[gnutls-devel] GnuTLS | gnutls_hash_copy() fails on SHA384 after gnutls_hash_output(). (#1257)

[Read-only notification of GnuTLS library development activities]

David Woodhouse created an issue: https://gitlab.com/gnutls/gnutls/-/issues/1257



In order to support TLSv1.3 with TPM-based keys (cf. #1235) I implemented RSA-PSS padding in my *application*: https://gitlab.com/openconnect/openconnect/-/commit/ff367965fcc13f6c1ba7fbda7a49d1467f1b39de

After generating the initial hash `M'` from `( 00 | 00 | 00 | 00 | 00 | 00 | 00 | 00 | mHash | Salt )` we then repeatedly generate hashes of `( M' | C )` where C is an incrementing 32-bit big-endian counter.

Instead of *repeatedly* feeding `M'` into the hash function, I chose to use `gnutls_hash_copy()` from a context where `M'` had already been hashed. Then each time round the loop it's just `gnutls_hash_copy()`, `gnutls_hash()` to add the four bytes of `C`, and `gnutls_hash_deinit()` of the copy.

This appears to work for SHA256 and SHA512, but fails for SHA384. It seems to have something to do with the fact that the hash context has already been used (to create `M'`). I thought that `gnutls_hash_output()` was supposed to reset it and leave it ready for re-use, but apparently not; calling `gnutls_hash_deinit()` and `gnutls_hash_init()` again does seem to reset it *harder* and make it work.

Tested with gnutls-3.6.16-1.fc33.x86_64

Test case:

```C
#
#include <gnutls/gnutls.h>
#include <gnutls/crypto.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <arpa/inet.h>

#define SHA512_SIZE 64

static unsigned char mHash[SHA512_SIZE] = {
	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
	0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
	0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
	0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
	0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
	0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f
};

static unsigned char mPrime[SHA512_SIZE];

static unsigned char h1[SHA512_SIZE];
static unsigned char h2[SHA512_SIZE];

static unsigned char h1p[SHA512_SIZE];
static unsigned char h2p[SHA512_SIZE];


static int mgf1(gnutls_hash_hd_t hctx, int count, unsigned char *buf, int diglen)
{
	gnutls_hash_hd_t ctx2 = gnutls_hash_copy(hctx);

	if (!ctx2)
		return GNUTLS_E_PK_SIGN_FAILED;

	uint32_t be_count = htonl(count);
	int err = gnutls_hash(ctx2, &be_count, sizeof(be_count));
	if (err) {
		gnutls_hash_deinit(ctx2, NULL);
		return err;
	}
	gnutls_hash_deinit(ctx2, buf);

	int i;
	for (i = 0; i < diglen; i++) {
		if (!(i & 15)) printf("\n(%d) %04x:", count, i);
		printf(" %02x", buf[i]);
	}
	printf("\n");
	return 0;
}

int psstest(int dig)
{
	gnutls_hash_hd_t hashctx = NULL;
	int diglen = gnutls_hash_get_len(dig);
	int err = 0;

	if ((err = gnutls_hash_init(&hashctx, dig)) ||
	    (err = gnutls_hash(hashctx, "\0\0\0\0\0\0\0\0", 8)) ||
	    (err = gnutls_hash(hashctx, mHash, diglen)))
		goto out;

	gnutls_hash_output(hashctx, mPrime);

	/* gnutls_hash_output() is supposed to reset the state. For SHA256 at least
	 * it *does* seem to work, in
	 * http://git.infradead.org/users/dwmw2/openconnect.git/commitdiff/ff367965f
	 */
	if (err = gnutls_hash(hashctx, mPrime, diglen))
		goto out;

	if (err = mgf1(hashctx, 0, h1, diglen))
		goto out;

	if (err = mgf1(hashctx, 1, h2, diglen))
		goto out;

	/* This one really *does* reset it, and makes things work. But *should*
	 * be identical to the above?  */
	gnutls_hash_deinit(hashctx, NULL);
	if (err = gnutls_hash_init(&hashctx, dig))
		goto out;

	if (err = gnutls_hash(hashctx, mPrime, diglen))
		goto out;

	if (err = mgf1(hashctx, 0, h1p, diglen))
		goto out;

	if (err = mgf1(hashctx, 1, h2p, diglen))
		goto out;

	if (memcmp(h1, h1p, diglen) || memcmp(h2, h2p, diglen)) {
		printf("Error: mismatch\n");
		return 1;
	}

	return 0;
 out:
	printf("Error: %s\n", gnutls_strerror(err));
	return 1;
}

int main(void)
{
	//psstest(GNUTLS_DIG_SHA256);
	psstest(GNUTLS_DIG_SHA384);
	//psstest(GNUTLS_DIG_SHA512);
}

```

-- 
Reply to this email directly or view it on GitLab: https://gitlab.com/gnutls/gnutls/-/issues/1257
You're receiving this email because of your account on gitlab.com.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.gnupg.org/pipermail/gnutls-devel/attachments/20210727/246abd93/attachment-0001.html>