[PATCH 1/6] Add bulk processing for hash transform functions
Jussi Kivilinna
jussi.kivilinna at iki.fi
Tue Dec 17 15:37:11 CET 2013
* cipher/hash-common.c (_gcry_md_block_write): Preload 'hd->blocksize'
to stack, pass number of blocks to 'hd->bwrite'.
* cipher/hash-common.c (_gcry_md_block_write_t): Add 'nblks'.
* cipher/gostr3411-94.c: Rename 'transform' function to
'transform_blk', add new 'transform' function with 'nblks' as
additional input.
* cipher/md4.c: Ditto.
* cipher/md5.c: Ditto.
* cipher/md4.c: Ditto.
* cipher/rmd160.c: Ditto.
* cipher/sha1.c: Ditto.
* cipher/sha256.c: Ditto.
* cipher/sha512.c: Ditto.
* cipher/stribog.c: Ditto.
* cipher/tiger.c: Ditto.
* cipher/whirlpool.c: Ditto.
--
Pass number of blocks to algorithm for futher optimizations.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/gostr3411-94.c | 21 +++++++++++++++++++--
cipher/hash-common.c | 25 ++++++++++++++-----------
cipher/hash-common.h | 3 ++-
cipher/md4.c | 21 ++++++++++++++++++---
cipher/md5.c | 23 +++++++++++++++++++----
cipher/rmd160.c | 24 ++++++++++++++++++++----
cipher/sha1.c | 30 +++++++++++++++++++++++-------
cipher/sha256.c | 20 ++++++++++++++------
cipher/sha512.c | 31 ++++++++++++++++++++++---------
cipher/stribog.c | 27 +++++++++++++++++++++------
cipher/tiger.c | 22 +++++++++++++++++++---
cipher/whirlpool.c | 19 +++++++++++++++++--
12 files changed, 208 insertions(+), 58 deletions(-)
diff --git a/cipher/gostr3411-94.c b/cipher/gostr3411-94.c
index 1267216..b3326aa 100644
--- a/cipher/gostr3411-94.c
+++ b/cipher/gostr3411-94.c
@@ -41,7 +41,7 @@ typedef struct {
} GOSTR3411_CONTEXT;
static unsigned int
-transform (void *c, const unsigned char *data);
+transform (void *c, const unsigned char *data, size_t nblks);
static void
gost3411_init (void *context)
@@ -211,7 +211,7 @@ do_hash_step (GOST28147_context *hd, unsigned char *h, unsigned char *m)
static unsigned int
-transform (void *ctx, const unsigned char *data)
+transform_blk (void *ctx, const unsigned char *data)
{
GOSTR3411_CONTEXT *hd = ctx;
byte m[32];
@@ -224,6 +224,23 @@ transform (void *ctx, const unsigned char *data)
return /* burn_stack */ burn + 3 * sizeof(void*) + 32 + 2 * sizeof(void*);
}
+
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+ unsigned int burn;
+
+ do
+ {
+ burn = transform_blk (c, data);
+ data += 32;
+ }
+ while (--nblks);
+
+ return burn;
+}
+
+
/*
The routine finally terminates the computation and returns the
digest. The handle is prepared for a new cycle, but adding bytes
diff --git a/cipher/hash-common.c b/cipher/hash-common.c
index ffbc39e..ed63a0b 100644
--- a/cipher/hash-common.c
+++ b/cipher/hash-common.c
@@ -102,16 +102,18 @@ _gcry_md_block_write (void *context, const void *inbuf_arg, size_t inlen)
const unsigned char *inbuf = inbuf_arg;
gcry_md_block_ctx_t *hd = context;
unsigned int stack_burn = 0;
+ const unsigned int blocksize = hd->blocksize;
+ size_t inblocks;
- if (sizeof(hd->buf) < hd->blocksize)
+ if (sizeof(hd->buf) < blocksize)
BUG();
if (hd->buf == NULL || hd->bwrite == NULL)
return;
- if (hd->count == hd->blocksize) /* Flush the buffer. */
+ if (hd->count == blocksize) /* Flush the buffer. */
{
- stack_burn = hd->bwrite (hd, hd->buf);
+ stack_burn = hd->bwrite (hd, hd->buf, 1);
_gcry_burn_stack (stack_burn);
stack_burn = 0;
hd->count = 0;
@@ -123,23 +125,24 @@ _gcry_md_block_write (void *context, const void *inbuf_arg, size_t inlen)
if (hd->count)
{
- for (; inlen && hd->count < hd->blocksize; inlen--)
+ for (; inlen && hd->count < blocksize; inlen--)
hd->buf[hd->count++] = *inbuf++;
_gcry_md_block_write (hd, NULL, 0);
if (!inlen)
return;
}
- while (inlen >= hd->blocksize)
+ if (inlen >= blocksize)
{
- stack_burn = hd->bwrite (hd, inbuf);
+ inblocks = inlen / blocksize;
+ stack_burn = hd->bwrite (hd, inbuf, inblocks);
hd->count = 0;
- if (!++hd->nblocks)
- hd->nblocks_high++;
- inlen -= hd->blocksize;
- inbuf += hd->blocksize;
+ hd->nblocks_high += (hd->nblocks + inblocks < inblocks);
+ hd->nblocks += inblocks;
+ inlen -= inblocks * blocksize;
+ inbuf += inblocks * blocksize;
}
_gcry_burn_stack (stack_burn);
- for (; inlen && hd->count < hd->blocksize; inlen--)
+ for (; inlen && hd->count < blocksize; inlen--)
hd->buf[hd->count++] = *inbuf++;
}
diff --git a/cipher/hash-common.h b/cipher/hash-common.h
index aa95365..c949191 100644
--- a/cipher/hash-common.h
+++ b/cipher/hash-common.h
@@ -30,7 +30,8 @@ const char * _gcry_hash_selftest_check_one
/* Type for the md_write helper function. */
typedef unsigned int (*_gcry_md_block_write_t) (void *c,
- const unsigned char *buf);
+ const unsigned char *blks,
+ size_t nblks);
#if defined(HAVE_U64_TYPEDEF) && (defined(USE_SHA512) || defined(USE_WHIRLPOOL))
/* SHA-512 needs u64 and larger buffer. Whirlpool needs u64. */
diff --git a/cipher/md4.c b/cipher/md4.c
index b9a1a95..40dc058 100644
--- a/cipher/md4.c
+++ b/cipher/md4.c
@@ -66,7 +66,7 @@ typedef struct {
} MD4_CONTEXT;
static unsigned int
-transform ( void *c, const unsigned char *data );
+transform ( void *c, const unsigned char *data, size_t nblks );
static void
md4_init( void *context )
@@ -94,7 +94,7 @@ md4_init( void *context )
* transform 64 bytes
*/
static unsigned int
-transform ( void *c, const unsigned char *data )
+transform_blk ( void *c, const unsigned char *data )
{
MD4_CONTEXT *ctx = c;
u32 in[16];
@@ -181,6 +181,21 @@ transform ( void *c, const unsigned char *data )
}
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+ unsigned int burn;
+
+ do
+ {
+ burn = transform_blk (c, data);
+ data += 64;
+ }
+ while (--nblks);
+
+ return burn;
+}
+
/* The routine final terminates the message-digest computation and
* ends with the desired message digest in mdContext->digest[0...15].
@@ -234,7 +249,7 @@ md4_final( void *context )
/* append the 64 bit count */
buf_put_le32(hd->bctx.buf + 56, lsb);
buf_put_le32(hd->bctx.buf + 60, msb);
- burn = transform( hd, hd->bctx.buf );
+ burn = transform ( hd, hd->bctx.buf, 1 );
_gcry_burn_stack (burn);
p = hd->bctx.buf;
diff --git a/cipher/md5.c b/cipher/md5.c
index 79b6e87..d06d3f7 100644
--- a/cipher/md5.c
+++ b/cipher/md5.c
@@ -50,7 +50,7 @@ typedef struct {
} MD5_CONTEXT;
static unsigned int
-transform ( void *ctx, const unsigned char *data );
+transform ( void *ctx, const unsigned char *data, size_t datalen );
static void
md5_init( void *context )
@@ -81,10 +81,10 @@ md5_init( void *context )
/****************
- * transform n*64 bytes
+ * transform 64 bytes
*/
static unsigned int
-transform ( void *c, const unsigned char *data )
+transform_blk ( void *c, const unsigned char *data )
{
MD5_CONTEXT *ctx = c;
u32 correct_words[16];
@@ -205,6 +205,21 @@ transform ( void *c, const unsigned char *data )
}
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+ unsigned int burn;
+
+ do
+ {
+ burn = transform_blk (c, data);
+ data += 64;
+ }
+ while (--nblks);
+
+ return burn;
+}
+
/* The routine final terminates the message-digest computation and
* ends with the desired message digest in mdContext->digest[0...15].
@@ -258,7 +273,7 @@ md5_final( void *context)
/* append the 64 bit count */
buf_put_le32(hd->bctx.buf + 56, lsb);
buf_put_le32(hd->bctx.buf + 60, msb);
- burn = transform( hd, hd->bctx.buf );
+ burn = transform ( hd, hd->bctx.buf, 1 );
_gcry_burn_stack (burn);
p = hd->bctx.buf;
diff --git a/cipher/rmd160.c b/cipher/rmd160.c
index a6d9a80..224694f 100644
--- a/cipher/rmd160.c
+++ b/cipher/rmd160.c
@@ -141,7 +141,7 @@
*/
static unsigned int
-transform ( void *ctx, const unsigned char *data );
+transform ( void *ctx, const unsigned char *data, size_t nblks );
void
_gcry_rmd160_init (void *context)
@@ -167,7 +167,7 @@ _gcry_rmd160_init (void *context)
* Transform the message X which consists of 16 32-bit-words
*/
static unsigned int
-transform ( void *ctx, const unsigned char *data )
+transform_blk ( void *ctx, const unsigned char *data )
{
RMD160_CONTEXT *hd = ctx;
register u32 a,b,c,d,e;
@@ -386,6 +386,22 @@ transform ( void *ctx, const unsigned char *data )
}
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+ unsigned int burn;
+
+ do
+ {
+ burn = transform_blk (c, data);
+ data += 64;
+ }
+ while (--nblks);
+
+ return burn;
+}
+
+
/****************
* Apply the rmd160 transform function on the buffer which must have
* a length 64 bytes. Do not use this function together with the
@@ -397,7 +413,7 @@ _gcry_rmd160_mixblock ( RMD160_CONTEXT *hd, void *blockof64byte )
{
char *p = blockof64byte;
- transform ( hd, blockof64byte );
+ transform ( hd, blockof64byte, 64 );
#define X(a) do { *(u32*)p = hd->h##a ; p += 4; } while(0)
X(0);
X(1);
@@ -457,7 +473,7 @@ rmd160_final( void *context )
/* append the 64 bit count */
buf_put_le32(hd->bctx.buf + 56, lsb);
buf_put_le32(hd->bctx.buf + 60, msb);
- burn = transform( hd, hd->bctx.buf );
+ burn = transform ( hd, hd->bctx.buf, 1 );
_gcry_burn_stack (burn);
p = hd->bctx.buf;
diff --git a/cipher/sha1.c b/cipher/sha1.c
index 18b6daa..53f7538 100644
--- a/cipher/sha1.c
+++ b/cipher/sha1.c
@@ -70,7 +70,7 @@ typedef struct
} SHA1_CONTEXT;
static unsigned int
-transform (void *c, const unsigned char *data);
+transform (void *c, const unsigned char *data, size_t nblks);
static void
@@ -122,7 +122,7 @@ sha1_init (void *context)
* Transform NBLOCKS of each 64 bytes (16 32-bit words) at DATA.
*/
static unsigned int
-_transform (void *ctx, const unsigned char *data)
+transform_blk (void *ctx, const unsigned char *data)
{
SHA1_CONTEXT *hd = ctx;
const u32 *idata = (const void *)data;
@@ -239,17 +239,33 @@ _gcry_sha1_transform_amd64_ssse3 (void *state, const unsigned char *data);
static unsigned int
-transform (void *ctx, const unsigned char *data)
+transform (void *ctx, const unsigned char *data, size_t nblks)
{
SHA1_CONTEXT *hd = ctx;
+ unsigned int burn;
#ifdef USE_SSSE3
if (hd->use_ssse3)
- return _gcry_sha1_transform_amd64_ssse3 (&hd->h0, data)
- + 4 * sizeof(void*);
+ {
+ do
+ {
+ burn = _gcry_sha1_transform_amd64_ssse3 (&hd->h0, data);
+ data += 64;
+ }
+ while (--nblks);
+
+ return burn + 4 * sizeof(void*);
+ }
#endif
- return _transform (hd, data);
+ do
+ {
+ burn = transform_blk (ctx, data);
+ data += 64;
+ }
+ while (--nblks);
+
+ return burn;
}
@@ -306,7 +322,7 @@ sha1_final(void *context)
/* append the 64 bit count */
buf_put_be32(hd->bctx.buf + 56, msb);
buf_put_be32(hd->bctx.buf + 60, lsb);
- burn = transform( hd, hd->bctx.buf );
+ burn = transform( hd, hd->bctx.buf, 1 );
_gcry_burn_stack (burn);
p = hd->bctx.buf;
diff --git a/cipher/sha256.c b/cipher/sha256.c
index f3c1d62..c2045b8 100644
--- a/cipher/sha256.c
+++ b/cipher/sha256.c
@@ -66,7 +66,7 @@ typedef struct {
static unsigned int
-transform (void *c, const unsigned char *data);
+transform (void *c, const unsigned char *data, size_t nblks);
static void
@@ -170,7 +170,7 @@ Sum1 (u32 x)
static unsigned int
-_transform (void *ctx, const unsigned char *data)
+transform_blk (void *ctx, const unsigned char *data)
{
SHA256_CONTEXT *hd = ctx;
static const u32 K[64] = {
@@ -283,17 +283,25 @@ unsigned int _gcry_sha256_transform_amd64_ssse3(const void *input_data,
static unsigned int
-transform (void *ctx, const unsigned char *data)
+transform (void *ctx, const unsigned char *data, size_t nblks)
{
SHA256_CONTEXT *hd = ctx;
+ unsigned int burn;
#ifdef USE_SSSE3
if (hd->use_ssse3)
- return _gcry_sha256_transform_amd64_ssse3 (data, &hd->h0, 1)
+ return _gcry_sha256_transform_amd64_ssse3 (data, &hd->h0, nblks)
+ 4 * sizeof(void*);
#endif
- return _transform (hd, data);
+ do
+ {
+ burn = transform_blk (hd, data);
+ data += 64;
+ }
+ while (--nblks);
+
+ return burn;
}
@@ -348,7 +356,7 @@ sha256_final(void *context)
/* append the 64 bit count */
buf_put_be32(hd->bctx.buf + 56, msb);
buf_put_be32(hd->bctx.buf + 60, lsb);
- burn = transform (hd, hd->bctx.buf);
+ burn = transform (hd, hd->bctx.buf, 1);
_gcry_burn_stack (burn);
p = hd->bctx.buf;
diff --git a/cipher/sha512.c b/cipher/sha512.c
index 586c809..215e8ed 100644
--- a/cipher/sha512.c
+++ b/cipher/sha512.c
@@ -116,7 +116,7 @@ typedef struct
} SHA512_CONTEXT;
static unsigned int
-transform (void *context, const unsigned char *data);
+transform (void *context, const unsigned char *data, size_t nblks);
static void
sha512_init (void *context)
@@ -273,7 +273,7 @@ static const u64 k[] =
* Transform the message W which consists of 16 64-bit-words
*/
static unsigned int
-__transform (SHA512_STATE *hd, const unsigned char *data)
+transform_blk (SHA512_STATE *hd, const unsigned char *data)
{
u64 a, b, c, d, e, f, g, h;
u64 w[16];
@@ -561,32 +561,38 @@ unsigned int _gcry_sha512_transform_amd64_avx2(const void *input_data,
static unsigned int
-transform (void *context, const unsigned char *data)
+transform (void *context, const unsigned char *data, size_t nblks)
{
SHA512_CONTEXT *ctx = context;
+ unsigned int burn;
#ifdef USE_AVX2
if (ctx->use_avx2)
- return _gcry_sha512_transform_amd64_avx2 (data, &ctx->state, 1)
+ return _gcry_sha512_transform_amd64_avx2 (data, &ctx->state, nblks)
+ 4 * sizeof(void*);
#endif
#ifdef USE_AVX
if (ctx->use_avx)
- return _gcry_sha512_transform_amd64_avx (data, &ctx->state, 1)
+ return _gcry_sha512_transform_amd64_avx (data, &ctx->state, nblks)
+ 4 * sizeof(void*);
#endif
#ifdef USE_SSSE3
if (ctx->use_ssse3)
- return _gcry_sha512_transform_amd64_ssse3 (data, &ctx->state, 1)
+ return _gcry_sha512_transform_amd64_ssse3 (data, &ctx->state, nblks)
+ 4 * sizeof(void*);
#endif
#ifdef USE_ARM_NEON_ASM
if (ctx->use_neon)
{
- _gcry_sha512_transform_armv7_neon (&ctx->state, data, k);
+ do
+ {
+ _gcry_sha512_transform_armv7_neon (&ctx->state, data, k);
+ data += 128;
+ }
+ while (--nblks);
/* _gcry_sha512_transform_armv7_neon does not store sensitive data
* to stack. */
@@ -594,7 +600,14 @@ transform (void *context, const unsigned char *data)
}
#endif
- return __transform (&ctx->state, data) + 3 * sizeof(void*);
+ do
+ {
+ burn = transform_blk (&ctx->state, data) + 3 * sizeof(void*);
+ data += 128;
+ }
+ while (--nblks);
+
+ return burn;
}
@@ -652,7 +665,7 @@ sha512_final (void *context)
/* append the 128 bit count */
buf_put_be64(hd->bctx.buf + 112, msb);
buf_put_be64(hd->bctx.buf + 120, lsb);
- stack_burn_depth = transform (hd, hd->bctx.buf);
+ stack_burn_depth = transform (hd, hd->bctx.buf, 1);
_gcry_burn_stack (stack_burn_depth);
p = hd->bctx.buf;
diff --git a/cipher/stribog.c b/cipher/stribog.c
index a3cab36..6d1d342 100644
--- a/cipher/stribog.c
+++ b/cipher/stribog.c
@@ -1194,7 +1194,7 @@ static inline void g (u64 *h, u64 *m, u64 *N)
static unsigned int
-transform64 (void *context, const unsigned char *inbuf_arg);
+transform (void *context, const unsigned char *inbuf_arg, size_t datalen);
static void
@@ -1205,7 +1205,7 @@ stribog_init_512 (void *context)
memset (hd, 0, sizeof (*hd));
hd->bctx.blocksize = 64;
- hd->bctx.bwrite = transform64;
+ hd->bctx.bwrite = transform;
}
static void
@@ -1217,7 +1217,7 @@ stribog_init_256 (void *context)
}
static void
-transform (STRIBOG_CONTEXT *hd, const unsigned char *data, unsigned count)
+transform_bits (STRIBOG_CONTEXT *hd, const unsigned char *data, unsigned count)
{
u64 M[8];
u64 l;
@@ -1248,15 +1248,30 @@ transform (STRIBOG_CONTEXT *hd, const unsigned char *data, unsigned count)
}
static unsigned int
-transform64 (void *context, const unsigned char *inbuf_arg)
+transform_blk (void *context, const unsigned char *inbuf_arg)
{
STRIBOG_CONTEXT *hd = context;
- transform (hd, inbuf_arg, 64 * 8);
+ transform_bits (hd, inbuf_arg, 64 * 8);
return /* burn_stack */ 768;
}
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+ unsigned int burn;
+
+ do
+ {
+ burn = transform_blk (c, data);
+ data += 64;
+ }
+ while (--nblks);
+
+ return burn;
+}
+
/*
The routine finally terminates the computation and returns the
digest. The handle is prepared for a new cycle, but adding bytes
@@ -1276,7 +1291,7 @@ stribog_final (void *context)
hd->bctx.buf[i++] = 1;
while (i < 64)
hd->bctx.buf[i++] = 0;
- transform (hd, hd->bctx.buf, hd->bctx.count * 8);
+ transform_bits (hd, hd->bctx.buf, hd->bctx.count * 8);
g (hd->h, hd->N, Z);
g (hd->h, hd->Sigma, Z);
diff --git a/cipher/tiger.c b/cipher/tiger.c
index 9b8d0ef..17c4119 100644
--- a/cipher/tiger.c
+++ b/cipher/tiger.c
@@ -590,7 +590,7 @@ static u64 sbox4[256] = {
};
static unsigned int
-transform ( void *ctx, const unsigned char *data );
+transform ( void *ctx, const unsigned char *data, size_t nblks );
static void
do_init (void *context, int variant)
@@ -695,7 +695,7 @@ key_schedule( u64 *x )
* Transform the message DATA which consists of 512 bytes (8 words)
*/
static unsigned int
-transform ( void *ctx, const unsigned char *data )
+transform_blk ( void *ctx, const unsigned char *data )
{
TIGER_CONTEXT *hd = ctx;
u64 a,b,c,aa,bb,cc;
@@ -729,6 +729,22 @@ transform ( void *ctx, const unsigned char *data )
}
+static unsigned int
+transform ( void *c, const unsigned char *data, size_t nblks )
+{
+ unsigned int burn;
+
+ do
+ {
+ burn = transform_blk (c, data);
+ data += 64;
+ }
+ while (--nblks);
+
+ return burn;
+}
+
+
/* The routine terminates the computation
*/
@@ -779,7 +795,7 @@ tiger_final( void *context )
/* append the 64 bit count */
buf_put_le32(hd->bctx.buf + 56, lsb);
buf_put_le32(hd->bctx.buf + 60, msb);
- burn = transform( hd, hd->bctx.buf );
+ burn = transform( hd, hd->bctx.buf, 1 );
_gcry_burn_stack (burn);
p = hd->bctx.buf;
diff --git a/cipher/whirlpool.c b/cipher/whirlpool.c
index e562781..57ca882 100644
--- a/cipher/whirlpool.c
+++ b/cipher/whirlpool.c
@@ -1161,7 +1161,7 @@ static const u64 C7[256] =
static unsigned int
-whirlpool_transform (void *ctx, const unsigned char *data);
+whirlpool_transform (void *ctx, const unsigned char *data, size_t nblks);
@@ -1181,7 +1181,7 @@ whirlpool_init (void *ctx)
* Transform block.
*/
static unsigned int
-whirlpool_transform (void *ctx, const unsigned char *data)
+whirlpool_transform_blk (void *ctx, const unsigned char *data)
{
whirlpool_context_t *context = ctx;
whirlpool_block_t data_block;
@@ -1280,6 +1280,21 @@ whirlpool_transform (void *ctx, const unsigned char *data)
4 * sizeof(void*);
}
+static unsigned int
+whirlpool_transform ( void *c, const unsigned char *data, size_t nblks )
+{
+ unsigned int burn;
+
+ do
+ {
+ burn = whirlpool_transform_blk (c, data);
+ data += BLOCK_SIZE;
+ }
+ while (--nblks);
+
+ return burn;
+}
+
static void
whirlpool_write (void *ctx, const void *buffer, size_t buffer_n)
{
More information about the Gcrypt-devel
mailing list