[PATCH 3/3] Process CCM/EAX/GCM/Poly1305 AEAD cipher modes input in 24 KiB chucks
Jussi Kivilinna
jussi.kivilinna at iki.fi
Sat Dec 29 23:39:11 CET 2018
* cipher/cipher-ccm.c (_gcry_cipher_ccm_encrypt)
(_gcry_cipher_ccm_decrypt): Process data in 24 KiB chunks.
* cipher/cipher-eax.c (_gcry_cipher_eax_encrypt)
(_gcry_cipher_eax_decrypt): Ditto.
* cipher/cipher-gcm.c (_gcry_cipher_gcm_encrypt)
(_gcry_cipher_gcm_decrypt): Ditto.
* cipher/cipher-poly1305.c (_gcry_cipher_poly1305_encrypt)
(_gcry_cipher_poly1305_decrypt): Ditto.
--
Patch changes AEAD modes to process input in 24 KiB chuncks to improve
cache locality when processing large buffers.
Huge buffer test in tests/benchmark show 0.7% improvement for AES-CCM
and AES-EAX, 6% for AES-GCM and 4% for Chacha20-Poly1305 on Intel Core
i7-4790K.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
0 files changed
diff --git a/cipher/cipher-ccm.c b/cipher/cipher-ccm.c
index e71c6f159..fd284caa5 100644
--- a/cipher/cipher-ccm.c
+++ b/cipher/cipher-ccm.c
@@ -319,7 +319,9 @@ _gcry_cipher_ccm_encrypt (gcry_cipher_hd_t c, unsigned char *outbuf,
size_t outbuflen, const unsigned char *inbuf,
size_t inbuflen)
{
- unsigned int burn;
+ gcry_err_code_t err = 0;
+ unsigned int burn = 0;
+ unsigned int nburn;
if (outbuflen < inbuflen)
return GPG_ERR_BUFFER_TOO_SHORT;
@@ -329,12 +331,32 @@ _gcry_cipher_ccm_encrypt (gcry_cipher_hd_t c, unsigned char *outbuf,
if (inbuflen > c->u_mode.ccm.encryptlen)
return GPG_ERR_INV_LENGTH;
- c->u_mode.ccm.encryptlen -= inbuflen;
- burn = do_cbc_mac (c, inbuf, inbuflen, 0);
+ while (inbuflen)
+ {
+ size_t currlen = inbuflen;
+
+ /* Since checksumming is done before encryption, process input in 24KiB
+ * chunks to keep data loaded in L1 cache for encryption. */
+ if (currlen > 24 * 1024)
+ currlen = 24 * 1024;
+
+ c->u_mode.ccm.encryptlen -= currlen;
+ nburn = do_cbc_mac (c, inbuf, currlen, 0);
+ burn = nburn > burn ? nburn : burn;
+
+ err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, currlen);
+ if (err)
+ break;
+
+ outbuf += currlen;
+ inbuf += currlen;
+ outbuflen -= currlen;
+ inbuflen -= currlen;
+ }
+
if (burn)
_gcry_burn_stack (burn + sizeof(void *) * 5);
-
- return _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, inbuflen);
+ return err;
}
@@ -343,8 +365,9 @@ _gcry_cipher_ccm_decrypt (gcry_cipher_hd_t c, unsigned char *outbuf,
size_t outbuflen, const unsigned char *inbuf,
size_t inbuflen)
{
- gcry_err_code_t err;
- unsigned int burn;
+ gcry_err_code_t err = 0;
+ unsigned int burn = 0;
+ unsigned int nburn;
if (outbuflen < inbuflen)
return GPG_ERR_BUFFER_TOO_SHORT;
@@ -354,14 +377,30 @@ _gcry_cipher_ccm_decrypt (gcry_cipher_hd_t c, unsigned char *outbuf,
if (inbuflen > c->u_mode.ccm.encryptlen)
return GPG_ERR_INV_LENGTH;
- err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, inbuflen);
- if (err)
- return err;
+ while (inbuflen)
+ {
+ size_t currlen = inbuflen;
+
+ /* Since checksumming is done after decryption, process input in 24KiB
+ * chunks to keep data loaded in L1 cache for checksumming. */
+ if (currlen > 24 * 1024)
+ currlen = 24 * 1024;
+
+ err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, currlen);
+ if (err)
+ break;
+
+ c->u_mode.ccm.encryptlen -= currlen;
+ nburn = do_cbc_mac (c, outbuf, currlen, 0);
+ burn = nburn > burn ? nburn : burn;
+
+ outbuf += currlen;
+ inbuf += currlen;
+ outbuflen -= currlen;
+ inbuflen -= currlen;
+ }
- c->u_mode.ccm.encryptlen -= inbuflen;
- burn = do_cbc_mac (c, outbuf, inbuflen, 0);
if (burn)
_gcry_burn_stack (burn + sizeof(void *) * 5);
-
return err;
}
diff --git a/cipher/cipher-eax.c b/cipher/cipher-eax.c
index 3b17bb648..08f815a9e 100644
--- a/cipher/cipher-eax.c
+++ b/cipher/cipher-eax.c
@@ -48,11 +48,31 @@ _gcry_cipher_eax_encrypt (gcry_cipher_hd_t c,
return err;
}
- err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, inbuflen);
- if (err != 0)
- return err;
+ while (inbuflen)
+ {
+ size_t currlen = inbuflen;
+
+ /* Since checksumming is done after encryption, process input in 24KiB
+ * chunks to keep data loaded in L1 cache for checksumming. */
+ if (currlen > 24 * 1024)
+ currlen = 24 * 1024;
+
+ err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, currlen);
+ if (err != 0)
+ return err;
- return _gcry_cmac_write (c, &c->u_mode.eax.cmac_ciphertext, outbuf, inbuflen);
+ err = _gcry_cmac_write (c, &c->u_mode.eax.cmac_ciphertext, outbuf,
+ currlen);
+ if (err != 0)
+ return err;
+
+ outbuf += currlen;
+ inbuf += currlen;
+ outbuflen -= currlen;
+ inbuflen -= currlen;
+ }
+
+ return 0;
}
@@ -75,11 +95,31 @@ _gcry_cipher_eax_decrypt (gcry_cipher_hd_t c,
return err;
}
- err = _gcry_cmac_write (c, &c->u_mode.eax.cmac_ciphertext, inbuf, inbuflen);
- if (err != 0)
- return err;
+ while (inbuflen)
+ {
+ size_t currlen = inbuflen;
+
+ /* Since checksumming is done before decryption, process input in 24KiB
+ * chunks to keep data loaded in L1 cache for decryption. */
+ if (currlen > 24 * 1024)
+ currlen = 24 * 1024;
+
+ err = _gcry_cmac_write (c, &c->u_mode.eax.cmac_ciphertext, inbuf,
+ currlen);
+ if (err != 0)
+ return err;
- return _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, inbuflen);
+ err = _gcry_cipher_ctr_encrypt (c, outbuf, outbuflen, inbuf, currlen);
+ if (err != 0)
+ return err;
+
+ outbuf += currlen;
+ inbuf += currlen;
+ outbuflen -= currlen;
+ inbuflen -= currlen;
+ }
+
+ return 0;
}
diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c
index 32ec9fa06..f9ddbc568 100644
--- a/cipher/cipher-gcm.c
+++ b/cipher/cipher-gcm.c
@@ -666,11 +666,26 @@ _gcry_cipher_gcm_encrypt (gcry_cipher_hd_t c,
return GPG_ERR_INV_LENGTH;
}
- err = gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, inbuflen);
- if (err != 0)
- return err;
+ while (inbuflen)
+ {
+ size_t currlen = inbuflen;
- do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, outbuf, inbuflen, 0);
+ /* Since checksumming is done after encryption, process input in 24KiB
+ * chunks to keep data loaded in L1 cache for checksumming. */
+ if (currlen > 24 * 1024)
+ currlen = 24 * 1024;
+
+ err = gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, currlen);
+ if (err != 0)
+ return err;
+
+ do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, outbuf, currlen, 0);
+
+ outbuf += currlen;
+ inbuf += currlen;
+ outbuflen -= currlen;
+ inbuflen -= currlen;
+ }
return 0;
}
@@ -682,6 +697,7 @@ _gcry_cipher_gcm_decrypt (gcry_cipher_hd_t c,
const byte *inbuf, size_t inbuflen)
{
static const unsigned char zerobuf[MAX_BLOCKSIZE];
+ gcry_err_code_t err;
if (c->spec->blocksize != GCRY_GCM_BLOCK_LEN)
return GPG_ERR_CIPHER_ALGO;
@@ -711,9 +727,28 @@ _gcry_cipher_gcm_decrypt (gcry_cipher_hd_t c,
return GPG_ERR_INV_LENGTH;
}
- do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, inbuf, inbuflen, 0);
+ while (inbuflen)
+ {
+ size_t currlen = inbuflen;
+
+ /* Since checksumming is done before decryption, process input in
+ * 24KiB chunks to keep data loaded in L1 cache for decryption. */
+ if (currlen > 24 * 1024)
+ currlen = 24 * 1024;
- return gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, inbuflen);
+ do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, inbuf, currlen, 0);
+
+ err = gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, currlen);
+ if (err)
+ return err;
+
+ outbuf += currlen;
+ inbuf += currlen;
+ outbuflen -= currlen;
+ inbuflen -= currlen;
+ }
+
+ return 0;
}
diff --git a/cipher/cipher-poly1305.c b/cipher/cipher-poly1305.c
index 82537aa4d..607586b55 100644
--- a/cipher/cipher-poly1305.c
+++ b/cipher/cipher-poly1305.c
@@ -164,9 +164,24 @@ _gcry_cipher_poly1305_encrypt (gcry_cipher_hd_t c,
return GPG_ERR_INV_LENGTH;
}
- c->spec->stencrypt(&c->context.c, outbuf, (byte*)inbuf, inbuflen);
+ while (inbuflen)
+ {
+ size_t currlen = inbuflen;
+
+ /* Since checksumming is done after encryption, process input in 24KiB
+ * chunks to keep data loaded in L1 cache for checksumming. */
+ if (currlen > 24 * 1024)
+ currlen = 24 * 1024;
+
+ c->spec->stencrypt(&c->context.c, outbuf, (byte*)inbuf, currlen);
- _gcry_poly1305_update (&c->u_mode.poly1305.ctx, outbuf, inbuflen);
+ _gcry_poly1305_update (&c->u_mode.poly1305.ctx, outbuf, currlen);
+
+ outbuf += currlen;
+ inbuf += currlen;
+ outbuflen -= currlen;
+ inbuflen -= currlen;
+ }
return 0;
}
@@ -202,9 +217,25 @@ _gcry_cipher_poly1305_decrypt (gcry_cipher_hd_t c,
return GPG_ERR_INV_LENGTH;
}
- _gcry_poly1305_update (&c->u_mode.poly1305.ctx, inbuf, inbuflen);
+ while (inbuflen)
+ {
+ size_t currlen = inbuflen;
+
+ /* Since checksumming is done before decryption, process input in 24KiB
+ * chunks to keep data loaded in L1 cache for decryption. */
+ if (currlen > 24 * 1024)
+ currlen = 24 * 1024;
+
+ _gcry_poly1305_update (&c->u_mode.poly1305.ctx, inbuf, currlen);
+
+ c->spec->stdecrypt(&c->context.c, outbuf, (byte*)inbuf, currlen);
+
+ outbuf += currlen;
+ inbuf += currlen;
+ outbuflen -= currlen;
+ inbuflen -= currlen;
+ }
- c->spec->stdecrypt(&c->context.c, outbuf, (byte*)inbuf, inbuflen);
return 0;
}
More information about the Gcrypt-devel
mailing list