[PATCH 3/9] Avoid division by spec->blocksize in cipher mode handlers
Jussi Kivilinna
jussi.kivilinna at iki.fi
Tue Jun 19 17:51:15 CEST 2018
* cipher/cipher-internal.h (_gcry_blocksize_shift): New.
* cipher/cipher-cbc.c (_gcry_cipher_cbc_encrypt)
(_gcry_cipherp_cbc_decrypt): Use bit-level operations instead of
division to get number of blocks and check input length against
blocksize.
* cipher/cipher-cfb.c (_gcry_cipher_cfb_encrypt)
(_gcry_cipher_cfb_decrypt): Ditto.
* cipher/cipher-cmac.c (_gcry_cmac_write): Ditto.
* cipher/cipher-ctr.c (_gcry_cipher_ctr_crypt): Ditto.
* cipher/cipher-ofb.c (_gcry_cipher_ofb_encrypt)
(_gcry_cipher_ofb_decrypt): Ditto.
--
Integer division was causing 10 to 20 cycles per call overhead
for cipher modes on x86-64.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/cipher-cbc.c | 50 ++++++++++++++++++----------------------
cipher/cipher-cfb.c | 32 ++++++++++---------------
cipher/cipher-cmac.c | 16 +++++--------
cipher/cipher-ctr.c | 16 +++++--------
cipher/cipher-internal.h | 10 ++++++++
cipher/cipher-ofb.c | 8 ++-----
6 files changed, 58 insertions(+), 74 deletions(-)
diff --git a/cipher/cipher-cbc.c b/cipher/cipher-cbc.c
index 95c49b2b6..7951f34b6 100644
--- a/cipher/cipher-cbc.c
+++ b/cipher/cipher-cbc.c
@@ -39,20 +39,17 @@ _gcry_cipher_cbc_encrypt (gcry_cipher_hd_t c,
size_t n;
unsigned char *ivp;
int i;
- size_t blocksize = c->spec->blocksize;
+ size_t blocksize_shift = _gcry_blocksize_shift(c);
+ size_t blocksize = 1 << blocksize_shift;
+ size_t blocksize_mask = blocksize - 1;
gcry_cipher_encrypt_t enc_fn = c->spec->encrypt;
- size_t nblocks = inbuflen / blocksize;
+ size_t nblocks = inbuflen >> blocksize_shift;
unsigned int burn, nburn;
- /* Tell compiler that we require a cipher with a 64bit or 128 bit block
- * length, to allow better optimization of this function. */
- if (blocksize > 16 || blocksize < 8 || blocksize & (8 - 1))
- return GPG_ERR_INV_LENGTH;
-
- if (outbuflen < ((c->flags & GCRY_CIPHER_CBC_MAC)? blocksize : inbuflen))
+ if (outbuflen < ((c->flags & GCRY_CIPHER_CBC_MAC) ? blocksize : inbuflen))
return GPG_ERR_BUFFER_TOO_SHORT;
- if ((inbuflen % blocksize)
+ if ((inbuflen & blocksize_mask)
&& !(inbuflen > blocksize
&& (c->flags & GCRY_CIPHER_CBC_CTS)))
return GPG_ERR_INV_LENGTH;
@@ -61,7 +58,7 @@ _gcry_cipher_cbc_encrypt (gcry_cipher_hd_t c,
if ((c->flags & GCRY_CIPHER_CBC_CTS) && inbuflen > blocksize)
{
- if ((inbuflen % blocksize) == 0)
+ if ((inbuflen & blocksize_mask) == 0)
nblocks--;
}
@@ -69,9 +66,9 @@ _gcry_cipher_cbc_encrypt (gcry_cipher_hd_t c,
{
c->bulk.cbc_enc (&c->context.c, c->u_iv.iv, outbuf, inbuf, nblocks,
(c->flags & GCRY_CIPHER_CBC_MAC));
- inbuf += nblocks * blocksize;
+ inbuf += nblocks << blocksize_shift;
if (!(c->flags & GCRY_CIPHER_CBC_MAC))
- outbuf += nblocks * blocksize;
+ outbuf += nblocks << blocksize_shift;
}
else
{
@@ -83,7 +80,7 @@ _gcry_cipher_cbc_encrypt (gcry_cipher_hd_t c,
nburn = enc_fn ( &c->context.c, outbuf, outbuf );
burn = nburn > burn ? nburn : burn;
ivp = outbuf;
- inbuf += blocksize;
+ inbuf += blocksize;
if (!(c->flags & GCRY_CIPHER_CBC_MAC))
outbuf += blocksize;
}
@@ -99,10 +96,10 @@ _gcry_cipher_cbc_encrypt (gcry_cipher_hd_t c,
size_t restbytes;
unsigned char b;
- if ((inbuflen % blocksize) == 0)
+ if ((inbuflen & blocksize_mask) == 0)
restbytes = blocksize;
else
- restbytes = inbuflen % blocksize;
+ restbytes = inbuflen & blocksize_mask;
outbuf -= blocksize;
for (ivp = c->u_iv.iv, i = 0; i < restbytes; i++)
@@ -133,20 +130,17 @@ _gcry_cipher_cbc_decrypt (gcry_cipher_hd_t c,
{
size_t n;
int i;
- size_t blocksize = c->spec->blocksize;
+ size_t blocksize_shift = _gcry_blocksize_shift(c);
+ size_t blocksize = 1 << blocksize_shift;
+ size_t blocksize_mask = blocksize - 1;
gcry_cipher_decrypt_t dec_fn = c->spec->decrypt;
- size_t nblocks = inbuflen / blocksize;
+ size_t nblocks = inbuflen >> blocksize_shift;
unsigned int burn, nburn;
- /* Tell compiler that we require a cipher with a 64bit or 128 bit block
- * length, to allow better optimization of this function. */
- if (blocksize > 16 || blocksize < 8 || blocksize & (8 - 1))
- return GPG_ERR_INV_LENGTH;
-
if (outbuflen < inbuflen)
return GPG_ERR_BUFFER_TOO_SHORT;
- if ((inbuflen % blocksize)
+ if ((inbuflen & blocksize_mask)
&& !(inbuflen > blocksize
&& (c->flags & GCRY_CIPHER_CBC_CTS)))
return GPG_ERR_INV_LENGTH;
@@ -156,7 +150,7 @@ _gcry_cipher_cbc_decrypt (gcry_cipher_hd_t c,
if ((c->flags & GCRY_CIPHER_CBC_CTS) && inbuflen > blocksize)
{
nblocks--;
- if ((inbuflen % blocksize) == 0)
+ if ((inbuflen & blocksize_mask) == 0)
nblocks--;
buf_cpy (c->lastiv, c->u_iv.iv, blocksize);
}
@@ -164,8 +158,8 @@ _gcry_cipher_cbc_decrypt (gcry_cipher_hd_t c,
if (c->bulk.cbc_dec)
{
c->bulk.cbc_dec (&c->context.c, c->u_iv.iv, outbuf, inbuf, nblocks);
- inbuf += nblocks * blocksize;
- outbuf += nblocks * blocksize;
+ inbuf += nblocks << blocksize_shift;
+ outbuf += nblocks << blocksize_shift;
}
else
{
@@ -186,10 +180,10 @@ _gcry_cipher_cbc_decrypt (gcry_cipher_hd_t c,
{
size_t restbytes;
- if ((inbuflen % blocksize) == 0)
+ if ((inbuflen & blocksize_mask) == 0)
restbytes = blocksize;
else
- restbytes = inbuflen % blocksize;
+ restbytes = inbuflen & blocksize_mask;
buf_cpy (c->lastiv, c->u_iv.iv, blocksize ); /* Save Cn-2. */
buf_cpy (c->u_iv.iv, inbuf + blocksize, restbytes ); /* Save Cn. */
diff --git a/cipher/cipher-cfb.c b/cipher/cipher-cfb.c
index c888e70a8..7f00aee5c 100644
--- a/cipher/cipher-cfb.c
+++ b/cipher/cipher-cfb.c
@@ -37,15 +37,11 @@ _gcry_cipher_cfb_encrypt (gcry_cipher_hd_t c,
{
unsigned char *ivp;
gcry_cipher_encrypt_t enc_fn = c->spec->encrypt;
- size_t blocksize = c->spec->blocksize;
+ size_t blocksize_shift = _gcry_blocksize_shift(c);
+ size_t blocksize = 1 << blocksize_shift;
size_t blocksize_x_2 = blocksize + blocksize;
unsigned int burn, nburn;
- /* Tell compiler that we require a cipher with a 64bit or 128 bit block
- * length, to allow better optimization of this function. */
- if (blocksize > 16 || blocksize < 8 || blocksize & (8 - 1))
- return GPG_ERR_INV_LENGTH;
-
if (outbuflen < inbuflen)
return GPG_ERR_BUFFER_TOO_SHORT;
@@ -77,11 +73,11 @@ _gcry_cipher_cfb_encrypt (gcry_cipher_hd_t c,
also allows to use a bulk encryption function if available. */
if (inbuflen >= blocksize_x_2 && c->bulk.cfb_enc)
{
- size_t nblocks = inbuflen / blocksize;
+ size_t nblocks = inbuflen >> blocksize_shift;
c->bulk.cfb_enc (&c->context.c, c->u_iv.iv, outbuf, inbuf, nblocks);
- outbuf += nblocks * blocksize;
- inbuf += nblocks * blocksize;
- inbuflen -= nblocks * blocksize;
+ outbuf += nblocks << blocksize_shift;
+ inbuf += nblocks << blocksize_shift;
+ inbuflen -= nblocks << blocksize_shift;
}
else
{
@@ -139,15 +135,11 @@ _gcry_cipher_cfb_decrypt (gcry_cipher_hd_t c,
{
unsigned char *ivp;
gcry_cipher_encrypt_t enc_fn = c->spec->encrypt;
- size_t blocksize = c->spec->blocksize;
+ size_t blocksize_shift = _gcry_blocksize_shift(c);
+ size_t blocksize = 1 << blocksize_shift;
size_t blocksize_x_2 = blocksize + blocksize;
unsigned int burn, nburn;
- /* Tell compiler that we require a cipher with a 64bit or 128 bit block
- * length, to allow better optimization of this function. */
- if (blocksize > 16 || blocksize < 8 || blocksize & (8 - 1))
- return GPG_ERR_INV_LENGTH;
-
if (outbuflen < inbuflen)
return GPG_ERR_BUFFER_TOO_SHORT;
@@ -179,11 +171,11 @@ _gcry_cipher_cfb_decrypt (gcry_cipher_hd_t c,
also allows to use a bulk encryption function if available. */
if (inbuflen >= blocksize_x_2 && c->bulk.cfb_dec)
{
- size_t nblocks = inbuflen / blocksize;
+ size_t nblocks = inbuflen >> blocksize_shift;
c->bulk.cfb_dec (&c->context.c, c->u_iv.iv, outbuf, inbuf, nblocks);
- outbuf += nblocks * blocksize;
- inbuf += nblocks * blocksize;
- inbuflen -= nblocks * blocksize;
+ outbuf += nblocks << blocksize_shift;
+ inbuf += nblocks << blocksize_shift;
+ inbuflen -= nblocks << blocksize_shift;
}
else
{
diff --git a/cipher/cipher-cmac.c b/cipher/cipher-cmac.c
index 30567b7fc..321ab9eab 100644
--- a/cipher/cipher-cmac.c
+++ b/cipher/cipher-cmac.c
@@ -38,7 +38,8 @@ _gcry_cmac_write (gcry_cipher_hd_t c, gcry_cmac_context_t *ctx,
const byte * inbuf, size_t inlen)
{
gcry_cipher_encrypt_t enc_fn = c->spec->encrypt;
- const unsigned int blocksize = c->spec->blocksize;
+ size_t blocksize_shift = _gcry_blocksize_shift(c);
+ size_t blocksize = 1 << blocksize_shift;
byte outbuf[MAX_BLOCKSIZE];
unsigned int burn = 0;
unsigned int nblocks;
@@ -46,11 +47,6 @@ _gcry_cmac_write (gcry_cipher_hd_t c, gcry_cmac_context_t *ctx,
if (ctx->tag)
return GPG_ERR_INV_STATE;
- /* Tell compiler that we require a cipher with a 64bit or 128 bit block
- * length, to allow better optimization of this function. */
- if (blocksize > 16 || blocksize < 8 || blocksize & (8 - 1))
- return GPG_ERR_INV_CIPHER_MODE;
-
if (!inbuf)
return GPG_ERR_INV_ARG;
@@ -78,12 +74,12 @@ _gcry_cmac_write (gcry_cipher_hd_t c, gcry_cmac_context_t *ctx,
if (c->bulk.cbc_enc && inlen > blocksize)
{
- nblocks = inlen / blocksize;
- nblocks -= (nblocks * blocksize == inlen);
+ nblocks = inlen >> blocksize_shift;
+ nblocks -= ((nblocks << blocksize_shift) == inlen);
c->bulk.cbc_enc (&c->context.c, ctx->u_iv.iv, outbuf, inbuf, nblocks, 1);
- inbuf += nblocks * blocksize;
- inlen -= nblocks * blocksize;
+ inbuf += nblocks << blocksize_shift;
+ inlen -= nblocks << blocksize_shift;
wipememory (outbuf, sizeof (outbuf));
}
diff --git a/cipher/cipher-ctr.c b/cipher/cipher-ctr.c
index f9cb6b577..b54fb5a73 100644
--- a/cipher/cipher-ctr.c
+++ b/cipher/cipher-ctr.c
@@ -38,15 +38,11 @@ _gcry_cipher_ctr_encrypt (gcry_cipher_hd_t c,
size_t n;
int i;
gcry_cipher_encrypt_t enc_fn = c->spec->encrypt;
- unsigned int blocksize = c->spec->blocksize;
+ size_t blocksize_shift = _gcry_blocksize_shift(c);
+ size_t blocksize = 1 << blocksize_shift;
size_t nblocks;
unsigned int burn, nburn;
- /* Tell compiler that we require a cipher with a 64bit or 128 bit block
- * length, to allow better optimization of this function. */
- if (blocksize > 16 || blocksize < 8 || blocksize & (8 - 1))
- return GPG_ERR_INV_LENGTH;
-
if (outbuflen < inbuflen)
return GPG_ERR_BUFFER_TOO_SHORT;
@@ -66,13 +62,13 @@ _gcry_cipher_ctr_encrypt (gcry_cipher_hd_t c,
}
/* Use a bulk method if available. */
- nblocks = inbuflen / blocksize;
+ nblocks = inbuflen >> blocksize_shift;
if (nblocks && c->bulk.ctr_enc)
{
c->bulk.ctr_enc (&c->context.c, c->u_ctr.ctr, outbuf, inbuf, nblocks);
- inbuf += nblocks * blocksize;
- outbuf += nblocks * blocksize;
- inbuflen -= nblocks * blocksize;
+ inbuf += nblocks << blocksize_shift;
+ outbuf += nblocks << blocksize_shift;
+ inbuflen -= nblocks << blocksize_shift;
}
/* If we don't have a bulk method use the standard method. We also
diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h
index a0ede5e03..a5bb0ad2f 100644
--- a/cipher/cipher-internal.h
+++ b/cipher/cipher-internal.h
@@ -563,4 +563,14 @@ ocb_get_l (gcry_cipher_hd_t c, u64 n)
return c->u_mode.ocb.L[ntz];
}
+
+/* Return bit-shift of blocksize. */
+static inline unsigned int _gcry_blocksize_shift(gcry_cipher_hd_t c)
+{
+ /* Only blocksizes 8 and 16 are used. Return value in such way
+ * that compiler can optimize calling functions based on this. */
+ return c->spec->blocksize == 8 ? 3 : 4;
+}
+
+
#endif /*G10_CIPHER_INTERNAL_H*/
diff --git a/cipher/cipher-ofb.c b/cipher/cipher-ofb.c
index f821d1bec..419a8d085 100644
--- a/cipher/cipher-ofb.c
+++ b/cipher/cipher-ofb.c
@@ -37,14 +37,10 @@ _gcry_cipher_ofb_encrypt (gcry_cipher_hd_t c,
{
unsigned char *ivp;
gcry_cipher_encrypt_t enc_fn = c->spec->encrypt;
- size_t blocksize = c->spec->blocksize;
+ size_t blocksize_shift = _gcry_blocksize_shift(c);
+ size_t blocksize = 1 << blocksize_shift;
unsigned int burn, nburn;
- /* Tell compiler that we require a cipher with a 64bit or 128 bit block
- * length, to allow better optimization of this function. */
- if (blocksize > 16 || blocksize < 8 || blocksize & (8 - 1))
- return GPG_ERR_INV_LENGTH;
-
if (outbuflen < inbuflen)
return GPG_ERR_BUFFER_TOO_SHORT;
--
2.17.1
More information about the Gcrypt-devel
mailing list