[PATCH 3/8] camellia: accelerate ECB (for benchmarking)

[Jussi Kivilinna]

* cipher/bulkhelp.h (bulk_ecb_crypt_128): New.
* cipher/camellia-glue.c (_gcry_camellia_ecb_crypt): New.
(camellia_setkey): Select ECB bulk function with AESNI/AVX2, VAES/AVX2
and GFNI/AVX2.
--

Benchmark on AMD Ryzen 9 7900X:

Before:
 CAMELLIA128    |  nanosecs/byte   mebibytes/sec   cycles/byte  auto Mhz
        ECB enc |      3.27 ns/B     291.8 MiB/s     18.38 c/B      5625
        ECB dec |      3.25 ns/B     293.3 MiB/s     18.29 c/B      5625

After (OCB for reference):
 CAMELLIA128    |  nanosecs/byte   mebibytes/sec   cycles/byte  auto Mhz
        ECB enc |     0.146 ns/B      6533 MiB/s     0.803 c/B      5500
        ECB dec |     0.149 ns/B      6384 MiB/s     0.822 c/B      5500
        OCB enc |     0.170 ns/B      5608 MiB/s     0.957 c/B      5625
        OCB dec |     0.175 ns/B      5452 MiB/s     0.984 c/B      5625

GnuPG-bug-id: T6242
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/bulkhelp.h      | 19 +++++++++++++++++++
 cipher/camellia-glue.c | 38 ++++++++++++++++++++++++++++++++++----
 2 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/cipher/bulkhelp.h b/cipher/bulkhelp.h
index 444973ab..b86abc27 100644
--- a/cipher/bulkhelp.h
+++ b/cipher/bulkhelp.h
@@ -470,5 +470,24 @@ bulk_xts_crypt_128 (void *priv, bulk_crypt_fn_t crypt_fn, byte *outbuf,
   return burn_depth;
 }
 
+static inline unsigned int
+bulk_ecb_crypt_128 (void *priv, bulk_crypt_fn_t crypt_fn, byte *outbuf,
+		    const byte *inbuf, size_t nblocks, size_t fn_max_nblocks)
+{
+  unsigned int burn_depth = 0;
+  unsigned int nburn;
+
+  while (nblocks >= 1)
+    {
+      size_t curr_blks = nblocks > fn_max_nblocks ? fn_max_nblocks : nblocks;
+      nburn = crypt_fn (priv, outbuf, inbuf, curr_blks);
+      burn_depth = nburn > burn_depth ? nburn : burn_depth;
+      inbuf += curr_blks * 16;
+      outbuf += curr_blks * 16;
+      nblocks -= curr_blks;
+    }
+
+  return burn_depth;
+}
 
 #endif /*GCRYPT_BULKHELP_H*/
diff --git a/cipher/camellia-glue.c b/cipher/camellia-glue.c
index b2a50233..a81d586a 100644
--- a/cipher/camellia-glue.c
+++ b/cipher/camellia-glue.c
@@ -405,11 +405,14 @@ static void _gcry_camellia_cfb_dec (void *context, unsigned char *iv,
 				    void *outbuf_arg, const void *inbuf_arg,
 				    size_t nblocks);
 static void _gcry_camellia_xts_crypt (void *context, unsigned char *tweak,
-                                      void *outbuf_arg, const void *inbuf_arg,
-                                      size_t nblocks, int encrypt);
+				      void *outbuf_arg, const void *inbuf_arg,
+				      size_t nblocks, int encrypt);
+static void _gcry_camellia_ecb_crypt (void *context, void *outbuf_arg,
+				      const void *inbuf_arg, size_t nblocks,
+				      int encrypt);
 static void _gcry_camellia_ctr32le_enc (void *context, unsigned char *ctr,
-                                        void *outbuf_arg, const void *inbuf_arg,
-                                        size_t nblocks);
+					void *outbuf_arg, const void *inbuf_arg,
+					size_t nblocks);
 static size_t _gcry_camellia_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
 					const void *inbuf_arg, size_t nblocks,
 					int encrypt);
@@ -474,10 +477,12 @@ camellia_setkey(void *c, const byte *key, unsigned keylen,
   if (ctx->use_aesni_avx2 || ctx->use_vaes_avx2 || ctx->use_gfni_avx2)
     {
       bulk_ops->xts_crypt = _gcry_camellia_xts_crypt;
+      bulk_ops->ecb_crypt = _gcry_camellia_ecb_crypt;
       bulk_ops->ctr32le_enc = _gcry_camellia_ctr32le_enc;
     }
 #else
   (void)_gcry_camellia_xts_crypt;
+  (void)_gcry_camellia_ecb_crypt;
   (void)_gcry_camellia_ctr32le_enc;
 #endif
 
@@ -1126,6 +1131,31 @@ _gcry_camellia_cfb_dec(void *context, unsigned char *iv,
     _gcry_burn_stack(burn_stack_depth);
 }
 
+/* Bulk encryption/decryption in ECB mode. */
+static void
+_gcry_camellia_ecb_crypt (void *context, void *outbuf_arg,
+			  const void *inbuf_arg, size_t nblocks, int encrypt)
+{
+  CAMELLIA_context *ctx = context;
+  unsigned char *outbuf = outbuf_arg;
+  const unsigned char *inbuf = inbuf_arg;
+  int burn_stack_depth = 0;
+
+  /* Process remaining blocks. */
+  if (nblocks)
+    {
+      size_t nburn;
+
+      nburn = bulk_ecb_crypt_128(ctx, encrypt ? camellia_encrypt_blk1_64
+                                              : camellia_decrypt_blk1_64,
+                                 outbuf, inbuf, nblocks, 64);
+      burn_stack_depth = nburn > burn_stack_depth ? nburn : burn_stack_depth;
+    }
+
+  if (burn_stack_depth)
+    _gcry_burn_stack(burn_stack_depth);
+}
+
 /* Bulk encryption/decryption of complete blocks in XTS mode. */
 static void
 _gcry_camellia_xts_crypt (void *context, unsigned char *tweak,
-- 
2.37.2