[PATCH 11/12] cipher-gcm-riscv-zvkg: use fractional LMUL for POLYVAL byte reversal

[Jussi Kivilinna]

* cipher/cipher-gcm-riscv-zvkg.c (cast_u32m1_u8m1, bswap128_u32m1):
Remove.
(cast_u8mf2_u32m1, cast_u8mf4_u32m1, cast_u8mf8_u32m1, bswap128_arr)
(POLYVAL_LOOP): New.
(_gcry_polyval_riscv_zvkg): Use POLYVAL_LOOP with LMUL selected by
VLEN.
--

Byte reversal of POLYVAL input blocks was done with vrgather at LMUL=1,
which is slow with VLEN larger than 128 bits. Now vrgather is done with
LMUL that gives 128-bit register group, selected by VLEN.

Benchmark on SpacemiT K3, X100 core (out-of-order, VLEN=256):

 Before:
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
    GCM-SIV enc |     0.586 ns/B      1627 MiB/s      1.17 c/B
    GCM-SIV dec |     0.634 ns/B      1504 MiB/s      1.27 c/B
   GCM-SIV auth |     0.224 ns/B      4249 MiB/s     0.449 c/B

 After (GCM-SIV auth ~1.7x faster):
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
    GCM-SIV enc |     0.494 ns/B      1929 MiB/s     0.989 c/B
    GCM-SIV dec |     0.537 ns/B      1777 MiB/s      1.07 c/B
   GCM-SIV auth |     0.129 ns/B      7418 MiB/s     0.257 c/B

Benchmark on SpacemiT K3, A100 core (in-order, VLEN=1024):

 Before:
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
    GCM-SIV enc |      3.71 ns/B     256.7 MiB/s      6.69 c/B
    GCM-SIV dec |      3.92 ns/B     243.5 MiB/s      7.05 c/B
   GCM-SIV auth |      3.44 ns/B     276.9 MiB/s      6.20 c/B

 After (GCM-SIV auth ~3.0x faster):
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
    GCM-SIV enc |      1.53 ns/B     622.4 MiB/s      2.76 c/B
    GCM-SIV dec |      1.54 ns/B     617.3 MiB/s      2.78 c/B
   GCM-SIV auth |      1.13 ns/B     842.8 MiB/s      2.04 c/B

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/cipher-gcm-riscv-zvkg.c | 57 ++++++++++++++++++++--------------
 1 file changed, 34 insertions(+), 23 deletions(-)

diff --git a/cipher/cipher-gcm-riscv-zvkg.c b/cipher/cipher-gcm-riscv-zvkg.c
index 1f0b3797..76be974f 100644
--- a/cipher/cipher-gcm-riscv-zvkg.c
+++ b/cipher/cipher-gcm-riscv-zvkg.c
@@ -38,7 +38,9 @@
 
 
 #define cast_u8m1_u32m1(a) __riscv_vreinterpret_v_u8m1_u32m1(a)
-#define cast_u32m1_u8m1(a) __riscv_vreinterpret_v_u32m1_u8m1(a)
+#define cast_u8mf2_u32m1(a) cast_u8m1_u32m1(__riscv_vlmul_ext_v_u8mf2_u8m1(a))
+#define cast_u8mf4_u32m1(a) cast_u8m1_u32m1(__riscv_vlmul_ext_v_u8mf4_u8m1(a))
+#define cast_u8mf8_u32m1(a) cast_u8m1_u32m1(__riscv_vlmul_ext_v_u8mf8_u8m1(a))
 
 
 static ASM_FUNC_ATTR_INLINE vuint32m1_t
@@ -52,19 +54,6 @@ unaligned_load_u32m1(const void *ptr, size_t vl_u32)
 #endif
 }
 
-static ASM_FUNC_ATTR_INLINE vuint32m1_t
-bswap128_u32m1(vuint32m1_t vec, size_t vl_u32)
-{
-  static const byte bswap128_arr[16] =
-    { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
-  size_t vl_bytes = vl_u32 * 4;
-  vuint8m1_t bswap128 = __riscv_vle8_v_u8m1(bswap128_arr, vl_bytes);
-
-  return cast_u8m1_u32m1(
-	    __riscv_vrgather_vv_u8m1(cast_u32m1_u8m1(vec), bswap128, vl_bytes));
-}
-
-
 ASM_FUNC_ATTR_NOINLINE int
 _gcry_ghash_setup_riscv_zvkg(gcry_cipher_hd_t c)
 {
@@ -104,24 +93,46 @@ _gcry_ghash_riscv_zvkg(gcry_cipher_hd_t c, byte *result, const byte *buf,
   return 0;
 }
 
+static const byte bswap128_arr[16] =
+  { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
+
+#define POLYVAL_LOOP(lmul) \
+  do \
+    { \
+      vuint8##lmul##_t bswap128 = __riscv_vle8_v_u8##lmul(bswap128_arr, 16); \
+      \
+      while (nblocks) \
+	{ \
+	  vuint8##lmul##_t data = __riscv_vle8_v_u8##lmul(buf, 16); \
+	  buf += 16; \
+	  nblocks--; \
+	  \
+	  data = __riscv_vrgather_vv_u8##lmul(data, bswap128, 16); \
+	  rhash = __riscv_vghsh_vv_u32m1(rhash, rh1, \
+					 cast_u8##lmul##_u32m1(data), vl); \
+	} \
+    } \
+  while (0)
+
 ASM_FUNC_ATTR_NOINLINE unsigned int
 _gcry_polyval_riscv_zvkg(gcry_cipher_hd_t c, byte *result, const byte *buf,
-		       size_t nblocks)
+			 size_t nblocks)
 {
   u32 *result_u32 = (void *)result;
   const u32 *key_u32 = (void *)c->u_mode.gcm.u_ghash_key.key;
   size_t vl = 4;
+  size_t vlenb = __riscv_vsetvlmax_e8m1();
   vuint32m1_t rhash = __riscv_vle32_v_u32m1(result_u32, vl);
   vuint32m1_t rh1 = __riscv_vle32_v_u32m1(key_u32, vl);
 
-  while (nblocks)
-    {
-      vuint32m1_t data = bswap128_u32m1(unaligned_load_u32m1(buf, vl), vl);
-      buf += 16;
-      nblocks--;
-
-      rhash = __riscv_vghsh_vv_u32m1(rhash, rh1, data, vl);
-    }
+  if (vlenb >= 128)
+    POLYVAL_LOOP(mf8);
+  else if (vlenb >= 64)
+    POLYVAL_LOOP(mf4);
+  else if (vlenb >= 32)
+    POLYVAL_LOOP(mf2);
+  else
+    POLYVAL_LOOP(m1);
 
   __riscv_vse32_v_u32m1(result_u32, rhash, vl);
 
-- 
2.53.0