[PATCH 1/2] rijndael-riscv-zvkned: shorten CBC/CFB encryption dependency chain

[Jussi Kivilinna]

* cipher/rijndael-riscv-zvkned.c (_gcry_aes_riscv_zvkned_cfb_enc)
(_gcry_aes_riscv_zvkned_cbc_enc): Merge first round key to input data
and use AES_CRYPT_NO_RK0.
--

Previously first round key was applied with vaesz in serial dependency
chain of each block. Now it is merged to input data outside the chain.

Benchmark on SpacemiT K3, X100 core (out-of-order, VLEN=256):

 Before         |  AES128 cycles/byte  AES256 cycles/byte
        CBC enc |            1.63 c/B            2.13 c/B
        CFB enc |            1.63 c/B            2.13 c/B

 After          |  AES128 cycles/byte  AES256 cycles/byte
        CBC enc |            1.51 c/B            2.00 c/B
        CFB enc |            1.44 c/B            1.94 c/B
  CBC enc: AES128 ~8%, AES256 ~6% faster
  CFB enc: AES128 ~13%, AES256 ~10% faster

Benchmark on SpacemiT K3, A100 core (in-order, VLEN=1024):

 Before         |  AES128 cycles/byte  AES256 cycles/byte
        CBC enc |            5.81 c/B            7.81 c/B
        CFB enc |            6.01 c/B            8.01 c/B

 After          |  AES128 cycles/byte  AES256 cycles/byte
        CBC enc |            5.55 c/B            7.56 c/B
        CFB enc |            5.60 c/B            7.60 c/B
  CBC enc: AES128 ~5%, AES256 ~3% faster
  CFB enc: AES128 ~7%, AES256 ~5% faster

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/rijndael-riscv-zvkned.c | 38 ++++++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/cipher/rijndael-riscv-zvkned.c b/cipher/rijndael-riscv-zvkned.c
index 80338509..81c79f78 100644
--- a/cipher/rijndael-riscv-zvkned.c
+++ b/cipher/rijndael-riscv-zvkned.c
@@ -852,23 +852,35 @@ _gcry_aes_riscv_zvkned_cfb_enc (void *context, unsigned char *iv_arg,
   if (nblocks)
     {
       vuint32m1_t data = unaligned_load_u32m1(inbuf, vl);
+      vuint32m1_t next;
+
+      data = __riscv_vxor_vv_u32m1(data, rk0, vl);
+      iv = __riscv_vxor_vv_u32m1(iv, rk0, vl);
 
       while (--nblocks)
 	{
-	  AES_CRYPT(e, m1, GET_PRELOADED_KEY, rk, rounds, iv, vl);
+	  /* Load early and XOR late to hide load latency on in-order cores.  */
+	  inbuf += BLOCKSIZE;
+	  next = unaligned_load_u32m1(inbuf, vl);
+	  memory_barrier_with_vec(next);
 
-	  iv = __riscv_vxor_vv_u32m1(iv, data, vl);
+	  AES_CRYPT_NO_RK0(e, m1, GET_PRELOADED_KEY, rk, rounds, iv, vl);
 
-	  inbuf += BLOCKSIZE;
-	  data = unaligned_load_u32m1(inbuf, vl);
+	  iv = __riscv_vxor_vv_u32m1(iv, data, vl);
+	  memory_barrier_with_vec(iv);
+	  memory_barrier_with_vec(next);
+	  data = __riscv_vxor_vv_u32m1(next, rk0, vl);
+	  memory_barrier_with_vec(data);
+	  memory_barrier_with_vec(iv);
 
-	  unaligned_store_u32m1(outbuf, iv, vl);
+	  unaligned_store_u32m1(outbuf, __riscv_vxor_vv_u32m1(iv, rk0, vl), vl);
 	  outbuf += BLOCKSIZE;
 	}
 
-      AES_CRYPT(e, m1, GET_PRELOADED_KEY, rk, rounds, iv, vl);
+      AES_CRYPT_NO_RK0(e, m1, GET_PRELOADED_KEY, rk, rounds, iv, vl);
 
       iv = __riscv_vxor_vv_u32m1(iv, data, vl);
+      iv = __riscv_vxor_vv_u32m1(iv, rk0, vl);
 
       unaligned_store_u32m1(outbuf, iv, vl);
     }
@@ -902,23 +914,31 @@ _gcry_aes_riscv_zvkned_cbc_enc (void *context, unsigned char *iv_arg,
   if (nblocks)
     {
       vuint32m1_t data = unaligned_load_u32m1(inbuf, vl);
+      vuint32m1_t next;
+
+      data = __riscv_vxor_vv_u32m1(data, rk0, vl);
 
       while (--nblocks)
 	{
 	  iv = __riscv_vxor_vv_u32m1(data, iv, vl);
 
+	  /* Load early and XOR late to hide load latency on in-order cores.  */
 	  inbuf += BLOCKSIZE;
-	  data = unaligned_load_u32m1(inbuf, vl);
+	  next = unaligned_load_u32m1(inbuf, vl);
+	  memory_barrier_with_vec(next);
 
-	  AES_CRYPT(e, m1, GET_PRELOADED_KEY, rk, rounds, iv, vl);
+	  AES_CRYPT_NO_RK0(e, m1, GET_PRELOADED_KEY, rk, rounds, iv, vl);
 
 	  unaligned_store_u32m1(outbuf, iv, vl);
 	  outbuf += outbuf_add;
+
+	  memory_barrier_with_vec(next);
+	  data = __riscv_vxor_vv_u32m1(next, rk0, vl);
 	}
 
       iv = __riscv_vxor_vv_u32m1(data, iv, vl);
 
-      AES_CRYPT(e, m1, GET_PRELOADED_KEY, rk, rounds, iv, vl);
+      AES_CRYPT_NO_RK0(e, m1, GET_PRELOADED_KEY, rk, rounds, iv, vl);
 
       unaligned_store_u32m1(outbuf, iv, vl);
     }
-- 
2.53.0