[PATCH 2/2] rijndael-riscv-zvkned: use Zvbc for XTS tweak multiplication
Jussi Kivilinna
jussi.kivilinna at iki.fi
Sat Oct 17 17:58:51 CEST 2026
* cipher/Makefile.am (riscv_vector_crypto_aes_cflags): Add Zvbc.
* cipher/rijndael-riscv-zvkned.c (DEFINE_XTS_GFMUL_POW): Add 'mulc' and
'use_zvbc' parameters; use vclmul for reduction if 'use_zvbc'.
(aes_riscv_xts_crypt): Add 'use_zvbc' parameter; prepare 'mulc'
vectors.
(_gcry_aes_riscv_zvkned_xts_crypt): Pass zero 'use_zvbc'.
(_gcry_aes_riscv_zvkned_zvbc_xts_crypt): New.
* cipher/rijndael.c (_gcry_aes_riscv_zvkned_zvbc_xts_crypt): New.
(do_setkey) [USE_RISCV_V_CRYPTO]: Use Zvbc XTS implementation if
supported by HW.
* configure.ac (GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST): Add vclmul
intrinsics test.
* doc/gcrypt.texi: Add "riscv-zvbc".
* src/g10lib.h [HAVE_CPU_ARCH_RISCV] (HWF_RISCV_ZVBC): New.
(HWF_RISCV_ZVKB, HWF_RISCV_ZVKG, HWF_RISCV_ZVKNED, HWF_RISCV_ZVKNHA)
(HWF_RISCV_ZVKNHB, HWF_RISCV_ZVKSED, HWF_RISCV_ZVKSH): Move to next bit.
* src/hwf-riscv.c (HWF_RISCV_HWPROBE_EXT_ZVBC): New.
(hwprobe_features): Add Zvbc.
* src/hwfeatures.c (hwflist) [HAVE_CPU_ARCH_RISCV]: Add "riscv-zvbc".
--
Previously 0x87 reduction in XTS tweak multiplication was done with
shifts and masked XOR. With Zvbc, this is single vclmul. Zvbc is not
part of Zvkng, so separate XTS function is used when Zvbc is available.
Benchmark on SpacemiT K3, X100 core (out-of-order, VLEN=256):
Before | AES128 cycles/byte AES256 cycles/byte
XTS enc | 1.23 c/B 1.48 c/B
XTS dec | 1.23 c/B 1.48 c/B
After | AES128 cycles/byte AES256 cycles/byte
XTS enc | 1.07 c/B 1.32 c/B
XTS dec | 1.07 c/B 1.32 c/B
XTS: AES128 ~15%, AES256 ~12% faster
Benchmark on SpacemiT K3, A100 core (in-order, VLEN=1024):
Before | AES128 cycles/byte AES256 cycles/byte
XTS enc | 1.10 c/B 1.35 c/B
XTS dec | 1.11 c/B 1.37 c/B
After | AES128 cycles/byte AES256 cycles/byte
XTS enc | 1.08 c/B 1.35 c/B
XTS dec | 1.10 c/B 1.36 c/B
XTS: no change
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/Makefile.am | 2 +-
cipher/rijndael-riscv-zvkned.c | 86 +++++++++++++++++++++++++---------
cipher/rijndael.c | 6 +++
configure.ac | 9 ++++
doc/gcrypt.texi | 1 +
src/g10lib.h | 15 +++---
src/hwf-riscv.c | 2 +
src/hwfeatures.c | 1 +
8 files changed, 91 insertions(+), 31 deletions(-)
diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index d2007815..0b94c23f 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -495,7 +495,7 @@ rijndael-vp-riscv-mf8.lo: $(srcdir)/rijndael-vp-riscv-mf8.c Makefile
if ENABLE_RISCV_VECTOR_CRYPTO_INTRINSICS_EXTRA_CFLAGS
riscv_vector_crypto_zvkb_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkb
-riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkned_zvkb
+riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkned_zvkb_zvbc
riscv_vector_crypto_sha256_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknha_zvkb
riscv_vector_crypto_sha512_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknhb_zvkb
riscv_vector_crypto_sha512_m1_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknhb_zvkb_zvl256b
diff --git a/cipher/rijndael-riscv-zvkned.c b/cipher/rijndael-riscv-zvkned.c
index 81c79f78..f38998a4 100644
--- a/cipher/rijndael-riscv-zvkned.c
+++ b/cipher/rijndael-riscv-zvkned.c
@@ -1968,8 +1968,9 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg,
/* Multiply each 128-bit lane pair by alpha^S. */
#define DEFINE_XTS_GFMUL_POW(MX, BOOL) \
static ASM_FUNC_ATTR_INLINE vuint64##MX##_t \
- xts_gfmul_pow1_##MX (vuint64##MX##_t vec_in, vbool##BOOL##_t even, size_t s, \
- size_t vl) \
+ xts_gfmul_pow1_##MX (vuint64##MX##_t vec_in, vbool##BOOL##_t even, \
+ vuint64##MX##_t mulc, size_t s, size_t vl, \
+ int use_zvbc) \
{ \
vuint64##MX##_t hi = __riscv_vsrl_vx_u64##MX(vec_in, 64 - s, vl); \
vuint64##MX##_t lo = __riscv_vsll_vx_u64##MX(vec_in, s, vl); \
@@ -1977,28 +1978,38 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg,
vuint64##MX##_t acc; \
\
sw = __riscv_vslidedown_vx_u64##MX##_mu(even, sw, hi, 1, vl); \
- acc = __riscv_vsll_vx_u64##MX(sw, 5, vl); \
- acc = __riscv_vxor_vv_u64##MX(acc, sw, vl); \
- acc = __riscv_vsll_vx_u64##MX(acc, 1, vl); \
- acc = __riscv_vxor_vv_u64##MX(acc, sw, vl); \
- acc = __riscv_vsll_vx_u64##MX(acc, 1, vl); \
- sw = __riscv_vxor_vv_u64##MX##_mu(even, sw, sw, acc, vl); \
+ if (use_zvbc) \
+ { \
+ sw = __riscv_vclmul_vv_u64##MX(sw, mulc, vl); \
+ } \
+ else \
+ { \
+ acc = __riscv_vsll_vx_u64##MX(sw, 5, vl); \
+ acc = __riscv_vxor_vv_u64##MX(acc, sw, vl); \
+ acc = __riscv_vsll_vx_u64##MX(acc, 1, vl); \
+ acc = __riscv_vxor_vv_u64##MX(acc, sw, vl); \
+ acc = __riscv_vsll_vx_u64##MX(acc, 1, vl); \
+ sw = __riscv_vxor_vv_u64##MX##_mu(even, sw, sw, acc, vl); \
+ } \
return __riscv_vxor_vv_u64##MX(lo, sw, vl); \
} \
\
static ASM_FUNC_ATTR_INLINE vuint32##MX##_t \
- xts_gfmul_pow_##MX (vuint32##MX##_t vec_in, vbool##BOOL##_t even, size_t s, \
- size_t vl_u32) \
+ xts_gfmul_pow_##MX (vuint32##MX##_t vec_in, vbool##BOOL##_t even, \
+ vuint64##MX##_t mulc, size_t s, size_t vl_u32, \
+ int use_zvbc) \
{ \
vuint64##MX##_t vec = cast_u32##MX##_u64##MX(vec_in); \
size_t vl = vl_u32 / 2; \
\
while (UNLIKELY(s > XTS_GFMUL_POW_MAX_SHIFT)) \
{ \
- vec = xts_gfmul_pow1_##MX(vec, even, XTS_GFMUL_POW_MAX_SHIFT, vl); \
+ vec = xts_gfmul_pow1_##MX(vec, even, mulc, XTS_GFMUL_POW_MAX_SHIFT, \
+ vl, use_zvbc); \
s -= XTS_GFMUL_POW_MAX_SHIFT; \
} \
- return cast_u64##MX##_u32##MX(xts_gfmul_pow1_##MX(vec, even, s, vl)); \
+ return cast_u64##MX##_u32##MX(xts_gfmul_pow1_##MX(vec, even, mulc, s, vl, \
+ use_zvbc)); \
}
DEFINE_XTS_GFMUL_POW(m1, 64)
@@ -2007,7 +2018,8 @@ DEFINE_XTS_GFMUL_POW(m4, 16)
static ASM_FUNC_ATTR_INLINE FUNC_ATTR_OPT_O2 void
aes_riscv_xts_crypt (void *context, unsigned char *tweak_arg, void *outbuf_arg,
- const void *inbuf_arg, size_t nblocks, int encrypt)
+ const void *inbuf_arg, size_t nblocks, int encrypt,
+ int use_zvbc)
{
RIJNDAEL_context *ctx = context;
unsigned char *outbuf = outbuf_arg;
@@ -2021,6 +2033,7 @@ aes_riscv_xts_crypt (void *context, unsigned char *tweak_arg, void *outbuf_arg,
size_t vl64_m1 = max_blocks_m1 * 2;
size_t vl_m1 = max_blocks_m1 * 4;
vbool64_t even_m1;
+ vuint64m1_t mulc_m1;
vuint32m1_t tweak;
size_t k;
ROUND_KEY_VARIABLES;
@@ -2041,10 +2054,15 @@ aes_riscv_xts_crypt (void *context, unsigned char *tweak_arg, void *outbuf_arg,
__riscv_vand_vx_u64m1(__riscv_vid_v_u64m1(vl64_m1), 1, vl64_m1), 0,
vl64_m1);
+ /* Zvbc multiplier: 0x87 reduces low halves, 1 keeps high halves. */
+ mulc_m1 = __riscv_vmerge_vxm_u64m1(__riscv_vmv_v_x_u64m1(1, vl64_m1), 0x87,
+ even_m1, vl64_m1);
+
/* Double the tweak run until m1 is full. */
for (k = 1; k < max_blocks_m1; k *= 2)
{
- vuint32m1_t next = xts_gfmul_pow_m1(tweak, even_m1, k, k * blk_vl32);
+ vuint32m1_t next = xts_gfmul_pow_m1(tweak, even_m1, mulc_m1, k,
+ k * blk_vl32, use_zvbc);
tweak = __riscv_vslideup_vx_u32m1(tweak, next, k * blk_vl32,
2 * k * blk_vl32);
@@ -2057,13 +2075,16 @@ aes_riscv_xts_crypt (void *context, unsigned char *tweak_arg, void *outbuf_arg,
vbool32_t even_m2 = __riscv_vmseq_vx_u64m2_b32(
__riscv_vand_vx_u64m2(__riscv_vid_v_u64m2(vl64_m2), 1, vl64_m2), 0,
vl64_m2);
+ vuint64m2_t mulc_m2 = __riscv_vmerge_vxm_u64m2(
+ __riscv_vmv_v_x_u64m2(1, vl64_m2), 0x87, even_m2, vl64_m2);
vuint32m2_t tweaks_m2;
/* Continue doubling from m1 to m2. */
tweaks_m2 = __riscv_vset_v_u32m1_u32m2(__riscv_vundefined_u32m2(), 0,
tweak);
tweaks_m2 = __riscv_vset_v_u32m1_u32m2(tweaks_m2, 1,
- xts_gfmul_pow_m1(tweak, even_m1, max_blocks_m1, vl_m1));
+ xts_gfmul_pow_m1(tweak, even_m1, mulc_m1, max_blocks_m1, vl_m1,
+ use_zvbc));
if (nblocks >= max_blocks_m4)
{
@@ -2072,13 +2093,16 @@ aes_riscv_xts_crypt (void *context, unsigned char *tweak_arg, void *outbuf_arg,
vbool16_t even_m4 = __riscv_vmseq_vx_u64m4_b16(
__riscv_vand_vx_u64m4(__riscv_vid_v_u64m4(vl64_m4), 1, vl64_m4),
0, vl64_m4);
+ vuint64m4_t mulc_m4 = __riscv_vmerge_vxm_u64m4(
+ __riscv_vmv_v_x_u64m4(1, vl64_m4), 0x87, even_m4, vl64_m4);
vuint32m4_t tweaks_m4;
/* Continue doubling from m2 to m4. */
tweaks_m4 = __riscv_vset_v_u32m2_u32m4(
__riscv_vundefined_u32m4(), 0, tweaks_m2);
tweaks_m4 = __riscv_vset_v_u32m2_u32m4(tweaks_m4, 1,
- xts_gfmul_pow_m2(tweaks_m2, even_m2, max_blocks_m2, vl_m2));
+ xts_gfmul_pow_m2(tweaks_m2, even_m2, mulc_m2, max_blocks_m2,
+ vl_m2, use_zvbc));
do
{
@@ -2095,8 +2119,8 @@ aes_riscv_xts_crypt (void *context, unsigned char *tweak_arg, void *outbuf_arg,
unaligned_store_u32m4(outbuf, data_blks, vl_m4);
- tweaks_m4 = xts_gfmul_pow_m4(tweaks_m4, even_m4, max_blocks_m4,
- vl_m4);
+ tweaks_m4 = xts_gfmul_pow_m4(tweaks_m4, even_m4, mulc_m4,
+ max_blocks_m4, vl_m4, use_zvbc);
inbuf += max_blocks_m4 * BLOCKSIZE;
outbuf += max_blocks_m4 * BLOCKSIZE;
@@ -2125,8 +2149,8 @@ aes_riscv_xts_crypt (void *context, unsigned char *tweak_arg, void *outbuf_arg,
unaligned_store_u32m2(outbuf, data_blks, vl_m2);
- tweaks_m2 = xts_gfmul_pow_m2(tweaks_m2, even_m2, max_blocks_m2,
- vl_m2);
+ tweaks_m2 = xts_gfmul_pow_m2(tweaks_m2, even_m2, mulc_m2,
+ max_blocks_m2, vl_m2, use_zvbc);
inbuf += max_blocks_m2 * BLOCKSIZE;
outbuf += max_blocks_m2 * BLOCKSIZE;
@@ -2158,7 +2182,8 @@ aes_riscv_xts_crypt (void *context, unsigned char *tweak_arg, void *outbuf_arg,
unaligned_store_u32m1(outbuf, data_blks, vl_m1);
- tweak = xts_gfmul_pow_m1(tweak, even_m1, curr_nblks, vl_m1);
+ tweak = xts_gfmul_pow_m1(tweak, even_m1, mulc_m1, curr_nblks, vl_m1,
+ use_zvbc);
inbuf += curr_nblks * BLOCKSIZE;
outbuf += curr_nblks * BLOCKSIZE;
@@ -2176,9 +2201,24 @@ _gcry_aes_riscv_zvkned_xts_crypt (void *context, unsigned char *tweak_arg,
size_t nblocks, int encrypt)
{
if (encrypt)
- aes_riscv_xts_crypt(context, tweak_arg, outbuf_arg, inbuf_arg, nblocks, 1);
+ aes_riscv_xts_crypt(context, tweak_arg, outbuf_arg, inbuf_arg, nblocks, 1,
+ 0);
+ else
+ aes_riscv_xts_crypt(context, tweak_arg, outbuf_arg, inbuf_arg, nblocks, 0,
+ 0);
+}
+
+ASM_FUNC_ATTR_NOINLINE FUNC_ATTR_OPT_O2 void
+_gcry_aes_riscv_zvkned_zvbc_xts_crypt (void *context, unsigned char *tweak_arg,
+ void *outbuf_arg, const void *inbuf_arg,
+ size_t nblocks, int encrypt)
+{
+ if (encrypt)
+ aes_riscv_xts_crypt(context, tweak_arg, outbuf_arg, inbuf_arg, nblocks, 1,
+ 1);
else
- aes_riscv_xts_crypt(context, tweak_arg, outbuf_arg, inbuf_arg, nblocks, 0);
+ aes_riscv_xts_crypt(context, tweak_arg, outbuf_arg, inbuf_arg, nblocks, 0,
+ 1);
}
#endif /* HAVE_COMPATIBLE_CC_RISCV_VECTOR_INTRINSICS */
diff --git a/cipher/rijndael.c b/cipher/rijndael.c
index 0586bfd2..b99d520d 100644
--- a/cipher/rijndael.c
+++ b/cipher/rijndael.c
@@ -287,6 +287,10 @@ extern void
_gcry_aes_riscv_zvkned_xts_crypt (void *context, unsigned char *tweak,
void *outbuf_arg, const void *inbuf_arg,
size_t nblocks, int encrypt);
+extern void
+_gcry_aes_riscv_zvkned_zvbc_xts_crypt (void *context, unsigned char *tweak,
+ void *outbuf_arg, const void *inbuf_arg,
+ size_t nblocks, int encrypt);
#endif
#ifdef USE_PADLOCK
@@ -816,6 +820,8 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->ocb_auth = _gcry_aes_riscv_zvkned_ocb_auth;
bulk_ops->ecb_crypt = _gcry_aes_riscv_zvkned_ecb_crypt;
bulk_ops->xts_crypt = _gcry_aes_riscv_zvkned_xts_crypt;
+ if (hwfeatures & HWF_RISCV_ZVBC)
+ bulk_ops->xts_crypt = _gcry_aes_riscv_zvkned_zvbc_xts_crypt;
}
#endif
#ifdef USE_VP_RISCV
diff --git a/configure.ac b/configure.ac
index 9ef73ee4..e5c85010 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3042,6 +3042,14 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST],
vuint32m1_t d = __riscv_vghsh_vv_u32m1(a, b, c, vl);
__riscv_vse32_v_u32m1(ptr + 0 * vl, d, vl);
}
+ void test_clmul(unsigned int *ptr)
+ {
+ int vl = __riscv_vsetvl_e64m1 (2);
+ vuint64m1_t a = __riscv_vle64_v_u64m1((void *)(ptr + 0 * vl), vl);
+ vuint64m1_t b = __riscv_vle64_v_u64m1((void *)(ptr + 2 * vl), vl);
+ vuint64m1_t c = __riscv_vclmul_vv_u64m1(a, b, vl);
+ __riscv_vse64_v_u64m1((void *)(ptr + 0 * vl), c, vl);
+ }
void test_sm3(unsigned int *ptr)
{
int vl = __riscv_vsetvl_e32m2 (8);
@@ -3105,6 +3113,7 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST],
test_aes_key(buf);
test_aes_crypt(buf);
test_ghash(buf);
+ test_clmul(buf);
test_sm3(buf);
test_sm4(buf);
test_rev8(buf);
diff --git a/doc/gcrypt.texi b/doc/gcrypt.texi
index 18a6f24c..9ddf09e6 100644
--- a/doc/gcrypt.texi
+++ b/doc/gcrypt.texi
@@ -618,6 +618,7 @@ are
@item riscv-v
@item riscv-zbb
@item riscv-zbc
+ at item riscv-zvbc
@item riscv-zvkb
@item riscv-zvkg
@item riscv-zvkned
diff --git a/src/g10lib.h b/src/g10lib.h
index a9c18db6..9584e84d 100644
--- a/src/g10lib.h
+++ b/src/g10lib.h
@@ -283,13 +283,14 @@ char **_gcry_strtokenize (const char *string, const char *delim);
#define HWF_RISCV_V (1 << 2)
#define HWF_RISCV_ZBB (1 << 3)
#define HWF_RISCV_ZBC (1 << 4)
-#define HWF_RISCV_ZVKB (1 << 5)
-#define HWF_RISCV_ZVKG (1 << 6)
-#define HWF_RISCV_ZVKNED (1 << 7)
-#define HWF_RISCV_ZVKNHA (1 << 8)
-#define HWF_RISCV_ZVKNHB (1 << 9)
-#define HWF_RISCV_ZVKSED (1 << 10)
-#define HWF_RISCV_ZVKSH (1 << 11)
+#define HWF_RISCV_ZVBC (1 << 5)
+#define HWF_RISCV_ZVKB (1 << 6)
+#define HWF_RISCV_ZVKG (1 << 7)
+#define HWF_RISCV_ZVKNED (1 << 8)
+#define HWF_RISCV_ZVKNHA (1 << 9)
+#define HWF_RISCV_ZVKNHB (1 << 10)
+#define HWF_RISCV_ZVKSED (1 << 11)
+#define HWF_RISCV_ZVKSH (1 << 12)
#endif
diff --git a/src/hwf-riscv.c b/src/hwf-riscv.c
index 192ab031..505a4e4d 100644
--- a/src/hwf-riscv.c
+++ b/src/hwf-riscv.c
@@ -192,6 +192,7 @@ detect_riscv_at_hwcap(void)
#define HWF_RISCV_HWPROBE_EXT_ZBS (1U << 5)
#define HWF_RISCV_HWPROBE_EXT_ZBC (1U << 7)
#define HWF_RISCV_HWPROBE_EXT_ZVBB (1U << 17)
+#define HWF_RISCV_HWPROBE_EXT_ZVBC (1U << 18)
#define HWF_RISCV_HWPROBE_EXT_ZVKB (1U << 19)
#define HWF_RISCV_HWPROBE_EXT_ZVKG (1U << 20)
#define HWF_RISCV_HWPROBE_EXT_ZVKNED (1U << 21)
@@ -223,6 +224,7 @@ static const struct hwprobe_feature_map_s hwprobe_features[] =
{ HWF_RISCV_HWPROBE_EXT_ZBA
| HWF_RISCV_HWPROBE_EXT_ZBB
| HWF_RISCV_HWPROBE_EXT_ZBS, HWF_RISCV_B },
+ { HWF_RISCV_HWPROBE_EXT_ZVBC, HWF_RISCV_ZVBC },
{ HWF_RISCV_HWPROBE_EXT_ZVKB, HWF_RISCV_ZVKB },
{ HWF_RISCV_HWPROBE_EXT_ZVBB, HWF_RISCV_ZVKB }, /* Zvkb is subset of Zvbb */
{ HWF_RISCV_HWPROBE_EXT_ZVKG, HWF_RISCV_ZVKG },
diff --git a/src/hwfeatures.c b/src/hwfeatures.c
index 3067ee86..c748948a 100644
--- a/src/hwfeatures.c
+++ b/src/hwfeatures.c
@@ -122,6 +122,7 @@ static struct
{ HWF_RISCV_V, "riscv-v" },
{ HWF_RISCV_ZBB, "riscv-zbb" },
{ HWF_RISCV_ZBC, "riscv-zbc" },
+ { HWF_RISCV_ZVBC, "riscv-zvbc" },
{ HWF_RISCV_ZVKB, "riscv-zvkb" },
{ HWF_RISCV_ZVKG, "riscv-zvkg" },
{ HWF_RISCV_ZVKNED, "riscv-zvkned" },
--
2.53.0
More information about the Gcrypt-devel
mailing list