[PATCH 06/12] sha512-riscv-zvknhb-zvkb: use LMUL=1 with VLEN>=256
Jussi Kivilinna
jussi.kivilinna at iki.fi
Sun Oct 11 19:23:15 CEST 2026
* cipher/Makefile.am (EXTRA_libcipher_la_SOURCES): Add
'sha512-riscv-zvknhb-zvkb.h' and 'sha512-riscv-zvknhb-zvkb-m1.c'.
(riscv_vector_crypto_sha_cflags): Remove.
(riscv_vector_crypto_sha256_cflags)
(riscv_vector_crypto_sha512_cflags)
(riscv_vector_crypto_sha512_m1_cflags): New.
* cipher/sha512-riscv-zvknhb-zvkb.c (working_vsha2cl_vv_u64m2)
(QUAD_ROUND_W_SCHED, QUAD_ROUND_NO_SCHED, load_and_swap)
(sha512_transform_zvknhb_zvkb): Move to 'sha512-riscv-zvknhb-zvkb.h'.
(_gcry_sha512_transform_riscv_zvknhb_zvkb): Use LMUL=1 variant when
VLEN>=256.
* cipher/sha512-riscv-zvknhb-zvkb-m1.c: New.
* cipher/sha512-riscv-zvknhb-zvkb.h: New template for LMUL=1 and LMUL=2
transform functions; prepare K+W for next quad-round ahead of use.
* configure.ac: Add 'sha512-riscv-zvknhb-zvkb-m1.lo'.
(GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST): Add LMUL=1 SHA-512
intrinsics to 'test_sha2'.
(MARCH_RVA23U64_WITH_VEC_CRYPTO): Add 'zvl256b'.
--
Previously LMUL=2 was used for all VLEN. With VLEN>=256, four 64-bit
words fit in LMUL=1 register and on SpacemiT K3 some LMUL=2 operations
cost more than LMUL=1 even with vl=4.
clang requires zvl256b for LMUL=1 SHA-512 intrinsics, so LMUL=1
variant is placed in separate file compiled with zvl256b. LMUL=2
variant and code-path selection are compiled without zvl256b as
they run on VLEN=128.
Benchmark on SpacemiT K3, X100 core (out-of-order, VLEN=256):
Before | cycles/byte
SHA512 | 2.48 c/B
After | cycles/byte
SHA512 | 2.05 c/B
~1.21x faster
Benchmark on SpacemiT K3, A100 core (in-order, VLEN=1024):
Before | cycles/byte
SHA512 | 9.58 c/B
After | cycles/byte
SHA512 | 7.34 c/B
~1.31x faster
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/Makefile.am | 28 +++--
cipher/sha512-riscv-zvknhb-zvkb-m1.c | 60 +++++++++
cipher/sha512-riscv-zvknhb-zvkb.c | 153 +++-------------------
cipher/sha512-riscv-zvknhb-zvkb.h | 181 +++++++++++++++++++++++++++
configure.ac | 12 +-
5 files changed, 293 insertions(+), 141 deletions(-)
create mode 100644 cipher/sha512-riscv-zvknhb-zvkb-m1.c
create mode 100644 cipher/sha512-riscv-zvknhb-zvkb.h
diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index b685a5f7..6c51ee94 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -154,7 +154,9 @@ EXTRA_libcipher_la_SOURCES = \
sha512-avx2-bmi2-amd64.S sha512-avx512-amd64.S \
sha512-intel-shaext.c \
sha512-armv7-neon.S sha512-armv8-aarch64-ce.S sha512-arm.S \
- sha512-ppc.c sha512-riscv-zvknhb-zvkb.c sha512-ssse3-i386.c \
+ sha512-ppc.c sha512-riscv-zvknhb-zvkb.c sha512-riscv-zvknhb-zvkb.h \
+ sha512-riscv-zvknhb-zvkb-m1.c \
+ sha512-ssse3-i386.c \
sm3.c sm3-avx-bmi2-amd64.S sm3-intel-avx2-amd64.S sm3-aarch64.S \
sm3-armv8-aarch64-ce.S \
keccak.c keccak_permute_32.h keccak_permute_64.h \
@@ -466,12 +468,16 @@ rijndael-vp-riscv.lo: $(srcdir)/rijndael-vp-riscv.c Makefile
if ENABLE_RISCV_VECTOR_CRYPTO_INTRINSICS_EXTRA_CFLAGS
riscv_vector_crypto_zvkb_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkb
riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkned_zvkb
-riscv_vector_crypto_sha_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknha_zvknhb_zvkb
+riscv_vector_crypto_sha256_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknha_zvkb
+riscv_vector_crypto_sha512_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknhb_zvkb
+riscv_vector_crypto_sha512_m1_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknhb_zvkb_zvl256b
riscv_vector_crypto_gcm_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkg_zvkb
else
riscv_vector_crypto_zvkb_cflags =
riscv_vector_crypto_aes_cflags =
-riscv_vector_crypto_sha_cflags =
+riscv_vector_crypto_sha256_cflags =
+riscv_vector_crypto_sha512_cflags =
+riscv_vector_crypto_sha512_m1_cflags =
riscv_vector_crypto_gcm_cflags =
endif
@@ -500,17 +506,25 @@ rijndael-riscv-zvkned.lo: $(srcdir)/rijndael-riscv-zvkned.c Makefile
$(custom_depfile_lo)
sha256-riscv-zvknha-zvkb.o: $(srcdir)/sha256-riscv-zvknha-zvkb.c Makefile
- `echo $(COMPILE) $(riscv_vector_crypto_sha_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ `echo $(COMPILE) $(riscv_vector_crypto_sha256_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
$(custom_depfile_o)
sha256-riscv-zvknha-zvkb.lo: $(srcdir)/sha256-riscv-zvknha-zvkb.c Makefile
- `echo $(LTCOMPILE) $(riscv_vector_crypto_sha_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ `echo $(LTCOMPILE) $(riscv_vector_crypto_sha256_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
$(custom_depfile_lo)
sha512-riscv-zvknhb-zvkb.o: $(srcdir)/sha512-riscv-zvknhb-zvkb.c Makefile
- `echo $(COMPILE) $(riscv_vector_crypto_sha_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ `echo $(COMPILE) $(riscv_vector_crypto_sha512_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
$(custom_depfile_o)
sha512-riscv-zvknhb-zvkb.lo: $(srcdir)/sha512-riscv-zvknhb-zvkb.c Makefile
- `echo $(LTCOMPILE) $(riscv_vector_crypto_sha_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ `echo $(LTCOMPILE) $(riscv_vector_crypto_sha512_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ $(custom_depfile_lo)
+
+sha512-riscv-zvknhb-zvkb-m1.o: $(srcdir)/sha512-riscv-zvknhb-zvkb-m1.c Makefile
+ `echo $(COMPILE) $(riscv_vector_crypto_sha512_m1_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ $(custom_depfile_o)
+
+sha512-riscv-zvknhb-zvkb-m1.lo: $(srcdir)/sha512-riscv-zvknhb-zvkb-m1.c Makefile
+ `echo $(LTCOMPILE) $(riscv_vector_crypto_sha512_m1_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
$(custom_depfile_lo)
diff --git a/cipher/sha512-riscv-zvknhb-zvkb-m1.c b/cipher/sha512-riscv-zvknhb-zvkb-m1.c
new file mode 100644
index 00000000..be1e984a
--- /dev/null
+++ b/cipher/sha512-riscv-zvknhb-zvkb-m1.c
@@ -0,0 +1,60 @@
+/* sha512-riscv-zvknhb-zvkb-m1.c - RISC-V vector SHA-512, VLEN>=256 variant
+ * Copyright (C) 2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+
+#if defined (__riscv) && \
+ defined(HAVE_COMPATIBLE_CC_RISCV_VECTOR_INTRINSICS) && \
+ defined(HAVE_COMPATIBLE_CC_RISCV_VECTOR_CRYPTO_INTRINSICS) && \
+ defined(USE_SHA512)
+
+#include "g10lib.h"
+#include "simd-common-riscv.h"
+#include <riscv_vector.h>
+
+
+#define ALWAYS_INLINE inline __attribute__((always_inline))
+#define NO_INLINE __attribute__((noinline))
+#define NO_INSTRUMENT_FUNCTION __attribute__((no_instrument_function))
+
+#define ASM_FUNC_ATTR NO_INSTRUMENT_FUNCTION
+#define ASM_FUNC_ATTR_INLINE ASM_FUNC_ATTR ALWAYS_INLINE
+#define ASM_FUNC_ATTR_NOINLINE ASM_FUNC_ATTR NO_INLINE
+
+#ifdef HAVE_GCC_ATTRIBUTE_OPTIMIZE
+# define FUNC_ATTR_OPT_O2 __attribute__((optimize("-O2")))
+#else
+# define FUNC_ATTR_OPT_O2
+#endif
+
+
+#define SHA512_LMUL m1
+#define SHA512_MASK_T vbool64_t
+#include "sha512-riscv-zvknhb-zvkb.h"
+
+unsigned int
+_gcry_sha512_transform_riscv_zvknhb_zvkb_m1(u64 state[8],
+ const unsigned char *input_data,
+ size_t num_blks,
+ const u64 k[80])
+{
+ return sha512_transform_zvknhb_zvkb_m1(state, input_data, num_blks, k);
+}
+
+#endif /* HAVE_COMPATIBLE_CC_RISCV_VECTOR_INTRINSICS */
diff --git a/cipher/sha512-riscv-zvknhb-zvkb.c b/cipher/sha512-riscv-zvknhb-zvkb.c
index fea8f32a..dd316b42 100644
--- a/cipher/sha512-riscv-zvknhb-zvkb.c
+++ b/cipher/sha512-riscv-zvknhb-zvkb.c
@@ -1,5 +1,5 @@
/* sha512-riscv-zvknhb-zvkb.c - RISC-V vector crypto implementation of SHA-512
- * Copyright (C) 2025 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ * Copyright (C) 2025-2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
*
* This file is part of Libgcrypt.
*
@@ -37,149 +37,36 @@
#define ASM_FUNC_ATTR_INLINE ASM_FUNC_ATTR ALWAYS_INLINE
#define ASM_FUNC_ATTR_NOINLINE ASM_FUNC_ATTR NO_INLINE
-
-static ASM_FUNC_ATTR_INLINE vuint64m2_t
-working_vsha2cl_vv_u64m2(vuint64m2_t hgcd, vuint64m2_t feba,
- vuint64m2_t kw, size_t vl)
-{
-#ifdef HAVE_BROKEN_VSHA2CL_INTRINSIC
- asm (
- "vsetvli zero,%3,e64,m2,ta,ma;\n\t"
- "vsha2cl.vv %0,%1,%2;\n\t"
- : "+vr" (hgcd)
- : "vr" (feba), "vr" (kw), "r" (vl)
- : "vl", "vtype"
- );
- return hgcd;
-#else
- return __riscv_vsha2cl_vv_u64m2(hgcd, feba, kw, vl);
-#endif
-}
-
-
-/* Quad-round with message expansion (rounds 0-63) */
-#define QUAD_ROUND_W_SCHED(w0, w1, w2, w3) \
- k_tmp = k; \
- asm ("" : "+r" (k_tmp) :: "memory"); \
- v_k = __riscv_vle64_v_u64m2(k_tmp, vl); \
- k += 4; \
- v_kw = __riscv_vadd_vv_u64m2(v_k, w0, vl); \
- v_hgcd_work = working_vsha2cl_vv_u64m2(v_hgcd_work, v_feba_work, v_kw, vl); \
- v_feba_work = __riscv_vsha2ch_vv_u64m2(v_feba_work, v_hgcd_work, v_kw, vl); \
- v_w_merged = __riscv_vmerge_vvm_u64m2(w2, w1, merge_mask, vl); \
- w0 = __riscv_vsha2ms_vv_u64m2(w0, v_w_merged, w3, vl);
-
-/* Quad-round without message expansion (rounds 64-79) */
-#define QUAD_ROUND_NO_SCHED(w0) \
- k_tmp = k; \
- asm ("" : "+r" (k_tmp) :: "memory"); \
- v_k = __riscv_vle64_v_u64m2(k_tmp, vl); \
- k += 4; \
- v_kw = __riscv_vadd_vv_u64m2(v_k, w0, vl); \
- v_hgcd_work = working_vsha2cl_vv_u64m2(v_hgcd_work, v_feba_work, v_kw, vl); \
- v_feba_work = __riscv_vsha2ch_vv_u64m2(v_feba_work, v_hgcd_work, v_kw, vl);
-
-
-static ASM_FUNC_ATTR_INLINE vuint64m2_t
-load_and_swap(const byte *p, size_t vl)
-{
-#ifdef RVV_UNALIGNED_NOT_ALLOWED
- vuint8m2_t temp = __riscv_vle8_v_u8m2(p, vl * 8);
- return __riscv_vrev8_v_u64m2(__riscv_vreinterpret_v_u8m2_u64m2(temp), vl);
-#else
- return __riscv_vrev8_v_u64m2(__riscv_vle64_v_u64m2((const void *)p, vl), vl);
-#endif
-}
-
-
-static ASM_FUNC_ATTR_INLINE void
-sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data,
- size_t nblocks, const u64 k_const[80])
-{
- static const u64 feba_hgcd_indices[4] = { 40, 32, 8, 0 };
- static const int feba_offset = 0;
- static const int hgcd_offset = 16 / sizeof(u64);
- size_t vl;
- vuint64m2_t idx;
- vuint64m2_t v_feba_work, v_feba;
- vuint64m2_t v_hgcd_work, v_hgcd;
- vuint64m2_t w0, w1, w2, w3;
- vuint64m2_t v_k, v_kw, v_w_merged;
- vbool32_t merge_mask;
- vuint64m2_t v_feba_hgcd_idx;
-
- vl = 4;
- idx = __riscv_vid_v_u64m2(vl);
- merge_mask = __riscv_vmseq_vx_u64m2_b32(idx, 0, vl);
-
- v_feba_hgcd_idx = __riscv_vle64_v_u64m2(feba_hgcd_indices, vl);
-
- v_feba = __riscv_vluxei64_v_u64m2(state + feba_offset, v_feba_hgcd_idx, vl);
- v_hgcd = __riscv_vluxei64_v_u64m2(state + hgcd_offset, v_feba_hgcd_idx, vl);
-
- while (nblocks > 0)
- {
- const u64 *k = k_const;
- const u64 *k_tmp;
-
- v_feba_work = v_feba;
- v_hgcd_work = v_hgcd;
-
- w0 = load_and_swap(data + 0, vl);
- w1 = load_and_swap(data + 32, vl);
- w2 = load_and_swap(data + 64, vl);
- w3 = load_and_swap(data + 96, vl);
-
- QUAD_ROUND_W_SCHED(w0, w1, w2, w3);
- QUAD_ROUND_W_SCHED(w1, w2, w3, w0);
- QUAD_ROUND_W_SCHED(w2, w3, w0, w1);
- QUAD_ROUND_W_SCHED(w3, w0, w1, w2);
- QUAD_ROUND_W_SCHED(w0, w1, w2, w3);
- QUAD_ROUND_W_SCHED(w1, w2, w3, w0);
- QUAD_ROUND_W_SCHED(w2, w3, w0, w1);
- QUAD_ROUND_W_SCHED(w3, w0, w1, w2);
- QUAD_ROUND_W_SCHED(w0, w1, w2, w3);
- QUAD_ROUND_W_SCHED(w1, w2, w3, w0);
- QUAD_ROUND_W_SCHED(w2, w3, w0, w1);
- QUAD_ROUND_W_SCHED(w3, w0, w1, w2);
- QUAD_ROUND_W_SCHED(w0, w1, w2, w3);
- QUAD_ROUND_W_SCHED(w1, w2, w3, w0);
- QUAD_ROUND_W_SCHED(w2, w3, w0, w1);
- QUAD_ROUND_W_SCHED(w3, w0, w1, w2);
-
- QUAD_ROUND_NO_SCHED(w0);
- QUAD_ROUND_NO_SCHED(w1);
- QUAD_ROUND_NO_SCHED(w2);
- QUAD_ROUND_NO_SCHED(w3);
-
- v_feba = __riscv_vadd_vv_u64m2(v_feba, v_feba_work, vl);
- v_hgcd = __riscv_vadd_vv_u64m2(v_hgcd, v_hgcd_work, vl);
-
- data += 128;
- nblocks--;
- }
-
- __riscv_vsuxei64_v_u64m2(state + feba_offset, v_feba_hgcd_idx, v_feba, vl);
- __riscv_vsuxei64_v_u64m2(state + hgcd_offset, v_feba_hgcd_idx, v_hgcd, vl);
-
- clear_vec_regs();
-}
-
-
#ifdef HAVE_GCC_ATTRIBUTE_OPTIMIZE
# define FUNC_ATTR_OPT_O2 __attribute__((optimize("-O2")))
#else
# define FUNC_ATTR_OPT_O2
#endif
-unsigned int ASM_FUNC_ATTR FUNC_ATTR_OPT_O2
+
+/* m2 variant for VLEN == 128 */
+#define SHA512_LMUL m2
+#define SHA512_MASK_T vbool32_t
+#include "sha512-riscv-zvknhb-zvkb.h"
+
+/* m1 variant for VLEN >= 256, compiled separately with zvl256b */
+extern unsigned int
+_gcry_sha512_transform_riscv_zvknhb_zvkb_m1(u64 state[8],
+ const unsigned char *input_data,
+ size_t num_blks,
+ const u64 k[80]);
+
+unsigned int
_gcry_sha512_transform_riscv_zvknhb_zvkb(u64 state[8],
const unsigned char *input_data,
size_t num_blks,
const u64 k[80])
{
- sha512_transform_zvknhb_zvkb(state, input_data, num_blks, k);
- return 0;
+ if (__riscv_vsetvlmax_e8m1() >= 32)
+ return _gcry_sha512_transform_riscv_zvknhb_zvkb_m1(state, input_data,
+ num_blks, k);
+ else
+ return sha512_transform_zvknhb_zvkb_m2(state, input_data, num_blks, k);
}
unsigned int ASM_FUNC_ATTR FUNC_ATTR_OPT_O2
diff --git a/cipher/sha512-riscv-zvknhb-zvkb.h b/cipher/sha512-riscv-zvknhb-zvkb.h
new file mode 100644
index 00000000..e5a0b3a2
--- /dev/null
+++ b/cipher/sha512-riscv-zvknhb-zvkb.h
@@ -0,0 +1,181 @@
+/* sha512-riscv-zvknhb-zvkb.h - RISC-V vector crypto SHA-512 LMUL template
+ * Copyright (C) 2025-2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* Included once for each LMUL with SHA512_LMUL and SHA512_MASK_T set. */
+
+#define CAT_(a, b) a##b
+#define CAT(a, b) CAT_(a, b)
+
+#define FN(name) CAT(name##_, SHA512_LMUL)
+#define V(name) CAT(__riscv_##name##_u64, SHA512_LMUL)
+#define V8(name) CAT(__riscv_##name##_u8, SHA512_LMUL)
+#define VEC_T CAT(CAT(vuint64, SHA512_LMUL), _t)
+#define MASK_T SHA512_MASK_T
+
+
+#ifdef HAVE_BROKEN_VSHA2CL_INTRINSIC
+static ASM_FUNC_ATTR_INLINE VEC_T
+FN(asm_vsha2cl) (VEC_T vd, VEC_T vs2, VEC_T vs1, size_t vl)
+{
+ asm ("vsetvli zero,%3,e64," STR2(SHA512_LMUL) ",ta,ma;\n\t"
+ "vsha2cl.vv %0,%1,%2;\n\t"
+ : "+vr" (vd)
+ : "vr" (vs2), "vr" (vs1), "r" (vl)
+ : "vl", "vtype");
+ return vd;
+}
+# define VSHA2CL FN(asm_vsha2cl)
+#else
+# define VSHA2CL __riscv_vsha2cl
+#endif
+
+
+#define LOAD_K() \
+ k_tmp = k; \
+ asm ("" : "+r" (k_tmp) :: "memory"); \
+ v_k = V(vle64_v)(k_tmp, vl); \
+ k += 4;
+
+/* K+W for next quad-round and K for the one after that are prepared
+ * between vsha2cl and vsha2ch, ahead of use. */
+
+/* Quad-round with message expansion (rounds 0-63) */
+#define QUAD_ROUND_W_SCHED(kw, kw_next, w0, w1, w2, w3) \
+ v_hgcd_work = VSHA2CL(v_hgcd_work, v_feba_work, kw, vl); \
+ kw_next = __riscv_vadd(v_k, w1, vl); \
+ LOAD_K(); \
+ v_feba_work = __riscv_vsha2ch(v_feba_work, v_hgcd_work, kw, vl); \
+ v_w_merged = __riscv_vmerge(w2, w1, merge_mask, vl); \
+ w0 = __riscv_vsha2ms(w0, v_w_merged, w3, vl);
+
+/* Quad-round without message expansion (rounds 64-75) */
+#define QUAD_ROUND_NO_SCHED(kw, kw_next, w_next, load_k) \
+ v_hgcd_work = VSHA2CL(v_hgcd_work, v_feba_work, kw, vl); \
+ kw_next = __riscv_vadd(v_k, w_next, vl); \
+ load_k; \
+ v_feba_work = __riscv_vsha2ch(v_feba_work, v_hgcd_work, kw, vl);
+
+/* Last quad-round (rounds 76-79) */
+#define QUAD_ROUND_LAST(kw) \
+ v_hgcd_work = VSHA2CL(v_hgcd_work, v_feba_work, kw, vl); \
+ v_feba_work = __riscv_vsha2ch(v_feba_work, v_hgcd_work, kw, vl);
+
+
+static ASM_FUNC_ATTR_INLINE VEC_T
+FN(load_and_swap) (const byte *p, size_t vl)
+{
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ return __riscv_vrev8(V(vreinterpret)(V8(vle8_v)(p, vl * 8)), vl);
+#else
+ return __riscv_vrev8(V(vle64_v)((const void *)p, vl), vl);
+#endif
+}
+
+static ASM_FUNC_ATTR_NOINLINE FUNC_ATTR_OPT_O2 unsigned int
+FN(sha512_transform_zvknhb_zvkb) (u64 state[8], const byte *data,
+ size_t nblocks, const u64 k_const[80])
+{
+ static const u64 feba_hgcd_indices[4] = { 40, 32, 8, 0 };
+ static const int feba_offset = 0;
+ static const int hgcd_offset = 16 / sizeof(u64);
+ size_t vl;
+ VEC_T idx;
+ VEC_T v_feba_work, v_feba;
+ VEC_T v_hgcd_work, v_hgcd;
+ VEC_T w0, w1, w2, w3;
+ VEC_T v_k, v_kw0, v_kw1, v_w_merged;
+ MASK_T merge_mask;
+ VEC_T v_feba_hgcd_idx;
+
+ vl = 4;
+ idx = V(vid_v)(vl);
+ merge_mask = __riscv_vmseq(idx, 0, vl);
+
+ v_feba_hgcd_idx = V(vle64_v)(feba_hgcd_indices, vl);
+
+ v_feba = V(vluxei64_v)(state + feba_offset, v_feba_hgcd_idx, vl);
+ v_hgcd = V(vluxei64_v)(state + hgcd_offset, v_feba_hgcd_idx, vl);
+
+ while (nblocks > 0)
+ {
+ const u64 *k = k_const;
+ const u64 *k_tmp;
+
+ v_feba_work = v_feba;
+ v_hgcd_work = v_hgcd;
+
+ w0 = FN(load_and_swap)(data + 0, vl);
+ w1 = FN(load_and_swap)(data + 32, vl);
+ w2 = FN(load_and_swap)(data + 64, vl);
+ w3 = FN(load_and_swap)(data + 96, vl);
+
+ LOAD_K();
+ v_kw0 = __riscv_vadd(v_k, w0, vl);
+ LOAD_K();
+
+ QUAD_ROUND_W_SCHED(v_kw0, v_kw1, w0, w1, w2, w3);
+ QUAD_ROUND_W_SCHED(v_kw1, v_kw0, w1, w2, w3, w0);
+ QUAD_ROUND_W_SCHED(v_kw0, v_kw1, w2, w3, w0, w1);
+ QUAD_ROUND_W_SCHED(v_kw1, v_kw0, w3, w0, w1, w2);
+ QUAD_ROUND_W_SCHED(v_kw0, v_kw1, w0, w1, w2, w3);
+ QUAD_ROUND_W_SCHED(v_kw1, v_kw0, w1, w2, w3, w0);
+ QUAD_ROUND_W_SCHED(v_kw0, v_kw1, w2, w3, w0, w1);
+ QUAD_ROUND_W_SCHED(v_kw1, v_kw0, w3, w0, w1, w2);
+ QUAD_ROUND_W_SCHED(v_kw0, v_kw1, w0, w1, w2, w3);
+ QUAD_ROUND_W_SCHED(v_kw1, v_kw0, w1, w2, w3, w0);
+ QUAD_ROUND_W_SCHED(v_kw0, v_kw1, w2, w3, w0, w1);
+ QUAD_ROUND_W_SCHED(v_kw1, v_kw0, w3, w0, w1, w2);
+ QUAD_ROUND_W_SCHED(v_kw0, v_kw1, w0, w1, w2, w3);
+ QUAD_ROUND_W_SCHED(v_kw1, v_kw0, w1, w2, w3, w0);
+ QUAD_ROUND_W_SCHED(v_kw0, v_kw1, w2, w3, w0, w1);
+ QUAD_ROUND_W_SCHED(v_kw1, v_kw0, w3, w0, w1, w2);
+
+ QUAD_ROUND_NO_SCHED(v_kw0, v_kw1, w1, LOAD_K());
+ QUAD_ROUND_NO_SCHED(v_kw1, v_kw0, w2, LOAD_K());
+ QUAD_ROUND_NO_SCHED(v_kw0, v_kw1, w3, (void)0);
+ QUAD_ROUND_LAST(v_kw1);
+
+ v_feba = __riscv_vadd(v_feba, v_feba_work, vl);
+ v_hgcd = __riscv_vadd(v_hgcd, v_hgcd_work, vl);
+
+ data += 128;
+ nblocks--;
+ }
+
+ __riscv_vsuxei64(state + feba_offset, v_feba_hgcd_idx, v_feba, vl);
+ __riscv_vsuxei64(state + hgcd_offset, v_feba_hgcd_idx, v_hgcd, vl);
+
+ clear_vec_regs();
+ return 0;
+}
+
+#undef QUAD_ROUND_LAST
+#undef QUAD_ROUND_NO_SCHED
+#undef QUAD_ROUND_W_SCHED
+#undef LOAD_K
+#undef VSHA2CL
+#undef MASK_T
+#undef VEC_T
+#undef V8
+#undef V
+#undef FN
+#undef CAT
+#undef CAT_
+#undef SHA512_MASK_T
+#undef SHA512_LMUL
diff --git a/configure.ac b/configure.ac
index 2a126041..e32519c9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2983,6 +2983,7 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST],
vuint32m1_t m;
vuint32m1_t idx = __riscv_vid_v_u32m1 (vl);
vbool32_t merge_mask = __riscv_vmseq_vx_u32m1_b32 (idx, 0, vl);
+ vuint64m1_t c, d, w4;
a = __riscv_vsha2cl_vv_u32m1(a, b, w0, vl);
b = __riscv_vsha2ch_vv_u32m1(a, b, w0, vl);
m = __riscv_vmerge_vvm_u32m1(w2, w1, merge_mask, vl);
@@ -2993,6 +2994,14 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST],
__riscv_vse32_v_u32m1(ptr + 3 * vl, w1, vl);
__riscv_vse32_v_u32m1(ptr + 4 * vl, w2, vl);
__riscv_vse32_v_u32m1(ptr + 5 * vl, w3, vl);
+ vl = __riscv_vsetvl_e64m1 (4);
+ c = __riscv_vle64_v_u64m1((void *)(ptr + 0 * vl), vl);
+ d = __riscv_vle64_v_u64m1((void *)(ptr + 2 * vl), vl);
+ w4 = __riscv_vle64_v_u64m1((void *)(ptr + 4 * vl), vl);
+ c = __riscv_vsha2cl_vv_u64m1(c, d, w4, vl);
+ d = __riscv_vsha2ch_vv_u64m1(c, d, w4, vl);
+ __riscv_vse64_v_u64m1((void *)(ptr + 0 * vl), c, vl);
+ __riscv_vse64_v_u64m1((void *)(ptr + 2 * vl), d, vl);
}
void test_aes_key(unsigned int *ptr)
{
@@ -3100,7 +3109,7 @@ if test "$gcry_cv_cc_riscv_zicclsm_enabled" = "yes" ; then
else
MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC}
fi
-MARCH_RVA23U64_WITH_VEC_CRYPTO=${MARCH_RVA23U64_BASE}_zvbc_zvkb_zvkg_zvkn_zvks
+MARCH_RVA23U64_WITH_VEC_CRYPTO=${MARCH_RVA23U64_BASE}_zvbc_zvkb_zvkg_zvkn_zvks_zvl256b
AC_SUBST([MARCH_RVA23U64_BASE])
CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO"
@@ -4178,6 +4187,7 @@ if test "$found" = "1" ; then
riscv64-*-*)
# Build with the RISC-V vector cryptography implementation
GCRYPT_ASM_DIGESTS="$GCRYPT_ASM_DIGESTS sha512-riscv-zvknhb-zvkb.lo"
+ GCRYPT_ASM_DIGESTS="$GCRYPT_ASM_DIGESTS sha512-riscv-zvknhb-zvkb-m1.lo"
;;
esac
--
2.53.0
More information about the Gcrypt-devel
mailing list