[PATCH 1/4] riscv: use unaligned vector memory access when supported
Jussi Kivilinna
jussi.kivilinna at iki.fi
Wed Jul 29 18:34:21 CEST 2026
* cipher/Makefile.am: Use -mstrict-align only for plain RVV
implementations; add Zvkb to RVV-crypto -march flags.
* cipher/chacha20-riscv-v.c (unaligned_load_u32m1)
(unaligned_store_u32m1): New.
(chacha20_rvv_blocks): Use unaligned load/store helpers.
* cipher/cipher-gcm-riscv-zvkg.c (unaligned_load_u32m1): Use element
width access when RVV_UNALIGNED_NOT_ALLOWED is not defined.
* cipher/rijndael-riscv-zvkned.c (unaligned_load_u64m1)
(unaligned_load_u32m1, unaligned_store_u32m1, unaligned_load_u32m2)
(unaligned_store_u32m2, unaligned_load_u32m4, unaligned_store_u32m4):
Likewise.
* cipher/sha256-riscv-zvknha-zvkb.c (load_and_swap): Likewise; drop
'vl_bytes' parameter.
(sha256_transform_zvknha_zvkb): Remove 'vl_bytes'.
* cipher/sha512-riscv-zvknhb-zvkb.c: Likewise.
* cipher/simd-common-riscv.h (RVV_UNALIGNED_NOT_ALLOWED): New.
* configure.ac: Replace -mstrict-align compiler support check with
Zicclsm enabled check; add Zicclsm to MARCH_RVA23U64_BASE and Zvkb to
MARCH_RVA23U64_WITH_VEC_CRYPTO; drop -mstrict-align from RVV-crypto
check flags; add vrev8 to RVV-crypto intrinsics test.
--
-mstrict-align was added for GCC-14 code generation bug, but is really
needed for SpacemiT K1 which does not support unaligned vector memory
access. K1 does not have vector cryptography extensions, so RVV-crypto
implementations can expect unaligned vector access to work. Detect
support through Zicclsm extension and keep -mstrict-align for plain
RVV implementations.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/Makefile.am | 22 +-
cipher/chacha20-riscv-v.c | 258 ++++++++++-----------
cipher/cipher-gcm-riscv-zvkg.c | 5 +-
cipher/rijndael-riscv-zvkned.c | 366 +++++++++++++++---------------
cipher/sha256-riscv-zvknha-zvkb.c | 21 +-
cipher/sha512-riscv-zvknhb-zvkb.c | 21 +-
cipher/simd-common-riscv.h | 13 ++
configure.ac | 62 +++--
8 files changed, 399 insertions(+), 369 deletions(-)
diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index 021a7ab6..0c00d181 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -368,16 +368,15 @@ serpent-avx512-x86.o: $(srcdir)/serpent-avx512-x86.c Makefile
serpent-avx512-x86.lo: $(srcdir)/serpent-avx512-x86.c Makefile
`echo $(LTCOMPILE) $(avx512f_cflags) -c $< | $(instrumentation_munging) `
-# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads)
if ENABLE_RISCV_VECTOR_INTRINSICS_EXTRA_CFLAGS
+# Note: -mstrict-align needed for SpacemiT K1 RVV bug (K1 does not support
+# unaligned vector memory access). This is enabled only for plain RVV
+# implementations, RVV-crypto extension implementations expect CPU with proper
+# support for unaligned vector memory access.
riscv_vector_cflags = -O2 -march=@MARCH_RVA22U64_WITH_VEC@ -mstrict-align
else
-if SUPPORT_CC_RISCV_MSTRICT_ALIGN
-riscv_vector_cflags = -O2 -mstrict-align
-else
riscv_vector_cflags =
endif
-endif
chacha20-riscv-v.o: $(srcdir)/chacha20-riscv-v.c Makefile
`echo $(COMPILE) $(riscv_vector_cflags) -c $< | $(instrumentation_munging) `
@@ -391,22 +390,15 @@ rijndael-vp-riscv.o: $(srcdir)/rijndael-vp-riscv.c Makefile
rijndael-vp-riscv.lo: $(srcdir)/rijndael-vp-riscv.c Makefile
`echo $(LTCOMPILE) $(riscv_vector_cflags) -c $< | $(instrumentation_munging) `
-# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads)
if ENABLE_RISCV_VECTOR_CRYPTO_INTRINSICS_EXTRA_CFLAGS
-riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkned -mstrict-align
-riscv_vector_crypto_sha_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknha_zvknhb_zvkb -mstrict-align
-riscv_vector_crypto_gcm_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkg -mstrict-align
-else
-if SUPPORT_CC_RISCV_MSTRICT_ALIGN
-riscv_vector_crypto_aes_cflags = -O2 -mstrict-align
-riscv_vector_crypto_sha_cflags = -O2 -mstrict-align
-riscv_vector_crypto_gcm_cflags = -O2 -mstrict-align
+riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkned_zvkb
+riscv_vector_crypto_sha_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknha_zvknhb_zvkb
+riscv_vector_crypto_gcm_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkg_zvkb
else
riscv_vector_crypto_aes_cflags =
riscv_vector_crypto_sha_cflags =
riscv_vector_crypto_gcm_cflags =
endif
-endif
cipher-gcm-riscv-zvkg.o: $(srcdir)/cipher-gcm-riscv-zvkg.c Makefile
`echo $(COMPILE) $(riscv_vector_crypto_gcm_cflags) -c $< | $(instrumentation_munging) `
diff --git a/cipher/chacha20-riscv-v.c b/cipher/chacha20-riscv-v.c
index 1304a333..94026305 100644
--- a/cipher/chacha20-riscv-v.c
+++ b/cipher/chacha20-riscv-v.c
@@ -151,85 +151,113 @@ gen_indexes(size_t vl, size_t stride)
return __riscv_vadd_vv_u16m2(idx_hi, idx_lo, vl * 4);
}
+static ASM_FUNC_ATTR_INLINE vuint32m1_t
+unaligned_load_u32m1(const void *src, size_t vl)
+{
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ return __riscv_vreinterpret_v_u8m1_u32m1(__riscv_vle8_v_u8m1(src, vl * 4));
+#else
+ return __riscv_vle32_v_u32m1(src, vl);
+#endif
+}
+
+static ASM_FUNC_ATTR_INLINE void
+unaligned_store_u32m1(void *dst, vuint32m1_t vec, size_t vl)
+{
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ __riscv_vse8_v_u8m1(dst, __riscv_vreinterpret_v_u32m1_u8m1(vec), vl * 4);
+#else
+ __riscv_vse32_v_u32m1(dst, vec, vl);
+#endif
+}
+
static ASM_FUNC_ATTR_INLINE vuint32m1x8_t
unaligned_vlsseg8e32_v_u32m1x8(const void *src, size_t vl)
{
- const byte *bsrc = src;
- vuint16m2_t indexes;
- vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7;
- vuint32m1x8_t data;
-
- if (LIKELY(((uintptr_t)src & 3) == 0))
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ if (UNLIKELY(((uintptr_t)src & 3) != 0))
+ {
+ const byte *bsrc = src;
+ vuint16m2_t indexes;
+ vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7;
+ vuint32m1x8_t data;
+
+ indexes = gen_indexes(4 * vl, 64);
+
+ b0 = __riscv_vluxei16_v_u8m1(bsrc + 0 * 4, indexes, vl * 4);
+ b1 = __riscv_vluxei16_v_u8m1(bsrc + 1 * 4, indexes, vl * 4);
+ b2 = __riscv_vluxei16_v_u8m1(bsrc + 2 * 4, indexes, vl * 4);
+ b3 = __riscv_vluxei16_v_u8m1(bsrc + 3 * 4, indexes, vl * 4);
+ b4 = __riscv_vluxei16_v_u8m1(bsrc + 4 * 4, indexes, vl * 4);
+ b5 = __riscv_vluxei16_v_u8m1(bsrc + 5 * 4, indexes, vl * 4);
+ b6 = __riscv_vluxei16_v_u8m1(bsrc + 6 * 4, indexes, vl * 4);
+ b7 = __riscv_vluxei16_v_u8m1(bsrc + 7 * 4, indexes, vl * 4);
+
+ data = __riscv_vundefined_u32m1x8();
+ data = __riscv_vset_v_u32m1_u32m1x8(
+ data, 0, __riscv_vreinterpret_v_u8m1_u32m1(b0));
+ data = __riscv_vset_v_u32m1_u32m1x8(
+ data, 1, __riscv_vreinterpret_v_u8m1_u32m1(b1));
+ data = __riscv_vset_v_u32m1_u32m1x8(
+ data, 2, __riscv_vreinterpret_v_u8m1_u32m1(b2));
+ data = __riscv_vset_v_u32m1_u32m1x8(
+ data, 3, __riscv_vreinterpret_v_u8m1_u32m1(b3));
+ data = __riscv_vset_v_u32m1_u32m1x8(
+ data, 4, __riscv_vreinterpret_v_u8m1_u32m1(b4));
+ data = __riscv_vset_v_u32m1_u32m1x8(
+ data, 5, __riscv_vreinterpret_v_u8m1_u32m1(b5));
+ data = __riscv_vset_v_u32m1_u32m1x8(
+ data, 6, __riscv_vreinterpret_v_u8m1_u32m1(b6));
+ data = __riscv_vset_v_u32m1_u32m1x8(
+ data, 7, __riscv_vreinterpret_v_u8m1_u32m1(b7));
+
+ return data;
+ }
+ else
+#endif
{
/* Fast path for 32-bit aligned loads. */
return __riscv_vlsseg8e32_v_u32m1x8(src, 64, vl);
}
-
- indexes = gen_indexes(4 * vl, 64);
-
- b0 = __riscv_vluxei16_v_u8m1(bsrc + 0 * 4, indexes, vl * 4);
- b1 = __riscv_vluxei16_v_u8m1(bsrc + 1 * 4, indexes, vl * 4);
- b2 = __riscv_vluxei16_v_u8m1(bsrc + 2 * 4, indexes, vl * 4);
- b3 = __riscv_vluxei16_v_u8m1(bsrc + 3 * 4, indexes, vl * 4);
- b4 = __riscv_vluxei16_v_u8m1(bsrc + 4 * 4, indexes, vl * 4);
- b5 = __riscv_vluxei16_v_u8m1(bsrc + 5 * 4, indexes, vl * 4);
- b6 = __riscv_vluxei16_v_u8m1(bsrc + 6 * 4, indexes, vl * 4);
- b7 = __riscv_vluxei16_v_u8m1(bsrc + 7 * 4, indexes, vl * 4);
-
- data = __riscv_vundefined_u32m1x8();
- data = __riscv_vset_v_u32m1_u32m1x8(
- data, 0, __riscv_vreinterpret_v_u8m1_u32m1(b0));
- data = __riscv_vset_v_u32m1_u32m1x8(
- data, 1, __riscv_vreinterpret_v_u8m1_u32m1(b1));
- data = __riscv_vset_v_u32m1_u32m1x8(
- data, 2, __riscv_vreinterpret_v_u8m1_u32m1(b2));
- data = __riscv_vset_v_u32m1_u32m1x8(
- data, 3, __riscv_vreinterpret_v_u8m1_u32m1(b3));
- data = __riscv_vset_v_u32m1_u32m1x8(
- data, 4, __riscv_vreinterpret_v_u8m1_u32m1(b4));
- data = __riscv_vset_v_u32m1_u32m1x8(
- data, 5, __riscv_vreinterpret_v_u8m1_u32m1(b5));
- data = __riscv_vset_v_u32m1_u32m1x8(
- data, 6, __riscv_vreinterpret_v_u8m1_u32m1(b6));
- data = __riscv_vset_v_u32m1_u32m1x8(
- data, 7, __riscv_vreinterpret_v_u8m1_u32m1(b7));
-
- return data;
}
static ASM_FUNC_ATTR_INLINE void
unaligned_vssseg8e32_v_u32m1x8(void *dst, vuint32m1x8_t data, size_t vl)
{
- byte *bdst = dst;
- vuint16m2_t indexes;
- vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7;
-
- if (LIKELY(((uintptr_t)dst & 3) == 0))
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ if (UNLIKELY(((uintptr_t)dst & 3) != 0))
+ {
+ byte *bdst = dst;
+ vuint16m2_t indexes;
+ vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7;
+
+ indexes = gen_indexes(4 * vl, 64);
+
+ b0 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 0));
+ b1 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 1));
+ b2 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 2));
+ b3 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 3));
+ b4 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 4));
+ b5 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 5));
+ b6 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 6));
+ b7 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 7));
+
+ __riscv_vsuxei16_v_u8m1(bdst + 0 * 4, indexes, b0, vl * 4);
+ __riscv_vsuxei16_v_u8m1(bdst + 1 * 4, indexes, b1, vl * 4);
+ __riscv_vsuxei16_v_u8m1(bdst + 2 * 4, indexes, b2, vl * 4);
+ __riscv_vsuxei16_v_u8m1(bdst + 3 * 4, indexes, b3, vl * 4);
+ __riscv_vsuxei16_v_u8m1(bdst + 4 * 4, indexes, b4, vl * 4);
+ __riscv_vsuxei16_v_u8m1(bdst + 5 * 4, indexes, b5, vl * 4);
+ __riscv_vsuxei16_v_u8m1(bdst + 6 * 4, indexes, b6, vl * 4);
+ __riscv_vsuxei16_v_u8m1(bdst + 7 * 4, indexes, b7, vl * 4);
+ }
+ else
+#endif
{
/* Fast path for 32-bit aligned stores. */
__riscv_vssseg8e32_v_u32m1x8(dst, 64, data, vl);
return;
}
-
- indexes = gen_indexes(4 * vl, 64);
-
- b0 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 0));
- b1 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 1));
- b2 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 2));
- b3 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 3));
- b4 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 4));
- b5 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 5));
- b6 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 6));
- b7 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 7));
-
- __riscv_vsuxei16_v_u8m1(bdst + 0 * 4, indexes, b0, vl * 4);
- __riscv_vsuxei16_v_u8m1(bdst + 1 * 4, indexes, b1, vl * 4);
- __riscv_vsuxei16_v_u8m1(bdst + 2 * 4, indexes, b2, vl * 4);
- __riscv_vsuxei16_v_u8m1(bdst + 3 * 4, indexes, b3, vl * 4);
- __riscv_vsuxei16_v_u8m1(bdst + 4 * 4, indexes, b4, vl * 4);
- __riscv_vsuxei16_v_u8m1(bdst + 5 * 4, indexes, b5, vl * 4);
- __riscv_vsuxei16_v_u8m1(bdst + 6 * 4, indexes, b6, vl * 4);
- __riscv_vsuxei16_v_u8m1(bdst + 7 * 4, indexes, b7, vl * 4);
}
static ASM_FUNC_ATTR_INLINE unsigned int
@@ -385,8 +413,8 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks)
vuint32m1_t v0, v1, v2, v3;
vuint32m1_t v4, v5, v6, v7;
vuint32m1_t state0, state1, state2, state3;
- vuint8m1_t i0, i1, i2, i3;
- vuint8m1_t i4, i5, i6, i7;
+ vuint32m1_t i0, i1, i2, i3;
+ vuint32m1_t i4, i5, i6, i7;
vuint16m1_t rot16 = gen_rot16(vl);
vuint8m1_t rot8 = gen_rot8(vl);
@@ -417,10 +445,10 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks)
v7 = state3;
v7 = ADD_U64(v7, one_u64);
- i0 = __riscv_vle8_v_u8m1(src + 0 * 16, vl * 4);
- i1 = __riscv_vle8_v_u8m1(src + 1 * 16, vl * 4);
- i2 = __riscv_vle8_v_u8m1(src + 2 * 16, vl * 4);
- i3 = __riscv_vle8_v_u8m1(src + 3 * 16, vl * 4);
+ i0 = unaligned_load_u32m1(src + 0 * 16, vl);
+ i1 = unaligned_load_u32m1(src + 1 * 16, vl);
+ i2 = unaligned_load_u32m1(src + 2 * 16, vl);
+ i3 = unaligned_load_u32m1(src + 3 * 16, vl);
for (i = 20; i > 0; i -= 2)
{
@@ -434,14 +462,10 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks)
v3 = __riscv_vadd_vv_u32m1(v3, state3, vl);
state3 = ADD_U64(state3, one_u64);
- v0 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i0),
- v0, vl);
- v1 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i1),
- v1, vl);
- v2 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i2),
- v2, vl);
- v3 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i3),
- v3, vl);
+ v0 = __riscv_vxor_vv_u32m1(i0, v0, vl);
+ v1 = __riscv_vxor_vv_u32m1(i1, v1, vl);
+ v2 = __riscv_vxor_vv_u32m1(i2, v2, vl);
+ v3 = __riscv_vxor_vv_u32m1(i3, v3, vl);
v4 = __riscv_vadd_vv_u32m1(v4, state0, vl);
v5 = __riscv_vadd_vv_u32m1(v5, state1, vl);
@@ -449,37 +473,25 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks)
v7 = __riscv_vadd_vv_u32m1(v7, state3, vl);
state3 = ADD_U64(state3, one_u64);
- i4 = __riscv_vle8_v_u8m1(src + 4 * 16, vl * 4);
- i5 = __riscv_vle8_v_u8m1(src + 5 * 16, vl * 4);
- i6 = __riscv_vle8_v_u8m1(src + 6 * 16, vl * 4);
- i7 = __riscv_vle8_v_u8m1(src + 7 * 16, vl * 4);
-
- __riscv_vse8_v_u8m1(dst + 0 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v0), vl * 4);
- __riscv_vse8_v_u8m1(dst + 1 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v1), vl * 4);
- __riscv_vse8_v_u8m1(dst + 2 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v2), vl * 4);
- __riscv_vse8_v_u8m1(dst + 3 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v3), vl * 4);
-
- v4 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i4),
- v4, vl);
- v5 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i5),
- v5, vl);
- v6 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i6),
- v6, vl);
- v7 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i7),
- v7, vl);
-
- __riscv_vse8_v_u8m1(dst + 4 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v4), vl * 4);
- __riscv_vse8_v_u8m1(dst + 5 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v5), vl * 4);
- __riscv_vse8_v_u8m1(dst + 6 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v6), vl * 4);
- __riscv_vse8_v_u8m1(dst + 7 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v7), vl * 4);
+ i4 = unaligned_load_u32m1(src + 4 * 16, vl);
+ i5 = unaligned_load_u32m1(src + 5 * 16, vl);
+ i6 = unaligned_load_u32m1(src + 6 * 16, vl);
+ i7 = unaligned_load_u32m1(src + 7 * 16, vl);
+
+ unaligned_store_u32m1(dst + 0 * 16, v0, vl);
+ unaligned_store_u32m1(dst + 1 * 16, v1, vl);
+ unaligned_store_u32m1(dst + 2 * 16, v2, vl);
+ unaligned_store_u32m1(dst + 3 * 16, v3, vl);
+
+ v4 = __riscv_vxor_vv_u32m1(i4, v4, vl);
+ v5 = __riscv_vxor_vv_u32m1(i5, v5, vl);
+ v6 = __riscv_vxor_vv_u32m1(i6, v6, vl);
+ v7 = __riscv_vxor_vv_u32m1(i7, v7, vl);
+
+ unaligned_store_u32m1(dst + 4 * 16, v4, vl);
+ unaligned_store_u32m1(dst + 5 * 16, v5, vl);
+ unaligned_store_u32m1(dst + 6 * 16, v6, vl);
+ unaligned_store_u32m1(dst + 7 * 16, v7, vl);
src += 2 * 64;
dst += 2 * 64;
@@ -495,10 +507,10 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks)
v2 = state2;
v3 = state3;
- i0 = __riscv_vle8_v_u8m1(src + 0 * 16, vl * 4);
- i1 = __riscv_vle8_v_u8m1(src + 1 * 16, vl * 4);
- i2 = __riscv_vle8_v_u8m1(src + 2 * 16, vl * 4);
- i3 = __riscv_vle8_v_u8m1(src + 3 * 16, vl * 4);
+ i0 = unaligned_load_u32m1(src + 0 * 16, vl);
+ i1 = unaligned_load_u32m1(src + 1 * 16, vl);
+ i2 = unaligned_load_u32m1(src + 2 * 16, vl);
+ i3 = unaligned_load_u32m1(src + 3 * 16, vl);
for (i = 20; i > 0; i -= 2)
{
@@ -513,22 +525,14 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks)
state3 = ADD_U64(state3, one_u64);
- v0 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i0),
- v0, vl);
- v1 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i1),
- v1, vl);
- v2 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i2),
- v2, vl);
- v3 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i3),
- v3, vl);
- __riscv_vse8_v_u8m1(dst + 0 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v0), vl * 4);
- __riscv_vse8_v_u8m1(dst + 1 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v1), vl * 4);
- __riscv_vse8_v_u8m1(dst + 2 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v2), vl * 4);
- __riscv_vse8_v_u8m1(dst + 3 * 16,
- __riscv_vreinterpret_v_u32m1_u8m1(v3), vl * 4);
+ v0 = __riscv_vxor_vv_u32m1(i0, v0, vl);
+ v1 = __riscv_vxor_vv_u32m1(i1, v1, vl);
+ v2 = __riscv_vxor_vv_u32m1(i2, v2, vl);
+ v3 = __riscv_vxor_vv_u32m1(i3, v3, vl);
+ unaligned_store_u32m1(dst + 0 * 16, v0, vl);
+ unaligned_store_u32m1(dst + 1 * 16, v1, vl);
+ unaligned_store_u32m1(dst + 2 * 16, v2, vl);
+ unaligned_store_u32m1(dst + 3 * 16, v3, vl);
src += 64;
dst += 64;
diff --git a/cipher/cipher-gcm-riscv-zvkg.c b/cipher/cipher-gcm-riscv-zvkg.c
index c459a6fb..1f0b3797 100644
--- a/cipher/cipher-gcm-riscv-zvkg.c
+++ b/cipher/cipher-gcm-riscv-zvkg.c
@@ -44,9 +44,12 @@
static ASM_FUNC_ATTR_INLINE vuint32m1_t
unaligned_load_u32m1(const void *ptr, size_t vl_u32)
{
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
size_t vl_bytes = vl_u32 * 4;
-
return cast_u8m1_u32m1(__riscv_vle8_v_u8m1(ptr, vl_bytes));
+#else
+ return __riscv_vle32_v_u32m1(ptr, vl_u32);
+#endif
}
static ASM_FUNC_ATTR_INLINE vuint32m1_t
diff --git a/cipher/rijndael-riscv-zvkned.c b/cipher/rijndael-riscv-zvkned.c
index ed3887ae..ee9c2de4 100644
--- a/cipher/rijndael-riscv-zvkned.c
+++ b/cipher/rijndael-riscv-zvkned.c
@@ -83,54 +83,74 @@ bswap128_u32m1(vuint32m1_t vec, size_t vl_u32)
__riscv_vrgather_vv_u8m1(cast_u32m1_u8m1(vec), bswap128, vl_bytes));
}
+static ASM_FUNC_ATTR_INLINE vuint64m1_t
+unaligned_load_u64m1(const void *ptr, size_t vl_u64)
+{
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ return cast_u8m1_u64m1(__riscv_vle8_v_u8m1(ptr, vl_u64 * 8));
+#else
+ return __riscv_vle64_v_u64m1(ptr, vl_u64);
+#endif
+}
+
static ASM_FUNC_ATTR_INLINE vuint32m1_t
unaligned_load_u32m1(const void *ptr, size_t vl_u32)
{
- size_t vl_bytes = vl_u32 * 4;
-
- return cast_u8m1_u32m1(__riscv_vle8_v_u8m1(ptr, vl_bytes));
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ return cast_u8m1_u32m1(__riscv_vle8_v_u8m1(ptr, vl_u32 * 4));
+#else
+ return __riscv_vle32_v_u32m1(ptr, vl_u32);
+#endif
}
static ASM_FUNC_ATTR_INLINE void
unaligned_store_u32m1(void *ptr, vuint32m1_t vec, size_t vl_u32)
{
- size_t vl_bytes = vl_u32 * 4;
-
- __riscv_vse8_v_u8m1(ptr, cast_u32m1_u8m1(vec), vl_bytes);
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ __riscv_vse8_v_u8m1(ptr, cast_u32m1_u8m1(vec), vl_u32 * 4);
+#else
+ __riscv_vse32_v_u32m1(ptr, vec, vl_u32);
+#endif
}
-static ASM_FUNC_ATTR_INLINE vuint32m4_t
-unaligned_load_u32m4(const void *ptr, size_t vl_u32)
+static ASM_FUNC_ATTR_INLINE vuint32m2_t
+unaligned_load_u32m2(const void *ptr, size_t vl_u32)
{
- size_t vl_bytes = vl_u32 * 4;
-
- return cast_u8m4_u32m4(__riscv_vle8_v_u8m4(ptr, vl_bytes));
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ return cast_u8m2_u32m2(__riscv_vle8_v_u8m2(ptr, vl_u32 * 4));
+#else
+ return __riscv_vle32_v_u32m2(ptr, vl_u32);
+#endif
}
static ASM_FUNC_ATTR_INLINE void
-unaligned_store_u32m4(void *ptr, vuint32m4_t vec, size_t vl_u32)
+unaligned_store_u32m2(void *ptr, vuint32m2_t vec, size_t vl_u32)
{
- size_t vl_bytes = vl_u32 * 4;
-
- __riscv_vse8_v_u8m4(ptr, cast_u32m4_u8m4(vec), vl_bytes);
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ __riscv_vse8_v_u8m2(ptr, cast_u32m2_u8m2(vec), vl_u32 * 4);
+#else
+ __riscv_vse32_v_u32m2(ptr, vec, vl_u32);
+#endif
}
-static ASM_FUNC_ATTR_INLINE vuint32m1_t
-vxor_u8_u32m1(vuint32m1_t a, vuint32m1_t b, size_t vl_u32)
+static ASM_FUNC_ATTR_INLINE vuint32m4_t
+unaligned_load_u32m4(const void *ptr, size_t vl_u32)
{
- size_t vl_bytes = vl_u32 * 4;
-
- return cast_u8m1_u32m1(__riscv_vxor_vv_u8m1(cast_u32m1_u8m1(a),
- cast_u32m1_u8m1(b), vl_bytes));
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ return cast_u8m4_u32m4(__riscv_vle8_v_u8m4(ptr, vl_u32 * 4));
+#else
+ return __riscv_vle32_v_u32m4(ptr, vl_u32);
+#endif
}
-static ASM_FUNC_ATTR_INLINE vuint32m4_t
-vxor_u8_u32m4(vuint32m4_t a, vuint32m4_t b, size_t vl_u32)
+static ASM_FUNC_ATTR_INLINE void
+unaligned_store_u32m4(void *ptr, vuint32m4_t vec, size_t vl_u32)
{
- size_t vl_bytes = vl_u32 * 4;
-
- return cast_u8m4_u32m4(__riscv_vxor_vv_u8m4(cast_u32m4_u8m4(a),
- cast_u32m4_u8m4(b), vl_bytes));
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ __riscv_vse8_v_u8m4(ptr, cast_u32m4_u8m4(vec), vl_u32 * 4);
+#else
+ __riscv_vse32_v_u32m4(ptr, vec, vl_u32);
+#endif
}
static ASM_FUNC_ATTR_INLINE vuint32m4_t
@@ -660,29 +680,28 @@ _gcry_aes_riscv_zvkned_cfb_enc (void *context, unsigned char *iv_arg,
const u32 *rk = ctx->keyschenc32[0];
int rounds = ctx->rounds;
size_t vl = 4;
- size_t vl_bytes = vl * 4;
vuint32m1_t iv;
ROUND_KEY_VARIABLES;
PRELOAD_ROUND_KEYS (rk, rounds, vl);
- iv = unaligned_load_u32m1(iv_arg, vl);
+ iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl);
for (; nblocks; nblocks--)
{
- vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes);
+ vuint32m1_t data = unaligned_load_u32m1(inbuf, vl);
AES_CRYPT(e, m1, rounds, iv, vl);
- data = __riscv_vxor_vv_u8m1(cast_u32m1_u8m1(iv), data, vl_bytes);
- __riscv_vse8_v_u8m1(outbuf, data, vl_bytes);
- iv = cast_u8m1_u32m1(data);
+ data = __riscv_vxor_vv_u32m1(iv, data, vl);
+ unaligned_store_u32m1(outbuf, data, vl);
+ iv = data;
outbuf += BLOCKSIZE;
inbuf += BLOCKSIZE;
}
- unaligned_store_u32m1(iv_arg, iv, vl);
+ __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl);
clear_vec_regs();
}
@@ -699,29 +718,27 @@ _gcry_aes_riscv_zvkned_cbc_enc (void *context, unsigned char *iv_arg,
const u32 *rk = ctx->keyschenc32[0];
int rounds = ctx->rounds;
size_t vl = 4;
- size_t vl_bytes = vl * 4;
vuint32m1_t iv;
ROUND_KEY_VARIABLES;
PRELOAD_ROUND_KEYS (rk, rounds, vl);
- iv = unaligned_load_u32m1(iv_arg, vl);
+ iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl);
for (; nblocks; nblocks--)
{
- vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes);
- iv = cast_u8m1_u32m1(
- __riscv_vxor_vv_u8m1(data, cast_u32m1_u8m1(iv), vl_bytes));
+ vuint32m1_t data = unaligned_load_u32m1(inbuf, vl);
+ iv = __riscv_vxor_vv_u32m1(data, iv, vl);
AES_CRYPT(e, m1, rounds, iv, vl);
- __riscv_vse8_v_u8m1(outbuf, cast_u32m1_u8m1(iv), vl_bytes);
+ unaligned_store_u32m1(outbuf, iv, vl);
inbuf += BLOCKSIZE;
outbuf += outbuf_add;
}
- unaligned_store_u32m1(iv_arg, iv, vl);
+ __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl);
clear_vec_regs();
}
@@ -731,7 +748,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
void *outbuf_arg, const void *inbuf_arg,
size_t nblocks)
{
- static const byte add_u8_array[4][16] =
+ static const byte add_u8_array[4][16] __attribute__ ((aligned (16))) =
{
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 },
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 },
@@ -745,7 +762,6 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
const u32 *rk = ctx->keyschenc32[0];
int rounds = ctx->rounds;
size_t vl = 4;
- size_t vl_bytes = vl * 4;
u64 ctrlow;
vuint32m1_t ctr;
vuint32m1_t add1;
@@ -753,20 +769,17 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
PRELOAD_ROUND_KEYS (rk, rounds, vl);
- add1 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[0], vl_bytes));
- ctr = unaligned_load_u32m1(ctr_arg, vl);
+ add1 = __riscv_vle32_v_u32m1((const void *)add_u8_array[0], vl);
+ ctr = __riscv_vle32_v_u32m1((void *)ctr_arg, vl);
ctrlow = __riscv_vmv_x_s_u64m1_u64(cast_u32m1_u64m1(bswap128_u32m1(ctr, vl)));
memory_barrier_with_vec(add1);
if (nblocks >= 4)
{
- vuint32m1_t add2 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[1],
- vl_bytes));
- vuint32m1_t add3 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[2],
- vl_bytes));
- vuint32m1_t add4 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[3],
- vl_bytes));
+ vuint32m1_t add2 = __riscv_vle32_v_u32m1((const void *)add_u8_array[1], vl);
+ vuint32m1_t add3 = __riscv_vle32_v_u32m1((const void *)add_u8_array[2], vl);
+ vuint32m1_t add4 = __riscv_vle32_v_u32m1((const void *)add_u8_array[3], vl);
memory_barrier_with_vec(add2);
memory_barrier_with_vec(add3);
@@ -774,7 +787,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
for (; nblocks >= 4; nblocks -= 4)
{
- vuint8m4_t data4blks;
+ vuint32m4_t data4blks;
vuint32m4_t ctr4blks;
/* detect if 8-bit carry handling is needed */
@@ -819,13 +832,12 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
ctr4blks = merge_4x_u32m1_to_u32m4(ctr0, ctr1, ctr2, ctr3);
}
- data4blks = __riscv_vle8_v_u8m4(inbuf, vl_bytes * 4);
+ data4blks = unaligned_load_u32m4(inbuf, vl * 4);
AES_CRYPT(e, m4, rounds, ctr4blks, vl * 4);
- data4blks = __riscv_vxor_vv_u8m4(cast_u32m4_u8m4(ctr4blks), data4blks,
- vl_bytes * 4);
- __riscv_vse8_v_u8m4(outbuf, data4blks, vl_bytes * 4);
+ data4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4);
+ unaligned_store_u32m4(outbuf, data4blks, vl * 4);
inbuf += 4 * BLOCKSIZE;
outbuf += 4 * BLOCKSIZE;
@@ -835,7 +847,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
for (; nblocks; nblocks--)
{
vuint32m1_t block = ctr;
- vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes);
+ vuint32m1_t data = unaligned_load_u32m1(inbuf, vl);
/* detect if 8-bit carry handling is needed */
if (UNLIKELY((++ctrlow & 0xff) == 0))
@@ -859,14 +871,14 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
AES_CRYPT(e, m1, rounds, block, vl);
- data = __riscv_vxor_vv_u8m1(cast_u32m1_u8m1(block), data, vl_bytes);
- __riscv_vse8_v_u8m1(outbuf, data, vl_bytes);
+ data = __riscv_vxor_vv_u32m1(block, data, vl);
+ unaligned_store_u32m1(outbuf, data, vl);
inbuf += BLOCKSIZE;
outbuf += BLOCKSIZE;
}
- unaligned_store_u32m1(ctr_arg, ctr, vl);
+ __riscv_vse32_v_u32m1((void *)ctr_arg, ctr, vl);
clear_vec_regs();
}
@@ -886,7 +898,6 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg,
const u32 *rk = ctx->keyschenc32[0];
int rounds = ctx->rounds;
size_t vl = 4;
- size_t vl_bytes = vl * 4;
vuint32m1_t ctr;
vuint32m1_t add1;
ROUND_KEY_VARIABLES;
@@ -894,7 +905,7 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg,
PRELOAD_ROUND_KEYS (rk, rounds, vl);
add1 = __riscv_vle32_v_u32m1(add_u32_array[0], vl);
- ctr = unaligned_load_u32m1(ctr_arg, vl);
+ ctr = __riscv_vle32_v_u32m1((void *)ctr_arg, vl);
memory_barrier_with_vec(add1);
@@ -914,18 +925,17 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg,
vuint32m1_t ctr2 = __riscv_vadd_vv_u32m1(ctr, add2, vl);
vuint32m1_t ctr3 = __riscv_vadd_vv_u32m1(ctr, add3, vl);
vuint32m4_t ctr4blks;
- vuint8m4_t data4blks;
+ vuint32m4_t data4blks;
ctr4blks = merge_4x_u32m1_to_u32m4(ctr, ctr1, ctr2, ctr3);
ctr = __riscv_vadd_vv_u32m1(ctr, add4, vl);
- data4blks = __riscv_vle8_v_u8m4(inbuf, vl_bytes * 4);
+ data4blks = unaligned_load_u32m4(inbuf, vl * 4);
AES_CRYPT(e, m4, rounds, ctr4blks, vl * 4);
- data4blks = __riscv_vxor_vv_u8m4(cast_u32m4_u8m4(ctr4blks), data4blks,
- vl_bytes * 4);
- __riscv_vse8_v_u8m4(outbuf, data4blks, vl_bytes * 4);
+ data4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4);
+ unaligned_store_u32m4(outbuf, data4blks, vl * 4);
inbuf += 4 * BLOCKSIZE;
outbuf += 4 * BLOCKSIZE;
@@ -935,20 +945,20 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg,
for (; nblocks; nblocks--)
{
vuint32m1_t block = ctr;
- vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes);
+ vuint32m1_t data = unaligned_load_u32m1(inbuf, vl);
ctr = __riscv_vadd_vv_u32m1(ctr, add1, vl);
AES_CRYPT(e, m1, rounds, block, vl);
- data = __riscv_vxor_vv_u8m1(cast_u32m1_u8m1(block), data, vl_bytes);
- __riscv_vse8_v_u8m1(outbuf, data, vl_bytes);
+ data = __riscv_vxor_vv_u32m1(block, data, vl);
+ unaligned_store_u32m1(outbuf, data, vl);
inbuf += BLOCKSIZE;
outbuf += BLOCKSIZE;
}
- unaligned_store_u32m1(ctr_arg, ctr, vl);
+ __riscv_vse32_v_u32m1((void *)ctr_arg, ctr, vl);
clear_vec_regs();
}
@@ -969,7 +979,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg,
PRELOAD_ROUND_KEYS (rk, rounds, vl);
- iv = unaligned_load_u32m1(iv_arg, vl);
+ iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl);
for (; nblocks >= 4; nblocks -= 4)
{
@@ -983,7 +993,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg,
AES_CRYPT(e, m4, rounds, iv4blks, vl * 4);
- data4blks = vxor_u8_u32m4(iv4blks, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(iv4blks, data4blks, vl * 4);
unaligned_store_u32m4(outbuf, data4blks, vl * 4);
inbuf += 4 * BLOCKSIZE;
@@ -997,7 +1007,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg,
AES_CRYPT(e, m1, rounds, iv, vl);
- data = vxor_u8_u32m1(iv, data, vl);
+ data = __riscv_vxor_vv_u32m1(iv, data, vl);
unaligned_store_u32m1(outbuf, data, vl);
iv = new_iv;
@@ -1005,7 +1015,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg,
outbuf += BLOCKSIZE;
}
- unaligned_store_u32m1(iv_arg, iv, vl);
+ __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl);
clear_vec_regs();
}
@@ -1032,7 +1042,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg,
PRELOAD_ROUND_KEYS (rk, rounds, vl);
- iv = unaligned_load_u32m1(iv_arg, vl);
+ iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl);
for (; nblocks >= 4; nblocks -= 4)
{
@@ -1045,7 +1055,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg,
AES_CRYPT(d, m4, rounds, data4blks, vl * 4);
- data4blks = vxor_u8_u32m4(iv4blks, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(iv4blks, data4blks, vl * 4);
unaligned_store_u32m4(outbuf, data4blks, vl * 4);
inbuf += 4 * BLOCKSIZE;
@@ -1059,7 +1069,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg,
AES_CRYPT(d, m1, rounds, data, vl);
- data = vxor_u8_u32m1(iv, data, vl);
+ data = __riscv_vxor_vv_u32m1(iv, data, vl);
unaligned_store_u32m1(outbuf, data, vl);
iv = new_iv;
@@ -1067,7 +1077,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg,
outbuf += BLOCKSIZE;
}
- unaligned_store_u32m1(iv_arg, iv, vl);
+ __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl);
clear_vec_regs();
}
@@ -1083,7 +1093,6 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg,
const u32 *rk = ctx->keyschenc32[0];
int rounds = ctx->rounds;
size_t vl = 4;
- size_t vl_bytes = vl * 4;
vuint32m1_t iv;
vuint32m1_t ctr;
ROUND_KEY_VARIABLES;
@@ -1091,8 +1100,8 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg,
PRELOAD_ROUND_KEYS (rk, rounds, vl);
/* Preload Offset and Checksum */
- iv = unaligned_load_u32m1(c->u_iv.iv, vl);
- ctr = unaligned_load_u32m1(c->u_ctr.ctr, vl);
+ iv = __riscv_vle32_v_u32m1((void *)c->u_iv.iv, vl);
+ ctr = __riscv_vle32_v_u32m1((void *)c->u_ctr.ctr, vl);
if (nblocks >= 4)
{
@@ -1101,44 +1110,44 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg,
for (; nblocks >= 4; nblocks -= 4)
{
- const unsigned char *l;
- vuint8m1_t l_ntzi;
+ const void *l;
+ vuint32m1_t l_ntzi;
vuint32m4_t data4blks = unaligned_load_u32m4(inbuf, vl * 4);
vuint32m1_t offset0, offset1, offset2, offset3;
vuint32m4_t offsets;
/* Checksum_i = Checksum_{i-1} xor P_i */
- ctr4blks = vxor_u8_u32m4(ctr4blks, data4blks, vl * 4);
+ ctr4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4);
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset0 = iv;
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset1 = iv;
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset2 = iv;
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset3 = iv;
offsets = merge_4x_u32m1_to_u32m4(offset0, offset1, offset2, offset3);
- data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4);
AES_CRYPT(e, m4, rounds, data4blks, vl * 4);
- data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4);
unaligned_store_u32m4(outbuf, data4blks, vl * 4);
@@ -1149,35 +1158,35 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg,
/* Checksum_i = Checksum_{i-1} xor P_i */
{
vuint32m1x4_t ctr0123 = split_u32m4_to_4x_u32m1(ctr4blks);
- ctr = vxor_u8_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0),
+ ctr = __riscv_vxor_vv_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0),
__riscv_vget_v_u32m1x4_u32m1(ctr0123, 1), vl);
- ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl);
- ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl);
+ ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl);
+ ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl);
}
}
for (; nblocks; nblocks--)
{
- const unsigned char *l;
- vuint8m1_t l_ntzi;
+ const void *l;
+ vuint32m1_t l_ntzi;
vuint32m1_t data;
data = unaligned_load_u32m1(inbuf, vl);
/* Checksum_i = Checksum_{i-1} xor P_i */
- ctr = vxor_u8_u32m1(ctr, data, vl);
+ ctr = __riscv_vxor_vv_u32m1(ctr, data, vl);
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
- data = vxor_u8_u32m1(data, iv, vl);
+ data = __riscv_vxor_vv_u32m1(data, iv, vl);
AES_CRYPT(e, m1, rounds, data, vl);
- data = vxor_u8_u32m1(iv, data, vl);
+ data = __riscv_vxor_vv_u32m1(iv, data, vl);
unaligned_store_u32m1(outbuf, data, vl);
inbuf += BLOCKSIZE;
@@ -1186,8 +1195,8 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg,
c->u_mode.ocb.data_nblocks = n;
- unaligned_store_u32m1(c->u_iv.iv, iv, vl);
- unaligned_store_u32m1(c->u_ctr.ctr, ctr, vl);
+ __riscv_vse32_v_u32m1((void *)c->u_iv.iv, iv, vl);
+ __riscv_vse32_v_u32m1((void *)c->u_ctr.ctr, ctr, vl);
clear_vec_regs();
@@ -1205,7 +1214,6 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg,
const u32 *rk = ctx->keyschdec32[0];
int rounds = ctx->rounds;
size_t vl = 4;
- size_t vl_bytes = vl * 4;
vuint32m1_t iv;
vuint32m1_t ctr;
ROUND_KEY_VARIABLES;
@@ -1219,8 +1227,8 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg,
PRELOAD_ROUND_KEYS (rk, rounds, vl);
/* Preload Offset and Checksum */
- iv = unaligned_load_u32m1(c->u_iv.iv, vl);
- ctr = unaligned_load_u32m1(c->u_ctr.ctr, vl);
+ iv = __riscv_vle32_v_u32m1((void *)c->u_iv.iv, vl);
+ ctr = __riscv_vle32_v_u32m1((void *)c->u_ctr.ctr, vl);
if (nblocks >= 4)
{
@@ -1229,8 +1237,8 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg,
for (; nblocks >= 4; nblocks -= 4)
{
- const unsigned char *l;
- vuint8m1_t l_ntzi;
+ const void *l;
+ vuint32m1_t l_ntzi;
vuint32m4_t data4blks = unaligned_load_u32m4(inbuf, vl * 4);
vuint32m1_t offset0, offset1, offset2, offset3;
vuint32m4_t offsets;
@@ -1238,37 +1246,37 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg,
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
/* P_i = Offset_i xor ENCIPHER(K, C_i xor Offset_i) */
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset0 = iv;
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset1 = iv;
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset2 = iv;
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset3 = iv;
offsets = merge_4x_u32m1_to_u32m4(offset0, offset1, offset2, offset3);
- data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4);
AES_CRYPT(d, m4, rounds, data4blks, vl * 4);
- data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4);
unaligned_store_u32m4(outbuf, data4blks, vl * 4);
/* Checksum_i = Checksum_{i-1} xor P_i */
- ctr4blks = vxor_u8_u32m4(ctr4blks, data4blks, vl * 4);
+ ctr4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4);
inbuf += 4 * BLOCKSIZE;
outbuf += 4 * BLOCKSIZE;
@@ -1277,37 +1285,34 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg,
/* Checksum_i = Checksum_{i-1} xor P_i */
{
vuint32m1x4_t ctr0123 = split_u32m4_to_4x_u32m1(ctr4blks);
- ctr = vxor_u8_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0),
+ ctr = __riscv_vxor_vv_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0),
__riscv_vget_v_u32m1x4_u32m1(ctr0123, 1), vl);
- ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl);
- ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl);
+ ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl);
+ ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl);
}
}
for (; nblocks; nblocks--)
{
- const unsigned char *l;
- vuint8m1_t l_ntzi;
- vuint8m1_t data;
- vuint32m1_t block;
-
- l = ocb_get_l(c, ++n);
+ const void *l;
+ vuint32m1_t l_ntzi;
+ vuint32m1_t data;
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
/* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- data = __riscv_vle8_v_u8m1(inbuf, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
- data = __riscv_vxor_vv_u8m1(data, cast_u32m1_u8m1(iv), vl_bytes);
- block = cast_u8m1_u32m1(data);
+ l = ocb_get_l(c, ++n);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ data = unaligned_load_u32m1(inbuf, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
+ data = __riscv_vxor_vv_u32m1(data, iv, vl);
- AES_CRYPT(d, m1, rounds, block, vl);
+ AES_CRYPT(d, m1, rounds, data, vl);
- block = vxor_u8_u32m1(iv, block, vl);
- unaligned_store_u32m1(outbuf, block, vl);
+ data = __riscv_vxor_vv_u32m1(iv, data, vl);
+ unaligned_store_u32m1(outbuf, data, vl);
/* Checksum_i = Checksum_{i-1} xor P_i */
- ctr = vxor_u8_u32m1(ctr, block, vl);
+ ctr = __riscv_vxor_vv_u32m1(ctr, data, vl);
inbuf += BLOCKSIZE;
outbuf += BLOCKSIZE;
@@ -1315,8 +1320,8 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg,
c->u_mode.ocb.data_nblocks = n;
- unaligned_store_u32m1(c->u_iv.iv, iv, vl);
- unaligned_store_u32m1(c->u_ctr.ctr, ctr, vl);
+ __riscv_vse32_v_u32m1((void *)c->u_iv.iv, iv, vl);
+ __riscv_vse32_v_u32m1((void *)c->u_ctr.ctr, ctr, vl);
clear_vec_regs();
@@ -1344,7 +1349,6 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg,
const u32 *rk = ctx->keyschenc32[0];
int rounds = ctx->rounds;
size_t vl = 4;
- size_t vl_bytes = vl * 4;
vuint32m1_t iv;
vuint32m1_t ctr;
ROUND_KEY_VARIABLES;
@@ -1352,8 +1356,8 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg,
PRELOAD_ROUND_KEYS (rk, rounds, vl);
/* Preload Offset and Sum */
- iv = unaligned_load_u32m1(c->u_mode.ocb.aad_offset, vl);
- ctr = unaligned_load_u32m1(c->u_mode.ocb.aad_sum, vl);
+ iv = __riscv_vle32_v_u32m1((void *)c->u_mode.ocb.aad_offset, vl);
+ ctr = __riscv_vle32_v_u32m1((void *)c->u_mode.ocb.aad_sum, vl);
if (nblocks >= 4)
{
@@ -1362,8 +1366,8 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg,
for (; nblocks >= 4; nblocks -= 4)
{
- const unsigned char *l;
- vuint8m1_t l_ntzi;
+ const void *l;
+ vuint32m1_t l_ntzi;
vuint32m4_t data4blks = unaligned_load_u32m4(abuf, vl * 4);
vuint32m1_t offset0, offset1, offset2, offset3;
vuint32m4_t offsets;
@@ -1371,32 +1375,32 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg,
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset0 = iv;
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset1 = iv;
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset2 = iv;
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
offset3 = iv;
offsets = merge_4x_u32m1_to_u32m4(offset0, offset1, offset2, offset3);
- data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4);
AES_CRYPT(e, m4, rounds, data4blks, vl * 4);
- ctr4blks = vxor_u8_u32m4(ctr4blks, data4blks, vl * 4);
+ ctr4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4);
abuf += 4 * BLOCKSIZE;
}
@@ -1404,17 +1408,17 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg,
/* Checksum_i = Checksum_{i-1} xor P_i */
{
vuint32m1x4_t ctr0123 = split_u32m4_to_4x_u32m1(ctr4blks);
- ctr = vxor_u8_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0),
+ ctr = __riscv_vxor_vv_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0),
__riscv_vget_v_u32m1x4_u32m1(ctr0123, 1), vl);
- ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl);
- ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl);
+ ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl);
+ ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl);
}
}
for (; nblocks; nblocks--)
{
- const unsigned char *l;
- vuint8m1_t l_ntzi;
+ const void *l;
+ vuint32m1_t l_ntzi;
vuint32m1_t data;
data = unaligned_load_u32m1(abuf, vl);
@@ -1422,22 +1426,22 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg,
/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */
l = ocb_get_l(c, ++n);
- l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes);
- iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl);
+ l_ntzi = __riscv_vle32_v_u32m1(l, vl);
+ iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl);
- data = vxor_u8_u32m1(data, iv, vl);
+ data = __riscv_vxor_vv_u32m1(data, iv, vl);
AES_CRYPT(e, m1, rounds, data, vl);
- ctr = vxor_u8_u32m1(ctr, data, vl);
+ ctr = __riscv_vxor_vv_u32m1(ctr, data, vl);
abuf += BLOCKSIZE;
}
c->u_mode.ocb.aad_nblocks = n;
- unaligned_store_u32m1(c->u_mode.ocb.aad_offset, iv, vl);
- unaligned_store_u32m1(c->u_mode.ocb.aad_sum, ctr, vl);
+ __riscv_vse32_v_u32m1((void *)c->u_mode.ocb.aad_offset, iv, vl);
+ __riscv_vse32_v_u32m1((void *)c->u_mode.ocb.aad_sum, ctr, vl);
clear_vec_regs();
@@ -1482,7 +1486,7 @@ aes_riscv_xts_enc (void *context, unsigned char *tweak_arg, void *outbuf_arg,
PRELOAD_ROUND_KEYS (rk, rounds, vl);
/* Preload tweak */
- tweak = unaligned_load_u32m1(tweak_arg, vl);
+ tweak = __riscv_vle32_v_u32m1((void *)tweak_arg, vl);
memory_barrier_with_vec(xts_gfmul);
memory_barrier_with_vec(xts_swap64);
@@ -1504,11 +1508,11 @@ aes_riscv_xts_enc (void *context, unsigned char *tweak_arg, void *outbuf_arg,
tweaks = merge_4x_u32m1_to_u32m4(tweak0, tweak1, tweak2, tweak3);
- data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4);
AES_CRYPT(e, m4, rounds, data4blks, vl * 4);
- data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4);
unaligned_store_u32m4(outbuf, data4blks, vl * 4);
@@ -1521,19 +1525,19 @@ aes_riscv_xts_enc (void *context, unsigned char *tweak_arg, void *outbuf_arg,
vuint32m1_t data = unaligned_load_u32m1(inbuf, vl);
vuint32m1_t tweak0 = tweak;
- data = vxor_u8_u32m1(data, tweak0, vl);
+ data = __riscv_vxor_vv_u32m1(data, tweak0, vl);
tweak = xts_gfmul_byA(tweak, xts_gfmul, xts_swap64, vl);
AES_CRYPT(e, m1, rounds, data, vl);
- data = vxor_u8_u32m1(data, tweak0, vl);
+ data = __riscv_vxor_vv_u32m1(data, tweak0, vl);
unaligned_store_u32m1(outbuf, data, vl);
inbuf += BLOCKSIZE;
outbuf += BLOCKSIZE;
}
- unaligned_store_u32m1(tweak_arg, tweak, vl);
+ __riscv_vse32_v_u32m1((void *)tweak_arg, tweak, vl);
clear_vec_regs();
}
@@ -1562,7 +1566,7 @@ aes_riscv_xts_dec (void *context, unsigned char *tweak_arg, void *outbuf_arg,
PRELOAD_ROUND_KEYS (rk, rounds, vl);
/* Preload tweak */
- tweak = unaligned_load_u32m1(tweak_arg, vl);
+ tweak = __riscv_vle32_v_u32m1((void *)tweak_arg, vl);
memory_barrier_with_vec(xts_gfmul);
memory_barrier_with_vec(xts_swap64);
@@ -1584,11 +1588,11 @@ aes_riscv_xts_dec (void *context, unsigned char *tweak_arg, void *outbuf_arg,
tweaks = merge_4x_u32m1_to_u32m4(tweak0, tweak1, tweak2, tweak3);
- data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4);
AES_CRYPT(d, m4, rounds, data4blks, vl * 4);
- data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4);
+ data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4);
unaligned_store_u32m4(outbuf, data4blks, vl * 4);
@@ -1601,19 +1605,19 @@ aes_riscv_xts_dec (void *context, unsigned char *tweak_arg, void *outbuf_arg,
vuint32m1_t data = unaligned_load_u32m1(inbuf, vl);
vuint32m1_t tweak0 = tweak;
- data = vxor_u8_u32m1(data, tweak0, vl);
+ data = __riscv_vxor_vv_u32m1(data, tweak0, vl);
tweak = xts_gfmul_byA(tweak, xts_gfmul, xts_swap64, vl);
AES_CRYPT(d, m1, rounds, data, vl);
- data = vxor_u8_u32m1(data, tweak0, vl);
+ data = __riscv_vxor_vv_u32m1(data, tweak0, vl);
unaligned_store_u32m1(outbuf, data, vl);
inbuf += BLOCKSIZE;
outbuf += BLOCKSIZE;
}
- unaligned_store_u32m1(tweak_arg, tweak, vl);
+ __riscv_vse32_v_u32m1((void *)tweak_arg, tweak, vl);
clear_vec_regs();
}
diff --git a/cipher/sha256-riscv-zvknha-zvkb.c b/cipher/sha256-riscv-zvknha-zvkb.c
index 6375f9aa..0ff80cf7 100644
--- a/cipher/sha256-riscv-zvknha-zvkb.c
+++ b/cipher/sha256-riscv-zvknha-zvkb.c
@@ -76,11 +76,14 @@ working_vsha2cl_vv_u32m1(vuint32m1_t hgcd, vuint32m1_t feba,
static ASM_FUNC_ATTR_INLINE vuint32m1_t
-load_and_swap (const byte * p, size_t vl, size_t vl_bytes)
+load_and_swap (const byte * p, size_t vl)
{
- vuint8m1_t temp_bytes = __riscv_vle8_v_u8m1(p, vl_bytes);
- return __riscv_vrev8_v_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(temp_bytes),
- vl);
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ vuint8m1_t temp = __riscv_vle8_v_u8m1(p, vl * 4);
+ return __riscv_vrev8_v_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(temp), vl);
+#else
+ return __riscv_vrev8_v_u32m1(__riscv_vle32_v_u32m1((const void *)p, vl), vl);
+#endif
}
@@ -111,7 +114,6 @@ sha256_transform_zvknha_zvkb (u32 state[8], const uint8_t * data,
static const int feba_offset = 0;
static const int hgcd_offset = 8 / sizeof(u32);
size_t vl;
- size_t vl_bytes;
vuint32m1_t idx;
vuint32m1_t v_feba_work, v_feba;
vuint32m1_t v_hgcd_work, v_hgcd;
@@ -121,7 +123,6 @@ sha256_transform_zvknha_zvkb (u32 state[8], const uint8_t * data,
vuint32m1_t v_feba_hgcd_idx;
vl = 4;
- vl_bytes = vl * 4;
idx = __riscv_vid_v_u32m1(vl);
merge_mask = __riscv_vmseq_vx_u32m1_b32(idx, 0, vl);
@@ -137,10 +138,10 @@ sha256_transform_zvknha_zvkb (u32 state[8], const uint8_t * data,
v_feba_work = v_feba;
v_hgcd_work = v_hgcd;
- w0 = load_and_swap(data + 0, vl, vl_bytes);
- w1 = load_and_swap(data + 16, vl, vl_bytes);
- w2 = load_and_swap(data + 32, vl, vl_bytes);
- w3 = load_and_swap(data + 48, vl, vl_bytes);
+ w0 = load_and_swap(data + 0, vl);
+ w1 = load_and_swap(data + 16, vl);
+ w2 = load_and_swap(data + 32, vl);
+ w3 = load_and_swap(data + 48, vl);
QUAD_ROUND_W_SCHED(w0, w1, w2, w3);
QUAD_ROUND_W_SCHED(w1, w2, w3, w0);
diff --git a/cipher/sha512-riscv-zvknhb-zvkb.c b/cipher/sha512-riscv-zvknhb-zvkb.c
index 5f5d483a..fea8f32a 100644
--- a/cipher/sha512-riscv-zvknhb-zvkb.c
+++ b/cipher/sha512-riscv-zvknhb-zvkb.c
@@ -81,11 +81,14 @@ working_vsha2cl_vv_u64m2(vuint64m2_t hgcd, vuint64m2_t feba,
static ASM_FUNC_ATTR_INLINE vuint64m2_t
-load_and_swap(const byte *p, size_t vl, size_t vl_bytes)
+load_and_swap(const byte *p, size_t vl)
{
- vuint8m2_t temp_bytes = __riscv_vle8_v_u8m2(p, vl_bytes);
- return __riscv_vrev8_v_u64m2(__riscv_vreinterpret_v_u8m2_u64m2(temp_bytes),
- vl);
+#ifdef RVV_UNALIGNED_NOT_ALLOWED
+ vuint8m2_t temp = __riscv_vle8_v_u8m2(p, vl * 8);
+ return __riscv_vrev8_v_u64m2(__riscv_vreinterpret_v_u8m2_u64m2(temp), vl);
+#else
+ return __riscv_vrev8_v_u64m2(__riscv_vle64_v_u64m2((const void *)p, vl), vl);
+#endif
}
@@ -97,7 +100,6 @@ sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data,
static const int feba_offset = 0;
static const int hgcd_offset = 16 / sizeof(u64);
size_t vl;
- size_t vl_bytes;
vuint64m2_t idx;
vuint64m2_t v_feba_work, v_feba;
vuint64m2_t v_hgcd_work, v_hgcd;
@@ -107,7 +109,6 @@ sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data,
vuint64m2_t v_feba_hgcd_idx;
vl = 4;
- vl_bytes = vl * 8;
idx = __riscv_vid_v_u64m2(vl);
merge_mask = __riscv_vmseq_vx_u64m2_b32(idx, 0, vl);
@@ -124,10 +125,10 @@ sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data,
v_feba_work = v_feba;
v_hgcd_work = v_hgcd;
- w0 = load_and_swap(data + 0, vl, vl_bytes);
- w1 = load_and_swap(data + 32, vl, vl_bytes);
- w2 = load_and_swap(data + 64, vl, vl_bytes);
- w3 = load_and_swap(data + 96, vl, vl_bytes);
+ w0 = load_and_swap(data + 0, vl);
+ w1 = load_and_swap(data + 32, vl);
+ w2 = load_and_swap(data + 64, vl);
+ w3 = load_and_swap(data + 96, vl);
QUAD_ROUND_W_SCHED(w0, w1, w2, w3);
QUAD_ROUND_W_SCHED(w1, w2, w3, w0);
diff --git a/cipher/simd-common-riscv.h b/cipher/simd-common-riscv.h
index 8381000f..8ddd5056 100644
--- a/cipher/simd-common-riscv.h
+++ b/cipher/simd-common-riscv.h
@@ -23,6 +23,19 @@
#include <config.h>
+#undef RVV_UNALIGNED_NOT_ALLOWED
+#if defined(HAVE_COMPATIBLE_CC_RISCV_VECTOR_INTRINSICS_WITH_CFLAGS) || \
+ !(defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000)) || \
+ defined(__riscv_misaligned_avoid)
+/* Zicclsm extension means that unaligned vector (and scalar) memory accesses
+ * are allowed. If build configuration is such that RVV intrinsics support had
+ * to be enabled with custom CFLAGS, assume that vector unaligned accesses are
+ * not allowed. This is to workaround broken build setups with RVA22
+ * non-compliant Spacemit K1 (unaligned scalar allowed, unaligned vector
+ * not allowed). */
+#define RVV_UNALIGNED_NOT_ALLOWED 1
+#endif
+
#define memory_barrier_with_vec(a) __asm__("" : "+vr"(a) :: "memory")
#define clear_vec_regs() __asm__ volatile("vsetvli zero, %0, e8, m1, ta, ma;\n" \
diff --git a/configure.ac b/configure.ac
index 631ed81e..4ac3450a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2802,28 +2802,26 @@ fi
#
-# Check whether compiler supports RISC-V -mstrict-align flag
+# Check whether compiler has RISC-V Zicclsm extension enable
#
-_gcc_cflags_save=$CFLAGS
-# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads)
-CFLAGS="$CFLAGS -mstrict-align"
-
-AC_CACHE_CHECK([whether compiler supports RISC-V -mstrict-align flag],
- [gcry_cv_cc_riscv_mstrict_align],
+AC_CACHE_CHECK([whether compiler has RISC-V Zicclsm extension enabled],
+ [gcry_cv_cc_riscv_zicclsm_enabled],
[if test "$mpi_cpu_arch" != "riscv64" ||
test "$try_asm_modules" != "yes" ; then
- gcry_cv_cc_riscv_mstrict_align="n/a"
+ gcry_cv_cc_riscv_zicclsm_enabled="n/a"
else
- gcry_cv_cc_riscv_mstrict_align=no
- AC_COMPILE_IFELSE(
- [AC_LANG_SOURCE([[void testfn(void) { }]])],
- [gcry_cv_cc_riscv_mstrict_align=yes])
+ gcry_cv_cc_riscv_zicclsm_enabled=no
+ AC_LINK_IFELSE([AC_LANG_PROGRAM(
+ [[#if defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000)
+ int testfunc(void) { return 0; }
+ #endif
+ ]], [ testfunc(); ])],
+ [gcry_cv_cc_riscv_zicclsm_enabled=yes])
fi])
-AM_CONDITIONAL(SUPPORT_CC_RISCV_MSTRICT_ALIGN,
- test "$gcry_cv_cc_riscv_mstrict_align" = "yes")
-
-# Restore flags.
-CFLAGS=$_gcc_cflags_save;
+if test "$gcry_cv_cc_riscv_zicclsm_enabled" = "yes" ; then
+ AC_DEFINE(HAVE_COMPATIBLE_CC_RISCV_ZICCLSM_ENABLED,1,
+ [Defined if underlying compiler has RISC-V Zicclsm extension enabled])
+fi
#
@@ -2896,7 +2894,10 @@ _gcc_cflags_save=$CFLAGS
# better code generation for RISC-V vector implementations.
MARCH_RVA22U64_WITH_VEC=rv64imafdcv_zba_zbb_zbs
AC_SUBST([MARCH_RVA22U64_WITH_VEC])
-# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads)
+# Note: -mstrict-align needed for SpacemiT K1 RVV bug (K1 does not support
+# unaligned vector memory access). This is enabled only for plain RVV
+# implementations, RVV-crypto extension implementations detect support for
+# unaligned vector memory access based on Zicclsm extension.
CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA22U64_WITH_VEC -mstrict-align"
if test "$gcry_cv_cc_riscv_vector_intrinsics" = "no" &&
@@ -2992,6 +2993,13 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST],
vuint32m1_t d = __riscv_vghsh_vv_u32m1(a, b, c, vl);
__riscv_vse32_v_u32m1(ptr + 0 * vl, d, vl);
}
+ void test_rev8(unsigned int *ptr)
+ {
+ int vl = __riscv_vsetvl_e32m1 (4);
+ vuint32m1_t a = __riscv_vle32_v_u32m1(ptr + 0 * vl, vl);
+ vuint32m1_t b = __riscv_vrev8_v_u32m1(a, vl);
+ __riscv_vse32_v_u32m1(ptr + 0 * vl, b, vl);
+ }
void test_inline_vec_asm(unsigned int *ptr)
{
int vl = __riscv_vsetvl_e32m1 (4);
@@ -3013,6 +3021,7 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST],
test_aes_key(buf);
test_aes_crypt(buf);
test_ghash(buf);
+ test_rev8(buf);
test_inline_vec_asm(buf);
return 0;
}
@@ -3039,16 +3048,19 @@ fi
_gcc_cflags_save=$CFLAGS
# Enable B extension (Zba+Zbb+Zbs) to align with RVA23U64 profile and for
# better code generation for RISC-V vector implementations.
-MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC}
-MARCH_RVA23U64_WITH_VEC_CRYPTO=${MARCH_RVA23U64_BASE}_zvbc_zvkg_zvkn_zvks
+if test "$gcry_cv_cc_riscv_zicclsm_enabled" = "yes" ; then
+ MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC}_zicclsm
+else
+ MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC}
+fi
+MARCH_RVA23U64_WITH_VEC_CRYPTO=${MARCH_RVA23U64_BASE}_zvbc_zvkb_zvkg_zvkn_zvks
AC_SUBST([MARCH_RVA23U64_BASE])
-# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads)
-CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO -mstrict-align"
+CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO"
if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics" = "no" &&
test "$mpi_cpu_arch" = "riscv64" &&
test "$try_asm_modules" = "yes" ; then
- AC_CACHE_CHECK([whether compiler supports RISC-V vector intrinsics with extra GCC flags],
+ AC_CACHE_CHECK([whether compiler supports RISC-V vector cryptography intrinsics with extra GCC flags],
[gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags],
[gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags=no
AC_LINK_IFELSE(
@@ -3082,7 +3094,7 @@ if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics" = "yes" ||
# Setup flags for test if needed.
_gcc_cflags_save=$CFLAGS
if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags" = "yes"; then
- CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO -mstrict-align"
+ CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO"
fi
# Disable LTO optimization for following check to generate
@@ -3131,7 +3143,7 @@ if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics" = "yes" ||
# Setup flags for test.
_gcc_cflags_save=$CFLAGS
if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags" = "yes"; then
- CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO -mstrict-align"
+ CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO"
else
CFLAGS="$CFLAGS -O2"
fi
--
2.53.0
More information about the Gcrypt-devel
mailing list