[PATCH 10/12] rijndael-vp-riscv: use LMUL that gives 128-bit register group
Jussi Kivilinna
jussi.kivilinna at iki.fi
Sun Oct 11 19:23:19 CEST 2026
* cipher/Makefile.am: Add 'rijndael-vp-riscv.h',
'rijndael-vp-riscv-mf2.c', 'rijndael-vp-riscv-mf4.c' and
'rijndael-vp-riscv-mf8.c'.
* cipher/rijndael-internal.h (rijndael_setkeyfn_t): New.
* cipher/rijndael-vp-riscv-mf2.c: New.
* cipher/rijndael-vp-riscv-mf4.c: New.
* cipher/rijndael-vp-riscv-mf8.c: New.
* cipher/rijndael-vp-riscv.h: New.
* cipher/rijndael-vp-riscv.c: Move SIMD128 macros and function names to
'rijndael-vp-riscv.h' and build LMUL=1 variant.
(_gcry_aes_vp_riscv_setup_acceleration): Add 'hw_setkey' and 'bulk_ops'
parameters and setup functions from variant selected by VLEN.
* cipher/rijndael-vp-simd128.h [HAVE_SIMD256]: Remove.
(FUNC_CFB_ENC, FUNC_CBC_ENC, FUNC_CTR_ENC, FUNC_CTR32LE_ENC)
(FUNC_CFB_DEC, FUNC_CBC_DEC): Use bulk function prototypes.
(aes_simd128_ecb_enc, aes_simd128_ecb_dec): Make static.
* cipher/rijndael.c [USE_VP_RISCV] (_gcry_aes_vp_riscv_do_setkey)
(_gcry_aes_vp_riscv_prepare_decryption, _gcry_aes_vp_riscv_encrypt)
(_gcry_aes_vp_riscv_decrypt, _gcry_aes_vp_riscv_cfb_enc)
(_gcry_aes_vp_riscv_cbc_enc, _gcry_aes_vp_riscv_ctr_enc)
(_gcry_aes_vp_riscv_ctr32le_enc, _gcry_aes_vp_riscv_cfb_dec)
(_gcry_aes_vp_riscv_cbc_dec, _gcry_aes_vp_riscv_ocb_crypt)
(_gcry_aes_vp_riscv_ocb_auth, _gcry_aes_vp_riscv_ecb_crypt)
(_gcry_aes_vp_riscv_xts_crypt): Remove.
(_gcry_aes_vp_riscv_setup_acceleration): Add 'hw_setkey' and 'bulk_ops'
parameters.
(do_setkey) [USE_VP_RISCV]: Move function setup to
_gcry_aes_vp_riscv_setup_acceleration.
* configure.ac: Add 'rijndael-vp-riscv-mf2.lo',
'rijndael-vp-riscv-mf4.lo' and 'rijndael-vp-riscv-mf8.lo'.
--
Vector permute AES used LMUL=1 for 128-bit vectors and, with VLEN>=256,
LMUL=1 for 256-bit vectors. On SpacemiT K1 and K3, vrgather cost grows
with register group size, which made shuffles slow with VLEN larger
than 128 bits. Now implementation is built for LMUL=1, 1/2, 1/4 and 1/8
and variant that gives 128-bit register group is selected by VLEN.
256-bit code-path is removed as it was slower on SpacemiT K1.
Benchmark on SpacemiT K1 (in-order, VLEN=256):
Before:
AES | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
ECB enc | 16.79 ns/B 56.80 MiB/s 26.82 c/B 1597
ECB dec | 20.12 ns/B 47.41 MiB/s 32.15 c/B 1598
CBC enc | 32.05 ns/B 29.76 MiB/s 51.18 c/B 1597
CBC dec | 20.71 ns/B 46.05 MiB/s 33.07 c/B 1597
CFB enc | 32.09 ns/B 29.72 MiB/s 51.25 c/B 1597
CFB dec | 16.86 ns/B 56.56 MiB/s 26.97 c/B 1600
CTR enc | 17.12 ns/B 55.70 MiB/s 27.37 c/B 1599
XTS enc | 17.46 ns/B 54.63 MiB/s 27.93 c/B 1600
OCB enc | 17.19 ns/B 55.49 MiB/s 27.50 c/B 1600
After (ECB enc ~1.2x, CTR ~1.3x, CBC enc ~1.6x faster):
AES | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
ECB enc | 13.53 ns/B 70.51 MiB/s 21.64 c/B 1600
ECB dec | 16.32 ns/B 58.43 MiB/s 26.12 c/B 1600
CBC enc | 19.36 ns/B 49.27 MiB/s 30.96 c/B 1600
CBC dec | 16.74 ns/B 56.98 MiB/s 26.76 c/B 1599
CFB enc | 19.54 ns/B 48.81 MiB/s 31.26 c/B 1600
CFB dec | 12.66 ns/B 75.33 MiB/s 20.25 c/B 1600
CTR enc | 13.07 ns/B 72.96 MiB/s 20.91 c/B 1600
XTS enc | 14.15 ns/B 67.39 MiB/s 22.62 c/B 1599
OCB enc | 14.39 ns/B 66.28 MiB/s 23.02 c/B 1600
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/Makefile.am | 26 ++
cipher/rijndael-internal.h | 2 +
cipher/rijndael-vp-riscv-mf2.c | 29 ++
cipher/rijndael-vp-riscv-mf4.c | 29 ++
cipher/rijndael-vp-riscv-mf8.c | 29 ++
cipher/rijndael-vp-riscv.c | 268 ++----------
cipher/rijndael-vp-riscv.h | 190 +++++++++
cipher/rijndael-vp-simd128.h | 743 ++-------------------------------
cipher/rijndael.c | 73 +---
configure.ac | 3 +
10 files changed, 375 insertions(+), 1017 deletions(-)
create mode 100644 cipher/rijndael-vp-riscv-mf2.c
create mode 100644 cipher/rijndael-vp-riscv-mf4.c
create mode 100644 cipher/rijndael-vp-riscv-mf8.c
create mode 100644 cipher/rijndael-vp-riscv.h
diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index 23c4655c..65b03d76 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -128,6 +128,8 @@ EXTRA_libcipher_la_SOURCES = \
rijndael-riscv-zvkned.c \
rijndael-s390x.c \
rijndael-vp-aarch64.c rijndael-vp-riscv.c \
+ rijndael-vp-riscv.h rijndael-vp-riscv-mf2.c \
+ rijndael-vp-riscv-mf4.c rijndael-vp-riscv-mf8.c \
rijndael-vp-simd128.h \
rmd160.c \
rsa.c \
@@ -466,6 +468,30 @@ rijndael-vp-riscv.lo: $(srcdir)/rijndael-vp-riscv.c Makefile
`echo $(LTCOMPILE) $(riscv_vector_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
$(custom_depfile_lo)
+rijndael-vp-riscv-mf2.o: $(srcdir)/rijndael-vp-riscv-mf2.c Makefile
+ `echo $(COMPILE) $(riscv_vector_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ $(custom_depfile_o)
+
+rijndael-vp-riscv-mf2.lo: $(srcdir)/rijndael-vp-riscv-mf2.c Makefile
+ `echo $(LTCOMPILE) $(riscv_vector_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ $(custom_depfile_lo)
+
+rijndael-vp-riscv-mf4.o: $(srcdir)/rijndael-vp-riscv-mf4.c Makefile
+ `echo $(COMPILE) $(riscv_vector_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ $(custom_depfile_o)
+
+rijndael-vp-riscv-mf4.lo: $(srcdir)/rijndael-vp-riscv-mf4.c Makefile
+ `echo $(LTCOMPILE) $(riscv_vector_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ $(custom_depfile_lo)
+
+rijndael-vp-riscv-mf8.o: $(srcdir)/rijndael-vp-riscv-mf8.c Makefile
+ `echo $(COMPILE) $(riscv_vector_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ $(custom_depfile_o)
+
+rijndael-vp-riscv-mf8.lo: $(srcdir)/rijndael-vp-riscv-mf8.c Makefile
+ `echo $(LTCOMPILE) $(riscv_vector_cflags) $(custom_depflags) -c $< | $(instrumentation_munging) `
+ $(custom_depfile_lo)
+
if ENABLE_RISCV_VECTOR_CRYPTO_INTRINSICS_EXTRA_CFLAGS
riscv_vector_crypto_zvkb_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkb
riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkned_zvkb
diff --git a/cipher/rijndael-internal.h b/cipher/rijndael-internal.h
index bb8f97a0..87b630ce 100644
--- a/cipher/rijndael-internal.h
+++ b/cipher/rijndael-internal.h
@@ -181,6 +181,8 @@ typedef unsigned int (*rijndael_cryptfn_t)(const struct RIJNDAEL_context_s *ctx,
const unsigned char *ax);
typedef void (*rijndael_prefetchfn_t)(void);
typedef void (*rijndael_prepare_decfn_t)(struct RIJNDAEL_context_s *ctx);
+typedef void (*rijndael_setkeyfn_t)(struct RIJNDAEL_context_s *ctx,
+ const byte *key);
/* Our context object. */
typedef struct RIJNDAEL_context_s
diff --git a/cipher/rijndael-vp-riscv-mf2.c b/cipher/rijndael-vp-riscv-mf2.c
new file mode 100644
index 00000000..cf9aa577
--- /dev/null
+++ b/cipher/rijndael-vp-riscv-mf2.c
@@ -0,0 +1,29 @@
+/* rijndael-vp-riscv-mf2.c - RISC-V vector permutation AES, VLEN=256 variant
+ * Copyright (C) 2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+#include "rijndael-internal.h"
+
+#ifdef USE_VP_RISCV
+
+#define VP_LMUL128 mf2
+
+#include "rijndael-vp-riscv.h"
+
+#endif /* USE_VP_RISCV */
diff --git a/cipher/rijndael-vp-riscv-mf4.c b/cipher/rijndael-vp-riscv-mf4.c
new file mode 100644
index 00000000..a0fb95ab
--- /dev/null
+++ b/cipher/rijndael-vp-riscv-mf4.c
@@ -0,0 +1,29 @@
+/* rijndael-vp-riscv-mf4.c - RISC-V vector permutation AES, VLEN=512 variant
+ * Copyright (C) 2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+#include "rijndael-internal.h"
+
+#ifdef USE_VP_RISCV
+
+#define VP_LMUL128 mf4
+
+#include "rijndael-vp-riscv.h"
+
+#endif /* USE_VP_RISCV */
diff --git a/cipher/rijndael-vp-riscv-mf8.c b/cipher/rijndael-vp-riscv-mf8.c
new file mode 100644
index 00000000..337e0098
--- /dev/null
+++ b/cipher/rijndael-vp-riscv-mf8.c
@@ -0,0 +1,29 @@
+/* rijndael-vp-riscv-mf8.c - RISC-V vector permutation AES, VLEN>=1024 variant
+ * Copyright (C) 2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+#include "rijndael-internal.h"
+
+#ifdef USE_VP_RISCV
+
+#define VP_LMUL128 mf8
+
+#include "rijndael-vp-riscv.h"
+
+#endif /* USE_VP_RISCV */
diff --git a/cipher/rijndael-vp-riscv.c b/cipher/rijndael-vp-riscv.c
index 7a1da94f..28a5f698 100644
--- a/cipher/rijndael-vp-riscv.c
+++ b/cipher/rijndael-vp-riscv.c
@@ -1,5 +1,5 @@
/* RISC-V vector permutation AES for Libgcrypt
- * Copyright (C) 2025 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ * Copyright (C) 2025-2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
*
* This file is part of Libgcrypt.
*
@@ -35,251 +35,45 @@
*/
#include <config.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h> /* for memcmp() */
-
-#include "types.h" /* for byte and u32 typedefs */
-#include "g10lib.h"
-#include "cipher.h"
-#include "bufhelp.h"
#include "rijndael-internal.h"
-#include "cipher-internal.h"
-
#ifdef USE_VP_RISCV
+#define VP_LMUL128 m1
-/**********************************************************************
- AT&T x86 asm to intrinsics conversion macros (RISC-V)
- **********************************************************************/
-
-#include <riscv_vector.h>
-#include "simd-common-riscv.h"
-
-/*
- * SIMD128
- */
-
-typedef vuint8m1_t __m128i;
-
-#define cast_m128i_to_s8(a) (__riscv_vreinterpret_v_u8m1_i8m1(a))
-#define cast_m128i_to_u32(a) (__riscv_vreinterpret_v_u8m1_u32m1(a))
-#define cast_m128i_to_u64(a) (__riscv_vreinterpret_v_u8m1_u64m1(a))
-#define cast_m128i_to_s64(a) (__riscv_vreinterpret_v_u64m1_i64m1(cast_m128i_to_u64(a)))
-
-#define cast_s8_to_m128i(a) (__riscv_vreinterpret_v_i8m1_u8m1(a))
-#define cast_u32_to_m128i(a) (__riscv_vreinterpret_v_u32m1_u8m1(a))
-#define cast_u64_to_m128i(a) (__riscv_vreinterpret_v_u64m1_u8m1(a))
-#define cast_s64_to_m128i(a) (cast_u64_to_m128i(__riscv_vreinterpret_v_i64m1_u64m1(a)))
-
-#define pand128(a, o) (o = __riscv_vand_vv_u8m1((o), (a), 16))
-#define pandn128(a, o) (o = __riscv_vand_vv_u8m1(__riscv_vnot_v_u8m1((o), 16), (a), 16))
-#define pxor128(a, o) (o = __riscv_vxor_vv_u8m1((o), (a), 16))
-#define paddb128(a, o) (o = __riscv_vadd_vv_u8m1((o), (a), 16))
-#define paddd128(a, o) (o = cast_u32_to_m128i(__riscv_vadd_vv_u32m1( \
- cast_m128i_to_u32(o), \
- cast_m128i_to_u32(a), 4)))
-#define paddq128(a, o) (o = cast_u64_to_m128i(__riscv_vadd_vv_u64m1( \
- cast_m128i_to_u64(o), \
- cast_m128i_to_u64(a), 2)))
-
-#define psrld128(s, o) (o = cast_u32_to_m128i(__riscv_vsrl_vx_u32m1(cast_m128i_to_u32(o), (s), 4))
-#define psraq128(s, o) (o = cast_s64_to_m128i(__riscv_vsra_vx_i64m1(cast_m128i_to_s64(o), (s), 2)))
-#define psrldq128(s, o) (o = __riscv_vslidedown_vx_u8m1((o), (s), 16))
-#define pslldq128(s, o) ({ vuint8m1_t __tmp = __riscv_vmv_v_x_u8m1(0, 16); \
- o = __riscv_vslideup_vx_u8m1(__tmp, (o), (s), 16); })
-#define psrl_byte_128(s, o) (o = __riscv_vsrl_vx_u8m1((o), (s), 16))
-
-#define pshufb128(m8, o) (o = __riscv_vrgather_vv_u8m1((o), (m8), 16))
-#define pshufd128(m32, a, o) ({ static const __m128i_const __tmp1 = PSHUFD_MASK_TO_PSHUFB_MASK(m32); \
- __m128i __tmp2; \
- movdqa128(a, o); \
- movdqa128_memld(&__tmp1, __tmp2); \
- pshufb128(__tmp2, o); })
-
-#define pshufd128_0x93(a, o) pshufd128(0x93, a, o)
-#define pshufd128_0xFF(a, o) (o = cast_u32_to_m128i(__riscv_vrgather_vx_u32m1(cast_m128i_to_u32(a), 3, 4)))
-#define pshufd128_0xFE(a, o) pshufd128(0xFE, a, o)
-#define pshufd128_0x4E(a, o) pshufd128(0x4E, a, o)
-
-#define palignr128(s, a, o) (o = __riscv_vslideup_vx_u8m1(__riscv_vslidedown_vx_u8m1((a), (s), 16), (o), 16 - (s), 16))
-
-#define movdqa128(a, o) (o = (a))
-
-#define movdqa128_memld(a, o) (o = __riscv_vle8_v_u8m1((const void *)(a), 16))
-
-#define pand128_amemld(m, o) pand128(__riscv_vle8_v_u8m1((const void *)(m), 16), (o))
-#define pxor128_amemld(m, o) pxor128(__riscv_vle8_v_u8m1((const void *)(m), 16), (o))
-#define paddq128_amemld(m, o) paddq128(__riscv_vle8_v_u8m1((const void *)(m), 16), (o))
-#define paddd128_amemld(m, o) paddd128(__riscv_vle8_v_u8m1((const void *)(m), 16), (o))
-#define pshufb128_amemld(m, o) pshufb128(__riscv_vle8_v_u8m1((const void *)(m), 16), (o))
-
-/* Following operations may have unaligned memory input */
-#define movdqu128_memld(a, o) (o = __riscv_vle8_v_u8m1((const void *)(a), 16))
-
-/* Following operations may have unaligned memory output */
-#define movdqu128_memst(a, o) (__riscv_vse8_v_u8m1((void *)(o), (a), 16))
-
-/*
- * SIMD256
- */
-
-#define PSHUFD256_MASK_TO_PSHUFB256_MASK(m32) { \
- M128I_BYTE(((((m32) >> 0) & 0x03) * 4) + 0, \
- ((((m32) >> 0) & 0x03) * 4) + 1, \
- ((((m32) >> 0) & 0x03) * 4) + 2, \
- ((((m32) >> 0) & 0x03) * 4) + 3, \
- ((((m32) >> 2) & 0x03) * 4) + 0, \
- ((((m32) >> 2) & 0x03) * 4) + 1, \
- ((((m32) >> 2) & 0x03) * 4) + 2, \
- ((((m32) >> 2) & 0x03) * 4) + 3, \
- ((((m32) >> 4) & 0x03) * 4) + 0, \
- ((((m32) >> 4) & 0x03) * 4) + 1, \
- ((((m32) >> 4) & 0x03) * 4) + 2, \
- ((((m32) >> 4) & 0x03) * 4) + 3, \
- ((((m32) >> 6) & 0x03) * 4) + 0, \
- ((((m32) >> 6) & 0x03) * 4) + 1, \
- ((((m32) >> 6) & 0x03) * 4) + 2, \
- ((((m32) >> 6) & 0x03) * 4) + 3), \
- M128I_BYTE(((((m32) >> 0) & 0x03) * 4) + 1 + 16, \
- ((((m32) >> 0) & 0x03) * 4) + 1 + 16, \
- ((((m32) >> 0) & 0x03) * 4) + 2 + 16, \
- ((((m32) >> 0) & 0x03) * 4) + 3 + 16, \
- ((((m32) >> 2) & 0x03) * 4) + 0 + 16, \
- ((((m32) >> 2) & 0x03) * 4) + 1 + 16, \
- ((((m32) >> 2) & 0x03) * 4) + 2 + 16, \
- ((((m32) >> 2) & 0x03) * 4) + 3 + 16, \
- ((((m32) >> 4) & 0x03) * 4) + 0 + 16, \
- ((((m32) >> 4) & 0x03) * 4) + 1 + 16, \
- ((((m32) >> 4) & 0x03) * 4) + 2 + 16, \
- ((((m32) >> 4) & 0x03) * 4) + 3 + 16, \
- ((((m32) >> 6) & 0x03) * 4) + 0 + 16, \
- ((((m32) >> 6) & 0x03) * 4) + 1 + 16, \
- ((((m32) >> 6) & 0x03) * 4) + 2 + 16, \
- ((((m32) >> 6) & 0x03) * 4) + 3 + 16) }
-
-typedef vuint8m1_t __m256i;
-
-#define HAVE_SIMD256 1
-
-#define check_simd256_support() (__riscv_vsetvl_e8m1(32) == 32)
-
-#define cast_m256i_to_s8(a) cast_m128i_to_s8(a)
-#define cast_m256i_to_u32(a) cast_m128i_to_u32(a)
-#define cast_m256i_to_u64(a) cast_m128i_to_u64(a)
-#define cast_m256i_to_s64(a) cast_m128i_to_s64(a)
-
-#define cast_s8_to_m256i(a) (__riscv_vreinterpret_v_i8m1_u8m1(a))
-#define cast_u32_to_m256i(a) (__riscv_vreinterpret_v_u32m1_u8m1(a))
-#define cast_u64_to_m256i(a) (__riscv_vreinterpret_v_u64m1_u8m1(a))
-#define cast_s64_to_m256i(a) (cast_u64_to_m128i(__riscv_vreinterpret_v_i64m1_u64m1(a)))
-
-#define pand256(a, o) (o = __riscv_vand_vv_u8m1((o), (a), 32))
-#define pandn256(a, o) (o = __riscv_vand_vv_u8m1(__riscv_vnot_v_u8m1((o), 32), (a), 32))
-#define pxor256(a, o) (o = __riscv_vxor_vv_u8m1((o), (a), 32))
-#define paddb256(a, o) (o = __riscv_vadd_vv_u8m1((o), (a), 32))
-#define paddd256(a, o) (o = cast_u32_to_m256i(__riscv_vadd_vv_u32m1( \
- cast_m256i_to_u32(o), \
- cast_m256i_to_u32(a), 8)))
-#define paddq256(a, o) (o = cast_u64_to_m256i(__riscv_vadd_vv_u64m1( \
- cast_m256i_to_u64(o), \
- cast_m256i_to_u64(a), 4)))
-
-#define psrld256(s, o) (o = cast_u32_to_m256i(__riscv_vsrl_vx_u32m1(cast_m256i_to_u32(o), (s), 8))
-#define psraq256(s, o) (o = cast_s64_to_m256i(__riscv_vsra_vx_i64m1(cast_m256i_to_s64(o), (s), 4)))
-#define psrl_byte_256(s, o) (o = __riscv_vsrl_vx_u8m1((o), (s), 32))
-
-/* Note: these are not PSHUFB equavalent as full 256-bit vector is used as
- * 32 byte table. 256-bit PSHUFB on x86 handles 128-bit lanes separately as
- * 128-bit 16 byte tables. */
-
-/* tab32 variant: indexes have values 0..31. Used when 'm8' is constant and
- * variable data is in 'o'. */
-#define pshufb256_tab32(m8, o) (o = __riscv_vrgather_vv_u8m1((o), (m8), 32))
-
-/* tab16 variant: indexes have values 0..16 and only low 128-bit of 'o' is
- * used. Used when 'o' is constant and variable data is in 'm8'. */
-#define pshufb256_tab16(m8, o) (o = __riscv_vrgather_vv_u8m1((o), (m8), 32))
-
-/* Load 16 byte mask for 'pshufb256_tab32' usage as if 256-bit PSHUFB was to be
- * used as on x86 (two separate 128-bit lanes). */
-#define load_tab32_mask(m, o) ({ __m128i __tmp_lo128; \
- __m128i __tmp_hi128; \
- movdqu128_memld(m, __tmp_lo128); \
- __tmp_hi128 = __riscv_vadd_vx_u8m1(__tmp_lo128, 16, 16); \
- o = __riscv_vslideup_vx_u8m1(__tmp_lo128, __tmp_hi128, 16, 32); })
-
-#define broadcast128_256(a, o) (o = __riscv_vslideup_vx_u8m1((a), (a), 16, 32))
-
-/* Load 16 byte table for 'pshufb256_tab16' usage. On x86 this would splat
- * 128-bit table from memory to both 128-bit lanes of 256-bit register.
- * On RISC-V this just loads memory to lower 128-bits. */
-#define load_tab16_table(m, o) movdqu128_memld(m, o)
-
-#define pshufd256(m32, a, o) ({ static const __m128i_const __tmp1 = PSHUFD_MASK_TO_PSHUFB_MASK(m32); \
- __m256i __tmp2; \
- movdqa256(a, o); \
- load_tab32_mask(&__tmp1, __tmp2); \
- pshufb256_tab32(__tmp2, o); })
-
-#define pshufd256_0x93(a, o) pshufd256(0x93, a, o)
-
-#define insert256_hi128(x, o) (o = __riscv_vslideup_vx_u8m1((o), (x), 16, 32))
-#define extract256_hi128(y, o) (o = __riscv_vslidedown_vx_u8m1((y), 16, 32))
-
-#define movdqa256(a, o) (o = (a))
-
-#define movdqa128_256(a, o) (o = __riscv_vslideup_vx_u8m1((a), __riscv_vmv_v_x_u8m1(0, 16), 16, 32))
-#define movdqa256_128(a, o) (o = (a))
-
-#define movdqa256_memld(a, o) (o = __riscv_vle8_v_u8m1((const void *)(a), 32))
-
-#define pand256_amemld(m, o) pand128(__riscv_vle8_v_u8m1((const void *)(m), 32), (o))
-#define pxor256_amemld(m, o) pxor128(__riscv_vle8_v_u8m1((const void *)(m), 32), (o))
-#define paddq256_amemld(m, o) paddq128(__riscv_vle8_v_u8m1((const void *)(m), 32), (o))
-#define paddd256_amemld(m, o) paddd128(__riscv_vle8_v_u8m1((const void *)(m), 32), (o))
-#define pshufb256_amemld(m, o) pshufb128(__riscv_vle8_v_u8m1((const void *)(m), 32), (o))
-#define broadcast128_256_amemld(m, o) \
- broadcast128_256(__riscv_vle8_v_u8m1((const void *)(m), 32), (o))
-
-/* Following operations may have unaligned memory input */
-#define movdqu256_memld(a, o) (o = __riscv_vle8_v_u8m1((const void *)(a), 32))
-
-/* Following operations may have unaligned memory output */
-#define movdqu256_memst(a, o) (__riscv_vse8_v_u8m1((void *)(o), (a), 32))
-
-
-#ifdef HAVE_GCC_ATTRIBUTE_OPTIMIZE
-# define FUNC_ATTR_OPT __attribute__((optimize("-O2")))
-#else
-# define FUNC_ATTR_OPT
-#endif
-
-#define SIMD128_OPT_ATTR FUNC_ATTR_OPT
-
-#define FUNC_ENCRYPT _gcry_aes_vp_riscv_encrypt
-#define FUNC_DECRYPT _gcry_aes_vp_riscv_decrypt
-#define FUNC_CFB_ENC _gcry_aes_vp_riscv_cfb_enc
-#define FUNC_CFB_DEC _gcry_aes_vp_riscv_cfb_dec
-#define FUNC_CBC_ENC _gcry_aes_vp_riscv_cbc_enc
-#define FUNC_CBC_DEC _gcry_aes_vp_riscv_cbc_dec
-#define FUNC_CTR_ENC _gcry_aes_vp_riscv_ctr_enc
-#define FUNC_CTR32LE_ENC _gcry_aes_vp_riscv_ctr32le_enc
-#define FUNC_OCB_CRYPT _gcry_aes_vp_riscv_ocb_crypt
-#define FUNC_OCB_AUTH _gcry_aes_vp_riscv_ocb_auth
-#define FUNC_ECB_CRYPT _gcry_aes_vp_riscv_ecb_crypt
-#define FUNC_XTS_CRYPT _gcry_aes_vp_riscv_xts_crypt
-#define FUNC_SETKEY _gcry_aes_vp_riscv_do_setkey
-#define FUNC_PREPARE_DEC _gcry_aes_vp_riscv_prepare_decryption
+#include "rijndael-vp-riscv.h"
-#include "rijndael-vp-simd128.h"
+extern void _gcry_aes_vp_riscv_mf2_setup (RIJNDAEL_context *ctx,
+ rijndael_setkeyfn_t *hw_setkey,
+ cipher_bulk_ops_t *bulk_ops);
+extern void _gcry_aes_vp_riscv_mf4_setup (RIJNDAEL_context *ctx,
+ rijndael_setkeyfn_t *hw_setkey,
+ cipher_bulk_ops_t *bulk_ops);
+extern void _gcry_aes_vp_riscv_mf8_setup (RIJNDAEL_context *ctx,
+ rijndael_setkeyfn_t *hw_setkey,
+ cipher_bulk_ops_t *bulk_ops);
int
-_gcry_aes_vp_riscv_setup_acceleration(RIJNDAEL_context *ctx)
+_gcry_aes_vp_riscv_setup_acceleration (RIJNDAEL_context *ctx,
+ rijndael_setkeyfn_t *hw_setkey,
+ cipher_bulk_ops_t *bulk_ops)
{
- (void)ctx;
- return (__riscv_vsetvl_e8m1(16) == 16);
+ size_t vlenb;
+
+ if (__riscv_vsetvl_e8m1(16) != 16)
+ return 0;
+
+ vlenb = __riscv_vsetvlmax_e8m1 ();
+ if (vlenb >= 128)
+ _gcry_aes_vp_riscv_mf8_setup (ctx, hw_setkey, bulk_ops);
+ else if (vlenb >= 64)
+ _gcry_aes_vp_riscv_mf4_setup (ctx, hw_setkey, bulk_ops);
+ else if (vlenb >= 32)
+ _gcry_aes_vp_riscv_mf2_setup (ctx, hw_setkey, bulk_ops);
+ else
+ _gcry_aes_vp_riscv_m1_setup (ctx, hw_setkey, bulk_ops);
+
+ return 1;
}
#endif /* USE_VP_RISCV */
diff --git a/cipher/rijndael-vp-riscv.h b/cipher/rijndael-vp-riscv.h
new file mode 100644
index 00000000..149ac70d
--- /dev/null
+++ b/cipher/rijndael-vp-riscv.h
@@ -0,0 +1,190 @@
+/* RISC-V vector permutation AES for Libgcrypt
+ * Copyright (C) 2025-2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * The code is based on the public domain library libvpaes version 0.5
+ * available at http://crypto.stanford.edu/vpaes/ and which carries
+ * this notice:
+ *
+ * libvpaes: constant-time SSSE3 AES encryption and decryption.
+ * version 0.5
+ *
+ * By Mike Hamburg, Stanford University, 2009. Public domain.
+ * I wrote essentially all of this code. I did not write the test
+ * vectors; they are the NIST known answer tests. I hereby release all
+ * the code and documentation here that I wrote into the public domain.
+ *
+ * This is an implementation of AES following my paper,
+ * "Accelerating AES with Vector Permute Instructions"
+ * CHES 2009; http://shiftleft.org/papers/vector_aes/
+ */
+
+#ifndef GCRY_RIJNDAEL_VP_RISCV_H
+#define GCRY_RIJNDAEL_VP_RISCV_H
+
+#include <config.h>
+#include "types.h"
+#include "g10lib.h"
+#include "cipher.h"
+#include "rijndael-internal.h"
+#include "cipher-internal.h"
+
+#ifdef VP_LMUL128
+
+/**********************************************************************
+ AT&T x86 asm to intrinsics conversion macros (RISC-V)
+ **********************************************************************/
+
+/* VP_LMUL128 is LMUL that gives 128-bit register group. */
+
+#include <riscv_vector.h>
+#include "simd-common-riscv.h"
+
+#define VP_CAT_(a, b) a##b
+#define VP_CAT(a, b) VP_CAT_(a, b)
+#define VP_CAT3_(a, b, c) a##b##c
+#define VP_CAT3(a, b, c) VP_CAT3_(a, b, c)
+
+#define VP_U8(fn, lmul) VP_CAT3(__riscv_, fn, VP_CAT(_u8, lmul))
+
+#define VP_TO_M1_m1(a) (a)
+#define VP_TO_M1_mf2(a) __riscv_vlmul_ext_v_u8mf2_u8m1(a)
+#define VP_TO_M1_mf4(a) __riscv_vlmul_ext_v_u8mf4_u8m1(a)
+#define VP_TO_M1_mf8(a) __riscv_vlmul_ext_v_u8mf8_u8m1(a)
+#define VP_FROM_M1_m1(a) (a)
+#define VP_FROM_M1_mf2(a) __riscv_vlmul_trunc_v_u8m1_u8mf2(a)
+#define VP_FROM_M1_mf4(a) __riscv_vlmul_trunc_v_u8m1_u8mf4(a)
+#define VP_FROM_M1_mf8(a) __riscv_vlmul_trunc_v_u8m1_u8mf8(a)
+#define VP_TO_M1(lmul, a) VP_CAT(VP_TO_M1_, lmul)(a)
+#define VP_FROM_M1(lmul, a) VP_CAT(VP_FROM_M1_, lmul)(a)
+
+/*
+ * SIMD128
+ */
+
+typedef VP_CAT3(vuint8, VP_LMUL128, _t) __m128i;
+
+#define V128(fn) VP_U8(fn, VP_LMUL128)
+
+#define cast_m128i_to_u32(a) (__riscv_vreinterpret_v_u8m1_u32m1(VP_TO_M1(VP_LMUL128, a)))
+#define cast_m128i_to_u64(a) (__riscv_vreinterpret_v_u8m1_u64m1(VP_TO_M1(VP_LMUL128, a)))
+#define cast_m128i_to_s64(a) (__riscv_vreinterpret_v_u64m1_i64m1(cast_m128i_to_u64(a)))
+
+#define cast_u32_to_m128i(a) (VP_FROM_M1(VP_LMUL128, __riscv_vreinterpret_v_u32m1_u8m1(a)))
+#define cast_u64_to_m128i(a) (VP_FROM_M1(VP_LMUL128, __riscv_vreinterpret_v_u64m1_u8m1(a)))
+#define cast_s64_to_m128i(a) (cast_u64_to_m128i(__riscv_vreinterpret_v_i64m1_u64m1(a)))
+
+#define pand128(a, o) (o = V128(vand_vv)((o), (a), 16))
+#define pandn128(a, o) (o = V128(vand_vv)(V128(vnot_v)((o), 16), (a), 16))
+#define pxor128(a, o) (o = V128(vxor_vv)((o), (a), 16))
+#define paddb128(a, o) (o = V128(vadd_vv)((o), (a), 16))
+#define paddd128(a, o) (o = cast_u32_to_m128i(__riscv_vadd_vv_u32m1( \
+ cast_m128i_to_u32(o), \
+ cast_m128i_to_u32(a), 4)))
+#define paddq128(a, o) (o = cast_u64_to_m128i(__riscv_vadd_vv_u64m1( \
+ cast_m128i_to_u64(o), \
+ cast_m128i_to_u64(a), 2)))
+
+#define psrld128(s, o) (o = cast_u32_to_m128i(__riscv_vsrl_vx_u32m1(cast_m128i_to_u32(o), (s), 4))
+#define psraq128(s, o) (o = cast_s64_to_m128i(__riscv_vsra_vx_i64m1(cast_m128i_to_s64(o), (s), 2)))
+#define psrldq128(s, o) (o = V128(vslidedown_vx)((o), (s), 16))
+#define pslldq128(s, o) ({ __m128i __tmp = V128(vmv_v_x)(0, 16); \
+ o = V128(vslideup_vx)(__tmp, (o), (s), 16); })
+#define psrl_byte_128(s, o) (o = V128(vsrl_vx)((o), (s), 16))
+
+#define pshufb128(m8, o) (o = V128(vrgather_vv)((o), (m8), 16))
+#define pshufd128(m32, a, o) ({ static const __m128i_const __tmp1 = PSHUFD_MASK_TO_PSHUFB_MASK(m32); \
+ __m128i __tmp2; \
+ movdqa128(a, o); \
+ movdqa128_memld(&__tmp1, __tmp2); \
+ pshufb128(__tmp2, o); })
+
+#define pshufd128_0x93(a, o) pshufd128(0x93, a, o)
+#define pshufd128_0xFF(a, o) pshufd128(0xFF, a, o)
+#define pshufd128_0xFE(a, o) pshufd128(0xFE, a, o)
+#define pshufd128_0x4E(a, o) pshufd128(0x4E, a, o)
+
+#define palignr128(s, a, o) (o = V128(vslideup_vx)(V128(vslidedown_vx)((a), (s), 16), (o), 16 - (s), 16))
+
+#define movdqa128(a, o) (o = (a))
+
+#define movdqa128_memld(a, o) (o = V128(vle8_v)((const void *)(a), 16))
+
+#define pand128_amemld(m, o) pand128(V128(vle8_v)((const void *)(m), 16), (o))
+#define pxor128_amemld(m, o) pxor128(V128(vle8_v)((const void *)(m), 16), (o))
+#define paddq128_amemld(m, o) paddq128(V128(vle8_v)((const void *)(m), 16), (o))
+#define paddd128_amemld(m, o) paddd128(V128(vle8_v)((const void *)(m), 16), (o))
+#define pshufb128_amemld(m, o) pshufb128(V128(vle8_v)((const void *)(m), 16), (o))
+
+/* Following operations may have unaligned memory input */
+#define movdqu128_memld(a, o) (o = V128(vle8_v)((const void *)(a), 16))
+
+/* Following operations may have unaligned memory output */
+#define movdqu128_memst(a, o) (V128(vse8_v)((void *)(o), (a), 16))
+
+#ifdef HAVE_GCC_ATTRIBUTE_OPTIMIZE
+# define FUNC_ATTR_OPT __attribute__((optimize("-O2")))
+#else
+# define FUNC_ATTR_OPT
+#endif
+
+#define SIMD128_OPT_ATTR FUNC_ATTR_OPT
+
+#define VP_FUNC(name) VP_CAT3(_gcry_aes_vp_riscv_, VP_LMUL128, _##name)
+
+#define FUNC_ENCRYPT VP_FUNC(encrypt)
+#define FUNC_DECRYPT VP_FUNC(decrypt)
+#define FUNC_CFB_ENC VP_FUNC(cfb_enc)
+#define FUNC_CFB_DEC VP_FUNC(cfb_dec)
+#define FUNC_CBC_ENC VP_FUNC(cbc_enc)
+#define FUNC_CBC_DEC VP_FUNC(cbc_dec)
+#define FUNC_CTR_ENC VP_FUNC(ctr_enc)
+#define FUNC_CTR32LE_ENC VP_FUNC(ctr32le_enc)
+#define FUNC_OCB_CRYPT VP_FUNC(ocb_crypt)
+#define FUNC_OCB_AUTH VP_FUNC(ocb_auth)
+#define FUNC_ECB_CRYPT VP_FUNC(ecb_crypt)
+#define FUNC_XTS_CRYPT VP_FUNC(xts_crypt)
+#define FUNC_SETKEY VP_FUNC(do_setkey)
+#define FUNC_PREPARE_DEC VP_FUNC(prepare_decryption)
+
+#include "rijndael-vp-simd128.h"
+
+void
+VP_FUNC(setup) (RIJNDAEL_context *ctx, rijndael_setkeyfn_t *hw_setkey,
+ cipher_bulk_ops_t *bulk_ops)
+{
+ *hw_setkey = FUNC_SETKEY;
+ ctx->encrypt_fn = FUNC_ENCRYPT;
+ ctx->decrypt_fn = FUNC_DECRYPT;
+ ctx->prepare_decryption = FUNC_PREPARE_DEC;
+
+ bulk_ops->cfb_enc = FUNC_CFB_ENC;
+ bulk_ops->cfb_dec = FUNC_CFB_DEC;
+ bulk_ops->cbc_enc = FUNC_CBC_ENC;
+ bulk_ops->cbc_dec = FUNC_CBC_DEC;
+ bulk_ops->ctr16be_enc = FUNC_CTR_ENC;
+ bulk_ops->ctr32le_enc = FUNC_CTR32LE_ENC;
+ bulk_ops->ocb_crypt = FUNC_OCB_CRYPT;
+ bulk_ops->ocb_auth = FUNC_OCB_AUTH;
+ bulk_ops->ecb_crypt = FUNC_ECB_CRYPT;
+ bulk_ops->xts_crypt = FUNC_XTS_CRYPT;
+}
+
+#endif /* VP_LMUL128 */
+
+#endif /* GCRY_RIJNDAEL_VP_RISCV_H */
diff --git a/cipher/rijndael-vp-simd128.h b/cipher/rijndael-vp-simd128.h
index 84dab316..092f6ec5 100644
--- a/cipher/rijndael-vp-simd128.h
+++ b/cipher/rijndael-vp-simd128.h
@@ -1160,119 +1160,6 @@ aes_encrypt_core_2blks(__m128i *pxmm0_a, __m128i *pxmm0_b,
*pxmm0_b = xmm0_b;
}
-#ifdef HAVE_SIMD256
-
-static ASM_FUNC_ATTR_INLINE void
-aes_encrypt_core_4blks_simd256(__m256i *pymm0_a, __m256i *pymm0_b,
- struct vp_aes_config_s config,
- __m128i xmm9, __m128i xmm10, __m128i xmm11,
- __m128i xmm12, __m128i xmm13, __m128i xmm14,
- __m128i xmm15)
-{
- __m256i ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15;
- __m256i ymm0_a, ymm0_b;
- __m256i ymm1_a, ymm2_a, ymm3_a, ymm4_a;
- __m256i ymm1_b, ymm2_b, ymm3_b, ymm4_b;
- __m256i ymm5, ymm6;
- const byte *end_keys = config.sched_keys + 16 * config.nround;
- unsigned int mc_pos = 1;
-
- broadcast128_256(xmm9, ymm9);
- movdqa128_256(xmm10, ymm10);
- movdqa128_256(xmm11, ymm11);
- movdqa128_256(xmm12, ymm12);
- movdqa128_256(xmm13, ymm13);
- movdqa128_256(xmm14, ymm14);
- movdqa128_256(xmm15, ymm15);
-
- ymm0_a = *pymm0_a;
- ymm0_b = *pymm0_b;
-
- load_tab16_table(&k_iptlo, ymm2_a); movdqa256(ymm2_a, ymm2_b);
- movdqa256(ymm9, ymm1_a); movdqa256(ymm9, ymm1_b);
- pandn256(ymm0_a, ymm1_a); pandn256(ymm0_b, ymm1_b);
- psrl_byte_256(4, ymm1_a); psrl_byte_256(4, ymm1_b);
- pand256(ymm9, ymm0_a); pand256(ymm9, ymm0_b);
- pshufb256_tab16(ymm0_a, ymm2_a); pshufb256_tab16(ymm0_b, ymm2_b);
- load_tab16_table(&k_ipthi, ymm0_a); movdqa256(ymm0_a, ymm0_b);
-
- pshufb256_tab16(ymm1_a, ymm0_a); pshufb256_tab16(ymm1_b, ymm0_b);
- broadcast128_256_amemld(config.sched_keys, ymm5);
- pxor256(ymm5, ymm2_a); pxor256(ymm5, ymm2_b);
- pxor256(ymm2_a, ymm0_a); pxor256(ymm2_b, ymm0_b);
-
- config.sched_keys += 16;
-
- while (1)
- {
- /* top of round */
- movdqa256(ymm9, ymm1_a); movdqa256(ymm9, ymm1_b);
- pandn256(ymm0_a, ymm1_a); pandn256(ymm0_b, ymm1_b);
- psrl_byte_256(4, ymm1_a); psrl_byte_256(4, ymm1_b);
- pand256(ymm9, ymm0_a); pand256(ymm9, ymm0_b);
- movdqa256(ymm11, ymm2_a); movdqa256(ymm11, ymm2_b);
- pshufb256_tab16(ymm0_a, ymm2_a); pshufb256_tab16(ymm0_b, ymm2_b);
- pxor256(ymm1_a, ymm0_a); pxor256(ymm1_b, ymm0_b);
- movdqa256(ymm10, ymm3_a); movdqa256(ymm10, ymm3_b);
- pshufb256_tab16(ymm1_a, ymm3_a); pshufb256_tab16(ymm1_b, ymm3_b);
- pxor256(ymm2_a, ymm3_a); pxor256(ymm2_b, ymm3_b);
- movdqa256(ymm10, ymm4_a); movdqa256(ymm10, ymm4_b);
- pshufb256_tab16(ymm0_a, ymm4_a); pshufb256_tab16(ymm0_b, ymm4_b);
- pxor256(ymm2_a, ymm4_a); pxor256(ymm2_b, ymm4_b);
- movdqa256(ymm10, ymm2_a); movdqa256(ymm10, ymm2_b);
- pshufb256_tab16(ymm3_a, ymm2_a); pshufb256_tab16(ymm3_b, ymm2_b);
- pxor256(ymm0_a, ymm2_a); pxor256(ymm0_b, ymm2_b);
- movdqa256(ymm10, ymm3_a); movdqa256(ymm10, ymm3_b);
- pshufb256_tab16(ymm4_a, ymm3_a); pshufb256_tab16(ymm4_b, ymm3_b);
- pxor256(ymm1_a, ymm3_a); pxor256(ymm1_b, ymm3_b);
-
- if (config.sched_keys == end_keys)
- break;
-
- /* middle of middle round */
- movdqa256(ymm13, ymm4_a); movdqa256(ymm13, ymm4_b);
- pshufb256_tab16(ymm2_a, ymm4_a); pshufb256_tab16(ymm2_b, ymm4_b);
- broadcast128_256_amemld(config.sched_keys, ymm5);
- pxor256(ymm5, ymm4_a); pxor256(ymm5, ymm4_b);
- movdqa256(ymm12, ymm0_a); movdqa256(ymm12, ymm0_b);
- pshufb256_tab16(ymm3_a, ymm0_a); pshufb256_tab16(ymm3_b, ymm0_b);
- pxor256(ymm4_a, ymm0_a); pxor256(ymm4_b, ymm0_b);
- movdqa256(ymm15, ymm4_a); movdqa256(ymm15, ymm4_b);
- pshufb256_tab16(ymm2_a, ymm4_a); pshufb256_tab16(ymm2_b, ymm4_b);
- load_tab32_mask(&k_mc_forward[mc_pos], ymm6);
- movdqa256(ymm14, ymm2_a); movdqa256(ymm14, ymm2_b);
- pshufb256_tab16(ymm3_a, ymm2_a); pshufb256_tab16(ymm3_b, ymm2_b);
- pxor256(ymm4_a, ymm2_a); pxor256(ymm4_b, ymm2_b);
- movdqa256(ymm0_a, ymm3_a); movdqa256(ymm0_b, ymm3_b);
- pshufb256_tab32(ymm6, ymm0_a); pshufb256_tab32(ymm6, ymm0_b);
- pxor256(ymm2_a, ymm0_a); pxor256(ymm2_b, ymm0_b);
- load_tab32_mask(&k_mc_backward[mc_pos], ymm5);
- pshufb256_tab32(ymm5, ymm3_a); pshufb256_tab32(ymm5, ymm3_b);
- pxor256(ymm0_a, ymm3_a); pxor256(ymm0_b, ymm3_b);
- pshufb256_tab32(ymm6, ymm0_a); pshufb256_tab32(ymm6, ymm0_b);
- pxor256(ymm3_a, ymm0_a); pxor256(ymm3_b, ymm0_b);
-
- config.sched_keys += 16;
- mc_pos = (mc_pos + 1) % 4; /* next mc mod 4 */
- }
-
- /* middle of last round */
- movdqa256_memld(&k_sbou, ymm4_a); movdqa256_memld(&k_sbou, ymm4_b);
- pshufb256_tab16(ymm2_a, ymm4_a); pshufb256_tab16(ymm2_b, ymm4_b);
- broadcast128_256_amemld(config.sched_keys, ymm5);
- pxor256(ymm5, ymm4_a); pxor256(ymm5, ymm4_b);
- movdqa256_memld(&k_sbot, ymm0_a); movdqa256_memld(&k_sbot, ymm0_b);
- pshufb256_tab16(ymm3_a, ymm0_a); pshufb256_tab16(ymm3_b, ymm0_b);
- pxor256(ymm4_a, ymm0_a); pxor256(ymm4_b, ymm0_b);
- load_tab32_mask(&k_sr[mc_pos], ymm5);
- pshufb256_tab32(ymm5, ymm0_a); pshufb256_tab32(ymm5, ymm0_b);
-
- *pymm0_a = ymm0_a;
- *pymm0_b = ymm0_b;
-}
-
-#endif /* HAVE_SIMD256 */
-
static ASM_FUNC_ATTR_INLINE __m128i
aes_decrypt_core(__m128i xmm0, struct vp_aes_config_s config,
__m128i xmm9, __m128i xmm10, __m128i xmm11, __m128i xmm12,
@@ -1480,130 +1367,6 @@ aes_decrypt_core_2blks(__m128i *pxmm0_a, __m128i *pxmm0_b,
*pxmm0_b = xmm0_b;
}
-#ifdef HAVE_SIMD256
-
-static ASM_FUNC_ATTR_INLINE void
-aes_decrypt_core_4blks_simd256(__m256i *pymm0_a, __m256i *pymm0_b,
- struct vp_aes_config_s config,
- __m128i xmm9, __m128i xmm10, __m128i xmm11,
- __m128i xmm12, __m128i xmm13, __m128i xmm14,
- __m128i xmm15, __m128i xmm8)
-{
- __m256i ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15, ymm8;
- __m256i ymm0_a, ymm0_b;
- __m256i ymm1_a, ymm2_a, ymm3_a, ymm4_a;
- __m256i ymm1_b, ymm2_b, ymm3_b, ymm4_b;
- __m256i ymm5, ymm6;
- const byte *end_keys = config.sched_keys + 16 * config.nround;
- unsigned int mc_pos = config.nround % 4;
-
- broadcast128_256(xmm9, ymm9);
- movdqa128_256(xmm10, ymm10);
- movdqa128_256(xmm11, ymm11);
- movdqa128_256(xmm12, ymm12);
- movdqa128_256(xmm13, ymm13);
- movdqa128_256(xmm14, ymm14);
- movdqa128_256(xmm15, ymm15);
- movdqa128_256(xmm8, ymm8);
-
- ymm0_a = *pymm0_a;
- ymm0_b = *pymm0_b;
-
- load_tab16_table(&k_dipt[0], ymm2_a); movdqa256(ymm2_a, ymm2_b);
- movdqa256(ymm9, ymm1_a); movdqa256(ymm9, ymm1_b);
- pandn256(ymm0_a, ymm1_a); pandn256(ymm0_b, ymm1_b);
- psrl_byte_256(4, ymm1_a); psrl_byte_256(4, ymm1_b);
- pand256(ymm9, ymm0_a); pand256(ymm9, ymm0_b);
- pshufb256_tab16(ymm0_a, ymm2_a); pshufb256_tab16(ymm0_b, ymm2_b);
- load_tab16_table(&k_dipt[1], ymm0_a); movdqa256(ymm0_a, ymm0_b);
- pshufb256_tab16(ymm1_a, ymm0_a); pshufb256_tab16(ymm1_b, ymm0_b);
- broadcast128_256_amemld(config.sched_keys, ymm6);
- pxor256(ymm6, ymm2_a); pxor256(ymm6, ymm2_b);
- pxor256(ymm2_a, ymm0_a); pxor256(ymm2_b, ymm0_b);
- load_tab32_mask(&k_mc_forward[3], ymm5);
-
- config.sched_keys += 16;
-
- while (1)
- {
- /* top of round */
- movdqa256(ymm9, ymm1_a); movdqa256(ymm9, ymm1_b);
- pandn256(ymm0_a, ymm1_a); pandn256(ymm0_b, ymm1_b);
- psrl_byte_256(4, ymm1_a); psrl_byte_256(4, ymm1_b);
- pand256(ymm9, ymm0_a); pand256(ymm9, ymm0_b);
- movdqa256(ymm11, ymm2_a); movdqa256(ymm11, ymm2_b);
- pshufb256_tab16(ymm0_a, ymm2_a); pshufb256_tab16(ymm0_b, ymm2_b);
- pxor256(ymm1_a, ymm0_a); pxor256(ymm1_b, ymm0_b);
- movdqa256(ymm10, ymm3_a); movdqa256(ymm10, ymm3_b);
- pshufb256_tab16(ymm1_a, ymm3_a); pshufb256_tab16(ymm1_b, ymm3_b);
- pxor256(ymm2_a, ymm3_a); pxor256(ymm2_b, ymm3_b);
- movdqa256(ymm10, ymm4_a); movdqa256(ymm10, ymm4_b);
- pshufb256_tab16(ymm0_a, ymm4_a); pshufb256_tab16(ymm0_b, ymm4_b);
- pxor256(ymm2_a, ymm4_a); pxor256(ymm2_b, ymm4_b);
- movdqa256(ymm10, ymm2_a); movdqa256(ymm10, ymm2_b);
- pshufb256_tab16(ymm3_a, ymm2_a); pshufb256_tab16(ymm3_b, ymm2_b);
- pxor256(ymm0_a, ymm2_a); pxor256(ymm0_b, ymm2_b);
- movdqa256(ymm10, ymm3_a); movdqa256(ymm10, ymm3_b);
- pshufb256_tab16(ymm4_a, ymm3_a); pshufb256_tab16(ymm4_b, ymm3_b);
- pxor256(ymm1_a, ymm3_a); pxor256(ymm1_b, ymm3_b);
-
- if (config.sched_keys == end_keys)
- break;
-
- /* Inverse mix columns */
- movdqa256(ymm13, ymm4_a); movdqa256(ymm13, ymm4_b);
- pshufb256_tab16(ymm2_a, ymm4_a); pshufb256_tab16(ymm2_b, ymm4_b);
- broadcast128_256_amemld(config.sched_keys, ymm6);
- pxor256(ymm6, ymm4_a); pxor256(ymm6, ymm4_b);
- movdqa256(ymm12, ymm0_a); movdqa256(ymm12, ymm0_b);
- pshufb256_tab16(ymm3_a, ymm0_a); pshufb256_tab16(ymm3_b, ymm0_b);
- load_tab16_table(&k_dsbd[1], ymm1_a); movdqa256(ymm1_a, ymm1_b);
- pxor256(ymm4_a, ymm0_a); pxor256(ymm4_b, ymm0_b);
-
- pshufb256_tab32(ymm5, ymm0_a); pshufb256_tab32(ymm5, ymm0_b);
- movdqa256(ymm15, ymm4_a); movdqa256(ymm15, ymm4_b);
- pshufb256_tab16(ymm2_a, ymm4_a); pshufb256_tab16(ymm2_b, ymm4_b);
- pxor256(ymm0_a, ymm4_a); pxor256(ymm0_b, ymm4_b);
- pshufb256_tab16(ymm3_a, ymm1_a); pshufb256_tab16(ymm3_b, ymm1_b);
- pxor256(ymm4_a, ymm1_a); pxor256(ymm4_b, ymm1_b);
-
- pshufb256_tab32(ymm5, ymm1_a); pshufb256_tab32(ymm5, ymm1_b);
- movdqa256(ymm14, ymm4_a); movdqa256(ymm14, ymm4_b);
- pshufb256_tab16(ymm2_a, ymm4_a); pshufb256_tab16(ymm2_b, ymm4_b);
- pxor256(ymm1_a, ymm4_a); pxor256(ymm1_b, ymm4_b);
- load_tab16_table(&k_dsbb[1], ymm0_a); movdqa256(ymm0_a, ymm0_b);
- pshufb256_tab16(ymm3_a, ymm0_a); pshufb256_tab16(ymm3_b, ymm0_b);
- pxor256(ymm4_a, ymm0_a); pxor256(ymm4_b, ymm0_b);
-
- pshufb256_tab32(ymm5, ymm0_a); pshufb256_tab32(ymm5, ymm0_b);
- movdqa256(ymm8, ymm4_a); movdqa256(ymm8, ymm4_b);
- pshufb256_tab16(ymm2_a, ymm4_a); pshufb256_tab16(ymm2_b, ymm4_b);
- pshufd256_0x93(ymm5, ymm5);
- pxor256(ymm0_a, ymm4_a); pxor256(ymm0_b, ymm4_b);
- load_tab16_table(&k_dsbe[1], ymm0_a); movdqa256(ymm0_a, ymm0_b);
- pshufb256_tab16(ymm3_a, ymm0_a); pshufb256_tab16(ymm3_b, ymm0_b);
- pxor256(ymm4_a, ymm0_a); pxor256(ymm4_b, ymm0_b);
-
- config.sched_keys += 16;
- }
-
- /* middle of last round */
- load_tab16_table(&k_dsbo[0], ymm4_a); movdqa256(ymm4_a, ymm4_b);
- pshufb256_tab16(ymm2_a, ymm4_a); pshufb256_tab16(ymm2_b, ymm4_b);
- broadcast128_256_amemld(config.sched_keys, ymm6);
- pxor256(ymm6, ymm4_a); pxor256(ymm6, ymm4_b);
- load_tab16_table(&k_dsbo[1], ymm0_a); movdqa256(ymm0_a, ymm0_b);
- pshufb256_tab16(ymm3_a, ymm0_a); pshufb256_tab16(ymm3_b, ymm0_b);
- pxor256(ymm4_a, ymm0_a); pxor256(ymm4_b, ymm0_b);
- load_tab32_mask(&k_sr[mc_pos], ymm5);
- pshufb256_tab16(ymm5, ymm0_a); pshufb256_tab16(ymm5, ymm0_b);
-
- *pymm0_a = ymm0_a;
- *pymm0_b = ymm0_b;
-}
-
-#endif /* HAVE_SIMD256 */
-
ASM_FUNC_ATTR_NOINLINE unsigned int
FUNC_ENCRYPT (const RIJNDAEL_context *ctx, unsigned char *dst,
const unsigned char *src)
@@ -1653,10 +1416,12 @@ FUNC_DECRYPT (const RIJNDAEL_context *ctx, unsigned char *dst,
}
ASM_FUNC_ATTR_NOINLINE void
-FUNC_CFB_ENC (RIJNDAEL_context *ctx, unsigned char *iv,
- unsigned char *outbuf, const unsigned char *inbuf,
- size_t nblocks)
+FUNC_CFB_ENC (void *context, unsigned char *iv, void *outbuf_arg,
+ const void *inbuf_arg, size_t nblocks)
{
+ RIJNDAEL_context *ctx = context;
+ unsigned char *outbuf = outbuf_arg;
+ const unsigned char *inbuf = inbuf_arg;
__m128i xmm0, xmm1, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15;
struct vp_aes_config_s config;
@@ -1686,10 +1451,12 @@ FUNC_CFB_ENC (RIJNDAEL_context *ctx, unsigned char *iv,
}
ASM_FUNC_ATTR_NOINLINE void
-FUNC_CBC_ENC (RIJNDAEL_context *ctx, unsigned char *iv,
- unsigned char *outbuf, const unsigned char *inbuf,
- size_t nblocks, int cbc_mac)
+FUNC_CBC_ENC (void *context, unsigned char *iv, void *outbuf_arg,
+ const void *inbuf_arg, size_t nblocks, int cbc_mac)
{
+ RIJNDAEL_context *ctx = context;
+ unsigned char *outbuf = outbuf_arg;
+ const unsigned char *inbuf = inbuf_arg;
__m128i xmm0, xmm7, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15;
struct vp_aes_config_s config;
size_t outbuf_add = (!cbc_mac) * BLOCKSIZE;
@@ -1722,10 +1489,12 @@ FUNC_CBC_ENC (RIJNDAEL_context *ctx, unsigned char *iv,
}
ASM_FUNC_ATTR_NOINLINE void
-FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
- unsigned char *outbuf, const unsigned char *inbuf,
- size_t nblocks)
+FUNC_CTR_ENC (void *context, unsigned char *ctr, void *outbuf_arg,
+ const void *inbuf_arg, size_t nblocks)
{
+ RIJNDAEL_context *ctx = context;
+ unsigned char *outbuf = outbuf_arg;
+ const unsigned char *inbuf = inbuf_arg;
__m128i xmm0, xmm1, xmm2, xmm3, xmm6, xmm7, xmm8;
__m128i xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15;
static const __m128i_const be_mask =
@@ -1745,62 +1514,6 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
movdqu128_memld(ctr, xmm7); /* Preload CTR */
movdqa128_memld(&be_mask, xmm6); /* Preload mask */
-#ifdef HAVE_SIMD256
- if (check_simd256_support())
- {
- __m256i ymm0, ymm1, ymm2, ymm3;
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- movdqa128_256(xmm7, ymm0);
-
- /* detect if 8-bit carry handling is needed */
- if (UNLIKELY(((ctrlow += 4) & 0xff) <= 3))
- {
- pshufb128(xmm6, xmm7);
- paddd128_amemld(&littleendian_add, xmm7);
- movdqa128(xmm7, xmm2);
- pshufb128(xmm6, xmm2);
- insert256_hi128(xmm2, ymm0);
- paddd128_amemld(&littleendian_add, xmm7);
- movdqa128(xmm7, xmm2);
- pshufb128(xmm6, xmm2);
- movdqa128_256(xmm2, ymm1);
- paddd128_amemld(&littleendian_add, xmm7);
- movdqa128(xmm7, xmm2);
- pshufb128(xmm6, xmm2);
- insert256_hi128(xmm2, ymm1);
- paddd128_amemld(&littleendian_add, xmm7);
- pshufb128(xmm6, xmm7);
- }
- else
- {
- paddb128(xmm8, xmm7);
- insert256_hi128(xmm7, ymm0);
- paddb128(xmm8, xmm7);
- movdqa128_256(xmm7, ymm1);
- paddb128(xmm8, xmm7);
- insert256_hi128(xmm7, ymm1);
- paddb128(xmm8, xmm7);
- }
-
- aes_encrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
- xmm15);
-
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm2);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm3);
- pxor256(ymm2, ymm0);
- pxor256(ymm3, ymm1);
- movdqu256_memst(ymm0, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- outbuf += 4 * BLOCKSIZE;
- inbuf += 4 * BLOCKSIZE;
- }
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
movdqa128(xmm7, xmm0);
@@ -1869,10 +1582,12 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
}
ASM_FUNC_ATTR_NOINLINE void
-FUNC_CTR32LE_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
- unsigned char *outbuf, const unsigned char *inbuf,
- size_t nblocks)
+FUNC_CTR32LE_ENC (void *context, unsigned char *ctr, void *outbuf_arg,
+ const void *inbuf_arg, size_t nblocks)
{
+ RIJNDAEL_context *ctx = context;
+ unsigned char *outbuf = outbuf_arg;
+ const unsigned char *inbuf = inbuf_arg;
__m128i xmm0, xmm1, xmm2, xmm3, xmm7, xmm8;
__m128i xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15;
static const __m128i_const add_one = M128I_U64(1, 0);
@@ -1887,44 +1602,6 @@ FUNC_CTR32LE_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
movdqa128_memld(&add_one, xmm8); /* Preload byte add */
movdqu128_memld(ctr, xmm7); /* Preload CTR */
-#ifdef HAVE_SIMD256
- if (check_simd256_support())
- {
- __m256i ymm0, ymm1, ymm2, ymm3;
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- movdqa128(xmm7, xmm0);
- movdqa128(xmm7, xmm1);
- paddd128(xmm8, xmm1);
- paddd128_amemld(&add_two, xmm7);
- movdqa128_256(xmm0, ymm0);
- insert256_hi128(xmm1, ymm0);
-
- movdqa128(xmm7, xmm1);
- movdqa128(xmm7, xmm2);
- paddd128(xmm8, xmm2);
- paddd128_amemld(&add_two, xmm7);
- movdqa128_256(xmm1, ymm1);
- insert256_hi128(xmm2, ymm1);
-
- aes_encrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15);
-
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm2);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm3);
- pxor256(ymm2, ymm0);
- pxor256(ymm3, ymm1);
- movdqu256_memst(ymm0, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- outbuf += 4 * BLOCKSIZE;
- inbuf += 4 * BLOCKSIZE;
- }
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
movdqa128(xmm7, xmm0);
@@ -1968,10 +1645,12 @@ FUNC_CTR32LE_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
}
ASM_FUNC_ATTR_NOINLINE void
-FUNC_CFB_DEC (RIJNDAEL_context *ctx, unsigned char *iv,
- unsigned char *outbuf, const unsigned char *inbuf,
- size_t nblocks)
+FUNC_CFB_DEC (void *context, unsigned char *iv, void *outbuf_arg,
+ const void *inbuf_arg, size_t nblocks)
{
+ RIJNDAEL_context *ctx = context;
+ unsigned char *outbuf = outbuf_arg;
+ const unsigned char *inbuf = inbuf_arg;
__m128i xmm0, xmm1, xmm2, xmm6, xmm9;
__m128i xmm10, xmm11, xmm12, xmm13, xmm14, xmm15;
struct vp_aes_config_s config;
@@ -1983,36 +1662,6 @@ FUNC_CFB_DEC (RIJNDAEL_context *ctx, unsigned char *iv,
movdqu128_memld(iv, xmm0);
-#ifdef HAVE_SIMD256
- if (check_simd256_support())
- {
- __m256i ymm6, ymm1, ymm2, ymm3;
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- movdqa128_256(xmm0, ymm6);
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm2);
- movdqa256_128(ymm2, xmm2);
- insert256_hi128(xmm2, ymm6);
- movdqu256_memld(inbuf + 1 * BLOCKSIZE, ymm1);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm3);
- extract256_hi128(ymm3, xmm0);
-
- aes_encrypt_core_4blks_simd256(&ymm6, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15);
-
- pxor256(ymm2, ymm6);
- pxor256(ymm3, ymm1);
- movdqu256_memst(ymm6, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- outbuf += 4 * BLOCKSIZE;
- inbuf += 4 * BLOCKSIZE;
- }
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
movdqa128(xmm0, xmm1);
@@ -2052,10 +1701,12 @@ FUNC_CFB_DEC (RIJNDAEL_context *ctx, unsigned char *iv,
}
ASM_FUNC_ATTR_NOINLINE void
-FUNC_CBC_DEC (RIJNDAEL_context *ctx, unsigned char *iv,
- unsigned char *outbuf, const unsigned char *inbuf,
- size_t nblocks)
+FUNC_CBC_DEC (void *context, unsigned char *iv, void *outbuf_arg,
+ const void *inbuf_arg, size_t nblocks)
{
+ RIJNDAEL_context *ctx = context;
+ unsigned char *outbuf = outbuf_arg;
+ const unsigned char *inbuf = inbuf_arg;
__m128i xmm0, xmm1, xmm5, xmm6, xmm7;
__m128i xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm8;
struct vp_aes_config_s config;
@@ -2073,36 +1724,6 @@ FUNC_CBC_DEC (RIJNDAEL_context *ctx, unsigned char *iv,
movdqu128_memld(iv, xmm7);
-#ifdef HAVE_SIMD256
- if (check_simd256_support())
- {
- __m256i ymm0, ymm1, ymm2, ymm3;
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm0);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm1);
- movdqa256_128(ymm0, xmm0);
- movdqa128_256(xmm7, ymm2);
- insert256_hi128(xmm0, ymm2);
- movdqu256_memld(inbuf + 1 * BLOCKSIZE, ymm3);
- extract256_hi128(ymm1, xmm7);
-
- aes_decrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15, xmm8);
-
- pxor256(ymm2, ymm0);
- pxor256(ymm3, ymm1);
- movdqu256_memst(ymm0, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- outbuf += 4 * BLOCKSIZE;
- inbuf += 4 * BLOCKSIZE;
- }
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
movdqu128_memld(inbuf, xmm0);
@@ -2167,68 +1788,6 @@ aes_simd128_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg,
movdqu128_memld(c->u_iv.iv, xmm7);
movdqu128_memld(c->u_ctr.ctr, xmm6);
-#ifdef HAVE_SIMD256
- if (check_simd256_support() && nblocks >= 4)
- {
- __m256i ymm0, ymm1, ymm3, ymm6, ymm8;
-
- movdqa128_256(xmm6, ymm6);
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- const unsigned char *l;
-
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm0);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm1);
-
- /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
- /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- movdqa128_256(xmm7, ymm3);
-
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- insert256_hi128(xmm7, ymm3);
-
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- movdqa128_256(xmm7, ymm8);
-
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- insert256_hi128(xmm7, ymm8);
-
- /* Checksum_i = Checksum_{i-1} xor P_i */
- pxor256(ymm0, ymm6);
- pxor256(ymm1, ymm6);
-
- pxor256(ymm3, ymm0);
- pxor256(ymm8, ymm1);
-
- aes_encrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15);
-
- pxor256(ymm3, ymm0);
- pxor256(ymm8, ymm1);
- movdqu256_memst(ymm0, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- inbuf += 4 * BLOCKSIZE;
- outbuf += 4 * BLOCKSIZE;
- }
-
- extract256_hi128(ymm6, xmm0);
- movdqa256_128(ymm6, xmm6);
- pxor128(xmm0, xmm6);
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
const unsigned char *l;
@@ -2328,69 +1887,6 @@ aes_simd128_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg,
movdqu128_memld(c->u_iv.iv, xmm7);
movdqu128_memld(c->u_ctr.ctr, xmm6);
-#ifdef HAVE_SIMD256
- if (check_simd256_support() && nblocks >= 4)
- {
- __m256i ymm0, ymm1, ymm3, ymm6, ymm8;
-
- movdqa128_256(xmm6, ymm6);
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- const unsigned char *l;
-
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm0);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm1);
-
- /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
- /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- movdqa128_256(xmm7, ymm3);
-
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- insert256_hi128(xmm7, ymm3);
-
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- movdqa128_256(xmm7, ymm8);
-
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- insert256_hi128(xmm7, ymm8);
-
- pxor256(ymm3, ymm0);
- pxor256(ymm8, ymm1);
-
- aes_decrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15, xmm8);
-
- pxor256(ymm3, ymm0);
- pxor256(ymm8, ymm1);
-
- /* Checksum_i = Checksum_{i-1} xor P_i */
- pxor256(ymm0, ymm6);
- pxor256(ymm1, ymm6);
-
- movdqu256_memst(ymm0, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- inbuf += 4 * BLOCKSIZE;
- outbuf += 4 * BLOCKSIZE;
- }
-
- extract256_hi128(ymm6, xmm0);
- movdqa256_128(ymm6, xmm6);
- pxor128(xmm0, xmm6);
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
const unsigned char *l;
@@ -2493,61 +1989,6 @@ FUNC_OCB_AUTH(gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks)
movdqu128_memld(c->u_mode.ocb.aad_offset, xmm7);
movdqu128_memld(c->u_mode.ocb.aad_sum, xmm6);
-#ifdef HAVE_SIMD256
- if (check_simd256_support() && nblocks >= 4)
- {
- __m256i ymm0, ymm1, ymm3, ymm6, ymm8;
-
- movdqa128_256(xmm6, ymm6);
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- const unsigned char *l;
-
- movdqu256_memld(abuf + 0 * BLOCKSIZE, ymm0);
- movdqu256_memld(abuf + 2 * BLOCKSIZE, ymm1);
-
- /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
- /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- movdqa128_256(xmm7, ymm3);
-
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- insert256_hi128(xmm7, ymm3);
-
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- movdqa128_256(xmm7, ymm8);
-
- l = ocb_get_l(c, ++n);
- movdqu128_memld(l, xmm2);
- pxor128(xmm2, xmm7);
- insert256_hi128(xmm7, ymm8);
-
- pxor256(ymm3, ymm0);
- pxor256(ymm8, ymm1);
-
- aes_encrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15);
-
- pxor256(ymm0, ymm6);
- pxor256(ymm1, ymm6);
-
- abuf += 4 * BLOCKSIZE;
- }
-
- extract256_hi128(ymm6, xmm0);
- movdqa256_128(ymm6, xmm6);
- pxor128(xmm0, xmm6);
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
const unsigned char *l;
@@ -2606,7 +2047,7 @@ FUNC_OCB_AUTH(gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks)
return 0;
}
-ASM_FUNC_ATTR_NOINLINE void
+static ASM_FUNC_ATTR_NOINLINE void
aes_simd128_ecb_enc (void *context, void *outbuf_arg, const void *inbuf_arg,
size_t nblocks)
{
@@ -2621,29 +2062,6 @@ aes_simd128_ecb_enc (void *context, void *outbuf_arg, const void *inbuf_arg,
enc_preload(xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15);
-#ifdef HAVE_SIMD256
- if (check_simd256_support())
- {
- __m256i ymm0, ymm1;
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm0);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm1);
-
- aes_encrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15);
-
- movdqu256_memst(ymm0, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- inbuf += 4 * BLOCKSIZE;
- outbuf += 4 * BLOCKSIZE;
- }
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
movdqu128_memld(inbuf + 0 * BLOCKSIZE, xmm0);
@@ -2677,7 +2095,7 @@ aes_simd128_ecb_enc (void *context, void *outbuf_arg, const void *inbuf_arg,
clear_vec_regs();
}
-ASM_FUNC_ATTR_NOINLINE void
+static ASM_FUNC_ATTR_NOINLINE void
aes_simd128_ecb_dec (void *context, void *outbuf_arg, const void *inbuf_arg,
size_t nblocks)
{
@@ -2698,29 +2116,6 @@ aes_simd128_ecb_dec (void *context, void *outbuf_arg, const void *inbuf_arg,
dec_preload(xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, xmm8);
-#ifdef HAVE_SIMD256
- if (check_simd256_support())
- {
- __m256i ymm0, ymm1;
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm0);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm1);
-
- aes_decrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15, xmm8);
-
- movdqu256_memst(ymm0, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- inbuf += 4 * BLOCKSIZE;
- outbuf += 4 * BLOCKSIZE;
- }
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
movdqu128_memld(inbuf + 0 * BLOCKSIZE, xmm0);
@@ -2796,43 +2191,6 @@ aes_simd128_xts_enc (void *context, unsigned char *tweak, void *outbuf_arg,
movdqu128_memld(tweak, xmm7); /* Preload tweak */
-#ifdef HAVE_SIMD256
- if (check_simd256_support())
- {
- __m256i ymm0, ymm1, ymm2, ymm3;
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm0);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm1);
-
- movdqa128_256(xmm7, ymm2);
- xmm7 = xts_gfmul_byA(xmm7);
- insert256_hi128(xmm7, ymm2);
- xmm7 = xts_gfmul_byA(xmm7);
- movdqa128_256(xmm7, ymm3);
- xmm7 = xts_gfmul_byA(xmm7);
- insert256_hi128(xmm7, ymm3);
- xmm7 = xts_gfmul_byA(xmm7);
-
- pxor256(ymm2, ymm0);
- pxor256(ymm3, ymm1);
-
- aes_encrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15);
-
- pxor256(ymm2, ymm0);
- pxor256(ymm3, ymm1);
- movdqu256_memst(ymm0, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- outbuf += 4 * BLOCKSIZE;
- inbuf += 4 * BLOCKSIZE;
- }
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
movdqu128_memld(inbuf, xmm0);
@@ -2902,43 +2260,6 @@ aes_simd128_xts_dec (void *context, unsigned char *tweak, void *outbuf_arg,
movdqu128_memld(tweak, xmm7); /* Preload tweak */
-#ifdef HAVE_SIMD256
- if (check_simd256_support())
- {
- __m256i ymm0, ymm1, ymm2, ymm3;
-
- for (; nblocks >= 4; nblocks -= 4)
- {
- movdqu256_memld(inbuf + 0 * BLOCKSIZE, ymm0);
- movdqu256_memld(inbuf + 2 * BLOCKSIZE, ymm1);
-
- movdqa128_256(xmm7, ymm2);
- xmm7 = xts_gfmul_byA(xmm7);
- insert256_hi128(xmm7, ymm2);
- xmm7 = xts_gfmul_byA(xmm7);
- movdqa128_256(xmm7, ymm3);
- xmm7 = xts_gfmul_byA(xmm7);
- insert256_hi128(xmm7, ymm3);
- xmm7 = xts_gfmul_byA(xmm7);
-
- pxor256(ymm2, ymm0);
- pxor256(ymm3, ymm1);
-
- aes_decrypt_core_4blks_simd256(&ymm0, &ymm1, config,
- xmm9, xmm10, xmm11, xmm12, xmm13,
- xmm14, xmm15, xmm8);
-
- pxor256(ymm2, ymm0);
- pxor256(ymm3, ymm1);
- movdqu256_memst(ymm0, outbuf + 0 * BLOCKSIZE);
- movdqu256_memst(ymm1, outbuf + 2 * BLOCKSIZE);
-
- outbuf += 4 * BLOCKSIZE;
- inbuf += 4 * BLOCKSIZE;
- }
- }
-#endif /* HAVE_SIMD256 */
-
for (; nblocks >= 2; nblocks -= 2)
{
movdqu128_memld(inbuf, xmm0);
diff --git a/cipher/rijndael.c b/cipher/rijndael.c
index 082b20ab..0586bfd2 100644
--- a/cipher/rijndael.c
+++ b/cipher/rijndael.c
@@ -227,58 +227,9 @@ extern void _gcry_aes_vp_aarch64_xts_crypt (void *context, unsigned char *tweak,
#ifdef USE_VP_RISCV
/* RISC-V vector permutation implementation of AES */
-extern int _gcry_aes_vp_riscv_setup_acceleration(RIJNDAEL_context *ctx);
-
-extern void _gcry_aes_vp_riscv_do_setkey(RIJNDAEL_context *ctx,
- const byte *key);
-extern void _gcry_aes_vp_riscv_prepare_decryption(RIJNDAEL_context *ctx);
-
-extern unsigned int _gcry_aes_vp_riscv_encrypt (const RIJNDAEL_context *ctx,
- unsigned char *dst,
- const unsigned char *src);
-extern unsigned int _gcry_aes_vp_riscv_decrypt (const RIJNDAEL_context *ctx,
- unsigned char *dst,
- const unsigned char *src);
-extern void _gcry_aes_vp_riscv_cfb_enc (void *context, unsigned char *iv,
- void *outbuf_arg,
- const void *inbuf_arg,
- size_t nblocks);
-extern void _gcry_aes_vp_riscv_cbc_enc (void *context, unsigned char *iv,
- void *outbuf_arg,
- const void *inbuf_arg,
- size_t nblocks,
- int cbc_mac);
-extern void _gcry_aes_vp_riscv_ctr_enc (void *context, unsigned char *ctr,
- void *outbuf_arg,
- const void *inbuf_arg,
- size_t nblocks);
-extern void _gcry_aes_vp_riscv_ctr32le_enc (void *context, unsigned char *ctr,
- void *outbuf_arg,
- const void *inbuf_arg,
- size_t nblocks);
-extern void _gcry_aes_vp_riscv_cfb_dec (void *context, unsigned char *iv,
- void *outbuf_arg,
- const void *inbuf_arg,
- size_t nblocks);
-extern void _gcry_aes_vp_riscv_cbc_dec (void *context, unsigned char *iv,
- void *outbuf_arg,
- const void *inbuf_arg,
- size_t nblocks);
-extern size_t _gcry_aes_vp_riscv_ocb_crypt (gcry_cipher_hd_t c,
- void *outbuf_arg,
- const void *inbuf_arg,
- size_t nblocks,
- int encrypt);
-extern size_t _gcry_aes_vp_riscv_ocb_auth (gcry_cipher_hd_t c,
- const void *abuf_arg,
- size_t nblocks);
-extern void _gcry_aes_vp_riscv_ecb_crypt (void *context, void *outbuf_arg,
- const void *inbuf_arg,
- size_t nblocks, int encrypt);
-extern void _gcry_aes_vp_riscv_xts_crypt (void *context, unsigned char *tweak,
- void *outbuf_arg,
- const void *inbuf_arg,
- size_t nblocks, int encrypt);
+extern int _gcry_aes_vp_riscv_setup_acceleration(RIJNDAEL_context *ctx,
+ rijndael_setkeyfn_t *hw_setkey,
+ cipher_bulk_ops_t *bulk_ops);
#endif
#ifdef USE_RISCV_V_CRYPTO
@@ -871,26 +822,10 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
else if ((hwfeatures & HWF_RISCV_IMAFDC)
&& (hwfeatures & HWF_RISCV_B) /* Mandatory in RVA22U64 */
&& (hwfeatures & HWF_RISCV_V) /* Optional in RVA22U64 */
- && _gcry_aes_vp_riscv_setup_acceleration(ctx))
+ && _gcry_aes_vp_riscv_setup_acceleration(ctx, &hw_setkey, bulk_ops))
{
- hw_setkey = _gcry_aes_vp_riscv_do_setkey;
- ctx->encrypt_fn = _gcry_aes_vp_riscv_encrypt;
- ctx->decrypt_fn = _gcry_aes_vp_riscv_decrypt;
ctx->prefetch_enc_fn = NULL;
ctx->prefetch_dec_fn = NULL;
- ctx->prepare_decryption = _gcry_aes_vp_riscv_prepare_decryption;
-
- /* Setup vector permute RISC-V bulk encryption routines. */
- bulk_ops->cfb_enc = _gcry_aes_vp_riscv_cfb_enc;
- bulk_ops->cfb_dec = _gcry_aes_vp_riscv_cfb_dec;
- bulk_ops->cbc_enc = _gcry_aes_vp_riscv_cbc_enc;
- bulk_ops->cbc_dec = _gcry_aes_vp_riscv_cbc_dec;
- bulk_ops->ctr16be_enc = _gcry_aes_vp_riscv_ctr_enc;
- bulk_ops->ctr32le_enc = _gcry_aes_vp_riscv_ctr32le_enc;
- bulk_ops->ocb_crypt = _gcry_aes_vp_riscv_ocb_crypt;
- bulk_ops->ocb_auth = _gcry_aes_vp_riscv_ocb_auth;
- bulk_ops->ecb_crypt = _gcry_aes_vp_riscv_ecb_crypt;
- bulk_ops->xts_crypt = _gcry_aes_vp_riscv_xts_crypt;
}
#endif
#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
diff --git a/configure.ac b/configure.ac
index b10af44c..4299b433 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3787,6 +3787,9 @@ if test "$found" = "1" ; then
riscv64-*-*)
# Build with the vector permute SIMD128 implementation
GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS rijndael-vp-riscv.lo"
+ GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS rijndael-vp-riscv-mf2.lo"
+ GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS rijndael-vp-riscv-mf4.lo"
+ GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS rijndael-vp-riscv-mf8.lo"
# Build with the RISC-V vector cryptography implementation
GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS rijndael-riscv-zvkned.lo"
--
2.53.0
More information about the Gcrypt-devel
mailing list