[PATCH v2 3/3] Add SM4 ARMv9 SVE CE assembly implementation
Tianjia Zhang
tianjia.zhang at linux.alibaba.com
Thu Jul 21 08:32:17 CEST 2022
* cipher/Makefile.am: Add 'sm4-armv9-aarch64-sve-ce.S'.
* cipher/sm4-armv9-aarch64-sve-ce.S: New.
* cipher/sm4.c (USE_ARM_SVE_CE): New.
(SM4_context) [USE_ARM_SVE_CE]: Add 'use_arm_sve_ce'.
(_gcry_sm4_armv9_sve_ce_crypt, _gcry_sm4_armv9_sve_ce_ctr_enc)
(_gcry_sm4_armv9_sve_ce_cbc_dec, _gcry_sm4_armv9_sve_ce_cfb_dec)
(sm4_armv9_sve_ce_crypt_blk1_16): New.
(sm4_setkey): Enable ARMv9 SVE CE if supported by HW.
(sm4_get_crypt_blk1_16_fn) [USE_ARM_SVE_CE]: Add ARMv9 SVE CE
bulk functions.
(_gcry_sm4_ctr_enc, _gcry_sm4_cbc_dec, _gcry_sm4_cfb_dec)
[USE_ARM_SVE_CE]: Add ARMv9 SVE CE bulk functions.
* configure.ac: Add 'sm4-armv9-aarch64-sve-ce.lo'.
--
Signed-off-by: Tianjia Zhang <tianjia.zhang at linux.alibaba.com>
---
cipher/Makefile.am | 1 +
cipher/sm4-armv9-aarch64-sve-ce.S | 967 ++++++++++++++++++++++++++++++
cipher/sm4.c | 86 +++
configure.ac | 1 +
4 files changed, 1055 insertions(+)
create mode 100644 cipher/sm4-armv9-aarch64-sve-ce.S
diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index 042dc0a7170d..97823cb48bd3 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -120,6 +120,7 @@ EXTRA_libcipher_la_SOURCES = \
serpent.c serpent-sse2-amd64.S \
sm4.c sm4-aesni-avx-amd64.S sm4-aesni-avx2-amd64.S sm4-aarch64.S \
sm4-armv8-aarch64-ce.S sm4-gfni-avx2-amd64.S \
+ sm4-armv9-aarch64-sve-ce.S \
serpent-avx2-amd64.S serpent-armv7-neon.S \
sha1.c sha1-ssse3-amd64.S sha1-avx-amd64.S sha1-avx-bmi2-amd64.S \
sha1-avx2-bmi2-amd64.S sha1-armv7-neon.S sha1-armv8-aarch32-ce.S \
diff --git a/cipher/sm4-armv9-aarch64-sve-ce.S b/cipher/sm4-armv9-aarch64-sve-ce.S
new file mode 100644
index 000000000000..21e34e6ffc84
--- /dev/null
+++ b/cipher/sm4-armv9-aarch64-sve-ce.S
@@ -0,0 +1,967 @@
+/* sm4-armv9-aarch64-sve-ce.S - ARMv9/AArch64 SVE Cryptography accelerated SM4
+ *
+ * Copyright (C) 2022 Alibaba Group.
+ * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang at linux.alibaba.com>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "asm-common-aarch64.h"
+
+#if defined(__AARCH64EL__) && \
+ defined(HAVE_COMPATIBLE_GCC_AARCH64_PLATFORM_AS) && \
+ defined(HAVE_GCC_INLINE_ASM_AARCH64_CRYPTO) && \
+ defined(HAVE_GCC_INLINE_ASM_AARCH64_SVE) && \
+ defined(HAVE_GCC_INLINE_ASM_AARCH64_SVE2) && \
+ defined(USE_SM4)
+
+.cpu generic+simd+crypto+sve+sve2
+
+/* Constants */
+
+.text
+.align 4
+ELF(.type _gcry_sm4_armv9_svesm4_consts, at object)
+_gcry_sm4_armv9_svesm4_consts:
+.Lbswap128_mask:
+ .byte 0x0c, 0x0d, 0x0e, 0x0f, 0x08, 0x09, 0x0a, 0x0b
+ .byte 0x04, 0x05, 0x06, 0x07, 0x00, 0x01, 0x02, 0x03
+ .byte 0x1c, 0x1d, 0x1e, 0x1f, 0x18, 0x19, 0x1a, 0x1b
+ .byte 0x14, 0x15, 0x16, 0x17, 0x10, 0x11, 0x12, 0x13
+ .byte 0x2c, 0x2d, 0x2e, 0x2f, 0x28, 0x29, 0x2a, 0x2b
+ .byte 0x24, 0x25, 0x26, 0x27, 0x20, 0x21, 0x22, 0x23
+ .byte 0x3c, 0x3d, 0x3e, 0x3f, 0x38, 0x39, 0x3a, 0x3b
+ .byte 0x34, 0x35, 0x36, 0x37, 0x30, 0x31, 0x32, 0x33
+ .byte 0x4c, 0x4d, 0x4e, 0x4f, 0x48, 0x49, 0x4a, 0x4b
+ .byte 0x44, 0x45, 0x46, 0x47, 0x40, 0x41, 0x42, 0x43
+ .byte 0x5c, 0x5d, 0x5e, 0x5f, 0x58, 0x59, 0x5a, 0x5b
+ .byte 0x54, 0x55, 0x56, 0x57, 0x50, 0x51, 0x52, 0x53
+ .byte 0x6c, 0x6d, 0x6e, 0x6f, 0x68, 0x69, 0x6a, 0x6b
+ .byte 0x64, 0x65, 0x66, 0x67, 0x60, 0x61, 0x62, 0x63
+ .byte 0x7c, 0x7d, 0x7e, 0x7f, 0x78, 0x79, 0x7a, 0x7b
+ .byte 0x74, 0x75, 0x76, 0x77, 0x70, 0x71, 0x72, 0x73
+ .byte 0x8c, 0x8d, 0x8e, 0x8f, 0x88, 0x89, 0x8a, 0x8b
+ .byte 0x84, 0x85, 0x86, 0x87, 0x80, 0x81, 0x82, 0x83
+ .byte 0x9c, 0x9d, 0x9e, 0x9f, 0x98, 0x99, 0x9a, 0x9b
+ .byte 0x94, 0x95, 0x96, 0x97, 0x90, 0x91, 0x92, 0x93
+ .byte 0xac, 0xad, 0xae, 0xaf, 0xa8, 0xa9, 0xaa, 0xab
+ .byte 0xa4, 0xa5, 0xa6, 0xa7, 0xa0, 0xa1, 0xa2, 0xa3
+ .byte 0xbc, 0xbd, 0xbe, 0xbf, 0xb8, 0xb9, 0xba, 0xbb
+ .byte 0xb4, 0xb5, 0xb6, 0xb7, 0xb0, 0xb1, 0xb2, 0xb3
+ .byte 0xcc, 0xcd, 0xce, 0xcf, 0xc8, 0xc9, 0xca, 0xcb
+ .byte 0xc4, 0xc5, 0xc6, 0xc7, 0xc0, 0xc1, 0xc2, 0xc3
+ .byte 0xdc, 0xdd, 0xde, 0xdf, 0xd8, 0xd9, 0xda, 0xdb
+ .byte 0xd4, 0xd5, 0xd6, 0xd7, 0xd0, 0xd1, 0xd2, 0xd3
+ .byte 0xec, 0xed, 0xee, 0xef, 0xe8, 0xe9, 0xea, 0xeb
+ .byte 0xe4, 0xe5, 0xe6, 0xe7, 0xe0, 0xe1, 0xe2, 0xe3
+ .byte 0xfc, 0xfd, 0xfe, 0xff, 0xf8, 0xf9, 0xfa, 0xfb
+ .byte 0xf4, 0xf5, 0xf6, 0xf7, 0xf0, 0xf1, 0xf2, 0xf3
+
+.Lle128_inc:
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+ELF(.size _gcry_sm4_armv9_svesm4_consts,.-_gcry_sm4_armv9_svesm4_consts)
+
+/* Register macros */
+
+#define RCTR z16
+#define RCTRv v16
+#define RIV z16
+#define RIVv v16
+#define RSWAP128 z17
+#define RZERO z18
+#define RLE128_INC z19
+
+#define RTMP0 z20
+#define RTMP1 z21
+#define RTMP2 z22
+#define RTMP3 z23
+#define RTMP0v v20
+
+#define vecnum_z0 0
+#define vecnum_z1 1
+#define vecnum_z2 2
+#define vecnum_z3 3
+#define vecnum_z4 4
+#define vecnum_z5 5
+#define vecnum_z6 6
+#define vecnum_z7 7
+#define vecnum_z8 8
+#define vecnum_z9 9
+#define vecnum_z10 10
+#define vecnum_z11 11
+#define vecnum_z12 12
+#define vecnum_z13 13
+#define vecnum_z14 14
+#define vecnum_z15 15
+#define vecnum_z16 16
+#define vecnum_z24 24
+#define vecnum_z25 25
+#define vecnum_z26 26
+#define vecnum_z27 27
+#define vecnum_z28 28
+#define vecnum_z29 29
+#define vecnum_z30 30
+#define vecnum_z31 31
+
+#define vecnum_v0 0
+#define vecnum_v15 15
+#define vecnum_v24 24
+#define vecnum_v25 25
+#define vecnum_v26 26
+#define vecnum_v27 27
+#define vecnum_v28 28
+#define vecnum_v29 29
+#define vecnum_v30 30
+#define vecnum_v31 31
+
+#define sm4e_ce(vd, vn) \
+ .inst (0xcec08400 | (vecnum_##vn << 5) | vecnum_##vd)
+
+#define sm4e_sve(zd, zm) \
+ .inst (0x4523e000 | (vecnum_##zm << 5) | vecnum_##zd)
+
+/* Helper macros. */
+
+#define PREPARE() \
+ GET_LOCAL_POINTER(x7, .Lbswap128_mask); \
+ ptrue p0.b, ALL; \
+ rdvl x5, #1; \
+ ld1b {RSWAP128.b}, p0/z, [x7]; \
+ \
+ ld1 {v24.16b-v27.16b}, [x0], #64; \
+ ld1 {v28.16b-v31.16b}, [x0]; \
+ dup z24.q, z24.q[0]; \
+ dup z25.q, z25.q[0]; \
+ dup z26.q, z26.q[0]; \
+ dup z27.q, z27.q[0]; \
+ dup z28.q, z28.q[0]; \
+ dup z29.q, z29.q[0]; \
+ dup z30.q, z30.q[0]; \
+ dup z31.q, z31.q[0];
+
+
+#define SM4_SVE_CE_CRYPT_BLK(b0) \
+ revb b0.s, p0/m, b0.s; \
+ sm4e_sve(b0, z24); \
+ sm4e_sve(b0, z25); \
+ sm4e_sve(b0, z26); \
+ sm4e_sve(b0, z27); \
+ sm4e_sve(b0, z28); \
+ sm4e_sve(b0, z29); \
+ sm4e_sve(b0, z30); \
+ sm4e_sve(b0, z31); \
+ tbl b0.b, {b0.b}, RSWAP128.b; \
+ revb b0.s, p0/m, b0.s;
+
+
+#define SM4_SVE_CE_CRYPT_BLK4(b0, b1, b2, b3) \
+ revb b0.s, p0/m, b0.s; \
+ revb b1.s, p0/m, b1.s; \
+ revb b2.s, p0/m, b2.s; \
+ revb b3.s, p0/m, b3.s; \
+ sm4e_sve(b0, z24); \
+ sm4e_sve(b1, z24); \
+ sm4e_sve(b2, z24); \
+ sm4e_sve(b3, z24); \
+ sm4e_sve(b0, z25); \
+ sm4e_sve(b1, z25); \
+ sm4e_sve(b2, z25); \
+ sm4e_sve(b3, z25); \
+ sm4e_sve(b0, z26); \
+ sm4e_sve(b1, z26); \
+ sm4e_sve(b2, z26); \
+ sm4e_sve(b3, z26); \
+ sm4e_sve(b0, z27); \
+ sm4e_sve(b1, z27); \
+ sm4e_sve(b2, z27); \
+ sm4e_sve(b3, z27); \
+ sm4e_sve(b0, z28); \
+ sm4e_sve(b1, z28); \
+ sm4e_sve(b2, z28); \
+ sm4e_sve(b3, z28); \
+ sm4e_sve(b0, z29); \
+ sm4e_sve(b1, z29); \
+ sm4e_sve(b2, z29); \
+ sm4e_sve(b3, z29); \
+ sm4e_sve(b0, z30); \
+ sm4e_sve(b1, z30); \
+ sm4e_sve(b2, z30); \
+ sm4e_sve(b3, z30); \
+ sm4e_sve(b0, z31); \
+ sm4e_sve(b1, z31); \
+ sm4e_sve(b2, z31); \
+ sm4e_sve(b3, z31); \
+ tbl b0.b, {b0.b}, RSWAP128.b; \
+ tbl b1.b, {b1.b}, RSWAP128.b; \
+ tbl b2.b, {b2.b}, RSWAP128.b; \
+ tbl b3.b, {b3.b}, RSWAP128.b; \
+ revb b0.s, p0/m, b0.s; \
+ revb b1.s, p0/m, b1.s; \
+ revb b2.s, p0/m, b2.s; \
+ revb b3.s, p0/m, b3.s;
+
+
+#define SM4_SVE_CE_CRYPT_BLK8(b0, b1, b2, b3, b4, b5, b6, b7) \
+ revb b0.s, p0/m, b0.s; \
+ revb b1.s, p0/m, b1.s; \
+ revb b2.s, p0/m, b2.s; \
+ revb b3.s, p0/m, b3.s; \
+ revb b4.s, p0/m, b4.s; \
+ revb b5.s, p0/m, b5.s; \
+ revb b6.s, p0/m, b6.s; \
+ revb b7.s, p0/m, b7.s; \
+ sm4e_sve(b0, z24); \
+ sm4e_sve(b1, z24); \
+ sm4e_sve(b2, z24); \
+ sm4e_sve(b3, z24); \
+ sm4e_sve(b4, z24); \
+ sm4e_sve(b5, z24); \
+ sm4e_sve(b6, z24); \
+ sm4e_sve(b7, z24); \
+ sm4e_sve(b0, z25); \
+ sm4e_sve(b1, z25); \
+ sm4e_sve(b2, z25); \
+ sm4e_sve(b3, z25); \
+ sm4e_sve(b4, z25); \
+ sm4e_sve(b5, z25); \
+ sm4e_sve(b6, z25); \
+ sm4e_sve(b7, z25); \
+ sm4e_sve(b0, z26); \
+ sm4e_sve(b1, z26); \
+ sm4e_sve(b2, z26); \
+ sm4e_sve(b3, z26); \
+ sm4e_sve(b4, z26); \
+ sm4e_sve(b5, z26); \
+ sm4e_sve(b6, z26); \
+ sm4e_sve(b7, z26); \
+ sm4e_sve(b0, z27); \
+ sm4e_sve(b1, z27); \
+ sm4e_sve(b2, z27); \
+ sm4e_sve(b3, z27); \
+ sm4e_sve(b4, z27); \
+ sm4e_sve(b5, z27); \
+ sm4e_sve(b6, z27); \
+ sm4e_sve(b7, z27); \
+ sm4e_sve(b0, z28); \
+ sm4e_sve(b1, z28); \
+ sm4e_sve(b2, z28); \
+ sm4e_sve(b3, z28); \
+ sm4e_sve(b4, z28); \
+ sm4e_sve(b5, z28); \
+ sm4e_sve(b6, z28); \
+ sm4e_sve(b7, z28); \
+ sm4e_sve(b0, z29); \
+ sm4e_sve(b1, z29); \
+ sm4e_sve(b2, z29); \
+ sm4e_sve(b3, z29); \
+ sm4e_sve(b4, z29); \
+ sm4e_sve(b5, z29); \
+ sm4e_sve(b6, z29); \
+ sm4e_sve(b7, z29); \
+ sm4e_sve(b0, z30); \
+ sm4e_sve(b1, z30); \
+ sm4e_sve(b2, z30); \
+ sm4e_sve(b3, z30); \
+ sm4e_sve(b4, z30); \
+ sm4e_sve(b5, z30); \
+ sm4e_sve(b6, z30); \
+ sm4e_sve(b7, z30); \
+ sm4e_sve(b0, z31); \
+ sm4e_sve(b1, z31); \
+ sm4e_sve(b2, z31); \
+ sm4e_sve(b3, z31); \
+ sm4e_sve(b4, z31); \
+ sm4e_sve(b5, z31); \
+ sm4e_sve(b6, z31); \
+ sm4e_sve(b7, z31); \
+ tbl b0.b, {b0.b}, RSWAP128.b; \
+ tbl b1.b, {b1.b}, RSWAP128.b; \
+ tbl b2.b, {b2.b}, RSWAP128.b; \
+ tbl b3.b, {b3.b}, RSWAP128.b; \
+ tbl b4.b, {b4.b}, RSWAP128.b; \
+ tbl b5.b, {b5.b}, RSWAP128.b; \
+ tbl b6.b, {b6.b}, RSWAP128.b; \
+ tbl b7.b, {b7.b}, RSWAP128.b; \
+ revb b0.s, p0/m, b0.s; \
+ revb b1.s, p0/m, b1.s; \
+ revb b2.s, p0/m, b2.s; \
+ revb b3.s, p0/m, b3.s; \
+ revb b4.s, p0/m, b4.s; \
+ revb b5.s, p0/m, b5.s; \
+ revb b6.s, p0/m, b6.s; \
+ revb b7.s, p0/m, b7.s;
+
+
+#define SM4_CE_CRYPT_BLK(b0) \
+ rev32 b0.16b, b0.16b; \
+ sm4e_ce(b0, v24); \
+ sm4e_ce(b0, v25); \
+ sm4e_ce(b0, v26); \
+ sm4e_ce(b0, v27); \
+ sm4e_ce(b0, v28); \
+ sm4e_ce(b0, v29); \
+ sm4e_ce(b0, v30); \
+ sm4e_ce(b0, v31); \
+ rev64 b0.4s, b0.4s; \
+ ext b0.16b, b0.16b, b0.16b, #8; \
+ rev32 b0.16b, b0.16b;
+
+
+.align 3
+.global _gcry_sm4_armv9_sve_ce_crypt
+ELF(.type _gcry_sm4_armv9_sve_ce_crypt,%function;)
+_gcry_sm4_armv9_sve_ce_crypt:
+ /* input:
+ * x0: round key array, CTX
+ * x1: dst
+ * x2: src
+ * x3: nblocks
+ */
+ CFI_STARTPROC();
+
+ PREPARE();
+
+.Lcrypt_loop_blks:
+ sub x3, x3, x5, LSR #1; /* x3 - (8 * VL) */
+ tbnz x3, #63, .Lcrypt_tail8;
+
+ ld1b {z0.b}, p0/z, [x2];
+ ld1b {z1.b}, p0/z, [x2, #1, MUL VL];
+ ld1b {z2.b}, p0/z, [x2, #2, MUL VL];
+ ld1b {z3.b}, p0/z, [x2, #3, MUL VL];
+ ld1b {z4.b}, p0/z, [x2, #4, MUL VL];
+ ld1b {z5.b}, p0/z, [x2, #5, MUL VL];
+ ld1b {z6.b}, p0/z, [x2, #6, MUL VL];
+ ld1b {z7.b}, p0/z, [x2, #7, MUL VL];
+ addvl x2, x2, #8;
+
+ SM4_SVE_CE_CRYPT_BLK8(z0, z1, z2, z3, z4, z5, z6, z7);
+
+ st1b {z0.b}, p0, [x1];
+ st1b {z1.b}, p0, [x1, #1, MUL VL];
+ st1b {z2.b}, p0, [x1, #2, MUL VL];
+ st1b {z3.b}, p0, [x1, #3, MUL VL];
+ st1b {z4.b}, p0, [x1, #4, MUL VL];
+ st1b {z5.b}, p0, [x1, #5, MUL VL];
+ st1b {z6.b}, p0, [x1, #6, MUL VL];
+ st1b {z7.b}, p0, [x1, #7, MUL VL];
+ addvl x1, x1, #8;
+
+ cbz x3, .Lcrypt_end;
+ b .Lcrypt_loop_blks;
+
+.Lcrypt_tail8:
+ add x3, x3, x5, LSR #1;
+ cmp x3, x5, LSR #2;
+ blt .Lcrypt_tail4;
+
+ sub x3, x3, x5, LSR #2; /* x3 - (4 * VL) */
+
+ ld1b {z0.b}, p0/z, [x2];
+ ld1b {z1.b}, p0/z, [x2, #1, MUL VL];
+ ld1b {z2.b}, p0/z, [x2, #2, MUL VL];
+ ld1b {z3.b}, p0/z, [x2, #3, MUL VL];
+ addvl x2, x2, #4;
+
+ SM4_SVE_CE_CRYPT_BLK4(z0, z1, z2, z3);
+
+ st1b {z0.b}, p0, [x1];
+ st1b {z1.b}, p0, [x1, #1, MUL VL];
+ st1b {z2.b}, p0, [x1, #2, MUL VL];
+ st1b {z3.b}, p0, [x1, #3, MUL VL];
+ addvl x1, x1, #4;
+
+ cbz x3, .Lcrypt_end;
+
+.Lcrypt_tail4:
+ cmp x3, x5, LSR #4;
+ blt .Lcrypt_tail;
+
+ sub x3, x3, x5, LSR #4; /* x3 - VL */
+
+ ld1b {z0.b}, p0/z, [x2];
+ addvl x2, x2, #1;
+
+ SM4_SVE_CE_CRYPT_BLK(z0);
+
+ st1b {z0.b}, p0, [x1];
+ addvl x1, x1, #1;
+
+ cbz x3, .Lcrypt_end;
+
+.Lcrypt_tail:
+ sub x3, x3, #1;
+
+ ld1 {v0.16b}, [x2], #16;
+ SM4_CE_CRYPT_BLK(v0);
+ st1 {v0.16b}, [x1], #16;
+
+ cbnz x3, .Lcrypt_tail;
+
+.Lcrypt_end:
+ ret_spec_stop;
+ CFI_ENDPROC();
+ELF(.size _gcry_sm4_armv9_sve_ce_crypt,.-_gcry_sm4_armv9_sve_ce_crypt;)
+
+.align 3
+.global _gcry_sm4_armv9_sve_ce_cbc_dec
+ELF(.type _gcry_sm4_armv9_sve_ce_cbc_dec,%function;)
+_gcry_sm4_armv9_sve_ce_cbc_dec:
+ /* input:
+ * x0: round key array, CTX
+ * x1: dst
+ * x2: src
+ * x3: iv (big endian, 128 bit)
+ * x4: nblocks
+ */
+ CFI_STARTPROC();
+ VPUSH_ABI;
+
+ PREPARE();
+ ld1 {RIVv.16b}, [x3];
+ ext RIV.b, RIV.b, RIV.b, #16;
+
+.Lcbc_loop_blks:
+ sub x4, x4, x5, LSR #1; /* x4 - (8 * VL) */
+ tbnz x4, #63, .Lcbc_tail8;
+
+ ld1b {z15.b}, p0/z, [x2];
+ ld1b {z14.b}, p0/z, [x2, #1, MUL VL];
+ ld1b {z13.b}, p0/z, [x2, #2, MUL VL];
+ ld1b {z12.b}, p0/z, [x2, #3, MUL VL];
+ ld1b {z11.b}, p0/z, [x2, #4, MUL VL];
+ ld1b {z10.b}, p0/z, [x2, #5, MUL VL];
+ ld1b {z9.b}, p0/z, [x2, #6, MUL VL];
+ ld1b {z8.b}, p0/z, [x2, #7, MUL VL];
+ rev z0.b, z15.b;
+ rev z1.b, z14.b;
+ rev z2.b, z13.b;
+ rev z3.b, z12.b;
+ rev z4.b, z11.b;
+ rev z5.b, z10.b;
+ rev z6.b, z9.b;
+ rev z7.b, z8.b;
+ rev RTMP0.b, RIV.b;
+ ext z7.b, z7.b, z6.b, #16;
+ ext z6.b, z6.b, z5.b, #16;
+ ext z5.b, z5.b, z4.b, #16;
+ ext z4.b, z4.b, z3.b, #16;
+ ext z3.b, z3.b, z2.b, #16;
+ ext z2.b, z2.b, z1.b, #16;
+ ext z1.b, z1.b, z0.b, #16;
+ ext z0.b, z0.b, RTMP0.b, #16;
+ rev z7.b, z7.b;
+ rev z6.b, z6.b;
+ rev z5.b, z5.b;
+ rev z4.b, z4.b;
+ rev z3.b, z3.b;
+ rev z2.b, z2.b;
+ rev z1.b, z1.b;
+ rev z0.b, z0.b;
+ mov RIV.d, z8.d;
+
+ SM4_SVE_CE_CRYPT_BLK8(z15, z14, z13, z12, z11, z10, z9, z8);
+
+ eor z0.d, z0.d, z15.d;
+ eor z1.d, z1.d, z14.d;
+ eor z2.d, z2.d, z13.d;
+ eor z3.d, z3.d, z12.d;
+ eor z4.d, z4.d, z11.d;
+ eor z5.d, z5.d, z10.d;
+ eor z6.d, z6.d, z9.d;
+ eor z7.d, z7.d, z8.d;
+ st1b {z0.b}, p0, [x1];
+ st1b {z1.b}, p0, [x1, #1, MUL VL];
+ st1b {z2.b}, p0, [x1, #2, MUL VL];
+ st1b {z3.b}, p0, [x1, #3, MUL VL];
+ st1b {z4.b}, p0, [x1, #4, MUL VL];
+ st1b {z5.b}, p0, [x1, #5, MUL VL];
+ st1b {z6.b}, p0, [x1, #6, MUL VL];
+ st1b {z7.b}, p0, [x1, #7, MUL VL];
+ addvl x2, x2, #8;
+ addvl x1, x1, #8;
+
+ cbz x4, .Lcbc_end;
+ b .Lcbc_loop_blks;
+
+.Lcbc_tail8:
+ add x4, x4, x5, LSR #1;
+ cmp x4, x5, LSR #2;
+ blt .Lcbc_tail4;
+
+ sub x4, x4, x5, LSR #2; /* x4 - (4 * VL) */
+
+ ld1b {z15.b}, p0/z, [x2];
+ ld1b {z14.b}, p0/z, [x2, #1, MUL VL];
+ ld1b {z13.b}, p0/z, [x2, #2, MUL VL];
+ ld1b {z12.b}, p0/z, [x2, #3, MUL VL];
+ rev z0.b, z15.b;
+ rev z1.b, z14.b;
+ rev z2.b, z13.b;
+ rev z3.b, z12.b;
+ rev RTMP0.b, RIV.b;
+ ext z3.b, z3.b, z2.b, #16;
+ ext z2.b, z2.b, z1.b, #16;
+ ext z1.b, z1.b, z0.b, #16;
+ ext z0.b, z0.b, RTMP0.b, #16;
+ rev z3.b, z3.b;
+ rev z2.b, z2.b;
+ rev z1.b, z1.b;
+ rev z0.b, z0.b;
+ mov RIV.d, z12.d;
+
+ SM4_SVE_CE_CRYPT_BLK4(z15, z14, z13, z12);
+
+ eor z0.d, z0.d, z15.d;
+ eor z1.d, z1.d, z14.d;
+ eor z2.d, z2.d, z13.d;
+ eor z3.d, z3.d, z12.d;
+ st1b {z0.b}, p0, [x1];
+ st1b {z1.b}, p0, [x1, #1, MUL VL];
+ st1b {z2.b}, p0, [x1, #2, MUL VL];
+ st1b {z3.b}, p0, [x1, #3, MUL VL];
+ addvl x2, x2, #4;
+ addvl x1, x1, #4;
+
+ cbz x4, .Lcbc_end;
+
+.Lcbc_tail4:
+ cmp x4, x5, LSR #4;
+ blt .Lcbc_tail_ce;
+
+ sub x4, x4, x5, LSR #4; /* x4 - VL */
+
+ ld1b {z15.b}, p0/z, [x2];
+ rev RTMP0.b, RIV.b;
+ rev z0.b, z15.b;
+ ext z0.b, z0.b, RTMP0.b, #16;
+ rev z0.b, z0.b;
+ mov RIV.d, z15.d;
+
+ SM4_SVE_CE_CRYPT_BLK(z15);
+
+ eor z0.d, z0.d, z15.d;
+ st1b {z0.b}, p0, [x1];
+ addvl x2, x2, #1;
+ addvl x1, x1, #1;
+
+ cbz x4, .Lcbc_end;
+ b .Lcbc_tail4;
+
+.Lcbc_tail_ce:
+ rev RIV.s, RIV.s;
+ tbl RIV.b, {RIV.b}, RSWAP128.b;
+
+.Lcbc_tail:
+ sub x4, x4, #1;
+
+ ld1 {v15.16b}, [x2], #16;
+ mov v0.16b, RIVv.16b;
+ mov RIVv.16b, v15.16b;
+ SM4_CE_CRYPT_BLK(v15);
+ eor v0.16b, v0.16b, v15.16b;
+ st1 {v0.16b}, [x1], #16;
+
+ cbnz x4, .Lcbc_tail;
+
+ ext RIV.b, RIV.b, RIV.b, #16;
+
+.Lcbc_end:
+ /* store new IV */
+ rev RIV.s, RIV.s;
+ tbl RIV.b, {RIV.b}, RSWAP128.b;
+ st1 {RIVv.16b}, [x3];
+
+ VPOP_ABI;
+ ret_spec_stop;
+ CFI_ENDPROC();
+ELF(.size _gcry_sm4_armv9_sve_ce_cbc_dec,.-_gcry_sm4_armv9_sve_ce_cbc_dec;)
+
+.align 3
+.global _gcry_sm4_armv9_sve_ce_cfb_dec
+ELF(.type _gcry_sm4_armv9_sve_ce_cfb_dec,%function;)
+_gcry_sm4_armv9_sve_ce_cfb_dec:
+ /* input:
+ * x0: round key array, CTX
+ * x1: dst
+ * x2: src
+ * x3: iv (big endian, 128 bit)
+ * x4: nblocks
+ */
+ CFI_STARTPROC();
+ VPUSH_ABI;
+
+ PREPARE();
+ ld1 {RIVv.16b}, [x3];
+ ext RIV.b, RIV.b, RIV.b, #16;
+
+.Lcfb_loop_blks:
+ sub x4, x4, x5, LSR #1; /* x4 - (8 * VL) */
+ tbnz x4, #63, .Lcfb_tail8;
+
+ ld1b {z15.b}, p0/z, [x2];
+ ld1b {z14.b}, p0/z, [x2, #1, MUL VL];
+ ld1b {z13.b}, p0/z, [x2, #2, MUL VL];
+ ld1b {z12.b}, p0/z, [x2, #3, MUL VL];
+ ld1b {z11.b}, p0/z, [x2, #4, MUL VL];
+ ld1b {z10.b}, p0/z, [x2, #5, MUL VL];
+ ld1b {z9.b}, p0/z, [x2, #6, MUL VL];
+ ld1b {z8.b}, p0/z, [x2, #7, MUL VL];
+ rev z0.b, z15.b;
+ rev z1.b, z14.b;
+ rev z2.b, z13.b;
+ rev z3.b, z12.b;
+ rev z4.b, z11.b;
+ rev z5.b, z10.b;
+ rev z6.b, z9.b;
+ rev z7.b, z8.b;
+ rev RTMP0.b, RIV.b;
+ ext z7.b, z7.b, z6.b, #16;
+ ext z6.b, z6.b, z5.b, #16;
+ ext z5.b, z5.b, z4.b, #16;
+ ext z4.b, z4.b, z3.b, #16;
+ ext z3.b, z3.b, z2.b, #16;
+ ext z2.b, z2.b, z1.b, #16;
+ ext z1.b, z1.b, z0.b, #16;
+ ext z0.b, z0.b, RTMP0.b, #16;
+ rev z7.b, z7.b;
+ rev z6.b, z6.b;
+ rev z5.b, z5.b;
+ rev z4.b, z4.b;
+ rev z3.b, z3.b;
+ rev z2.b, z2.b;
+ rev z1.b, z1.b;
+ rev z0.b, z0.b;
+ mov RIV.d, z8.d;
+
+ SM4_SVE_CE_CRYPT_BLK8(z0, z1, z2, z3, z4, z5, z6, z7);
+
+ eor z0.d, z0.d, z15.d;
+ eor z1.d, z1.d, z14.d;
+ eor z2.d, z2.d, z13.d;
+ eor z3.d, z3.d, z12.d;
+ eor z4.d, z4.d, z11.d;
+ eor z5.d, z5.d, z10.d;
+ eor z6.d, z6.d, z9.d;
+ eor z7.d, z7.d, z8.d;
+ st1b {z0.b}, p0, [x1];
+ st1b {z1.b}, p0, [x1, #1, MUL VL];
+ st1b {z2.b}, p0, [x1, #2, MUL VL];
+ st1b {z3.b}, p0, [x1, #3, MUL VL];
+ st1b {z4.b}, p0, [x1, #4, MUL VL];
+ st1b {z5.b}, p0, [x1, #5, MUL VL];
+ st1b {z6.b}, p0, [x1, #6, MUL VL];
+ st1b {z7.b}, p0, [x1, #7, MUL VL];
+ addvl x2, x2, #8;
+ addvl x1, x1, #8;
+
+ cbz x4, .Lcfb_end;
+ b .Lcfb_loop_blks;
+
+.Lcfb_tail8:
+ add x4, x4, x5, LSR #1;
+ cmp x4, x5, LSR #2;
+ blt .Lcfb_tail4;
+
+ sub x4, x4, x5, LSR #2; /* x4 - (4 * VL) */
+
+ ld1b {z15.b}, p0/z, [x2];
+ ld1b {z14.b}, p0/z, [x2, #1, MUL VL];
+ ld1b {z13.b}, p0/z, [x2, #2, MUL VL];
+ ld1b {z12.b}, p0/z, [x2, #3, MUL VL];
+ rev z0.b, z15.b;
+ rev z1.b, z14.b;
+ rev z2.b, z13.b;
+ rev z3.b, z12.b;
+ rev RTMP0.b, RIV.b;
+ ext z3.b, z3.b, z2.b, #16;
+ ext z2.b, z2.b, z1.b, #16;
+ ext z1.b, z1.b, z0.b, #16;
+ ext z0.b, z0.b, RTMP0.b, #16;
+ rev z3.b, z3.b;
+ rev z2.b, z2.b;
+ rev z1.b, z1.b;
+ rev z0.b, z0.b;
+ mov RIV.d, z12.d;
+
+ SM4_SVE_CE_CRYPT_BLK4(z0, z1, z2, z3);
+
+ eor z0.d, z0.d, z15.d;
+ eor z1.d, z1.d, z14.d;
+ eor z2.d, z2.d, z13.d;
+ eor z3.d, z3.d, z12.d;
+ st1b {z0.b}, p0, [x1];
+ st1b {z1.b}, p0, [x1, #1, MUL VL];
+ st1b {z2.b}, p0, [x1, #2, MUL VL];
+ st1b {z3.b}, p0, [x1, #3, MUL VL];
+ addvl x2, x2, #4;
+ addvl x1, x1, #4;
+
+ cbz x4, .Lcfb_end;
+
+.Lcfb_tail4:
+ cmp x4, x5, LSR #4;
+ blt .Lcfb_tail_ce;
+
+ sub x4, x4, x5, LSR #4; /* x4 - VL */
+
+ ld1b {z15.b}, p0/z, [x2];
+ rev RTMP0.b, RIV.b;
+ rev z0.b, z15.b;
+ ext z0.b, z0.b, RTMP0.b, #16;
+ rev z0.b, z0.b;
+ mov RIV.d, z15.d;
+
+ SM4_SVE_CE_CRYPT_BLK(z0);
+
+ eor z0.d, z0.d, z15.d;
+ st1b {z0.b}, p0, [x1];
+ addvl x2, x2, #1;
+ addvl x1, x1, #1;
+
+ cbz x4, .Lcfb_end;
+ b .Lcfb_tail4;
+
+.Lcfb_tail_ce:
+ rev RIV.s, RIV.s;
+ tbl RIV.b, {RIV.b}, RSWAP128.b;
+
+.Lcfb_tail:
+ sub x4, x4, #1;
+
+ ld1 {v15.16b}, [x2], #16;
+ mov v0.16b, RIVv.16b;
+ mov RIVv.16b, v15.16b;
+ SM4_CE_CRYPT_BLK(v0);
+ eor v0.16b, v0.16b, v15.16b;
+ st1 {v0.16b}, [x1], #16;
+
+ cbnz x4, .Lcfb_tail;
+
+ ext RIV.b, RIV.b, RIV.b, #16;
+
+.Lcfb_end:
+ /* store new IV */
+ rev RIV.s, RIV.s;
+ tbl RIV.b, {RIV.b}, RSWAP128.b;
+ st1 {RIVv.16b}, [x3];
+
+ VPOP_ABI;
+ ret_spec_stop;
+ CFI_ENDPROC();
+ELF(.size _gcry_sm4_armv9_sve_ce_cfb_dec,.-_gcry_sm4_armv9_sve_ce_cfb_dec;)
+
+.align 3
+.global _gcry_sm4_armv9_sve_ce_ctr_enc
+ELF(.type _gcry_sm4_armv9_sve_ce_ctr_enc,%function;)
+_gcry_sm4_armv9_sve_ce_ctr_enc:
+ /* input:
+ * x0: round key array, CTX
+ * x1: dst
+ * x2: src
+ * x3: ctr (big endian, 128 bit)
+ * x4: nblocks
+ */
+ CFI_STARTPROC();
+
+ PREPARE();
+
+ dup RZERO.d, #0;
+ GET_LOCAL_POINTER(x6, .Lle128_inc);
+ ld1b {RLE128_INC.b}, p0/z, [x6];
+
+ ldp x7, x8, [x3];
+ rev x7, x7;
+ rev x8, x8;
+
+#define inc_le128(zctr) \
+ mov RCTRv.d[1], x8; \
+ mov RCTRv.d[0], x7; \
+ mov zctr.d, RLE128_INC.d; \
+ dup RCTR.q, RCTR.q[0]; \
+ adds x8, x8, x5, LSR #4; \
+ adc x7, x7, xzr; \
+ adclt zctr.d, RCTR.d, RZERO.d; \
+ adclt RCTR.d, zctr.d, RZERO.d; \
+ trn1 zctr.d, RCTR.d, zctr.d; \
+ revb zctr.d, p0/m, zctr.d;
+
+.Lctr_loop_blks:
+ sub x4, x4, x5, LSR #1; /* x4 - (8 * VL) */
+ tbnz x4, #63, .Lctr_tail8;
+
+ inc_le128(z0);
+ inc_le128(z1);
+ inc_le128(z2);
+ inc_le128(z3);
+ inc_le128(z4);
+ inc_le128(z5);
+ inc_le128(z6);
+ inc_le128(z7);
+
+ SM4_SVE_CE_CRYPT_BLK8(z0, z1, z2, z3, z4, z5, z6, z7);
+
+ ld1b {RTMP0.b}, p0/z, [x2];
+ ld1b {RTMP1.b}, p0/z, [x2, #1, MUL VL];
+ ld1b {RTMP2.b}, p0/z, [x2, #2, MUL VL];
+ ld1b {RTMP3.b}, p0/z, [x2, #3, MUL VL];
+ eor z0.d, z0.d, RTMP0.d;
+ eor z1.d, z1.d, RTMP1.d;
+ eor z2.d, z2.d, RTMP2.d;
+ eor z3.d, z3.d, RTMP3.d;
+ ld1b {RTMP0.b}, p0/z, [x2, #4, MUL VL];
+ ld1b {RTMP1.b}, p0/z, [x2, #5, MUL VL];
+ ld1b {RTMP2.b}, p0/z, [x2, #6, MUL VL];
+ ld1b {RTMP3.b}, p0/z, [x2, #7, MUL VL];
+ eor z4.d, z4.d, RTMP0.d;
+ eor z5.d, z5.d, RTMP1.d;
+ eor z6.d, z6.d, RTMP2.d;
+ eor z7.d, z7.d, RTMP3.d;
+ addvl x2, x2, #8;
+
+ st1b {z0.b}, p0, [x1];
+ st1b {z1.b}, p0, [x1, #1, MUL VL];
+ st1b {z2.b}, p0, [x1, #2, MUL VL];
+ st1b {z3.b}, p0, [x1, #3, MUL VL];
+ st1b {z4.b}, p0, [x1, #4, MUL VL];
+ st1b {z5.b}, p0, [x1, #5, MUL VL];
+ st1b {z6.b}, p0, [x1, #6, MUL VL];
+ st1b {z7.b}, p0, [x1, #7, MUL VL];
+ addvl x1, x1, #8;
+
+ cbz x4, .Lctr_end;
+ b .Lctr_loop_blks;
+
+.Lctr_tail8:
+ add x4, x4, x5, LSR #1;
+ cmp x4, x5, LSR #2;
+ blt .Lctr_tail4;
+
+ sub x4, x4, x5, LSR #2; /* x4 - (4 * VL) */
+
+ inc_le128(z0);
+ inc_le128(z1);
+ inc_le128(z2);
+ inc_le128(z3);
+
+ SM4_SVE_CE_CRYPT_BLK4(z0, z1, z2, z3);
+
+ ld1b {RTMP0.b}, p0/z, [x2];
+ ld1b {RTMP1.b}, p0/z, [x2, #1, MUL VL];
+ ld1b {RTMP2.b}, p0/z, [x2, #2, MUL VL];
+ ld1b {RTMP3.b}, p0/z, [x2, #3, MUL VL];
+ eor z0.d, z0.d, RTMP0.d;
+ eor z1.d, z1.d, RTMP1.d;
+ eor z2.d, z2.d, RTMP2.d;
+ eor z3.d, z3.d, RTMP3.d;
+ st1b {z0.b}, p0, [x1];
+ st1b {z1.b}, p0, [x1, #1, MUL VL];
+ st1b {z2.b}, p0, [x1, #2, MUL VL];
+ st1b {z3.b}, p0, [x1, #3, MUL VL];
+ addvl x2, x2, #4;
+ addvl x1, x1, #4;
+
+ cbz x4, .Lctr_end;
+
+.Lctr_tail4:
+ cmp x4, x5, LSR #4;
+ blt .Lctr_tail;
+
+ sub x4, x4, x5, LSR #4; /* x4 - VL */
+
+ inc_le128(z0);
+ SM4_SVE_CE_CRYPT_BLK(z0);
+ ld1b {RTMP0.b}, p0/z, [x2];
+ eor z0.d, z0.d, RTMP0.d;
+ st1b {z0.b}, p0, [x1];
+ addvl x2, x2, #1;
+ addvl x1, x1, #1;
+
+ cbz x4, .Lctr_end;
+ b .Lctr_tail4;
+
+.Lctr_tail:
+ sub x4, x4, #1;
+
+ /* inc_le128 for CE */
+ mov v0.d[1], x8;
+ mov v0.d[0], x7;
+ adds x8, x8, #1;
+ adc x7, x7, xzr;
+ rev64 v0.16b, v0.16b;
+
+ SM4_CE_CRYPT_BLK(v0);
+ ld1 {RTMP0v.16b}, [x2], #16;
+ eor v0.16b, v0.16b, RTMP0v.16b;
+ st1 {v0.16b}, [x1], #16;
+
+ cbnz x4, .Lctr_tail;
+
+.Lctr_end:
+ /* store new CTR */
+ rev x7, x7;
+ rev x8, x8;
+ stp x7, x8, [x3];
+
+ ret_spec_stop;
+ CFI_ENDPROC();
+ELF(.size _gcry_sm4_armv9_sve_ce_ctr_enc,.-_gcry_sm4_armv9_sve_ce_ctr_enc;)
+
+.align 3
+.global _gcry_sm4_armv9_sve_get_vl
+ELF(.type _gcry_sm4_armv9_sve_get_vl,%function;)
+_gcry_sm4_armv9_sve_get_vl:
+ CFI_STARTPROC();
+
+ /* VL in bytes */
+ rdvl x0, #1;
+
+ ret_spec_stop;
+ CFI_ENDPROC();
+ELF(.size _gcry_sm4_armv9_sve_get_vl,.-_gcry_sm4_armv9_sve_get_vl;)
+
+#endif
diff --git a/cipher/sm4.c b/cipher/sm4.c
index 1c54b339db82..062a14f4c670 100644
--- a/cipher/sm4.c
+++ b/cipher/sm4.c
@@ -94,6 +94,17 @@
# endif
#endif
+#undef USE_ARM_SVE_CE
+#ifdef ENABLE_SVE_SUPPORT
+# if defined(__AARCH64EL__) && \
+ defined(HAVE_COMPATIBLE_GCC_AARCH64_PLATFORM_AS) && \
+ defined(HAVE_GCC_INLINE_ASM_AARCH64_CRYPTO) && \
+ defined(HAVE_GCC_INLINE_ASM_AARCH64_SVE) && \
+ defined(HAVE_GCC_INLINE_ASM_AARCH64_SVE2)
+# define USE_ARM_SVE_CE 1
+# endif
+#endif
+
static const char *sm4_selftest (void);
static void _gcry_sm4_ctr_enc (void *context, unsigned char *ctr,
@@ -133,6 +144,9 @@ typedef struct
#ifdef USE_ARM_CE
unsigned int use_arm_ce:1;
#endif
+#ifdef USE_ARM_SVE_CE
+ unsigned int use_arm_sve_ce:1;
+#endif
} SM4_context;
typedef unsigned int (*crypt_blk1_16_fn_t) (const void *ctx, byte *out,
@@ -448,6 +462,37 @@ sm4_armv8_ce_crypt_blk1_16(const void *rk, byte *out, const byte *in,
#endif /* USE_ARM_CE */
+#ifdef USE_ARM_SVE_CE
+extern void _gcry_sm4_armv9_sve_ce_crypt(const u32 *rk, byte *out,
+ const byte *in,
+ size_t nblocks);
+
+extern void _gcry_sm4_armv9_sve_ce_ctr_enc(const u32 *rk_enc, byte *out,
+ const byte *in,
+ byte *ctr,
+ size_t nblocks);
+
+extern void _gcry_sm4_armv9_sve_ce_cbc_dec(const u32 *rk_dec, byte *out,
+ const byte *in,
+ byte *iv,
+ size_t nblocks);
+
+extern void _gcry_sm4_armv9_sve_ce_cfb_dec(const u32 *rk_enc, byte *out,
+ const byte *in,
+ byte *iv,
+ size_t nblocks);
+
+static inline unsigned int
+sm4_armv9_sve_ce_crypt_blk1_16(const void *rk, byte *out, const byte *in,
+ unsigned int num_blks)
+{
+ _gcry_sm4_armv9_sve_ce_crypt(rk, out, in, num_blks);
+ return 0;
+}
+
+extern unsigned int _gcry_sm4_armv9_sve_get_vl(void);
+#endif /* USE_ARM_SVE_CE */
+
static inline void prefetch_sbox_table(void)
{
const volatile byte *vtab = (void *)&sbox_table;
@@ -606,6 +651,11 @@ sm4_setkey (void *context, const byte *key, const unsigned keylen,
#ifdef USE_ARM_CE
ctx->use_arm_ce = !!(hwf & HWF_ARM_SM4);
#endif
+#ifdef USE_ARM_SVE_CE
+ /* Only enabled when the SVE vector length is greater than 128 bits */
+ ctx->use_arm_sve_ce = (hwf & HWF_ARM_SVE2) && (hwf & HWF_ARM_SVESM4)
+ && _gcry_sm4_armv9_sve_get_vl() > 16;
+#endif
#ifdef USE_GFNI_AVX2
if (ctx->use_gfni_avx2)
@@ -802,6 +852,12 @@ sm4_get_crypt_blk1_16_fn(SM4_context *ctx)
return &sm4_aesni_avx_crypt_blk1_16;
}
#endif
+#ifdef USE_ARM_SVE_CE
+ else if (ctx->use_arm_sve_ce)
+ {
+ return &sm4_armv9_sve_ce_crypt_blk1_16;
+ }
+#endif
#ifdef USE_ARM_CE
else if (ctx->use_arm_ce)
{
@@ -879,6 +935,16 @@ _gcry_sm4_ctr_enc(void *context, unsigned char *ctr,
}
#endif
+#ifdef USE_ARM_SVE_CE
+ if (ctx->use_arm_sve_ce)
+ {
+ /* Process all blocks at a time. */
+ _gcry_sm4_armv9_sve_ce_ctr_enc(ctx->rkey_enc, outbuf, inbuf,
+ ctr, nblocks);
+ nblocks = 0;
+ }
+#endif
+
#ifdef USE_ARM_CE
if (ctx->use_arm_ce)
{
@@ -990,6 +1056,16 @@ _gcry_sm4_cbc_dec(void *context, unsigned char *iv,
}
#endif
+#ifdef USE_ARM_SVE_CE
+ if (ctx->use_arm_sve_ce)
+ {
+ /* Process all blocks at a time. */
+ _gcry_sm4_armv9_sve_ce_cbc_dec(ctx->rkey_dec, outbuf, inbuf,
+ iv, nblocks);
+ nblocks = 0;
+ }
+#endif
+
#ifdef USE_ARM_CE
if (ctx->use_arm_ce)
{
@@ -1101,6 +1177,16 @@ _gcry_sm4_cfb_dec(void *context, unsigned char *iv,
}
#endif
+#ifdef USE_ARM_SVE_CE
+ if (ctx->use_arm_sve_ce)
+ {
+ /* Process all blocks at a time. */
+ _gcry_sm4_armv9_sve_ce_cfb_dec(ctx->rkey_enc, outbuf, inbuf,
+ iv, nblocks);
+ nblocks = 0;
+ }
+#endif
+
#ifdef USE_ARM_CE
if (ctx->use_arm_ce)
{
diff --git a/configure.ac b/configure.ac
index 31bcd77e3c75..b55510d81eec 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2957,6 +2957,7 @@ if test "$found" = "1" ; then
# Build with the assembly implementation
GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-aarch64.lo"
GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-armv8-aarch64-ce.lo"
+ GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-armv9-aarch64-sve-ce.lo"
esac
fi
--
2.24.3 (Apple Git-128)
More information about the Gcrypt-devel
mailing list