[PATCH 3/3] Add SM4 ARMv9 SVE CE assembly implementation

[Jussi Kivilinna]

Hello,

On 20.7.2022 11.25, Tianjia Zhang wrote:
> * cipher/Makefile.am: Add 'sm4-armv9-aarch64-sve-ce.S'.
> * cipher/sm4-armv9-aarch64-sve-ce.S: New.
> * cipher/sm4.c (USE_ARM_SVE_CE): New.
> (SM4_context) [USE_ARM_SVE_CE]: Add 'use_arm_sve_ce'.
> (_gcry_sm4_armv9_sve_ce_crypt, _gcry_sm4_armv9_sve_ce_ctr_enc)
> (_gcry_sm4_armv9_sve_ce_cbc_dec, _gcry_sm4_armv9_sve_ce_cfb_dec)
> (sm4_armv9_sve_ce_crypt_blk1_16): New.
> (sm4_setkey): Enable ARMv9 SVE CE if supported by HW.
> (sm4_get_crypt_blk1_16_fn) [USE_ARM_SVE_CE]: Add ARMv9 SVE CE
> bulk functions.
> (_gcry_sm4_ctr_enc, _gcry_sm4_cbc_dec, _gcry_sm4_cfb_dec)
> [USE_ARM_SVE_CE]: Add ARMv9 SVE CE bulk functions.
> * configure.ac: Add 'sm4-armv9-aarch64-sve-ce.lo'.
> --
> 
> Signed-off-by: Tianjia Zhang <tianjia.zhang at linux.alibaba.com>
> ---
>   cipher/Makefile.am                |   1 +
>   cipher/sm4-armv9-aarch64-sve-ce.S | 966 ++++++++++++++++++++++++++++++
>   cipher/sm4.c                      |  85 +++
>   configure.ac                      |   1 +
>   4 files changed, 1053 insertions(+)
>   create mode 100644 cipher/sm4-armv9-aarch64-sve-ce.S
> 
> diff --git a/cipher/Makefile.am b/cipher/Makefile.am
> index 042dc0a7170d..97823cb48bd3 100644
> --- a/cipher/Makefile.am
> +++ b/cipher/Makefile.am
> @@ -120,6 +120,7 @@ EXTRA_libcipher_la_SOURCES = \
>   	serpent.c serpent-sse2-amd64.S \
>   	sm4.c sm4-aesni-avx-amd64.S sm4-aesni-avx2-amd64.S sm4-aarch64.S \
>   	sm4-armv8-aarch64-ce.S sm4-gfni-avx2-amd64.S \
> +	sm4-armv9-aarch64-sve-ce.S \
>   	serpent-avx2-amd64.S serpent-armv7-neon.S \
>   	sha1.c sha1-ssse3-amd64.S sha1-avx-amd64.S sha1-avx-bmi2-amd64.S \
>   	sha1-avx2-bmi2-amd64.S sha1-armv7-neon.S sha1-armv8-aarch32-ce.S \
> diff --git a/cipher/sm4-armv9-aarch64-sve-ce.S b/cipher/sm4-armv9-aarch64-sve-ce.S
> new file mode 100644
> index 000000000000..2f4cfcc9ecab
> --- /dev/null
> +++ b/cipher/sm4-armv9-aarch64-sve-ce.S
> @@ -0,0 +1,966 @@
> +/* sm4-armv9-aarch64-sve-ce.S - ARMv9/AArch64 SVE Cryptography accelerated SM4
> + *
> + * Copyright (C) 2022 Alibaba Group.
> + * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang at linux.alibaba.com>
> + *
> + * This file is part of Libgcrypt.
> + *
> + * Libgcrypt is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU Lesser General Public License as
> + * published by the Free Software Foundation; either version 2.1 of
> + * the License, or (at your option) any later version.
> + *
> + * Libgcrypt is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this program; if not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include "asm-common-aarch64.h"
> +
> +#if defined(__AARCH64EL__) && \
> +    defined(HAVE_COMPATIBLE_GCC_AARCH64_PLATFORM_AS) && \
> +    defined(HAVE_GCC_INLINE_ASM_AARCH64_CRYPTO) && \
> +    defined(HAVE_GCC_INLINE_ASM_AARCH64_SVE) && \
> +    defined(USE_SM4)
> +
> +.cpu generic+simd+crypto+sve+sve2

Should we add "gcry_cv_gcc_inline_asm_aarch64_sve2"/"HAVE_GCC_INLINE_ASM_AARCH64_SVE2" to configure.ac as SVE2 instructions are being used in assembly below?

> +
> +/* Constants */
> +
<snip>> +#define inc_le128(zctr)                             \
> +        mov         RCTRv.d[1], x8;                 \
> +        mov         RCTRv.d[0], x7;                 \
> +        mov         zctr.d, RLE128_INC.d;           \
> +        dup         RCTR.q, RCTR.q[0];              \
> +        adds        x8, x8, x5, LSR #4;             \
> +        adc         x7, x7, xzr;                    \
> +        adclt       zctr.d, RCTR.d, RZERO.d;        \
> +        adclt       RCTR.d, zctr.d, RZERO.d;        \
> +        trn1        zctr.d, RCTR.d, zctr.d;         \
> +        revb        zctr.d, p0/m, zctr.d;

'adclt' here is SVE2.

> diff --git a/cipher/sm4.c b/cipher/sm4.c
> index 1c54b339db82..bd56be0ebd7a 100644
> --- a/cipher/sm4.c
> +++ b/cipher/sm4.c
> @@ -94,6 +94,16 @@
>   # endif
>   #endif
>   
> +#undef USE_ARM_SVE_CE
> +#ifdef ENABLE_SVE_SUPPORT
> +# if defined(__AARCH64EL__) && \
> +     defined(HAVE_COMPATIBLE_GCC_AARCH64_PLATFORM_AS) && \
> +     defined(HAVE_GCC_INLINE_ASM_AARCH64_CRYPTO) && \
> +     defined(HAVE_GCC_INLINE_ASM_AARCH64_SVE)
> +#   define USE_ARM_SVE_CE 1
> +# endif
> +#endif
<snip>
> @@ -606,6 +650,11 @@ sm4_setkey (void *context, const byte *key, const unsigned keylen,
>   #ifdef USE_ARM_CE
>     ctx->use_arm_ce = !!(hwf & HWF_ARM_SM4);
>   #endif
> +#ifdef USE_ARM_SVE_CE
> +  /* Only enabled when the SVE vector length is greater than 128 bits */
> +  ctx->use_arm_sve_ce = (hwf & HWF_ARM_SVESM4)
> +		&& _gcry_sm4_armv9_sve_get_vl() > 16;
> +#endif

Should this also check for HWF_ARM_SVE2 or does HWF_ARM_SVESM4 imply that SVE2 is also available?


Otherwise patches look good. I tested them with QEMU which I found out had SVE2 and SVE-SM4 support available.

-Jussi