[PATCH 3/3] Add SM4 ARMv9 SVE CE assembly implementation

[Tianjia Zhang]

Hi Jussi,

On 7/21/22 3:22 AM, Jussi Kivilinna wrote:
> Hello,
> 
> On 20.7.2022 11.25, Tianjia Zhang wrote:
>> * cipher/Makefile.am: Add 'sm4-armv9-aarch64-sve-ce.S'.
>> * cipher/sm4-armv9-aarch64-sve-ce.S: New.
>> * cipher/sm4.c (USE_ARM_SVE_CE): New.
>> (SM4_context) [USE_ARM_SVE_CE]: Add 'use_arm_sve_ce'.
>> (_gcry_sm4_armv9_sve_ce_crypt, _gcry_sm4_armv9_sve_ce_ctr_enc)
>> (_gcry_sm4_armv9_sve_ce_cbc_dec, _gcry_sm4_armv9_sve_ce_cfb_dec)
>> (sm4_armv9_sve_ce_crypt_blk1_16): New.
>> (sm4_setkey): Enable ARMv9 SVE CE if supported by HW.
>> (sm4_get_crypt_blk1_16_fn) [USE_ARM_SVE_CE]: Add ARMv9 SVE CE
>> bulk functions.
>> (_gcry_sm4_ctr_enc, _gcry_sm4_cbc_dec, _gcry_sm4_cfb_dec)
>> [USE_ARM_SVE_CE]: Add ARMv9 SVE CE bulk functions.
>> * configure.ac: Add 'sm4-armv9-aarch64-sve-ce.lo'.
>> -- 
>>
>> Signed-off-by: Tianjia Zhang <tianjia.zhang at linux.alibaba.com>
>> ---
>>   cipher/Makefile.am                |   1 +
>>   cipher/sm4-armv9-aarch64-sve-ce.S | 966 ++++++++++++++++++++++++++++++
>>   cipher/sm4.c                      |  85 +++
>>   configure.ac                      |   1 +
>>   4 files changed, 1053 insertions(+)
>>   create mode 100644 cipher/sm4-armv9-aarch64-sve-ce.S
>>
>> diff --git a/cipher/Makefile.am b/cipher/Makefile.am
>> index 042dc0a7170d..97823cb48bd3 100644
>> --- a/cipher/Makefile.am
>> +++ b/cipher/Makefile.am
>> @@ -120,6 +120,7 @@ EXTRA_libcipher_la_SOURCES = \
>>       serpent.c serpent-sse2-amd64.S \
>>       sm4.c sm4-aesni-avx-amd64.S sm4-aesni-avx2-amd64.S sm4-aarch64.S \
>>       sm4-armv8-aarch64-ce.S sm4-gfni-avx2-amd64.S \
>> +    sm4-armv9-aarch64-sve-ce.S \
>>       serpent-avx2-amd64.S serpent-armv7-neon.S \
>>       sha1.c sha1-ssse3-amd64.S sha1-avx-amd64.S sha1-avx-bmi2-amd64.S \
>>       sha1-avx2-bmi2-amd64.S sha1-armv7-neon.S sha1-armv8-aarch32-ce.S \
>> diff --git a/cipher/sm4-armv9-aarch64-sve-ce.S 
>> b/cipher/sm4-armv9-aarch64-sve-ce.S
>> new file mode 100644
>> index 000000000000..2f4cfcc9ecab
>> --- /dev/null
>> +++ b/cipher/sm4-armv9-aarch64-sve-ce.S
>> @@ -0,0 +1,966 @@
>> +/* sm4-armv9-aarch64-sve-ce.S - ARMv9/AArch64 SVE Cryptography 
>> accelerated SM4
>> + *
>> + * Copyright (C) 2022 Alibaba Group.
>> + * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang at linux.alibaba.com>
>> + *
>> + * This file is part of Libgcrypt.
>> + *
>> + * Libgcrypt is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU Lesser General Public License as
>> + * published by the Free Software Foundation; either version 2.1 of
>> + * the License, or (at your option) any later version.
>> + *
>> + * Libgcrypt is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU Lesser General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU Lesser General Public
>> + * License along with this program; if not, see 
>> <http://www.gnu.org/licenses/>.
>> + */
>> +
>> +#include "asm-common-aarch64.h"
>> +
>> +#if defined(__AARCH64EL__) && \
>> +    defined(HAVE_COMPATIBLE_GCC_AARCH64_PLATFORM_AS) && \
>> +    defined(HAVE_GCC_INLINE_ASM_AARCH64_CRYPTO) && \
>> +    defined(HAVE_GCC_INLINE_ASM_AARCH64_SVE) && \
>> +    defined(USE_SM4)
>> +
>> +.cpu generic+simd+crypto+sve+sve2
> 
> Should we add 
> "gcry_cv_gcc_inline_asm_aarch64_sve2"/"HAVE_GCC_INLINE_ASM_AARCH64_SVE2" 
> to configure.ac as SVE2 instructions are being used in assembly below?
> 

Will be added in the next patch.

>> +
>> +/* Constants */
>> +
> <snip>> +#define inc_le128(zctr)                             \
>> +        mov         RCTRv.d[1], x8;                 \
>> +        mov         RCTRv.d[0], x7;                 \
>> +        mov         zctr.d, RLE128_INC.d;           \
>> +        dup         RCTR.q, RCTR.q[0];              \
>> +        adds        x8, x8, x5, LSR #4;             \
>> +        adc         x7, x7, xzr;                    \
>> +        adclt       zctr.d, RCTR.d, RZERO.d;        \
>> +        adclt       RCTR.d, zctr.d, RZERO.d;        \
>> +        trn1        zctr.d, RCTR.d, zctr.d;         \
>> +        revb        zctr.d, p0/m, zctr.d;
> 
> 'adclt' here is SVE2.
> 
>> diff --git a/cipher/sm4.c b/cipher/sm4.c
>> index 1c54b339db82..bd56be0ebd7a 100644
>> --- a/cipher/sm4.c
>> +++ b/cipher/sm4.c
>> @@ -94,6 +94,16 @@
>>   # endif
>>   #endif
>> +#undef USE_ARM_SVE_CE
>> +#ifdef ENABLE_SVE_SUPPORT
>> +# if defined(__AARCH64EL__) && \
>> +     defined(HAVE_COMPATIBLE_GCC_AARCH64_PLATFORM_AS) && \
>> +     defined(HAVE_GCC_INLINE_ASM_AARCH64_CRYPTO) && \
>> +     defined(HAVE_GCC_INLINE_ASM_AARCH64_SVE)
>> +#   define USE_ARM_SVE_CE 1
>> +# endif
>> +#endif
> <snip>
>> @@ -606,6 +650,11 @@ sm4_setkey (void *context, const byte *key, const 
>> unsigned keylen,
>>   #ifdef USE_ARM_CE
>>     ctx->use_arm_ce = !!(hwf & HWF_ARM_SM4);
>>   #endif
>> +#ifdef USE_ARM_SVE_CE
>> +  /* Only enabled when the SVE vector length is greater than 128 bits */
>> +  ctx->use_arm_sve_ce = (hwf & HWF_ARM_SVESM4)
>> +        && _gcry_sm4_armv9_sve_get_vl() > 16;
>> +#endif
> 
> Should this also check for HWF_ARM_SVE2 or does HWF_ARM_SVESM4 imply 
> that SVE2 is also available?
> 
> 
> Otherwise patches look good. I tested them with QEMU which I found out 
> had SVE2 and SVE-SM4 support available.
> 
> -Jussi

Thanks for your suggestion and test.

HWF_ARM_SVESM4 already implies the existence of HWF_ARM_SVE2, but it is
still a good practice to check SVE2 explicitly, I will add this check.

Best regards,
Tianjia