From jussi.kivilinna at iki.fi Sat Jul 4 07:37:32 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Sat, 4 Jul 2026 08:37:32 +0300 Subject: [PATCH 1/2] sm4: tail-call ARM and PPC crypt_blk1_x helpers Message-ID: <20260704053733.3618370-1-jussi.kivilinna@iki.fi> * cipher/sm4-aarch64.S (sm4_aarch64_crypt_blk1_4) (_gcry_sm4_aarch64_crypt_blk1_8): Return zero stack burn size. * cipher/sm4-armv8-aarch64-ce.S (sm4_armv8_ce_crypt_blk1_4) (_gcry_sm4_armv8_ce_crypt_blk1_8): Likewise. * cipher/sm4-armv9-aarch64-sve-ce.S (_gcry_sm4_armv9_sve_ce_crypt): Likewise. * cipher/sm4-ppc.c (sm4_ppc_crypt_blk1_16, _gcry_sm4_ppc8le_crypt_blk1_16) (_gcry_sm4_ppc9le_crypt_blk1_16): Likewise. * cipher/sm4.c (_gcry_sm4_aarch64_crypt_blk1_8) (_gcry_sm4_armv8_ce_crypt_blk1_8, _gcry_sm4_armv9_sve_ce_crypt) (_gcry_sm4_ppc8le_crypt_blk1_16, _gcry_sm4_ppc9le_crypt_blk1_16): Likewise. (sm4_aarch64_crypt_blk1_16, sm4_armv8_ce_crypt_blk1_16) (sm4_armv9_sve_ce_crypt_blk1_16, sm4_ppc8le_crypt_blk1_16) (sm4_ppc9le_crypt_blk1_16): Tail-call the underlying blk1_x helper. -- These wrappers called the helper and then returned a constant zero, which prevents the compiler from emitting a tail call. Signed-off-by: Jussi Kivilinna --- cipher/sm4-aarch64.S | 2 ++ cipher/sm4-armv8-aarch64-ce.S | 2 ++ cipher/sm4-armv9-aarch64-sve-ce.S | 1 + cipher/sm4-ppc.c | 14 +++++----- cipher/sm4.c | 43 ++++++++++++++----------------- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/cipher/sm4-aarch64.S b/cipher/sm4-aarch64.S index bab4b4df..c71bf3ca 100644 --- a/cipher/sm4-aarch64.S +++ b/cipher/sm4-aarch64.S @@ -222,6 +222,7 @@ sm4_aarch64_crypt_blk1_4: .Lblk4_store_output_done: VPOP_ABI; + mov x0, #0; ret_spec_stop; CFI_ENDPROC(); ELF(.size sm4_aarch64_crypt_blk1_4,.-sm4_aarch64_crypt_blk1_4;) @@ -396,6 +397,7 @@ _gcry_sm4_aarch64_crypt_blk1_8: CFI_ADJUST_CFA_OFFSET(-16); CFI_RESTORE(x29); CFI_RESTORE(x30); + mov x0, #0; ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_sm4_aarch64_crypt_blk1_8,.-_gcry_sm4_aarch64_crypt_blk1_8;) diff --git a/cipher/sm4-armv8-aarch64-ce.S b/cipher/sm4-armv8-aarch64-ce.S index 01f3df92..5917ad5b 100644 --- a/cipher/sm4-armv8-aarch64-ce.S +++ b/cipher/sm4-armv8-aarch64-ce.S @@ -335,6 +335,7 @@ sm4_armv8_ce_crypt_blk1_4: st1 {v3.16b}, [x1]; .Lblk4_store_output_done: + mov x0, #0; ret_spec_stop; CFI_ENDPROC(); ELF(.size sm4_armv8_ce_crypt_blk1_4,.-sm4_armv8_ce_crypt_blk1_4;) @@ -385,6 +386,7 @@ _gcry_sm4_armv8_ce_crypt_blk1_8: .Lblk8_store_output_done: CLEAR_ALL_REGS(); + mov x0, #0; ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_sm4_armv8_ce_crypt_blk1_8,.-_gcry_sm4_armv8_ce_crypt_blk1_8;) diff --git a/cipher/sm4-armv9-aarch64-sve-ce.S b/cipher/sm4-armv9-aarch64-sve-ce.S index 7367cd28..ba9bde84 100644 --- a/cipher/sm4-armv9-aarch64-sve-ce.S +++ b/cipher/sm4-armv9-aarch64-sve-ce.S @@ -444,6 +444,7 @@ _gcry_sm4_armv9_sve_ce_crypt: cbnz x3, .Lcrypt_tail; .Lcrypt_end: + mov x0, #0; ret_spec_stop; CFI_ENDPROC(); ELF(.size _gcry_sm4_armv9_sve_ce_crypt,.-_gcry_sm4_armv9_sve_ce_crypt;) diff --git a/cipher/sm4-ppc.c b/cipher/sm4-ppc.c index 2b26c39d..9d9227c0 100644 --- a/cipher/sm4-ppc.c +++ b/cipher/sm4-ppc.c @@ -293,7 +293,7 @@ sm4_ppc_crypt_blk1_4(u32 *rk, byte *out, const byte *in, size_t nblks) vec_xst((vector16x_u8)vec_revb(ra0), 0, out + 3 * 16); } -static ASM_FUNC_ATTR_INLINE void +static ASM_FUNC_ATTR_INLINE unsigned int sm4_ppc_crypt_blk1_16(u32 *rk, byte *out, const byte *in, size_t nblks) { if (nblks >= 16) @@ -321,25 +321,27 @@ sm4_ppc_crypt_blk1_16(u32 *rk, byte *out, const byte *in, size_t nblks) } clear_vec_regs(); + + return 0; } -ASM_FUNC_ATTR_NOINLINE FUNC_ATTR_TARGET_P8 void +ASM_FUNC_ATTR_NOINLINE FUNC_ATTR_TARGET_P8 unsigned int _gcry_sm4_ppc8le_crypt_blk1_16(u32 *rk, byte *out, const byte *in, size_t nblks) { - sm4_ppc_crypt_blk1_16(rk, out, in, nblks); + return sm4_ppc_crypt_blk1_16(rk, out, in, nblks); } -ASM_FUNC_ATTR_NOINLINE FUNC_ATTR_TARGET_P9 void +ASM_FUNC_ATTR_NOINLINE FUNC_ATTR_TARGET_P9 unsigned int _gcry_sm4_ppc9le_crypt_blk1_16(u32 *rk, byte *out, const byte *in, size_t nblks) { #ifdef HAVE_FUNC_ATTR_TARGET /* Inline for POWER9 target optimization. */ - sm4_ppc_crypt_blk1_16(rk, out, in, nblks); + return sm4_ppc_crypt_blk1_16(rk, out, in, nblks); #else /* Target selecting not working, just call the other noinline function. */ - _gcry_sm4_ppc8le_crypt_blk1_16(rk, out, in, nblks); + return _gcry_sm4_ppc8le_crypt_blk1_16(rk, out, in, nblks); #endif } diff --git a/cipher/sm4.c b/cipher/sm4.c index 165a66b2..cc98b676 100644 --- a/cipher/sm4.c +++ b/cipher/sm4.c @@ -635,9 +635,9 @@ extern void _gcry_sm4_aarch64_cfb_dec(const u32 *rk_enc, byte *out, byte *iv, size_t nblocks); -extern void _gcry_sm4_aarch64_crypt_blk1_8(u32 *rk, byte *out, - const byte *in, - size_t num_blocks); +extern unsigned int _gcry_sm4_aarch64_crypt_blk1_8(u32 *rk, byte *out, + const byte *in, + size_t num_blocks); static inline unsigned int sm4_aarch64_crypt_blk1_16(void *rk, byte *out, const byte *in, @@ -651,8 +651,7 @@ sm4_aarch64_crypt_blk1_16(void *rk, byte *out, const byte *in, num_blks -= 8; } - _gcry_sm4_aarch64_crypt_blk1_8(rk, out, in, num_blks); - return 0; + return _gcry_sm4_aarch64_crypt_blk1_8(rk, out, in, num_blks); } #endif /* USE_AARCH64_SIMD */ @@ -686,9 +685,9 @@ extern void _gcry_sm4_armv8_ce_xts_crypt(const u32 *rk, byte *out, byte *tweak, size_t nblocks); -extern void _gcry_sm4_armv8_ce_crypt_blk1_8(u32 *rk, byte *out, - const byte *in, - size_t num_blocks); +extern unsigned int _gcry_sm4_armv8_ce_crypt_blk1_8(u32 *rk, byte *out, + const byte *in, + size_t num_blocks); static inline unsigned int sm4_armv8_ce_crypt_blk1_16(void *rk, byte *out, const byte *in, @@ -702,16 +701,15 @@ sm4_armv8_ce_crypt_blk1_16(void *rk, byte *out, const byte *in, num_blks -= 8; } - _gcry_sm4_armv8_ce_crypt_blk1_8(rk, out, in, num_blks); - return 0; + return _gcry_sm4_armv8_ce_crypt_blk1_8(rk, out, in, num_blks); } #endif /* USE_ARM_CE */ #ifdef USE_ARM_SVE_CE -extern void _gcry_sm4_armv9_sve_ce_crypt(u32 *rk, byte *out, - const byte *in, - size_t nblocks); +extern unsigned int _gcry_sm4_armv9_sve_ce_crypt(u32 *rk, byte *out, + const byte *in, + size_t nblocks); extern void _gcry_sm4_armv9_sve_ce_ctr_enc(const u32 *rk_enc, byte *out, const byte *in, @@ -732,32 +730,31 @@ static inline unsigned int sm4_armv9_sve_ce_crypt_blk1_16(void *rk, byte *out, const byte *in, size_t num_blks) { - _gcry_sm4_armv9_sve_ce_crypt(rk, out, in, num_blks); - return 0; + return _gcry_sm4_armv9_sve_ce_crypt(rk, out, in, num_blks); } extern unsigned int _gcry_sm4_armv9_sve_get_vl(void); #endif /* USE_ARM_SVE_CE */ #ifdef USE_PPC_CRYPTO -extern void _gcry_sm4_ppc8le_crypt_blk1_16(u32 *rk, byte *out, const byte *in, - size_t num_blks); +extern unsigned int _gcry_sm4_ppc8le_crypt_blk1_16(u32 *rk, byte *out, + const byte *in, + size_t num_blks); -extern void _gcry_sm4_ppc9le_crypt_blk1_16(u32 *rk, byte *out, const byte *in, - size_t num_blks); +extern unsigned int _gcry_sm4_ppc9le_crypt_blk1_16(u32 *rk, byte *out, + const byte *in, + size_t num_blks); static inline unsigned int sm4_ppc8le_crypt_blk1_16(void *rk, byte *out, const byte *in, size_t num_blks) { - _gcry_sm4_ppc8le_crypt_blk1_16(rk, out, in, num_blks); - return 0; + return _gcry_sm4_ppc8le_crypt_blk1_16(rk, out, in, num_blks); } static inline unsigned int sm4_ppc9le_crypt_blk1_16(void *rk, byte *out, const byte *in, size_t num_blks) { - _gcry_sm4_ppc9le_crypt_blk1_16(rk, out, in, num_blks); - return 0; + return _gcry_sm4_ppc9le_crypt_blk1_16(rk, out, in, num_blks); } #endif /* USE_PPC_CRYPTO */ -- 2.53.0 From jussi.kivilinna at iki.fi Sat Jul 4 07:37:33 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Sat, 4 Jul 2026 08:37:33 +0300 Subject: [PATCH 2/2] cipher: limit table prefetch to the look-up table arrays In-Reply-To: <20260704053733.3618370-1-jussi.kivilinna@iki.fi> References: <20260704053733.3618370-1-jussi.kivilinna@iki.fi> Message-ID: <20260704053733.3618370-2-jussi.kivilinna@iki.fi> * cipher/aria.c (prefetch_sboxes): Compute unshare counter once and prefetch only look-up arrays. * cipher/cipher-gcm.c (do_prefetch_tables): Likewise. * cipher/rijndael.c (prefetch_enc, prefetch_dec): Likewise. * cipher/sm4.c (prefetch_sbox_table): Likewise. * cipher/rijndael-tables.h (dec_tables): Rename 'inv_sbox' field to 'inv_sboxT'. -- Signed-off-by: Jussi Kivilinna --- cipher/aria.c | 7 ++++--- cipher/cipher-gcm.c | 7 ++++--- cipher/rijndael-tables.h | 4 ++-- cipher/rijndael.c | 15 +++++++++------ cipher/sm4.c | 7 ++++--- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/cipher/aria.c b/cipher/aria.c index 26546a63..bb67ed03 100644 --- a/cipher/aria.c +++ b/cipher/aria.c @@ -627,11 +627,12 @@ prefetch_sboxes(void) * of look-up table are shared between processes. Modifying counters also * causes checksums for pages to change and hint same-page merging algorithm * that these pages are frequently changing. */ - sboxes.counter_head++; - sboxes.counter_tail++; + u32 counter = sboxes.counter_head + 1; + sboxes.counter_head = counter; + sboxes.counter_tail = counter; /* Prefetch look-up tables to cache. */ - prefetch_table((const void *)&sboxes, sizeof(sboxes)); + prefetch_table((const void *)&sboxes.s1[0], sizeof(sboxes.s1) * 4); } diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c index 2db371b8..1627cd1c 100644 --- a/cipher/cipher-gcm.c +++ b/cipher/cipher-gcm.c @@ -255,12 +255,13 @@ do_prefetch_tables (const void *gcmM, size_t gcmM_size) * of look-up table are shared between processes. Modifying counters also * causes checksums for pages to change and hint same-page merging algorithm * that these pages are frequently changing. */ - gcm_table.counter_head++; - gcm_table.counter_tail++; + u32 counter = gcm_table.counter_head + 1; + gcm_table.counter_head = counter; + gcm_table.counter_tail = counter; /* Prefetch look-up tables to cache. */ prefetch_table(gcmM, gcmM_size); - prefetch_table(&gcm_table, sizeof(gcm_table)); + prefetch_table(&gcm_table.R, sizeof(gcm_table.R)); } #ifdef GCM_TABLES_USE_U64 diff --git a/cipher/rijndael-tables.h b/cipher/rijndael-tables.h index e46ce08c..52b9518f 100644 --- a/cipher/rijndael-tables.h +++ b/cipher/rijndael-tables.h @@ -107,7 +107,7 @@ static struct volatile u32 counter_head; u32 cacheline_align[64 / 4 - 1]; u32 T[256]; - byte inv_sbox[256]; + byte inv_sboxT[256]; volatile u32 counter_tail; } dec_tables ATTR_ALIGNED_64 = { @@ -217,4 +217,4 @@ static struct }; #define decT dec_tables.T -#define inv_sbox dec_tables.inv_sbox +#define inv_sbox dec_tables.inv_sboxT diff --git a/cipher/rijndael.c b/cipher/rijndael.c index f3daf35a..645c0e2f 100644 --- a/cipher/rijndael.c +++ b/cipher/rijndael.c @@ -597,11 +597,12 @@ static void prefetch_enc(void) * of look-up table are shared between processes. Modifying counters also * causes checksums for pages to change and hint same-page merging algorithm * that these pages are frequently changing. */ - enc_tables.counter_head++; - enc_tables.counter_tail++; + u32 counter = enc_tables.counter_head + 1; + enc_tables.counter_head = counter; + enc_tables.counter_tail = counter; /* Prefetch look-up tables to cache. */ - prefetch_table((const void *)&enc_tables, sizeof(enc_tables)); + prefetch_table((const void *)&enc_tables.T[0], sizeof(enc_tables.T)); } static void prefetch_dec(void) @@ -610,11 +611,13 @@ static void prefetch_dec(void) * of look-up table are shared between processes. Modifying counters also * causes checksums for pages to change and hint same-page merging algorithm * that these pages are frequently changing. */ - dec_tables.counter_head++; - dec_tables.counter_tail++; + u32 counter = dec_tables.counter_head + 1; + dec_tables.counter_head = counter; + dec_tables.counter_tail = counter; /* Prefetch look-up tables to cache. */ - prefetch_table((const void *)&dec_tables, sizeof(dec_tables)); + prefetch_table((const void *)&dec_tables.T[0], + sizeof(dec_tables.T) + sizeof(dec_tables.inv_sboxT)); } diff --git a/cipher/sm4.c b/cipher/sm4.c index cc98b676..70b4901c 100644 --- a/cipher/sm4.c +++ b/cipher/sm4.c @@ -760,14 +760,15 @@ sm4_ppc9le_crypt_blk1_16(void *rk, byte *out, const byte *in, size_t num_blks) static inline void prefetch_sbox_table(void) { - const volatile byte *vtab = (void *)&sbox_table; + const volatile byte *vtab = (void *)&sbox_table.S[0]; /* Modify counters to trigger copy-on-write and unsharing if physical pages * of look-up table are shared between processes. Modifying counters also * causes checksums for pages to change and hint same-page merging algorithm * that these pages are frequently changing. */ - sbox_table.counter_head++; - sbox_table.counter_tail++; + u32 counter = sbox_table.counter_head + 1; + sbox_table.counter_head = counter; + sbox_table.counter_tail = counter; /* Prefetch look-up table to cache. */ (void)vtab[0 * 32]; -- 2.53.0 From davegalos at google.com Fri Jul 10 01:11:18 2026 From: davegalos at google.com (David Galos) Date: Thu, 9 Jul 2026 19:11:18 -0400 Subject: [PATCH 2/2] Fix intermittent failure in rsa_decrypt when p>q. In-Reply-To: <20260709231752.1332171-1-davegalos@google.com> References: <20260709231752.1332171-1-davegalos@google.com> Message-ID: <20260709231752.1332171-3-davegalos@google.com> --- cipher/rsa.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cipher/rsa.c b/cipher/rsa.c index 58058d6a..ff679d95 100644 --- a/cipher/rsa.c +++ b/cipher/rsa.c @@ -1141,9 +1141,12 @@ secret_core_crt (gcry_mpi_t M, gcry_mpi_t C, mpi_free ( D_blind ); /* h = u * ( m2 - m1 ) mod q */ - mpi_sub ( h, m2, m1 ); - if ( mpi_has_sign ( h ) ) - mpi_add ( h, h, Q ); + /* NB: prior code used non-modular subtraction and a sign check, + * adding Q to the result when negative, which does not work if + * P > Q + 1. That is not technically a bug because RFC 4880 requires + * that P < Q, but there are non-compliant keys out there, and is + * a more direct rendition of the equation anyway. */ + mpi_subm ( h, m2, m1, Q ); mpi_mulm ( h, U, h, Q ); /* m = m1 + h * p */ -- 2.55.0.795.g602f6c329a-goog From davegalos at google.com Fri Jul 10 01:11:16 2026 From: davegalos at google.com (David Galos) Date: Thu, 9 Jul 2026 19:11:16 -0400 Subject: [PATCH 0/2] Changes to rsa_decrypt behavior on bad keys Message-ID: <20260709231752.1332171-1-davegalos@google.com> Apologies for the repost if you're subscribed to gnupg-devel as well as this list. When reading the HACKING file I didn't realize that I could've sent right to this list, so my bad. There are two following patches, and I think at least one of them is pretty important because the failure rate of a gcrypt-based crypto oracle could probably leak info about factor-proximity based on failure rate. Below is the same text I sent to gnupg-devel (which was the wrong list) but I think it still provides good explanation and context: What follows are two patches based on some surprising behavior I experienced dealing with some OpenPGP keys generated outside of GnuPG. I'll start with the second patch first, a key with p > q but a correctly calculated u will work for decryption about 80% of the time, but will overflow some percentage of the time, yielding an incorrect result, and a confusing error message from gpg -d. There are two better choices in this case: 1. error out. 2. always work. The patch attached choses the second option, because it would quietly make some things work, rather than breaking them. There is an argument for the first: broken things should break, and while from an RFC4880 point of view, the key is bad, RSA itself doesn't specify which key needs to be bigger. In the first patch I just validate the relationship between p, q, and u, because I have no idea how the code would behave if that relationship doesn't hold, but ERR_BAD_KEY seems like the right move. If Option 1 from above is chosen above, I will expand this patch to ensure p < q. David Galos (2): Validate CRT factors in rsa_decrypt Fix intermittent failure in rsa_decrypt when p>q. cipher/rsa.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) -- 2.55.0.795.g602f6c329a-goog From davegalos at google.com Fri Jul 10 01:11:17 2026 From: davegalos at google.com (David Galos) Date: Thu, 9 Jul 2026 19:11:17 -0400 Subject: [PATCH 1/2] Validate CRT factors in rsa_decrypt In-Reply-To: <20260709231752.1332171-1-davegalos@google.com> References: <20260709231752.1332171-1-davegalos@google.com> Message-ID: <20260709231752.1332171-2-davegalos@google.com> --- cipher/rsa.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cipher/rsa.c b/cipher/rsa.c index 44a44bdc..58058d6a 100644 --- a/cipher/rsa.c +++ b/cipher/rsa.c @@ -30,6 +30,7 @@ #include #include "g10lib.h" +#include "gcrypt.h" #include "mpi.h" #include "cipher.h" #include "pubkey-internal.h" @@ -1449,6 +1450,18 @@ rsa_encrypt (gcry_sexp_t *r_ciph, gcry_sexp_t s_data, gcry_sexp_t keyparms) } +static gcry_err_code_t +crt_validate_factors (RSA_secret_key *sk) +{ + int cmp; + gcry_mpi_t result = mpi_alloc_secure (0); + + mpi_mulm (result, sk->u, sk->p, sk->q); + cmp = mpi_cmp (result, GCRYMPI_CONST_ONE); + mpi_free (result); + return cmp == 0 ? 0 : GPG_ERR_BAD_SECKEY; +} + static gcry_err_code_t rsa_decrypt (gcry_sexp_t *r_plain, gcry_sexp_t s_data, gcry_sexp_t keyparms) @@ -1505,6 +1518,10 @@ rsa_decrypt (gcry_sexp_t *r_plain, gcry_sexp_t s_data, gcry_sexp_t keyparms) } } + rc = crt_validate_factors (&sk); + if (rc) + goto leave; + /* Better make sure that there are no superfluous leading zeroes in the input and it has not been "padded" using multiples of N. This mitigates side-channel attacks (CVE-2013-4576). */ -- 2.55.0.795.g602f6c329a-goog From dtsen at us.ibm.com Tue Jul 14 13:41:47 2026 From: dtsen at us.ibm.com (Danny Tsen) Date: Tue, 14 Jul 2026 11:41:47 +0000 Subject: [PATCH 0/5] dilithium-kyber: Optimized (i)NTT support for In-Reply-To: References: <20260224002753.151873-1-dtsen@us.ibm.com> <87tsu1bxn0.fsf@haruna.fsij.org> Message-ID: Hi NIIBE, It's been a while. Do you have any progress on the new setup? BTW, ZETA layout may be implementation or platform specific due to optimization approach. Thanks. -Danny ________________________________ From: Danny Tsen Sent: Monday, March 30, 2026 7:28 AM To: NIIBE Yutaka ; gcrypt-devel at gnupg.org Subject: Re: [EXTERNAL] Re: [PATCH 0/5] dilithium-kyber: Optimized (i)NTT support for Hi, That's seems like a good idea. Let me know when you have the framework available. I can follow that. Thanks. -Danny ________________________________ From: NIIBE Yutaka Sent: Friday, March 27, 2026 1:21 AM To: Danny Tsen ; gcrypt-devel at gnupg.org Subject: [EXTERNAL] Re: [PATCH 0/5] dilithium-kyber: Optimized (i)NTT support for Hello, Sorry for late reply. Danny Tsen wrote: > Added optimized (i)NTT algorithm support for ppc64le (Power 8 and > above). Defined ENABLE_PPC_DILITHIUM and ENABLE_PPC_KYBER for > dilithium (ML-DSA) and kyber (ML-KEM) NTT and inverse NTT. Thank you for your work. The approach of optimizing NTT functions looks good. Let me start a discussion about Kyber. Then, we can apply the result to Dilithium. I wonder if we can do a bit more, so that we can avoid the duplication of the ZETA constant among NTT implementation and kyber-common.c. I'm considering about factoring following five functions from kyber-common.c: void _gcry_poly_ntt(poly *r); void _gcry_poly_invntt_tomont(poly *r); void _gcry_poly_reduce(poly *r) void _gcry_poly_tomont(poly *r); void _gcry_poly_basemul_montgomery(poly *r, const poly *a, const poly *b); into, say, kyber-common-generic.c. And provide archtecture specific kyber-common--.S for optimized version(s). This way, NTT functions are covered and ZETA is placed inside kyber-common-*. How do you think? I'll try with the optimized implementation of AVX2 in the reference code. https://urldefense.proofpoint.com/v2/url?u=https-3A__www.pq-2Dcrystals.org_kyber_&d=DwIBAg&c=BSDicqBQBDjDI9RkVyTcHQ&r=zspFcGYEyUrRywX_TdjlLwwrCx0eBFnzcs6XZVVVMh0&m=cyhP1gGDXWh9JCIn4z5NrebvLkC7bN89aMGL_HFl26R2f9h7kqRDsaD6W5C2Q8tQ&s=guKkLMabJUVbm4bjm61GKleAAubKCEyFJJobD1MSghQ&e= -- -------------- next part -------------- An HTML attachment was scrubbed... URL: From tagrain at gmail.com Tue Jul 14 16:20:40 2026 From: tagrain at gmail.com (Thomas Grainger) Date: Tue, 14 Jul 2026 15:20:40 +0100 Subject: DCO declaration -- Thomas Grainger Message-ID: -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 Libgcrypt Developer's Certificate of Origin. Version 1.0 ========================================================= By making a contribution to the Libgcrypt project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the free software license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate free software license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same free software license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the free software license(s) involved. Signed-off-by: Thomas Grainger -----BEGIN PGP SIGNATURE----- iHUEARYKAB0WIQTymEMfVHoEZC8HGEPdpItcR/vIyAUCalZEHAAKCRDdpItcR/vI yJpmAP9MT4pOFQZ6y8dcwmE/zmtrUZnkaZl3bx0MrH1hFZdUFgEAy2U2MgukpNtz uAzfLt34Jf5HfQsa3BxpKAeP4TjUOgE= =30xb -----END PGP SIGNATURE----- From tagrain at gmail.com Tue Jul 14 16:43:14 2026 From: tagrain at gmail.com (Thomas Grainger) Date: Tue, 14 Jul 2026 15:43:14 +0100 Subject: DCO declaration -- Thomas Grainger Message-ID: -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 Libgcrypt Developer's Certificate of Origin. Version 1.0 ========================================================= By making a contribution to the Libgcrypt project, I certify that: (a) The contribution was created in whole or in part by me and I have the right to submit it under the free software license indicated in the file; or (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate free software license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same free software license (unless I am permitted to submit under a different license), as indicated in the file; or (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the free software license(s) involved. Signed-off-by: Thomas Grainger -----BEGIN PGP SIGNATURE----- iHUEARYKAB0WIQTymEMfVHoEZC8HGEPdpItcR/vIyAUCalZEHAAKCRDdpItcR/vI yJpmAP9MT4pOFQZ6y8dcwmE/zmtrUZnkaZl3bx0MrH1hFZdUFgEAy2U2MgukpNtz uAzfLt34Jf5HfQsa3BxpKAeP4TjUOgE= =30xb -----END PGP SIGNATURE----- From gniibe at fsij.org Fri Jul 17 11:02:34 2026 From: gniibe at fsij.org (NIIBE Yutaka) Date: Fri, 17 Jul 2026 18:02:34 +0900 Subject: [PATCH 0/2] Changes to rsa_decrypt behavior on bad keys In-Reply-To: <20260709231752.1332171-1-davegalos@google.com> References: <20260709231752.1332171-1-davegalos@google.com> Message-ID: <87cxwm7zv9.fsf@haruna.fsij.org> Hello, Sorry for late reply. I misunderstood/confused as your message were sent to gnupg-devel. FYI, I replied to gnupg-devel about the issue of P >= Q: https://lists.gnupg.org/pipermail/gnupg-devel/2026-July/036329.html Our ticket is: https://dev.gnupg.org/T8171 Repeat: Such a key should not be used by GnuPG. David Galos wrote: > There are two following patches, and I think at least one of them is > pretty important because the failure rate of a gcrypt-based crypto > oracle could probably leak info about factor-proximity based on failure > rate. Thank you for your work. It is important. I agree. My concern is: If a key (P >= Q) has been used and (many) signatures are already available out there, such a key would not be safe. Case by case judgement and manual handling would be needed for such a key. I don't think [PATCH 2/2] is good fix for a library, when a library assumes/claims "p is smaller than q" in other parts. It would introduce another confusion. For libgcrypt, mpi_subm is computationally heavy than mpi_sub+mpi_add, we don't modify like that. For [PATCH 1/2], for libgcrypt, I think that it is better for an application to use standard function of gcry_pk_testkey to check a key (or validate values before creating SEXP of libgcrypt). It should not be within a specific operation. I mean, it should be done **beforehand**. In libgcrypt, for some reason, p < q has been not tested by gcry_pk_testkey. It is enabled with ENABLE_STRONGER_CHECK now. -- From jussi.kivilinna at iki.fi Thu Jul 23 13:30:48 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Thu, 23 Jul 2026 14:30:48 +0300 Subject: [PATCH] cipher: prefer 128-bit blocksize in inline block helpers Message-ID: <20260723113048.2098312-1-jussi.kivilinna@iki.fi> * cipher/cipher-internal.h (_gcry_blocksize_shift, cipher_block_add) (cipher_block_cpy, cipher_block_xor, cipher_block_xor_2dst) (cipher_block_xor_n_copy_2, cipher_block_bswap): Mark 8-byte blocksize branch with UNLIKELY. -- Inline block helpers branch on blocksize 8 versus 16. 128-bit ciphers dominate, so mark 8-byte branch unlikely and let compiler lay out 16-byte path as fall-through. Signed-off-by: Jussi Kivilinna --- cipher/cipher-internal.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h index 7a10de6e..8e6cbffc 100644 --- a/cipher/cipher-internal.h +++ b/cipher/cipher-internal.h @@ -804,7 +804,7 @@ static inline unsigned int _gcry_blocksize_shift(gcry_cipher_hd_t c) { /* Only blocksizes 8 and 16 are used. Return value in such way * that compiler can optimize calling functions based on this. */ - return c->spec->blocksize == 8 ? 3 : 4; + return UNLIKELY(c->spec->blocksize == 8) ? 3 : 4; } @@ -839,7 +839,7 @@ cipher_block_add(void *_dstsrc, unsigned int add, size_t blocksize) byte *dstsrc = _dstsrc; u64 s[2]; - if (blocksize == 8) + if (UNLIKELY(blocksize == 8)) { buf_put_be64(dstsrc + 0, buf_get_be64(dstsrc + 0) + add); } @@ -863,7 +863,7 @@ cipher_block_cpy(void *_dst, const void *_src, size_t blocksize) const byte *src = _src; u64 s[2]; - if (blocksize == 8) + if (UNLIKELY(blocksize == 8)) { buf_put_he64(dst + 0, buf_get_he64(src + 0)); } @@ -888,7 +888,7 @@ cipher_block_xor(void *_dst, const void *_src1, const void *_src2, u64 s1[2]; u64 s2[2]; - if (blocksize == 8) + if (UNLIKELY(blocksize == 8)) { buf_put_he64(dst + 0, buf_get_he64(src1 + 0) ^ buf_get_he64(src2 + 0)); } @@ -924,7 +924,7 @@ cipher_block_xor_2dst(void *_dst1, void *_dst2, const void *_src, u64 d2[2]; u64 s[2]; - if (blocksize == 8) + if (UNLIKELY(blocksize == 8)) { d2[0] = buf_get_he64(dst2 + 0) ^ buf_get_he64(src + 0); buf_put_he64(dst2 + 0, d2[0]); @@ -961,7 +961,7 @@ cipher_block_xor_n_copy_2(void *_dst_xor, const void *_src_xor, u64 sx[2]; u64 sdc[2]; - if (blocksize == 8) + if (UNLIKELY(blocksize == 8)) { sc[0] = buf_get_he64(src_cpy + 0); buf_put_he64(dst_xor + 0, @@ -995,7 +995,7 @@ cipher_block_bswap (void *_dst_bswap, const void *_src_bswap, const byte *src_bswap = _src_bswap; u64 t[2]; - if (blocksize == 8) + if (UNLIKELY(blocksize == 8)) { buf_put_le64(dst_bswap, buf_get_be64(src_bswap)); } -- 2.53.0 From jussi.kivilinna at iki.fi Thu Jul 23 13:32:03 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Thu, 23 Jul 2026 14:32:03 +0300 Subject: [PATCH 1/2] Detect nonstring attribute via __has_attribute Message-ID: <20260723113204.2101375-1-jussi.kivilinna@iki.fi> * src/gcrypt.h.in (_GCRY_GCC_ATTR_NONSTRING): Detect with __has_attribute instead of GCC version check. -- Clang reports __GNUC__ as 4, so _GCRY_GCC_VERSION >= 80000 check never enabled nonstring attribute there, and clang 21 warns about unterminated string initialization in byte arrays. Probe the attribute with __has_attribute, which works on both GCC and clang regardless of reported version. Signed-off-by: Jussi Kivilinna --- src/gcrypt.h.in | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/gcrypt.h.in b/src/gcrypt.h.in index 8867b4e6..bb6f340d 100644 --- a/src/gcrypt.h.in +++ b/src/gcrypt.h.in @@ -98,8 +98,10 @@ extern "C" { #define _GCRY_GCC_ATTR_SENTINEL(a) __attribute__ ((sentinel(a))) #endif -#if _GCRY_GCC_VERSION >= 80000 -#define _GCRY_GCC_ATTR_NONSTRING __attribute__((__nonstring__)) +#if defined __has_attribute +# if __has_attribute (__nonstring__) +# define _GCRY_GCC_ATTR_NONSTRING __attribute__((__nonstring__)) +# endif #endif #endif /*__GNUC__*/ -- 2.53.0 From jussi.kivilinna at iki.fi Thu Jul 23 13:32:04 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Thu, 23 Jul 2026 14:32:04 +0300 Subject: [PATCH 2/2] secmem: use nested #if for __has_feature test In-Reply-To: <20260723113204.2101375-1-jussi.kivilinna@iki.fi> References: <20260723113204.2101375-1-jussi.kivilinna@iki.fi> Message-ID: <20260723113204.2101375-2-jussi.kivilinna@iki.fi> * src/secmem.c (lock_pool_pages): Test __has_feature with a nested #if instead of a single combined expression. -- GCC documents nested #if as portable form for the __has_* operators, and flags combined defined(X) && X(...) as valid only where operator is supported. GCC and clang both provide __has_feature, so nothing is broken in practice, but use documented form. Select no-mlock path through a macro so nested test keeps its else branch. Signed-off-by: Jussi Kivilinna --- src/secmem.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/secmem.c b/src/secmem.c index 7c5cac76..67fef0e6 100644 --- a/src/secmem.c +++ b/src/secmem.c @@ -296,7 +296,12 @@ lock_pool_pages (void *p, size_t n) uid = getuid (); #ifdef HAVE_BROKEN_MLOCK -#if defined(__has_feature) && __has_feature(address_sanitizer) +#if defined __has_feature +# if __has_feature (address_sanitizer) +# define GCRY_ASAN_NO_MLOCK 1 +# endif +#endif +#ifdef GCRY_ASAN_NO_MLOCK /* Address sanitizer and mlock is incompatible. We can't use mlock * with address sanitizer enabled. Proceed with NO_MLOCK=1. */ (void)p; -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 08:56:54 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 09:56:54 +0300 Subject: [PATCH] rijndael-vp-riscv: clear upper half when moving 128-bit to 256-bit Message-ID: <20260724065654.1074699-1-jussi.kivilinna@iki.fi> * cipher/rijndael-vp-riscv.c (movdqa128_256): Clear high 128-bit half of target 256-bit vector register. -- movdqa128_256 left high 128-bit half of target register uninitialized. With vector tail-agnostic all-ones policy (as with QEMU_CPU "max,vlen=256,rvv_ta_all_1s=true") upper half read back as all-ones and corrupted OCB checksum. Zero it explicitly. Signed-off-by: Jussi Kivilinna --- cipher/rijndael-vp-riscv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cipher/rijndael-vp-riscv.c b/cipher/rijndael-vp-riscv.c index 24ae3a7c..7a1da94f 100644 --- a/cipher/rijndael-vp-riscv.c +++ b/cipher/rijndael-vp-riscv.c @@ -230,7 +230,7 @@ typedef vuint8m1_t __m256i; #define movdqa256(a, o) (o = (a)) -#define movdqa128_256(a, o) (o = (a)) +#define movdqa128_256(a, o) (o = __riscv_vslideup_vx_u8m1((a), __riscv_vmv_v_x_u8m1(0, 16), 16, 32)) #define movdqa256_128(a, o) (o = (a)) #define movdqa256_memld(a, o) (o = __riscv_vle8_v_u8m1((const void *)(a), 32)) -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:48:00 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:48:00 +0300 Subject: [PATCH 1/7] cipher: fix spurious AEAD byte-counter carry for 4 GiB adds Message-ID: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> * cipher/cipher-internal.h (cipher_bytecounter_add): New. * cipher/cipher-gcm.c (gcm_bytecounter_add): Use shared helper. * cipher/cipher-gcm-siv.c (gcm_siv_bytecounter_add): Likewise. * cipher/cipher-poly1305.c (poly1305_bytecounter_add): Likewise. * tests/Makefile.am (tests_bin): Add 't-cipher-internal'. (t_cipher_internal_CPPFLAGS): New. * tests/t-cipher-internal.c: New test. -- Byte-counter helpers added high half of size_t addend, then detected low-word carry by comparing updated low word against full 64-bit addend. For single add of 4 GiB or more this comparison always caused carry and incremented high word second time. Adding 4 GiB to zero counter gave high word 2 instead of 1. Detect carry against truncated low addend instead and use single shared helper for all three modes. New test checks arithmetic across 32/64-bit boundaries. Reported-by: JEAN Jeremy Signed-off-by: Jussi Kivilinna --- cipher/cipher-gcm-siv.c | 11 +-- cipher/cipher-gcm.c | 11 +-- cipher/cipher-internal.h | 24 ++++++ cipher/cipher-poly1305.c | 17 +---- tests/Makefile.am | 4 +- tests/t-cipher-internal.c | 157 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 187 insertions(+), 37 deletions(-) create mode 100644 tests/t-cipher-internal.c diff --git a/cipher/cipher-gcm-siv.c b/cipher/cipher-gcm-siv.c index b92ae45d..6e0ce347 100644 --- a/cipher/cipher-gcm-siv.c +++ b/cipher/cipher-gcm-siv.c @@ -50,16 +50,7 @@ mulx_ghash (byte *a) static inline void gcm_siv_bytecounter_add (u32 ctr[2], size_t add) { - if (sizeof(add) > sizeof(u32)) - { - u32 high_add = ((add >> 31) >> 1) & 0xffffffff; - ctr[1] += high_add; - } - - ctr[0] += add; - if (ctr[0] >= add) - return; - ++ctr[1]; + cipher_bytecounter_add (ctr, add); } diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c index 1627cd1c..c04683f8 100644 --- a/cipher/cipher-gcm.c +++ b/cipher/cipher-gcm.c @@ -692,16 +692,7 @@ setupM (gcry_cipher_hd_t c) static inline void gcm_bytecounter_add (u32 ctr[2], size_t add) { - if (sizeof(add) > sizeof(u32)) - { - u32 high_add = ((add >> 31) >> 1) & 0xffffffff; - ctr[1] += high_add; - } - - ctr[0] += add; - if (ctr[0] >= add) - return; - ++ctr[1]; + cipher_bytecounter_add (ctr, add); } diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h index eabb7ea9..8e6cbffc 100644 --- a/cipher/cipher-internal.h +++ b/cipher/cipher-internal.h @@ -808,6 +808,30 @@ static inline unsigned int _gcry_blocksize_shift(gcry_cipher_hd_t c) } +/* Add 32-bit or 64-bit size_t to 2x32-bit byte counter. */ +static inline int +cipher_bytecounter_add (u32 ctr[2], size_t add) +{ + int overflow = 0; + u32 low_add = add; + + if (sizeof(add) > sizeof(u32)) + { + u32 high_add = ((add >> 31) >> 1) & 0xffffffff; + ctr[1] += high_add; + if (ctr[1] < high_add) + overflow = 1; + } + + ctr[0] += low_add; + if (ctr[0] >= low_add) + return overflow; + + ctr[1] += 1; + return (ctr[1] < 1) || overflow; +} + + /* Optimized function for adding value to cipher block. */ static inline void cipher_block_add(void *_dstsrc, unsigned int add, size_t blocksize) diff --git a/cipher/cipher-poly1305.c b/cipher/cipher-poly1305.c index c76dd9a4..c7441971 100644 --- a/cipher/cipher-poly1305.c +++ b/cipher/cipher-poly1305.c @@ -33,22 +33,7 @@ static inline int poly1305_bytecounter_add (u32 ctr[2], size_t add) { - int overflow = 0; - - if (sizeof(add) > sizeof(u32)) - { - u32 high_add = ((add >> 31) >> 1) & 0xffffffff; - ctr[1] += high_add; - if (ctr[1] < high_add) - overflow = 1; - } - - ctr[0] += add; - if (ctr[0] >= add) - return overflow; - - ctr[1] += 1; - return (ctr[1] < 1) || overflow; + return cipher_bytecounter_add (ctr, add); } diff --git a/tests/Makefile.am b/tests/Makefile.am index 9392dd36..8fcde4fc 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -25,7 +25,8 @@ tests_bin = \ version t-secmem mpitests t-sexp t-convert \ t-mpi-bit t-mpi-point t-lock \ prime basic keygen pubkey hmac hashtest t-kdf keygrip \ - aeswrap random t-kem t-thread-local t-fips-service-ind + aeswrap random t-kem t-thread-local t-fips-service-ind \ + t-cipher-internal if USE_RSA tests_bin += pkcs1v2 t-rsa-pss t-rsa-15 t-rsa-testparm @@ -104,6 +105,7 @@ t_lock_CFLAGS = $(GPG_ERROR_MT_CFLAGS) t_thread_local_LDADD = $(standard_ldadd) $(GPG_ERROR_MT_LIBS) @LDADD_FOR_TESTS_KLUDGE@ t_thread_local_CFLAGS = $(GPG_ERROR_MT_CFLAGS) testdrv_LDADD = $(LDADD_FOR_TESTS_KLUDGE) +t_cipher_internal_CPPFLAGS = $(AM_CPPFLAGS) -I$(top_srcdir)/cipher # Build a version of the test driver for the build platform. testdrv-build: testdrv.c diff --git a/tests/t-cipher-internal.c b/tests/t-cipher-internal.c new file mode 100644 index 00000000..6a9cdc7d --- /dev/null +++ b/tests/t-cipher-internal.c @@ -0,0 +1,157 @@ +/* t-cipher-internal.c - Regression tests for cipher-internal.h helpers + * Copyright (C) 2026 Jussi Kivilinna + * + * This file is part of Libgcrypt. + * + * Libgcrypt is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * Libgcrypt is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, see . + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "../src/g10lib.h" +#include "../cipher/cipher-internal.h" + +#define PGM "t-cipher-internal" + +static int verbose; +static int debug; +static int error_count; + +static void +print_hilo (const char *label, u64 v) +{ + fprintf (stderr, "%s=%08lx%08lx", label, + (unsigned long)(u32)(v >> 32), (unsigned long)(u32)v); +} + +static void +check_bytecounter_add (u64 start, u64 add) +{ + u32 ctr[2]; + u64 want = start + add; + int want_ovf = add > (~(u64)0 - start); + int got_ovf; + + ctr[0] = (u32)start; + ctr[1] = (u32)(start >> 32); + + got_ovf = cipher_bytecounter_add (ctr, (size_t)add); + + if ((((u64)ctr[1] << 32) | ctr[0]) != want || !!got_ovf != !!want_ovf) + { + error_count++; + print_hilo ("FAIL: start", start); + print_hilo (" add", add); + print_hilo (" -> got", ((u64)ctr[1] << 32) | ctr[0]); + fprintf (stderr, " ovf=%d,", !!got_ovf); + print_hilo (" want", want); + fprintf (stderr, " ovf=%d\n", !!want_ovf); + } + else if (debug) + { + print_hilo ("ok: start", start); + print_hilo (" add", add); + print_hilo (" -> ", want); + fprintf (stderr, " ovf=%d\n", !!want_ovf); + } +} + +static void +test_bytecounter_add (void) +{ + static const u64 starts[] = { + U64_C (0x0000000000000000), U64_C (0x0000000000000001), + U64_C (0x0000000090000000), U64_C (0x00000000ffffffff), + U64_C (0x00000001ffffffff), U64_C (0xfffffffe00000000), + U64_C (0x123456789abcdef0) + }; + static const u64 adds[] = { + U64_C (0), U64_C (1), U64_C (15), U64_C (16), + U64_C (0x000000007fffffff), U64_C (0x0000000080000000), + U64_C (0x00000000ffffffff), U64_C (0x0000000100000000), + U64_C (0x0000000100000010), U64_C (0x0000000180000000), + U64_C (0x0000001000000000), U64_C (0x000000fffffffff0), + U64_C (0xffffffffffffffff) + }; + unsigned int s, a; + + if (verbose) + fprintf (stderr, " checking cipher_bytecounter_add\n"); + + for (s = 0; s < DIM (starts); s++) + for (a = 0; a < DIM (adds); a++) + { + if (sizeof (size_t) < 8 && adds[a] > U64_C (0xffffffff)) + continue; + check_bytecounter_add (starts[s], adds[a]); + } + + if (sizeof (size_t) > 4) + { + u32 ctr[2] = { 0, 0 }; + cipher_bytecounter_add (ctr, (size_t)U64_C (0x100000000)); + if (ctr[0] != 0 || ctr[1] != 1) + { + error_count++; + fprintf (stderr, "FAIL: 4 GiB add gave %08lx:%08lx," + " expected 00000001:00000000\n", + (unsigned long)ctr[1], (unsigned long)ctr[0]); + } + } +} + +int +main (int argc, char **argv) +{ + int last_argc = -1; + + if (argc) + { argc--; argv++; } + + while (argc && last_argc != argc) + { + last_argc = argc; + if (!strcmp (*argv, "--")) + { + argc--; argv++; + break; + } + else if (!strcmp (*argv, "--verbose")) + { + verbose++; + argc--; argv++; + } + else if (!strcmp (*argv, "--debug")) + { + verbose = debug = 1; + argc--; argv++; + } + } + + if (verbose) + fprintf (stderr, "Starting cipher-internal checks.\n"); + + test_bytecounter_add (); + + if (error_count) + fprintf (stderr, PGM ": %d test(s) failed\n", error_count); + + return !!error_count; +} -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:48:02 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:48:02 +0300 Subject: [PATCH 3/7] gost28147: fix out-of-bounds read in IMIT MAC verify In-Reply-To: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> References: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> Message-ID: <20260724184806.3842171-3-jussi.kivilinna@iki.fi> * cipher/gost28147.c (gost_imit_verify): Reject tag longer than MAC. * tests/basic.c (check_mac_gost_imit_verify_len): New. (check_mac): Call check_mac_gost_imit_verify_len. -- gost_imit_verify built 8-byte MAC in stack buffer but compared caller supplied number of bytes without bounding it. Verify request longer than 8 bytes read past buffer into adjacent stack storage. Reject length above MAC size with GPG_ERR_INV_LENGTH, as HMAC verify does. Shorter lengths still verify for truncated tags. Reported-by: JEAN Jeremy Signed-off-by: Jussi Kivilinna --- cipher/gost28147.c | 3 +++ tests/basic.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/cipher/gost28147.c b/cipher/gost28147.c index f094d5ba..10383c23 100644 --- a/cipher/gost28147.c +++ b/cipher/gost28147.c @@ -485,6 +485,9 @@ gost_imit_verify (gcry_mac_hd_t h, const unsigned char *buf, size_t buflen) { unsigned char tbuf[8]; + if (buflen > sizeof(tbuf)) + return GPG_ERR_INV_LENGTH; + gost_imit_finish (h); buf_put_le32 (tbuf+0, h->u.imit.n1); diff --git a/tests/basic.c b/tests/basic.c index b62a9a90..0dceebd7 100644 --- a/tests/basic.c +++ b/tests/basic.c @@ -16159,6 +16159,68 @@ out: gcry_mac_close (hd); } +#if USE_GOST28147 +static void +check_mac_gost_imit_verify_len (void) +{ + static const char key[32] _GCRY_GCC_ATTR_NONSTRING = + "\x9d\x05\xb7\x9e\x90\xca\xd0\x0a\x2c\xda\xd2\x2e\xf4\xe8\x6f\x5c" + "\xf5\xdc\x37\x68\x19\x85\xb3\xbf\xaa\x18\xc1\xc3\x05\x0a\x91\xa2"; + static const char data[16] _GCRY_GCC_ATTR_NONSTRING = + "\xb5\xa1\xf0\xe3\xce\x2f\x02\x1d\x67\x61\x94\x34\x5c\x41\xe3\x6e"; + static const size_t bad_lens[] = { 9, 16, 64, 4096 }; + unsigned char tag[8]; + unsigned char big[4096]; + size_t taglen = sizeof (tag); + gcry_mac_hd_t hd; + gcry_error_t err; + unsigned int i; + + if (gcry_mac_test_algo (GCRY_MAC_GOST28147_IMIT)) + return; + + if (verbose) + fprintf (stderr, " checking GOST28147 IMIT verify length handling\n"); + + err = gcry_mac_open (&hd, GCRY_MAC_GOST28147_IMIT, 0, NULL); + if (err) + { + fail ("gost imit verify: gcry_mac_open failed: %s\n", gpg_strerror (err)); + return; + } + + if (!(err = gcry_mac_setkey (hd, key, sizeof (key)))) + err = gcry_mac_write (hd, data, sizeof (data)); + if (!err) + err = gcry_mac_read (hd, tag, &taglen); + if (err || taglen != sizeof (tag)) + { + fail ("gost imit verify: setup failed: %s\n", gpg_strerror (err)); + goto out; + } + + err = gcry_mac_verify (hd, tag, sizeof (tag)); + if (err) + fail ("gost imit verify: 8-byte tag rejected: %s\n", gpg_strerror (err)); + err = gcry_mac_verify (hd, tag, 4); + if (err) + fail ("gost imit verify: 4-byte tag rejected: %s\n", gpg_strerror (err)); + + memset (big, 0, sizeof (big)); + memcpy (big, tag, sizeof (tag)); + for (i = 0; i < DIM (bad_lens); i++) + { + err = gcry_mac_verify (hd, big, bad_lens[i]); + if (gcry_err_code (err) != GPG_ERR_INV_LENGTH) + fail ("gost imit verify: oversized len %d not rejected: %s\n", + (int)bad_lens[i], gpg_strerror (err)); + } + +out: + gcry_mac_close (hd); +} +#endif /* USE_GOST28147 */ + static void check_mac (void) { @@ -17110,6 +17172,10 @@ check_mac (void) algos[i].expect, 1); } +#if USE_GOST28147 + check_mac_gost_imit_verify_len (); +#endif + if (verbose) fprintf (stderr, "Completed MAC checks.\n"); } -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:48:04 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:48:04 +0300 Subject: [PATCH 5/7] cipher-gcm: assert 32 KiB input cap in gcm_ctr_encrypt In-Reply-To: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> References: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> Message-ID: <20260724184806.3842171-5-jussi.kivilinna@iki.fi> * cipher/cipher-gcm.c (gcm_ctr_encrypt): Assert input length cap that keeps CTR-wrap chunk math from overflowing. -- gcm_ctr_encrypt handles the 32-bit CTR wrap by computing a chunk size in u32 arithmetic that could overflow for a single multi-gigabyte input. This was reported as a reachable assertion failure, but sole caller gcm_crypt_inner caps input at 32 KiB for cache locality, so num_ctr_increments and the chunk length stay small and the overflow is unreachable. Assert the cap to document that instead of changing the arithmetic. Reported-by: JEAN Jeremy Signed-off-by: Jussi Kivilinna --- cipher/cipher-gcm.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c index c04683f8..e6a4df0f 100644 --- a/cipher/cipher-gcm.c +++ b/cipher/cipher-gcm.c @@ -815,6 +815,11 @@ gcm_ctr_encrypt (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen, { gcry_err_code_t err = 0; + /* Input is capped to 32KiB by gcm_crypt_inner() for better cache + * locality. As side-effect, CTR overflow checks below do not + * overflow. */ + gcry_assert(inbuflen <= 32 * 1024); + while (inbuflen) { u32 nblocks_to_overflow; -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:48:06 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:48:06 +0300 Subject: [PATCH 7/7] rijndael-armv8-aarch32-ce: clear q15 holding last round key In-Reply-To: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> References: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> Message-ID: <20260724184806.3842171-7-jussi.kivilinna@iki.fi> * cipher/rijndael-armv8-aarch32-ce.S (_gcry_aes_cbc_enc_armv8_ce) (_gcry_aes_cbc_dec_armv8_ce, _gcry_aes_ecb_enc_armv8_ce) (_gcry_aes_ecb_dec_armv8_ce, _gcry_aes_cfb_enc_armv8_ce) (_gcry_aes_cfb_dec_armv8_ce, _gcry_aes_ctr_enc_armv8_ce) (_gcry_aes_ctr32le_enc_armv8_ce, _gcry_aes_ocb_enc_armv8_ce) (_gcry_aes_ocb_dec_armv8_ce, _gcry_aes_ocb_auth_armv8_ce) (_gcry_aes_xts_enc_armv8_ce, _gcry_aes_xts_dec_armv8_ce): Clear q15 on function exit. -- q15 holds last round key in aarch32 AES macros. Bulk functions cleared q8-q14 on exit but not q15, leaving round key in vector register. Add q15 to clear list. Signed-off-by: Jussi Kivilinna --- cipher/rijndael-armv8-aarch32-ce.S | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cipher/rijndael-armv8-aarch32-ce.S b/cipher/rijndael-armv8-aarch32-ce.S index 3c4149b3..7d85aff9 100644 --- a/cipher/rijndael-armv8-aarch32-ce.S +++ b/cipher/rijndael-armv8-aarch32-ce.S @@ -541,6 +541,7 @@ _gcry_aes_cbc_enc_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lcbc_enc_skip: pop {r4-r6,pc} @@ -647,6 +648,7 @@ _gcry_aes_cbc_dec_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lcbc_dec_skip: pop {r4-r6,pc} @@ -732,6 +734,7 @@ _gcry_aes_ecb_enc_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lecb_enc_skip: pop {r4-r6,pc} @@ -790,6 +793,7 @@ _gcry_aes_ecb_dec_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lecb_dec_skip: pop {r4-r6,pc} @@ -866,6 +870,7 @@ _gcry_aes_cfb_enc_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lcfb_enc_skip: pop {r4-r6,pc} @@ -974,6 +979,7 @@ _gcry_aes_cfb_dec_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lcfb_dec_skip: pop {r4-r6,pc} @@ -1139,6 +1145,7 @@ _gcry_aes_ctr_enc_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lctr_enc_skip: pop {r4-r12,lr} @@ -1272,6 +1279,7 @@ _gcry_aes_ctr32le_enc_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lctr32le_enc_skip: pop {r4-r12,lr} @@ -1447,6 +1455,7 @@ _gcry_aes_ocb_enc_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) mov r0, #0 pop {r4-r12,lr} @@ -1622,6 +1631,7 @@ _gcry_aes_ocb_dec_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) mov r0, #0 pop {r4-r12,lr} @@ -1776,6 +1786,7 @@ _gcry_aes_ocb_auth_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) mov r0, #0 pop {r4-r12,lr} @@ -1932,6 +1943,7 @@ _gcry_aes_xts_enc_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lxts_enc_skip: pop {r4-r12,lr} @@ -2087,6 +2099,7 @@ _gcry_aes_xts_dec_armv8_ce: CLEAR_REG(q12) CLEAR_REG(q13) CLEAR_REG(q14) + CLEAR_REG(q15) .Lxts_dec_skip: pop {r4-r12,lr} -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:48:03 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:48:03 +0300 Subject: [PATCH 4/7] cipher-ocb: fix split-AAD abort at L-table wrap boundary In-Reply-To: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> References: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> Message-ID: <20260724184806.3842171-4-jussi.kivilinna@iki.fi> * cipher/cipher-ocb.c (_gcry_cipher_ocb_authenticate): Pass aad_nblocks to ocb_get_L_big in buffered-block path, not aad_nblocks + 1. * tests/basic.c (ocb_aad_tag, check_ocb_cipher_aad_tablewrap): New. (check_ocb_cipher): Call check_ocb_cipher_aad_tablewrap. -- Buffered-AAD path incremented aad_nblocks and then, at each L-table wrap (every 65536 blocks), asked ocb_get_L_big for aad_nblocks + 1. That helper needs a wrap-aligned block number, so the off-by-one hits "Assertion `ntz >= 16' failed" and aborts whenever split AAD finishes the wrap block through buffered path. Full-blocks path already passes the correct count. New test feeds about 1 MiB of AAD split so the wrap block finishes in buffered path, comparing tag against one-shot. Reported-by: JEAN Jeremy Signed-off-by: Jussi Kivilinna --- cipher/cipher-ocb.c | 2 +- tests/basic.c | 60 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/cipher/cipher-ocb.c b/cipher/cipher-ocb.c index 05e56f44..8d4f576e 100644 --- a/cipher/cipher-ocb.c +++ b/cipher/cipher-ocb.c @@ -281,7 +281,7 @@ _gcry_cipher_ocb_authenticate (gcry_cipher_hd_t c, const unsigned char *abuf, if ((c->u_mode.ocb.aad_nblocks % table_maxblks) == 0) { /* Table overflow, L needs to be generated. */ - ocb_get_L_big(c, c->u_mode.ocb.aad_nblocks + 1, l_tmp); + ocb_get_L_big(c, c->u_mode.ocb.aad_nblocks, l_tmp); } else { diff --git a/tests/basic.c b/tests/basic.c index 0dceebd7..1f4273ea 100644 --- a/tests/basic.c +++ b/tests/basic.c @@ -9474,6 +9474,65 @@ check_ocb_cipher_largebuf (int algo, int keylen, const char *tagexpect) } +static void +ocb_aad_tag (const unsigned char key[16], const unsigned char nonce[12], + const unsigned char *aad, size_t len0, size_t len1, + unsigned char tag[16]) +{ + gcry_cipher_hd_t hd; + gcry_error_t err; + unsigned char out[16]; + + err = gcry_cipher_open (&hd, GCRY_CIPHER_AES, GCRY_CIPHER_MODE_OCB, 0); + if (!err) + err = gcry_cipher_setkey (hd, key, 16); + if (!err) + err = gcry_cipher_setiv (hd, nonce, 12); + if (!err) + err = gcry_cipher_authenticate (hd, aad, len0); + if (!err && len1) + err = gcry_cipher_authenticate (hd, aad + len0, len1); + if (!err) + err = gcry_cipher_final (hd); + if (!err) + err = gcry_cipher_encrypt (hd, out, sizeof out, aad, sizeof out); + if (!err) + err = gcry_cipher_gettag (hd, tag, 16); + gcry_cipher_close (hd); + if (err) + fail ("ocb aad tablewrap: %s\n", gpg_strerror (err)); +} + + +static void +check_ocb_cipher_aad_tablewrap (void) +{ + static const unsigned char key[16] _GCRY_GCC_ATTR_NONSTRING = + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"; + static const unsigned char nonce[12] _GCRY_GCC_ATTR_NONSTRING = + "\xbb\xaa\x99\x88\x77\x66\x55\x44\x33\x22\x11\x0d"; + const size_t aadlen = (size_t)65536 * 16; /* fills one OCB L-table wrap */ + unsigned char tag_ref[16], tag_split[16]; + unsigned char *aad; + size_t i; + + if (verbose) + fprintf (stderr, " checking OCB AAD split across L-table boundary\n"); + + aad = xmalloc (aadlen); + for (i = 0; i < aadlen; i++) + aad[i] = (unsigned char)i; + + ocb_aad_tag (key, nonce, aad, aadlen, 0, tag_ref); + ocb_aad_tag (key, nonce, aad, aadlen - 15, 15, tag_split); + + if (memcmp (tag_ref, tag_split, 16)) + fail ("ocb aad tablewrap: split tag mismatch\n"); + + xfree (aad); +} + + static void check_ocb_cipher_splitaad (void) { @@ -9766,6 +9825,7 @@ check_ocb_cipher (void) /* Check that the AAD data is correctly buffered. */ check_ocb_cipher_splitaad (); + check_ocb_cipher_aad_tablewrap (); } -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:48:05 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:48:05 +0300 Subject: [PATCH 6/7] cipher: fold AEAD byte-counter helper to 64-bit accumulator In-Reply-To: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> References: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> Message-ID: <20260724184806.3842171-6-jussi.kivilinna@iki.fi> * cipher/cipher-internal.h (cipher_bytecounter_add): Rewrite using 64-bit accumulator. -- cipher_bytecounter_add split size_t addend into high and low halves with separate carry handling. Load 32-bit counter pair into 64-bit accumulator, add in one step and split back. Signed-off-by: Jussi Kivilinna --- cipher/cipher-internal.h | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h index 8e6cbffc..bc023c76 100644 --- a/cipher/cipher-internal.h +++ b/cipher/cipher-internal.h @@ -812,23 +812,11 @@ static inline unsigned int _gcry_blocksize_shift(gcry_cipher_hd_t c) static inline int cipher_bytecounter_add (u32 ctr[2], size_t add) { - int overflow = 0; - u32 low_add = add; - - if (sizeof(add) > sizeof(u32)) - { - u32 high_add = ((add >> 31) >> 1) & 0xffffffff; - ctr[1] += high_add; - if (ctr[1] < high_add) - overflow = 1; - } - - ctr[0] += low_add; - if (ctr[0] >= low_add) - return overflow; - - ctr[1] += 1; - return (ctr[1] < 1) || overflow; + u64 bytecounter = ((u64)ctr[1] << 32) + ctr[0]; + bytecounter += add; + ctr[0] = bytecounter; + ctr[1] = bytecounter >> 32; + return bytecounter < add; } -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:48:01 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:48:01 +0300 Subject: [PATCH 2/7] cipher-cmac: fix CMAC block-count truncation for 64 GiB writes In-Reply-To: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> References: <20260724184806.3842171-1-jussi.kivilinna@iki.fi> Message-ID: <20260724184806.3842171-2-jussi.kivilinna@iki.fi> * cipher/cipher-cmac.c (_gcry_cmac_write): Use size_t for nblocks. -- _gcry_cmac_write derived number of complete blocks from size_t input length but stored it in unsigned int. On 64-bit build, single write of 64 GiB or more truncated block count given to bulk CBC routine, and pointer and length updates used same truncated count. Resulting CMAC covered wrong data and gave wrong tag. Reported-by: JEAN Jeremy Signed-off-by: Jussi Kivilinna --- cipher/cipher-cmac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cipher/cipher-cmac.c b/cipher/cipher-cmac.c index 4efd1e19..c8bedc08 100644 --- a/cipher/cipher-cmac.c +++ b/cipher/cipher-cmac.c @@ -42,7 +42,7 @@ _gcry_cmac_write (gcry_cipher_hd_t c, gcry_cmac_context_t *ctx, size_t blocksize = 1 << blocksize_shift; byte outbuf[MAX_BLOCKSIZE]; unsigned int burn = 0; - unsigned int nblocks; + size_t nblocks; size_t n; if (ctx->tag) -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:50:12 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:12 +0300 Subject: [PATCH 05/10] cast5: reduce CTR bulk counter carry handling to 16 bits In-Reply-To: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> References: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> Message-ID: <20260724185017.3847717-5-jussi.kivilinna@iki.fi> * cipher/cast5.c (cast_setkey): Assign ctr16be_enc bulk op. (_gcry_cast5_ctr_enc): Use cipher_block_add_be16. * cipher/cast5-arm.S (_gcry_cast5_arm_ctr_enc): Drop 64-bit counter carry propagation in IV construction. -- CAST5 is 64-bit block cipher, so ARM CTR bulk function built IVs by incrementing 64-bit counter with carry from low to high 32-bit word. Under ctr16be_enc contract low 16 bits never overflow inside bulk call, so low-word add cannot carry into high word. Drop carry propagation. Moves cast5 off transitional ctr_enc alias. Signed-off-by: Jussi Kivilinna --- cipher/cast5-arm.S | 16 ++++++++-------- cipher/cast5.c | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cipher/cast5-arm.S b/cipher/cast5-arm.S index ae53e6b4..75cc4a85 100644 --- a/cipher/cast5-arm.S +++ b/cipher/cast5-arm.S @@ -603,17 +603,17 @@ _gcry_cast5_arm_ctr_enc: mov lr, r3; - /* Load IV (big => host endian) */ - read_block_aligned(lr, 0, RL0, RR0, be_to_host, RT1); + /* Load IV */ + read_block_aligned(lr, 0, RL0, RR0, host_to_be, RT0); /* Construct IVs */ - adds RR1, RR0, #1; /* +1 */ - adc RL1, RL0, #0; - adds r6, RR1, #1; /* +2 */ - adc r5, RL1, #0; + mov RL1, RL0; + add RR1, RR0, #1; /* +1 */ + add r6, RR0, #2; /* +2 */ - /* Store new IV (host => big-endian) */ - write_block_aligned(lr, 0, r5, r6, host_to_be, RT1); + /* Store new IV */ + be_to_host(r6, RT0); + str r6, [lr, #4]; bl _gcry_cast5_arm_enc_blk2; /* result in RR0:RL0, RR1:RL1 = r4:r3, r10:r9 */ diff --git a/cipher/cast5.c b/cipher/cast5.c index 34cf2e58..19a7f6a6 100644 --- a/cipher/cast5.c +++ b/cipher/cast5.c @@ -814,9 +814,9 @@ _gcry_cast5_ctr_enc(void *context, unsigned char *ctr, void *outbuf_arg, cipher_block_cpy (tmpbuf + 0, ctr, CAST5_BLOCKSIZE); cipher_block_cpy (tmpbuf + 8, ctr, CAST5_BLOCKSIZE); cipher_block_cpy (tmpbuf + 16, ctr, CAST5_BLOCKSIZE); - cipher_block_add (tmpbuf + 8, 1, CAST5_BLOCKSIZE); - cipher_block_add (tmpbuf + 16, 2, CAST5_BLOCKSIZE); - cipher_block_add (ctr, 3, CAST5_BLOCKSIZE); + cipher_block_add_be16 (tmpbuf + 8, 1, CAST5_BLOCKSIZE); + cipher_block_add_be16 (tmpbuf + 16, 2, CAST5_BLOCKSIZE); + cipher_block_add_be16 (ctr, 3, CAST5_BLOCKSIZE); /* Encrypt the counter. */ do_encrypt_block_3(ctx, tmpbuf, tmpbuf); /* XOR the input with the encrypted counter and store in output. */ @@ -835,7 +835,7 @@ _gcry_cast5_ctr_enc(void *context, unsigned char *ctr, void *outbuf_arg, outbuf += CAST5_BLOCKSIZE; inbuf += CAST5_BLOCKSIZE; /* Increment the counter. */ - cipher_block_add (ctr, 1, CAST5_BLOCKSIZE); + cipher_block_add_be16 (ctr, 1, CAST5_BLOCKSIZE); } wipememory(tmpbuf, sizeof(tmpbuf)); @@ -1171,7 +1171,7 @@ cast_setkey (void *context, const byte *key, unsigned keylen, memset (bulk_ops, 0, sizeof(*bulk_ops)); bulk_ops->cfb_dec = _gcry_cast5_cfb_dec; bulk_ops->cbc_dec = _gcry_cast5_cbc_dec; - bulk_ops->ctr_enc = _gcry_cast5_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_cast5_ctr_enc; return rc; } -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:50:17 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:17 +0300 Subject: [PATCH 10/10] cipher: remove transitional ctr_enc alias In-Reply-To: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> References: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> Message-ID: <20260724185017.3847717-10-jussi.kivilinna@iki.fi> * cipher/cipher-internal.h (ctr_enc): Remove transitional alias. * cipher/rijndael-s390x.c (_gcry_aes_s390x_setup_acceleration): Assign ctr16be_enc bulk op. -- All ciphers now assign ctr16be_enc directly, so drop transitional ctr_enc alias. s390x was last user. Its KMCTR hardware path does full 128-bit counter addition, correct under contract, so only rename needed. Signed-off-by: Jussi Kivilinna --- cipher/cipher-internal.h | 5 ----- cipher/rijndael-s390x.c | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h index c99bd189..2872b656 100644 --- a/cipher/cipher-internal.h +++ b/cipher/cipher-internal.h @@ -209,11 +209,6 @@ typedef struct cipher_bulk_ops const void *inbuf_arg, size_t nblocks, int encrypt); } cipher_bulk_ops_t; -/* Temporary alias for transition period from full 128-bit big-endian - * counter addition in bulk processing function to 16-bit big-endian - * addition. */ -#define ctr_enc ctr16be_enc - /* A VIA processor with the Padlock engine as well as the Intel AES_NI instructions require an alignment of most data on a 16 byte diff --git a/cipher/rijndael-s390x.c b/cipher/rijndael-s390x.c index 0a26020e..f0885179 100644 --- a/cipher/rijndael-s390x.c +++ b/cipher/rijndael-s390x.c @@ -1138,7 +1138,7 @@ int _gcry_aes_s390x_setup_acceleration(RIJNDAEL_context *ctx, if (ctx->kma_func) { - bulk_ops->ctr_enc = aes_s390x_ctr128_enc; + bulk_ops->ctr16be_enc = aes_s390x_ctr128_enc; if (kimd_query () & km_function_to_mask (KMID_FUNCTION_GHASH)) { -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:50:15 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:15 +0300 Subject: [PATCH 08/10] twofish: reduce CTR bulk counter carry handling to 16 bits In-Reply-To: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> References: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> Message-ID: <20260724185017.3847717-8-jussi.kivilinna@iki.fi> * cipher/twofish.c (twofish_setkey): Assign ctr16be_enc bulk op. (_gcry_twofish_ctr_enc): Use cipher_block_add_be16. * cipher/twofish-amd64.S (RT0bl, RT1bl): New. (_gcry_twofish_amd64_ctr_enc): Increment low 16 counter bits in place, drop full-width carry and byteswap. * cipher/twofish-avx2-amd64.S (inc_le128): Remove. (_gcry_twofish_avx2_ctr_enc): Add to low 16 counter bits only, drop full-width carry handling. -- Twofish CTR built IVs with full 128-bit increment and byteswap round-trip. Under ctr16be_enc contract generic ctr code splits work at low 16-bit overflow, so bulk function adds only to low 16 counter bits. Scalar amd64 path now increments low counter bytes in place, dropping byteswap. AVX2 path uses 16-bit stepping and drops full-width carry. Moves twofish off transitional ctr_enc alias. Signed-off-by: Jussi Kivilinna --- cipher/twofish-amd64.S | 53 +++++++++++++-------------- cipher/twofish-avx2-amd64.S | 71 +++++++------------------------------ cipher/twofish.c | 4 +-- 3 files changed, 39 insertions(+), 89 deletions(-) diff --git a/cipher/twofish-amd64.S b/cipher/twofish-amd64.S index 913b252d..beafd10f 100644 --- a/cipher/twofish-amd64.S +++ b/cipher/twofish-amd64.S @@ -359,6 +359,9 @@ ELF(.size _gcry_twofish_amd64_encrypt_block,.-_gcry_twofish_amd64_encrypt_block; #define RT0d %edx #define RT1d %esi +#define RT0bl %dl +#define RT1bl %sil + #define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \ movzbl ab ## bl, tmp2 ## d; \ movzbl ab ## bh, tmp1 ## d; \ @@ -650,35 +653,27 @@ _gcry_twofish_amd64_ctr_enc: movq %rdx, (7 * 8)(%rsp); movq %rcx, RX0; - /* load IV and byteswap */ - movq 8(RX0), RT0; - movq 0(RX0), RT1; - movq RT0, RCD0; - movq RT1, RAB0; - bswapq RT0; - bswapq RT1; - - /* construct IVs */ - movq RT0, RCD1; - movq RT1, RAB1; - movq RT0, RCD2; - movq RT1, RAB2; - addq $1, RCD1; - adcq $0, RAB1; - bswapq RCD1; - bswapq RAB1; - addq $2, RCD2; - adcq $0, RAB2; - bswapq RCD2; - bswapq RAB2; - addq $3, RT0; - adcq $0, RT1; - bswapq RT0; - bswapq RT1; - - /* store new IV */ - movq RT0, 8(RX0); - movq RT1, 0(RX0); + /* Construct and update IV */ + movzbl 15(RX0), RT0d; + movzbl 14(RX0), RT1d; + movq 8(RX0), RCD0; + movq 0(RX0), RAB0; + movq RAB0, RAB1; + movq RAB0, RAB2; + addb $1, RT0bl; + adcb $0, RT1bl; + movb RT0bl, 15(RX0); + movb RT1bl, 14(RX0); + movq 8(RX0), RCD1; + addb $1, RT0bl; + adcb $0, RT1bl; + movb RT0bl, 15(RX0); + movb RT1bl, 14(RX0); + movq 8(RX0), RCD2; + addb $1, RT0bl; + adcb $0, RT1bl; + movb RT0bl, 15(RX0); + movb RT1bl, 14(RX0); call __twofish_enc_blk3; diff --git a/cipher/twofish-avx2-amd64.S b/cipher/twofish-avx2-amd64.S index 67c1e52e..72a39d17 100644 --- a/cipher/twofish-avx2-amd64.S +++ b/cipher/twofish-avx2-amd64.S @@ -553,12 +553,6 @@ _gcry_twofish_avx2_blk16: CFI_ENDPROC(); ELF(.size _gcry_twofish_avx2_blk16,.-_gcry_twofish_avx2_blk16;) -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - .align 16 .globl _gcry_twofish_avx2_ctr_enc ELF(.type _gcry_twofish_avx2_ctr_enc, at function;) @@ -571,79 +565,40 @@ _gcry_twofish_avx2_ctr_enc: */ CFI_STARTPROC(); - movq 8(%rcx), %rax; - bswapq %rax; - vzeroupper; vbroadcasti128 .Lbswap128_mask rRIP, RTMP3; vpcmpeqd RNOT, RNOT, RNOT; - vpsrldq $8, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */ - vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */ + vpsrldq $14, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */ + vpaddw RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */ /* load IV and byteswap */ vmovdqu (%rcx), RTMP4x; vpshufb RTMP3x, RTMP4x, RTMP4x; vmovdqa RTMP4x, RTMP0x; - inc_le128(RTMP4x, RNOTx, RTMP1x); + vpsubw RNOTx, RTMP4x, RTMP4x; vinserti128 $1, RTMP4x, RTMP0, RTMP0; vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */ - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 16), %rax; - ja .Lhandle_ctr_carry; + /* Update IV */ + addb $16, 15(%rcx); + adcb $0, 14(%rcx); /* construct IVs */ - vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +3 ; +2 */ vpshufb RTMP3, RTMP0, RB0; - vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +5 ; +4 */ vpshufb RTMP3, RTMP0, RC0; - vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +7 ; +6 */ vpshufb RTMP3, RTMP0, RD0; - vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +9 ; +8 */ vpshufb RTMP3, RTMP0, RA1; - vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +11 ; +10 */ vpshufb RTMP3, RTMP0, RB1; - vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +13 ; +12 */ vpshufb RTMP3, RTMP0, RC1; - vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +15 ; +14 */ vpshufb RTMP3, RTMP0, RD1; - vpsubq RTMP2, RTMP0, RTMP0; /* +16 */ - vpshufb RTMP3x, RTMP0x, RTMP0x; - - jmp .Lctr_carry_done; - -.Lhandle_ctr_carry: - /* construct IVs */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB0; /* +3 ; +2 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RC0; /* +5 ; +4 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RD0; /* +7 ; +6 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RA1; /* +9 ; +8 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RC1; /* +13 ; +12 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RD1; /* +15 ; +14 */ - inc_le128(RTMP0, RNOT, RTMP1); - vextracti128 $1, RTMP0, RTMP0x; - vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */ - -.align 4 -.Lctr_carry_done: - /* store new IV */ - vmovdqu RTMP0x, (%rcx); call __twofish_enc_blk16; diff --git a/cipher/twofish.c b/cipher/twofish.c index 11a6e251..4341a7fb 100644 --- a/cipher/twofish.c +++ b/cipher/twofish.c @@ -774,7 +774,7 @@ twofish_setkey (void *context, const byte *key, unsigned int keylen, memset (bulk_ops, 0, sizeof(*bulk_ops)); bulk_ops->cbc_dec = _gcry_twofish_cbc_dec; bulk_ops->cfb_dec = _gcry_twofish_cfb_dec; - bulk_ops->ctr_enc = _gcry_twofish_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_twofish_ctr_enc; bulk_ops->ocb_crypt = _gcry_twofish_ocb_crypt; bulk_ops->ocb_auth = _gcry_twofish_ocb_auth; bulk_ops->xts_crypt = _gcry_twofish_xts_crypt; @@ -1197,7 +1197,7 @@ _gcry_twofish_ctr_enc(void *context, unsigned char *ctr, void *outbuf_arg, outbuf += TWOFISH_BLOCKSIZE; inbuf += TWOFISH_BLOCKSIZE; /* Increment the counter. */ - cipher_block_add(ctr, 1, TWOFISH_BLOCKSIZE); + cipher_block_add_be16(ctr, 1, TWOFISH_BLOCKSIZE); } wipememory(tmpbuf, sizeof(tmpbuf)); -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:50:13 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:13 +0300 Subject: [PATCH 06/10] serpent: reduce CTR bulk counter carry handling to 16 bits In-Reply-To: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> References: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> Message-ID: <20260724185017.3847717-6-jussi.kivilinna@iki.fi> * cipher/serpent.c (serpent_setkey): Assign ctr16be_enc bulk op. (_gcry_serpent_ctr_enc): Use cipher_block_add_be16. * cipher/serpent-avx2-amd64.S (inc_le128): Remove. (_gcry_serpent_avx2_ctr_enc): Add to low 16 counter bits only, drop full-width overflow path. * cipher/serpent-sse2-amd64.S (_gcry_serpent_sse2_ctr_enc): Add to low 16 counter bits only, drop full-width overflow path. * cipher/serpent-avx512-x86.c (ctr_generate): Do 16-bit big-endian counter addition, add carry slow path with vpaddw/vpshufb. * cipher/serpent-armv7-neon.S (_gcry_serpent_neon_ctr_enc): Drop 64-bit counter overflow path. * cipher/Makefile.am (avx512f_cflags): Add -mavx512bw. * configure.ac: Add -mavx512bw and _mm512_shuffle_epi8 to AVX512 intrinsics check. -- Serpent CTR keystream generators did full 128-bit counter increment with separate 64-bit overflow path. Under ctr16be_enc contract generic ctr code splits work at low 16-bit overflow, so bulk function adds only to low 16 counter bits. Drop overflow paths, use 16-bit stepping. AVX512 slow path now does 16-bit vector addition and byte shuffle (vpaddw, vpshufb), which need AVX512BW. Add -mavx512bw to AVX512 intrinsics build flags and probe _mm512_shuffle_epi8 in configure. Moves serpent off transitional ctr_enc alias. Signed-off-by: Jussi Kivilinna --- cipher/Makefile.am | 2 +- cipher/serpent-armv7-neon.S | 39 ------------- cipher/serpent-avx2-amd64.S | 71 +++++------------------- cipher/serpent-avx512-x86.c | 108 ++++++++++++++++-------------------- cipher/serpent-sse2-amd64.S | 67 +++++----------------- cipher/serpent.c | 4 +- configure.ac | 3 +- 7 files changed, 80 insertions(+), 214 deletions(-) diff --git a/cipher/Makefile.am b/cipher/Makefile.am index 8f54ea47..021a7ab6 100644 --- a/cipher/Makefile.am +++ b/cipher/Makefile.am @@ -357,7 +357,7 @@ rijndael-vp-aarch64.lo: $(srcdir)/rijndael-vp-aarch64.c Makefile `echo $(LTCOMPILE) $(aarch64_simd_cflags) -c $< | $(instrumentation_munging) ` if ENABLE_X86_AVX512_INTRINSICS_EXTRA_CFLAGS -avx512f_cflags = -mavx512f +avx512f_cflags = -mavx512f -mavx512bw else avx512f_cflags = endif diff --git a/cipher/serpent-armv7-neon.S b/cipher/serpent-armv7-neon.S index 4179ba2c..c8b9660b 100644 --- a/cipher/serpent-armv7-neon.S +++ b/cipher/serpent-armv7-neon.S @@ -702,10 +702,6 @@ _gcry_serpent_neon_ctr_enc: vmov RB3d0, RT0d0; vmov RT2d0, RT0d0; - /* check need for handling 64-bit overflow and carry */ - beq .Ldo_ctr_carry; - -.Lctr_carry_done: /* le => be */ vrev64.u8 RA1, RA1; vrev64.u8 RA2, RA2; @@ -757,41 +753,6 @@ _gcry_serpent_neon_ctr_enc: veor RB3, RB3; pop {r4,pc}; - -.Ldo_ctr_carry: - cmp r4, #-8; - blo .Lctr_carry_done; - beq .Lcarry_RT2; - - cmp r4, #-6; - blo .Lcarry_RB3; - beq .Lcarry_RB2; - - cmp r4, #-4; - blo .Lcarry_RB1; - beq .Lcarry_RB0; - - cmp r4, #-2; - blo .Lcarry_RA3; - beq .Lcarry_RA2; - - vsub.u64 RA1d0, RT1d0; -.Lcarry_RA2: - vsub.u64 RA2d0, RT1d0; -.Lcarry_RA3: - vsub.u64 RA3d0, RT1d0; -.Lcarry_RB0: - vsub.u64 RB0d0, RT1d0; -.Lcarry_RB1: - vsub.u64 RB1d0, RT1d0; -.Lcarry_RB2: - vsub.u64 RB2d0, RT1d0; -.Lcarry_RB3: - vsub.u64 RB3d0, RT1d0; -.Lcarry_RT2: - vsub.u64 RT2d0, RT1d0; - - b .Lctr_carry_done; .size _gcry_serpent_neon_ctr_enc,.-_gcry_serpent_neon_ctr_enc; .align 3 diff --git a/cipher/serpent-avx2-amd64.S b/cipher/serpent-avx2-amd64.S index 7aba235f..a419eb29 100644 --- a/cipher/serpent-avx2-amd64.S +++ b/cipher/serpent-avx2-amd64.S @@ -633,12 +633,6 @@ _gcry_serpent_avx2_blk16: CFI_ENDPROC(); ELF(.size _gcry_serpent_avx2_blk16,.-_gcry_serpent_avx2_blk16;) -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - .align 16 .globl _gcry_serpent_avx2_ctr_enc ELF(.type _gcry_serpent_avx2_ctr_enc, at function;) @@ -651,79 +645,40 @@ _gcry_serpent_avx2_ctr_enc: */ CFI_STARTPROC(); - movq 8(%rcx), %rax; - bswapq %rax; - vzeroupper; vbroadcasti128 .Lbswap128_mask rRIP, RTMP3; vpcmpeqd RNOT, RNOT, RNOT; - vpsrldq $8, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */ - vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */ + vpsrldq $14, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */ + vpaddw RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */ /* load IV and byteswap */ vmovdqu (%rcx), RTMP4x; vpshufb RTMP3x, RTMP4x, RTMP4x; vmovdqa RTMP4x, RTMP0x; - inc_le128(RTMP4x, RNOTx, RTMP1x); + vpsubw RNOTx, RTMP4x, RTMP4x; vinserti128 $1, RTMP4x, RTMP0, RTMP0; vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */ - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 16), %rax; - ja .Lhandle_ctr_carry; - /* construct IVs */ - vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +3 ; +2 */ vpshufb RTMP3, RTMP0, RA1; - vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +5 ; +4 */ vpshufb RTMP3, RTMP0, RA2; - vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +7 ; +6 */ vpshufb RTMP3, RTMP0, RA3; - vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +9 ; +8 */ vpshufb RTMP3, RTMP0, RB0; - vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +11 ; +10 */ vpshufb RTMP3, RTMP0, RB1; - vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +13 ; +12 */ vpshufb RTMP3, RTMP0, RB2; - vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +15 ; +14 */ vpshufb RTMP3, RTMP0, RB3; - vpsubq RTMP2, RTMP0, RTMP0; /* +16 */ - vpshufb RTMP3x, RTMP0x, RTMP0x; - - jmp .Lctr_carry_done; -.Lhandle_ctr_carry: - /* construct IVs */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RA1; /* +3 ; +2 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RA2; /* +5 ; +4 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RA3; /* +7 ; +6 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB0; /* +9 ; +8 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB2; /* +13 ; +12 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB3; /* +15 ; +14 */ - inc_le128(RTMP0, RNOT, RTMP1); - vextracti128 $1, RTMP0, RTMP0x; - vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */ - -.align 4 -.Lctr_carry_done: - /* store new IV */ - vmovdqu RTMP0x, (%rcx); + /* Update IV */ + addb $16, 15(%rcx); + adcb $0, 14(%rcx); call __serpent_enc_blk16; diff --git a/cipher/serpent-avx512-x86.c b/cipher/serpent-avx512-x86.c index 5b5c2483..dfbe2432 100644 --- a/cipher/serpent-avx512-x86.c +++ b/cipher/serpent-avx512-x86.c @@ -618,74 +618,62 @@ ctr_generate(unsigned char *ctr, __m512i vin[8]) 4LL << 56, 0, 4LL << 56, 0, 4LL << 56, 0); - const __m512i add4567 = _mm512_add_epi32(add0123, add4444); - const __m512i add8888 = _mm512_add_epi32(add4444, add4444); + const __m512i add4567 = _mm512_add_epi8(add0123, add4444); + const __m512i add8888 = _mm512_add_epi8(add4444, add4444); // Fast path without carry handling. __m512i vctr = _mm512_broadcast_i32x4(_mm_loadu_si128((const void *)ctr)); - cipher_block_add(ctr, 32, blocksize); - vin[0] = _mm512_add_epi32(vctr, add0123); - vin[1] = _mm512_add_epi32(vctr, add4567); - vin[2] = _mm512_add_epi32(vin[0], add8888); - vin[3] = _mm512_add_epi32(vin[1], add8888); - vin[4] = _mm512_add_epi32(vin[2], add8888); - vin[5] = _mm512_add_epi32(vin[3], add8888); - vin[6] = _mm512_add_epi32(vin[4], add8888); - vin[7] = _mm512_add_epi32(vin[5], add8888); + cipher_block_add_be16(ctr, 32, blocksize); + vin[0] = _mm512_add_epi8(vctr, add0123); + vin[1] = _mm512_add_epi8(vctr, add4567); + vin[2] = _mm512_add_epi8(vin[0], add8888); + vin[3] = _mm512_add_epi8(vin[1], add8888); + vin[4] = _mm512_add_epi8(vin[2], add8888); + vin[5] = _mm512_add_epi8(vin[3], add8888); + vin[6] = _mm512_add_epi8(vin[4], add8888); + vin[7] = _mm512_add_epi8(vin[5], add8888); } else { - // Slow path. - u32 blocks[4][blocksize / sizeof(u32)]; - - cipher_block_cpy(blocks[0], ctr, blocksize); - cipher_block_cpy(blocks[1], ctr, blocksize); - cipher_block_cpy(blocks[2], ctr, blocksize); - cipher_block_cpy(blocks[3], ctr, blocksize); - cipher_block_add(ctr, 32, blocksize); - cipher_block_add(blocks[1], 1, blocksize); - cipher_block_add(blocks[2], 2, blocksize); - cipher_block_add(blocks[3], 3, blocksize); - vin[0] = _mm512_loadu_epi32 (blocks); - cipher_block_add(blocks[0], 4, blocksize); - cipher_block_add(blocks[1], 4, blocksize); - cipher_block_add(blocks[2], 4, blocksize); - cipher_block_add(blocks[3], 4, blocksize); - vin[1] = _mm512_loadu_epi32 (blocks); - cipher_block_add(blocks[0], 4, blocksize); - cipher_block_add(blocks[1], 4, blocksize); - cipher_block_add(blocks[2], 4, blocksize); - cipher_block_add(blocks[3], 4, blocksize); - vin[2] = _mm512_loadu_epi32 (blocks); - cipher_block_add(blocks[0], 4, blocksize); - cipher_block_add(blocks[1], 4, blocksize); - cipher_block_add(blocks[2], 4, blocksize); - cipher_block_add(blocks[3], 4, blocksize); - vin[3] = _mm512_loadu_epi32 (blocks); - cipher_block_add(blocks[0], 4, blocksize); - cipher_block_add(blocks[1], 4, blocksize); - cipher_block_add(blocks[2], 4, blocksize); - cipher_block_add(blocks[3], 4, blocksize); - vin[4] = _mm512_loadu_epi32 (blocks); - cipher_block_add(blocks[0], 4, blocksize); - cipher_block_add(blocks[1], 4, blocksize); - cipher_block_add(blocks[2], 4, blocksize); - cipher_block_add(blocks[3], 4, blocksize); - vin[5] = _mm512_loadu_epi32 (blocks); - cipher_block_add(blocks[0], 4, blocksize); - cipher_block_add(blocks[1], 4, blocksize); - cipher_block_add(blocks[2], 4, blocksize); - cipher_block_add(blocks[3], 4, blocksize); - vin[6] = _mm512_loadu_epi32 (blocks); - cipher_block_add(blocks[0], 4, blocksize); - cipher_block_add(blocks[1], 4, blocksize); - cipher_block_add(blocks[2], 4, blocksize); - cipher_block_add(blocks[3], 4, blocksize); - vin[7] = _mm512_loadu_epi32 (blocks); - - wipememory(blocks, sizeof(blocks)); + const __m512i add0123 = _mm512_set_epi64(0, 3, + 0, 2, + 0, 1, + 0, 0); + const __m512i add4444 = _mm512_set_epi64(0, 4, + 0, 4, + 0, 4, + 0, 4); + const __m512i add4567 = _mm512_add_epi16(add0123, add4444); + const __m512i add8888 = _mm512_add_epi16(add4444, add4444); + static const unsigned char bswap_mask[16] __attribute__ ((aligned (16))) = + { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }; + const __m512i vbswap = + _mm512_broadcast_i32x4(_mm_loadu_si128((const void *)bswap_mask)); + + // Slow path with 16-bit carry. + __m512i vctr = + _mm512_broadcast_i32x4(_mm_loadu_si128((const void *)ctr)); + + cipher_block_add_be16(ctr, 32, blocksize); + vctr = _mm512_shuffle_epi8(vctr, vbswap); + vin[0] = _mm512_add_epi16(vctr, add0123); + vin[1] = _mm512_add_epi16(vctr, add4567); + vin[2] = _mm512_add_epi16(vin[0], add8888); + vin[3] = _mm512_add_epi16(vin[1], add8888); + vin[4] = _mm512_add_epi16(vin[2], add8888); + vin[5] = _mm512_add_epi16(vin[3], add8888); + vin[6] = _mm512_add_epi16(vin[4], add8888); + vin[7] = _mm512_add_epi16(vin[5], add8888); + vin[0] = _mm512_shuffle_epi8(vin[0], vbswap); + vin[1] = _mm512_shuffle_epi8(vin[1], vbswap); + vin[2] = _mm512_shuffle_epi8(vin[2], vbswap); + vin[3] = _mm512_shuffle_epi8(vin[3], vbswap); + vin[4] = _mm512_shuffle_epi8(vin[4], vbswap); + vin[5] = _mm512_shuffle_epi8(vin[5], vbswap); + vin[6] = _mm512_shuffle_epi8(vin[6], vbswap); + vin[7] = _mm512_shuffle_epi8(vin[7], vbswap); } } diff --git a/cipher/serpent-sse2-amd64.S b/cipher/serpent-sse2-amd64.S index 885c2bf1..93912935 100644 --- a/cipher/serpent-sse2-amd64.S +++ b/cipher/serpent-sse2-amd64.S @@ -688,68 +688,27 @@ _gcry_serpent_sse2_ctr_enc: pbswap(RTMP0, RTMP1); /* be => le */ pcmpeqd RNOT, RNOT; - psrldq $8, RNOT; /* low: -1, high: 0 */ + psrldq $14, RNOT; /* low: -1, high: 0 */ movdqa RNOT, RTMP2; - paddq RTMP2, RTMP2; /* low: -2, high: 0 */ + paddw RTMP2, RTMP2; /* low: -2, high: 0 */ /* construct IVs */ movdqa RTMP0, RTMP1; - psubq RNOT, RTMP0; /* +1 */ + psubw RNOT, RTMP0; /* +1 */ movdqa RTMP0, RA1; - psubq RTMP2, RTMP1; /* +2 */ + psubw RTMP2, RTMP1; /* +2 */ movdqa RTMP1, RA2; - psubq RTMP2, RTMP0; /* +3 */ + psubw RTMP2, RTMP0; /* +3 */ movdqa RTMP0, RA3; - psubq RTMP2, RTMP1; /* +4 */ + psubw RTMP2, RTMP1; /* +4 */ movdqa RTMP1, RB0; - psubq RTMP2, RTMP0; /* +5 */ + psubw RTMP2, RTMP0; /* +5 */ movdqa RTMP0, RB1; - psubq RTMP2, RTMP1; /* +6 */ + psubw RTMP2, RTMP1; /* +6 */ movdqa RTMP1, RB2; - psubq RTMP2, RTMP0; /* +7 */ + psubw RTMP2, RTMP0; /* +7 */ movdqa RTMP0, RB3; - psubq RTMP2, RTMP1; /* +8 */ - - /* check need for handling 64-bit overflow and carry */ - cmpl $0xffffffff, 8(%rcx); - jne .Lno_ctr_carry; - - movl 12(%rcx), %eax; - bswapl %eax; - cmpl $-8, %eax; - jb .Lno_ctr_carry; - pslldq $8, RNOT; /* low: 0, high: -1 */ - je .Lcarry_RTMP0; - - cmpl $-6, %eax; - jb .Lcarry_RB3; - je .Lcarry_RB2; - - cmpl $-4, %eax; - jb .Lcarry_RB1; - je .Lcarry_RB0; - - cmpl $-2, %eax; - jb .Lcarry_RA3; - je .Lcarry_RA2; - - psubq RNOT, RA1; -.Lcarry_RA2: - psubq RNOT, RA2; -.Lcarry_RA3: - psubq RNOT, RA3; -.Lcarry_RB0: - psubq RNOT, RB0; -.Lcarry_RB1: - psubq RNOT, RB1; -.Lcarry_RB2: - psubq RNOT, RB2; -.Lcarry_RB3: - psubq RNOT, RB3; -.Lcarry_RTMP0: - psubq RNOT, RTMP1; - -.Lno_ctr_carry: + /* le => be */ pbswap(RA1, RTMP0); pbswap(RA2, RTMP0); @@ -759,8 +718,10 @@ _gcry_serpent_sse2_ctr_enc: pbswap(RB2, RTMP0); pbswap(RB3, RTMP0); pbswap(RTMP1, RTMP0); - /* store new IV */ - movdqu RTMP1, (%rcx); + + /* Update IV */ + addb $8, 15(%rcx); + adcb $0, 14(%rcx); call __serpent_enc_blk8; diff --git a/cipher/serpent.c b/cipher/serpent.c index 72a2287d..37e0c63a 100644 --- a/cipher/serpent.c +++ b/cipher/serpent.c @@ -854,7 +854,7 @@ serpent_setkey (void *ctx, memset (bulk_ops, 0, sizeof(*bulk_ops)); bulk_ops->cbc_dec = _gcry_serpent_cbc_dec; bulk_ops->cfb_dec = _gcry_serpent_cfb_dec; - bulk_ops->ctr_enc = _gcry_serpent_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_serpent_ctr_enc; bulk_ops->ocb_crypt = _gcry_serpent_ocb_crypt; bulk_ops->ocb_auth = _gcry_serpent_ocb_auth; bulk_ops->xts_crypt = _gcry_serpent_xts_crypt; @@ -1126,7 +1126,7 @@ _gcry_serpent_ctr_enc(void *context, unsigned char *ctr, outbuf += sizeof(serpent_block_t); inbuf += sizeof(serpent_block_t); /* Increment the counter. */ - cipher_block_add(ctr, 1, sizeof(serpent_block_t)); + cipher_block_add_be16(ctr, 1, sizeof(serpent_block_t)); } wipememory(tmpbuf, sizeof(tmpbuf)); diff --git a/configure.ac b/configure.ac index 604f063e..631ed81e 100644 --- a/configure.ac +++ b/configure.ac @@ -1834,7 +1834,7 @@ fi # Check whether compiler supports x86/AVX512 intrinsics # _gcc_cflags_save=$CFLAGS -CFLAGS="$CFLAGS -mavx512f" +CFLAGS="$CFLAGS -mavx512f -mavx512bw" AC_CACHE_CHECK([whether compiler supports x86/AVX512 intrinsics], [gcry_cv_cc_x86_avx512_intrinsics], @@ -1851,6 +1851,7 @@ AC_CACHE_CHECK([whether compiler supports x86/AVX512 intrinsics], x = _mm512_loadu_epi32 (in); /* check the GCC bug 90980. */ x = _mm512_maskz_loadu_epi32(_cvtu32_mask16(0xfff0), in) ^ _mm512_castsi128_si512(y); + x = _mm512_shuffle_epi8(x, x); asm volatile ("vinserti32x4 \$3, %0, %%zmm6, %%zmm6;\n\t" "vpxord %%zmm6, %%zmm6, %%zmm6" ::"x"(y),"r"(in):"memory","xmm6"); -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:50:10 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:10 +0300 Subject: [PATCH 03/10] blowfish: reduce CTR bulk counter carry handling to 16 bits In-Reply-To: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> References: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> Message-ID: <20260724185017.3847717-3-jussi.kivilinna@iki.fi> * cipher/blowfish.c (bf_setkey): Assign ctr16be_enc bulk op. (_gcry_blowfish_ctr_enc): Use cipher_block_add_be16. * cipher/blowfish-arm.S (_gcry_blowfish_arm_ctr_enc): Drop 64-bit counter carry propagation in IV construction. -- Blowfish is 64-bit block cipher, so ARM CTR bulk function built IVs by incrementing 64-bit counter with carry from low to high 32-bit word. Under ctr16be_enc contract low 16 bits never overflow inside bulk call, so low-word add cannot carry into high word. Drop carry propagation. Moves blowfish off transitional ctr_enc alias. Signed-off-by: Jussi Kivilinna --- cipher/blowfish-arm.S | 16 ++++++++-------- cipher/blowfish.c | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cipher/blowfish-arm.S b/cipher/blowfish-arm.S index a5101b5c..372c3ebc 100644 --- a/cipher/blowfish-arm.S +++ b/cipher/blowfish-arm.S @@ -629,17 +629,17 @@ _gcry_blowfish_arm_ctr_enc: mov lr, r3; - /* Load IV (big => host endian) */ - read_block_aligned(lr, 0, RL0, RR0, be_to_host, RT0); + /* Load IV */ + read_block_aligned(lr, 0, RL0, RR0, host_to_be, RT0); /* Construct IVs */ - adds RR1, RR0, #1; /* +1 */ - adc RL1, RL0, #0; - adds r6, RR1, #1; /* +2 */ - adc r5, RL1, #0; + mov RL1, RL0; + add RR1, RR0, #1; /* +1 */ + add r6, RR0, #2; /* +2 */ - /* Store new IV (host => big-endian) */ - write_block_aligned(lr, 0, r5, r6, host_to_be, RT0); + /* Store new IV */ + be_to_host(r6, RT0); + str r6, [lr, #4]; bl _gcry_blowfish_arm_enc_blk2; /* result in RR0:RL0, RR1:RL1 = r4:r3, r10:r9 */ diff --git a/cipher/blowfish.c b/cipher/blowfish.c index 87abd563..534d994f 100644 --- a/cipher/blowfish.c +++ b/cipher/blowfish.c @@ -678,9 +678,9 @@ _gcry_blowfish_ctr_enc(void *context, unsigned char *ctr, void *outbuf_arg, cipher_block_cpy (tmpbuf + 0, ctr, BLOWFISH_BLOCKSIZE); cipher_block_cpy (tmpbuf + 8, ctr, BLOWFISH_BLOCKSIZE); cipher_block_cpy (tmpbuf + 16, ctr, BLOWFISH_BLOCKSIZE); - cipher_block_add (tmpbuf + 8, 1, BLOWFISH_BLOCKSIZE); - cipher_block_add (tmpbuf + 16, 2, BLOWFISH_BLOCKSIZE); - cipher_block_add (ctr, 3, BLOWFISH_BLOCKSIZE); + cipher_block_add_be16 (tmpbuf + 8, 1, BLOWFISH_BLOCKSIZE); + cipher_block_add_be16 (tmpbuf + 16, 2, BLOWFISH_BLOCKSIZE); + cipher_block_add_be16 (ctr, 3, BLOWFISH_BLOCKSIZE); /* Encrypt the counter. */ do_encrypt_3(ctx, tmpbuf, tmpbuf); /* XOR the input with the encrypted counter and store in output. */ @@ -699,7 +699,7 @@ _gcry_blowfish_ctr_enc(void *context, unsigned char *ctr, void *outbuf_arg, outbuf += BLOWFISH_BLOCKSIZE; inbuf += BLOWFISH_BLOCKSIZE; /* Increment the counter. */ - cipher_block_add (ctr, 1, BLOWFISH_BLOCKSIZE); + cipher_block_add_be16 (ctr, 1, BLOWFISH_BLOCKSIZE); } wipememory(tmpbuf, sizeof(tmpbuf)); @@ -1074,7 +1074,7 @@ bf_setkey (void *context, const byte *key, unsigned keylen, memset (bulk_ops, 0, sizeof(*bulk_ops)); bulk_ops->cfb_dec = _gcry_blowfish_cfb_dec; bulk_ops->cbc_dec = _gcry_blowfish_cbc_dec; - bulk_ops->ctr_enc = _gcry_blowfish_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_blowfish_ctr_enc; return rc; } -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:50:16 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:16 +0300 Subject: [PATCH 09/10] des: reduce CTR bulk counter carry handling to 16 bits In-Reply-To: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> References: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> Message-ID: <20260724185017.3847717-9-jussi.kivilinna@iki.fi> * cipher/des.c (_gcry_3des_ctr_enc): Use cipher_block_add_be16. (do_tripledes_setkey): Assign ctr16be_enc bulk op. -- 3DES CTR increment used full-width cipher_block_add. Under ctr16be_enc contract generic ctr code splits work at low 16-bit overflow, so low 16-bit addition suffices. Moves 3DES off transitional ctr_enc alias. Signed-off-by: Jussi Kivilinna --- cipher/des.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cipher/des.c b/cipher/des.c index 4b3f9a1e..c28fcf5a 100644 --- a/cipher/des.c +++ b/cipher/des.c @@ -911,7 +911,7 @@ _gcry_3des_ctr_enc(void *context, unsigned char *ctr, void *outbuf_arg, outbuf += DES_BLOCKSIZE; inbuf += DES_BLOCKSIZE; /* Increment the counter. */ - cipher_block_add(ctr, 1, DES_BLOCKSIZE); + cipher_block_add_be16(ctr, 1, DES_BLOCKSIZE); } wipememory(tmpbuf, sizeof(tmpbuf)); @@ -1250,7 +1250,7 @@ do_tripledes_setkey ( void *context, const byte *key, unsigned keylen, memset (bulk_ops, 0, sizeof(*bulk_ops)); bulk_ops->cbc_dec = _gcry_3des_cbc_dec; bulk_ops->cfb_dec = _gcry_3des_cfb_dec; - bulk_ops->ctr_enc = _gcry_3des_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_3des_ctr_enc; tripledes_set3keys ( ctx, key, key+8, key+16); -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:50:11 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:11 +0300 Subject: [PATCH 04/10] camellia: reduce CTR bulk counter carry handling to 16 bits In-Reply-To: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> References: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> Message-ID: <20260724185017.3847717-4-jussi.kivilinna@iki.fi> * cipher/camellia-glue.c (camellia_setkey): Assign ctr16be_enc bulk op. * cipher/camellia-aesni-avx-amd64.S (inc_le128): Remove. (_gcry_camellia_aesni_avx_ctr_enc): Add to low 16 counter bits only, drop full-width overflow path. * cipher/camellia-aesni-avx2-amd64.h: Likewise. * cipher/camellia-gfni-avx512-amd64.S: Likewise. -- Camellia CTR keystream generators did full 128-bit little-endian counter increment with separate 64-bit overflow path. Under ctr16be_enc contract generic ctr code splits work at low 16-bit overflow, so bulk function adds only to low 16 counter bits. Drop inc_le128/add_le128 and overflow path, use 16-bit stepping. Moves camellia off transitional ctr_enc alias. Signed-off-by: Jussi Kivilinna --- cipher/camellia-aesni-avx-amd64.S | 62 +++++-------- cipher/camellia-aesni-avx2-amd64.h | 129 +++++----------------------- cipher/camellia-gfni-avx512-amd64.S | 101 +++++----------------- cipher/camellia-glue.c | 2 +- 4 files changed, 64 insertions(+), 230 deletions(-) diff --git a/cipher/camellia-aesni-avx-amd64.S b/cipher/camellia-aesni-avx-amd64.S index 25354195..a29b9023 100644 --- a/cipher/camellia-aesni-avx-amd64.S +++ b/cipher/camellia-aesni-avx-amd64.S @@ -878,12 +878,6 @@ __camellia_dec_blk16: CFI_ENDPROC(); ELF(.size __camellia_dec_blk16,.-__camellia_dec_blk16;) -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - .align 16 .globl _gcry_camellia_aesni_avx_ctr_enc ELF(.type _gcry_camellia_aesni_avx_ctr_enc, at function;) @@ -924,48 +918,48 @@ _gcry_camellia_aesni_avx_ctr_enc: vpshufb %xmm14, %xmm15, %xmm0; /* be => le */ vpcmpeqd %xmm15, %xmm15, %xmm15; - vpsrldq $8, %xmm15, %xmm15; /* low: -1, high: 0 */ + vpsrldq $14, %xmm15, %xmm15; /* low: -1, high: 0 */ /* construct IVs */ - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm13; vmovdqu %xmm13, 14 * 16(%rax); - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm13; vmovdqu %xmm13, 13 * 16(%rax); - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm12; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm11; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm10; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm9; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm8; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm7; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm6; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm5; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm4; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm3; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm2; - inc_le128(%xmm0, %xmm15, %xmm13); + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm1; - inc_le128(%xmm0, %xmm15, %xmm13); - vmovdqa %xmm0, %xmm13; + vpsubw %xmm15, %xmm0, %xmm0; vpshufb %xmm14, %xmm0, %xmm0; - inc_le128(%xmm13, %xmm15, %xmm14); - vpshufb .Lbswap128_mask rRIP, %xmm13, %xmm13; /* le => be */ - vmovdqu %xmm13, (%rcx); .align 8 .Lload_ctr_done: + /* Update IV */ + addb $16, 15(%rcx); + adcb $0, 14(%rcx); + /* inpack16_pre: */ vmovq (key_table)(CTX), %xmm15; vpshufb .Lpack_bswap rRIP, %xmm15, %xmm15; @@ -1016,24 +1010,8 @@ _gcry_camellia_aesni_avx_ctr_enc: ret_spec_stop; .align 8 -.Lctr_byteadd_full_ctr_carry: - movq 8(%rcx), %r11; - movq (%rcx), %r10; - bswapq %r11; - bswapq %r10; - addq $16, %r11; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%rcx); - movq %r10, (%rcx); - jmp .Lctr_byteadd_xmm; -.align 8 .Lctr_byteadd: vmovdqu (%rcx), %xmm15; - je .Lctr_byteadd_full_ctr_carry; - addb $16, 15(%rcx); -.Lctr_byteadd_xmm: vmovdqa %xmm15, %xmm0; vpaddb .Lbige_addb_1 rRIP, %xmm15, %xmm14; vmovdqu %xmm15, 15 * 16(%rax); diff --git a/cipher/camellia-aesni-avx2-amd64.h b/cipher/camellia-aesni-avx2-amd64.h index 4c3fb4b2..21933c34 100644 --- a/cipher/camellia-aesni-avx2-amd64.h +++ b/cipher/camellia-aesni-avx2-amd64.h @@ -1164,12 +1164,6 @@ FUNC_NAME(dec_blk32): CFI_ENDPROC(); ELF(.size FUNC_NAME(dec_blk32),.-FUNC_NAME(dec_blk32);) -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - .align 16 .globl FUNC_NAME(ctr_enc) ELF(.type FUNC_NAME(ctr_enc), at function;) @@ -1200,123 +1194,60 @@ FUNC_NAME(ctr_enc): cmpb $(0x100 - 32), 15(%rcx); jbe .Lctr_byteadd; - movq 8(%rcx), %r11; - bswapq %r11; - vpcmpeqd %ymm15, %ymm15, %ymm15; - vpsrldq $8, %ymm15, %ymm15; /* ab: -1:0 ; cd: -1:0 */ + vpsrldq $14, %ymm15, %ymm15; /* ab: -1:0 ; cd: -1:0 */ /* load IV and byteswap */ vmovdqu (%rcx), %xmm0; vpshufb .Lbswap128_mask rRIP, %xmm0, %xmm0; vmovdqa %xmm0, %xmm1; - inc_le128(%xmm0, %xmm15, %xmm14); + vpsubw %xmm15, %xmm0, %xmm0; vbroadcasti128 .Lbswap128_mask rRIP, %ymm14; vinserti128 $1, %xmm0, %ymm1, %ymm0; vpshufb %ymm14, %ymm0, %ymm13; vmovdqu %ymm13, 15 * 32(%rax); - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 32), %r11; - ja .Lload_ctr_carry; - /* construct IVs */ - vpaddq %ymm15, %ymm15, %ymm15; /* ab: -2:0 ; cd: -2:0 */ - vpsubq %ymm15, %ymm0, %ymm0; + vpaddw %ymm15, %ymm15, %ymm15; /* ab: -2:0 ; cd: -2:0 */ + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm13; vmovdqu %ymm13, 14 * 32(%rax); - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm13; vmovdqu %ymm13, 13 * 32(%rax); - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm12; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm11; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm10; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm9; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm8; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm7; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm6; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm5; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm4; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm3; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm2; - vpsubq %ymm15, %ymm0, %ymm0; + vpsubw %ymm15, %ymm0, %ymm0; vpshufb %ymm14, %ymm0, %ymm1; - vpsubq %ymm15, %ymm0, %ymm0; /* +30 ; +31 */ - vpsubq %xmm15, %xmm0, %xmm13; /* +32 */ + vpsubw %ymm15, %ymm0, %ymm0; /* +30 ; +31 */ vpshufb %ymm14, %ymm0, %ymm0; - vpshufb %xmm14, %xmm13, %xmm13; - vmovdqu %xmm13, (%rcx); - - jmp .Lload_ctr_done; - -.align 4 -.Lload_ctr_carry: - /* construct IVs */ - inc_le128(%ymm0, %ymm15, %ymm13); /* ab: le1 ; cd: le2 */ - inc_le128(%ymm0, %ymm15, %ymm13); /* ab: le2 ; cd: le3 */ - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 14 * 32(%rax); - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 13 * 32(%rax); - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm12; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm11; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm10; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm9; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm8; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm7; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm6; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm5; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm4; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm3; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm2; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vpshufb %ymm14, %ymm0, %ymm1; - inc_le128(%ymm0, %ymm15, %ymm13); - inc_le128(%ymm0, %ymm15, %ymm13); - vextracti128 $1, %ymm0, %xmm13; - vpshufb %ymm14, %ymm0, %ymm0; - inc_le128(%xmm13, %xmm15, %xmm14); - vpshufb .Lbswap128_mask rRIP, %xmm13, %xmm13; - vmovdqu %xmm13, (%rcx); .align 8 .Lload_ctr_done: + /* Update IV */ + addb $32, 15(%rcx); + adcb $0, 14(%rcx); + /* inpack32_pre: */ vpbroadcastq (key_table)(CTX), %ymm15; vpshufb .Lpack_bswap rRIP, %ymm15, %ymm15; @@ -1367,24 +1298,8 @@ FUNC_NAME(ctr_enc): ret_spec_stop; .align 8 -.Lctr_byteadd_full_ctr_carry: - movq 8(%rcx), %r11; - movq (%rcx), %r10; - bswapq %r11; - bswapq %r10; - addq $32, %r11; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%rcx); - movq %r10, (%rcx); - jmp .Lctr_byteadd_ymm; -.align 8 .Lctr_byteadd: vbroadcasti128 (%rcx), %ymm8; - je .Lctr_byteadd_full_ctr_carry; - addb $32, 15(%rcx); -.Lctr_byteadd_ymm: vpaddb .Lbige_addb_16_16 rRIP, %ymm8, %ymm0; vpaddb .Lbige_addb_0_1 rRIP, %ymm8, %ymm15; vpaddb .Lbige_addb_2_3 rRIP, %ymm8, %ymm14; diff --git a/cipher/camellia-gfni-avx512-amd64.S b/cipher/camellia-gfni-avx512-amd64.S index 22ae43d9..4725b06a 100644 --- a/cipher/camellia-gfni-avx512-amd64.S +++ b/cipher/camellia-gfni-avx512-amd64.S @@ -578,8 +578,6 @@ ELF(.type _gcry_camellia_gfni_avx512__constants, at object;) .quad 8, 0 .Lcounter16161616_lo: .quad 16, 0 -.Lcounter1111_hi: - .quad 0, 1 .Lshufb_16x16b: .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) @@ -834,12 +832,6 @@ __camellia_gfni_avx512_dec_blk64: CFI_ENDPROC(); ELF(.size __camellia_gfni_avx512_dec_blk64,.-__camellia_gfni_avx512_dec_blk64;) -#define add_le128(out, in, lo_counter, hi_counter1) \ - vpaddq lo_counter, in, out; \ - vpcmpuq $1, lo_counter, out, %k1; \ - kaddb %k1, %k1, %k1; \ - vpaddq hi_counter1, out, out{%k1}; - .align 16 .globl _gcry_camellia_gfni_avx512_ctr_enc ELF(.type _gcry_camellia_gfni_avx512_ctr_enc, at function;) @@ -867,69 +859,34 @@ _gcry_camellia_gfni_avx512_ctr_enc: vbroadcasti64x2 .Lcounter4444_lo rRIP, %zmm22; vbroadcasti64x2 .Lcounter8888_lo rRIP, %zmm23; vbroadcasti64x2 .Lcounter16161616_lo rRIP, %zmm24; - vbroadcasti64x2 .Lcounter1111_hi rRIP, %zmm25; /* load IV and byteswap */ - movq 8(%rcx), %r11; - movq (%rcx), %r10; - bswapq %r11; - bswapq %r10; vbroadcasti64x2 (%rcx), %zmm0; vpshufb %zmm19, %zmm0, %zmm0; - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 64), %r11; - ja .Lload_ctr_carry; - /* construct IVs */ - vpaddq %zmm21, %zmm0, %zmm15; /* +0:+1:+2:+3 */ - vpaddq %zmm22, %zmm15, %zmm14; /* +4:+5:+6:+7 */ - vpaddq %zmm23, %zmm15, %zmm13; /* +8:+9:+10:+11 */ - vpaddq %zmm23, %zmm14, %zmm12; /* +12:+13:+14:+15 */ - vpaddq %zmm24, %zmm15, %zmm11; /* +16... */ - vpaddq %zmm24, %zmm14, %zmm10; /* +20... */ - vpaddq %zmm24, %zmm13, %zmm9; /* +24... */ - vpaddq %zmm24, %zmm12, %zmm8; /* +28... */ - vpaddq %zmm24, %zmm11, %zmm7; /* +32... */ - vpaddq %zmm24, %zmm10, %zmm6; /* +36... */ - vpaddq %zmm24, %zmm9, %zmm5; /* +40... */ - vpaddq %zmm24, %zmm8, %zmm4; /* +44... */ - vpaddq %zmm24, %zmm7, %zmm3; /* +48... */ - vpaddq %zmm24, %zmm6, %zmm2; /* +52... */ - vpaddq %zmm24, %zmm5, %zmm1; /* +56... */ - vpaddq %zmm24, %zmm4, %zmm0; /* +60... */ - jmp .Lload_ctr_done; - -.align 4 -.Lload_ctr_carry: - /* construct IVs */ - add_le128(%zmm15, %zmm0, %zmm21, %zmm25); /* +0:+1:+2:+3 */ - add_le128(%zmm14, %zmm15, %zmm22, %zmm25); /* +4:+5:+6:+7 */ - add_le128(%zmm13, %zmm15, %zmm23, %zmm25); /* +8:+9:+10:+11 */ - add_le128(%zmm12, %zmm14, %zmm23, %zmm25); /* +12:+13:+14:+15 */ - add_le128(%zmm11, %zmm15, %zmm24, %zmm25); /* +16... */ - add_le128(%zmm10, %zmm14, %zmm24, %zmm25); /* +20... */ - add_le128(%zmm9, %zmm13, %zmm24, %zmm25); /* +24... */ - add_le128(%zmm8, %zmm12, %zmm24, %zmm25); /* +28... */ - add_le128(%zmm7, %zmm11, %zmm24, %zmm25); /* +32... */ - add_le128(%zmm6, %zmm10, %zmm24, %zmm25); /* +36... */ - add_le128(%zmm5, %zmm9, %zmm24, %zmm25); /* +40... */ - add_le128(%zmm4, %zmm8, %zmm24, %zmm25); /* +44... */ - add_le128(%zmm3, %zmm7, %zmm24, %zmm25); /* +48... */ - add_le128(%zmm2, %zmm6, %zmm24, %zmm25); /* +52... */ - add_le128(%zmm1, %zmm5, %zmm24, %zmm25); /* +56... */ - add_le128(%zmm0, %zmm4, %zmm24, %zmm25); /* +60... */ - kxorq %k1, %k1, %k1; - -.align 4 -.Lload_ctr_done: + vpaddw %zmm21, %zmm0, %zmm15; /* +0:+1:+2:+3 */ + vpaddw %zmm22, %zmm15, %zmm14; /* +4:+5:+6:+7 */ + vpaddw %zmm23, %zmm15, %zmm13; /* +8:+9:+10:+11 */ + vpaddw %zmm23, %zmm14, %zmm12; /* +12:+13:+14:+15 */ + vpaddw %zmm24, %zmm15, %zmm11; /* +16... */ + vpaddw %zmm24, %zmm14, %zmm10; /* +20... */ + vpaddw %zmm24, %zmm13, %zmm9; /* +24... */ + vpaddw %zmm24, %zmm12, %zmm8; /* +28... */ + vpaddw %zmm24, %zmm11, %zmm7; /* +32... */ + vpaddw %zmm24, %zmm10, %zmm6; /* +36... */ + vpaddw %zmm24, %zmm9, %zmm5; /* +40... */ + vpaddw %zmm24, %zmm8, %zmm4; /* +44... */ + vpaddw %zmm24, %zmm7, %zmm3; /* +48... */ + vpaddw %zmm24, %zmm6, %zmm2; /* +52... */ + vpaddw %zmm24, %zmm5, %zmm1; /* +56... */ + vpaddw %zmm24, %zmm4, %zmm0; /* +60... */ + vbroadcasti64x2 .Lpack_bswap rRIP, %zmm17; vpbroadcastq (key_table)(CTX), %zmm16; vpshufb %zmm17, %zmm16, %zmm16; /* Byte-swap IVs and update counter. */ - addq $64, %r11; - adcq $0, %r10; vpshufb %zmm19, %zmm15, %zmm15; vpshufb %zmm19, %zmm14, %zmm14; vpshufb %zmm19, %zmm13, %zmm13; @@ -938,8 +895,6 @@ _gcry_camellia_gfni_avx512_ctr_enc: vpshufb %zmm19, %zmm10, %zmm10; vpshufb %zmm19, %zmm9, %zmm9; vpshufb %zmm19, %zmm8, %zmm8; - bswapq %r11; - bswapq %r10; vpshufb %zmm19, %zmm7, %zmm7; vpshufb %zmm19, %zmm6, %zmm6; vpshufb %zmm19, %zmm5, %zmm5; @@ -948,11 +903,13 @@ _gcry_camellia_gfni_avx512_ctr_enc: vpshufb %zmm19, %zmm2, %zmm2; vpshufb %zmm19, %zmm1, %zmm1; vpshufb %zmm19, %zmm0, %zmm0; - movq %r11, 8(%rcx); - movq %r10, (%rcx); .align 16 .Lctr_inpack64_pre: + /* Update IV */ + addb $64, 15(%rcx); + adcb $0, 14(%rcx); + /* inpack64_pre: */ vpxorq %zmm0, %zmm16, %zmm0; vpxorq %zmm1, %zmm16, %zmm1; @@ -999,24 +956,8 @@ _gcry_camellia_gfni_avx512_ctr_enc: ret_spec_stop; .align 16 -.Lctr_byteadd_full_ctr_carry: - movq 8(%rcx), %r11; - movq (%rcx), %r10; - bswapq %r11; - bswapq %r10; - addq $64, %r11; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%rcx); - movq %r10, (%rcx); - jmp .Lctr_byteadd_zmm; -.align 16 .Lctr_byteadd: vbroadcasti64x2 (%rcx), %zmm12; - je .Lctr_byteadd_full_ctr_carry; - addb $64, 15(%rcx); -.Lctr_byteadd_zmm: vbroadcasti64x2 .Lbige_addb_16 rRIP, %zmm16; vmovdqa64 .Lbige_addb_0_1 rRIP, %zmm17; vmovdqa64 .Lbige_addb_4_5 rRIP, %zmm18; diff --git a/cipher/camellia-glue.c b/cipher/camellia-glue.c index 78ff22b9..b656d964 100644 --- a/cipher/camellia-glue.c +++ b/cipher/camellia-glue.c @@ -611,7 +611,7 @@ camellia_setkey(void *c, const byte *key, unsigned keylen, memset (bulk_ops, 0, sizeof(*bulk_ops)); bulk_ops->cbc_dec = _gcry_camellia_cbc_dec; bulk_ops->cfb_dec = _gcry_camellia_cfb_dec; - bulk_ops->ctr_enc = _gcry_camellia_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_camellia_ctr_enc; bulk_ops->ocb_crypt = _gcry_camellia_ocb_crypt; bulk_ops->ocb_auth = _gcry_camellia_ocb_auth; bulk_ops->xts_crypt = _gcry_camellia_xts_crypt; -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:50:14 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:14 +0300 Subject: [PATCH 07/10] sm4: reduce CTR bulk counter carry handling to 16 bits In-Reply-To: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> References: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> Message-ID: <20260724185017.3847717-7-jussi.kivilinna@iki.fi> * cipher/sm4.c (sm4_setkey): Assign ctr16be_enc bulk op. * cipher/sm4-aesni-avx-amd64.S (inc_le128): Remove. (_gcry_sm4_aesni_avx_ctr_enc): Add to low 16 counter bits only, drop full-width carry handling. * cipher/sm4-avx2-amd64.h: Likewise. * cipher/sm4-avx512-amd64.h (ctr_enc_blk32): Add to low 16 counter bits only, drop full-width carry handling. * cipher/sm4-intel-avx512-amd64.S (add_le128): Remove. * cipher/sm4-gfni-avx512-amd64.S (add_le128): Remove. (_gcry_sm4_gfni_avx512_ctr_enc): Add to low 16 counter bits only, drop full-width carry handling. * cipher/sm4-aarch64.S: Likewise. * cipher/sm4-armv8-aarch64-ce.S: Likewise. * cipher/sm4-armv9-aarch64-sve-ce.S: Likewise. -- SM4 CTR implementations did full 128-bit counter increment with separate 64-bit overflow path. Under ctr16be_enc contract generic ctr code splits work at low 16-bit overflow, so bulk function adds only to low 16 counter bits. Drop overflow paths and use 16-bit stepping. Moves sm4 off transitional ctr_enc alias. Signed-off-by: Jussi Kivilinna --- cipher/sm4-aarch64.S | 40 +++++++------- cipher/sm4-aesni-avx-amd64.S | 49 +++++------------ cipher/sm4-armv8-aarch64-ce.S | 39 +++++++------- cipher/sm4-armv9-aarch64-sve-ce.S | 88 ++++++++++++++----------------- cipher/sm4-avx2-amd64.h | 85 +++++------------------------ cipher/sm4-avx512-amd64.h | 77 +++++---------------------- cipher/sm4-gfni-avx512-amd64.S | 81 +++++----------------------- cipher/sm4-intel-avx512-amd64.S | 6 --- cipher/sm4.c | 2 +- 9 files changed, 132 insertions(+), 335 deletions(-) diff --git a/cipher/sm4-aarch64.S b/cipher/sm4-aarch64.S index c71bf3ca..0e754a88 100644 --- a/cipher/sm4-aarch64.S +++ b/cipher/sm4-aarch64.S @@ -595,30 +595,31 @@ _gcry_sm4_aarch64_ctr_enc: preload_sbox(x5); - ldp x7, x8, [x3]; - rev x7, x7; - rev x8, x8; + ld1 {RIV.16b}, [x3]; + rev16 RIV.16b, RIV.16b; + + mov w7, #1; .Lctr_loop_blk: subs x4, x4, #8; bmi .Lctr_end; -#define inc_le128(vctr) \ - mov vctr.d[1], x8; \ - mov vctr.d[0], x7; \ - adds x8, x8, #1; \ - adc x7, x7, xzr; \ - rev64 vctr.16b, vctr.16b; +#define inc_be16(vctr) \ + rev16 vctr.16b, RIV.16b; \ + add RIV.8h, RIV.8h, RTMP0.8h; + + movi RTMP0.16b, #0; + mov RTMP0.b[14], w7; /* construct CTRs */ - inc_le128(v0); /* +0 */ - inc_le128(v1); /* +1 */ - inc_le128(v2); /* +2 */ - inc_le128(v3); /* +3 */ - inc_le128(v4); /* +4 */ - inc_le128(v5); /* +5 */ - inc_le128(v6); /* +6 */ - inc_le128(v7); /* +7 */ + inc_be16(v0); /* +0 */ + inc_be16(v1); /* +1 */ + inc_be16(v2); /* +2 */ + inc_be16(v3); /* +3 */ + inc_be16(v4); /* +4 */ + inc_be16(v5); /* +5 */ + inc_be16(v6); /* +6 */ + inc_be16(v7); /* +7 */ bl __sm4_crypt_blk8; @@ -640,9 +641,8 @@ _gcry_sm4_aarch64_ctr_enc: .Lctr_end: /* store new CTR */ - rev x7, x7; - rev x8, x8; - stp x7, x8, [x3]; + rev16 RIV.16b, RIV.16b; + st1 {RIV.16b}, [x3]; VPOP_ABI; clear_volatile_vec_regs(); diff --git a/cipher/sm4-aesni-avx-amd64.S b/cipher/sm4-aesni-avx-amd64.S index ca9be44a..643ab276 100644 --- a/cipher/sm4-aesni-avx-amd64.S +++ b/cipher/sm4-aesni-avx-amd64.S @@ -571,37 +571,30 @@ _gcry_sm4_aesni_avx_ctr_enc: vpshufb RBSWAP, RA0, RTMP0; /* be => le */ vpcmpeqd RNOT, RNOT, RNOT; - vpsrldq $8, RNOT, RNOT; /* low: -1, high: 0 */ - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; + vpsrldq $14, RNOT, RNOT; /* low: -1, high: 0 */ /* construct IVs */ - inc_le128(RTMP0, RNOT, RTMP2); /* +1 */ + vpsubw RNOT, RTMP0, RTMP0; /* +1 */ vpshufb RBSWAP, RTMP0, RA1; - inc_le128(RTMP0, RNOT, RTMP2); /* +2 */ + vpsubw RNOT, RTMP0, RTMP0; /* +2 */ vpshufb RBSWAP, RTMP0, RA2; - inc_le128(RTMP0, RNOT, RTMP2); /* +3 */ + vpsubw RNOT, RTMP0, RTMP0; /* +3 */ vpshufb RBSWAP, RTMP0, RA3; - inc_le128(RTMP0, RNOT, RTMP2); /* +4 */ + vpsubw RNOT, RTMP0, RTMP0; /* +4 */ vpshufb RBSWAP, RTMP0, RB0; - inc_le128(RTMP0, RNOT, RTMP2); /* +5 */ + vpsubw RNOT, RTMP0, RTMP0; /* +5 */ vpshufb RBSWAP, RTMP0, RB1; - inc_le128(RTMP0, RNOT, RTMP2); /* +6 */ + vpsubw RNOT, RTMP0, RTMP0; /* +6 */ vpshufb RBSWAP, RTMP0, RB2; - inc_le128(RTMP0, RNOT, RTMP2); /* +7 */ + vpsubw RNOT, RTMP0, RTMP0; /* +7 */ vpshufb RBSWAP, RTMP0, RB3; - inc_le128(RTMP0, RNOT, RTMP2); /* +8 */ - vpshufb RBSWAP, RTMP0, RTMP1; - - /* store new IV */ - vmovdqu RTMP1, (%rcx); .align 8 .Lload_ctr_done: + /* Update counter */ + addb $8, 15(%rcx); + adcb $0, 14(%rcx); + call __sm4_crypt_blk8; vpxor (0 * 16)(%rdx), RA0, RA0; @@ -625,26 +618,10 @@ _gcry_sm4_aesni_avx_ctr_enc: vzeroall; ret_spec_stop; - .align 8 - -.Lctr_byteadd_full_ctr_carry: - movq 8(%rcx), %r11; - movq (%rcx), %r10; - bswapq %r11; - bswapq %r10; - addq $8, %r11; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%rcx); - movq %r10, (%rcx); - jmp .Lctr_byteadd_xmm; + .align 8 .Lctr_byteadd: vmovdqu (%rcx), RA0; - je .Lctr_byteadd_full_ctr_carry; - addb $8, 15(%rcx); -.Lctr_byteadd_xmm: vpaddb .Lbige_addb_1 rRIP, RA0, RA1; vpaddb .Lbige_addb_2 rRIP, RA0, RA2; vpaddb .Lbige_addb_3 rRIP, RA0, RA3; diff --git a/cipher/sm4-armv8-aarch64-ce.S b/cipher/sm4-armv8-aarch64-ce.S index 5917ad5b..cd4a505f 100644 --- a/cipher/sm4-armv8-aarch64-ce.S +++ b/cipher/sm4-armv8-aarch64-ce.S @@ -546,30 +546,30 @@ _gcry_sm4_armv8_ce_ctr_enc: load_rkey(x0); - ldp x7, x8, [x3]; - rev x7, x7; - rev x8, x8; + ld1 {RIV.16b}, [x3]; + rev16 RIV.16b, RIV.16b; + + mov w7, #1; + movi RMASK.16b, #0; + mov RMASK.b[14], w7; .Lctr_loop_blk: subs x4, x4, #8; bmi .Lctr_end; -#define inc_le128(vctr) \ - mov vctr.d[1], x8; \ - mov vctr.d[0], x7; \ - adds x8, x8, #1; \ - adc x7, x7, xzr; \ - rev64 vctr.16b, vctr.16b; +#define inc_be16(vctr) \ + rev16 vctr.16b, RIV.16b; \ + add RIV.8h, RIV.8h, RMASK.8h; /* construct CTRs */ - inc_le128(v0); /* +0 */ - inc_le128(v1); /* +1 */ - inc_le128(v2); /* +2 */ - inc_le128(v3); /* +3 */ - inc_le128(v4); /* +4 */ - inc_le128(v5); /* +5 */ - inc_le128(v6); /* +6 */ - inc_le128(v7); /* +7 */ + inc_be16(v0); /* +0 */ + inc_be16(v1); /* +1 */ + inc_be16(v2); /* +2 */ + inc_be16(v3); /* +3 */ + inc_be16(v4); /* +4 */ + inc_be16(v5); /* +5 */ + inc_be16(v6); /* +6 */ + inc_be16(v7); /* +7 */ crypt_blk8(v0, v1, v2, v3, v4, v5, v6, v7); @@ -591,9 +591,8 @@ _gcry_sm4_armv8_ce_ctr_enc: .Lctr_end: /* store new CTR */ - rev x7, x7; - rev x8, x8; - stp x7, x8, [x3]; + rev16 RIV.16b, RIV.16b; + st1 {RIV.16b}, [x3]; CLEAR_ALL_REGS(); ret_spec_stop; diff --git a/cipher/sm4-armv9-aarch64-sve-ce.S b/cipher/sm4-armv9-aarch64-sve-ce.S index ba9bde84..ca6e51a3 100644 --- a/cipher/sm4-armv9-aarch64-sve-ce.S +++ b/cipher/sm4-armv9-aarch64-sve-ce.S @@ -70,7 +70,8 @@ _gcry_sm4_armv9_svesm4_consts: .byte 0xfc, 0xfd, 0xfe, 0xff, 0xf8, 0xf9, 0xfa, 0xfb .byte 0xf4, 0xf5, 0xf6, 0xf7, 0xf0, 0xf1, 0xf2, 0xf3 -.Lle128_inc: +.Lbe16_inc: + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 .byte 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 @@ -102,18 +103,15 @@ _gcry_sm4_armv9_svesm4_consts: .byte 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 .byte 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 ELF(.size _gcry_sm4_armv9_svesm4_consts,.-_gcry_sm4_armv9_svesm4_consts) /* Register macros */ -#define RCTR z16 -#define RCTRv v16 #define RIV z16 #define RIVv v16 #define RSWAP128 z17 -#define RZERO z18 -#define RLE128_INC z19 +#define RBE16_INC z18 +#define RADDv v19 #define RTMP0 z20 #define RTMP1 z21 @@ -814,38 +812,35 @@ _gcry_sm4_armv9_sve_ce_ctr_enc: PREPARE(); - dup RZERO.d, #0; - GET_DATA_POINTER(x6, .Lle128_inc); - ld1b {RLE128_INC.b}, p0/z, [x6]; - - ldp x7, x8, [x3]; - rev x7, x7; - rev x8, x8; - -#define inc_le128(zctr) \ - mov RCTRv.d[1], x8; \ - mov RCTRv.d[0], x7; \ - mov zctr.d, RLE128_INC.d; \ - dup RCTR.q, RCTR.q[0]; \ - adds x8, x8, x5, LSR #4; \ - adc x7, x7, xzr; \ - adclt zctr.d, RCTR.d, RZERO.d; \ - adclt RCTR.d, zctr.d, RZERO.d; \ - trn1 zctr.d, RCTR.d, zctr.d; \ + GET_DATA_POINTER(x6, .Lbe16_inc); + ld1b {RBE16_INC.b}, p0/z, [x6]; + + ld1 {RIVv.16b}, [x3]; + rev64 RIVv.16b, RIVv.16b; + + lsr x6, x5, #4; + movi RADDv.16b, #0; + mov RADDv.d[1], x6; + mov x6, #1; + +#define inc_be16(zctr) \ + dup zctr.q, RIV.q[0]; \ + add RIVv.8h, RIVv.8h, RADDv.8h; \ + add zctr.h, zctr.h, RBE16_INC.h; \ revb zctr.d, p0/m, zctr.d; .Lctr_loop_blks: sub x4, x4, x5, LSR #1; /* x4 - (8 * VL) */ tbnz x4, #63, .Lctr_tail8; - inc_le128(z0); - inc_le128(z1); - inc_le128(z2); - inc_le128(z3); - inc_le128(z4); - inc_le128(z5); - inc_le128(z6); - inc_le128(z7); + inc_be16(z0); + inc_be16(z1); + inc_be16(z2); + inc_be16(z3); + inc_be16(z4); + inc_be16(z5); + inc_be16(z6); + inc_be16(z7); SM4_SVE_CE_CRYPT_BLK8(z0, z1, z2, z3, z4, z5, z6, z7); @@ -887,10 +882,10 @@ _gcry_sm4_armv9_sve_ce_ctr_enc: sub x4, x4, x5, LSR #2; /* x4 - (4 * VL) */ - inc_le128(z0); - inc_le128(z1); - inc_le128(z2); - inc_le128(z3); + inc_be16(z0); + inc_be16(z1); + inc_be16(z2); + inc_be16(z3); SM4_SVE_CE_CRYPT_BLK4(z0, z1, z2, z3); @@ -917,7 +912,7 @@ _gcry_sm4_armv9_sve_ce_ctr_enc: sub x4, x4, x5, LSR #4; /* x4 - VL */ - inc_le128(z0); + inc_be16(z0); SM4_SVE_CE_CRYPT_BLK(z0); ld1b {RTMP0.b}, p0/z, [x2]; eor z0.d, z0.d, RTMP0.d; @@ -929,27 +924,26 @@ _gcry_sm4_armv9_sve_ce_ctr_enc: b .Lctr_tail4; .Lctr_tail: + mov RADDv.d[1], x6; + +.Lctr_tail_loop: sub x4, x4, #1; - /* inc_le128 for CE */ - mov v0.d[1], x8; - mov v0.d[0], x7; - adds x8, x8, #1; - adc x7, x7, xzr; - rev64 v0.16b, v0.16b; + /* increment counter for CE */ + rev64 v0.16b, RIVv.16b; + add RIVv.8h, RIVv.8h, RADDv.8h; SM4_CE_CRYPT_BLK(v0); ld1 {RTMP0v.16b}, [x2], #16; eor v0.16b, v0.16b, RTMP0v.16b; st1 {v0.16b}, [x1], #16; - cbnz x4, .Lctr_tail; + cbnz x4, .Lctr_tail_loop; .Lctr_end: /* store new CTR */ - rev x7, x7; - rev x8, x8; - stp x7, x8, [x3]; + rev64 RIVv.16b, RIVv.16b; + st1 {RIVv.16b}, [x3]; CLEAR_ALL_REGS(); ret_spec_stop; diff --git a/cipher/sm4-avx2-amd64.h b/cipher/sm4-avx2-amd64.h index 6d3740dd..a1b68833 100644 --- a/cipher/sm4-avx2-amd64.h +++ b/cipher/sm4-avx2-amd64.h @@ -56,12 +56,6 @@ FUNC_NAME(cipher_mode_consts): .text -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - .align 16 .globl FUNC_NAME(ctr_enc) ELF(.type FUNC_NAME(ctr_enc), at function;) @@ -82,74 +76,39 @@ FUNC_NAME(ctr_enc): vbroadcasti128 .Lbswap128_mask rRIP, RTMP3; vpcmpeqd RNOT, RNOT, RNOT; - vpsrldq $8, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */ - vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */ + vpsrldq $14, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */ + vpaddw RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */ /* load IV and byteswap */ vmovdqu (%rcx), RTMP4x; vpshufb RTMP3x, RTMP4x, RTMP4x; vmovdqa RTMP4x, RTMP0x; - inc_le128(RTMP4x, RNOTx, RTMP1x); + vpsubw RNOTx, RTMP4x, RTMP4x; vinserti128 $1, RTMP4x, RTMP0, RTMP0; vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */ - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 16), %rax; - ja .Lhandle_ctr_carry; - /* construct IVs */ - vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +3 ; +2 */ vpshufb RTMP3, RTMP0, RA1; - vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +5 ; +4 */ vpshufb RTMP3, RTMP0, RA2; - vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +7 ; +6 */ vpshufb RTMP3, RTMP0, RA3; - vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +9 ; +8 */ vpshufb RTMP3, RTMP0, RB0; - vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +11 ; +10 */ vpshufb RTMP3, RTMP0, RB1; - vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +13 ; +12 */ vpshufb RTMP3, RTMP0, RB2; - vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */ + vpsubw RTMP2, RTMP0, RTMP0; /* +15 ; +14 */ vpshufb RTMP3, RTMP0, RB3; - vpsubq RTMP2, RTMP0, RTMP0; /* +16 */ - vpshufb RTMP3x, RTMP0x, RTMP0x; - - jmp .Lctr_carry_done; - -.Lhandle_ctr_carry: - /* construct IVs */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RA1; /* +3 ; +2 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RA2; /* +5 ; +4 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RA3; /* +7 ; +6 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB0; /* +9 ; +8 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB2; /* +13 ; +12 */ - inc_le128(RTMP0, RNOT, RTMP1); - inc_le128(RTMP0, RNOT, RTMP1); - vpshufb RTMP3, RTMP0, RB3; /* +15 ; +14 */ - inc_le128(RTMP0, RNOT, RTMP1); - vextracti128 $1, RTMP0, RTMP0x; - vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */ - -.Lctr_carry_done: - /* store new IV */ - vmovdqu RTMP0x, (%rcx); .align 8 .Lload_ctr_done: + /* Update counter */ + addb $16, 15(%rcx); + adcb $0, 14(%rcx); + call SM4_CRYPT_BLK16; vpxor (0 * 32)(%rdx), RA0, RA0; @@ -175,24 +134,8 @@ FUNC_NAME(ctr_enc): ret_spec_stop; .align 8 -.Lctr_byteadd_full_ctr_carry: - movq 8(%rcx), %r11; - movq (%rcx), %r10; - bswapq %r11; - bswapq %r10; - addq $16, %r11; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%rcx); - movq %r10, (%rcx); - jmp .Lctr_byteadd_ymm; -.align 8 .Lctr_byteadd: vbroadcasti128 (%rcx), RB3; - je .Lctr_byteadd_full_ctr_carry; - addb $16, 15(%rcx); -.Lctr_byteadd_ymm: vpaddb .Lbige_addb_0_1 rRIP, RB3, RA0; vpaddb .Lbige_addb_2_3 rRIP, RB3, RA1; vpaddb .Lbige_addb_4_5 rRIP, RB3, RA2; diff --git a/cipher/sm4-avx512-amd64.h b/cipher/sm4-avx512-amd64.h index 3ee5a4f6..05e032de 100644 --- a/cipher/sm4-avx512-amd64.h +++ b/cipher/sm4-avx512-amd64.h @@ -70,8 +70,6 @@ FUNC_NAME(cipher_mode_consts): .quad 8, 0 .Lcounter16161616_lo: .quad 16, 0 -.Lcounter1111_hi: - .quad 0, 1 .text @@ -138,57 +136,19 @@ FUNC_NAME(ctr_enc_blk32): vbroadcasti64x2 .Lcounter16161616_lo rRIP, RTMP4z; /* load IV and byteswap */ - movq 8(%rcx), %r11; - bswapq %r11; vbroadcasti64x2 (%rcx), RB3z; vpshufb RTMP0z, RB3z, RB3z; - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 32), %r11; - ja .Lhandle_ctr_carry_blk32; - - /* construct IVs */ - vpaddq RTMP1z, RB3z, RA0z; /* +0:+1:+2:+3 */ - vpaddq RTMP2z, RA0z, RA1z; /* +4:+5:+6:+7 */ - vpaddq RTMP3z, RA0z, RA2z; /* +8:+9:+10:+11 */ - vpaddq RTMP3z, RA1z, RA3z; /* +12:+13:+14:+15 */ - vpaddq RTMP4z, RA0z, RB0z; /* +16... */ - vpaddq RTMP4z, RA1z, RB1z; /* +20... */ - vpaddq RTMP4z, RA2z, RB2z; /* +24... */ - vpaddq RTMP4z, RA3z, RB3z; /* +28... */ - - /* Update counter */ - leaq 32(%r11), %r11; - bswapq %r11; - movq %r11, 8(%rcx); - - jmp .Lctr_carry_done_blk32; - -.Lhandle_ctr_carry_blk32: - vbroadcasti64x2 .Lcounter1111_hi rRIP, RNOTz; - /* construct IVs */ - add_le128(RA0z, RB3z, RTMP1z, RNOTz); /* +0:+1:+2:+3 */ - add_le128(RA1z, RA0z, RTMP2z, RNOTz); /* +4:+5:+6:+7 */ - add_le128(RA2z, RA0z, RTMP3z, RNOTz); /* +8:+9:+10:+11 */ - add_le128(RA3z, RA1z, RTMP3z, RNOTz); /* +12:+13:+14:+15 */ - add_le128(RB0z, RA0z, RTMP4z, RNOTz); /* +16... */ - add_le128(RB1z, RA1z, RTMP4z, RNOTz); /* +20... */ - add_le128(RB2z, RA2z, RTMP4z, RNOTz); /* +24... */ - add_le128(RB3z, RA3z, RTMP4z, RNOTz); /* +28... */ + vpaddw RTMP1z, RB3z, RA0z; /* +0:+1:+2:+3 */ + vpaddw RTMP2z, RA0z, RA1z; /* +4:+5:+6:+7 */ + vpaddw RTMP3z, RA0z, RA2z; /* +8:+9:+10:+11 */ + vpaddw RTMP3z, RA1z, RA3z; /* +12:+13:+14:+15 */ + vpaddw RTMP4z, RA0z, RB0z; /* +16... */ + vpaddw RTMP4z, RA1z, RB1z; /* +20... */ + vpaddw RTMP4z, RA2z, RB2z; /* +24... */ + vpaddw RTMP4z, RA3z, RB3z; /* +28... */ - /* Update counter */ - addq $32, %r11; - movq (%rcx), %r10; - bswapq %r10; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%rcx); - movq %r10, (%rcx); - -.align 16 -.Lctr_carry_done_blk32: /* Byte-swap IVs. */ vpshufb RTMP0z, RA0z, RA0z; vpshufb RTMP0z, RA1z, RA1z; @@ -201,6 +161,10 @@ FUNC_NAME(ctr_enc_blk32): .align 16 .Lload_ctr_done32: + /* Update counter */ + addb $32, 15(%rcx); + adcb $0, 14(%rcx); + call SM4_CRYPT_BLK32; vpxord (0 * 64)(%rdx), RA0z, RA0z; @@ -222,29 +186,12 @@ FUNC_NAME(ctr_enc_blk32): vmovdqu32 RB3z, (7 * 64)(%rsi); vzeroall; - kxorq %k1, %k1, %k1; ret_spec_stop; .align 16 -.Lctr_byteadd_full_ctr_carry32: - movq 8(%rcx), %r11; - movq (%rcx), %r10; - bswapq %r11; - bswapq %r10; - addq $32, %r11; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%rcx); - movq %r10, (%rcx); - jmp .Lctr_byteadd_zmm32; -.align 16 .Lctr_byteadd32: vbroadcasti64x2 (%rcx), RA3z; - je .Lctr_byteadd_full_ctr_carry32; - addb $32, 15(%rcx); -.Lctr_byteadd_zmm32: vbroadcasti64x2 .Lbige_addb_16 rRIP, RB3z; vpaddb RB3z, RA3z, RB3z; vpaddb .Lbige_addb_0_1 rRIP, RA3z, RA0z; diff --git a/cipher/sm4-gfni-avx512-amd64.S b/cipher/sm4-gfni-avx512-amd64.S index 8bc8bd81..e69b2ab5 100644 --- a/cipher/sm4-gfni-avx512-amd64.S +++ b/cipher/sm4-gfni-avx512-amd64.S @@ -590,12 +590,6 @@ _gcry_sm4_gfni_avx512_crypt_blk1_16: CFI_ENDPROC(); ELF(.size _gcry_sm4_gfni_avx512_crypt_blk1_16,.-_gcry_sm4_gfni_avx512_crypt_blk1_16;) -#define add_le128(out, in, lo_counter, hi_counter1) \ - vpaddq lo_counter, in, out; \ - vpcmpuq $1, lo_counter, out, %k1; \ - kaddb %k1, %k1, %k1; \ - vpaddq hi_counter1, out, out{%k1}; - .align 16 .globl _gcry_sm4_gfni_avx512_ctr_enc ELF(.type _gcry_sm4_gfni_avx512_ctr_enc, at function;) @@ -619,57 +613,19 @@ _gcry_sm4_gfni_avx512_ctr_enc: vbroadcasti128 .Lcounter8888_lo rRIP, RTMP4; /* load IV and byteswap */ - movq 8(%rcx), %r11; - bswapq %r11; vbroadcasti128 (%rcx), RB3; vpshufb RTMP0, RB3, RB3; - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 16), %r11; - ja .Lhandle_ctr_carry_blk16; - - /* construct IVs */ - vpaddq RTMP1, RB3, RA0; /* +0:+1 */ - vpaddq RTMP2, RA0, RA1; /* +2:+3 */ - vpaddq RTMP3, RA0, RA2; /* +4:+5 */ - vpaddq RTMP3, RA1, RA3; /* +6:+7 */ - vpaddq RTMP4, RA0, RB0; /* +8... */ - vpaddq RTMP4, RA1, RB1; /* +10... */ - vpaddq RTMP4, RA2, RB2; /* +12... */ - vpaddq RTMP4, RA3, RB3; /* +14... */ - - /* Update counter */ - leaq 16(%r11), %r11; - bswapq %r11; - movq %r11, 8(%rcx); - - jmp .Lctr_carry_done_blk16; - -.Lhandle_ctr_carry_blk16: - vbroadcasti128 .Lcounter1111_hi rRIP, RNOT; - /* construct IVs */ - add_le128(RA0, RB3, RTMP1, RNOT); /* +0:+1 */ - add_le128(RA1, RA0, RTMP2, RNOT); /* +2:+3 */ - add_le128(RA2, RA0, RTMP3, RNOT); /* +4:+5 */ - add_le128(RA3, RA1, RTMP3, RNOT); /* +6:+7 */ - add_le128(RB0, RA0, RTMP4, RNOT); /* +8... */ - add_le128(RB1, RA1, RTMP4, RNOT); /* +10... */ - add_le128(RB2, RA2, RTMP4, RNOT); /* +12... */ - add_le128(RB3, RA3, RTMP4, RNOT); /* +14... */ + vpaddw RTMP1, RB3, RA0; /* +0:+1 */ + vpaddw RTMP2, RA0, RA1; /* +2:+3 */ + vpaddw RTMP3, RA0, RA2; /* +4:+5 */ + vpaddw RTMP3, RA1, RA3; /* +6:+7 */ + vpaddw RTMP4, RA0, RB0; /* +8... */ + vpaddw RTMP4, RA1, RB1; /* +10... */ + vpaddw RTMP4, RA2, RB2; /* +12... */ + vpaddw RTMP4, RA3, RB3; /* +14... */ - /* Update counter */ - addq $16, %r11; - movq (%rcx), %r10; - bswapq %r10; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%rcx); - movq %r10, (%rcx); - -.align 16 -.Lctr_carry_done_blk16: /* Byte-swap IVs. */ vpshufb RTMP0, RA0, RA0; vpshufb RTMP0, RA1, RA1; @@ -682,6 +638,10 @@ _gcry_sm4_gfni_avx512_ctr_enc: .align 16 .Lload_ctr_done16: + /* Update counter */ + addb $16, 15(%rcx); + adcb $0, 14(%rcx); + call __sm4_gfni_crypt_blk16; vpxor (0 * 32)(%rdx), RA0, RA0; @@ -703,29 +663,12 @@ _gcry_sm4_gfni_avx512_ctr_enc: vmovdqu RB3, (7 * 32)(%rsi); vzeroall; - kxorq %k1, %k1, %k1; ret_spec_stop; .align 16 -.Lctr_byteadd_full_ctr_carry16: - movq 8(%rcx), %r11; - movq (%rcx), %r10; - bswapq %r11; - bswapq %r10; - addq $16, %r11; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%rcx); - movq %r10, (%rcx); - jmp .Lctr_byteadd_ymm16; -.align 16 .Lctr_byteadd16: vbroadcasti128 (%rcx), RB3; - je .Lctr_byteadd_full_ctr_carry16; - addb $16, 15(%rcx); -.Lctr_byteadd_ymm16: vpaddb .Lbige_addb_0_1 rRIP, RB3, RA0; vpaddb .Lbige_addb_2_3 rRIP, RB3, RA1; vpaddb .Lbige_addb_4_5 rRIP, RB3, RA2; diff --git a/cipher/sm4-intel-avx512-amd64.S b/cipher/sm4-intel-avx512-amd64.S index a4963db0..c5eaa723 100644 --- a/cipher/sm4-intel-avx512-amd64.S +++ b/cipher/sm4-intel-avx512-amd64.S @@ -160,12 +160,6 @@ __sm4_intel_crypt_blk32: CFI_ENDPROC(); ELF(.size __sm4_intel_crypt_blk32,.-__sm4_intel_crypt_blk32;) -#define add_le128(out, in, lo_counter, hi_counter1) \ - vpaddq lo_counter, in, out; \ - vpcmpuq $1, lo_counter, out, %k1; \ - kaddb %k1, %k1, %k1; \ - vpaddq hi_counter1, out, out{%k1}; - #define FUNC_NAME(func) _gcry_sm4_intel_avx512_ ## func #define SM4_CRYPT_BLK32 __sm4_intel_crypt_blk32 #include "sm4-avx512-amd64.h" diff --git a/cipher/sm4.c b/cipher/sm4.c index 70b4901c..482a7983 100644 --- a/cipher/sm4.c +++ b/cipher/sm4.c @@ -995,7 +995,7 @@ sm4_setkey (void *context, const byte *key, const unsigned keylen, memset (bulk_ops, 0, sizeof(*bulk_ops)); bulk_ops->cbc_dec = _gcry_sm4_cbc_dec; bulk_ops->cfb_dec = _gcry_sm4_cfb_dec; - bulk_ops->ctr_enc = _gcry_sm4_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_sm4_ctr_enc; bulk_ops->xts_crypt = _gcry_sm4_xts_crypt; bulk_ops->ecb_crypt = _gcry_sm4_ecb_crypt; bulk_ops->ctr32le_enc = _gcry_sm4_ctr32le_enc; -- 2.53.0 From jussi.kivilinna at iki.fi Fri Jul 24 20:50:09 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:09 +0300 Subject: [PATCH 02/10] aria: reduce CTR bulk counter carry handling to 16 bits In-Reply-To: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> References: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> Message-ID: <20260724185017.3847717-2-jussi.kivilinna@iki.fi> * cipher/aria.c (aria_setkey): Assign ctr16be_enc bulk op. * cipher/aria-aesni-avx-amd64.S (inc_le128): Remove. (__aria_aesni_avx_ctr_gen_keystream_16way): Add to low 16 counter bits only, drop full-width overflow path. * cipher/aria-aesni-avx2-amd64.S: Likewise. * cipher/aria-gfni-avx512-amd64.S: Likewise. -- ARIA CTR keystream generators did full 128-bit little-endian counter increment with separate 64-bit overflow path. Under ctr16be_enc contract generic ctr code splits work at low 16-bit overflow, so bulk function adds only to low 16 counter bits. Drop inc_le128 and overflow path, use 16-bit stepping. Moves ARIA off transitional ctr_enc alias. Signed-off-by: Jussi Kivilinna --- cipher/aria-aesni-avx-amd64.S | 86 ++++++---------- cipher/aria-aesni-avx2-amd64.S | 170 ++++++-------------------------- cipher/aria-gfni-avx512-amd64.S | 107 +++++--------------- cipher/aria.c | 2 +- 4 files changed, 86 insertions(+), 279 deletions(-) diff --git a/cipher/aria-aesni-avx-amd64.S b/cipher/aria-aesni-avx-amd64.S index 29597488..b177ca5e 100644 --- a/cipher/aria-aesni-avx-amd64.S +++ b/cipher/aria-aesni-avx-amd64.S @@ -66,12 +66,6 @@ ((l0) << (7 * 8)) ) /* asm macros */ -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - #define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \ vpand x, mask4bit, tmp0; \ vpandn x, mask4bit, x; \ @@ -1097,82 +1091,52 @@ __aria_aesni_avx_ctr_gen_keystream_16way: vpshufb %xmm1, %xmm8, %xmm3; /* be => le */ vpcmpeqd %xmm0, %xmm0, %xmm0; - vpsrldq $8, %xmm0, %xmm0; /* low: -1, high: 0 */ + vpsrldq $14, %xmm0, %xmm0; /* low: -1, high: 0 */ /* construct IVs */ - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm9; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ - vpshufb %xmm1, %xmm3, %xmm10; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ + vpshufb %xmm1, %xmm3, %xmm2; + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm11; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ - vpshufb %xmm1, %xmm3, %xmm12; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ - vpshufb %xmm1, %xmm3, %xmm13; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ - vpshufb %xmm1, %xmm3, %xmm14; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ - vpshufb %xmm1, %xmm3, %xmm15; + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ + vpshufb %xmm1, %xmm3, %xmm4; + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ + vpshufb %xmm1, %xmm3, %xmm5; + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ + vpshufb %xmm1, %xmm3, %xmm6; + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ + vpshufb %xmm1, %xmm3, %xmm7; vmovdqu %xmm8, (0 * 16)(%rcx); vmovdqu %xmm9, (1 * 16)(%rcx); - vmovdqu %xmm10, (2 * 16)(%rcx); vmovdqu %xmm11, (3 * 16)(%rcx); - vmovdqu %xmm12, (4 * 16)(%rcx); - vmovdqu %xmm13, (5 * 16)(%rcx); - vmovdqu %xmm14, (6 * 16)(%rcx); - vmovdqu %xmm15, (7 * 16)(%rcx); - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm8; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm9; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm10; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm11; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm12; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm13; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm14; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ + vpsubw %xmm0, %xmm3, %xmm3; /* +1 */ vpshufb %xmm1, %xmm3, %xmm15; - inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */ - vpshufb %xmm1, %xmm3, %xmm4; - vmovdqu %xmm4, (%r8); vmovdqu (0 * 16)(%rcx), %xmm0; vmovdqu (1 * 16)(%rcx), %xmm1; - vmovdqu (2 * 16)(%rcx), %xmm2; vmovdqu (3 * 16)(%rcx), %xmm3; - vmovdqu (4 * 16)(%rcx), %xmm4; - vmovdqu (5 * 16)(%rcx), %xmm5; - vmovdqu (6 * 16)(%rcx), %xmm6; - vmovdqu (7 * 16)(%rcx), %xmm7; - ret_spec_stop; + jmp .Lctr_gen_keystream_done; .align 8 -.Lctr_byteadd_full_ctr_carry: - addb $16, 15(%r8); - pushq %rcx; - CFI_ADJUST_CFA_OFFSET(8); - movl $14, %ecx; - 1: - adcb $0, (%r8, %rcx); - jnc 2f; - loop 1b; - 2: - popq %rcx; - CFI_ADJUST_CFA_OFFSET(-8); - jmp .Lctr_byteadd_xmm; -.align 8 .Lctr_byteadd: - je .Lctr_byteadd_full_ctr_carry; - addb $16, 15(%r8); -.Lctr_byteadd_xmm: vmovdqa %xmm8, %xmm0; vpaddb .Lbige_addb_1 rRIP, %xmm8, %xmm1; vpaddb .Lbige_addb_2 rRIP, %xmm8, %xmm2; @@ -1190,6 +1154,12 @@ __aria_aesni_avx_ctr_gen_keystream_16way: vpaddb .Lbige_addb_14 rRIP, %xmm0, %xmm14; vpaddb .Lbige_addb_15 rRIP, %xmm0, %xmm15; +.align 16 +.Lctr_gen_keystream_done: + /* Update IV */ + addb $16, 15(%r8); + adcb $0, 14(%r8); + ret_spec_stop; CFI_ENDPROC(); ELF(.size __aria_aesni_avx_ctr_gen_keystream_16way,.-__aria_aesni_avx_ctr_gen_keystream_16way;) diff --git a/cipher/aria-aesni-avx2-amd64.S b/cipher/aria-aesni-avx2-amd64.S index 08bf5a0e..99eca542 100644 --- a/cipher/aria-aesni-avx2-amd64.S +++ b/cipher/aria-aesni-avx2-amd64.S @@ -86,12 +86,6 @@ ((l0) << (7 * 8)) ) /* asm macros */ -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - #define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \ vpand x, mask4bit, tmp0; \ vpandn x, mask4bit, x; \ @@ -1246,171 +1240,63 @@ __aria_aesni_avx2_ctr_gen_keystream_32way: cmpb $(0x100 - 32), 15(%r8); jbe .Lctr_byteadd; - movq 8(%r8), %r11; - bswapq %r11; - vbroadcasti128 .Lbswap128_mask rRIP, %ymm6; vpcmpeqd %ymm0, %ymm0, %ymm0; - vpsrldq $8, %ymm0, %ymm0; /* ab: -1:0 ; cd: -1:0 */ - vpaddq %ymm0, %ymm0, %ymm5; /* ab: -2:0 ; cd: -2:0 */ + vpsrldq $14, %ymm0, %ymm0; /* ab: -1:0 ; cd: -1:0 */ + vpaddw %ymm0, %ymm0, %ymm5; /* ab: -2:0 ; cd: -2:0 */ - /* load IV and byteswap */ + /* load IV and byteswap. */ vmovdqu (%r8), %xmm7; vpshufb %xmm6, %xmm7, %xmm7; vmovdqa %xmm7, %xmm3; - inc_le128(%xmm7, %xmm0, %xmm4); + vpsubw %xmm0, %xmm7, %xmm7; vinserti128 $1, %xmm7, %ymm3, %ymm3; - vpshufb %ymm6, %ymm3, %ymm8; /* +1 ; +0 */ - - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 32), %r11; - ja .Lhandle_ctr_carry; + vpshufb %ymm6, %ymm3, %ymm0; /* +1 ; +0 */ /* construct IVs */ - vpsubq %ymm5, %ymm3, %ymm3; /* +3 ; +2 */ - vpshufb %ymm6, %ymm3, %ymm9; - vpsubq %ymm5, %ymm3, %ymm3; /* +5 ; +4 */ - vpshufb %ymm6, %ymm3, %ymm10; - vpsubq %ymm5, %ymm3, %ymm3; /* +7 ; +6 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +3 ; +2 */ + vpshufb %ymm6, %ymm3, %ymm1; + vpsubw %ymm5, %ymm3, %ymm3; /* +5 ; +4 */ + vpshufb %ymm6, %ymm3, %ymm2; + vpsubw %ymm5, %ymm3, %ymm3; /* +7 ; +6 */ vpshufb %ymm6, %ymm3, %ymm11; - vpsubq %ymm5, %ymm3, %ymm3; /* +9 ; +8 */ - vpshufb %ymm6, %ymm3, %ymm12; - vpsubq %ymm5, %ymm3, %ymm3; /* +11 ; +10 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +9 ; +8 */ + vpshufb %ymm6, %ymm3, %ymm4; + vpsubw %ymm5, %ymm3, %ymm3; /* +11 ; +10 */ vpshufb %ymm6, %ymm3, %ymm13; - vpsubq %ymm5, %ymm3, %ymm3; /* +13 ; +12 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +13 ; +12 */ vpshufb %ymm6, %ymm3, %ymm14; - vpsubq %ymm5, %ymm3, %ymm3; /* +15 ; +14 */ - vpshufb %ymm6, %ymm3, %ymm15; - vmovdqu %ymm8, (0 * 32)(%rcx); - vmovdqu %ymm9, (1 * 32)(%rcx); - vmovdqu %ymm10, (2 * 32)(%rcx); + vpsubw %ymm5, %ymm3, %ymm3; /* +15 ; +14 */ + vpshufb %ymm6, %ymm3, %ymm7; vmovdqu %ymm11, (3 * 32)(%rcx); - vmovdqu %ymm12, (4 * 32)(%rcx); vmovdqu %ymm13, (5 * 32)(%rcx); vmovdqu %ymm14, (6 * 32)(%rcx); - vmovdqu %ymm15, (7 * 32)(%rcx); - vpsubq %ymm5, %ymm3, %ymm3; /* +17 ; +16 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +17 ; +16 */ vpshufb %ymm6, %ymm3, %ymm8; - vpsubq %ymm5, %ymm3, %ymm3; /* +19 ; +18 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +19 ; +18 */ vpshufb %ymm6, %ymm3, %ymm9; - vpsubq %ymm5, %ymm3, %ymm3; /* +21 ; +20 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +21 ; +20 */ vpshufb %ymm6, %ymm3, %ymm10; - vpsubq %ymm5, %ymm3, %ymm3; /* +23 ; +22 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +23 ; +22 */ vpshufb %ymm6, %ymm3, %ymm11; - vpsubq %ymm5, %ymm3, %ymm3; /* +25 ; +24 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +25 ; +24 */ vpshufb %ymm6, %ymm3, %ymm12; - vpsubq %ymm5, %ymm3, %ymm3; /* +27 ; +26 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +27 ; +26 */ vpshufb %ymm6, %ymm3, %ymm13; - vpsubq %ymm5, %ymm3, %ymm3; /* +29 ; +28 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +29 ; +28 */ vpshufb %ymm6, %ymm3, %ymm14; - vpsubq %ymm5, %ymm3, %ymm3; /* +31 ; +30 */ + vpsubw %ymm5, %ymm3, %ymm3; /* +31 ; +30 */ vpshufb %ymm6, %ymm3, %ymm15; - vpsubq %ymm5, %ymm3, %ymm3; /* +32 */ - vpshufb %xmm6, %xmm3, %xmm3; - vmovdqu %xmm3, (%r8); - vmovdqu (0 * 32)(%rcx), %ymm0; - vmovdqu (1 * 32)(%rcx), %ymm1; - vmovdqu (2 * 32)(%rcx), %ymm2; - vmovdqu (3 * 32)(%rcx), %ymm3; - vmovdqu (4 * 32)(%rcx), %ymm4; - vmovdqu (5 * 32)(%rcx), %ymm5; - vmovdqu (6 * 32)(%rcx), %ymm6; - vmovdqu (7 * 32)(%rcx), %ymm7; - jmp .Lctr_carry_done; - - .Lhandle_ctr_carry: - /* construct IVs */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm9; /* +3 ; +2 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm10; /* +5 ; +4 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm11; /* +7 ; +6 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm12; /* +9 ; +8 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm13; /* +11 ; +10 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm14; /* +13 ; +12 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm15; /* +15 ; +14 */ - vmovdqu %ymm8, (0 * 32)(%rcx); - vmovdqu %ymm9, (1 * 32)(%rcx); - vmovdqu %ymm10, (2 * 32)(%rcx); - vmovdqu %ymm11, (3 * 32)(%rcx); - vmovdqu %ymm12, (4 * 32)(%rcx); - vmovdqu %ymm13, (5 * 32)(%rcx); - vmovdqu %ymm14, (6 * 32)(%rcx); - vmovdqu %ymm15, (7 * 32)(%rcx); - - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm8; /* +17 ; +16 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm9; /* +19 ; +18 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm10; /* +21 ; +20 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm11; /* +23 ; +22 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm12; /* +25 ; +24 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm13; /* +27 ; +26 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm14; /* +29 ; +28 */ - inc_le128(%ymm3, %ymm0, %ymm4); - inc_le128(%ymm3, %ymm0, %ymm4); - vpshufb %ymm6, %ymm3, %ymm15; /* +31 ; +30 */ - inc_le128(%ymm3, %ymm0, %ymm4); - vextracti128 $1, %ymm3, %xmm3; - vpshufb %xmm6, %xmm3, %xmm3; /* +32 */ - vmovdqu %xmm3, (%r8); - vmovdqu (0 * 32)(%rcx), %ymm0; - vmovdqu (1 * 32)(%rcx), %ymm1; - vmovdqu (2 * 32)(%rcx), %ymm2; vmovdqu (3 * 32)(%rcx), %ymm3; - vmovdqu (4 * 32)(%rcx), %ymm4; vmovdqu (5 * 32)(%rcx), %ymm5; vmovdqu (6 * 32)(%rcx), %ymm6; - vmovdqu (7 * 32)(%rcx), %ymm7; -.Lctr_carry_done: - ret_spec_stop; + jmp .Lctr_gen_keystream_done; .align 8 -.Lctr_byteadd_full_ctr_carry: - addb $32, 15(%r8); - pushq %rcx; - CFI_ADJUST_CFA_OFFSET(8); - movl $14, %ecx; - 1: - adcb $0, (%r8, %rcx); - jnc 2f; - loop 1b; - 2: - popq %rcx; - CFI_ADJUST_CFA_OFFSET(-8); - jmp .Lctr_byteadd_ymm; -.align 8 .Lctr_byteadd: vbroadcasti128 (%r8), %ymm8; - je .Lctr_byteadd_full_ctr_carry; - addb $32, 15(%r8); -.Lctr_byteadd_ymm: vpaddb .Lbige_addb_16_16 rRIP, %ymm8, %ymm15; vpaddb .Lbige_addb_0_1 rRIP, %ymm8, %ymm0; vpaddb .Lbige_addb_2_3 rRIP, %ymm8, %ymm1; @@ -1429,6 +1315,12 @@ __aria_aesni_avx2_ctr_gen_keystream_32way: vpaddb .Lbige_addb_12_13 rRIP, %ymm15, %ymm14; vpaddb .Lbige_addb_14_15 rRIP, %ymm15, %ymm15; +.align 16 +.Lctr_gen_keystream_done: + /* Update IV */ + addb $32, 15(%r8); + adcb $0, 14(%r8); + ret_spec_stop; CFI_ENDPROC(); ELF(.size __aria_aesni_avx2_ctr_gen_keystream_32way, diff --git a/cipher/aria-gfni-avx512-amd64.S b/cipher/aria-gfni-avx512-amd64.S index a497b395..c25caf86 100644 --- a/cipher/aria-gfni-avx512-amd64.S +++ b/cipher/aria-gfni-avx512-amd64.S @@ -75,16 +75,9 @@ clear_vec4(%ymm19, %ymm23, %ymm27, %ymm31) #define clear_regs() \ - kxorq %k1, %k1, %k1; \ vzeroall; \ clear_zmm16_zmm31() -#define add_le128(out, in, lo_counter, hi_counter1) \ - vpaddq lo_counter, in, out; \ - vpcmpuq $1, lo_counter, out, %k1; \ - kaddb %k1, %k1, %k1; \ - vpaddq hi_counter1, out, out{%k1}; - #define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \ vpandq x, mask4bit, tmp0; \ vpandqn x, mask4bit, x; \ @@ -583,8 +576,6 @@ SECTION_RODATA .quad 8, 0 .Lcounter16161616_lo: .quad 16, 0 -.Lcounter1111_hi: - .quad 0, 1 /* For CTR-mode IV byteswap */ .Lbswap128_mask: @@ -815,62 +806,30 @@ __aria_gfni_avx512_ctr_gen_keystream_64way: vbroadcasti64x2 .Lcounter4444_lo rRIP, %zmm22; vbroadcasti64x2 .Lcounter8888_lo rRIP, %zmm23; vbroadcasti64x2 .Lcounter16161616_lo rRIP, %zmm24; - vbroadcasti64x2 .Lcounter1111_hi rRIP, %zmm25; - /* load IV and byteswap */ - movq 8(%r8), %r11; - movq (%r8), %r10; - bswapq %r11; - bswapq %r10; + /* load IV and byteswap. */ vbroadcasti64x2 (%r8), %zmm20; vpshufb %zmm19, %zmm20, %zmm20; - /* check need for handling 64-bit overflow and carry */ - cmpq $(0xffffffffffffffff - 64), %r11; - ja .Lload_ctr_carry; - - /* construct IVs */ - vpaddq %zmm21, %zmm20, %zmm0; /* +0:+1:+2:+3 */ - vpaddq %zmm22, %zmm0, %zmm1; /* +4:+5:+6:+7 */ - vpaddq %zmm23, %zmm0, %zmm2; /* +8:+9:+10:+11 */ - vpaddq %zmm23, %zmm1, %zmm3; /* +12:+13:+14:+15 */ - vpaddq %zmm24, %zmm0, %zmm4; /* +16... */ - vpaddq %zmm24, %zmm1, %zmm5; /* +20... */ - vpaddq %zmm24, %zmm2, %zmm6; /* +24... */ - vpaddq %zmm24, %zmm3, %zmm7; /* +28... */ - vpaddq %zmm24, %zmm4, %zmm8; /* +32... */ - vpaddq %zmm24, %zmm5, %zmm9; /* +36... */ - vpaddq %zmm24, %zmm6, %zmm10; /* +40... */ - vpaddq %zmm24, %zmm7, %zmm11; /* +44... */ - vpaddq %zmm24, %zmm8, %zmm12; /* +48... */ - vpaddq %zmm24, %zmm9, %zmm13; /* +52... */ - vpaddq %zmm24, %zmm10, %zmm14; /* +56... */ - vpaddq %zmm24, %zmm11, %zmm15; /* +60... */ - jmp .Lload_ctr_done; - -.Lload_ctr_carry: /* construct IVs */ - add_le128(%zmm0, %zmm20, %zmm21, %zmm25); /* +0:+1:+2:+3 */ - add_le128(%zmm1, %zmm0, %zmm22, %zmm25); /* +4:+5:+6:+7 */ - add_le128(%zmm2, %zmm0, %zmm23, %zmm25); /* +8:+9:+10:+11 */ - add_le128(%zmm3, %zmm1, %zmm23, %zmm25); /* +12:+13:+14:+15 */ - add_le128(%zmm4, %zmm0, %zmm24, %zmm25); /* +16... */ - add_le128(%zmm5, %zmm1, %zmm24, %zmm25); /* +20... */ - add_le128(%zmm6, %zmm2, %zmm24, %zmm25); /* +24... */ - add_le128(%zmm7, %zmm3, %zmm24, %zmm25); /* +28... */ - add_le128(%zmm8, %zmm4, %zmm24, %zmm25); /* +32... */ - add_le128(%zmm9, %zmm5, %zmm24, %zmm25); /* +36... */ - add_le128(%zmm10, %zmm6, %zmm24, %zmm25); /* +40... */ - add_le128(%zmm11, %zmm7, %zmm24, %zmm25); /* +44... */ - add_le128(%zmm12, %zmm8, %zmm24, %zmm25); /* +48... */ - add_le128(%zmm13, %zmm9, %zmm24, %zmm25); /* +52... */ - add_le128(%zmm14, %zmm10, %zmm24, %zmm25); /* +56... */ - add_le128(%zmm15, %zmm11, %zmm24, %zmm25); /* +60... */ - -.Lload_ctr_done: - /* Byte-swap IVs and update counter. */ - addq $64, %r11; - adcq $0, %r10; + vpaddw %zmm21, %zmm20, %zmm0; /* +0:+1:+2:+3 */ + vpaddw %zmm22, %zmm0, %zmm1; /* +4:+5:+6:+7 */ + vpaddw %zmm23, %zmm0, %zmm2; /* +8:+9:+10:+11 */ + vpaddw %zmm23, %zmm1, %zmm3; /* +12:+13:+14:+15 */ + vpaddw %zmm24, %zmm0, %zmm4; /* +16... */ + vpaddw %zmm24, %zmm1, %zmm5; /* +20... */ + vpaddw %zmm24, %zmm2, %zmm6; /* +24... */ + vpaddw %zmm24, %zmm3, %zmm7; /* +28... */ + vpaddw %zmm24, %zmm4, %zmm8; /* +32... */ + vpaddw %zmm24, %zmm5, %zmm9; /* +36... */ + vpaddw %zmm24, %zmm6, %zmm10; /* +40... */ + vpaddw %zmm24, %zmm7, %zmm11; /* +44... */ + vpaddw %zmm24, %zmm8, %zmm12; /* +48... */ + vpaddw %zmm24, %zmm9, %zmm13; /* +52... */ + vpaddw %zmm24, %zmm10, %zmm14; /* +56... */ + vpaddw %zmm24, %zmm11, %zmm15; /* +60... */ + + /* Byte-swap IVs. */ vpshufb %zmm19, %zmm15, %zmm15; vpshufb %zmm19, %zmm14, %zmm14; vpshufb %zmm19, %zmm13, %zmm13; @@ -879,8 +838,6 @@ __aria_gfni_avx512_ctr_gen_keystream_64way: vpshufb %zmm19, %zmm10, %zmm10; vpshufb %zmm19, %zmm9, %zmm9; vpshufb %zmm19, %zmm8, %zmm8; - bswapq %r11; - bswapq %r10; vpshufb %zmm19, %zmm7, %zmm7; vpshufb %zmm19, %zmm6, %zmm6; vpshufb %zmm19, %zmm5, %zmm5; @@ -889,30 +846,12 @@ __aria_gfni_avx512_ctr_gen_keystream_64way: vpshufb %zmm19, %zmm2, %zmm2; vpshufb %zmm19, %zmm1, %zmm1; vpshufb %zmm19, %zmm0, %zmm0; - movq %r11, 8(%r8); - movq %r10, (%r8); - ret_spec_stop; + jmp .Lctr_gen_keystream_done; .align 16 -.Lctr_byteadd_full_ctr_carry: - movq 8(%r8), %r11; - movq (%r8), %r10; - bswapq %r11; - bswapq %r10; - addq $64, %r11; - adcq $0, %r10; - bswapq %r11; - bswapq %r10; - movq %r11, 8(%r8); - movq %r10, (%r8); - jmp .Lctr_byteadd_zmm; -.align 16 .Lctr_byteadd: vbroadcasti64x2 (%r8), %zmm3; - je .Lctr_byteadd_full_ctr_carry; - addb $64, 15(%r8); -.Lctr_byteadd_zmm: vbroadcasti64x2 .Lbige_addb_16 rRIP, %zmm16; vmovdqa64 .Lbige_addb_0_1 rRIP, %zmm17; vmovdqa64 .Lbige_addb_4_5 rRIP, %zmm18; @@ -938,6 +877,12 @@ __aria_gfni_avx512_ctr_gen_keystream_64way: vpaddb %zmm19, %zmm15, %zmm14; vpaddb %zmm20, %zmm15, %zmm15; +.align 16 +.Lctr_gen_keystream_done: + /* Update IV */ + addb $64, 15(%r8); + adcb $0, 14(%r8); + ret_spec_stop; CFI_ENDPROC(); ELF(.size __aria_gfni_avx512_ctr_gen_keystream_64way, diff --git a/cipher/aria.c b/cipher/aria.c index bb67ed03..227affb4 100644 --- a/cipher/aria.c +++ b/cipher/aria.c @@ -1656,7 +1656,7 @@ aria_setkey(void *c, const byte *key, unsigned keylen, bulk_ops->cbc_dec = _gcry_aria_cbc_dec; bulk_ops->cfb_enc = _gcry_aria_cfb_enc; bulk_ops->cfb_dec = _gcry_aria_cfb_dec; - bulk_ops->ctr_enc = _gcry_aria_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aria_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aria_ctr32le_enc; bulk_ops->ecb_crypt = _gcry_aria_ecb_crypt; bulk_ops->xts_crypt = _gcry_aria_xts_crypt; -- 2.53.0 From davegalos at google.com Fri Jul 24 19:14:36 2026 From: davegalos at google.com (David Galos) Date: Fri, 24 Jul 2026 13:14:36 -0400 Subject: [PATCH 0/2] Changes to rsa_decrypt behavior on bad keys In-Reply-To: <87cxwm7zv9.fsf@haruna.fsij.org> References: <20260709231752.1332171-1-davegalos@google.com> <87cxwm7zv9.fsf@haruna.fsij.org> Message-ID: Thank you for your response on this. I apologise for my somewhat delayed message. It was also to the wrong mailing list and I wanted to avoid double-posting. I can see that what I "figured out" was actually realized weeks ago by another user of the ProtonMail crypto library, and it's very heartening to see that changes have been made in the right direction. I believe their changes to that go library are incomplete, because I think the library will still p-q flip when it's reading a key. That said, gcrypt-wise I don't think theres anything I need to push for. From jussi.kivilinna at iki.fi Fri Jul 24 20:50:08 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Fri, 24 Jul 2026 21:50:08 +0300 Subject: [PATCH 01/10] cipher: reduce CTR bulk counter carry handling to 16 bits Message-ID: <20260724185017.3847717-1-jussi.kivilinna@iki.fi> * cipher/cipher-internal.h (cipher_bulk_ops_t): Rename 'ctr_enc' member to 'ctr16be_enc'. (ctr_enc): New transitional alias macro. (cipher_block_add): Widen 'add' to u64. (cipher_block_add_be16): New. * cipher/cipher-ctr.c (_gcry_cipher_ctr_encrypt_ctx): Split bulk work at low 16-bit counter overflow and do full-width carry addition here. * cipher/bulkhelp.h (bulk_ctr_enc_128): Use cipher_block_add_be16. * cipher/rijndael.c (do_setkey): Assign ctr16be_enc bulk op. (_gcry_aes_ctr_enc): Use cipher_block_add_be16. * cipher/rijndael-aesni.c (do_aesni_ctr, do_aesni_ctr_4) (do_aesni_ctr_8): Drop full-width counter carry handling. * cipher/rijndael-ssse3-amd64.c (_gcry_aes_ssse3_ctr_enc): Likewise. * cipher/rijndael-vp-simd128.h (FUNC_CTR_ENC): Likewise. * cipher/rijndael-armv8-aarch32-ce.S (_gcry_aes_ctr_enc_armv8_ce): Likewise. * cipher/rijndael-armv8-aarch64-ce.S (_gcry_aes_ctr_enc_armv8_ce): Likewise. * cipher/rijndael-ppc-functions.h (CTR_ENC_FUNC): Likewise. * cipher/rijndael-riscv-zvkned.c (_gcry_aes_riscv_zvkned_ctr_enc): Likewise. * cipher/rijndael-vaes-avx2-amd64.S (_gcry_vaes_avx2_ctr_enc_amd64): Likewise. * cipher/rijndael-vaes-avx2-i386.S (_gcry_vaes_avx2_ctr_enc_i386): Likewise. * cipher/rijndael-vaes-avx512-amd64.S (_gcry_vaes_avx512_ctr_enc_amd64): Likewise. * tests/basic.c (cipher_cbc_bulk_test, cipher_cfb_bulk_test) (cipher_ctr_bulk_test): Add verbose output. (cipher_ctr16_overflow_test, check_ctr16_overflow): New. (check_cipher_modes): Call check_ctr16_overflow. -- Every SIMD CTR implementation carried code to propagate big-endian counter carry across bytes, up to full 128 bits. Move carry out of assembly: bulk functions now add only to low 16 counter bits, and generic ctr code splits work at each 16-bit overflow to do full-width carry addition in C. Low 16-bit overflow never happens inside bulk call, so per-implementation carry handling drops to low-halfword increment. ctr_enc bulk op renamed to ctr16be_enc. Temporary ctr_enc alias keeps not-yet-converted ciphers building. Rijndael converted as example. New test drives single CTR call over 0x10000+ blocks across several start counters, checking keystream and counter state against ECB reference. Signed-off-by: Jussi Kivilinna --- cipher/bulkhelp.h | 4 +- cipher/cipher-ctr.c | 50 ++++++++- cipher/cipher-internal.h | 26 ++++- cipher/rijndael-aesni.c | 123 ++++----------------- cipher/rijndael-armv8-aarch32-ce.S | 104 +++++------------- cipher/rijndael-armv8-aarch64-ce.S | 82 +++++++------- cipher/rijndael-ppc-common.h | 10 ++ cipher/rijndael-ppc-functions.h | 33 +++--- cipher/rijndael-riscv-zvkned.c | 103 +++++++----------- cipher/rijndael-ssse3-amd64.c | 13 +-- cipher/rijndael-vaes-avx2-amd64.S | 130 ++++++++-------------- cipher/rijndael-vaes-avx2-i386.S | 161 +++++++++------------------- cipher/rijndael-vaes-avx512-amd64.S | 89 +++++---------- cipher/rijndael-vp-simd128.h | 56 ++-------- cipher/rijndael.c | 24 ++--- tests/basic.c | 154 ++++++++++++++++++++++++++ 16 files changed, 524 insertions(+), 638 deletions(-) diff --git a/cipher/bulkhelp.h b/cipher/bulkhelp.h index 833262e2..1834a9fb 100644 --- a/cipher/bulkhelp.h +++ b/cipher/bulkhelp.h @@ -155,9 +155,9 @@ bulk_ctr_enc_128 (void *priv, bulk_crypt_fn_t crypt_fn, byte *outbuf, for (i = 1; i < curr_blks; i++) { cipher_block_cpy (&tmpbuf[i * 16], ctr, 16); - cipher_block_add (&tmpbuf[i * 16], i, 16); + cipher_block_add_be16 (&tmpbuf[i * 16], i, 16); } - cipher_block_add (ctr, curr_blks, 16); + cipher_block_add_be16 (ctr, curr_blks, 16); nburn = crypt_fn (priv, tmpbuf, tmpbuf, curr_blks); burn_depth = nburn > burn_depth ? nburn : burn_depth; diff --git a/cipher/cipher-ctr.c b/cipher/cipher-ctr.c index 98363334..1ddaa7e6 100644 --- a/cipher/cipher-ctr.c +++ b/cipher/cipher-ctr.c @@ -64,12 +64,52 @@ _gcry_cipher_ctr_encrypt_ctx (gcry_cipher_hd_t c, /* Use a bulk method if available. */ nblocks = inbuflen >> blocksize_shift; - if (nblocks && c->bulk.ctr_enc) + if (nblocks && c->bulk.ctr16be_enc) { - c->bulk.ctr_enc (algo_context, c->u_ctr.ctr, outbuf, inbuf, nblocks); - inbuf += nblocks << blocksize_shift; - outbuf += nblocks << blocksize_shift; - inbuflen -= nblocks << blocksize_shift; + byte ctr_copy[MAX_BLOCKSIZE]; + + do + { + /* Bulk CTR function only handles 16-bit big-endian addition of + * counter as optimization. Let bulk function to process only up to + * next 16-bit overflow. */ + u32 ctr_low32 = buf_get_be32(&c->u_ctr.ctr[blocksize - sizeof(u32)]); + unsigned int ctr_low16 = ctr_low32 & 0xffffU; + size_t blks_to_overflow = (size_t)0x10000U - ctr_low16; + size_t curr_blks = nblocks; + int ctr_overflows = 0; + + if (nblocks >= blks_to_overflow) + { + curr_blks = blks_to_overflow; + + ctr_overflows = 1; + cipher_block_cpy(ctr_copy, c->u_ctr.ctr, blocksize); + } + + c->bulk.ctr16be_enc (algo_context, c->u_ctr.ctr, outbuf, inbuf, + curr_blks); + + if (ctr_overflows) + { + /* Lower 16-bits of CTR should now be zero. */ + ctr_low32 = buf_get_be32(&c->u_ctr.ctr[blocksize - sizeof(u32)]); + ctr_low16 = ctr_low32 & 0xffffU; + gcry_assert(ctr_low16 == 0); + + /* Handle full blocksize addition. */ + cipher_block_cpy(c->u_ctr.ctr, ctr_copy, blocksize); + cipher_block_add(c->u_ctr.ctr, curr_blks, blocksize); + + wipememory(ctr_copy, sizeof(ctr_copy)); + } + + inbuf += curr_blks << blocksize_shift; + outbuf += curr_blks << blocksize_shift; + inbuflen -= curr_blks << blocksize_shift; + nblocks -= curr_blks; + } + while (nblocks); } /* If we don't have a bulk method use the standard method. We also diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h index bc023c76..c99bd189 100644 --- a/cipher/cipher-internal.h +++ b/cipher/cipher-internal.h @@ -196,8 +196,8 @@ typedef struct cipher_bulk_ops const void *inbuf_arg, size_t nblocks); void (*ofb_enc)(void *context, unsigned char *iv, void *outbuf_arg, const void *inbuf_arg, size_t nblocks); - void (*ctr_enc)(void *context, unsigned char *iv, void *outbuf_arg, - const void *inbuf_arg, size_t nblocks); + void (*ctr16be_enc)(void *context, unsigned char *iv, void *outbuf_arg, + const void *inbuf_arg, size_t nblocks); void (*ctr32le_enc)(void *context, unsigned char *iv, void *outbuf_arg, const void *inbuf_arg, size_t nblocks); size_t (*ocb_crypt)(gcry_cipher_hd_t c, void *outbuf_arg, @@ -209,6 +209,11 @@ typedef struct cipher_bulk_ops const void *inbuf_arg, size_t nblocks, int encrypt); } cipher_bulk_ops_t; +/* Temporary alias for transition period from full 128-bit big-endian + * counter addition in bulk processing function to 16-bit big-endian + * addition. */ +#define ctr_enc ctr16be_enc + /* A VIA processor with the Padlock engine as well as the Intel AES_NI instructions require an alignment of most data on a 16 byte @@ -820,9 +825,9 @@ cipher_bytecounter_add (u32 ctr[2], size_t add) } -/* Optimized function for adding value to cipher block. */ +/* Optimized function for block-size big-endian addition. */ static inline void -cipher_block_add(void *_dstsrc, unsigned int add, size_t blocksize) +cipher_block_add(void *_dstsrc, u64 add, size_t blocksize) { byte *dstsrc = _dstsrc; u64 s[2]; @@ -843,6 +848,19 @@ cipher_block_add(void *_dstsrc, unsigned int add, size_t blocksize) } +/* Optimized function for 16-bit big-endian addition to cipher block. */ +static inline void +cipher_block_add_be16(void *_dstsrc, unsigned int add, size_t blocksize) +{ + byte *dstsrc = _dstsrc; + byte add_lo = add & 0xff; + byte add_hi = add >> 8; + + dstsrc[blocksize - 1] += add_lo; + dstsrc[blocksize - 2] += add_hi + (dstsrc[blocksize - 1] < add_lo); +} + + /* Optimized function for cipher block copying */ static inline void cipher_block_cpy(void *_dst, const void *_src, size_t blocksize) diff --git a/cipher/rijndael-aesni.c b/cipher/rijndael-aesni.c index dea78fd0..0093aaec 100644 --- a/cipher/rijndael-aesni.c +++ b/cipher/rijndael-aesni.c @@ -1073,26 +1073,10 @@ do_aesni_ctr (const RIJNDAEL_context *ctx, #define aesenc_xmm1_xmm0 ".byte 0x66, 0x0f, 0x38, 0xdc, 0xc1\n\t" #define aesenclast_xmm1_xmm0 ".byte 0x66, 0x0f, 0x38, 0xdd, 0xc1\n\t" - asm volatile ("movdqa %%xmm5, %%xmm0\n\t" /* xmm0 := CTR (xmm5) */ - "pcmpeqd %%xmm1, %%xmm1\n\t" - "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */ - - "pshufb %%xmm6, %%xmm5\n\t" - "psubq %%xmm1, %%xmm5\n\t" /* xmm5++ (big endian) */ - - /* detect if 64-bit carry handling is needed */ - "cmpl $0xffffffff, 8(%[ctr])\n\t" - "jne .Lno_carry%=\n\t" - "cmpl $0xffffffff, 12(%[ctr])\n\t" - "jne .Lno_carry%=\n\t" - - "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */ - "psubq %%xmm1, %%xmm5\n\t" /* add carry to upper 64bits */ + asm volatile ("movdqa (%[ctr]), %%xmm0\n\t" /* xmm0 := CTR (mem) */ - ".Lno_carry%=:\n\t" - - "pshufb %%xmm6, %%xmm5\n\t" - "movdqa %%xmm5, (%[ctr])\n\t" /* Update CTR (mem). */ + "addb $1, 15(%[ctr])\n\t" /* Update CTR (mem). */ + "adcb $0, 14(%[ctr])\n\t" "pxor (%[key]), %%xmm0\n\t" /* xmm1 ^= key[0] */ "movdqa 0x10(%[key]), %%xmm1\n\t" @@ -1194,45 +1178,21 @@ do_aesni_ctr_4 (const RIJNDAEL_context *ctx, "jmp .Ldone_ctr%=\n\t" ".Ladd32bit%=:\n\t" - "movdqa %%xmm5, (%[ctr])\n\t" /* Restore CTR. */ + "addb $1, 14(%[ctr])\n\t" /* Update CTR. */ "movdqa %%xmm5, %%xmm0\n\t" /* xmm0, xmm2 := CTR (xmm5) */ "movdqa %%xmm0, %%xmm2\n\t" "pcmpeqd %%xmm1, %%xmm1\n\t" - "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */ + "psrldq $14, %%xmm1\n\t" /* xmm1 = -1 */ "pshufb %%xmm6, %%xmm2\n\t" /* xmm2 := le(xmm2) */ - "psubq %%xmm1, %%xmm2\n\t" /* xmm2++ */ + "psubw %%xmm1, %%xmm2\n\t" /* xmm2++ */ "movdqa %%xmm2, %%xmm3\n\t" /* xmm3 := xmm2 */ - "psubq %%xmm1, %%xmm3\n\t" /* xmm3++ */ + "psubw %%xmm1, %%xmm3\n\t" /* xmm3++ */ "movdqa %%xmm3, %%xmm4\n\t" /* xmm4 := xmm3 */ - "psubq %%xmm1, %%xmm4\n\t" /* xmm4++ */ + "psubw %%xmm1, %%xmm4\n\t" /* xmm4++ */ "movdqa %%xmm4, %%xmm5\n\t" /* xmm5 := xmm4 */ - "psubq %%xmm1, %%xmm5\n\t" /* xmm5++ */ - - /* detect if 64-bit carry handling is needed */ - "cmpl $0xffffffff, 8(%[ctr])\n\t" - "jne .Lno_carry%=\n\t" - "movl 12(%[ctr]), %%esi\n\t" - "bswapl %%esi\n\t" - "cmpl $0xfffffffc, %%esi\n\t" - "jb .Lno_carry%=\n\t" /* no carry */ - - "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */ - "je .Lcarry_xmm5%=\n\t" /* esi == 0xfffffffc */ - "cmpl $0xfffffffe, %%esi\n\t" - "jb .Lcarry_xmm4%=\n\t" /* esi == 0xfffffffd */ - "je .Lcarry_xmm3%=\n\t" /* esi == 0xfffffffe */ - /* esi == 0xffffffff */ - - "psubq %%xmm1, %%xmm2\n\t" - ".Lcarry_xmm3%=:\n\t" - "psubq %%xmm1, %%xmm3\n\t" - ".Lcarry_xmm4%=:\n\t" - "psubq %%xmm1, %%xmm4\n\t" - ".Lcarry_xmm5%=:\n\t" - "psubq %%xmm1, %%xmm5\n\t" - - ".Lno_carry%=:\n\t" + "psubw %%xmm1, %%xmm5\n\t" /* xmm5++ */ + "movdqa (%[key]), %%xmm1\n\t" /* xmm1 := key[0] */ "pshufb %%xmm6, %%xmm2\n\t" /* xmm2 := be(xmm2) */ @@ -1240,8 +1200,6 @@ do_aesni_ctr_4 (const RIJNDAEL_context *ctx, "pshufb %%xmm6, %%xmm4\n\t" /* xmm4 := be(xmm4) */ "pshufb %%xmm6, %%xmm5\n\t" /* xmm5 := be(xmm5) */ - "movdqa %%xmm5, (%[ctr])\n\t" /* Update CTR (mem). */ - ".Ldone_ctr%=:\n\t" : : [ctr] "r" (ctr), @@ -1447,67 +1405,29 @@ do_aesni_ctr_8 (const RIJNDAEL_context *ctx, "jmp .Ldone_ctr%=\n\t" ".Ladd32bit%=:\n\t" - "movdqa %%xmm5, (%[ctr])\n\t" /* Restore CTR. */ + "addb $1, 14(%[ctr])\n\t" /* Update CTR. */ "movdqa %%xmm5, %%xmm0\n\t" /* xmm0, xmm2 := CTR (xmm5) */ "movdqa %%xmm0, %%xmm2\n\t" "pcmpeqd %%xmm1, %%xmm1\n\t" - "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */ + "psrldq $14, %%xmm1\n\t" /* xmm1 = -1 */ "pshufb %%xmm6, %%xmm2\n\t" /* xmm2 := le(xmm2) */ - "psubq %%xmm1, %%xmm2\n\t" /* xmm2++ */ + "psubw %%xmm1, %%xmm2\n\t" /* xmm2++ */ "movdqa %%xmm2, %%xmm3\n\t" /* xmm3 := xmm2 */ - "psubq %%xmm1, %%xmm3\n\t" /* xmm3++ */ + "psubw %%xmm1, %%xmm3\n\t" /* xmm3++ */ "movdqa %%xmm3, %%xmm4\n\t" /* xmm4 := xmm3 */ - "psubq %%xmm1, %%xmm4\n\t" /* xmm4++ */ + "psubw %%xmm1, %%xmm4\n\t" /* xmm4++ */ "movdqa %%xmm4, %%xmm8\n\t" /* xmm8 := xmm4 */ - "psubq %%xmm1, %%xmm8\n\t" /* xmm8++ */ + "psubw %%xmm1, %%xmm8\n\t" /* xmm8++ */ "movdqa %%xmm8, %%xmm9\n\t" /* xmm9 := xmm8 */ - "psubq %%xmm1, %%xmm9\n\t" /* xmm9++ */ + "psubw %%xmm1, %%xmm9\n\t" /* xmm9++ */ "movdqa %%xmm9, %%xmm10\n\t" /* xmm10 := xmm9 */ - "psubq %%xmm1, %%xmm10\n\t" /* xmm10++ */ + "psubw %%xmm1, %%xmm10\n\t" /* xmm10++ */ "movdqa %%xmm10, %%xmm11\n\t" /* xmm11 := xmm10 */ - "psubq %%xmm1, %%xmm11\n\t" /* xmm11++ */ + "psubw %%xmm1, %%xmm11\n\t" /* xmm11++ */ "movdqa %%xmm11, %%xmm5\n\t" /* xmm5 := xmm11 */ - "psubq %%xmm1, %%xmm5\n\t" /* xmm5++ */ - - /* detect if 64-bit carry handling is needed */ - "cmpl $0xffffffff, 8(%[ctr])\n\t" - "jne .Lno_carry%=\n\t" - "movl 12(%[ctr]), %%esi\n\t" - "bswapl %%esi\n\t" - "cmpl $0xfffffff8, %%esi\n\t" - "jb .Lno_carry%=\n\t" /* no carry */ - - "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */ - "je .Lcarry_xmm5%=\n\t" /* esi == 0xfffffff8 */ - "cmpl $0xfffffffa, %%esi\n\t" - "jb .Lcarry_xmm11%=\n\t" /* esi == 0xfffffff9 */ - "je .Lcarry_xmm10%=\n\t" /* esi == 0xfffffffa */ - "cmpl $0xfffffffc, %%esi\n\t" - "jb .Lcarry_xmm9%=\n\t" /* esi == 0xfffffffb */ - "je .Lcarry_xmm8%=\n\t" /* esi == 0xfffffffc */ - "cmpl $0xfffffffe, %%esi\n\t" - "jb .Lcarry_xmm4%=\n\t" /* esi == 0xfffffffd */ - "je .Lcarry_xmm3%=\n\t" /* esi == 0xfffffffe */ - /* esi == 0xffffffff */ - - "psubq %%xmm1, %%xmm2\n\t" - ".Lcarry_xmm3%=:\n\t" - "psubq %%xmm1, %%xmm3\n\t" - ".Lcarry_xmm4%=:\n\t" - "psubq %%xmm1, %%xmm4\n\t" - ".Lcarry_xmm8%=:\n\t" - "psubq %%xmm1, %%xmm8\n\t" - ".Lcarry_xmm9%=:\n\t" - "psubq %%xmm1, %%xmm9\n\t" - ".Lcarry_xmm10%=:\n\t" - "psubq %%xmm1, %%xmm10\n\t" - ".Lcarry_xmm11%=:\n\t" - "psubq %%xmm1, %%xmm11\n\t" - ".Lcarry_xmm5%=:\n\t" - "psubq %%xmm1, %%xmm5\n\t" - - ".Lno_carry%=:\n\t" + "psubw %%xmm1, %%xmm5\n\t" /* xmm5++ */ + "movdqa (%[key]), %%xmm1\n\t" /* xmm1 := key[0] */ "movdqa 16(%[key]), %%xmm7\n\t" /* xmm7 := key[1] */ @@ -1536,7 +1456,6 @@ do_aesni_ctr_8 (const RIJNDAEL_context *ctx, "aesenc %%xmm7, %%xmm11\n\t" "pshufb %%xmm6, %%xmm5\n\t" /* xmm5 := be(xmm5) */ - "movdqa %%xmm5, (%[ctr])\n\t" /* Update CTR (mem). */ ".align 16\n\t" ".Ldone_ctr%=:\n\t" diff --git a/cipher/rijndael-armv8-aarch32-ce.S b/cipher/rijndael-armv8-aarch32-ce.S index 7d85aff9..7a0623cf 100644 --- a/cipher/rijndael-armv8-aarch32-ce.S +++ b/cipher/rijndael-armv8-aarch32-ce.S @@ -1007,19 +1007,18 @@ _gcry_aes_ctr_enc_armv8_ce: */ vpush {q4-q7} - push {r4-r12,lr} /* 4*16 + 4*10 = 104b */ - ldr r4, [sp, #(104+0)] - ldr r5, [sp, #(104+4)] + push {r4-r7,lr} /* 4*16 + 4*5 = 84b */ + ldr r4, [sp, #(84+0)] + ldr r5, [sp, #(84+4)] cmp r4, #0 beq .Lctr_enc_skip cmp r5, #12 - ldm r3, {r7-r10} + ldr r7, [r3, #12] vld1.8 {q0}, [r3] /* load IV */ rev r7, r7 - rev r8, r8 - rev r9, r9 - rev r10, r10 + + mov r5, #1 aes_preload_keys(r0, r6); @@ -1032,58 +1031,22 @@ _gcry_aes_ctr_enc_armv8_ce: blo .Lctr_enc_loop_##bits; \ \ .Lctr_enc_loop4_##bits: \ - cmp r10, #0xfffffffc; \ + vmov.i8 q2, #0; \ sub r4, r4, #4; \ - blo .Lctr_enc_loop4_##bits##_nocarry; \ - cmp r9, #0xffffffff; \ - bne .Lctr_enc_loop4_##bits##_nocarry; \ - \ - adds r10, #1; \ - vmov q1, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ - vmov.32 d1[1], r11; \ - \ - adds r10, #1; \ - vmov q2, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ - vmov.32 d1[1], r11; \ - \ - adds r10, #1; \ - vmov q3, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ - vmov.32 d1[1], r11; \ - \ - adds r10, #1; \ - vmov q4, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ - vmov.32 d1[1], r11; \ - \ - b .Lctr_enc_loop4_##bits##_store_ctr; \ - \ - .Lctr_enc_loop4_##bits##_nocarry: \ - \ - veor q2, q2; \ - vrev64.8 q1, q0; \ - vceq.u32 d5, d5; \ - vadd.u64 q3, q2, q2; \ - vadd.u64 q4, q3, q2; \ - vadd.u64 q0, q3, q3; \ - vsub.u64 q2, q1, q2; \ - vsub.u64 q3, q1, q3; \ - vsub.u64 q4, q1, q4; \ - vsub.u64 q0, q1, q0; \ - vrev64.8 q1, q1; \ - vrev64.8 q2, q2; \ - vrev64.8 q3, q3; \ - vrev64.8 q0, q0; \ - vrev64.8 q4, q4; \ - add r10, #4; \ - \ - .Lctr_enc_loop4_##bits##_store_ctr: \ + vmov.16 d5[3], r5; \ + vrev16.8 q1, q0; \ + \ + vadd.u16 q0, q2, q2; \ + vadd.u16 q2, q1, q2; \ + vadd.u16 q3, q1, q0; \ + vrev16.8 q1, q1; \ + vadd.u16 q4, q2, q0; \ + vrev16.8 q2, q2; \ + vadd.u16 q0, q3, q0; \ + vrev16.8 q3, q3; \ + vrev16.8 q4, q4; \ + vrev16.8 q0, q0; \ + add r7, #4; \ \ vst1.8 {q0}, [r3]; \ cmp r4, #4; \ @@ -1109,13 +1072,12 @@ _gcry_aes_ctr_enc_armv8_ce: \ .Lctr_enc_loop_##bits: \ \ - adds r10, #1; \ + add r7, #1; \ vmov q1, q0; \ - blcs .Lctr_overflow_one; \ - rev r11, r10; \ + rev r5, r7; \ subs r4, r4, #1; \ vld1.8 {q2}, [r2]!; /* load ciphertext */ \ - vmov.32 d1[1], r11; \ + vmov.32 d1[1], r5; \ \ do_aes_one##bits(e, mc, q1, q1, ##__VA_ARGS__); \ \ @@ -1148,21 +1110,9 @@ _gcry_aes_ctr_enc_armv8_ce: CLEAR_REG(q15) .Lctr_enc_skip: - pop {r4-r12,lr} + pop {r4-r7,lr} vpop {q4-q7} bx lr - -.Lctr_overflow_one: - adcs r9, #0 - adcs r8, #0 - adc r7, #0 - rev r11, r9 - rev r12, r8 - vmov.32 d1[0], r11 - rev r11, r7 - vmov.32 d0[1], r12 - vmov.32 d0[0], r11 - bx lr .size _gcry_aes_ctr_enc_armv8_ce,.-_gcry_aes_ctr_enc_armv8_ce; @@ -1208,7 +1158,7 @@ _gcry_aes_ctr32le_enc_armv8_ce: blo .Lctr32le_enc_loop_##bits; \ \ .Lctr32le_enc_loop4_##bits: \ - veor q2, q2; \ + vmov.i8 q2, #0; \ sub r4, r4, #4; \ vmov.i64 d4, #0xffffffff; /* q2 <= -1:0:0:0 */ \ vmov q1, q0; \ @@ -1244,7 +1194,7 @@ _gcry_aes_ctr32le_enc_armv8_ce: \ .Lctr32le_enc_loop_##bits: \ \ - veor q2, q2; \ + vmov.i8 q2, #0; \ vmov q1, q0; \ vmov.i64 d4, #0xffffffff; /* q2 <= -1:0:0:0 */ \ subs r4, r4, #1; \ diff --git a/cipher/rijndael-armv8-aarch64-ce.S b/cipher/rijndael-armv8-aarch64-ce.S index 64f67fbe..5152fab7 100644 --- a/cipher/rijndael-armv8-aarch64-ce.S +++ b/cipher/rijndael-armv8-aarch64-ce.S @@ -734,14 +734,11 @@ _gcry_aes_ctr_enc_armv8_ce: movi v16.16b, #0 mov v16.S[3], w6 /* 1 */ - /* load IV */ - ldp x9, x10, [x3] + /* load IV, x9 gets counter low byte at bits 56:63 for carry detection */ + ldr x9, [x3, #8] ld1 {v0.16b}, [x3] - rev x9, x9 - rev x10, x10 - mov x12, #(4 << 56) - lsl x11, x10, #56 + mov x10, #(4 << 56) aes_preload_keys(x0, w5); @@ -751,25 +748,25 @@ _gcry_aes_ctr_enc_armv8_ce: #define CTR_ENC(bits) \ .Lctr_enc_entry_##bits: \ cmp x4, #4; \ - b.lo .Lctr_enc_loop_##bits; \ + b.lo .Lctr_enc_loop_entry_##bits; \ \ st1 {v8.16b-v11.16b}, [sp]; /* store callee saved registers */ \ \ - adds x11, x11, x12; \ - add v9.4s, v16.4s, v16.4s; /* 2 */ \ - add v10.4s, v16.4s, v9.4s; /* 3 */ \ - add v11.4s, v9.4s, v9.4s; /* 4 */ \ + adds x9, x9, x10; \ + add v9.16b, v16.16b, v16.16b; /* 2 */ \ + add v10.16b, v16.16b, v9.16b; /* 3 */ \ + add v11.16b, v9.16b, v9.16b; /* 4 */ \ mov x7, #1; \ sub x4, x4, #4; \ ld1 {v5.16b-v8.16b}, [x2], #64; /* preload ciphertext */ \ b.cs .Lctr_enc_carry4_##bits; \ \ + /* 8-bit addition */ \ mov v1.16b, v0.16b; \ - add x10, x10, #4; \ add v2.16b, v0.16b, v16.16b; \ - add v3.8h, v0.8h, v9.8h; \ - add v4.4s, v0.4s, v10.4s; \ - add v0.2d, v0.2d, v11.2d; \ + add v3.16b, v0.16b, v9.16b; \ + add v4.16b, v0.16b, v10.16b; \ + add v0.16b, v0.16b, v11.16b; \ \ .Lctr_enc_entry4_##bits##_carry_done: \ mov x7, #0; \ @@ -786,16 +783,16 @@ _gcry_aes_ctr_enc_armv8_ce: eor v8.16b, v8.16b, vklast.16b; \ do_aes_4_part2_##bits(e, mc, v12, v13, v14, v15, v1, v2, v3, v4, v5, v6, v7, v8); \ ld1 {v5.16b-v8.16b}, [x2], #64; /* preload ciphertext */ \ - adds x11, x11, x12; \ + adds x9, x9, x10; \ sub x4, x4, #4; \ b.cs .Lctr_enc_carry4_##bits; \ \ + /* 8-bit addition */ \ mov v1.16b, v0.16b; \ - add x10, x10, #4; \ add v2.16b, v0.16b, v16.16b; \ - add v3.8h, v0.8h, v9.8h; \ - add v4.4s, v0.4s, v10.4s; \ - add v0.2d, v0.2d, v11.2d; \ + add v3.16b, v0.16b, v9.16b; \ + add v4.16b, v0.16b, v10.16b; \ + add v0.16b, v0.16b, v11.16b; \ \ .Lctr_enc_loop4_##bits##_carry_done: \ cmp x4, #4; \ @@ -815,7 +812,6 @@ _gcry_aes_ctr_enc_armv8_ce: \ st1 {v5.16b-v8.16b}, [x1], #64; /* store plaintext */ \ \ - CLEAR_REG(v3); \ CLEAR_REG(v4); \ ld1 {v8.16b-v11.16b}, [sp]; /* restore callee saved registers */ \ CLEAR_REG(v5); \ @@ -823,16 +819,17 @@ _gcry_aes_ctr_enc_armv8_ce: CLEAR_REG(v7); \ cbz x4, .Lctr_enc_done; \ \ + .Lctr_enc_loop_entry_##bits: \ + rev16 v3.16b, v16.16b; \ + \ .Lctr_enc_loop_##bits: \ \ - adds x10, x10, #1; \ mov v1.16b, v0.16b; \ - adc x9, x9, xzr; \ - dup v0.2d, x10; \ + rev16 v0.16b, v0.16b; /* be->le */ \ + add v0.8h, v0.8h, v3.8h; \ sub x4, x4, #1; \ - ins v0.D[0], x9; \ ld1 {v2.16b}, [x2], #16; /* load ciphertext */ \ - rev64 v0.16b, v0.16b; \ + rev16 v0.16b, v0.16b; \ \ do_aes_one_part1(e, mc, v1, vk0); \ eor v2.16b, v2.16b, vklast.16b; \ @@ -846,27 +843,19 @@ _gcry_aes_ctr_enc_armv8_ce: \ .Lctr_enc_carry4_##bits: \ \ - adds x13, x10, #1; \ + /* 16-bit addition */ \ + rev16 v4.16b, v0.16b; /* be->le */ \ + rev16 v3.16b, v16.16b; \ mov v1.16b, v0.16b; \ - adc x14, x9, xzr; \ - dup v2.2d, x13; \ - adds x13, x10, #2; \ - ins v2.D[0], x14; \ - adc x14, x9, xzr; \ - rev64 v2.16b, v2.16b; \ - dup v3.2d, x13; \ - adds x13, x10, #3; \ - ins v3.D[0], x14; \ - adc x14, x9, xzr; \ - rev64 v3.16b, v3.16b; \ - dup v4.2d, x13; \ - adds x10, x10, #4; \ - ins v4.D[0], x14; \ - adc x9, x9, xzr; \ - rev64 v4.16b, v4.16b; \ - dup v0.2d, x10; \ - ins v0.D[0], x9; \ - rev64 v0.16b, v0.16b; \ + rev16 v0.16b, v9.16b; \ + add v2.8h, v4.8h, v3.8h; /* ctr+1 */ \ + add v3.8h, v4.8h, v0.8h; /* ctr+2 */ \ + add v4.8h, v0.8h, v2.8h; /* ctr+1+2 */ \ + add v0.8h, v0.8h, v3.8h; /* ctr+2+2 */ \ + rev16 v2.16b, v2.16b; /* le->be */ \ + rev16 v3.16b, v3.16b; /* le->be */ \ + rev16 v4.16b, v4.16b; /* le->be */ \ + rev16 v0.16b, v0.16b; /* le->be */ \ \ cbz x7, .Lctr_enc_loop4_##bits##_carry_done; \ b .Lctr_enc_entry4_##bits##_carry_done; @@ -885,6 +874,7 @@ _gcry_aes_ctr_enc_armv8_ce: CLEAR_REG(v0) CLEAR_REG(v1) CLEAR_REG(v2) + CLEAR_REG(v3) CLEAR_REG(v16) add sp, sp, #128; diff --git a/cipher/rijndael-ppc-common.h b/cipher/rijndael-ppc-common.h index 611b5871..d7a35c26 100644 --- a/cipher/rijndael-ppc-common.h +++ b/cipher/rijndael-ppc-common.h @@ -232,6 +232,16 @@ asm_add_uint64(block a, block b) return res; } +static ASM_FUNC_ATTR_INLINE block +asm_add_uint16(block a, block b) +{ + block res; + __asm__ volatile ("vadduhm %0,%1,%2\n\t" + : "=v" (res) + : "v" (a), "v" (b)); + return res; +} + static ASM_FUNC_ATTR_INLINE block asm_sra_int64(block a, block b) { diff --git a/cipher/rijndael-ppc-functions.h b/cipher/rijndael-ppc-functions.h index eb39717d..31b75aee 100644 --- a/cipher/rijndael-ppc-functions.h +++ b/cipher/rijndael-ppc-functions.h @@ -888,25 +888,24 @@ CTR_ENC_FUNC (void *context, unsigned char *ctr_arg, void *outbuf_arg, { block in0, in1, in2, in3, in4, in5, in6, in7; block b0, b1, b2, b3, b4, b5, b6, b7; - block two, three, four; + block two, four; block rkey; - two = asm_add_uint128 (one, one); - three = asm_add_uint128 (two, one); - four = asm_add_uint128 (two, two); + two = asm_add_uint16 (one, one); + four = asm_add_uint16 (two, two); for (; nblocks >= 8; nblocks -= 8) { - b1 = asm_add_uint128 (ctr, one); - b2 = asm_add_uint128 (ctr, two); - b3 = asm_add_uint128 (ctr, three); - b4 = asm_add_uint128 (ctr, four); - b5 = asm_add_uint128 (b1, four); - b6 = asm_add_uint128 (b2, four); - b7 = asm_add_uint128 (b3, four); + b1 = asm_add_uint16 (ctr, one); + b2 = asm_add_uint16 (ctr, two); + b3 = asm_add_uint16 (b1, two); + b4 = asm_add_uint16 (b2, two); + b5 = asm_add_uint16 (b1, four); + b6 = asm_add_uint16 (b2, four); + b7 = asm_add_uint16 (b3, four); b0 = asm_xor (rkey0, ctr); rkey = ALIGNED_LOAD (rk, 1); - ctr = asm_add_uint128 (b4, four); + ctr = asm_add_uint16 (b4, four); b1 = asm_xor (rkey0, b1); b2 = asm_xor (rkey0, b2); b3 = asm_xor (rkey0, b3); @@ -1012,11 +1011,11 @@ CTR_ENC_FUNC (void *context, unsigned char *ctr_arg, void *outbuf_arg, if (nblocks >= 4) { - b1 = asm_add_uint128 (ctr, one); - b2 = asm_add_uint128 (ctr, two); - b3 = asm_add_uint128 (ctr, three); + b1 = asm_add_uint16 (ctr, one); + b2 = asm_add_uint16 (ctr, two); + b3 = asm_add_uint16 (b1, two); b0 = asm_xor (rkey0, ctr); - ctr = asm_add_uint128 (ctr, four); + ctr = asm_add_uint16 (b2, two); b1 = asm_xor (rkey0, b1); b2 = asm_xor (rkey0, b2); b3 = asm_xor (rkey0, b3); @@ -1080,7 +1079,7 @@ CTR_ENC_FUNC (void *context, unsigned char *ctr_arg, void *outbuf_arg, for (; nblocks; nblocks--) { b = ctr; - ctr = asm_add_uint128 (ctr, one); + ctr = asm_add_uint16 (ctr, one); rkeylast = rkeylast_orig ^ VEC_LOAD_BE (in, 0, bige_const); AES_ENCRYPT (b, rounds); diff --git a/cipher/rijndael-riscv-zvkned.c b/cipher/rijndael-riscv-zvkned.c index 9472a99e..ed3887ae 100644 --- a/cipher/rijndael-riscv-zvkned.c +++ b/cipher/rijndael-riscv-zvkned.c @@ -738,8 +738,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 }, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4 } }; - static const u64 carry_add[2] = { 1, 1 }; - static const u64 nocarry_add[2] = { 1, 0 }; + static const u32 le_add[4] = { 1, 0, 0, 0 }; RIJNDAEL_context *ctx = context; unsigned char *outbuf = outbuf_arg; const unsigned char *inbuf = inbuf_arg; @@ -749,12 +748,12 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, size_t vl_bytes = vl * 4; u64 ctrlow; vuint32m1_t ctr; - vuint8m1_t add1; + vuint32m1_t add1; ROUND_KEY_VARIABLES; PRELOAD_ROUND_KEYS (rk, rounds, vl); - add1 = __riscv_vle8_v_u8m1(add_u8_array[0], vl_bytes); + add1 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[0], vl_bytes)); ctr = unaligned_load_u32m1(ctr_arg, vl); ctrlow = __riscv_vmv_x_s_u64m1_u64(cast_u32m1_u64m1(bswap128_u32m1(ctr, vl))); @@ -762,9 +761,12 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, if (nblocks >= 4) { - vuint8m1_t add2 = __riscv_vle8_v_u8m1(add_u8_array[1], vl_bytes); - vuint8m1_t add3 = __riscv_vle8_v_u8m1(add_u8_array[2], vl_bytes); - vuint8m1_t add4 = __riscv_vle8_v_u8m1(add_u8_array[3], vl_bytes); + vuint32m1_t add2 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[1], + vl_bytes)); + vuint32m1_t add3 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[2], + vl_bytes)); + vuint32m1_t add4 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[3], + vl_bytes)); memory_barrier_with_vec(add2); memory_barrier_with_vec(add3); @@ -778,67 +780,43 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, /* detect if 8-bit carry handling is needed */ if (UNLIKELY(((ctrlow += 4) & 0xff) <= 3)) { - static const u64 *adders[5][4] = - { - { nocarry_add, nocarry_add, nocarry_add, carry_add }, - { nocarry_add, nocarry_add, carry_add, nocarry_add }, - { nocarry_add, carry_add, nocarry_add, nocarry_add }, - { carry_add, nocarry_add, nocarry_add, nocarry_add }, - { nocarry_add, nocarry_add, nocarry_add, nocarry_add } - }; - unsigned int idx = ctrlow <= 3 ? ctrlow : 4; - vuint64m1_t ctr_u64; - vuint32m1_t ctr_u32_1; - vuint32m1_t ctr_u32_2; - vuint32m1_t ctr_u32_3; - vuint32m1_t ctr_u32_4; - vuint64m1_t add_u64; + vuint32m1_t add_1 = __riscv_vle32_v_u32m1(le_add, vl); + vuint32m1_t add_2 = __riscv_vadd_vv_u32m1(add_1, add_1, vl); + vuint32m1_t ctr_le; + vuint32m1_t ctr_1; + vuint32m1_t ctr_2; + vuint32m1_t ctr_3; + vuint32m1_t ctr_4; /* Byte swap counter */ - ctr_u64 = cast_u32m1_u64m1(bswap128_u32m1(ctr, vl)); + ctr_le = bswap128_u32m1(ctr, vl); /* Addition with carry handling */ - add_u64 = __riscv_vle64_v_u64m1(adders[idx][0], vl / 2); - ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2); - ctr_u32_1 = cast_u64m1_u32m1(ctr_u64); - - add_u64 = __riscv_vle64_v_u64m1(adders[idx][1], vl / 2); - ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2); - ctr_u32_2 = cast_u64m1_u32m1(ctr_u64); - - add_u64 = __riscv_vle64_v_u64m1(adders[idx][2], vl / 2); - ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2); - ctr_u32_3 = cast_u64m1_u32m1(ctr_u64); - - add_u64 = __riscv_vle64_v_u64m1(adders[idx][3], vl / 2); - ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2); - ctr_u32_4 = cast_u64m1_u32m1(ctr_u64); + ctr_1 = __riscv_vadd_vv_u32m1(ctr_le, add_1, vl); + ctr_2 = __riscv_vadd_vv_u32m1(ctr_le, add_2, vl); + ctr_3 = __riscv_vadd_vv_u32m1(ctr_1, add_2, vl); + ctr_4 = __riscv_vadd_vv_u32m1(ctr_2, add_2, vl); /* Byte swap counters */ - ctr_u32_1 = bswap128_u32m1(ctr_u32_1, vl); - ctr_u32_2 = bswap128_u32m1(ctr_u32_2, vl); - ctr_u32_3 = bswap128_u32m1(ctr_u32_3, vl); - ctr_u32_4 = bswap128_u32m1(ctr_u32_4, vl); - - ctr4blks = merge_4x_u32m1_to_u32m4(ctr, ctr_u32_1, ctr_u32_2, - ctr_u32_3); - ctr = ctr_u32_4; + ctr_1 = bswap128_u32m1(ctr_1, vl); + ctr_2 = bswap128_u32m1(ctr_2, vl); + ctr_3 = bswap128_u32m1(ctr_3, vl); + ctr_4 = bswap128_u32m1(ctr_4, vl); + + ctr4blks = merge_4x_u32m1_to_u32m4(ctr, ctr_1, ctr_2, ctr_3); + ctr = ctr_4; } else { /* Fast path addition without carry handling */ - vuint8m1_t ctr_u8 = cast_u32m1_u8m1(ctr); - vuint8m1_t ctr1 = __riscv_vadd_vv_u8m1(ctr_u8, add1, vl_bytes); - vuint8m1_t ctr2 = __riscv_vadd_vv_u8m1(ctr_u8, add2, vl_bytes); - vuint8m1_t ctr3 = __riscv_vadd_vv_u8m1(ctr_u8, add3, vl_bytes); - - ctr = cast_u8m1_u32m1(__riscv_vadd_vv_u8m1(ctr_u8, add4, - vl_bytes)); - - ctr4blks = merge_4x_u32m1_to_u32m4(cast_u8m1_u32m1(ctr_u8), - cast_u8m1_u32m1(ctr1), - cast_u8m1_u32m1(ctr2), - cast_u8m1_u32m1(ctr3)); + vuint32m1_t ctr0 = ctr; + vuint32m1_t ctr1 = __riscv_vadd_vv_u32m1(ctr, add1, vl); + vuint32m1_t ctr2 = __riscv_vadd_vv_u32m1(ctr, add2, vl); + vuint32m1_t ctr3 = __riscv_vadd_vv_u32m1(ctr, add3, vl); + + ctr = __riscv_vadd_vv_u32m1(ctr, add4, vl); + + ctr4blks = merge_4x_u32m1_to_u32m4(ctr0, ctr1, ctr2, ctr3); } data4blks = __riscv_vle8_v_u8m4(inbuf, vl_bytes * 4); @@ -862,15 +840,13 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, /* detect if 8-bit carry handling is needed */ if (UNLIKELY((++ctrlow & 0xff) == 0)) { - const u64 *add_arr = UNLIKELY(ctrlow == 0) ? carry_add : nocarry_add; - vuint64m1_t add_val = __riscv_vle64_v_u64m1(add_arr, vl / 2); + vuint32m1_t add_val = __riscv_vle32_v_u32m1(le_add, vl); /* Byte swap counter */ ctr = bswap128_u32m1(ctr, vl); /* Addition with carry handling */ - ctr = cast_u64m1_u32m1(__riscv_vadd_vv_u64m1(cast_u32m1_u64m1(ctr), - add_val, vl / 2)); + ctr = __riscv_vadd_vv_u32m1(ctr, add_val, vl); /* Byte swap counter */ ctr = bswap128_u32m1(ctr, vl); @@ -878,8 +854,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, else { /* Fast path addition without carry handling */ - ctr = cast_u8m1_u32m1(__riscv_vadd_vv_u8m1(cast_u32m1_u8m1(ctr), - add1, vl_bytes)); + ctr = __riscv_vadd_vv_u32m1(ctr, add1, vl); } AES_CRYPT(e, m1, rounds, block, vl); diff --git a/cipher/rijndael-ssse3-amd64.c b/cipher/rijndael-ssse3-amd64.c index 0f0abf62..d39173a3 100644 --- a/cipher/rijndael-ssse3-amd64.c +++ b/cipher/rijndael-ssse3-amd64.c @@ -375,19 +375,10 @@ _gcry_aes_ssse3_ctr_enc (RIJNDAEL_context *ctx, unsigned char *ctr, { asm volatile ("movdqa %%xmm7, %%xmm0\n\t" /* xmm0 := CTR (xmm7) */ "pcmpeqd %%xmm1, %%xmm1\n\t" - "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */ + "psrldq $14, %%xmm1\n\t" /* xmm1 = -1 */ "pshufb %%xmm6, %%xmm7\n\t" - "psubq %%xmm1, %%xmm7\n\t" /* xmm7++ (big endian) */ - - /* detect if 64-bit carry handling is needed */ - "incq %q[ctrlow]\n\t" - "jnz .Lno_carry%=\n\t" - - "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */ - "psubq %%xmm1, %%xmm7\n\t" /* add carry to upper 64bits */ - - ".Lno_carry%=:\n\t" + "psubw %%xmm1, %%xmm7\n\t" /* xmm7++ (big endian) */ "pshufb %%xmm6, %%xmm7\n\t" : [ctrlow] "+r" (ctrlow) diff --git a/cipher/rijndael-vaes-avx2-amd64.S b/cipher/rijndael-vaes-avx2-amd64.S index 07e6f1ca..65fd79fe 100644 --- a/cipher/rijndael-vaes-avx2-amd64.S +++ b/cipher/rijndael-vaes-avx2-amd64.S @@ -714,40 +714,14 @@ _gcry_vaes_avx2_ctr_enc_amd64: */ CFI_STARTPROC(); - movq 8(%rsi), %r10; - movq 0(%rsi), %r11; - bswapq %r10; - bswapq %r11; + movzbl 15(%rsi), %r10d; + movzbl 14(%rsi), %r11d; vpcmpeqd %ymm15, %ymm15, %ymm15; - vpsrldq $8, %ymm15, %ymm15; // 0:-1 - vpaddq %ymm15, %ymm15, %ymm14; // 0:-2 + vpsrldq $14, %ymm15, %ymm15; // 0:-1 + vpaddw %ymm15, %ymm15, %ymm14; // 0:-2 vbroadcasti128 .Lbswap128_mask rRIP, %ymm13; -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ - vpcmpeqq minus_one, x, tmp1; \ - vpcmpeqq minus_two, x, tmp2; \ - vpor tmp1, tmp2, tmp2; \ - vpsubq minus_two, x, x; \ - vpslldq $8, tmp2, tmp2; \ - vpsubq tmp2, x, x; - -#define handle_ctr_128bit_add(nblks) \ - addq $(nblks), %r10; \ - adcq $0, %r11; \ - bswapq %r10; \ - bswapq %r11; \ - movq %r10, 8(%rsi); \ - movq %r11, 0(%rsi); \ - bswapq %r10; \ - bswapq %r11; - /* Process 16 blocks per loop. */ .align 8 .Lctr_enc_blk16: @@ -760,11 +734,9 @@ _gcry_vaes_avx2_ctr_enc_amd64: vbroadcasti128 (0 * 16)(%rdi), %ymm8; /* detect if carry handling is needed */ - addb $16, 15(%rsi); + addb $16, %r10b; jc .Lctr_enc_blk16_handle_carry; - leaq 16(%r10), %r10; - .Lctr_enc_blk16_byte_bige_add: /* Increment counters. */ vpaddb .Lbige_addb_0 rRIP, %ymm7, %ymm0; @@ -777,6 +749,7 @@ _gcry_vaes_avx2_ctr_enc_amd64: vpaddb .Lbige_addb_14 rRIP, %ymm7, %ymm7; .Lctr_enc_blk16_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7); vbroadcasti128 (1 * 16)(%rdi), %ymm8; @@ -841,34 +814,30 @@ _gcry_vaes_avx2_ctr_enc_amd64: jmp .Lctr_enc_blk16; - .align 8 - .Lctr_enc_blk16_handle_only_ctr_carry: - handle_ctr_128bit_add(16); - jmp .Lctr_enc_blk16_byte_bige_add; - .align 8 .Lctr_enc_blk16_handle_carry: - jz .Lctr_enc_blk16_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk16_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vpshufb %xmm13, %xmm7, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm15, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm7; /* ctr: +1:+0 */ + vpsubw %xmm15, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm7; /* ctr: +1:+0 */ vpshufb %ymm13, %ymm7, %ymm0; - handle_ctr_128bit_add(16); - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +3:+2 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +3:+2 */ vpshufb %ymm13, %ymm7, %ymm1; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +5:+4 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +5:+4 */ vpshufb %ymm13, %ymm7, %ymm2; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +7:+6 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +7:+6 */ vpshufb %ymm13, %ymm7, %ymm3; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +9:+8 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +9:+8 */ vpshufb %ymm13, %ymm7, %ymm4; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +11:+10 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +11:+10 */ vpshufb %ymm13, %ymm7, %ymm5; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +13:+12 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +13:+12 */ vpshufb %ymm13, %ymm7, %ymm6; - add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +15:+14 */ + vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +15:+14 */ vpshufb %ymm13, %ymm7, %ymm7; jmp .Lctr_enc_blk16_rounds; @@ -885,11 +854,9 @@ _gcry_vaes_avx2_ctr_enc_amd64: vbroadcasti128 (0 * 16)(%rdi), %ymm4; /* detect if carry handling is needed */ - addb $8, 15(%rsi); + addb $8, %r10b; jc .Lctr_enc_blk8_handle_carry; - leaq 8(%r10), %r10; - .Lctr_enc_blk8_byte_bige_add: /* Increment counters. */ vpaddb .Lbige_addb_0 rRIP, %ymm3, %ymm0; @@ -898,6 +865,7 @@ _gcry_vaes_avx2_ctr_enc_amd64: vpaddb .Lbige_addb_6 rRIP, %ymm3, %ymm3; .Lctr_enc_blk8_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3); vbroadcasti128 (1 * 16)(%rdi), %ymm4; @@ -950,26 +918,22 @@ _gcry_vaes_avx2_ctr_enc_amd64: jmp .Lctr_enc_blk4; - .align 8 - .Lctr_enc_blk8_handle_only_ctr_carry: - handle_ctr_128bit_add(8); - jmp .Lctr_enc_blk8_byte_bige_add; - .align 8 .Lctr_enc_blk8_handle_carry: - jz .Lctr_enc_blk8_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk8_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vpshufb %xmm13, %xmm3, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm15, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */ + vpsubw %xmm15, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm3; /* ctr: +1:+0 */ vpshufb %ymm13, %ymm3, %ymm0; - handle_ctr_128bit_add(8); - add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +3:+2 */ + vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +3:+2 */ vpshufb %ymm13, %ymm3, %ymm1; - add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +5:+4 */ + vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +5:+4 */ vpshufb %ymm13, %ymm3, %ymm2; - add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +7:+6 */ + vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +7:+6 */ vpshufb %ymm13, %ymm3, %ymm3; jmp .Lctr_enc_blk8_rounds; @@ -986,17 +950,16 @@ _gcry_vaes_avx2_ctr_enc_amd64: vbroadcasti128 (0 * 16)(%rdi), %ymm4; /* detect if carry handling is needed */ - addb $4, 15(%rsi); + addb $4, %r10b; jc .Lctr_enc_blk4_handle_carry; - leaq 4(%r10), %r10; - .Lctr_enc_blk4_byte_bige_add: /* Increment counters. */ vpaddb .Lbige_addb_0 rRIP, %ymm3, %ymm0; vpaddb .Lbige_addb_2 rRIP, %ymm3, %ymm1; .Lctr_enc_blk4_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR2(%ymm4, %ymm0, %ymm1); vbroadcasti128 (1 * 16)(%rdi), %ymm4; @@ -1043,22 +1006,18 @@ _gcry_vaes_avx2_ctr_enc_amd64: jmp .Lctr_enc_blk1; - .align 8 - .Lctr_enc_blk4_handle_only_ctr_carry: - handle_ctr_128bit_add(4); - jmp .Lctr_enc_blk4_byte_bige_add; - .align 8 .Lctr_enc_blk4_handle_carry: - jz .Lctr_enc_blk4_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk4_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vpshufb %xmm13, %xmm3, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm15, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */ + vpsubw %xmm15, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm3; /* ctr: +1:+0 */ vpshufb %ymm13, %ymm3, %ymm0; - handle_ctr_128bit_add(4); - add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +3:+2 */ + vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +3:+2 */ vpshufb %ymm13, %ymm3, %ymm1; jmp .Lctr_enc_blk4_rounds; @@ -1073,13 +1032,16 @@ _gcry_vaes_avx2_ctr_enc_amd64: /* Load and increament counter. */ vmovdqu (%rsi), %xmm0; - handle_ctr_128bit_add(1); + addb $1, %r10b; + adcb $0, %r11b; /* AES rounds. */ vpxor (0 * 16)(%rdi), %xmm0, %xmm0; vaesenc (1 * 16)(%rdi), %xmm0, %xmm0; vaesenc (2 * 16)(%rdi), %xmm0, %xmm0; vaesenc (3 * 16)(%rdi), %xmm0, %xmm0; + movb %r10b, 15(%rsi); + movb %r11b, 14(%rsi); vaesenc (4 * 16)(%rdi), %xmm0, %xmm0; vaesenc (5 * 16)(%rdi), %xmm0, %xmm0; vaesenc (6 * 16)(%rdi), %xmm0, %xmm0; diff --git a/cipher/rijndael-vaes-avx2-i386.S b/cipher/rijndael-vaes-avx2-i386.S index 245e8443..d1bf8bb5 100644 --- a/cipher/rijndael-vaes-avx2-i386.S +++ b/cipher/rijndael-vaes-avx2-i386.S @@ -556,15 +556,15 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): movl %esp, %ebp; CFI_DEF_CFA_REGISTER(%ebp); - subl $(3 * 32 + 3 * 4), %esp; + subl $(1 * 32 + 3 * 4), %esp; andl $-32, %esp; - movl %edi, (3 * 32 + 0 * 4)(%esp); - CFI_REG_ON_STACK(edi, 3 * 32 + 0 * 4); - movl %esi, (3 * 32 + 1 * 4)(%esp); - CFI_REG_ON_STACK(esi, 3 * 32 + 1 * 4); - movl %ebx, (3 * 32 + 2 * 4)(%esp); - CFI_REG_ON_STACK(ebx, 3 * 32 + 2 * 4); + movl %edi, (1 * 32 + 0 * 4)(%esp); + CFI_REG_ON_STACK(edi, 1 * 32 + 0 * 4); + movl %esi, (1 * 32 + 1 * 4)(%esp); + CFI_REG_ON_STACK(esi, 1 * 32 + 1 * 4); + movl %ebx, (1 * 32 + 2 * 4)(%esp); + CFI_REG_ON_STACK(ebx, 1 * 32 + 2 * 4); movl %eax, %ebx; movl 4+4(%ebp), %edi; @@ -574,50 +574,8 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): #define prepare_ctr_const(minus_one, minus_two) \ vpcmpeqd minus_one, minus_one, minus_one; \ - vpsrldq $8, minus_one, minus_one; /* 0:-1 */ \ - vpaddq minus_one, minus_one, minus_two; /* 0:-2 */ - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ - vpcmpeqq minus_one, x, tmp1; \ - vpcmpeqq minus_two, x, tmp2; \ - vpor tmp1, tmp2, tmp2; \ - vpsubq minus_two, x, x; \ - vpslldq $8, tmp2, tmp2; \ - vpsubq tmp2, x, x; - -#define handle_ctr_128bit_add(nblks) \ - movl 12(%esi), %eax; \ - bswapl %eax; \ - addl $nblks, %eax; \ - bswapl %eax; \ - movl %eax, 12(%esi); \ - jnc 1f; \ - \ - movl 8(%esi), %eax; \ - bswapl %eax; \ - adcl $0, %eax; \ - bswapl %eax; \ - movl %eax, 8(%esi); \ - \ - movl 4(%esi), %eax; \ - bswapl %eax; \ - adcl $0, %eax; \ - bswapl %eax; \ - movl %eax, 4(%esi); \ - \ - movl 0(%esi), %eax; \ - bswapl %eax; \ - adcl $0, %eax; \ - bswapl %eax; \ - movl %eax, 0(%esi); \ - .align 8; \ - 1:; + vpsrldq $14, minus_one, minus_one; /* 0:-1 */ \ + vpaddw minus_one, minus_one, minus_two; /* 0:-2 */ cmpl $12, 4+20(%ebp); jae .Lctr_enc_blk12_loop; @@ -631,10 +589,8 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): vbroadcasti128 (%esi), %ymm6; /* detect if carry handling is needed */ - movl 12(%esi), %eax; - addl $(12 << 24), %eax; + addb $12, 15(%esi); jc .Lctr_enc_blk12_handle_carry; - movl %eax, 12(%esi); .Lctr_enc_blk12_byte_bige_add: /* Increment counters. */ @@ -707,39 +663,32 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): jae .Lctr_enc_blk12_loop; jmp .Lctr_enc_blk4; - .align 8 - .Lctr_enc_blk12_handle_only_ctr_carry: - handle_ctr_128bit_add(12); - jmp .Lctr_enc_blk12_byte_bige_add; - .align 8 .Lctr_enc_blk12_handle_carry: - jz .Lctr_enc_blk12_handle_only_ctr_carry; - /* Increment counters (handle carry). */ - prepare_ctr_const(%ymm4, %ymm7); - vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm2; - vpshufb %xmm2, %xmm6, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm4, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm6; /* ctr: +1:+0 */ - handle_ctr_128bit_add(12); - vpshufb %ymm2, %ymm6, %ymm0; + movzbl 14(%esi), %eax; + leal 1(%eax), %eax; + movb %al, 14(%esi); + jz .Lctr_enc_blk12_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ + prepare_ctr_const(%ymm2, %ymm7); + vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm4; + vpshufb %xmm4, %xmm6, %xmm1; /* be => le */ + vpsubw %xmm2, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm6; /* ctr: +1:+0 */ + vpshufb %ymm4, %ymm6, %ymm0; vmovdqa %ymm0, (0 * 32)(%esp); - add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +3:+2 */ - vpshufb %ymm2, %ymm6, %ymm0; - vmovdqa %ymm0, (1 * 32)(%esp); - add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +5:+4 */ - vpshufb %ymm2, %ymm6, %ymm0; - vmovdqa %ymm0, (2 * 32)(%esp); - add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +7:+6 */ - vpshufb %ymm2, %ymm6, %ymm3; - add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +9:+8 */ - vpshufb %ymm2, %ymm6, %ymm5; - add2_le128(%ymm6, %ymm4, %ymm7, %ymm2, %ymm1); /* ctr: +11:+10 */ + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +3:+2 */ + vpshufb %ymm4, %ymm6, %ymm1; + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +5:+4 */ + vpshufb %ymm4, %ymm6, %ymm2; + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +7:+6 */ + vpshufb %ymm4, %ymm6, %ymm3; + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +9:+8 */ + vpshufb %ymm4, %ymm6, %ymm5; + vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +11:+10 */ vmovdqa (0 * 32)(%esp), %ymm0; - vmovdqa (1 * 32)(%esp), %ymm1; - vmovdqa (2 * 32)(%esp), %ymm2; - vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm6, %ymm6; + vpshufb %ymm4, %ymm6, %ymm6; jmp .Lctr_enc_blk12_rounds; @@ -754,10 +703,8 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): vbroadcasti128 (%esi), %ymm3; /* detect if carry handling is needed */ - movl 12(%esi), %eax; - addl $(4 << 24), %eax; + addb $4, 15(%esi); jc .Lctr_enc_blk4_handle_carry; - movl %eax, 12(%esi); .Lctr_enc_blk4_byte_bige_add: /* Increment counters. */ @@ -812,24 +759,22 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): jmp .Lctr_enc_blk1; - .align 8 - .Lctr_enc_blk4_handle_only_ctr_carry: - handle_ctr_128bit_add(4); - jmp .Lctr_enc_blk4_byte_bige_add; - .align 8 .Lctr_enc_blk4_handle_carry: - jz .Lctr_enc_blk4_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + movzbl 14(%esi), %eax; + leal 1(%eax), %eax; + movb %al, 14(%esi); + jz .Lctr_enc_blk4_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ prepare_ctr_const(%ymm4, %ymm7); - vpshufb CADDR(.Lbswap128_mask, %ebx), %xmm3, %xmm1; /* be => le */ - vmovdqa %xmm1, %xmm0; - inc_le128(%xmm1, %xmm4, %xmm5); - vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */ - vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm0; - handle_ctr_128bit_add(4); - add2_le128(%ymm3, %ymm4, %ymm7, %ymm5, %ymm6); /* ctr: +3:+2 */ - vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm1; + vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm2; + vpshufb %xmm2, %xmm3, %xmm1; /* be => le */ + vpsubw %xmm4, %xmm1, %xmm0; + vinserti128 $1, %xmm0, %ymm1, %ymm3; /* ctr: +1:+0 */ + vpshufb %ymm2, %ymm3, %ymm0; + vpsubw %ymm7, %ymm3, %ymm3; /* ctr: +3:+2 */ + vpshufb %ymm2, %ymm3, %ymm1; jmp .Lctr_enc_blk4_rounds; @@ -841,15 +786,16 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): subl $1, 4+20(%ebp); - /* Load and increament counter. */ + /* Load counter. */ vmovdqu (%esi), %xmm0; - handle_ctr_128bit_add(1); /* AES rounds. */ vpxor (0 * 16)(%edi), %xmm0, %xmm0; vaesenc (1 * 16)(%edi), %xmm0, %xmm0; vaesenc (2 * 16)(%edi), %xmm0, %xmm0; vaesenc (3 * 16)(%edi), %xmm0, %xmm0; + addb $1, 15(%esi); /* Increament counter. */ + adcb $0, 14(%esi); vaesenc (4 * 16)(%edi), %xmm0, %xmm0; vaesenc (5 * 16)(%edi), %xmm0, %xmm0; vaesenc (6 * 16)(%edi), %xmm0, %xmm0; @@ -880,18 +826,17 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386): .align 8 .Ldone_ctr_enc: vpxor %ymm0, %ymm0, %ymm0; - movl (3 * 32 + 0 * 4)(%esp), %edi; + movl (1 * 32 + 0 * 4)(%esp), %edi; CFI_RESTORE(edi); - movl (3 * 32 + 1 * 4)(%esp), %esi; + movl (1 * 32 + 1 * 4)(%esp), %esi; CFI_RESTORE(esi); - movl (3 * 32 + 2 * 4)(%esp), %ebx; + movl (1 * 32 + 2 * 4)(%esp), %ebx; CFI_RESTORE(ebx); vmovdqa %ymm0, (0 * 32)(%esp); - vmovdqa %ymm0, (1 * 32)(%esp); - vmovdqa %ymm0, (2 * 32)(%esp); leave; CFI_LEAVE(); vzeroall; + xorl %eax, %eax; ret_spec_stop CFI_ENDPROC(); ELF(.size SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386), diff --git a/cipher/rijndael-vaes-avx512-amd64.S b/cipher/rijndael-vaes-avx512-amd64.S index f20998b0..96cbdd0a 100644 --- a/cipher/rijndael-vaes-avx512-amd64.S +++ b/cipher/rijndael-vaes-avx512-amd64.S @@ -519,16 +519,14 @@ _gcry_vaes_avx512_ctr_enc_amd64: spec_stop_avx512; - movq 8(%rsi), %r10; - movq 0(%rsi), %r11; - bswapq %r10; - bswapq %r11; - vmovdqa32 .Lbige_addb_0 rRIP, %zmm20; vmovdqa32 .Lbige_addb_4 rRIP, %zmm21; vmovdqa32 .Lbige_addb_8 rRIP, %zmm22; vmovdqa32 .Lbige_addb_12 rRIP, %zmm23; + movzbl 15(%rsi), %r10d; + movzbl 14(%rsi), %r11d; + /* Load first and last key. */ leal (, %r9d, 4), %eax; vbroadcasti32x4 (%rdi), %zmm30; @@ -542,22 +540,6 @@ _gcry_vaes_avx512_ctr_enc_amd64: vmovdqa32 .Lbige_addb_24 rRIP, %zmm26; vmovdqa32 .Lbige_addb_28 rRIP, %zmm27; -#define add_le128(out, in, lo_counter, hi_counter1) \ - vpaddq lo_counter, in, out; \ - vpcmpuq $1, lo_counter, out, %k1; \ - kaddb %k1, %k1, %k1; \ - vpaddq hi_counter1, out, out{%k1}; - -#define handle_ctr_128bit_add(nblks) \ - addq $(nblks), %r10; \ - adcq $0, %r11; \ - bswapq %r10; \ - bswapq %r11; \ - movq %r10, 8(%rsi); \ - movq %r11, 0(%rsi); \ - bswapq %r10; \ - bswapq %r11; - /* Process 32 blocks per loop. */ .align 16 .Lctr_enc_blk32: @@ -567,11 +549,9 @@ _gcry_vaes_avx512_ctr_enc_amd64: vbroadcasti32x4 (1 * 16)(%rdi), %zmm8; /* detect if carry handling is needed */ - addb $32, 15(%rsi); + addb $32, %r10b; jc .Lctr_enc_blk32_handle_carry; - leaq 32(%r10), %r10; - .Lctr_enc_blk32_byte_bige_add: /* Increment counters. */ vpaddb %zmm20, %zmm7, %zmm0; @@ -584,6 +564,7 @@ _gcry_vaes_avx512_ctr_enc_amd64: vpaddb %zmm27, %zmm7, %zmm7; .Lctr_enc_blk32_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR8(%zmm30, %zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7); VAESENC8(%zmm8, %zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7); @@ -656,35 +637,31 @@ _gcry_vaes_avx512_ctr_enc_amd64: jmp .Lctr_enc_blk16; - .align 16 - .Lctr_enc_blk32_handle_only_ctr_carry: - handle_ctr_128bit_add(32); - jmp .Lctr_enc_blk32_byte_bige_add; - .align 16 .Lctr_enc_blk32_handle_carry: - jz .Lctr_enc_blk32_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk32_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vbroadcasti32x4 .Lbswap128_mask rRIP, %zmm15; vpmovzxbq .Lcounter0_1_2_3_lo_bq rRIP, %zmm10; - vpmovzxbq .Lcounter1_1_1_1_hi_bq rRIP, %zmm13; vpshufb %zmm15, %zmm7, %zmm7; /* be => le */ vpmovzxbq .Lcounter4_4_4_4_lo_bq rRIP, %zmm11; vpmovzxbq .Lcounter8_8_8_8_lo_bq rRIP, %zmm12; - handle_ctr_128bit_add(32); - add_le128(%zmm0, %zmm7, %zmm10, %zmm13); /* +0:+1:+2:+3 */ - add_le128(%zmm1, %zmm0, %zmm11, %zmm13); /* +4:+5:+6:+7 */ - add_le128(%zmm2, %zmm0, %zmm12, %zmm13); /* +8:... */ + vpaddw %zmm10, %zmm7, %zmm0; /* +0:+1:+2:+3 */ + vpaddw %zmm11, %zmm0, %zmm1; /* +4:+5:+6:+7 */ + vpaddw %zmm12, %zmm0, %zmm2; /* +8:... */ vpshufb %zmm15, %zmm0, %zmm0; /* le => be */ - add_le128(%zmm3, %zmm1, %zmm12, %zmm13); /* +12:... */ + vpaddw %zmm12, %zmm1, %zmm3; /* +12:... */ vpshufb %zmm15, %zmm1, %zmm1; /* le => be */ - add_le128(%zmm4, %zmm2, %zmm12, %zmm13); /* +16:... */ + vpaddw %zmm12, %zmm2, %zmm4; /* +16:... */ vpshufb %zmm15, %zmm2, %zmm2; /* le => be */ - add_le128(%zmm5, %zmm3, %zmm12, %zmm13); /* +20:... */ + vpaddw %zmm12, %zmm3, %zmm5; /* +20:... */ vpshufb %zmm15, %zmm3, %zmm3; /* le => be */ - add_le128(%zmm6, %zmm4, %zmm12, %zmm13); /* +24:... */ + vpaddw %zmm12, %zmm4, %zmm6; /* +24:... */ vpshufb %zmm15, %zmm4, %zmm4; /* le => be */ - add_le128(%zmm7, %zmm5, %zmm12, %zmm13); /* +28:... */ + vpaddw %zmm12, %zmm5, %zmm7; /* +28:... */ vpshufb %zmm15, %zmm5, %zmm5; /* le => be */ vpshufb %zmm15, %zmm6, %zmm6; /* le => be */ vpshufb %zmm15, %zmm7, %zmm7; /* le => be */ @@ -703,11 +680,9 @@ _gcry_vaes_avx512_ctr_enc_amd64: vbroadcasti32x4 (1 * 16)(%rdi), %zmm4; /* detect if carry handling is needed */ - addb $16, 15(%rsi); + addb $16, %r10b; jc .Lctr_enc_blk16_handle_carry; - leaq 16(%r10), %r10; - .Lctr_enc_blk16_byte_bige_add: /* Increment counters. */ vpaddb %zmm20, %zmm3, %zmm0; @@ -716,6 +691,7 @@ _gcry_vaes_avx512_ctr_enc_amd64: vpaddb %zmm23, %zmm3, %zmm3; .Lctr_enc_blk16_rounds: + movb %r10b, 15(%rsi); /* AES rounds */ XOR4(%zmm30, %zmm0, %zmm1, %zmm2, %zmm3); VAESENC4(%zmm4, %zmm0, %zmm1, %zmm2, %zmm3); @@ -767,27 +743,23 @@ _gcry_vaes_avx512_ctr_enc_amd64: jmp .Lctr_enc_tail; - .align 16 - .Lctr_enc_blk16_handle_only_ctr_carry: - handle_ctr_128bit_add(16); - jmp .Lctr_enc_blk16_byte_bige_add; - .align 16 .Lctr_enc_blk16_handle_carry: - jz .Lctr_enc_blk16_handle_only_ctr_carry; - /* Increment counters (handle carry). */ + leal 1(%r11d), %r11d; + movb %r11b, 14(%rsi); + jz .Lctr_enc_blk16_byte_bige_add; + + /* Increment counters (handle 16-bit carry). */ vbroadcasti32x4 .Lbswap128_mask rRIP, %zmm15; vpmovzxbq .Lcounter0_1_2_3_lo_bq rRIP, %zmm10; - vpmovzxbq .Lcounter1_1_1_1_hi_bq rRIP, %zmm13; vpshufb %zmm15, %zmm3, %zmm3; /* be => le */ vpmovzxbq .Lcounter4_4_4_4_lo_bq rRIP, %zmm11; vpmovzxbq .Lcounter8_8_8_8_lo_bq rRIP, %zmm12; - handle_ctr_128bit_add(16); - add_le128(%zmm0, %zmm3, %zmm10, %zmm13); /* +0:+1:+2:+3 */ - add_le128(%zmm1, %zmm0, %zmm11, %zmm13); /* +4:+5:+6:+7 */ - add_le128(%zmm2, %zmm0, %zmm12, %zmm13); /* +8:... */ + vpaddw %zmm10, %zmm3, %zmm0; /* +0:+1:+2:+3 */ + vpaddw %zmm11, %zmm0, %zmm1; /* +4:+5:+6:+7 */ + vpaddw %zmm12, %zmm0, %zmm2; /* +8:... */ vpshufb %zmm15, %zmm0, %zmm0; /* le => be */ - add_le128(%zmm3, %zmm1, %zmm12, %zmm13); /* +12:... */ + vpaddw %zmm12, %zmm1, %zmm3; /* +12:... */ vpshufb %zmm15, %zmm1, %zmm1; /* le => be */ vpshufb %zmm15, %zmm2, %zmm2; /* le => be */ vpshufb %zmm15, %zmm3, %zmm3; /* le => be */ @@ -806,7 +778,6 @@ _gcry_vaes_avx512_ctr_enc_amd64: vpxord %ymm23, %ymm23, %ymm23; vpxord %ymm30, %ymm30, %ymm30; vpxord %ymm31, %ymm31, %ymm31; - kxorq %k1, %k1, %k1; vzeroall; .align 16 @@ -2462,8 +2433,6 @@ _gcry_vaes_avx512_consts: .byte 16, 0, 16, 0, 16, 0, 16, 0 .Lcounter32_32_32_32_lo_bq: .byte 32, 0, 32, 0, 32, 0, 32, 0 -.Lcounter1_1_1_1_hi_bq: - .byte 0, 1, 0, 1, 0, 1, 0, 1 ELF(.size _gcry_vaes_avx512_consts,.-_gcry_vaes_avx512_consts) diff --git a/cipher/rijndael-vp-simd128.h b/cipher/rijndael-vp-simd128.h index 805faa6f..84dab316 100644 --- a/cipher/rijndael-vp-simd128.h +++ b/cipher/rijndael-vp-simd128.h @@ -1732,8 +1732,7 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr, M128I_BYTE(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); static const __m128i_const bigendian_add = M128I_BYTE(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1); - static const __m128i_const carry_add = M128I_U64(1, 1); - static const __m128i_const nocarry_add = M128I_U64(1, 0); + static const __m128i_const littleendian_add = M128I_U64(1, 0); u64 ctrlow = buf_get_be64(ctr + 8); struct vp_aes_config_s config; @@ -1758,32 +1757,20 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr, /* detect if 8-bit carry handling is needed */ if (UNLIKELY(((ctrlow += 4) & 0xff) <= 3)) { - static const __m128i_const *adders[5][4] = - { - { &nocarry_add, &nocarry_add, &nocarry_add, &carry_add }, - { &nocarry_add, &nocarry_add, &carry_add, &nocarry_add }, - { &nocarry_add, &carry_add, &nocarry_add, &nocarry_add }, - { &carry_add, &nocarry_add, &nocarry_add, &nocarry_add }, - { &nocarry_add, &nocarry_add, &nocarry_add, &nocarry_add } - }; - unsigned int idx = ctrlow <= 3 ? ctrlow : 4; - pshufb128(xmm6, xmm7); - - paddq128_amemld(adders[idx][0], xmm7); + paddd128_amemld(&littleendian_add, xmm7); movdqa128(xmm7, xmm2); pshufb128(xmm6, xmm2); insert256_hi128(xmm2, ymm0); - paddq128_amemld(adders[idx][1], xmm7); + paddd128_amemld(&littleendian_add, xmm7); movdqa128(xmm7, xmm2); pshufb128(xmm6, xmm2); movdqa128_256(xmm2, ymm1); - paddq128_amemld(adders[idx][2], xmm7); + paddd128_amemld(&littleendian_add, xmm7); movdqa128(xmm7, xmm2); pshufb128(xmm6, xmm2); insert256_hi128(xmm2, ymm1); - paddq128_amemld(adders[idx][3], xmm7); - + paddd128_amemld(&littleendian_add, xmm7); pshufb128(xmm6, xmm7); } else @@ -1822,30 +1809,10 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr, if (UNLIKELY(((ctrlow += 2) & 0xff) <= 1)) { pshufb128(xmm6, xmm7); - - /* detect if 64-bit carry handling is needed */ - if (UNLIKELY(ctrlow == 1)) - { - paddq128_amemld(&carry_add, xmm7); - movdqa128(xmm7, xmm1); - pshufb128(xmm6, xmm1); - paddq128_amemld(&nocarry_add, xmm7); - } - else if (UNLIKELY(ctrlow == 0)) - { - paddq128_amemld(&nocarry_add, xmm7); - movdqa128(xmm7, xmm1); - pshufb128(xmm6, xmm1); - paddq128_amemld(&carry_add, xmm7); - } - else - { - paddq128_amemld(&nocarry_add, xmm7); - movdqa128(xmm7, xmm1); - pshufb128(xmm6, xmm1); - paddq128_amemld(&nocarry_add, xmm7); - } - + paddd128_amemld(&littleendian_add, xmm7); + movdqa128(xmm7, xmm1); + paddq128_amemld(&littleendian_add, xmm7); + pshufb128(xmm6, xmm1); pshufb128(xmm6, xmm7); } else @@ -1877,10 +1844,7 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr, if (UNLIKELY((++ctrlow & 0xff) == 0)) { pshufb128(xmm6, xmm7); - - /* detect if 64-bit carry handling is needed */ - paddq128_amemld(UNLIKELY(ctrlow == 0) ? &carry_add : &nocarry_add, xmm7); - + paddd128_amemld(&littleendian_add, xmm7); pshufb128(xmm6, xmm7); } else diff --git a/cipher/rijndael.c b/cipher/rijndael.c index 645c0e2f..cf7f4c23 100644 --- a/cipher/rijndael.c +++ b/cipher/rijndael.c @@ -691,7 +691,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_ctr_enc; bulk_ops->ocb_crypt = _gcry_aes_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_ocb_auth; bulk_ops->xts_crypt = _gcry_aes_xts_crypt; @@ -719,7 +719,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_aesni_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_aesni_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_aesni_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_aesni_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_aesni_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_aesni_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_aesni_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_aesni_ocb_auth; @@ -737,7 +737,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, /* Setup VAES bulk encryption routines. */ bulk_ops->cfb_dec = _gcry_aes_vaes_cfb_dec; bulk_ops->cbc_dec = _gcry_aes_vaes_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_vaes_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_vaes_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_vaes_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_vaes_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_vaes_ocb_auth; @@ -752,7 +752,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, /* Setup VAES bulk encryption routines. */ bulk_ops->cfb_dec = _gcry_aes_vaes_cfb_dec; bulk_ops->cbc_dec = _gcry_aes_vaes_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_vaes_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_vaes_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_vaes_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_vaes_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_vaes_ocb_auth; @@ -788,7 +788,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_ssse3_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_ssse3_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_ssse3_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_ssse3_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_ssse3_ctr_enc; bulk_ops->ocb_crypt = _gcry_aes_ssse3_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_ssse3_ocb_auth; } @@ -808,7 +808,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_armv8_ce_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_armv8_ce_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_armv8_ce_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_armv8_ce_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_armv8_ce_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_armv8_ce_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_armv8_ce_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_armv8_ce_ocb_auth; @@ -831,7 +831,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_vp_aarch64_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_vp_aarch64_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_vp_aarch64_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_vp_aarch64_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_vp_aarch64_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_vp_aarch64_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_vp_aarch64_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_vp_aarch64_ocb_auth; @@ -858,7 +858,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_riscv_zvkned_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_riscv_zvkned_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_riscv_zvkned_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_riscv_zvkned_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_riscv_zvkned_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_riscv_zvkned_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_riscv_zvkned_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_riscv_zvkned_ocb_auth; @@ -884,7 +884,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_vp_riscv_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_vp_riscv_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_vp_riscv_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_vp_riscv_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_vp_riscv_ctr_enc; bulk_ops->ctr32le_enc = _gcry_aes_vp_riscv_ctr32le_enc; bulk_ops->ocb_crypt = _gcry_aes_vp_riscv_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_vp_riscv_ocb_auth; @@ -908,7 +908,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_ppc9le_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_ppc9le_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_ppc9le_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_ppc9le_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_ppc9le_ctr_enc; bulk_ops->ocb_crypt = _gcry_aes_ppc9le_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_ppc9le_ocb_auth; bulk_ops->xts_crypt = _gcry_aes_ppc9le_xts_crypt; @@ -939,7 +939,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen, bulk_ops->cfb_dec = _gcry_aes_ppc8_cfb_dec; bulk_ops->cbc_enc = _gcry_aes_ppc8_cbc_enc; bulk_ops->cbc_dec = _gcry_aes_ppc8_cbc_dec; - bulk_ops->ctr_enc = _gcry_aes_ppc8_ctr_enc; + bulk_ops->ctr16be_enc = _gcry_aes_ppc8_ctr_enc; bulk_ops->ocb_crypt = _gcry_aes_ppc8_ocb_crypt; bulk_ops->ocb_auth = _gcry_aes_ppc8_ocb_auth; bulk_ops->xts_crypt = _gcry_aes_ppc8_xts_crypt; @@ -1348,7 +1348,7 @@ _gcry_aes_ctr_enc (void *context, unsigned char *ctr, outbuf += BLOCKSIZE; inbuf += BLOCKSIZE; /* Increment the counter. */ - cipher_block_add(ctr, 1, BLOCKSIZE); + cipher_block_add_be16(ctr, 1, BLOCKSIZE); } wipememory(&tmp, sizeof(tmp)); diff --git a/tests/basic.c b/tests/basic.c index 1f4273ea..9ec1b596 100644 --- a/tests/basic.c +++ b/tests/basic.c @@ -12666,6 +12666,10 @@ cipher_cbc_bulk_test (int cipher_algo) return -1; } + if (verbose) + fprintf (stderr, " checking CBC bulk encryption for %s [%i]\n", + cipher, cipher_algo); + memsize = (blocksize * 2) + (blocksize * nblocks * 3) + 16 + (blocksize + 1); mem = xcalloc (1, memsize); @@ -12906,6 +12910,10 @@ cipher_cfb_bulk_test (int cipher_algo) return -1; } + if (verbose) + fprintf (stderr, " checking CFB bulk encryption for %s [%i]\n", + cipher, cipher_algo); + memsize = (blocksize * 2) + (blocksize * nblocks * 3) + 16 + (blocksize + 1); mem = xcalloc (1, memsize); @@ -13135,6 +13143,10 @@ cipher_ctr_bulk_test (int cipher_algo) return -1; } + if (verbose) + fprintf (stderr, " checking CTR bulk encryption for %s [%i]\n", + cipher, cipher_algo); + memsize = (blocksize * 2) + (blocksize * nblocks * 4) + 16 + (blocksize + 1); mem = xcalloc (1, memsize); @@ -13440,6 +13452,132 @@ cipher_ctr_bulk_test (int cipher_algo) } +/* Regression test for the 16-bit-overflow split in bulk CTR encryption: a + single call spanning >= 0x10000 blocks must not reuse keystream nor drop the + counter carry. Reference keystream is built from ECB with a full-width + byte-wise counter, correct independent of the split logic. */ +static int +cipher_ctr16_overflow_test (int cipher_algo) +{ + static const unsigned int low16_start[] = + { 0, 1, 0x7fff, 0x8000, 0xfffe, 0xffff }; + const size_t nblocks = 2 * 0x10000 + 5; + int blocksize; + const char *cipher; + gcry_cipher_hd_t hd_ecb = NULL; + gcry_cipher_hd_t hd_ctr = NULL; + unsigned char *plaintext = NULL; + unsigned char *reftext = NULL; + unsigned char *enctext = NULL; + unsigned char iv[16]; + unsigned char getctr[17]; + size_t buflen, i, j, tc; + unsigned int keylen; + static const unsigned char key[32] = { + 0x06,0x9A,0x00,0x7F,0xC7,0x6A,0x45,0x9F, + 0x98,0xBA,0xF9,0x17,0xFE,0xDF,0x95,0x21, + 0x06,0x9A,0x00,0x7F,0xC7,0x6A,0x45,0x9F, + 0x98,0xBA,0xF9,0x17,0xFE,0xDF,0x95,0x21 + }; + + if (gcry_cipher_test_algo (cipher_algo)) + return 0; + blocksize = gcry_cipher_get_algo_blklen (cipher_algo); + if (blocksize < 8 || blocksize > 16) + return 0; + cipher = gcry_cipher_algo_name (cipher_algo); + keylen = gcry_cipher_get_algo_keylen (cipher_algo); + if (keylen > sizeof(key)) + return 0; + + if (verbose) + fprintf (stderr, " checking CTR 16-bit overflow for %s [%i]\n", + cipher, cipher_algo); + + buflen = nblocks * (size_t)blocksize; + plaintext = xmalloc (buflen); + reftext = xmalloc (buflen); + enctext = xmalloc (buflen); + + if (gcry_cipher_open (&hd_ecb, cipher_algo, GCRY_CIPHER_MODE_ECB, 0) + || gcry_cipher_open (&hd_ctr, cipher_algo, GCRY_CIPHER_MODE_CTR, 0) + || gcry_cipher_setkey (hd_ecb, key, keylen) + || gcry_cipher_setkey (hd_ctr, key, keylen)) + { + fail ("%s-CTR-%d ctr16 test failed (setup)", cipher, blocksize * 8); + goto leave; + } + + for (i = 0; i < buflen; i++) + plaintext[i] = (unsigned char)i; + + for (tc = 0; tc < DIM (low16_start); tc++) + { + unsigned int low16 = low16_start[tc]; + + /* Fixed non-zero upper bytes exercise carry past the low 16 bits. */ + memset (iv, 0x5a, blocksize); + iv[blocksize - 3] = 0x40; + iv[blocksize - 2] = (low16 >> 8) & 0xff; + iv[blocksize - 1] = low16 & 0xff; + + if (gcry_cipher_setctr (hd_ctr, iv, blocksize)) + { + fail ("%s-CTR-%d ctr16 test failed (setctr, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + + /* iv ends holding start + nblocks, the reference final counter. */ + for (i = 0; i < buflen; i += blocksize) + { + memcpy (&reftext[i], iv, blocksize); + for (j = blocksize; j > 0; j--) + if (++iv[j - 1]) + break; + } + if (gcry_cipher_encrypt (hd_ecb, reftext, buflen, NULL, 0)) + { + fail ("%s-CTR-%d ctr16 test failed (ecb, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + for (i = 0; i < buflen; i++) + reftext[i] ^= plaintext[i]; + + if (gcry_cipher_encrypt (hd_ctr, enctext, buflen, plaintext, buflen)) + { + fail ("%s-CTR-%d ctr16 test failed (ctr, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + if (memcmp (enctext, reftext, buflen)) + { + fail ("%s-CTR-%d ctr16 test failed (keystream, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + + if (gcry_cipher_ctl (hd_ctr, PRIV_CIPHERCTL_GET_COUNTER, getctr, + blocksize + 1) + || getctr[0] != blocksize + || memcmp (getctr + 1, iv, blocksize)) + { + fail ("%s-CTR-%d ctr16 test failed (counter, low16=0x%04x)", + cipher, blocksize * 8, low16); + goto leave; + } + } + +leave: + gcry_cipher_close (hd_ecb); + gcry_cipher_close (hd_ctr); + xfree (plaintext); + xfree (reftext); + xfree (enctext); + return 0; +} + static void check_ciphers (void) @@ -13603,6 +13741,20 @@ check_ciphers (void) } +static void +check_ctr16_overflow (void) +{ + /* One 128-bit and one 64-bit block cipher cover both blocksize code paths + in _gcry_cipher_ctr_encrypt_ctx. */ + if (verbose) + fprintf (stderr, " Starting CTR 16-bit overflow checks.\n"); + cipher_ctr16_overflow_test (GCRY_CIPHER_AES); + cipher_ctr16_overflow_test (GCRY_CIPHER_BLOWFISH); + if (verbose) + fprintf (stderr, " Completed CTR 16-bit overflow checks.\n"); +} + + static void check_cipher_modes(void) { @@ -13613,6 +13765,8 @@ check_cipher_modes(void) check_aes128_cbc_cts_cipher (); check_cbc_mac_cipher (); check_ctr_cipher (); + check_ctr16_overflow (); + check_cfb_cipher (); check_ofb_cipher (); check_ccm_cipher (); -- 2.53.0 From jussi.kivilinna at iki.fi Wed Jul 29 18:34:22 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Wed, 29 Jul 2026 19:34:22 +0300 Subject: [PATCH 2/4] riscv: use memory operands for inline assembly load/store In-Reply-To: <20260729163424.420661-1-jussi.kivilinna@iki.fi> References: <20260729163424.420661-1-jussi.kivilinna@iki.fi> Message-ID: <20260729163424.420661-2-jussi.kivilinna@iki.fi> * cipher/cipher-gcm-riscv-zbb-zbc.c (load_aligned_u64x2) (store_aligned_u64x2): Use memory operand instead of register operand with offset in assembly template and 'memory' clobber. (load_unaligned_u64x2): Skip alignment check when Zicclsm is enabled. * cipher/crc-riscv-zbb-zbc.c (load_aligned_u32, load_aligned_u64x2): Use memory operand instead of register operand with offset in assembly template and 'memory' clobber. -- 'memory' clobber forced compiler to spill values around every load and store. Memory operand describes accessed object exactly, so clobber is not needed and compiler can select addressing mode itself. Signed-off-by: Jussi Kivilinna --- cipher/cipher-gcm-riscv-zbb-zbc.c | 36 ++++++++++++++----------------- cipher/crc-riscv-zbb-zbc.c | 15 ++++++------- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/cipher/cipher-gcm-riscv-zbb-zbc.c b/cipher/cipher-gcm-riscv-zbb-zbc.c index e32bfafe..1e97cb5c 100644 --- a/cipher/cipher-gcm-riscv-zbb-zbc.c +++ b/cipher/cipher-gcm-riscv-zbb-zbc.c @@ -43,14 +43,12 @@ load_aligned_u64x2(const void *ptr) { u64x2 vec; - asm ("ld %0, 0(%1)" + asm ("ld %0, %1" : "=r" (vec.val[0]) - : "r" (ptr) - : "memory"); - asm ("ld %0, 8(%1)" + : "m" (((const bufhelp_u64_t *)ptr)[0])); + asm ("ld %0, %1" : "=r" (vec.val[1]) - : "r" (ptr) - : "memory"); + : "m" (((const bufhelp_u64_t *)ptr)[1])); return vec; } @@ -58,12 +56,8 @@ load_aligned_u64x2(const void *ptr) static ASM_FUNC_ATTR_INLINE u64x2 load_unaligned_u64x2(const void *ptr) { - if (((uintptr_t)ptr & 7) == 0) - { - /* aligned load */ - return load_aligned_u64x2(ptr); - } - else +#if !(defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000)) + if (UNLIKELY(((uintptr_t)ptr & 7) != 0)) { /* unaligned load */ const bufhelp_u64_t *ptr_u64 = ptr; @@ -72,19 +66,21 @@ load_unaligned_u64x2(const void *ptr) vec.val[1] = ptr_u64[1].a; return vec; } +#endif + + /* aligned load */ + return load_aligned_u64x2(ptr); } static ASM_FUNC_ATTR_INLINE void store_aligned_u64x2(void *ptr, u64x2 vec) { - asm ("sd %0, 0(%1)" - : - : "r" (vec.val[0]), "r" (ptr) - : "memory"); - asm ("sd %0, 8(%1)" - : - : "r" (vec.val[1]), "r" (ptr) - : "memory"); + asm ("sd %1, %0" + : "=m" (((bufhelp_u64_t *)ptr)[0]) + : "r" (vec.val[0])); + asm ("sd %1, %0" + : "=m" (((bufhelp_u64_t *)ptr)[1]) + : "r" (vec.val[1])); } static ASM_FUNC_ATTR_INLINE u64 diff --git a/cipher/crc-riscv-zbb-zbc.c b/cipher/crc-riscv-zbb-zbc.c index 0915fa6c..bec6806f 100644 --- a/cipher/crc-riscv-zbb-zbc.c +++ b/cipher/crc-riscv-zbb-zbc.c @@ -152,10 +152,9 @@ static ASM_FUNC_ATTR_INLINE u64 load_aligned_u32(const void *ptr) { u64 out; - asm ("lw %0, 0(%1)" + asm ("lw %0, %1" : "=r" (out) - : "r" (ptr) - : "memory"); + : "m" (((const bufhelp_u32_t *)ptr)[0])); return out; } @@ -164,14 +163,12 @@ load_aligned_u64x2(const void *ptr) { u64x2 vec; - asm ("ld %0, 0(%1)" + asm ("ld %0, %1" : "=r" (vec.lo) - : "r" (ptr) - : "memory"); - asm ("ld %0, 8(%1)" + : "m" (((const bufhelp_u64_t *)ptr)[0])); + asm ("ld %0, %1" : "=r" (vec.hi) - : "r" (ptr) - : "memory"); + : "m" (((const bufhelp_u64_t *)ptr)[1])); return vec; } -- 2.53.0 From jussi.kivilinna at iki.fi Wed Jul 29 18:34:24 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Wed, 29 Jul 2026 19:34:24 +0300 Subject: [PATCH 4/4] bufhelp: use aligned type on all targets when alignment is known In-Reply-To: <20260729163424.420661-1-jussi.kivilinna@iki.fi> References: <20260729163424.420661-1-jussi.kivilinna@iki.fi> Message-ID: <20260729163424.420661-4-jussi.kivilinna@iki.fi> * cipher/bufhelp.h (buf_load32_maybe_unaligned) (buf_store32_maybe_unaligned, buf_load64_maybe_unaligned) (buf_store64_maybe_unaligned): New. (buf_load32, buf_store32, buf_load64, buf_store64): Move compile-time alignment check out of RISC-V specific path. -- Compile-time alignment check was only used on RISC-V with Zicclsm. Move check to common code, so that other targets also get access through naturally aligned type when compiler can prove alignment. Signed-off-by: Jussi Kivilinna --- cipher/bufhelp.h | 133 +++++++++++++++++++++++++---------------------- 1 file changed, 72 insertions(+), 61 deletions(-) diff --git a/cipher/bufhelp.h b/cipher/bufhelp.h index 53516508..7b8ef209 100644 --- a/cipher/bufhelp.h +++ b/cipher/bufhelp.h @@ -145,96 +145,107 @@ typedef struct bufhelp_u64_aligned_s (defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000)) && \ defined(HAVE_GCC_INLINE_ASM_RISCV) -static inline u32 buf_load32(const void *_buf) -{ - if (CONSTANT_P(((uintptr_t)_buf & 3) == 0) && ((uintptr_t)_buf & 3) == 0) - { - /* Compiler could determinate that this pointer is always aligned. */ - return ((const bufhelp_u32_aligned_t *)_buf)->a; - } - else - { - u64 val; - asm ("lw %0, %1" - : "=r" (val) - : "m" (*((const bufhelp_u32_t *)_buf))); - return val; - } +/* Functions for loading and storing potentially unaligned u64/u32 values on + RISC-V with Zicclsm extension. */ +static inline u32 buf_load32_maybe_unaligned(const void *buf) +{ + u64 val; + asm ("lw %0, %1" + : "=r" (val) + : "m" (*((const bufhelp_u32_t *)buf))); + return val; } -static inline void buf_store32(void *_buf, u32 val) +static inline void buf_store32_maybe_unaligned(void *buf, u32 val) { - if (CONSTANT_P(((uintptr_t)_buf & 3) == 0) && ((uintptr_t)_buf & 3) == 0) - { - /* Compiler could determinate that this pointer is always aligned. */ - ((bufhelp_u32_aligned_t *)_buf)->a = val; - } - else - { - asm ("sw %1, %0" - : "=m" (*((bufhelp_u32_t *)_buf)) - : "r" (val)); - } + asm ("sw %1, %0" + : "=m" (*((bufhelp_u32_t *)buf)) + : "r" (val)); } -static inline u64 buf_load64(const void *_buf) +static inline u64 buf_load64_maybe_unaligned(const void *buf) { - if (CONSTANT_P(((uintptr_t)_buf & 7) == 0) && ((uintptr_t)_buf & 7) == 0) - { - /* Compiler could determinate that this pointer is always aligned. */ - return ((const bufhelp_u64_aligned_t *)_buf)->a; - } - else - { - u64 val; - asm ("ld %0, %1" - : "=r" (val) - : "m" (*((const bufhelp_u64_t *)_buf))); - return val; - } + u64 val; + asm ("ld %0, %1" + : "=r" (val) + : "m" (*((const bufhelp_u64_t *)buf))); + return val; } -static inline void buf_store64(void *_buf, u64 val) +static inline void buf_store64_maybe_unaligned(void *buf, u64 val) { - if (CONSTANT_P(((uintptr_t)_buf & 7) == 0) && ((uintptr_t)_buf & 7) == 0) - { - /* Compiler could determinate that this pointer is always aligned. */ - ((bufhelp_u64_aligned_t *)_buf)->a = val; - } - else - { - asm ("sd %1, %0" - : "=m" (*((bufhelp_u64_t *)_buf)) - : "r" (val)); - } + asm ("sd %1, %0" + : "=m" (*((bufhelp_u64_t *)buf)) + : "r" (val)); } #else -static inline u32 buf_load32(const void *_buf) +/* Functions for loading and storing potentially unaligned u64/u32 values. */ +static inline u32 buf_load32_maybe_unaligned(const void *buf) { - return ((const bufhelp_u32_t *)_buf)->a; + return ((const bufhelp_u32_t *)buf)->a; } -static inline void buf_store32(void *_buf, u32 val) +static inline void buf_store32_maybe_unaligned(void *buf, u32 val) { - bufhelp_u32_t *out = _buf; + bufhelp_u32_t *out = buf; out->a = val; } -static inline u64 buf_load64(const void *_buf) +static inline u64 buf_load64_maybe_unaligned(const void *buf) { - return ((const bufhelp_u64_t *)_buf)->a; + return ((const bufhelp_u64_t *)buf)->a; } -static inline void buf_store64(void *_buf, u64 val) +static inline void buf_store64_maybe_unaligned(void *buf, u64 val) { - bufhelp_u64_t *out = _buf; + bufhelp_u64_t *out = buf; out->a = val; } #endif + +/* Functions for loading and storing u64/u32 values, with aligned path + when pointer alignment can be detected at compile time. */ +static inline u32 buf_load32(const void *buf) +{ + /* Check if pointer alignment is known at compile time. */ + if (CONSTANT_P(((uintptr_t)buf & 3) == 0) && ((uintptr_t)buf & 3) == 0) + return ((const bufhelp_u32_aligned_t *)buf)->a; + else + return buf_load32_maybe_unaligned(buf); +} + +static inline void buf_store32(void *buf, u32 val) +{ + /* Check if pointer alignment is known at compile time. */ + if (CONSTANT_P(((uintptr_t)buf & 3) == 0) && ((uintptr_t)buf & 3) == 0) + ((bufhelp_u32_aligned_t *)buf)->a = val; + else + buf_store32_maybe_unaligned(buf, val); +} + +static inline u64 buf_load64(const void *buf) +{ + /* Check if pointer alignment is known at compile time. */ + if (CONSTANT_P(((uintptr_t)buf & 7) == 0) && ((uintptr_t)buf & 7) == 0) + return ((const bufhelp_u64_aligned_t *)buf)->a; + else + return buf_load64_maybe_unaligned(buf); +} + +static inline void buf_store64(void *buf, u64 val) +{ + /* Check if pointer alignment is known at compile time. */ + if (CONSTANT_P(((uintptr_t)buf & 7) == 0) && ((uintptr_t)buf & 7) == 0) + ((bufhelp_u64_aligned_t *)buf)->a = val; + else + buf_store64_maybe_unaligned(buf, val); +} + + /* Functions for loading and storing unaligned u32 values of different endianness. */ static inline u32 buf_get_be32(const void *_buf) -- 2.53.0 From jussi.kivilinna at iki.fi Wed Jul 29 18:34:23 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Wed, 29 Jul 2026 19:34:23 +0300 Subject: [PATCH 3/4] bufhelp: avoid byte-wise load/store on RISC-V with Zicclsm In-Reply-To: <20260729163424.420661-1-jussi.kivilinna@iki.fi> References: <20260729163424.420661-1-jussi.kivilinna@iki.fi> Message-ID: <20260729163424.420661-3-jussi.kivilinna@iki.fi> * cipher/bufhelp.h (bufhelp_u32_aligned_t, bufhelp_u64_aligned_t) (buf_load32, buf_store32, buf_load64, buf_store64): New. (buf_get_be32, buf_get_le32, buf_put_be32, buf_put_le32) (buf_get_be64, buf_get_le64, buf_put_be64, buf_put_le64): Use 'buf_load32'/'buf_load64' and 'buf_store32'/'buf_store64'. -- GCC expands accesses through 'aligned(1)' type to byte load/store sequences when tuning model marks unaligned access slow, which is case with default -mtune. This happens with aligned buffers too and neither Zicclsm in -march nor -mno-strict-align changes it. Use inline assembly with memory operand to get single instruction access, and naturally aligned type when compiler can prove pointer alignment. Benchmark on SpacemiT K1 (1600 Mhz), CAMELLIA128 cycles/byte: | before after speedup ECB enc | 33.62 31.39 1.07x CBC enc | 39.91 32.28 1.24x CBC dec | 38.50 32.12 1.20x CFB enc | 40.06 31.99 1.25x CFB dec | 37.20 32.14 1.16x OFB enc | 38.78 32.11 1.21x CTR enc | 37.18 32.08 1.16x XTS enc | 40.09 32.45 1.24x Signed-off-by: Jussi Kivilinna --- cipher/bufhelp.h | 134 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 117 insertions(+), 17 deletions(-) diff --git a/cipher/bufhelp.h b/cipher/bufhelp.h index 6dcc622a..53516508 100644 --- a/cipher/bufhelp.h +++ b/cipher/bufhelp.h @@ -126,58 +126,158 @@ typedef struct bufhelp_u32_s u32 a; } __attribute__((packed, aligned(1), may_alias)) bufhelp_u32_t; +typedef struct bufhelp_u32_aligned_s +{ + u32 a; +} __attribute__(( aligned(4), may_alias)) bufhelp_u32_aligned_t; + +typedef struct bufhelp_u64_s +{ + u64 a; +} __attribute__((packed, aligned(1), may_alias)) bufhelp_u64_t; + +typedef struct bufhelp_u64_aligned_s +{ + u64 a; +} __attribute__((aligned(8), may_alias)) bufhelp_u64_aligned_t; + +#if defined(__riscv) && \ + (defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000)) && \ + defined(HAVE_GCC_INLINE_ASM_RISCV) + +static inline u32 buf_load32(const void *_buf) +{ + if (CONSTANT_P(((uintptr_t)_buf & 3) == 0) && ((uintptr_t)_buf & 3) == 0) + { + /* Compiler could determinate that this pointer is always aligned. */ + return ((const bufhelp_u32_aligned_t *)_buf)->a; + } + else + { + u64 val; + asm ("lw %0, %1" + : "=r" (val) + : "m" (*((const bufhelp_u32_t *)_buf))); + return val; + } +} + +static inline void buf_store32(void *_buf, u32 val) +{ + if (CONSTANT_P(((uintptr_t)_buf & 3) == 0) && ((uintptr_t)_buf & 3) == 0) + { + /* Compiler could determinate that this pointer is always aligned. */ + ((bufhelp_u32_aligned_t *)_buf)->a = val; + } + else + { + asm ("sw %1, %0" + : "=m" (*((bufhelp_u32_t *)_buf)) + : "r" (val)); + } +} + +static inline u64 buf_load64(const void *_buf) +{ + if (CONSTANT_P(((uintptr_t)_buf & 7) == 0) && ((uintptr_t)_buf & 7) == 0) + { + /* Compiler could determinate that this pointer is always aligned. */ + return ((const bufhelp_u64_aligned_t *)_buf)->a; + } + else + { + u64 val; + asm ("ld %0, %1" + : "=r" (val) + : "m" (*((const bufhelp_u64_t *)_buf))); + return val; + } +} + +static inline void buf_store64(void *_buf, u64 val) +{ + if (CONSTANT_P(((uintptr_t)_buf & 7) == 0) && ((uintptr_t)_buf & 7) == 0) + { + /* Compiler could determinate that this pointer is always aligned. */ + ((bufhelp_u64_aligned_t *)_buf)->a = val; + } + else + { + asm ("sd %1, %0" + : "=m" (*((bufhelp_u64_t *)_buf)) + : "r" (val)); + } +} + +#else + +static inline u32 buf_load32(const void *_buf) +{ + return ((const bufhelp_u32_t *)_buf)->a; +} + +static inline void buf_store32(void *_buf, u32 val) +{ + bufhelp_u32_t *out = _buf; + out->a = val; +} + +static inline u64 buf_load64(const void *_buf) +{ + return ((const bufhelp_u64_t *)_buf)->a; +} + +static inline void buf_store64(void *_buf, u64 val) +{ + bufhelp_u64_t *out = _buf; + out->a = val; +} + +#endif + /* Functions for loading and storing unaligned u32 values of different endianness. */ static inline u32 buf_get_be32(const void *_buf) { - return be_bswap32(((const bufhelp_u32_t *)_buf)->a); + return be_bswap32(buf_load32(_buf)); } static inline u32 buf_get_le32(const void *_buf) { - return le_bswap32(((const bufhelp_u32_t *)_buf)->a); + return le_bswap32(buf_load32(_buf)); } static inline void buf_put_be32(void *_buf, u32 val) { - bufhelp_u32_t *out = _buf; - out->a = be_bswap32(val); + buf_store32(_buf, be_bswap32(val)); } static inline void buf_put_le32(void *_buf, u32 val) { - bufhelp_u32_t *out = _buf; - out->a = le_bswap32(val); + buf_store32(_buf, le_bswap32(val)); } -typedef struct bufhelp_u64_s -{ - u64 a; -} __attribute__((packed, aligned(1), may_alias)) bufhelp_u64_t; - /* Functions for loading and storing unaligned u64 values of different endianness. */ static inline u64 buf_get_be64(const void *_buf) { - return be_bswap64(((const bufhelp_u64_t *)_buf)->a); + return be_bswap64(buf_load64(_buf)); } static inline u64 buf_get_le64(const void *_buf) { - return le_bswap64(((const bufhelp_u64_t *)_buf)->a); + return le_bswap64(buf_load64(_buf)); } static inline void buf_put_be64(void *_buf, u64 val) { - bufhelp_u64_t *out = _buf; - out->a = be_bswap64(val); + buf_store64(_buf, be_bswap64(val)); } static inline void buf_put_le64(void *_buf, u64 val) { - bufhelp_u64_t *out = _buf; - out->a = le_bswap64(val); + buf_store64(_buf, le_bswap64(val)); } #endif /*BUFHELP_UNALIGNED_ACCESS*/ -- 2.53.0 From jussi.kivilinna at iki.fi Wed Jul 29 18:34:21 2026 From: jussi.kivilinna at iki.fi (Jussi Kivilinna) Date: Wed, 29 Jul 2026 19:34:21 +0300 Subject: [PATCH 1/4] riscv: use unaligned vector memory access when supported Message-ID: <20260729163424.420661-1-jussi.kivilinna@iki.fi> * cipher/Makefile.am: Use -mstrict-align only for plain RVV implementations; add Zvkb to RVV-crypto -march flags. * cipher/chacha20-riscv-v.c (unaligned_load_u32m1) (unaligned_store_u32m1): New. (chacha20_rvv_blocks): Use unaligned load/store helpers. * cipher/cipher-gcm-riscv-zvkg.c (unaligned_load_u32m1): Use element width access when RVV_UNALIGNED_NOT_ALLOWED is not defined. * cipher/rijndael-riscv-zvkned.c (unaligned_load_u64m1) (unaligned_load_u32m1, unaligned_store_u32m1, unaligned_load_u32m2) (unaligned_store_u32m2, unaligned_load_u32m4, unaligned_store_u32m4): Likewise. * cipher/sha256-riscv-zvknha-zvkb.c (load_and_swap): Likewise; drop 'vl_bytes' parameter. (sha256_transform_zvknha_zvkb): Remove 'vl_bytes'. * cipher/sha512-riscv-zvknhb-zvkb.c: Likewise. * cipher/simd-common-riscv.h (RVV_UNALIGNED_NOT_ALLOWED): New. * configure.ac: Replace -mstrict-align compiler support check with Zicclsm enabled check; add Zicclsm to MARCH_RVA23U64_BASE and Zvkb to MARCH_RVA23U64_WITH_VEC_CRYPTO; drop -mstrict-align from RVV-crypto check flags; add vrev8 to RVV-crypto intrinsics test. -- -mstrict-align was added for GCC-14 code generation bug, but is really needed for SpacemiT K1 which does not support unaligned vector memory access. K1 does not have vector cryptography extensions, so RVV-crypto implementations can expect unaligned vector access to work. Detect support through Zicclsm extension and keep -mstrict-align for plain RVV implementations. Signed-off-by: Jussi Kivilinna --- cipher/Makefile.am | 22 +- cipher/chacha20-riscv-v.c | 258 ++++++++++----------- cipher/cipher-gcm-riscv-zvkg.c | 5 +- cipher/rijndael-riscv-zvkned.c | 366 +++++++++++++++--------------- cipher/sha256-riscv-zvknha-zvkb.c | 21 +- cipher/sha512-riscv-zvknhb-zvkb.c | 21 +- cipher/simd-common-riscv.h | 13 ++ configure.ac | 62 +++-- 8 files changed, 399 insertions(+), 369 deletions(-) diff --git a/cipher/Makefile.am b/cipher/Makefile.am index 021a7ab6..0c00d181 100644 --- a/cipher/Makefile.am +++ b/cipher/Makefile.am @@ -368,16 +368,15 @@ serpent-avx512-x86.o: $(srcdir)/serpent-avx512-x86.c Makefile serpent-avx512-x86.lo: $(srcdir)/serpent-avx512-x86.c Makefile `echo $(LTCOMPILE) $(avx512f_cflags) -c $< | $(instrumentation_munging) ` -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) if ENABLE_RISCV_VECTOR_INTRINSICS_EXTRA_CFLAGS +# Note: -mstrict-align needed for SpacemiT K1 RVV bug (K1 does not support +# unaligned vector memory access). This is enabled only for plain RVV +# implementations, RVV-crypto extension implementations expect CPU with proper +# support for unaligned vector memory access. riscv_vector_cflags = -O2 -march=@MARCH_RVA22U64_WITH_VEC@ -mstrict-align else -if SUPPORT_CC_RISCV_MSTRICT_ALIGN -riscv_vector_cflags = -O2 -mstrict-align -else riscv_vector_cflags = endif -endif chacha20-riscv-v.o: $(srcdir)/chacha20-riscv-v.c Makefile `echo $(COMPILE) $(riscv_vector_cflags) -c $< | $(instrumentation_munging) ` @@ -391,22 +390,15 @@ rijndael-vp-riscv.o: $(srcdir)/rijndael-vp-riscv.c Makefile rijndael-vp-riscv.lo: $(srcdir)/rijndael-vp-riscv.c Makefile `echo $(LTCOMPILE) $(riscv_vector_cflags) -c $< | $(instrumentation_munging) ` -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) if ENABLE_RISCV_VECTOR_CRYPTO_INTRINSICS_EXTRA_CFLAGS -riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkned -mstrict-align -riscv_vector_crypto_sha_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknha_zvknhb_zvkb -mstrict-align -riscv_vector_crypto_gcm_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkg -mstrict-align -else -if SUPPORT_CC_RISCV_MSTRICT_ALIGN -riscv_vector_crypto_aes_cflags = -O2 -mstrict-align -riscv_vector_crypto_sha_cflags = -O2 -mstrict-align -riscv_vector_crypto_gcm_cflags = -O2 -mstrict-align +riscv_vector_crypto_aes_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkned_zvkb +riscv_vector_crypto_sha_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvknha_zvknhb_zvkb +riscv_vector_crypto_gcm_cflags = -O2 -march=@MARCH_RVA23U64_BASE at _zvkg_zvkb else riscv_vector_crypto_aes_cflags = riscv_vector_crypto_sha_cflags = riscv_vector_crypto_gcm_cflags = endif -endif cipher-gcm-riscv-zvkg.o: $(srcdir)/cipher-gcm-riscv-zvkg.c Makefile `echo $(COMPILE) $(riscv_vector_crypto_gcm_cflags) -c $< | $(instrumentation_munging) ` diff --git a/cipher/chacha20-riscv-v.c b/cipher/chacha20-riscv-v.c index 1304a333..94026305 100644 --- a/cipher/chacha20-riscv-v.c +++ b/cipher/chacha20-riscv-v.c @@ -151,85 +151,113 @@ gen_indexes(size_t vl, size_t stride) return __riscv_vadd_vv_u16m2(idx_hi, idx_lo, vl * 4); } +static ASM_FUNC_ATTR_INLINE vuint32m1_t +unaligned_load_u32m1(const void *src, size_t vl) +{ +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return __riscv_vreinterpret_v_u8m1_u32m1(__riscv_vle8_v_u8m1(src, vl * 4)); +#else + return __riscv_vle32_v_u32m1(src, vl); +#endif +} + +static ASM_FUNC_ATTR_INLINE void +unaligned_store_u32m1(void *dst, vuint32m1_t vec, size_t vl) +{ +#ifdef RVV_UNALIGNED_NOT_ALLOWED + __riscv_vse8_v_u8m1(dst, __riscv_vreinterpret_v_u32m1_u8m1(vec), vl * 4); +#else + __riscv_vse32_v_u32m1(dst, vec, vl); +#endif +} + static ASM_FUNC_ATTR_INLINE vuint32m1x8_t unaligned_vlsseg8e32_v_u32m1x8(const void *src, size_t vl) { - const byte *bsrc = src; - vuint16m2_t indexes; - vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7; - vuint32m1x8_t data; - - if (LIKELY(((uintptr_t)src & 3) == 0)) +#ifdef RVV_UNALIGNED_NOT_ALLOWED + if (UNLIKELY(((uintptr_t)src & 3) != 0)) + { + const byte *bsrc = src; + vuint16m2_t indexes; + vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7; + vuint32m1x8_t data; + + indexes = gen_indexes(4 * vl, 64); + + b0 = __riscv_vluxei16_v_u8m1(bsrc + 0 * 4, indexes, vl * 4); + b1 = __riscv_vluxei16_v_u8m1(bsrc + 1 * 4, indexes, vl * 4); + b2 = __riscv_vluxei16_v_u8m1(bsrc + 2 * 4, indexes, vl * 4); + b3 = __riscv_vluxei16_v_u8m1(bsrc + 3 * 4, indexes, vl * 4); + b4 = __riscv_vluxei16_v_u8m1(bsrc + 4 * 4, indexes, vl * 4); + b5 = __riscv_vluxei16_v_u8m1(bsrc + 5 * 4, indexes, vl * 4); + b6 = __riscv_vluxei16_v_u8m1(bsrc + 6 * 4, indexes, vl * 4); + b7 = __riscv_vluxei16_v_u8m1(bsrc + 7 * 4, indexes, vl * 4); + + data = __riscv_vundefined_u32m1x8(); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 0, __riscv_vreinterpret_v_u8m1_u32m1(b0)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 1, __riscv_vreinterpret_v_u8m1_u32m1(b1)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 2, __riscv_vreinterpret_v_u8m1_u32m1(b2)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 3, __riscv_vreinterpret_v_u8m1_u32m1(b3)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 4, __riscv_vreinterpret_v_u8m1_u32m1(b4)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 5, __riscv_vreinterpret_v_u8m1_u32m1(b5)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 6, __riscv_vreinterpret_v_u8m1_u32m1(b6)); + data = __riscv_vset_v_u32m1_u32m1x8( + data, 7, __riscv_vreinterpret_v_u8m1_u32m1(b7)); + + return data; + } + else +#endif { /* Fast path for 32-bit aligned loads. */ return __riscv_vlsseg8e32_v_u32m1x8(src, 64, vl); } - - indexes = gen_indexes(4 * vl, 64); - - b0 = __riscv_vluxei16_v_u8m1(bsrc + 0 * 4, indexes, vl * 4); - b1 = __riscv_vluxei16_v_u8m1(bsrc + 1 * 4, indexes, vl * 4); - b2 = __riscv_vluxei16_v_u8m1(bsrc + 2 * 4, indexes, vl * 4); - b3 = __riscv_vluxei16_v_u8m1(bsrc + 3 * 4, indexes, vl * 4); - b4 = __riscv_vluxei16_v_u8m1(bsrc + 4 * 4, indexes, vl * 4); - b5 = __riscv_vluxei16_v_u8m1(bsrc + 5 * 4, indexes, vl * 4); - b6 = __riscv_vluxei16_v_u8m1(bsrc + 6 * 4, indexes, vl * 4); - b7 = __riscv_vluxei16_v_u8m1(bsrc + 7 * 4, indexes, vl * 4); - - data = __riscv_vundefined_u32m1x8(); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 0, __riscv_vreinterpret_v_u8m1_u32m1(b0)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 1, __riscv_vreinterpret_v_u8m1_u32m1(b1)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 2, __riscv_vreinterpret_v_u8m1_u32m1(b2)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 3, __riscv_vreinterpret_v_u8m1_u32m1(b3)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 4, __riscv_vreinterpret_v_u8m1_u32m1(b4)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 5, __riscv_vreinterpret_v_u8m1_u32m1(b5)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 6, __riscv_vreinterpret_v_u8m1_u32m1(b6)); - data = __riscv_vset_v_u32m1_u32m1x8( - data, 7, __riscv_vreinterpret_v_u8m1_u32m1(b7)); - - return data; } static ASM_FUNC_ATTR_INLINE void unaligned_vssseg8e32_v_u32m1x8(void *dst, vuint32m1x8_t data, size_t vl) { - byte *bdst = dst; - vuint16m2_t indexes; - vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7; - - if (LIKELY(((uintptr_t)dst & 3) == 0)) +#ifdef RVV_UNALIGNED_NOT_ALLOWED + if (UNLIKELY(((uintptr_t)dst & 3) != 0)) + { + byte *bdst = dst; + vuint16m2_t indexes; + vuint8m1_t b0, b1, b2, b3, b4, b5, b6, b7; + + indexes = gen_indexes(4 * vl, 64); + + b0 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 0)); + b1 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 1)); + b2 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 2)); + b3 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 3)); + b4 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 4)); + b5 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 5)); + b6 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 6)); + b7 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 7)); + + __riscv_vsuxei16_v_u8m1(bdst + 0 * 4, indexes, b0, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 1 * 4, indexes, b1, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 2 * 4, indexes, b2, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 3 * 4, indexes, b3, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 4 * 4, indexes, b4, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 5 * 4, indexes, b5, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 6 * 4, indexes, b6, vl * 4); + __riscv_vsuxei16_v_u8m1(bdst + 7 * 4, indexes, b7, vl * 4); + } + else +#endif { /* Fast path for 32-bit aligned stores. */ __riscv_vssseg8e32_v_u32m1x8(dst, 64, data, vl); return; } - - indexes = gen_indexes(4 * vl, 64); - - b0 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 0)); - b1 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 1)); - b2 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 2)); - b3 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 3)); - b4 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 4)); - b5 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 5)); - b6 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 6)); - b7 = __riscv_vreinterpret_v_u32m1_u8m1(__riscv_vget_v_u32m1x8_u32m1(data, 7)); - - __riscv_vsuxei16_v_u8m1(bdst + 0 * 4, indexes, b0, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 1 * 4, indexes, b1, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 2 * 4, indexes, b2, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 3 * 4, indexes, b3, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 4 * 4, indexes, b4, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 5 * 4, indexes, b5, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 6 * 4, indexes, b6, vl * 4); - __riscv_vsuxei16_v_u8m1(bdst + 7 * 4, indexes, b7, vl * 4); } static ASM_FUNC_ATTR_INLINE unsigned int @@ -385,8 +413,8 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) vuint32m1_t v0, v1, v2, v3; vuint32m1_t v4, v5, v6, v7; vuint32m1_t state0, state1, state2, state3; - vuint8m1_t i0, i1, i2, i3; - vuint8m1_t i4, i5, i6, i7; + vuint32m1_t i0, i1, i2, i3; + vuint32m1_t i4, i5, i6, i7; vuint16m1_t rot16 = gen_rot16(vl); vuint8m1_t rot8 = gen_rot8(vl); @@ -417,10 +445,10 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) v7 = state3; v7 = ADD_U64(v7, one_u64); - i0 = __riscv_vle8_v_u8m1(src + 0 * 16, vl * 4); - i1 = __riscv_vle8_v_u8m1(src + 1 * 16, vl * 4); - i2 = __riscv_vle8_v_u8m1(src + 2 * 16, vl * 4); - i3 = __riscv_vle8_v_u8m1(src + 3 * 16, vl * 4); + i0 = unaligned_load_u32m1(src + 0 * 16, vl); + i1 = unaligned_load_u32m1(src + 1 * 16, vl); + i2 = unaligned_load_u32m1(src + 2 * 16, vl); + i3 = unaligned_load_u32m1(src + 3 * 16, vl); for (i = 20; i > 0; i -= 2) { @@ -434,14 +462,10 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) v3 = __riscv_vadd_vv_u32m1(v3, state3, vl); state3 = ADD_U64(state3, one_u64); - v0 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i0), - v0, vl); - v1 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i1), - v1, vl); - v2 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i2), - v2, vl); - v3 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i3), - v3, vl); + v0 = __riscv_vxor_vv_u32m1(i0, v0, vl); + v1 = __riscv_vxor_vv_u32m1(i1, v1, vl); + v2 = __riscv_vxor_vv_u32m1(i2, v2, vl); + v3 = __riscv_vxor_vv_u32m1(i3, v3, vl); v4 = __riscv_vadd_vv_u32m1(v4, state0, vl); v5 = __riscv_vadd_vv_u32m1(v5, state1, vl); @@ -449,37 +473,25 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) v7 = __riscv_vadd_vv_u32m1(v7, state3, vl); state3 = ADD_U64(state3, one_u64); - i4 = __riscv_vle8_v_u8m1(src + 4 * 16, vl * 4); - i5 = __riscv_vle8_v_u8m1(src + 5 * 16, vl * 4); - i6 = __riscv_vle8_v_u8m1(src + 6 * 16, vl * 4); - i7 = __riscv_vle8_v_u8m1(src + 7 * 16, vl * 4); - - __riscv_vse8_v_u8m1(dst + 0 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v0), vl * 4); - __riscv_vse8_v_u8m1(dst + 1 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v1), vl * 4); - __riscv_vse8_v_u8m1(dst + 2 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v2), vl * 4); - __riscv_vse8_v_u8m1(dst + 3 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v3), vl * 4); - - v4 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i4), - v4, vl); - v5 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i5), - v5, vl); - v6 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i6), - v6, vl); - v7 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i7), - v7, vl); - - __riscv_vse8_v_u8m1(dst + 4 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v4), vl * 4); - __riscv_vse8_v_u8m1(dst + 5 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v5), vl * 4); - __riscv_vse8_v_u8m1(dst + 6 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v6), vl * 4); - __riscv_vse8_v_u8m1(dst + 7 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v7), vl * 4); + i4 = unaligned_load_u32m1(src + 4 * 16, vl); + i5 = unaligned_load_u32m1(src + 5 * 16, vl); + i6 = unaligned_load_u32m1(src + 6 * 16, vl); + i7 = unaligned_load_u32m1(src + 7 * 16, vl); + + unaligned_store_u32m1(dst + 0 * 16, v0, vl); + unaligned_store_u32m1(dst + 1 * 16, v1, vl); + unaligned_store_u32m1(dst + 2 * 16, v2, vl); + unaligned_store_u32m1(dst + 3 * 16, v3, vl); + + v4 = __riscv_vxor_vv_u32m1(i4, v4, vl); + v5 = __riscv_vxor_vv_u32m1(i5, v5, vl); + v6 = __riscv_vxor_vv_u32m1(i6, v6, vl); + v7 = __riscv_vxor_vv_u32m1(i7, v7, vl); + + unaligned_store_u32m1(dst + 4 * 16, v4, vl); + unaligned_store_u32m1(dst + 5 * 16, v5, vl); + unaligned_store_u32m1(dst + 6 * 16, v6, vl); + unaligned_store_u32m1(dst + 7 * 16, v7, vl); src += 2 * 64; dst += 2 * 64; @@ -495,10 +507,10 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) v2 = state2; v3 = state3; - i0 = __riscv_vle8_v_u8m1(src + 0 * 16, vl * 4); - i1 = __riscv_vle8_v_u8m1(src + 1 * 16, vl * 4); - i2 = __riscv_vle8_v_u8m1(src + 2 * 16, vl * 4); - i3 = __riscv_vle8_v_u8m1(src + 3 * 16, vl * 4); + i0 = unaligned_load_u32m1(src + 0 * 16, vl); + i1 = unaligned_load_u32m1(src + 1 * 16, vl); + i2 = unaligned_load_u32m1(src + 2 * 16, vl); + i3 = unaligned_load_u32m1(src + 3 * 16, vl); for (i = 20; i > 0; i -= 2) { @@ -513,22 +525,14 @@ chacha20_rvv_blocks(u32 *input, byte *dst, const byte *src, size_t nblks) state3 = ADD_U64(state3, one_u64); - v0 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i0), - v0, vl); - v1 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i1), - v1, vl); - v2 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i2), - v2, vl); - v3 = __riscv_vxor_vv_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(i3), - v3, vl); - __riscv_vse8_v_u8m1(dst + 0 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v0), vl * 4); - __riscv_vse8_v_u8m1(dst + 1 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v1), vl * 4); - __riscv_vse8_v_u8m1(dst + 2 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v2), vl * 4); - __riscv_vse8_v_u8m1(dst + 3 * 16, - __riscv_vreinterpret_v_u32m1_u8m1(v3), vl * 4); + v0 = __riscv_vxor_vv_u32m1(i0, v0, vl); + v1 = __riscv_vxor_vv_u32m1(i1, v1, vl); + v2 = __riscv_vxor_vv_u32m1(i2, v2, vl); + v3 = __riscv_vxor_vv_u32m1(i3, v3, vl); + unaligned_store_u32m1(dst + 0 * 16, v0, vl); + unaligned_store_u32m1(dst + 1 * 16, v1, vl); + unaligned_store_u32m1(dst + 2 * 16, v2, vl); + unaligned_store_u32m1(dst + 3 * 16, v3, vl); src += 64; dst += 64; diff --git a/cipher/cipher-gcm-riscv-zvkg.c b/cipher/cipher-gcm-riscv-zvkg.c index c459a6fb..1f0b3797 100644 --- a/cipher/cipher-gcm-riscv-zvkg.c +++ b/cipher/cipher-gcm-riscv-zvkg.c @@ -44,9 +44,12 @@ static ASM_FUNC_ATTR_INLINE vuint32m1_t unaligned_load_u32m1(const void *ptr, size_t vl_u32) { +#ifdef RVV_UNALIGNED_NOT_ALLOWED size_t vl_bytes = vl_u32 * 4; - return cast_u8m1_u32m1(__riscv_vle8_v_u8m1(ptr, vl_bytes)); +#else + return __riscv_vle32_v_u32m1(ptr, vl_u32); +#endif } static ASM_FUNC_ATTR_INLINE vuint32m1_t diff --git a/cipher/rijndael-riscv-zvkned.c b/cipher/rijndael-riscv-zvkned.c index ed3887ae..ee9c2de4 100644 --- a/cipher/rijndael-riscv-zvkned.c +++ b/cipher/rijndael-riscv-zvkned.c @@ -83,54 +83,74 @@ bswap128_u32m1(vuint32m1_t vec, size_t vl_u32) __riscv_vrgather_vv_u8m1(cast_u32m1_u8m1(vec), bswap128, vl_bytes)); } +static ASM_FUNC_ATTR_INLINE vuint64m1_t +unaligned_load_u64m1(const void *ptr, size_t vl_u64) +{ +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return cast_u8m1_u64m1(__riscv_vle8_v_u8m1(ptr, vl_u64 * 8)); +#else + return __riscv_vle64_v_u64m1(ptr, vl_u64); +#endif +} + static ASM_FUNC_ATTR_INLINE vuint32m1_t unaligned_load_u32m1(const void *ptr, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - return cast_u8m1_u32m1(__riscv_vle8_v_u8m1(ptr, vl_bytes)); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return cast_u8m1_u32m1(__riscv_vle8_v_u8m1(ptr, vl_u32 * 4)); +#else + return __riscv_vle32_v_u32m1(ptr, vl_u32); +#endif } static ASM_FUNC_ATTR_INLINE void unaligned_store_u32m1(void *ptr, vuint32m1_t vec, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - __riscv_vse8_v_u8m1(ptr, cast_u32m1_u8m1(vec), vl_bytes); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + __riscv_vse8_v_u8m1(ptr, cast_u32m1_u8m1(vec), vl_u32 * 4); +#else + __riscv_vse32_v_u32m1(ptr, vec, vl_u32); +#endif } -static ASM_FUNC_ATTR_INLINE vuint32m4_t -unaligned_load_u32m4(const void *ptr, size_t vl_u32) +static ASM_FUNC_ATTR_INLINE vuint32m2_t +unaligned_load_u32m2(const void *ptr, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - return cast_u8m4_u32m4(__riscv_vle8_v_u8m4(ptr, vl_bytes)); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return cast_u8m2_u32m2(__riscv_vle8_v_u8m2(ptr, vl_u32 * 4)); +#else + return __riscv_vle32_v_u32m2(ptr, vl_u32); +#endif } static ASM_FUNC_ATTR_INLINE void -unaligned_store_u32m4(void *ptr, vuint32m4_t vec, size_t vl_u32) +unaligned_store_u32m2(void *ptr, vuint32m2_t vec, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - __riscv_vse8_v_u8m4(ptr, cast_u32m4_u8m4(vec), vl_bytes); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + __riscv_vse8_v_u8m2(ptr, cast_u32m2_u8m2(vec), vl_u32 * 4); +#else + __riscv_vse32_v_u32m2(ptr, vec, vl_u32); +#endif } -static ASM_FUNC_ATTR_INLINE vuint32m1_t -vxor_u8_u32m1(vuint32m1_t a, vuint32m1_t b, size_t vl_u32) +static ASM_FUNC_ATTR_INLINE vuint32m4_t +unaligned_load_u32m4(const void *ptr, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - return cast_u8m1_u32m1(__riscv_vxor_vv_u8m1(cast_u32m1_u8m1(a), - cast_u32m1_u8m1(b), vl_bytes)); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + return cast_u8m4_u32m4(__riscv_vle8_v_u8m4(ptr, vl_u32 * 4)); +#else + return __riscv_vle32_v_u32m4(ptr, vl_u32); +#endif } -static ASM_FUNC_ATTR_INLINE vuint32m4_t -vxor_u8_u32m4(vuint32m4_t a, vuint32m4_t b, size_t vl_u32) +static ASM_FUNC_ATTR_INLINE void +unaligned_store_u32m4(void *ptr, vuint32m4_t vec, size_t vl_u32) { - size_t vl_bytes = vl_u32 * 4; - - return cast_u8m4_u32m4(__riscv_vxor_vv_u8m4(cast_u32m4_u8m4(a), - cast_u32m4_u8m4(b), vl_bytes)); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + __riscv_vse8_v_u8m4(ptr, cast_u32m4_u8m4(vec), vl_u32 * 4); +#else + __riscv_vse32_v_u32m4(ptr, vec, vl_u32); +#endif } static ASM_FUNC_ATTR_INLINE vuint32m4_t @@ -660,29 +680,28 @@ _gcry_aes_riscv_zvkned_cfb_enc (void *context, unsigned char *iv_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; ROUND_KEY_VARIABLES; PRELOAD_ROUND_KEYS (rk, rounds, vl); - iv = unaligned_load_u32m1(iv_arg, vl); + iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl); for (; nblocks; nblocks--) { - vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); + vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); AES_CRYPT(e, m1, rounds, iv, vl); - data = __riscv_vxor_vv_u8m1(cast_u32m1_u8m1(iv), data, vl_bytes); - __riscv_vse8_v_u8m1(outbuf, data, vl_bytes); - iv = cast_u8m1_u32m1(data); + data = __riscv_vxor_vv_u32m1(iv, data, vl); + unaligned_store_u32m1(outbuf, data, vl); + iv = data; outbuf += BLOCKSIZE; inbuf += BLOCKSIZE; } - unaligned_store_u32m1(iv_arg, iv, vl); + __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl); clear_vec_regs(); } @@ -699,29 +718,27 @@ _gcry_aes_riscv_zvkned_cbc_enc (void *context, unsigned char *iv_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; ROUND_KEY_VARIABLES; PRELOAD_ROUND_KEYS (rk, rounds, vl); - iv = unaligned_load_u32m1(iv_arg, vl); + iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl); for (; nblocks; nblocks--) { - vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); - iv = cast_u8m1_u32m1( - __riscv_vxor_vv_u8m1(data, cast_u32m1_u8m1(iv), vl_bytes)); + vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); + iv = __riscv_vxor_vv_u32m1(data, iv, vl); AES_CRYPT(e, m1, rounds, iv, vl); - __riscv_vse8_v_u8m1(outbuf, cast_u32m1_u8m1(iv), vl_bytes); + unaligned_store_u32m1(outbuf, iv, vl); inbuf += BLOCKSIZE; outbuf += outbuf_add; } - unaligned_store_u32m1(iv_arg, iv, vl); + __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl); clear_vec_regs(); } @@ -731,7 +748,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, void *outbuf_arg, const void *inbuf_arg, size_t nblocks) { - static const byte add_u8_array[4][16] = + static const byte add_u8_array[4][16] __attribute__ ((aligned (16))) = { { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 }, { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 }, @@ -745,7 +762,6 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; u64 ctrlow; vuint32m1_t ctr; vuint32m1_t add1; @@ -753,20 +769,17 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); - add1 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[0], vl_bytes)); - ctr = unaligned_load_u32m1(ctr_arg, vl); + add1 = __riscv_vle32_v_u32m1((const void *)add_u8_array[0], vl); + ctr = __riscv_vle32_v_u32m1((void *)ctr_arg, vl); ctrlow = __riscv_vmv_x_s_u64m1_u64(cast_u32m1_u64m1(bswap128_u32m1(ctr, vl))); memory_barrier_with_vec(add1); if (nblocks >= 4) { - vuint32m1_t add2 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[1], - vl_bytes)); - vuint32m1_t add3 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[2], - vl_bytes)); - vuint32m1_t add4 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[3], - vl_bytes)); + vuint32m1_t add2 = __riscv_vle32_v_u32m1((const void *)add_u8_array[1], vl); + vuint32m1_t add3 = __riscv_vle32_v_u32m1((const void *)add_u8_array[2], vl); + vuint32m1_t add4 = __riscv_vle32_v_u32m1((const void *)add_u8_array[3], vl); memory_barrier_with_vec(add2); memory_barrier_with_vec(add3); @@ -774,7 +787,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, for (; nblocks >= 4; nblocks -= 4) { - vuint8m4_t data4blks; + vuint32m4_t data4blks; vuint32m4_t ctr4blks; /* detect if 8-bit carry handling is needed */ @@ -819,13 +832,12 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, ctr4blks = merge_4x_u32m1_to_u32m4(ctr0, ctr1, ctr2, ctr3); } - data4blks = __riscv_vle8_v_u8m4(inbuf, vl_bytes * 4); + data4blks = unaligned_load_u32m4(inbuf, vl * 4); AES_CRYPT(e, m4, rounds, ctr4blks, vl * 4); - data4blks = __riscv_vxor_vv_u8m4(cast_u32m4_u8m4(ctr4blks), data4blks, - vl_bytes * 4); - __riscv_vse8_v_u8m4(outbuf, data4blks, vl_bytes * 4); + data4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); + unaligned_store_u32m4(outbuf, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; outbuf += 4 * BLOCKSIZE; @@ -835,7 +847,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, for (; nblocks; nblocks--) { vuint32m1_t block = ctr; - vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); + vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); /* detect if 8-bit carry handling is needed */ if (UNLIKELY((++ctrlow & 0xff) == 0)) @@ -859,14 +871,14 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg, AES_CRYPT(e, m1, rounds, block, vl); - data = __riscv_vxor_vv_u8m1(cast_u32m1_u8m1(block), data, vl_bytes); - __riscv_vse8_v_u8m1(outbuf, data, vl_bytes); + data = __riscv_vxor_vv_u32m1(block, data, vl); + unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; } - unaligned_store_u32m1(ctr_arg, ctr, vl); + __riscv_vse32_v_u32m1((void *)ctr_arg, ctr, vl); clear_vec_regs(); } @@ -886,7 +898,6 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t ctr; vuint32m1_t add1; ROUND_KEY_VARIABLES; @@ -894,7 +905,7 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); add1 = __riscv_vle32_v_u32m1(add_u32_array[0], vl); - ctr = unaligned_load_u32m1(ctr_arg, vl); + ctr = __riscv_vle32_v_u32m1((void *)ctr_arg, vl); memory_barrier_with_vec(add1); @@ -914,18 +925,17 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg, vuint32m1_t ctr2 = __riscv_vadd_vv_u32m1(ctr, add2, vl); vuint32m1_t ctr3 = __riscv_vadd_vv_u32m1(ctr, add3, vl); vuint32m4_t ctr4blks; - vuint8m4_t data4blks; + vuint32m4_t data4blks; ctr4blks = merge_4x_u32m1_to_u32m4(ctr, ctr1, ctr2, ctr3); ctr = __riscv_vadd_vv_u32m1(ctr, add4, vl); - data4blks = __riscv_vle8_v_u8m4(inbuf, vl_bytes * 4); + data4blks = unaligned_load_u32m4(inbuf, vl * 4); AES_CRYPT(e, m4, rounds, ctr4blks, vl * 4); - data4blks = __riscv_vxor_vv_u8m4(cast_u32m4_u8m4(ctr4blks), data4blks, - vl_bytes * 4); - __riscv_vse8_v_u8m4(outbuf, data4blks, vl_bytes * 4); + data4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); + unaligned_store_u32m4(outbuf, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; outbuf += 4 * BLOCKSIZE; @@ -935,20 +945,20 @@ _gcry_aes_riscv_zvkned_ctr32le_enc (void *context, unsigned char *ctr_arg, for (; nblocks; nblocks--) { vuint32m1_t block = ctr; - vuint8m1_t data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); + vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); ctr = __riscv_vadd_vv_u32m1(ctr, add1, vl); AES_CRYPT(e, m1, rounds, block, vl); - data = __riscv_vxor_vv_u8m1(cast_u32m1_u8m1(block), data, vl_bytes); - __riscv_vse8_v_u8m1(outbuf, data, vl_bytes); + data = __riscv_vxor_vv_u32m1(block, data, vl); + unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; } - unaligned_store_u32m1(ctr_arg, ctr, vl); + __riscv_vse32_v_u32m1((void *)ctr_arg, ctr, vl); clear_vec_regs(); } @@ -969,7 +979,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); - iv = unaligned_load_u32m1(iv_arg, vl); + iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl); for (; nblocks >= 4; nblocks -= 4) { @@ -983,7 +993,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg, AES_CRYPT(e, m4, rounds, iv4blks, vl * 4); - data4blks = vxor_u8_u32m4(iv4blks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(iv4blks, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; @@ -997,7 +1007,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg, AES_CRYPT(e, m1, rounds, iv, vl); - data = vxor_u8_u32m1(iv, data, vl); + data = __riscv_vxor_vv_u32m1(iv, data, vl); unaligned_store_u32m1(outbuf, data, vl); iv = new_iv; @@ -1005,7 +1015,7 @@ _gcry_aes_riscv_zvkned_cfb_dec (void *context, unsigned char *iv_arg, outbuf += BLOCKSIZE; } - unaligned_store_u32m1(iv_arg, iv, vl); + __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl); clear_vec_regs(); } @@ -1032,7 +1042,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); - iv = unaligned_load_u32m1(iv_arg, vl); + iv = __riscv_vle32_v_u32m1((void *)iv_arg, vl); for (; nblocks >= 4; nblocks -= 4) { @@ -1045,7 +1055,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg, AES_CRYPT(d, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(iv4blks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(iv4blks, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; @@ -1059,7 +1069,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg, AES_CRYPT(d, m1, rounds, data, vl); - data = vxor_u8_u32m1(iv, data, vl); + data = __riscv_vxor_vv_u32m1(iv, data, vl); unaligned_store_u32m1(outbuf, data, vl); iv = new_iv; @@ -1067,7 +1077,7 @@ _gcry_aes_riscv_zvkned_cbc_dec (void *context, unsigned char *iv_arg, outbuf += BLOCKSIZE; } - unaligned_store_u32m1(iv_arg, iv, vl); + __riscv_vse32_v_u32m1((void *)iv_arg, iv, vl); clear_vec_regs(); } @@ -1083,7 +1093,6 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; vuint32m1_t ctr; ROUND_KEY_VARIABLES; @@ -1091,8 +1100,8 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload Offset and Checksum */ - iv = unaligned_load_u32m1(c->u_iv.iv, vl); - ctr = unaligned_load_u32m1(c->u_ctr.ctr, vl); + iv = __riscv_vle32_v_u32m1((void *)c->u_iv.iv, vl); + ctr = __riscv_vle32_v_u32m1((void *)c->u_ctr.ctr, vl); if (nblocks >= 4) { @@ -1101,44 +1110,44 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, for (; nblocks >= 4; nblocks -= 4) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m4_t data4blks = unaligned_load_u32m4(inbuf, vl * 4); vuint32m1_t offset0, offset1, offset2, offset3; vuint32m4_t offsets; /* Checksum_i = Checksum_{i-1} xor P_i */ - ctr4blks = vxor_u8_u32m4(ctr4blks, data4blks, vl * 4); + ctr4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset0 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset1 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset2 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset3 = iv; offsets = merge_4x_u32m1_to_u32m4(offset0, offset1, offset2, offset3); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); AES_CRYPT(e, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); @@ -1149,35 +1158,35 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, /* Checksum_i = Checksum_{i-1} xor P_i */ { vuint32m1x4_t ctr0123 = split_u32m4_to_4x_u32m1(ctr4blks); - ctr = vxor_u8_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), + ctr = __riscv_vxor_vv_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), __riscv_vget_v_u32m1x4_u32m1(ctr0123, 1), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); } } for (; nblocks; nblocks--) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m1_t data; data = unaligned_load_u32m1(inbuf, vl); /* Checksum_i = Checksum_{i-1} xor P_i */ - ctr = vxor_u8_u32m1(ctr, data, vl); + ctr = __riscv_vxor_vv_u32m1(ctr, data, vl); /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); - data = vxor_u8_u32m1(data, iv, vl); + data = __riscv_vxor_vv_u32m1(data, iv, vl); AES_CRYPT(e, m1, rounds, data, vl); - data = vxor_u8_u32m1(iv, data, vl); + data = __riscv_vxor_vv_u32m1(iv, data, vl); unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; @@ -1186,8 +1195,8 @@ aes_riscv_ocb_enc (gcry_cipher_hd_t c, void *outbuf_arg, c->u_mode.ocb.data_nblocks = n; - unaligned_store_u32m1(c->u_iv.iv, iv, vl); - unaligned_store_u32m1(c->u_ctr.ctr, ctr, vl); + __riscv_vse32_v_u32m1((void *)c->u_iv.iv, iv, vl); + __riscv_vse32_v_u32m1((void *)c->u_ctr.ctr, ctr, vl); clear_vec_regs(); @@ -1205,7 +1214,6 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, const u32 *rk = ctx->keyschdec32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; vuint32m1_t ctr; ROUND_KEY_VARIABLES; @@ -1219,8 +1227,8 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload Offset and Checksum */ - iv = unaligned_load_u32m1(c->u_iv.iv, vl); - ctr = unaligned_load_u32m1(c->u_ctr.ctr, vl); + iv = __riscv_vle32_v_u32m1((void *)c->u_iv.iv, vl); + ctr = __riscv_vle32_v_u32m1((void *)c->u_ctr.ctr, vl); if (nblocks >= 4) { @@ -1229,8 +1237,8 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, for (; nblocks >= 4; nblocks -= 4) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m4_t data4blks = unaligned_load_u32m4(inbuf, vl * 4); vuint32m1_t offset0, offset1, offset2, offset3; vuint32m4_t offsets; @@ -1238,37 +1246,37 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* P_i = Offset_i xor ENCIPHER(K, C_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset0 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset1 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset2 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset3 = iv; offsets = merge_4x_u32m1_to_u32m4(offset0, offset1, offset2, offset3); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); AES_CRYPT(d, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); /* Checksum_i = Checksum_{i-1} xor P_i */ - ctr4blks = vxor_u8_u32m4(ctr4blks, data4blks, vl * 4); + ctr4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); inbuf += 4 * BLOCKSIZE; outbuf += 4 * BLOCKSIZE; @@ -1277,37 +1285,34 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, /* Checksum_i = Checksum_{i-1} xor P_i */ { vuint32m1x4_t ctr0123 = split_u32m4_to_4x_u32m1(ctr4blks); - ctr = vxor_u8_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), + ctr = __riscv_vxor_vv_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), __riscv_vget_v_u32m1x4_u32m1(ctr0123, 1), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); } } for (; nblocks; nblocks--) { - const unsigned char *l; - vuint8m1_t l_ntzi; - vuint8m1_t data; - vuint32m1_t block; - - l = ocb_get_l(c, ++n); + const void *l; + vuint32m1_t l_ntzi; + vuint32m1_t data; /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* P_i = Offset_i xor DECIPHER(K, C_i xor Offset_i) */ - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - data = __riscv_vle8_v_u8m1(inbuf, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); - data = __riscv_vxor_vv_u8m1(data, cast_u32m1_u8m1(iv), vl_bytes); - block = cast_u8m1_u32m1(data); + l = ocb_get_l(c, ++n); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + data = unaligned_load_u32m1(inbuf, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); + data = __riscv_vxor_vv_u32m1(data, iv, vl); - AES_CRYPT(d, m1, rounds, block, vl); + AES_CRYPT(d, m1, rounds, data, vl); - block = vxor_u8_u32m1(iv, block, vl); - unaligned_store_u32m1(outbuf, block, vl); + data = __riscv_vxor_vv_u32m1(iv, data, vl); + unaligned_store_u32m1(outbuf, data, vl); /* Checksum_i = Checksum_{i-1} xor P_i */ - ctr = vxor_u8_u32m1(ctr, block, vl); + ctr = __riscv_vxor_vv_u32m1(ctr, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; @@ -1315,8 +1320,8 @@ aes_riscv_ocb_dec (gcry_cipher_hd_t c, void *outbuf_arg, c->u_mode.ocb.data_nblocks = n; - unaligned_store_u32m1(c->u_iv.iv, iv, vl); - unaligned_store_u32m1(c->u_ctr.ctr, ctr, vl); + __riscv_vse32_v_u32m1((void *)c->u_iv.iv, iv, vl); + __riscv_vse32_v_u32m1((void *)c->u_ctr.ctr, ctr, vl); clear_vec_regs(); @@ -1344,7 +1349,6 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, const u32 *rk = ctx->keyschenc32[0]; int rounds = ctx->rounds; size_t vl = 4; - size_t vl_bytes = vl * 4; vuint32m1_t iv; vuint32m1_t ctr; ROUND_KEY_VARIABLES; @@ -1352,8 +1356,8 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload Offset and Sum */ - iv = unaligned_load_u32m1(c->u_mode.ocb.aad_offset, vl); - ctr = unaligned_load_u32m1(c->u_mode.ocb.aad_sum, vl); + iv = __riscv_vle32_v_u32m1((void *)c->u_mode.ocb.aad_offset, vl); + ctr = __riscv_vle32_v_u32m1((void *)c->u_mode.ocb.aad_sum, vl); if (nblocks >= 4) { @@ -1362,8 +1366,8 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, for (; nblocks >= 4; nblocks -= 4) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m4_t data4blks = unaligned_load_u32m4(abuf, vl * 4); vuint32m1_t offset0, offset1, offset2, offset3; vuint32m4_t offsets; @@ -1371,32 +1375,32 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset0 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset1 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset2 = iv; l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); offset3 = iv; offsets = merge_4x_u32m1_to_u32m4(offset0, offset1, offset2, offset3); - data4blks = vxor_u8_u32m4(offsets, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(offsets, data4blks, vl * 4); AES_CRYPT(e, m4, rounds, data4blks, vl * 4); - ctr4blks = vxor_u8_u32m4(ctr4blks, data4blks, vl * 4); + ctr4blks = __riscv_vxor_vv_u32m4(ctr4blks, data4blks, vl * 4); abuf += 4 * BLOCKSIZE; } @@ -1404,17 +1408,17 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, /* Checksum_i = Checksum_{i-1} xor P_i */ { vuint32m1x4_t ctr0123 = split_u32m4_to_4x_u32m1(ctr4blks); - ctr = vxor_u8_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), + ctr = __riscv_vxor_vv_u32m1(__riscv_vget_v_u32m1x4_u32m1(ctr0123, 0), __riscv_vget_v_u32m1x4_u32m1(ctr0123, 1), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); - ctr = vxor_u8_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 2), vl); + ctr = __riscv_vxor_vv_u32m1(ctr, __riscv_vget_v_u32m1x4_u32m1(ctr0123, 3), vl); } } for (; nblocks; nblocks--) { - const unsigned char *l; - vuint8m1_t l_ntzi; + const void *l; + vuint32m1_t l_ntzi; vuint32m1_t data; data = unaligned_load_u32m1(abuf, vl); @@ -1422,22 +1426,22 @@ _gcry_aes_riscv_zvkned_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, /* Offset_i = Offset_{i-1} xor L_{ntz(i)} */ /* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i) */ l = ocb_get_l(c, ++n); - l_ntzi = __riscv_vle8_v_u8m1(l, vl_bytes); - iv = vxor_u8_u32m1(iv, cast_u8m1_u32m1(l_ntzi), vl); + l_ntzi = __riscv_vle32_v_u32m1(l, vl); + iv = __riscv_vxor_vv_u32m1(iv, l_ntzi, vl); - data = vxor_u8_u32m1(data, iv, vl); + data = __riscv_vxor_vv_u32m1(data, iv, vl); AES_CRYPT(e, m1, rounds, data, vl); - ctr = vxor_u8_u32m1(ctr, data, vl); + ctr = __riscv_vxor_vv_u32m1(ctr, data, vl); abuf += BLOCKSIZE; } c->u_mode.ocb.aad_nblocks = n; - unaligned_store_u32m1(c->u_mode.ocb.aad_offset, iv, vl); - unaligned_store_u32m1(c->u_mode.ocb.aad_sum, ctr, vl); + __riscv_vse32_v_u32m1((void *)c->u_mode.ocb.aad_offset, iv, vl); + __riscv_vse32_v_u32m1((void *)c->u_mode.ocb.aad_sum, ctr, vl); clear_vec_regs(); @@ -1482,7 +1486,7 @@ aes_riscv_xts_enc (void *context, unsigned char *tweak_arg, void *outbuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload tweak */ - tweak = unaligned_load_u32m1(tweak_arg, vl); + tweak = __riscv_vle32_v_u32m1((void *)tweak_arg, vl); memory_barrier_with_vec(xts_gfmul); memory_barrier_with_vec(xts_swap64); @@ -1504,11 +1508,11 @@ aes_riscv_xts_enc (void *context, unsigned char *tweak_arg, void *outbuf_arg, tweaks = merge_4x_u32m1_to_u32m4(tweak0, tweak1, tweak2, tweak3); - data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4); AES_CRYPT(e, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); @@ -1521,19 +1525,19 @@ aes_riscv_xts_enc (void *context, unsigned char *tweak_arg, void *outbuf_arg, vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); vuint32m1_t tweak0 = tweak; - data = vxor_u8_u32m1(data, tweak0, vl); + data = __riscv_vxor_vv_u32m1(data, tweak0, vl); tweak = xts_gfmul_byA(tweak, xts_gfmul, xts_swap64, vl); AES_CRYPT(e, m1, rounds, data, vl); - data = vxor_u8_u32m1(data, tweak0, vl); + data = __riscv_vxor_vv_u32m1(data, tweak0, vl); unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; } - unaligned_store_u32m1(tweak_arg, tweak, vl); + __riscv_vse32_v_u32m1((void *)tweak_arg, tweak, vl); clear_vec_regs(); } @@ -1562,7 +1566,7 @@ aes_riscv_xts_dec (void *context, unsigned char *tweak_arg, void *outbuf_arg, PRELOAD_ROUND_KEYS (rk, rounds, vl); /* Preload tweak */ - tweak = unaligned_load_u32m1(tweak_arg, vl); + tweak = __riscv_vle32_v_u32m1((void *)tweak_arg, vl); memory_barrier_with_vec(xts_gfmul); memory_barrier_with_vec(xts_swap64); @@ -1584,11 +1588,11 @@ aes_riscv_xts_dec (void *context, unsigned char *tweak_arg, void *outbuf_arg, tweaks = merge_4x_u32m1_to_u32m4(tweak0, tweak1, tweak2, tweak3); - data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4); AES_CRYPT(d, m4, rounds, data4blks, vl * 4); - data4blks = vxor_u8_u32m4(tweaks, data4blks, vl * 4); + data4blks = __riscv_vxor_vv_u32m4(tweaks, data4blks, vl * 4); unaligned_store_u32m4(outbuf, data4blks, vl * 4); @@ -1601,19 +1605,19 @@ aes_riscv_xts_dec (void *context, unsigned char *tweak_arg, void *outbuf_arg, vuint32m1_t data = unaligned_load_u32m1(inbuf, vl); vuint32m1_t tweak0 = tweak; - data = vxor_u8_u32m1(data, tweak0, vl); + data = __riscv_vxor_vv_u32m1(data, tweak0, vl); tweak = xts_gfmul_byA(tweak, xts_gfmul, xts_swap64, vl); AES_CRYPT(d, m1, rounds, data, vl); - data = vxor_u8_u32m1(data, tweak0, vl); + data = __riscv_vxor_vv_u32m1(data, tweak0, vl); unaligned_store_u32m1(outbuf, data, vl); inbuf += BLOCKSIZE; outbuf += BLOCKSIZE; } - unaligned_store_u32m1(tweak_arg, tweak, vl); + __riscv_vse32_v_u32m1((void *)tweak_arg, tweak, vl); clear_vec_regs(); } diff --git a/cipher/sha256-riscv-zvknha-zvkb.c b/cipher/sha256-riscv-zvknha-zvkb.c index 6375f9aa..0ff80cf7 100644 --- a/cipher/sha256-riscv-zvknha-zvkb.c +++ b/cipher/sha256-riscv-zvknha-zvkb.c @@ -76,11 +76,14 @@ working_vsha2cl_vv_u32m1(vuint32m1_t hgcd, vuint32m1_t feba, static ASM_FUNC_ATTR_INLINE vuint32m1_t -load_and_swap (const byte * p, size_t vl, size_t vl_bytes) +load_and_swap (const byte * p, size_t vl) { - vuint8m1_t temp_bytes = __riscv_vle8_v_u8m1(p, vl_bytes); - return __riscv_vrev8_v_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(temp_bytes), - vl); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + vuint8m1_t temp = __riscv_vle8_v_u8m1(p, vl * 4); + return __riscv_vrev8_v_u32m1(__riscv_vreinterpret_v_u8m1_u32m1(temp), vl); +#else + return __riscv_vrev8_v_u32m1(__riscv_vle32_v_u32m1((const void *)p, vl), vl); +#endif } @@ -111,7 +114,6 @@ sha256_transform_zvknha_zvkb (u32 state[8], const uint8_t * data, static const int feba_offset = 0; static const int hgcd_offset = 8 / sizeof(u32); size_t vl; - size_t vl_bytes; vuint32m1_t idx; vuint32m1_t v_feba_work, v_feba; vuint32m1_t v_hgcd_work, v_hgcd; @@ -121,7 +123,6 @@ sha256_transform_zvknha_zvkb (u32 state[8], const uint8_t * data, vuint32m1_t v_feba_hgcd_idx; vl = 4; - vl_bytes = vl * 4; idx = __riscv_vid_v_u32m1(vl); merge_mask = __riscv_vmseq_vx_u32m1_b32(idx, 0, vl); @@ -137,10 +138,10 @@ sha256_transform_zvknha_zvkb (u32 state[8], const uint8_t * data, v_feba_work = v_feba; v_hgcd_work = v_hgcd; - w0 = load_and_swap(data + 0, vl, vl_bytes); - w1 = load_and_swap(data + 16, vl, vl_bytes); - w2 = load_and_swap(data + 32, vl, vl_bytes); - w3 = load_and_swap(data + 48, vl, vl_bytes); + w0 = load_and_swap(data + 0, vl); + w1 = load_and_swap(data + 16, vl); + w2 = load_and_swap(data + 32, vl); + w3 = load_and_swap(data + 48, vl); QUAD_ROUND_W_SCHED(w0, w1, w2, w3); QUAD_ROUND_W_SCHED(w1, w2, w3, w0); diff --git a/cipher/sha512-riscv-zvknhb-zvkb.c b/cipher/sha512-riscv-zvknhb-zvkb.c index 5f5d483a..fea8f32a 100644 --- a/cipher/sha512-riscv-zvknhb-zvkb.c +++ b/cipher/sha512-riscv-zvknhb-zvkb.c @@ -81,11 +81,14 @@ working_vsha2cl_vv_u64m2(vuint64m2_t hgcd, vuint64m2_t feba, static ASM_FUNC_ATTR_INLINE vuint64m2_t -load_and_swap(const byte *p, size_t vl, size_t vl_bytes) +load_and_swap(const byte *p, size_t vl) { - vuint8m2_t temp_bytes = __riscv_vle8_v_u8m2(p, vl_bytes); - return __riscv_vrev8_v_u64m2(__riscv_vreinterpret_v_u8m2_u64m2(temp_bytes), - vl); +#ifdef RVV_UNALIGNED_NOT_ALLOWED + vuint8m2_t temp = __riscv_vle8_v_u8m2(p, vl * 8); + return __riscv_vrev8_v_u64m2(__riscv_vreinterpret_v_u8m2_u64m2(temp), vl); +#else + return __riscv_vrev8_v_u64m2(__riscv_vle64_v_u64m2((const void *)p, vl), vl); +#endif } @@ -97,7 +100,6 @@ sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data, static const int feba_offset = 0; static const int hgcd_offset = 16 / sizeof(u64); size_t vl; - size_t vl_bytes; vuint64m2_t idx; vuint64m2_t v_feba_work, v_feba; vuint64m2_t v_hgcd_work, v_hgcd; @@ -107,7 +109,6 @@ sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data, vuint64m2_t v_feba_hgcd_idx; vl = 4; - vl_bytes = vl * 8; idx = __riscv_vid_v_u64m2(vl); merge_mask = __riscv_vmseq_vx_u64m2_b32(idx, 0, vl); @@ -124,10 +125,10 @@ sha512_transform_zvknhb_zvkb (u64 state[8], const byte *data, v_feba_work = v_feba; v_hgcd_work = v_hgcd; - w0 = load_and_swap(data + 0, vl, vl_bytes); - w1 = load_and_swap(data + 32, vl, vl_bytes); - w2 = load_and_swap(data + 64, vl, vl_bytes); - w3 = load_and_swap(data + 96, vl, vl_bytes); + w0 = load_and_swap(data + 0, vl); + w1 = load_and_swap(data + 32, vl); + w2 = load_and_swap(data + 64, vl); + w3 = load_and_swap(data + 96, vl); QUAD_ROUND_W_SCHED(w0, w1, w2, w3); QUAD_ROUND_W_SCHED(w1, w2, w3, w0); diff --git a/cipher/simd-common-riscv.h b/cipher/simd-common-riscv.h index 8381000f..8ddd5056 100644 --- a/cipher/simd-common-riscv.h +++ b/cipher/simd-common-riscv.h @@ -23,6 +23,19 @@ #include +#undef RVV_UNALIGNED_NOT_ALLOWED +#if defined(HAVE_COMPATIBLE_CC_RISCV_VECTOR_INTRINSICS_WITH_CFLAGS) || \ + !(defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000)) || \ + defined(__riscv_misaligned_avoid) +/* Zicclsm extension means that unaligned vector (and scalar) memory accesses + * are allowed. If build configuration is such that RVV intrinsics support had + * to be enabled with custom CFLAGS, assume that vector unaligned accesses are + * not allowed. This is to workaround broken build setups with RVA22 + * non-compliant Spacemit K1 (unaligned scalar allowed, unaligned vector + * not allowed). */ +#define RVV_UNALIGNED_NOT_ALLOWED 1 +#endif + #define memory_barrier_with_vec(a) __asm__("" : "+vr"(a) :: "memory") #define clear_vec_regs() __asm__ volatile("vsetvli zero, %0, e8, m1, ta, ma;\n" \ diff --git a/configure.ac b/configure.ac index 631ed81e..4ac3450a 100644 --- a/configure.ac +++ b/configure.ac @@ -2802,28 +2802,26 @@ fi # -# Check whether compiler supports RISC-V -mstrict-align flag +# Check whether compiler has RISC-V Zicclsm extension enable # -_gcc_cflags_save=$CFLAGS -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) -CFLAGS="$CFLAGS -mstrict-align" - -AC_CACHE_CHECK([whether compiler supports RISC-V -mstrict-align flag], - [gcry_cv_cc_riscv_mstrict_align], +AC_CACHE_CHECK([whether compiler has RISC-V Zicclsm extension enabled], + [gcry_cv_cc_riscv_zicclsm_enabled], [if test "$mpi_cpu_arch" != "riscv64" || test "$try_asm_modules" != "yes" ; then - gcry_cv_cc_riscv_mstrict_align="n/a" + gcry_cv_cc_riscv_zicclsm_enabled="n/a" else - gcry_cv_cc_riscv_mstrict_align=no - AC_COMPILE_IFELSE( - [AC_LANG_SOURCE([[void testfn(void) { }]])], - [gcry_cv_cc_riscv_mstrict_align=yes]) + gcry_cv_cc_riscv_zicclsm_enabled=no + AC_LINK_IFELSE([AC_LANG_PROGRAM( + [[#if defined(__riscv_zicclsm) && (__riscv_zicclsm >= 1000000) + int testfunc(void) { return 0; } + #endif + ]], [ testfunc(); ])], + [gcry_cv_cc_riscv_zicclsm_enabled=yes]) fi]) -AM_CONDITIONAL(SUPPORT_CC_RISCV_MSTRICT_ALIGN, - test "$gcry_cv_cc_riscv_mstrict_align" = "yes") - -# Restore flags. -CFLAGS=$_gcc_cflags_save; +if test "$gcry_cv_cc_riscv_zicclsm_enabled" = "yes" ; then + AC_DEFINE(HAVE_COMPATIBLE_CC_RISCV_ZICCLSM_ENABLED,1, + [Defined if underlying compiler has RISC-V Zicclsm extension enabled]) +fi # @@ -2896,7 +2894,10 @@ _gcc_cflags_save=$CFLAGS # better code generation for RISC-V vector implementations. MARCH_RVA22U64_WITH_VEC=rv64imafdcv_zba_zbb_zbs AC_SUBST([MARCH_RVA22U64_WITH_VEC]) -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) +# Note: -mstrict-align needed for SpacemiT K1 RVV bug (K1 does not support +# unaligned vector memory access). This is enabled only for plain RVV +# implementations, RVV-crypto extension implementations detect support for +# unaligned vector memory access based on Zicclsm extension. CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA22U64_WITH_VEC -mstrict-align" if test "$gcry_cv_cc_riscv_vector_intrinsics" = "no" && @@ -2992,6 +2993,13 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST], vuint32m1_t d = __riscv_vghsh_vv_u32m1(a, b, c, vl); __riscv_vse32_v_u32m1(ptr + 0 * vl, d, vl); } + void test_rev8(unsigned int *ptr) + { + int vl = __riscv_vsetvl_e32m1 (4); + vuint32m1_t a = __riscv_vle32_v_u32m1(ptr + 0 * vl, vl); + vuint32m1_t b = __riscv_vrev8_v_u32m1(a, vl); + __riscv_vse32_v_u32m1(ptr + 0 * vl, b, vl); + } void test_inline_vec_asm(unsigned int *ptr) { int vl = __riscv_vsetvl_e32m1 (4); @@ -3013,6 +3021,7 @@ m4_define([GCRY_RISCV_VECTOR_CRYPTO_INTRINSICS_TEST], test_aes_key(buf); test_aes_crypt(buf); test_ghash(buf); + test_rev8(buf); test_inline_vec_asm(buf); return 0; } @@ -3039,16 +3048,19 @@ fi _gcc_cflags_save=$CFLAGS # Enable B extension (Zba+Zbb+Zbs) to align with RVA23U64 profile and for # better code generation for RISC-V vector implementations. -MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC} -MARCH_RVA23U64_WITH_VEC_CRYPTO=${MARCH_RVA23U64_BASE}_zvbc_zvkg_zvkn_zvks +if test "$gcry_cv_cc_riscv_zicclsm_enabled" = "yes" ; then + MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC}_zicclsm +else + MARCH_RVA23U64_BASE=${MARCH_RVA22U64_WITH_VEC} +fi +MARCH_RVA23U64_WITH_VEC_CRYPTO=${MARCH_RVA23U64_BASE}_zvbc_zvkb_zvkg_zvkn_zvks AC_SUBST([MARCH_RVA23U64_BASE]) -# Note: -mstrict-align needed for GCC-14 bug (disable unaligned vector loads) -CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO -mstrict-align" +CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO" if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics" = "no" && test "$mpi_cpu_arch" = "riscv64" && test "$try_asm_modules" = "yes" ; then - AC_CACHE_CHECK([whether compiler supports RISC-V vector intrinsics with extra GCC flags], + AC_CACHE_CHECK([whether compiler supports RISC-V vector cryptography intrinsics with extra GCC flags], [gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags], [gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags=no AC_LINK_IFELSE( @@ -3082,7 +3094,7 @@ if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics" = "yes" || # Setup flags for test if needed. _gcc_cflags_save=$CFLAGS if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags" = "yes"; then - CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO -mstrict-align" + CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO" fi # Disable LTO optimization for following check to generate @@ -3131,7 +3143,7 @@ if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics" = "yes" || # Setup flags for test. _gcc_cflags_save=$CFLAGS if test "$gcry_cv_cc_riscv_vector_crypto_intrinsics_cflags" = "yes"; then - CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO -mstrict-align" + CFLAGS="$CFLAGS -O2 -march=$MARCH_RVA23U64_WITH_VEC_CRYPTO" else CFLAGS="$CFLAGS -O2" fi -- 2.53.0