[PATCH 1/6] cipher: remove unneeded vzeroupper from AVX implementations
Jussi Kivilinna
jussi.kivilinna at iki.fi
Tue Sep 8 13:46:37 CEST 2026
* cipher/blake2b-amd64-avx2.S (_gcry_blake2b_transform_amd64_avx2): Remove
vzeroupper from function entry.
* cipher/blake2s-amd64-avx.S (_gcry_blake2s_transform_amd64_avx): Likewise.
* cipher/camellia-aesni-avx-amd64.S (_gcry_camellia_aesni_avx_ctr_enc)
(_gcry_camellia_aesni_avx_ecb_enc, _gcry_camellia_aesni_avx_ecb_dec)
(_gcry_camellia_aesni_avx_cbc_dec, _gcry_camellia_aesni_avx_cfb_dec)
(_gcry_camellia_aesni_avx_ocb_enc, _gcry_camellia_aesni_avx_ocb_dec)
(_gcry_camellia_aesni_avx_ocb_auth, _gcry_camellia_aesni_avx_keygen):
Likewise.
* cipher/chacha20-amd64-avx2.S (_gcry_chacha20_amd64_avx2_blocks8)
(_gcry_chacha20_poly1305_amd64_avx2_blocks8): Likewise.
* cipher/serpent-avx2-amd64.S (_gcry_serpent_avx2_ctr_enc)
(_gcry_serpent_avx2_cbc_dec, _gcry_serpent_avx2_cfb_dec)
(_gcry_serpent_avx2_ocb_enc, _gcry_serpent_avx2_ocb_dec)
(_gcry_serpent_avx2_ocb_auth): Likewise.
* cipher/sha1-avx-amd64.S (_gcry_sha1_transform_amd64_avx): Likewise.
* cipher/sha1-avx-bmi2-amd64.S (_gcry_sha1_transform_amd64_avx_bmi2):
Likewise.
* cipher/sha1-avx2-bmi2-amd64.S (_gcry_sha1_transform_amd64_avx2_bmi2):
Likewise.
* cipher/sha256-avx-amd64.S (_gcry_sha256_transform_amd64_avx): Likewise.
* cipher/sha256-avx2-bmi2-amd64.S (_gcry_sha256_transform_amd64_avx2):
Likewise.
* cipher/sha512-avx-amd64.S (_gcry_sha512_transform_amd64_avx): Likewise.
* cipher/sha512-avx2-bmi2-amd64.S (_gcry_sha512_transform_amd64_avx2):
Likewise.
* cipher/sm3-avx-bmi2-amd64.S (_gcry_sm3_transform_amd64_avx_bmi2):
Likewise.
* cipher/twofish-avx2-amd64.S (_gcry_twofish_avx2_blk16)
(_gcry_twofish_avx2_ctr_enc, _gcry_twofish_avx2_cbc_dec)
(_gcry_twofish_avx2_cfb_dec, _gcry_twofish_avx2_ocb_enc)
(_gcry_twofish_avx2_ocb_dec, _gcry_twofish_avx2_ocb_auth): Likewise.
* cipher/chacha20-amd64-avx512.S (_gcry_chacha20_amd64_avx512_blocks):
Remove vzeroupper from register clearing.
--
vzeroupper is needed only ahead of legacy SSE code, to avoid AVX-SSE
transition penalty. These implementations use VEX/EVEX encoding throughout
and clear upper register state with vzeroall before return.
Measured on Tiger Lake, leaving upper register state dirty costs 1.54x for
legacy SSE loop, but nothing for VEX encoded xmm/ymm code. Zen5 shows no
penalty for either.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/blake2b-amd64-avx2.S | 2 --
cipher/blake2s-amd64-avx.S | 2 --
cipher/camellia-aesni-avx-amd64.S | 18 ------------------
cipher/chacha20-amd64-avx2.S | 4 ----
cipher/chacha20-amd64-avx512.S | 1 -
cipher/serpent-avx2-amd64.S | 12 ------------
cipher/sha1-avx-amd64.S | 2 --
cipher/sha1-avx-bmi2-amd64.S | 2 --
cipher/sha1-avx2-bmi2-amd64.S | 2 --
cipher/sha256-avx-amd64.S | 1 -
cipher/sha256-avx2-bmi2-amd64.S | 2 --
cipher/sha512-avx-amd64.S | 2 --
cipher/sha512-avx2-bmi2-amd64.S | 2 --
cipher/sm3-avx-bmi2-amd64.S | 2 --
cipher/twofish-avx2-amd64.S | 14 --------------
15 files changed, 68 deletions(-)
diff --git a/cipher/blake2b-amd64-avx2.S b/cipher/blake2b-amd64-avx2.S
index 43c2cce1..02ee80a4 100644
--- a/cipher/blake2b-amd64-avx2.S
+++ b/cipher/blake2b-amd64-avx2.S
@@ -210,8 +210,6 @@ _gcry_blake2b_transform_amd64_avx2:
*/
CFI_STARTPROC();
- vzeroupper;
-
addq $128, (STATE_T + 0)(RSTATE);
adcq $0, (STATE_T + 8)(RSTATE);
diff --git a/cipher/blake2s-amd64-avx.S b/cipher/blake2s-amd64-avx.S
index 44b82ab2..d2df67d6 100644
--- a/cipher/blake2s-amd64-avx.S
+++ b/cipher/blake2s-amd64-avx.S
@@ -196,8 +196,6 @@ _gcry_blake2s_transform_amd64_avx:
*/
CFI_STARTPROC();
- vzeroupper;
-
addq $64, (STATE_T + 0)(RSTATE);
vmovdqa .Lshuf_ror16 rRIP, R16;
diff --git a/cipher/camellia-aesni-avx-amd64.S b/cipher/camellia-aesni-avx-amd64.S
index a29b9023..14e11d18 100644
--- a/cipher/camellia-aesni-avx-amd64.S
+++ b/cipher/camellia-aesni-avx-amd64.S
@@ -896,8 +896,6 @@ _gcry_camellia_aesni_avx_ctr_enc:
movq %rsp, %rbp;
CFI_DEF_CFA_REGISTER(%rbp);
- vzeroupper;
-
cmpl $128, key_bitlength(CTX);
movl $32, %r8d;
movl $24, %eax;
@@ -1053,8 +1051,6 @@ _gcry_camellia_aesni_avx_ecb_enc:
movq %rsp, %rbp;
CFI_DEF_CFA_REGISTER(%rbp);
- vzeroupper;
-
cmpl $128, key_bitlength(CTX);
movl $32, %r8d;
movl $24, %eax;
@@ -1099,8 +1095,6 @@ _gcry_camellia_aesni_avx_ecb_dec:
movq %rsp, %rbp;
CFI_DEF_CFA_REGISTER(%rbp);
- vzeroupper;
-
cmpl $128, key_bitlength(CTX);
movl $32, %r8d;
movl $24, %eax;
@@ -1146,8 +1140,6 @@ _gcry_camellia_aesni_avx_cbc_dec:
movq %rsp, %rbp;
CFI_DEF_CFA_REGISTER(%rbp);
- vzeroupper;
-
movq %rcx, %r9;
cmpl $128, key_bitlength(CTX);
@@ -1219,8 +1211,6 @@ _gcry_camellia_aesni_avx_cfb_dec:
movq %rsp, %rbp;
CFI_DEF_CFA_REGISTER(%rbp);
- vzeroupper;
-
cmpl $128, key_bitlength(CTX);
movl $32, %r8d;
movl $24, %eax;
@@ -1303,8 +1293,6 @@ _gcry_camellia_aesni_avx_ocb_enc:
movq %rsp, %rbp;
CFI_DEF_CFA_REGISTER(%rbp);
- vzeroupper;
-
subq $(16 * 16 + 4 * 8), %rsp;
andq $~31, %rsp;
movq %rsp, %rax;
@@ -1455,8 +1443,6 @@ _gcry_camellia_aesni_avx_ocb_dec:
movq %rsp, %rbp;
CFI_DEF_CFA_REGISTER(%rbp);
- vzeroupper;
-
subq $(16 * 16 + 4 * 8), %rsp;
andq $~31, %rsp;
movq %rsp, %rax;
@@ -1625,8 +1611,6 @@ _gcry_camellia_aesni_avx_ocb_auth:
movq %rsp, %rbp;
CFI_DEF_CFA_REGISTER(%rbp);
- vzeroupper;
-
subq $(16 * 16 + 4 * 8), %rsp;
andq $~31, %rsp;
movq %rsp, %rax;
@@ -2720,8 +2704,6 @@ _gcry_camellia_aesni_avx_keygen:
*/
CFI_STARTPROC();
- vzeroupper;
-
vmovdqu (%rsi), %xmm0;
cmpl $24, %edx;
jb __camellia_avx_setup128;
diff --git a/cipher/chacha20-amd64-avx2.S b/cipher/chacha20-amd64-avx2.S
index 54e2ffab..b2311ab5 100644
--- a/cipher/chacha20-amd64-avx2.S
+++ b/cipher/chacha20-amd64-avx2.S
@@ -184,8 +184,6 @@ _gcry_chacha20_amd64_avx2_blocks8:
*/
CFI_STARTPROC();
- vzeroupper;
-
pushq %rbp;
CFI_PUSH(%rbp);
movq %rsp, %rbp;
@@ -356,8 +354,6 @@ _gcry_chacha20_poly1305_amd64_avx2_blocks8:
movq %rsp, %rbp;
CFI_DEF_CFA_REGISTER(%rbp);
- vzeroupper;
-
subq $(9 * 8) + STACK_MAX + 32, %rsp;
andq $~31, %rsp;
diff --git a/cipher/chacha20-amd64-avx512.S b/cipher/chacha20-amd64-avx512.S
index 2d140815..6d421bf2 100644
--- a/cipher/chacha20-amd64-avx512.S
+++ b/cipher/chacha20-amd64-avx512.S
@@ -625,7 +625,6 @@ _gcry_chacha20_amd64_avx512_blocks:
.Lskip4v:
/* clear AVX512 registers */
kxorq %k2, %k2, %k2;
- vzeroupper;
clear_zmm16_zmm31();
.align 16
diff --git a/cipher/serpent-avx2-amd64.S b/cipher/serpent-avx2-amd64.S
index a419eb29..9ec31f17 100644
--- a/cipher/serpent-avx2-amd64.S
+++ b/cipher/serpent-avx2-amd64.S
@@ -645,8 +645,6 @@ _gcry_serpent_avx2_ctr_enc:
*/
CFI_STARTPROC();
- vzeroupper;
-
vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
vpcmpeqd RNOT, RNOT, RNOT;
vpsrldq $14, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */
@@ -718,8 +716,6 @@ _gcry_serpent_avx2_cbc_dec:
*/
CFI_STARTPROC();
- vzeroupper;
-
vmovdqu (0 * 32)(%rdx), RA0;
vmovdqu (1 * 32)(%rdx), RA1;
vmovdqu (2 * 32)(%rdx), RA2;
@@ -771,8 +767,6 @@ _gcry_serpent_avx2_cfb_dec:
*/
CFI_STARTPROC();
- vzeroupper;
-
/* Load input */
vmovdqu (%rcx), RNOTx;
vinserti128 $1, (%rdx), RNOT, RA0;
@@ -829,8 +823,6 @@ _gcry_serpent_avx2_ocb_enc:
*/
CFI_STARTPROC();
- vzeroupper;
-
subq $(4 * 8), %rsp;
CFI_ADJUST_CFA_OFFSET(4 * 8);
@@ -943,8 +935,6 @@ _gcry_serpent_avx2_ocb_dec:
*/
CFI_STARTPROC();
- vzeroupper;
-
subq $(4 * 8), %rsp;
CFI_ADJUST_CFA_OFFSET(4 * 8);
@@ -1066,8 +1056,6 @@ _gcry_serpent_avx2_ocb_auth:
*/
CFI_STARTPROC();
- vzeroupper;
-
subq $(4 * 8), %rsp;
CFI_ADJUST_CFA_OFFSET(4 * 8);
diff --git a/cipher/sha1-avx-amd64.S b/cipher/sha1-avx-amd64.S
index e5e55684..ca81506c 100644
--- a/cipher/sha1-avx-amd64.S
+++ b/cipher/sha1-avx-amd64.S
@@ -222,8 +222,6 @@ _gcry_sha1_transform_amd64_avx:
cmpq $0, %rdx;
jz .Lret;
- vzeroupper;
-
movq %rdx, RNBLKS;
movq %rdi, RSTATE;
movq %rsi, RDATA;
diff --git a/cipher/sha1-avx-bmi2-amd64.S b/cipher/sha1-avx-bmi2-amd64.S
index 16a01bfd..85ba22b0 100644
--- a/cipher/sha1-avx-bmi2-amd64.S
+++ b/cipher/sha1-avx-bmi2-amd64.S
@@ -222,8 +222,6 @@ _gcry_sha1_transform_amd64_avx_bmi2:
cmpq $0, %rdx;
jz .Lret;
- vzeroupper;
-
movq %rdx, RNBLKS;
movq %rdi, RSTATE;
movq %rsi, RDATA;
diff --git a/cipher/sha1-avx2-bmi2-amd64.S b/cipher/sha1-avx2-bmi2-amd64.S
index 06ff92f0..190b9731 100644
--- a/cipher/sha1-avx2-bmi2-amd64.S
+++ b/cipher/sha1-avx2-bmi2-amd64.S
@@ -224,8 +224,6 @@ _gcry_sha1_transform_amd64_avx2_bmi2:
*/
CFI_STARTPROC();
- vzeroupper;
-
movq %rdx, RNBLKS;
movq %rdi, RSTATE;
movq %rsi, RDATA;
diff --git a/cipher/sha256-avx-amd64.S b/cipher/sha256-avx-amd64.S
index 8cfd0880..9c15f481 100644
--- a/cipher/sha256-avx-amd64.S
+++ b/cipher/sha256-avx-amd64.S
@@ -345,7 +345,6 @@ ELF(.type _gcry_sha256_transform_amd64_avx, at function;)
.align 16
_gcry_sha256_transform_amd64_avx:
CFI_STARTPROC()
- vzeroupper
push rbx
CFI_PUSH(rbx)
diff --git a/cipher/sha256-avx2-bmi2-amd64.S b/cipher/sha256-avx2-bmi2-amd64.S
index e2a5454c..ec99e57f 100644
--- a/cipher/sha256-avx2-bmi2-amd64.S
+++ b/cipher/sha256-avx2-bmi2-amd64.S
@@ -268,8 +268,6 @@ _gcry_sha256_transform_amd64_avx2:
push r15
CFI_PUSH(r15)
- vzeroupper
-
vmovdqa BYTE_FLIP_MASK, [.LPSHUFFLE_BYTE_FLIP_MASK ADD_RIP]
vmovdqa SHUF_00BA, [.L_SHUF_00BA ADD_RIP]
vmovdqa SHUF_DC00, [.L_SHUF_DC00 ADD_RIP]
diff --git a/cipher/sha512-avx-amd64.S b/cipher/sha512-avx-amd64.S
index 1bd38060..22b1058e 100644
--- a/cipher/sha512-avx-amd64.S
+++ b/cipher/sha512-avx-amd64.S
@@ -254,8 +254,6 @@ _gcry_sha512_transform_amd64_avx:
cmp msglen, 0
je .Lnowork
- vzeroupper
-
/* Allocate Stack Space */
sub rsp, frame_size
CFI_ADJUST_CFA_OFFSET(frame_size);
diff --git a/cipher/sha512-avx2-bmi2-amd64.S b/cipher/sha512-avx2-bmi2-amd64.S
index 7b60bf1d..c2275bb5 100644
--- a/cipher/sha512-avx2-bmi2-amd64.S
+++ b/cipher/sha512-avx2-bmi2-amd64.S
@@ -282,8 +282,6 @@ _gcry_sha512_transform_amd64_avx2:
cmp rdx, 0
je .Lnowork
- vzeroupper
-
/* Allocate Stack Space */
mov rax, rsp
CFI_DEF_CFA_REGISTER(rax);
diff --git a/cipher/sm3-avx-bmi2-amd64.S b/cipher/sm3-avx-bmi2-amd64.S
index ef923165..9252e279 100644
--- a/cipher/sm3-avx-bmi2-amd64.S
+++ b/cipher/sm3-avx-bmi2-amd64.S
@@ -354,8 +354,6 @@ _gcry_sm3_transform_amd64_avx_bmi2:
*/
CFI_STARTPROC();
- vzeroupper;
-
pushq %rbp;
CFI_PUSH(%rbp);
movq %rsp, %rbp;
diff --git a/cipher/twofish-avx2-amd64.S b/cipher/twofish-avx2-amd64.S
index 72a39d17..c448dfae 100644
--- a/cipher/twofish-avx2-amd64.S
+++ b/cipher/twofish-avx2-amd64.S
@@ -519,8 +519,6 @@ _gcry_twofish_avx2_blk16:
*/
CFI_STARTPROC();
- vzeroupper;
-
vmovdqu (0 * 32)(%rdx), RA0;
vmovdqu (1 * 32)(%rdx), RB0;
vmovdqu (2 * 32)(%rdx), RC0;
@@ -565,8 +563,6 @@ _gcry_twofish_avx2_ctr_enc:
*/
CFI_STARTPROC();
- vzeroupper;
-
vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
vpcmpeqd RNOT, RNOT, RNOT;
vpsrldq $14, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */
@@ -638,8 +634,6 @@ _gcry_twofish_avx2_cbc_dec:
*/
CFI_STARTPROC();
- vzeroupper;
-
vmovdqu (0 * 32)(%rdx), RA0;
vmovdqu (1 * 32)(%rdx), RB0;
vmovdqu (2 * 32)(%rdx), RC0;
@@ -691,8 +685,6 @@ _gcry_twofish_avx2_cfb_dec:
*/
CFI_STARTPROC();
- vzeroupper;
-
/* Load input */
vmovdqu (%rcx), RNOTx;
vinserti128 $1, (%rdx), RNOT, RA0;
@@ -749,8 +741,6 @@ _gcry_twofish_avx2_ocb_enc:
*/
CFI_STARTPROC();
- vzeroupper;
-
subq $(4 * 8), %rsp;
CFI_ADJUST_CFA_OFFSET(4 * 8);
@@ -863,8 +853,6 @@ _gcry_twofish_avx2_ocb_dec:
*/
CFI_STARTPROC();
- vzeroupper;
-
subq $(4 * 8), %rsp;
CFI_ADJUST_CFA_OFFSET(4 * 8);
@@ -987,8 +975,6 @@ _gcry_twofish_avx2_ocb_auth:
*/
CFI_STARTPROC();
- vzeroupper;
-
subq $(4 * 8), %rsp;
CFI_ADJUST_CFA_OFFSET(4 * 8);
--
2.53.0
More information about the Gcrypt-devel
mailing list