[PATCH 5/5] camellia-gfni-avx512: speed up 1-block implementation
Jussi Kivilinna
jussi.kivilinna at iki.fi
Thu Oct 22 19:38:44 CEST 2026
* cipher/camellia-gfni-avx512-amd64.S (camellia_f_gfni)
(preload_camellia_f_consts, do_camellia_f)
(preload_camellia_key_consts, add_roundkey_blk1, do_fls_blk1): Remove.
(RPOST14, RPOST2, RPOST3, RBSWAP, RCM14, RCM2, RCM3, RM1A, RM1B, RM2A)
(RM2B, RM3A, RM3B, RM4A, RM4B, RPRE, RPINV, RSEL, RCX, RKC)
(preload_blk1_consts, to_prefiltered, from_prefiltered, p_xor_blk1)
(roundsm_blk1_first, roundsm_blk1_last): New.
(roundsm_blk1, two_roundsm_blk1, enc_rounds_blk1, dec_rounds_blk1)
(fls_blk1, inpack_blk1, outunpack_blk1, clear_regs_blk1): Keep block
halves in pre-filtered form.
(_gcry_camellia_gfni_avx512_enc_blk1)
(_gcry_camellia_gfni_avx512_dec_blk1): Update for above.
(.Lsp1mask_swap32_gfni, .Lsp2mask_swap32_gfni, .Lsp3mask_swap32_gfni)
(.Lsp4mask_swap32_gfni): Remove.
(.Lsp1mask_a_gfni, .Lsp1mask_b_gfni, .Lsp2mask_a_gfni, .Lsp2mask_b_gfni)
(.Lsp3mask_a_gfni, .Lsp3mask_b_gfni, .Lsp4mask_a_gfni, .Lsp4mask_b_gfni)
(.Lsel_s4_hi, .Lbswap32_dup, .Lpre_s123_s4, .Lpre_inv_s123_s4)
(.Lpre_inv_const, .Lcm_s14, .Lcm_s2, .Lcm_s3, .Lpost_const): New.
--
Round function of 1-block implementation had two dependent GFNI
instructions per sbox and P-function output folded from two halves,
with six GFNI instructions per round competing for same execution ports.
Now block halves are kept in pre-filtered form, with s4 pre-filter for
bytes going to s4, so that each sbox is single GF2P8AFFINEINVQB with
post-filter composed with pre-filters of next round. This gives three
GFNI instructions per round as s1 and s4 share same matrices. P-function
shuffles route contributions directly to output bytes. Conversions to
normal form for FL-layers and output are done in first and last round
of each six round block, in parallel with sbox computation.
Benchmark on AMD Ryzen 9 9950X3D (zen5):
Before:
CAMELLIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.72 ns/B 256.3 MiB/s 20.56 c/B 5527
CFB enc | 3.72 ns/B 256.2 MiB/s 20.57 c/B 5525
CAMELLIA256 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 4.98 ns/B 191.7 MiB/s 27.49 c/B 5524
CFB enc | 4.97 ns/B 191.8 MiB/s 27.62 c/B 5555
After (~16% faster):
CAMELLIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.22 ns/B 296.6 MiB/s 17.75 c/B 5519
CFB enc | 3.20 ns/B 298.2 MiB/s 17.79 c/B 5562
CAMELLIA256 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 4.25 ns/B 224.2 MiB/s 23.49 c/B 5520
CFB enc | 4.25 ns/B 224.3 MiB/s 23.61 c/B 5551
Benchmark on AMD Ryzen 9 7900X (zen4):
Before:
CAMELLIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.36 ns/B 283.7 MiB/s 18.98 c/B 5646±19
CFB enc | 3.36 ns/B 283.4 MiB/s 19.14 c/B 5687±4
CAMELLIA256 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 4.48 ns/B 213.0 MiB/s 25.47 c/B 5687±4
CFB enc | 4.48 ns/B 213.0 MiB/s 25.47 c/B 5687±4
After (~10% faster):
CAMELLIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.06 ns/B 311.7 MiB/s 17.21 c/B 5625
CFB enc | 3.06 ns/B 311.6 MiB/s 17.21 c/B 5625
CAMELLIA256 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 4.13 ns/B 231.0 MiB/s 23.48 c/B 5687±4
CFB enc | 4.18 ns/B 228.3 MiB/s 23.81 c/B 5699
Benchmark on Intel Core i3-1115G4 (tigerlake):
Before:
CAMELLIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 4.34 ns/B 219.6 MiB/s 17.76 c/B 4090
CFB enc | 4.36 ns/B 218.6 MiB/s 17.84 c/B 4090
CAMELLIA256 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 5.81 ns/B 164.1 MiB/s 23.77 c/B 4090
CFB enc | 5.82 ns/B 163.9 MiB/s 23.79 c/B 4090
After (~9% faster):
CAMELLIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 4.02 ns/B 237.2 MiB/s 16.45 c/B 4090
CFB enc | 4.01 ns/B 237.8 MiB/s 16.40 c/B 4090
CAMELLIA256 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 5.33 ns/B 179.0 MiB/s 21.79 c/B 4090
CFB enc | 5.32 ns/B 179.2 MiB/s 21.77 c/B 4090
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/camellia-gfni-avx512-amd64.S | 362 ++++++++++++++++++----------
1 file changed, 234 insertions(+), 128 deletions(-)
diff --git a/cipher/camellia-gfni-avx512-amd64.S b/cipher/camellia-gfni-avx512-amd64.S
index 4725b06a..8b03d136 100644
--- a/cipher/camellia-gfni-avx512-amd64.S
+++ b/cipher/camellia-gfni-avx512-amd64.S
@@ -691,19 +691,66 @@ ELF(.type _gcry_camellia_gfni_avx512__constants, at object;)
.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16
.align 16
-/* Shuffling constants for AVX512+GFNI 1-way variant. */
-.Lsp1mask_swap32_gfni:
- .byte 0xff, 0x04, 0x04, 0x04, 0xff, 0x04, 0x04, 0x04
+/* Constants for AVX512+GFNI 1-way variant. */
+
+/* P-function shuffles, first ('a') and second ('b') byte from each sbox. */
+.Lsp1mask_a_gfni:
+ .byte 0xff, 0x04, 0x04, 0x04, 0xff, 0x0c, 0x04, 0x04
+ .byte 0xff, 0x04, 0x04, 0x04, 0xff, 0x0c, 0x04, 0x04
+.Lsp1mask_b_gfni:
+ .byte 0xff, 0x03, 0x03, 0x03, 0x03, 0xff, 0xff, 0x03
.byte 0xff, 0x03, 0x03, 0x03, 0x03, 0xff, 0xff, 0x03
-.Lsp2mask_swap32_gfni:
- .byte 0x07, 0x07, 0x07, 0xff, 0x07, 0x07, 0x07, 0xff
- .byte 0x02, 0x02, 0x02, 0xff, 0xff, 0xff, 0x02, 0x02
-.Lsp3mask_swap32_gfni:
- .byte 0x06, 0x06, 0xff, 0x06, 0x06, 0x06, 0xff, 0x06
- .byte 0x01, 0x01, 0xff, 0x01, 0xff, 0x01, 0x01, 0xff
-.Lsp4mask_swap32_gfni:
- .byte 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff
- .byte 0x05, 0xff, 0x05, 0x05, 0x05, 0xff, 0x05, 0x05
+.Lsp2mask_a_gfni:
+ .byte 0x0f, 0x07, 0x07, 0xff, 0x07, 0x0f, 0x07, 0xff
+ .byte 0x0f, 0x07, 0x07, 0xff, 0x07, 0x0f, 0x07, 0xff
+.Lsp2mask_b_gfni:
+ .byte 0x0a, 0x02, 0x02, 0xff, 0xff, 0xff, 0x02, 0x02
+ .byte 0x0a, 0x02, 0x02, 0xff, 0xff, 0xff, 0x02, 0x02
+.Lsp3mask_a_gfni:
+ .byte 0x0e, 0x06, 0xff, 0x06, 0x06, 0x0e, 0xff, 0x06
+ .byte 0x0e, 0x06, 0xff, 0x06, 0x06, 0x0e, 0xff, 0x06
+.Lsp3mask_b_gfni:
+ .byte 0x09, 0x01, 0xff, 0x01, 0xff, 0x09, 0x01, 0xff
+ .byte 0x09, 0x01, 0xff, 0x01, 0xff, 0x09, 0x01, 0xff
+.Lsp4mask_a_gfni:
+ .byte 0x08, 0xff, 0x00, 0x00, 0x00, 0x08, 0xff, 0xff
+ .byte 0x08, 0xff, 0x00, 0x00, 0x00, 0x08, 0xff, 0xff
+.Lsp4mask_b_gfni:
+ .byte 0x0d, 0xff, 0x05, 0x05, 0x05, 0xff, 0x05, 0x05
+ .byte 0x0d, 0xff, 0x05, 0x05, 0x05, 0xff, 0x05, 0x05
+
+/* Bytes 0 and 5 from high quadword, others from low quadword. */
+.Lsel_s4_hi:
+ .byte 0x08, 0x01, 0x02, 0x03, 0x04, 0x0d, 0x06, 0x07
+ .byte 0x08, 0x01, 0x02, 0x03, 0x04, 0x0d, 0x06, 0x07
+
+/* Byte-swap 32-bit words of low quadword to both quadwords. */
+.Lbswap32_dup:
+ .byte 3, 2, 1, 0, 7, 6, 5, 4, 3, 2, 1, 0, 7, 6, 5, 4
+
+/* Pre-filters of s1/s2/s3 and s4. */
+.Lpre_s123_s4:
+ .quad 0xb74c0bcd30253461, 0xdb2685e618921ab0
+
+/* Inverse pre-filters of s1/s2/s3 and s4, and their constants. */
+.Lpre_inv_s123_s4:
+ .quad 0xed4b50a28d9df0e7, 0x4b50a28d9df0e7ed
+.Lpre_inv_const:
+ .byte 0xe2, 0xc5, 0xc5, 0xc5, 0xc5, 0xe2, 0xc5, 0xc5
+ .byte 0xe2, 0xc5, 0xc5, 0xc5, 0xc5, 0xe2, 0xc5, 0xc5
+
+/* Post-filters composed with pre-filters of s1/s2/s3 and s4. */
+.Lcm_s14:
+ .quad 0xa85b3ee30b877604, 0x8961c5a7a92fcf33
+.Lcm_s2:
+ .quad 0x8961c5a7a92fcf33, 0x9997a013a5a62557
+.Lcm_s3:
+ .quad 0xc7916a7784405ca0, 0xa85b3ee30b877604
+
+/* Post-filter constants through P-function and pre-filters. */
+.Lpost_const:
+ .byte 0x00, 0x00, 0x00, 0x00, 0xcf, 0x0a, 0x63, 0xcf
+ .byte 0x00, 0x00, 0x00, 0x00, 0xcf, 0x0a, 0x63, 0xcf
ELF(.size _gcry_camellia_gfni_avx512__constants,.-_gcry_camellia_gfni_avx512__constants;)
@@ -1590,117 +1637,180 @@ ELF(.size _gcry_camellia_gfni_avx512_dec_blk64,.-_gcry_camellia_gfni_avx512_dec_
1-block non-parallel camellia
**********************************************************************/
-/* Camellia F function, AVX512+GFNI version */
-#define camellia_f_gfni(ab, cd, x1, x2, x3, x4, \
- pre_filter_bitmatrix_s123, pre_filter_bitmatrix_s4, \
- post_filter_bitmatrix_s14, post_filter_bitmatrix_s2, \
- post_filter_bitmatrix_s3, sp1mask, sp2mask, sp3mask, \
- sp4mask) \
- /* camellia sboxes s1, s2, s3, s4 */ \
+#define RPOST14 %xmm10
+#define RPOST2 %xmm11
+#define RPOST3 %xmm12
+#define RBSWAP %xmm15
+#define RCM14 %xmm16
+#define RCM2 %xmm17
+#define RCM3 %xmm18
+#define RM1A %xmm19
+#define RM1B %xmm20
+#define RM2A %xmm21
+#define RM2B %xmm22
+#define RM3A %xmm23
+#define RM3B %xmm24
+#define RM4A %xmm25
+#define RM4B %xmm26
+#define RPRE %xmm27
+#define RPINV %xmm28
+#define RSEL %xmm29
+#define RCX %xmm30
+#define RKC %xmm31
+
+#define preload_blk1_consts() \
+ vpbroadcastq .Lpost_filter_bitmatrix_s14 rRIP, RPOST14; \
+ vpbroadcastq .Lpost_filter_bitmatrix_s2 rRIP, RPOST2; \
+ vpbroadcastq .Lpost_filter_bitmatrix_s3 rRIP, RPOST3; \
+ vmovdqa .Lbswap32_dup rRIP, RBSWAP; \
+ vmovdqa64 .Lcm_s14 rRIP, RCM14; \
+ vmovdqa64 .Lcm_s2 rRIP, RCM2; \
+ vmovdqa64 .Lcm_s3 rRIP, RCM3; \
+ vmovdqa64 .Lsp1mask_a_gfni rRIP, RM1A; \
+ vmovdqa64 .Lsp1mask_b_gfni rRIP, RM1B; \
+ vmovdqa64 .Lsp2mask_a_gfni rRIP, RM2A; \
+ vmovdqa64 .Lsp2mask_b_gfni rRIP, RM2B; \
+ vmovdqa64 .Lsp3mask_a_gfni rRIP, RM3A; \
+ vmovdqa64 .Lsp3mask_b_gfni rRIP, RM3B; \
+ vmovdqa64 .Lsp4mask_a_gfni rRIP, RM4A; \
+ vmovdqa64 .Lsp4mask_b_gfni rRIP, RM4B; \
+ vmovdqa64 .Lpre_s123_s4 rRIP, RPRE; \
+ vmovdqa64 .Lpre_inv_s123_s4 rRIP, RPINV; \
+ vmovdqa64 .Lsel_s4_hi rRIP, RSEL; \
+ vmovdqa64 .Lpre_inv_const rRIP, RCX; \
+ vmovdqa64 .Lpost_const rRIP, RKC; \
+ movl $0x5, %eax; \
+ kmovb %eax, %k2; \
+ movl $0xa, %eax; \
+ kmovb %eax, %k3;
+
+/* Normal form to pre-filtered form. */
+#define to_prefiltered(x) \
+ vgf2p8affineqb $(pre_filter_constant_s1234), RPRE, x, x; \
+ vpshufb RSEL, x, x;
+
+/* Pre-filtered form to normal form. */
+#define from_prefiltered(x) \
+ vgf2p8affineqb $0, RPINV, x, x; \
+ vpshufb RSEL, x, x; \
+ vpxorq RCX, x, x;
+
+/* P-function on sbox outputs in 'z1' (s1), 'z4' (s4), %xmm3 (s2) and %xmm4
+ * (s3), xored to 'cd'. */
+#define p_xor_blk1(z1, z4, cd) \
+ vpshufb RM1A, z1, %xmm5; \
+ vpshufb RM1B, z1, %xmm6; \
+ vpshufb RM4A, z4, %xmm7; \
+ vpshufb RM4B, z4, %xmm2; \
+ vpshufb RM2A, %xmm3, %xmm8; \
+ vpshufb RM2B, %xmm3, %xmm3; \
+ vpshufb RM3A, %xmm4, %xmm9; \
+ vpshufb RM3B, %xmm4, %xmm4; \
+ vpternlogq $0x96, %xmm7, %xmm6, %xmm5; \
+ vpternlogq $0x96, %xmm8, %xmm3, %xmm2; \
+ vpternlogq $0x96, %xmm9, %xmm4, cd; \
+ vpternlogq $0x96, %xmm5, %xmm2, cd;
+
+/* cd ^= F(ab, key), with 'cd' and 'ab' in pre-filtered form. */
+#define roundsm_blk1(ab, cd, key) \
+ vpbroadcastq key, %xmm9; \
+ vgf2p8affineqb $0, RPRE, %xmm9, %xmm9; \
+ vpshufb RSEL, %xmm9, %xmm9; \
+ vpternlogq $0x96, RKC, %xmm9, cd; \
+ \
+ vgf2p8affineinvqb $0, RCM14, ab, %xmm2; \
+ vgf2p8affineinvqb $0, RCM2, ab, %xmm3; \
+ vgf2p8affineinvqb $0, RCM3, ab, %xmm4; \
+ p_xor_blk1(%xmm2, %xmm2, cd);
+
+/* As roundsm_blk1, but 'cd' and 'ab' are converted from normal form. */
+#define roundsm_blk1_first(ab, cd, key) \
vgf2p8affineqb $(pre_filter_constant_s1234), \
- pre_filter_bitmatrix_s4, ab, x4; \
- vgf2p8affineinvqb $(post_filter_constant_s14), \
- post_filter_bitmatrix_s14, x4, x4; \
+ .Lpre_filter_bitmatrix_s123 rRIP {1to2}, ab, %xmm13; \
vgf2p8affineqb $(pre_filter_constant_s1234), \
- pre_filter_bitmatrix_s123, ab, x1; \
- vgf2p8affineinvqb $(post_filter_constant_s2), \
- post_filter_bitmatrix_s2, x1, x2; \
- vgf2p8affineinvqb $(post_filter_constant_s3), \
- post_filter_bitmatrix_s3, x1, x3; \
- vgf2p8affineinvqb $(post_filter_constant_s14), \
- post_filter_bitmatrix_s14, x1, x1; \
+ .Lpre_filter_bitmatrix_s4 rRIP {1to2}, ab, %xmm14; \
+ to_prefiltered(ab); \
+ to_prefiltered(cd); \
+ vpbroadcastq key, %xmm9; \
+ vgf2p8affineqb $0, RPRE, %xmm9, %xmm9; \
+ vpshufb RSEL, %xmm9, %xmm9; \
+ vpternlogq $0x96, RKC, %xmm9, cd; \
\
- /* permutation */ \
- vpshufb sp4mask, x4, x4; \
- vpshufb sp2mask, x2, x2; \
- vpshufb sp3mask, x3, x3; \
- vpshufb sp1mask, x1, x1; \
- vpxor x4, x2, x2; \
- vpternlogd $0x96, x3, x1, x2; \
- vpsrldq $8, x2, x1; \
+ vgf2p8affineinvqb $0, RCM14, %xmm13, %xmm2; \
+ vgf2p8affineinvqb $0, RCM14, %xmm14, %xmm14; \
+ vgf2p8affineinvqb $0, RCM2, %xmm13, %xmm3; \
+ vgf2p8affineinvqb $0, RCM3, %xmm13, %xmm4; \
+ p_xor_blk1(%xmm2, %xmm14, cd);
+
+/* As roundsm_blk1, but 'cd' and 'ab' are converted to normal form. */
+#define roundsm_blk1_last(ab, cd, key) \
+ vgf2p8affineqb $0, RPINV, cd, cd; \
+ vpshufb RSEL, cd, cd; \
+ vpternlogq $0x96, key{1to2}, RCX, cd; \
\
- /* output xor */ \
- vpternlogd $0x96, x2, x1, cd;
-
-#define preload_camellia_f_consts() \
- vmovq .Lpre_filter_bitmatrix_s123 rRIP, %xmm14; \
- vmovq .Lpre_filter_bitmatrix_s4 rRIP, %xmm13; \
- vmovq .Lpost_filter_bitmatrix_s14 rRIP, %xmm12; \
- vmovq .Lpost_filter_bitmatrix_s2 rRIP, %xmm11; \
- vmovq .Lpost_filter_bitmatrix_s3 rRIP, %xmm10; \
- vmovdqa .Lsp1mask_swap32_gfni rRIP, %xmm9; \
- vmovdqa .Lsp2mask_swap32_gfni rRIP, %xmm8; \
- vmovdqa .Lsp3mask_swap32_gfni rRIP, %xmm7; \
- vmovdqa .Lsp4mask_swap32_gfni rRIP, %xmm6;
-
-#define do_camellia_f(ab, cd, t0, t1, t2, t3) \
- camellia_f_gfni(ab, cd, t0, t1, t2, t3, \
- %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, \
- %xmm9, %xmm8, %xmm7, %xmm6);
-
-#define preload_camellia_key_consts() \
- kxnorb %k1, %k1, %k1; \
- vmovdqa .Lpack_bswap rRIP, %xmm15; \
- kshiftrb $7, %k1, %k1;
-
-#define add_roundkey_blk1(cd, key) \
- vpxorq key, cd, cd{%k1}{z};
-
-#define do_fls_blk1(ll, lr, rl, rr, t0, t1, kll, klr, krl, krr) \
- vpternlogd $0x1E, krr, rr, rl{%k1}{z}; \
- vpandd kll, ll, t0{%k1}{z}; \
- vpandd krl, rl, t1{%k1}{z}; \
- vprold $1, t0, t0; \
- vprold $1, t1, t1; \
- vpxor t0, lr, lr; \
- vpxor t1, rr, rr; \
- vpternlogd $0x1E, klr, lr, ll{%k1}{z};
-
-#define clear_regs_blk1() \
- kxorq %k1, %k1, %k1; \
- vzeroall;
-
-#define roundsm_blk1(ab, cd, t0, t1, t2, t3, key) \
- add_roundkey_blk1(cd, key); \
- do_camellia_f(ab, cd, t0, t1, t2, t3);
-
-#define two_roundsm_blk1(ab, cd, t0, t1, t2, t3, i, dir, ctx) \
- roundsm_blk1(ab, cd, t0, t1, t2, t3, \
- (key_table + (i) * 8)(ctx)); \
- roundsm_blk1(cd, ab, t0, t1, t2, t3, \
- (key_table + ((i) + (dir)) * 8)(ctx));
-
-#define enc_rounds_blk1(ab, cd, t0, t1, t2, t3, i, ctx) \
- two_roundsm_blk1(ab, cd, t0, t1, t2, t3, (i) + 2, 1, ctx); \
- two_roundsm_blk1(ab, cd, t0, t1, t2, t3, (i) + 4, 1, ctx); \
- two_roundsm_blk1(ab, cd, t0, t1, t2, t3, (i) + 6, 1, ctx);
-
-#define dec_rounds_blk1(ab, cd, t0, t1, t2, t3, i, ctx) \
- two_roundsm_blk1(ab, cd, t0, t1, t2, t3, (i) + 7, -1, ctx); \
- two_roundsm_blk1(ab, cd, t0, t1, t2, t3, (i) + 5, -1, ctx); \
- two_roundsm_blk1(ab, cd, t0, t1, t2, t3, (i) + 3, -1, ctx);
-
-#define fls_blk1(ll_lr, rl_rr, tmp_lr, tmp_rr, t0, t1, kll, klr, krl, krr) \
- vpsrlq $32, rl_rr, tmp_rr; \
- vpsrlq $32, ll_lr, tmp_lr; \
- do_fls_blk1(ll_lr, tmp_lr, rl_rr, tmp_rr, t0, t1, kll, klr, krl, krr); \
- vpunpckldq tmp_lr, ll_lr, ll_lr; \
- vpunpckldq tmp_rr, rl_rr, rl_rr;
+ vgf2p8affineinvqb $(post_filter_constant_s14), RPOST14, ab, %xmm2; \
+ vgf2p8affineinvqb $(post_filter_constant_s2), RPOST2, ab, %xmm3; \
+ vgf2p8affineinvqb $(post_filter_constant_s3), RPOST3, ab, %xmm4; \
+ from_prefiltered(ab); \
+ p_xor_blk1(%xmm2, %xmm2, cd);
+
+#define two_roundsm_blk1(ab, cd, i, dir, ctx) \
+ roundsm_blk1(ab, cd, (key_table + (i) * 8)(ctx)); \
+ roundsm_blk1(cd, ab, (key_table + ((i) + (dir)) * 8)(ctx));
+
+#define enc_rounds_blk1(ab, cd, i, ctx) \
+ roundsm_blk1_first(ab, cd, (key_table + ((i) + 2) * 8)(ctx)); \
+ roundsm_blk1(cd, ab, (key_table + ((i) + 3) * 8)(ctx)); \
+ two_roundsm_blk1(ab, cd, (i) + 4, 1, ctx); \
+ roundsm_blk1(ab, cd, (key_table + ((i) + 6) * 8)(ctx)); \
+ roundsm_blk1_last(cd, ab, (key_table + ((i) + 7) * 8)(ctx));
+
+#define dec_rounds_blk1(ab, cd, i, ctx) \
+ roundsm_blk1_first(ab, cd, (key_table + ((i) + 7) * 8)(ctx)); \
+ roundsm_blk1(cd, ab, (key_table + ((i) + 6) * 8)(ctx)); \
+ two_roundsm_blk1(ab, cd, (i) + 5, -1, ctx); \
+ roundsm_blk1(ab, cd, (key_table + ((i) + 3) * 8)(ctx)); \
+ roundsm_blk1_last(cd, ab, (key_table + ((i) + 2) * 8)(ctx));
+
+/* FL on 'll_lr' and FL^-1 on 'rl_rr', on 32-bit halves of each
+ * quadword. */
+#define fls_blk1(ll_lr, rl_rr, kll, klr, krl, krr) \
+ vpsrlq $32, ll_lr, %xmm2; \
+ vpandd kll{1to4}, ll_lr, %xmm3{%k2}{z}; \
+ vpsllq $32, ll_lr, %xmm4; \
+ vpsllq $32, rl_rr, %xmm5; \
+ vpsrlq $32, rl_rr, %xmm6; \
+ vprold $1, %xmm3, %xmm3; \
+ vpandd kll{1to4}, %xmm4, %xmm4; \
+ vpternlogd $0x1e, krr{1to4}, rl_rr, %xmm5; \
+ vpternlogd $0x1e, krr{1to4}, %xmm6, rl_rr{%k2}; \
+ vpxord %xmm3, %xmm2, %xmm2; \
+ vprold $1, %xmm4, %xmm4; \
+ vpandd krl{1to4}, %xmm5, %xmm5{%k3}{z}; \
+ vpternlogd $0x1e, klr{1to4}, %xmm2, ll_lr{%k2}; \
+ vprold $1, %xmm5, %xmm5; \
+ vpxord %xmm4, ll_lr, ll_lr; \
+ vpxord %xmm5, rl_rr, rl_rr;
#define inpack_blk1(ab, cd, src, key) \
- vmovq 0(src), ab; \
- vmovq 8(src), cd; \
- vpshufb %xmm15, ab, ab; \
- vpshufb %xmm15, cd, cd; \
- add_roundkey_blk1(ab, key);
+ vpbroadcastq 0(src), ab; \
+ vpbroadcastq 8(src), cd; \
+ vpshufb RBSWAP, ab, ab; \
+ vpshufb RBSWAP, cd, cd; \
+ vpxorq key{1to2}, ab, ab;
#define outunpack_blk1(ab, cd, dst, key) \
- add_roundkey_blk1(cd, key); \
- vpshufb %xmm15, ab, ab; \
- vpshufb %xmm15, cd, cd; \
+ vpxorq key{1to2}, cd, cd; \
+ vpshufb RBSWAP, ab, ab; \
+ vpshufb RBSWAP, cd, cd; \
vmovq ab, 8(dst); \
vmovq cd, 0(dst);
+#define clear_regs_blk1() \
+ kxorb %k2, %k2, %k2; \
+ kxorb %k3, %k3, %k3; \
+ vzeroall;
+
.align 16
.globl _gcry_camellia_gfni_avx512_enc_blk1
ELF(.type _gcry_camellia_gfni_avx512_enc_blk1, at function;)
@@ -1714,7 +1824,7 @@ _gcry_camellia_gfni_avx512_enc_blk1:
CFI_STARTPROC();
spec_stop_avx512;
- preload_camellia_key_consts();
+ preload_blk1_consts();
cmpl $128, key_bitlength(CTX);
movl $32, %r8d;
@@ -1723,23 +1833,21 @@ _gcry_camellia_gfni_avx512_enc_blk1:
inpack_blk1(%xmm0, %xmm1, %rdx, (key_table)(CTX));
- preload_camellia_f_consts();
-
leaq (-8 * 8)(CTX, %r8, 8), %r8;
.align 16
.Lenc_loop_blk1:
- enc_rounds_blk1(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, 0, CTX);
+ enc_rounds_blk1(%xmm0, %xmm1, 0, CTX);
cmpq %r8, CTX;
je .Lenc_done_blk1;
leaq (8 * 8)(CTX), CTX;
- fls_blk1(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5,
+ fls_blk1(%xmm0, %xmm1,
((key_table) + 0)(CTX),
- ((key_table) + 4)(CTX),
- ((key_table) + 8)(CTX),
- ((key_table) + 12)(CTX));
+ ((key_table) + 4)(CTX),
+ ((key_table) + 8)(CTX),
+ ((key_table) + 12)(CTX));
jmp .Lenc_loop_blk1;
@@ -1766,7 +1874,7 @@ _gcry_camellia_gfni_avx512_dec_blk1:
CFI_STARTPROC();
spec_stop_avx512;
- preload_camellia_key_consts();
+ preload_blk1_consts();
cmpl $128, key_bitlength(CTX);
movl $32, %r8d;
@@ -1775,22 +1883,20 @@ _gcry_camellia_gfni_avx512_dec_blk1:
inpack_blk1(%xmm0, %xmm1, %rdx, (key_table)(CTX, %r8, 8));
- preload_camellia_f_consts();
-
leaq (-8 * 8)(CTX, %r8, 8), %rax;
.align 16
.Ldec_loop_blk1:
- dec_rounds_blk1(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, 0, %rax);
+ dec_rounds_blk1(%xmm0, %xmm1, 0, %rax);
cmpq CTX, %rax;
je .Ldec_done_blk1;
- fls_blk1(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5,
+ fls_blk1(%xmm0, %xmm1,
((key_table) + 8)(%rax),
- ((key_table) + 12)(%rax),
- ((key_table) + 0)(%rax),
- ((key_table) + 4)(%rax));
+ ((key_table) + 12)(%rax),
+ ((key_table) + 0)(%rax),
+ ((key_table) + 4)(%rax));
leaq (-8 * 8)(%rax), %rax;
jmp .Ldec_loop_blk1;
--
2.53.0
More information about the Gcrypt-devel
mailing list