[PATCH 4/4] aria: add x86_64 GFNI/AVX512 1-block implementation
Jussi Kivilinna
jussi.kivilinna at iki.fi
Fri Nov 6 09:07:23 CET 2026
* cipher/aria-gfni-avx512-amd64.S (_gcry_aria_gfni_avx512_crypt_blk1):
New.
* cipher/aria.c [USE_GFNI_AVX512] (_gcry_aria_gfni_avx512_crypt_blk1):
New prototype.
(aria_crypt_uses_sboxes) [USE_GFNI_AVX512]: Handle GFNI/AVX512
implementation.
(aria_crypt) [USE_GFNI_AVX512]: Use GFNI/AVX512 1-block implementation
if supported by CPU.
--
Benchmark on AMD Ryzen 9 9950X3D (zen5):
Generic C:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.26 ns/B 292.8 MiB/s 17.48 c/B 5365
CFB enc | 3.25 ns/B 293.2 MiB/s 17.52 c/B 5386±1
Before (GFNI/AVX):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 2.72 ns/B 351.2 MiB/s 14.13 c/B 5204
CFB enc | 2.72 ns/B 351.2 MiB/s 14.26 c/B 5252
After (~20% faster than GFNI/AVX, ~1.5x faster than generic C):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 2.15 ns/B 442.6 MiB/s 11.73 c/B 5442±2
CFB enc | 2.17 ns/B 439.0 MiB/s 11.85 c/B 5455±1
Benchmark on AMD Ryzen 9 7900X (zen4):
Generic C:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.87 ns/B 246.5 MiB/s 21.72 c/B 5614±10
CFB enc | 3.86 ns/B 247.3 MiB/s 21.46 c/B 5566
Before (GFNI/AVX):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 2.09 ns/B 455.6 MiB/s 11.51 c/B 5500
CFB enc | 2.09 ns/B 455.5 MiB/s 11.52 c/B 5500
After (~9% faster than GFNI/AVX, ~2.1x faster than generic C):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 1.92 ns/B 496.9 MiB/s 10.56 c/B 5502±1
CFB enc | 1.98 ns/B 482.1 MiB/s 10.90 c/B 5512±1
Benchmark on Intel Core i3-1115G4 (tigerlake):
Generic C:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 5.56 ns/B 171.4 MiB/s 22.76 c/B 4090
CFB enc | 5.57 ns/B 171.3 MiB/s 22.76 c/B 4090
Before (GFNI/AVX):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.04 ns/B 313.7 MiB/s 12.44 c/B 4090
CFB enc | 3.05 ns/B 312.7 MiB/s 12.47 c/B 4090
After (~15% faster than GFNI/AVX, ~2.1x faster than generic C):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 2.65 ns/B 359.5 MiB/s 10.85 c/B 4090
CFB enc | 2.66 ns/B 358.9 MiB/s 10.87 c/B 4090
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/aria-gfni-avx512-amd64.S | 172 ++++++++++++++++++++++++++++++++
cipher/aria.c | 14 +++
2 files changed, 186 insertions(+)
diff --git a/cipher/aria-gfni-avx512-amd64.S b/cipher/aria-gfni-avx512-amd64.S
index 47d93329..4fabe1bd 100644
--- a/cipher/aria-gfni-avx512-amd64.S
+++ b/cipher/aria-gfni-avx512-amd64.S
@@ -582,6 +582,38 @@ SECTION_RODATA
.byte 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08
.byte 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
+.align 16
+/* 1-block diffusion layer: first input byte from S1, S2, X1, X2 outputs,
+ * then second input byte from S1, S2, X1, X2 outputs. Odd rounds use
+ * same tables with S and X outputs swapped. */
+Lshuf_even_odd_blk1:
+ .byte 0x06, 0x02, 0x06, 0x0a, 0x02, 0x0a, 0x02, 0x06
+ .byte 0x0a, 0x06, 0x02, 0x02, 0x02, 0x06, 0x0e, 0x02
+ .byte 0x03, 0x07, 0x0b, 0x07, 0x0b, 0x03, 0x07, 0x03
+ .byte 0x07, 0x0b, 0x03, 0x03, 0x07, 0x03, 0x03, 0x0f
+ .byte 0x04, 0x08, 0x04, 0x00, 0x00, 0x04, 0x00, 0x08
+ .byte 0x00, 0x00, 0x08, 0x04, 0x0c, 0x00, 0x00, 0x04
+ .byte 0x09, 0x05, 0x01, 0x05, 0x05, 0x01, 0x09, 0x01
+ .byte 0x01, 0x01, 0x05, 0x09, 0x01, 0x0d, 0x05, 0x01
+ .byte 0x0e, 0x80, 0x0a, 0x0e, 0x0e, 0x0e, 0x0a, 0x80
+ .byte 0x80, 0x0e, 0x06, 0x0e, 0x06, 0x0a, 0x80, 0x0a
+ .byte 0x80, 0x0f, 0x0f, 0x0b, 0x0f, 0x0f, 0x80, 0x0b
+ .byte 0x0f, 0x80, 0x0f, 0x07, 0x0b, 0x07, 0x0b, 0x80
+ .byte 0x08, 0x0c, 0x0c, 0x80, 0x08, 0x80, 0x0c, 0x0c
+ .byte 0x04, 0x0c, 0x80, 0x0c, 0x80, 0x08, 0x04, 0x08
+ .byte 0x0d, 0x09, 0x80, 0x0d, 0x80, 0x09, 0x0d, 0x0d
+ .byte 0x0d, 0x05, 0x0d, 0x80, 0x09, 0x80, 0x09, 0x05
+/* 1-block last round: bytes from S1, S2, X1, X2 outputs. */
+.Lshuf_final_blk1:
+ .byte 0x80, 0x80, 0x02, 0x80, 0x80, 0x80, 0x06, 0x80
+ .byte 0x80, 0x80, 0x0a, 0x80, 0x80, 0x80, 0x0e, 0x80
+ .byte 0x80, 0x80, 0x80, 0x03, 0x80, 0x80, 0x80, 0x07
+ .byte 0x80, 0x80, 0x80, 0x0b, 0x80, 0x80, 0x80, 0x0f
+ .byte 0x00, 0x80, 0x80, 0x80, 0x04, 0x80, 0x80, 0x80
+ .byte 0x08, 0x80, 0x80, 0x80, 0x0c, 0x80, 0x80, 0x80
+ .byte 0x80, 0x01, 0x80, 0x80, 0x80, 0x05, 0x80, 0x80
+ .byte 0x80, 0x09, 0x80, 0x80, 0x80, 0x0d, 0x80, 0x80
+
.align 8
/* AES affine: */
#define tf_aff_const BV8(1, 1, 0, 0, 0, 1, 1, 0)
@@ -673,6 +705,10 @@ SECTION_RODATA
.text
+/**********************************************************************
+ 64-block parallel ARIA (GFNI/AVX512)
+ **********************************************************************/
+
.align 16
ELF(.type __aria_gfni_avx512_crypt_64way, at function;)
__aria_gfni_avx512_crypt_64way:
@@ -953,5 +989,141 @@ _gcry_aria_gfni_avx512_ctr_crypt_blk64:
ELF(.size _gcry_aria_gfni_avx512_ctr_crypt_blk64,
.-_gcry_aria_gfni_avx512_ctr_crypt_blk64;)
+/**********************************************************************
+ 1-block non-parallel ARIA (GFNI/AVX512)
+ **********************************************************************/
+
+#define RAFF %xmm8
+#define RS2 %xmm9
+#define RINV %xmm10
+#define RX2 %xmm11
+#define RID %xmm12
+
+#define SHUF_EVEN_0 %xmm13
+#define SHUF_EVEN_1 %xmm14
+#define SHUF_EVEN_2 %xmm15
+#define SHUF_EVEN_3 %xmm16
+#define SHUF_EVEN_4 %xmm17
+#define SHUF_EVEN_5 %xmm18
+#define SHUF_EVEN_6 %xmm19
+#define SHUF_EVEN_7 %xmm20
+#define SHUF_ODD_0 SHUF_EVEN_2
+#define SHUF_ODD_1 SHUF_EVEN_3
+#define SHUF_ODD_2 SHUF_EVEN_0
+#define SHUF_ODD_3 SHUF_EVEN_1
+#define SHUF_ODD_4 SHUF_EVEN_6
+#define SHUF_ODD_5 SHUF_EVEN_7
+#define SHUF_ODD_6 SHUF_EVEN_4
+#define SHUF_ODD_7 SHUF_EVEN_5
+
+#define preload_blk1_consts() \
+ vpbroadcastq .Ltf_aff_bitmatrix rRIP, RAFF; \
+ vpbroadcastq .Ltf_s2_bitmatrix rRIP, RS2; \
+ vpbroadcastq .Ltf_inv_bitmatrix rRIP, RINV; \
+ vpbroadcastq .Ltf_x2_bitmatrix rRIP, RX2; \
+ vpbroadcastq .Ltf_id_bitmatrix rRIP, RID; \
+ vmovdqa64 Lshuf_even_odd_blk1 + 0 * 16 rRIP, SHUF_EVEN_0; \
+ vmovdqa64 Lshuf_even_odd_blk1 + 1 * 16 rRIP, SHUF_EVEN_1; \
+ vmovdqa64 Lshuf_even_odd_blk1 + 2 * 16 rRIP, SHUF_EVEN_2; \
+ vmovdqa64 Lshuf_even_odd_blk1 + 3 * 16 rRIP, SHUF_EVEN_3; \
+ vmovdqa64 Lshuf_even_odd_blk1 + 4 * 16 rRIP, SHUF_EVEN_4; \
+ vmovdqa64 Lshuf_even_odd_blk1 + 5 * 16 rRIP, SHUF_EVEN_5; \
+ vmovdqa64 Lshuf_even_odd_blk1 + 6 * 16 rRIP, SHUF_EVEN_6; \
+ vmovdqa64 Lshuf_even_odd_blk1 + 7 * 16 rRIP, SHUF_EVEN_7;
+
+#define clear_blk1_consts() \
+ clear_vec4(%ymm16, %ymm17, %ymm18, %ymm19); \
+ vpxord %ymm20, %ymm20, %ymm20;
+
+#define aria_ark_blk1(x, rk, round) \
+ vpxor ((round) * 16)(rk), x, x;
+
+#define aria_sbox_blk1(x, s1, s2, x1, x2) \
+ vgf2p8affineqb $(tf_inv_const), RINV, x, x1; \
+ vgf2p8affineqb $(tf_x2_const), RX2, x, x2; \
+ vgf2p8affineinvqb $(tf_aff_const), RAFF, x, s1; \
+ vgf2p8affineinvqb $(tf_s2_const), RS2, x, s2; \
+ vgf2p8affineinvqb $0, RID, x1, x1; \
+ vgf2p8affineinvqb $0, RID, x2, x2;
+
+#define aria_diff_blk1(x, s1, s2, x1, x2, t0, t1, t2, shuf, rk, next_round) \
+ vpshufb shuf ## _0, s1, x; \
+ vpshufb shuf ## _4, s1, s1; \
+ vpshufb shuf ## _1, s2, t0; \
+ vpshufb shuf ## _5, s2, s2; \
+ vpshufb shuf ## _2, x1, t1; \
+ vpshufb shuf ## _6, x1, x1; \
+ vpshufb shuf ## _3, x2, t2; \
+ vpshufb shuf ## _7, x2, x2; \
+ vpternlogq $0x96, t0, s1, x; \
+ vpternlogq $0x96, ((next_round) * 16)(rk), s2, x; \
+ vpternlogq $0x96, t2, x1, t1; \
+ vpternlogq $0x96, x2, t1, x;
+
+#define aria_diff_final_blk1(x, s1, s2, x1, x2, rk, last_round) \
+ vpshufb .Lshuf_final_blk1 + 0 * 16 rRIP, s1, x; \
+ vpshufb .Lshuf_final_blk1 + 1 * 16 rRIP, s2, s2; \
+ vpshufb .Lshuf_final_blk1 + 2 * 16 rRIP, x1, x1; \
+ vpshufb .Lshuf_final_blk1 + 3 * 16 rRIP, x2, x2; \
+ vpternlogq $0x96, ((last_round) * 16)(rk), s2, x; \
+ vpternlogq $0x96, x2, x1, x;
+
+#define aria_fe_blk1(x, rk, next_round) \
+ aria_sbox_blk1(x, %xmm1, %xmm2, %xmm3, %xmm4); \
+ aria_diff_blk1(x, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, \
+ SHUF_EVEN, rk, next_round);
+
+#define aria_fo_blk1(x, rk, next_round) \
+ aria_sbox_blk1(x, %xmm1, %xmm2, %xmm3, %xmm4); \
+ aria_diff_blk1(x, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, \
+ SHUF_ODD, rk, next_round);
+
+#define aria_ff_blk1(x, rk, last_round) \
+ aria_sbox_blk1(x, %xmm1, %xmm2, %xmm3, %xmm4); \
+ aria_diff_final_blk1(x, %xmm1, %xmm2, %xmm3, %xmm4, rk, last_round);
+
+.align 16
+.globl _gcry_aria_gfni_avx512_crypt_blk1
+ELF(.type _gcry_aria_gfni_avx512_crypt_blk1, at function;)
+_gcry_aria_gfni_avx512_crypt_blk1:
+ /* input:
+ * %rdi: ctx, CTX
+ * %rsi: dst
+ * %rdx: src
+ * %rcx: round keys
+ */
+ CFI_STARTPROC();
+ spec_stop_avx512;
+
+ vmovdqu (%rdx), %xmm0;
+
+ preload_blk1_consts();
+
+ aria_ark_blk1(%xmm0, %rcx, 0);
+
+ movl ARIA_CTX_rounds(CTX), %r9d;
+ subl $2, %r9d;
+
+ aria_fo_blk1(%xmm0, %rcx, 1);
+ leaq 2*16(%rcx), %rcx;
+
+.align 16
+.Loop_blk1:
+ aria_fe_blk1(%xmm0, %rcx, 0);
+ aria_fo_blk1(%xmm0, %rcx, 1);
+ leaq 2*16(%rcx), %rcx;
+ subl $2, %r9d;
+ jnz .Loop_blk1;
+
+ aria_ff_blk1(%xmm0, %rcx, 0);
+
+ vmovdqu %xmm0, (%rsi);
+
+ clear_blk1_consts();
+ vzeroall;
+ ret_spec_stop;
+ CFI_ENDPROC();
+ELF(.size _gcry_aria_gfni_avx512_crypt_blk1,.-_gcry_aria_gfni_avx512_crypt_blk1;)
+
#endif /* ENABLE_AVX512_SUPPORT && ENABLE_GFNI_SUPPORT */
#endif /* __x86_64 */
diff --git a/cipher/aria.c b/cipher/aria.c
index 2f8d7685..85b972bd 100644
--- a/cipher/aria.c
+++ b/cipher/aria.c
@@ -575,6 +575,9 @@ aria_avx2_ctr_crypt_blk32(const ARIA_context *ctx, byte *out, const byte *in,
#endif /* USE_AESNI_AVX2 */
#ifdef USE_GFNI_AVX512
+extern void
+_gcry_aria_gfni_avx512_crypt_blk1(const void *ctx, byte *out, const byte *in,
+ const void *key) ASM_FUNC_ABI;
extern unsigned int
_gcry_aria_gfni_avx512_ecb_crypt_blk64(const void *ctx, byte *out,
const byte *in,
@@ -1001,6 +1004,10 @@ aria_set_decrypt_key(ARIA_context *ctx)
static inline int
aria_crypt_uses_sboxes(ARIA_context *ctx)
{
+#ifdef USE_GFNI_AVX512
+ if (ctx->use_gfni_avx512)
+ return 0;
+#endif
#ifdef USE_GFNI_AVX
if (ctx->use_gfni_avx)
return 0;
@@ -1021,6 +1028,13 @@ aria_crypt(ARIA_context *ctx, byte *out, const byte *in,
int rounds = ctx->rounds;
int rkidx = 0;
+#ifdef USE_GFNI_AVX512
+ if (ctx->use_gfni_avx512)
+ {
+ _gcry_aria_gfni_avx512_crypt_blk1(ctx, out, in, key);
+ return ASM_EXTRA_STACK;
+ }
+#endif
#ifdef USE_GFNI_AVX
if (ctx->use_gfni_avx)
{
--
2.53.0
More information about the Gcrypt-devel
mailing list