[PATCH 3/4] aria: add x86_64 GFNI/AVX 1-block implementation
Jussi Kivilinna
jussi.kivilinna at iki.fi
Fri Nov 6 09:07:22 CET 2026
* cipher/aria-aesni-avx-amd64.S (_gcry_aria_gfni_avx_crypt_blk1): New.
* cipher/aria.c [USE_GFNI_AVX] (_gcry_aria_gfni_avx_crypt_blk1): New
prototype.
(aria_crypt_uses_sboxes) [USE_GFNI_AVX]: Handle GFNI/AVX implementation.
(aria_crypt) [USE_GFNI_AVX]: Use GFNI/AVX 1-block implementation if
supported by CPU.
--
Benchmark on AMD Ryzen 9 9950X3D (zen5):
Generic C:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.26 ns/B 292.8 MiB/s 17.48 c/B 5365
CFB enc | 3.25 ns/B 293.2 MiB/s 17.52 c/B 5386±1
Before (AESNI/AVX):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.17 ns/B 301.2 MiB/s 16.92 c/B 5343±1
CFB enc | 3.17 ns/B 300.5 MiB/s 16.99 c/B 5352±2
After (~19% faster than AESNI/AVX, ~23% faster than generic C):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 2.62 ns/B 364.0 MiB/s 14.21 c/B 5425±2
CFB enc | 2.61 ns/B 365.7 MiB/s 14.17 c/B 5434±2
Benchmark on AMD Ryzen 9 7900X (zen4):
Generic C:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.87 ns/B 246.5 MiB/s 21.72 c/B 5614±10
CFB enc | 3.86 ns/B 247.3 MiB/s 21.46 c/B 5566
Before (AESNI/AVX):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 2.75 ns/B 346.6 MiB/s 15.13 c/B 5500
CFB enc | 2.73 ns/B 349.8 MiB/s 15.00 c/B 5500
After (~31% faster than AESNI/AVX, ~1.9x faster than generic C):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 2.09 ns/B 455.6 MiB/s 11.51 c/B 5500
CFB enc | 2.09 ns/B 455.5 MiB/s 11.52 c/B 5500
Benchmark on Intel Core i3-1115G4 (tigerlake):
Generic C:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 5.56 ns/B 171.4 MiB/s 22.76 c/B 4090
CFB enc | 5.57 ns/B 171.3 MiB/s 22.76 c/B 4090
Before (AESNI/AVX):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.51 ns/B 271.7 MiB/s 14.36 c/B 4090
CFB enc | 3.49 ns/B 273.3 MiB/s 14.27 c/B 4090
After (~14% faster than AESNI/AVX, ~1.8x faster than generic C):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.07 ns/B 311.1 MiB/s 12.54 c/B 4090
CFB enc | 3.08 ns/B 309.3 MiB/s 12.61 c/B 4090
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/aria-aesni-avx-amd64.S | 163 ++++++++++++++++++++++++++++++++++
cipher/aria.c | 14 +++
2 files changed, 177 insertions(+)
diff --git a/cipher/aria-aesni-avx-amd64.S b/cipher/aria-aesni-avx-amd64.S
index 8d3a38f2..22da7bf0 100644
--- a/cipher/aria-aesni-avx-amd64.S
+++ b/cipher/aria-aesni-avx-amd64.S
@@ -989,6 +989,37 @@ SECTION_RODATA
BV8(0, 0, 0, 0, 0, 1, 0, 0),
BV8(0, 0, 0, 0, 0, 0, 1, 0),
BV8(0, 0, 0, 0, 0, 0, 0, 1))
+
+.align 16
+/* 1-block diffusion layer for GFNI without shift rows. Odd rounds use same
+ * tables with S and X outputs swapped. */
+.Lshuf_even_odd_gfni_blk1:
+ .byte 0x06, 0x02, 0x06, 0x0a, 0x02, 0x0a, 0x02, 0x06
+ .byte 0x0a, 0x06, 0x02, 0x02, 0x02, 0x06, 0x0e, 0x02
+ .byte 0x03, 0x07, 0x0b, 0x07, 0x0b, 0x03, 0x07, 0x03
+ .byte 0x07, 0x0b, 0x03, 0x03, 0x07, 0x03, 0x03, 0x0f
+ .byte 0x04, 0x08, 0x04, 0x00, 0x00, 0x04, 0x00, 0x08
+ .byte 0x00, 0x00, 0x08, 0x04, 0x0c, 0x00, 0x00, 0x04
+ .byte 0x09, 0x05, 0x01, 0x05, 0x05, 0x01, 0x09, 0x01
+ .byte 0x01, 0x01, 0x05, 0x09, 0x01, 0x0d, 0x05, 0x01
+ .byte 0x0e, 0x80, 0x0a, 0x0e, 0x0e, 0x0e, 0x0a, 0x80
+ .byte 0x80, 0x0e, 0x06, 0x0e, 0x06, 0x0a, 0x80, 0x0a
+ .byte 0x80, 0x0f, 0x0f, 0x0b, 0x0f, 0x0f, 0x80, 0x0b
+ .byte 0x0f, 0x80, 0x0f, 0x07, 0x0b, 0x07, 0x0b, 0x80
+ .byte 0x08, 0x0c, 0x0c, 0x80, 0x08, 0x80, 0x0c, 0x0c
+ .byte 0x04, 0x0c, 0x80, 0x0c, 0x80, 0x08, 0x04, 0x08
+ .byte 0x0d, 0x09, 0x80, 0x0d, 0x80, 0x09, 0x0d, 0x0d
+ .byte 0x0d, 0x05, 0x0d, 0x80, 0x09, 0x80, 0x09, 0x05
+/* 1-block last round for GFNI. */
+.Lshuf_final_gfni_blk1:
+ .byte 0x80, 0x80, 0x02, 0x80, 0x80, 0x80, 0x06, 0x80
+ .byte 0x80, 0x80, 0x0a, 0x80, 0x80, 0x80, 0x0e, 0x80
+ .byte 0x80, 0x80, 0x80, 0x03, 0x80, 0x80, 0x80, 0x07
+ .byte 0x80, 0x80, 0x80, 0x0b, 0x80, 0x80, 0x80, 0x0f
+ .byte 0x00, 0x80, 0x80, 0x80, 0x04, 0x80, 0x80, 0x80
+ .byte 0x08, 0x80, 0x80, 0x80, 0x0c, 0x80, 0x80, 0x80
+ .byte 0x80, 0x01, 0x80, 0x80, 0x80, 0x05, 0x80, 0x80
+ .byte 0x80, 0x09, 0x80, 0x80, 0x80, 0x0d, 0x80, 0x80
#endif /* CONFIG_AS_GFNI */
/* 4-bit mask */
@@ -1629,5 +1660,137 @@ ELF(.size _gcry_aria_aesni_avx_crypt_blk1,.-_gcry_aria_aesni_avx_crypt_blk1;)
#undef SHUF_ODD_6
#undef SHUF_ODD_7
+#ifdef CONFIG_AS_GFNI
+/**********************************************************************
+ 1-block non-parallel ARIA (GFNI/AVX)
+ **********************************************************************/
+
+#define RAFF %xmm11
+#define RS2 %xmm12
+#define RINV %xmm13
+#define RX2 %xmm14
+#define RID %xmm15
+
+#define SHUF_EVEN_0 %xmm8
+#define SHUF_EVEN_1 .Lshuf_even_odd_gfni_blk1 + 1 * 16 rRIP
+#define SHUF_EVEN_2 %xmm9
+#define SHUF_EVEN_3 .Lshuf_even_odd_gfni_blk1 + 3 * 16 rRIP
+#define SHUF_EVEN_4 %xmm10
+#define SHUF_EVEN_5 .Lshuf_even_odd_gfni_blk1 + 5 * 16 rRIP
+#define SHUF_EVEN_6 .Lshuf_even_odd_gfni_blk1 + 6 * 16 rRIP
+#define SHUF_EVEN_7 .Lshuf_even_odd_gfni_blk1 + 7 * 16 rRIP
+#define SHUF_ODD_0 SHUF_EVEN_2
+#define SHUF_ODD_1 SHUF_EVEN_3
+#define SHUF_ODD_2 SHUF_EVEN_0
+#define SHUF_ODD_3 SHUF_EVEN_1
+#define SHUF_ODD_4 SHUF_EVEN_6
+#define SHUF_ODD_5 SHUF_EVEN_7
+#define SHUF_ODD_6 SHUF_EVEN_4
+#define SHUF_ODD_7 SHUF_EVEN_5
+
+#define preload_blk1_consts_gfni() \
+ vmovddup .Ltf_aff_bitmatrix rRIP, RAFF; \
+ vmovddup .Ltf_s2_bitmatrix rRIP, RS2; \
+ vmovddup .Ltf_inv_bitmatrix rRIP, RINV; \
+ vmovddup .Ltf_x2_bitmatrix rRIP, RX2; \
+ vmovddup .Ltf_id_bitmatrix rRIP, RID; \
+ vmovdqa .Lshuf_even_odd_gfni_blk1 + 0 * 16 rRIP, SHUF_EVEN_0; \
+ vmovdqa .Lshuf_even_odd_gfni_blk1 + 2 * 16 rRIP, SHUF_EVEN_2; \
+ vmovdqa .Lshuf_even_odd_gfni_blk1 + 4 * 16 rRIP, SHUF_EVEN_4;
+
+#define aria_sbox_blk1_gfni(x, s1, s2, x1, x2) \
+ vgf2p8affineqb $(tf_inv_const), RINV, x, x1; \
+ vgf2p8affineqb $(tf_x2_const), RX2, x, x2; \
+ vgf2p8affineinvqb $(tf_aff_const), RAFF, x, s1; \
+ vgf2p8affineinvqb $(tf_s2_const), RS2, x, s2; \
+ vgf2p8affineinvqb $0, RID, x1, x1; \
+ vgf2p8affineinvqb $0, RID, x2, x2;
+
+#define aria_diff_blk1_gfni(x, s1, s2, x1, x2, t0, t1, t2, \
+ shuf, rk, next_round) \
+ vpshufb shuf ## _0, s1, x; \
+ vpshufb shuf ## _4, s1, s1; \
+ vpshufb shuf ## _1, s2, t0; \
+ vpshufb shuf ## _5, s2, s2; \
+ vpshufb shuf ## _2, x1, t1; \
+ vpshufb shuf ## _6, x1, x1; \
+ vpshufb shuf ## _3, x2, t2; \
+ vpshufb shuf ## _7, x2, x2; \
+ vpxor s1, x, x; \
+ vpxor t0, s2, s2; \
+ vpxor ((next_round) * 16)(rk), x, x; \
+ vpxor s2, x, x; \
+ vpxor t1, x1, x1; \
+ vpxor t2, x2, x2; \
+ vpxor x1, x, x; \
+ vpxor x2, x, x;
+
+#define aria_diff_final_blk1_gfni(x, s1, s2, x1, x2, rk, last_round) \
+ vpshufb .Lshuf_final_gfni_blk1 + 0 * 16 rRIP, s1, s1; \
+ vpshufb .Lshuf_final_gfni_blk1 + 1 * 16 rRIP, s2, s2; \
+ vpshufb .Lshuf_final_gfni_blk1 + 2 * 16 rRIP, x1, x1; \
+ vpshufb .Lshuf_final_gfni_blk1 + 3 * 16 rRIP, x2, x2; \
+ vpxor ((last_round) * 16)(rk), s1, x; \
+ vpxor s2, x, x; \
+ vpxor x1, x, x; \
+ vpxor x2, x, x;
+
+#define aria_fe_blk1_gfni(x, rk, next_round) \
+ aria_sbox_blk1_gfni(x, %xmm1, %xmm2, %xmm3, %xmm4); \
+ aria_diff_blk1_gfni(x, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, \
+ %xmm7, SHUF_EVEN, rk, next_round);
+
+#define aria_fo_blk1_gfni(x, rk, next_round) \
+ aria_sbox_blk1_gfni(x, %xmm1, %xmm2, %xmm3, %xmm4); \
+ aria_diff_blk1_gfni(x, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, \
+ %xmm7, SHUF_ODD, rk, next_round);
+
+#define aria_ff_blk1_gfni(x, rk, last_round) \
+ aria_sbox_blk1_gfni(x, %xmm1, %xmm2, %xmm3, %xmm4); \
+ aria_diff_final_blk1_gfni(x, %xmm1, %xmm2, %xmm3, %xmm4, rk, \
+ last_round);
+
+.align 16
+.globl _gcry_aria_gfni_avx_crypt_blk1
+ELF(.type _gcry_aria_gfni_avx_crypt_blk1, at function;)
+_gcry_aria_gfni_avx_crypt_blk1:
+ /* input:
+ * %rdi: ctx, CTX
+ * %rsi: dst
+ * %rdx: src
+ * %rcx: round keys
+ */
+ CFI_STARTPROC();
+
+ vmovdqu (%rdx), %xmm0;
+
+ preload_blk1_consts_gfni();
+
+ aria_ark_blk1(%xmm0, %rcx, 0);
+
+ movl ARIA_CTX_rounds(CTX), %r9d;
+ subl $2, %r9d;
+
+ aria_fo_blk1_gfni(%xmm0, %rcx, 1);
+ leaq 2*16(%rcx), %rcx;
+
+.align 16
+.Loop_gfni_blk1:
+ aria_fe_blk1_gfni(%xmm0, %rcx, 0);
+ aria_fo_blk1_gfni(%xmm0, %rcx, 1);
+ leaq 2*16(%rcx), %rcx;
+ subl $2, %r9d;
+ jnz .Loop_gfni_blk1;
+
+ aria_ff_blk1_gfni(%xmm0, %rcx, 0);
+
+ vmovdqu %xmm0, (%rsi);
+
+ vzeroall;
+ ret_spec_stop;
+ CFI_ENDPROC();
+ELF(.size _gcry_aria_gfni_avx_crypt_blk1,.-_gcry_aria_gfni_avx_crypt_blk1;)
+#endif /* CONFIG_AS_GFNI */
+
#endif /* ENABLE_AVX_SUPPORT && ENABLE_AESNI_SUPPORT */
#endif /* __x86_64 */
diff --git a/cipher/aria.c b/cipher/aria.c
index 38d11e5a..2f8d7685 100644
--- a/cipher/aria.c
+++ b/cipher/aria.c
@@ -461,6 +461,9 @@ _gcry_aria_aesni_avx_ctr_crypt_blk16(const void *ctx, byte *out,
const byte *in, byte *iv) ASM_FUNC_ABI;
#ifdef USE_GFNI_AVX
+extern void
+_gcry_aria_gfni_avx_crypt_blk1(const void *ctx, byte *out, const byte *in,
+ const void *key) ASM_FUNC_ABI;
extern unsigned int
_gcry_aria_gfni_avx_ecb_crypt_blk1_16(const void *ctx, byte *out,
const byte *in, const void *key,
@@ -998,6 +1001,10 @@ aria_set_decrypt_key(ARIA_context *ctx)
static inline int
aria_crypt_uses_sboxes(ARIA_context *ctx)
{
+#ifdef USE_GFNI_AVX
+ if (ctx->use_gfni_avx)
+ return 0;
+#endif
#ifdef USE_AESNI_AVX
if (ctx->use_aesni_avx)
return 0;
@@ -1014,6 +1021,13 @@ aria_crypt(ARIA_context *ctx, byte *out, const byte *in,
int rounds = ctx->rounds;
int rkidx = 0;
+#ifdef USE_GFNI_AVX
+ if (ctx->use_gfni_avx)
+ {
+ _gcry_aria_gfni_avx_crypt_blk1(ctx, out, in, key);
+ return ASM_EXTRA_STACK;
+ }
+#endif
#ifdef USE_AESNI_AVX
if (ctx->use_aesni_avx)
{
--
2.53.0
More information about the Gcrypt-devel
mailing list