[PATCH 2/4] aria: add x86_64 AESNI/AVX 1-block implementation
Jussi Kivilinna
jussi.kivilinna at iki.fi
Fri Nov 6 09:07:21 CET 2026
* cipher/aria-aesni-avx-amd64.S (filter_8bit_split): New.
(filter_8bit): Use filter_8bit_split.
(_gcry_aria_aesni_avx_crypt_blk1): New.
* cipher/aria.c [USE_AESNI_AVX] (_gcry_aria_aesni_avx_crypt_blk1): New
prototype.
(aria_crypt_uses_sboxes): New.
(aria_crypt) [USE_AESNI_AVX]: Use AESNI/AVX 1-block implementation if
supported by CPU.
(aria_encrypt, aria_decrypt, _gcry_aria_cbc_enc, _gcry_aria_cfb_enc):
Prefetch sbox tables only if used.
--
Benchmark on AMD Ryzen 9 9950X3D (zen5):
Before:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.26 ns/B 292.8 MiB/s 17.48 c/B 5365
CFB enc | 3.25 ns/B 293.2 MiB/s 17.52 c/B 5386±1
After (~3% faster):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.17 ns/B 301.2 MiB/s 16.92 c/B 5343±1
CFB enc | 3.17 ns/B 300.5 MiB/s 16.99 c/B 5352±2
Benchmark on AMD Ryzen 9 7900X (zen4):
Before:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.87 ns/B 246.5 MiB/s 21.72 c/B 5614±10
CFB enc | 3.86 ns/B 247.3 MiB/s 21.46 c/B 5566
After (~1.4x faster):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 2.75 ns/B 346.6 MiB/s 15.13 c/B 5500
CFB enc | 2.73 ns/B 349.8 MiB/s 15.00 c/B 5500
Benchmark on AMD EPYC 7713 (zen3):
Before:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 6.66 ns/B 143.3 MiB/s 20.63 c/B 3100
CFB enc | 6.66 ns/B 143.1 MiB/s 20.66 c/B 3100
After (~1.4x faster):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 4.83 ns/B 197.5 MiB/s 14.97 c/B 3100
CFB enc | 4.84 ns/B 196.9 MiB/s 15.02 c/B 3100
Benchmark on AMD EPYC 7402P (zen2):
Before:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 6.43 ns/B 148.4 MiB/s 21.48 c/B 3344
CFB enc | 6.43 ns/B 148.4 MiB/s 21.49 c/B 3343
After (~1.6x faster):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.93 ns/B 242.6 MiB/s 13.14 c/B 3343
CFB enc | 3.94 ns/B 242.3 MiB/s 13.16 c/B 3344
Benchmark on AMD Ryzen 5 2600X (zen+):
Before:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 5.63 ns/B 169.4 MiB/s 22.83 c/B 4054
CFB enc | 5.68 ns/B 168.0 MiB/s 22.98 c/B 4049±1
After (~1.7x faster):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.37 ns/B 282.7 MiB/s 13.65 c/B 4045±3
CFB enc | 3.38 ns/B 281.9 MiB/s 13.72 c/B 4056
Benchmark on Intel Core i3-1115G4 (tigerlake):
Before:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 5.56 ns/B 171.4 MiB/s 22.76 c/B 4090
CFB enc | 5.57 ns/B 171.3 MiB/s 22.76 c/B 4090
After (~1.6x faster):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 3.51 ns/B 271.7 MiB/s 14.36 c/B 4090
CFB enc | 3.49 ns/B 273.3 MiB/s 14.27 c/B 4090
Benchmark on Intel Xeon E5-2620 v3 (haswell):
Before:
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 8.75 ns/B 109.0 MiB/s 22.72 c/B 2595
CFB enc | 8.75 ns/B 108.9 MiB/s 22.72 c/B 2595
After (~27% faster):
ARIA128 | nanosecs/byte mebibytes/sec cycles/byte auto Mhz
CBC enc | 6.87 ns/B 138.9 MiB/s 17.82 c/B 2595
CFB enc | 6.85 ns/B 139.2 MiB/s 17.77 c/B 2595
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/aria-aesni-avx-amd64.S | 225 +++++++++++++++++++++++++++++++++-
cipher/aria.c | 35 +++++-
2 files changed, 250 insertions(+), 10 deletions(-)
diff --git a/cipher/aria-aesni-avx-amd64.S b/cipher/aria-aesni-avx-amd64.S
index ee542b00..8d3a38f2 100644
--- a/cipher/aria-aesni-avx-amd64.S
+++ b/cipher/aria-aesni-avx-amd64.S
@@ -66,13 +66,17 @@
((l0) << (7 * 8)) )
/* asm macros */
-#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
- vpand x, mask4bit, tmp0; \
- vpandn x, mask4bit, x; \
- vpsrld $4, x, x; \
+/* lo_out must not alias x. */
+#define filter_8bit_split(lo_out, hi_out, x, lo_t, hi_t, mask4bit) \
+ vpand x, mask4bit, lo_out; \
+ vpandn x, mask4bit, hi_out; \
+ vpsrld $4, hi_out, hi_out; \
\
- vpshufb tmp0, lo_t, tmp0; \
- vpshufb x, hi_t, x; \
+ vpshufb lo_out, lo_t, lo_out; \
+ vpshufb hi_out, hi_t, hi_out;
+
+#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
+ filter_8bit_split(tmp0, x, x, lo_t, hi_t, mask4bit); \
vpxor tmp0, x, x;
#define transpose_4x4(x0, x1, x2, x3, t1, t2) \
@@ -816,6 +820,47 @@ SECTION_RODATA
.byte 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08
.byte 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
+/* 1-block diffusion layer, with AES (inverse) shift rows folded in. Tables
+ * shared by even and odd rounds (S1 and X1 outputs swapped in odd rounds),
+ * then S2 and X2 tables of even and odd rounds. */
+.Lshuf_even_odd_blk1:
+ .byte 0x0e, 0x0a, 0x0e, 0x02, 0x0a, 0x02, 0x0a, 0x0e
+ .byte 0x02, 0x0e, 0x0a, 0x0a, 0x0a, 0x0e, 0x06, 0x0a
+ .byte 0x04, 0x08, 0x04, 0x00, 0x00, 0x04, 0x00, 0x08
+ .byte 0x00, 0x00, 0x08, 0x04, 0x0c, 0x00, 0x00, 0x04
+ .byte 0x06, 0x80, 0x02, 0x06, 0x06, 0x06, 0x02, 0x80
+ .byte 0x80, 0x06, 0x0e, 0x06, 0x0e, 0x02, 0x80, 0x02
+ .byte 0x08, 0x0c, 0x0c, 0x80, 0x08, 0x80, 0x0c, 0x0c
+ .byte 0x04, 0x0c, 0x80, 0x0c, 0x80, 0x08, 0x04, 0x08
+.Lshuf_even_blk1:
+ .byte 0x07, 0x0b, 0x0f, 0x0b, 0x0f, 0x07, 0x0b, 0x07
+ .byte 0x0b, 0x0f, 0x07, 0x07, 0x0b, 0x07, 0x07, 0x03
+ .byte 0x0d, 0x09, 0x05, 0x09, 0x09, 0x05, 0x0d, 0x05
+ .byte 0x05, 0x05, 0x09, 0x0d, 0x05, 0x01, 0x09, 0x05
+ .byte 0x80, 0x03, 0x03, 0x0f, 0x03, 0x03, 0x80, 0x0f
+ .byte 0x03, 0x80, 0x03, 0x0b, 0x0f, 0x0b, 0x0f, 0x80
+ .byte 0x01, 0x0d, 0x80, 0x01, 0x80, 0x0d, 0x01, 0x01
+ .byte 0x01, 0x09, 0x01, 0x80, 0x0d, 0x80, 0x0d, 0x09
+.Lshuf_odd_blk1:
+ .byte 0x05, 0x01, 0x0d, 0x01, 0x01, 0x0d, 0x05, 0x0d
+ .byte 0x0d, 0x0d, 0x01, 0x05, 0x0d, 0x09, 0x01, 0x0d
+ .byte 0x0f, 0x03, 0x07, 0x03, 0x07, 0x0f, 0x03, 0x0f
+ .byte 0x03, 0x07, 0x0f, 0x0f, 0x03, 0x0f, 0x0f, 0x0b
+ .byte 0x09, 0x05, 0x80, 0x09, 0x80, 0x05, 0x09, 0x09
+ .byte 0x09, 0x01, 0x09, 0x80, 0x05, 0x80, 0x05, 0x01
+ .byte 0x80, 0x0b, 0x0b, 0x07, 0x0b, 0x0b, 0x80, 0x07
+ .byte 0x0b, 0x80, 0x0b, 0x03, 0x07, 0x03, 0x07, 0x80
+/* 1-block last round: bytes from S1, S2, X1, X2 outputs. */
+.Lshuf_final_blk1:
+ .byte 0x80, 0x80, 0x0a, 0x80, 0x80, 0x80, 0x0e, 0x80
+ .byte 0x80, 0x80, 0x02, 0x80, 0x80, 0x80, 0x06, 0x80
+ .byte 0x80, 0x80, 0x80, 0x07, 0x80, 0x80, 0x80, 0x0b
+ .byte 0x80, 0x80, 0x80, 0x0f, 0x80, 0x80, 0x80, 0x03
+ .byte 0x00, 0x80, 0x80, 0x80, 0x04, 0x80, 0x80, 0x80
+ .byte 0x08, 0x80, 0x80, 0x80, 0x0c, 0x80, 0x80, 0x80
+ .byte 0x80, 0x05, 0x80, 0x80, 0x80, 0x09, 0x80, 0x80
+ .byte 0x80, 0x0d, 0x80, 0x80, 0x80, 0x01, 0x80, 0x80
+
/* AES inverse affine and S2 combined:
* 1 1 0 0 0 0 0 1 x0 0
* 0 1 0 0 1 0 0 0 x1 0
@@ -953,6 +998,10 @@ SECTION_RODATA
.text
+/**********************************************************************
+ 16-block parallel ARIA (AESNI/AVX)
+ **********************************************************************/
+
.align 16
ELF(.type __aria_aesni_avx_crypt_16way, at function;)
__aria_aesni_avx_crypt_16way:
@@ -1228,6 +1277,11 @@ _gcry_aria_aesni_avx_ctr_crypt_blk16:
ELF(.size _gcry_aria_aesni_avx_ctr_crypt_blk16,.-_gcry_aria_aesni_avx_ctr_crypt_blk16;)
#ifdef CONFIG_AS_GFNI
+
+/**********************************************************************
+ 16-block parallel ARIA (GFNI/AVX)
+ **********************************************************************/
+
.align 16
ELF(.type __aria_gfni_avx_crypt_16way, at function;)
__aria_gfni_avx_crypt_16way:
@@ -1416,5 +1470,164 @@ _gcry_aria_gfni_avx_ctr_crypt_blk16:
ELF(.size _gcry_aria_gfni_avx_ctr_crypt_blk16,.-_gcry_aria_gfni_avx_ctr_crypt_blk16;)
#endif /* CONFIG_AS_GFNI */
+/**********************************************************************
+ 1-block non-parallel ARIA (AESNI/AVX)
+ **********************************************************************/
+
+#define MASK4BIT %xmm10
+#define TF_LO_S2 %xmm11
+#define TF_HI_S2 %xmm12
+#define TF_LO_X2 %xmm13
+#define TF_HI_X2 %xmm14
+#define ZERO %xmm15
+
+#define SHUF_EVEN_0 .Lshuf_even_odd_blk1 + 0 * 16 rRIP
+#define SHUF_EVEN_1 .Lshuf_even_blk1 + 0 * 16 rRIP
+#define SHUF_EVEN_2 .Lshuf_even_odd_blk1 + 1 * 16 rRIP
+#define SHUF_EVEN_3 .Lshuf_even_blk1 + 1 * 16 rRIP
+#define SHUF_EVEN_4 .Lshuf_even_odd_blk1 + 2 * 16 rRIP
+#define SHUF_EVEN_5 .Lshuf_even_blk1 + 2 * 16 rRIP
+#define SHUF_EVEN_6 .Lshuf_even_odd_blk1 + 3 * 16 rRIP
+#define SHUF_EVEN_7 .Lshuf_even_blk1 + 3 * 16 rRIP
+#define SHUF_ODD_0 SHUF_EVEN_2
+#define SHUF_ODD_1 .Lshuf_odd_blk1 + 0 * 16 rRIP
+#define SHUF_ODD_2 SHUF_EVEN_0
+#define SHUF_ODD_3 .Lshuf_odd_blk1 + 1 * 16 rRIP
+#define SHUF_ODD_4 SHUF_EVEN_6
+#define SHUF_ODD_5 .Lshuf_odd_blk1 + 2 * 16 rRIP
+#define SHUF_ODD_6 SHUF_EVEN_4
+#define SHUF_ODD_7 .Lshuf_odd_blk1 + 3 * 16 rRIP
+
+#define preload_blk1_consts() \
+ vbroadcastss .L0f0f0f0f rRIP, MASK4BIT; \
+ vmovdqa .Ltf_lo__inv_aff__and__s2 rRIP, TF_LO_S2; \
+ vmovdqa .Ltf_hi__inv_aff__and__s2 rRIP, TF_HI_S2; \
+ vmovdqa .Ltf_lo__x2__and__fwd_aff rRIP, TF_LO_X2; \
+ vmovdqa .Ltf_hi__x2__and__fwd_aff rRIP, TF_HI_X2; \
+ vpxor ZERO, ZERO, ZERO;
+
+#define aria_ark_blk1(x, rk, round) \
+ vpxor ((round) * 16)(rk), x, x;
+
+/* Outputs keep AES shift rows; S2 output left as lo/hi lookup halves. */
+#define aria_sbox_blk1(x, s1, s2lo, s2hi, x1, x2, t0) \
+ vaesenclast ZERO, x, s1; \
+ vaesdeclast ZERO, x, x1; \
+ \
+ /* affine transformation for S2 */ \
+ filter_8bit_split(s2lo, s2hi, s1, TF_LO_S2, TF_HI_S2, MASK4BIT); \
+ \
+ /* affine transformation for X2 */ \
+ filter_8bit(x, TF_LO_X2, TF_HI_X2, MASK4BIT, t0); \
+ vaesdeclast ZERO, x, x2;
+
+#define aria_diff_blk1(x, s1, s2lo, s2hi, x1, x2, t0, t1, t2, t3, \
+ shuf, rk, next_round) \
+ vpshufb shuf ## _0, s1, x; \
+ vpshufb shuf ## _4, s1, s1; \
+ vpshufb shuf ## _2, x1, t1; \
+ vpshufb shuf ## _6, x1, x1; \
+ vpshufb shuf ## _1, s2lo, t0; \
+ vpshufb shuf ## _5, s2lo, s2lo; \
+ vpshufb shuf ## _1, s2hi, t3; \
+ vpshufb shuf ## _5, s2hi, s2hi; \
+ vpshufb shuf ## _3, x2, t2; \
+ vpshufb shuf ## _7, x2, x2; \
+ vpxor s1, x, x; \
+ vpxor t1, x1, x1; \
+ vpxor ((next_round) * 16)(rk), x, x; \
+ vpxor x1, x, x; \
+ vpxor t0, s2lo, s2lo; \
+ vpxor s2lo, x, x; \
+ vpxor t3, s2hi, s2hi; \
+ vpxor s2hi, x, x; \
+ vpxor t2, x2, x2; \
+ vpxor x2, x, x;
+
+#define aria_diff_final_blk1(x, s1, s2lo, s2hi, x1, x2, rk, last_round) \
+ vpshufb .Lshuf_final_blk1 + 0 * 16 rRIP, s1, s1; \
+ vpshufb .Lshuf_final_blk1 + 2 * 16 rRIP, x1, x1; \
+ vpshufb .Lshuf_final_blk1 + 1 * 16 rRIP, s2lo, s2lo; \
+ vpshufb .Lshuf_final_blk1 + 1 * 16 rRIP, s2hi, s2hi; \
+ vpshufb .Lshuf_final_blk1 + 3 * 16 rRIP, x2, x2; \
+ vpxor ((last_round) * 16)(rk), s1, x; \
+ vpxor x1, x, x; \
+ vpxor s2lo, x, x; \
+ vpxor s2hi, x, x; \
+ vpxor x2, x, x;
+
+#define aria_fe_blk1(x, rk, next_round) \
+ aria_sbox_blk1(x, %xmm1, %xmm2, %xmm8, %xmm3, %xmm4, %xmm5); \
+ aria_diff_blk1(x, %xmm1, %xmm2, %xmm8, %xmm3, %xmm4, %xmm5, %xmm6, \
+ %xmm7, %xmm9, SHUF_EVEN, rk, next_round);
+
+#define aria_fo_blk1(x, rk, next_round) \
+ aria_sbox_blk1(x, %xmm1, %xmm2, %xmm8, %xmm3, %xmm4, %xmm5); \
+ aria_diff_blk1(x, %xmm1, %xmm2, %xmm8, %xmm3, %xmm4, %xmm5, %xmm6, \
+ %xmm7, %xmm9, SHUF_ODD, rk, next_round);
+
+#define aria_ff_blk1(x, rk, last_round) \
+ aria_sbox_blk1(x, %xmm1, %xmm2, %xmm8, %xmm3, %xmm4, %xmm5); \
+ aria_diff_final_blk1(x, %xmm1, %xmm2, %xmm8, %xmm3, %xmm4, rk, \
+ last_round);
+
+.align 16
+.globl _gcry_aria_aesni_avx_crypt_blk1
+ELF(.type _gcry_aria_aesni_avx_crypt_blk1, at function;)
+_gcry_aria_aesni_avx_crypt_blk1:
+ /* input:
+ * %rdi: ctx, CTX
+ * %rsi: dst
+ * %rdx: src
+ * %rcx: round keys
+ */
+ CFI_STARTPROC();
+
+ vmovdqu (%rdx), %xmm0;
+
+ preload_blk1_consts();
+
+ aria_ark_blk1(%xmm0, %rcx, 0);
+
+ movl ARIA_CTX_rounds(CTX), %r9d;
+ subl $2, %r9d;
+
+ aria_fo_blk1(%xmm0, %rcx, 1);
+ leaq 2*16(%rcx), %rcx;
+
+.align 16
+.Loop_aesni_blk1:
+ aria_fe_blk1(%xmm0, %rcx, 0);
+ aria_fo_blk1(%xmm0, %rcx, 1);
+ leaq 2*16(%rcx), %rcx;
+ subl $2, %r9d;
+ jnz .Loop_aesni_blk1;
+
+ aria_ff_blk1(%xmm0, %rcx, 0);
+
+ vmovdqu %xmm0, (%rsi);
+
+ vzeroall;
+ ret_spec_stop;
+ CFI_ENDPROC();
+ELF(.size _gcry_aria_aesni_avx_crypt_blk1,.-_gcry_aria_aesni_avx_crypt_blk1;)
+
+#undef SHUF_EVEN_0
+#undef SHUF_EVEN_1
+#undef SHUF_EVEN_2
+#undef SHUF_EVEN_3
+#undef SHUF_EVEN_4
+#undef SHUF_EVEN_5
+#undef SHUF_EVEN_6
+#undef SHUF_EVEN_7
+#undef SHUF_ODD_0
+#undef SHUF_ODD_1
+#undef SHUF_ODD_2
+#undef SHUF_ODD_3
+#undef SHUF_ODD_4
+#undef SHUF_ODD_5
+#undef SHUF_ODD_6
+#undef SHUF_ODD_7
+
#endif /* ENABLE_AVX_SUPPORT && ENABLE_AESNI_SUPPORT */
#endif /* __x86_64 */
diff --git a/cipher/aria.c b/cipher/aria.c
index 80fe185b..38d11e5a 100644
--- a/cipher/aria.c
+++ b/cipher/aria.c
@@ -449,6 +449,9 @@ static struct
};
#ifdef USE_AESNI_AVX
+extern void
+_gcry_aria_aesni_avx_crypt_blk1(const void *ctx, byte *out, const byte *in,
+ const void *key) ASM_FUNC_ABI;
extern unsigned int
_gcry_aria_aesni_avx_ecb_crypt_blk1_16(const void *ctx, byte *out,
const byte *in, const void *key,
@@ -991,6 +994,18 @@ aria_set_decrypt_key(ARIA_context *ctx)
}
}
+/* Returns true if aria_crypt uses sbox tables. */
+static inline int
+aria_crypt_uses_sboxes(ARIA_context *ctx)
+{
+#ifdef USE_AESNI_AVX
+ if (ctx->use_aesni_avx)
+ return 0;
+#endif
+ (void)ctx;
+ return 1;
+}
+
static NO_INLINE unsigned int
aria_crypt(ARIA_context *ctx, byte *out, const byte *in,
u32 key[][ARIA_RD_KEY_WORDS])
@@ -999,6 +1014,14 @@ aria_crypt(ARIA_context *ctx, byte *out, const byte *in,
int rounds = ctx->rounds;
int rkidx = 0;
+#ifdef USE_AESNI_AVX
+ if (ctx->use_aesni_avx)
+ {
+ _gcry_aria_aesni_avx_crypt_blk1(ctx, out, in, key);
+ return ASM_EXTRA_STACK;
+ }
+#endif
+
reg0 = buf_get_le32(in + 0);
reg1 = buf_get_le32(in + 4);
reg2 = buf_get_le32(in + 8);
@@ -1037,7 +1060,8 @@ aria_encrypt(void *c, byte *outbuf, const byte *inbuf)
{
ARIA_context *ctx = (ARIA_context *)c;
- prefetch_sboxes ();
+ if (aria_crypt_uses_sboxes (ctx))
+ prefetch_sboxes ();
return aria_crypt (ctx, outbuf, inbuf, ctx->enc_key);
}
@@ -1053,7 +1077,8 @@ aria_decrypt(void *c, byte *outbuf, const byte *inbuf)
ctx->decryption_prepared = 1;
}
- prefetch_sboxes ();
+ if (aria_crypt_uses_sboxes (ctx))
+ prefetch_sboxes ();
return aria_crypt (ctx, outbuf, inbuf, ctx->dec_key);
}
@@ -1323,7 +1348,8 @@ _gcry_aria_cbc_enc (void *context, unsigned char *iv,
unsigned char *last_iv;
unsigned int burn_depth = 0;
- prefetch_sboxes();
+ if (aria_crypt_uses_sboxes (ctx))
+ prefetch_sboxes();
last_iv = iv;
@@ -1396,7 +1422,8 @@ _gcry_aria_cfb_enc (void *context, unsigned char *iv,
const unsigned char *inbuf = inbuf_arg;
unsigned int burn_depth = 0;
- prefetch_sboxes();
+ if (aria_crypt_uses_sboxes (ctx))
+ prefetch_sboxes();
for (; nblocks; nblocks--)
{
--
2.53.0
More information about the Gcrypt-devel
mailing list