[PATCH 2/4] Add straight-line speculation hardening for aarch64 assembly
Jussi Kivilinna
jussi.kivilinna at iki.fi
Sat Jan 8 21:13:37 CET 2022
* cipher/asm-common-aarch64.h (ret_spec_stop): New.
* cipher/asm-poly1305-aarch64.h: Use 'ret_spec_stop' for 'ret'
instruction.
* cipher/camellia-aarch64.S: Likewise.
* cipher/chacha20-aarch64.S: Likewise.
* cipher/cipher-gcm-armv8-aarch64-ce.S: Likewise.
* cipher/crc-armv8-aarch64-ce.S: Likewise.
* cipher/rijndael-aarch64.S: Likewise.
* cipher/rijndael-armv8-aarch64-ce.S: Likewise.
* cipher/sha1-armv8-aarch64-ce.S: Likewise.
* cipher/sha256-armv8-aarch64-ce.S: Likewise.
* cipher/sm3-aarch64.S: Likewise.
* cipher/twofish-aarch64.S: Likewise.
* mpi/aarch64/mpih-add1.S: Likewise.
* mpi/aarch64/mpih-mul1.S: Likewise.
* mpi/aarch64/mpih-mul2.S: Likewise.
* mpi/aarch64/mpih-mul3.S: Likewise.
* mpi/aarch64/mpih-sub1.S: Likewise.
--
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/asm-common-aarch64.h | 4 ++++
cipher/asm-poly1305-aarch64.h | 2 +-
cipher/camellia-aarch64.S | 6 +++---
cipher/chacha20-aarch64.S | 4 ++--
cipher/cipher-gcm-armv8-aarch64-ce.S | 6 +++---
cipher/crc-armv8-aarch64-ce.S | 8 ++++----
cipher/rijndael-aarch64.S | 4 ++--
cipher/rijndael-armv8-aarch64-ce.S | 30 ++++++++++++++--------------
cipher/sha1-armv8-aarch64-ce.S | 2 +-
cipher/sha256-armv8-aarch64-ce.S | 2 +-
cipher/sm3-aarch64.S | 2 +-
cipher/twofish-aarch64.S | 4 ++--
mpi/aarch64/mpih-add1.S | 2 +-
mpi/aarch64/mpih-mul1.S | 2 +-
mpi/aarch64/mpih-mul2.S | 2 +-
mpi/aarch64/mpih-mul3.S | 4 ++--
mpi/aarch64/mpih-sub1.S | 2 +-
17 files changed, 45 insertions(+), 41 deletions(-)
diff --git a/cipher/asm-common-aarch64.h b/cipher/asm-common-aarch64.h
index cf0afe1f..6ce773f2 100644
--- a/cipher/asm-common-aarch64.h
+++ b/cipher/asm-common-aarch64.h
@@ -101,4 +101,8 @@
# define CFI_REG_ON_STACK(reg,rsp_offs)
#endif
+/* 'ret' instruction replacement for straight-line speculation mitigation */
+#define ret_spec_stop \
+ ret; b .; dsb sy; isb;
+
#endif /* GCRY_ASM_COMMON_AARCH64_H */
diff --git a/cipher/asm-poly1305-aarch64.h b/cipher/asm-poly1305-aarch64.h
index 90092709..2f05aae2 100644
--- a/cipher/asm-poly1305-aarch64.h
+++ b/cipher/asm-poly1305-aarch64.h
@@ -237,7 +237,7 @@ _gcry_poly1305_aarch64_blocks1:
mov x0, #0;
POLY1305_POP_REGS();
- ret;
+ ret_spec_stop;
CFI_ENDPROC()
ELF(.size _gcry_poly1305_aarch64_blocks1, .-_gcry_poly1305_aarch64_blocks1;)
#endif
diff --git a/cipher/camellia-aarch64.S b/cipher/camellia-aarch64.S
index f4980862..30b568d3 100644
--- a/cipher/camellia-aarch64.S
+++ b/cipher/camellia-aarch64.S
@@ -238,7 +238,7 @@ _gcry_camellia_arm_encrypt_block:
CFI_ADJUST_CFA_OFFSET(-16)
CFI_RESTORE(x19)
CFI_RESTORE(x30)
- ret;
+ ret_spec_stop;
CFI_RESTORE_STATE()
.ltorg
@@ -252,7 +252,7 @@ _gcry_camellia_arm_encrypt_block:
CFI_ADJUST_CFA_OFFSET(-16)
CFI_RESTORE(x19)
CFI_RESTORE(x30)
- ret;
+ ret_spec_stop;
CFI_ENDPROC()
.ltorg
ELF(.size _gcry_camellia_arm_encrypt_block,.-_gcry_camellia_arm_encrypt_block;)
@@ -299,7 +299,7 @@ _gcry_camellia_arm_decrypt_block:
CFI_ADJUST_CFA_OFFSET(-16)
CFI_RESTORE(x19)
CFI_RESTORE(x30)
- ret;
+ ret_spec_stop;
CFI_RESTORE_STATE()
.ltorg
diff --git a/cipher/chacha20-aarch64.S b/cipher/chacha20-aarch64.S
index 4f76834b..2a980b95 100644
--- a/cipher/chacha20-aarch64.S
+++ b/cipher/chacha20-aarch64.S
@@ -356,7 +356,7 @@ _gcry_chacha20_aarch64_blocks4:
clear(X15);
eor x0, x0, x0
- ret
+ ret_spec_stop
CFI_ENDPROC()
ELF(.size _gcry_chacha20_aarch64_blocks4, .-_gcry_chacha20_aarch64_blocks4;)
@@ -641,7 +641,7 @@ _gcry_chacha20_poly1305_aarch64_blocks4:
eor x0, x0, x0
POLY1305_POP_REGS()
- ret
+ ret_spec_stop
CFI_ENDPROC()
ELF(.size _gcry_chacha20_poly1305_aarch64_blocks4, .-_gcry_chacha20_poly1305_aarch64_blocks4;)
diff --git a/cipher/cipher-gcm-armv8-aarch64-ce.S b/cipher/cipher-gcm-armv8-aarch64-ce.S
index 2c619f9b..e6714249 100644
--- a/cipher/cipher-gcm-armv8-aarch64-ce.S
+++ b/cipher/cipher-gcm-armv8-aarch64-ce.S
@@ -365,7 +365,7 @@ _gcry_ghash_armv8_ce_pmull:
.Ldo_nothing:
mov x0, #0
- ret
+ ret_spec_stop
CFI_ENDPROC()
ELF(.size _gcry_ghash_armv8_ce_pmull,.-_gcry_ghash_armv8_ce_pmull;)
@@ -593,7 +593,7 @@ _gcry_polyval_armv8_ce_pmull:
.Lpolyval_do_nothing:
mov x0, #0
- ret
+ ret_spec_stop
CFI_ENDPROC()
ELF(.size _gcry_polyval_armv8_ce_pmull,.-_gcry_polyval_armv8_ce_pmull;)
@@ -645,7 +645,7 @@ _gcry_ghash_setup_armv8_ce_pmull:
st1 {rh2.16b-rh4.16b}, [x1], #(3*16)
st1 {rh5.16b-rh6.16b}, [x1]
- ret
+ ret_spec_stop
CFI_ENDPROC()
ELF(.size _gcry_ghash_setup_armv8_ce_pmull,.-_gcry_ghash_setup_armv8_ce_pmull;)
diff --git a/cipher/crc-armv8-aarch64-ce.S b/cipher/crc-armv8-aarch64-ce.S
index 060abdfe..7ac884af 100644
--- a/cipher/crc-armv8-aarch64-ce.S
+++ b/cipher/crc-armv8-aarch64-ce.S
@@ -227,7 +227,7 @@ _gcry_crc32r_armv8_ce_bulk:
/* store CRC */
st1 {v0.s}[2], [x0]
- ret
+ ret_spec_stop
CFI_ENDPROC()
ELF(.size _gcry_crc32r_armv8_ce_bulk,.-_gcry_crc32r_armv8_ce_bulk;)
@@ -260,7 +260,7 @@ _gcry_crc32r_armv8_ce_reduction_4:
mov w0, v0.s[1]
- ret
+ ret_spec_stop
CFI_ENDPROC()
ELF(.size _gcry_crc32r_armv8_ce_reduction_4,.-_gcry_crc32r_armv8_ce_reduction_4;)
@@ -457,7 +457,7 @@ _gcry_crc32_armv8_ce_bulk:
rev32 v0.8b, v0.8b /* byte swap */
st1 {v0.s}[0], [x0]
- ret
+ ret_spec_stop
CFI_ENDPROC()
ELF(.size _gcry_crc32_armv8_ce_bulk,.-_gcry_crc32_armv8_ce_bulk;)
@@ -490,7 +490,7 @@ _gcry_crc32_armv8_ce_reduction_4:
rev32 v0.8b, v0.8b /* Return in input endian */
mov w0, v0.s[0]
- ret
+ ret_spec_stop
CFI_ENDPROC()
ELF(.size _gcry_crc32_armv8_ce_reduction_4,.-_gcry_crc32_armv8_ce_reduction_4;)
diff --git a/cipher/rijndael-aarch64.S b/cipher/rijndael-aarch64.S
index e77dd4e0..184fcd20 100644
--- a/cipher/rijndael-aarch64.S
+++ b/cipher/rijndael-aarch64.S
@@ -263,7 +263,7 @@ _gcry_aes_arm_encrypt_block:
stp RC, RD, [RDST, #8];
mov x0, #(0);
- ret;
+ ret_spec_stop;
.ltorg
.Lenc_not_128:
@@ -486,7 +486,7 @@ _gcry_aes_arm_decrypt_block:
stp RC, RD, [RDST, #8];
mov x0, #(0);
- ret;
+ ret_spec_stop;
.ltorg
.Ldec_256:
diff --git a/cipher/rijndael-armv8-aarch64-ce.S b/cipher/rijndael-armv8-aarch64-ce.S
index 9f8d9d49..4fef0345 100644
--- a/cipher/rijndael-armv8-aarch64-ce.S
+++ b/cipher/rijndael-armv8-aarch64-ce.S
@@ -301,7 +301,7 @@ _gcry_aes_enc_armv8_ce:
CLEAR_REG(v0)
mov x0, #0
- ret
+ ret_spec_stop
.Lenc1_192:
do_aes_one192(e, mc, v0, v0, vk0);
@@ -365,7 +365,7 @@ _gcry_aes_dec_armv8_ce:
CLEAR_REG(v0)
mov x0, #0
- ret
+ ret_spec_stop
.Ldec1_192:
do_aes_one192(d, imc, v0, v0, vk0);
@@ -463,7 +463,7 @@ _gcry_aes_cbc_enc_armv8_ce:
CLEAR_REG(v0)
.Lcbc_enc_skip:
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_cbc_enc_armv8_ce,.-_gcry_aes_cbc_enc_armv8_ce;)
@@ -584,7 +584,7 @@ _gcry_aes_cbc_dec_armv8_ce:
CFI_ADJUST_CFA_OFFSET(-64);
.Lcbc_dec_skip:
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_cbc_dec_armv8_ce,.-_gcry_aes_cbc_dec_armv8_ce;)
@@ -777,7 +777,7 @@ _gcry_aes_ctr_enc_armv8_ce:
CFI_ADJUST_CFA_OFFSET(-128);
.Lctr_enc_skip:
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_ctr_enc_armv8_ce,.-_gcry_aes_ctr_enc_armv8_ce;)
@@ -924,7 +924,7 @@ _gcry_aes_ctr32le_enc_armv8_ce:
CFI_ADJUST_CFA_OFFSET(-128);
.Lctr32le_enc_skip:
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_ctr32le_enc_armv8_ce,.-_gcry_aes_ctr32le_enc_armv8_ce;)
@@ -1006,7 +1006,7 @@ _gcry_aes_cfb_enc_armv8_ce:
CLEAR_REG(v4)
.Lcfb_enc_skip:
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_cfb_enc_armv8_ce,.-_gcry_aes_cfb_enc_armv8_ce;)
@@ -1130,7 +1130,7 @@ _gcry_aes_cfb_dec_armv8_ce:
CFI_ADJUST_CFA_OFFSET(-64);
.Lcfb_dec_skip:
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_cfb_dec_armv8_ce,.-_gcry_aes_cfb_dec_armv8_ce;)
@@ -1379,7 +1379,7 @@ _gcry_aes_ocb_enc_armv8_ce:
add sp, sp, #128;
CFI_ADJUST_CFA_OFFSET(-128);
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_ocb_enc_armv8_ce,.-_gcry_aes_ocb_enc_armv8_ce;)
@@ -1458,7 +1458,7 @@ _gcry_aes_ocb_dec_armv8_ce:
add sp, sp, #128;
CFI_ADJUST_CFA_OFFSET(-128);
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_ocb_dec_armv8_ce,.-_gcry_aes_ocb_dec_armv8_ce;)
@@ -1605,7 +1605,7 @@ _gcry_aes_ocb_auth_armv8_ce:
CLEAR_REG(v2)
CLEAR_REG(v16)
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_ocb_auth_armv8_ce,.-_gcry_aes_ocb_auth_armv8_ce;)
@@ -1806,7 +1806,7 @@ _gcry_aes_xts_enc_armv8_ce:
CFI_ADJUST_CFA_OFFSET(-128);
.Lxts_enc_skip:
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_xts_enc_armv8_ce,.-_gcry_aes_xts_enc_armv8_ce;)
@@ -1874,7 +1874,7 @@ _gcry_aes_xts_dec_armv8_ce:
CFI_ADJUST_CFA_OFFSET(-128);
.Lxts_dec_skip:
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_xts_dec_armv8_ce,.-_gcry_aes_xts_dec_armv8_ce;)
@@ -1897,7 +1897,7 @@ _gcry_aes_sbox4_armv8_ce:
addv s0, v0.4s
mov w0, v0.S[0]
CLEAR_REG(v0)
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_sbox4_armv8_ce,.-_gcry_aes_sbox4_armv8_ce;)
@@ -1914,7 +1914,7 @@ _gcry_aes_invmixcol_armv8_ce:
aesimc v0.16b, v0.16b
st1 {v0.16b}, [x0]
CLEAR_REG(v0)
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_aes_invmixcol_armv8_ce,.-_gcry_aes_invmixcol_armv8_ce;)
diff --git a/cipher/sha1-armv8-aarch64-ce.S b/cipher/sha1-armv8-aarch64-ce.S
index 8ea1486b..ea26564b 100644
--- a/cipher/sha1-armv8-aarch64-ce.S
+++ b/cipher/sha1-armv8-aarch64-ce.S
@@ -194,7 +194,7 @@ _gcry_sha1_transform_armv8_ce:
.Ldo_nothing:
mov x0, #0
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_sha1_transform_armv8_ce,.-_gcry_sha1_transform_armv8_ce;)
diff --git a/cipher/sha256-armv8-aarch64-ce.S b/cipher/sha256-armv8-aarch64-ce.S
index 5c39e83e..d0fa6285 100644
--- a/cipher/sha256-armv8-aarch64-ce.S
+++ b/cipher/sha256-armv8-aarch64-ce.S
@@ -208,7 +208,7 @@ _gcry_sha256_transform_armv8_ce:
.Ldo_nothing:
mov x0, #0
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_sha256_transform_armv8_ce,.-_gcry_sha256_transform_armv8_ce;)
diff --git a/cipher/sm3-aarch64.S b/cipher/sm3-aarch64.S
index 77dba2ba..3fb89006 100644
--- a/cipher/sm3-aarch64.S
+++ b/cipher/sm3-aarch64.S
@@ -650,7 +650,7 @@ _gcry_sm3_transform_aarch64:
CFI_ADJUST_CFA_OFFSET(-16);
CFI_RESTORE(x28);
CFI_RESTORE(x29);
- ret
+ ret_spec_stop
CFI_ENDPROC();
ELF(.size _gcry_sm3_transform_aarch64, .-_gcry_sm3_transform_aarch64;)
diff --git a/cipher/twofish-aarch64.S b/cipher/twofish-aarch64.S
index 9f35b5cd..7941fe3a 100644
--- a/cipher/twofish-aarch64.S
+++ b/cipher/twofish-aarch64.S
@@ -262,7 +262,7 @@ _gcry_twofish_arm_encrypt_block:
str_output_le(RDST, RC, RD, RA, RB, RT0, RT1);
- ret;
+ ret_spec_stop;
CFI_ENDPROC();
.ltorg
ELF(.size _gcry_twofish_arm_encrypt_block,.-_gcry_twofish_arm_encrypt_block;)
@@ -313,7 +313,7 @@ _gcry_twofish_arm_decrypt_block:
str_output_le(RDST, RA, RB, RC, RD, RT0, RT1);
- ret;
+ ret_spec_stop;
CFI_ENDPROC();
ELF(.size _gcry_twofish_arm_decrypt_block,.-_gcry_twofish_arm_decrypt_block;)
diff --git a/mpi/aarch64/mpih-add1.S b/mpi/aarch64/mpih-add1.S
index cc356bce..24859b17 100644
--- a/mpi/aarch64/mpih-add1.S
+++ b/mpi/aarch64/mpih-add1.S
@@ -69,6 +69,6 @@ C_SYMBOL_NAME(_gcry_mpih_add_n):
.Lend:
adc x0, xzr, xzr;
- ret;
+ ret_spec_stop;
CFI_ENDPROC()
ELF(.size C_SYMBOL_NAME(_gcry_mpih_add_n),.-C_SYMBOL_NAME(_gcry_mpih_add_n);)
diff --git a/mpi/aarch64/mpih-mul1.S b/mpi/aarch64/mpih-mul1.S
index 0db54444..f34c13c5 100644
--- a/mpi/aarch64/mpih-mul1.S
+++ b/mpi/aarch64/mpih-mul1.S
@@ -94,6 +94,6 @@ C_SYMBOL_NAME(_gcry_mpih_mul_1):
.Lend:
mov x0, x4;
- ret;
+ ret_spec_stop;
CFI_ENDPROC()
ELF(.size C_SYMBOL_NAME(_gcry_mpih_mul_1),.-C_SYMBOL_NAME(_gcry_mpih_mul_1);)
diff --git a/mpi/aarch64/mpih-mul2.S b/mpi/aarch64/mpih-mul2.S
index b4cc6eeb..1880999d 100644
--- a/mpi/aarch64/mpih-mul2.S
+++ b/mpi/aarch64/mpih-mul2.S
@@ -106,6 +106,6 @@ C_SYMBOL_NAME(_gcry_mpih_addmul_1):
.Lend:
mov x0, x6;
- ret;
+ ret_spec_stop;
CFI_ENDPROC()
ELF(.size C_SYMBOL_NAME(_gcry_mpih_addmul_1),.-C_SYMBOL_NAME(_gcry_mpih_addmul_1);)
diff --git a/mpi/aarch64/mpih-mul3.S b/mpi/aarch64/mpih-mul3.S
index 47a189b6..e5faeddc 100644
--- a/mpi/aarch64/mpih-mul3.S
+++ b/mpi/aarch64/mpih-mul3.S
@@ -115,10 +115,10 @@ C_SYMBOL_NAME(_gcry_mpih_submul_1):
cbnz w2, .Large_loop;
mov x0, x7;
- ret;
+ ret_spec_stop;
.Loop_end:
cinc x0, x7, cc;
- ret;
+ ret_spec_stop;
CFI_ENDPROC()
ELF(.size C_SYMBOL_NAME(_gcry_mpih_submul_1),.-C_SYMBOL_NAME(_gcry_mpih_submul_1);)
diff --git a/mpi/aarch64/mpih-sub1.S b/mpi/aarch64/mpih-sub1.S
index 16b6c004..46908286 100644
--- a/mpi/aarch64/mpih-sub1.S
+++ b/mpi/aarch64/mpih-sub1.S
@@ -69,6 +69,6 @@ C_SYMBOL_NAME(_gcry_mpih_sub_n):
.Lend:
cset x0, cc;
- ret;
+ ret_spec_stop;
CFI_ENDPROC()
ELF(.size C_SYMBOL_NAME(_gcry_mpih_sub_n),.-C_SYMBOL_NAME(_gcry_mpih_sub_n);)
--
2.32.0
More information about the Gcrypt-devel
mailing list