[PATCH 2/4] Add straight-line speculation hardening for aarch64 assembly

Jussi Kivilinna jussi.kivilinna at iki.fi
Sat Jan 8 21:13:37 CET 2022


* cipher/asm-common-aarch64.h (ret_spec_stop): New.
* cipher/asm-poly1305-aarch64.h: Use 'ret_spec_stop' for 'ret'
instruction.
* cipher/camellia-aarch64.S: Likewise.
* cipher/chacha20-aarch64.S: Likewise.
* cipher/cipher-gcm-armv8-aarch64-ce.S: Likewise.
* cipher/crc-armv8-aarch64-ce.S: Likewise.
* cipher/rijndael-aarch64.S: Likewise.
* cipher/rijndael-armv8-aarch64-ce.S: Likewise.
* cipher/sha1-armv8-aarch64-ce.S: Likewise.
* cipher/sha256-armv8-aarch64-ce.S: Likewise.
* cipher/sm3-aarch64.S: Likewise.
* cipher/twofish-aarch64.S: Likewise.
* mpi/aarch64/mpih-add1.S: Likewise.
* mpi/aarch64/mpih-mul1.S: Likewise.
* mpi/aarch64/mpih-mul2.S: Likewise.
* mpi/aarch64/mpih-mul3.S: Likewise.
* mpi/aarch64/mpih-sub1.S: Likewise.
--

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/asm-common-aarch64.h          |  4 ++++
 cipher/asm-poly1305-aarch64.h        |  2 +-
 cipher/camellia-aarch64.S            |  6 +++---
 cipher/chacha20-aarch64.S            |  4 ++--
 cipher/cipher-gcm-armv8-aarch64-ce.S |  6 +++---
 cipher/crc-armv8-aarch64-ce.S        |  8 ++++----
 cipher/rijndael-aarch64.S            |  4 ++--
 cipher/rijndael-armv8-aarch64-ce.S   | 30 ++++++++++++++--------------
 cipher/sha1-armv8-aarch64-ce.S       |  2 +-
 cipher/sha256-armv8-aarch64-ce.S     |  2 +-
 cipher/sm3-aarch64.S                 |  2 +-
 cipher/twofish-aarch64.S             |  4 ++--
 mpi/aarch64/mpih-add1.S              |  2 +-
 mpi/aarch64/mpih-mul1.S              |  2 +-
 mpi/aarch64/mpih-mul2.S              |  2 +-
 mpi/aarch64/mpih-mul3.S              |  4 ++--
 mpi/aarch64/mpih-sub1.S              |  2 +-
 17 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/cipher/asm-common-aarch64.h b/cipher/asm-common-aarch64.h
index cf0afe1f..6ce773f2 100644
--- a/cipher/asm-common-aarch64.h
+++ b/cipher/asm-common-aarch64.h
@@ -101,4 +101,8 @@
 # define CFI_REG_ON_STACK(reg,rsp_offs)
 #endif
 
+/* 'ret' instruction replacement for straight-line speculation mitigation */
+#define ret_spec_stop \
+	ret; b .; dsb sy; isb;
+
 #endif /* GCRY_ASM_COMMON_AARCH64_H */
diff --git a/cipher/asm-poly1305-aarch64.h b/cipher/asm-poly1305-aarch64.h
index 90092709..2f05aae2 100644
--- a/cipher/asm-poly1305-aarch64.h
+++ b/cipher/asm-poly1305-aarch64.h
@@ -237,7 +237,7 @@ _gcry_poly1305_aarch64_blocks1:
 	mov x0, #0;
 
 	POLY1305_POP_REGS();
-	ret;
+	ret_spec_stop;
 	CFI_ENDPROC()
 ELF(.size _gcry_poly1305_aarch64_blocks1, .-_gcry_poly1305_aarch64_blocks1;)
 #endif
diff --git a/cipher/camellia-aarch64.S b/cipher/camellia-aarch64.S
index f4980862..30b568d3 100644
--- a/cipher/camellia-aarch64.S
+++ b/cipher/camellia-aarch64.S
@@ -238,7 +238,7 @@ _gcry_camellia_arm_encrypt_block:
 	CFI_ADJUST_CFA_OFFSET(-16)
 	CFI_RESTORE(x19)
 	CFI_RESTORE(x30)
-	ret;
+	ret_spec_stop;
 	CFI_RESTORE_STATE()
 .ltorg
 
@@ -252,7 +252,7 @@ _gcry_camellia_arm_encrypt_block:
 	CFI_ADJUST_CFA_OFFSET(-16)
 	CFI_RESTORE(x19)
 	CFI_RESTORE(x30)
-	ret;
+	ret_spec_stop;
 	CFI_ENDPROC()
 .ltorg
 ELF(.size _gcry_camellia_arm_encrypt_block,.-_gcry_camellia_arm_encrypt_block;)
@@ -299,7 +299,7 @@ _gcry_camellia_arm_decrypt_block:
 	CFI_ADJUST_CFA_OFFSET(-16)
 	CFI_RESTORE(x19)
 	CFI_RESTORE(x30)
-	ret;
+	ret_spec_stop;
 	CFI_RESTORE_STATE()
 .ltorg
 
diff --git a/cipher/chacha20-aarch64.S b/cipher/chacha20-aarch64.S
index 4f76834b..2a980b95 100644
--- a/cipher/chacha20-aarch64.S
+++ b/cipher/chacha20-aarch64.S
@@ -356,7 +356,7 @@ _gcry_chacha20_aarch64_blocks4:
 	clear(X15);
 
 	eor x0, x0, x0
-	ret
+	ret_spec_stop
 	CFI_ENDPROC()
 ELF(.size _gcry_chacha20_aarch64_blocks4, .-_gcry_chacha20_aarch64_blocks4;)
 
@@ -641,7 +641,7 @@ _gcry_chacha20_poly1305_aarch64_blocks4:
 
 	eor x0, x0, x0
 	POLY1305_POP_REGS()
-	ret
+	ret_spec_stop
 	CFI_ENDPROC()
 ELF(.size _gcry_chacha20_poly1305_aarch64_blocks4, .-_gcry_chacha20_poly1305_aarch64_blocks4;)
 
diff --git a/cipher/cipher-gcm-armv8-aarch64-ce.S b/cipher/cipher-gcm-armv8-aarch64-ce.S
index 2c619f9b..e6714249 100644
--- a/cipher/cipher-gcm-armv8-aarch64-ce.S
+++ b/cipher/cipher-gcm-armv8-aarch64-ce.S
@@ -365,7 +365,7 @@ _gcry_ghash_armv8_ce_pmull:
 
 .Ldo_nothing:
   mov x0, #0
-  ret
+  ret_spec_stop
   CFI_ENDPROC()
 ELF(.size _gcry_ghash_armv8_ce_pmull,.-_gcry_ghash_armv8_ce_pmull;)
 
@@ -593,7 +593,7 @@ _gcry_polyval_armv8_ce_pmull:
 
 .Lpolyval_do_nothing:
   mov x0, #0
-  ret
+  ret_spec_stop
   CFI_ENDPROC()
 ELF(.size _gcry_polyval_armv8_ce_pmull,.-_gcry_polyval_armv8_ce_pmull;)
 
@@ -645,7 +645,7 @@ _gcry_ghash_setup_armv8_ce_pmull:
   st1 {rh2.16b-rh4.16b}, [x1], #(3*16)
   st1 {rh5.16b-rh6.16b}, [x1]
 
-  ret
+  ret_spec_stop
   CFI_ENDPROC()
 ELF(.size _gcry_ghash_setup_armv8_ce_pmull,.-_gcry_ghash_setup_armv8_ce_pmull;)
 
diff --git a/cipher/crc-armv8-aarch64-ce.S b/cipher/crc-armv8-aarch64-ce.S
index 060abdfe..7ac884af 100644
--- a/cipher/crc-armv8-aarch64-ce.S
+++ b/cipher/crc-armv8-aarch64-ce.S
@@ -227,7 +227,7 @@ _gcry_crc32r_armv8_ce_bulk:
   /* store CRC */
   st1 {v0.s}[2], [x0]
 
-  ret
+  ret_spec_stop
   CFI_ENDPROC()
 ELF(.size _gcry_crc32r_armv8_ce_bulk,.-_gcry_crc32r_armv8_ce_bulk;)
 
@@ -260,7 +260,7 @@ _gcry_crc32r_armv8_ce_reduction_4:
 
   mov w0, v0.s[1]
 
-  ret
+  ret_spec_stop
   CFI_ENDPROC()
 ELF(.size _gcry_crc32r_armv8_ce_reduction_4,.-_gcry_crc32r_armv8_ce_reduction_4;)
 
@@ -457,7 +457,7 @@ _gcry_crc32_armv8_ce_bulk:
   rev32 v0.8b, v0.8b             /* byte swap */
   st1 {v0.s}[0], [x0]
 
-  ret
+  ret_spec_stop
   CFI_ENDPROC()
 ELF(.size _gcry_crc32_armv8_ce_bulk,.-_gcry_crc32_armv8_ce_bulk;)
 
@@ -490,7 +490,7 @@ _gcry_crc32_armv8_ce_reduction_4:
   rev32 v0.8b, v0.8b            /* Return in input endian */
   mov w0, v0.s[0]
 
-  ret
+  ret_spec_stop
   CFI_ENDPROC()
 ELF(.size _gcry_crc32_armv8_ce_reduction_4,.-_gcry_crc32_armv8_ce_reduction_4;)
 
diff --git a/cipher/rijndael-aarch64.S b/cipher/rijndael-aarch64.S
index e77dd4e0..184fcd20 100644
--- a/cipher/rijndael-aarch64.S
+++ b/cipher/rijndael-aarch64.S
@@ -263,7 +263,7 @@ _gcry_aes_arm_encrypt_block:
 	stp	RC, RD, [RDST, #8];
 
 	mov     x0, #(0);
-	ret;
+	ret_spec_stop;
 
 .ltorg
 .Lenc_not_128:
@@ -486,7 +486,7 @@ _gcry_aes_arm_decrypt_block:
 	stp	RC, RD, [RDST, #8];
 
 	mov     x0, #(0);
-	ret;
+	ret_spec_stop;
 
 .ltorg
 .Ldec_256:
diff --git a/cipher/rijndael-armv8-aarch64-ce.S b/cipher/rijndael-armv8-aarch64-ce.S
index 9f8d9d49..4fef0345 100644
--- a/cipher/rijndael-armv8-aarch64-ce.S
+++ b/cipher/rijndael-armv8-aarch64-ce.S
@@ -301,7 +301,7 @@ _gcry_aes_enc_armv8_ce:
   CLEAR_REG(v0)
 
   mov x0, #0
-  ret
+  ret_spec_stop
 
 .Lenc1_192:
   do_aes_one192(e, mc, v0, v0, vk0);
@@ -365,7 +365,7 @@ _gcry_aes_dec_armv8_ce:
   CLEAR_REG(v0)
 
   mov x0, #0
-  ret
+  ret_spec_stop
 
 .Ldec1_192:
   do_aes_one192(d, imc, v0, v0, vk0);
@@ -463,7 +463,7 @@ _gcry_aes_cbc_enc_armv8_ce:
   CLEAR_REG(v0)
 
 .Lcbc_enc_skip:
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_cbc_enc_armv8_ce,.-_gcry_aes_cbc_enc_armv8_ce;)
 
@@ -584,7 +584,7 @@ _gcry_aes_cbc_dec_armv8_ce:
   CFI_ADJUST_CFA_OFFSET(-64);
 
 .Lcbc_dec_skip:
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_cbc_dec_armv8_ce,.-_gcry_aes_cbc_dec_armv8_ce;)
 
@@ -777,7 +777,7 @@ _gcry_aes_ctr_enc_armv8_ce:
   CFI_ADJUST_CFA_OFFSET(-128);
 
 .Lctr_enc_skip:
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_ctr_enc_armv8_ce,.-_gcry_aes_ctr_enc_armv8_ce;)
 
@@ -924,7 +924,7 @@ _gcry_aes_ctr32le_enc_armv8_ce:
   CFI_ADJUST_CFA_OFFSET(-128);
 
 .Lctr32le_enc_skip:
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_ctr32le_enc_armv8_ce,.-_gcry_aes_ctr32le_enc_armv8_ce;)
 
@@ -1006,7 +1006,7 @@ _gcry_aes_cfb_enc_armv8_ce:
   CLEAR_REG(v4)
 
 .Lcfb_enc_skip:
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_cfb_enc_armv8_ce,.-_gcry_aes_cfb_enc_armv8_ce;)
 
@@ -1130,7 +1130,7 @@ _gcry_aes_cfb_dec_armv8_ce:
   CFI_ADJUST_CFA_OFFSET(-64);
 
 .Lcfb_dec_skip:
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_cfb_dec_armv8_ce,.-_gcry_aes_cfb_dec_armv8_ce;)
 
@@ -1379,7 +1379,7 @@ _gcry_aes_ocb_enc_armv8_ce:
   add sp, sp, #128;
   CFI_ADJUST_CFA_OFFSET(-128);
 
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_ocb_enc_armv8_ce,.-_gcry_aes_ocb_enc_armv8_ce;)
 
@@ -1458,7 +1458,7 @@ _gcry_aes_ocb_dec_armv8_ce:
   add sp, sp, #128;
   CFI_ADJUST_CFA_OFFSET(-128);
 
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_ocb_dec_armv8_ce,.-_gcry_aes_ocb_dec_armv8_ce;)
 
@@ -1605,7 +1605,7 @@ _gcry_aes_ocb_auth_armv8_ce:
   CLEAR_REG(v2)
   CLEAR_REG(v16)
 
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_ocb_auth_armv8_ce,.-_gcry_aes_ocb_auth_armv8_ce;)
 
@@ -1806,7 +1806,7 @@ _gcry_aes_xts_enc_armv8_ce:
   CFI_ADJUST_CFA_OFFSET(-128);
 
 .Lxts_enc_skip:
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_xts_enc_armv8_ce,.-_gcry_aes_xts_enc_armv8_ce;)
 
@@ -1874,7 +1874,7 @@ _gcry_aes_xts_dec_armv8_ce:
   CFI_ADJUST_CFA_OFFSET(-128);
 
 .Lxts_dec_skip:
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_xts_dec_armv8_ce,.-_gcry_aes_xts_dec_armv8_ce;)
 
@@ -1897,7 +1897,7 @@ _gcry_aes_sbox4_armv8_ce:
   addv s0, v0.4s
   mov w0, v0.S[0]
   CLEAR_REG(v0)
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_sbox4_armv8_ce,.-_gcry_aes_sbox4_armv8_ce;)
 
@@ -1914,7 +1914,7 @@ _gcry_aes_invmixcol_armv8_ce:
   aesimc v0.16b, v0.16b
   st1 {v0.16b}, [x0]
   CLEAR_REG(v0)
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_aes_invmixcol_armv8_ce,.-_gcry_aes_invmixcol_armv8_ce;)
 
diff --git a/cipher/sha1-armv8-aarch64-ce.S b/cipher/sha1-armv8-aarch64-ce.S
index 8ea1486b..ea26564b 100644
--- a/cipher/sha1-armv8-aarch64-ce.S
+++ b/cipher/sha1-armv8-aarch64-ce.S
@@ -194,7 +194,7 @@ _gcry_sha1_transform_armv8_ce:
 
 .Ldo_nothing:
   mov x0, #0
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_sha1_transform_armv8_ce,.-_gcry_sha1_transform_armv8_ce;)
 
diff --git a/cipher/sha256-armv8-aarch64-ce.S b/cipher/sha256-armv8-aarch64-ce.S
index 5c39e83e..d0fa6285 100644
--- a/cipher/sha256-armv8-aarch64-ce.S
+++ b/cipher/sha256-armv8-aarch64-ce.S
@@ -208,7 +208,7 @@ _gcry_sha256_transform_armv8_ce:
 
 .Ldo_nothing:
   mov x0, #0
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_sha256_transform_armv8_ce,.-_gcry_sha256_transform_armv8_ce;)
 
diff --git a/cipher/sm3-aarch64.S b/cipher/sm3-aarch64.S
index 77dba2ba..3fb89006 100644
--- a/cipher/sm3-aarch64.S
+++ b/cipher/sm3-aarch64.S
@@ -650,7 +650,7 @@ _gcry_sm3_transform_aarch64:
   CFI_ADJUST_CFA_OFFSET(-16);
   CFI_RESTORE(x28);
   CFI_RESTORE(x29);
-  ret
+  ret_spec_stop
   CFI_ENDPROC();
 ELF(.size _gcry_sm3_transform_aarch64, .-_gcry_sm3_transform_aarch64;)
 
diff --git a/cipher/twofish-aarch64.S b/cipher/twofish-aarch64.S
index 9f35b5cd..7941fe3a 100644
--- a/cipher/twofish-aarch64.S
+++ b/cipher/twofish-aarch64.S
@@ -262,7 +262,7 @@ _gcry_twofish_arm_encrypt_block:
 
 	str_output_le(RDST, RC, RD, RA, RB, RT0, RT1);
 
-	ret;
+	ret_spec_stop;
 	CFI_ENDPROC();
 .ltorg
 ELF(.size _gcry_twofish_arm_encrypt_block,.-_gcry_twofish_arm_encrypt_block;)
@@ -313,7 +313,7 @@ _gcry_twofish_arm_decrypt_block:
 
 	str_output_le(RDST, RA, RB, RC, RD, RT0, RT1);
 
-	ret;
+	ret_spec_stop;
 	CFI_ENDPROC();
 ELF(.size _gcry_twofish_arm_decrypt_block,.-_gcry_twofish_arm_decrypt_block;)
 
diff --git a/mpi/aarch64/mpih-add1.S b/mpi/aarch64/mpih-add1.S
index cc356bce..24859b17 100644
--- a/mpi/aarch64/mpih-add1.S
+++ b/mpi/aarch64/mpih-add1.S
@@ -69,6 +69,6 @@ C_SYMBOL_NAME(_gcry_mpih_add_n):
 
 .Lend:
 	adc	x0, xzr, xzr;
-	ret;
+	ret_spec_stop;
 	CFI_ENDPROC()
 ELF(.size C_SYMBOL_NAME(_gcry_mpih_add_n),.-C_SYMBOL_NAME(_gcry_mpih_add_n);)
diff --git a/mpi/aarch64/mpih-mul1.S b/mpi/aarch64/mpih-mul1.S
index 0db54444..f34c13c5 100644
--- a/mpi/aarch64/mpih-mul1.S
+++ b/mpi/aarch64/mpih-mul1.S
@@ -94,6 +94,6 @@ C_SYMBOL_NAME(_gcry_mpih_mul_1):
 
 .Lend:
 	mov	x0, x4;
-	ret;
+	ret_spec_stop;
 	CFI_ENDPROC()
 ELF(.size C_SYMBOL_NAME(_gcry_mpih_mul_1),.-C_SYMBOL_NAME(_gcry_mpih_mul_1);)
diff --git a/mpi/aarch64/mpih-mul2.S b/mpi/aarch64/mpih-mul2.S
index b4cc6eeb..1880999d 100644
--- a/mpi/aarch64/mpih-mul2.S
+++ b/mpi/aarch64/mpih-mul2.S
@@ -106,6 +106,6 @@ C_SYMBOL_NAME(_gcry_mpih_addmul_1):
 
 .Lend:
 	mov	x0, x6;
-	ret;
+	ret_spec_stop;
 	CFI_ENDPROC()
 ELF(.size C_SYMBOL_NAME(_gcry_mpih_addmul_1),.-C_SYMBOL_NAME(_gcry_mpih_addmul_1);)
diff --git a/mpi/aarch64/mpih-mul3.S b/mpi/aarch64/mpih-mul3.S
index 47a189b6..e5faeddc 100644
--- a/mpi/aarch64/mpih-mul3.S
+++ b/mpi/aarch64/mpih-mul3.S
@@ -115,10 +115,10 @@ C_SYMBOL_NAME(_gcry_mpih_submul_1):
 	cbnz	w2, .Large_loop;
 
 	mov	x0, x7;
-	ret;
+	ret_spec_stop;
 
 .Loop_end:
 	cinc	x0, x7, cc;
-	ret;
+	ret_spec_stop;
 	CFI_ENDPROC()
 ELF(.size C_SYMBOL_NAME(_gcry_mpih_submul_1),.-C_SYMBOL_NAME(_gcry_mpih_submul_1);)
diff --git a/mpi/aarch64/mpih-sub1.S b/mpi/aarch64/mpih-sub1.S
index 16b6c004..46908286 100644
--- a/mpi/aarch64/mpih-sub1.S
+++ b/mpi/aarch64/mpih-sub1.S
@@ -69,6 +69,6 @@ C_SYMBOL_NAME(_gcry_mpih_sub_n):
 
 .Lend:
 	cset	x0, cc;
-	ret;
+	ret_spec_stop;
 	CFI_ENDPROC()
 ELF(.size C_SYMBOL_NAME(_gcry_mpih_sub_n),.-C_SYMBOL_NAME(_gcry_mpih_sub_n);)
-- 
2.32.0




More information about the Gcrypt-devel mailing list