[PATCH v2] mceliece6688128f: fix UBSAN runtime errors

[Jussi Kivilinna]

* cipher/mceliece6688128f.c (cbrecursion): Fix left shift of signed integer
by too many places, by casting to unsigned integer before shift.
(syndrome): Fix misaligned memory loads by using 'bufhelp.h'.
--

There was multiple undefined behaviour sanitizer warnings with mceliece668128f
which have been fixed by this commit:

  $ tests/t-kem
  ../../cipher/mceliece6688128f.c:1766:37: runtime error: left shift of 73010 by 16 places cannot be represented in type 'int'
  ../../cipher/mceliece6688128f.c:1769:37: runtime error: left shift of 71034 by 16 places cannot be represented in type 'int'
  ../../cipher/mceliece6688128f.c:1792:39: runtime error: left shift of 72996 by 16 places cannot be represented in type 'int'
  ../../cipher/mceliece6688128f.c:1800:41: runtime error: left shift of 71344 by 16 places cannot be represented in type 'int'
  ../../cipher/mceliece6688128f.c:1807:43: runtime error: left shift of 68334 by 16 places cannot be represented in type 'int'
  ../../cipher/mceliece6688128f.c:1834:23: runtime error: left shift of 72247 by 16 places cannot be represented in type 'int'
  ../../cipher/mceliece6688128f.c:1833:19: runtime error: left shift of 136871 by 16 places cannot be represented in type 'int'
  ../../cipher/mceliece6688128f.c:1781:41: runtime error: left shift of 66551 by 20 places cannot be represented in type 'int'
  ../../cipher/mceliece6688128f.c:2261:15: runtime error: load of misaligned address 0x7ffd13d3ad84 for type 'const uint64_t', which requires 8 byte alignment
  0x7ffd13d3ad84: note: pointer points here
    d8 7f e7 3c 61 1d b1 60  68 9b ff 4e 95 da 54 31  4b ca b9 7f e2 3b 06 a5  51 e3 7f 74 14 99 81 44
                ^
t-kem: 70 tests done

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/mceliece6688128f.c | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/cipher/mceliece6688128f.c b/cipher/mceliece6688128f.c
index 6ad3eecb..ca1952b5 100644
--- a/cipher/mceliece6688128f.c
+++ b/cipher/mceliece6688128f.c
@@ -131,6 +131,7 @@
 #endif
 
 #include "g10lib.h"
+#include "bufhelp.h"
 #include "mceliece6688128f.h"
 
 static void
@@ -1763,10 +1764,10 @@ static void cbrecursion(unsigned char *out,long long pos,long long step,const in
   }
   /* B = (p<<16)+c */
 
-  for (x = 0;x < n;++x) A[x] = (A[x]<<16)|x; /* A = (pibar<<16)+id */
+  for (x = 0;x < n;++x) A[x] = ((u32)A[x]<<16)|x; /* A = (pibar<<16)+id */
   int32_sort(A,n); /* A = (id<<16)+pibar^-1 */
 
-  for (x = 0;x < n;++x) A[x] = (A[x]<<16)+(B[x]>>16); /* A = (pibar^(-1)<<16)+pibar */
+  for (x = 0;x < n;++x) A[x] = ((u32)A[x]<<16)+(B[x]>>16); /* A = (pibar^(-1)<<16)+pibar */
   int32_sort(A,n); /* A = (id<<16)+pibar^2 */
 
   if (w <= 10) {
@@ -1778,7 +1779,7 @@ static void cbrecursion(unsigned char *out,long long pos,long long step,const in
       for (x = 0;x < n;++x) A[x] = ((B[x]&~0x3ff)<<6)|x; /* A = (p<<16)+id */
       int32_sort(A,n); /* A = (id<<16)+p^{-1} */
 
-      for (x = 0;x < n;++x) A[x] = (A[x]<<20)|B[x]; /* A = (p^{-1}<<20)+(p<<10)+c */
+      for (x = 0;x < n;++x) A[x] = ((u32)A[x]<<20)|B[x]; /* A = (p^{-1}<<20)+(p<<10)+c */
       int32_sort(A,n); /* A = (id<<20)+(pp<<10)+cp */
 
       for (x = 0;x < n;++x) {
@@ -1789,7 +1790,7 @@ static void cbrecursion(unsigned char *out,long long pos,long long step,const in
     }
     for (x = 0;x < n;++x) B[x] &= 0x3ff;
   } else {
-    for (x = 0;x < n;++x) B[x] = (A[x]<<16)|(B[x]&0xffff);
+    for (x = 0;x < n;++x) B[x] = ((u32)A[x]<<16)|(B[x]&0xffff);
 
     for (i = 1;i < w-1;++i) {
       /* B = (p<<16)+c */
@@ -1797,14 +1798,14 @@ static void cbrecursion(unsigned char *out,long long pos,long long step,const in
       for (x = 0;x < n;++x) A[x] = (B[x]&~0xffff)|x;
       int32_sort(A,n); /* A = (id<<16)+p^(-1) */
 
-      for (x = 0;x < n;++x) A[x] = (A[x]<<16)|(B[x]&0xffff);
+      for (x = 0;x < n;++x) A[x] = ((u32)A[x]<<16)|(B[x]&0xffff);
       /* A = p^(-1)<<16+c */
 
       if (i < w-2) {
         for (x = 0;x < n;++x) B[x] = (A[x]&~0xffff)|(B[x]>>16);
         /* B = (p^(-1)<<16)+p */
         int32_sort(B,n); /* B = (id<<16)+p^(-2) */
-        for (x = 0;x < n;++x) B[x] = (B[x]<<16)|(A[x]&0xffff);
+        for (x = 0;x < n;++x) B[x] = ((u32)B[x]<<16)|(A[x]&0xffff);
         /* B = (p^(-2)<<16)+c */
       }
 
@@ -1830,8 +1831,8 @@ static void cbrecursion(unsigned char *out,long long pos,long long step,const in
     out[pos>>3] ^= fj<<(pos&7);
     pos += step;
 
-    B[lx] = (A[lx]<<16)|Fx;
-    B[lx+1] = (A[lx+1]<<16)|Fx1;
+    B[lx] = ((u32)A[lx]<<16)|Fx;
+    B[lx+1] = ((u32)A[lx+1]<<16)|Fx1;
   }
   /* B = (pi^(-1)<<16)+F */
 
@@ -2242,8 +2243,8 @@ static void syndrome(unsigned char *s, const unsigned char *pk, unsigned char *e
 {
 	uint64_t b;
 
-	const uint64_t *pk_ptr;
-	const uint64_t *e_ptr = ((uint64_t *) (e + SYND_BYTES));
+	const unsigned char *pk_ptr;
+	const unsigned char *e_ptr = (e + SYND_BYTES);
 
 	int i, j;
 
@@ -2254,13 +2255,13 @@ static void syndrome(unsigned char *s, const unsigned char *pk, unsigned char *e
 
 	for (i = 0; i < PK_NROWS; i++)
 	{
-		pk_ptr = ((uint64_t *) (pk + PK_ROW_BYTES * i));
+		pk_ptr = (pk + PK_ROW_BYTES * i);
 
 		b = 0;
 		for (j = 0; j < PK_NCOLS/64; j++)
-			b ^= pk_ptr[j] & e_ptr[j];
+			b ^= buf_get_he64(&pk_ptr[j*8]) & buf_get_he64(&e_ptr[j*8]);
 
-		b ^= ((uint32_t *) &pk_ptr[j])[0] & ((uint32_t *) &e_ptr[j])[0];
+		b ^= buf_get_he32(&pk_ptr[j*8]) & buf_get_he32(&e_ptr[j*8]);
 
 		b ^= b >> 32;
 		b ^= b >> 16;
-- 
2.48.1