[PATCH 1/2] mceliece6688128f: fix stack overflow crash on win64/wine

[Jussi Kivilinna]

* cipher/mceliece6688128f.c (pk_gen): Remove 'mat' array allocation and
rename function to ...
(pk_gen_mat): ... this.
(pk_gen): New warper for 'pk_gen_mat' with dynamic allocation of 'mat' array.
--

Huge array allocations from stack are not always guaranteed to work on
every target platform, so avoid allocating multi-megabyte 'mat' array
from stack.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/mceliece6688128f.c | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/cipher/mceliece6688128f.c b/cipher/mceliece6688128f.c
index ca1952b5..c15c2d14 100644
--- a/cipher/mceliece6688128f.c
+++ b/cipher/mceliece6688128f.c
@@ -3315,16 +3315,15 @@ static int mov_columns(uint64_t mat[][ (SYS_N + 63) / 64 ], int16_t * pi, uint64
 	return 0;
 }
 
-static int pk_gen(unsigned char * pk, const unsigned char * irr, uint32_t * perm, int16_t * pi, uint64_t * pivots)
-{
-	const int nblocks_H = (SYS_N + 63) / 64;
-	const int nblocks_I = (PK_NROWS + 63) / 64;
+#define nblocks_H ((SYS_N + 63) / 64)
+#define nblocks_I ((PK_NROWS + 63) / 64)
 
+static int pk_gen_mat(unsigned char * pk, const unsigned char * irr, uint32_t * perm, int16_t * pi, uint64_t * pivots,
+		      uint64_t mat[ PK_NROWS ][ nblocks_H ])
+{
 	int i, j, k;
 	int row, c;
 
-	uint64_t mat[ PK_NROWS ][ nblocks_H ];
-
 	uint64_t mask;
 
 	vec irr_int[2][ GFBITS ];
@@ -3460,6 +3459,16 @@ static int pk_gen(unsigned char * pk, const unsigned char * irr, uint32_t * perm
 }
 
 
+static int pk_gen(unsigned char * pk, const unsigned char * irr, uint32_t * perm, int16_t * pi, uint64_t * pivots)
+{
+	/* Allocate large array from heap to avoid stack overflow crash on Win32/Wine. */
+	uint64_t *mat = xmalloc(sizeof(uint64_t) * PK_NROWS * nblocks_H);
+	int ret = pk_gen_mat(pk, irr, perm, pi, pivots, (void *)mat);
+	xfree(mat);
+	return ret;
+}
+
+
 /* from libmceliece-20230612/crypto_kem/6688128f/vec/sk_gen.c */
 /*
   This file is for secret-key generation
-- 
2.48.1