fix ufndmq* algos heap-overflows

rurban · Nov 24, 2024 · db154cf · db154cf
1 parent 92aa3b8
commit db154cf
Show file tree

Hide file tree

Showing 6 changed files with 113 additions and 76 deletions.
diff --git a/GNUmakefile b/GNUmakefile
@@ -41,6 +41,9 @@ ifneq ($(ASSERT),1)
     endif
   endif
 endif
+ifeq ($(NOSHM),1)
+  CFLAGS += -DNOSHM
+endif
 ifneq ($(ARCH),$(MACHINE))
   ifeq ($(BINDIR),bin)
     BINDIR = bin/$(ARCH)
@@ -94,7 +97,7 @@ TESTS := $(shell shuf -n 10 good.lst)
 ifeq ($(TESTS),)
   TESTS = hor mp kmp musl1 tbm so ssm qf33 twfr3 fndm
 endif
-COMPILE = $(CC) $(CFLAGS)
+COMPILE = $(CC) -c $(CFLAGS)
 
 all: $(BINS) $(HELPERS) good.lst asan.lst
 
@@ -246,7 +249,7 @@ fuzz-repro:
 # emacs flymake-mode
 check-syntax:
 	test -n "$(CHK_SOURCES)" && \
-	  $(COMPILE) -c -o /dev/null -S $(CHK_SOURCES)
+	  $(COMPILE) -Wextra -o /dev/null -S $(CHK_SOURCES)
 .PHONY: check-syntax
 fmt:
 	clang-format -i `find source -name \*.c -o -name \*.h`

diff --git a/source/algocfg.c b/source/algocfg.c
@@ -262,13 +262,13 @@ const struct algocfg ALGOCFGS[] = {
   [_FSBNDMQ86] = {_FSBNDMQ86, FAIL, FAIL, VFY_PASS, 8, 26, 0, 0, 0, 0},
   [_IOM]   = {_IOM, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
   [_JOM]   = {_JOM, RNDCRASH, FAIL, VFY_PASS, 2, 0, 0, 0, 0, 0},
-  [_LWFR2] = {_LWFR2, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_LWFR3] = {_LWFR3, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_LWFR4] = {_LWFR4, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_LWFR5] = {_LWFR5, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_LWFR6] = {_LWFR6, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_LWFR7] = {_LWFR7, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_LWFR8] = {_LWFR8, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
+  [_LWFR2] = {_LWFR2, GOOD, ASAN, VFY_PASS, 2, 0, 0, 0, 0, 0},
+  [_LWFR3] = {_LWFR3, GOOD, ASAN, VFY_PASS, 3, 0, 0, 0, 0, 0},
+  [_LWFR4] = {_LWFR4, GOOD, ASAN, VFY_PASS, 4, 0, 0, 0, 0, 0},
+  [_LWFR5] = {_LWFR5, GOOD, ASAN, VFY_PASS, 5, 0, 0, 0, 0, 0},
+  [_LWFR6] = {_LWFR6, GOOD, ASAN, VFY_PASS, 6, 0, 0, 0, 0, 0},
+  [_LWFR7] = {_LWFR7, GOOD, ASAN, VFY_PASS, 7, 0, 0, 0, 0, 0},
+  [_LWFR8] = {_LWFR8, GOOD, ASAN, VFY_PASS, 8, 0, 0, 0, 0, 0},
   [_QF23] = {_QF23, GOOD, ASAN, VFY_PASS, 3, 0, 0, 0, 0, 0},
   [_QF24] = {_QF24, GOOD, ASAN, VFY_PASS, 3, 0, 0, 0, 0, 0},
   [_QF26] = {_QF26, GOOD, ASAN, VFY_PASS, 3, 0, 0, 0, 0, 0},
@@ -298,35 +298,35 @@ const struct algocfg ALGOCFGS[] = {
   [_TVSBS_W6] = {_TVSBS_W6, ASSERTS, FAIL, VFY_PASS, 2, XSIZE, 0, 0, 0, 0},
   [_TVSBS_W8] = {_TVSBS_W8, FAIL, FAIL, VFY_PASS, 2, XSIZE, 0, 0, 0, 0},
   [_WFR]  = {_WFR,  GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFR2] = {_WFR2, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFR3] = {_WFR3, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFR4] = {_WFR4, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFR5] = {_WFR5, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFR6] = {_WFR6, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFR7] = {_WFR7, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFR8] = {_WFR8, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFRQ2] = {_WFRQ2, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFRQ3] = {_WFRQ3, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFRQ4] = {_WFRQ4, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFRQ5] = {_WFRQ5, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFRQ6] = {_WFRQ6, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFRQ7] = {_WFRQ7, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_WFRQ8] = {_WFRQ8, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
+  [_WFR2] = {_WFR2, GOOD, ASAN, VFY_PASS, 2, 0, 0, 0, 0, 0},
+  [_WFR3] = {_WFR3, GOOD, ASAN, VFY_PASS, 3, 0, 0, 0, 0, 0},
+  [_WFR4] = {_WFR4, GOOD, ASAN, VFY_PASS, 4, 0, 0, 0, 0, 0},
+  [_WFR5] = {_WFR5, GOOD, ASAN, VFY_PASS, 5, 0, 0, 0, 0, 0},
+  [_WFR6] = {_WFR6, GOOD, ASAN, VFY_PASS, 6, 0, 0, 0, 0, 0},
+  [_WFR7] = {_WFR7, GOOD, ASAN, VFY_PASS, 7, 0, 0, 0, 0, 0},
+  [_WFR8] = {_WFR8, GOOD, ASAN, VFY_PASS, 8, 0, 0, 0, 0, 0},
+  [_WFRQ2] = {_WFRQ2, GOOD, ASAN, VFY_PASS, 2, 0, 0, 0, 0, 0},
+  [_WFRQ3] = {_WFRQ3, GOOD, ASAN, VFY_PASS, 3, 0, 0, 0, 0, 0},
+  [_WFRQ4] = {_WFRQ4, GOOD, ASAN, VFY_PASS, 4, 0, 0, 0, 0, 0},
+  [_WFRQ5] = {_WFRQ5, GOOD, ASAN, VFY_PASS, 5, 0, 0, 0, 0, 0},
+  [_WFRQ6] = {_WFRQ6, GOOD, ASAN, VFY_PASS, 6, 0, 0, 0, 0, 0},
+  [_WFRQ7] = {_WFRQ7, GOOD, ASAN, VFY_PASS, 7, 0, 0, 0, 0, 0},
+  [_WFRQ8] = {_WFRQ8, GOOD, ASAN, VFY_PASS, 8, 0, 0, 0, 0, 0},
   [_TWFR]  = {_TWFR,  GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFR2] = {_TWFR2, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFR3] = {_TWFR3, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFR4] = {_TWFR4, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFR5] = {_TWFR5, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFR6] = {_TWFR6, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFR7] = {_TWFR7, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFR8] = {_TWFR8, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFRQ2] = {_TWFRQ2, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFRQ3] = {_TWFRQ3, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFRQ4] = {_TWFRQ4, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFRQ5] = {_TWFRQ5, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFRQ6] = {_TWFRQ6, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFRQ7] = {_TWFRQ7, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
-  [_TWFRQ8] = {_TWFRQ8, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
+  [_TWFR2] = {_TWFR2, GOOD, ASAN, VFY_PASS, 2, 0, 0, 0, 0, 0},
+  [_TWFR3] = {_TWFR3, GOOD, ASAN, VFY_PASS, 3, 0, 0, 0, 0, 0},
+  [_TWFR4] = {_TWFR4, GOOD, ASAN, VFY_PASS, 4, 0, 0, 0, 0, 0},
+  [_TWFR5] = {_TWFR5, GOOD, ASAN, VFY_PASS, 5, 0, 0, 0, 0, 0},
+  [_TWFR6] = {_TWFR6, GOOD, ASAN, VFY_PASS, 6, 0, 0, 0, 0, 0},
+  [_TWFR7] = {_TWFR7, GOOD, ASAN, VFY_PASS, 7, 0, 0, 0, 0, 0},
+  [_TWFR8] = {_TWFR8, GOOD, ASAN, VFY_PASS, 8, 0, 0, 0, 0, 0},
+  [_TWFRQ2] = {_TWFRQ2, GOOD, ASAN, VFY_PASS, 2, 0, 0, 0, 0, 0},
+  [_TWFRQ3] = {_TWFRQ3, GOOD, ASAN, VFY_PASS, 3, 0, 0, 0, 0, 0},
+  [_TWFRQ4] = {_TWFRQ4, GOOD, ASAN, VFY_PASS, 4, 0, 0, 0, 0, 0},
+  [_TWFRQ5] = {_TWFRQ5, GOOD, ASAN, VFY_PASS, 5, 0, 0, 0, 0, 0},
+  [_TWFRQ6] = {_TWFRQ6, GOOD, ASAN, VFY_PASS, 6, 0, 0, 0, 0, 0},
+  [_TWFRQ7] = {_TWFRQ7, GOOD, ASAN, VFY_PASS, 7, 0, 0, 0, 0, 0},
+  [_TWFRQ8] = {_TWFRQ8, GOOD, ASAN, VFY_PASS, 8, 0, 0, 0, 0, 0},
   [_WC]     = {_WC,     GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
   [_WOM]    = {_WOM,    GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},
   [_DOUBLEHASH] = {_DOUBLEHASH, GOOD, ASAN, VFY_PASS, 0, 0, 0, 0, 0, 0},

diff --git a/source/algos/ufndmq2.c b/source/algos/ufndmq2.c
@@ -23,7 +23,7 @@
  * York, USA, (2009).
  * https://users.aalto.fi/~tarhio/papers/ale.pdf
  *
- * Constraints: requires m>=2
+ * Constraints: requires m>=2. required 2m space after y.
  */
 
 #define MIN_M 2
@@ -33,21 +33,24 @@
 
 #define GRAM2(i) ((B[y[i - 1]] << 1) | B[y[i]])
 
-int search_large(unsigned char *x, int m, unsigned char *y, int n);
+int search_large(unsigned char *x, int m, unsigned char *_y, int n);
 
-int search(unsigned char *x, int m, unsigned char *y, int n) {
+int search(unsigned char *x, int m, unsigned char *_y, int n) {
   unsigned int D, F, mm, mask, B[SIGMA], S;
   int i, j, k, mq;
   int count = 0;
   const int q = 2;
+  unsigned char *y;
 
   if (m < q)
-    return search_small(x, m, y, n);
+    return search_small(x, m, _y, n);
   if (m + q > WORD)
-    return search_large(x, m, y, n);
+    return search_large(x, m, _y, n);
 
   /* Preprocessing */
   BEGIN_PREPROCESSING
+  y = malloc(n + 2*m);
+  memcpy(y, _y, n);
   S = ~(1U << m);
   for (j = 0; j < SIGMA; ++j)
     B[j] = S;
@@ -78,6 +81,8 @@ int search(unsigned char *x, int m, unsigned char *y, int n) {
             if (y[k + j] != x[j])
               goto mismatch;
           if (k + m > n) {
+            if (y != _y)
+              free(y);
             END_SEARCHING
             return (count);
           }
@@ -100,19 +105,22 @@ int search(unsigned char *x, int m, unsigned char *y, int n) {
  * the pattern
  */
 
-int search_large(unsigned char *x, int m, unsigned char *y, int n) {
+int search_large(unsigned char *x, int m, unsigned char *_y, int n) {
   unsigned int D, F, mm, mask, B[SIGMA], S;
-  int i, j, k, mq, q, p_len;
+  int i, j, k, mq, p_len;
   int count = 0;
-  q = 2;
+  const int q = 2;
+  unsigned char *y;
 
+  /* Preprocessing */
+  BEGIN_PREPROCESSING
+  y = malloc(n + 2*m);
+  memcpy(y, _y, n);
   for (i = 0; i < m; i++)
     y[n + i] = y[n + m + i] = x[i];
   p_len = m;
   m = 32 - q;
 
-  /* Preprocessing */
-  BEGIN_PREPROCESSING
   S = ~(1U << m);
   for (j = 0; j < SIGMA; ++j)
     B[j] = S;
@@ -142,6 +150,8 @@ int search_large(unsigned char *x, int m, unsigned char *y, int n) {
             if (y[k + j] != x[j])
               goto mismatch;
           if (k + p_len > n) {
+            if (y != _y)
+              free(y);
             END_SEARCHING
             return (count);
           }

diff --git a/source/algos/ufndmq4.c b/source/algos/ufndmq4.c
@@ -22,7 +22,7 @@
  * Engineering and Experiments, ALENEX 2009, pp.29--37, SIAM, New York, New
  * York, USA, (2009).
  *
- * Constraints: requires m>=4
+ * Constraints: requires m>=4. required 2m space after y.
  */
 
 #define MIN_M 4
@@ -33,21 +33,24 @@
 #define GRAM4(i)                                                               \
   ((B[y[i - 3]] << 3) | (B[y[i - 2]] << 2) | (B[y[i - 1]] << 1) | B[y[i]])
 
-int search_large(unsigned char *x, int m, unsigned char *y, int n);
+int search_large(unsigned char *x, int m, unsigned char *_y, int n);
 
-int search(unsigned char *x, int m, unsigned char *y, int n) {
+int search(unsigned char *x, int m, unsigned char *_y, int n) {
   unsigned int D, F, mm, mask, B[SIGMA], S;
   int i, j, k, mq;
   int count = 0;
   const int q = 4;
+  unsigned char *y;
 
   if (m < q)
-    return search_small(x, m, y, n);
+    return search_small(x, m, _y, n);
   if (m + q > WORD)
-    return search_large(x, m, y, n);
+    return search_large(x, m, _y, n);
 
   /* Preprocessing */
   BEGIN_PREPROCESSING
+  y = malloc(n + 2*m);
+  memcpy(y, _y, n);
   S = ~((unsigned char)7 << m);
   for (j = 0; j < SIGMA; ++j)
     B[j] = S;
@@ -79,6 +82,7 @@ int search(unsigned char *x, int m, unsigned char *y, int n) {
             if (y[k + j] != x[j])
               goto mismatch;
           if (k + m > n) {
+            free(y);
             END_SEARCHING
             return (count);
           }
@@ -101,21 +105,24 @@ int search(unsigned char *x, int m, unsigned char *y, int n) {
  * the pattern
  */
 
-int search_large(unsigned char *x, int m, unsigned char *y, int n) {
+int search_large(unsigned char *x, int m, unsigned char *_y, int n) {
   unsigned int D, F, mm, mask, B[SIGMA], S;
-  int i, j, k, mq, q, p_len;
+  int i, j, k, mq, p_len;
   int count = 0;
-  q = 4;
+  const int q = 4;
+  unsigned char *y;
   if (m < q)
     return 0;
 
+  /* Preprocessing */
+  BEGIN_PREPROCESSING
+  y = malloc(n + 2*m);
+  memcpy(y, _y, n);
   for (i = 0; i < m; i++)
     y[n + i] = y[n + m + i] = x[i];
   p_len = m;
   m = 32 - q;
 
-  /* Preprocessing */
-  BEGIN_PREPROCESSING
   S = ~((unsigned char)7 << m);
   for (j = 0; j < SIGMA; ++j)
     B[j] = S;
@@ -144,6 +151,7 @@ int search_large(unsigned char *x, int m, unsigned char *y, int n) {
             if (y[k + j] != x[j])
               goto mismatch;
           if (k + p_len > n) {
+            free(y);
             END_SEARCHING
             return (count);
           }

diff --git a/source/algos/ufndmq6.c b/source/algos/ufndmq6.c
@@ -22,7 +22,7 @@
  * Engineering and Experiments, ALENEX 2009, pp.29--37, SIAM, New York, New
  * York, USA, (2009).
  *
- * Constraints: requires m>=6
+ * Constraints: requires m>=6. required 2m space after y.
  */
 
 #define MIN_M 6
@@ -36,19 +36,22 @@
 
 int search_large(unsigned char *x, int m, unsigned char *y, int n);
 
-int search(unsigned char *x, int m, unsigned char *y, int n) {
+int search(unsigned char *x, int m, unsigned char *_y, int n) {
   unsigned int D, F, mm, mask, B[SIGMA], S;
   int i, j, k, mq;
   int count = 0;
   const int q = 6;
+  unsigned char *y;
 
   if (m < q)
-    return search_small(x, m, y, n);
+    return search_small(x, m, _y, n);
   if (m + q > WORD)
-    return search_large(x, m, y, n);
+    return search_large(x, m, _y, n);
 
   /* Preprocessing */
   BEGIN_PREPROCESSING
+  y = malloc(n + 2*m);
+  memcpy(y, _y, n);
   S = ~((unsigned char)31U << m);
   for (j = 0; j < SIGMA; ++j)
     B[j] = S;
@@ -80,6 +83,7 @@ int search(unsigned char *x, int m, unsigned char *y, int n) {
             if (y[k + j] != x[j])
               goto mismatch;
           if (k + m > n) {
+            free(y);
             END_SEARCHING
             return (count);
           }
@@ -102,19 +106,22 @@ int search(unsigned char *x, int m, unsigned char *y, int n) {
  * the pattern
  */
 
-int search_large(unsigned char *x, int m, unsigned char *y, int n) {
+int search_large(unsigned char *x, int m, unsigned char *_y, int n) {
   unsigned int D, F, mm, mask, B[SIGMA], S;
-  int i, j, k, mq, q, p_len;
+  int i, j, k, mq, p_len;
   int count = 0;
-  q = 6;
+  const int q = 6;
+  unsigned char *y;
 
+  /* Preprocessing */
+  BEGIN_PREPROCESSING
+  y = malloc(n + 2*m);
+  memcpy(y, _y, n);
   for (i = 0; i < m; i++)
     y[n + i] = y[n + m + i] = x[i];
   p_len = m;
   m = 32 - q;
 
-  /* Preprocessing */
-  BEGIN_PREPROCESSING
   S = ~((unsigned char)31U << m);
   for (j = 0; j < SIGMA; ++j)
     B[j] = S;
@@ -143,6 +150,7 @@ int search_large(unsigned char *x, int m, unsigned char *y, int n) {
             if (y[k + j] != x[j])
               goto mismatch;
           if (k + p_len > n) {
+            free(y);
             END_SEARCHING
             return (count);
           }