[libc] Properly handle max malloc (=32764 bytes) in v7malloc

ghaerr · Nov 20, 2024 · a5730b8 · a5730b8
1 parent 5f5effd
commit a5730b8
Showing 1 changed file with 19 additions and 11 deletions.
diff --git a/libc/malloc/v7malloc.c b/libc/malloc/v7malloc.c
@@ -61,7 +61,7 @@ static	union store __wcnear *allocx;	/*for benefit of realloc*/
 #include <paths.h>
 #include <fcntl.h>
 #include <sys/sysctl.h>
-#define ASSERT(p)	if(!(p))malloc_assert_fail(#p);else
+#define ASSERT(p)	if(!(p))malloc_assert_fail(#p);else {}
 #define errmsg(str) write(STDERR_FILENO, str, sizeof(str) - 1)
 #define errstr(str) write(STDERR_FILENO, str, strlen(str))
 static void malloc_assert_fail(char *s);
@@ -97,7 +97,7 @@ void *
 malloc(size_t nbytes)
 {
 	union store __wcnear *p, __wcnear *q;
-	int nw, temp;
+	unsigned int nw, temp;
 
 #if DEBUG > 1
 	if (dbgout->fd < 0)
@@ -113,12 +113,20 @@ malloc(size_t nbytes)
 		allocp = (union store __wcnear *)&allocs[0];
 	}
 
-	debug("(%d)malloc(%d) ", getpid(), nbytes);
-	if (nbytes == 0)
+	debug("(%d)malloc(%u) ", getpid(), nbytes);
+	if (nbytes == 0) {
+		debug(" (malloc 0) = NULL\n");
 		return NULL;	/* ANSI std */
-
+	}
 	if (nbytes < MINALLOC)
 		nbytes = MINALLOC;
+
+	/* check INT overflow beyond 32764 (nbytes/WORD+WORD+1 > 0xFFFF/WORD/WORD)*/
+	if (nbytes > 0xFFFF/WORD-WORD-1) {      /* UINT_MAX = 0xFFFF */
+		debug(" (req too big) = NULL\n");
+		return(NULL);
+	}
+
 	nw = (nbytes+WORD+WORD-1)/WORD;			/* extra word for link ptr/size*/
 	ASSERT(allocp>=allocs && allocp<=alloct);
 	ASSERT(malloc_check_heap());
@@ -139,9 +147,9 @@ allocp = (union store __wcnear *)allocs;     /* experimental */
 			}
 			q = p;
 			p = clearbusy(p->ptr);
-			if(p>q)
+			if(p>q) {
 				ASSERT(p<=alloct);
-			else if(q!=alloct || p!=allocs) {
+			} else if(q!=alloct || p!=allocs) {
 				ASSERT(q==alloct&&p==allocs);
 				debug(" (corrupt) = NULL\n");
 				return(NULL);
@@ -161,9 +169,9 @@ allocp = (union store __wcnear *)allocs;     /* experimental */
 		if((INT)q & (sizeof(union store) - 1))
 			sbrk(4 - ((INT)q & (sizeof(union store) - 1)));
 
-		/* check possible wrap (>= 32k alloc)*/
+		/* check possible address wrap*/
 		if(q+temp+GRANULE < q) {
-			debug(" (req too big) = NULL\n");
+			debug(" (no more address space) = NULL\n");
 			return(NULL);
 		}
 
@@ -230,7 +238,7 @@ realloc(void *ptr, size_t nbytes)
 
 	if (p == 0)
 		return malloc(nbytes);
-	debug("(%d)realloc(%p,%d) ", getpid(), p-1, nbytes);
+	debug("(%d)realloc(%p,%u) ", getpid(), p-1, nbytes);
 
 	ASSERT(testbusy(p[-1].ptr));
 	if(testbusy(p[-1].ptr))
@@ -307,7 +315,7 @@ malloc_show_heap(void)
 		debug2("\n");
 	}
 	alloc += 2;
-	debug2("%2d: %p %4u  (top) ", n, alloct, 2);
+	debug2("%2d: %p %4u (top) ", n, alloct, 2);
 	debug("alloc %u, free %u, total %u\n", alloc, free, alloc+free);
 }
 #endif