Malloc fixes

elkscmd/test/libc/malloc.c

@@ -4,85 +4,87 @@
 #include <malloc.h>
 #include <string.h>
 
-TEST_CASE(malloc_malloc_free)
-{
-	void *p;
-
-	/* malloc(0) may return NULL or a pointer which can be passed to free */
-	errno = 0;
-	p = malloc(0);
-	EXPECT_EQ(errno, 0);
-	if (p != NULL) {
-		free(p);
-	}
-
-	/* TODO:BUG: causes hang at 100% CPU */
+TEST_CASE(malloc_malloc_free) {
+    void *p;
+
+    /* malloc(0) may return NULL or a pointer which can be passed to free */
+    errno = 0;
+    p = malloc(0);
+    EXPECT_EQ(errno, 0);
+    if (p != NULL) {
+        free(p);
+    }
+
+    /* TODO:BUG: causes hang at 100% CPU */
 #if 0
-	errno = 0;
-	p = malloc((size_t)-1);
-	if (p == NULL) {
-		EXPECT_EQ(errno, ENOMEM);
-	} else {
-		EXPECT_EQ(errno, 0);
-		memset(p, 0xff, (size_t)-1);
-		free(p);
-	}
+    errno = 0;
+    p = malloc((size_t) - 1);
+    if (p == NULL) {
+        EXPECT_EQ(errno, ENOMEM);
+    } else {
+        EXPECT_EQ(errno, 0);
+        memset(p, 0xff, (size_t) - 1);
+        free(p);
+    }
 #endif
 
-	/* strange sizes are fine; memory is writable and free-able */
-	for (int i = 1; i < 1024; i += 123) {
-		errno = 0;
-		p = malloc(i);
-		EXPECT_EQ(errno, 0);
-		ASSERT_NE_P(p, NULL);
-		memset(p, 0xff, i);
-		free(p);
-	}
+    /* strange sizes are fine; memory is writable and free-able */
+    for (int i = 1; i < 1024; i += 123) {
+        errno = 0;
+        p = malloc(i);
+        EXPECT_EQ(errno, 0);
+        ASSERT_NE_P(p, NULL);
+        memset(p, 0xff, i);
+
+        /* free must not change errno */
+        errno = 123;
+        free(p);
+        EXPECT_EQ(errno, 123);
+    }
 }
 
-TEST_CASE(malloc_calloc)
-{
-	void *p;
+TEST_CASE(malloc_calloc) {
+    void *p;
 
-	errno = 0;
-	p = calloc(0, 1);
-	EXPECT_EQ(errno, 0);
-	if (p != NULL) {
-		free(p);
-	}
+    errno = 0;
+    p = calloc(0, 1);
+    EXPECT_EQ(errno, 0);
+    if (p != NULL) {
+        free(p);
+    }
 
-	/* TODO check for mult overflow */
+    p = calloc(((unsigned)-1)>>2, 5);
+    EXPECT_EQ(errno, ENOMEM);
+    EXPECT_EQ_P(p, NULL);
 }
 
-TEST_CASE(malloc_realloc)
-{
-	char *p;
+TEST_CASE(malloc_realloc) {
+    char *p;
 
-	p = realloc(NULL, 32);
-	ASSERT_NE_P(p, NULL);
-	memset(p, 'A', 32);
+    p = realloc(NULL, 32);
+    ASSERT_NE_P(p, NULL);
+    memset(p, 'A', 32);
 
-	/* shrink */
-	p = realloc(p, 1);
-	ASSERT_NE_P(p, NULL);
-	memset(p, 'B', 1);
+    /* shrink */
+    p = realloc(p, 1);
+    ASSERT_NE_P(p, NULL);
+    memset(p, 'B', 1);
 
-	/* grow */
-	p = realloc(p, 64);
-	ASSERT_NE_P(p, NULL);
-	EXPECT_EQ(p[0], 'B');
-	memset(p, 'C', 64);
+    /* grow */
+    p = realloc(p, 64);
+    ASSERT_NE_P(p, NULL);
+    EXPECT_EQ(p[0], 'B');
+    memset(p, 'C', 64);
 
-	free(p);
+    free(p);
 }
 
-TEST_CASE(malloc_alloca)
-{
-	void *p;
+TEST_CASE(malloc_alloca) {
+    void *p;
 
-	p = alloca(1);
-	EXPECT_NE_P(p, NULL);
+    p = alloca(1);
+    EXPECT_NE_P(p, NULL);
 
-	/* TODO nested function calls */
-	/* TODO vs setjmp / longjmp */
+    /* TODO nested function calls */
+    /* TODO vs setjmp / longjmp */
 }

libc/malloc/calloc.c

@@ -1,15 +1,29 @@
+#include <errno.h>
 #include <malloc.h>
 #include <string.h>
 
 void *
 calloc(unsigned int elm, unsigned int sz)
 {
-	register unsigned int v;
-	register void *ptr;
+    unsigned int v;
+    void *ptr;
 
-	ptr = malloc(v = elm * sz);
-	if(ptr)
-		memset(ptr, 0, v);
+#ifdef __GNUC__
+    if (__builtin_umul_overflow(elm, sz, &v)) {
+        errno = ENOMEM;
+        return 0;
+    }
+#else
+    v = elm * sz;
+    if (sz != 0 && v / sz != elm) {
+        errno = ENOMEM;
+        return 0;
+    }
+#endif
 
-	return ptr;
+    ptr = malloc(v);
+    if (ptr)
+        memset(ptr, 0, v);
+
+    return ptr;
 }

libc/malloc/noise.c

@@ -1,37 +1,40 @@
 #if defined(VERBOSE)
-#	include <string.h>
-# include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
 
-#	include "_malloc.h"
+#include "_malloc.h"
 
 /* NB: Careful here, stdio may use malloc - so we can't */
 static void
 phex(int val)
 {
-	static char hex[] = "0123456789ABCDEF";
-	int i;
+    static char hex[] = "0123456789ABCDEF";
+    int i;
 
-	for (i = sizeof(int) * 8 - 4; i >= 0; i -= 4)
-		write(2, hex + ((val >> i) & 0xF), 1);
+    for (i = sizeof(int) * 8 - 4; i >= 0; i -= 4)
+        write(2, hex + ((val >> i) & 0xF), 1);
 }
 
 void
-__noise(char *y, mem *x)
+__noise(char *y, mem * x)
 {
-	write(2, "Malloc ", 7);
-	phex((int)x);
-	write(2, " sz ", 4);
-	if(x)
-		phex(m_size(x));
-	else
-		phex(0);
-	write(2, " nxt ", 5);
-	if (x)
-		phex((int)m_next(x));
-	else
-		phex(0);
-	write(2, " is ", 4);
-	write(2, y, strlen(y));
-	write(2, "\n", 1);
+    int saved_errno = errno;
+    write(2, "Malloc ", 7);
+    phex((int)x);
+    write(2, " sz ", 4);
+    if (x)
+        phex(m_size(x));
+    else
+        phex(0);
+    write(2, " nxt ", 5);
+    if (x)
+        phex((int)m_next(x));
+    else
+        phex(0);
+    write(2, " is ", 4);
+    write(2, y, strlen(y));
+    write(2, "\n", 1);
+    errno = saved_errno;
 }
 #endif