Fix errno values. Fix MALLOC_GLIBC_COMPAT handling in malloc/malloc.c,

which was reversed.  Provide more consistancy between implementations.
Handle it when people do stupid things like malloc(-1);

libc/stdlib/malloc-930716/malloc.c

@@ -15,6 +15,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <errno.h>
 #include "malloc.h"
 
 #ifdef __UCLIBC_HAS_THREADS__
@@ -161,19 +162,22 @@ void * __malloc_unlocked (size_t size)
     struct list *next;
 
 #if defined(__MALLOC_GLIBC_COMPAT__)
-    if (size == 0)
+    if (unlikely(size == 0))
 	size++;
 #else
     /* Some programs will call malloc (0).  Lets be strict and return NULL */
-    if (size == 0)
-	return NULL;
+    if (unlikely(size == 0))
+	goto oom;
 #endif
+    /* Check if they are doing something dumb like malloc(-1) */
+    if (unlikely(((unsigned long)size > (unsigned long)(sizeof (struct list)*-2))))
+	goto oom;
 
-    if (size < sizeof (struct list))
+    if (unlikely(size < sizeof (struct list)))
 	size = sizeof (struct list);
 
     if (!initialized && !initialize()) {
-	return NULL;
+	goto oom;
     }
 
     /* Determine the allocation policy based on the request size. */
@@ -204,7 +208,7 @@ void * __malloc_unlocked (size_t size)
 	       and break it into fragments, returning the first. */
 	    result = __malloc_unlocked(BLOCKSIZE);
 	    if (!result) {
-		return NULL;
+		goto oom;
 	    }
 	    ++_fragblocks[log];
 
@@ -255,7 +259,7 @@ void * __malloc_unlocked (size_t size)
 		}
 		result = morecore(blocks * BLOCKSIZE);
 		if (!result) {
-		    return NULL;
+		    goto oom;
 		}
 		block = BLOCK(result);
 		_heapinfo[block].busy.type = 0;
@@ -293,6 +297,10 @@ void * __malloc_unlocked (size_t size)
     }
 
     return result;
+
+oom:
+    __set_errno(ENOMEM);
+    return NULL;
 }
 
 /* Return memory to the heap. */

libc/stdlib/malloc-930716/realloc.c

@@ -15,6 +15,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <errno.h>
 #include "malloc.h"
 
 #ifdef __UCLIBC_HAS_THREADS__
@@ -44,8 +45,7 @@ void * realloc (void *ptr, size_t size)
 	LOCK;
 	__free_unlocked(ptr);
 	result = __malloc_unlocked(0);
-	UNLOCK;
-	return(result);
+	goto alldone;
     }
 
     LOCK;
@@ -59,8 +59,7 @@ void * realloc (void *ptr, size_t size)
 		    memcpy(result, ptr, size);
 		    __free_unlocked(ptr);
 		}
-		UNLOCK;
-		return result;
+		goto alldone;
 	    }
 
 	    /* The new size is a large allocation as well; see if
@@ -74,12 +73,12 @@ void * realloc (void *ptr, size_t size)
 		    = _heapinfo[block].busy.info.size - blocks;
 		_heapinfo[block].busy.info.size = blocks;
 		__free_unlocked(ADDRESS(block + blocks));
-		UNLOCK;
-		return ptr;
+		result = ptr;
+		goto alldone;
 	    } else if (blocks == _heapinfo[block].busy.info.size) {
 		/* No size change necessary. */
-		UNLOCK;
-		return ptr;
+		result = ptr;
+		goto alldone;
 	    } else {
 		/* Won't fit, so allocate a new region that will.  Free
 		   the old region first in case there is sufficient adjacent
@@ -102,13 +101,11 @@ void * realloc (void *ptr, size_t size)
 			__malloc_unlocked(blocks * BLOCKSIZE);
 			__free_unlocked(previous);
 		    }	    
-		    UNLOCK;
-		    return NULL;
+		    goto oom;
 		}
 		if (ptr != result)
 		    memmove(result, ptr, blocks * BLOCKSIZE);
-		UNLOCK;
-		return result;
+		goto alldone;
 	    }
 	    break;
 
@@ -117,24 +114,29 @@ void * realloc (void *ptr, size_t size)
 	       the fragment size. */
 	    if ((size > 1 << (type - 1)) && (size <= 1 << type)) {
 		/* New size is the same kind of fragment. */
-		UNLOCK;
-		return ptr;
+		result = ptr;
+		goto alldone;
 	    }
 	    else {
 		/* New size is different; allocate a new space, and copy
 		   the lesser of the new size and the old. */
 		result = __malloc_unlocked(size);
 		if (!result) {
-		    UNLOCK;
-		    return NULL;
+		    goto oom;
 		}
 		memcpy(result, ptr, MIN(size, (size_t)(1 << type)));
 		__free_unlocked(ptr);
-		UNLOCK;
-		return result;
+		goto alldone;
 	    }
 	    break;
     }
+alldone:
     UNLOCK;
+    return result;
+
+oom:
+    UNLOCK;
+    __set_errno(ENOMEM);
+    return NULL;
 }
 

libc/stdlib/malloc/malloc.c

@@ -13,6 +13,7 @@
 
 #include <stdlib.h>
 #include <unistd.h>
+#include <errno.h>
 #include <sys/mman.h>
 
 #include "malloc.h"
@@ -173,6 +174,7 @@ malloc_from_heap (size_t size, struct heap *heap)
 void *
 malloc (size_t size)
 {
+  void *mem;
 #ifdef MALLOC_DEBUGGING
   static int debugging_initialized = 0;
   if (! debugging_initialized)
@@ -185,12 +187,22 @@ malloc (size_t size)
 #endif
 
 #if defined(__MALLOC_GLIBC_COMPAT__)
-  if (size == 0)
-    return 0;
-#else
-  if (size == 0)
+  if (unlikely(size == 0))
       size++;
+#else
+  /* Some programs will call malloc (0).  Lets be strict and return NULL */
+  if (unlikely(size == 0))
+      goto oom;
 #endif
-
-  return malloc_from_heap (size, &__malloc_heap);
+  /* Check if they are doing something dumb like malloc(-1) */
+  if (unlikely(((unsigned long)size > (unsigned long)(MALLOC_HEADER_SIZE*-2))))
+      goto oom;
+
+  mem = malloc_from_heap (size, &__malloc_heap);
+  if (unlikely(!mem)) {
+oom:
+      __set_errno(ENOMEM);
+      return NULL;
+  }
+  return mem;
 }

libc/stdlib/malloc/realloc.c

@@ -13,6 +13,7 @@
 
 #include <stdlib.h>
 #include <string.h>
+#include <errno.h>
 
 #include "malloc.h"
 #include "heap.h"
@@ -25,13 +26,12 @@ realloc (void *mem, size_t new_size)
   char *base_mem;
 
   /* Check for special cases.  */
-  if (! new_size)
-    {
+  if (!mem)
+      return malloc(new_size);
+  if (!new_size) {
       free (mem);
-      return 0;
-    }
-  else if (! mem)
-    return malloc (new_size);
+      return (malloc(new_size));
+  }
 
   /* Normal realloc.  */