runtime: correct return value checks for mmap on darwin/freebsd

On Darwin and FreeBSD, the mmap syscall return value is returned
unmodified. This means that the return value will either be a
valid address or a positive error number.

Also check return value from mmap in SysReserve - the callers of
SysReserve expect nil to be returned if the allocation failed.

R=golang-dev, rsc
CC=golang-dev
https://golang.org/cl/7871043

src/pkg/runtime/mem_darwin.c

@@ -37,7 +37,12 @@ runtime·SysFree(void *v, uintptr n)
 void*
 runtime·SysReserve(void *v, uintptr n)
 {
-	return runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0);
+	void *p;
+
+	p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0);
+	if(p < (void*)4096)
+		return nil;
+	return p;
 }
 
 enum
@@ -52,7 +57,7 @@ runtime·SysMap(void *v, uintptr n)
 	
 	mstats.sys += n;
 	p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0);
-	if(p == (void*)-ENOMEM)
+	if(p == (void*)ENOMEM)
 		runtime·throw("runtime: out of memory");
 	if(p != v)
 		runtime·throw("runtime: cannot map pages in arena address space");

src/pkg/runtime/mem_freebsd.c

@@ -8,6 +8,11 @@
 #include "os_GOOS.h"
 #include "malloc.h"
 
+enum
+{
+	ENOMEM = 12,
+};
+
 void*
 runtime·SysAlloc(uintptr n)
 {
@@ -36,20 +41,20 @@ runtime·SysFree(void *v, uintptr n)
 void*
 runtime·SysReserve(void *v, uintptr n)
 {
+	void *p;
+
 	// On 64-bit, people with ulimit -v set complain if we reserve too
 	// much address space.  Instead, assume that the reservation is okay
 	// and check the assumption in SysMap.
 	if(sizeof(void*) == 8)
 		return v;
 	
-	return runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0);
+	p = runtime·mmap(v, n, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0);
+	if(p < (void*)4096)
+		return nil;
+	return p;
 }
 
-enum
-{
-	ENOMEM = 12,
-};
-
 void
 runtime·SysMap(void *v, uintptr n)
 {
@@ -60,7 +65,7 @@ runtime·SysMap(void *v, uintptr n)
 	// On 64-bit, we don't actually have v reserved, so tread carefully.
 	if(sizeof(void*) == 8) {
 		p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0);
-		if(p == (void*)-ENOMEM)
+		if(p == (void*)ENOMEM)
 			runtime·throw("runtime: out of memory");
 		if(p != v) {
 			runtime·printf("runtime: address space conflict: map(%p) = %p\n", v, p);
@@ -70,7 +75,7 @@ runtime·SysMap(void *v, uintptr n)
 	}
 
 	p = runtime·mmap(v, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_FIXED|MAP_PRIVATE, -1, 0);
-	if(p == (void*)-ENOMEM)
+	if(p == (void*)ENOMEM)
 		runtime·throw("runtime: out of memory");
 	if(p != v)
 		runtime·throw("runtime: cannot map pages in arena address space");