First pieces of malloc.

R=r
DELTA=756  (754 added, 0 deleted, 2 changed)
OCL=19266
CL=19378

src/runtime/runtime.c

@@ -63,7 +63,7 @@ enum
 	MAP_SHARED	= 0x0001,
 	MAP_PRIVATE	= 0x0002,
 	MAP_FIXED	= 0x0010,
-	MAP_ANON	= 0x1000,
+	MAP_ANON	= 0x1000,	// not on Linux - TODO(rsc)
 };
 
 void

src/runtime/runtime.h

@@ -15,6 +15,7 @@ typedef	signed long long int	int64;
 typedef	unsigned long long int	uint64;
 typedef	float			float32;
 typedef	double			float64;
+typedef	uint64		uintptr;
 
 /*
  * get rid of C types
@@ -69,6 +70,10 @@ enum
 	true	= 1,
 	false	= 0,
 };
+enum
+{
+	SmallFreeClasses = 168,	// number of small free lists in malloc
+};
 
 /*
  * structures
@@ -103,7 +108,7 @@ struct	Array
 {				// must not move anything
 	byte*	array;		// actual data
 	uint32	nel;		// number of elements
-	uint32	cap;		// allocate3d number of elements
+	uint32	cap;		// allocated number of elements
 	byte	b[8];		// actual array - may not be contig
 };
 struct	Gobuf
@@ -152,6 +157,7 @@ struct	M
 	M*	schedlink;
 	Mem	mem;
 	uint32	machport;	// Return address for Mach IPC (OS X)
+	void*	freelist[SmallFreeClasses];
 };
 struct	Stktop
 {

usr/rsc/mem/Makefile

+# Copyright 2009 The Go Authors.  All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+CC=6c -w
+GC=6g
+LD=6l
+O=6
+
+TARG=testrandom testrepeat testsizetoclass
+
+default: $(TARG)
+
+%.$O: %.c malloc.h
+	$(CC) $*.c
+
+%.$O: %.go
+	$(GC) $*.go
+
+OFILES=\
+	allocator.$O\
+	malloc.$O\
+	pagemap.$O\
+	triv.$O\
+
+testrandom.$O: allocator.$O
+testrepeat.$O: allocator.$O
+
+test%: test%.$O $(OFILES)
+	$(LD) -o $@ $^
+
+clean:
+	rm -f *.$O $(TARG)

usr/rsc/mem/allocator.go

+// Copyright 2009 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package allocator
+
+export func free(*byte)
+export func malloc(int) *byte
+export func memset(*byte, int, int)
+export var footprint int64
+export var frozen bool
+export func testsizetoclass()

usr/rsc/mem/malloc.c

+// Copyright 2009 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// General C malloc/free, but intended for Go.
+// Same design as tcmalloc:
+// see https://www/eng/designdocs/tcmalloc/tcmalloc.html
+
+// TODO:
+//	* Central free lists.
+//	* Thread cache stealing.
+//	* Return memory to the OS.
+//	* Memory footprint during testrandom is too big.
+//	* Need to coalesce adjacent free spans.
+//
+//	*** Some way to avoid the ``malloc overflows the stack
+//	    during the stack overflow malloc'' problem.
+
+#include "malloc.h"
+
+typedef struct Span Span;
+typedef struct Central Central;
+
+// A Span contains metadata about a range of pages.
+enum {
+	SpanInUse = 0,	// span has been handed out by allocator
+	SpanFree = 1,	// span is in central free list
+};
+struct Span
+{
+	Span *next;	// in free lists
+	byte *base;	// first byte in span
+	uintptr length;	// number of pages in span
+	int32 cl;
+	int32 state;	// state (enum above)
+//	int ref;	// reference count if state == SpanInUse (for GC)
+//	void *type;	// object type if state == SpanInUse (for GC)
+};
+
+// The Central cache contains a list of free spans,
+// as well as free lists of small blocks.
+struct Central
+{
+	Lock;
+	Span *free[256];
+	Span *large;	// free spans >= MaxPage pages
+};
+
+static Central central;
+static PageMap spanmap;
+
+// Insert a new span into the map.
+static void
+insertspan(Span *s)
+{
+	int32 i;
+	uintptr base;
+
+	// TODO: This is likely too slow for large spans.
+	base = (uintptr)s->base >> PageShift;
+	for(i=0; i<s->length; i++)
+		pminsert(&spanmap, base+i, s);
+}
+
+// Record that a span has gotten smaller.
+static void
+shrinkspan(Span *s, int32 newlength)
+{
+	int32 i;
+	uintptr base;
+
+	// TODO: This is unnecessary, because an insertspan is next.
+	base = (uintptr)s->base >> PageShift;
+	for(i=newlength; i<s->length; i++)
+		pminsert(&spanmap, base+i, nil);
+
+	s->length = newlength;
+}
+
+// Find the span for a given pointer.
+static Span*
+spanofptr(void *v)
+{
+	return pmlookup(&spanmap, (uintptr)v >> PageShift);
+}
+
+static void freespan(Span*);
+
+// Allocate a span of at least n pages.
+static Span*
+allocspan(int32 npage)
+{
+	Span *s, **l, *s1;
+	int32 allocnpage, i;
+
+	// Look in the n-page free lists for big enough n.
+	for(i=npage; i<nelem(central.free); i++) {
+		s = central.free[i];
+		if(s != nil) {
+			central.free[i] = s->next;
+			goto havespan;
+		}
+	}
+
+	// Look in the large list, which has large runs of pages.
+	for(l=&central.large; (s=*l) != nil; l=&s->next) {
+		if(s->length >= npage) {
+			*l = s->next;
+			s->next = nil;
+if(s->length > npage) {
+prints("Chop span");
+sys·printint(s->length);
+prints(" for ");
+sys·printint(npage);
+prints("\n");
+}
+			goto havespan;
+		}
+	}
+
+	// Otherwise we need more memory.
+	// TODO: Could try to release this lock while asking for memory.
+	s = trivalloc(sizeof *s);
+	allocnpage = npage;
+	if(allocnpage < (1<<20>>PageShift))	// TODO: Tune
+		allocnpage = (1<<20>>PageShift);
+	s->length = allocnpage;
+prints("New span ");
+sys·printint(allocnpage);
+prints(" for ");
+sys·printint(npage);
+prints("\n");
+	s->base = trivalloc(allocnpage<<PageShift);
+	insertspan(s);
+
+havespan:
+	// If span is bigger than needed, redistribute the remainder.
+	if(s->length > npage) {
+		s1 = trivalloc(sizeof *s);
+		s1->base = s->base + (npage << PageShift);
+		s1->length = s->length - npage;
+		shrinkspan(s, npage);
+		insertspan(s1);
+		freespan(s1);
+	}
+	s->state = SpanInUse;
+	return s;
+}
+
+// Free a span.
+static void
+freespan(Span *s)
+{
+	Span **l;
+	Span *ss;
+
+	s->state = SpanFree;
+	if(s->length < nelem(central.free)) {
+		s->next = central.free[s->length];
+		central.free[s->length] = s;
+	} else {
+		// Keep central.large sorted in
+		// increasing size for best-fit allocation.
+		for(l = &central.large; (ss=*l) != nil; l=&ss->next)
+			if(ss->length >= s->length)
+				break;
+		s->next = *l;
+		*l = s;
+	}
+}
+
+// Small objects are kept on per-size free lists in the M.
+// There are SmallFreeClasses (defined in runtime.h) different lists.
+static int32 classtosize[SmallFreeClasses] = {
+	/*
+	seq 8 8 127 | sed 's/$/,/' | fmt
+	seq 128 16 255 | sed 's/$/,/' | fmt
+	seq 256 32 511 | sed 's/$/,/' | fmt
+	seq 512 64 1023 | sed 's/$/,/' | fmt
+	seq 1024 128 2047 | sed 's/$/,/' | fmt
+	seq 2048 256 32768 | sed 's/$/,/' | fmt
+	*/
+	8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120,
+	128, 144, 160, 176, 192, 208, 224, 240,
+	256, 288, 320, 352, 384, 416, 448, 480,
+	512, 576, 640, 704, 768, 832, 896, 960,
+	1024, 1152, 1280, 1408, 1536, 1664, 1792, 1920,
+	2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4352, 4608,
+	4864, 5120, 5376, 5632, 5888, 6144, 6400, 6656, 6912, 7168, 7424,
+	7680, 7936, 8192, 8448, 8704, 8960, 9216, 9472, 9728, 9984, 10240,
+	10496, 10752, 11008, 11264, 11520, 11776, 12032, 12288, 12544,
+	12800, 13056, 13312, 13568, 13824, 14080, 14336, 14592, 14848,
+	15104, 15360, 15616, 15872, 16128, 16384, 16640, 16896, 17152,
+	17408, 17664, 17920, 18176, 18432, 18688, 18944, 19200, 19456,
+	19712, 19968, 20224, 20480, 20736, 20992, 21248, 21504, 21760,
+	22016, 22272, 22528, 22784, 23040, 23296, 23552, 23808, 24064,
+	24320, 24576, 24832, 25088, 25344, 25600, 25856, 26112, 26368,
+	26624, 26880, 27136, 27392, 27648, 27904, 28160, 28416, 28672,
+	28928, 29184, 29440, 29696, 29952, 30208, 30464, 30720, 30976,
+	31232, 31488, 31744, 32000, 32256, 32512, 32768,
+};
+enum {
+	LargeSize = 32768
+};
+
+// Trigger compile error if nelem(classtosize) != SmallFreeClasses.
+static int32 zzz1[SmallFreeClasses-nelem(classtosize)+1];
+static int32 zzz2[nelem(classtosize)-SmallFreeClasses+1];
+
+static int32
+sizetoclass(int32 siz)
+{
+	if(siz <= 0)
+		return 0;
+	if(siz <= 128)
+		return (siz-1) >> 3;
+	if(siz <= 256)
+		return ((siz-1) >> 4) + 8;
+	if(siz <= 512)
+		return ((siz-1) >> 5) + 16;
+	if(siz <= 1024)
+		return ((siz-1) >> 6) + 24;
+	if(siz <= 2048)
+		return ((siz-1) >> 7) + 32;
+	if(siz <= 32768)
+		return ((siz-1) >> 8) + 40;
+	throw("sizetoclass - invalid size");
+	return -1;
+}
+
+void
+allocator·testsizetoclass(void)
+{
+	int32 i, n;
+
+	n = 0;
+	for(i=0; i<nelem(classtosize); i++) {
+		for(; n <= classtosize[i]; n++) {
+			if(sizetoclass(n) != i) {
+				prints("sizetoclass ");
+				sys·printint(n);
+				prints(" = ");
+				sys·printint(sizetoclass(n));
+				prints(" want ");
+				sys·printint(i);
+				prints("\n");
+				throw("testsizetoclass");
+			}
+		}
+	}
+	if (n != 32768+1) {
+		prints("testsizetoclass stopped at ");
+		sys·printint(n);
+		prints("\n");
+		throw("testsizetoclass");
+	}
+}
+
+// Grab a bunch of objects of size class cl off the central free list.
+// Set *pn to the number of objects returned.
+static void*
+centralgrab(int32 cl, int32 *pn)
+{
+	byte *p;
+	Span *s;
+	int32 chunk, i, n, siz;
+
+	// For now there is no central free list.
+	// Fall back to allocating a new span
+	// and chopping it up.
+	chunk = classtosize[cl] * 1024;
+	if(chunk > 1<<20) {
+		chunk = 1<<20;
+	}
+	chunk = (chunk+PageMask) & ~PageMask;
+	s = allocspan(chunk>>PageShift);
+prints("New Class ");
+sys·printint(cl);
+prints("\n");
+	s->state = SpanInUse;
+	s->cl = cl;
+	siz = classtosize[cl];
+	n = chunk/siz;
+	p = s->base;
+	for(i=0; i<n-1; i++) {
+		*(void**)p = p+siz;
+		p += siz;
+	}
+	*pn = n;
+	return p;
+}
+
+// Allocate a small object of size class cl.
+void*
+allocsmall(int32 cl)
+{
+	void **p;
+	int32 n;
+
+	if(cl < 0 || cl >= SmallFreeClasses)
+		throw("allocsmall - invalid class");
+
+	// try m-local cache.
+	p = m->freelist[cl];
+	if(p == nil) {
+		// otherwise grab some blocks from central cache.
+		lock(&central);
+		p = centralgrab(cl, &n);
+		// TODO: update local counters using n
+		unlock(&central);
+	}
+
+	// advance linked list.
+	m->freelist[cl] = *p;
+
+	// Blocks on free list are zeroed except for
+	// the linked list pointer that we just used.  Zero it.
+	*p = 0;
+
+	return p;
+}
+
+// Allocate large object of np pages.
+void*
+alloclarge(int32 np)
+{
+	Span *s;
+
+	lock(&central);
+//prints("Alloc span ");
+//sys·printint(np);
+//prints("\n");
+	s = allocspan(np);
+	unlock(&central);
+	s->state = SpanInUse;
+	s->cl = -1;
+	return s->base;
+}
+
+// Allocate object of n bytes.
+void*
+alloc(int32 n)
+{
+	int32 cl, np;
+
+	if(n < LargeSize) {
+		cl = sizetoclass(n);
+		if(cl < 0 || cl >= SmallFreeClasses) {
+			sys·printint(n);
+			prints(" -> ");
+			sys·printint(cl);
+			prints("\n");
+			throw("alloc - logic error");
+		}
+		return allocsmall(sizetoclass(n));
+	}
+
+	// count number of pages; careful about overflow for big n.
+	np = (n>>PageShift) + (((n&PageMask)+PageMask)>>PageShift);
+	return alloclarge(np);
+}
+
+void
+allocator·malloc(int32 n, byte *out)
+{
+	out = alloc(n);
+	FLUSH(&out);
+}
+
+// Free object with base pointer v.
+void
+free(void *v)
+{
+	void **p;
+	Span *s;
+	int32 siz, off;
+
+	s = spanofptr(v);
+	if(s->state != SpanInUse)
+		throw("free - invalid pointer1");
+
+	// Big object should be s->base.
+	if(s->cl < 0) {
+		if(v != s->base)
+			throw("free - invalid pointer2");
+		// TODO: For large spans, maybe just return the
+		// memory to the operating system and let it zero it.
+		sys·memclr(s->base, s->length << PageShift);
+//prints("Free big ");
+//sys·printint(s->length);
+//prints("\n");
+		lock(&central);
+		freespan(s);
+		unlock(&central);
+		return;
+	}
+
+	// Small object should be aligned properly.
+	siz = classtosize[s->cl];
+	off = (byte*)v - (byte*)s->base;
+	if(off%siz)
+		throw("free - invalid pointer3");
+
+	// Zero and add to free list.
+	sys·memclr(v, siz);
+	p = v;
+	*p = m->freelist[s->cl];
+	m->freelist[s->cl] = p;
+}
+
+void
+allocator·free(byte *v)
+{
+	free(v);
+}
+
+void
+allocator·memset(byte *v, int32 c, int32 n)
+{
+	int32 i;
+
+	for(i=0; i<n; i++)
+		v[i] = c;
+}
+
+// Allocate stack segment.
+// Must be done without holding locks, because
+// calling any function might trigger another stack segment allocation.
+void*
+allocstack(int32 n)
+{
+	// TODO
+	USED(n);
+	return nil;
+}
+
+void
+freestack(void *v)
+{
+	// TODO
+	USED(v);
+}
+

usr/rsc/mem/malloc.h

+// Copyright 2009 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "../../../src/runtime/runtime.h"
+
+typedef struct PageMap PageMap;
+
+enum
+{
+	PageShift = 12,
+	PageMask = (1<<PageShift) - 1,
+};
+
+enum {
+	PMBits = 64 - PageShift,
+	PMLevels = 4,
+	PMLevelBits = 13,
+	PMLevelSize = 1<<PMLevelBits,
+	PMLevelMask = PMLevelSize - 1,
+};
+struct PageMap
+{
+	void *level0[PMLevelSize];
+};
+
+extern int64 allocator·footprint;
+extern bool allocator·frozen;
+
+void* trivalloc(int32);
+void* pmlookup(PageMap*, uintptr);
+void* pminsert(PageMap*, uintptr, void*);

usr/rsc/mem/pagemap.c

+// Copyright 2009 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "malloc.h"
+
+// A PageMap maps page numbers to void* pointers.
+// The AMD64 has 64-bit addresses and 4096-byte pages, so
+// the page numbers are 52 bits.  We use a four-level radix tree,
+// with 13 bits for each level.  This requires 32 kB per level or
+// 128 kB for a table with one entry.  Moving to three levels of 18 bits
+// would require 3 MB for a table with one entry, which seems too expensive.
+// This is easy to change.
+// It may be that a balanced tree would be better anyway.
+
+// Return the entry for page number pn in m.
+void*
+pmlookup(PageMap *m, uintptr pn)
+{
+	int32 i, x;
+	void **v;
+
+	v = &m->level0[0];
+	for(i=0; i<PMLevels; i++) {
+		// Pick off top PMLevelBits bits as index and shift up.
+		x = (pn >> (PMBits - PMLevelBits)) & PMLevelMask;
+		pn <<= PMLevelBits;
+
+		// Walk down using index.
+		v = v[x];
+		if(v == nil)
+			return nil;
+	}
+	return v;
+}
+
+// Set the entry for page number pn in m to s.
+// Return the old value.
+void*
+pminsert(PageMap *m, uintptr pn, void *value)
+{
+	int32 i, x;
+	void **v, **l;
+
+	l = nil;	// shut up 6c
+	v = &m->level0[0];
+	for(i=0; i<PMLevels; i++) {
+		// Pick off top PMLevelBits bits as index and shift up.
+		x = (pn >> (PMBits - PMLevelBits)) & PMLevelMask;
+		pn <<= PMLevelBits;
+
+		// Walk down using index, but remember location of pointer.
+		l = &v[x];
+		v = *l;
+
+		// Allocate new level if needed.
+		if(v == nil && i < PMLevels-1) {
+			v = trivalloc(PMLevelSize * sizeof v[0]);
+			*l = v;
+		}
+	}
+
+	// Record new value and return old.
+	*l = value;
+	return v;
+}

usr/rsc/mem/testrandom.go

+// Copyright 2009 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"allocator";
+	"rand"
+)
+
+var footprint int64;
+var allocated int64;
+func bigger() {
+	if footprint < allocator.footprint {
+		footprint = allocator.footprint;
+		println("Footprint", footprint, " for ", allocated);
+		if footprint > 1e9 {
+			panicln("too big");
+		}
+	}
+}
+
+// Prime the data structures by allocating one of
+// each block in order.  After this, there should be
+// little reason to ask for more memory from the OS.
+func prime() {
+	for i := 0; i < 16; i++ {
+		b := allocator.malloc(1<<uint(i));
+		allocator.free(b);
+	}
+	for i := 0; i < 256; i++ {
+		b := allocator.malloc(i<<12);
+		allocator.free(b);
+	}
+}
+
+func main() {
+//	prime();
+	var blocks [1] struct { base *byte; siz int; };
+	for i := 0; i < 1 << 20; i++ {
+		if i%(1<<10) == 0 {
+			println(i);
+		}
+		b := rand.rand() % len(blocks);
+		if blocks[b].base != nil {
+		//	println("Free", blocks[b].siz, blocks[b].base);
+			allocator.free(blocks[b].base);
+			blocks[b].base = nil;
+			allocated -= int64(blocks[b].siz);
+			continue
+		}
+		siz := rand.rand() >> (11 + rand.urand32() % 20);
+		base := allocator.malloc(siz);
+		blocks[b].base = base;
+		blocks[b].siz = siz;
+		allocated += int64(siz);
+	//	println("Alloc", siz, base);
+		allocator.memset(base, 0xbb, siz);
+		bigger();
+	}
+}

usr/rsc/mem/testrepeat.go

+// Copyright 2009 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"allocator"
+)
+
+var footprint int64
+func bigger() {
+	if footprint < allocator.footprint {
+		footprint = allocator.footprint;
+		println("Footprint", footprint);
+	}
+}
+
+func main() {
+	for i := 0; i < 1<<16; i++ {
+		for j := 1; j <= 1<<22; j<<=1 {
+			if i == 0 {
+				println("First alloc:", j);
+			}
+			b := allocator.malloc(j);
+			allocator.free(b);
+			bigger();
+		}
+		if i%(1<<10) == 0 {
+			println(i);
+		}
+		if i == 0 {
+			println("Primed", i);
+			allocator.frozen = true;
+		}
+	}
+}

usr/rsc/mem/testsizetoclass.go

+// Copyright 2009 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "allocator"
+
+func main() {
+	allocator.testsizetoclass()
+}

usr/rsc/mem/triv.c

+// Copyright 2009 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Trivial base allocator.
+
+#include "malloc.h"
+
+// TODO: The call to sys·mmap should be a call to an assembly
+// function sys·mmapnew that takes only a size parameter.
+enum
+{
+	PROT_NONE	= 0x00,
+	PROT_READ	= 0x01,
+	PROT_WRITE	= 0x02,
+	PROT_EXEC	= 0x04,
+
+	MAP_FILE	= 0x0000,
+	MAP_SHARED	= 0x0001,
+	MAP_PRIVATE	= 0x0002,
+	MAP_FIXED	= 0x0010,
+	MAP_ANON	= 0x1000,
+};
+
+// Allocate and return zeroed memory.
+// Simple allocator for small things like Span structures,
+// and also used to grab large amounts of memory for
+// the real allocator to hand out.
+enum
+{
+	Round = 15,
+};
+void*
+trivalloc(int32 size)
+{
+	static byte *p;
+	static int32 n;
+	byte *v;
+
+	if(allocator·frozen)
+		throw("allocator frozen");
+
+//prints("Newmem: ");
+//sys·printint(size);
+//prints("\n");
+
+	if(size < 4096) {	// TODO: Tune constant.
+		size = (size + Round) & ~Round;
+		if(size > n) {
+			n = 1<<20;	// TODO: Tune constant.
+			p = sys·mmap(nil, n, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, 0, 0);
+			allocator·footprint += n;
+		}
+		v = p;
+		p += size;
+		return v;
+	}
+	if(size & PageMask)
+		size += (1<<PageShift) - (size & PageMask);
+	v = sys·mmap(nil, size, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, 0, 0);
+	allocator·footprint += size;
+	return v;
+}
+