cmd/5g, cmd/5l, cmd/6g, cmd/6l, cmd/8g, cmd/8l, cmd/gc, runtime: generate…

cmd/5g, cmd/5l, cmd/6g, cmd/6l, cmd/8g, cmd/8l, cmd/gc, runtime: generate pointer maps by liveness analysis

This change allows the garbage collector to examine stack
slots that are determined as live and containing a pointer
value by the garbage collector.  This results in a mean
reduction of 65% in the number of stack slots scanned during
an invocation of "GOGC=1 all.bash".

Unfortunately, this does not yet allow garbage collection to
be precise for the stack slots computed as live.  Pointers
confound the determination of what definitions reach a given
instruction.  In general, this problem is not solvable without
runtime cost but some advanced cooperation from the compiler
might mitigate common cases.

R=golang-dev, rsc, cshapiro
CC=golang-dev
https://golang.org/cl/14430048

src/cmd/5g/ggen.c

@@ -9,15 +9,11 @@
 #include "gg.h"
 #include "opt.h"
 
-static Prog* appendp(Prog*, int, int, int, int32, int, int, int32);
-
 void
-defframe(Prog *ptxt, Bvec *bv)
+defframe(Prog *ptxt)
 {
-	int i, j, first;
 	uint32 frame;
-	Prog *p, *p1;
-	
+
 	// fill in argument size
 	ptxt->to.type = D_CONST2;
 	ptxt->to.offset2 = rnd(curfn->type->argwid, widthptr);
@@ -28,59 +24,6 @@ defframe(Prog *ptxt, Bvec *bv)
 	frame = rnd(maxstksize+maxarg, widthptr);
 	ptxt->to.offset = frame;
 	maxstksize = 0;
-
-	// insert code to clear pointered part of the frame,
-	// so that garbage collector only sees initialized values
-	// when it looks for pointers.
-	p = ptxt;
-	while(p->link->as == AFUNCDATA || p->link->as == APCDATA || p->link->as == ATYPE)
-		p = p->link;
-	if(stkzerosize >= 8*widthptr) {
-		p = appendp(p, AMOVW, D_CONST, NREG, 0, D_REG, 0, 0);
-		p = appendp(p, AADD, D_CONST, NREG, 4+frame-stkzerosize, D_REG, 1, 0);
-		p->reg = REGSP;
-		p = appendp(p, AADD, D_CONST, NREG, stkzerosize, D_REG, 2, 0);
-		p->reg = 1;
-		p1 = p = appendp(p, AMOVW, D_REG, 0, 0, D_OREG, 1, 4);
-		p->scond |= C_PBIT;
-		p = appendp(p, ACMP, D_REG, 1, 0, D_NONE, 0, 0);
-		p->reg = 2;
-		p = appendp(p, ABNE, D_NONE, NREG, 0, D_BRANCH, NREG, 0);
-		patch(p, p1);
-	} else {
-		first = 1;
-		j = (stkptrsize - stkzerosize)/widthptr * 2;
-		for(i=0; i<stkzerosize; i+=widthptr) {
-			if(bvget(bv, j) || bvget(bv, j+1)) {
-				if(first) {
-					p = appendp(p, AMOVW, D_CONST, NREG, 0, D_REG, 0, 0);
-					first = 0;
-				}
-				p = appendp(p, AMOVW, D_REG, 0, 0, D_OREG, REGSP, 4+frame-stkzerosize+i);
-			}
-			j += 2;
-		}
-	}
-}
-
-static Prog*
-appendp(Prog *p, int as, int ftype, int freg, int32 foffset, int ttype, int treg, int32 toffset)
-{
-	Prog *q;
-	
-	q = mal(sizeof(*q));
-	clearp(q);
-	q->as = as;
-	q->lineno = p->lineno;
-	q->from.type = ftype;
-	q->from.reg = freg;
-	q->from.offset = foffset;
-	q->to.type = ttype;
-	q->to.reg = treg;
-	q->to.offset = toffset;
-	q->link = p->link;
-	p->link = q;
-	return q;
 }
 
 // Sweep the prog list to mark any used nodes.
@@ -829,6 +772,8 @@ clearfat(Node *nl)
 	if(componentgen(N, nl))
 		return;
 
+	gfatvardef(nl);
+
 	c = w % 4;	// bytes
 	q = w / 4;	// quads
 

src/cmd/5g/peep.c

@@ -1164,6 +1164,7 @@ copyu(Prog *p, Adr *v, Adr *s)
 
 	case APCDATA:
 	case AFUNCDATA:
+	case AFATVARDEF:
 		return 0;
 	}
 }

src/cmd/5g/prog.c

@@ -29,6 +29,7 @@ static ProgInfo progtable[ALAST] = {
 	[AUNDEF]=	{OK},
 	[AUSEFIELD]=	{OK},
 	[ACHECKNIL]=	{LeftRead},
+	[AFATVARDEF]=	{Pseudo | RightWrite},
 
 	// NOP is an internal no-op that also stands
 	// for USED and SET annotations, not the Intel opcode.

src/cmd/5g/reg.c

@@ -168,8 +168,7 @@ regopt(Prog *firstp)
 		fmtinstall('Q', Qconv);
 		first = 0;
 	}
-	
-	fixjmp(firstp);
+
 	mergetemp(firstp);
 
 	/*

src/cmd/5l/5.out.h

@@ -198,6 +198,7 @@ enum	as
 	AFUNCDATA,
 	APCDATA,
 	ACHECKNIL,
+	AFATVARDEF,
 
 	ALAST,
 };

src/cmd/6g/ggen.c

@@ -9,14 +9,10 @@
 #include "gg.h"
 #include "opt.h"
 
-static Prog* appendp(Prog*, int, int, vlong, int, vlong);
-
 void
-defframe(Prog *ptxt, Bvec *bv)
+defframe(Prog *ptxt)
 {
-	int i, j;
 	uint32 frame;
-	Prog *p;
 
 	// fill in argument size
 	ptxt->to.offset = rnd(curfn->type->argwid, widthptr);
@@ -25,43 +21,6 @@ defframe(Prog *ptxt, Bvec *bv)
 	ptxt->to.offset <<= 32;
 	frame = rnd(stksize+maxarg, widthptr);
 	ptxt->to.offset |= frame;
-
-	// insert code to clear pointered part of the frame,
-	// so that garbage collector only sees initialized values
-	// when it looks for pointers.
-	p = ptxt;
-	if(stkzerosize >= 8*widthptr) {
-		p = appendp(p, AMOVQ, D_CONST, 0, D_AX, 0);
-		p = appendp(p, AMOVQ, D_CONST, stkzerosize/widthptr, D_CX, 0);
-		p = appendp(p, ALEAQ, D_SP+D_INDIR, frame-stkzerosize, D_DI, 0);
-		p = appendp(p, AREP, D_NONE, 0, D_NONE, 0);
-		appendp(p, ASTOSQ, D_NONE, 0, D_NONE, 0);
-	} else {
-		j = (stkptrsize - stkzerosize)/widthptr * 2;
-		for(i=0; i<stkzerosize; i+=widthptr) {
-			if(bvget(bv, j) || bvget(bv, j+1))
-				p = appendp(p, AMOVQ, D_CONST, 0, D_SP+D_INDIR, frame-stkzerosize+i);
-			j += 2;
-		}
-	}
-}
-
-static Prog*
-appendp(Prog *p, int as, int ftype, vlong foffset, int ttype, vlong toffset)
-{
-	Prog *q;
-	
-	q = mal(sizeof(*q));
-	clearp(q);
-	q->as = as;
-	q->lineno = p->lineno;
-	q->from.type = ftype;
-	q->from.offset = foffset;
-	q->to.type = ttype;
-	q->to.offset = toffset;
-	q->link = p->link;
-	p->link = q;
-	return q;
 }
 
 // Sweep the prog list to mark any used nodes.
@@ -1037,6 +996,8 @@ clearfat(Node *nl)
 	if(componentgen(N, nl))
 		return;
 
+	gfatvardef(nl);
+
 	c = w % 8;	// bytes
 	q = w / 8;	// quads
 

src/cmd/6g/prog.c

@@ -41,6 +41,7 @@ static ProgInfo progtable[ALAST] = {
 	[AUNDEF]=	{OK},
 	[AUSEFIELD]=	{OK},
 	[ACHECKNIL]=	{LeftRead},
+	[AFATVARDEF]=	{Pseudo | RightWrite},
 
 	// NOP is an internal no-op that also stands
 	// for USED and SET annotations, not the Intel opcode.

src/cmd/6g/reg.c

@@ -155,9 +155,8 @@ regopt(Prog *firstp)
 		first = 0;
 	}
 
-	fixjmp(firstp);
 	mergetemp(firstp);
-	
+
 	/*
 	 * control flow is more complicated in generated go code
 	 * than in generated c code.  define pseudo-variables for

src/cmd/6l/6.out.h

@@ -762,6 +762,7 @@ enum	as
 	AFUNCDATA,
 	APCDATA,
 	ACHECKNIL,
+	AFATVARDEF,
 	
 	ALAST
 };

src/cmd/6l/optab.c

@@ -1352,8 +1352,11 @@ Optab optab[] =
 	{ APCLMULQDQ,	yxshuf,	Pq, 0x3a,0x44,0 },
 
 	{ AUSEFIELD,	ynop,	Px, 0,0 },
+	{ ATYPE },
 	{ AFUNCDATA,	yfuncdata,	Px, 0,0 },
 	{ APCDATA,	ypcdata,	Px, 0,0 },
+	{ ACHECKNIL },
+	{ AFATVARDEF },
 
 	{ AEND },
 	0

src/cmd/8g/ggen.c

@@ -9,14 +9,10 @@
 #include "gg.h"
 #include "opt.h"
 
-static Prog* appendp(Prog*, int, int, int32, int, int32);
-
 void
-defframe(Prog *ptxt, Bvec *bv)
+defframe(Prog *ptxt)
 {
 	uint32 frame;
-	Prog *p;
-	int i, j;
 
 	// fill in argument size
 	ptxt->to.offset2 = rnd(curfn->type->argwid, widthptr);
@@ -27,43 +23,6 @@ defframe(Prog *ptxt, Bvec *bv)
 	frame = rnd(maxstksize+maxarg, widthptr);
 	ptxt->to.offset = frame;
 	maxstksize = 0;
-
-	// insert code to clear pointered part of the frame,
-	// so that garbage collector only sees initialized values
-	// when it looks for pointers.
-	p = ptxt;
-	if(stkzerosize >= 8*widthptr) {
-		p = appendp(p, AMOVL, D_CONST, 0, D_AX, 0);
-		p = appendp(p, AMOVL, D_CONST, stkzerosize/widthptr, D_CX, 0);
-		p = appendp(p, ALEAL, D_SP+D_INDIR, frame-stkzerosize, D_DI, 0);
-		p = appendp(p, AREP, D_NONE, 0, D_NONE, 0);
-		appendp(p, ASTOSL, D_NONE, 0, D_NONE, 0);
-	} else {
-		j = (stkptrsize - stkzerosize)/widthptr * 2;
-		for(i=0; i<stkzerosize; i+=widthptr) {
-			if(bvget(bv, j) || bvget(bv, j+1))
-				p = appendp(p, AMOVL, D_CONST, 0, D_SP+D_INDIR, frame-stkzerosize+i);
-			j += 2;
-		}
-	}
-}
-
-static Prog*
-appendp(Prog *p, int as, int ftype, int32 foffset, int ttype, int32 toffset)
-{
-	Prog *q;
-	
-	q = mal(sizeof(*q));
-	clearp(q);
-	q->as = as;
-	q->lineno = p->lineno;
-	q->from.type = ftype;
-	q->from.offset = foffset;
-	q->to.type = ttype;
-	q->to.offset = toffset;
-	q->link = p->link;
-	p->link = q;
-	return q;
 }
 
 // Sweep the prog list to mark any used nodes.
@@ -119,6 +78,8 @@ clearfat(Node *nl)
 	if(componentgen(N, nl))
 		return;
 
+	gfatvardef(nl);
+
 	c = w % 4;	// bytes
 	q = w / 4;	// quads
 

src/cmd/8g/prog.c

@@ -41,6 +41,7 @@ static ProgInfo progtable[ALAST] = {
 	[AUNDEF]=	{OK},
 	[AUSEFIELD]=	{OK},
 	[ACHECKNIL]=	{LeftRead},
+	[AFATVARDEF]=	{Pseudo | RightWrite},
 
 	// NOP is an internal no-op that also stands
 	// for USED and SET annotations, not the Intel opcode.

src/cmd/8g/reg.c

@@ -124,8 +124,7 @@ regopt(Prog *firstp)
 		exregoffset = D_DI;	// no externals
 		first = 0;
 	}
-	
-	fixjmp(firstp);
+
 	mergetemp(firstp);
 
 	/*

src/cmd/8l/8.out.h

@@ -578,6 +578,7 @@ enum	as
 	AFUNCDATA,
 	APCDATA,
 	ACHECKNIL,
+	AFATVARDEF,
 	
 	ALAST
 };

src/cmd/8l/optab.c

@@ -1025,6 +1025,8 @@ Optab optab[] =
 	{ ATYPE },
 	{ AFUNCDATA,	yfuncdata,	Px, 0,0 },
 	{ APCDATA,	ypcdata,	Px, 0,0 },
+	{ ACHECKNIL },
+	{ AFATVARDEF },
 
 	0
 };

src/cmd/cc/pgen.c

@@ -35,6 +35,25 @@ enum { BitsPerPointer = 2 };
 
 static void dumpgcargs(Type *fn, Sym *sym);
 
+static Sym*
+makefuncdatasym(char *namefmt, int64 funcdatakind)
+{
+	Node nod;
+	Sym *sym;
+	static int32 nsym;
+	static char namebuf[40];
+
+	snprint(namebuf, sizeof(namebuf), namefmt, nsym++);
+	sym = slookup(namebuf);
+	sym->class = CSTATIC;
+	memset(&nod, 0, sizeof nod);
+	nod.op = ONAME;
+	nod.sym = sym;
+	nod.class = CSTATIC;
+	gins(AFUNCDATA, nodconst(funcdatakind), &nod);
+	return sym;
+}
+
 int
 hasdotdotdot(void)
 {
@@ -80,10 +99,10 @@ void
 codgen(Node *n, Node *nn)
 {
 	Prog *sp;
-	Node *n1, nod, nod1, nod2;
-	Sym *gcsym, *gclocalssym;
-	static int ngcsym, ngclocalssym;
-	static char namebuf[40];
+	Node *n1, nod, nod1;
+	Sym *gcargs;
+	Sym *gclocals;
+	int isvarargs;
 
 	cursafe = 0;
 	curarg = 0;
@@ -109,25 +128,11 @@ codgen(Node *n, Node *nn)
 	 * generate funcdata symbol for this function.
 	 * data is filled in at the end of codgen().
 	 */
-	snprint(namebuf, sizeof namebuf, "gc·%d", ngcsym++);
-	gcsym = slookup(namebuf);
-	gcsym->class = CSTATIC;
-
-	memset(&nod, 0, sizeof nod);
-	nod.op = ONAME;
-	nod.sym = gcsym;
-	nod.class = CSTATIC;
-	gins(AFUNCDATA, nodconst(FUNCDATA_GCArgs), &nod);
-
-	snprint(namebuf, sizeof(namebuf), "gclocalssym·%d", ngclocalssym++);
-	gclocalssym = slookup(namebuf);
-	gclocalssym->class = CSTATIC;
-
-	memset(&nod2, 0, sizeof(nod2));
-	nod2.op = ONAME;
-	nod2.sym = gclocalssym;
-	nod2.class = CSTATIC;
-	gins(AFUNCDATA, nodconst(FUNCDATA_GCLocals), &nod2);
+	isvarargs = hasdotdotdot();
+	gcargs = nil;
+	if(!isvarargs)
+		gcargs = makefuncdatasym("gcargs·%d", FUNCDATA_ArgsPointerMaps);
+	gclocals = makefuncdatasym("gclocals·%d", FUNCDATA_LocalsPointerMaps);
 
 	/*
 	 * isolate first argument
@@ -166,7 +171,8 @@ codgen(Node *n, Node *nn)
 		maxargsafe = xround(maxargsafe, 8);
 	sp->to.offset += maxargsafe;
 
-	dumpgcargs(thisfn, gcsym);
+	if(!isvarargs)
+		dumpgcargs(thisfn, gcargs);
 
 	// TODO(rsc): "stkoff" is not right. It does not account for
 	// the possibility of data stored in .safe variables.
@@ -177,9 +183,9 @@ codgen(Node *n, Node *nn)
 	// area its own section.
 	// That said, we've been using stkoff for months
 	// and nothing too terrible has happened.
-	gextern(gclocalssym, nodconst(-stkoff), 0, 4); // locals
-	gclocalssym->type = typ(0, T);
-	gclocalssym->type->width = 4;
+	gextern(gclocals, nodconst(-stkoff), 0, 4); // locals
+	gclocals->type = typ(0, T);
+	gclocals->type->width = 4;
 }
 
 void
@@ -715,39 +721,39 @@ dumpgcargs(Type *fn, Sym *sym)
 	int32 argbytes;
 	int32 symoffset, argoffset;
 
-	if(hasdotdotdot()) {
-		// give up for C vararg functions.
-		// TODO: maybe make a map just for the args we do know?
-		gextern(sym, nodconst(0), 0, 4); // nptrs=0
-		symoffset = 4;
-	} else {
-		argbytes = (argsize() + ewidth[TIND] - 1);
-		bv = bvalloc((argbytes  / ewidth[TIND]) * BitsPerPointer);
-		argoffset = align(0, fn->link, Aarg0, nil);
-		if(argoffset > 0) {
-			// The C calling convention returns structs by
-			// copying them to a location pointed to by a
-			// hidden first argument.  This first argument
-			// is a pointer.
-			if(argoffset != ewidth[TIND])
-				yyerror("passbyptr arg not the right size");
-			bvset(bv, 0);
-		}
-		for(t = fn->down; t != T; t = t->down) {
-			if(t->etype == TVOID)
-				continue;
-			argoffset = align(argoffset, t, Aarg1, nil);
-			walktype1(t, argoffset, bv, 1);
-			argoffset = align(argoffset, t, Aarg2, nil);
-		}
-		gextern(sym, nodconst(bv->n), 0, 4);
-		symoffset = 4;
-		for(i = 0; i < bv->n; i += 32) {
-			gextern(sym, nodconst(bv->b[i/32]), symoffset, 4);
-			symoffset += 4;
-		}
-		free(bv);
+	// Dump the length of the bitmap array.  This value is always one for
+	// functions written in C.
+	symoffset = 0;
+	gextern(sym, nodconst(1), symoffset, 4);
+	symoffset += 4;
+	argbytes = (argsize() + ewidth[TIND] - 1);
+	bv = bvalloc((argbytes  / ewidth[TIND]) * BitsPerPointer);
+	argoffset = align(0, fn->link, Aarg0, nil);
+	if(argoffset > 0) {
+		// The C calling convention returns structs by copying them to a
+		// location pointed to by a hidden first argument.  This first
+		// argument is a pointer.
+		if(argoffset != ewidth[TIND])
+			yyerror("passbyptr arg not the right size");
+		bvset(bv, 0);
+	}
+	for(t = fn->down; t != T; t = t->down) {
+		if(t->etype == TVOID)
+			continue;
+		argoffset = align(argoffset, t, Aarg1, nil);
+		walktype1(t, argoffset, bv, 1);
+		argoffset = align(argoffset, t, Aarg2, nil);
+	}
+	// Dump the length of the bitmap.
+	gextern(sym, nodconst(bv->n), symoffset, 4);
+	symoffset += 4;
+	// Dump the words of the bitmap.
+	for(i = 0; i < bv->n; i += 32) {
+		gextern(sym, nodconst(bv->b[i/32]), symoffset, 4);
+		symoffset += 4;
 	}
+	free(bv);
+	// Finalize the gc symbol.
 	sym->type = typ(0, T);
 	sym->type->width = symoffset;
 }

src/cmd/dist/build.c

@@ -452,7 +452,7 @@ static char *proto_gccargs[] = {
 	// Fix available at http://patchwork.ozlabs.org/patch/64562/.
 	"-O1",
 #else
-	"-O2",
+	"-O0",
 #endif
 };
 
@@ -500,6 +500,7 @@ static struct {
 	{"cmd/gc", {
 		"-cplx.c",
 		"-pgen.c",
+		"-plive.c",
 		"-popt.c",
 		"-y1.tab.c",  // makefile dreg
 		"opnames.h",
@@ -525,6 +526,7 @@ static struct {
 	{"cmd/5g", {
 		"../gc/cplx.c",
 		"../gc/pgen.c",
+		"../gc/plive.c",
 		"../gc/popt.c",
 		"../gc/popt.h",
 		"../5l/enam.c",
@@ -533,6 +535,7 @@ static struct {
 	{"cmd/6g", {
 		"../gc/cplx.c",
 		"../gc/pgen.c",
+		"../gc/plive.c",
 		"../gc/popt.c",
 		"../gc/popt.h",
 		"../6l/enam.c",
@@ -541,6 +544,7 @@ static struct {
 	{"cmd/8g", {
 		"../gc/cplx.c",
 		"../gc/pgen.c",
+		"../gc/plive.c",
 		"../gc/popt.c",
 		"../gc/popt.h",
 		"../8l/enam.c",

src/cmd/gc/array.c

+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include <u.h>
+#include <libc.h>
+#include "go.h"
+
+enum {
+	DEFAULTCAPACITY = 16,
+};
+
+struct Array
+{
+	int32	length;  // number of elements
+	int32	size;  // element size
+	int32	capacity;  // size of data in elements
+	char	*data;  // element storage
+};
+
+Array*
+arraynew(int32 capacity, int32 size)
+{
+	Array *result;
+
+	if(capacity < 0)
+		fatal("arraynew: capacity %d is not positive", capacity);
+	if(size < 0)
+		fatal("arraynew: size %d is not positive\n", size);
+	result = malloc(sizeof(*result));
+	if(result == nil)
+		fatal("arraynew: malloc failed\n");
+	result->length = 0;
+	result->size = size;
+	result->capacity = capacity == 0 ? DEFAULTCAPACITY : capacity;
+	result->data = malloc(result->capacity * result->size);
+	if(result->data == nil)
+		fatal("arraynew: malloc failed\n");
+	return result;
+}
+
+void
+arrayfree(Array *array)
+{
+	if(array == nil)
+		return;
+	free(array->data);
+	free(array);
+}
+
+int32
+arraylength(Array *array)
+{
+	return array->length;
+}
+
+void*
+arrayget(Array *array, int32 index)
+{
+	if(array == nil)
+		fatal("arrayget: array is nil\n");
+	if(index < 0 || index >= array->length)
+		fatal("arrayget: index %d is out of bounds for length %d\n", index, array->length);
+	return array->data + index * array->size;
+}
+
+void
+arrayset(Array *array, int32 index, void *element)
+{
+	if(array == nil)
+		fatal("arrayset: array is nil\n");
+	if(element == nil)
+		fatal("arrayset: element is nil\n");
+	if(index < 0 || index >= array->length)
+		fatal("arrayget: index %d is out of bounds for length %d\n", index, array->length);
+	memmove(array->data + index * array->size, element, array->size);
+}
+
+static void
+ensurecapacity(Array *array, int32 capacity)
+{
+	int32 newcapacity;
+	char *newdata;
+
+	if(array == nil)
+		fatal("ensurecapacity: array is nil\n");
+	if(capacity < 0)
+		fatal("ensurecapacity: capacity %d is not positive", capacity);
+	if(capacity >= array->capacity) {
+		newcapacity = capacity + (capacity >> 1);
+		newdata = realloc(array->data, newcapacity * array->size);
+		if(newdata == nil)
+			fatal("ensurecapacity: realloc failed\n");
+		array->capacity = newcapacity;
+		array->data = newdata;
+	}
+}
+
+void
+arrayadd(Array *array, void *element)
+{
+	if(array == nil)
+		fatal("arrayset: array is nil\n");
+	if(element == nil)
+		fatal("arrayset: element is nil\n");
+	ensurecapacity(array, array->length + 1);
+	array->length++;
+	arrayset(array, array->length - 1, element);
+}
+
+int32
+arrayindexof(Array *array, void *element)
+{
+	void *p;
+	int32 i;
+
+	for(i = 0; i < array->length; i++) {
+		p = arrayget(array, i);
+		if(memcmp(p, &element, array->size) == 0)
+			return i;
+	}
+	return -1;
+}
+
+void
+arraysort(Array *array, int (*cmp)(const void*, const void*))
+{
+	qsort(array->data, array->length, array->size, cmp);
+}

src/cmd/gc/bv.c

@@ -11,12 +11,24 @@ enum {
 	WORDBITS = 32,
 };
 
-uintptr
+static uintptr
 bvsize(uintptr n)
 {
 	return ((n + WORDBITS - 1) / WORDBITS) * WORDSIZE;
 }
 
+int32
+bvbits(Bvec *bv)
+{
+	return bv->n;
+}
+
+int32
+bvwords(Bvec *bv)
+{
+	return (bv->n + WORDBITS - 1) / WORDBITS;
+}
+
 Bvec*
 bvalloc(int32 n)
 {
@@ -34,26 +46,49 @@ bvalloc(int32 n)
 	return bv;
 }
 
+/* difference */
 void
-bvset(Bvec *bv, int32 i)
+bvandnot(Bvec *dst, Bvec *src1, Bvec *src2)
 {
-	uint32 mask;
+	int32 i, w;
 
-	if(i < 0 || i >= bv->n)
-		fatal("bvset: index %d is out of bounds with length %d\n", i, bv->n);
-	mask = 1U << (i % WORDBITS);
-	bv->b[i / WORDBITS] |= mask;
+	if(dst->n != src1->n || dst->n != src2->n)
+		fatal("bvand: lengths %d, %d, and %d are not equal", dst->n, src1->n, src2->n);
+	for(i = 0, w = 0; i < dst->n; i += WORDBITS, w++)
+		dst->b[w] = src1->b[w] & ~src2->b[w];
+}
+
+int
+bvcmp(Bvec *bv1, Bvec *bv2)
+{
+	uintptr nbytes;
+
+	if(bv1->n != bv2->n)
+		fatal("bvequal: lengths %d and %d are not equal", bv1->n, bv2->n);
+	nbytes = bvsize(bv1->n);
+	return memcmp(bv1->b, bv2->b, nbytes);
 }
 
 void
-bvres(Bvec *bv, int32 i)
+bvcopy(Bvec *dst, Bvec *src)
 {
-	uint32 mask;
+	memmove(dst->b, src->b, bvsize(dst->n));
+}
 
-	if(i < 0 || i >= bv->n)
-		fatal("bvres: index %d is out of bounds with length %d\n", i, bv->n);
-	mask = ~(1 << (i % WORDBITS));
-	bv->b[i / WORDBITS] &= mask;
+Bvec*
+bvconcat(Bvec *src1, Bvec *src2)
+{
+	Bvec *dst;
+	int32 i;
+
+	dst = bvalloc(src1->n + src2->n);
+	for(i = 0; i < src1->n; i++)
+		if(bvget(src1, i))
+			bvset(dst, i);
+	for(i = 0; i < src2->n; i++)
+		if(bvget(src2, i))
+			bvset(dst, i + src1->n);
+	return dst;
 }
 
 int
@@ -78,3 +113,62 @@ bvisempty(Bvec *bv)
 			return 0;
 	return 1;
 }
+
+void
+bvnot(Bvec *bv)
+{
+	int32 i, w;
+
+	for(i = 0, w = 0; i < bv->n; i += WORDBITS, w++)
+		bv->b[w] = ~bv->b[w];
+}
+
+/* union */
+void
+bvor(Bvec *dst, Bvec *src1, Bvec *src2)
+{
+	int32 i, w;
+
+	if(dst->n != src1->n || dst->n != src2->n)
+		fatal("bvor: lengths %d, %d, and %d are not equal", dst->n, src1->n, src2->n);
+	for(i = 0, w = 0; i < dst->n; i += WORDBITS, w++)
+		dst->b[w] = src1->b[w] | src2->b[w];
+}
+
+void
+bvprint(Bvec *bv)
+{
+	int32 i;
+
+	print("#*");
+	for(i = 0; i < bv->n; i++)
+		print("%d", bvget(bv, i));
+}
+
+void
+bvreset(Bvec *bv, int32 i)
+{
+	uint32 mask;
+
+	if(i < 0 || i >= bv->n)
+		fatal("bvreset: index %d is out of bounds with length %d\n", i, bv->n);
+	mask = ~(1 << (i % WORDBITS));
+	bv->b[i / WORDBITS] &= mask;
+}
+
+void
+bvresetall(Bvec *bv)
+{
+	memset(bv->b, 0x00, bvsize(bv->n));
+}
+
+void
+bvset(Bvec *bv, int32 i)
+{
+	uint32 mask;
+
+	if(i < 0 || i >= bv->n)
+		fatal("bvset: index %d is out of bounds with length %d\n", i, bv->n);
+	mask = 1U << (i % WORDBITS);
+	bv->b[i / WORDBITS] |= mask;
+}

src/cmd/gc/gen.c

@@ -767,6 +767,8 @@ cgen_eface(Node *n, Node *res)
 	 * so it's important that it is done first
 	 */
 	Node dst;
+
+	gfatvardef(res);
 	dst = *res;
 	dst.type = types[tptr];
 	dst.xoffset += widthptr;
@@ -795,6 +797,8 @@ cgen_slice(Node *n, Node *res)
 	if(n->list->next->next)
 		offs = n->list->next->next->n;
 
+	gfatvardef(res);
+
 	// dst.len = hi [ - lo ]
 	dst = *res;
 	dst.xoffset += Array_nel;

src/cmd/gc/go.h

@@ -129,6 +129,7 @@ struct	Val
 	} u;
 };
 
+typedef	struct	Array	Array;
 typedef	struct	Bvec	Bvec;
 typedef	struct	Pkg Pkg;
 typedef	struct	Sym	Sym;
@@ -1003,6 +1004,18 @@ void	resumecheckwidth(void);
 vlong	rnd(vlong o, vlong r);
 void	typeinit(void);
 
+/*
+ *	array.c
+ */
+Array*	arraynew(int32 capacity, int32 size);
+void	arrayfree(Array *array);
+int32	arraylength(Array *array);
+void*	arrayget(Array *array, int32 index);
+void	arrayset(Array *array, int32 index, void *element);
+void	arrayadd(Array *array, void *element);
+int32	arrayindexof(Array* array, void *element);
+void	arraysort(Array* array, int (*cmp)(const void*, const void*));
+
 /*
  *	bits.c
  */
@@ -1021,11 +1034,18 @@ int	bset(Bits a, uint n);
  *	bv.c
  */
 Bvec*	bvalloc(int32 n);
-void	bvset(Bvec *bv, int32 i);
-void	bvres(Bvec *bv, int32 i);
+void	bvandnot(Bvec *dst, Bvec *src1, Bvec *src2);
+int	bvcmp(Bvec *bv1, Bvec *bv2);
+void	bvcopy(Bvec *dst, Bvec *src);
+Bvec*	bvconcat(Bvec *src1, Bvec *src2);
 int	bvget(Bvec *bv, int32 i);
 int	bvisempty(Bvec *bv);
-int	bvcmp(Bvec *bv1, Bvec *bv2);
+void	bvnot(Bvec *bv);
+void	bvor(Bvec *dst, Bvec *src1, Bvec *src2);
+void	bvprint(Bvec *bv);
+void	bvreset(Bvec *bv, int32 i);
+void	bvresetall(Bvec *bv);
+void	bvset(Bvec *bv, int32 i);
 
 /*
  *	closure.c
@@ -1439,7 +1459,7 @@ Node*	conv(Node*, Type*);
 int	candiscard(Node*);
 
 /*
- *	arch-specific ggen.c/gsubr.c/gobj.c/pgen.c
+ *	arch-specific ggen.c/gsubr.c/gobj.c/pgen.c/plive.c
  */
 #define	P	((Prog*)0)
 
@@ -1477,7 +1497,7 @@ void	cgen_checknil(Node*);
 void	cgen_ret(Node *n);
 void	clearfat(Node *n);
 void	compile(Node*);
-void	defframe(Prog*, Bvec*);
+void	defframe(Prog*);
 int	dgostringptr(Sym*, int off, char *str);
 int	dgostrlitptr(Sym*, int off, Strlit*);
 int	dstringptr(Sym *s, int off, char *str);
@@ -1491,10 +1511,12 @@ void	gdatacomplex(Node*, Mpcplx*);
 void	gdatastring(Node*, Strlit*);
 void	ggloblnod(Node *nam);
 void	ggloblsym(Sym *s, int32 width, int dupok, int rodata);
+void	gfatvardef(Node*);
 Prog*	gjmp(Prog*);
 void	gused(Node*);
 void	movelarge(NodeList*);
 int	isfat(Type*);
+void	liveness(Node*, Prog*, Sym*, Sym*, Sym*);
 void	markautoused(Prog*);
 Plist*	newplist(void);
 Node*	nodarg(Type*, int);

src/cmd/gc/pgen.c

@@ -12,26 +12,66 @@
 #include	"opt.h"
 #include	"../../pkg/runtime/funcdata.h"
 
-enum { BitsPerPointer = 2 };
-
 static void allocauto(Prog* p);
-static void dumpgcargs(Node*, Sym*);
-static Bvec* dumpgclocals(Node*, Sym*);
+
+static Sym*
+makefuncdatasym(char *namefmt, int64 funcdatakind)
+{
+	Node nod;
+	Node *pnod;
+	Sym *sym;
+	static int32 nsym;
+
+	snprint(namebuf, sizeof(namebuf), namefmt, nsym++);
+	sym = lookup(namebuf);
+	pnod = newname(sym);
+	pnod->class = PEXTERN;
+	nodconst(&nod, types[TINT32], funcdatakind);
+	gins(AFUNCDATA, &nod, pnod);
+	return sym;
+}
+
+void
+gfatvardef(Node *n)
+{
+	if(n == N || !isfat(n->type))
+		fatal("gfatvardef: node is not fat");
+	switch(n->class) {
+	case PAUTO:
+	case PPARAM:
+	case PPARAMOUT:
+		gins(AFATVARDEF, N, n);
+	}
+}
+
+static void
+removefatvardef(Prog *firstp)
+{
+	Prog *p;
+
+	for(p = firstp; p != P; p = p->link) {
+		while(p->link != P && p->link->as == AFATVARDEF)
+			p->link = p->link->link;
+		if(p->to.type == D_BRANCH)
+			while(p->to.u.branch != P && p->to.u.branch->as == AFATVARDEF)
+				p->to.u.branch = p->to.u.branch->link;
+	}
+}
 
 void
 compile(Node *fn)
 {
-	Bvec *bv;
 	Plist *pl;
-	Node nod1, *n, *gcargsnod, *gclocalsnod;
+	Node nod1, *n;
 	Prog *ptxt, *p, *p1;
 	int32 lno;
 	Type *t;
 	Iter save;
 	vlong oldstksize;
 	NodeList *l;
-	Sym *gcargssym, *gclocalssym;
-	static int ngcargs, ngclocals;
+	Sym *gcargs;
+	Sym *gclocals;
+	Sym *gcdead;
 
 	if(newproc == N) {
 		newproc = sysfunc("newproc");
@@ -111,21 +151,9 @@ compile(Node *fn)
 
 	ginit();
 
-	snprint(namebuf, sizeof namebuf, "gcargs·%d", ngcargs++);
-	gcargssym = lookup(namebuf);
-	gcargsnod = newname(gcargssym);
-	gcargsnod->class = PEXTERN;
-
-	nodconst(&nod1, types[TINT32], FUNCDATA_GCArgs);
-	gins(AFUNCDATA, &nod1, gcargsnod);
-
-	snprint(namebuf, sizeof(namebuf), "gclocals·%d", ngclocals++);
-	gclocalssym = lookup(namebuf);
-	gclocalsnod = newname(gclocalssym);
-	gclocalsnod->class = PEXTERN;
-
-	nodconst(&nod1, types[TINT32], FUNCDATA_GCLocals);
-	gins(AFUNCDATA, &nod1, gclocalsnod);
+	gcargs = makefuncdatasym("gcargs·%d", FUNCDATA_ArgsPointerMaps);
+	gclocals = makefuncdatasym("gclocals·%d", FUNCDATA_LocalsPointerMaps);
+	gcdead = makefuncdatasym("gcdead·%d", FUNCDATA_DeadPointerMaps);
 
 	for(t=curfn->paramfld; t; t=t->down)
 		gtrack(tracksym(t->type));
@@ -184,6 +212,7 @@ compile(Node *fn)
 	pc->as = ARET;	// overwrite AEND
 	pc->lineno = lineno;
 
+	fixjmp(ptxt);
 	if(!debug['N'] || debug['R'] || debug['P']) {
 		regopt(ptxt);
 		nilopt(ptxt);
@@ -203,184 +232,19 @@ compile(Node *fn)
 	}
 
 	// Emit garbage collection symbols.
-	dumpgcargs(fn, gcargssym);
-	bv = dumpgclocals(curfn, gclocalssym);
+	liveness(curfn, ptxt, gcargs, gclocals, gcdead);
 
-	defframe(ptxt, bv);
-	free(bv);
+	defframe(ptxt);
 
 	if(0)
 		frame(0);
 
+	// Remove leftover instrumentation from the instruction stream.
+	removefatvardef(ptxt);
 ret:
 	lineno = lno;
 }
 
-static void
-walktype1(Type *t, vlong *xoffset, Bvec *bv)
-{
-	vlong fieldoffset, i, o;
-	Type *t1;
-
-	if(t->align > 0 && (*xoffset % t->align) != 0)
-	 	fatal("walktype1: invalid initial alignment, %T", t);
-
-	switch(t->etype) {
-	case TINT8:
-	case TUINT8:
-	case TINT16:
-	case TUINT16:
-	case TINT32:
-	case TUINT32:
-	case TINT64:
-	case TUINT64:
-	case TINT:
-	case TUINT:
-	case TUINTPTR:
-	case TBOOL:
-	case TFLOAT32:
-	case TFLOAT64:
-	case TCOMPLEX64:
-	case TCOMPLEX128:
-		*xoffset += t->width;
-		break;
-
-	case TPTR32:
-	case TPTR64:
-	case TUNSAFEPTR:
-	case TFUNC:
-	case TCHAN:
-	case TMAP:
-		if(*xoffset % widthptr != 0)
-			fatal("walktype1: invalid alignment, %T", t);
-		bvset(bv, (*xoffset / widthptr) * BitsPerPointer);
-		*xoffset += t->width;
-		break;
-
-	case TSTRING:
-		// struct { byte *str; intgo len; }
-		if(*xoffset % widthptr != 0)
-			fatal("walktype1: invalid alignment, %T", t);
-		bvset(bv, (*xoffset / widthptr) * BitsPerPointer);
-		*xoffset += t->width;
-		break;
-
-	case TINTER:
-		// struct { Itab* tab;  union { void* ptr, uintptr val } data; }
-		// or, when isnilinter(t)==true:
-		// struct { Type* type; union { void* ptr, uintptr val } data; }
-		if(*xoffset % widthptr != 0)
-			fatal("walktype1: invalid alignment, %T", t);
-		bvset(bv, ((*xoffset / widthptr) * BitsPerPointer) + 1);
-		if(isnilinter(t))
-			bvset(bv, ((*xoffset / widthptr) * BitsPerPointer));
-		*xoffset += t->width;
-		break;
-
-	case TARRAY:
-		// The value of t->bound is -1 for slices types and >0 for
-		// for fixed array types.  All other values are invalid.
-		if(t->bound < -1)
-			fatal("walktype1: invalid bound, %T", t);
-		if(isslice(t)) {
-			// struct { byte* array; uintgo len; uintgo cap; }
-			if(*xoffset % widthptr != 0)
-				fatal("walktype1: invalid TARRAY alignment, %T", t);
-			bvset(bv, (*xoffset / widthptr) * BitsPerPointer);
-			*xoffset += t->width;
-		} else if(!haspointers(t->type))
-				*xoffset += t->width;
-		else
-			for(i = 0; i < t->bound; ++i)
-				walktype1(t->type, xoffset, bv);
-		break;
-
-	case TSTRUCT:
-		o = 0;
-		for(t1 = t->type; t1 != T; t1 = t1->down) {
-			fieldoffset = t1->width;
-			*xoffset += fieldoffset - o;
-			walktype1(t1->type, xoffset, bv);
-			o = fieldoffset + t1->type->width;
-		}
-		*xoffset += t->width - o;
-		break;
-
-	default:
-		fatal("walktype1: unexpected type, %T", t);
-	}
-}
-
-static void
-walktype(Type *type, Bvec *bv)
-{
-	vlong xoffset;
-
-	// Start the walk at offset 0.  The correct offset will be
-	// filled in by the first type encountered during the walk.
-	xoffset = 0;
-	walktype1(type, &xoffset, bv);
-}
-
-// Compute a bit vector to describe the pointer-containing locations
-// in the in and out argument list and dump the bitvector length and
-// data to the provided symbol.
-static void
-dumpgcargs(Node *fn, Sym *sym)
-{
-	Type *thistype, *inargtype, *outargtype;
-	Bvec *bv;
-	int32 i;
-	int off;
-
-	thistype = getthisx(fn->type);
-	inargtype = getinargx(fn->type);
-	outargtype = getoutargx(fn->type);
-	bv = bvalloc((fn->type->argwid / widthptr) * BitsPerPointer);
-	if(thistype != nil)
-		walktype(thistype, bv);
-	if(inargtype != nil)
-		walktype(inargtype, bv);
-	if(outargtype != nil)
-		walktype(outargtype, bv);
-	off = duint32(sym, 0, bv->n);
-	for(i = 0; i < bv->n; i += 32)
-		off = duint32(sym, off, bv->b[i/32]);
-	free(bv);
-	ggloblsym(sym, off, 0, 1);
-}
-
-// Compute a bit vector to describe the pointer-containing locations
-// in local variables and dump the bitvector length and data out to
-// the provided symbol. Return the vector for use and freeing by caller.
-static Bvec*
-dumpgclocals(Node* fn, Sym *sym)
-{
-	Bvec *bv;
-	NodeList *ll;
-	Node *node;
-	vlong xoffset;
-	int32 i;
-	int off;
-
-	bv = bvalloc((stkptrsize / widthptr) * BitsPerPointer);
-	for(ll = fn->dcl; ll != nil; ll = ll->next) {
-		node = ll->n;
-		if(node->class == PAUTO && node->op == ONAME) {
-			if(haspointers(node->type)) {
-				xoffset = node->xoffset + stkptrsize;
-				walktype1(node->type, &xoffset, bv);
-			}
-		}
-	}
-	off = duint32(sym, 0, bv->n);
-	for(i = 0; i < bv->n; i += 32) {
-		off = duint32(sym, off, bv->b[i/32]);
-	}
-	ggloblsym(sym, off, 0, 1);
-	return bv;
-}
-
 // Sort the list of stack variables. Autos after anything else,
 // within autos, unused after used, within used, things with
 // pointers first, zeroed things first, and then decreasing size.

src/pkg/runtime/funcdata.h

@@ -8,9 +8,11 @@
 // as well as the compilers.
 
 #define PCDATA_ArgSize 0 /* argument size at CALL instruction */
+#define PCDATA_StackMapIndex 1
 
-#define FUNCDATA_GCArgs 0 /* garbage collector blocks */
-#define FUNCDATA_GCLocals 1
+#define FUNCDATA_ArgsPointerMaps 2 /* garbage collector blocks */
+#define FUNCDATA_LocalsPointerMaps 3
+#define FUNCDATA_DeadPointerMaps 4
 
 // To be used in assembly.
 #define ARGSIZE(n) PCDATA $PCDATA_ArgSize, $n

src/pkg/runtime/mgc0.c

@@ -1367,6 +1367,33 @@ struct BitVector
 	uint32 data[];
 };
 
+typedef struct StackMap StackMap;
+struct StackMap
+{
+	int32 n;
+	uint32 data[];
+};
+
+static BitVector*
+stackmapdata(StackMap *stackmap, int32 n)
+{
+	BitVector *bv;
+	uint32 *ptr;
+	uint32 words;
+	int32 i;
+
+	if(n < 0 || n >= stackmap->n) {
+		runtime·throw("stackmapdata: index out of range");
+	}
+	ptr = stackmap->data;
+	for(i = 0; i < n; i++) {
+		bv = (BitVector*)ptr;
+		words = ((bv->n + 31) / 32) + 1;
+		ptr += words;
+	}
+	return (BitVector*)ptr;
+}
+
 // Scans an interface data value when the interface type indicates
 // that it is a pointer.
 static void
@@ -1422,101 +1449,69 @@ scanbitvector(byte *scanp, BitVector *bv, bool afterprologue, Scanbuf *sbuf)
 	}
 }
 
-static void
-addstackroots(G *gp)
-{
-	M *mp;
-	int32 n;
-	Stktop *stk;
-	uintptr sp, guard;
-	void *base;
-	uintptr size;
-
-	if(gp == g)
-		runtime·throw("can't scan our own stack");
-	if((mp = gp->m) != nil && mp->helpgc)
-		runtime·throw("can't scan gchelper stack");
-	if(gp->syscallstack != (uintptr)nil) {
-		// Scanning another goroutine that is about to enter or might
-		// have just exited a system call. It may be executing code such
-		// as schedlock and may have needed to start a new stack segment.
-		// Use the stack segment and stack pointer at the time of
-		// the system call instead, since that won't change underfoot.
-		sp = gp->syscallsp;
-		stk = (Stktop*)gp->syscallstack;
-		guard = gp->syscallguard;
-	} else {
-		// Scanning another goroutine's stack.
-		// The goroutine is usually asleep (the world is stopped).
-		sp = gp->sched.sp;
-		stk = (Stktop*)gp->stackbase;
-		guard = gp->stackguard;
-		// For function about to start, context argument is a root too.
-		if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil))
-			addroot((Obj){base, size, 0});
-	}
-	if(ScanStackByFrames) {
-		USED(sp);
-		USED(stk);
-		USED(guard);
-		addroot((Obj){(byte*)gp, PtrSize, (uintptr)gptrProg});
-	} else {
-		n = 0;
-		while(stk) {
-			if(sp < guard-StackGuard || (uintptr)stk < sp) {
-				runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
-				runtime·throw("scanstack");
-			}
-			addroot((Obj){(byte*)sp, (uintptr)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
-			sp = stk->gobuf.sp;
-			guard = stk->stackguard;
-			stk = (Stktop*)stk->stackbase;
-			n++;
-		}
-	}
-}
-
+// Scan a stack frame: local variables and function arguments/results.
 static void
 scanframe(Stkframe *frame, void *arg)
 {
-	BitVector *args, *locals;
+	Func *f;
 	Scanbuf *sbuf;
+	StackMap *stackmap;
+	BitVector *bv;
 	uintptr size;
+	uintptr targetpc;
+	int32 pcdata;
 	bool afterprologue;
 
+	f = frame->fn;
+	targetpc = frame->pc;
+	if(targetpc != f->entry)
+		targetpc--;
+	pcdata = runtime·pcdatavalue(f, PCDATA_StackMapIndex, targetpc);
+	if(pcdata == -1) {
+		// We do not have a valid pcdata value but there might be a
+		// stackmap for this function.  It is likely that we are looking
+		// at the function prologue, assume so and hope for the best.
+		pcdata = 0;
+	}
+
 	sbuf = arg;
 	// Scan local variables if stack frame has been allocated.
 	// Use pointer information if known.
 	afterprologue = (frame->varp > (byte*)frame->sp);
 	if(afterprologue) {
-		locals = runtime·funcdata(frame->fn, FUNCDATA_GCLocals);
-		if(locals == nil) {
+		stackmap = runtime·funcdata(f, FUNCDATA_LocalsPointerMaps);
+		if(stackmap == nil) {
 			// No locals information, scan everything.
 			size = frame->varp - (byte*)frame->sp;
 			*sbuf->obj.pos++ = (Obj){frame->varp - size, size, 0};
 			if(sbuf->obj.pos == sbuf->obj.end)
 				flushobjbuf(sbuf);
-		} else if(locals->n < 0) {
-			// Locals size information, scan just the
-			// locals.
-			size = -locals->n;
+		} else if(stackmap->n < 0) {
+			// Locals size information, scan just the locals.
+			size = -stackmap->n;
 			*sbuf->obj.pos++ = (Obj){frame->varp - size, size, 0};
 			if(sbuf->obj.pos == sbuf->obj.end)
-				flushobjbuf(sbuf);
-		} else if(locals->n > 0) {
-			// Locals bitmap information, scan just the
-			// pointers in locals.
-			size = (locals->n*PtrSize) / BitsPerPointer;
-			scanbitvector(frame->varp - size, locals, afterprologue, sbuf);
+				flushobjbuf(sbuf);		} else if(stackmap->n > 0) {
+			// Locals bitmap information, scan just the pointers in
+			// locals.
+			if(pcdata < 0 || pcdata >= stackmap->n) {
+				// don't know where we are
+				runtime·printf("pcdata is %d and %d stack map entries\n", pcdata, stackmap->n);
+				runtime·throw("addframeroots: bad symbol table");
+			}
+			bv = stackmapdata(stackmap, pcdata);
+			size = (bv->n * PtrSize) / BitsPerPointer;
+			scanbitvector(frame->varp - size, bv, afterprologue, sbuf);
 		}
 	}
 
 	// Scan arguments.
 	// Use pointer information if known.
-	args = runtime·funcdata(frame->fn, FUNCDATA_GCArgs);
-	if(args != nil && args->n > 0)
-		scanbitvector(frame->argp, args, false, sbuf);
-	else {
+	stackmap = runtime·funcdata(f, FUNCDATA_ArgsPointerMaps);
+	if(stackmap != nil) {
+		bv = stackmapdata(stackmap, pcdata);
+		scanbitvector(frame->argp, bv, false, sbuf);
+	} else {
 		*sbuf->obj.pos++ = (Obj){frame->argp, frame->arglen, 0};
 		if(sbuf->obj.pos == sbuf->obj.end)
 			flushobjbuf(sbuf);
@@ -1549,6 +1544,60 @@ scanstack(G* gp, void *scanbuf)
 	runtime·gentraceback(pc, sp, lr, gp, 0, nil, 0x7fffffff, scanframe, scanbuf, false);
 }
 
+static void
+addstackroots(G *gp)
+{
+	M *mp;
+	int32 n;
+	Stktop *stk;
+	uintptr sp, guard;
+	void *base;
+	uintptr size;
+
+	if(gp == g)
+		runtime·throw("can't scan our own stack");
+	if((mp = gp->m) != nil && mp->helpgc)
+		runtime·throw("can't scan gchelper stack");
+	if(gp->syscallstack != (uintptr)nil) {
+		// Scanning another goroutine that is about to enter or might
+		// have just exited a system call. It may be executing code such
+		// as schedlock and may have needed to start a new stack segment.
+		// Use the stack segment and stack pointer at the time of
+		// the system call instead, since that won't change underfoot.
+		sp = gp->syscallsp;
+		stk = (Stktop*)gp->syscallstack;
+		guard = gp->syscallguard;
+	} else {
+		// Scanning another goroutine's stack.
+		// The goroutine is usually asleep (the world is stopped).
+		sp = gp->sched.sp;
+		stk = (Stktop*)gp->stackbase;
+		guard = gp->stackguard;
+		// For function about to start, context argument is a root too.
+		if(gp->sched.ctxt != 0 && runtime·mlookup(gp->sched.ctxt, &base, &size, nil))
+			addroot((Obj){base, size, 0});
+	}
+	if(ScanStackByFrames) {
+		USED(sp);
+		USED(stk);
+		USED(guard);
+		addroot((Obj){(byte*)gp, PtrSize, (uintptr)gptrProg});
+	} else {
+		n = 0;
+		while(stk) {
+			if(sp < guard-StackGuard || (uintptr)stk < sp) {
+				runtime·printf("scanstack inconsistent: g%D#%d sp=%p not in [%p,%p]\n", gp->goid, n, sp, guard-StackGuard, stk);
+				runtime·throw("scanstack");
+			}
+			addroot((Obj){(byte*)sp, (uintptr)stk - sp, (uintptr)defaultProg | PRECISE | LOOP});
+			sp = stk->gobuf.sp;
+			guard = stk->stackguard;
+			stk = (Stktop*)stk->stackbase;
+			n++;
+		}
+	}
+}
+
 static void
 addfinroots(void *v)
 {