cmd/gc: add 2-, 3-, 4-word write barrier specializations

Assignments of 2-, 3-, and 4-word values were handled
by individual MOV instructions (and for scalars still are).
But if there are pointers involved, those assignments now
go through the write barrier routine. Before this CL, they
went to writebarrierfat, which calls memmove.
Memmove is too much overhead for these small
amounts of data.

Instead, call writebarrierfat{2,3,4}, which are specialized
for the specific amount of data being copied.
Today the write barrier does not care which words are
pointers, so size alone is enough to distinguish the cases.
If we keep these distinctions in Go 1.5 we will need to
expand them for all the pointer-vs-scalar possibilities,
so the current 3 functions will become 3+7+15 = 25,
still not a large burden (we deleted more morestack
functions than that when we dropped segmented stacks).

BenchmarkBinaryTree17           3250972583  3123910344  -3.91%
BenchmarkFannkuch11             3067605223  2964737839  -3.35%
BenchmarkFmtFprintfEmpty        101         96.0        -4.95%
BenchmarkFmtFprintfString       267         235         -11.99%
BenchmarkFmtFprintfInt          261         253         -3.07%
BenchmarkFmtFprintfIntInt       444         402         -9.46%
BenchmarkFmtFprintfPrefixedInt  374         346         -7.49%
BenchmarkFmtFprintfFloat        472         449         -4.87%
BenchmarkFmtManyArgs            1537        1476        -3.97%
BenchmarkGobDecode              13986528    12432985    -11.11%
BenchmarkGobEncode              13120323    12537420    -4.44%
BenchmarkGzip                   451925758   437500578   -3.19%
BenchmarkGunzip                 113267612   110053644   -2.84%
BenchmarkHTTPClientServer       103151      77100       -25.26%
BenchmarkJSONEncode             25002733    23435278    -6.27%
BenchmarkJSONDecode             94213717    82568789    -12.36%
BenchmarkMandelbrot200          4804246     4713070     -1.90%
BenchmarkGoParse                4646114     4379456     -5.74%
BenchmarkRegexpMatchEasy0_32    163         158         -3.07%
BenchmarkRegexpMatchEasy0_1K    433         391         -9.70%
BenchmarkRegexpMatchEasy1_32    154         138         -10.39%
BenchmarkRegexpMatchEasy1_1K    1481        1132        -23.57%
BenchmarkRegexpMatchMedium_32   282         270         -4.26%
BenchmarkRegexpMatchMedium_1K   92421       86149       -6.79%
BenchmarkRegexpMatchHard_32     5209        4718        -9.43%
BenchmarkRegexpMatchHard_1K     158141      147921      -6.46%
BenchmarkRevcomp                699818791   642222464   -8.23%
BenchmarkTemplate               132402383   108269713   -18.23%
BenchmarkTimeParse              509         478         -6.09%
BenchmarkTimeFormat             462         456         -1.30%

LGTM=r
R=r
CC=golang-codereviews
https://golang.org/cl/156200043

src/cmd/gc/builtin.c

@@ -84,9 +84,12 @@ char *runtimeimport =
 	"func @\"\".chansend1 (@\"\".chanType·1 *byte, @\"\".hchan·2 chan<- any, @\"\".elem·3 *any)\n"
 	"func @\"\".closechan (@\"\".hchan·1 any)\n"
 	"func @\"\".writebarrierptr (@\"\".dst·1 *any, @\"\".src·2 any)\n"
-	"func @\"\".writebarrieriface (@\"\".dst·1 *any, @\"\".src·2 any)\n"
 	"func @\"\".writebarrierstring (@\"\".dst·1 *any, @\"\".src·2 any)\n"
 	"func @\"\".writebarrierslice (@\"\".dst·1 *any, @\"\".src·2 any)\n"
+	"func @\"\".writebarrieriface (@\"\".dst·1 *any, @\"\".src·2 any)\n"
+	"func @\"\".writebarrierfat2 (@\"\".dst·1 *any, @\"\".src·2 any)\n"
+	"func @\"\".writebarrierfat3 (@\"\".dst·1 *any, @\"\".src·2 any)\n"
+	"func @\"\".writebarrierfat4 (@\"\".dst·1 *any, @\"\".src·2 any)\n"
 	"func @\"\".writebarrierfat (@\"\".typ·1 *byte, @\"\".dst·2 *any, @\"\".src·3 *any)\n"
 	"func @\"\".selectnbsend (@\"\".chanType·2 *byte, @\"\".hchan·3 chan<- any, @\"\".elem·4 *any) (? bool)\n"
 	"func @\"\".selectnbrecv (@\"\".chanType·2 *byte, @\"\".elem·3 *any, @\"\".hchan·4 <-chan any) (? bool)\n"

src/cmd/gc/runtime.go

@@ -112,6 +112,9 @@ func writebarrierptr(dst *any, src any)
 func writebarrierstring(dst *any, src any)
 func writebarrierslice(dst *any, src any)
 func writebarrieriface(dst *any, src any)
+func writebarrierfat2(dst *any, src any)
+func writebarrierfat3(dst *any, src any)
+func writebarrierfat4(dst *any, src any)
 func writebarrierfat(typ *byte, dst *any, src *any)
 
 func selectnbsend(chanType *byte, hchan chan<- any, elem *any) bool

src/cmd/gc/walk.c

@@ -2040,21 +2040,32 @@ static Node*
 applywritebarrier(Node *n, NodeList **init)
 {
 	Node *l, *r;
+	Type *t;
 
 	if(n->left && n->right && needwritebarrier(n->left, n->right)) {
+		t = n->left->type;
 		l = nod(OADDR, n->left, N);
 		l->etype = 1; // addr does not escape
-		if(n->left->type->width == widthptr) {
-			n = mkcall1(writebarrierfn("writebarrierptr", n->left->type, n->right->type), T, init,
+		if(t->width == widthptr) {
+			n = mkcall1(writebarrierfn("writebarrierptr", t, n->right->type), T, init,
+				l, n->right);
+		} else if(t->etype == TSTRING) {
+			n = mkcall1(writebarrierfn("writebarrierstring", t, n->right->type), T, init,
+				l, n->right);
+		} else if(isslice(t)) {
+			n = mkcall1(writebarrierfn("writebarrierslice", t, n->right->type), T, init,
+				l, n->right);
+		} else if(isinter(t)) {
+			n = mkcall1(writebarrierfn("writebarrieriface", t, n->right->type), T, init,
 				l, n->right);
-		} else if(n->left->type->etype == TSTRING) {
-			n = mkcall1(writebarrierfn("writebarrierstring", n->left->type, n->right->type), T, init,
+		} else if(t->width == 2*widthptr) {
+			n = mkcall1(writebarrierfn("writebarrierfat2", t, n->right->type), T, init,
 				l, n->right);
-		} else if(isslice(n->left->type)) {
-			n = mkcall1(writebarrierfn("writebarrierslice", n->left->type, n->right->type), T, init,
+		} else if(t->width == 3*widthptr) {
+			n = mkcall1(writebarrierfn("writebarrierfat3", t, n->right->type), T, init,
 				l, n->right);
-		} else if(isinter(n->left->type)) {
-			n = mkcall1(writebarrierfn("writebarrieriface", n->left->type, n->right->type), T, init,
+		} else if(t->width == 4*widthptr) {
+			n = mkcall1(writebarrierfn("writebarrierfat4", t, n->right->type), T, init,
 				l, n->right);
 		} else {
 			r = n->right;
@@ -2062,8 +2073,9 @@ applywritebarrier(Node *n, NodeList **init)
 				r = r->left;
 			r = nod(OADDR, r, N);
 			r->etype = 1; // addr does not escape
-			n = mkcall1(writebarrierfn("writebarrierfat", n->left->type, r->left->type), T, init,
-				typename(n->left->type), l, r);
+			//warnl(n->lineno, "writebarrierfat %T %N", t, r);
+			n = mkcall1(writebarrierfn("writebarrierfat", t, r->left->type), T, init,
+				typename(t), l, r);
 		}
 	}
 	return n;

src/runtime/mgc0.go

@@ -109,6 +109,27 @@ func writebarrieriface(dst *[2]uintptr, src [2]uintptr) {
 	dst[1] = src[1]
 }
 
+//go:nosplit
+func writebarrierfat2(dst *[2]uintptr, src [2]uintptr) {
+	dst[0] = src[0]
+	dst[1] = src[1]
+}
+
+//go:nosplit
+func writebarrierfat3(dst *[3]uintptr, src [3]uintptr) {
+	dst[0] = src[0]
+	dst[1] = src[1]
+	dst[2] = src[2]
+}
+
+//go:nosplit
+func writebarrierfat4(dst *[4]uintptr, src [4]uintptr) {
+	dst[0] = src[0]
+	dst[1] = src[1]
+	dst[2] = src[2]
+	dst[3] = src[3]
+}
+
 //go:nosplit
 func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
 	memmove(dst, src, typ.size)