runtime: convert slice operations to Go.

LGTM=bradfitz, dvyukov
R=golang-codereviews, bradfitz, dvyukov
CC=golang-codereviews
https://golang.org/cl/120190044

src/cmd/api/goapi.go

@@ -378,7 +378,7 @@ func (w *Walker) parseFile(dir, file string) (*ast.File, error) {
 	}
 	if w.context != nil && file == fmt.Sprintf("zruntime_defs_%s_%s.go", w.context.GOOS, w.context.GOARCH) {
 		// Just enough to keep the api checker happy.
-		src := "package runtime; type maptype struct{}; type _type struct{}; type alg struct{}; type mspan struct{}; type m struct{}; type lock struct{}"
+		src := "package runtime; type maptype struct{}; type _type struct{}; type alg struct{}; type mspan struct{}; type m struct{}; type lock struct{}; type slicetype struct{};"
 		f, err = parser.ParseFile(fset, filename, src, 0)
 		if err != nil {
 			log.Fatalf("incorrect generated file: %s", err)

src/cmd/gc/builtin.c

@@ -39,7 +39,7 @@ char *runtimeimport =
 	"func @\"\".stringtoslicerune (? string) (? []rune)\n"
 	"func @\"\".stringiter (? string, ? int) (? int)\n"
 	"func @\"\".stringiter2 (? string, ? int) (@\"\".retk·1 int, @\"\".retv·2 rune)\n"
-	"func @\"\".copy (@\"\".to·2 any, @\"\".fr·3 any, @\"\".wid·4 uintptr) (? int)\n"
+	"func @\"\".slicecopy (@\"\".to·2 any, @\"\".fr·3 any, @\"\".wid·4 uintptr) (? int)\n"
 	"func @\"\".slicestringcopy (@\"\".to·2 any, @\"\".fr·3 any) (? int)\n"
 	"func @\"\".typ2Itab (@\"\".typ·2 *byte, @\"\".typ2·3 *byte, @\"\".cache·4 **byte) (@\"\".ret·1 *byte)\n"
 	"func @\"\".convI2E (@\"\".elem·2 any) (@\"\".ret·1 any)\n"

src/cmd/gc/runtime.go

@@ -53,7 +53,7 @@ func stringtoslicebyte(string) []byte
 func stringtoslicerune(string) []rune
 func stringiter(string, int) int
 func stringiter2(string, int) (retk int, retv rune)
-func copy(to any, fr any, wid uintptr) int
+func slicecopy(to any, fr any, wid uintptr) int
 func slicestringcopy(to any, fr any) int
 
 // interface conversions

src/cmd/gc/walk.c

@@ -2623,7 +2623,7 @@ appendslice(Node *n, NodeList **init)
 		if(l2->type->etype == TSTRING)
 			fn = syslook("slicestringcopy", 1);
 		else
-			fn = syslook("copy", 1);
+			fn = syslook("slicecopy", 1);
 		argtype(fn, l1->type);
 		argtype(fn, l2->type);
 		nt = mkcall1(fn, types[TINT], &l,
@@ -2761,7 +2761,7 @@ copyany(Node *n, NodeList **init, int runtimecall)
 		if(n->right->type->etype == TSTRING)
 			fn = syslook("slicestringcopy", 1);
 		else
-			fn = syslook("copy", 1);
+			fn = syslook("slicecopy", 1);
 		argtype(fn, n->left->type);
 		argtype(fn, n->right->type);
 		return mkcall1(fn, n->type, init,

src/pkg/runtime/alg.goc

@@ -286,7 +286,7 @@ runtime·c128hash(uintptr *h, uintptr s, void *a)
 }
 
 void
-runtime·slicecopy(uintptr s, void *a, void *b)
+runtime·algslicecopy(uintptr s, void *a, void *b)
 {
 	USED(s);
 	if(b == nil) {
@@ -445,7 +445,7 @@ runtime·algarray[] =
 [ASTRING]	{ runtime·strhash, runtime·strequal, runtime·strprint, runtime·strcopy },
 [AINTER]	{ runtime·interhash, runtime·interequal, runtime·interprint, runtime·intercopy },
 [ANILINTER]	{ runtime·nilinterhash, runtime·nilinterequal, runtime·nilinterprint, runtime·nilintercopy },
-[ASLICE]	{ runtime·nohash, runtime·noequal, runtime·memprint, runtime·slicecopy },
+[ASLICE]	{ runtime·nohash, runtime·noequal, runtime·memprint, runtime·algslicecopy },
 [AFLOAT32]	{ runtime·f32hash, runtime·f32equal, runtime·memprint, runtime·memcopy },
 [AFLOAT64]	{ runtime·f64hash, runtime·f64equal, runtime·memprint, runtime·memcopy },
 [ACPLX64]	{ runtime·c64hash, runtime·c64equal, runtime·memprint, runtime·memcopy },

src/pkg/runtime/malloc.go

@@ -232,6 +232,12 @@ func newarray(typ *_type, n uintptr) unsafe.Pointer {
 	return gomallocgc(uintptr(typ.size)*n, typ, flags)
 }
 
+// rawmem returns a chunk of pointerless memory.  It is
+// not zeroed.
+func rawmem(size uintptr) unsafe.Pointer {
+	return gomallocgc(size, nil, flagNoScan|flagNoZero)
+}
+
 // round size up to next size class
 func goroundupsize(size uintptr) uintptr {
 	if size < maxSmallSize {

src/pkg/runtime/slice.c

+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "runtime.h"
+#include "arch_GOARCH.h"
+
+void
+runtime·printslice_m(G *gp)
+{
+	void *array;
+	uintptr len, cap;
+
+	array = g->m->ptrarg[0];
+	g->m->ptrarg[0] = nil;
+	len = g->m->scalararg[0];
+	cap = g->m->scalararg[1];
+
+	runtime·prints("[");
+	runtime·printint(len);
+	runtime·prints("/");
+	runtime·printint(cap);
+	runtime·prints("]");
+	runtime·printpointer(array);
+
+	runtime·gogo(&gp->sched);
+}

src/pkg/runtime/slice.go

+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+import (
+	"unsafe"
+)
+
+type sliceStruct struct {
+	array unsafe.Pointer
+	len   int
+	cap   int
+}
+
+// TODO: take uintptrs instead of int64s?
+func makeslice(t *slicetype, len64 int64, cap64 int64) sliceStruct {
+	// NOTE: The len > MaxMem/elemsize check here is not strictly necessary,
+	// but it produces a 'len out of range' error instead of a 'cap out of range' error
+	// when someone does make([]T, bignumber). 'cap out of range' is true too,
+	// but since the cap is only being supplied implicitly, saying len is clearer.
+	// See issue 4085.
+	len := int(len64)
+	if len64 < 0 || int64(len) != len64 || t.elem.size > 0 && len > int(maxMem/uintptr(t.elem.size)) {
+		panic(errorString("makeslice: len out of range"))
+	}
+	cap := int(cap64)
+	if cap < len || int64(cap) != cap64 || t.elem.size > 0 && cap > int(maxMem/uintptr(t.elem.size)) {
+		panic(errorString("makeslice: cap out of range"))
+	}
+	p := newarray(t.elem, uintptr(cap))
+	return sliceStruct{p, len, cap}
+}
+
+// TODO: take uintptr instead of int64?
+func growslice(t *slicetype, old sliceStruct, n int64) sliceStruct {
+	if n < 1 {
+		panic(errorString("growslice: invalid n"))
+	}
+
+	cap64 := int64(old.cap) + n
+	cap := int(cap64)
+
+	if int64(cap) != cap64 || cap < old.cap || t.elem.size > 0 && cap > int(maxMem/uintptr(t.elem.size)) {
+		panic(errorString("growslice: cap out of range"))
+	}
+
+	if raceenabled {
+		callerpc := gogetcallerpc(unsafe.Pointer(&t))
+		fn := growslice
+		pc := **(**uintptr)(unsafe.Pointer(&fn))
+		racereadrangepc(old.array, old.len*int(t.elem.size), callerpc, pc)
+	}
+
+	et := t.elem
+	if et.size == 0 {
+		return sliceStruct{old.array, old.len, cap}
+	}
+
+	newcap := old.cap
+	if newcap+newcap < cap {
+		newcap = cap
+	} else {
+		for {
+			if old.len < 1024 {
+				newcap += newcap
+			} else {
+				newcap += newcap / 4
+			}
+			if newcap >= cap {
+				break
+			}
+		}
+	}
+
+	if newcap >= int(maxMem/uintptr(et.size)) {
+		panic(errorString("growslice: cap out of range"))
+	}
+	lenmem := uintptr(old.len) * uintptr(et.size)
+	capmem := goroundupsize(uintptr(newcap) * uintptr(et.size))
+	newcap = int(capmem / uintptr(et.size))
+	var p unsafe.Pointer
+	if et.kind&kindNoPointers != 0 {
+		p = rawmem(capmem)
+		memclr(add(p, lenmem), capmem-lenmem)
+	} else {
+		// Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan unitialized memory
+		p = newarray(et, uintptr(newcap))
+	}
+	memmove(p, old.array, lenmem)
+
+	return sliceStruct{p, old.len, newcap}
+}
+
+func slicecopy(to sliceStruct, fm sliceStruct, width uintptr) int {
+	if fm.len == 0 || to.len == 0 || width == 0 {
+		return 0
+	}
+
+	n := fm.len
+	if to.len < n {
+		n = to.len
+	}
+
+	if raceenabled {
+		callerpc := gogetcallerpc(unsafe.Pointer(&to))
+		fn := slicecopy
+		pc := **(**uintptr)(unsafe.Pointer(&fn))
+		racewriterangepc(to.array, n*int(width), callerpc, pc)
+		racereadrangepc(fm.array, n*int(width), callerpc, pc)
+	}
+
+	size := uintptr(n) * width
+	if size == 1 { // common case worth about 2x to do here
+		// TODO: is this still worth it with new memmove impl?
+		*(*byte)(to.array) = *(*byte)(fm.array) // known to be a byte pointer
+	} else {
+		memmove(to.array, fm.array, size)
+	}
+	return int(n)
+}
+
+func slicestringcopy(to []byte, fm string) int {
+	if len(fm) == 0 || len(to) == 0 {
+		return 0
+	}
+
+	n := len(fm)
+	if len(to) < n {
+		n = len(to)
+	}
+
+	if raceenabled {
+		callerpc := gogetcallerpc(unsafe.Pointer(&to))
+		fn := slicestringcopy
+		pc := **(**uintptr)(unsafe.Pointer(&fn))
+		racewriterangepc(unsafe.Pointer(&to[0]), n, callerpc, pc)
+	}
+
+	memmove(unsafe.Pointer(&to[0]), unsafe.Pointer((*stringStruct)(unsafe.Pointer(&fm)).str), uintptr(n))
+	return n
+}
+
+var printslice_m byte
+
+func printslice(a sliceStruct) {
+	mp := acquirem()
+	mp.ptrarg[0] = a.array
+	mp.scalararg[0] = uint(a.len)
+	mp.scalararg[1] = uint(a.cap)
+	mcall(&printslice_m)
+	releasem(mp)
+}

src/pkg/runtime/slice.goc

-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package runtime
-#include "runtime.h"
-#include "arch_GOARCH.h"
-#include "type.h"
-#include "typekind.h"
-#include "malloc.h"
-#include "race.h"
-#include "stack.h"
-#include "../../cmd/ld/textflag.h"
-
-enum
-{
-	debug = 0
-};
-
-static	void	makeslice1(SliceType*, intgo, intgo, Slice*);
-static	void	growslice1(SliceType*, Slice, intgo, Slice *);
-
-// see also unsafe·NewArray
-func makeslice(t *SliceType, len int64, cap int64) (ret Slice) {
-	// NOTE: The len > MaxMem/elemsize check here is not strictly necessary,
-	// but it produces a 'len out of range' error instead of a 'cap out of range' error
-	// when someone does make([]T, bignumber). 'cap out of range' is true too,
-	// but since the cap is only being supplied implicitly, saying len is clearer.
-	// See issue 4085.
-	if(len < 0 || (intgo)len != len || t->elem->size > 0 && len > MaxMem / t->elem->size)
-		runtime·panicstring("makeslice: len out of range");
-
-	if(cap < len || (intgo)cap != cap || t->elem->size > 0 && cap > MaxMem / t->elem->size)
-		runtime·panicstring("makeslice: cap out of range");
-
-	makeslice1(t, len, cap, &ret);
-
-	if(debug) {
-		runtime·printf("makeslice(%S, %D, %D); ret=",
-			*t->string, len, cap);
-		runtime·printslice(ret);
-	}
-}
-
-// Dummy word to use as base pointer for make([]T, 0).
-// Since you cannot take the address of such a slice,
-// you can't tell that they all have the same base pointer.
-uintptr runtime·zerobase;
-
-static void
-makeslice1(SliceType *t, intgo len, intgo cap, Slice *ret)
-{
-	ret->len = len;
-	ret->cap = cap;
-	ret->array = runtime·cnewarray(t->elem, cap);
-}
-
-// growslice(type *Type, x, []T, n int64) []T
-func growslice(t *SliceType, old Slice, n int64) (ret Slice) {
-	int64 cap;
-	void *pc;
-
-	if(n < 1)
-		runtime·panicstring("growslice: invalid n");
-
-	cap = old.cap + n;
-
-	if((intgo)cap != cap || cap < (int64)old.cap || (t->elem->size > 0 && cap > MaxMem/t->elem->size))
-		runtime·panicstring("growslice: cap out of range");
-
-	if(raceenabled) {
-		pc = runtime·getcallerpc(&t);
-		runtime·racereadrangepc(old.array, old.len*t->elem->size, pc, runtime·growslice);
-	}
-
-	growslice1(t, old, cap, &ret);
-
-	if(debug) {
-		runtime·printf("growslice(%S,", *t->string);
-		runtime·printslice(old);
-		runtime·printf(", new cap=%D) =", cap);
-		runtime·printslice(ret);
-	}
-}
-
-static void
-growslice1(SliceType *t, Slice x, intgo newcap, Slice *ret)
-{
-	intgo newcap1;
-	uintptr capmem, lenmem;
-	int32 flag;
-	Type *typ;
-
-	typ = t->elem;
-	if(typ->size == 0) {
-		*ret = x;
-		ret->cap = newcap;
-		return;
-	}
-
-	newcap1 = x.cap;
-	
-	// Using newcap directly for m+m < newcap handles
-	// both the case where m == 0 and also the case where
-	// m+m/4 wraps around, in which case the loop
-	// below might never terminate.
-	if(newcap1+newcap1 < newcap)
-		newcap1 = newcap;
-	else {
-		do {
-			if(x.len < 1024)
-				newcap1 += newcap1;
-			else
-				newcap1 += newcap1/4;
-		} while(newcap1 < newcap);
-	}
-
-	if(newcap1 > MaxMem/typ->size)
-		runtime·panicstring("growslice: cap out of range");
-	// Try to use all memory that malloc will give us...
-	capmem = runtime·roundupsize(newcap1*typ->size);
-	// ...but don't ask for fractional number of elements (that can confuse GC).
-	newcap1 = capmem/typ->size;
-	capmem = newcap1*typ->size;
-	flag = 0;
-	// Can't use FlagNoZero w/o FlagNoScan, because otherwise GC can scan unitialized memory.
-	if(typ->kind&KindNoPointers)
-		flag = FlagNoScan|FlagNoZero;
-	ret->array = runtime·mallocgc(capmem, typ, flag);
-	ret->len = x.len;
-	ret->cap = newcap1;
-	lenmem = x.len*typ->size;
-	runtime·memmove(ret->array, x.array, lenmem);
-	if(typ->kind&KindNoPointers)
-		runtime·memclr(ret->array+lenmem, capmem-lenmem);
-}
-
-#pragma textflag NOSPLIT
-func copy(to Slice, fm Slice, width uintptr) (ret int) {
-	void *pc;
-
-	if(fm.len == 0 || to.len == 0 || width == 0) {
-		ret = 0;
-		goto out;
-	}
-
-	ret = fm.len;
-	if(to.len < ret)
-		ret = to.len;
-
-	if(raceenabled) {
-		pc = runtime·getcallerpc(&to);
-		runtime·racewriterangepc(to.array, ret*width, pc, runtime·copy);
-		runtime·racereadrangepc(fm.array, ret*width, pc, runtime·copy);
-	}
-
-	if(ret == 1 && width == 1) {	// common case worth about 2x to do here
-		*to.array = *fm.array;	// known to be a byte pointer
-	} else {
-		runtime·memmove(to.array, fm.array, ret*width);
-	}
-
-out:
-
-	if(debug) {
-		runtime·prints("main·copy: to=");
-		runtime·printslice(to);
-		runtime·prints("; fm=");
-		runtime·printslice(fm);
-		runtime·prints("; width=");
-		runtime·printint(width);
-		runtime·prints("; ret=");
-		runtime·printint(ret);
-		runtime·prints("\n");
-	}
-}
-
-#pragma textflag NOSPLIT
-func slicestringcopy(to Slice, fm String) (ret int) {
-	void *pc;
-
-	if(fm.len == 0 || to.len == 0) {
-		ret = 0;
-		goto out;
-	}
-
-	ret = fm.len;
-	if(to.len < ret)
-		ret = to.len;
-
-	if(raceenabled) {
-		pc = runtime·getcallerpc(&to);
-		runtime·racewriterangepc(to.array, ret, pc, runtime·slicestringcopy);
-	}
-
-	runtime·memmove(to.array, fm.str, ret);
-
-out:;
-}
-
-func printslice(a Slice) {
-	runtime·prints("[");
-	runtime·printint(a.len);
-	runtime·prints("/");
-	runtime·printint(a.cap);
-	runtime·prints("]");
-	runtime·printpointer(a.array);
-}

src/pkg/runtime/stubs.go

@@ -27,6 +27,9 @@ func racewritepc(addr unsafe.Pointer, callpc, pc uintptr)
 //go:noescape
 func racereadrangepc(addr unsafe.Pointer, len int, callpc, pc uintptr)
 
+//go:noescape
+func racewriterangepc(addr unsafe.Pointer, len int, callpc, pc uintptr)
+
 // Should be a built-in for unsafe.Pointer?
 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
 	return unsafe.Pointer(uintptr(p) + x)