runtime: new profile buffer implementation supporting label pointers

The existing CPU profiling buffer is a slice of uintptr, but we want to
start including profiling label data in the profiles, and those labels need
to be pointers in order to let them describe rich information.

This CL implements a new profBuf type that holds both a slice of uint64
for data and a slice of unsafe.Pointer for profiling labels (aka tags).
Making the runtime use these buffers will happen in followup CLs.

Change-Id: I9ff16b532d8edaf4ce0cbba1098229a561834efc
Reviewed-on: https://go-review.googlesource.com/36713
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

src/runtime/export_test.go

@@ -248,3 +248,26 @@ func CountPagesInUse() (pagesInUse, counted uintptr) {
 
 func Fastrand() uint32          { return fastrand() }
 func Fastrandn(n uint32) uint32 { return fastrandn(n) }
+
+type ProfBuf profBuf
+
+func NewProfBuf(hdrsize, bufwords, tags int) *ProfBuf {
+	return (*ProfBuf)(newProfBuf(hdrsize, bufwords, tags))
+}
+
+func (p *ProfBuf) Write(tag *unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) {
+	(*profBuf)(p).write(tag, now, hdr, stk)
+}
+
+const (
+	ProfBufBlocking    = profBufBlocking
+	ProfBufNonBlocking = profBufNonBlocking
+)
+
+func (p *ProfBuf) Read(mode profBufReadMode) ([]uint64, []unsafe.Pointer, bool) {
+	return (*profBuf)(p).read(profBufReadMode(mode))
+}
+
+func (p *ProfBuf) Close() {
+	(*profBuf)(p).close()
+}

src/runtime/mbarrier.go

@@ -149,6 +149,11 @@ func gcmarkwb_m(slot *uintptr, ptr uintptr) {
 		// combine the read and the write. Checking inheap is
 		// insufficient since we need to track changes to
 		// roots outside the heap.
+		//
+		// Note: profbuf.go omits a barrier during signal handler
+		// profile logging; that's safe only because this deletion barrier exists.
+		// If we remove the deletion barrier, we'll have to work out
+		// a new way to handle the profile logging.
 		if slot1 := uintptr(unsafe.Pointer(slot)); slot1 >= minPhysPageSize {
 			if optr := *slot; optr != 0 {
 				shade(optr)

src/runtime/profbuf_test.go

+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime_test
+
+import (
+	"reflect"
+	. "runtime"
+	"testing"
+	"time"
+	"unsafe"
+)
+
+func TestProfBuf(t *testing.T) {
+	const hdrSize = 2
+
+	write := func(t *testing.T, b *ProfBuf, tag unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) {
+		b.Write(&tag, now, hdr, stk)
+	}
+	read := func(t *testing.T, b *ProfBuf, data []uint64, tags []unsafe.Pointer) {
+		rdata, rtags, eof := b.Read(ProfBufNonBlocking)
+		if !reflect.DeepEqual(rdata, data) || !reflect.DeepEqual(rtags, tags) {
+			t.Fatalf("unexpected profile read:\nhave data %#x\nwant data %#x\nhave tags %#x\nwant tags %#x", rdata, data, rtags, tags)
+		}
+		if eof {
+			t.Fatalf("unexpected eof")
+		}
+	}
+	readBlock := func(t *testing.T, b *ProfBuf, data []uint64, tags []unsafe.Pointer) func() {
+		c := make(chan int)
+		go func() {
+			eof := data == nil
+			rdata, rtags, reof := b.Read(ProfBufBlocking)
+			if !reflect.DeepEqual(rdata, data) || !reflect.DeepEqual(rtags, tags) || reof != eof {
+				// Errorf, not Fatalf, because called in goroutine.
+				t.Errorf("unexpected profile read:\nhave data %#x\nwant data %#x\nhave tags %#x\nwant tags %#x\nhave eof=%v, want %v", rdata, data, rtags, tags, reof, eof)
+			}
+			c <- 1
+		}()
+		time.Sleep(10 * time.Millisecond) // let goroutine run and block
+		return func() {
+			select {
+			case <-c:
+			case <-time.After(1 * time.Second):
+				t.Fatalf("timeout waiting for blocked read")
+			}
+		}
+	}
+	readEOF := func(t *testing.T, b *ProfBuf) {
+		rdata, rtags, eof := b.Read(ProfBufBlocking)
+		if rdata != nil || rtags != nil || !eof {
+			t.Errorf("unexpected profile read: %#x, %#x, eof=%v; want nil, nil, eof=true", rdata, rtags, eof)
+		}
+		rdata, rtags, eof = b.Read(ProfBufNonBlocking)
+		if rdata != nil || rtags != nil || !eof {
+			t.Errorf("unexpected profile read (non-blocking): %#x, %#x, eof=%v; want nil, nil, eof=true", rdata, rtags, eof)
+		}
+	}
+
+	myTags := make([]byte, 100)
+	t.Logf("myTags is %p", &myTags[0])
+
+	t.Run("BasicWriteRead", func(t *testing.T) {
+		b := NewProfBuf(2, 11, 1)
+		write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+		read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])})
+		read(t, b, nil, nil) // release data returned by previous read
+		write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204})
+		read(t, b, []uint64{8, 99, 101, 102, 201, 202, 203, 204}, []unsafe.Pointer{unsafe.Pointer(&myTags[2])})
+	})
+
+	t.Run("ReadMany", func(t *testing.T) {
+		b := NewProfBuf(2, 50, 50)
+		write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+		write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204})
+		write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 504}, []uintptr{506})
+		read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 99, 101, 102, 201, 202, 203, 204, 5, 500, 502, 504, 506}, []unsafe.Pointer{unsafe.Pointer(&myTags[0]), unsafe.Pointer(&myTags[2]), unsafe.Pointer(&myTags[1])})
+	})
+
+	t.Run("ReadManyShortData", func(t *testing.T) {
+		b := NewProfBuf(2, 50, 50)
+		write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+		write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204})
+		read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 99, 101, 102, 201, 202, 203, 204}, []unsafe.Pointer{unsafe.Pointer(&myTags[0]), unsafe.Pointer(&myTags[2])})
+	})
+
+	t.Run("ReadManyShortTags", func(t *testing.T) {
+		b := NewProfBuf(2, 50, 50)
+		write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+		write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204})
+		read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 99, 101, 102, 201, 202, 203, 204}, []unsafe.Pointer{unsafe.Pointer(&myTags[0]), unsafe.Pointer(&myTags[2])})
+	})
+
+	t.Run("ReadAfterOverflow1", func(t *testing.T) {
+		// overflow record synthesized by write
+		b := NewProfBuf(2, 16, 5)
+		write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})           // uses 10
+		read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) // reads 10 but still in use until next read
+		write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5})                       // uses 6
+		read(t, b, []uint64{6, 1, 2, 3, 4, 5}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])})              // reads 6 but still in use until next read
+		// now 10 available
+		write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204, 205, 206, 207, 208, 209}) // no room
+		for i := 0; i < 299; i++ {
+			write(t, b, unsafe.Pointer(&myTags[3]), int64(100+i), []uint64{101, 102}, []uintptr{201, 202, 203, 204}) // no room for overflow+this record
+		}
+		write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 504}, []uintptr{506}) // room for overflow+this record
+		read(t, b, []uint64{5, 99, 0, 0, 300, 5, 500, 502, 504, 506}, []unsafe.Pointer{nil, unsafe.Pointer(&myTags[1])})
+	})
+
+	t.Run("ReadAfterOverflow2", func(t *testing.T) {
+		// overflow record synthesized by read
+		b := NewProfBuf(2, 16, 5)
+		write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+		write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213})
+		for i := 0; i < 299; i++ {
+			write(t, b, unsafe.Pointer(&myTags[3]), 100, []uint64{101, 102}, []uintptr{201, 202, 203, 204})
+		}
+		read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])}) // reads 10 but still in use until next read
+		write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 504}, []uintptr{})                     // still overflow
+		read(t, b, []uint64{5, 99, 0, 0, 301}, []unsafe.Pointer{nil})                                     // overflow synthesized by read
+		write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 505}, []uintptr{506})                  // written
+		read(t, b, []uint64{5, 500, 502, 505, 506}, []unsafe.Pointer{unsafe.Pointer(&myTags[1])})
+	})
+
+	t.Run("ReadAtEndAfterOverflow", func(t *testing.T) {
+		b := NewProfBuf(2, 12, 5)
+		write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+		write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204})
+		for i := 0; i < 299; i++ {
+			write(t, b, unsafe.Pointer(&myTags[3]), 100, []uint64{101, 102}, []uintptr{201, 202, 203, 204})
+		}
+		read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])})
+		read(t, b, []uint64{5, 99, 0, 0, 300}, []unsafe.Pointer{nil})
+		write(t, b, unsafe.Pointer(&myTags[1]), 500, []uint64{502, 504}, []uintptr{506})
+		read(t, b, []uint64{5, 500, 502, 504, 506}, []unsafe.Pointer{unsafe.Pointer(&myTags[1])})
+	})
+
+	t.Run("BlockingWriteRead", func(t *testing.T) {
+		b := NewProfBuf(2, 11, 1)
+		wait := readBlock(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])})
+		write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+		wait()
+		wait = readBlock(t, b, []uint64{8, 99, 101, 102, 201, 202, 203, 204}, []unsafe.Pointer{unsafe.Pointer(&myTags[2])})
+		time.Sleep(10 * time.Millisecond)
+		write(t, b, unsafe.Pointer(&myTags[2]), 99, []uint64{101, 102}, []uintptr{201, 202, 203, 204})
+		wait()
+		wait = readBlock(t, b, nil, nil)
+		b.Close()
+		wait()
+		wait = readBlock(t, b, nil, nil)
+		wait()
+		readEOF(t, b)
+	})
+
+	t.Run("DataWraparound", func(t *testing.T) {
+		b := NewProfBuf(2, 16, 1024)
+		for i := 0; i < 10; i++ {
+			write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+			read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])})
+			read(t, b, nil, nil) // release data returned by previous read
+		}
+	})
+
+	t.Run("TagWraparound", func(t *testing.T) {
+		b := NewProfBuf(2, 1024, 2)
+		for i := 0; i < 10; i++ {
+			write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+			read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])})
+			read(t, b, nil, nil) // release data returned by previous read
+		}
+	})
+
+	t.Run("BothWraparound", func(t *testing.T) {
+		b := NewProfBuf(2, 16, 2)
+		for i := 0; i < 10; i++ {
+			write(t, b, unsafe.Pointer(&myTags[0]), 1, []uint64{2, 3}, []uintptr{4, 5, 6, 7, 8, 9})
+			read(t, b, []uint64{10, 1, 2, 3, 4, 5, 6, 7, 8, 9}, []unsafe.Pointer{unsafe.Pointer(&myTags[0])})
+			read(t, b, nil, nil) // release data returned by previous read
+		}
+	})
+}

src/runtime/runtime2.go

@@ -373,10 +373,9 @@ type g struct {
 	gopc           uintptr // pc of go statement that created this goroutine
 	startpc        uintptr // pc of goroutine function
 	racectx        uintptr
-	waiting        *sudog    // sudog structures this g is waiting on (that have a valid elem ptr); in lock order
-	cgoCtxt        []uintptr // cgo traceback context
-
-	labels unsafe.Pointer // profiler labels
+	waiting        *sudog         // sudog structures this g is waiting on (that have a valid elem ptr); in lock order
+	cgoCtxt        []uintptr      // cgo traceback context
+	labels         unsafe.Pointer // profiler labels
 
 	// Per-G GC state