runtime/pprof/internal: add package protopprof

This change adds code, originally written by Russ Cox <rsc@golang.org>
and open-sourced by Google, that converts from the "legacy"
binary pprof profile format to a struct representation of the
new protocol buffer pprof profile format.

This code reads the entire binary format for conversion to the
protobuf format. In a future change, we will update the code
to incrementally read and convert segments of the binary format,
so that the entire profile does not need to be stored in memory.

This change also contains contributions by Daria Kolistratova
<daria.kolistratova@intel.com> from the rolled-back change
golang.org/cl/30556 adapting the code to be used by the package
runtime/pprof.

This code also appeared in the change golang.org/cl/32257, which was based
on Daria Kolistratova's change, but was also rolled back.

Updates #16093

Change-Id: I5c768b1134bc15408d80a3ccc7ed867db9a1c63d
Reviewed-on: https://go-review.googlesource.com/32811
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>

src/go/build/deps_test.go

@@ -93,7 +93,7 @@ var pkgDeps = map[string][]string{
 	// L3 adds reflection and some basic utility packages
 	// and interface definitions, but nothing that makes
 	// system calls.
-	"crypto":              {"L2", "hash"},          // interfaces
+	"crypto":              {"L2", "hash"}, // interfaces
 	"crypto/cipher":       {"L2", "crypto/subtle"},
 	"crypto/subtle":       {},
 	"encoding/base32":     {"L2"},
@@ -171,13 +171,14 @@ var pkgDeps = map[string][]string{
 	"log": {"L1", "os", "fmt", "time"},
 
 	// Packages used by testing must be low-level (L2+fmt).
-	"regexp":                       {"L2", "regexp/syntax"},
-	"regexp/syntax":                {"L2"},
-	"runtime/debug":                {"L2", "fmt", "io/ioutil", "os", "time"},
-	"runtime/pprof/internal/gzip0": {"L2"},
-	"runtime/pprof":                {"L2", "fmt", "os", "text/tabwriter"},
-	"runtime/trace":                {"L0"},
-	"text/tabwriter":               {"L2"},
+	"regexp":                            {"L2", "regexp/syntax"},
+	"regexp/syntax":                     {"L2"},
+	"runtime/debug":                     {"L2", "fmt", "io/ioutil", "os", "time"},
+	"runtime/pprof/internal/gzip0":      {"L2"},
+	"runtime/pprof/internal/protopprof": {"L2", "fmt", "internal/pprof/profile", "os", "time"},
+	"runtime/pprof":                     {"L2", "fmt", "os", "text/tabwriter"},
+	"runtime/trace":                     {"L0"},
+	"text/tabwriter":                    {"L2"},
 
 	"testing":          {"L2", "context", "flag", "fmt", "internal/race", "os", "runtime/debug", "runtime/pprof", "runtime/trace", "time"},
 	"testing/iotest":   {"L2", "log"},

src/runtime/pprof/internal/protopprof/protopprof.go

+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package protopprof converts the runtime's raw profile logs
+// to Profile structs containing a representation of the pprof
+// protocol buffer profile format.
+package protopprof
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"time"
+	"unsafe"
+
+	"internal/pprof/profile"
+)
+
+// TranslateCPUProfile parses binary CPU profiling stack trace data
+// generated by runtime.CPUProfile() into a profile struct.
+func TranslateCPUProfile(b []byte, startTime time.Time) (*profile.Profile, error) {
+	const wordSize = unsafe.Sizeof(uintptr(0))
+	const minRawProfile = 5 * wordSize // Need a minimum of 5 words.
+	if uintptr(len(b)) < minRawProfile {
+		return nil, fmt.Errorf("truncated profile")
+	}
+	n := int(uintptr(len(b)) / wordSize)
+	data := ((*[1 << 28]uintptr)(unsafe.Pointer(&b[0])))[:n:n]
+	period := data[3]
+	data = data[5:] // skip header
+
+	// profile initialization taken from pprof tool
+	p := &profile.Profile{
+		Period:     int64(period) * 1000,
+		PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
+		SampleType: []*profile.ValueType{
+			{Type: "samples", Unit: "count"},
+			{Type: "cpu", Unit: "nanoseconds"},
+		},
+		TimeNanos:     int64(startTime.UnixNano()),
+		DurationNanos: time.Since(startTime).Nanoseconds(),
+	}
+	// Parse CPU samples from the profile.
+	locs := make(map[uint64]*profile.Location)
+	for len(b) > 0 {
+		if len(data) < 2 || uintptr(len(data)) < 2+data[1] {
+			return nil, fmt.Errorf("truncated profile")
+		}
+		count := data[0]
+		nstk := data[1]
+		fmt.Printf("count:%v nstk: %v\n", count, nstk)
+		if uintptr(len(data)) < 2+nstk {
+			return nil, fmt.Errorf("truncated profile")
+		}
+		stk := data[2 : 2+nstk]
+		data = data[2+nstk:]
+
+		if count == 0 && nstk == 1 && stk[0] == 0 {
+			// end of data marker
+			break
+		}
+
+		sloc := make([]*profile.Location, len(stk))
+		for i, addr := range stk {
+			addr := uint64(addr)
+			// Addresses from stack traces point to the next instruction after
+			// each call.  Adjust by -1 to land somewhere on the actual call
+			// (except for the leaf, which is not a call).
+			if i > 0 {
+				addr--
+			}
+			loc := locs[addr]
+			if loc == nil {
+				loc = &profile.Location{
+					ID:      uint64(len(p.Location) + 1),
+					Address: addr,
+				}
+				locs[addr] = loc
+				p.Location = append(p.Location, loc)
+			}
+			sloc[i] = loc
+		}
+		p.Sample = append(p.Sample, &profile.Sample{
+			Value:    []int64{int64(count), int64(count) * int64(p.Period)},
+			Location: sloc,
+		})
+	}
+
+	if runtime.GOOS == "linux" {
+		if err := addMappings(p); err != nil {
+			return nil, err
+		}
+	}
+	return p, nil
+}
+
+func addMappings(p *profile.Profile) error {
+	// Parse memory map from /proc/self/maps
+	f, err := os.Open("/proc/self/maps")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return p.ParseMemoryMap(f)
+}

src/runtime/pprof/internal/protopprof/protopprof_test.go

+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package protopprof
+
+import (
+	"bytes"
+	"fmt"
+	"internal/pprof/profile"
+	"io"
+	"io/ioutil"
+	"reflect"
+	"runtime"
+	"runtime/pprof"
+	"testing"
+	"time"
+	"unsafe"
+)
+
+// Profile collects a CPU utilization profile and
+// writes it to w as a compressed profile.proto. It's used by
+// TestProfileParse.
+func Profile(w io.Writer, seconds int) error {
+	var buf bytes.Buffer
+	// Collect the CPU profile in legacy format in buf.
+	startTime := time.Now()
+	if err := pprof.StartCPUProfile(&buf); err != nil {
+		return fmt.Errorf("Could not enable CPU profiling: %s\n", err)
+	}
+	time.Sleep(time.Duration(seconds) * time.Second)
+	pprof.StopCPUProfile()
+
+	const untagged = false
+	p, err := TranslateCPUProfile(buf.Bytes(), startTime)
+	if err != nil {
+		return err
+	}
+	return p.Write(w)
+}
+
+// Helper function to initialize empty cpu profile with sampling period provided.
+func createEmptyProfileWithPeriod(t *testing.T, periodMs uint64) bytes.Buffer {
+	// Mock the sample header produced by cpu profiler. Write a sample
+	// period of 2000 microseconds, followed by no samples.
+	buf := new(bytes.Buffer)
+	// Profile header is as follows:
+	// The first, third and fifth words are 0. The second word is 3.
+	// The fourth word is the period.
+	// EOD marker:
+	// The sixth word -- count is initialized to 0 above.
+	// The code below sets the seventh word -- nstk to 1
+	// The eighth word -- addr is initialized to 0 above.
+	words := []int{0, 3, 0, int(periodMs), 0, 0, 1, 0}
+	n := int(unsafe.Sizeof(0)) * len(words)
+	data := ((*[1 << 29]byte)(unsafe.Pointer(&words[0])))[:n:n]
+	if _, err := buf.Write(data); err != nil {
+		t.Fatalf("createEmptyProfileWithPeriod failed: %v", err)
+	}
+	return *buf
+}
+
+// Helper function to initialize cpu profile with two sample values.
+func createProfileWithTwoSamples(t *testing.T, periodMs uintptr, count1 uintptr, count2 uintptr,
+	address1 uintptr, address2 uintptr) bytes.Buffer {
+	// Mock the sample header produced by cpu profiler. Write a sample
+	// period of 2000 microseconds, followed by no samples.
+	buf := new(bytes.Buffer)
+	words := []uint64{0, 3, 0, uint64(periodMs), 0, uint64(count1), 2,
+		uint64(address1), uint64(address1 + 2),
+		uint64(count2), 2, uint64(address2), uint64(address2 + 2),
+		0, uint64(1), 0}
+	for _, n := range words {
+		var err error
+		switch unsafe.Sizeof(int(0)) {
+		case 8:
+			_, err = buf.Write((*[8]byte)(unsafe.Pointer(&n))[:8:8])
+		case 4:
+			_, err = buf.Write((*[4]byte)(unsafe.Pointer(&n))[:4:4])
+		}
+		if err != nil {
+			t.Fatalf("createProfileWithTwoSamples failed: %v", err)
+		}
+	}
+	return *buf
+}
+
+// Tests that server creates a cpu profile handler that outputs a parsable Profile profile.
+func TestCPUProfileParse(t *testing.T) {
+	var before, after runtime.MemStats
+	runtime.ReadMemStats(&before)
+	var buf bytes.Buffer
+	if err := Profile(&buf, 30); err != nil {
+		t.Fatalf("Profile failed: %v", err)
+	}
+	runtime.ReadMemStats(&after)
+	_, err := profile.Parse(&buf)
+	if err != nil {
+		t.Fatalf("Could not parse Profile profile: %v", err)
+	}
+}
+
+// Tests TranslateCPUProfile parses correct sampling period in an otherwise empty cpu profile.
+func TestTranlateCPUProfileSamplingPeriod(t *testing.T) {
+	// A test server with mock cpu profile data.
+	var buf bytes.Buffer
+
+	startTime := time.Now()
+	b := createEmptyProfileWithPeriod(t, 2000)
+	p, err := TranslateCPUProfile(b.Bytes(), startTime)
+	if err != nil {
+		t.Fatalf("translate failed: %v", err)
+	}
+	if err := p.Write(&buf); err != nil {
+		t.Fatalf("write failed: %v", err)
+	}
+
+	p, err = profile.Parse(&buf)
+	if err != nil {
+		t.Fatalf("Could not parse Profile profile: %v", err)
+	}
+
+	// Expected PeriodType and SampleType.
+	expectedPeriodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}
+	expectedSampleType := []*profile.ValueType{
+		{Type: "samples", Unit: "count"},
+		{Type: "cpu", Unit: "nanoseconds"},
+	}
+	if p.Period != 2000*1000 || !reflect.DeepEqual(p.PeriodType, expectedPeriodType) ||
+		!reflect.DeepEqual(p.SampleType, expectedSampleType) || p.Sample != nil {
+		t.Fatalf("Unexpected Profile fields")
+	}
+}
+
+func getSampleAsString(sample []*profile.Sample) string {
+	var str string
+	for _, x := range sample {
+		for _, y := range x.Location {
+			if y.Mapping != nil {
+				str += fmt.Sprintf("Mapping:%v\n", *y.Mapping)
+			}
+			str += fmt.Sprintf("Location:%v\n", y)
+		}
+		str += fmt.Sprintf("Sample:%v\n", *x)
+	}
+	return str
+}
+
+// Tests TranslateCPUProfile parses a cpu profile with sample values present.
+func TestTranslateCPUProfileWithSamples(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("test requires a system with /proc/self/maps")
+	}
+	// Figure out two addresses from /proc/self/maps.
+	mmap, err := ioutil.ReadFile("/proc/self/maps")
+	if err != nil {
+		t.Fatal("Cannot read /proc/self/maps")
+	}
+	rd := bytes.NewReader(mmap)
+	mprof := &profile.Profile{}
+	if err = mprof.ParseMemoryMap(rd); err != nil {
+		t.Fatalf("Cannot parse /proc/self/maps")
+	}
+	if len(mprof.Mapping) < 2 {
+		t.Fatalf("Less than two mappings")
+	}
+	address1 := mprof.Mapping[0].Start
+	address2 := mprof.Mapping[1].Start
+	// A test server with mock cpu profile data.
+
+	startTime := time.Now()
+	b := createProfileWithTwoSamples(t, 2000, 20, 40, uintptr(address1), uintptr(address2))
+	p, err := TranslateCPUProfile(b.Bytes(), startTime)
+
+	if err != nil {
+		t.Fatalf("Could not parse Profile profile: %v", err)
+	}
+	// Expected PeriodType, SampleType and Sample.
+	expectedPeriodType := &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}
+	expectedSampleType := []*profile.ValueType{
+		{Type: "samples", Unit: "count"},
+		{Type: "cpu", Unit: "nanoseconds"},
+	}
+	expectedSample := []*profile.Sample{
+		{Value: []int64{20, 20 * 2000 * 1000}, Location: []*profile.Location{
+			{ID: 1, Mapping: mprof.Mapping[0], Address: address1},
+			{ID: 2, Mapping: mprof.Mapping[0], Address: address1 + 1},
+		}},
+		{Value: []int64{40, 40 * 2000 * 1000}, Location: []*profile.Location{
+			{ID: 3, Mapping: mprof.Mapping[1], Address: address2},
+			{ID: 4, Mapping: mprof.Mapping[1], Address: address2 + 1},
+		}},
+	}
+	if p.Period != 2000*1000 {
+		t.Fatalf("Sampling periods do not match")
+	}
+	if !reflect.DeepEqual(p.PeriodType, expectedPeriodType) {
+		t.Fatalf("Period types do not match")
+	}
+	if !reflect.DeepEqual(p.SampleType, expectedSampleType) {
+		t.Fatalf("Sample types do not match")
+	}
+	if !reflect.DeepEqual(p.Sample, expectedSample) {
+		t.Fatalf("Samples do not match: Expected: %v, Got:%v", getSampleAsString(expectedSample),
+			getSampleAsString(p.Sample))
+	}
+}