Skip to content

G
golang
Commit 9e227acf authored by Michael Matloob's avatar Michael Matloob
Browse files
Options

Revert "runtime/pprof: write profiles in protobuf format." 

This reverts commit b33030a7.

Reason for revert: We're going to try to get the code in this change
submitted in smaller, more carefully reviewed changes.

Change-Id: I4175f4b297f0e69fb78b11f9dc0bd82f27865be7
Reviewed-on: https://go-review.googlesource.com/32441Reviewed-by: 's avatarRuss Cox <rsc@golang.org>
parent 61ffec4b
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 28 additions and 1110 deletions
src/go/build/deps_test.go
View file @ 9e227acf
......@@ -170,15 +170,12 @@ var pkgDeps = map[string][]string{
"log": {"L1", "os", "fmt", "time"},
// Packages used by testing must be low-level (L2+fmt).
"regexp": {"L2", "regexp/syntax"},
"regexp/syntax": {"L2"},
"runtime/debug": {"L2", "fmt", "io/ioutil", "os", "time"},
"runtime/pprof/internal/gzip0": {"L2"},
"runtime/pprof/internal/profile": {"L2"},
"runtime/pprof/internal/protopprof": {"L2", "fmt", "runtime/pprof/internal/profile", "os", "time"},
"runtime/pprof": {"L2", "fmt", "runtime/pprof/internal/profile", "runtime/pprof/internal/protopprof", "time", "text/tabwriter"},
"runtime/trace": {"L0"},
"text/tabwriter": {"L2"},
"regexp": {"L2", "regexp/syntax"},
"regexp/syntax": {"L2"},
"runtime/debug": {"L2", "fmt", "io/ioutil", "os", "time"},
"runtime/pprof": {"L2", "fmt", "os", "text/tabwriter"},
"runtime/trace": {"L0"},
"text/tabwriter": {"L2"},
"testing": {"L2", "flag", "fmt", "os", "runtime/debug", "runtime/pprof", "runtime/trace", "time"},
"testing/iotest": {"L2", "log"},
......
src/runtime/crash_cgo_test.go
View file @ 9e227acf
......@@ -271,11 +271,10 @@ func testCgoPprof(t *testing.T, buildArg, runArg string) {
if err != nil {
t.Fatal(err)
}
fn := strings.TrimSpace(string(got))
defer os.Remove(fn)
cmd := testEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-top", "-nodecount=1", "-symbolize=force", exe, fn))
cmd := testEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-top", "-nodecount=1", exe, fn))
found := false
for i, e := range cmd.Env {
......
src/runtime/pprof/decoder_for_test.go deleted 100644 → 0
View file @ 61ffec4b
This diff is collapsed.
src/runtime/pprof/internal/profile/encode.go deleted 100644 → 0
View file @ 61ffec4b
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package profile
import (
"sort"
)
// preEncode populates the unexported fields to be used by encode
// (with suffix X) from the corresponding exported fields. The
// exported fields are cleared up to facilitate testing.
func (p *Profile) preEncode() {
strings := make(map[string]int)
addString(strings, "")
for _, st := range p.SampleType {
st.typeX = addString(strings, st.Type)
st.unitX = addString(strings, st.Unit)
}
for _, s := range p.Sample {
s.labelX = nil
var keys []string
for k := range s.Label {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
vs := s.Label[k]
for _, v := range vs {
s.labelX = append(s.labelX,
Label{
keyX: addString(strings, k),
strX: addString(strings, v),
},
)
}
}
var numKeys []string
for k := range s.NumLabel {
numKeys = append(numKeys, k)
}
sort.Strings(numKeys)
for _, k := range numKeys {
vs := s.NumLabel[k]
for _, v := range vs {
s.labelX = append(s.labelX,
Label{
keyX: addString(strings, k),
numX: v,
},
)
}
}
s.locationIDX = nil
for _, l := range s.Location {
s.locationIDX = append(s.locationIDX, l.ID)
}
}
for _, m := range p.Mapping {
m.fileX = addString(strings, m.File)
m.buildIDX = addString(strings, m.BuildID)
}
for _, l := range p.Location {
for i, ln := range l.Line {
if ln.Function != nil {
l.Line[i].functionIDX = ln.Function.ID
} else {
l.Line[i].functionIDX = 0
}
}
if l.Mapping != nil {
l.mappingIDX = l.Mapping.ID
} else {
l.mappingIDX = 0
}
}
for _, f := range p.Function {
f.nameX = addString(strings, f.Name)
f.systemNameX = addString(strings, f.SystemName)
f.filenameX = addString(strings, f.Filename)
}
if pt := p.PeriodType; pt != nil {
pt.typeX = addString(strings, pt.Type)
pt.unitX = addString(strings, pt.Unit)
}
p.stringTable = make([]string, len(strings))
for s, i := range strings {
p.stringTable[i] = s
}
}
func (p *Profile) encode(b *buffer) {
for _, x := range p.SampleType {
encodeMessage(b, 1, x)
}
for _, x := range p.Sample {
encodeMessage(b, 2, x)
}
for _, x := range p.Mapping {
encodeMessage(b, 3, x)
}
for _, x := range p.Location {
encodeMessage(b, 4, x)
}
for _, x := range p.Function {
encodeMessage(b, 5, x)
}
encodeStrings(b, 6, p.stringTable)
encodeInt64Opt(b, 9, p.TimeNanos)
encodeInt64Opt(b, 10, p.DurationNanos)
if pt := p.PeriodType; pt != nil && (pt.typeX != 0 || pt.unitX != 0) {
encodeMessage(b, 11, p.PeriodType)
}
encodeInt64Opt(b, 12, p.Period)
}
func (p *ValueType) encode(b *buffer) {
encodeInt64Opt(b, 1, p.typeX)
encodeInt64Opt(b, 2, p.unitX)
}
func (p *Sample) encode(b *buffer) {
encodeUint64s(b, 1, p.locationIDX)
for _, x := range p.Value {
encodeInt64(b, 2, x)
}
for _, x := range p.labelX {
encodeMessage(b, 3, x)
}
}
func (p Label) encode(b *buffer) {
encodeInt64Opt(b, 1, p.keyX)
encodeInt64Opt(b, 2, p.strX)
encodeInt64Opt(b, 3, p.numX)
}
func (p *Mapping) encode(b *buffer) {
encodeUint64Opt(b, 1, p.ID)
encodeUint64Opt(b, 2, p.Start)
encodeUint64Opt(b, 3, p.Limit)
encodeUint64Opt(b, 4, p.Offset)
encodeInt64Opt(b, 5, p.fileX)
encodeInt64Opt(b, 6, p.buildIDX)
encodeBoolOpt(b, 7, p.HasFunctions)
encodeBoolOpt(b, 8, p.HasFilenames)
encodeBoolOpt(b, 9, p.HasLineNumbers)
encodeBoolOpt(b, 10, p.HasInlineFrames)
}
func (p *Location) encode(b *buffer) {
encodeUint64Opt(b, 1, p.ID)
encodeUint64Opt(b, 2, p.mappingIDX)
encodeUint64Opt(b, 3, p.Address)
for i := range p.Line {
encodeMessage(b, 4, &p.Line[i])
}
}
func (p *Line) encode(b *buffer) {
encodeUint64Opt(b, 1, p.functionIDX)
encodeInt64Opt(b, 2, p.Line)
}
func (p *Function) encode(b *buffer) {
encodeUint64Opt(b, 1, p.ID)
encodeInt64Opt(b, 2, p.nameX)
encodeInt64Opt(b, 3, p.systemNameX)
encodeInt64Opt(b, 4, p.filenameX)
encodeInt64Opt(b, 5, p.StartLine)
}
func addString(strings map[string]int, s string) int64 {
i, ok := strings[s]
if !ok {
i = len(strings)
strings[s] = i
}
return int64(i)
}
src/runtime/pprof/internal/profile/profile.go deleted 100644 → 0
View file @ 61ffec4b
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Package profile provides a representation of profile.proto and
// methods to encode/decode profiles in this format.
package profile
import (
"io"
)
// Profile is an in-memory representation of profile.proto.
type Profile struct {
SampleType []*ValueType
Sample []*Sample
Mapping []*Mapping
Location []*Location
Function []*Function
TimeNanos int64
DurationNanos int64
PeriodType *ValueType
Period int64
stringTable []string
}
// ValueType corresponds to Profile.ValueType
type ValueType struct {
Type string // cpu, wall, inuse_space, etc
Unit string // seconds, nanoseconds, bytes, etc
typeX int64
unitX int64
}
// Sample corresponds to Profile.Sample
type Sample struct {
Location []*Location
Value []int64
Label map[string][]string
NumLabel map[string][]int64
locationIDX []uint64
labelX []Label
}
// Label corresponds to Profile.Label
type Label struct {
keyX int64
// Exactly one of the two following values must be set
strX int64
numX int64 // Integer value for this label
}
// Mapping corresponds to Profile.Mapping
type Mapping struct {
ID uint64
Start uint64
Limit uint64
Offset uint64
File string
BuildID string
HasFunctions bool
HasFilenames bool
HasLineNumbers bool
HasInlineFrames bool
fileX int64
buildIDX int64
}
// Location corresponds to Profile.Location
type Location struct {
ID uint64
Mapping *Mapping
Address uint64
Line []Line
mappingIDX uint64
}
// Line corresponds to Profile.Line
type Line struct {
Function *Function
Line int64
functionIDX uint64
}
// Function corresponds to Profile.Function
type Function struct {
ID uint64
Name string
SystemName string
Filename string
StartLine int64
nameX int64
systemNameX int64
filenameX int64
}
// Write writes the profile as a gzip-compressed marshaled protobuf.
func (p *Profile) Write(w io.Writer) error {
p.preEncode()
var b buffer
p.encode(&b)
_, err := w.Write(b.data)
return err
}
src/runtime/pprof/internal/profile/profile_memmap.go deleted 100644 → 0
View file @ 61ffec4b
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package profile
import (
"bufio"
"errors"
"io"
"strconv"
"strings"
)
var errUnrecognized = errors.New("unrecognized profile format")
func hasLibFile(file string) string {
ix := strings.Index(file, "so")
if ix < 1 {
return ""
}
start := ix - 1
end := ix + 2
s := file[start:end]
if end < len(file) {
endalt := end
if file[endalt] != '.' && file[endalt] != '_' {
return s
}
endalt++
for file[endalt] >= '0' && file[endalt] <= '9' {
endalt++
}
if endalt < end+2 {
return s
}
return s[start:endalt]
}
return s
}
// massageMappings applies heuristic-based changes to the profile
// mappings to account for quirks of some environments.
func (p *Profile) massageMappings() {
// Merge adjacent regions with matching names, checking that the offsets match
if len(p.Mapping) > 1 {
mappings := []*Mapping{p.Mapping[0]}
for _, m := range p.Mapping[1:] {
lm := mappings[len(mappings)-1]
if offset := lm.Offset + (lm.Limit - lm.Start); lm.Limit == m.Start &&
offset == m.Offset &&
(lm.File == m.File || lm.File == "") {
lm.File = m.File
lm.Limit = m.Limit
if lm.BuildID == "" {
lm.BuildID = m.BuildID
}
p.updateLocationMapping(m, lm)
continue
}
mappings = append(mappings, m)
}
p.Mapping = mappings
}
// Use heuristics to identify main binary and move it to the top of the list of mappings
for i, m := range p.Mapping {
file := strings.TrimSpace(strings.Replace(m.File, "(deleted)", "", -1))
if len(file) == 0 {
continue
}
if len(hasLibFile(file)) > 0 {
continue
}
if strings.HasPrefix(file, "[") {
continue
}
// Swap what we guess is main to position 0.
p.Mapping[0], p.Mapping[i] = p.Mapping[i], p.Mapping[0]
break
}
// Keep the mapping IDs neatly sorted
for i, m := range p.Mapping {
m.ID = uint64(i + 1)
}
}
func (p *Profile) updateLocationMapping(from, to *Mapping) {
for _, l := range p.Location {
if l.Mapping == from {
l.Mapping = to
}
}
}
// remapLocationIDs ensures there is a location for each address
// referenced by a sample, and remaps the samples to point to the new
// location ids.
func (p *Profile) remapLocationIDs() {
seen := make(map[*Location]bool, len(p.Location))
var locs []*Location
for _, s := range p.Sample {
for _, l := range s.Location {
if seen[l] {
continue
}
l.ID = uint64(len(locs) + 1)
locs = append(locs, l)
seen[l] = true
}
}
p.Location = locs
}
func (p *Profile) remapFunctionIDs() {
seen := make(map[*Function]bool, len(p.Function))
var fns []*Function
for _, l := range p.Location {
for _, ln := range l.Line {
fn := ln.Function
if fn == nil || seen[fn] {
continue
}
fn.ID = uint64(len(fns) + 1)
fns = append(fns, fn)
seen[fn] = true
}
}
p.Function = fns
}
// remapMappingIDs matches location addresses with existing mappings
// and updates them appropriately. This is O(N*M), if this ever shows
// up as a bottleneck, evaluate sorting the mappings and doing a
// binary search, which would make it O(N*log(M)).
func (p *Profile) remapMappingIDs() {
// Some profile handlers will incorrectly set regions for the main
// executable if its section is remapped. Fix them through heuristics.
if len(p.Mapping) > 0 {
// Remove the initial mapping if named '/anon_hugepage' and has a
// consecutive adjacent mapping.
if m := p.Mapping[0]; strings.HasPrefix(m.File, "/anon_hugepage") {
if len(p.Mapping) > 1 && m.Limit == p.Mapping[1].Start {
p.Mapping = p.Mapping[1:]
}
}
}
// Subtract the offset from the start of the main mapping if it
// ends up at a recognizable start address.
if len(p.Mapping) > 0 {
const expectedStart = 0x400000
if m := p.Mapping[0]; m.Start-m.Offset == expectedStart {
m.Start = expectedStart
m.Offset = 0
}
}
// Associate each location with an address to the corresponding
// mapping. Create fake mapping if a suitable one isn't found.
var fake *Mapping
nextLocation:
for _, l := range p.Location {
a := l.Address
if l.Mapping != nil || a == 0 {
continue
}
for _, m := range p.Mapping {
if m.Start <= a && a < m.Limit {
l.Mapping = m
continue nextLocation
}
}
// Work around legacy handlers failing to encode the first
// part of mappings split into adjacent ranges.
for _, m := range p.Mapping {
if m.Offset != 0 && m.Start-m.Offset <= a && a < m.Start {
m.Start -= m.Offset
m.Offset = 0
l.Mapping = m
continue nextLocation
}
}
// If there is still no mapping, create a fake one.
// This is important for the Go legacy handler, which produced
// no mappings.
if fake == nil {
fake = &Mapping{
ID: 1,
Limit: ^uint64(0),
}
p.Mapping = append(p.Mapping, fake)
}
l.Mapping = fake
}
// Reset all mapping IDs.
for i, m := range p.Mapping {
m.ID = uint64(i + 1)
}
}
func (p *Profile) RemapAll() {
p.remapLocationIDs()
p.remapFunctionIDs()
p.remapMappingIDs()
}
// ParseProcMaps parses a memory map in the format of /proc/self/maps.
// ParseMemoryMap should be called after setting on a profile to
// associate locations to the corresponding mapping based on their
// address.
func ParseProcMaps(rd io.Reader) ([]*Mapping, error) {
var mapping []*Mapping
b := bufio.NewReader(rd)
var attrs []string
var r *strings.Replacer
const delimiter = "="
for {
l, err := b.ReadString('\n')
if err != nil {
if err != io.EOF {
return nil, err
}
if l == "" {
break
}
}
if l = strings.TrimSpace(l); l == "" {
continue
}
if r != nil {
l = r.Replace(l)
}
m, err := parseMappingEntry(l)
if err != nil {
if err == errUnrecognized {
// Recognize assignments of the form: attr=value, and replace
// $attr with value on subsequent mappings.
if attr := strings.SplitN(l, delimiter, 2); len(attr) == 2 {
attrs = append(attrs, "$"+strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1]))
r = strings.NewReplacer(attrs...)
}
// Ignore any unrecognized entries
continue
}
return nil, err
}
if m == nil {
continue
}
mapping = append(mapping, m)
}
return mapping, nil
}
// ParseMemoryMap parses a memory map in the format of
// /proc/self/maps, and overrides the mappings in the current profile.
// It renumbers the samples and locations in the profile correspondingly.
func (p *Profile) ParseMemoryMap(rd io.Reader) error {
mapping, err := ParseProcMaps(rd)
if err != nil {
return err
}
p.Mapping = append(p.Mapping, mapping...)
p.massageMappings()
p.RemapAll()
return nil
}
func parseMappingEntry(l string) (*Mapping, error) {
mapping := &Mapping{}
var err error
fields := strings.Fields(l)
// fmt.Println(len(me), me)
if len(fields) == 6 {
if !strings.Contains(fields[1], "x") {
// Skip non-executable entries.
return nil, nil
}
addrRange := strings.Split(fields[0], "-")
if mapping.Start, err = strconv.ParseUint(addrRange[0], 16, 64); err != nil {
return nil, errUnrecognized
}
if mapping.Limit, err = strconv.ParseUint(addrRange[1], 16, 64); err != nil {
return nil, errUnrecognized
}
offset := fields[2]
if offset != "" {
if mapping.Offset, err = strconv.ParseUint(offset, 16, 64); err != nil {
return nil, errUnrecognized
}
}
mapping.File = fields[5]
return mapping, nil
}
return nil, errUnrecognized
}
src/runtime/pprof/internal/profile/proto.go deleted 100644 → 0
View file @ 61ffec4b
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// This file is a simple protocol buffer encoder and decoder.
//
// A protocol message must implement the message interface:
// decoder() []decoder
// encode(*buffer)
//
// The decode method returns a slice indexed by field number that gives the
// function to decode that field.
// The encode method encodes its receiver into the given buffer.
//
// The two methods are simple enough to be implemented by hand rather than
// by using a protocol compiler.
//
// See profile.go for examples of messages implementing this interface.
//
// There is no support for groups, message sets, or "has" bits.
package profile
type buffer struct {
field int
typ int
u64 uint64
data []byte
tmp [16]byte
}
type message interface {
encode(*buffer)
}
func encodeVarint(b *buffer, x uint64) {
for x >= 128 {
b.data = append(b.data, byte(x)|0x80)
x >>= 7
}
b.data = append(b.data, byte(x))
}
func encodeLength(b *buffer, tag int, len int) {
encodeVarint(b, uint64(tag)<<3|2)
encodeVarint(b, uint64(len))
}
func encodeUint64(b *buffer, tag int, x uint64) {
// append varint to b.data
encodeVarint(b, uint64(tag)<<3|0)
encodeVarint(b, x)
}
func encodeUint64s(b *buffer, tag int, x []uint64) {
if len(x) > 2 {
// Use packed encoding
n1 := len(b.data)
for _, u := range x {
encodeVarint(b, u)
}
n2 := len(b.data)
encodeLength(b, tag, n2-n1)
n3 := len(b.data)
copy(b.tmp[:], b.data[n2:n3])
copy(b.data[n1+(n3-n2):], b.data[n1:n2])
copy(b.data[n1:], b.tmp[:n3-n2])
return
}
for _, u := range x {
encodeUint64(b, tag, u)
}
}
func encodeUint64Opt(b *buffer, tag int, x uint64) {
if x == 0 {
return
}
encodeUint64(b, tag, x)
}
func encodeInt64(b *buffer, tag int, x int64) {
u := uint64(x)
encodeUint64(b, tag, u)
}
func encodeInt64Opt(b *buffer, tag int, x int64) {
if x == 0 {
return
}
encodeInt64(b, tag, x)
}
func encodeInt64s(b *buffer, tag int, x []int64) {
if len(x) > 2 {
// Use packed encoding
n1 := len(b.data)
for _, u := range x {
encodeVarint(b, uint64(u))
}
n2 := len(b.data)
encodeLength(b, tag, n2-n1)
n3 := len(b.data)
copy(b.tmp[:], b.data[n2:n3])
copy(b.data[n1+(n3-n2):], b.data[n1:n2])
copy(b.data[n1:], b.tmp[:n3-n2])
return
}
for _, u := range x {
encodeInt64(b, tag, u)
}
}
func encodeString(b *buffer, tag int, x string) {
encodeLength(b, tag, len(x))
b.data = append(b.data, x...)
}
func encodeStrings(b *buffer, tag int, x []string) {
for _, s := range x {
encodeString(b, tag, s)
}
}
func encodeStringOpt(b *buffer, tag int, x string) {
if x == "" {
return
}
encodeString(b, tag, x)
}
func encodeBool(b *buffer, tag int, x bool) {
if x {
encodeUint64(b, tag, 1)
} else {
encodeUint64(b, tag, 0)
}
}
func encodeBoolOpt(b *buffer, tag int, x bool) {
if x == false {
return
}
encodeBool(b, tag, x)
}
func encodeMessage(b *buffer, tag int, m message) {
n1 := len(b.data)
m.encode(b)
n2 := len(b.data)
encodeLength(b, tag, n2-n1)
n3 := len(b.data)
copy(b.tmp[:], b.data[n2:n3])
copy(b.data[n1+(n3-n2):], b.data[n1:n2])
copy(b.data[n1:], b.tmp[:n3-n2])
}
src/runtime/pprof/internal/protopprof/protopprof.go deleted 100644 → 0
View file @ 61ffec4b
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package protopprof
import (
"fmt"
"os"
"runtime"
"strings"
"time"
"runtime/pprof/internal/profile"
)
// Copied from encoding/binary package, which can't be imported due to
// dependency cycles
// LittleEndian is the little-endian implementation of ByteOrder.
var lEndian littleEndian
// BigEndian is the big-endian implementation of ByteOrder.
var bEndian bigEndian
type littleEndian struct{}
type bigEndian struct{}
func (bigEndian) uint32(b []byte) uint32 {
_ = b[3] // bounds check hint to compiler; see golang.org/issue/14808
return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
}
func (bigEndian) uint64(b []byte) uint64 {
_ = b[7] // bounds check hint to compiler; see golang.org/issue/14808
return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 |
uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56
}
func (littleEndian) uint32(b []byte) uint32 {
_ = b[3] // bounds check hint to compiler; see golang.org/issue/14808
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
}
func (littleEndian) uint64(b []byte) uint64 {
_ = b[7] // bounds check hint to compiler; see golang.org/issue/14808
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
}
func big32(b []byte) (uint64, []byte) {
if len(b) < 4 {
return 0, nil
}
return uint64(bEndian.uint32(b)), b[4:]
}
func little32(b []byte) (uint64, []byte) {
if len(b) < 4 {
return 0, nil
}
return uint64(lEndian.uint32(b)), b[4:]
}
func big64(b []byte) (uint64, []byte) {
if len(b) < 8 {
return 0, nil
}
return bEndian.uint64(b), b[8:]
}
func little64(b []byte) (uint64, []byte) {
if len(b) < 8 {
return 0, nil
}
return lEndian.uint64(b), b[8:]
}
// End of copy from encoding/binary package
type parser func([]byte) (uint64, []byte)
var parsers = []parser{
big32,
big64,
little32,
little64,
}
// parse returns a parsing function to parse native integers from a buffer.
func findParser(b []byte) parser {
for _, p := range parsers {
// If the second word decodes as 3, we have the right parser.
_, rest := p(b) // first word
n, _ := p(rest) // second word
if n == 3 {
return p
}
}
return nil
}
// decodeHeader parses binary CPU profiling stack trace data
// generated by runtime.CPUProfile() and returns the sample period,
// the rest of the profile and a parse function for parsing the profile. The
// function detects whether the legacy profile format is in little or big
// endian and whether it was generated by a 32-bit or 64-bit machine.
func decodeHeader(b []byte) (period uint64, parse parser, rest []byte, err error) {
const minRawProfile = 12 // Need a minimum of 3 words, at least 32-bit each.
if len(b) < minRawProfile {
return 0, nil, nil, fmt.Errorf("truncated raw profile: len %d", len(b))
}
if parse = findParser(b); parse == nil {
return 0, nil, nil, fmt.Errorf("cannot parse raw profile: header %v", b[:minRawProfile])
}
// skip 5-word header; 4th word is period
_, rest = parse(b)
_, rest = parse(rest)
_, rest = parse(rest)
period, rest = parse(rest)
_, rest = parse(rest)
if rest == nil {
return 0, nil, nil, fmt.Errorf("profile too short")
}
return period, parse, rest, nil
}
// translateCPUProfile parses binary CPU profiling stack trace data
// generated by runtime.CPUProfile() into a profile struct.
func TranslateCPUProfile(b []byte, startTime time.Time) (*profile.Profile, error) {
// Get the sample period from the header.
var n4 uint64
var getInt parser
var err error
n4, getInt, b, err = decodeHeader(b)
if err != nil {
return nil, err
}
// profile initialization taken from pprof tool
p := &profile.Profile{
Period: int64(n4) * 1000,
PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
SampleType: []*profile.ValueType{
{Type: "samples", Unit: "count"},
{Type: "cpu", Unit: "nanoseconds"},
},
TimeNanos: int64(startTime.UnixNano()),
DurationNanos: time.Since(startTime).Nanoseconds(),
}
// Parse CPU samples from the profile.
locs := make(map[uint64]*profile.Location)
for len(b) > 0 {
var count, nstk uint64
count, b = getInt(b)
nstk, b = getInt(b)
if b == nil {
return nil, fmt.Errorf("unrecognized profile format")
}
var sloc []*profile.Location
addrs := make([]uint64, nstk)
for i := 0; i < int(nstk); i++ {
if b == nil {
return nil, fmt.Errorf("unrecognized profile format")
}
addrs[i], b = getInt(b)
}
// End of data marker, can return
if count == 0 && nstk == 1 && addrs[0] == 0 {
if runtime.GOOS == "linux" {
if err := addMappings(p); err != nil {
return nil, err
}
}
return p, nil
}
for i, addr := range addrs {
// Addresses from stack traces point to the next instruction after
// each call. Adjust by -1 to land somewhere on the actual call
// (except for the leaf, which is not a call).
if i > 0 {
addr--
}
loc := locs[addr]
if loc == nil {
loc = &profile.Location{
ID: uint64(len(p.Location) + 1),
Address: addr,
}
locs[addr] = loc
p.Location = append(p.Location, loc)
}
sloc = append(sloc, loc)
}
p.Sample = append(p.Sample, &profile.Sample{
Value: []int64{int64(count), int64(count) * int64(p.Period)},
Location: sloc,
})
}
return nil, fmt.Errorf("unrecognized profile format")
}
func addMappings(p *profile.Profile) error {
// Parse memory map from /proc/self/maps
f, err := os.Open("/proc/self/maps")
if err != nil {
return err
}
defer f.Close()
return p.ParseMemoryMap(f)
}
// Symbolization enables adding names to locations.
func Symbolize(p *profile.Profile) {
fns := profileFunctionMap{}
for _, l := range p.Location {
pc := uintptr(l.Address)
f := runtime.FuncForPC(pc)
if f == nil {
continue
}
file, lineno := f.FileLine(pc)
if l.Mapping != nil {
if f.Name() != "" {
l.Mapping.HasFunctions = true
}
if file != "" {
l.Mapping.HasFilenames = true
}
if lineno != 0 {
l.Mapping.HasLineNumbers = true
}
}
l.Line = []profile.Line{
{
Function: fns.findOrAddFunction(f.Name(), file, p),
Line: int64(lineno),
},
}
}
// Trim runtime functions. Always hide runtime.goexit. Other runtime
// functions are only hidden for heapz when they appear at the beginning.
isHeapz := p.PeriodType != nil && p.PeriodType.Type == "space"
for _, s := range p.Sample {
show := !isHeapz
var i int
for _, l := range s.Location {
if (len(l.Line) > 0) && (l.Line[0].Function != nil) {
name := l.Line[0].Function.Name
if (name == "runtime.goexit") || (!show && strings.HasPrefix(name, "runtime.")) {
continue
}
}
show = true
s.Location[i] = l
i++
}
s.Location = s.Location[:i]
}
}
type profileFunctionMap map[profile.Function]*profile.Function
func (fns profileFunctionMap) findOrAddFunction(name, filename string, p *profile.Profile) *profile.Function {
f := profile.Function{
Name: name,
SystemName: name,
Filename: filename,
}
if fp := fns[f]; fp != nil {
return fp
}
fp := new(profile.Function)
fns[f] = fp
*fp = f
fp.ID = uint64(len(p.Function) + 1)
p.Function = append(p.Function, fp)
return fp
}
func CleanupDuplicateLocations(p *profile.Profile) {
// The profile handler may duplicate the leaf frame, because it gets
// its address both from stack unwinding and from the signal
// context. Detect this and delete the duplicate, which has been
// adjusted by -1. The leaf address should not be adjusted as it is
// not a call.
for _, s := range p.Sample {
if len(s.Location) > 1 && s.Location[0].Address == s.Location[1].Address+1 {
s.Location = append(s.Location[:1], s.Location[2:]...)
}
}
}
src/runtime/pprof/mprof_test.go
View file @ 9e227acf
......@@ -6,8 +6,8 @@ package pprof_test
import (
"bytes"
"math"
"reflect"
"fmt"
"regexp"
"runtime"
. "runtime/pprof"
"testing"
......@@ -71,48 +71,26 @@ func TestMemoryProfiler(t *testing.T) {
memoryProfilerRun++
r := bytes.NewReader(buf.Bytes())
p, err := Parse(r)
if err != nil {
t.Fatalf("can't parse pprof profile: %v", err)
tests := []string{
fmt.Sprintf(`%v: %v \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
# 0x[0-9,a-f]+ runtime/pprof_test\.allocatePersistent1K\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test\.go:40
# 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test\.go:63
`, 32*memoryProfilerRun, 1024*memoryProfilerRun, 32*memoryProfilerRun, 1024*memoryProfilerRun),
fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
# 0x[0-9,a-f]+ runtime/pprof_test\.allocateTransient1M\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:21
# 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:61
`, (1<<10)*memoryProfilerRun, (1<<20)*memoryProfilerRun),
fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+
# 0x[0-9,a-f]+ runtime/pprof_test\.allocateTransient2M\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:27
# 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:62
`, memoryProfilerRun, (2<<20)*memoryProfilerRun),
}
if len(p.Sample) < 3 {
t.Fatalf("few samples, got: %d", len(p.Sample))
}
testSample := make(map[int][]int64)
testSample[0] = scaleHeapSample((int64)(32*memoryProfilerRun), (int64)(1024*memoryProfilerRun), p.Period)
testSample[0] = append(testSample[0], testSample[0][0], testSample[0][1])
testSample[1] = scaleHeapSample((int64)((1<<10)*memoryProfilerRun), (int64)((1<<20)*memoryProfilerRun), p.Period)
testSample[1] = append([]int64{0, 0}, testSample[1][0], testSample[1][1])
testSample[2] = scaleHeapSample((int64)(memoryProfilerRun), (int64)((2<<20)*memoryProfilerRun), p.Period)
testSample[2] = append([]int64{0, 0}, testSample[2][0], testSample[2][1])
for _, value := range testSample {
found := false
for i := range p.Sample {
if reflect.DeepEqual(p.Sample[i].Value, value) {
found = true
break
}
}
if !found {
t.Fatalf("the entry did not match any sample:\n%v\n", value)
}
}
}
func scaleHeapSample(count, size, rate int64) []int64 {
if count == 0 || size == 0 {
return []int64{0, 0}
}
if rate <= 1 {
// if rate==1 all samples were collected so no adjustment is needed.
// if rate<1 treat as unknown and skip scaling.
return []int64{count, size}
for _, test := range tests {
if !regexp.MustCompile(test).Match(buf.Bytes()) {
t.Fatalf("The entry did not match:\n%v\n\nProfile:\n%v\n", test, buf.String())
}
}
avgSize := float64(size) / float64(count)
scale := 1 / (1 - math.Exp(-avgSize/float64(rate)))
return []int64{int64(float64(count) * scale), int64(float64(size) * scale)}
}
src/runtime/pprof/pprof.go
View file @ 9e227acf
This diff is collapsed.
src/runtime/pprof/pprof_test.go
View file @ 9e227acf
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment