cmd/trace: add new command

Trace command allows to visualize and analyze traces.
Run as:
$ go tool trace binary trace.file
The commands opens web browser with the main page,
which contains links for trace visualization,
blocking profiler, network IO profiler and per-goroutine
traces.

Also move trace parser from runtime/pprof/trace_parser_test.go
to internal/trace/parser.go, so that it can be shared between
tests and the command.

Change-Id: Ic97ed59ad6e4c7e1dc9eca5e979701a2b4aed7cf
Reviewed-on: https://go-review.googlesource.com/3601Reviewed-by: Andrew Gerrand <adg@golang.org>

misc/trace/README.md

+This directory contains helper file for trace viewer (go tool trace).
+
+trace_viewer_lean.html was generated following instructions in:
+https://github.com/google/trace-viewer/wiki/Embedding
+on revision 895aa74558d19d91906fb720df6458244ef160c6 using:
+trace-viewer$ ./vulcanize_trace_viewer --config=lean

src/cmd/go/pkg.go

@@ -400,6 +400,7 @@ var goTools = map[string]targetDir{
 	"cmd/objdump":                          toTool,
 	"cmd/pack":                             toTool,
 	"cmd/pprof":                            toTool,
+	"cmd/trace":                            toTool,
 	"cmd/yacc":                             toTool,
 	"golang.org/x/tools/cmd/cover":         toTool,
 	"golang.org/x/tools/cmd/godoc":         toBin,

src/cmd/trace/goroutines.go

+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Goroutine-related profiles.
+
+package main
+
+import (
+	"fmt"
+	"html/template"
+	"internal/trace"
+	"net/http"
+	"sort"
+	"strconv"
+)
+
+func init() {
+	http.HandleFunc("/goroutines", httpGoroutines)
+	http.HandleFunc("/goroutine", httpGoroutine)
+}
+
+// gtype describes a group of goroutines grouped by start PC.
+type gtype struct {
+	ID       uint64 // Unique identifier (PC).
+	Name     string // Start function.
+	N        int    // Total number of goroutines in this group.
+	ExecTime int64  // Total execution time of all goroutines in this group.
+}
+
+type gtypeList []gtype
+
+func (l gtypeList) Len() int {
+	return len(l)
+}
+
+func (l gtypeList) Less(i, j int) bool {
+	return l[i].ExecTime > l[j].ExecTime
+}
+
+func (l gtypeList) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}
+
+// gdesc desribes a single goroutine.
+type gdesc struct {
+	ID         uint64
+	Name       string
+	PC         uint64
+	CreateTime int64
+	StartTime  int64
+	EndTime    int64
+	LastStart  int64
+
+	ExecTime      int64
+	SchedWaitTime int64
+	IOTime        int64
+	BlockTime     int64
+	SyscallTime   int64
+	GCTime        int64
+	SweepTime     int64
+	TotalTime     int64
+
+	blockNetTime     int64
+	blockSyncTime    int64
+	blockSyscallTime int64
+	blockSweepTime   int64
+	blockGCTime      int64
+	blockSchedTime   int64
+}
+
+type gdescList []*gdesc
+
+func (l gdescList) Len() int {
+	return len(l)
+}
+
+func (l gdescList) Less(i, j int) bool {
+	return l[i].TotalTime > l[j].TotalTime
+}
+
+func (l gdescList) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}
+
+var gs = make(map[uint64]*gdesc)
+
+// analyzeGoroutines generates list gdesc's from the trace and stores it in gs.
+func analyzeGoroutines(events []*trace.Event) {
+	if len(gs) > 0 { //!!! racy
+		return
+	}
+	var lastTs int64
+	var gcStartTime int64
+	for _, ev := range events {
+		lastTs = ev.Ts
+		switch ev.Type {
+		case trace.EvGoCreate:
+			g := &gdesc{CreateTime: ev.Ts}
+			g.blockSchedTime = ev.Ts
+			gs[ev.Args[0]] = g
+		case trace.EvGoStart:
+			g := gs[ev.G]
+			if g.PC == 0 {
+				g.PC = ev.Stk[0].PC
+				g.Name = ev.Stk[0].Fn
+			}
+			g.LastStart = ev.Ts
+			if g.StartTime == 0 {
+				g.StartTime = ev.Ts
+			}
+			if g.blockSchedTime != 0 {
+				g.SchedWaitTime += ev.Ts - g.blockSchedTime
+				g.blockSchedTime = 0
+			}
+		case trace.EvGoEnd, trace.EvGoStop:
+			g := gs[ev.G]
+			g.ExecTime += ev.Ts - g.LastStart
+			g.TotalTime = ev.Ts - g.CreateTime
+			g.EndTime = ev.Ts
+		case trace.EvGoBlockSend, trace.EvGoBlockRecv, trace.EvGoBlockSelect,
+			trace.EvGoBlockSync, trace.EvGoBlockCond:
+			g := gs[ev.G]
+			g.ExecTime += ev.Ts - g.LastStart
+			g.blockSyncTime = ev.Ts
+		case trace.EvGoSched, trace.EvGoPreempt:
+			g := gs[ev.G]
+			g.ExecTime += ev.Ts - g.LastStart
+			g.blockSchedTime = ev.Ts
+		case trace.EvGoSleep, trace.EvGoBlock:
+			g := gs[ev.G]
+			g.ExecTime += ev.Ts - g.LastStart
+		case trace.EvGoBlockNet:
+			g := gs[ev.G]
+			g.ExecTime += ev.Ts - g.LastStart
+			g.blockNetTime = ev.Ts
+		case trace.EvGoUnblock:
+			g := gs[ev.Args[0]]
+			if g.blockNetTime != 0 {
+				g.IOTime += ev.Ts - g.blockNetTime
+				g.blockNetTime = 0
+			}
+			if g.blockSyncTime != 0 {
+				g.BlockTime += ev.Ts - g.blockSyncTime
+				g.blockSyncTime = 0
+			}
+			g.blockSchedTime = ev.Ts
+		case trace.EvGoSysBlock:
+			g := gs[ev.G]
+			g.ExecTime += ev.Ts - g.LastStart
+			g.blockSyscallTime = ev.Ts
+		case trace.EvGoSysExit:
+			g := gs[ev.G]
+			if g.blockSyscallTime != 0 {
+				g.SyscallTime += ev.Ts - g.blockSyscallTime
+				g.blockSyscallTime = 0
+			}
+			g.blockSchedTime = ev.Ts
+		case trace.EvGCSweepStart:
+			g := gs[ev.G]
+			if g != nil {
+				// Sweep can happen during GC on system goroutine.
+				g.blockSweepTime = ev.Ts
+			}
+		case trace.EvGCSweepDone:
+			g := gs[ev.G]
+			if g != nil && g.blockSweepTime != 0 {
+				g.SweepTime += ev.Ts - g.blockSweepTime
+				g.blockSweepTime = 0
+			}
+		case trace.EvGCStart:
+			gcStartTime = ev.Ts
+		case trace.EvGCDone:
+			for _, g := range gs {
+				if g.EndTime == 0 {
+					g.GCTime += ev.Ts - gcStartTime
+				}
+			}
+		}
+	}
+
+	for _, g := range gs {
+		if g.TotalTime == 0 {
+			g.TotalTime = lastTs - g.CreateTime
+		}
+		if g.EndTime == 0 {
+			g.EndTime = lastTs
+		}
+		if g.blockNetTime != 0 {
+			g.IOTime += lastTs - g.blockNetTime
+			g.blockNetTime = 0
+		}
+		if g.blockSyncTime != 0 {
+			g.BlockTime += lastTs - g.blockSyncTime
+			g.blockSyncTime = 0
+		}
+		if g.blockSyscallTime != 0 {
+			g.SyscallTime += lastTs - g.blockSyscallTime
+			g.blockSyscallTime = 0
+		}
+		if g.blockSchedTime != 0 {
+			g.SchedWaitTime += lastTs - g.blockSchedTime
+			g.blockSchedTime = 0
+		}
+	}
+}
+
+// httpGoroutines serves list of goroutine groups.
+func httpGoroutines(w http.ResponseWriter, r *http.Request) {
+	events, err := parseEvents()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	analyzeGoroutines(events)
+	gss := make(map[uint64]gtype)
+	for _, g := range gs {
+		gs1 := gss[g.PC]
+		gs1.ID = g.PC
+		gs1.Name = g.Name
+		gs1.N++
+		gs1.ExecTime += g.ExecTime
+		gss[g.PC] = gs1
+	}
+	var glist gtypeList
+	for k, v := range gss {
+		v.ID = k
+		glist = append(glist, v)
+	}
+	sort.Sort(glist)
+	templGoroutines.Execute(w, glist)
+}
+
+var templGoroutines = template.Must(template.New("").Parse(`
+<html>
+<body>
+Goroutines: <br>
+{{range $}}
+  <a href="/goroutine?id={{.ID}}">{{.Name}}</a> N={{.N}} <br>
+{{end}}
+</body>
+</html>
+`))
+
+// httpGoroutine serves list of goroutines in a particular group.
+func httpGoroutine(w http.ResponseWriter, r *http.Request) {
+	events, err := parseEvents()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	pc, err := strconv.ParseUint(r.FormValue("id"), 10, 64)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to parse id parameter '%v': %v", r.FormValue("id"), err), http.StatusInternalServerError)
+		return
+	}
+	analyzeGoroutines(events)
+	var glist gdescList
+	for gid, g := range gs {
+		if g.PC != pc || g.ExecTime == 0 {
+			continue
+		}
+		g.ID = gid
+		glist = append(glist, g)
+	}
+	sort.Sort(glist)
+	err = templGoroutine.Execute(w, glist)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to execute template: %v", err), http.StatusInternalServerError)
+		return
+	}
+}
+
+var templGoroutine = template.Must(template.New("").Parse(`
+<html>
+<body>
+<table border="1" sortable="1">
+<tr>
+<th> Goroutine </th>
+<th> Total time, ns </th>
+<th> Execution time, ns </th>
+<th> Network wait time, ns </th>
+<th> Sync block time, ns </th>
+<th> Blocking syscall time, ns </th>
+<th> Scheduler wait time, ns </th>
+<th> GC sweeping time, ns </th>
+<th> GC pause time, ns </th>
+</tr>
+{{range $}}
+  <tr>
+    <td> <a href="/trace?goid={{.ID}}">{{.ID}}</a> </td>
+    <td> {{.TotalTime}} </td>
+    <td> {{.ExecTime}} </td>
+    <td> {{.IOTime}} </td>
+    <td> {{.BlockTime}} </td>
+    <td> {{.SyscallTime}} </td>
+    <td> {{.SchedWaitTime}} </td>
+    <td> {{.SweepTime}} </td>
+    <td> {{.GCTime}} </td>
+  </tr>
+{{end}}
+</table>
+</body>
+</html>
+`))
+
+// relatedGoroutines finds set of related goroutines that we need to include
+// into trace for goroutine goid.
+func relatedGoroutines(events []*trace.Event, goid uint64) map[uint64]bool {
+	// BFS of depth 2 over "unblock" edges
+	// (what goroutines unblock goroutine goid?).
+	gmap := make(map[uint64]bool)
+	gmap[goid] = true
+	for i := 0; i < 2; i++ {
+		gmap1 := make(map[uint64]bool)
+		for g := range gmap {
+			gmap1[g] = true
+		}
+		for _, ev := range events {
+			if ev.Type == trace.EvGoUnblock && gmap[ev.Args[0]] {
+				gmap1[ev.G] = true
+			}
+		}
+		gmap = gmap1
+	}
+	gmap[0] = true // for GC events
+	return gmap
+}

src/cmd/trace/main.go

+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+Trace is a tool for viewing trace files.
+
+Trace files can be generated with:
+	- runtime/pprof.StartTrace
+	- net/http/pprof package
+	- go test -trace
+
+Example usage:
+Generate a trace file with 'go test':
+	go test -trace trace.out pkg
+View the trace in a web browser:
+	go tool trace pkg.test trace.out
+*/
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"internal/trace"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"runtime"
+	"sync"
+)
+
+const usageMessage = "" +
+	`Usage of 'go tool trace':
+Given a trace file produced by 'go test':
+	go test -trace=trace.out pkg
+
+Open a web browser displaying trace:
+	go tool trace [flags] pkg.test trace.out
+
+Flags:
+	-http=addr: HTTP service address (e.g., ':6060')
+`
+
+var (
+	httpFlag = flag.String("http", "localhost:0", "HTTP service address (e.g., ':6060')")
+
+	// The binary file name, left here for serveSVGProfile.
+	programBinary string
+	traceFile     string
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, usageMessage)
+		os.Exit(2)
+	}
+	flag.Parse()
+
+	// Usage information when no arguments.
+	if flag.NArg() != 2 {
+		flag.Usage()
+	}
+	programBinary = flag.Arg(0)
+	traceFile = flag.Arg(1)
+
+	ln, err := net.Listen("tcp", *httpFlag)
+	if err != nil {
+		dief("failed to create server socket: %v\n", err)
+	}
+	// Open browser.
+	if !startBrowser("http://" + ln.Addr().String()) {
+		dief("failed to start browser\n")
+	}
+
+	// Parse and symbolize trace asynchronously while browser opens.
+	go parseEvents()
+
+	// Start http server.
+	http.HandleFunc("/", httpMain)
+	err = http.Serve(ln, nil)
+	dief("failed to start http server: %v\n", err)
+}
+
+var loader struct {
+	once   sync.Once
+	events []*trace.Event
+	err    error
+}
+
+func parseEvents() ([]*trace.Event, error) {
+	loader.once.Do(func() {
+		tracef, err := os.Open(flag.Arg(1))
+		if err != nil {
+			loader.err = fmt.Errorf("failed to open trace file: %v", err)
+			return
+		}
+		defer tracef.Close()
+
+		// Parse and symbolize.
+		events, err := trace.Parse(bufio.NewReader(tracef))
+		if err != nil {
+			loader.err = fmt.Errorf("failed to parse trace: %v", err)
+			return
+		}
+		err = trace.Symbolize(events, programBinary)
+		if err != nil {
+			loader.err = fmt.Errorf("failed to symbolize trace: %v", err)
+			return
+		}
+		loader.events = events
+	})
+	return loader.events, loader.err
+}
+
+// httpMain serves the starting page.
+func httpMain(w http.ResponseWriter, r *http.Request) {
+	w.Write(templMain)
+}
+
+var templMain = []byte(`
+<html>
+<body>
+<a href="/trace">View trace</a><br>
+<a href="/goroutines">Goroutine analysis</a><br>
+<a href="/io">IO blocking profile</a><br>
+<a href="/block">Synchronization blocking profile</a><br>
+<a href="/syscall">Syscall blocking profile</a><br>
+<a href="/sched">Scheduler latency profile</a><br>
+</body>
+</html>
+`)
+
+// startBrowser tries to open the URL in a browser
+// and reports whether it succeeds.
+// Note: copied from x/tools/cmd/cover/html.go
+func startBrowser(url string) bool {
+	// try to start the browser
+	var args []string
+	switch runtime.GOOS {
+	case "darwin":
+		args = []string{"open"}
+	case "windows":
+		args = []string{"cmd", "/c", "start"}
+	default:
+		args = []string{"xdg-open"}
+	}
+	cmd := exec.Command(args[0], append(args[1:], url)...)
+	return cmd.Start() == nil
+}
+
+func dief(msg string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, msg, args...)
+	os.Exit(1)
+}

src/cmd/trace/pprof.go

+// Copyright 2014 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Serving of pprof-like profiles.
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"internal/trace"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+)
+
+func init() {
+	http.HandleFunc("/io", httpIO)
+	http.HandleFunc("/block", httpBlock)
+	http.HandleFunc("/syscall", httpSyscall)
+	http.HandleFunc("/sched", httpSched)
+}
+
+// Record represents one entry in pprof-like profiles.
+type Record struct {
+	stk  []*trace.Frame
+	n    uint64
+	time int64
+}
+
+// httpIO serves IO pprof-like profile (time spent in IO wait).
+func httpIO(w http.ResponseWriter, r *http.Request) {
+	events, err := parseEvents()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	prof := make(map[uint64]Record)
+	for _, ev := range events {
+		if ev.Type != trace.EvGoBlockNet || ev.Link == nil || ev.StkID == 0 || len(ev.Stk) == 0 {
+			continue
+		}
+		rec := prof[ev.StkID]
+		rec.stk = ev.Stk
+		rec.n++
+		rec.time += ev.Link.Ts - ev.Ts
+		prof[ev.StkID] = rec
+	}
+	serveSVGProfile(w, r, prof)
+}
+
+// httpBlock serves blocking pprof-like profile (time spent blocked on synchronization primitives).
+func httpBlock(w http.ResponseWriter, r *http.Request) {
+	events, err := parseEvents()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	prof := make(map[uint64]Record)
+	for _, ev := range events {
+		switch ev.Type {
+		case trace.EvGoBlockSend, trace.EvGoBlockRecv, trace.EvGoBlockSelect,
+			trace.EvGoBlockSync, trace.EvGoBlockCond:
+		default:
+			continue
+		}
+		if ev.Link == nil || ev.StkID == 0 || len(ev.Stk) == 0 {
+			continue
+		}
+		rec := prof[ev.StkID]
+		rec.stk = ev.Stk
+		rec.n++
+		rec.time += ev.Link.Ts - ev.Ts
+		prof[ev.StkID] = rec
+	}
+	serveSVGProfile(w, r, prof)
+}
+
+// httpSyscall serves syscall pprof-like profile (time spent blocked in syscalls).
+func httpSyscall(w http.ResponseWriter, r *http.Request) {
+	events, err := parseEvents()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	prof := make(map[uint64]Record)
+	for _, ev := range events {
+		if ev.Type != trace.EvGoSysCall || ev.Link == nil || ev.StkID == 0 || len(ev.Stk) == 0 {
+			continue
+		}
+		rec := prof[ev.StkID]
+		rec.stk = ev.Stk
+		rec.n++
+		rec.time += ev.Link.Ts - ev.Ts
+		prof[ev.StkID] = rec
+	}
+	serveSVGProfile(w, r, prof)
+}
+
+// httpSched serves scheduler latency pprof-like profile
+// (time between a goroutine become runnable and actually scheduled for execution).
+func httpSched(w http.ResponseWriter, r *http.Request) {
+	events, err := parseEvents()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	prof := make(map[uint64]Record)
+	for _, ev := range events {
+		if (ev.Type != trace.EvGoUnblock && ev.Type != trace.EvGoCreate) ||
+			ev.Link == nil || ev.StkID == 0 || len(ev.Stk) == 0 {
+			continue
+		}
+		rec := prof[ev.StkID]
+		rec.stk = ev.Stk
+		rec.n++
+		rec.time += ev.Link.Ts - ev.Ts
+		prof[ev.StkID] = rec
+	}
+	serveSVGProfile(w, r, prof)
+}
+
+// generateSVGProfile generates pprof-like profile stored in prof and writes in to w.
+func serveSVGProfile(w http.ResponseWriter, r *http.Request, prof map[uint64]Record) {
+	blockf, err := ioutil.TempFile("", "block")
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to create temp file: %v", err), http.StatusInternalServerError)
+		return
+	}
+	defer os.Remove(blockf.Name())
+	blockb := bufio.NewWriter(blockf)
+	fmt.Fprintf(blockb, "--- contention:\ncycles/second=1000000000\n")
+	for _, rec := range prof {
+		fmt.Fprintf(blockb, "%v %v @", rec.time, rec.n)
+		for _, f := range rec.stk {
+			fmt.Fprintf(blockb, " 0x%x", f.PC)
+		}
+		fmt.Fprintf(blockb, "\n")
+	}
+	err = blockb.Flush()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to flush temp file: %v", err), http.StatusInternalServerError)
+		return
+	}
+	err = blockf.Close()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to close temp file: %v", err), http.StatusInternalServerError)
+		return
+	}
+
+	svgFilename := blockf.Name() + ".svg"
+	_, err = exec.Command("go", "tool", "pprof", "-svg", "-output", svgFilename, programBinary, blockf.Name()).CombinedOutput()
+	if err != nil {
+		http.Error(w, fmt.Sprintf("failed to execute go tool pprof: %v", err), http.StatusInternalServerError)
+		return
+	}
+	defer os.Remove(svgFilename)
+	w.Header().Set("Content-Type", "image/svg+xml")
+	http.ServeFile(w, r, svgFilename)
+}

src/runtime/pprof/trace_test.go

@@ -6,6 +6,7 @@ package pprof_test
 
 import (
 	"bytes"
+	"internal/trace"
 	"net"
 	"os"
 	"runtime"
@@ -66,7 +67,7 @@ func TestTrace(t *testing.T) {
 		t.Fatalf("failed to start tracing: %v", err)
 	}
 	StopTrace()
-	_, err := parseTrace(buf)
+	_, err := trace.Parse(buf)
 	if err != nil {
 		t.Fatalf("failed to parse trace: %v", err)
 	}
@@ -198,12 +199,153 @@ func TestTraceStress(t *testing.T) {
 	runtime.GOMAXPROCS(procs)
 
 	StopTrace()
-	_, err = parseTrace(buf)
+	_, err = trace.Parse(buf)
 	if err != nil {
 		t.Fatalf("failed to parse trace: %v", err)
 	}
 }
 
+// Do a bunch of various stuff (timers, GC, network, etc) in a separate goroutine.
+// And concurrently with all that start/stop trace 3 times.
+func TestTraceStressStartStop(t *testing.T) {
+	skipTraceTestsIfNeeded(t)
+
+	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(8))
+	outerDone := make(chan bool)
+
+	go func() {
+		defer func() {
+			outerDone <- true
+		}()
+
+		var wg sync.WaitGroup
+		done := make(chan bool)
+
+		wg.Add(1)
+		go func() {
+			<-done
+			wg.Done()
+		}()
+
+		rp, wp, err := os.Pipe()
+		if err != nil {
+			t.Fatalf("failed to create pipe: %v", err)
+		}
+		defer func() {
+			rp.Close()
+			wp.Close()
+		}()
+		wg.Add(1)
+		go func() {
+			var tmp [1]byte
+			rp.Read(tmp[:])
+			<-done
+			wg.Done()
+		}()
+		time.Sleep(time.Millisecond)
+
+		go func() {
+			runtime.LockOSThread()
+			for {
+				select {
+				case <-done:
+					return
+				default:
+					runtime.Gosched()
+				}
+			}
+		}()
+
+		runtime.GC()
+		// Trigger GC from malloc.
+		for i := 0; i < 1e3; i++ {
+			_ = make([]byte, 1<<20)
+		}
+
+		// Create a bunch of busy goroutines to load all Ps.
+		for p := 0; p < 10; p++ {
+			wg.Add(1)
+			go func() {
+				// Do something useful.
+				tmp := make([]byte, 1<<16)
+				for i := range tmp {
+					tmp[i]++
+				}
+				_ = tmp
+				<-done
+				wg.Done()
+			}()
+		}
+
+		// Block in syscall.
+		wg.Add(1)
+		go func() {
+			var tmp [1]byte
+			rp.Read(tmp[:])
+			<-done
+			wg.Done()
+		}()
+
+		runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))
+
+		// Test timers.
+		timerDone := make(chan bool)
+		go func() {
+			time.Sleep(time.Millisecond)
+			timerDone <- true
+		}()
+		<-timerDone
+
+		// A bit of network.
+		ln, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			t.Fatalf("listen failed: %v", err)
+		}
+		defer ln.Close()
+		go func() {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			time.Sleep(time.Millisecond)
+			var buf [1]byte
+			c.Write(buf[:])
+			c.Close()
+		}()
+		c, err := net.Dial("tcp", ln.Addr().String())
+		if err != nil {
+			t.Fatalf("dial failed: %v", err)
+		}
+		var tmp [1]byte
+		c.Read(tmp[:])
+		c.Close()
+
+		go func() {
+			runtime.Gosched()
+			select {}
+		}()
+
+		// Unblock helper goroutines and wait them to finish.
+		wp.Write(tmp[:])
+		wp.Write(tmp[:])
+		close(done)
+		wg.Wait()
+	}()
+
+	for i := 0; i < 3; i++ {
+		buf := new(bytes.Buffer)
+		if err := StartTrace(buf); err != nil {
+			t.Fatalf("failed to start tracing: %v", err)
+		}
+		time.Sleep(time.Millisecond)
+		StopTrace()
+		if _, err := trace.Parse(buf); err != nil {
+			t.Fatalf("failed to parse trace: %v", err)
+		}
+	}
+	<-outerDone
+}
+
 func TestTraceSymbolize(t *testing.T) {
 	skipTraceTestsIfNeeded(t)
 	if runtime.GOOS == "nacl" {
@@ -215,24 +357,24 @@ func TestTraceSymbolize(t *testing.T) {
 	}
 	runtime.GC()
 	StopTrace()
-	events, err := parseTrace(buf)
+	events, err := trace.Parse(buf)
 	if err != nil {
 		t.Fatalf("failed to parse trace: %v", err)
 	}
-	err = symbolizeTrace(events, os.Args[0])
+	err = trace.Symbolize(events, os.Args[0])
 	if err != nil {
 		t.Fatalf("failed to symbolize trace: %v", err)
 	}
 	found := false
 eventLoop:
 	for _, ev := range events {
-		if ev.typ != traceEvGCStart {
+		if ev.Type != trace.EvGCStart {
 			continue
 		}
-		for _, f := range ev.stk {
-			if strings.HasSuffix(f.file, "trace_test.go") &&
-				strings.HasSuffix(f.fn, "pprof_test.TestTraceSymbolize") &&
-				f.line == 216 {
+		for _, f := range ev.Stk {
+			if strings.HasSuffix(f.File, "trace_test.go") &&
+				strings.HasSuffix(f.Fn, "pprof_test.TestTraceSymbolize") &&
+				f.Line == 358 {
 				found = true
 				break eventLoop
 			}

src/runtime/trace.go

@@ -190,7 +190,6 @@ func StopTrace() {
 	}
 
 	traceGoSched()
-	traceGoStart()
 
 	for _, p := range &allp {
 		if p == nil {