cmd/trace: generate jsontrace data in a streaming fashion

Update #21870

The Sys went down to 4.25G from 6.2G.

$ DEBUG_MEMORY_USAGE=1 go tool trace trace.out
2018/03/07 08:49:01 Parsing trace...
after parsing trace
 Alloc:	3385757184 Bytes
 Sys:	3661195896 Bytes
 HeapReleased:	0 Bytes
 HeapSys:	3488841728 Bytes
 HeapInUse:	3426516992 Bytes
 HeapAlloc:	3385757184 Bytes
Enter to continue...
2018/03/07 08:49:18 Splitting trace...
after spliting trace
 Alloc:	2352071904 Bytes
 Sys:	4243825464 Bytes
 HeapReleased:	0 Bytes
 HeapSys:	4025712640 Bytes
 HeapInUse:	2377703424 Bytes
 HeapAlloc:	2352071904 Bytes
Enter to continue...
after httpJsonTrace
 Alloc:	3228697832 Bytes
 Sys:	4250379064 Bytes
 HeapReleased:	0 Bytes
 HeapSys:	4025647104 Bytes
 HeapInUse:	3260014592 Bytes
 HeapAlloc:	3228697832 Bytes

Change-Id: I546f26bdbc68b1e58f1af1235a0e299dc0ff115e
Reviewed-on: https://go-review.googlesource.com/92375
Run-TryBot: Hyang-Ah Hana Kim <hyangah@gmail.com>
Reviewed-by: Peter Weinberger <pjw@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/cmd/trace/main.go

@@ -120,19 +120,8 @@ func main() {
 	}
 	reportMemoryUsage("after parsing trace")
 
-	log.Print("Serializing trace...")
-	params := &traceParams{
-		parsed:  res,
-		endTime: int64(1<<63 - 1),
-	}
-	data, err := generateTrace(params)
-	if err != nil {
-		dief("%v\n", err)
-	}
-	reportMemoryUsage("after generating trace")
-
 	log.Print("Splitting trace...")
-	ranges = splitTrace(data)
+	ranges = splitTrace(res)
 	reportMemoryUsage("after spliting trace")
 
 	addr := "http://" + ln.Addr().String()

src/cmd/trace/trace_test.go

@@ -6,6 +6,7 @@ package main
 
 import (
 	"internal/trace"
+	"io/ioutil"
 	"strings"
 	"testing"
 )
@@ -68,12 +69,10 @@ func TestGoroutineCount(t *testing.T) {
 		endTime: int64(1<<63 - 1),
 	}
 
-	// If the counts drop below 0, generateTrace will return an error.
-	viewerData, err := generateTrace(params)
-	if err != nil {
-		t.Fatalf("generateTrace failed: %v", err)
-	}
-	for _, ev := range viewerData.Events {
+	// Use the default viewerDataTraceConsumer but replace
+	// consumeViewerEvent to intercept the ViewerEvents for testing.
+	c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1)
+	c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) {
 		if ev.Name == "Goroutines" {
 			cnt := ev.Arg.(*goroutineCountersArg)
 			if cnt.Runnable+cnt.Running > 2 {
@@ -82,6 +81,11 @@ func TestGoroutineCount(t *testing.T) {
 			t.Logf("read %+v %+v", ev, cnt)
 		}
 	}
+
+	// If the counts drop below 0, generateTrace will return an error.
+	if err := generateTrace(params, c); err != nil {
+		t.Fatalf("generateTrace failed: %v", err)
+	}
 }
 
 func TestGoroutineFilter(t *testing.T) {
@@ -120,8 +124,8 @@ func TestGoroutineFilter(t *testing.T) {
 		gs:      map[uint64]bool{10: true},
 	}
 
-	_, err = generateTrace(params)
-	if err != nil {
+	c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1)
+	if err := generateTrace(params, c); err != nil {
 		t.Fatalf("generateTrace failed: %v", err)
 	}
 }
@@ -152,17 +156,18 @@ func TestPreemptedMarkAssist(t *testing.T) {
 		endTime: int64(1<<63 - 1),
 	}
 
-	viewerData, err := generateTrace(params)
-	if err != nil {
-		t.Fatalf("generateTrace failed: %v", err)
-	}
+	c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1)
 
 	marks := 0
-	for _, ev := range viewerData.Events {
+	c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) {
 		if strings.Contains(ev.Name, "MARK ASSIST") {
 			marks++
 		}
 	}
+	if err := generateTrace(params, c); err != nil {
+		t.Fatalf("generateTrace failed: %v", err)
+	}
+
 	if marks != 2 {
 		t.Errorf("Got %v MARK ASSIST events, want %v", marks, 2)
 	}

src/cmd/trace/trace_unix_test.go

@@ -9,6 +9,7 @@ package main
 import (
 	"bytes"
 	"internal/trace"
+	"io/ioutil"
 	"runtime"
 	rtrace "runtime/trace"
 	"sync"
@@ -79,14 +80,8 @@ func TestGoroutineInSyscall(t *testing.T) {
 
 	// Check only one thread for the pipe read goroutine is
 	// considered in-syscall.
-	viewerData, err := generateTrace(&traceParams{
-		parsed:  res,
-		endTime: int64(1<<63 - 1),
-	})
-	if err != nil {
-		t.Fatalf("failed to generate ViewerData: %v", err)
-	}
-	for _, ev := range viewerData.Events {
+	c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1)
+	c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) {
 		if ev.Name == "Threads" {
 			arg := ev.Arg.(*threadCountersArg)
 			if arg.InSyscall > 1 {
@@ -94,4 +89,12 @@ func TestGoroutineInSyscall(t *testing.T) {
 			}
 		}
 	}
+
+	param := &traceParams{
+		parsed:  res,
+		endTime: int64(1<<63 - 1),
+	}
+	if err := generateTrace(param, c); err != nil {
+		t.Fatalf("failed to generate ViewerData: %v", err)
+	}
 }