Commit 3e2dc457 authored by Russ Cox's avatar Russ Cox

cmd/test2json: go tool test2json converts test output to JSON

Also add cmd/internal/test2json, the actual implementation,
which will be called directly from cmd/go in addition to being
a standalone command (like cmd/buildid and cmd/internal/buildid).

For #2981.

Change-Id: I244ce36d665f424bbf13f5ae00ece10b705d367d
Reviewed-on: https://go-review.googlesource.com/76872
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: 's avatarBrad Fitzpatrick <bradfitz@golang.org>
parent 7badae85
This diff is collapsed.
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package test2json
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"path/filepath"
"reflect"
"strings"
"testing"
"unicode/utf8"
)
var update = flag.Bool("update", false, "rewrite testdata/*.json files")
func TestGolden(t *testing.T) {
files, err := filepath.Glob("testdata/*.test")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
name := strings.TrimSuffix(filepath.Base(file), ".test")
t.Run(name, func(t *testing.T) {
orig, err := ioutil.ReadFile(file)
if err != nil {
t.Fatal(err)
}
// Test one line written to c at a time.
// Assume that's the most likely to be handled correctly.
var buf bytes.Buffer
c := NewConverter(&buf, "", 0)
in := append([]byte{}, orig...)
for _, line := range bytes.SplitAfter(in, []byte("\n")) {
writeAndKill(c, line)
}
c.Close()
if *update {
js := strings.TrimSuffix(file, ".test") + ".json"
t.Logf("rewriting %s", js)
if err := ioutil.WriteFile(js, buf.Bytes(), 0666); err != nil {
t.Fatal(err)
}
return
}
want, err := ioutil.ReadFile(strings.TrimSuffix(file, ".test") + ".json")
if err != nil {
t.Fatal(err)
}
diffJSON(t, buf.Bytes(), want)
if t.Failed() {
// If the line-at-a-time conversion fails, no point testing boundary conditions.
return
}
// Write entire input in bulk.
t.Run("bulk", func(t *testing.T) {
buf.Reset()
c = NewConverter(&buf, "", 0)
in = append([]byte{}, orig...)
writeAndKill(c, in)
c.Close()
diffJSON(t, buf.Bytes(), want)
})
// Write 2 bytes at a time on even boundaries.
t.Run("even2", func(t *testing.T) {
buf.Reset()
c = NewConverter(&buf, "", 0)
in = append([]byte{}, orig...)
for i := 0; i < len(in); i += 2 {
if i+2 <= len(in) {
writeAndKill(c, in[i:i+2])
} else {
writeAndKill(c, in[i:])
}
}
c.Close()
diffJSON(t, buf.Bytes(), want)
})
// Write 2 bytes at a time on odd boundaries.
t.Run("odd2", func(t *testing.T) {
buf.Reset()
c = NewConverter(&buf, "", 0)
in = append([]byte{}, orig...)
if len(in) > 0 {
writeAndKill(c, in[:1])
}
for i := 1; i < len(in); i += 2 {
if i+2 <= len(in) {
writeAndKill(c, in[i:i+2])
} else {
writeAndKill(c, in[i:])
}
}
c.Close()
diffJSON(t, buf.Bytes(), want)
})
// Test with very small output buffers, to check that
// UTF8 sequences are not broken up.
for b := 5; b <= 8; b++ {
t.Run(fmt.Sprintf("tiny%d", b), func(t *testing.T) {
oldIn := inBuffer
oldOut := outBuffer
defer func() {
inBuffer = oldIn
outBuffer = oldOut
}()
inBuffer = 64
outBuffer = b
buf.Reset()
c = NewConverter(&buf, "", 0)
in = append([]byte{}, orig...)
writeAndKill(c, in)
c.Close()
diffJSON(t, buf.Bytes(), want)
})
}
})
}
}
// writeAndKill writes b to w and then fills b with Zs.
// The filling makes sure that if w is holding onto b for
// future use, that future use will have obviously wrong data.
func writeAndKill(w io.Writer, b []byte) {
w.Write(b)
for i := range b {
b[i] = 'Z'
}
}
// diffJSON diffs the stream we have against the stream we want
// and fails the test with a useful message if they don't match.
func diffJSON(t *testing.T, have, want []byte) {
t.Helper()
type event map[string]interface{}
// Parse into events, one per line.
parseEvents := func(b []byte) ([]event, []string) {
t.Helper()
var events []event
var lines []string
for _, line := range bytes.SplitAfter(b, []byte("\n")) {
if len(line) > 0 {
line = bytes.TrimSpace(line)
var e event
err := json.Unmarshal(line, &e)
if err != nil {
t.Errorf("unmarshal %s: %v", b, err)
continue
}
events = append(events, e)
lines = append(lines, string(line))
}
}
return events, lines
}
haveEvents, haveLines := parseEvents(have)
wantEvents, wantLines := parseEvents(want)
if t.Failed() {
return
}
// Make sure the events we have match the events we want.
// At each step we're matching haveEvents[i] against wantEvents[j].
// i and j can move independently due to choices about exactly
// how to break up text in "output" events.
i := 0
j := 0
// Fail reports a failure at the current i,j and stops the test.
// It shows the events around the current positions,
// with the current positions marked.
fail := func() {
var buf bytes.Buffer
show := func(i int, lines []string) {
for k := -2; k < 5; k++ {
marker := ""
if k == 0 {
marker = "» "
}
if 0 <= i+k && i+k < len(lines) {
fmt.Fprintf(&buf, "\t%s%s\n", marker, lines[i+k])
}
}
if i >= len(lines) {
// show marker after end of input
fmt.Fprintf(&buf, "\t» \n")
}
}
fmt.Fprintf(&buf, "have:\n")
show(i, haveLines)
fmt.Fprintf(&buf, "want:\n")
show(j, wantLines)
t.Fatal(buf.String())
}
var outputTest string // current "Test" key in "output" events
var wantOutput, haveOutput string // collected "Output" of those events
// getTest returns the "Test" setting, or "" if it is missing.
getTest := func(e event) string {
s, _ := e["Test"].(string)
return s
}
// checkOutput collects output from the haveEvents for the current outputTest
// and then checks that the collected output matches the wanted output.
checkOutput := func() {
for i < len(haveEvents) && haveEvents[i]["Action"] == "output" && getTest(haveEvents[i]) == outputTest {
haveOutput += haveEvents[i]["Output"].(string)
i++
}
if haveOutput != wantOutput {
t.Errorf("output mismatch for Test=%q:\nhave %q\nwant %q", outputTest, haveOutput, wantOutput)
fail()
}
haveOutput = ""
wantOutput = ""
}
// Walk through wantEvents matching against haveEvents.
for j = range wantEvents {
e := wantEvents[j]
if e["Action"] == "output" && getTest(e) == outputTest {
wantOutput += e["Output"].(string)
continue
}
checkOutput()
if e["Action"] == "output" {
outputTest = getTest(e)
wantOutput += e["Output"].(string)
continue
}
if i >= len(haveEvents) {
t.Errorf("early end of event stream: missing event")
fail()
}
if !reflect.DeepEqual(haveEvents[i], e) {
t.Errorf("events out of sync")
fail()
}
i++
}
checkOutput()
if i < len(haveEvents) {
t.Errorf("extra events in stream")
fail()
}
}
func TestTrimUTF8(t *testing.T) {
s := "hello α ☺ 😂 world" // α is 2-byte, ☺ is 3-byte, 😂 is 4-byte
b := []byte(s)
for i := 0; i < len(s); i++ {
j := trimUTF8(b[:i])
u := string([]rune(s[:j])) + string([]rune(s[j:]))
if u != s {
t.Errorf("trimUTF8(%q) = %d (-%d), not at boundary (split: %q %q)", s[:i], j, i-j, s[:j], s[j:])
}
if utf8.FullRune(b[j:i]) {
t.Errorf("trimUTF8(%q) = %d (-%d), too early (missed: %q)", s[:j], j, i-j, s[j:i])
}
}
}
{"Action":"run","Test":"TestAscii"}
{"Action":"output","Test":"TestAscii","Output":"=== RUN TestAscii\n"}
{"Action":"output","Test":"TestAscii","Output":"I can eat glass, and it doesn't hurt me. I can eat glass, and it doesn't hurt me.\n"}
{"Action":"output","Test":"TestAscii","Output":"I CAN EAT GLASS, AND IT DOESN'T HURT ME. I CAN EAT GLASS, AND IT DOESN'T HURT ME.\n"}
{"Action":"output","Test":"TestAscii","Output":"--- PASS: TestAscii\n"}
{"Action":"output","Test":"TestAscii","Output":" i can eat glass, and it doesn't hurt me. i can eat glass, and it doesn't hurt me.\n"}
{"Action":"output","Test":"TestAscii","Output":" V PNA RNG TYNFF, NAQ VG QBRFA'G UHEG ZR. V PNA RNG TYNFF, NAQ VG QBRFA'G UHEG ZR.\n"}
{"Action":"pass","Test":"TestAscii"}
{"Action":"output","Output":"PASS\n"}
{"Action":"pass"}
=== RUN TestAscii
I can eat glass, and it doesn't hurt me. I can eat glass, and it doesn't hurt me.
I CAN EAT GLASS, AND IT DOESN'T HURT ME. I CAN EAT GLASS, AND IT DOESN'T HURT ME.
--- PASS: TestAscii
i can eat glass, and it doesn't hurt me. i can eat glass, and it doesn't hurt me.
V PNA RNG TYNFF, NAQ VG QBRFA'G UHEG ZR. V PNA RNG TYNFF, NAQ VG QBRFA'G UHEG ZR.
PASS
This diff is collapsed.
=== RUN Test☺☹
=== PAUSE Test☺☹
=== RUN Test☺☹Asm
=== PAUSE Test☺☹Asm
=== RUN Test☺☹Dirs
=== PAUSE Test☺☹Dirs
=== RUN TestTags
=== PAUSE TestTags
=== RUN Test☺☹Verbose
=== PAUSE Test☺☹Verbose
=== CONT Test☺☹
=== CONT TestTags
=== CONT Test☺☹Verbose
=== RUN TestTags/testtag
=== PAUSE TestTags/testtag
=== CONT Test☺☹Dirs
=== CONT Test☺☹Asm
=== RUN Test☺☹/0
=== PAUSE Test☺☹/0
=== RUN Test☺☹/1
=== PAUSE Test☺☹/1
=== RUN Test☺☹/2
=== PAUSE Test☺☹/2
=== RUN Test☺☹/3
=== PAUSE Test☺☹/3
=== RUN Test☺☹/4
=== RUN TestTags/x_testtag_y
=== PAUSE Test☺☹/4
=== RUN Test☺☹/5
=== PAUSE Test☺☹/5
=== PAUSE TestTags/x_testtag_y
=== RUN Test☺☹/6
=== RUN TestTags/x,testtag,y
=== PAUSE TestTags/x,testtag,y
=== RUN Test☺☹Dirs/testingpkg
=== PAUSE Test☺☹/6
=== CONT TestTags/x,testtag,y
=== PAUSE Test☺☹Dirs/testingpkg
=== RUN Test☺☹Dirs/divergent
=== RUN Test☺☹/7
=== PAUSE Test☺☹/7
=== PAUSE Test☺☹Dirs/divergent
=== CONT TestTags/x_testtag_y
=== CONT TestTags/testtag
=== RUN Test☺☹Dirs/buildtag
=== PAUSE Test☺☹Dirs/buildtag
=== CONT Test☺☹/0
=== CONT Test☺☹/4
=== RUN Test☺☹Dirs/incomplete
=== PAUSE Test☺☹Dirs/incomplete
=== RUN Test☺☹Dirs/cgo
=== PAUSE Test☺☹Dirs/cgo
=== CONT Test☺☹/7
=== CONT Test☺☹/6
--- PASS: Test☺☹Verbose (0.04s)
=== CONT Test☺☹/5
=== CONT Test☺☹/3
=== CONT Test☺☹/2
--- PASS: TestTags (0.00s)
--- PASS: TestTags/x_testtag_y (0.04s)
vet_test.go:187: -tags=x testtag y
--- PASS: TestTags/x,testtag,y (0.04s)
vet_test.go:187: -tags=x,testtag,y
--- PASS: TestTags/testtag (0.04s)
vet_test.go:187: -tags=testtag
=== CONT Test☺☹/1
=== CONT Test☺☹Dirs/testingpkg
=== CONT Test☺☹Dirs/buildtag
=== CONT Test☺☹Dirs/divergent
=== CONT Test☺☹Dirs/incomplete
=== CONT Test☺☹Dirs/cgo
--- PASS: Test☺☹ (0.39s)
--- PASS: Test☺☹/5 (0.07s)
vet_test.go:114: φιλεσ: ["testdata/copylock_func.go" "testdata/rangeloop.go"]
--- PASS: Test☺☹/3 (0.07s)
vet_test.go:114: φιλεσ: ["testdata/composite.go" "testdata/nilfunc.go"]
--- PASS: Test☺☹/6 (0.07s)
vet_test.go:114: φιλεσ: ["testdata/copylock_range.go" "testdata/shadow.go"]
--- PASS: Test☺☹/2 (0.07s)
vet_test.go:114: φιλεσ: ["testdata/bool.go" "testdata/method.go" "testdata/unused.go"]
--- PASS: Test☺☹/0 (0.13s)
vet_test.go:114: φιλεσ: ["testdata/assign.go" "testdata/httpresponse.go" "testdata/structtag.go"]
--- PASS: Test☺☹/4 (0.16s)
vet_test.go:114: φιλεσ: ["testdata/copylock.go" "testdata/print.go"]
--- PASS: Test☺☹/1 (0.07s)
vet_test.go:114: φιλεσ: ["testdata/atomic.go" "testdata/lostcancel.go" "testdata/unsafeptr.go"]
--- PASS: Test☺☹/7 (0.19s)
vet_test.go:114: φιλεσ: ["testdata/deadcode.go" "testdata/shift.go"]
--- PASS: Test☺☹Dirs (0.01s)
--- PASS: Test☺☹Dirs/testingpkg (0.06s)
--- PASS: Test☺☹Dirs/divergent (0.05s)
--- PASS: Test☺☹Dirs/buildtag (0.06s)
--- PASS: Test☺☹Dirs/incomplete (0.05s)
--- PASS: Test☺☹Dirs/cgo (0.04s)
--- PASS: Test☺☹Asm (0.75s)
PASS
ok cmd/vet (cached)
{"Action":"run","Test":"TestUnicode"}
{"Action":"output","Test":"TestUnicode","Output":"=== RUN TestUnicode\n"}
{"Action":"output","Test":"TestUnicode","Output":"Μπορώ να φάω σπασμένα γυαλιά χωρίς να πάθω τίποτα. Μπορώ να φάω σπασμένα γυαλιά χωρίς να πάθω τίποτα.\n"}
{"Action":"output","Test":"TestUnicode","Output":"私はガラスを食べられます。それは私を傷つけません。私はガラスを食べられます。それは私を傷つけません。\n"}
{"Action":"output","Test":"TestUnicode","Output":"--- PASS: TestUnicode\n"}
{"Action":"output","Test":"TestUnicode","Output":" ฉันกินกระจกได้ แต่มันไม่ทำให้ฉันเจ็บ ฉันกินกระจกได้ แต่มันไม่ทำให้ฉันเจ็บ\n"}
{"Action":"output","Test":"TestUnicode","Output":" אני יכול לאכול זכוכית וזה לא מזיק לי. אני יכול לאכול זכוכית וזה לא מזיק לי.\n"}
{"Action":"pass","Test":"TestUnicode"}
{"Action":"output","Output":"PASS\n"}
{"Action":"pass"}
=== RUN TestUnicode
Μπορώ να φάω σπασμένα γυαλιά χωρίς να πάθω τίποτα. Μπορώ να φάω σπασμένα γυαλιά χωρίς να πάθω τίποτα.
私はガラスを食べられますそれは私を傷つけません私はガラスを食べられますそれは私を傷つけません
--- PASS: TestUnicode
นกนกระจกได แตนไมทำใหนเจ นกนกระจกได แตนไมทำใหนเจ
אני יכול לאכול זכוכית וזה לא מזיק לי. אני יכול לאכול זכוכית וזה לא מזיק לי.
PASS
This diff is collapsed.
=== RUN TestVet
=== PAUSE TestVet
=== RUN TestVetAsm
=== PAUSE TestVetAsm
=== RUN TestVetDirs
=== PAUSE TestVetDirs
=== RUN TestTags
=== PAUSE TestTags
=== RUN TestVetVerbose
=== PAUSE TestVetVerbose
=== CONT TestVet
=== CONT TestTags
=== CONT TestVetVerbose
=== RUN TestTags/testtag
=== PAUSE TestTags/testtag
=== CONT TestVetDirs
=== CONT TestVetAsm
=== RUN TestVet/0
=== PAUSE TestVet/0
=== RUN TestVet/1
=== PAUSE TestVet/1
=== RUN TestVet/2
=== PAUSE TestVet/2
=== RUN TestVet/3
=== PAUSE TestVet/3
=== RUN TestVet/4
=== RUN TestTags/x_testtag_y
=== PAUSE TestVet/4
=== RUN TestVet/5
=== PAUSE TestVet/5
=== PAUSE TestTags/x_testtag_y
=== RUN TestVet/6
=== RUN TestTags/x,testtag,y
=== PAUSE TestTags/x,testtag,y
=== RUN TestVetDirs/testingpkg
=== PAUSE TestVet/6
=== CONT TestTags/x,testtag,y
=== PAUSE TestVetDirs/testingpkg
=== RUN TestVetDirs/divergent
=== RUN TestVet/7
=== PAUSE TestVet/7
=== PAUSE TestVetDirs/divergent
=== CONT TestTags/x_testtag_y
=== CONT TestTags/testtag
=== RUN TestVetDirs/buildtag
=== PAUSE TestVetDirs/buildtag
=== CONT TestVet/0
=== CONT TestVet/4
=== RUN TestVetDirs/incomplete
=== PAUSE TestVetDirs/incomplete
=== RUN TestVetDirs/cgo
=== PAUSE TestVetDirs/cgo
=== CONT TestVet/7
=== CONT TestVet/6
--- PASS: TestVetVerbose (0.04s)
=== CONT TestVet/5
=== CONT TestVet/3
=== CONT TestVet/2
--- PASS: TestTags (0.00s)
--- PASS: TestTags/x_testtag_y (0.04s)
vet_test.go:187: -tags=x testtag y
--- PASS: TestTags/x,testtag,y (0.04s)
vet_test.go:187: -tags=x,testtag,y
--- PASS: TestTags/testtag (0.04s)
vet_test.go:187: -tags=testtag
=== CONT TestVet/1
=== CONT TestVetDirs/testingpkg
=== CONT TestVetDirs/buildtag
=== CONT TestVetDirs/divergent
=== CONT TestVetDirs/incomplete
=== CONT TestVetDirs/cgo
--- PASS: TestVet (0.39s)
--- PASS: TestVet/5 (0.07s)
vet_test.go:114: files: ["testdata/copylock_func.go" "testdata/rangeloop.go"]
--- PASS: TestVet/3 (0.07s)
vet_test.go:114: files: ["testdata/composite.go" "testdata/nilfunc.go"]
--- PASS: TestVet/6 (0.07s)
vet_test.go:114: files: ["testdata/copylock_range.go" "testdata/shadow.go"]
--- PASS: TestVet/2 (0.07s)
vet_test.go:114: files: ["testdata/bool.go" "testdata/method.go" "testdata/unused.go"]
--- PASS: TestVet/0 (0.13s)
vet_test.go:114: files: ["testdata/assign.go" "testdata/httpresponse.go" "testdata/structtag.go"]
--- PASS: TestVet/4 (0.16s)
vet_test.go:114: files: ["testdata/copylock.go" "testdata/print.go"]
--- PASS: TestVet/1 (0.07s)
vet_test.go:114: files: ["testdata/atomic.go" "testdata/lostcancel.go" "testdata/unsafeptr.go"]
--- PASS: TestVet/7 (0.19s)
vet_test.go:114: files: ["testdata/deadcode.go" "testdata/shift.go"]
--- PASS: TestVetDirs (0.01s)
--- PASS: TestVetDirs/testingpkg (0.06s)
--- PASS: TestVetDirs/divergent (0.05s)
--- PASS: TestVetDirs/buildtag (0.06s)
--- PASS: TestVetDirs/incomplete (0.05s)
--- PASS: TestVetDirs/cgo (0.04s)
--- PASS: TestVetAsm (0.75s)
PASS
ok cmd/vet (cached)
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Test2json converts go test output to a machine-readable JSON stream.
//
// Usage:
//
// go test ... | go tool test2json [-p pkg] [-t]
// ./test.out 2>&1 | go tool test2json [-p pkg] [-t]
//
// Test2json expects to read go test output from standard input.
// It writes a corresponding stream of JSON events to standard output.
// There is no unnecessary input or output buffering, so that
// the JSON stream can be read for “live updates” of test status.
//
// The -p flag sets the package reported in each test event.
//
// The -t flag requests that time stamps be added to each test event.
//
// Output Format
//
// The JSON stream is a newline-separated sequence of TestEvent objects
// corresponding to the Go struct:
//
// type TestEvent struct {
// Time time.Time
// Event string
// Package string
// Test string
// Elapsed float64 // seconds
// Output string
// }
//
// The Time field holds the time the event happened.
// It is conventionally omitted for cached test results.
//
// The Event field is one of a fixed set of event descriptions:
//
// run - the test has started running
// pause - the test has been paused
// cont - the test has continued running
// pass - the test passed
// fail - the test failed
// output - the test printed output
//
// The Package field, if present, specifies the package being tested.
// When the go command runs parallel tests in -json mode, events from
// different tests are interlaced; the Package field allows readers to
// separate them.
//
// The Test field, if present, specifies the test or example, or benchmark
// function that caused the event. Events for the overall package test
// do not set Test.
//
// The Elapsed field is set for "pass" and "fail" events. It gives the time
// elapsed for the specific test or the overall package test that passed or failed.
//
// The Output field is set for Event == "output" and is a portion of the test's output
// (standard output and standard error merged together). The output is
// unmodified except that invalid UTF-8 output from a test is coerced
// into valid UTF-8 by use of replacement characters. With that one exception,
// the concatenation of the Output fields of all output events is the exact
// output of the test execution.
//
package main
import (
"flag"
"fmt"
"io"
"os"
"cmd/internal/test2json"
)
var (
flagP = flag.String("p", "", "report `pkg` as the package being tested in each event")
flagT = flag.Bool("t", false, "include timestamps in events")
)
func usage() {
fmt.Fprintf(os.Stderr, "usage: go test ... | go tool test2json [-p pkg] [-t]\n")
os.Exit(2)
}
func main() {
flag.Usage = usage
flag.Parse()
if flag.NArg() > 0 {
usage()
}
var mode test2json.Mode
if *flagT {
mode |= test2json.Timestamp
}
c := test2json.NewConverter(os.Stdout, *flagP, mode)
defer c.Close()
io.Copy(c, os.Stdin)
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment