test: expand run.go's errorcheck, make clear which bugs run

Today, if run.go doesn't understand a test header line it just ignores
the test, making it too easy to write or edit tests that are not actually
being run.

- expand errorcheck to accept flags, so that bounds.go and escape*.go can run.
- create a whitelist of skippable tests in run.go; skipping others is an error.
- mark all skipped tests at top of file.

Update #4139.

R=golang-dev, bradfitz
CC=golang-dev
https://golang.org/cl/6549054

test/args.go

 // $G $F.go && $L $F.$A && ./$A.out arg1 arg2
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/bounds.go

-// errchk -0 $G -m -l $D/$F.go
+// errorcheck -0 -m -l
 
 // Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style

test/bugs/bug395.go

 // echo bug395 is broken  # takes 90+ seconds to break
 // # $G $D/$F.go || echo bug395
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/bugs/bug434.go

 // $G $D/$F.dir/one.go && $G $D/$F.dir/two.go || echo BUG:bug434
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/ddd3.go

 // $G $D/ddd2.go && $G $D/$F.go && $L $F.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2010 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/dwarf/main.go

 // $G $D/$F.go $D/z*.go && $L $F.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/escape2.go

-// errchk -0 $G -m -l $D/$F.go
+// errorcheck -0 -m -l
 
 // Copyright 2010 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style

test/escape4.go

-// errchk -0 $G -m $D/$F.go
+// errorcheck -0 -m
 
 // Copyright 2010 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style

test/fixedbugs/bug083.go

 // $G $D/$F.dir/bug0.go && errchk $G $D/$F.dir/bug1.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug133.go

 // $G $D/$F.dir/bug0.go && $G $D/$F.dir/bug1.go && errchk $G $D/$F.dir/bug2.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug160.go

 // $G $D/bug160.dir/x.go && $G $D/bug160.dir/y.go && $L y.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug191.go

 // $G $D/bug191.dir/a.go && $G $D/bug191.dir/b.go && $G $D/$F.go && $L $F.$A
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug248.go

@@ -5,6 +5,9 @@
 // $L bug2.$A &&
 // ./$A.out || echo BUG: failed to compile
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug302.go

 // $G $D/bug302.dir/p.go && pack grc pp.a p.$A && $G $D/bug302.dir/main.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2010 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug313.go

 // errchk $G -e $D/$F.dir/[ab].go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2010 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug322.go

 // $G $D/$F.dir/lib.go && $G $D/$F.dir/main.go && $L main.$A && ./$A.out || echo BUG: fails incorrectly
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug324.go

 // $G $D/$F.dir/p.go && $G $D/$F.dir/main.go && $L main.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug345.go

 // $G $D/$F.dir/io.go && errchk $G -e $D/$F.dir/main.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug367.go

 // $G $D/$F.dir/p.go && $G $D/$F.dir/main.go && $L main.$A && ./$A.out || echo BUG: should not fail
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug369.go

@@ -2,6 +2,9 @@
 // $G -o fast.$A $D/bug369.dir/pkg.go &&
 // run
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug382.go

 // $G $D/$F.dir/pkg.go && $G $D/$F.go || echo "Bug 382"
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file

test/fixedbugs/bug385_32.go

 // [ $A == 6 ] || errchk $G -e $D/$F.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug385_64.go

 // [ $A != 6 ]  || errchk $G -e $D/$F.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug414.go

 // $G $D/$F.dir/p1.go && $G $D/$F.dir/main.go && $L main.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug424.go

 // $G $D/$F.dir/lib.go && $G $D/$F.go && $L $F.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug429.go

 // $G $D/$F.go && $L $F.$A && ! ./$A.out || echo BUG: bug429
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/fixedbugs/bug437.go

 // $G $D/$F.dir/one.go && $G $D/$F.dir/two.go && $G $D/$F.go && $L $F.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/import3.go

 // $G $D/import2.go && $G $D/$F.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2010 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/import4.go

 // $G $D/empty.go && errchk $G $D/$F.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/index.go

@@ -4,6 +4,9 @@
 // ./$A.out -pass 2 >tmp.go && errchk $G -e tmp.go
 // rm -f tmp.go $A.out1
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2010 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/interface/embed1.go

 // $G $D/embed0.go && $G $D/$F.go && $L $F.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/interface/private.go

 // $G $D/${F}1.go && errchk $G $D/$F.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/interface/recursive2.go

 // $G $D/recursive1.go && $G $D/$F.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/linkx.go

 // $G $D/$F.go && $L -X main.tbd hello $F.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/method4.go

 // $G $D/method4a.go && $G $D/$F.go && $L $F.$A && ./$A.out
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2012 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/nul1.go

@@ -3,6 +3,9 @@
 // errchk $G -e tmp.go
 // rm -f tmp.go
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/rotate.go

@@ -2,6 +2,9 @@
 // ./$A.out >tmp.go && $G tmp.go && $L -o $A.out1 tmp.$A && ./$A.out1
 // rm -f tmp.go $A.out1
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/run.go

@@ -20,6 +20,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -109,15 +110,17 @@ func main() {
 		<-test.donec
 		_, isSkip := test.err.(skipError)
 		errStr := "pass"
-		if isSkip {
-			errStr = "skip"
-		}
 		if test.err != nil {
 			errStr = test.err.Error()
 			if !isSkip {
 				failed = true
 			}
 		}
+		if isSkip && !skipOkay[path.Join(test.dir, test.gofile)] {
+			errStr = "unexpected skip for " + path.Join(test.dir, test.gofile) + ": " + errStr
+			isSkip = false
+			failed = true
+		}
 		resCount[errStr]++
 		if isSkip && !*verbose && !*showSkips {
 			continue
@@ -251,7 +254,8 @@ func (t *test) run() {
 		action = action[2:]
 	}
 
-	var args []string
+	var args, flags []string
+	wantError := false
 	f := strings.Fields(action)
 	if len(f) > 0 {
 		action = f[0]
@@ -262,8 +266,19 @@ func (t *test) run() {
 	case "cmpout":
 		action = "run" // the run case already looks for <dir>/<test>.out files
 		fallthrough
-	case "compile", "compiledir", "build", "run", "errorcheck", "runoutput":
+	case "compile", "compiledir", "build", "run", "runoutput":
+		t.action = action
+	case "errorcheck":
 		t.action = action
+		wantError = true
+		for len(args) > 0 && strings.HasPrefix(args[0], "-") {
+			if args[0] == "-0" {
+				wantError = false
+			} else {
+				flags = append(flags, args[0])
+			}
+			args = args[1:]
+		}
 	case "skip":
 		t.action = "skip"
 		return
@@ -302,7 +317,21 @@ func (t *test) run() {
 		t.err = fmt.Errorf("unimplemented action %q", action)
 
 	case "errorcheck":
-		out, _ := runcmd("go", "tool", gc, "-e", "-o", "a."+letter, long)
+		cmdline := []string{"go", "tool", gc, "-e", "-o", "a." + letter}
+		cmdline = append(cmdline, flags...)
+		cmdline = append(cmdline, long)
+		out, err := runcmd(cmdline...)
+		if wantError {
+			if err == nil {
+				t.err = fmt.Errorf("compilation succeeded unexpectedly\n%s", out)
+				return
+			}
+		} else {
+			if err != nil {
+				t.err = fmt.Errorf("%s\n%s", err, out)
+				return
+			}
+		}
 		t.err = t.errorCheck(string(out), long, t.gofile)
 		return
 
@@ -400,7 +429,7 @@ func (t *test) errorCheck(outStr string, full, short string) (err error) {
 	// 6g error messages continue onto additional lines with leading tabs.
 	// Split the output at the beginning of each line that doesn't begin with a tab.
 	for _, line := range strings.Split(outStr, "\n") {
-		if strings.HasSuffix(line, "\r") {	// remove '\r', output by compiler on windows
+		if strings.HasSuffix(line, "\r") { // remove '\r', output by compiler on windows
 			line = line[:len(line)-1]
 		}
 		if strings.HasPrefix(line, "\t") {
@@ -517,3 +546,62 @@ func (t *test) wantedErrors() (errs []wantedError) {
 
 	return
 }
+
+var skipOkay = map[string]bool{
+	"args.go":                 true,
+	"ddd3.go":                 true,
+	"import3.go":              true,
+	"import4.go":              true,
+	"index.go":                true,
+	"linkx.go":                true,
+	"method4.go":              true,
+	"nul1.go":                 true,
+	"rotate.go":               true,
+	"sigchld.go":              true,
+	"sinit.go":                true,
+	"interface/embed1.go":     true,
+	"interface/private.go":    true,
+	"interface/recursive2.go": true,
+	"dwarf/main.go":           true,
+	"dwarf/z1.go":             true,
+	"dwarf/z10.go":            true,
+	"dwarf/z11.go":            true,
+	"dwarf/z12.go":            true,
+	"dwarf/z13.go":            true,
+	"dwarf/z14.go":            true,
+	"dwarf/z15.go":            true,
+	"dwarf/z16.go":            true,
+	"dwarf/z17.go":            true,
+	"dwarf/z18.go":            true,
+	"dwarf/z19.go":            true,
+	"dwarf/z2.go":             true,
+	"dwarf/z20.go":            true,
+	"dwarf/z3.go":             true,
+	"dwarf/z4.go":             true,
+	"dwarf/z5.go":             true,
+	"dwarf/z6.go":             true,
+	"dwarf/z7.go":             true,
+	"dwarf/z8.go":             true,
+	"dwarf/z9.go":             true,
+	"fixedbugs/bug083.go":     true,
+	"fixedbugs/bug133.go":     true,
+	"fixedbugs/bug160.go":     true,
+	"fixedbugs/bug191.go":     true,
+	"fixedbugs/bug248.go":     true,
+	"fixedbugs/bug302.go":     true,
+	"fixedbugs/bug313.go":     true,
+	"fixedbugs/bug322.go":     true,
+	"fixedbugs/bug324.go":     true,
+	"fixedbugs/bug345.go":     true,
+	"fixedbugs/bug367.go":     true,
+	"fixedbugs/bug369.go":     true,
+	"fixedbugs/bug382.go":     true,
+	"fixedbugs/bug385_32.go":  true,
+	"fixedbugs/bug385_64.go":  true,
+	"fixedbugs/bug414.go":     true,
+	"fixedbugs/bug424.go":     true,
+	"fixedbugs/bug429.go":     true,
+	"fixedbugs/bug437.go":     true,
+	"bugs/bug395.go":          true,
+	"bugs/bug434.go":          true,
+}

test/sigchld.go

 // [ "$GOOS" == windows ] ||
 // ($G $D/$F.go && $L $F.$A && ./$A.out 2>&1 | cmp - $D/$F.out)
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/sinit.go

 // $G -S $D/$F.go | egrep initdone >/dev/null && echo BUG sinit || true
 
+// NOTE: This test is not run by 'run.go' and so not run by all.bash.
+// To run this test you must use the ./run shell script.
+
 // Copyright 2010 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.

test/testlib

@@ -48,7 +48,12 @@ cmpout() {
 }
 
 errorcheck() {
-	errchk $G -e $D/$F.go
+	zero=""
+	if [ "$1" = "-0" ]; then
+		zero="-0"
+		shift
+	fi
+	errchk $zero $G -e $* $D/$F.go
 }
 
 skip() {