go/printer: implement SourcePos mode

If a printer is configured with the SourcePos mode
set, it will emit //-line comments as necessary to
ensure that the result - if reparsed - reflects the
original source position information.

This change required a bit of reworking of the
output section in printer.go. Specifically:

- Introduced new Config mode 'SourcePos'.

- Introduced new position 'out' which tracks the
position of the generated output if it were read
in again. If there is a discrepancy between out
and the current AST/source position, a //line
comment is emitted to correct for it.

- Lazy emission of indentation so that //line
comments can be placed correctly. As a result,
the trimmer will have to do less work.

- Merged writeItem into writeString.

- Merged writeByteN into writeByte.

- Use a []byte instead of a byte.Buffer both in the
printer and in the trimmer (eliminates dependency).

Also: introduced explicit printer.Mode type (in
sync w/ parser.Mode, scanner.Mode, etc.)

Runs all tests. Applied gofmt to src, misc w/o changes.

Fixes #1047.
Fixes #2697.

R=rsc, rsc
CC=golang-dev
https://golang.org/cl/5643066

doc/go1.html

@@ -1049,6 +1049,15 @@ The <code>Duration</code> flag is new and affects no existing code.
 Several packages under <code>go</code> have slightly revised APIs.
 </p>
 
+<p>
+A concrete <code>Mode</code> type was introduced for configuration mode flags
+in the packages
+<a href="/pkg/go/scanner/"><code>go/scanner</code></a>,
+<a href="/pkg/go/parser/"><code>go/parser</code></a>,
+<a href="/pkg/go/printer/"><code>go/printer</code></a>, and
+<a href="/pkg/go/doc/"><code>go/doc</code></a>.
+</p>
+
 <p>
 The modes <code>AllowIllegalChars</code> and <code>InsertSemis</code> have been removed
 from the <a href="/pkg/go/scanner/"><code>go/scanner</code></a> package. They were mostly
@@ -1075,6 +1084,17 @@ convenience functions <a href="/pkg/go/parser/#ParseDir"><code>ParseDir</code></
 and <a href="/pkg/go/parser/#ParseExpr"><code>ParseExpr</code></a>.
 </p>
 
+<p>
+The <a href="/pkg/go/printer/"><code>go/printer</code></a> package supports an additional
+configuration mode <a href="/pkg/go/printer/#Mode"><code>SourcePos</code></a>;
+if set, the printer will emit <code>//line</code> comments such that the generated
+output contains the original source code position information. The new type
+<a href="/pkg/go/printer/#CommentedNode"><code>CommentedNode</code></a> can be
+used to provide comments associated with an arbitrary
+<a href="/pkg/go/ast/#Node"><code>ast.Node</code></a> (until now only
+<a href="/pkg/go/ast/#File"><code>ast.File</code></a> carried comment information).
+</p>
+
 <p>
 The type names of the <a href="/pkg/go/doc/"><code>go/doc</code></a> package have been
 streamlined by removing the <code>Doc</code> suffix: <code>PackageDoc</code>

doc/go1.tmpl

@@ -952,6 +952,15 @@ The <code>Duration</code> flag is new and affects no existing code.
 Several packages under <code>go</code> have slightly revised APIs.
 </p>
 
+<p>
+A concrete <code>Mode</code> type was introduced for configuration mode flags
+in the packages
+<a href="/pkg/go/scanner/"><code>go/scanner</code></a>,
+<a href="/pkg/go/parser/"><code>go/parser</code></a>,
+<a href="/pkg/go/printer/"><code>go/printer</code></a>, and
+<a href="/pkg/go/doc/"><code>go/doc</code></a>.
+</p>
+
 <p>
 The modes <code>AllowIllegalChars</code> and <code>InsertSemis</code> have been removed
 from the <a href="/pkg/go/scanner/"><code>go/scanner</code></a> package. They were mostly
@@ -978,6 +987,17 @@ convenience functions <a href="/pkg/go/parser/#ParseDir"><code>ParseDir</code></
 and <a href="/pkg/go/parser/#ParseExpr"><code>ParseExpr</code></a>.
 </p>
 
+<p>
+The <a href="/pkg/go/printer/"><code>go/printer</code></a> package supports an additional
+configuration mode <a href="/pkg/go/printer/#Mode"><code>SourcePos</code></a>;
+if set, the printer will emit <code>//line</code> comments such that the generated
+output contains the original source code position information. The new type
+<a href="/pkg/go/printer/#CommentedNode"><code>CommentedNode</code></a> can be
+used to provide comments associated with an arbitrary
+<a href="/pkg/go/ast/#Node"><code>ast.Node</code></a> (until now only
+<a href="/pkg/go/ast/#File"><code>ast.File</code></a> carried comment information).
+</p>
+
 <p>
 The type names of the <a href="/pkg/go/doc/"><code>go/doc</code></a> package have been
 streamlined by removing the <code>Doc</code> suffix: <code>PackageDoc</code>

src/cmd/cgo/godefs.go

@@ -109,7 +109,7 @@ func (p *Package) godefs(f *File, srcfile string) string {
 		}
 	}
 
-	printer.Fprint(&buf, fset, f.AST)
+	conf.Fprint(&buf, fset, f.AST)
 
 	return buf.String()
 }

src/cmd/cgo/out.go

@@ -17,6 +17,8 @@ import (
 	"strings"
 )
 
+var conf = printer.Config{Mode: printer.SourcePos, Tabwidth: 8}
+
 // writeDefs creates output files to be compiled by 6g, 6c, and gcc.
 // (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.)
 func (p *Package) writeDefs() {
@@ -57,7 +59,7 @@ func (p *Package) writeDefs() {
 
 	for name, def := range typedef {
 		fmt.Fprintf(fgo2, "type %s ", name)
-		printer.Fprint(fgo2, fset, def)
+		conf.Fprint(fgo2, fset, def)
 		fmt.Fprintf(fgo2, "\n\n")
 	}
 	fmt.Fprintf(fgo2, "type _Ctype_void [0]byte\n")
@@ -87,7 +89,7 @@ func (p *Package) writeDefs() {
 		fmt.Fprintf(fc, "\n")
 
 		fmt.Fprintf(fgo2, "var %s ", n.Mangle)
-		printer.Fprint(fgo2, fset, &ast.StarExpr{X: n.Type.Go})
+		conf.Fprint(fgo2, fset, &ast.StarExpr{X: n.Type.Go})
 		fmt.Fprintf(fgo2, "\n")
 	}
 	fmt.Fprintf(fc, "\n")
@@ -255,7 +257,7 @@ func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name) {
 		Name: ast.NewIdent(n.Mangle),
 		Type: gtype,
 	}
-	printer.Fprint(fgo2, fset, d)
+	conf.Fprint(fgo2, fset, d)
 	if *gccgo {
 		fmt.Fprintf(fgo2, " __asm__(\"%s\")\n", n.C)
 	} else {
@@ -327,8 +329,7 @@ func (p *Package) writeOutput(f *File, srcfile string) {
 
 	// Write Go output: Go input with rewrites of C.xxx to _C_xxx.
 	fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n\n")
-	fmt.Fprintf(fgo1, "//line %s:1\n", srcfile)
-	printer.Fprint(fgo1, fset, f.AST)
+	conf.Fprint(fgo1, fset, f.AST)
 
 	// While we process the vars and funcs, also write 6c and gcc output.
 	// Gcc output starts with the preamble.
@@ -542,11 +543,11 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) {
 		// a Go wrapper function.
 		if fn.Recv != nil {
 			fmt.Fprintf(fgo2, "func %s(recv ", goname)
-			printer.Fprint(fgo2, fset, fn.Recv.List[0].Type)
+			conf.Fprint(fgo2, fset, fn.Recv.List[0].Type)
 			forFieldList(fntype.Params,
 				func(i int, atype ast.Expr) {
 					fmt.Fprintf(fgo2, ", p%d ", i)
-					printer.Fprint(fgo2, fset, atype)
+					conf.Fprint(fgo2, fset, atype)
 				})
 			fmt.Fprintf(fgo2, ")")
 			if gccResult != "void" {
@@ -556,7 +557,7 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) {
 						if i > 0 {
 							fmt.Fprint(fgo2, ", ")
 						}
-						printer.Fprint(fgo2, fset, atype)
+						conf.Fprint(fgo2, fset, atype)
 					})
 				fmt.Fprint(fgo2, ")")
 			}

src/cmd/gofmt/gofmt.go

@@ -45,7 +45,7 @@ var (
 	exitCode    = 0
 	rewrite     func(*ast.File) *ast.File
 	parserMode  parser.Mode
-	printerMode uint
+	printerMode printer.Mode
 )
 
 func report(err error) {

src/pkg/go/printer/printer_test.go

@@ -223,7 +223,8 @@ func TestBadNodes(t *testing.T) {
 	}
 }
 
-// Print and parse f with 
+// testComment verifies that f can be parsed again after printing it
+// with its first comment set to comment at any possible source offset.
 func testComment(t *testing.T, f *ast.File, srclen int, comment *ast.Comment) {
 	f.Comments[0].List[0] = comment
 	var buf bytes.Buffer
@@ -280,3 +281,128 @@ func fibo(n int) {
 	testComment(t, f, len(src), &ast.Comment{pos, "/*-style \n comment */"})
 	testComment(t, f, len(src), &ast.Comment{pos, "/*-style comment \n\n\n */"})
 }
+
+type visitor chan *ast.Ident
+
+func (v visitor) Visit(n ast.Node) (w ast.Visitor) {
+	if ident, ok := n.(*ast.Ident); ok {
+		v <- ident
+	}
+	return v
+}
+
+// idents is an iterator that returns all idents in f via the result channel.
+func idents(f *ast.File) <-chan *ast.Ident {
+	v := make(visitor)
+	go func() {
+		ast.Walk(v, f)
+		close(v)
+	}()
+	return v
+}
+
+// identCount returns the number of identifiers found in f.
+func identCount(f *ast.File) int {
+	n := 0
+	for _ = range idents(f) {
+		n++
+	}
+	return n
+}
+
+// Verify that the SourcePos mode emits correct //line comments
+// by testing that position information for matching identifiers
+// is maintained.
+func TestSourcePos(t *testing.T) {
+	const src = `
+package p
+import ( "go/printer"; "math" )
+const pi = 3.14; var x = 0
+type t struct{ x, y, z int; u, v, w float32 }
+func (t *t) foo(a, b, c int) int {
+	return a*t.x + b*t.y +
+		// two extra lines here
+		// ...
+		c*t.z
+}
+`
+
+	// parse original
+	f1, err := parser.ParseFile(fset, "src", src, parser.ParseComments)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// pretty-print original
+	var buf bytes.Buffer
+	err = (&Config{Mode: UseSpaces | SourcePos, Tabwidth: 8}).Fprint(&buf, fset, f1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// parse pretty printed original
+	// (//line comments must be interpreted even w/o parser.ParseComments set)
+	f2, err := parser.ParseFile(fset, "", buf.Bytes(), 0)
+	if err != nil {
+		t.Fatalf("%s\n%s", err, buf.Bytes())
+	}
+
+	// At this point the position information of identifiers in f2 should
+	// match the position information of corresponding identifiers in f1.
+
+	// number of identifiers must be > 0 (test should run) and must match
+	n1 := identCount(f1)
+	n2 := identCount(f2)
+	if n1 == 0 {
+		t.Fatal("got no idents")
+	}
+	if n2 != n1 {
+		t.Errorf("got %d idents; want %d", n2, n1)
+	}
+
+	// verify that all identifiers have correct line information
+	i2range := idents(f2)
+	for i1 := range idents(f1) {
+		i2 := <-i2range
+
+		if i2.Name != i1.Name {
+			t.Errorf("got ident %s; want %s", i2.Name, i1.Name)
+		}
+
+		l1 := fset.Position(i1.Pos()).Line
+		l2 := fset.Position(i2.Pos()).Line
+		if l2 != l1 {
+			t.Errorf("got line %d; want %d for %s", l2, l1, i1.Name)
+		}
+	}
+
+	if t.Failed() {
+		t.Logf("\n%s", buf.Bytes())
+	}
+}
+
+// TextX is a skeleton test that can be filled in for debugging one-off cases.
+// Do not remove.
+func TestX(t *testing.T) {
+	const src = `
+package p
+func _() {}
+`
+	// parse original
+	f, err := parser.ParseFile(fset, "src", src, parser.ParseComments)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// pretty-print original
+	var buf bytes.Buffer
+	if err = (&Config{Mode: UseSpaces, Tabwidth: 8}).Fprint(&buf, fset, f); err != nil {
+		t.Fatal(err)
+	}
+
+	// parse pretty printed original
+	if _, err := parser.ParseFile(fset, "", buf.Bytes(), 0); err != nil {
+		t.Fatalf("%s\n%s", err, buf.Bytes())
+	}
+
+}