cmd/godoc: use go/build to determine package and example files

Also:
- faster code for example extraction
- simplify handling of command documentation:
  all "main" packages are treated as commands
- various minor cleanups along the way

For commands written in Go, any doc.go file containing
documentation must now be part of package main (rather
then package documentation), otherwise the documentation
won't show up in godoc (it will still build, though).

For commands written in C, documentation may still be
in doc.go files defining package documentation, but the
recommended way is to explicitly ignore those files with
a +build ignore constraint to define package main.

Fixes #4806.

R=adg, rsc, dave, bradfitz
CC=golang-dev
https://golang.org/cl/7333046

lib/godoc/package.html

@@ -10,7 +10,7 @@
 	correspond to Go identifiers).
 -->
 {{with .PDoc}}
-	{{if $.IsPkg}}
+	{{if not $.IsMain}}
 		<div id="short-nav">
 			<dl>
 			<dd><code>import "{{html .ImportPath}}"</code></dd>

lib/godoc/package.txt

@@ -2,7 +2,7 @@
 
 ---------------------------------------
 
-*/}}{{with .PDoc}}{{if $.IsPkg}}PACKAGE
+*/}}{{with .PDoc}}{{if not $.IsMain}}PACKAGE
 
 package {{.Name}}
     import "{{.ImportPath}}"

misc/dashboard/builder/doc.go

@@ -55,4 +55,4 @@ If the Google Code credentials are not provided the archival step
 will be skipped.
 
 */
-package documentation
+package main

misc/goplay/doc.go

@@ -20,4 +20,4 @@
 // security mechanisms. Do not deploy it in untrusted environments.
 // By default, goplay listens only on localhost. This can be overridden with
 // the -http parameter. Do so at your own risk.
-package documentation
+package main

src/cmd/5a/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 5a is a version of the Plan 9 assembler.  The original is documented at
@@ -11,4 +13,4 @@
 Its target architecture is the ARM, referred to by these tools as arm.
 
 */
-package documentation
+package main

src/cmd/5c/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 5c is a version of the Plan 9 C compiler.  The original is documented at
@@ -11,4 +13,4 @@
 Its target architecture is the ARM, referred to by these tools as arm.
 
 */
-package documentation
+package main

src/cmd/5g/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 5g is the version of the gc compiler for the ARM.
@@ -10,4 +12,4 @@ The $GOARCH for these tools is arm.
 It reads .go files and outputs .5 files. The flags are documented in ../gc/doc.go.
 
 */
-package documentation
+package main

src/cmd/5l/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 5l is the linker for the ARM.
@@ -10,4 +12,4 @@ The $GOARCH for these tools is arm.
 The flags are documented in ../ld/doc.go.
 
 */
-package documentation
+package main

src/cmd/6a/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 6a is a version of the Plan 9 assembler.  The original is documented at
@@ -11,4 +13,4 @@
 Its target architecture is the x86-64, referred to by these tools as amd64.
 
 */
-package documentation
+package main

src/cmd/6c/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 6c is a version of the Plan 9 C compiler.  The original is documented at
@@ -11,4 +13,4 @@
 Its target architecture is the x86-64, referred to by these tools as amd64.
 
 */
-package documentation
+package main

src/cmd/6g/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 6g is the version of the gc compiler for the x86-64.
@@ -10,4 +12,4 @@ The $GOARCH for these tools is amd64.
 It reads .go files and outputs .6 files. The flags are documented in ../gc/doc.go.
 
 */
-package documentation
+package main

src/cmd/6l/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 6l is the linker for the x86-64.
@@ -10,4 +12,4 @@ The $GOARCH for these tools is amd64.
 The flags are documented in ../ld/doc.go.
 
 */
-package documentation
+package main

src/cmd/8a/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 8a is a version of the Plan 9 assembler.  The original is documented at
@@ -11,4 +13,4 @@
 Its target architecture is the x86, referred to by these tools for historical reasons as 386.
 
 */
-package documentation
+package main

src/cmd/8c/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 8c is a version of the Plan 9 C compiler.  The original is documented at
@@ -11,4 +13,4 @@
 Its target architecture is the x86, referred to by these tools for historical reasons as 386.
 
 */
-package documentation
+package main

src/cmd/8g/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 8g is the version of the gc compiler for the x86.
@@ -10,4 +12,4 @@ The $GOARCH for these tools is 386.
 It reads .go files and outputs .8 files. The flags are documented in ../gc/doc.go.
 
 */
-package documentation
+package main

src/cmd/8l/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 8l is the linker for the 32-bit x86.
@@ -10,4 +12,4 @@ The $GOARCH for these tools is 386.
 The flags are documented in ../ld/doc.go.
 
 */
-package documentation
+package main

src/cmd/cc/doc.go

@@ -2,10 +2,12 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 This directory contains the portable section of the Plan 9 C compilers.
 See ../6c, ../8c, and ../5c for more information.
 
 */
-package documentation
+package main

src/cmd/cgo/doc.go

@@ -133,4 +133,4 @@ Cgo does not yet work with gccgo.
 See "C? Go? Cgo!" for an introduction to using cgo:
 http://golang.org/doc/articles/c_go_cgo.html
 */
-package documentation
+package main

src/cmd/cov/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 Cov is a rudimentary code coverage tool.
@@ -31,4 +33,4 @@ The options are:
 The program is the same for all architectures: 386, amd64, and arm.
 
 */
-package documentation
+package main

src/cmd/fix/doc.go

@@ -33,4 +33,4 @@ Fix does not make backup copies of the files that it edits.
 Instead, use a version control system's ``diff'' functionality to inspect
 the changes that fix makes before committing them.
 */
-package documentation
+package main

src/cmd/gc/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 Gc is the generic label for the family of Go compilers
@@ -86,4 +88,4 @@ in Go) does not allow any of the pointers passed as arguments to escape into the
 heap or into the values returned from the function. This information can be used as
 during the compiler's escape analysis of Go code calling the function.
 */
-package documentation
+package main

src/cmd/go/doc.go

@@ -812,6 +812,6 @@ See the documentation of the testing package for more information.
 
 
 */
-package documentation
+package main
 
 // NOTE: cmdDoc is in fmt.go.

src/cmd/go/main.go

@@ -200,7 +200,7 @@ var documentationTemplate = `// Copyright 2011 The Go Authors.  All rights reser
 
 
 {{end}}*/
-package documentation
+package main
 
 // NOTE: cmdDoc is in fmt.go.
 `

src/cmd/godoc/dirtrees.go

@@ -74,7 +74,7 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i
 	// determine number of subdirectories and if there are package files
 	ndirs := 0
 	hasPkgFiles := false
-	var synopses [4]string // prioritized package documentation (0 == highest priority)
+	var synopses [3]string // prioritized package documentation (0 == highest priority)
 	for _, d := range list {
 		switch {
 		case isPkgDir(d):
@@ -95,12 +95,10 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i
 						switch file.Name.Name {
 						case name:
 							i = 0 // normal case: directory name matches package name
-						case fakePkgName:
-							i = 1 // synopses for commands
 						case "main":
-							i = 2 // directory contains a main package
+							i = 1 // directory contains a main package
 						default:
-							i = 3 // none of the above
+							i = 2 // none of the above
 						}
 						if 0 <= i && i < len(synopses) && synopses[i] == "" {
 							synopses[i] = doc.Synopsis(file.Doc.Text())

src/cmd/godoc/doc.go

@@ -127,4 +127,4 @@ See "Godoc: documenting Go code" for how to write good comments for godoc:
 http://golang.org/doc/articles/godoc_documenting_go_code.html
 
 */
-package documentation
+package main

src/cmd/godoc/parser.go

@@ -2,10 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// This file contains support functions for parsing .go files.
-// Similar functionality is found in package go/parser but the
-// functions here operate using godoc's file system fs instead
-// of calling the OS's file operations directly.
+// This file contains support functions for parsing .go files
+// accessed via godoc's file system fs.
 
 package main
 
@@ -13,7 +11,6 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
-	"os"
 	pathpkg "path"
 )
 
@@ -25,44 +22,16 @@ func parseFile(fset *token.FileSet, filename string, mode parser.Mode) (*ast.Fil
 	return parser.ParseFile(fset, filename, src, mode)
 }
 
-func parseFiles(fset *token.FileSet, filenames []string) (pkgs map[string]*ast.Package, first error) {
-	pkgs = make(map[string]*ast.Package)
-	for _, filename := range filenames {
-		file, err := parseFile(fset, filename, parser.ParseComments)
+func parseFiles(fset *token.FileSet, abspath string, localnames []string) (map[string]*ast.File, error) {
+	files := make(map[string]*ast.File)
+	for _, f := range localnames {
+		absname := pathpkg.Join(abspath, f)
+		file, err := parseFile(fset, absname, parser.ParseComments)
 		if err != nil {
-			if first == nil {
-				first = err
-			}
-			continue
-		}
-
-		name := file.Name.Name
-		pkg, found := pkgs[name]
-		if !found {
-			// TODO(gri) Use NewPackage here; reconsider ParseFiles API.
-			pkg = &ast.Package{Name: name, Files: make(map[string]*ast.File)}
-			pkgs[name] = pkg
-		}
-		pkg.Files[filename] = file
-	}
-	return
-}
-
-func parseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool) (map[string]*ast.Package, error) {
-	list, err := fs.ReadDir(path)
-	if err != nil {
-		return nil, err
-	}
-
-	filenames := make([]string, len(list))
-	i := 0
-	for _, d := range list {
-		if filter == nil || filter(d) {
-			filenames[i] = pathpkg.Join(path, d.Name())
-			i++
+			return nil, err
 		}
+		files[absname] = file
 	}
-	filenames = filenames[0:i]
 
-	return parseFiles(fset, filenames)
+	return files, nil
 }

src/cmd/gofmt/doc.go

@@ -72,6 +72,6 @@ To convert the package tree from explicit slice upper bounds to implicit ones:
 
 	gofmt -r 'α[β:len(α)] -> α[β:]' -w $GOROOT/src/pkg
 */
-package documentation
+package main
 
 // BUG(rsc): The implementation of -r is a bit slow.

src/cmd/ld/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 Ld is the portable code for a modified version of the Plan 9 linker.  The original is documented at
@@ -70,4 +72,4 @@ Options new in this version:
 		calls, not false positives caused by dead temporaries stored in
 		the current function call.
 */
-package documentation
+package main

src/cmd/nm/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 Nm is a version of the Plan 9 nm command.  The original is documented at
@@ -18,4 +20,4 @@ Usage:
 	go tool nm [-aghnsTu] file
 
 */
-package documentation
+package main

src/cmd/pack/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 Pack is a variant of the Plan 9 ar tool.  The original is documented at
@@ -24,4 +26,4 @@ The new option 'P' causes pack to remove the given prefix
 from file names in the line number information in object files
 that are already stored in or added to the archive.
 */
-package documentation
+package main

src/cmd/prof/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build ignore
+
 /*
 
 Prof is a rudimentary real-time profiler.
@@ -44,4 +46,4 @@ every 100ms until the program completes.
 It is installed as go tool prof and is architecture-independent.
 
 */
-package documentation
+package main

src/cmd/vet/doc.go

@@ -67,4 +67,4 @@ The other flags are:
 			-printfuncs=Warn:1,Warnf:1
 
 */
-package documentation
+package main

src/cmd/yacc/doc.go

@@ -48,4 +48,4 @@ referenced by yacc's generated code.  Setting it to distinct values
 allows multiple grammars to be placed in a single package.
 
 */
-package documentation
+package main

src/pkg/exp/ebnflint/doc.go

@@ -19,4 +19,4 @@ The --start flag specifies the name of the start production for
 the grammar; it defaults to "Start".
 
 */
-package documentation
+package main

src/pkg/exp/gotype/doc.go

@@ -58,6 +58,6 @@ To verify the output of a pipe:
 	echo "package foo" | gotype
 
 */
-package documentation
+package main
 
 // BUG(gri): At the moment, only single-file scope analysis is performed.