go/doc: clean rewrite of go/doc internals

The implementation is divided into 4 phases:
1) export filtering of an incoming AST if necessary (exports.go)
2) reading of a possibly filtered AST (reader.go: type reader)
3) method set computation (reader.go)
4) sorting and creation of final documentation (reader.go)

In contrast to the old implementation, the presentation data
(Names, Docs, Decls, etc.) are created immediately upon reading
the respective AST node. Also, all types are collected (embedded
or not) in a uniform way.

Once the entire AST has been processed, all methods and types
have been collected and the method sets for each type can be
computed (phase 3).

To produce the final documentation, the method sets and value
maps are sorted.

There are no API changes. Passes the existing test suite unchanged.

R=rsc, rogpeppe
CC=golang-dev
https://golang.org/cl/5554044

src/pkg/go/doc/doc.go

@@ -7,7 +7,7 @@ package doc
 
 import (
 	"go/ast"
-	"sort"
+	"go/token"
 )
 
 // Package is the documentation for an entire package.
@@ -35,11 +35,12 @@ type Value struct {
 	order int
 }
 
+// Method is the documentation for a method declaration.
 type Method struct {
 	*Func
 	// TODO(gri) The following fields are not set at the moment. 
 	Origin *Type // original receiver base type
-	Level  int   // embedding level; 0 means Func is not embedded
+	Level  int   // embedding level; 0 means Method is not embedded
 }
 
 // Type is the documentation for type declaration.
@@ -54,9 +55,7 @@ type Type struct {
 	Funcs   []*Func   // sorted list of functions returning this type
 	Methods []*Method // sorted list of methods (including embedded ones) of this type
 
-	methods  []*Func   // top-level methods only
-	embedded methodSet // embedded methods only
-	order    int
+	order int
 }
 
 // Func is the documentation for a func declaration.
@@ -77,27 +76,22 @@ const (
 	AllDecls Mode = 1 << iota
 )
 
-// New computes the package documentation for the given package.
-func New(pkg *ast.Package, importpath string, mode Mode) *Package {
-	var r docReader
-	r.init(pkg.Name, mode)
-	filenames := make([]string, len(pkg.Files))
-	// sort package files before reading them so that the
-	// result is the same on different machines (32/64bit)
-	i := 0
-	for filename := range pkg.Files {
-		filenames[i] = filename
-		i++
-	}
-	sort.Strings(filenames)
-
-	// process files in sorted order
-	for _, filename := range filenames {
-		f := pkg.Files[filename]
-		if mode&AllDecls == 0 {
-			r.fileExports(f)
-		}
-		r.addFile(f)
+// New computes the package documentation for the given package AST.
+func New(pkg *ast.Package, importPath string, mode Mode) *Package {
+	var r reader
+	r.readPackage(pkg, mode)
+	r.computeMethodSets()
+	r.cleanupTypes()
+	return &Package{
+		Doc:        r.doc,
+		Name:       pkg.Name,
+		ImportPath: importPath,
+		Imports:    sortedKeys(r.imports),
+		Filenames:  r.filenames,
+		Bugs:       r.bugs,
+		Consts:     sortedValues(r.values, token.CONST),
+		Types:      sortedTypes(r.types),
+		Vars:       sortedValues(r.values, token.VAR),
+		Funcs:      r.funcs.sortedFuncs(),
 	}
-	return r.newDoc(importpath, filenames)
 }

src/pkg/go/doc/exports.go

@@ -8,6 +8,9 @@ package doc
 
 import "go/ast"
 
+// filterIdentList removes unexported names from list in place
+// and returns the resulting list.
+//
 func filterIdentList(list []*ast.Ident) []*ast.Ident {
 	j := 0
 	for _, x := range list {
@@ -19,54 +22,46 @@ func filterIdentList(list []*ast.Ident) []*ast.Ident {
 	return list[0:j]
 }
 
-func baseName(x ast.Expr) *ast.Ident {
-	switch t := x.(type) {
-	case *ast.Ident:
-		return t
-	case *ast.SelectorExpr:
-		if _, ok := t.X.(*ast.Ident); ok {
-			return t.Sel
-		}
-	case *ast.StarExpr:
-		return baseName(t.X)
-	}
-	return nil
-}
-
-func (doc *docReader) filterFieldList(tinfo *typeInfo, fields *ast.FieldList) (removedFields bool) {
+// filterFieldList removes unexported fields (field names) from the field list
+// in place and returns true if fields were removed. Removed fields that are
+// anonymous (embedded) fields are added as embedded types to base. filterType
+// is called with the types of all remaining fields.
+//
+func (r *reader) filterFieldList(base *baseType, fields *ast.FieldList) (removedFields bool) {
 	if fields == nil {
-		return false
+		return
 	}
 	list := fields.List
 	j := 0
-	for _, f := range list {
+	for _, field := range list {
 		keepField := false
-		if len(f.Names) == 0 {
+		if n := len(field.Names); n == 0 {
 			// anonymous field
-			name := baseName(f.Type)
-			if name != nil && name.IsExported() {
-				// we keep the field - in this case doc.addDecl
+			name, imp := baseTypeName(field.Type)
+			if ast.IsExported(name) {
+				// we keep the field - in this case r.readDecl
 				// will take care of adding the embedded type
 				keepField = true
-			} else if tinfo != nil {
+			} else if base != nil && !imp {
 				// we don't keep the field - add it as an embedded
 				// type so we won't loose its methods, if any
-				if embedded := doc.lookupTypeInfo(name.Name); embedded != nil {
-					_, ptr := f.Type.(*ast.StarExpr)
-					tinfo.addEmbeddedType(embedded, ptr)
+				if embedded := r.lookupType(name); embedded != nil {
+					_, ptr := field.Type.(*ast.StarExpr)
+					base.addEmbeddedType(embedded, ptr)
 				}
 			}
 		} else {
-			n := len(f.Names)
-			f.Names = filterIdentList(f.Names)
-			if len(f.Names) < n {
+			field.Names = filterIdentList(field.Names)
+			if len(field.Names) < n {
 				removedFields = true
 			}
-			keepField = len(f.Names) > 0
+			if len(field.Names) > 0 {
+				keepField = true
+			}
 		}
 		if keepField {
-			doc.filterType(nil, f.Type)
-			list[j] = f
+			r.filterType(nil, field.Type)
+			list[j] = field
 			j++
 		}
 	}
@@ -77,52 +72,48 @@ func (doc *docReader) filterFieldList(tinfo *typeInfo, fields *ast.FieldList) (r
 	return
 }
 
-func (doc *docReader) filterParamList(fields *ast.FieldList) bool {
-	if fields == nil {
-		return false
-	}
-	var b bool
-	for _, f := range fields.List {
-		if doc.filterType(nil, f.Type) {
-			b = true
+// filterParamList applies filterType to each parameter type in fields.
+//
+func (r *reader) filterParamList(fields *ast.FieldList) {
+	if fields != nil {
+		for _, f := range fields.List {
+			r.filterType(nil, f.Type)
 		}
 	}
-	return b
 }
 
-func (doc *docReader) filterType(tinfo *typeInfo, typ ast.Expr) bool {
+// filterType strips any unexported struct fields or method types from typ
+// in place. If fields (or methods) have been removed, the corresponding
+// struct or interface type has the Incomplete field set to true. 
+//
+func (r *reader) filterType(base *baseType, typ ast.Expr) {
 	switch t := typ.(type) {
 	case *ast.Ident:
-		return ast.IsExported(t.Name)
+		// nothing to do
 	case *ast.ParenExpr:
-		return doc.filterType(nil, t.X)
+		r.filterType(nil, t.X)
 	case *ast.ArrayType:
-		return doc.filterType(nil, t.Elt)
+		r.filterType(nil, t.Elt)
 	case *ast.StructType:
-		if doc.filterFieldList(tinfo, t.Fields) {
+		if r.filterFieldList(base, t.Fields) {
 			t.Incomplete = true
 		}
-		return len(t.Fields.List) > 0
 	case *ast.FuncType:
-		b1 := doc.filterParamList(t.Params)
-		b2 := doc.filterParamList(t.Results)
-		return b1 || b2
+		r.filterParamList(t.Params)
+		r.filterParamList(t.Results)
 	case *ast.InterfaceType:
-		if doc.filterFieldList(tinfo, t.Methods) {
+		if r.filterFieldList(base, t.Methods) {
 			t.Incomplete = true
 		}
-		return len(t.Methods.List) > 0
 	case *ast.MapType:
-		b1 := doc.filterType(nil, t.Key)
-		b2 := doc.filterType(nil, t.Value)
-		return b1 || b2
+		r.filterType(nil, t.Key)
+		r.filterType(nil, t.Value)
 	case *ast.ChanType:
-		return doc.filterType(nil, t.Value)
+		r.filterType(nil, t.Value)
 	}
-	return false
 }
 
-func (doc *docReader) filterSpec(spec ast.Spec) bool {
+func (r *reader) filterSpec(spec ast.Spec) bool {
 	switch s := spec.(type) {
 	case *ast.ImportSpec:
 		// always keep imports so we can collect them
@@ -130,22 +121,22 @@ func (doc *docReader) filterSpec(spec ast.Spec) bool {
 	case *ast.ValueSpec:
 		s.Names = filterIdentList(s.Names)
 		if len(s.Names) > 0 {
-			doc.filterType(nil, s.Type)
+			r.filterType(nil, s.Type)
 			return true
 		}
 	case *ast.TypeSpec:
 		if ast.IsExported(s.Name.Name) {
-			doc.filterType(doc.lookupTypeInfo(s.Name.Name), s.Type)
+			r.filterType(r.lookupType(s.Name.Name), s.Type)
 			return true
 		}
 	}
 	return false
 }
 
-func (doc *docReader) filterSpecList(list []ast.Spec) []ast.Spec {
+func (r *reader) filterSpecList(list []ast.Spec) []ast.Spec {
 	j := 0
 	for _, s := range list {
-		if doc.filterSpec(s) {
+		if r.filterSpec(s) {
 			list[j] = s
 			j++
 		}
@@ -153,10 +144,10 @@ func (doc *docReader) filterSpecList(list []ast.Spec) []ast.Spec {
 	return list[0:j]
 }
 
-func (doc *docReader) filterDecl(decl ast.Decl) bool {
+func (r *reader) filterDecl(decl ast.Decl) bool {
 	switch d := decl.(type) {
 	case *ast.GenDecl:
-		d.Specs = doc.filterSpecList(d.Specs)
+		d.Specs = r.filterSpecList(d.Specs)
 		return len(d.Specs) > 0
 	case *ast.FuncDecl:
 		return ast.IsExported(d.Name.Name)
@@ -164,18 +155,15 @@ func (doc *docReader) filterDecl(decl ast.Decl) bool {
 	return false
 }
 
-// fileExports trims the AST for a Go file in place such that
-// only exported nodes remain. fileExports returns true if
-// there are exported declarations; otherwise it returns false.
+// fileExports removes unexported declarations from src in place.
 //
-func (doc *docReader) fileExports(src *ast.File) bool {
+func (r *reader) fileExports(src *ast.File) {
 	j := 0
 	for _, d := range src.Decls {
-		if doc.filterDecl(d) {
+		if r.filterDecl(d) {
 			src.Decls[j] = d
 			j++
 		}
 	}
 	src.Decls = src.Decls[0:j]
-	return j > 0
 }

src/pkg/go/doc/testdata/e.0.golden

+// 
+PACKAGE e
+
+IMPORTPATH
+	testdata/e
+
+FILENAMES
+	testdata/e.go
+
+TYPES
+	// T1 has no (top-level) M method due to conflict. 
+	type T1 struct {
+		// contains filtered or unexported fields
+	}
+
+	// T2 has only M as top-level method. 
+	type T2 struct {
+		// contains filtered or unexported fields
+	}
+
+	// T2.M should appear as method of T2. 
+	func (T2) M()
+
+	// T3 has only M as top-level method. 
+	type T3 struct {
+		// contains filtered or unexported fields
+	}
+
+	// T3.M should appear as method of T3. 
+	func (T3) M()
+

src/pkg/go/doc/testdata/e.1.golden

+// 
+PACKAGE e
+
+IMPORTPATH
+	testdata/e
+
+FILENAMES
+	testdata/e.go
+
+TYPES
+	// T1 has no (top-level) M method due to conflict. 
+	type T1 struct {
+		t1
+		t2
+	}
+
+	// T2 has only M as top-level method. 
+	type T2 struct {
+		t1
+	}
+
+	// T2.M should appear as method of T2. 
+	func (T2) M()
+
+	// T3 has only M as top-level method. 
+	type T3 struct {
+		t1e
+		t2e
+	}
+
+	// T3.M should appear as method of T3. 
+	func (T3) M()
+
+	// 
+	type t1 struct{}
+
+	// t1.M should not appear as method in a Tx type. 
+	func (t1) M()
+
+	// 
+	type t1e struct {
+		t1
+	}
+
+	// t1.M should not appear as method in a Tx type. 
+	func (t1e) M()
+
+	// 
+	type t2 struct{}
+
+	// t2.M should not appear as method in a Tx type. 
+	func (t2) M()
+
+	// 
+	type t2e struct {
+		t2
+	}
+
+	// t2.M should not appear as method in a Tx type. 
+	func (t2e) M()
+

src/pkg/go/doc/testdata/e.go

+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Embedding tests.
+// TODO(gri): This should be comprehensive.
+
+package e
+
+// ----------------------------------------------------------------------------
+// Conflicting methods M must not show up.
+
+type t1 struct{}
+
+// t1.M should not appear as method in a Tx type.
+func (t1) M() {}
+
+type t2 struct{}
+
+// t2.M should not appear as method in a Tx type.
+func (t2) M() {}
+
+// T1 has no (top-level) M method due to conflict.
+type T1 struct {
+	t1
+	t2
+}
+
+// ----------------------------------------------------------------------------
+// Higher-level method M wins over lower-level method M.
+
+// T2 has only M as top-level method.
+type T2 struct {
+	t1
+}
+
+// T2.M should appear as method of T2.
+func (T2) M() {}
+
+// ----------------------------------------------------------------------------
+// Higher-level method M wins over lower-level conflicting methods M.
+
+type t1e struct {
+	t1
+}
+
+type t2e struct {
+	t2
+}
+
+// T3 has only M as top-level method.
+type T3 struct {
+	t1e
+	t2e
+}
+
+// T3.M should appear as method of T3.
+func (T3) M() {}