go/build: replace FindTree, ScanDir, Tree, DirInfo with Import, Package

This is an API change, but one I have been promising would
happen when it was clear what the go command needed.

This is basically a complete replacement of what used to be here.

build.Tree is gone.

build.DirInfo is expanded and now called build.Package.

build.FindTree is now build.Import(package, srcDir, build.FindOnly).
The returned *Package contains information that FindTree returned,
but applicable only to a single package.

build.ScanDir is now build.ImportDir.

build.FindTree+build.ScanDir is now build.Import.

The new Import API allows specifying the source directory,
in order to resolve local imports (import "./foo") and also allows
scanning of packages outside of $GOPATH.  They will come back
with less information in the Package, but they will still work.

The old go/build API exposed both too much and too little.
This API is much closer to what the go command needs,
and it works well enough in the other places where it is
used.  Path is gone, so it can no longer be misused.  (Fixes issue 2749.)

This CL updates clients of go/build other than the go command.
The go command changes are in a separate CL, to be submitted
at the same time.

R=golang-dev, r, alex.brainman, adg
CC=golang-dev
https://golang.org/cl/5713043

src/cmd/api/goapi.go

@@ -74,16 +74,10 @@ func main() {
 		pkgs = strings.Fields(string(stds))
 	}
 
-	tree, _, err := build.FindTree("os") // some known package
-	if err != nil {
-		log.Fatalf("failed to find tree: %v", err)
-	}
-
 	var featureCtx = make(map[string]map[string]bool) // feature -> context name -> true
 	for _, context := range contexts {
 		w := NewWalker()
 		w.context = context
-		w.tree = tree
 
 		for _, pkg := range pkgs {
 			w.wantedPkg[pkg] = true
@@ -95,7 +89,7 @@ func main() {
 				strings.HasPrefix(pkg, "old/") {
 				continue
 			}
-			if !tree.HasSrc(pkg) {
+			if fi, err := os.Stat(filepath.Join(w.root, pkg)); err != nil || !fi.IsDir() {
 				log.Fatalf("no source in tree for package %q", pkg)
 			}
 			w.WalkPackage(pkg)
@@ -165,7 +159,7 @@ type pkgSymbol struct {
 
 type Walker struct {
 	context         *build.Context
-	tree            *build.Tree
+	root            string
 	fset            *token.FileSet
 	scope           []string
 	features        map[string]bool // set
@@ -191,6 +185,7 @@ func NewWalker() *Walker {
 		selectorFullPkg: make(map[string]string),
 		wantedPkg:       make(map[string]bool),
 		prevConstType:   make(map[pkgSymbol]string),
+		root:            filepath.Join(build.Default.GOROOT, "src/pkg"),
 	}
 }
 
@@ -252,15 +247,13 @@ func (w *Walker) WalkPackage(name string) {
 	defer func() {
 		w.packageState[name] = loaded
 	}()
-	dir := filepath.Join(w.tree.SrcDir(), filepath.FromSlash(name))
+	dir := filepath.Join(w.root, filepath.FromSlash(name))
 
-	var info *build.DirInfo
-	var err error
-	if ctx := w.context; ctx != nil {
-		info, err = ctx.ScanDir(dir)
-	} else {
-		info, err = build.ScanDir(dir)
+	ctxt := w.context
+	if ctxt == nil {
+		ctxt = &build.Default
 	}
+	info, err := ctxt.ImportDir(dir, 0)
 	if err != nil {
 		if strings.Contains(err.Error(), "no Go source files") {
 			return

src/cmd/api/goapi_test.go

@@ -7,7 +7,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"go/build"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -36,7 +35,7 @@ func TestGolden(t *testing.T) {
 		w := NewWalker()
 		w.wantedPkg[fi.Name()] = true
 
-		w.tree = &build.Tree{Path: "testdata", Goroot: true}
+		w.root = "testdata/src/pkg"
 		goldenFile := filepath.Join("testdata", "src", "pkg", fi.Name(), "golden.txt")
 		w.WalkPackage(fi.Name())
 

src/cmd/godoc/godoc.go

@@ -15,6 +15,7 @@ import (
 	"go/printer"
 	"go/token"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
@@ -90,11 +91,11 @@ var (
 
 func initHandlers() {
 	paths := filepath.SplitList(*pkgPath)
-	for _, t := range build.Path {
-		if t.Goroot {
-			continue
+	gorootSrc := filepath.Join(build.Default.GOROOT, "src", "pkg")
+	for _, p := range build.Default.SrcDirs() {
+		if p != gorootSrc {
+			paths = append(paths, p)
 		}
-		paths = append(paths, t.SrcDir())
 	}
 	fsMap.Init(paths)
 
@@ -1002,11 +1003,13 @@ func fsReadDir(dir string) ([]os.FileInfo, error) {
 	return fs.ReadDir(dir)
 }
 
-// fsReadFile implements ReadFile for the go/build package.
-func fsReadFile(dir, name string) (path string, data []byte, err error) {
-	path = filepath.Join(dir, name)
-	data, err = ReadFile(fs, path)
-	return
+// fsOpenFile implements OpenFile for the go/build package.
+func fsOpenFile(name string) (r io.ReadCloser, err error) {
+	data, err := ReadFile(fs, name)
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.NopCloser(bytes.NewReader(data)), nil
 }
 
 func inList(name string, list []string) bool {
@@ -1039,10 +1042,11 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf
 		// To use different pair, such as if we allowed the user
 		// to choose, set ctxt.GOOS and ctxt.GOARCH before
 		// calling ctxt.ScanDir.
-		ctxt := build.DefaultContext
+		ctxt := build.Default
+		ctxt.IsAbsPath = path.IsAbs
 		ctxt.ReadDir = fsReadDir
-		ctxt.ReadFile = fsReadFile
-		dir, err := ctxt.ScanDir(abspath)
+		ctxt.OpenFile = fsOpenFile
+		dir, err := ctxt.ImportDir(abspath, 0)
 		if err == nil {
 			pkgFiles = append(dir.GoFiles, dir.CgoFiles...)
 		}

src/cmd/godoc/main.go

@@ -388,9 +388,9 @@ func main() {
 	}
 	relpath := path
 	abspath := path
-	if t, pkg, err := build.FindTree(path); err == nil {
-		relpath = pkg
-		abspath = filepath.Join(t.SrcDir(), pkg)
+	if bp, _ := build.Import(path, "", build.FindOnly); bp.Dir != "" && bp.ImportPath != "" {
+		relpath = bp.ImportPath
+		abspath = bp.Dir
 	} else if !filepath.IsAbs(path) {
 		abspath = absolutePath(path, pkgHandler.fsRoot)
 	} else {

src/pkg/exp/types/gcimporter.go

@@ -18,6 +18,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"text/scanner"
 )
 
@@ -39,11 +40,14 @@ func findPkg(path string) (filename, id string) {
 	switch path[0] {
 	default:
 		// "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x"
-		tree, pkg, err := build.FindTree(path)
-		if err != nil {
+		bp, _ := build.Import(path, "", build.FindOnly)
+		if bp.PkgObj == "" {
 			return
 		}
-		noext = filepath.Join(tree.PkgDir(), pkg)
+		noext = bp.PkgObj
+		if strings.HasSuffix(noext, ".a") {
+			noext = noext[:len(noext)-2]
+		}
 
 	case '.':
 		// "./x" -> "/this/directory/x.ext", "/this/directory/x"
@@ -742,7 +746,7 @@ func (p *gcParser) parseVarDecl() {
 }
 
 // FuncBody = "{" ... "}" .
-// 
+//
 func (p *gcParser) parseFuncBody() {
 	p.expect('{')
 	for i := 1; i > 0; p.next() {

src/pkg/go/build/build_test.go

@@ -5,83 +5,14 @@
 package build
 
 import (
+	"os"
 	"path/filepath"
-	"reflect"
 	"runtime"
-	"sort"
 	"testing"
 )
 
-func sortstr(x []string) []string {
-	sort.Strings(x)
-	return x
-}
-
-var buildPkgs = []struct {
-	dir  string
-	info *DirInfo
-}{
-	{
-		"go/build/pkgtest",
-		&DirInfo{
-			GoFiles:      []string{"pkgtest.go"},
-			SFiles:       []string{"sqrt_" + runtime.GOARCH + ".s"},
-			Package:      "pkgtest",
-			Imports:      []string{"bytes"},
-			TestImports:  []string{"fmt", "pkgtest"},
-			TestGoFiles:  sortstr([]string{"sqrt_test.go", "sqrt_" + runtime.GOARCH + "_test.go"}),
-			XTestGoFiles: []string{"xsqrt_test.go"},
-		},
-	},
-	{
-		"go/build/cmdtest",
-		&DirInfo{
-			GoFiles:     []string{"main.go"},
-			Package:     "main",
-			Imports:     []string{"go/build/pkgtest"},
-			TestImports: []string{},
-		},
-	},
-	{
-		"go/build/cgotest",
-		&DirInfo{
-			CgoFiles:    ifCgo([]string{"cgotest.go"}),
-			CFiles:      []string{"cgotest.c"},
-			HFiles:      []string{"cgotest.h"},
-			Imports:     []string{"C", "unsafe"},
-			TestImports: []string{},
-			Package:     "cgotest",
-		},
-	},
-}
-
-func ifCgo(x []string) []string {
-	if DefaultContext.CgoEnabled {
-		return x
-	}
-	return nil
-}
-
-func TestBuild(t *testing.T) {
-	for _, tt := range buildPkgs {
-		tree := Path[0] // Goroot
-		dir := filepath.Join(tree.SrcDir(), tt.dir)
-		info, err := ScanDir(dir)
-		if err != nil {
-			t.Errorf("ScanDir(%#q): %v", tt.dir, err)
-			continue
-		}
-		// Don't bother testing import positions.
-		tt.info.ImportPos, tt.info.TestImportPos = info.ImportPos, info.TestImportPos
-		if !reflect.DeepEqual(info, tt.info) {
-			t.Errorf("ScanDir(%#q) = %#v, want %#v\n", tt.dir, info, tt.info)
-			continue
-		}
-	}
-}
-
 func TestMatch(t *testing.T) {
-	ctxt := DefaultContext
+	ctxt := Default
 	what := "default"
 	match := func(tag string) {
 		if !ctxt.match(tag) {
@@ -106,3 +37,40 @@ func TestMatch(t *testing.T) {
 	match(runtime.GOOS + "," + runtime.GOARCH + ",!bar")
 	nomatch(runtime.GOOS + "," + runtime.GOARCH + ",bar")
 }
+
+func TestDotSlashImport(t *testing.T) {
+	p, err := ImportDir("testdata/other", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(p.Imports) != 1 || p.Imports[0] != "./file" {
+		t.Fatalf("testdata/other: Imports=%v, want [./file]", p.Imports)
+	}
+
+	p1, err := Import("./file", "testdata/other", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p1.Name != "file" {
+		t.Fatalf("./file: Name=%q, want %q", p1.Name, "file")
+	}
+	dir := filepath.Clean("testdata/other/file") // Clean to use \ on Windows
+	if p1.Dir != dir {
+		t.Fatalf("./file: Dir=%q, want %q", p1.Name, dir)
+	}
+}
+
+func TestLocalDirectory(t *testing.T) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p, err := ImportDir(cwd, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if p.ImportPath != "go/build" {
+		t.Fatalf("ImportPath=%q, want %q", p.ImportPath, "go/build")
+	}
+}

src/pkg/go/build/cgotest/cgotest.c

-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-int
-Add(int x, int y, int *sum)
-{
-	sum = x+y;
-}

src/pkg/go/build/cgotest/cgotest.go

-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package cgotest
-
-/*
-char* greeting = "hello, world";
-*/
-// #include "cgotest.h"
-import "C"
-import "unsafe"
-
-var Greeting = C.GoString(C.greeting)
-
-func DoAdd(x, y int) (sum int) {
-	C.Add(C.int(x), C.int(y), (*C.int)(unsafe.Pointer(&sum)))
-	return
-}

src/pkg/go/build/cgotest/cgotest.h

-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-extern int Add(int, int, int *);

src/pkg/go/build/cmdtest/main.go

-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import "go/build/pkgtest"
-
-func main() {
-	pkgtest.Foo()
-	print(int(pkgtest.Sqrt(9)))
-}

src/pkg/go/build/path.go

-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package build
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime"
-)
-
-// Path is a validated list of Trees derived from $GOROOT and $GOPATH at init.
-var Path []*Tree
-
-// Tree describes a Go source tree, either $GOROOT or one from $GOPATH.
-type Tree struct {
-	Path   string
-	Goroot bool
-}
-
-func newTree(p string) (*Tree, error) {
-	if !filepath.IsAbs(p) {
-		return nil, errors.New("must be absolute")
-	}
-	ep, err := filepath.EvalSymlinks(p)
-	if err != nil {
-		return nil, err
-	}
-	return &Tree{Path: ep}, nil
-}
-
-// SrcDir returns the tree's package source directory.
-func (t *Tree) SrcDir() string {
-	if t.Goroot {
-		return filepath.Join(t.Path, "src", "pkg")
-	}
-	return filepath.Join(t.Path, "src")
-}
-
-// PkgDir returns the tree's package object directory.
-func (t *Tree) PkgDir() string {
-	goos, goarch := runtime.GOOS, runtime.GOARCH
-	if e := os.Getenv("GOOS"); e != "" {
-		goos = e
-	}
-	if e := os.Getenv("GOARCH"); e != "" {
-		goarch = e
-	}
-	return filepath.Join(t.Path, "pkg", goos+"_"+goarch)
-}
-
-// BinDir returns the tree's binary executable directory.
-func (t *Tree) BinDir() string {
-	if t.Goroot {
-		if gobin := os.Getenv("GOBIN"); gobin != "" {
-			return filepath.Clean(gobin)
-		}
-	}
-	return filepath.Join(t.Path, "bin")
-}
-
-// HasSrc returns whether the given package's
-// source can be found inside this Tree.
-func (t *Tree) HasSrc(pkg string) bool {
-	fi, err := os.Stat(filepath.Join(t.SrcDir(), pkg))
-	if err != nil {
-		return false
-	}
-	return fi.IsDir()
-}
-
-// HasPkg returns whether the given package's
-// object file can be found inside this Tree.
-func (t *Tree) HasPkg(pkg string) bool {
-	fi, err := os.Stat(filepath.Join(t.PkgDir(), pkg+".a"))
-	if err != nil {
-		return false
-	}
-	return !fi.IsDir()
-}
-
-var (
-	ErrNotFound     = errors.New("package could not be found locally")
-	ErrTreeNotFound = errors.New("no valid GOROOT or GOPATH could be found")
-)
-
-// FindTree takes an import or filesystem path and returns the
-// tree where the package source should be and the package import path.
-func FindTree(path string) (tree *Tree, pkg string, err error) {
-	if isLocalPath(path) {
-		if path, err = filepath.Abs(path); err != nil {
-			return
-		}
-		if path, err = filepath.EvalSymlinks(path); err != nil {
-			return
-		}
-		for _, t := range Path {
-			tpath := t.SrcDir() + string(filepath.Separator)
-			if !filepath.HasPrefix(path, tpath) {
-				continue
-			}
-			tree = t
-			pkg = filepath.ToSlash(path[len(tpath):])
-			return
-		}
-		err = fmt.Errorf("path %q not inside a GOPATH", path)
-		return
-	}
-	tree = defaultTree
-	pkg = filepath.ToSlash(path)
-	for _, t := range Path {
-		if t.HasSrc(pkg) {
-			tree = t
-			return
-		}
-	}
-	if tree == nil {
-		err = ErrTreeNotFound
-	} else {
-		err = ErrNotFound
-	}
-	return
-}
-
-var (
-	// argument lists used by the build's gc and ld methods
-	gcImportArgs []string
-	ldImportArgs []string
-
-	// default tree for remote packages
-	defaultTree *Tree
-)
-
-// set up Path: parse and validate GOROOT and GOPATH variables
-func init() {
-	root := runtime.GOROOT()
-	t, err := newTree(root)
-	if err == nil {
-		t.Goroot = true
-		Path = []*Tree{t}
-	}
-
-	for _, p := range filepath.SplitList(os.Getenv("GOPATH")) {
-		if p == "" {
-			continue
-		}
-		t, err := newTree(p)
-		if err != nil {
-			continue
-		}
-
-		Path = append(Path, t)
-		gcImportArgs = append(gcImportArgs, "-I", t.PkgDir())
-		ldImportArgs = append(ldImportArgs, "-L", t.PkgDir())
-
-		// select first GOPATH entry as default
-		if defaultTree == nil {
-			defaultTree = t
-		}
-	}
-
-	// use GOROOT if no valid GOPATH specified
-	if defaultTree == nil && len(Path) > 0 {
-		defaultTree = Path[0]
-	}
-}

src/pkg/go/build/pkgtest/pkgtest.go

-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package pkgtest
-
-import "bytes"
-
-func Foo() *bytes.Buffer {
-	return nil
-}
-
-func Sqrt(x float64) float64

src/pkg/go/build/pkgtest/sqrt_386.s

-// Copyright 2009 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// func Sqrt(x float64) float64	
-TEXT ·Sqrt(SB),7,$0
-	FMOVD   x+0(FP),F0
-	FSQRT
-	FMOVDP  F0,r+8(FP)
-	RET

src/pkg/go/build/pkgtest/sqrt_386_test.go

-package pkgtest

src/pkg/go/build/pkgtest/sqrt_amd64.s

-// Copyright 2009 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// func Sqrt(x float64) float64
-TEXT ·Sqrt(SB),7,$0
-	SQRTSD x+0(FP), X0
-	MOVSD X0, r+8(FP)
-	RET

src/pkg/go/build/pkgtest/sqrt_amd64_test.go

-package pkgtest

src/pkg/go/build/pkgtest/sqrt_arm.s

-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// func Sqrt(x float64) float64	
-TEXT ·Sqrt(SB),7,$0
-	MOVD   x+0(FP),F0
-	SQRTD  F0,F0
-	MOVD  F0,r+8(FP)
-	RET

src/pkg/go/build/pkgtest/sqrt_arm_test.go

-package pkgtest

src/pkg/go/build/pkgtest/sqrt_test.go

-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package pkgtest
-
-import "fmt"
-
-var _ = fmt.Printf

src/pkg/go/build/pkgtest/xsqrt_test.go

-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package pkgtest_test
-
-import "pkgtest"
-
-var _ = pkgtest.Foo

src/pkg/go/build/syslist_test.go

@@ -55,7 +55,7 @@ var tests = []GoodFileTest{
 
 func TestGoodOSArch(t *testing.T) {
 	for _, test := range tests {
-		if DefaultContext.goodOSArchFile(test.name) != test.result {
+		if Default.goodOSArchFile(test.name) != test.result {
 			t.Fatalf("goodOSArchFile(%q) != %v", test.name, test.result)
 		}
 	}

src/pkg/go/build/testdata/other/file/file.go

+// Test data - not compiled.
+
+package file
+
+func F() {}

src/pkg/go/build/testdata/other/main.go

+// Test data - not compiled.
+
+package main
+
+import (
+	"./file"
+)
+
+func main() {
+	file.F()
+}