cmd/go: split out cmd/go/internal/load

This is one CL in a long sequence of changes to break up the
go command from one package into a plausible group of packages.

This sequence is concerned only with moving code, not changing
or cleaning up code. There will still be more cleanup after this sequence.

The entire sequence will be submitted together: it is not a goal
for the tree to build at every step.

For #18653.

Change-Id: Ic802483e50598def638f1e2e706d5fdf7822d32d
Reviewed-on: https://go-review.googlesource.com/36196Reviewed-by: David Crawshaw <crawshaw@golang.org>

src/cmd/go/bug.go

@@ -6,8 +6,8 @@ package main
 
 import (
 	"bytes"
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
 	"fmt"
 	"io"
 	"io/ioutil"

src/cmd/go/clean.go

@@ -5,8 +5,9 @@
 package main
 
 import (
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/load"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -77,12 +78,12 @@ func init() {
 }
 
 func runClean(cmd *base.Command, args []string) {
-	for _, pkg := range packagesAndErrors(args) {
+	for _, pkg := range load.PackagesAndErrors(args) {
 		clean(pkg)
 	}
 }
 
-var cleaned = map[*Package]bool{}
+var cleaned = map[*load.Package]bool{}
 
 // TODO: These are dregs left by Makefile-based builds.
 // Eventually, can stop deleting these.
@@ -107,7 +108,7 @@ var cleanExt = map[string]bool{
 	".so": true,
 }
 
-func clean(p *Package) {
+func clean(p *load.Package) {
 	if cleaned[p] {
 		return
 	}
@@ -209,17 +210,17 @@ func clean(p *Package) {
 		}
 	}
 
-	if cleanI && p.target != "" {
+	if cleanI && p.Internal.Target != "" {
 		if cfg.BuildN || cfg.BuildX {
-			b.showcmd("", "rm -f %s", p.target)
+			b.showcmd("", "rm -f %s", p.Internal.Target)
 		}
 		if !cfg.BuildN {
-			removeFile(p.target)
+			removeFile(p.Internal.Target)
 		}
 	}
 
 	if cleanR {
-		for _, p1 := range p.imports {
+		for _, p1 := range p.Internal.Imports {
 			clean(p1)
 		}
 	}

src/cmd/go/doc.go

@@ -7,8 +7,8 @@
 package main
 
 import (
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
 )
 
 var cmdDoc = &base.Command{

src/cmd/go/env.go

@@ -5,8 +5,9 @@
 package main
 
 import (
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/load"
 	"fmt"
 	"os"
 	"runtime"
@@ -33,14 +34,14 @@ func mkEnv() []cfg.EnvVar {
 
 	env := []cfg.EnvVar{
 		{"GOARCH", cfg.Goarch},
-		{"GOBIN", gobin},
+		{"GOBIN", cfg.GOBIN},
 		{"GOEXE", cfg.ExeSuffix},
 		{"GOHOSTARCH", runtime.GOARCH},
 		{"GOHOSTOS", runtime.GOOS},
 		{"GOOS", cfg.Goos},
 		{"GOPATH", cfg.BuildContext.GOPATH},
 		{"GORACE", os.Getenv("GORACE")},
-		{"GOROOT", goroot},
+		{"GOROOT", cfg.GOROOT},
 		{"GOTOOLDIR", base.ToolDir},
 
 		// disable escape codes in clang errors
@@ -88,7 +89,7 @@ func findEnv(env []cfg.EnvVar, name string) string {
 func extraEnvVars() []cfg.EnvVar {
 	var b builder
 	b.init()
-	cppflags, cflags, cxxflags, fflags, ldflags := b.cflags(&Package{})
+	cppflags, cflags, cxxflags, fflags, ldflags := b.cflags(&load.Package{})
 	return []cfg.EnvVar{
 		{"PKG_CONFIG", b.pkgconfigCmd()},
 		{"CGO_CFLAGS", strings.Join(cflags, " ")},

src/cmd/go/fix.go

@@ -5,8 +5,9 @@
 package main
 
 import (
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/load"
 	"cmd/go/internal/str"
 )
 
@@ -27,10 +28,10 @@ See also: go fmt, go vet.
 }
 
 func runFix(cmd *base.Command, args []string) {
-	for _, pkg := range packages(args) {
+	for _, pkg := range load.Packages(args) {
 		// Use pkg.gofiles instead of pkg.Dir so that
 		// the command only applies to this package,
 		// not to packages in subdirectories.
-		base.Run(str.StringList(cfg.BuildToolexec, base.Tool("fix"), base.RelPaths(pkg.allgofiles)))
+		base.Run(str.StringList(cfg.BuildToolexec, base.Tool("fix"), base.RelPaths(pkg.Internal.AllGoFiles)))
 	}
 }

src/cmd/go/fmt.go

@@ -5,8 +5,9 @@
 package main
 
 import (
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/load"
 	"cmd/go/internal/str"
 	"os"
 	"path/filepath"
@@ -38,11 +39,11 @@ See also: go fix, go vet.
 
 func runFmt(cmd *base.Command, args []string) {
 	gofmt := gofmtPath()
-	for _, pkg := range packages(args) {
+	for _, pkg := range load.Packages(args) {
 		// Use pkg.gofiles instead of pkg.Dir so that
 		// the command only applies to this package,
 		// not to packages in subdirectories.
-		base.Run(str.StringList(gofmt, "-l", "-w", base.RelPaths(pkg.allgofiles)))
+		base.Run(str.StringList(gofmt, "-l", "-w", base.RelPaths(pkg.Internal.AllGoFiles)))
 	}
 }
 
@@ -52,12 +53,12 @@ func gofmtPath() string {
 		gofmt += base.ToolWindowsExtension
 	}
 
-	gofmtPath := filepath.Join(gobin, gofmt)
+	gofmtPath := filepath.Join(cfg.GOBIN, gofmt)
 	if _, err := os.Stat(gofmtPath); err == nil {
 		return gofmtPath
 	}
 
-	gofmtPath = filepath.Join(goroot, "bin", gofmt)
+	gofmtPath = filepath.Join(cfg.GOROOT, "bin", gofmt)
 	if _, err := os.Stat(gofmtPath); err == nil {
 		return gofmtPath
 	}

src/cmd/go/generate.go

@@ -7,8 +7,9 @@ package main
 import (
 	"bufio"
 	"bytes"
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/load"
 	"fmt"
 	"io"
 	"log"
@@ -138,7 +139,7 @@ func init() {
 }
 
 func runGenerate(cmd *base.Command, args []string) {
-	ignoreImports = true
+	load.IgnoreImports = true
 
 	if generateRunFlag != "" {
 		var err error
@@ -148,8 +149,8 @@ func runGenerate(cmd *base.Command, args []string) {
 		}
 	}
 	// Even if the arguments are .go files, this loop suffices.
-	for _, pkg := range packages(args) {
-		for _, file := range pkg.gofiles {
+	for _, pkg := range load.Packages(args) {
+		for _, file := range pkg.Internal.GoFiles {
 			if !generate(pkg.Name, file) {
 				break
 			}

src/cmd/go/internal/base/base.go

@@ -7,10 +7,13 @@
 package base
 
 import (
+	"bytes"
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/str"
+	"errors"
 	"flag"
 	"fmt"
+	"go/scanner"
 	"log"
 	"os"
 	"os/exec"
@@ -145,3 +148,25 @@ func RunStdin(cmdline []string) {
 // Usage is the usage-reporting function, filled in by package main
 // but here for reference by other packages.
 var Usage func()
+
+// ExpandScanner expands a scanner.List error into all the errors in the list.
+// The default Error method only shows the first error
+// and does not shorten paths.
+func ExpandScanner(err error) error {
+	// Look for parser errors.
+	if err, ok := err.(scanner.ErrorList); ok {
+		// Prepare error with \n before each message.
+		// When printed in something like context: %v
+		// this will put the leading file positions each on
+		// its own line. It will also show all the errors
+		// instead of just the first, as err.Error does.
+		var buf bytes.Buffer
+		for _, e := range err {
+			e.Pos.Filename = ShortPath(e.Pos.Filename)
+			buf.WriteString("\n")
+			buf.WriteString(e.Error())
+		}
+		return errors.New(buf.String())
+	}
+	return err
+}

src/cmd/go/internal/cfg/cfg.go

@@ -9,6 +9,8 @@ package cfg
 import (
 	"flag"
 	"go/build"
+	"os"
+	"path/filepath"
 	"runtime"
 )
 
@@ -64,3 +66,11 @@ func AddBuildFlagsNX(flags *flag.FlagSet) {
 	flags.BoolVar(&BuildN, "n", false, "")
 	flags.BoolVar(&BuildX, "x", false, "")
 }
+
+var (
+	GOROOT    = filepath.Clean(runtime.GOROOT())
+	GOBIN     = os.Getenv("GOBIN")
+	GOROOTbin = filepath.Join(GOROOT, "bin")
+	GOROOTpkg = filepath.Join(GOROOT, "pkg")
+	GOROOTsrc = filepath.Join(GOROOT, "src")
+)

src/cmd/go/match_test.go → src/cmd/go/internal/load/match_test.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package load
 
 import "testing"
 

src/cmd/go/internal/load/path.go

+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package load
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+// hasSubdir reports whether dir is a subdirectory of
+// (possibly multiple levels below) root.
+// If so, it sets rel to the path fragment that must be
+// appended to root to reach dir.
+func hasSubdir(root, dir string) (rel string, ok bool) {
+	if p, err := filepath.EvalSymlinks(root); err == nil {
+		root = p
+	}
+	if p, err := filepath.EvalSymlinks(dir); err == nil {
+		dir = p
+	}
+	const sep = string(filepath.Separator)
+	root = filepath.Clean(root)
+	if !strings.HasSuffix(root, sep) {
+		root += sep
+	}
+	dir = filepath.Clean(dir)
+	if !strings.HasPrefix(dir, root) {
+		return "", false
+	}
+	return filepath.ToSlash(dir[len(root):]), true
+}
+
+// hasPathPrefix reports whether the path s begins with the
+// elements in prefix.
+func hasPathPrefix(s, prefix string) bool {
+	switch {
+	default:
+		return false
+	case len(s) == len(prefix):
+		return s == prefix
+	case len(s) > len(prefix):
+		if prefix != "" && prefix[len(prefix)-1] == '/' {
+			return strings.HasPrefix(s, prefix)
+		}
+		return s[len(prefix)] == '/' && s[:len(prefix)] == prefix
+	}
+}
+
+// expandPath returns the symlink-expanded form of path.
+func expandPath(p string) string {
+	x, err := filepath.EvalSymlinks(p)
+	if err == nil {
+		return x
+	}
+	return p
+}
+
+// hasFilePathPrefix reports whether the filesystem path s begins with the
+// elements in prefix.
+func hasFilePathPrefix(s, prefix string) bool {
+	sv := strings.ToUpper(filepath.VolumeName(s))
+	pv := strings.ToUpper(filepath.VolumeName(prefix))
+	s = s[len(sv):]
+	prefix = prefix[len(pv):]
+	switch {
+	default:
+		return false
+	case sv != pv:
+		return false
+	case len(s) == len(prefix):
+		return s == prefix
+	case len(s) > len(prefix):
+		if prefix != "" && prefix[len(prefix)-1] == filepath.Separator {
+			return strings.HasPrefix(s, prefix)
+		}
+		return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix
+	}
+}

src/cmd/go/internal/load/search.go

+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package load
+
+import (
+	"cmd/go/internal/cfg"
+	"fmt"
+	"go/build"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// allPackages returns all the packages that can be found
+// under the $GOPATH directories and $GOROOT matching pattern.
+// The pattern is either "all" (all packages), "std" (standard packages),
+// "cmd" (standard commands), or a path including "...".
+func allPackages(pattern string) []string {
+	pkgs := MatchPackages(pattern)
+	if len(pkgs) == 0 {
+		fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
+	}
+	return pkgs
+}
+
+// allPackagesInFS is like allPackages but is passed a pattern
+// beginning ./ or ../, meaning it should scan the tree rooted
+// at the given directory. There are ... in the pattern too.
+func allPackagesInFS(pattern string) []string {
+	pkgs := MatchPackagesInFS(pattern)
+	if len(pkgs) == 0 {
+		fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
+	}
+	return pkgs
+}
+
+// MatchPackages returns a list of package paths matching pattern
+// (see go help packages for pattern syntax).
+func MatchPackages(pattern string) []string {
+	match := func(string) bool { return true }
+	treeCanMatch := func(string) bool { return true }
+	if !IsMetaPackage(pattern) {
+		match = matchPattern(pattern)
+		treeCanMatch = treeCanMatchPattern(pattern)
+	}
+
+	have := map[string]bool{
+		"builtin": true, // ignore pseudo-package that exists only for documentation
+	}
+	if !cfg.BuildContext.CgoEnabled {
+		have["runtime/cgo"] = true // ignore during walk
+	}
+	var pkgs []string
+
+	for _, src := range cfg.BuildContext.SrcDirs() {
+		if (pattern == "std" || pattern == "cmd") && src != cfg.GOROOTsrc {
+			continue
+		}
+		src = filepath.Clean(src) + string(filepath.Separator)
+		root := src
+		if pattern == "cmd" {
+			root += "cmd" + string(filepath.Separator)
+		}
+		filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
+			if err != nil || !fi.IsDir() || path == src {
+				return nil
+			}
+
+			// Avoid .foo, _foo, and testdata directory trees.
+			_, elem := filepath.Split(path)
+			if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
+				return filepath.SkipDir
+			}
+
+			name := filepath.ToSlash(path[len(src):])
+			if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") {
+				// The name "std" is only the standard library.
+				// If the name is cmd, it's the root of the command tree.
+				return filepath.SkipDir
+			}
+			if !treeCanMatch(name) {
+				return filepath.SkipDir
+			}
+			if have[name] {
+				return nil
+			}
+			have[name] = true
+			if !match(name) {
+				return nil
+			}
+			_, err = cfg.BuildContext.ImportDir(path, 0)
+			if err != nil {
+				if _, noGo := err.(*build.NoGoError); noGo {
+					return nil
+				}
+			}
+			pkgs = append(pkgs, name)
+			return nil
+		})
+	}
+	return pkgs
+}
+
+// MatchPackagesInFS returns a list of package paths matching pattern,
+// which must begin with ./ or ../
+// (see go help packages for pattern syntax).
+func MatchPackagesInFS(pattern string) []string {
+	// Find directory to begin the scan.
+	// Could be smarter but this one optimization
+	// is enough for now, since ... is usually at the
+	// end of a path.
+	i := strings.Index(pattern, "...")
+	dir, _ := path.Split(pattern[:i])
+
+	// pattern begins with ./ or ../.
+	// path.Clean will discard the ./ but not the ../.
+	// We need to preserve the ./ for pattern matching
+	// and in the returned import paths.
+	prefix := ""
+	if strings.HasPrefix(pattern, "./") {
+		prefix = "./"
+	}
+	match := matchPattern(pattern)
+
+	var pkgs []string
+	filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil || !fi.IsDir() {
+			return nil
+		}
+		if path == dir {
+			// filepath.Walk starts at dir and recurses. For the recursive case,
+			// the path is the result of filepath.Join, which calls filepath.Clean.
+			// The initial case is not Cleaned, though, so we do this explicitly.
+			//
+			// This converts a path like "./io/" to "io". Without this step, running
+			// "cd $GOROOT/src; go list ./io/..." would incorrectly skip the io
+			// package, because prepending the prefix "./" to the unclean path would
+			// result in "././io", and match("././io") returns false.
+			path = filepath.Clean(path)
+		}
+
+		// Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..".
+		_, elem := filepath.Split(path)
+		dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".."
+		if dot || strings.HasPrefix(elem, "_") || elem == "testdata" {
+			return filepath.SkipDir
+		}
+
+		name := prefix + filepath.ToSlash(path)
+		if !match(name) {
+			return nil
+		}
+
+		// We keep the directory if we can import it, or if we can't import it
+		// due to invalid Go source files. This means that directories containing
+		// parse errors will be built (and fail) instead of being silently skipped
+		// as not matching the pattern. Go 1.5 and earlier skipped, but that
+		// behavior means people miss serious mistakes.
+		// See golang.org/issue/11407.
+		if p, err := cfg.BuildContext.ImportDir(path, 0); err != nil && (p == nil || len(p.InvalidGoFiles) == 0) {
+			if _, noGo := err.(*build.NoGoError); !noGo {
+				log.Print(err)
+			}
+			return nil
+		}
+		pkgs = append(pkgs, name)
+		return nil
+	})
+	return pkgs
+}
+
+// treeCanMatchPattern(pattern)(name) reports whether
+// name or children of name can possibly match pattern.
+// Pattern is the same limited glob accepted by matchPattern.
+func treeCanMatchPattern(pattern string) func(name string) bool {
+	wildCard := false
+	if i := strings.Index(pattern, "..."); i >= 0 {
+		wildCard = true
+		pattern = pattern[:i]
+	}
+	return func(name string) bool {
+		return len(name) <= len(pattern) && hasPathPrefix(pattern, name) ||
+			wildCard && strings.HasPrefix(name, pattern)
+	}
+}
+
+// matchPattern(pattern)(name) reports whether
+// name matches pattern. Pattern is a limited glob
+// pattern in which '...' means 'any string' and there
+// is no other special syntax.
+func matchPattern(pattern string) func(name string) bool {
+	re := regexp.QuoteMeta(pattern)
+	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
+	// Special case: foo/... matches foo too.
+	if strings.HasSuffix(re, `/.*`) {
+		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
+	}
+	reg := regexp.MustCompile(`^` + re + `$`)
+	return func(name string) bool {
+		return reg.MatchString(name)
+	}
+}
+
+// ImportPaths returns the import paths to use for the given command line.
+func ImportPaths(args []string) []string {
+	args = ImportPathsNoDotExpansion(args)
+	var out []string
+	for _, a := range args {
+		if strings.Contains(a, "...") {
+			if build.IsLocalImport(a) {
+				out = append(out, allPackagesInFS(a)...)
+			} else {
+				out = append(out, allPackages(a)...)
+			}
+			continue
+		}
+		out = append(out, a)
+	}
+	return out
+}
+
+// ImportPathsNoDotExpansion returns the import paths to use for the given
+// command line, but it does no ... expansion.
+func ImportPathsNoDotExpansion(args []string) []string {
+	if len(args) == 0 {
+		return []string{"."}
+	}
+	var out []string
+	for _, a := range args {
+		// Arguments are supposed to be import paths, but
+		// as a courtesy to Windows developers, rewrite \ to /
+		// in command-line arguments. Handles .\... and so on.
+		if filepath.Separator == '\\' {
+			a = strings.Replace(a, `\`, `/`, -1)
+		}
+
+		// Put argument in canonical form, but preserve leading ./.
+		if strings.HasPrefix(a, "./") {
+			a = "./" + path.Clean(a)
+			if a == "./." {
+				a = "."
+			}
+		} else {
+			a = path.Clean(a)
+		}
+		if IsMetaPackage(a) {
+			out = append(out, allPackages(a)...)
+			continue
+		}
+		out = append(out, a)
+	}
+	return out
+}
+
+// isMetaPackage checks if name is a reserved package name that expands to multiple packages.
+func IsMetaPackage(name string) bool {
+	return name == "std" || name == "cmd" || name == "all"
+}

src/cmd/go/testgo.go → src/cmd/go/internal/load/testgo.go

@@ -10,7 +10,7 @@
 
 // +build testgo
 
-package main
+package load
 
 import "os"
 

src/cmd/go/list.go

@@ -6,8 +6,9 @@ package main
 
 import (
 	"bufio"
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/load"
 	"encoding/json"
 	"io"
 	"os"
@@ -152,9 +153,9 @@ func runList(cmd *base.Command, args []string) {
 	out := newTrackingWriter(os.Stdout)
 	defer out.w.Flush()
 
-	var do func(*Package)
+	var do func(*load.PackagePublic)
 	if *listJson {
-		do = func(p *Package) {
+		do = func(p *load.PackagePublic) {
 			b, err := json.MarshalIndent(p, "", "\t")
 			if err != nil {
 				out.Flush()
@@ -179,7 +180,7 @@ func runList(cmd *base.Command, args []string) {
 		if err != nil {
 			base.Fatalf("%s", err)
 		}
-		do = func(p *Package) {
+		do = func(p *load.PackagePublic) {
 			if err := tmpl.Execute(out, p); err != nil {
 				out.Flush()
 				base.Fatalf("%s", err)
@@ -190,17 +191,17 @@ func runList(cmd *base.Command, args []string) {
 		}
 	}
 
-	load := packages
+	loadpkgs := load.Packages
 	if *listE {
-		load = packagesAndErrors
+		loadpkgs = load.PackagesAndErrors
 	}
 
-	for _, pkg := range load(args) {
+	for _, pkg := range loadpkgs(args) {
 		// Show vendor-expanded paths in listing
-		pkg.TestImports = pkg.vendored(pkg.TestImports)
-		pkg.XTestImports = pkg.vendored(pkg.XTestImports)
+		pkg.TestImports = pkg.Vendored(pkg.TestImports)
+		pkg.XTestImports = pkg.Vendored(pkg.XTestImports)
 
-		do(pkg)
+		do(&pkg.PackagePublic)
 	}
 }
 

src/cmd/go/main.go

@@ -7,12 +7,9 @@ package main
 import (
 	"flag"
 	"fmt"
-	"go/build"
 	"log"
 	"os"
-	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"strings"
 
@@ -89,8 +86,8 @@ func main() {
 		}
 	}
 
-	if fi, err := os.Stat(goroot); err != nil || !fi.IsDir() {
-		fmt.Fprintf(os.Stderr, "go: cannot find GOROOT directory: %v\n", goroot)
+	if fi, err := os.Stat(cfg.GOROOT); err != nil || !fi.IsDir() {
+		fmt.Fprintf(os.Stderr, "go: cannot find GOROOT directory: %v\n", cfg.GOROOT)
 		os.Exit(2)
 	}
 
@@ -145,57 +142,6 @@ func mainUsage() {
 	os.Exit(2)
 }
 
-// importPathsNoDotExpansion returns the import paths to use for the given
-// command line, but it does no ... expansion.
-func importPathsNoDotExpansion(args []string) []string {
-	if len(args) == 0 {
-		return []string{"."}
-	}
-	var out []string
-	for _, a := range args {
-		// Arguments are supposed to be import paths, but
-		// as a courtesy to Windows developers, rewrite \ to /
-		// in command-line arguments. Handles .\... and so on.
-		if filepath.Separator == '\\' {
-			a = strings.Replace(a, `\`, `/`, -1)
-		}
-
-		// Put argument in canonical form, but preserve leading ./.
-		if strings.HasPrefix(a, "./") {
-			a = "./" + path.Clean(a)
-			if a == "./." {
-				a = "."
-			}
-		} else {
-			a = path.Clean(a)
-		}
-		if isMetaPackage(a) {
-			out = append(out, allPackages(a)...)
-			continue
-		}
-		out = append(out, a)
-	}
-	return out
-}
-
-// importPaths returns the import paths to use for the given command line.
-func importPaths(args []string) []string {
-	args = importPathsNoDotExpansion(args)
-	var out []string
-	for _, a := range args {
-		if strings.Contains(a, "...") {
-			if build.IsLocalImport(a) {
-				out = append(out, allPackagesInFS(a)...)
-			} else {
-				out = append(out, allPackages(a)...)
-			}
-			continue
-		}
-		out = append(out, a)
-	}
-	return out
-}
-
 // envForDir returns a copy of the environment
 // suitable for running in the given directory.
 // The environment is the current process's environment
@@ -225,235 +171,3 @@ NextVar:
 	}
 	return out
 }
-
-// matchPattern(pattern)(name) reports whether
-// name matches pattern. Pattern is a limited glob
-// pattern in which '...' means 'any string' and there
-// is no other special syntax.
-func matchPattern(pattern string) func(name string) bool {
-	re := regexp.QuoteMeta(pattern)
-	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
-	// Special case: foo/... matches foo too.
-	if strings.HasSuffix(re, `/.*`) {
-		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
-	}
-	reg := regexp.MustCompile(`^` + re + `$`)
-	return func(name string) bool {
-		return reg.MatchString(name)
-	}
-}
-
-// hasPathPrefix reports whether the path s begins with the
-// elements in prefix.
-func hasPathPrefix(s, prefix string) bool {
-	switch {
-	default:
-		return false
-	case len(s) == len(prefix):
-		return s == prefix
-	case len(s) > len(prefix):
-		if prefix != "" && prefix[len(prefix)-1] == '/' {
-			return strings.HasPrefix(s, prefix)
-		}
-		return s[len(prefix)] == '/' && s[:len(prefix)] == prefix
-	}
-}
-
-// hasFilePathPrefix reports whether the filesystem path s begins with the
-// elements in prefix.
-func hasFilePathPrefix(s, prefix string) bool {
-	sv := strings.ToUpper(filepath.VolumeName(s))
-	pv := strings.ToUpper(filepath.VolumeName(prefix))
-	s = s[len(sv):]
-	prefix = prefix[len(pv):]
-	switch {
-	default:
-		return false
-	case sv != pv:
-		return false
-	case len(s) == len(prefix):
-		return s == prefix
-	case len(s) > len(prefix):
-		if prefix != "" && prefix[len(prefix)-1] == filepath.Separator {
-			return strings.HasPrefix(s, prefix)
-		}
-		return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix
-	}
-}
-
-// expandPath returns the symlink-expanded form of path.
-func expandPath(p string) string {
-	x, err := filepath.EvalSymlinks(p)
-	if err == nil {
-		return x
-	}
-	return p
-}
-
-// treeCanMatchPattern(pattern)(name) reports whether
-// name or children of name can possibly match pattern.
-// Pattern is the same limited glob accepted by matchPattern.
-func treeCanMatchPattern(pattern string) func(name string) bool {
-	wildCard := false
-	if i := strings.Index(pattern, "..."); i >= 0 {
-		wildCard = true
-		pattern = pattern[:i]
-	}
-	return func(name string) bool {
-		return len(name) <= len(pattern) && hasPathPrefix(pattern, name) ||
-			wildCard && strings.HasPrefix(name, pattern)
-	}
-}
-
-// allPackages returns all the packages that can be found
-// under the $GOPATH directories and $GOROOT matching pattern.
-// The pattern is either "all" (all packages), "std" (standard packages),
-// "cmd" (standard commands), or a path including "...".
-func allPackages(pattern string) []string {
-	pkgs := matchPackages(pattern)
-	if len(pkgs) == 0 {
-		fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
-	}
-	return pkgs
-}
-
-func matchPackages(pattern string) []string {
-	match := func(string) bool { return true }
-	treeCanMatch := func(string) bool { return true }
-	if !isMetaPackage(pattern) {
-		match = matchPattern(pattern)
-		treeCanMatch = treeCanMatchPattern(pattern)
-	}
-
-	have := map[string]bool{
-		"builtin": true, // ignore pseudo-package that exists only for documentation
-	}
-	if !cfg.BuildContext.CgoEnabled {
-		have["runtime/cgo"] = true // ignore during walk
-	}
-	var pkgs []string
-
-	for _, src := range cfg.BuildContext.SrcDirs() {
-		if (pattern == "std" || pattern == "cmd") && src != gorootSrc {
-			continue
-		}
-		src = filepath.Clean(src) + string(filepath.Separator)
-		root := src
-		if pattern == "cmd" {
-			root += "cmd" + string(filepath.Separator)
-		}
-		filepath.Walk(root, func(path string, fi os.FileInfo, err error) error {
-			if err != nil || !fi.IsDir() || path == src {
-				return nil
-			}
-
-			// Avoid .foo, _foo, and testdata directory trees.
-			_, elem := filepath.Split(path)
-			if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
-				return filepath.SkipDir
-			}
-
-			name := filepath.ToSlash(path[len(src):])
-			if pattern == "std" && (!isStandardImportPath(name) || name == "cmd") {
-				// The name "std" is only the standard library.
-				// If the name is cmd, it's the root of the command tree.
-				return filepath.SkipDir
-			}
-			if !treeCanMatch(name) {
-				return filepath.SkipDir
-			}
-			if have[name] {
-				return nil
-			}
-			have[name] = true
-			if !match(name) {
-				return nil
-			}
-			_, err = cfg.BuildContext.ImportDir(path, 0)
-			if err != nil {
-				if _, noGo := err.(*build.NoGoError); noGo {
-					return nil
-				}
-			}
-			pkgs = append(pkgs, name)
-			return nil
-		})
-	}
-	return pkgs
-}
-
-// allPackagesInFS is like allPackages but is passed a pattern
-// beginning ./ or ../, meaning it should scan the tree rooted
-// at the given directory. There are ... in the pattern too.
-func allPackagesInFS(pattern string) []string {
-	pkgs := matchPackagesInFS(pattern)
-	if len(pkgs) == 0 {
-		fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
-	}
-	return pkgs
-}
-
-func matchPackagesInFS(pattern string) []string {
-	// Find directory to begin the scan.
-	// Could be smarter but this one optimization
-	// is enough for now, since ... is usually at the
-	// end of a path.
-	i := strings.Index(pattern, "...")
-	dir, _ := path.Split(pattern[:i])
-
-	// pattern begins with ./ or ../.
-	// path.Clean will discard the ./ but not the ../.
-	// We need to preserve the ./ for pattern matching
-	// and in the returned import paths.
-	prefix := ""
-	if strings.HasPrefix(pattern, "./") {
-		prefix = "./"
-	}
-	match := matchPattern(pattern)
-
-	var pkgs []string
-	filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || !fi.IsDir() {
-			return nil
-		}
-		if path == dir {
-			// filepath.Walk starts at dir and recurses. For the recursive case,
-			// the path is the result of filepath.Join, which calls filepath.Clean.
-			// The initial case is not Cleaned, though, so we do this explicitly.
-			//
-			// This converts a path like "./io/" to "io". Without this step, running
-			// "cd $GOROOT/src; go list ./io/..." would incorrectly skip the io
-			// package, because prepending the prefix "./" to the unclean path would
-			// result in "././io", and match("././io") returns false.
-			path = filepath.Clean(path)
-		}
-
-		// Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..".
-		_, elem := filepath.Split(path)
-		dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".."
-		if dot || strings.HasPrefix(elem, "_") || elem == "testdata" {
-			return filepath.SkipDir
-		}
-
-		name := prefix + filepath.ToSlash(path)
-		if !match(name) {
-			return nil
-		}
-
-		// We keep the directory if we can import it, or if we can't import it
-		// due to invalid Go source files. This means that directories containing
-		// parse errors will be built (and fail) instead of being silently skipped
-		// as not matching the pattern. Go 1.5 and earlier skipped, but that
-		// behavior means people miss serious mistakes.
-		// See golang.org/issue/11407.
-		if p, err := cfg.BuildContext.ImportDir(path, 0); err != nil && (p == nil || len(p.InvalidGoFiles) == 0) {
-			if _, noGo := err.(*build.NoGoError); !noGo {
-				log.Print(err)
-			}
-			return nil
-		}
-		pkgs = append(pkgs, name)
-		return nil
-	})
-	return pkgs
-}

src/cmd/go/pkg_test.go

@@ -5,8 +5,9 @@
 package main
 
 import (
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/load"
 	"cmd/go/internal/str"
 	"io/ioutil"
 	"os"
@@ -87,71 +88,79 @@ func TestParseMetaGoImports(t *testing.T) {
 	}
 }
 
+func pkgImportPath(path string) *load.Package {
+	return &load.Package{
+		PackagePublic: load.PackagePublic{
+			ImportPath: path,
+		},
+	}
+}
+
 func TestSharedLibName(t *testing.T) {
 	// TODO(avdva) - make these values platform-specific
 	prefix := "lib"
 	suffix := ".so"
 	testData := []struct {
 		args      []string
-		pkgs      []*Package
+		pkgs      []*load.Package
 		expected  string
 		expectErr bool
 		rootedAt  string
 	}{
 		{
 			args:     []string{"std"},
-			pkgs:     []*Package{},
+			pkgs:     []*load.Package{},
 			expected: "std",
 		},
 		{
 			args:     []string{"std", "cmd"},
-			pkgs:     []*Package{},
+			pkgs:     []*load.Package{},
 			expected: "std,cmd",
 		},
 		{
 			args:     []string{},
-			pkgs:     []*Package{&Package{ImportPath: "gopkg.in/somelib"}},
+			pkgs:     []*load.Package{pkgImportPath("gopkg.in/somelib")},
 			expected: "gopkg.in-somelib",
 		},
 		{
 			args:     []string{"./..."},
-			pkgs:     []*Package{&Package{ImportPath: "somelib"}},
+			pkgs:     []*load.Package{pkgImportPath("somelib")},
 			expected: "somelib",
 			rootedAt: "somelib",
 		},
 		{
 			args:     []string{"../somelib", "../somelib"},
-			pkgs:     []*Package{&Package{ImportPath: "somelib"}},
+			pkgs:     []*load.Package{pkgImportPath("somelib")},
 			expected: "somelib",
 		},
 		{
 			args:     []string{"../lib1", "../lib2"},
-			pkgs:     []*Package{&Package{ImportPath: "gopkg.in/lib1"}, &Package{ImportPath: "gopkg.in/lib2"}},
+			pkgs:     []*load.Package{pkgImportPath("gopkg.in/lib1"), pkgImportPath("gopkg.in/lib2")},
 			expected: "gopkg.in-lib1,gopkg.in-lib2",
 		},
 		{
 			args: []string{"./..."},
-			pkgs: []*Package{
-				&Package{ImportPath: "gopkg.in/dir/lib1"},
-				&Package{ImportPath: "gopkg.in/lib2"},
-				&Package{ImportPath: "gopkg.in/lib3"},
+			pkgs: []*load.Package{
+				pkgImportPath("gopkg.in/dir/lib1"),
+				pkgImportPath("gopkg.in/lib2"),
+				pkgImportPath("gopkg.in/lib3"),
 			},
 			expected: "gopkg.in",
 			rootedAt: "gopkg.in",
 		},
 		{
 			args:      []string{"std", "../lib2"},
-			pkgs:      []*Package{},
+			pkgs:      []*load.Package{},
 			expectErr: true,
 		},
 		{
 			args:      []string{"all", "./"},
-			pkgs:      []*Package{},
+			pkgs:      []*load.Package{},
 			expectErr: true,
 		},
 		{
 			args:      []string{"cmd", "fmt"},
-			pkgs:      []*Package{},
+			pkgs:      []*load.Package{},
 			expectErr: true,
 		},
 	}

src/cmd/go/run.go

@@ -5,8 +5,9 @@
 package main
 
 import (
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/load"
 	"cmd/go/internal/str"
 	"fmt"
 	"os"
@@ -87,17 +88,17 @@ func runRun(cmd *base.Command, args []string) {
 			base.Fatalf("go run: cannot run *_test.go files (%s)", file)
 		}
 	}
-	p := goFilesPackage(files)
+	p := load.GoFilesPackage(files)
 	if p.Error != nil {
 		base.Fatalf("%s", p.Error)
 	}
-	p.omitDWARF = true
+	p.Internal.OmitDWARF = true
 	if len(p.DepsErrors) > 0 {
 		// Since these are errors in dependencies,
 		// the same error might show up multiple times,
 		// once in each package that depends on it.
 		// Only print each once.
-		printed := map[*PackageError]bool{}
+		printed := map[*load.PackageError]bool{}
 		for _, err := range p.DepsErrors {
 			if !printed[err] {
 				printed[err] = true
@@ -109,7 +110,7 @@ func runRun(cmd *base.Command, args []string) {
 	if p.Name != "main" {
 		base.Fatalf("go run: cannot run non-main package")
 	}
-	p.target = "" // must build - not up to date
+	p.Internal.Target = "" // must build - not up to date
 	var src string
 	if len(p.GoFiles) > 0 {
 		src = p.GoFiles[0]
@@ -124,7 +125,7 @@ func runRun(cmd *base.Command, args []string) {
 		}
 		base.Fatalf("go run: no suitable source files%s", hint)
 	}
-	p.exeName = src[:len(src)-len(".go")] // name temporary executable for first go file
+	p.Internal.ExeName = src[:len(src)-len(".go")] // name temporary executable for first go file
 	a1 := b.action(modeBuild, modeBuild, p)
 	a := &action{f: (*builder).runProgram, args: cmdArgs, deps: []*action{a1}}
 	b.do(a)

src/cmd/go/testflag.go

@@ -5,8 +5,8 @@
 package main
 
 import (
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
 	"flag"
 	"fmt"
 	"os"

src/cmd/go/tool.go

@@ -11,8 +11,8 @@ import (
 	"sort"
 	"strings"
 
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
 )
 
 var cmdTool = &base.Command{
@@ -72,7 +72,7 @@ func runTool(cmd *base.Command, args []string) {
 		Stdout: os.Stdout,
 		Stderr: os.Stderr,
 		// Set $GOROOT, mainly for go tool dist.
-		Env: mergeEnvLists([]string{"GOROOT=" + goroot}, os.Environ()),
+		Env: mergeEnvLists([]string{"GOROOT=" + cfg.GOROOT}, os.Environ()),
 	}
 	err := toolCmd.Run()
 	if err != nil {

src/cmd/go/vet.go

@@ -7,8 +7,9 @@ package main
 import (
 	"path/filepath"
 
-	"cmd/go/internal/cfg"
 	"cmd/go/internal/base"
+	"cmd/go/internal/cfg"
+	"cmd/go/internal/load"
 	"cmd/go/internal/str"
 )
 
@@ -38,7 +39,7 @@ See also: go fmt, go fix.
 }
 
 func runVet(cmd *base.Command, args []string) {
-	for _, p := range packages(args) {
+	for _, p := range load.Packages(args) {
 		// Vet expects to be given a set of files all from the same package.
 		// Run once for package p and once for package p_test.
 		if len(p.GoFiles)+len(p.CgoFiles)+len(p.TestGoFiles) > 0 {
@@ -50,7 +51,7 @@ func runVet(cmd *base.Command, args []string) {
 	}
 }
 
-func runVetFiles(p *Package, files []string) {
+func runVetFiles(p *load.Package, files []string) {
 	for i := range files {
 		files[i] = filepath.Join(p.Dir, files[i])
 	}