go: implement build, install, run

clean is gone; all the intermediate files are created
in a temporary tree that is wiped when the command ends.

Not using go/build's Script because it is not well aligned
with this API.  The various builder methods are copied from
go/build and adapted.  Probably once we delete goinstall
we can delete the Script API too.

R=rogpeppe, adg, adg
CC=golang-dev
https://golang.org/cl/5483069

src/cmd/cgo/gcc.go

@@ -729,7 +729,9 @@ func (p *Package) gccMachine() []string {
 	return nil
 }
 
-var gccTmp = objDir + "_cgo_.o"
+func gccTmp() string {
+	return *objDir + "_cgo_.o"
+}
 
 // gccCmd returns the gcc command line to use for compiling
 // the input.
@@ -738,7 +740,7 @@ func (p *Package) gccCmd() []string {
 		p.gccName(),
 		"-Wall",                             // many warnings
 		"-Werror",                           // warnings are errors
-		"-o" + gccTmp,                       // write object to tmp
+		"-o" + gccTmp(),                     // write object to tmp
 		"-gdwarf-2",                         // generate DWARF v2 debugging symbols
 		"-fno-eliminate-unused-debug-types", // gets rid of e.g. untyped enum otherwise
 		"-c",                                // do not link
@@ -755,10 +757,10 @@ func (p *Package) gccCmd() []string {
 func (p *Package) gccDebug(stdin []byte) (*dwarf.Data, binary.ByteOrder, []byte) {
 	runGcc(stdin, p.gccCmd())
 
-	if f, err := macho.Open(gccTmp); err == nil {
+	if f, err := macho.Open(gccTmp()); err == nil {
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp, err)
+			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
 		}
 		var data []byte
 		if f.Symtab != nil {
@@ -784,23 +786,23 @@ func (p *Package) gccDebug(stdin []byte) (*dwarf.Data, binary.ByteOrder, []byte)
 	// Can skip debug data block in ELF and PE for now.
 	// The DWARF information is complete.
 
-	if f, err := elf.Open(gccTmp); err == nil {
+	if f, err := elf.Open(gccTmp()); err == nil {
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp, err)
+			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
 		}
 		return d, f.ByteOrder, nil
 	}
 
-	if f, err := pe.Open(gccTmp); err == nil {
+	if f, err := pe.Open(gccTmp()); err == nil {
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp, err)
+			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
 		}
 		return d, binary.LittleEndian, nil
 	}
 
-	fatalf("cannot parse gcc output %s as ELF, Mach-O, PE object", gccTmp)
+	fatalf("cannot parse gcc output %s as ELF, Mach-O, PE object", gccTmp())
 	panic("not reached")
 }
 

src/cmd/cgo/main.go

@@ -123,12 +123,14 @@ var cPrefix string
 var fset = token.NewFileSet()
 
 var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file")
+var dynout = flag.String("dynout", "", "write -dynobj output to this file")
 
 // These flags are for bootstrapping a new Go implementation,
 // to generate Go and C headers that match the data layout and
 // constant values used in the host's C libraries and system calls.
 var godefs = flag.Bool("godefs", false, "for bootstrap: write Go definitions for C file to standard output")
 var cdefs = flag.Bool("cdefs", false, "for bootstrap: write C definitions for C file to standard output")
+var objDir = flag.String("objdir", "", "object directory")
 
 var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo")
 
@@ -202,9 +204,13 @@ func main() {
 		fs[i] = f
 	}
 
-	// make sure that _obj directory exists, so that we can write
-	// all the output files there.
-	os.Mkdir("_obj", 0777)
+	if *objDir == "" {
+		// make sure that _obj directory exists, so that we can write
+		// all the output files there.
+		os.Mkdir("_obj", 0777)
+		*objDir = "_obj"
+	}
+	*objDir += string(filepath.Separator)
 
 	for i, input := range goFiles {
 		f := fs[i]

src/cmd/cgo/out.go

@@ -14,20 +14,17 @@ import (
 	"go/printer"
 	"go/token"
 	"os"
-	"path/filepath"
 	"strings"
 )
 
-var objDir = "_obj" + string(filepath.Separator)
-
 // writeDefs creates output files to be compiled by 6g, 6c, and gcc.
 // (The comments here say 6g and 6c but the code applies to the 8 and 5 tools too.)
 func (p *Package) writeDefs() {
-	fgo2 := creat(objDir + "_cgo_gotypes.go")
-	fc := creat(objDir + "_cgo_defun.c")
-	fm := creat(objDir + "_cgo_main.c")
+	fgo2 := creat(*objDir + "_cgo_gotypes.go")
+	fc := creat(*objDir + "_cgo_defun.c")
+	fm := creat(*objDir + "_cgo_main.c")
 
-	fflg := creat(objDir + "_cgo_flags")
+	fflg := creat(*objDir + "_cgo_flags")
 	for k, v := range p.CgoFlags {
 		fmt.Fprintf(fflg, "_CGO_%s=%s\n", k, v)
 	}
@@ -109,6 +106,15 @@ func (p *Package) writeDefs() {
 }
 
 func dynimport(obj string) {
+	stdout := os.Stdout
+	if *dynout != "" {
+		f, err := os.Create(*dynout)
+		if err != nil {
+			fatalf("%s", err)
+		}
+		stdout = f
+	}
+
 	if f, err := elf.Open(obj); err == nil {
 		sym, err := f.ImportedSymbols()
 		if err != nil {
@@ -119,14 +125,14 @@ func dynimport(obj string) {
 			if s.Version != "" {
 				targ += "@" + s.Version
 			}
-			fmt.Printf("#pragma dynimport %s %s %q\n", s.Name, targ, s.Library)
+			fmt.Fprintf(stdout, "#pragma dynimport %s %s %q\n", s.Name, targ, s.Library)
 		}
 		lib, err := f.ImportedLibraries()
 		if err != nil {
 			fatalf("cannot load imported libraries from ELF file %s: %v", obj, err)
 		}
 		for _, l := range lib {
-			fmt.Printf("#pragma dynimport _ _ %q\n", l)
+			fmt.Fprintf(stdout, "#pragma dynimport _ _ %q\n", l)
 		}
 		return
 	}
@@ -140,14 +146,14 @@ func dynimport(obj string) {
 			if len(s) > 0 && s[0] == '_' {
 				s = s[1:]
 			}
-			fmt.Printf("#pragma dynimport %s %s %q\n", s, s, "")
+			fmt.Fprintf(stdout, "#pragma dynimport %s %s %q\n", s, s, "")
 		}
 		lib, err := f.ImportedLibraries()
 		if err != nil {
 			fatalf("cannot load imported libraries from Mach-O file %s: %v", obj, err)
 		}
 		for _, l := range lib {
-			fmt.Printf("#pragma dynimport _ _ %q\n", l)
+			fmt.Fprintf(stdout, "#pragma dynimport _ _ %q\n", l)
 		}
 		return
 	}
@@ -159,7 +165,7 @@ func dynimport(obj string) {
 		}
 		for _, s := range sym {
 			ss := strings.Split(s, ":")
-			fmt.Printf("#pragma dynimport %s %s %q\n", ss[0], ss[0], strings.ToLower(ss[1]))
+			fmt.Fprintf(stdout, "#pragma dynimport %s %s %q\n", ss[0], ss[0], strings.ToLower(ss[1]))
 		}
 		return
 	}
@@ -307,8 +313,8 @@ func (p *Package) writeOutput(f *File, srcfile string) {
 		base = base[0 : len(base)-3]
 	}
 	base = strings.Map(slashToUnderscore, base)
-	fgo1 := creat(objDir + base + ".cgo1.go")
-	fgcc := creat(objDir + base + ".cgo2.c")
+	fgo1 := creat(*objDir + base + ".cgo1.go")
+	fgcc := creat(*objDir + base + ".cgo2.c")
 
 	p.GoFiles = append(p.GoFiles, base+".cgo1.go")
 	p.GccFiles = append(p.GccFiles, base+".cgo2.c")
@@ -383,7 +389,7 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
 // Write out the various stubs we need to support functions exported
 // from Go so that they are callable from C.
 func (p *Package) writeExports(fgo2, fc, fm *os.File) {
-	fgcc := creat(objDir + "_cgo_export.c")
+	fgcc := creat(*objDir + "_cgo_export.c")
 	fgcch := creat("_cgo_export.h")
 
 	fmt.Fprintf(fgcch, "/* Created by cgo - DO NOT EDIT. */\n")

src/cmd/go/Makefile

@@ -7,7 +7,6 @@ include ../../Make.inc
 TARG=go
 GOFILES=\
 	build.go\
-	clean.go\
 	fix.go\
 	get.go\
 	fmt.go\
@@ -15,6 +14,7 @@ GOFILES=\
 	list.go\
 	main.go\
 	pkg.go\
+	run.go\
 	test.go\
 	version.go\
 	vet.go\

src/cmd/go/clean.go

-// Copyright 2011 The Go Authors.  All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-var cmdClean = &Command{
-	Run:       runClean,
-	UsageLine: "clean [-nuke] [importpath...]",
-	Short:     "remove intermediate objects",
-	Long: `
-Clean removes intermediate object files generated during
-the compilation of the packages named by the import paths,
-but by default it does not remove the installed package binaries.
-
-The -nuke flag causes clean to remove the installed package binaries too.
-
-TODO: Clean does not clean dependencies of the packages.
-TODO: Rename -nuke.
-
-For more about import paths, see 'go help importpath'.
-	`,
-}
-
-var cleanNuke = cmdClean.Flag.Bool("nuke", false, "")
-
-func runClean(cmd *Command, args []string) {
-	args = importPaths(args)
-	_ = args
-	panic("nuke not implemented")
-}

src/cmd/go/list.go

@@ -58,12 +58,19 @@ func init() {
 
 var listFmt = cmdList.Flag.String("f", "{{.Name}} {{.Dir}}", "")
 var listJson = cmdList.Flag.Bool("json", false, "")
+var nl = []byte{'\n'}
 
 func runList(cmd *Command, args []string) {
 	var do func(*Package)
 	if *listJson {
-		enc := json.NewEncoder(os.Stdout)
-		do = func(p *Package) { enc.Encode(p) }
+		do = func(p *Package) {
+			b, err := json.MarshalIndent(p, "", "\t")
+			if err != nil {
+				fatalf("%s", err)
+			}
+			os.Stdout.Write(b)
+			os.Stdout.Write(nl)
+		}
 	} else {
 		tmpl, err := template.New("main").Parse(*listFmt + "\n")
 		if err != nil {

src/cmd/go/main.go

@@ -7,10 +7,12 @@ package main
 import (
 	"flag"
 	"fmt"
+	"go/build"
 	"io"
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"text/template"
 )
@@ -56,13 +58,13 @@ func (c *Command) Usage() {
 // The order here is the order in which they are printed by 'go help'.
 var commands = []*Command{
 	cmdBuild,
-	cmdClean,
 	cmdDoc,
 	cmdFix,
 	cmdFmt,
 	cmdGet,
 	cmdInstall,
 	cmdList,
+	cmdRun,
 	cmdTest,
 	cmdVersion,
 	cmdVet,
@@ -95,7 +97,7 @@ func main() {
 			cmd.Flag.Parse(args[1:])
 			args = cmd.Flag.Args()
 			cmd.Run(cmd, args)
-			os.Exit(exitStatus)
+			exit()
 			return
 		}
 	}
@@ -173,16 +175,31 @@ func help(args []string) {
 
 // importPaths returns the import paths to use for the given command line.
 func importPaths(args []string) []string {
-	// TODO: "all"
+	if len(args) == 1 && args[0] == "all" {
+		return allPackages()
+	}
 	if len(args) == 0 {
 		return []string{"."}
 	}
 	return args
 }
 
+var atexitFuncs []func()
+
+func atexit(f func()) {
+	atexitFuncs = append(atexitFuncs, f)
+}
+
+func exit() {
+	for _, f := range atexitFuncs {
+		f()
+	}
+	os.Exit(exitStatus)
+}
+
 func fatalf(format string, args ...interface{}) {
-	log.Printf(format, args...)
-	os.Exit(1)
+	errorf(format, args...)
+	exit()
 }
 
 func errorf(format string, args ...interface{}) {
@@ -192,7 +209,7 @@ func errorf(format string, args ...interface{}) {
 
 func exitIfErrors() {
 	if exitStatus != 0 {
-		os.Exit(exitStatus)
+		exit()
 	}
 }
 
@@ -204,3 +221,52 @@ func run(cmdline ...string) {
 		errorf("%v", err)
 	}
 }
+
+// allPackages returns all the packages that can be found
+// under the $GOPATH directories and $GOROOT.
+func allPackages() []string {
+	have := make(map[string]bool)
+	var pkgs []string
+	runtime := filepath.Join(build.Path[0].SrcDir(), "runtime") + string(filepath.Separator)
+	for _, t := range build.Path {
+		src := t.SrcDir() + string(filepath.Separator)
+		filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
+			if err != nil || !fi.IsDir() {
+				return nil
+			}
+
+			// Avoid testdata directory trees.
+			if strings.HasSuffix(path, string(filepath.Separator)+"testdata") {
+				return filepath.SkipDir
+			}
+			// Avoid runtime subdirectories.
+			if strings.HasPrefix(path, runtime) {
+				switch path {
+				case runtime + "darwin", runtime + "freebsd", runtime + "linux", runtime + "netbsd", runtime + "openbsd", runtime + "windows":
+					return filepath.SkipDir
+				}
+			}
+
+			_, err = build.ScanDir(path)
+			if err != nil {
+				return nil
+			}
+			name := path[len(src):]
+			if have[name] {
+				return nil
+			}
+			pkgs = append(pkgs, name)
+			have[name] = true
+
+			// Avoid go/build test data.
+			if path == filepath.Join(build.Path[0].SrcDir(), "go/build") {
+				return filepath.SkipDir
+			}
+
+			return nil
+		})
+
+		// TODO: Commands.
+	}
+	return pkgs
+}

src/cmd/go/pkg.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"go/build"
 	"go/doc"
+	"os"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -40,6 +41,7 @@ type Package struct {
 	info    *build.DirInfo
 	imports []*Package
 	gofiles []string // GoFiles+CgoFiles
+	targ    string
 }
 
 // packageCache is a lookup cache for loadPackage,
@@ -66,10 +68,21 @@ func loadPackage(arg string) (*Package, error) {
 
 	// Find basic information about package path.
 	t, importPath, err := build.FindTree(arg)
+	// Maybe it is a standard command.
+	if err != nil && !filepath.IsAbs(arg) && !strings.HasPrefix(arg, ".") {
+		goroot := build.Path[0]
+		p := filepath.Join(goroot.Path, "src/cmd", arg)
+		if st, err1 := os.Stat(p); err1 == nil && st.IsDir() {
+			t = goroot
+			importPath = "../cmd/" + arg
+			err = nil
+		}
+	}
 	if err != nil {
 		return nil, err
 	}
-	dir := filepath.Join(t.SrcDir(), importPath)
+
+	dir := filepath.Join(t.SrcDir(), filepath.FromSlash(importPath))
 
 	// Maybe we know the package by its directory.
 	if p := packageCache[dir]; p != nil {
@@ -79,13 +92,25 @@ func loadPackage(arg string) (*Package, error) {
 		return p, nil
 	}
 
+	return scanPackage(&build.DefaultContext, t, arg, importPath, dir)
+}
+
+func scanPackage(ctxt *build.Context, t *build.Tree, arg, importPath, dir string) (*Package, error) {
 	// Read the files in the directory to learn the structure
 	// of the package.
-	info, err := build.ScanDir(dir)
+	info, err := ctxt.ScanDir(dir)
 	if err != nil {
 		return nil, err
 	}
 
+	var targ string
+	if info.Package == "main" {
+		_, elem := filepath.Split(importPath)
+		targ = filepath.Join(t.BinDir(), elem)
+	} else {
+		targ = filepath.Join(t.PkgDir(), filepath.FromSlash(importPath)+".a")
+	}
+
 	p := &Package{
 		Name:       info.Package,
 		Doc:        doc.CommentText(info.PackageComment),
@@ -97,6 +122,9 @@ func loadPackage(arg string) (*Package, error) {
 		SFiles:     info.SFiles,
 		CgoFiles:   info.CgoFiles,
 		Standard:   t.Goroot && !strings.Contains(importPath, "."),
+		targ:       targ,
+		t:          t,
+		info:       info,
 	}
 
 	// Build list of full paths to all Go files in the package,
@@ -123,7 +151,7 @@ func loadPackage(arg string) (*Package, error) {
 		}
 		p1, err := loadPackage(path)
 		if err != nil {
-			delete(packageCache, arg)
+			delete(packageCache, dir)
 			delete(packageCache, importPath)
 			// Add extra error detail to show full import chain.
 			// Always useful, but especially useful in import loops.

src/cmd/go/run.go

+// Copyright 2011 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import ()
+
+// Break init loop.
+func init() {
+	cmdRun.Run = runRun
+}
+
+var cmdRun = &Command{
+	UsageLine: "run [-a] [-n] [-v] gofiles...",
+	Short:     "compile and run Go program",
+	Long: `
+Run compiles and runs the main package comprising the named Go source files.
+
+The -a flag forces reinstallation of packages that are already up-to-date.
+The -n flag prints the commands but does not run them.
+The -v flag prints the commands.
+
+See also: go build.
+	`,
+}
+
+var runA = cmdRun.Flag.Bool("a", false, "")
+var runN = cmdRun.Flag.Bool("n", false, "")
+var runV = cmdRun.Flag.Bool("v", false, "")
+
+func runRun(cmd *Command, args []string) {
+	var b builder
+	b.init(*runA, *runN, *runV)
+	p := goFilesPackage(args, "")
+	p.targ = "" // force rebuild - no up-to-date copy anywhere
+	a1 := b.action(modeBuild, modeBuild, p)
+	a := &action{f: (*builder).runProgram, deps: []*action{a1}}
+	b.do(a)
+}
+
+// runProgram is the action for running a binary that has already
+// been compiled.  We ignore exit status.
+func (b *builder) runProgram(a *action) error {
+	run(a.deps[0].pkgbin)
+	return nil
+}