cmd/go: use -importcfg to invoke compiler, linker

This is a step toward using cached build artifacts: the importcfg
will direct the compiler and linker to read them right from the cache
if necessary. However, this CL does not have a cache yet, so it still
reads them from the usual install location or build location.
Even so, this fixes a long-standing issue that -I and -L (no longer used)
are not expressive enough to describe complex GOPATH setups.

Shared libraries are handled enough that all.bash passes, but
there may still be more work to do here. If so, tests and fixes
can be added in follow-up CLs.

Gccgo will need updating to support -importcfg as well.

Fixes #14271.

Change-Id: I5c52a0a5df0ffbf7436e1130c74e9e24fceff80f
Reviewed-on: https://go-review.googlesource.com/56279
Run-TryBot: Russ Cox <rsc@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: David Crawshaw <crawshaw@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

misc/cgo/testshared/shared_test.go

@@ -47,7 +47,7 @@ func run(t *testing.T, msg string, args ...string) {
 func goCmd(t *testing.T, args ...string) {
 	newargs := []string{args[0], "-installsuffix=" + suffix}
 	if testing.Verbose() {
-		newargs = append(newargs, "-v")
+		newargs = append(newargs, "-x")
 	}
 	newargs = append(newargs, args[1:]...)
 	c := exec.Command("go", newargs...)
@@ -58,6 +58,7 @@ func goCmd(t *testing.T, args ...string) {
 		c.Stdout = os.Stdout
 		c.Stderr = os.Stderr
 		err = c.Run()
+		output = []byte("(output above)")
 	} else {
 		output, err = c.CombinedOutput()
 	}

src/cmd/go/internal/cfg/cfg.go

@@ -128,3 +128,28 @@ func isGOROOT(path string) bool {
 	}
 	return stat.IsDir()
 }
+
+// ExternalLinkingForced reports whether external linking is being
+// forced even for programs that do not use cgo.
+func ExternalLinkingForced() bool {
+	if !BuildContext.CgoEnabled {
+		return false
+	}
+	// Currently build modes c-shared, pie (on systems that do not
+	// support PIE with internal linking mode (currently all
+	// systems: issue #18968)), plugin, and -linkshared force
+	// external linking mode, as of course does
+	// -ldflags=-linkmode=external. External linking mode forces
+	// an import of runtime/cgo.
+	pieCgo := BuildBuildmode == "pie"
+	linkmodeExternal := false
+	for i, a := range BuildLdflags {
+		if a == "-linkmode=external" {
+			linkmodeExternal = true
+		}
+		if a == "-linkmode" && i+1 < len(BuildLdflags) && BuildLdflags[i+1] == "external" {
+			linkmodeExternal = true
+		}
+	}
+	return BuildBuildmode == "c-shared" || BuildBuildmode == "plugin" || pieCgo || BuildLinkshared || linkmodeExternal
+}

src/cmd/go/internal/load/pkg.go

@@ -99,6 +99,7 @@ type PackageInternal struct {
 	SFiles       []string
 	AllGoFiles   []string             // gofiles + IgnoredGoFiles, absolute paths
 	Target       string               // installed file for this package (may be executable)
+	Pkgfile      string               // where package will be (or is already) built or installed
 	Fake         bool                 // synthesized package
 	External     bool                 // synthesized external test package
 	ForceLibrary bool                 // this package is a library (even if named "main")
@@ -951,26 +952,8 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
 		importPaths = append(importPaths, "syscall")
 	}
 
-	if cfg.BuildContext.CgoEnabled && p.Name == "main" && !p.Goroot {
-		// Currently build modes c-shared, pie (on systems that do not
-		// support PIE with internal linking mode (currently all
-		// systems: issue #18968)), plugin, and -linkshared force
-		// external linking mode, as of course does
-		// -ldflags=-linkmode=external. External linking mode forces
-		// an import of runtime/cgo.
-		pieCgo := cfg.BuildBuildmode == "pie"
-		linkmodeExternal := false
-		for i, a := range cfg.BuildLdflags {
-			if a == "-linkmode=external" {
-				linkmodeExternal = true
-			}
-			if a == "-linkmode" && i+1 < len(cfg.BuildLdflags) && cfg.BuildLdflags[i+1] == "external" {
-				linkmodeExternal = true
-			}
-		}
-		if cfg.BuildBuildmode == "c-shared" || cfg.BuildBuildmode == "plugin" || pieCgo || cfg.BuildLinkshared || linkmodeExternal {
-			importPaths = append(importPaths, "runtime/cgo")
-		}
+	if p.Name == "main" && !p.Goroot && cfg.ExternalLinkingForced() {
+		importPaths = append(importPaths, "runtime/cgo")
 	}
 
 	// Everything depends on runtime, except runtime, its internal

src/cmd/go/internal/test/test.go

@@ -887,7 +887,11 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin
 
 	// The generated main also imports testing, regexp, and os.
 	stk.Push("testmain")
-	for _, dep := range testMainDeps {
+	deps := testMainDeps
+	if cfg.ExternalLinkingForced() {
+		deps = str.StringList(deps, "runtime/cgo")
+	}
+	for _, dep := range deps {
 		if dep == ptest.ImportPath {
 			pmain.Internal.Imports = append(pmain.Internal.Imports, ptest)
 		} else {

src/cmd/link/internal/ld/ld.go

@@ -116,6 +116,16 @@ func findlib(ctxt *Link, lib string) (string, bool) {
 			pname = name
 		} else {
 			pkg := pkgname(lib)
+			// Add .a if needed; the new -importcfg modes
+			// do not put .a into the package name anymore.
+			// This only matters when people try to mix
+			// compiles using -importcfg with links not using -importcfg,
+			// such as when running quick things like
+			// 'go tool compile x.go && go tool link x.o'
+			// by hand against a standard library built using -importcfg.
+			if !strings.HasSuffix(name, ".a") && !strings.HasSuffix(name, ".o") {
+				name += ".a"
+			}
 			// try dot, -L "libdir", and then goroot.
 			for _, dir := range ctxt.Libdir {
 				if *FlagLinkshared {
@@ -163,14 +173,15 @@ func addlib(ctxt *Link, src string, obj string, lib string) *Library {
  *	objref: object file referring to package
  *	file: object file, e.g., /home/rsc/go/pkg/container/vector.a
  *	pkg: package import path, e.g. container/vector
+ *	shlib: path to shared library, or .shlibname file holding path
  */
-func addlibpath(ctxt *Link, srcref string, objref string, file string, pkg string, shlibnamefile string) *Library {
+func addlibpath(ctxt *Link, srcref string, objref string, file string, pkg string, shlib string) *Library {
 	if l := ctxt.LibraryByPkg[pkg]; l != nil {
 		return l
 	}
 
 	if ctxt.Debugvlog > 1 {
-		ctxt.Logf("%5.2f addlibpath: srcref: %s objref: %s file: %s pkg: %s shlibnamefile: %s\n", Cputime(), srcref, objref, file, pkg, shlibnamefile)
+		ctxt.Logf("%5.2f addlibpath: srcref: %s objref: %s file: %s pkg: %s shlib: %s\n", Cputime(), srcref, objref, file, pkg, shlib)
 	}
 
 	l := &Library{}
@@ -180,12 +191,15 @@ func addlibpath(ctxt *Link, srcref string, objref string, file string, pkg strin
 	l.Srcref = srcref
 	l.File = file
 	l.Pkg = pkg
-	if shlibnamefile != "" {
-		shlibbytes, err := ioutil.ReadFile(shlibnamefile)
-		if err != nil {
-			Errorf(nil, "cannot read %s: %v", shlibnamefile, err)
+	if shlib != "" {
+		if strings.HasSuffix(shlib, ".shlibname") {
+			data, err := ioutil.ReadFile(shlib)
+			if err != nil {
+				Errorf(nil, "cannot read %s: %v", shlib, err)
+			}
+			shlib = strings.TrimSpace(string(data))
 		}
-		l.Shlib = strings.TrimSpace(string(shlibbytes))
+		l.Shlib = shlib
 	}
 	return l
 }

src/cmd/link/internal/ld/lib.go

@@ -338,8 +338,8 @@ func errorexit() {
 
 func loadinternal(ctxt *Link, name string) *Library {
 	if *FlagLinkshared && ctxt.PackageShlib != nil {
-		if shlibname := ctxt.PackageShlib[name]; shlibname != "" {
-			return addlibpath(ctxt, "internal", "internal", "", name, shlibname)
+		if shlib := ctxt.PackageShlib[name]; shlib != "" {
+			return addlibpath(ctxt, "internal", "internal", "", name, shlib)
 		}
 	}
 	if ctxt.PackageFile != nil {
@@ -423,22 +423,23 @@ func (ctxt *Link) loadlib() {
 		loadinternal(ctxt, "runtime/msan")
 	}
 
-	var i int
-	for i = 0; i < len(ctxt.Library); i++ {
-		if ctxt.Library[i].Shlib == "" {
+	// ctxt.Library grows during the loop, so not a range loop.
+	for i := 0; i < len(ctxt.Library); i++ {
+		lib := ctxt.Library[i]
+		if lib.Shlib == "" {
 			if ctxt.Debugvlog > 1 {
-				ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), ctxt.Library[i].File, ctxt.Library[i].Objref)
+				ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.File, lib.Objref)
 			}
-			objfile(ctxt, ctxt.Library[i])
+			objfile(ctxt, lib)
 		}
 	}
 
-	for i = 0; i < len(ctxt.Library); i++ {
-		if ctxt.Library[i].Shlib != "" {
+	for _, lib := range ctxt.Library {
+		if lib.Shlib != "" {
 			if ctxt.Debugvlog > 1 {
-				ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), ctxt.Library[i].Shlib, ctxt.Library[i].Objref)
+				ctxt.Logf("%5.2f autolib: %s (from %s)\n", Cputime(), lib.Shlib, lib.Objref)
 			}
-			ldshlibsyms(ctxt, ctxt.Library[i].Shlib)
+			ldshlibsyms(ctxt, lib.Shlib)
 		}
 	}
 
@@ -461,21 +462,19 @@ func (ctxt *Link) loadlib() {
 		toc.Type = SDYNIMPORT
 	}
 
-	if Linkmode == LinkExternal && !iscgo {
+	if Linkmode == LinkExternal && !iscgo && ctxt.LibraryByPkg["runtime/cgo"] == nil {
 		// This indicates a user requested -linkmode=external.
 		// The startup code uses an import of runtime/cgo to decide
 		// whether to initialize the TLS.  So give it one. This could
 		// be handled differently but it's an unusual case.
-		loadinternal(ctxt, "runtime/cgo")
-
-		if i < len(ctxt.Library) {
-			if ctxt.Library[i].Shlib != "" {
-				ldshlibsyms(ctxt, ctxt.Library[i].Shlib)
+		if lib := loadinternal(ctxt, "runtime/cgo"); lib != nil {
+			if lib.Shlib != "" {
+				ldshlibsyms(ctxt, lib.Shlib)
 			} else {
 				if Buildmode == BuildmodeShared || *FlagLinkshared {
 					Exitf("cannot implicitly include runtime/cgo in a shared library")
 				}
-				objfile(ctxt, ctxt.Library[i])
+				objfile(ctxt, lib)
 			}
 		}
 	}
@@ -633,9 +632,9 @@ func (ctxt *Link) loadlib() {
 	// If package versioning is required, generate a hash of the
 	// the packages used in the link.
 	if Buildmode == BuildmodeShared || Buildmode == BuildmodePlugin || ctxt.Syms.ROLookup("plugin.Open", 0) != nil {
-		for i = 0; i < len(ctxt.Library); i++ {
-			if ctxt.Library[i].Shlib == "" {
-				genhash(ctxt, ctxt.Library[i])
+		for _, lib := range ctxt.Library {
+			if lib.Shlib == "" {
+				genhash(ctxt, lib)
 			}
 		}
 	}
@@ -1538,6 +1537,9 @@ func readnote(f *elf.File, name []byte, typ int32) ([]byte, error) {
 }
 
 func findshlib(ctxt *Link, shlib string) string {
+	if filepath.IsAbs(shlib) {
+		return shlib
+	}
 	for _, libdir := range ctxt.Libdir {
 		libpath := filepath.Join(libdir, shlib)
 		if _, err := os.Stat(libpath); err == nil {
@@ -1549,9 +1551,15 @@ func findshlib(ctxt *Link, shlib string) string {
 }
 
 func ldshlibsyms(ctxt *Link, shlib string) {
-	libpath := findshlib(ctxt, shlib)
-	if libpath == "" {
-		return
+	var libpath string
+	if filepath.IsAbs(shlib) {
+		libpath = shlib
+		shlib = filepath.Base(shlib)
+	} else {
+		libpath = findshlib(ctxt, shlib)
+		if libpath == "" {
+			return
+		}
 	}
 	for _, processedlib := range ctxt.Shlibs {
 		if processedlib.Path == libpath {
@@ -1580,7 +1588,22 @@ func ldshlibsyms(ctxt *Link, shlib string) {
 		Errorf(nil, "cannot read dep list from shared library %s: %v", libpath, err)
 		return
 	}
-	deps := strings.Split(string(depsbytes), "\n")
+	var deps []string
+	for _, dep := range strings.Split(string(depsbytes), "\n") {
+		if dep == "" {
+			continue
+		}
+		if !filepath.IsAbs(dep) {
+			// If the dep can be interpreted as a path relative to the shlib
+			// in which it was found, do that. Otherwise, we will leave it
+			// to be resolved by libdir lookup.
+			abs := filepath.Join(filepath.Dir(libpath), dep)
+			if _, err := os.Stat(abs); err == nil {
+				dep = abs
+			}
+		}
+		deps = append(deps, dep)
+	}
 
 	syms, err := f.DynamicSymbols()
 	if err != nil {