cmd/dist: change mkdeps to be more merge-friendly

In addition to the obvious formatting change, this also drops
from deps.go any indirect dependencies, so that when you add
a new import to one package, the resulting diff only affects that
one package, not every package that imports that package
directly or indirectly. That makes the file a bit easier to understand,
if you need to debug it or deal with a possible merge conflict.

The code to trim the import lists (but not too much) was more
than I wanted to do in shell, so I rewrote mkdeps in Go.

The shell script is left behind for backwards-compatibility with
people who have learned to run ./mkdeps.bash (or documentation
suggesting the same).

Change-Id: I0bf27b5b27d0440e11ea830b00735c73f58eae03
Reviewed-on: https://go-review.googlesource.com/67650Reviewed-by: David Crawshaw <crawshaw@golang.org>

src/cmd/dist/mkdeps.bash

@@ -3,50 +3,12 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-set -e
-
-output="$1"
-if test -z "$output"; then
-    output=deps.go
-fi
-
-# We need to test enough GOOS/GOARCH combinations to pick up all the
-# package dependencies.
-gooslist="windows linux darwin solaris"
-goarchlist="386 amd64 arm arm64 ppc64"
-
-echo NOTE: errors about loading internal/syscall/windows are ok
+# This script regenerates deps.go.
+# The script used to do all the work, but now a Go program does.
+# The script has been preserved so that people who learned to type
+# ./mkdeps.bash don't have to relearn a new method.
+# It's fine to run "go run mkdeps.go" directly instead.
 
-deps_of() {
-	for goos in $gooslist
-	do
-		for goarch in $goarchlist
-		do
-			GOOS=$goos GOARCH=$goarch go list -tags cmd_go_bootstrap -f '{{range .Deps}}{{$.ImportPath}} {{.}}
-{{end}}' $*
-		done
-	done | sort -u | grep . | grep -v ' unsafe$'
-}
-
-all="$(deps_of cmd/go | awk '{print $2}') cmd/go"
-deps_of $all >tmp.all.deps
-
-(
-	echo '// Code generated by mkdeps.bash; DO NOT EDIT.'
-	echo
-	echo 'package main'
-	echo
-	echo 'var builddeps = map[string][]string{'
-	for pkg in $all
-	do
-		echo -n "\"$pkg\": {"
-		for dep in $(awk -v pkg=$pkg '$1==pkg {print $2}' tmp.all.deps)
-		do
-			echo -n "\"$dep\","
-		done
-		echo '},'
-	done
-	echo '}'
-) |gofmt >$output
-
-rm -f tmp.all.deps
+set -e
+go run mkdeps.go -- "$@"
+exit 0

src/cmd/dist/mkdeps.go

+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This program generates deps.go.
+// Run as "go run mkdeps.go" or, to redirect the output, "go run mkdeps.go x.txt".
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+)
+
+// We need to test enough GOOS/GOARCH combinations
+// to find all the package dependencies of cmd/go on all systems.
+var targetList = strings.Fields(`
+	linux/386
+	linux/amd64
+	windows/amd64
+`)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "usage: mkdeps [deps.go]\n")
+	os.Exit(2)
+}
+
+func main() {
+	log.SetPrefix("mkdeps: ")
+	log.SetFlags(0)
+	flag.Usage = usage
+	flag.Parse()
+	if flag.NArg() > 1 {
+		usage()
+	}
+
+	outfile := "deps.go"
+	if flag.NArg() == 1 {
+		outfile = flag.Arg(0)
+	}
+
+	_, deps := importsAndDepsOf("cmd/go")
+	all := deps["cmd/go"]
+	all = append(all, "cmd/go")
+	imports, deps := importsAndDepsOf(all...)
+
+	// Sort topologically, then by import path.
+	var topo []string
+	walked := make(map[string]bool)
+	var walk func(string)
+	walk = func(p string) {
+		if walked[p] {
+			return
+		}
+		walked[p] = true
+		sort.Strings(deps[p])
+		for _, d := range deps[p] {
+			walk(d)
+		}
+		topo = append(topo, p)
+	}
+
+	// We're only going to print imports, not deps,
+	// in hopes of making deps.go intelligible to people
+	// who need to debug it or attempt to resolve merge conflicts.
+	// For the most part, deps is just the transitive closure of imports,
+	// but sometimes there are implicit deps supplied by the go command
+	// that are not derivable from imports.
+	// Find those (if any) and copy them explicitly into imports.
+	for _, p := range topo {
+		for _, dp := range deps[p] {
+			found := false
+			for _, ip := range imports[p] {
+				if dp == ip || inList(deps[ip], dp) {
+					found = true
+					break
+				}
+			}
+			if !found {
+				imports[p] = append(imports[p], dp)
+			}
+		}
+		sort.Strings(imports[p])
+	}
+	sort.Strings(all)
+
+	// Print table.
+	var buf bytes.Buffer
+	fmt.Fprintf(&buf, "// Code generated by mkdeps.bash; DO NOT EDIT.\n\n")
+	fmt.Fprintf(&buf, "package main\n\n")
+	fmt.Fprintf(&buf, "var builddeps = map[string][]string{\n")
+	for _, p := range all {
+		if p == "unsafe" { // unsafe should not be built
+			continue
+		}
+		// We're printing a multiline format here to make the output more
+		// intelligible both to people and to merge tools.
+		// We put the name of the parent package as a comment on every line
+		// to keep a merge tool from applying the diff for one package
+		// to the dependency list for a different package.
+		// The extra blank line at the start stops any attempt by gofmt at
+		// lining up the slice literals from different packages,
+		// even if they are empty slices (on a single line with the key).
+		fmt.Fprintf(&buf, "\n\t%q: {\n", p)
+		for _, d := range imports[p] {
+			if d != "unsafe" {
+				fmt.Fprintf(&buf, "\t\t%q, // %s\n", d, p)
+			}
+		}
+		fmt.Fprintf(&buf, "\t},\n")
+	}
+	fmt.Fprintf(&buf, "\n}\n")
+
+	// Run the installed gofmt instead of using go/format,
+	// because, on the off chance they disagree,
+	// the installed gofmt binary is by definition the correct one.
+	cmd := exec.Command("gofmt")
+	cmd.Stdin = &buf
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	if err := cmd.Run(); err != nil {
+		log.Fatalf("gofmt: %v", err)
+	}
+
+	if err := ioutil.WriteFile(outfile, out.Bytes(), 0666); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func inList(xs []string, s string) bool {
+	for _, x := range xs {
+		if x == s {
+			return true
+		}
+	}
+	return false
+}
+
+// importsAndDepsOf returns two maps, one giving the imports for each package in pkgs,
+// and one giving the dependencies for each package in pkgs.
+// Both the keys and the entries in the value slices are full import paths.
+func importsAndDepsOf(pkgs ...string) (map[string][]string, map[string][]string) {
+	imports := make(map[string][]string)
+	deps := make(map[string][]string)
+	for _, target := range targetList {
+		args := []string{"list", "-tags", "cmd_go_bootstrap", "-f", "{{range .Imports}}import {{$.ImportPath}} {{.}}\n{{end}}{{range .Deps}}dep {{$.ImportPath}} {{.}}\n{{end}}"}
+		args = append(args, pkgs...)
+		cmd := exec.Command("go", args...)
+		cmd.Env = append(os.Environ(), "GOOS="+t[0], "GOARCH="+t[1])
+		out, err := cmd.Output()
+		if err != nil {
+			log.Fatalf("GOOS=%s GOARCH=%s go list: %v", t[0], t[1], err)
+		}
+		helped := false
+		for _, line := range strings.Split(string(out), "\n") {
+			f := strings.Fields(line)
+			if len(f) != 3 {
+				continue
+			}
+			if f[0] == "import" && !inList(imports[f[1]], f[2]) {
+				helped = true
+				imports[f[1]] = append(imports[f[1]], f[2])
+			}
+			if f[0] == "dep" && !inList(deps[f[1]], f[2]) {
+				helped = true
+				deps[f[1]] = append(deps[f[1]], f[2])
+			}
+		}
+		if !helped {
+			fmt.Fprintf(os.Stderr, "mkdeps: note: %s did not contribute any new dependencies\n", target)
+		}
+	}
+	return imports, deps
+}