cmd/compile: always write pack files

By always writing out pack files, the object file format can be
simplified somewhat. In particular, the export data format will no
longer require escaping, because the pack file provides appropriate
framing.

This CL does not affect build systems that use -pack, which includes
all major Go build systems (cmd/go, gb, bazel).

Also, existing package import logic already distinguishes pack/object
files based on file contents rather than file extension.

The only exception is cmd/pack, which specially handled object files
created by cmd/compile when used with the 'c' mode. This mode is
extended to now recognize the pack files produced by cmd/compile and
handle them as before.

Passes toolstash-check.

Updates #21705.
Updates #24512.

Change-Id: Idf131013bfebd73a5cde7e087eb19964503a9422
Reviewed-on: https://go-review.googlesource.com/102236
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

src/cmd/compile/internal/gc/main.go

@@ -211,7 +211,7 @@ func Main(archInit func(*Arch)) {
 	flag.BoolVar(&nolocalimports, "nolocalimports", false, "reject local (relative) imports")
 	flag.StringVar(&outfile, "o", "", "write output to `file`")
 	flag.StringVar(&myimportpath, "p", "", "set expected package import `path`")
-	flag.BoolVar(&writearchive, "pack", false, "write package file instead of object file")
+	flag.BoolVar(&writearchive, "pack", false, "write to file.a instead of file.o")
 	objabi.Flagcount("r", "debug generated wrappers", &Debug['r'])
 	flag.BoolVar(&flag_race, "race", false, "enable race detector")
 	objabi.Flagcount("s", "warn about composite literals that can be simplified", &Debug['s'])

src/cmd/compile/internal/gc/obj.go

@@ -62,69 +62,68 @@ func dumpobj1(outfile string, mode int) {
 		errorexit()
 	}
 	defer bout.Close()
+	bout.WriteString("!<arch>\n")
 
-	startobj := int64(0)
-	var arhdr [ArhdrSize]byte
-	if writearchive {
-		bout.WriteString("!<arch>\n")
-		arhdr = [ArhdrSize]byte{}
-		bout.Write(arhdr[:])
-		startobj = bout.Offset()
+	if mode&modeCompilerObj != 0 {
+		start := startArchiveEntry(bout)
+		dumpCompilerObj(bout)
+		finishArchiveEntry(bout, start, "__.PKGDEF")
 	}
+	if mode&modeLinkerObj != 0 {
+		start := startArchiveEntry(bout)
+		dumpLinkerObj(bout)
+		finishArchiveEntry(bout, start, "_go_.o")
+	}
+}
 
-	printheader := func() {
-		fmt.Fprintf(bout, "go object %s %s %s %s\n", objabi.GOOS, objabi.GOARCH, objabi.Version, objabi.Expstring())
-		if buildid != "" {
-			fmt.Fprintf(bout, "build id %q\n", buildid)
-		}
-		if localpkg.Name == "main" {
-			fmt.Fprintf(bout, "main\n")
-		}
-		if safemode {
-			fmt.Fprintf(bout, "safe\n")
-		} else {
-			fmt.Fprintf(bout, "----\n") // room for some other tool to write "safe"
-		}
-		fmt.Fprintf(bout, "\n") // header ends with blank line
+func printObjHeader(bout *bio.Writer) {
+	fmt.Fprintf(bout, "go object %s %s %s %s\n", objabi.GOOS, objabi.GOARCH, objabi.Version, objabi.Expstring())
+	if buildid != "" {
+		fmt.Fprintf(bout, "build id %q\n", buildid)
+	}
+	if localpkg.Name == "main" {
+		fmt.Fprintf(bout, "main\n")
+	}
+	if safemode {
+		fmt.Fprintf(bout, "safe\n")
+	} else {
+		fmt.Fprintf(bout, "----\n") // room for some other tool to write "safe"
 	}
+	fmt.Fprintf(bout, "\n") // header ends with blank line
+}
 
-	printheader()
+func startArchiveEntry(bout *bio.Writer) int64 {
+	var arhdr [ArhdrSize]byte
+	bout.Write(arhdr[:])
+	return bout.Offset()
+}
 
-	if mode&modeCompilerObj != 0 {
-		dumpexport(bout)
+func finishArchiveEntry(bout *bio.Writer, start int64, name string) {
+	bout.Flush()
+	size := bout.Offset() - start
+	if size&1 != 0 {
+		bout.WriteByte(0)
 	}
+	bout.Seek(start-ArhdrSize, 0)
 
-	if writearchive {
-		bout.Flush()
-		size := bout.Offset() - startobj
-		if size&1 != 0 {
-			bout.WriteByte(0)
-		}
-		bout.Seek(startobj-ArhdrSize, 0)
-		formathdr(arhdr[:], "__.PKGDEF", size)
-		bout.Write(arhdr[:])
-		bout.Flush()
-		bout.Seek(startobj+size+(size&1), 0)
-	}
+	var arhdr [ArhdrSize]byte
+	formathdr(arhdr[:], name, size)
+	bout.Write(arhdr[:])
+	bout.Flush()
+	bout.Seek(start+size+(size&1), 0)
+}
 
-	if mode&modeLinkerObj == 0 {
-		return
-	}
+func dumpCompilerObj(bout *bio.Writer) {
+	printObjHeader(bout)
+	dumpexport(bout)
+}
 
-	if writearchive {
-		// start object file
-		arhdr = [ArhdrSize]byte{}
-		bout.Write(arhdr[:])
-		startobj = bout.Offset()
-		printheader()
-	}
+func dumpLinkerObj(bout *bio.Writer) {
+	printObjHeader(bout)
 
 	if pragcgobuf != "" {
-		if writearchive {
-			// write empty export section; must be before cgo section
-			fmt.Fprintf(bout, "\n$$\n\n$$\n\n")
-		}
-
+		// write empty export section; must be before cgo section
+		fmt.Fprintf(bout, "\n$$\n\n$$\n\n")
 		fmt.Fprintf(bout, "\n$$  // cgo\n")
 		fmt.Fprintf(bout, "%s\n$$\n\n", pragcgobuf)
 	}
@@ -158,17 +157,6 @@ func dumpobj1(outfile string, mode int) {
 	addGCLocals()
 
 	obj.WriteObjFile(Ctxt, bout.Writer)
-
-	if writearchive {
-		bout.Flush()
-		size := bout.Offset() - startobj
-		if size&1 != 0 {
-			bout.WriteByte(0)
-		}
-		bout.Seek(startobj-ArhdrSize, 0)
-		formathdr(arhdr[:], "_go_.o", size)
-		bout.Write(arhdr[:])
-	}
 }
 
 func addptabs() {

src/cmd/pack/pack.go

@@ -5,13 +5,11 @@
 package main
 
 import (
-	"bufio"
-	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"log"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -332,11 +330,26 @@ func (ar *Archive) addFiles() {
 		if verbose {
 			fmt.Printf("%s\n", file)
 		}
-		fd, err := os.Open(file)
-		if err != nil {
-			log.Fatal(err)
+
+		if !isGoCompilerObjFile(file) {
+			fd, err := os.Open(file)
+			if err != nil {
+				log.Fatal(err)
+			}
+			ar.addFile(fd)
+			continue
 		}
-		ar.addFile(fd)
+
+		aro := archive(file, os.O_RDONLY, nil)
+		aro.scan(func(entry *Entry) {
+			if entry.name != "_go_.o" {
+				aro.skip(entry)
+				return
+			}
+			ar.startFile(filepath.Base(file), 0, 0, 0, 0644, entry.size)
+			aro.output(entry, ar.fd)
+			ar.endFile()
+		})
 	}
 	ar.files = nil
 }
@@ -397,61 +410,29 @@ func (ar *Archive) endFile() {
 // from the first Go object file on the file list, if any.
 // The archive is known to be empty.
 func (ar *Archive) addPkgdef() {
+	done := false
 	for _, file := range ar.files {
-		pkgdef, err := readPkgdef(file)
-		if err != nil {
+		if !isGoCompilerObjFile(file) {
 			continue
 		}
-		if verbose {
-			fmt.Printf("__.PKGDEF # %s\n", file)
-		}
-		ar.startFile("__.PKGDEF", 0, 0, 0, 0644, int64(len(pkgdef)))
-		_, err = ar.fd.Write(pkgdef)
-		if err != nil {
-			log.Fatal("writing __.PKGDEF: ", err)
-		}
-		ar.endFile()
-		break
-	}
-}
-
-// readPkgdef extracts the __.PKGDEF data from a Go object file.
-func readPkgdef(file string) (data []byte, err error) {
-	f, err := os.Open(file)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	// Read from file, collecting header for __.PKGDEF.
-	// The header is from the beginning of the file until a line
-	// containing just "!". The first line must begin with "go object ".
-	//
-	// Note: It's possible for "\n!\n" to appear within the binary
-	// package export data format. To avoid truncating the package
-	// definition prematurely (issue 21703), we keep keep track of
-	// how many "$$" delimiters we've seen.
-
-	rbuf := bufio.NewReader(f)
-	var wbuf bytes.Buffer
-	markers := 0
-	for {
-		line, err := rbuf.ReadBytes('\n')
-		if err != nil {
-			return nil, err
-		}
-		if wbuf.Len() == 0 && !bytes.HasPrefix(line, []byte("go object ")) {
-			return nil, errors.New("not a Go object file")
-		}
-		if markers%2 == 0 && bytes.Equal(line, []byte("!\n")) {
+		aro := archive(file, os.O_RDONLY, nil)
+		aro.scan(func(entry *Entry) {
+			if entry.name != "__.PKGDEF" {
+				aro.skip(entry)
+				return
+			}
+			if verbose {
+				fmt.Printf("__.PKGDEF # %s\n", file)
+			}
+			ar.startFile("__.PKGDEF", 0, 0, 0, 0644, entry.size)
+			aro.output(entry, ar.fd)
+			ar.endFile()
+			done = true
+		})
+		if done {
 			break
 		}
-		if bytes.HasPrefix(line, []byte("$$")) {
-			markers++
-		}
-		wbuf.Write(line)
 	}
-	return wbuf.Bytes(), nil
 }
 
 // exactly16Bytes truncates the string if necessary so it is at most 16 bytes long,
@@ -514,3 +495,67 @@ func (ar *Archive) extractContents(entry *Entry) {
 		ar.skip(entry)
 	}
 }
+
+// isGoCompilerObjFile reports whether file is an object file created
+// by the Go compiler.
+func isGoCompilerObjFile(file string) bool {
+	fd, err := os.Open(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Check for "!<arch>\n" header.
+	buf := make([]byte, len(arHeader))
+	_, err = io.ReadFull(fd, buf)
+	if err != nil {
+		if err == io.EOF {
+			return false
+		}
+		log.Fatal(err)
+	}
+	if string(buf) != arHeader {
+		return false
+	}
+
+	// Check for exactly two entries: "__.PKGDEF" and "_go_.o".
+	match := []string{"__.PKGDEF", "_go_.o"}
+	buf = make([]byte, entryLen)
+	for {
+		_, err := io.ReadFull(fd, buf)
+		if err != nil {
+			if err == io.EOF {
+				// No entries left.
+				return true
+			}
+			log.Fatal(err)
+		}
+		if buf[entryLen-2] != '`' || buf[entryLen-1] != '\n' {
+			return false
+		}
+
+		name := strings.TrimRight(string(buf[:16]), " ")
+		for {
+			if len(match) == 0 {
+				return false
+			}
+			var next string
+			next, match = match[0], match[1:]
+			if name == next {
+				break
+			}
+		}
+
+		size, err := strconv.ParseInt(strings.TrimRight(string(buf[48:58]), " "), 10, 64)
+		if err != nil {
+			return false
+		}
+		if size&1 != 0 {
+			size++
+		}
+
+		_, err = fd.Seek(size, io.SeekCurrent)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+}