cmd/compile/internal/gc: compact binary export format

The binary import/export format is significantly more compact than the existing textual format. It should also be faster to read and write (to be measured). Use -newexport to enable, for instance: export GO_GCFLAGS=-newexport; make.bash The compiler can import packages using both the old and the new format ("mixed mode"). Missing: export info for inlined functions bodies (performance issue, does not affect correctness). Disabled by default until we have inlined function bodies and confirmation of no regression and equality of binaries. For #6110. For #1909. This change depends on: https://go-review.googlesource.com/16220 https://go-review.googlesource.com/16222 (already submitted) for all.bash to work. Some initial export data sizes for std lib packages. This data is without exported functions with inlineable function bodies. Package old new new/old archive/tar.................................13875.....3883 28% archive/zip.................................19464.....5046 26% bufio....................................... 7733.....2222 29% bytes.......................................10342.....3347 32% cmd/addr2line.................................242.......26 11% cmd/api.....................................39305....10368 26% cmd/asm/internal/arch.......................27732.....7939 29% cmd/asm/internal/asm........................35264....10295 29% cmd/asm/internal/flags........................629......178 28% cmd/asm/internal/lex........................39248....11128 28% cmd/asm.......................................306.......26 8% cmd/cgo.....................................40197....10570 26% cmd/compile/internal/amd64...................1106......214 19% cmd/compile/internal/arm....................27891.....7710 28% cmd/compile/internal/arm64....................891......153 17% cmd/compile/internal/big....................21637.....8336 39% cmd/compile/internal/gc....................109845....29727 27% cmd/compile/internal/mips64...................972......168 17% cmd/compile/internal/ppc64....................972......168 17% cmd/compile/internal/x86.....................1104......195 18% cmd/compile...................................329.......26 8% cmd/cover...................................12986.....3749 29% cmd/dist......................................477.......67 14% cmd/doc.....................................23043.....6793 29% cmd/expdump...................................167.......26 16% cmd/fix......................................1190......208 17% cmd/go......................................26399.....5629 21% cmd/gofmt.....................................499.......26 5% cmd/internal/gcprog..........................1342......490 37% cmd/internal/goobj...........................2690......980 36% cmd/internal/obj/arm........................32740....10057 31% cmd/internal/obj/arm64......................46542....15364 33% cmd/internal/obj/mips.......................42140....13731 33% cmd/internal/obj/ppc64......................42140....13731 33% cmd/internal/obj/x86........................52732....19015 36% cmd/internal/obj............................36729....11690 32% cmd/internal/objfile........................36365....10287 28% cmd/link/internal/amd64.....................45893....12220 27% cmd/link/internal/arm.........................307.......96 31% cmd/link/internal/arm64.......................345.......98 28% cmd/link/internal/ld.......................109300....46326 42% cmd/link/internal/ppc64.......................344.......99 29% cmd/link/internal/x86.........................334......107 32% cmd/link......................................314.......26 8% cmd/newlink..................................8110.....2544 31% cmd/nm........................................210.......26 12% cmd/objdump...................................244.......26 11% cmd/pack....................................14248.....4066 29% cmd/pprof/internal/commands..................5239.....1285 25% cmd/pprof/internal/driver...................37967.....8860 23% cmd/pprof/internal/fetch....................30962.....7337 24% cmd/pprof/internal/plugin...................47734.....7719 16% cmd/pprof/internal/profile..................22286.....6922 31% cmd/pprof/internal/report...................31187.....7838 25% cmd/pprof/internal/svg.......................4315......965 22% cmd/pprof/internal/symbolizer...............30051.....7397 25% cmd/pprof/internal/symbolz..................28545.....6949 24% cmd/pprof/internal/tempfile.................12550.....3356 27% cmd/pprof.....................................563.......26 5% cmd/trace....................................1455......636 44% cmd/vendor/golang.org/x/arch/arm/armasm....168035....64737 39% cmd/vendor/golang.org/x/arch/x86/x86asm.....26871.....8578 32% cmd/vet.....................................38980.....9913 25% cmd/vet/whitelist.............................102.......49 48% cmd/yacc.....................................2518......926 37% compress/bzip2...............................6326......129 2% compress/flate...............................7069.....2541 36% compress/gzip...............................20143.....5069 25% compress/lzw..................................828......295 36% compress/zlib...............................10676.....2692 25% container/heap................................523......181 35% container/list...............................3517......740 21% container/ring................................881......229 26% crypto/aes....................................550......187 34% crypto/cipher................................1966......825 42% crypto.......................................1836......646 35% crypto/des....................................632......235 37% crypto/dsa..................................18718.....5035 27% crypto/ecdsa................................23131.....6097 26% crypto/elliptic.............................20790.....5740 28% crypto/hmac...................................455......186 41% crypto/md5...................................1375......171 12% crypto/rand.................................18132.....4748 26% crypto/rc4....................................561......240 43% crypto/rsa..................................22094.....6380 29% crypto/sha1..................................1416......172 12% crypto/sha256.................................551......238 43% crypto/sha512.................................839......378 45% crypto/subtle................................1153......250 22% crypto/tls..................................58203....17984 31% crypto/x509/pkix............................29447.....8161 28% database/sql/driver..........................3318.....1096 33% database/sql................................11258.....3942 35% debug/dwarf.................................18416.....7006 38% debug/elf...................................57530....21014 37% debug/gosym..................................4992.....2058 41% debug/macho.................................23037.....6538 28% debug/pe....................................21063.....6619 31% debug/plan9obj...............................2467......802 33% encoding/ascii85.............................1523......360 24% encoding/asn1................................1718......527 31% encoding/base32..............................2642......686 26% encoding/base64..............................3077......800 26% encoding/binary..............................4727.....1040 22% encoding/csv................................12223.....2850 23% encoding......................................383......217 57% encoding/gob................................37563....10113 27% encoding/hex.................................1327......390 29% encoding/json...............................30897.....7804 25% encoding/pem..................................595......200 34% encoding/xml................................37798.....9336 25% errors........................................274.......36 13% expvar.......................................3155.....1021 32% flag........................................19860.....2849 14% fmt..........................................3137.....1263 40% go/ast......................................44729....13422 30% go/build....................................16336.....4657 29% go/constant..................................3703......846 23% go/doc.......................................9877.....2807 28% go/format....................................5472.....1575 29% go/importer..................................4980.....1301 26% go/internal/gccgoimporter....................5587.....1525 27% go/internal/gcimporter.......................8979.....2186 24% go/parser...................................20692.....5304 26% go/printer...................................7015.....2029 29% go/scanner...................................9719.....2824 29% go/token.....................................7933.....2465 31% go/types....................................64569....19978 31% hash/adler32.................................1176......176 15% hash/crc32...................................1663......360 22% hash/crc64...................................1587......306 19% hash/fnv.....................................3964......260 7% hash..........................................591......278 47% html..........................................217.......74 34% html/template...............................69623....12588 18% image/color/palette...........................315.......98 31% image/color..................................5565.....1036 19% image/draw...................................6917.....1028 15% image/gif....................................8894.....1654 19% image/internal/imageutil.....................9112.....1476 16% image/jpeg...................................6647.....1026 15% image/png....................................6906.....1069 15% image.......................................28992.....6139 21% index/suffixarray...........................17106.....4773 28% internal/singleflight........................1614......506 31% internal/testenv............................12212.....3152 26% internal/trace...............................2762.....1323 48% io/ioutil...................................13502.....3682 27% io...........................................6765.....2482 37% log.........................................11620.....3317 29% log/syslog..................................13516.....3821 28% math/big....................................21819.....8320 38% math/cmplx...................................2816......438 16% math/rand....................................2317......929 40% math.........................................7511.....2444 33% mime/multipart..............................12679.....3360 27% mime/quotedprintable.........................5458.....1235 23% mime.........................................6076.....1628 27% net/http/cgi................................59796....17173 29% net/http/cookiejar..........................14781.....3739 25% net/http/fcgi...............................57861....16426 28% net/http/httptest...........................84100....24365 29% net/http/httputil...........................67763....18869 28% net/http/internal............................6907......637 9% net/http/pprof..............................57945....16316 28% net/http....................................95391....30210 32% net/internal/socktest........................4555.....1453 32% net/mail....................................14481.....3608 25% net/rpc/jsonrpc.............................33335......988 3% net/rpc.....................................79950....23106 29% net/smtp....................................57790....16468 28% net/textproto...............................11356.....3248 29% net/url......................................3123.....1009 32% os/exec.....................................20738.....5769 28% os/signal.....................................437......167 38% os..........................................24875.....6668 27% path/filepath...............................11340.....2826 25% path..........................................778......285 37% reflect.....................................15469.....5198 34% regexp......................................13627.....4661 34% regexp/syntax................................5539.....2249 41% runtime/debug................................9275.....2322 25% runtime/pprof................................1355......477 35% runtime/race...................................39.......17 44% runtime/trace.................................228.......92 40% runtime.....................................13498.....1821 13% sort.........................................2848......842 30% strconv......................................2947.....1252 42% strings......................................7983.....2456 31% sync/atomic..................................2666.....1149 43% sync.........................................2568......845 33% syscall.....................................81252....38398 47% testing/iotest...............................2444......302 12% testing/quick...............................18890.....5076 27% testing.....................................16502.....4800 29% text/scanner.................................6849.....2052 30% text/tabwriter...............................6607.....1863 28% text/template/parse.........................22978.....6183 27% text/template...............................64153....11518 18% time........................................12103.....3546 29% unicode......................................9706.....3320 34% unicode/utf16................................1055......148 14% unicode/utf8.................................1118......513 46% vendor/golang.org/x/net/http2/hpack..........8905.....2636 30% All packages 3518505 1017774 29% Change-Id: Id657334f276383ff1e6fa91472d3d1db5a03349c Reviewed-on: https://go-review.googlesource.com/13937 Run-TryBot: Robert Griesemer <gri@golang.org> Reviewed-by: Chris Manghane <cmang@golang.org>

cmd/compile/internal/gc: compact binary export format
The binary import/export format is significantly more compact than the existing textual format. It should also be faster to read and write (to be measured). Use -newexport to enable, for instance: export GO_GCFLAGS=-newexport; make.bash The compiler can import packages using both the old and the new format ("mixed mode"). Missing: export info for inlined functions bodies (performance issue, does not affect correctness). Disabled by default until we have inlined function bodies and confirmation of no regression and equality of binaries. For #6110. For #1909. This change depends on: https://go-review.googlesource.com/16220 https://go-review.googlesource.com/16222 (already submitted) for all.bash to work. Some initial export data sizes for std lib packages. This data is without exported functions with inlineable function bodies. Package old new new/old archive/tar.................................13875.....3883 28% archive/zip.................................19464.....5046 26% bufio....................................... 7733.....2222 29% bytes.......................................10342.....3347 32% cmd/addr2line.................................242.......26 11% cmd/api.....................................39305....10368 26% cmd/asm/internal/arch.......................27732.....7939 29% cmd/asm/internal/asm........................35264....10295 29% cmd/asm/internal/flags........................629......178 28% cmd/asm/internal/lex........................39248....11128 28% cmd/asm.......................................306.......26 8% cmd/cgo.....................................40197....10570 26% cmd/compile/internal/amd64...................1106......214 19% cmd/compile/internal/arm....................27891.....7710 28% cmd/compile/internal/arm64....................891......153 17% cmd/compile/internal/big....................21637.....8336 39% cmd/compile/internal/gc....................109845....29727 27% cmd/compile/internal/mips64...................972......168 17% cmd/compile/internal/ppc64....................972......168 17% cmd/compile/internal/x86.....................1104......195 18% cmd/compile...................................329.......26 8% cmd/cover...................................12986.....3749 29% cmd/dist......................................477.......67 14% cmd/doc.....................................23043.....6793 29% cmd/expdump...................................167.......26 16% cmd/fix......................................1190......208 17% cmd/go......................................26399.....5629 21% cmd/gofmt.....................................499.......26 5% cmd/internal/gcprog..........................1342......490 37% cmd/internal/goobj...........................2690......980 36% cmd/internal/obj/arm........................32740....10057 31% cmd/internal/obj/arm64......................46542....15364 33% cmd/internal/obj/mips.......................42140....13731 33% cmd/internal/obj/ppc64......................42140....13731 33% cmd/internal/obj/x86........................52732....19015 36% cmd/internal/obj............................36729....11690 32% cmd/internal/objfile........................36365....10287 28% cmd/link/internal/amd64.....................45893....12220 27% cmd/link/internal/arm.........................307.......96 31% cmd/link/internal/arm64.......................345.......98 28% cmd/link/internal/ld.......................109300....46326 42% cmd/link/internal/ppc64.......................344.......99 29% cmd/link/internal/x86.........................334......107 32% cmd/link......................................314.......26 8% cmd/newlink..................................8110.....2544 31% cmd/nm........................................210.......26 12% cmd/objdump...................................244.......26 11% cmd/pack....................................14248.....4066 29% cmd/pprof/internal/commands..................5239.....1285 25% cmd/pprof/internal/driver...................37967.....8860 23% cmd/pprof/internal/fetch....................30962.....7337 24% cmd/pprof/internal/plugin...................47734.....7719 16% cmd/pprof/internal/profile..................22286.....6922 31% cmd/pprof/internal/report...................31187.....7838 25% cmd/pprof/internal/svg.......................4315......965 22% cmd/pprof/internal/symbolizer...............30051.....7397 25% cmd/pprof/internal/symbolz..................28545.....6949 24% cmd/pprof/internal/tempfile.................12550.....3356 27% cmd/pprof.....................................563.......26 5% cmd/trace....................................1455......636 44% cmd/vendor/golang.org/x/arch/arm/armasm....168035....64737 39% cmd/vendor/golang.org/x/arch/x86/x86asm.....26871.....8578 32% cmd/vet.....................................38980.....9913 25% cmd/vet/whitelist.............................102.......49 48% cmd/yacc.....................................2518......926 37% compress/bzip2...............................6326......129 2% compress/flate...............................7069.....2541 36% compress/gzip...............................20143.....5069 25% compress/lzw..................................828......295 36% compress/zlib...............................10676.....2692 25% container/heap................................523......181 35% container/list...............................3517......740 21% container/ring................................881......229 26% crypto/aes....................................550......187 34% crypto/cipher................................1966......825 42% crypto.......................................1836......646 35% crypto/des....................................632......235 37% crypto/dsa..................................18718.....5035 27% crypto/ecdsa................................23131.....6097 26% crypto/elliptic.............................20790.....5740 28% crypto/hmac...................................455......186 41% crypto/md5...................................1375......171 12% crypto/rand.................................18132.....4748 26% crypto/rc4....................................561......240 43% crypto/rsa..................................22094.....6380 29% crypto/sha1..................................1416......172 12% crypto/sha256.................................551......238 43% crypto/sha512.................................839......378 45% crypto/subtle................................1153......250 22% crypto/tls..................................58203....17984 31% crypto/x509/pkix............................29447.....8161 28% database/sql/driver..........................3318.....1096 33% database/sql................................11258.....3942 35% debug/dwarf.................................18416.....7006 38% debug/elf...................................57530....21014 37% debug/gosym..................................4992.....2058 41% debug/macho.................................23037.....6538 28% debug/pe....................................21063.....6619 31% debug/plan9obj...............................2467......802 33% encoding/ascii85.............................1523......360 24% encoding/asn1................................1718......527 31% encoding/base32..............................2642......686 26% encoding/base64..............................3077......800 26% encoding/binary..............................4727.....1040 22% encoding/csv................................12223.....2850 23% encoding......................................383......217 57% encoding/gob................................37563....10113 27% encoding/hex.................................1327......390 29% encoding/json...............................30897.....7804 25% encoding/pem..................................595......200 34% encoding/xml................................37798.....9336 25% errors........................................274.......36 13% expvar.......................................3155.....1021 32% flag........................................19860.....2849 14% fmt..........................................3137.....1263 40% go/ast......................................44729....13422 30% go/build....................................16336.....4657 29% go/constant..................................3703......846 23% go/doc.......................................9877.....2807 28% go/format....................................5472.....1575 29% go/importer..................................4980.....1301 26% go/internal/gccgoimporter....................5587.....1525 27% go/internal/gcimporter.......................8979.....2186 24% go/parser...................................20692.....5304 26% go/printer...................................7015.....2029 29% go/scanner...................................9719.....2824 29% go/token.....................................7933.....2465 31% go/types....................................64569....19978 31% hash/adler32.................................1176......176 15% hash/crc32...................................1663......360 22% hash/crc64...................................1587......306 19% hash/fnv.....................................3964......260 7% hash..........................................591......278 47% html..........................................217.......74 34% html/template...............................69623....12588 18% image/color/palette...........................315.......98 31% image/color..................................5565.....1036 19% image/draw...................................6917.....1028 15% image/gif....................................8894.....1654 19% image/internal/imageutil.....................9112.....1476 16% image/jpeg...................................6647.....1026 15% image/png....................................6906.....1069 15% image.......................................28992.....6139 21% index/suffixarray...........................17106.....4773 28% internal/singleflight........................1614......506 31% internal/testenv............................12212.....3152 26% internal/trace...............................2762.....1323 48% io/ioutil...................................13502.....3682 27% io...........................................6765.....2482 37% log.........................................11620.....3317 29% log/syslog..................................13516.....3821 28% math/big....................................21819.....8320 38% math/cmplx...................................2816......438 16% math/rand....................................2317......929 40% math.........................................7511.....2444 33% mime/multipart..............................12679.....3360 27% mime/quotedprintable.........................5458.....1235 23% mime.........................................6076.....1628 27% net/http/cgi................................59796....17173 29% net/http/cookiejar..........................14781.....3739 25% net/http/fcgi...............................57861....16426 28% net/http/httptest...........................84100....24365 29% net/http/httputil...........................67763....18869 28% net/http/internal............................6907......637 9% net/http/pprof..............................57945....16316 28% net/http....................................95391....30210 32% net/internal/socktest........................4555.....1453 32% net/mail....................................14481.....3608 25% net/rpc/jsonrpc.............................33335......988 3% net/rpc.....................................79950....23106 29% net/smtp....................................57790....16468 28% net/textproto...............................11356.....3248 29% net/url......................................3123.....1009 32% os/exec.....................................20738.....5769 28% os/signal.....................................437......167 38% os..........................................24875.....6668 27% path/filepath...............................11340.....2826 25% path..........................................778......285 37% reflect.....................................15469.....5198 34% regexp......................................13627.....4661 34% regexp/syntax................................5539.....2249 41% runtime/debug................................9275.....2322 25% runtime/pprof................................1355......477 35% runtime/race...................................39.......17 44% runtime/trace.................................228.......92 40% runtime.....................................13498.....1821 13% sort.........................................2848......842 30% strconv......................................2947.....1252 42% strings......................................7983.....2456 31% sync/atomic..................................2666.....1149 43% sync.........................................2568......845 33% syscall.....................................81252....38398 47% testing/iotest...............................2444......302 12% testing/quick...............................18890.....5076 27% testing.....................................16502.....4800 29% text/scanner.................................6849.....2052 30% text/tabwriter...............................6607.....1863 28% text/template/parse.........................22978.....6183 27% text/template...............................64153....11518 18% time........................................12103.....3546 29% unicode......................................9706.....3320 34% unicode/utf16................................1055......148 14% unicode/utf8.................................1118......513 46% vendor/golang.org/x/net/http2/hpack..........8905.....2636 30% All packages 3518505 1017774 29% Change-Id: Id657334f276383ff1e6fa91472d3d1db5a03349c Reviewed-on: https://go-review.googlesource.com/13937 Run-TryBot: Robert Griesemer <gri@golang.org> Reviewed-by: Chris Manghane <cmang@golang.org>
ae2f54a7 · Robert Griesemer · 28ef4c38 · ae2f54a7 · ae2f54a7 · ae2f54a7
Commit ae2f54a7 authored Aug 14, 2015 by Robert Griesemer
14 changed files
--- a/src/cmd/compile/internal/gc/bexport.go
+++ b/src/cmd/compile/internal/gc/bexport.go
--- a/src/cmd/compile/internal/gc/bimport.go
+++ b/src/cmd/compile/internal/gc/bimport.go
--- a/src/cmd/compile/internal/gc/dcl.go
+++ b/src/cmd/compile/internal/gc/dcl.go
@@ -874,11 +874,18 @@ func checkdupfields(t *Type, what string) {
 * a type for struct/interface/arglist
 */
 func tostruct(l *NodeList) *Type {
-	var f *Type
 	t := typ(TSTRUCT)
+	tostruct0(t, l)
+	return t
+}
+
+func tostruct0(t *Type, l *NodeList) {
+	if t == nil || t.Etype != TSTRUCT {
+		Fatalf("struct expected")
+	}

 	for tp := &t.Type; l != nil; l = l.Next {
-		f = structfield(l.N)
+		f := structfield(l.N)

 		*tp = f
 		tp = &f.Down
@@ -896,8 +903,6 @@ func tostruct(l *NodeList) *Type {
 	if !t.Broke {
 		checkwidth(t)
 	}
-
-	return t
 }

 func tofunargs(l *NodeList) *Type {
@@ -996,18 +1001,23 @@ func interfacefield(n *Node) *Type {
 }

 func tointerface(l *NodeList) *Type {
-	var f *Type
-	var t1 *Type
-
 	t := typ(TINTER)
+	tointerface0(t, l)
+	return t
+}
+
+func tointerface0(t *Type, l *NodeList) *Type {
+	if t == nil || t.Etype != TINTER {
+		Fatalf("interface expected")
+	}

 	tp := &t.Type
 	for ; l != nil; l = l.Next {
-		f = interfacefield(l.N)
+		f := interfacefield(l.N)

 		if l.N.Left == nil && f.Type.Etype == TINTER {
 			// embedded interface, inline methods
-			for t1 = f.Type.Type; t1 != nil; t1 = t1.Down {
+			for t1 := f.Type.Type; t1 != nil; t1 = t1.Down {
 				f = typ(TFIELD)
 				f.Type = t1.Type
 				f.Broke = t1.Broke
@@ -1200,6 +1210,14 @@ func isifacemethod(f *Type) bool {
 */
 func functype(this *Node, in *NodeList, out *NodeList) *Type {
 	t := typ(TFUNC)
+	functype0(t, this, in, out)
+	return t
+}
+
+func functype0(t *Type, this *Node, in *NodeList, out *NodeList) {
+	if t == nil || t.Etype != TFUNC {
+		Fatalf("function type expected")
+	}

 	var rcvr *NodeList
 	if this != nil {
@@ -1230,8 +1248,6 @@ func functype(this *Node, in *NodeList, out *NodeList) *Type {
 			t.Outnamed = true
 		}
 	}
-
-	return t
 }

 var methodsym_toppkg *Pkg

--- a/src/cmd/compile/internal/gc/export.go
+++ b/src/cmd/compile/internal/gc/export.go
@@ -5,6 +5,7 @@
 package gc

 import (
+	"bytes"
 	"cmd/internal/obj"
 	"fmt"
 	"sort"
@@ -12,6 +13,20 @@ import (
 	"unicode/utf8"
 )

+var (
+	newexport    int // if set, use new export format
+	Debug_export int // if set, print debugging information about export data
+	exportsize   int
+)
+
+func exportf(format string, args ...interface{}) {
+	n, _ := fmt.Fprintf(bout, format, args...)
+	exportsize += n
+	if Debug_export != 0 {
+		fmt.Printf(format, args...)
+	}
+}
+
 var asmlist *NodeList

 // Mark n's symbol as exported
@@ -35,8 +50,8 @@ func exportsym(n *Node) {
 }

 func exportname(s string) bool {
-	if s[0] < utf8.RuneSelf {
-		return 'A' <= s[0] && s[0] <= 'Z'
+	if r := s[0]; r < utf8.RuneSelf {
+		return 'A' <= r && r <= 'Z'
 	}
 	r, _ := utf8.DecodeRuneInString(s)
 	return unicode.IsUpper(r)
@@ -87,7 +102,7 @@ func dumppkg(p *Pkg) {
 	if !p.Direct {
 		suffix = " // indirect"
 	}
-	fmt.Fprintf(bout, "\timport %s %q%s\n", p.Name, p.Path, suffix)
+	exportf("\timport %s %q%s\n", p.Name, p.Path, suffix)
 }

 // Look for anything we need for the inline body
@@ -128,7 +143,7 @@ func reexportdep(n *Node) {
 			}
 		}

-		// Local variables in the bodies need their type.
+	// Local variables in the bodies need their type.
 	case ODCL:
 		t := n.Left.Type

@@ -167,7 +182,7 @@ func reexportdep(n *Node) {
 			exportlist = append(exportlist, n)
 		}

-		// for operations that need a type when rendered, put the type on the export list.
+	// for operations that need a type when rendered, put the type on the export list.
 	case OCONV,
 		OCONVIFACE,
 		OCONVNOP,
@@ -216,9 +231,9 @@ func dumpexportconst(s *Sym) {
 	dumpexporttype(t)

 	if t != nil && !isideal(t) {
-		fmt.Fprintf(bout, "\tconst %v %v = %v\n", Sconv(s, obj.FmtSharp), Tconv(t, obj.FmtSharp), Vconv(n.Val(), obj.FmtSharp))
+		exportf("\tconst %v %v = %v\n", Sconv(s, obj.FmtSharp), Tconv(t, obj.FmtSharp), Vconv(n.Val(), obj.FmtSharp))
 	} else {
-		fmt.Fprintf(bout, "\tconst %v = %v\n", Sconv(s, obj.FmtSharp), Vconv(n.Val(), obj.FmtSharp))
+		exportf("\tconst %v = %v\n", Sconv(s, obj.FmtSharp), Vconv(n.Val(), obj.FmtSharp))
 	}
 }

@@ -242,14 +257,14 @@ func dumpexportvar(s *Sym) {
 			}

 			// NOTE: The space after %#S here is necessary for ld's export data parser.
-			fmt.Fprintf(bout, "\tfunc %v %v { %v }\n", Sconv(s, obj.FmtSharp), Tconv(t, obj.FmtShort|obj.FmtSharp), Hconv(n.Func.Inl, obj.FmtSharp))
+			exportf("\tfunc %v %v { %v }\n", Sconv(s, obj.FmtSharp), Tconv(t, obj.FmtShort|obj.FmtSharp), Hconv(n.Func.Inl, obj.FmtSharp))

 			reexportdeplist(n.Func.Inl)
 		} else {
-			fmt.Fprintf(bout, "\tfunc %v %v\n", Sconv(s, obj.FmtSharp), Tconv(t, obj.FmtShort|obj.FmtSharp))
+			exportf("\tfunc %v %v\n", Sconv(s, obj.FmtSharp), Tconv(t, obj.FmtShort|obj.FmtSharp))
 		}
 	} else {
-		fmt.Fprintf(bout, "\tvar %v %v\n", Sconv(s, obj.FmtSharp), Tconv(t, obj.FmtSharp))
+		exportf("\tvar %v %v\n", Sconv(s, obj.FmtSharp), Tconv(t, obj.FmtSharp))
 	}
 }

@@ -287,10 +302,10 @@ func dumpexporttype(t *Type) {
 	}
 	sort.Sort(methodbyname(m))

-	fmt.Fprintf(bout, "\ttype %v %v\n", Sconv(t.Sym, obj.FmtSharp), Tconv(t, obj.FmtSharp|obj.FmtLong))
+	exportf("\ttype %v %v\n", Sconv(t.Sym, obj.FmtSharp), Tconv(t, obj.FmtSharp|obj.FmtLong))
 	for _, f := range m {
 		if f.Nointerface {
-			fmt.Fprintf(bout, "\t//go:nointerface\n")
+			exportf("\t//go:nointerface\n")
 		}
 		if f.Type.Nname != nil && f.Type.Nname.Func.Inl != nil { // nname was set by caninl

@@ -299,10 +314,10 @@ func dumpexporttype(t *Type) {
 			if Debug['l'] < 2 {
 				typecheckinl(f.Type.Nname)
 			}
-			fmt.Fprintf(bout, "\tfunc (%v) %v %v { %v }\n", Tconv(getthisx(f.Type).Type, obj.FmtSharp), Sconv(f.Sym, obj.FmtShort|obj.FmtByte|obj.FmtSharp), Tconv(f.Type, obj.FmtShort|obj.FmtSharp), Hconv(f.Type.Nname.Func.Inl, obj.FmtSharp))
+			exportf("\tfunc (%v) %v %v { %v }\n", Tconv(getthisx(f.Type).Type, obj.FmtSharp), Sconv(f.Sym, obj.FmtShort|obj.FmtByte|obj.FmtSharp), Tconv(f.Type, obj.FmtShort|obj.FmtSharp), Hconv(f.Type.Nname.Func.Inl, obj.FmtSharp))
 			reexportdeplist(f.Type.Nname.Func.Inl)
 		} else {
-			fmt.Fprintf(bout, "\tfunc (%v) %v %v\n", Tconv(getthisx(f.Type).Type, obj.FmtSharp), Sconv(f.Sym, obj.FmtShort|obj.FmtByte|obj.FmtSharp), Tconv(f.Type, obj.FmtShort|obj.FmtSharp))
+			exportf("\tfunc (%v) %v %v\n", Tconv(getthisx(f.Type).Type, obj.FmtSharp), Sconv(f.Sym, obj.FmtShort|obj.FmtByte|obj.FmtSharp), Tconv(f.Type, obj.FmtShort|obj.FmtSharp))
 		}
 	}
 }
@@ -341,33 +356,83 @@ func dumpsym(s *Sym) {
 }

 func dumpexport() {
-	lno := lineno
-
 	if buildid != "" {
-		fmt.Fprintf(bout, "build id %q\n", buildid)
-	}
-	fmt.Fprintf(bout, "\n$$\npackage %s", localpkg.Name)
-	if safemode != 0 {
-		fmt.Fprintf(bout, " safe")
-	}
-	fmt.Fprintf(bout, "\n")
+		exportf("build id %q\n", buildid)
+	}
+
+	size := 0 // size of export section without enclosing markers
+	if forceNewExport || newexport != 0 {
+		// binary export
+		// The linker also looks for the $$ marker - use char after $$ to distinguish format.
+		exportf("\n$$B\n")        // indicate binary format
+		const verifyExport = true // enable to check format changes
+		if verifyExport {
+			// save a copy of the export data
+			var copy bytes.Buffer
+			bcopy := obj.Binitw(&copy)
+			size = Export(bcopy, Debug_export != 0)
+			bcopy.Flush() // flushing to bytes.Buffer cannot fail
+			if n, err := bout.Write(copy.Bytes()); n != size || err != nil {
+				Fatalf("error writing export data: got %d bytes, want %d bytes, err = %v", n, size, err)
+			}
+
+			// verify there's no "\n$$\n" inside the export data
+			// TODO(gri) fragile - the end marker needs to be fixed
+			// TODO(gri) investigate if exporting a string containing "\n$$\n"
+			//           causes problems (old and new format)
+			if bytes.Index(copy.Bytes(), []byte("\n$$\n")) >= 0 {
+				Fatalf("export data contains end marker in its midst")
+			}

-	for _, p := range pkgs {
-		if p.Direct {
-			dumppkg(p)
+			// verify that we can read the copied export data back in
+			// (use empty package map to avoid collisions)
+			savedPkgMap := pkgMap
+			savedPkgs := pkgs
+			pkgMap = make(map[string]*Pkg)
+			pkgs = nil
+			importpkg = mkpkg("")
+			Import(obj.Binitr(&copy)) // must not die
+			importpkg = nil
+			pkgs = savedPkgs
+			pkgMap = savedPkgMap
+		} else {
+			size = Export(bout, Debug_export != 0)
+		}
+		exportf("\n$$\n")
+	} else {
+		// textual export
+		lno := lineno
+
+		exportf("\n$$\n") // indicate textual format
+		exportsize = 0
+		exportf("package %s", localpkg.Name)
+		if safemode != 0 {
+			exportf(" safe")
+		}
+		exportf("\n")
+
+		for _, p := range pkgs {
+			if p.Direct {
+				dumppkg(p)
+			}
 		}
-	}

-	// exportlist grows during iteration - cannot use range
-	for len(exportlist) > 0 {
-		n := exportlist[0]
-		exportlist = exportlist[1:]
-		lineno = n.Lineno
-		dumpsym(n.Sym)
+		// exportlist grows during iteration - cannot use range
+		for len(exportlist) > 0 {
+			n := exportlist[0]
+			exportlist = exportlist[1:]
+			lineno = n.Lineno
+			dumpsym(n.Sym)
+		}
+
+		size = exportsize
+		exportf("\n$$\n")
+		lineno = lno
 	}

-	fmt.Fprintf(bout, "\n$$\n")
-	lineno = lno
+	if Debug_export != 0 {
+		fmt.Printf("export data size = %d bytes\n", size)
+	}
 }

 /*

--- a/src/cmd/compile/internal/gc/go.go
+++ b/src/cmd/compile/internal/gc/go.go
@@ -288,7 +288,7 @@ const (

 	TFUNC
 	TARRAY
-	T_old_DARRAY
+	T_old_DARRAY // Doesn't seem to be used in existing code. Used now for Isddd export (see bexport.go). TODO(gri) rename.
 	TSTRUCT
 	TCHAN
 	TMAP

--- a/src/cmd/compile/internal/gc/go.y
+++ b/src/cmd/compile/internal/gc/go.y
@@ -254,6 +254,7 @@ import_stmt:
 			break;
 		}
 		if my.Name == "init" {
+			lineno = int32($1)
 			Yyerror("cannot import package as init - init must be a func");
 			break;
 		}

--- a/src/cmd/compile/internal/gc/lex.go
+++ b/src/cmd/compile/internal/gc/lex.go
@@ -58,6 +58,7 @@ var debugtab = []struct {
 	{"slice", &Debug_slice},           // print information about slice compilation
 	{"typeassert", &Debug_typeassert}, // print information about type assertion inlining
 	{"wb", &Debug_wb},                 // print information about write barriers
+	{"export", &Debug_export},         // print export data
 }

 const (
@@ -201,6 +202,7 @@ func Main() {
 	obj.Flagcount("live", "debug liveness analysis", &debuglive)
 	obj.Flagcount("m", "print optimization decisions", &Debug['m'])
 	obj.Flagcount("msan", "build code compatible with C/C++ memory sanitizer", &flag_msan)
+	obj.Flagcount("newexport", "use new export format", &newexport) // TODO remove eventually
 	obj.Flagcount("nolocalimports", "reject local (relative) imports", &nolocalimports)
 	obj.Flagstr("o", "write output to `file`", &outfile)
 	obj.Flagstr("p", "set expected package import `path`", &myimportpath)
@@ -655,6 +657,7 @@ func fakeimport() {
 	cannedimports("fake.o", "$$\n")
 }

+// TODO(gri) line argument doesn't appear to be used
 func importfile(f *Val, line int) {
 	if _, ok := f.U.(string); !ok {
 		Yyerror("import statement not a string")
@@ -786,39 +789,58 @@ func importfile(f *Val, line int) {
 	// so don't record the full path.
 	linehistpragma(file[len(file)-len(path_)-2:]) // acts as #pragma lib

-	/*
-	 * position the input right
-	 * after $$ and return
-	 */
-	pushedio = curio
-
-	curio.bin = imp
-	curio.peekc = 0
-	curio.peekc1 = 0
-	curio.infile = file
-	curio.nlsemi = false
-	typecheckok = true
+	// In the importfile, if we find:
+	// $$\n  (old format): position the input right after $$\n and return
+	// $$B\n (new format): import directly, then feed the lexer a dummy statement

+	// look for $$
+	var c int
 	for {
-		c := getc()
-		if c == EOF {
-			break
-		}
-		if c != '$' {
-			continue
-		}
-		c = getc()
-		if c == EOF {
+		c = obj.Bgetc(imp)
+		if c < 0 {
 			break
 		}
-		if c != '$' {
-			continue
+		if c == '$' {
+			c = obj.Bgetc(imp)
+			if c == '$' || c < 0 {
+				break
+			}
 		}
-		return
 	}

-	Yyerror("no import in %q", f.U.(string))
-	unimportfile()
+	// get character after $$
+	if c >= 0 {
+		c = obj.Bgetc(imp)
+	}
+
+	switch c {
+	case '\n':
+		// old export format
+		pushedio = curio
+
+		curio.bin = imp
+		curio.peekc = 0
+		curio.peekc1 = 0
+		curio.infile = file
+		curio.nlsemi = false
+		typecheckok = true
+
+	case 'B':
+		// new export format
+		obj.Bgetc(imp) // skip \n after $$B
+		Import(imp)
+
+		// continue as if the package was imported before (see above)
+		tag := ""
+		if importpkg.Safe {
+			tag = "safe"
+		}
+		p := fmt.Sprintf("package %s %s\n$$\n", importpkg.Name, tag)
+		cannedimports(file, p)
+
+	default:
+		Yyerror("no import in %q", f.U.(string))
+	}
 }

 func unimportfile() {

--- a/src/cmd/compile/internal/gc/typecheck.go
+++ b/src/cmd/compile/internal/gc/typecheck.go
@@ -2605,6 +2605,8 @@ func hasddd(t *Type) bool {
 	return false
 }

+// downcount is the same as countfield
+// TODO decide if we want both (for semantic reasons)
 func downcount(t *Type) int {
 	n := 0
 	for tl := t.Type; tl != nil; tl = tl.Down {

--- a/src/cmd/compile/internal/gc/y.go
+++ b/src/cmd/compile/internal/gc/y.go
--- a/src/cmd/internal/obj/util.go
+++ b/src/cmd/internal/obj/util.go
@@ -53,6 +53,10 @@ func Binitw(w io.Writer) *Biobuf {
 	return &Biobuf{w: bufio.NewWriter(w)}
 }

+func Binitr(r io.Reader) *Biobuf {
+	return &Biobuf{r: bufio.NewReader(r)}
+}
+
 func (b *Biobuf) Write(p []byte) (int, error) {
 	return b.w.Write(p)
 }

--- a/src/go/internal/gcimporter/bimport.go
+++ b/src/go/internal/gcimporter/bimport.go
--- a/src/go/internal/gcimporter/exportdata.go
+++ b/src/go/internal/gcimporter/exportdata.go
@@ -39,14 +39,16 @@ func readGopackHeader(r *bufio.Reader) (name string, size int, err error) {
 // FindExportData positions the reader r at the beginning of the
 // export data section of an underlying GC-created object/archive
 // file by reading from it. The reader must be positioned at the
-// start of the file before calling this function.
+// start of the file before calling this function. The hdr result
+// is the string before the export data, either "$$" or "$$B".
 //
-func FindExportData(r *bufio.Reader) (err error) {
+func FindExportData(r *bufio.Reader) (hdr string, err error) {
 	// Read first line to make sure this is an object file.
 	line, err := r.ReadSlice('\n')
 	if err != nil {
 		return
 	}
+
 	if string(line) == "!<arch>\n" {
 		// Archive file. Scan to __.PKGDEF.
 		var name string
@@ -97,12 +99,13 @@ func FindExportData(r *bufio.Reader) (err error) {
 	}

 	// Skip over object header to export data.
-	// Begins after first line with $$.
+	// Begins after first line starting with $$.
 	for line[0] != '$' {
 		if line, err = r.ReadSlice('\n'); err != nil {
 			return
 		}
 	}
+	hdr = string(line)

 	return
 }
--- a/src/go/internal/gcimporter/gcimporter.go
+++ b/src/go/internal/gcimporter/gcimporter.go
@@ -12,6 +12,7 @@ import (
 	"go/build"
 	"go/token"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -149,12 +150,25 @@ func Import(packages map[string]*types.Package, path string) (pkg *types.Package
 		}
 	}()

+	var hdr string
 	buf := bufio.NewReader(f)
-	if err = FindExportData(buf); err != nil {
+	if hdr, err = FindExportData(buf); err != nil {
 		return
 	}

-	pkg, err = ImportData(packages, filename, id, buf)
+	switch hdr {
+	case "$$\n":
+		return ImportData(packages, filename, id, buf)
+	case "$$B\n":
+		var data []byte
+		data, err = ioutil.ReadAll(buf)
+		if err == nil {
+			_, pkg, err = BImportData(packages, data, path)
+			return
+		}
+	default:
+		err = fmt.Errorf("unknown export data header: %q", hdr)
+	}

 	return
 }

--- a/src/go/internal/gcimporter/gcimporter_test.go
+++ b/src/go/internal/gcimporter/gcimporter_test.go
@@ -46,13 +46,23 @@ func compile(t *testing.T, dirname, filename string) string {
 	return filepath.Join(dirname, filename[:len(filename)-2]+"o")
 }

-// Use the same global imports map for all tests. The effect is
-// as if all tested packages were imported into a single package.
-var imports = make(map[string]*types.Package)
+// TODO(gri) Remove this function once we switched to new export format by default.
+func compileNewExport(t *testing.T, dirname, filename string) string {
+	testenv.MustHaveGoBuild(t)
+	cmd := exec.Command("go", "tool", "compile", "-newexport", filename)
+	cmd.Dir = dirname
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Logf("%s", out)
+		t.Fatalf("go tool compile %s failed: %s", filename, err)
+	}
+	// filename should end with ".go"
+	return filepath.Join(dirname, filename[:len(filename)-2]+"o")
+}

 func testPath(t *testing.T, path string) *types.Package {
 	t0 := time.Now()
-	pkg, err := Import(imports, path)
+	pkg, err := Import(make(map[string]*types.Package), path)
 	if err != nil {
 		t.Errorf("testPath(%s): %s", path, err)
 		return nil
@@ -92,7 +102,7 @@ func testDir(t *testing.T, dir string, endTime time.Time) (nimports int) {
 	return
 }

-func TestImport(t *testing.T) {
+func TestImportTestdata(t *testing.T) {
 	// This package only handles gc export data.
 	if runtime.Compiler != "gc" {
 		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
@@ -103,20 +113,58 @@ func TestImport(t *testing.T) {
 		defer os.Remove(outFn)
 	}

-	nimports := 0
 	if pkg := testPath(t, "./testdata/exports"); pkg != nil {
-		nimports++
-		// The package's Imports should include all the types
-		// referenced by the exportdata, which may be more than
-		// the import statements in the package's source, but
-		// fewer than the transitive closure of dependencies.
-		want := `[package ast ("go/ast") package token ("go/token") package runtime ("runtime")]`
+		// The package's Imports list must include all packages
+		// explicitly imported by exports.go, plus all packages
+		// referenced indirectly via exported objects in exports.go.
+		// With the textual export format, the list may also include
+		// additional packages that are not strictly required for
+		// import processing alone (they are exported to err "on
+		// the safe side").
+		got := fmt.Sprint(pkg.Imports())
+		for _, want := range []string{"go/ast", "go/token"} {
+			if !strings.Contains(got, want) {
+				t.Errorf(`Package("exports").Imports() = %s, does not contain %s`, got, want)
+			}
+		}
+	}
+}
+
+// TODO(gri) Remove this function once we switched to new export format by default
+//           (and update the comment and want list in TestImportTestdata).
+func TestImportTestdataNewExport(t *testing.T) {
+	// This package only handles gc export data.
+	if runtime.Compiler != "gc" {
+		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
+		return
+	}
+
+	if outFn := compileNewExport(t, "testdata", "exports.go"); outFn != "" {
+		defer os.Remove(outFn)
+	}
+
+	if pkg := testPath(t, "./testdata/exports"); pkg != nil {
+		// The package's Imports list must include all packages
+		// explicitly imported by exports.go, plus all packages
+		// referenced indirectly via exported objects in exports.go.
+		want := `[package ast ("go/ast") package token ("go/token")]`
 		got := fmt.Sprint(pkg.Imports())
 		if got != want {
 			t.Errorf(`Package("exports").Imports() = %s, want %s`, got, want)
 		}
 	}
-	nimports += testDir(t, "", time.Now().Add(maxTime)) // installed packages
+}
+
+func TestImportStdLib(t *testing.T) {
+	skipSpecialPlatforms(t)
+
+	// This package only handles gc export data.
+	if runtime.Compiler != "gc" {
+		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
+		return
+	}
+
+	nimports := testDir(t, "", time.Now().Add(maxTime)) // installed packages
 	t.Logf("tested %d imports", nimports)
 }

@@ -148,7 +196,7 @@ func TestImportedTypes(t *testing.T) {
 		importPath := s[0]
 		objName := s[1]

-		pkg, err := Import(imports, importPath)
+		pkg, err := Import(make(map[string]*types.Package), importPath)
 		if err != nil {
 			t.Error(err)
 			continue