Skip to content

G
golang
Commit 4d0f3a1c authored by Russ Cox's avatar Russ Cox
Browse files
Options

cmd/internal/gc, runtime: use 1-bit bitmap for stack frames, data, bss 

The bitmaps were 2 bits per pointer because we needed to distinguish
scalar, pointer, multiword, and we used the leftover value to distinguish
uninitialized from scalar, even though the garbage collector (GC) didn't care.

Now that there are no multiword structures from the GC's point of view,
cut the bitmaps down to 1 bit per pointer, recording just live pointer vs not.

The GC assumes the same layout for stack frames and for the maps
describing the global data and bss sections, so change them all in one CL.

The code still refers to 4-bit heap bitmaps and 2-bit "type bitmaps", since
the 2-bit representation lives (at least for now) in some of the reflect data.

Because these stack frame bitmaps are stored directly in the rodata in
the binary, this CL reduces the size of the 6g binary by about 1.1%.

Performance change is basically a wash, but using less memory,
and smaller binaries, and enables other bitmap reductions.

name                                      old mean                new mean        delta
BenchmarkBinaryTree17                13.2s × (0.97,1.03)     13.0s × (0.99,1.01)  -0.93% (p=0.005)
BenchmarkBinaryTree17-2              9.69s × (0.96,1.05)     9.51s × (0.96,1.03)  -1.86% (p=0.001)
BenchmarkBinaryTree17-4              10.1s × (0.97,1.05)     10.0s × (0.96,1.05)  ~ (p=0.141)
BenchmarkFannkuch11                  4.35s × (0.99,1.01)     4.43s × (0.98,1.04)  +1.75% (p=0.001)
BenchmarkFannkuch11-2                4.31s × (0.99,1.03)     4.32s × (1.00,1.00)  ~ (p=0.095)
BenchmarkFannkuch11-4                4.32s × (0.99,1.02)     4.38s × (0.98,1.04)  +1.38% (p=0.008)
BenchmarkFmtFprintfEmpty            83.5ns × (0.97,1.10)    87.3ns × (0.92,1.11)  +4.55% (p=0.014)
BenchmarkFmtFprintfEmpty-2          81.8ns × (0.98,1.04)    82.5ns × (0.97,1.08)  ~ (p=0.364)
BenchmarkFmtFprintfEmpty-4          80.9ns × (0.99,1.01)    82.6ns × (0.97,1.08)  +2.12% (p=0.010)
BenchmarkFmtFprintfString            320ns × (0.95,1.04)     322ns × (0.97,1.05)  ~ (p=0.368)
BenchmarkFmtFprintfString-2          303ns × (0.97,1.04)     304ns × (0.97,1.04)  ~ (p=0.484)
BenchmarkFmtFprintfString-4          305ns × (0.97,1.05)     306ns × (0.98,1.05)  ~ (p=0.543)
BenchmarkFmtFprintfInt               311ns × (0.98,1.03)     319ns × (0.97,1.03)  +2.63% (p=0.000)
BenchmarkFmtFprintfInt-2             297ns × (0.98,1.04)     301ns × (0.97,1.04)  +1.19% (p=0.023)
BenchmarkFmtFprintfInt-4             302ns × (0.98,1.02)     304ns × (0.97,1.03)  ~ (p=0.126)
BenchmarkFmtFprintfIntInt            554ns × (0.96,1.05)     554ns × (0.97,1.03)  ~ (p=0.975)
BenchmarkFmtFprintfIntInt-2          520ns × (0.98,1.03)     517ns × (0.98,1.02)  ~ (p=0.153)
BenchmarkFmtFprintfIntInt-4          524ns × (0.98,1.02)     525ns × (0.98,1.03)  ~ (p=0.597)
BenchmarkFmtFprintfPrefixedInt       433ns × (0.97,1.06)     434ns × (0.97,1.06)  ~ (p=0.804)
BenchmarkFmtFprintfPrefixedInt-2     413ns × (0.98,1.04)     413ns × (0.98,1.03)  ~ (p=0.881)
BenchmarkFmtFprintfPrefixedInt-4     420ns × (0.97,1.03)     421ns × (0.97,1.03)  ~ (p=0.561)
BenchmarkFmtFprintfFloat             620ns × (0.99,1.03)     636ns × (0.97,1.03)  +2.57% (p=0.000)
BenchmarkFmtFprintfFloat-2           601ns × (0.98,1.02)     617ns × (0.98,1.03)  +2.58% (p=0.000)
BenchmarkFmtFprintfFloat-4           613ns × (0.98,1.03)     626ns × (0.98,1.02)  +2.15% (p=0.000)
BenchmarkFmtManyArgs                2.19µs × (0.96,1.04)    2.23µs × (0.97,1.02)  +1.65% (p=0.000)
BenchmarkFmtManyArgs-2              2.08µs × (0.98,1.03)    2.10µs × (0.99,1.02)  +0.79% (p=0.019)
BenchmarkFmtManyArgs-4              2.10µs × (0.98,1.02)    2.13µs × (0.98,1.02)  +1.72% (p=0.000)
BenchmarkGobDecode                  21.3ms × (0.97,1.05)    21.1ms × (0.97,1.04)  -1.36% (p=0.025)
BenchmarkGobDecode-2                20.0ms × (0.97,1.03)    19.2ms × (0.97,1.03)  -4.00% (p=0.000)
BenchmarkGobDecode-4                19.5ms × (0.99,1.02)    19.0ms × (0.99,1.01)  -2.39% (p=0.000)
BenchmarkGobEncode                  18.3ms × (0.95,1.07)    18.1ms × (0.96,1.08)  ~ (p=0.305)
BenchmarkGobEncode-2                16.8ms × (0.97,1.02)    16.4ms × (0.98,1.02)  -2.79% (p=0.000)
BenchmarkGobEncode-4                15.4ms × (0.98,1.02)    15.4ms × (0.98,1.02)  ~ (p=0.465)
BenchmarkGzip                        650ms × (0.98,1.03)     655ms × (0.97,1.04)  ~ (p=0.075)
BenchmarkGzip-2                      652ms × (0.98,1.03)     655ms × (0.98,1.02)  ~ (p=0.337)
BenchmarkGzip-4                      656ms × (0.98,1.04)     653ms × (0.98,1.03)  ~ (p=0.291)
BenchmarkGunzip                      143ms × (1.00,1.01)     143ms × (1.00,1.01)  ~ (p=0.507)
BenchmarkGunzip-2                    143ms × (1.00,1.01)     143ms × (1.00,1.01)  ~ (p=0.313)
BenchmarkGunzip-4                    143ms × (1.00,1.01)     143ms × (1.00,1.01)  ~ (p=0.312)
BenchmarkHTTPClientServer            110µs × (0.98,1.03)     109µs × (0.99,1.02)  -1.40% (p=0.000)
BenchmarkHTTPClientServer-2          154µs × (0.90,1.08)     149µs × (0.90,1.08)  -3.43% (p=0.007)
BenchmarkHTTPClientServer-4          138µs × (0.97,1.04)     138µs × (0.96,1.04)  ~ (p=0.670)
BenchmarkJSONEncode                 40.2ms × (0.98,1.02)    40.2ms × (0.98,1.05)  ~ (p=0.828)
BenchmarkJSONEncode-2               35.1ms × (0.99,1.02)    35.2ms × (0.98,1.03)  ~ (p=0.392)
BenchmarkJSONEncode-4               35.3ms × (0.98,1.03)    35.3ms × (0.98,1.02)  ~ (p=0.813)
BenchmarkJSONDecode                  119ms × (0.97,1.02)     117ms × (0.98,1.02)  -1.80% (p=0.000)
BenchmarkJSONDecode-2                115ms × (0.99,1.02)     114ms × (0.98,1.02)  -1.18% (p=0.000)
BenchmarkJSONDecode-4                116ms × (0.98,1.02)     114ms × (0.98,1.02)  -1.43% (p=0.000)
BenchmarkMandelbrot200              6.03ms × (1.00,1.01)    6.03ms × (1.00,1.01)  ~ (p=0.985)
BenchmarkMandelbrot200-2            6.03ms × (1.00,1.01)    6.02ms × (1.00,1.01)  ~ (p=0.320)
BenchmarkMandelbrot200-4            6.03ms × (1.00,1.01)    6.03ms × (1.00,1.01)  ~ (p=0.799)
BenchmarkGoParse                    8.63ms × (0.89,1.10)    8.58ms × (0.93,1.09)  ~ (p=0.667)
BenchmarkGoParse-2                  8.20ms × (0.97,1.04)    8.37ms × (0.97,1.04)  +1.96% (p=0.001)
BenchmarkGoParse-4                  8.00ms × (0.98,1.02)    8.14ms × (0.99,1.02)  +1.75% (p=0.000)
BenchmarkRegexpMatchEasy0_32         162ns × (1.00,1.01)     164ns × (0.98,1.04)  +1.35% (p=0.011)
BenchmarkRegexpMatchEasy0_32-2       161ns × (1.00,1.01)     161ns × (1.00,1.00)  ~ (p=0.185)
BenchmarkRegexpMatchEasy0_32-4       161ns × (1.00,1.00)     161ns × (1.00,1.00)  -0.19% (p=0.001)
BenchmarkRegexpMatchEasy0_1K         540ns × (0.99,1.02)     566ns × (0.98,1.04)  +4.98% (p=0.000)
BenchmarkRegexpMatchEasy0_1K-2       540ns × (0.99,1.01)     557ns × (0.99,1.01)  +3.21% (p=0.000)
BenchmarkRegexpMatchEasy0_1K-4       541ns × (0.99,1.01)     559ns × (0.99,1.01)  +3.26% (p=0.000)
BenchmarkRegexpMatchEasy1_32         139ns × (0.98,1.04)     139ns × (0.99,1.03)  ~ (p=0.979)
BenchmarkRegexpMatchEasy1_32-2       139ns × (0.99,1.04)     139ns × (0.99,1.02)  ~ (p=0.777)
BenchmarkRegexpMatchEasy1_32-4       139ns × (0.98,1.04)     139ns × (0.99,1.04)  ~ (p=0.771)
BenchmarkRegexpMatchEasy1_1K         890ns × (0.99,1.03)     885ns × (1.00,1.01)  -0.50% (p=0.004)
BenchmarkRegexpMatchEasy1_1K-2       888ns × (0.99,1.01)     885ns × (0.99,1.01)  -0.37% (p=0.004)
BenchmarkRegexpMatchEasy1_1K-4       890ns × (0.99,1.02)     884ns × (1.00,1.00)  -0.70% (p=0.000)
BenchmarkRegexpMatchMedium_32        252ns × (0.99,1.01)     251ns × (0.99,1.01)  ~ (p=0.081)
BenchmarkRegexpMatchMedium_32-2      254ns × (0.99,1.04)     252ns × (0.99,1.01)  -0.78% (p=0.027)
BenchmarkRegexpMatchMedium_32-4      253ns × (0.99,1.04)     252ns × (0.99,1.01)  -0.70% (p=0.022)
BenchmarkRegexpMatchMedium_1K       72.9µs × (0.99,1.01)    72.7µs × (1.00,1.00)  ~ (p=0.064)
BenchmarkRegexpMatchMedium_1K-2     74.1µs × (0.98,1.05)    72.9µs × (1.00,1.01)  -1.61% (p=0.001)
BenchmarkRegexpMatchMedium_1K-4     73.6µs × (0.99,1.05)    72.8µs × (1.00,1.00)  -1.13% (p=0.007)
BenchmarkRegexpMatchHard_32         3.88µs × (0.99,1.03)    3.92µs × (0.98,1.05)  ~ (p=0.143)
BenchmarkRegexpMatchHard_32-2       3.89µs × (0.99,1.03)    3.93µs × (0.98,1.09)  ~ (p=0.278)
BenchmarkRegexpMatchHard_32-4       3.90µs × (0.99,1.05)    3.93µs × (0.98,1.05)  ~ (p=0.252)
BenchmarkRegexpMatchHard_1K          118µs × (0.99,1.01)     117µs × (0.99,1.02)  -0.54% (p=0.003)
BenchmarkRegexpMatchHard_1K-2        118µs × (0.99,1.01)     118µs × (0.99,1.03)  ~ (p=0.581)
BenchmarkRegexpMatchHard_1K-4        118µs × (0.99,1.02)     117µs × (0.99,1.01)  -0.54% (p=0.002)
BenchmarkRevcomp                     991ms × (0.95,1.10)     989ms × (0.94,1.08)  ~ (p=0.879)
BenchmarkRevcomp-2                   978ms × (0.95,1.11)     962ms × (0.96,1.08)  ~ (p=0.257)
BenchmarkRevcomp-4                   979ms × (0.96,1.07)     974ms × (0.96,1.11)  ~ (p=0.678)
BenchmarkTemplate                    141ms × (0.99,1.02)     145ms × (0.99,1.02)  +2.75% (p=0.000)
BenchmarkTemplate-2                  135ms × (0.98,1.02)     138ms × (0.99,1.02)  +2.34% (p=0.000)
BenchmarkTemplate-4                  136ms × (0.98,1.02)     140ms × (0.99,1.02)  +2.71% (p=0.000)
BenchmarkTimeParse                   640ns × (0.99,1.01)     622ns × (0.99,1.01)  -2.88% (p=0.000)
BenchmarkTimeParse-2                 640ns × (0.99,1.01)     622ns × (1.00,1.00)  -2.81% (p=0.000)
BenchmarkTimeParse-4                 640ns × (1.00,1.01)     622ns × (0.99,1.01)  -2.82% (p=0.000)
BenchmarkTimeFormat                  730ns × (0.98,1.02)     731ns × (0.98,1.03)  ~ (p=0.767)
BenchmarkTimeFormat-2                709ns × (0.99,1.02)     707ns × (0.99,1.02)  ~ (p=0.347)
BenchmarkTimeFormat-4                717ns × (0.98,1.01)     718ns × (0.98,1.02)  ~ (p=0.793)

Change-Id: Ie779c47e912bf80eb918bafa13638bd8dfd6c2d9
Reviewed-on: https://go-review.googlesource.com/9406Reviewed-by: 's avatarRick Hudson <rlh@golang.org>
parent e9ab343f
Hide whitespace changes
Inline Side-by-side
Showing with 237 additions and 238 deletions
src/cmd/internal/gc/pgen.go
View file @ 4d0f3a1c
......@@ -142,12 +142,12 @@ func emitptrargsmap() {
var xoffset int64
if Curfn.Type.Thistuple > 0 {
xoffset = 0
twobitwalktype1(getthisx(Curfn.Type), &xoffset, bv)
onebitwalktype1(getthisx(Curfn.Type), &xoffset, bv)
}
if Curfn.Type.Intuple > 0 {
xoffset = 0
twobitwalktype1(getinargx(Curfn.Type), &xoffset, bv)
onebitwalktype1(getinargx(Curfn.Type), &xoffset, bv)
}
for j := 0; int32(j) < bv.n; j += 32 {
......@@ -155,7 +155,7 @@ func emitptrargsmap() {
}
if Curfn.Type.Outtuple > 0 {
xoffset = 0
twobitwalktype1(getoutargx(Curfn.Type), &xoffset, bv)
onebitwalktype1(getoutargx(Curfn.Type), &xoffset, bv)
for j := 0; int32(j) < bv.n; j += 32 {
off = duint32(sym, off, bv.b[j/32])
}
......
src/cmd/internal/gc/plive.go
View file @ 4d0f3a1c
......@@ -886,11 +886,11 @@ func checkptxt(fn *Node, firstp *obj.Prog) {
// NOTE: The bitmap for a specific type t should be cached in t after the first run
// and then simply copied into bv at the correct offset on future calls with
// the same type t. On https://rsc.googlecode.com/hg/testdata/slow.go, twobitwalktype1
// the same type t. On https://rsc.googlecode.com/hg/testdata/slow.go, onebitwalktype1
// accounts for 40% of the 6g execution time.
func twobitwalktype1(t *Type, xoffset *int64, bv Bvec) {
func onebitwalktype1(t *Type, xoffset *int64, bv Bvec) {
if t.Align > 0 && *xoffset&int64(t.Align-1) != 0 {
Fatal("twobitwalktype1: invalid initial alignment, %v", t)
Fatal("onebitwalktype1: invalid initial alignment, %v", t)
}
switch t.Etype {
......@@ -910,10 +910,6 @@ func twobitwalktype1(t *Type, xoffset *int64, bv Bvec) {
TFLOAT64,
TCOMPLEX64,
TCOMPLEX128:
for i := int64(0); i < t.Width; i++ {
bvset(bv, int32(((*xoffset+i)/int64(Widthptr))*obj.BitsPerPointer)) // 1 = live scalar (BitsScalar)
}
*xoffset += t.Width
case TPTR32,
......@@ -923,46 +919,46 @@ func twobitwalktype1(t *Type, xoffset *int64, bv Bvec) {
TCHAN,
TMAP:
if *xoffset&int64(Widthptr-1) != 0 {
Fatal("twobitwalktype1: invalid alignment, %v", t)
Fatal("onebitwalktype1: invalid alignment, %v", t)
}
bvset(bv, int32((*xoffset/int64(Widthptr))*obj.BitsPerPointer+1)) // 2 = live ptr (BitsPointer)
bvset(bv, int32(*xoffset/int64(Widthptr))) // pointer
*xoffset += t.Width
// struct { byte *str; intgo len; }
case TSTRING:
// struct { byte *str; intgo len; }
if *xoffset&int64(Widthptr-1) != 0 {
Fatal("twobitwalktype1: invalid alignment, %v", t)
Fatal("onebitwalktype1: invalid alignment, %v", t)
}
bvset(bv, int32((*xoffset/int64(Widthptr))*obj.BitsPerPointer+1)) // 2 = live ptr in first slot (BitsPointer)
bvset(bv, int32(*xoffset/int64(Widthptr))) //pointer in first slot
*xoffset += t.Width
// struct { Itab *tab; union { void *ptr, uintptr val } data; }
// or, when isnilinter(t)==true:
// struct { Type *type; union { void *ptr, uintptr val } data; }
case TINTER:
// struct { Itab *tab; void *data; }
// or, when isnilinter(t)==true:
// struct { Type *type; void *data; }
if *xoffset&int64(Widthptr-1) != 0 {
Fatal("twobitwalktype1: invalid alignment, %v", t)
Fatal("onebitwalktype1: invalid alignment, %v", t)
}
bvset(bv, int32((*xoffset/int64(Widthptr))*obj.BitsPerPointer+1)) // 2 = live ptr in first slot (BitsPointer)
bvset(bv, int32((*xoffset/int64(Widthptr))*obj.BitsPerPointer+3)) // 2 = live ptr in second slot (BitsPointer)
bvset(bv, int32(*xoffset/int64(Widthptr))) // pointer in first slot
bvset(bv, int32(*xoffset/int64(Widthptr)+1)) // pointer in second slot
*xoffset += t.Width
// The value of t->bound is -1 for slices types and >0 for
// for fixed array types. All other values are invalid.
case TARRAY:
// The value of t->bound is -1 for slices types and >0 for
// for fixed array types. All other values are invalid.
if t.Bound < -1 {
Fatal("twobitwalktype1: invalid bound, %v", t)
Fatal("onebitwalktype1: invalid bound, %v", t)
}
if Isslice(t) {
// struct { byte *array; uintgo len; uintgo cap; }
if *xoffset&int64(Widthptr-1) != 0 {
Fatal("twobitwalktype1: invalid TARRAY alignment, %v", t)
Fatal("onebitwalktype1: invalid TARRAY alignment, %v", t)
}
bvset(bv, int32((*xoffset/int64(Widthptr))*obj.BitsPerPointer+1)) // 2 = live ptr in first slot (BitsPointer)
bvset(bv, int32(*xoffset/int64(Widthptr))) // pointer in first slot (BitsPointer)
*xoffset += t.Width
} else {
for i := int64(0); i < t.Bound; i++ {
twobitwalktype1(t.Type, xoffset, bv)
onebitwalktype1(t.Type, xoffset, bv)
}
}
......@@ -972,14 +968,14 @@ func twobitwalktype1(t *Type, xoffset *int64, bv Bvec) {
for t1 := t.Type; t1 != nil; t1 = t1.Down {
fieldoffset = t1.Width
*xoffset += fieldoffset - o
twobitwalktype1(t1.Type, xoffset, bv)
onebitwalktype1(t1.Type, xoffset, bv)
o = fieldoffset + t1.Type.Width
}
*xoffset += t.Width - o
default:
Fatal("twobitwalktype1: unexpected type, %v", t)
Fatal("onebitwalktype1: unexpected type, %v", t)
}
}
......@@ -996,7 +992,7 @@ func argswords() int32 {
// Generates live pointer value maps for arguments and local variables. The
// this argument and the in arguments are always assumed live. The vars
// argument is an array of Node*s.
func twobitlivepointermap(lv *Liveness, liveout Bvec, vars []*Node, args Bvec, locals Bvec) {
func onebitlivepointermap(lv *Liveness, liveout Bvec, vars []*Node, args Bvec, locals Bvec) {
var node *Node
var xoffset int64
......@@ -1009,11 +1005,11 @@ func twobitlivepointermap(lv *Liveness, liveout Bvec, vars []*Node, args Bvec, l
switch node.Class {
case PAUTO:
xoffset = node.Xoffset + stkptrsize
twobitwalktype1(node.Type, &xoffset, locals)
onebitwalktype1(node.Type, &xoffset, locals)
case PPARAM, PPARAMOUT:
xoffset = node.Xoffset
twobitwalktype1(node.Type, &xoffset, args)
onebitwalktype1(node.Type, &xoffset, args)
}
}
......@@ -1025,13 +1021,13 @@ func twobitlivepointermap(lv *Liveness, liveout Bvec, vars []*Node, args Bvec, l
if thisargtype != nil {
xoffset = 0
twobitwalktype1(thisargtype, &xoffset, args)
onebitwalktype1(thisargtype, &xoffset, args)
}
inargtype := getinargx(lv.fn.Type)
if inargtype != nil {
xoffset = 0
twobitwalktype1(inargtype, &xoffset, args)
onebitwalktype1(inargtype, &xoffset, args)
}
}
......@@ -1202,15 +1198,15 @@ func livenesssolve(lv *Liveness) {
func islive(n *Node, args Bvec, locals Bvec) bool {
switch n.Class {
case PPARAM, PPARAMOUT:
for i := 0; int64(i) < n.Type.Width/int64(Widthptr)*obj.BitsPerPointer; i++ {
if bvget(args, int32(n.Xoffset/int64(Widthptr)*obj.BitsPerPointer+int64(i))) != 0 {
for i := 0; int64(i) < n.Type.Width/int64(Widthptr); i++ {
if bvget(args, int32(n.Xoffset/int64(Widthptr)+int64(i))) != 0 {
return true
}
}
case PAUTO:
for i := 0; int64(i) < n.Type.Width/int64(Widthptr)*obj.BitsPerPointer; i++ {
if bvget(locals, int32((n.Xoffset+stkptrsize)/int64(Widthptr)*obj.BitsPerPointer+int64(i))) != 0 {
for i := 0; int64(i) < n.Type.Width/int64(Widthptr); i++ {
if bvget(locals, int32((n.Xoffset+stkptrsize)/int64(Widthptr)+int64(i))) != 0 {
return true
}
}
......@@ -1239,7 +1235,7 @@ func livenessepilogue(lv *Liveness) {
avarinit := bvalloc(nvars)
any := bvalloc(nvars)
all := bvalloc(nvars)
ambig := bvalloc(localswords() * obj.BitsPerPointer)
ambig := bvalloc(localswords())
nmsg := int32(0)
startmsg := int32(0)
......@@ -1294,7 +1290,7 @@ func livenessepilogue(lv *Liveness) {
// Record in 'ambiguous' bitmap.
xoffset = n.Xoffset + stkptrsize
twobitwalktype1(n.Type, &xoffset, ambig)
onebitwalktype1(n.Type, &xoffset, ambig)
}
}
}
......@@ -1303,10 +1299,10 @@ func livenessepilogue(lv *Liveness) {
// value we are tracking.
// Live stuff first.
args = bvalloc(argswords() * obj.BitsPerPointer)
args = bvalloc(argswords())
lv.argslivepointers = append(lv.argslivepointers, args)
locals = bvalloc(localswords() * obj.BitsPerPointer)
locals = bvalloc(localswords())
lv.livepointers = append(lv.livepointers, locals)
if debuglive >= 3 {
......@@ -1319,7 +1315,7 @@ func livenessepilogue(lv *Liveness) {
// because the any/all calculation requires walking forward
// over the block (as this loop does), while the liveout
// requires walking backward (as the next loop does).
twobitlivepointermap(lv, any, lv.vars, args, locals)
onebitlivepointermap(lv, any, lv.vars, args, locals)
}
if p == bb.last {
......@@ -1394,7 +1390,7 @@ func livenessepilogue(lv *Liveness) {
args = lv.argslivepointers[pos]
locals = lv.livepointers[pos]
twobitlivepointermap(lv, liveout, lv.vars, args, locals)
onebitlivepointermap(lv, liveout, lv.vars, args, locals)
// Ambiguously live variables are zeroed immediately after
// function entry. Mark them live for all the non-entry bitmaps
......@@ -1727,7 +1723,7 @@ func livenessprintdebug(lv *Liveness) {
// length of the bitmaps. All bitmaps are assumed to be of equal length. The
// words that are followed are the raw bitmap words. The arr argument is an
// array of Node*s.
func twobitwritesymbol(arr []Bvec, sym *Sym) {
func onebitwritesymbol(arr []Bvec, sym *Sym) {
var i int
var j int
var word uint32
......@@ -1816,9 +1812,9 @@ func liveness(fn *Node, firstp *obj.Prog, argssym *Sym, livesym *Sym) {
}
// Emit the live pointer map data structures
twobitwritesymbol(lv.livepointers, livesym)
onebitwritesymbol(lv.livepointers, livesym)
twobitwritesymbol(lv.argslivepointers, argssym)
onebitwritesymbol(lv.argslivepointers, argssym)
// Free everything.
for l := fn.Func.Dcl; l != nil; l = l.Next {
......
src/cmd/internal/gc/reflect.go
View file @ 4d0f3a1c
......@@ -1404,7 +1404,7 @@ func gengcmask(t *Type, gcmask []byte) {
xoffset := int64(0)
vec := bvalloc(2 * int32(Widthptr) * 8)
twobitwalktype1(t, &xoffset, vec)
onebitwalktype1(t, &xoffset, vec)
// Unfold the mask for the GC bitmap format:
// 4 bits per word, 2 high bits encode pointer info.
......@@ -1419,13 +1419,11 @@ func gengcmask(t *Type, gcmask []byte) {
var bits uint8
for j := int64(0); j <= (nptr % 2); j++ {
for i = 0; i < nptr; i++ {
bits = uint8(bvget(vec, int32(i*obj.BitsPerPointer)) | bvget(vec, int32(i*obj.BitsPerPointer+1))<<1)
// Some fake types (e.g. Hmap) has missing fileds.
// twobitwalktype1 generates BitsDead for that holes,
// replace BitsDead with BitsScalar.
if bits == obj.BitsDead {
// convert 0=scalar / 1=pointer to GC bit encoding
if bvget(vec, int32(i)) == 0 {
bits = obj.BitsScalar
} else {
bits = obj.BitsPointer
}
bits <<= 2
if half {
......
src/cmd/internal/gc/walk.go
View file @ 4d0f3a1c
......@@ -2243,30 +2243,23 @@ func applywritebarrier(n *Node, init **NodeList) *Node {
} else if t.Width <= int64(4*Widthptr) {
x := int64(0)
if applywritebarrier_bv.b == nil {
applywritebarrier_bv = bvalloc(obj.BitsPerPointer * 4)
applywritebarrier_bv = bvalloc(4)
}
bvresetall(applywritebarrier_bv)
twobitwalktype1(t, &x, applywritebarrier_bv)
const (
PtrBit = 1
)
// The bvgets are looking for BitsPointer in successive slots.
if obj.BitsPointer != 1<<PtrBit {
Fatal("wrong PtrBit")
}
onebitwalktype1(t, &x, applywritebarrier_bv)
var name string
switch t.Width / int64(Widthptr) {
default:
Fatal("found writebarrierfat for %d-byte object of type %v", int(t.Width), t)
case 2:
name = fmt.Sprintf("writebarrierfat%d%d", bvget(applywritebarrier_bv, PtrBit), bvget(applywritebarrier_bv, obj.BitsPerPointer+PtrBit))
name = fmt.Sprintf("writebarrierfat%d%d", bvget(applywritebarrier_bv, 0), bvget(applywritebarrier_bv, 1))
case 3:
name = fmt.Sprintf("writebarrierfat%d%d%d", bvget(applywritebarrier_bv, PtrBit), bvget(applywritebarrier_bv, obj.BitsPerPointer+PtrBit), bvget(applywritebarrier_bv, 2*obj.BitsPerPointer+PtrBit))
name = fmt.Sprintf("writebarrierfat%d%d%d", bvget(applywritebarrier_bv, 0), bvget(applywritebarrier_bv, 1), bvget(applywritebarrier_bv, 2))
case 4:
name = fmt.Sprintf("writebarrierfat%d%d%d%d", bvget(applywritebarrier_bv, PtrBit), bvget(applywritebarrier_bv, obj.BitsPerPointer+PtrBit), bvget(applywritebarrier_bv, 2*obj.BitsPerPointer+PtrBit), bvget(applywritebarrier_bv, 3*obj.BitsPerPointer+PtrBit))
name = fmt.Sprintf("writebarrierfat%d%d%d%d", bvget(applywritebarrier_bv, 0), bvget(applywritebarrier_bv, 1), bvget(applywritebarrier_bv, 2), bvget(applywritebarrier_bv, 3))
}
n = mkcall1(writebarrierfn(name, t, n.Right.Type), nil, init, l, Nodintconst(0), n.Right)
......
src/runtime/gcinfo_test.go
View file @ 4d0f3a1c
......@@ -12,21 +12,21 @@ import (
// TestGCInfo tests that various objects in heap, data and bss receive correct GC pointer type info.
func TestGCInfo(t *testing.T) {
verifyGCInfo(t, "bss ScalarPtr", &bssScalarPtr, nonStackInfo(infoScalarPtr))
verifyGCInfo(t, "bss PtrScalar", &bssPtrScalar, nonStackInfo(infoPtrScalar))
verifyGCInfo(t, "bss BigStruct", &bssBigStruct, nonStackInfo(infoBigStruct()))
verifyGCInfo(t, "bss string", &bssString, nonStackInfo(infoString))
verifyGCInfo(t, "bss slice", &bssSlice, nonStackInfo(infoSlice))
verifyGCInfo(t, "bss eface", &bssEface, nonStackInfo(infoEface))
verifyGCInfo(t, "bss iface", &bssIface, nonStackInfo(infoIface))
verifyGCInfo(t, "data ScalarPtr", &dataScalarPtr, nonStackInfo(infoScalarPtr))
verifyGCInfo(t, "data PtrScalar", &dataPtrScalar, nonStackInfo(infoPtrScalar))
verifyGCInfo(t, "data BigStruct", &dataBigStruct, nonStackInfo(infoBigStruct()))
verifyGCInfo(t, "data string", &dataString, nonStackInfo(infoString))
verifyGCInfo(t, "data slice", &dataSlice, nonStackInfo(infoSlice))
verifyGCInfo(t, "data eface", &dataEface, nonStackInfo(infoEface))
verifyGCInfo(t, "data iface", &dataIface, nonStackInfo(infoIface))
verifyGCInfo(t, "bss ScalarPtr", &bssScalarPtr, infoScalarPtr)
verifyGCInfo(t, "bss PtrScalar", &bssPtrScalar, infoPtrScalar)
verifyGCInfo(t, "bss BigStruct", &bssBigStruct, infoBigStruct())
verifyGCInfo(t, "bss string", &bssString, infoString)
verifyGCInfo(t, "bss slice", &bssSlice, infoSlice)
verifyGCInfo(t, "bss eface", &bssEface, infoEface)
verifyGCInfo(t, "bss iface", &bssIface, infoIface)
verifyGCInfo(t, "data ScalarPtr", &dataScalarPtr, infoScalarPtr)
verifyGCInfo(t, "data PtrScalar", &dataPtrScalar, infoPtrScalar)
verifyGCInfo(t, "data BigStruct", &dataBigStruct, infoBigStruct())
verifyGCInfo(t, "data string", &dataString, infoString)
verifyGCInfo(t, "data slice", &dataSlice, infoSlice)
verifyGCInfo(t, "data eface", &dataEface, infoEface)
verifyGCInfo(t, "data iface", &dataIface, infoIface)
verifyGCInfo(t, "stack ScalarPtr", new(ScalarPtr), infoScalarPtr)
verifyGCInfo(t, "stack PtrScalar", new(PtrScalar), infoPtrScalar)
......@@ -37,12 +37,12 @@ func TestGCInfo(t *testing.T) {
verifyGCInfo(t, "stack iface", new(Iface), infoIface)
for i := 0; i < 10; i++ {
verifyGCInfo(t, "heap ScalarPtr", escape(new(ScalarPtr)), nonStackInfo(infoScalarPtr))
verifyGCInfo(t, "heap PtrScalar", escape(new(PtrScalar)), nonStackInfo(infoPtrScalar))
verifyGCInfo(t, "heap BigStruct", escape(new(BigStruct)), nonStackInfo(infoBigStruct()))
verifyGCInfo(t, "heap string", escape(new(string)), nonStackInfo(infoString))
verifyGCInfo(t, "heap eface", escape(new(interface{})), nonStackInfo(infoEface))
verifyGCInfo(t, "heap iface", escape(new(Iface)), nonStackInfo(infoIface))
verifyGCInfo(t, "heap ScalarPtr", escape(new(ScalarPtr)), infoScalarPtr)
verifyGCInfo(t, "heap PtrScalar", escape(new(PtrScalar)), infoPtrScalar)
verifyGCInfo(t, "heap BigStruct", escape(new(BigStruct)), infoBigStruct())
verifyGCInfo(t, "heap string", escape(new(string)), infoString)
verifyGCInfo(t, "heap eface", escape(new(interface{})), infoEface)
verifyGCInfo(t, "heap iface", escape(new(Iface)), infoIface)
}
}
......@@ -130,25 +130,25 @@ func infoBigStruct() []byte {
return []byte{
typePointer, // q *int
typeScalar, typeScalar, typeScalar, typeScalar, typeScalar, // w byte; e [17]byte
typePointer, typeDead, typeDead, // r []byte
typePointer, typeScalar, typeScalar, // r []byte
typeScalar, typeScalar, typeScalar, typeScalar, // t int; y uint16; u uint64
typePointer, typeDead, // i string
typePointer, typeScalar, // i string
}
case "arm64", "amd64", "ppc64", "ppc64le":
return []byte{
typePointer, // q *int
typeScalar, typeScalar, typeScalar, // w byte; e [17]byte
typePointer, typeDead, typeDead, // r []byte
typePointer, typeScalar, typeScalar, // r []byte
typeScalar, typeScalar, typeScalar, // t int; y uint16; u uint64
typePointer, typeDead, // i string
typePointer, typeScalar, // i string
}
case "amd64p32":
return []byte{
typePointer, // q *int
typeScalar, typeScalar, typeScalar, typeScalar, typeScalar, // w byte; e [17]byte
typePointer, typeDead, typeDead, // r []byte
typeScalar, typeScalar, typeDead, typeScalar, typeScalar, // t int; y uint16; u uint64
typePointer, typeDead, // i string
typePointer, typeScalar, typeScalar, // r []byte
typeScalar, typeScalar, typeScalar, typeScalar, typeScalar, // t int; y uint16; u uint64
typePointer, typeScalar, // i string
}
default:
panic("unknown arch")
......@@ -183,8 +183,8 @@ var (
dataEface interface{} = 42
dataIface Iface = IfaceImpl(42)
infoString = []byte{typePointer, typeDead}
infoSlice = []byte{typePointer, typeDead, typeDead}
infoString = []byte{typePointer, typeScalar}
infoSlice = []byte{typePointer, typeScalar, typeScalar}
infoEface = []byte{typePointer, typePointer}
infoIface = []byte{typePointer, typePointer}
)
src/runtime/heapdump.go
View file @ 4d0f3a1c
......@@ -237,18 +237,10 @@ type childInfo struct {
// dump kinds & offsets of interesting fields in bv
func dumpbv(cbv *bitvector, offset uintptr) {
bv := gobv(*cbv)
for i := uintptr(0); i < uintptr(bv.n); i += typeBitsWidth {
switch bv.bytedata[i/8] >> (i % 8) & typeMask {
default:
throw("unexpected pointer bits")
case typeDead:
// typeDead has already been processed in makeheapobjbv.
// We should only see it in stack maps, in which case we should continue processing.
case typeScalar:
// ok
case typePointer:
for i := uintptr(0); i < uintptr(bv.n); i++ {
if bv.bytedata[i/8]>>(i%8)&1 == 1 {
dumpint(fieldKindPtr)
dumpint(uint64(offset + i/typeBitsWidth*ptrSize))
dumpint(uint64(offset + i*ptrSize))
}
}
}
......@@ -278,7 +270,7 @@ func dumpframe(s *stkframe, arg unsafe.Pointer) bool {
var bv bitvector
if stkmap != nil && stkmap.n > 0 {
bv = stackmapdata(stkmap, pcdata)
dumpbvtypes(&bv, unsafe.Pointer(s.varp-uintptr(bv.n/typeBitsWidth*ptrSize)))
dumpbvtypes(&bv, unsafe.Pointer(s.varp-uintptr(bv.n*ptrSize)))
} else {
bv.n = -1
}
......@@ -326,7 +318,7 @@ func dumpframe(s *stkframe, arg unsafe.Pointer) bool {
} else if stkmap.n > 0 {
// Locals bitmap information, scan just the pointers in
// locals.
dumpbv(&bv, s.varp-uintptr(bv.n)/typeBitsWidth*ptrSize-s.sp)
dumpbv(&bv, s.varp-uintptr(bv.n)*ptrSize-s.sp)
}
dumpint(fieldKindEol)
......@@ -651,7 +643,7 @@ func dumpmemprof() {
}
}
var dumphdr = []byte("go1.4 heap dump\n")
var dumphdr = []byte("go1.5 heap dump\n")
func mdump() {
// make sure we're done sweeping
......@@ -720,18 +712,21 @@ func dumpbvtypes(bv *bitvector, base unsafe.Pointer) {
func makeheapobjbv(p uintptr, size uintptr) bitvector {
// Extend the temp buffer if necessary.
nptr := size / ptrSize
if uintptr(len(tmpbuf)) < nptr*typeBitsWidth/8+1 {
if uintptr(len(tmpbuf)) < nptr/8+1 {
if tmpbuf != nil {
sysFree(unsafe.Pointer(&tmpbuf[0]), uintptr(len(tmpbuf)), &memstats.other_sys)
}
n := nptr*typeBitsWidth/8 + 1
n := nptr/8 + 1
p := sysAlloc(n, &memstats.other_sys)
if p == nil {
throw("heapdump: out of memory")
}
tmpbuf = (*[1 << 30]byte)(p)[:n]
}
// Convert heap bitmap to type bitmap.
// Convert heap bitmap to pointer bitmap.
for i := uintptr(0); i < nptr/8+1; i++ {
tmpbuf[i] = 0
}
i := uintptr(0)
hbits := heapBitsForAddr(p)
for ; i < nptr; i++ {
......@@ -740,8 +735,9 @@ func makeheapobjbv(p uintptr, size uintptr) bitvector {
break // end of object
}
hbits = hbits.next()
tmpbuf[i*typeBitsWidth/8] &^= (typeMask << ((i * typeBitsWidth) % 8))
tmpbuf[i*typeBitsWidth/8] |= bits << ((i * typeBitsWidth) % 8)
if bits == typePointer {
tmpbuf[i/8] |= 1 << (i % 8)
}
}
return bitvector{int32(i * typeBitsWidth), &tmpbuf[0]}
return bitvector{int32(i), &tmpbuf[0]}
}
src/runtime/mbitmap.go
View file @ 4d0f3a1c
......@@ -4,9 +4,20 @@
// Garbage collector: type and heap bitmaps.
//
// Stack, data, and bss bitmaps
//
// Not handled in this file, but worth mentioning: stack frames and global data
// in the data and bss sections are described by 1-bit bitmaps in which 0 means
// scalar or uninitialized or dead and 1 means pointer to visit during GC.
//
// Comparing this 1-bit form with the 2-bit form described below, 0 represents
// both the 2-bit 00 and 01, while 1 represents the 2-bit 10.
// Therefore conversions between the two (until the 2-bit form is gone)
// can be done by x>>1 for 2-bit to 1-bit and x+1 for 1-bit to 2-bit.
//
// Type bitmaps
//
// The global variables (in the data and bss sections) and types that aren't too large
// Types that aren't too large
// record information about the layout of their memory words using a type bitmap.
// The bitmap holds two bits for each pointer-sized word. The two-bit values are:
//
......@@ -17,7 +28,6 @@
//
// typeDead only appears in type bitmaps in Go type descriptors
// and in type bitmaps embedded in the heap bitmap (see below).
// It is not used in the type bitmap for the global variables.
//
// Heap bitmap
//
......@@ -71,9 +81,8 @@ const (
typePointer = 2
typePointerCheckmarked = 3
typeBitsWidth = 2 // # of type bits per pointer-sized word
typeMask = 1<<typeBitsWidth - 1
typeBitmapScale = ptrSize * (8 / typeBitsWidth) // number of data bytes per type bitmap byte
typeBitsWidth = 2 // # of type bits per pointer-sized word
typeMask = 1<<typeBitsWidth - 1
heapBitsWidth = 4
heapBitmapScale = ptrSize * (8 / heapBitsWidth) // number of data bytes per heap bitmap byte
......@@ -600,7 +609,7 @@ const (
// mask is where to store the result.
// If inplace is true, store the result not in mask but in the heap bitmap for mask.
// ppos is a pointer to position in mask, in bits.
// sparse says to generate 4-bits per word mask for heap (2-bits for data/bss otherwise).
// sparse says to generate 4-bits per word mask for heap (1-bit for data/bss otherwise).
//go:nowritebarrier
func unrollgcprog1(maskp *byte, prog *byte, ppos *uintptr, inplace, sparse bool) *byte {
pos := *ppos
......@@ -635,10 +644,10 @@ func unrollgcprog1(maskp *byte, prog *byte, ppos *uintptr, inplace, sparse bool)
mask[pos/8] |= v
pos += heapBitsWidth
} else {
// 2-bits per word
v <<= pos % 8
mask[pos/8] |= v
pos += typeBitsWidth
// 1 bit per word, for data/bss bitmap
v >>= 1 // convert typePointer to 1, others to 0
mask[pos/8] |= v << (pos % 8)
pos++
}
}
prog = addb(prog, round(uintptr(siz)*typeBitsWidth, 8)/8)
......@@ -668,13 +677,13 @@ func unrollgcprog1(maskp *byte, prog *byte, ppos *uintptr, inplace, sparse bool)
// Unrolls GC program prog for data/bss, returns dense GC mask.
func unrollglobgcprog(prog *byte, size uintptr) bitvector {
masksize := round(round(size, ptrSize)/ptrSize*typeBitsWidth, 8) / 8
masksize := round(round(size, ptrSize)/ptrSize, 8) / 8
mask := (*[1 << 30]byte)(persistentalloc(masksize+1, 0, &memstats.gc_sys))
mask[masksize] = 0xa1
pos := uintptr(0)
prog = unrollgcprog1(&mask[0], prog, &pos, false, false)
if pos != size/ptrSize*typeBitsWidth {
print("unrollglobgcprog: bad program size, got ", pos, ", expect ", size/ptrSize*typeBitsWidth, "\n")
if pos != size/ptrSize {
print("unrollglobgcprog: bad program size, got ", pos, ", expect ", size/ptrSize, "\n")
throw("unrollglobgcprog: bad program size")
}
if *prog != insEnd {
......@@ -744,8 +753,6 @@ func getgcmask(p unsafe.Pointer, t *_type, mask **byte, len *uintptr) {
*mask = nil
*len = 0
const typeBitsPerByte = 8 / typeBitsWidth
// data
for datap := &firstmoduledata; datap != nil; datap = datap.next {
if datap.data <= uintptr(p) && uintptr(p) < datap.edata {
......@@ -754,8 +761,9 @@ func getgcmask(p unsafe.Pointer, t *_type, mask **byte, len *uintptr) {
*mask = &make([]byte, *len)[0]
for i := uintptr(0); i < n; i += ptrSize {
off := (uintptr(p) + i - datap.data) / ptrSize
bits := (*(*byte)(add(unsafe.Pointer(datap.gcdatamask.bytedata), off/typeBitsPerByte)) >> ((off % typeBitsPerByte) * typeBitsWidth)) & typeMask
*(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits
bits := (*addb(datap.gcdatamask.bytedata, off/8) >> (off % 8)) & 1
bits += 1 // convert 1-bit to 2-bit
*addb(*mask, i/ptrSize) = bits
}
return
}
......@@ -767,8 +775,9 @@ func getgcmask(p unsafe.Pointer, t *_type, mask **byte, len *uintptr) {
*mask = &make([]byte, *len)[0]
for i := uintptr(0); i < n; i += ptrSize {
off := (uintptr(p) + i - datap.bss) / ptrSize
bits := (*(*byte)(add(unsafe.Pointer(datap.gcbssmask.bytedata), off/typeBitsPerByte)) >> ((off % typeBitsPerByte) * typeBitsWidth)) & typeMask
*(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits
bits := (*addb(datap.gcbssmask.bytedata, off/8) >> (off % 8)) & 1
bits += 1 // convert 1-bit to 2-bit
*addb(*mask, i/ptrSize) = bits
}
return
}
......@@ -782,7 +791,7 @@ func getgcmask(p unsafe.Pointer, t *_type, mask **byte, len *uintptr) {
*mask = &make([]byte, *len)[0]
for i := uintptr(0); i < n; i += ptrSize {
bits := heapBitsForAddr(base + i).typeBits()
*(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits
*addb(*mask, i/ptrSize) = bits
}
return
}
......@@ -810,14 +819,15 @@ func getgcmask(p unsafe.Pointer, t *_type, mask **byte, len *uintptr) {
return
}
bv := stackmapdata(stkmap, pcdata)
size := uintptr(bv.n) / typeBitsWidth * ptrSize
size := uintptr(bv.n) * ptrSize
n := (*ptrtype)(unsafe.Pointer(t)).elem.size
*len = n / ptrSize
*mask = &make([]byte, *len)[0]
for i := uintptr(0); i < n; i += ptrSize {
off := (uintptr(p) + i - frame.varp + size) / ptrSize
bits := ((*(*byte)(add(unsafe.Pointer(bv.bytedata), off*typeBitsWidth/8))) >> ((off * typeBitsWidth) % 8)) & typeMask
*(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits
bits := (*addb(bv.bytedata, off/8) >> (off % 8)) & 1
bits += 1 // convert 1-bit to 2-bit
*addb(*mask, i/ptrSize) = bits
}
}
}
src/runtime/mfinal.go
View file @ 4d0f3a1c
......@@ -20,7 +20,7 @@ var finlock mutex // protects the following variables
var fing *g // goroutine that runs finalizers
var finq *finblock // list of finalizers that are to be executed
var finc *finblock // cache of free blocks
var finptrmask [_FinBlockSize / typeBitmapScale]byte
var finptrmask [_FinBlockSize / ptrSize / 8]byte
var fingwait bool
var fingwake bool
var allfin *finblock // list of all blocks
......@@ -35,25 +35,31 @@ type finalizer struct {
}
var finalizer1 = [...]byte{
// Each Finalizer is 5 words, ptr ptr uintptr ptr ptr.
// Each byte describes 4 words.
// Need 4 Finalizers described by 5 bytes before pattern repeats:
// ptr ptr uintptr ptr ptr
// ptr ptr uintptr ptr ptr
// ptr ptr uintptr ptr ptr
// ptr ptr uintptr ptr ptr
// Each Finalizer is 5 words, ptr ptr INT ptr ptr (INT = uintptr here)
// Each byte describes 8 words.
// Need 8 Finalizers described by 5 bytes before pattern repeats:
// ptr ptr INT ptr ptr
// ptr ptr INT ptr ptr
// ptr ptr INT ptr ptr
// ptr ptr INT ptr ptr
// ptr ptr INT ptr ptr
// ptr ptr INT ptr ptr
// ptr ptr INT ptr ptr
// ptr ptr INT ptr ptr
// aka
// ptr ptr uintptr ptr
// ptr ptr ptr uintptr
// ptr ptr ptr ptr
// uintptr ptr ptr ptr
// ptr uintptr ptr ptr
//
// ptr ptr INT ptr ptr ptr ptr INT
// ptr ptr ptr ptr INT ptr ptr ptr
// ptr INT ptr ptr ptr ptr INT ptr
// ptr ptr ptr INT ptr ptr ptr ptr
// INT ptr ptr ptr ptr INT ptr ptr
//
// Assumptions about Finalizer layout checked below.
typePointer | typePointer<<2 | typeScalar<<4 | typePointer<<6,
typePointer | typePointer<<2 | typePointer<<4 | typeScalar<<6,
typePointer | typePointer<<2 | typePointer<<4 | typePointer<<6,
typeScalar | typePointer<<2 | typePointer<<4 | typePointer<<6,
typePointer | typeScalar<<2 | typePointer<<4 | typePointer<<6,
1<<0 | 1<<1 | 0<<2 | 1<<3 | 1<<4 | 1<<5 | 1<<6 | 0<<7,
1<<0 | 1<<1 | 1<<2 | 1<<3 | 0<<4 | 1<<5 | 1<<6 | 1<<7,
1<<0 | 0<<1 | 1<<2 | 1<<3 | 1<<4 | 1<<5 | 0<<6 | 1<<7,
1<<0 | 1<<1 | 1<<2 | 0<<3 | 1<<4 | 1<<5 | 1<<6 | 1<<7,
0<<0 | 1<<1 | 1<<2 | 1<<3 | 1<<4 | 0<<5 | 1<<6 | 1<<7,
}
func queuefinalizer(p unsafe.Pointer, fn *funcval, nret uintptr, fint *_type, ot *ptrtype) {
......@@ -72,8 +78,7 @@ func queuefinalizer(p unsafe.Pointer, fn *funcval, nret uintptr, fint *_type, ot
unsafe.Offsetof(finalizer{}.arg) != ptrSize ||
unsafe.Offsetof(finalizer{}.nret) != 2*ptrSize ||
unsafe.Offsetof(finalizer{}.fint) != 3*ptrSize ||
unsafe.Offsetof(finalizer{}.ot) != 4*ptrSize ||
typeBitsWidth != 2) {
unsafe.Offsetof(finalizer{}.ot) != 4*ptrSize) {
throw("finalizer out of sync")
}
for i := range finptrmask {
......
src/runtime/mgcmark.go
View file @ 4d0f3a1c
......@@ -51,7 +51,7 @@ func gcscan_m() {
}
// ptrmask for an allocation containing a single pointer.
var oneptr = [...]uint8{typePointer}
var oneptrmask = [...]uint8{1}
//go:nowritebarrier
func markroot(desc *parfor, i uint32) {
......@@ -98,9 +98,9 @@ func markroot(desc *parfor, i uint32) {
// A finalizer can be set for an inner byte of an object, find object beginning.
p := uintptr(s.start<<_PageShift) + uintptr(spf.special.offset)/s.elemsize*s.elemsize
if gcphase != _GCscan {
scanblock(p, s.elemsize, nil, &gcw) // scanned during mark phase
scanobject(p, &gcw) // scanned during mark termination
}
scanblock(uintptr(unsafe.Pointer(&spf.fn)), ptrSize, &oneptr[0], &gcw)
scanblock(uintptr(unsafe.Pointer(&spf.fn)), ptrSize, &oneptrmask[0], &gcw)
}
}
......@@ -383,7 +383,7 @@ func scanframeworker(frame *stkframe, unused unsafe.Pointer, gcw *gcWork) {
throw("scanframe: bad symbol table")
}
bv := stackmapdata(stkmap, pcdata)
size = (uintptr(bv.n) / typeBitsWidth) * ptrSize
size = uintptr(bv.n) * ptrSize
scanblock(frame.varp-size, size, bv.bytedata, gcw)
}
......@@ -405,7 +405,7 @@ func scanframeworker(frame *stkframe, unused unsafe.Pointer, gcw *gcWork) {
}
bv = stackmapdata(stkmap, pcdata)
}
scanblock(frame.argp, uintptr(bv.n)/typeBitsWidth*ptrSize, bv.bytedata, gcw)
scanblock(frame.argp, uintptr(bv.n)*ptrSize, bv.bytedata, gcw)
}
}
......@@ -447,7 +447,7 @@ func gcDrain(gcw *gcWork, flushScanCredit int64) {
// out of the wbuf passed in + a single object placed
// into an empty wbuf in scanobject so there could be
// a performance hit as we keep fetching fresh wbufs.
scanobject(b, 0, nil, gcw)
scanobject(b, gcw)
// Flush background scan work credit to the global
// account if we've accumulated enough locally so
......@@ -499,7 +499,7 @@ func gcDrainUntilPreempt(gcw *gcWork, flushScanCredit int64) {
// No more work
break
}
scanobject(b, 0, nil, gcw)
scanobject(b, gcw)
// Flush background scan work credit to the global
// account if we've accumulated enough locally so
......@@ -534,12 +534,12 @@ func gcDrainN(gcw *gcWork, scanWork int64) {
if b == 0 {
return
}
scanobject(b, 0, nil, gcw)
scanobject(b, gcw)
}
}
// scanblock scans b as scanobject would.
// If the gcphase is GCscan, scanblock performs additional checks.
// scanblock scans b as scanobject would, but using an explicit
// pointer bitmap instead of the heap bitmap.
//go:nowritebarrier
func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork) {
// Use local copies of original parameters, so that a stack trace
......@@ -548,59 +548,69 @@ func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork) {
b := b0
n := n0
// ptrmask can have 2 possible values:
// 1. nil - obtain pointer mask from GC bitmap.
// 2. pointer to a compact mask (for stacks and data).
arena_start := mheap_.arena_start
arena_used := mheap_.arena_used
scanWork := int64(0)
scanobject(b, n, ptrmask, gcw)
if gcphase == _GCscan {
if inheap(b) && ptrmask == nil {
// b is in heap, we are in GCscan so there should be a ptrmask.
throw("scanblock: In GCscan phase and inheap is true.")
for i := uintptr(0); i < n; {
// Find bits for the next word.
bits := uint32(*addb(ptrmask, i/(ptrSize*8)))
if bits == 0 {
i += ptrSize * 8
continue
}
for j := 0; j < 8 && i < n; j++ {
if bits&1 != 0 {
// Same work as in scanobject; see comments there.
obj := *(*uintptr)(unsafe.Pointer(b + i))
scanWork++
if obj != 0 && arena_start <= obj && obj < arena_used {
if mheap_.shadow_enabled && debug.wbshadow >= 2 && debug.gccheckmark > 0 && useCheckmark {
checkwbshadow((*uintptr)(unsafe.Pointer(b + i)))
}
if obj, hbits, span := heapBitsForObject(obj); obj != 0 {
greyobject(obj, b, i, hbits, span, gcw)
}
}
}
bits >>= 1
i += ptrSize
}
}
gcw.bytesMarked += uint64(n)
gcw.scanWork += scanWork
}
// scanobject scans memory starting at b, adding pointers to gcw.
// If ptrmask != nil, it specifies the pointer mask starting at b and
// n specifies the number of bytes to scan.
// If ptrmask == nil, b must point to the beginning of a heap object
// and scanobject consults the GC bitmap for the pointer mask and the
// spans for the size of the object (it ignores n).
// scanobject scans the object starting at b, adding pointers to gcw.
// b must point to the beginning of a heap object; scanobject consults
// the GC bitmap for the pointer mask and the spans for the size of the
// object (it ignores n).
//go:nowritebarrier
func scanobject(b, n uintptr, ptrmask *uint8, gcw *gcWork) {
func scanobject(b uintptr, gcw *gcWork) {
arena_start := mheap_.arena_start
arena_used := mheap_.arena_used
scanWork := int64(0)
// Find bits of the beginning of the object.
var hbits heapBits
if ptrmask == nil {
// b must point to the beginning of a heap object, so
// we can get its bits and span directly.
hbits = heapBitsForAddr(b)
s := spanOfUnchecked(b)
n = s.elemsize
if n == 0 {
throw("scanobject n == 0")
}
// b must point to the beginning of a heap object, so
// we can get its bits and span directly.
hbits := heapBitsForAddr(b)
s := spanOfUnchecked(b)
n := s.elemsize
if n == 0 {
throw("scanobject n == 0")
}
for i := uintptr(0); i < n; i += ptrSize {
// Find bits for this word.
var bits uintptr
if ptrmask != nil {
// dense mask (stack or data)
bits = (uintptr(*(*byte)(add(unsafe.Pointer(ptrmask), (i/ptrSize)/4))) >> (((i / ptrSize) % 4) * typeBitsWidth)) & typeMask
} else {
if i != 0 {
// Avoid needless hbits.next() on last iteration.
hbits = hbits.next()
}
bits = uintptr(hbits.typeBits())
if bits == typeDead {
break // no more pointers in this object
}
if i != 0 {
// Avoid needless hbits.next() on last iteration.
hbits = hbits.next()
}
bits := uintptr(hbits.typeBits())
if bits == typeDead {
break // no more pointers in this object
}
if bits <= typeScalar { // typeScalar, typeDead, typeScalarMarked
......@@ -608,10 +618,13 @@ func scanobject(b, n uintptr, ptrmask *uint8, gcw *gcWork) {
}
if bits&typePointer != typePointer {
print("gc useCheckmark=", useCheckmark, " b=", hex(b), " ptrmask=", ptrmask, "\n")
print("gc useCheckmark=", useCheckmark, " b=", hex(b), "\n")
throw("unexpected garbage collection bits")
}
// Work here is duplicated in scanblock.
// If you make changes here, make changes there too.
obj := *(*uintptr)(unsafe.Pointer(b + i))
// Track the scan work performed as a way to estimate
......@@ -626,17 +639,15 @@ func scanobject(b, n uintptr, ptrmask *uint8, gcw *gcWork) {
// At this point we have extracted the next potential pointer.
// Check if it points into heap.
if obj == 0 || obj < arena_start || obj >= arena_used {
continue
}
if mheap_.shadow_enabled && debug.wbshadow >= 2 && debug.gccheckmark > 0 && useCheckmark {
checkwbshadow((*uintptr)(unsafe.Pointer(b + i)))
}
if obj != 0 && arena_start <= obj && obj < arena_used {
if mheap_.shadow_enabled && debug.wbshadow >= 2 && debug.gccheckmark > 0 && useCheckmark {
checkwbshadow((*uintptr)(unsafe.Pointer(b + i)))
}
// Mark the object.
if obj, hbits, span := heapBitsForObject(obj); obj != 0 {
greyobject(obj, b, i, hbits, span, gcw)
// Mark the object.
if obj, hbits, span := heapBitsForObject(obj); obj != 0 {
greyobject(obj, b, i, hbits, span, gcw)
}
}
}
gcw.bytesMarked += uint64(n)
......
src/runtime/stack1.go
View file @ 4d0f3a1c
......@@ -298,10 +298,9 @@ func stackfree(stk stack) {
var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real
var mapnames = []string{
typeDead: "---",
typeScalar: "scalar",
typePointer: "ptr",
var ptrnames = []string{
0: "scalar",
1: "ptr",
}
// Stack frame layout
......@@ -365,8 +364,8 @@ func gobv(bv bitvector) gobitvector {
}
}
func ptrbits(bv *gobitvector, i uintptr) uint8 {
return (bv.bytedata[i/4] >> ((i & 3) * 2)) & 3
func ptrbit(bv *gobitvector, i uintptr) uint8 {
return (bv.bytedata[i/8] >> (i % 8)) & 1
}
// bv describes the memory starting at address scanp.
......@@ -376,21 +375,12 @@ func adjustpointers(scanp unsafe.Pointer, cbv *bitvector, adjinfo *adjustinfo, f
minp := adjinfo.old.lo
maxp := adjinfo.old.hi
delta := adjinfo.delta
num := uintptr(bv.n) / typeBitsWidth
num := uintptr(bv.n)
for i := uintptr(0); i < num; i++ {
if stackDebug >= 4 {
print(" ", add(scanp, i*ptrSize), ":", mapnames[ptrbits(&bv, i)], ":", hex(*(*uintptr)(add(scanp, i*ptrSize))), " # ", i, " ", bv.bytedata[i/4], "\n")
print(" ", add(scanp, i*ptrSize), ":", ptrnames[ptrbit(&bv, i)], ":", hex(*(*uintptr)(add(scanp, i*ptrSize))), " # ", i, " ", bv.bytedata[i/4], "\n")
}
switch ptrbits(&bv, i) {
default:
throw("unexpected pointer bits")
case typeDead:
if debug.gcdead != 0 {
*(*unsafe.Pointer)(add(scanp, i*ptrSize)) = unsafe.Pointer(uintptr(poisonStack))
}
case typeScalar:
// ok
case typePointer:
if ptrbit(&bv, i) == 1 {
p := *(*unsafe.Pointer)(add(scanp, i*ptrSize))
up := uintptr(p)
if f != nil && 0 < up && up < _PageSize && debug.invalidptr != 0 || up == poisonStack {
......@@ -461,7 +451,7 @@ func adjustframe(frame *stkframe, arg unsafe.Pointer) bool {
throw("bad symbol table")
}
bv = stackmapdata(stackmap, pcdata)
size = (uintptr(bv.n) / typeBitsWidth) * ptrSize
size = uintptr(bv.n) * ptrSize
if stackDebug >= 3 {
print(" locals ", pcdata, "/", stackmap.n, " ", size/ptrSize, " words ", bv.bytedata, "\n")
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment