runtime: count black allocations toward scan work

Currently we count black allocations toward the scannable heap size,
but not toward the scan work we've done so far. This is clearly
inconsistent (we have, in effect, scanned these allocations and since
they're already black, we're not going to scan them again). Worse, it
means we don't count black allocations toward the scannable heap size
as of the *next* GC because this is based on the amount of scan work
we did in this cycle.

Fix this by counting black allocations as scan work. Currently the GC
spends very little time in allocate-black mode, so this probably
hasn't been a problem, but this will become important when we switch
to always allocating black.

Change-Id: If6ff693b070c385b65b6ecbbbbf76283a0f9d990
Reviewed-on: https://go-review.googlesource.com/22119Reviewed-by: Rick Hudson <rlh@golang.org>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>

src/runtime/malloc.go

@@ -655,6 +655,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
 		size = s.elemsize
 	}
 
+	var scanSize uintptr
 	if noscan {
 		// All objects are pre-marked as noscan. Nothing to do.
 	} else {
@@ -673,11 +674,12 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
 			// pointers, GC has to scan to the last
 			// element.
 			if typ.ptrdata != 0 {
-				c.local_scan += dataSize - typ.size + typ.ptrdata
+				scanSize = dataSize - typ.size + typ.ptrdata
 			}
 		} else {
-			c.local_scan += typ.ptrdata
+			scanSize = typ.ptrdata
 		}
+		c.local_scan += scanSize
 
 		// Ensure that the stores above that initialize x to
 		// type-safe memory and set the heap bits occur before
@@ -694,7 +696,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
 	// a race marking the bit.
 	if gcphase == _GCmarktermination || gcBlackenPromptly {
 		systemstack(func() {
-			gcmarknewobject_m(uintptr(x), size)
+			gcmarknewobject_m(uintptr(x), size, scanSize)
 		})
 	}
 

src/runtime/mgc.go

@@ -304,7 +304,8 @@ type gcControllerState struct {
 	// scanWork is the total scan work performed this cycle. This
 	// is updated atomically during the cycle. Updates occur in
 	// bounded batches, since it is both written and read
-	// throughout the cycle.
+	// throughout the cycle. At the end of the cycle, this is how
+	// much of the retained heap is scannable.
 	//
 	// Currently this is the bytes of heap scanned. For most uses,
 	// this is an opaque unit of work, but for estimation the
@@ -1578,9 +1579,13 @@ func gcMark(start_time int64) {
 	work.markrootDone = true
 
 	for i := 0; i < int(gomaxprocs); i++ {
-		if !allp[i].gcw.empty() {
+		gcw := &allp[i].gcw
+		if !gcw.empty() {
 			throw("P has cached GC work at end of mark termination")
 		}
+		if gcw.scanWork != 0 || gcw.bytesMarked != 0 {
+			throw("P has unflushed stats at end of mark termination")
+		}
 	}
 
 	if trace.enabled {

src/runtime/mgcmark.go

@@ -1134,12 +1134,14 @@ func gcDumpObject(label string, obj, off uintptr) {
 
 // If gcBlackenPromptly is true we are in the second mark phase phase so we allocate black.
 //go:nowritebarrier
-func gcmarknewobject_m(obj, size uintptr) {
+func gcmarknewobject_m(obj, size, scanSize uintptr) {
 	if useCheckmark && !gcBlackenPromptly { // The world should be stopped so this should not happen.
 		throw("gcmarknewobject called while doing checkmark")
 	}
 	heapBitsForAddr(obj).setMarked()
 	atomic.Xadd64(&work.bytesMarked, int64(size))
+	gcw := &getg().m.p.ptr().gcw
+	gcw.scanWork += int64(scanSize)
 }
 
 // Checkmarking