runtime: assist harder if GC exceeds the estimated marked heap

Currently, the GC controller computes the mutator assist ratio at the
beginning of the cycle by estimating that the marked heap size this
cycle will be the same as it was the previous cycle. It then uses that
assist ratio for the rest of the cycle. However, this means that if
the mutator is quickly growing its reachable heap, the heap size is
likely to exceed the heap goal and currently there's no additional
pressure on mutator assists when this happens. For example, 6g (with
GOMAXPROCS=1) frequently exceeds the goal heap size by ~25% because of
this.

This change makes GC revise its work estimate and the resulting assist
ratio every 10ms during the concurrent mark. Instead of
unconditionally using the marked heap size from the last cycle as an
estimate for this cycle, it takes the minimum of the previously marked
heap and the currently marked heap. As a result, as the cycle
approaches or exceeds its heap goal, this will increase the assist
ratio to put more pressure on the mutator assist to bring the cycle to
an end. For 6g, this causes the GC to always finish within 5% and
often within 1% of its heap goal.

Change-Id: I4333b92ad0878c704964be42c655c38a862b4224
Reviewed-on: https://go-review.googlesource.com/9070Reviewed-by: Rick Hudson <rlh@golang.org>
Run-TryBot: Austin Clements <austin@google.com>

src/runtime/mgc.go

@@ -263,6 +263,14 @@ type gcControllerState struct {
 	// that the background mark phase started.
 	bgMarkStartTime int64
 
+	// initialHeapLive is the value of memstats.heap_live at the
+	// beginning of this cycle.
+	initialHeapLive uint64
+
+	// heapGoal is the goal memstats.heap_live for when this cycle
+	// ends. This is computed at the beginning of each cycle.
+	heapGoal uint64
+
 	// dedicatedMarkWorkersNeeded is the number of dedicated mark
 	// workers that need to be started. This is computed at the
 	// beginning of each cycle and decremented atomically as
@@ -314,6 +322,7 @@ func (c *gcControllerState) startCycle() {
 	c.dedicatedMarkTime = 0
 	c.fractionalMarkTime = 0
 	c.idleMarkTime = 0
+	c.initialHeapLive = memstats.heap_live
 
 	// If this is the first GC cycle or we're operating on a very
 	// small heap, fake heap_marked so it looks like next_gc is
@@ -325,24 +334,8 @@ func (c *gcControllerState) startCycle() {
 		memstats.heap_marked = uint64(float64(memstats.next_gc) / (1 + c.triggerRatio))
 	}
 
-	// Compute the expected work based on last cycle's marked bytes.
-	scanWorkExpected := uint64(float64(memstats.heap_marked) * c.workRatioAvg)
-
-	// Compute the mutator assist ratio so by the time the mutator
-	// allocates the remaining heap bytes up to next_gc, it will
-	// have done (or stolen) the estimated amount of scan work.
-	heapGoal := memstats.heap_marked + memstats.heap_marked*uint64(gcpercent)/100
-	heapDistance := int64(heapGoal) - int64(memstats.heap_live)
-	if heapDistance <= 1024*1024 {
-		// heapDistance can be negative if GC start is delayed
-		// or if the allocation that pushed heap_live over
-		// next_gc is large or if the trigger is really close
-		// to GOGC. We don't want to set the assist negative
-		// (or divide by zero, or set it really high), so
-		// enforce a minimum on the distance.
-		heapDistance = 1024 * 1024
-	}
-	c.assistRatio = float64(scanWorkExpected) / float64(heapDistance)
+	// Compute the heap goal for this cycle
+	c.heapGoal = memstats.heap_marked + memstats.heap_marked*uint64(gcpercent)/100
 
 	// Compute the total mark utilization goal and divide it among
 	// dedicated and fractional workers.
@@ -355,6 +348,10 @@ func (c *gcControllerState) startCycle() {
 		c.fractionalMarkWorkersNeeded = 0
 	}
 
+	// Compute initial values for controls that are updated
+	// throughout the cycle.
+	c.revise()
+
 	// Clear per-P state
 	for _, p := range &allp {
 		if p == nil {
@@ -366,6 +363,42 @@ func (c *gcControllerState) startCycle() {
 	return
 }
 
+// revise updates the assist ratio during the GC cycle to account for
+// improved estimates. This should be called periodically during
+// concurrent mark.
+func (c *gcControllerState) revise() {
+	// Estimate the size of the marked heap. We don't have much to
+	// go on, so at the beginning of the cycle this uses the
+	// marked heap size from last cycle. If the reachable heap has
+	// grown since last cycle, we'll eventually mark more than
+	// this and we can revise our estimate. This way, if we
+	// overshoot our initial estimate, the assist ratio will climb
+	// smoothly and put more pressure on mutator assists to finish
+	// the cycle.
+	heapMarkedEstimate := memstats.heap_marked
+	if heapMarkedEstimate < work.bytesMarked {
+		heapMarkedEstimate = work.bytesMarked
+	}
+
+	// Compute the expected work based on this estimate.
+	scanWorkExpected := uint64(float64(heapMarkedEstimate) * c.workRatioAvg)
+
+	// Compute the mutator assist ratio so by the time the mutator
+	// allocates the remaining heap bytes up to next_gc, it will
+	// have done (or stolen) the estimated amount of scan work.
+	heapDistance := int64(c.heapGoal) - int64(c.initialHeapLive)
+	if heapDistance <= 1024*1024 {
+		// heapDistance can be negative if GC start is delayed
+		// or if the allocation that pushed heap_live over
+		// next_gc is large or if the trigger is really close
+		// to GOGC. We don't want to set the assist negative
+		// (or divide by zero, or set it really high), so
+		// enforce a minimum on the distance.
+		heapDistance = 1024 * 1024
+	}
+	c.assistRatio = float64(scanWorkExpected) / float64(heapDistance)
+}
+
 // endCycle updates the GC controller state at the end of the
 // concurrent part of the GC cycle.
 func (c *gcControllerState) endCycle() {
@@ -725,7 +758,9 @@ func gc(mode int) {
 		if debug.gctrace > 0 {
 			tMark = nanotime()
 		}
-		notetsleepg(&work.bgMarkNote, -1)
+		for !notetsleepg(&work.bgMarkNote, 10*1000*1000) {
+			gcController.revise()
+		}
 		noteclear(&work.bgMarkNote)
 
 		// Begin mark termination.