runtime: mark tiny blocks at GC start

The hybrid barrier requires allocate-black, but there's one case where
we don't currently allocate black: the tiny allocator. If we allocate
a *new* tiny alloc block during GC, it will be allocated black, but if
we allocated the current block before GC, it won't be black, and the
further allocations from it won't mark it, which means we may free a
reachable tiny block during sweeping.

Fix this by passing over all mcaches at the beginning of mark, while
the world is still stopped, and greying their tiny blocks.

Updates #17503.

Change-Id: I04d4df7cc2f553f8f7b1e4cb0b52e2946588111a
Reviewed-on: https://go-review.googlesource.com/31456Reviewed-by: Rick Hudson <rlh@golang.org>

src/runtime/mgc.go

@@ -1025,6 +1025,13 @@ func gcStart(mode gcMode, forceTrigger bool) {
 		gcBgMarkPrepare() // Must happen before assist enable.
 		gcMarkRootPrepare()
 
+		// Mark all active tinyalloc blocks. Since we're
+		// allocating from these, they need to be black like
+		// other allocations. The alternative is to blacken
+		// the tiny block on every allocation from it, which
+		// would slow down the tiny allocator.
+		gcMarkTinyAllocs()
+
 		// At this point all Ps have enabled the write
 		// barrier, thus maintaining the no white to
 		// black invariant. Enable mutator assists to

src/runtime/mgcmark.go

@@ -1396,6 +1396,27 @@ func gcmarknewobject(obj, size, scanSize uintptr) {
 	}
 }
 
+// gcMarkTinyAllocs greys all active tiny alloc blocks.
+//
+// The world must be stopped.
+func gcMarkTinyAllocs() {
+	for _, p := range &allp {
+		if p == nil || p.status == _Pdead {
+			break
+		}
+		c := p.mcache
+		if c == nil || c.tiny == 0 {
+			continue
+		}
+		_, hbits, span, objIndex := heapBitsForObject(c.tiny, 0, 0)
+		gcw := &p.gcw
+		greyobject(c.tiny, 0, 0, hbits, span, gcw, objIndex)
+		if gcBlackenPromptly {
+			gcw.dispose()
+		}
+	}
+}
+
 // Checkmarking
 
 // To help debug the concurrent GC we remark with the world