Skip to content

G
golang
Commit f7aa454c authored by Josh Bleecher Snyder's avatar Josh Bleecher Snyder
Browse files
Options

runtime: mask shifts in map implementation on x86 

This slightly improves the generated code on x86 architectures,
including on many hot paths.

It is a no-op on other architectures.

Change-Id: I86336fd846bc5805a27bbec572e8c73dcbd0d567
Reviewed-on: https://go-review.googlesource.com/57411
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: 's avatarKeith Randall <khr@golang.org>
parent 5df1fe52
Hide whitespace changes
Inline Side-by-side
Showing with 38 additions and 25 deletions
src/runtime/hashmap.go
View file @ f7aa454c
......@@ -170,6 +170,19 @@ type hiter struct {
checkBucket uintptr
}
// bucketShift returns 1<<b, optimized for code generation.
func bucketShift(b uint8) uintptr {
if sys.GoarchAmd64|sys.GoarchAmd64p32|sys.Goarch386 != 0 {
b &= sys.PtrSize*8 - 1 // help x86 archs remove shift overflow checks
}
return uintptr(1) << b
}
// bucketMask returns 1<<b - 1, optimized for code generation.
func bucketMask(b uint8) uintptr {
return bucketShift(b) - 1
}
// tophash calculates the tophash value for hash.
func tophash(hash uintptr) uint8 {
top := uint8(hash >> (sys.PtrSize*8 - 8))
......@@ -374,7 +387,7 @@ func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
}
alg := t.key.alg
hash := alg.hash(key, uintptr(h.hash0))
m := uintptr(1)<<h.B - 1
m := bucketMask(h.B)
b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
......@@ -429,7 +442,7 @@ func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool)
}
alg := t.key.alg
hash := alg.hash(key, uintptr(h.hash0))
m := uintptr(1)<<h.B - 1
m := bucketMask(h.B)
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
......@@ -473,7 +486,7 @@ func mapaccessK(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, unsafe
}
alg := t.key.alg
hash := alg.hash(key, uintptr(h.hash0))
m := uintptr(1)<<h.B - 1
m := bucketMask(h.B)
b := (*bmap)(unsafe.Pointer(uintptr(h.buckets) + (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
......@@ -555,7 +568,7 @@ func mapassign(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
}
again:
bucket := hash & (uintptr(1)<<h.B - 1)
bucket := hash & bucketMask(h.B)
if h.growing() {
growWork(t, h, bucket)
}
......@@ -662,7 +675,7 @@ func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
// in which case we have not actually done a write (delete).
h.flags |= hashWriting
bucket := hash & (uintptr(1)<<h.B - 1)
bucket := hash & bucketMask(h.B)
if h.growing() {
growWork(t, h, bucket)
}
......@@ -758,7 +771,7 @@ func mapiterinit(t *maptype, h *hmap, it *hiter) {
if h.B > 31-bucketCntBits {
r += uintptr(fastrand()) << 31
}
it.startBucket = r & (uintptr(1)<<h.B - 1)
it.startBucket = r & bucketMask(h.B)
it.offset = uint8(r >> h.B & (bucketCnt - 1))
// iterator state
......@@ -817,7 +830,7 @@ next:
checkBucket = noCheck
}
bucket++
if bucket == uintptr(1)<<it.B {
if bucket == bucketShift(it.B) {
bucket = 0
it.wrapped = true
}
......@@ -845,7 +858,7 @@ next:
// If the item in the oldbucket is not destined for
// the current new bucket in the iteration, skip it.
hash := alg.hash(k, uintptr(h.hash0))
if hash&(uintptr(1)<<it.B-1) != checkBucket {
if hash&bucketMask(it.B) != checkBucket {
continue
}
} else {
......@@ -901,7 +914,7 @@ next:
}
func makeBucketArray(t *maptype, b uint8) (buckets unsafe.Pointer, nextOverflow *bmap) {
base := uintptr(1 << b)
base := bucketShift(b)
nbuckets := base
// For small b, overflow buckets are unlikely.
// Avoid the overhead of the calculation.
......@@ -909,7 +922,7 @@ func makeBucketArray(t *maptype, b uint8) (buckets unsafe.Pointer, nextOverflow
// Add on the estimated number of overflow buckets
// required to insert the median number of elements
// used with this value of b.
nbuckets += 1 << (b - 4)
nbuckets += bucketShift(b - 4)
sz := t.bucket.size * nbuckets
up := roundupsize(sz)
if up != sz {
......@@ -975,7 +988,7 @@ func hashGrow(t *maptype, h *hmap) {
// overLoadFactor reports whether count items placed in 1<<B buckets is over loadFactor.
func overLoadFactor(count int, B uint8) bool {
return count >= bucketCnt && uint64(count) >= loadFactorNum*((uint64(1)<<B)/loadFactorDen)
return count >= bucketCnt && uintptr(count) >= loadFactorNum*(bucketShift(B)/loadFactorDen)
}
// tooManyOverflowBuckets reports whether noverflow buckets is too many for a map with 1<<B buckets.
......@@ -1009,7 +1022,7 @@ func (h *hmap) noldbuckets() uintptr {
if !h.sameSizeGrow() {
oldB--
}
return uintptr(1) << oldB
return bucketShift(oldB)
}
// oldbucketmask provides a mask that can be applied to calculate n % noldbuckets().
......
src/runtime/hashmap_fast.go
View file @ f7aa454c
......@@ -26,7 +26,7 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
b = (*bmap)(h.buckets)
} else {
hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0))
m := uintptr(1)<<h.B - 1
m := bucketMask(h.B)
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
......@@ -69,7 +69,7 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
b = (*bmap)(h.buckets)
} else {
hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0))
m := uintptr(1)<<h.B - 1
m := bucketMask(h.B)
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
......@@ -112,7 +112,7 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
b = (*bmap)(h.buckets)
} else {
hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0))
m := uintptr(1)<<h.B - 1
m := bucketMask(h.B)
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
......@@ -155,7 +155,7 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
b = (*bmap)(h.buckets)
} else {
hash := t.key.alg.hash(noescape(unsafe.Pointer(&key)), uintptr(h.hash0))
m := uintptr(1)<<h.B - 1
m := bucketMask(h.B)
b = (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
......@@ -243,7 +243,7 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
}
dohash:
hash := t.key.alg.hash(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0))
m := uintptr(1)<<h.B - 1
m := bucketMask(h.B)
b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
......@@ -335,7 +335,7 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
}
dohash:
hash := t.key.alg.hash(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0))
m := uintptr(1)<<h.B - 1
m := bucketMask(h.B)
b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
if c := h.oldbuckets; c != nil {
if !h.sameSizeGrow() {
......@@ -386,7 +386,7 @@ func mapassign_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
}
again:
bucket := hash & (uintptr(1)<<h.B - 1)
bucket := hash & bucketMask(h.B)
if h.growing() {
growWork(t, h, bucket)
}
......@@ -471,7 +471,7 @@ func mapassign_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
}
again:
bucket := hash & (uintptr(1)<<h.B - 1)
bucket := hash & bucketMask(h.B)
if h.growing() {
growWork(t, h, bucket)
}
......@@ -557,7 +557,7 @@ func mapassign_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
}
again:
bucket := hash & (uintptr(1)<<h.B - 1)
bucket := hash & bucketMask(h.B)
if h.growing() {
growWork(t, h, bucket)
}
......@@ -642,7 +642,7 @@ func mapdelete_fast32(t *maptype, h *hmap, key uint32) {
// Set hashWriting after calling alg.hash for consistency with mapdelete
h.flags |= hashWriting
bucket := hash & (uintptr(1)<<h.B - 1)
bucket := hash & bucketMask(h.B)
if h.growing() {
growWork(t, h, bucket)
}
......@@ -695,7 +695,7 @@ func mapdelete_fast64(t *maptype, h *hmap, key uint64) {
// Set hashWriting after calling alg.hash for consistency with mapdelete
h.flags |= hashWriting
bucket := hash & (uintptr(1)<<h.B - 1)
bucket := hash & bucketMask(h.B)
if h.growing() {
growWork(t, h, bucket)
}
......@@ -749,7 +749,7 @@ func mapdelete_faststr(t *maptype, h *hmap, ky string) {
// Set hashWriting after calling alg.hash for consistency with mapdelete
h.flags |= hashWriting
bucket := hash & (uintptr(1)<<h.B - 1)
bucket := hash & bucketMask(h.B)
if h.growing() {
growWork(t, h, bucket)
}
......
src/runtime/runtime_test.go
View file @ f7aa454c
......@@ -369,7 +369,7 @@ func TestIntendedInlining(t *testing.T) {
t.Parallel()
// want is the list of function names that should be inlined.
want := []string{"tophash", "add", "(*bmap).keys"}
want := []string{"tophash", "add", "(*bmap).keys", "bucketShift", "bucketMask"}
m := make(map[string]bool, len(want))
for _, s := range want {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment