runtime: native xadd for 386/amd64

benchmark                          old ns/op    new ns/op    delta
BenchmarkSemaUncontended               37.40        34.10   -8.82%
BenchmarkSemaUncontended-2             18.90        17.70   -6.35%
BenchmarkSemaUncontended-4             11.90        10.90   -8.40%
BenchmarkSemaUncontended-8              6.26         5.19  -17.09%
BenchmarkSemaUncontended-16             4.39         3.91  -10.93%
BenchmarkSemaSyntNonblock              38.00        35.30   -7.11%
BenchmarkSemaSyntNonblock-2            83.00        46.70  -43.73%
BenchmarkSemaSyntNonblock-4           124.00       101.00  -18.55%
BenchmarkSemaSyntNonblock-8           124.00       116.00   -6.45%
BenchmarkSemaSyntNonblock-16          148.00       114.00  -22.97%

(on HP Z600 2 x Xeon E5620, 8 HT cores, 2.40GHz)

R=golang-dev, rsc
CC=golang-dev
https://golang.org/cl/4755041

src/pkg/runtime/386/asm.s

@@ -318,6 +318,19 @@ TEXT runtime·casp(SB), 7, $0
 	MOVL	$1, AX
 	RET
 
+// uint32 xadd(uint32 volatile *val, int32 delta)
+// Atomically:
+//	*val += delta;
+//	return *val;
+TEXT runtime·xadd(SB), 7, $0
+	MOVL	4(SP), BX
+	MOVL	8(SP), AX
+	MOVL	AX, CX
+	LOCK
+	XADDL	AX, 0(BX)
+	ADDL	CX, AX
+	RET
+
 TEXT runtime·atomicstorep(SB), 7, $0
 	MOVL	4(SP), BX
 	MOVL	8(SP), AX

src/pkg/runtime/amd64/asm.s

@@ -364,6 +364,19 @@ TEXT runtime·casp(SB), 7, $0
 	MOVL	$1, AX
 	RET
 
+// uint32 xadd(uint32 volatile *val, int32 delta)
+// Atomically:
+//	*val += delta;
+//	return *val;
+TEXT runtime·xadd(SB), 7, $0
+	MOVQ	8(SP), BX
+	MOVL	16(SP), AX
+	MOVL	AX, CX
+	LOCK
+	XADDL	AX, 0(BX)
+	ADDL	CX, AX
+	RET
+
 TEXT runtime·atomicstorep(SB), 7, $0
 	MOVQ	8(SP), BX
 	MOVQ	16(SP), AX

src/pkg/runtime/arm/atomic.c

@@ -4,6 +4,21 @@
 
 #include "runtime.h"
 
+// Atomic add and return new value.
+#pragma textflag 7
+uint32
+runtime·xadd(uint32 volatile *val, int32 delta)
+{
+	uint32 oval, nval;
+
+	for(;;){
+		oval = *val;
+		nval = oval + delta;
+		if(runtime·cas(val, oval, nval))
+			return nval;
+	}
+}
+
 #pragma textflag 7
 uint32
 runtime·atomicload(uint32 volatile* addr)

src/pkg/runtime/runtime.c

@@ -207,20 +207,6 @@ runtime·goenvs_unix(void)
 	os·Envs.cap = n;
 }
 
-// Atomic add and return new value.
-uint32
-runtime·xadd(uint32 volatile *val, int32 delta)
-{
-	uint32 oval, nval;
-
-	for(;;){
-		oval = *val;
-		nval = oval + delta;
-		if(runtime·cas(val, oval, nval))
-			return nval;
-	}
-}
-
 byte*
 runtime·getenv(int8 *s)
 {