crypto/elliptic: utilize faster z14 multiply/square instructions (when available)

In the s390x assembly implementation of NIST P-256 curve, utilize faster multiply/square
instructions introduced in the z14. These new instructions are designed for crypto
and are constant time. The algorithm is unchanged except for faster
multiplication when run on a z14 or later. On z13, the original mutiplication
(also constant time) is used.

P-256 performance is critical in many applications, such as Blockchain.

name            old time      new time     delta
BaseMultP256    24396 ns/op   21564 ns/op  1.13x
ScalarMultP256  87546 ns/op   72813 ns/op. 1.20x

Change-Id: I7e6d8b420fac56d5f9cc13c9423e2080df854bac
Reviewed-on: https://go-review.googlesource.com/c/146022Reviewed-by: Michael Munday <mike.munday@ibm.com>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Michael Munday <mike.munday@ibm.com>

src/crypto/elliptic/p256_s390x.go

@@ -8,7 +8,14 @@ package elliptic
 
 import (
 	"crypto/subtle"
+	"internal/cpu"
 	"math/big"
+	"unsafe"
+)
+
+const (
+	offsetS390xHasVX  = unsafe.Offsetof(cpu.S390X.HasVX)
+	offsetS390xHasVE1 = unsafe.Offsetof(cpu.S390X.HasVE1)
 )
 
 type p256CurveFast struct {
@@ -26,14 +33,26 @@ var (
 	p256PreFast *[37][64]p256Point
 )
 
-// hasVectorFacility reports whether the machine has the z/Architecture
-// vector facility installed and enabled.
-func hasVectorFacility() bool
+//go:noescape
+func p256MulInternalTrampolineSetup()
+
+//go:noescape
+func p256SqrInternalTrampolineSetup()
+
+//go:noescape
+func p256MulInternalVX()
+
+//go:noescape
+func p256MulInternalVMSL()
 
-var hasVX = hasVectorFacility()
+//go:noescape
+func p256SqrInternalVX()
+
+//go:noescape
+func p256SqrInternalVMSL()
 
 func initP256Arch() {
-	if hasVX {
+	if cpu.S390X.HasVX {
 		p256 = p256CurveFast{p256Params}
 		initTable()
 		return
@@ -51,12 +70,15 @@ func (curve p256CurveFast) Params() *CurveParams {
 // Functions implemented in p256_asm_s390x.s
 // Montgomery multiplication modulo P256
 //
+//go:noescape
+func p256SqrAsm(res, in1 []byte)
+
 //go:noescape
 func p256MulAsm(res, in1, in2 []byte)
 
 // Montgomery square modulo P256
 func p256Sqr(res, in []byte) {
-	p256MulAsm(res, in, in)
+	p256SqrAsm(res, in)
 }
 
 // Montgomery multiplication by 1

src/internal/cpu/cpu.go

@@ -126,6 +126,7 @@ type s390x struct {
 	HasSHA256       bool // K{I,L}MD-SHA-256 functions
 	HasSHA512       bool // K{I,L}MD-SHA-512 functions
 	HasVX           bool // vector facility. Note: the runtime sets this when it processes auxv records.
+	HasVE1          bool // vector-enhancement 1
 	_               CacheLinePad
 }
 

src/internal/cpu/cpu_s390x.go

@@ -71,6 +71,9 @@ const (
 	msa5 facility = 57  // message-security-assist extension 5
 	msa8 facility = 146 // message-security-assist extension 8
 
+	// vector facilities
+	ve1 facility = 135 // vector-enhancements 1
+
 	// Note: vx and highgprs are excluded because they require
 	// kernel support and so must be fetched from HWCAP.
 )
@@ -115,6 +118,7 @@ func doinit() {
 		{Name: "dfp", Feature: &S390X.HasDFP},
 		{Name: "etf3eh", Feature: &S390X.HasETF3Enhanced},
 		{Name: "vx", Feature: &S390X.HasVX},
+		{Name: "ve1", Feature: &S390X.HasVE1},
 	}
 
 	aes := []function{aes128, aes192, aes256}
@@ -150,4 +154,7 @@ func doinit() {
 		S390X.HasSHA512 = kimd.Has(sha512) && klmd.Has(sha512)
 		S390X.HasGHASH = kimd.Has(ghash) // KLMD-GHASH does not exist
 	}
+	if S390X.HasVX {
+		S390X.HasVE1 = facilities.Has(ve1)
+	}
 }