crypto/tls: improve error message for unsupported certificates in TLS 1.3

Fixes #28960

Change-Id: I0d049d4776dc42ef165a1da15f63de08677fbb85
Reviewed-on: https://go-review.googlesource.com/c/151661
Run-TryBot: Filippo Valsorda <filippo@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Reviewed-by: Adam Langley <agl@golang.org>

src/crypto/tls/auth.go

@@ -143,7 +143,7 @@ func signatureSchemesForCertificate(version uint16, cert *Certificate) []Signatu
 		return nil
 	}
 
-	switch priv := priv.Public().(type) {
+	switch pub := priv.Public().(type) {
 	case *ecdsa.PublicKey:
 		if version != VersionTLS13 {
 			// In TLS 1.2 and earlier, ECDSA algorithms are not
@@ -155,7 +155,7 @@ func signatureSchemesForCertificate(version uint16, cert *Certificate) []Signatu
 				ECDSAWithSHA1,
 			}
 		}
-		switch priv.Curve {
+		switch pub.Curve {
 		case elliptic.P256():
 			return []SignatureScheme{ECDSAWithP256AndSHA256}
 		case elliptic.P384():
@@ -187,3 +187,35 @@ func signatureSchemesForCertificate(version uint16, cert *Certificate) []Signatu
 		return nil
 	}
 }
+
+// unsupportedCertificateError returns a helpful error for certificates with
+// an unsupported private key.
+func unsupportedCertificateError(cert *Certificate) error {
+	switch cert.PrivateKey.(type) {
+	case rsa.PrivateKey, ecdsa.PrivateKey:
+		return fmt.Errorf("tls: unsupported certificate: private key is %T, expected *%T",
+			cert.PrivateKey, cert.PrivateKey)
+	}
+
+	signer, ok := cert.PrivateKey.(crypto.Signer)
+	if !ok {
+		return fmt.Errorf("tls: certificate private key (%T) does not implement crypto.Signer",
+			cert.PrivateKey)
+	}
+
+	switch pub := signer.Public().(type) {
+	case *ecdsa.PublicKey:
+		switch pub.Curve {
+		case elliptic.P256():
+		case elliptic.P384():
+		case elliptic.P521():
+		default:
+			return fmt.Errorf("tls: unsupported certificate curve (%s)", pub.Curve.Params().Name)
+		}
+	case *rsa.PublicKey:
+	default:
+		return fmt.Errorf("tls: unsupported certificate key (%T)", pub)
+	}
+
+	return fmt.Errorf("tls: internal error: unsupported key (%T)", cert.PrivateKey)
+}

src/crypto/tls/handshake_client_tls13.go

@@ -10,7 +10,6 @@ import (
 	"crypto/hmac"
 	"crypto/rsa"
 	"errors"
-	"fmt"
 	"hash"
 	"sync/atomic"
 	"time"
@@ -559,7 +558,7 @@ func (hs *clientHandshakeStateTLS13) sendClientCertificate() error {
 	supportedAlgs := signatureSchemesForCertificate(c.vers, cert)
 	if supportedAlgs == nil {
 		c.sendAlert(alertInternalError)
-		return fmt.Errorf("tls: unsupported certificate key (%T)", cert.PrivateKey)
+		return unsupportedCertificateError(cert)
 	}
 	// Pick signature scheme in server preference order, as the client
 	// preference order is not configurable.

src/crypto/tls/handshake_server_tls13.go

@@ -10,7 +10,6 @@ import (
 	"crypto/hmac"
 	"crypto/rsa"
 	"errors"
-	"fmt"
 	"hash"
 	"io"
 	"sync/atomic"
@@ -372,7 +371,7 @@ func (hs *serverHandshakeStateTLS13) pickCertificate() error {
 	supportedAlgs := signatureSchemesForCertificate(c.vers, certificate)
 	if supportedAlgs == nil {
 		c.sendAlert(alertInternalError)
-		return fmt.Errorf("tls: unsupported certificate key (%T)", certificate.PrivateKey)
+		return unsupportedCertificateError(certificate)
 	}
 	// Pick signature scheme in client preference order, as the server
 	// preference order is not configurable.