math: Dim, Max, Min - allow more bit patterns for NaN

Fixes #9919

Change-Id: Ib443c762f727d4986ca7f8a404362f92b0e91aff
Reviewed-on: https://go-review.googlesource.com/5553Reviewed-by: Minux Ma <minux@golang.org>

src/math/all_test.go

@@ -991,6 +991,24 @@ var vffdimSC = [][2]float64{
 	{NaN(), Inf(1)},
 	{NaN(), NaN()},
 }
+var nan = Float64frombits(0xFFF8000000000000) // SSE2 DIVSD 0/0
+var vffdim2SC = [][2]float64{
+	{Inf(-1), Inf(-1)},
+	{Inf(-1), Inf(1)},
+	{Inf(-1), nan},
+	{Copysign(0, -1), Copysign(0, -1)},
+	{Copysign(0, -1), 0},
+	{0, Copysign(0, -1)},
+	{0, 0},
+	{Inf(1), Inf(-1)},
+	{Inf(1), Inf(1)},
+	{Inf(1), nan},
+	{nan, Inf(-1)},
+	{nan, Copysign(0, -1)},
+	{nan, 0},
+	{nan, Inf(1)},
+	{nan, nan},
+}
 var fdimSC = []float64{
 	NaN(),
 	0,
@@ -2015,6 +2033,11 @@ func TestDim(t *testing.T) {
 			t.Errorf("Dim(%g, %g) = %g, want %g", vffdimSC[i][0], vffdimSC[i][1], f, fdimSC[i])
 		}
 	}
+	for i := 0; i < len(vffdim2SC); i++ {
+		if f := Dim(vffdim2SC[i][0], vffdim2SC[i][1]); !alike(fdimSC[i], f) {
+			t.Errorf("Dim(%g, %g) = %g, want %g", vffdim2SC[i][0], vffdim2SC[i][1], f, fdimSC[i])
+		}
+	}
 }
 
 func TestFloor(t *testing.T) {
@@ -2041,6 +2064,11 @@ func TestMax(t *testing.T) {
 			t.Errorf("Max(%g, %g) = %g, want %g", vffdimSC[i][0], vffdimSC[i][1], f, fmaxSC[i])
 		}
 	}
+	for i := 0; i < len(vffdim2SC); i++ {
+		if f := Max(vffdim2SC[i][0], vffdim2SC[i][1]); !alike(fmaxSC[i], f) {
+			t.Errorf("Max(%g, %g) = %g, want %g", vffdim2SC[i][0], vffdim2SC[i][1], f, fmaxSC[i])
+		}
+	}
 }
 
 func TestMin(t *testing.T) {
@@ -2054,6 +2082,11 @@ func TestMin(t *testing.T) {
 			t.Errorf("Min(%g, %g) = %g, want %g", vffdimSC[i][0], vffdimSC[i][1], f, fminSC[i])
 		}
 	}
+	for i := 0; i < len(vffdim2SC); i++ {
+		if f := Min(vffdim2SC[i][0], vffdim2SC[i][1]); !alike(fminSC[i], f) {
+			t.Errorf("Min(%g, %g) = %g, want %g", vffdim2SC[i][0], vffdim2SC[i][1], f, fminSC[i])
+		}
+	}
 }
 
 func TestMod(t *testing.T) {

src/math/dim_amd64.s

@@ -26,13 +26,13 @@ dim2:	// (-Inf, -Inf) special case
 	JEQ     bothInf
 dim3:	// (NaN, x) or (x, NaN)
 	MOVQ    $~(1<<63), DX
-	MOVQ    $NaN, AX
+	MOVQ    $PosInf, AX
 	ANDQ    DX, BX // x = |x|
 	CMPQ    AX, BX
-	JLE     isDimNaN
+	JLT     isDimNaN
 	ANDQ    DX, CX // y = |y|
 	CMPQ    AX, CX
-	JLE     isDimNaN
+	JLT     isDimNaN
 
 	MOVSD x+0(FP), X0
 	SUBSD y+8(FP), X0
@@ -41,8 +41,8 @@ dim3:	// (NaN, x) or (x, NaN)
 	MOVSD X0, ret+16(FP)
 	RET
 bothInf: // Dim(-Inf, -Inf) or Dim(+Inf, +Inf)
-	MOVQ    $NaN, AX
 isDimNaN:
+	MOVQ    $NaN, AX
 	MOVQ    AX, ret+16(FP)
 	RET
 
@@ -58,15 +58,15 @@ TEXT ·Max(SB),NOSPLIT,$0
 	JEQ     isPosInf
 	// NaN special cases
 	MOVQ    $~(1<<63), DX // bit mask
-	MOVQ    $NaN, AX
+	MOVQ    $PosInf, AX
 	MOVQ    R8, BX
 	ANDQ    DX, BX // x = |x|
 	CMPQ    AX, BX
-	JLE     isMaxNaN
+	JLT     isMaxNaN
 	MOVQ    R9, CX
 	ANDQ    DX, CX // y = |y|
 	CMPQ    AX, CX
-	JLE     isMaxNaN
+	JLT     isMaxNaN
 	// ±0 special cases
 	ORQ     CX, BX
 	JEQ     isMaxZero
@@ -77,6 +77,7 @@ TEXT ·Max(SB),NOSPLIT,$0
 	MOVSD   X0, ret+16(FP)
 	RET
 isMaxNaN: // return NaN
+	MOVQ	$NaN, AX
 isPosInf: // return +Inf
 	MOVQ    AX, ret+16(FP)
 	RET
@@ -89,16 +90,6 @@ isMaxZero:
 	MOVQ    R9, ret+16(FP) // return other 0
 	RET
 
-/*
-	MOVQ    $0, AX
-	CMPQ    AX, R8
-	JNE     +3(PC)
-	MOVQ    R8, ret+16(FP) // return 0
-	RET
-	MOVQ    R9, ret+16(FP) // return other 0
-	RET
-*/
-
 // func Min(x, y float64) float64
 TEXT ·Min(SB),NOSPLIT,$0
 	// -Inf special cases
@@ -111,15 +102,15 @@ TEXT ·Min(SB),NOSPLIT,$0
 	JEQ     isNegInf
 	// NaN special cases
 	MOVQ    $~(1<<63), DX
-	MOVQ    $NaN, AX
+	MOVQ    $PosInf, AX
 	MOVQ    R8, BX
 	ANDQ    DX, BX // x = |x|
 	CMPQ    AX, BX
-	JLE     isMinNaN
+	JLT     isMinNaN
 	MOVQ    R9, CX
 	ANDQ    DX, CX // y = |y|
 	CMPQ    AX, CX
-	JLE     isMinNaN
+	JLT     isMinNaN
 	// ±0 special cases
 	ORQ     CX, BX
 	JEQ     isMinZero
@@ -130,6 +121,7 @@ TEXT ·Min(SB),NOSPLIT,$0
 	MOVSD X0, ret+16(FP)
 	RET
 isMinNaN: // return NaN
+	MOVQ	$NaN, AX
 isNegInf: // return -Inf
 	MOVQ    AX, ret+16(FP)
 	RET