math/big: fix Exp(x, x, x) for certain large x

Fixes #13907.

Change-Id: Ieaa5183f399b12a9177372212adf481c8f0b4a0d
Reviewed-on: https://go-review.googlesource.com/18491Reviewed-by: Robert Griesemer <gri@golang.org>
Reviewed-by: Vlad Krasnov <vlad@cloudflare.com>
Reviewed-by: Adam Langley <agl@golang.org>

src/math/big/int_test.go

@@ -566,6 +566,12 @@ var expTests = []struct {
 		"0xAC6BDB41324A9A9BF166DE5E1389582FAF72B6651987EE07FC3192943DB56050A37329CBB4A099ED8193E0757767A13DD52312AB4B03310DCD7F48A9DA04FD50E8083969EDB767B0CF6095179A163AB3661A05FBD5FAAAE82918A9962F0B93B855F97993EC975EEAA80D740ADBF4FF747359D041D5C33EA71D281E446B14773BCA97B43A23FB801676BD207A436C6481F1D2B9078717461A5B9D32E688F87748544523B524B0D57D5EA77A2775D2ECFA032CFBDBF52FB3786160279004E57AE6AF874E7303CE53299CCC041C7BC308D82A5698F3A8D0C38271AE35F8E9DBFBB694B5C803D89F7AE435DE236D525F54759B65E372FCD68EF20FA7111F9E4AFF73",
 		"21484252197776302499639938883777710321993113097987201050501182909581359357618579566746556372589385361683610524730509041328855066514963385522570894839035884713051640171474186548713546686476761306436434146475140156284389181808675016576845833340494848283681088886584219750554408060556769486628029028720727393293111678826356480455433909233520504112074401376133077150471237549474149190242010469539006449596611576612573955754349042329130631128234637924786466585703488460540228477440853493392086251021228087076124706778899179648655221663765993962724699135217212118535057766739392069738618682722216712319320435674779146070442",
 	},
+
+	// test cases for issue 13907
+	{"0xffffffff00000001", "0xffffffff00000001", "0xffffffff00000001", "0"},
+	{"0xffffffffffffffff00000001", "0xffffffffffffffff00000001", "0xffffffffffffffff00000001", "0"},
+	{"0xffffffffffffffffffffffff00000001", "0xffffffffffffffffffffffff00000001", "0xffffffffffffffffffffffff00000001", "0"},
+	{"0xffffffffffffffffffffffffffffffff00000001", "0xffffffffffffffffffffffffffffffff00000001", "0xffffffffffffffffffffffffffffffff00000001", "0"},
 }
 
 func TestExp(t *testing.T) {
@@ -593,7 +599,7 @@ func TestExp(t *testing.T) {
 			t.Errorf("#%d: %v is not normalized", i, *z1)
 		}
 		if z1.Cmp(out) != 0 {
-			t.Errorf("#%d: got %s want %s", i, z1, out)
+			t.Errorf("#%d: got %x want %x", i, z1, out)
 		}
 
 		if m == nil {
@@ -602,7 +608,7 @@ func TestExp(t *testing.T) {
 			m = &Int{abs: nat{}} // m != nil && len(m.abs) == 0
 			z2 := new(Int).Exp(x, y, m)
 			if z2.Cmp(z1) != 0 {
-				t.Errorf("#%d: got %s want %s", i, z2, z1)
+				t.Errorf("#%d: got %x want %x", i, z2, z1)
 			}
 		}
 	}

src/math/big/nat.go

@@ -1126,6 +1126,23 @@ func (z nat) expNNMontgomery(x, y, m nat) nat {
 	}
 	// convert to regular number
 	zz = zz.montgomery(z, one, m, k0, numWords)
+
+	// One last reduction, just in case.
+	// See golang.org/issue/13907.
+	if zz.cmp(m) >= 0 {
+		// Common case is m has high bit set; in that case,
+		// since zz is the same length as m, there can be just
+		// one multiple of m to remove. Just subtract.
+		// We think that the subtract should be sufficient in general,
+		// so do that unconditionally, but double-check,
+		// in case our beliefs are wrong.
+		// The div is not expected to be reached.
+		zz = zz.sub(zz, m)
+		if zz.cmp(m) >= 0 {
+			_, zz = nat(nil).div(nil, zz, m)
+		}
+	}
+
 	return zz.norm()
 }