net: deflake lookup tests

The build dashboard is dotted with net test failures.
We cannot declare all builders to have flaky networks,
although all fundamentally do.

Instead, add a simple retry/backoff loop to the ones that
show up most commonly on the dashboard at this moment.

If this approach works well in practice, we can
incrementally apply it to other flaky net tests.

Change-Id: I69c1ca6ce5b347ad549c7eb18d0438373f6e2489
Reviewed-on: https://go-review.googlesource.com/102397
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

src/net/lookup_test.go

@@ -60,6 +60,8 @@ var lookupGoogleSRVTests = []struct {
 	},
 }
 
+var backoffDuration = [...]time.Duration{time.Second, 5 * time.Second, 30 * time.Second}
+
 func TestLookupGoogleSRV(t *testing.T) {
 	if testenv.Builder() == "" {
 		testenv.MustHaveExternalNetwork(t)
@@ -69,10 +71,20 @@ func TestLookupGoogleSRV(t *testing.T) {
 		t.Skip("IPv4 is required")
 	}
 
-	for _, tt := range lookupGoogleSRVTests {
+	attempts := 0
+	for i := 0; i < len(lookupGoogleSRVTests); i++ {
+		tt := lookupGoogleSRVTests[i]
 		cname, srvs, err := LookupSRV(tt.service, tt.proto, tt.name)
 		if err != nil {
 			testenv.SkipFlakyNet(t)
+			if attempts < len(backoffDuration) {
+				dur := backoffDuration[attempts]
+				t.Logf("backoff %v after failure %v\n", dur, err)
+				time.Sleep(dur)
+				attempts++
+				i--
+				continue
+			}
 			t.Fatal(err)
 		}
 		if len(srvs) == 0 {
@@ -107,9 +119,20 @@ func TestLookupGmailMX(t *testing.T) {
 
 	defer dnsWaitGroup.Wait()
 
-	for _, tt := range lookupGmailMXTests {
+	attempts := 0
+	for i := 0; i < len(lookupGmailMXTests); i++ {
+		tt := lookupGmailMXTests[i]
 		mxs, err := LookupMX(tt.name)
 		if err != nil {
+			testenv.SkipFlakyNet(t)
+			if attempts < len(backoffDuration) {
+				dur := backoffDuration[attempts]
+				t.Logf("backoff %v after failure %v\n", dur, err)
+				time.Sleep(dur)
+				attempts++
+				i--
+				continue
+			}
 			t.Fatal(err)
 		}
 		if len(mxs) == 0 {
@@ -176,9 +199,20 @@ func TestLookupGmailTXT(t *testing.T) {
 
 	defer dnsWaitGroup.Wait()
 
-	for _, tt := range lookupGmailTXTTests {
+	attempts := 0
+	for i := 0; i < len(lookupGmailTXTTests); i++ {
+		tt := lookupGmailTXTTests[i]
 		txts, err := LookupTXT(tt.name)
 		if err != nil {
+			testenv.SkipFlakyNet(t)
+			if attempts < len(backoffDuration) {
+				dur := backoffDuration[attempts]
+				t.Logf("backoff %v after failure %v\n", dur, err)
+				time.Sleep(dur)
+				attempts++
+				i--
+				continue
+			}
 			t.Fatal(err)
 		}
 		if len(txts) == 0 {