storage: retry on REFUSED_STREAM

Under high load, a REFUSED_STREAM error can be returned by the
Go http2 implementation prior to Go 1.10. We retry the error
with exponential backoff, as we do with other retryable errors.

Fixes #701.

Change-Id: I6e459876760d6666b0ab8e30cc4b5cf57b426f04
Reviewed-on: https://code-review.googlesource.com/15630
Reviewed-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Michael Darakananda <pongad@google.com>

storage/go110.go

@@ -0,0 +1,30 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.10
+
+package storage
+
+import "google.golang.org/api/googleapi"
+
+func shouldRetry(err error) bool {
+	switch e := err.(type) {
+	case *googleapi.Error:
+		// Retry on 429 and 5xx, according to
+		// https://cloud.google.com/storage/docs/exponential-backoff.
+		return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
+	default:
+		return false
+	}
+}

storage/invoke.go

@@ -18,7 +18,6 @@ import (
 	"cloud.google.com/go/internal"
 	gax "github.com/googleapis/gax-go"
 	"golang.org/x/net/context"
-	"google.golang.org/api/googleapi"
 )
 
 // runWithRetry calls the function until it returns nil or a non-retryable error, or
@@ -29,13 +28,7 @@ func runWithRetry(ctx context.Context, call func() error) error {
 		if err == nil {
 			return true, nil
 		}
-		e, ok := err.(*googleapi.Error)
-		if !ok {
-			return true, err
-		}
-		// Retry on 429 and 5xx, according to
-		// https://cloud.google.com/storage/docs/exponential-backoff.
-		if e.Code == 429 || (e.Code >= 500 && e.Code < 600) {
+		if shouldRetry(err) {
 			return false, nil
 		}
 		return true, err

storage/not_go110.go

@@ -0,0 +1,40 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// -build go1.10
+
+package storage
+
+import (
+	"net/url"
+	"strings"
+
+	"google.golang.org/api/googleapi"
+)
+
+func shouldRetry(err error) bool {
+	switch e := err.(type) {
+	case *googleapi.Error:
+		// Retry on 429 and 5xx, according to
+		// https://cloud.google.com/storage/docs/exponential-backoff.
+		return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
+	case *url.Error:
+		// Retry on REFUSED_STREAM.
+		// Unfortunately the error type is unexported, so we resort to string
+		// matching.
+		return strings.Contains(e.Error(), "REFUSED_STREAM")
+	default:
+		return false
+	}
+}