chore(gensupport): add retry invocation headers

go.mod

@@ -5,6 +5,7 @@ go 1.15
 require (
 	cloud.google.com/go/compute v1.7.0
 	github.com/google/go-cmp v0.5.8
+	github.com/google/uuid v1.1.2
 	github.com/googleapis/enterprise-certificate-proxy v0.1.0
 	github.com/googleapis/gax-go/v2 v2.4.0
 	go.opencensus.io v0.23.0

go.sum

@@ -160,6 +160,7 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
 github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw=

internal/gensupport/resumable.go

@@ -10,8 +10,12 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/api/internal"
 )
 
 // ResumableUpload is used by the generated APIs to provide resumable uploads.
@@ -38,6 +42,11 @@ type ResumableUpload struct {
 	// ChunkRetryDeadline configures the per-chunk deadline after which no further
 	// retries should happen.
 	ChunkRetryDeadline time.Duration
+
+	// Track current request invocation ID and attempt count for retry metric
+	// headers.
+	invocationID string
+	attempts     int
 }
 
 // Progress returns the number of bytes uploaded at this point.
@@ -72,6 +81,10 @@ func (rx *ResumableUpload) doUploadRequest(ctx context.Context, data io.Reader,
 	req.Header.Set("Content-Type", rx.MediaType)
 	req.Header.Set("User-Agent", rx.UserAgent)
 
+	baseXGoogHeader := "gl-go/" + GoVersion() + " gdcl/" + internal.Version
+	invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", rx.invocationID, rx.attempts)
+	req.Header.Set("X-Goog-Api-Client", strings.Join([]string{baseXGoogHeader, invocationHeader}, " "))
+
 	// Google's upload endpoint uses status code 308 for a
 	// different purpose than the "308 Permanent Redirect"
 	// since-standardized in RFC 7238. Because of the conflict in
@@ -178,9 +191,11 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err
 	for {
 		var pause time.Duration
 
-		// Each chunk gets its own initialized-at-zero backoff.
+		// Each chunk gets its own initialized-at-zero backoff and invocation ID.
 		bo := rx.Retry.backoff()
 		quitAfter := time.After(retryDeadline)
+		rx.attempts = 1
+		rx.invocationID = uuid.New().String()
 
 		// Retry loop for a single chunk.
 		for {
@@ -223,6 +238,7 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err
 				break
 			}
 
+			rx.attempts++
 			pause = bo.Pause()
 			if resp != nil && resp.Body != nil {
 				resp.Body.Close()

internal/gensupport/send.go

@@ -8,9 +8,12 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/googleapis/gax-go/v2"
 )
 
@@ -71,6 +74,9 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r
 
 	var resp *http.Response
 	var err error
+	attempts := 1
+	invocationID := uuid.New().String()
+	baseXGoogHeader := req.Header.Get("X-Goog-Api-Client")
 
 	// Loop to retry the request, up to the context deadline.
 	var pause time.Duration
@@ -109,6 +115,9 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r
 			}
 			return resp, err
 		}
+		invocationHeader := fmt.Sprintf("gccl-invocation-id/%s gccl-attempt-count/%d", invocationID, attempts)
+		xGoogHeader := strings.Join([]string{invocationHeader, baseXGoogHeader}, " ")
+		req.Header.Set("X-Goog-Api-Client", xGoogHeader)
 
 		resp, err = client.Do(req.WithContext(ctx))
 
@@ -123,6 +132,7 @@ func sendAndRetry(ctx context.Context, client *http.Client, req *http.Request, r
 		if req.GetBody == nil || !errorFunc(status, err) {
 			break
 		}
+		attempts++
 		var errBody error
 		req.Body, errBody = req.GetBody()
 		if errBody != nil {