feat(storage): find GoogleAccessID when using impersonated creds

storage/bucket.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"strings"
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
@@ -159,22 +160,17 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (
 }
 
 // SignedURL returns a URL for the specified object. Signed URLs allow anyone
-// access to a restricted resource for a limited time without needing a
-// Google account or signing in. For more information about signed URLs, see
-// https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication
+// access to a restricted resource for a limited time without needing a Google
+// account or signing in.
+// For more information about signed URLs, see "[Overview of access control]."
 //
-// This method only requires the Method and Expires fields in the specified
-// SignedURLOptions opts to be non-nil. If not provided, it attempts to fill the
-// GoogleAccessID and PrivateKey from the GOOGLE_APPLICATION_CREDENTIALS environment variable.
-// If you are authenticating with a custom HTTP client, Service Account based
-// auto-detection will be hindered.
+// This method requires the Method and Expires fields in the specified
+// SignedURLOptions to be non-nil. You may need to set the GoogleAccessID and
+// PrivateKey fields in some cases. Read more on the [automatic detection of credentials]
+// for this method.
 //
-// If no private key is found, it attempts to use the GoogleAccessID to sign the URL.
-// This requires the IAM Service Account Credentials API to be enabled
-// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview)
-// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account.
-// If you do not want these fields set for you, you may pass them in through opts or use
-// SignedURL(bucket, name string, opts *SignedURLOptions) instead.
+// [Overview of access control]: https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication
+// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4]
 func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) {
 	if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
 		return SignedURL(b.name, object, opts)
@@ -212,18 +208,11 @@ func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string,
 // GenerateSignedPostPolicyV4 generates a PostPolicyV4 value from bucket, object and opts.
 // The generated URL and fields will then allow an unauthenticated client to perform multipart uploads.
 //
-// This method only requires the Expires field in the specified PostPolicyV4Options
-// to be non-nil. If not provided, it attempts to fill the GoogleAccessID and PrivateKey
-// from the GOOGLE_APPLICATION_CREDENTIALS environment variable.
-// If you are authenticating with a custom HTTP client, Service Account based
-// auto-detection will be hindered.
+// This method requires the Expires field in the specified PostPolicyV4Options
+// to be non-nil. You may need to set the GoogleAccessID and PrivateKey fields
+// in some cases. Read more on the [automatic detection of credentials] for this method.
 //
-// If no private key is found, it attempts to use the GoogleAccessID to sign the URL.
-// This requires the IAM Service Account Credentials API to be enabled
-// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview)
-// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account.
-// If you do not want these fields set for you, you may pass them in through opts or use
-// GenerateSignedPostPolicyV4(bucket, name string, opts *PostPolicyV4Options) instead.
+// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4]
 func (b *BucketHandle) GenerateSignedPostPolicyV4(object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) {
 	if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
 		return GenerateSignedPostPolicyV4(b.name, object, opts)
@@ -263,17 +252,27 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {
 
 	if b.c.creds != nil && len(b.c.creds.JSON) > 0 {
 		var sa struct {
-			ClientEmail string `json:"client_email"`
+			ClientEmail        string `json:"client_email"`
+			SAImpersonationURL string `json:"service_account_impersonation_url"`
+			CredType           string `json:"type"`
 		}
+
 		err := json.Unmarshal(b.c.creds.JSON, &sa)
-		if err == nil && sa.ClientEmail != "" {
-			return sa.ClientEmail, nil
-		} else if err != nil {
+		if err != nil {
 			returnErr = err
+		} else if sa.CredType == "impersonated_service_account" {
+			start, end := strings.LastIndex(sa.SAImpersonationURL, "/"), strings.LastIndex(sa.SAImpersonationURL, ":")
+
+			if end <= start {
+				returnErr = errors.New("error parsing impersonated service account credentials")
+			} else {
+				return sa.SAImpersonationURL[start+1 : end], nil
+			}
+		} else if sa.CredType == "service_account" && sa.ClientEmail != "" {
+			return sa.ClientEmail, nil
 		} else {
-			returnErr = errors.New("storage: empty client email in credentials")
+			returnErr = errors.New("unable to parse credentials; only service_account and impersonated_service_account credentials are supported")
 		}
-
 	}
 
 	// Don't error out if we can't unmarshal, fallback to GCE check.
@@ -284,11 +283,11 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {
 		} else if err != nil {
 			returnErr = err
 		} else {
-			returnErr = errors.New("got empty email from GCE metadata service")
+			returnErr = errors.New("empty email from GCE metadata service")
 		}
 
 	}
-	return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %v", returnErr)
+	return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %w. Please provide the GoogleAccessID or use a supported means for autodetecting it (see https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4])", returnErr)
 }
 
 func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) {

storage/bucket_test.go

@@ -15,12 +15,14 @@
 package storage
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
 	"cloud.google.com/go/internal/testutil"
 	"github.com/google/go-cmp/cmp"
 	gax "github.com/googleapis/gax-go/v2"
+	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
 	raw "google.golang.org/api/storage/v1"
 )
@@ -782,3 +784,96 @@ func TestBucketRetryer(t *testing.T) {
 		})
 	}
 }
+
+func TestDetectDefaultGoogleAccessID(t *testing.T) {
+	testCases := []struct {
+		name           string
+		serviceAccount string
+		creds          func(string) string
+		expectSuccess  bool
+	}{
+		{
+			name:           "impersonated creds",
+			serviceAccount: "default@my-project.iam.gserviceaccount.com",
+			creds: func(sa string) string {
+				return fmt.Sprintf(`{
+					"delegates": [],
+					"service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/%s:generateAccessToken",
+					"source_credentials": {
+					  "client_id": "id",
+					  "client_secret": "secret",
+					  "refresh_token": "token",
+					  "type": "authorized_user"
+					},
+					"type": "impersonated_service_account"
+				  }`, sa)
+			},
+			expectSuccess: true,
+		},
+		{
+			name:           "gcloud ADC creds",
+			serviceAccount: "default@my-project.iam.gserviceaccount.com",
+			creds: func(sa string) string {
+				return fmt.Sprint(`{
+					"client_id": "my-id.apps.googleusercontent.com",
+					"client_secret": "secret",
+					"quota_project_id": "",
+					"refresh_token": "token",
+					"type": "authorized_user"
+				}`)
+			},
+			expectSuccess: false,
+		},
+		{
+			name:           "ADC private key",
+			serviceAccount: "default@my-project.iam.gserviceaccount.com",
+			creds: func(sa string) string {
+				return fmt.Sprintf(`{
+					"type": "service_account",
+					"project_id": "my-project",
+					"private_key_id": "my1",
+					"private_key": "-----BEGIN PRIVATE KEY-----\nkey\n-----END PRIVATE KEY-----\n",
+					"client_email": "%s",
+					"client_id": "01",
+					"auth_uri": "https://accounts.google.com/o/oauth2/auth",
+					"token_uri": "https://oauth2.googleapis.com/token",
+					"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+					"client_x509_cert_url": "cert"
+				}`, sa)
+			},
+			expectSuccess: true,
+		},
+		{
+			name: "no creds",
+			creds: func(_ string) string {
+				return ""
+			},
+			expectSuccess: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			bucket := BucketHandle{
+				c: &Client{
+					creds: &google.Credentials{
+						JSON: []byte(tc.creds(tc.serviceAccount)),
+					},
+				},
+				name: "my-bucket",
+			}
+
+			id, err := bucket.detectDefaultGoogleAccessID()
+			if tc.expectSuccess {
+				if err != nil {
+					t.Fatal(err)
+				}
+				if id != tc.serviceAccount {
+					t.Errorf("service account not found correctly; got: %s, want: %s", id, tc.serviceAccount)
+				}
+			} else if err == nil {
+				t.Error("expected error but detectDefaultGoogleAccessID did not return one")
+			}
+		})
+	}
+}

storage/doc.go

@@ -214,8 +214,8 @@ since you read it. Here is how to express that:
 
 You can obtain a URL that lets anyone read or write an object for a limited time.
 Signing a URL requires credentials authorized to sign a URL. To use the same
-authentication that was used when instantiating the Storage client, use the
-BucketHandle.SignedURL method.
+authentication that was used when instantiating the Storage client, use
+[BucketHandle.SignedURL].
 
 	url, err := client.Bucket(bucketName).SignedURL(objectName, opts)
 	if err != nil {
@@ -224,7 +224,7 @@ BucketHandle.SignedURL method.
 	fmt.Println(url)
 
 You can also sign a URL wihout creating a client. See the documentation of
-SignedURL for details.
+[SignedURL] for details.
 
 	url, err := storage.SignedURL(bucketName, "shared-object", opts)
 	if err != nil {
@@ -247,6 +247,26 @@ as the documentation of BucketHandle.GenerateSignedPostPolicyV4.
 	}
 	fmt.Printf("URL: %s\nFields; %v\n", pv4.URL, pv4.Fields)
 
+# Credential requirements for [BucketHandle.SignedURL] and [BucketHandle.GenerateSignedPostPolicyV4]
+
+If the GoogleAccessID and PrivateKey option fields are not provided, they will
+be automatically detected if any of the following are true:
+  - you are authenticated to the Storage Client with a service account's
+    downloaded private key, either directly in code or by setting the
+    GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]),
+  - your application is running on Google Compute Engine (GCE), or
+  - you are logged into [gcloud using application default credentials]
+    with [impersonation enabled].
+
+Detecting GoogleAccessID may not be possible if you are authenticated using a
+token source or using [option.WithHTTPClient]. In this case, you can provide a
+service account email for GoogleAccessID and the client will attempt to sign
+the URL or Post Policy using that service account.
+
+To generate the signature, you must have:
+- iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account, and
+- the [IAM Service Account Credentials API] enabled (unless authenticating with a downloaded private key).
+
 # Errors
 
 Errors returned by this client are often of the type googleapi.Error.
@@ -296,5 +316,10 @@ client (using Client.SetRetry). For example:
 	if err := o.Delete(ctx); err != nil {
 		// Handle err.
 	}
+
+[Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth
+[gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login
+[impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account
+[IAM Service Account Credentials API]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview
 */
 package storage // import "cloud.google.com/go/storage"