Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2290 from berndverst/azblobbindingtrack2
Azure Blobstorage Binding and State Store: Migrate to Track2 Azure SDK
- Loading branch information
Showing
35 changed files
with
760 additions
and
612 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| /* | ||
| Copyright 2021 The Dapr Authors | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package blobstorage | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "net/url" | ||
| "time" | ||
|
|
||
| "github.com/Azure/azure-sdk-for-go/sdk/azcore" | ||
| "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" | ||
| "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" | ||
| "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" | ||
|
|
||
| azauth "github.com/dapr/components-contrib/internal/authentication/azure" | ||
| mdutils "github.com/dapr/components-contrib/metadata" | ||
| "github.com/dapr/kit/logger" | ||
| ) | ||
|
|
||
| const ( | ||
| // Specifies the maximum number of HTTP requests that will be made to retry blob operations. A value | ||
| // of zero means that no additional HTTP requests will be made. | ||
| defaultBlobRetryCount = 3 | ||
| ) | ||
|
|
||
| func CreateContainerStorageClient(log logger.Logger, meta map[string]string) (*container.Client, *BlobStorageMetadata, error) { | ||
| m, err := parseMetadata(meta) | ||
| if err != nil { | ||
| return nil, nil, err | ||
| } | ||
|
|
||
| userAgent := "dapr-" + logger.DaprVersion | ||
| options := container.ClientOptions{ | ||
| ClientOptions: azcore.ClientOptions{ | ||
| Retry: policy.RetryOptions{ | ||
| MaxRetries: m.RetryCount, | ||
| }, | ||
| Telemetry: policy.TelemetryOptions{ | ||
| ApplicationID: userAgent, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| settings, err := azauth.NewEnvironmentSettings("storage", meta) | ||
| if err != nil { | ||
| return nil, nil, err | ||
| } | ||
| var customEndpoint string | ||
| if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageEndpointKeys...); ok && val != "" { | ||
| customEndpoint = val | ||
| } | ||
| var URL *url.URL | ||
| if customEndpoint != "" { | ||
| var parseErr error | ||
| URL, parseErr = url.Parse(fmt.Sprintf("%s/%s/%s", customEndpoint, m.AccountName, m.ContainerName)) | ||
| if parseErr != nil { | ||
| return nil, nil, parseErr | ||
| } | ||
| } else { | ||
| env := settings.AzureEnvironment | ||
| URL, _ = url.Parse(fmt.Sprintf("https://%s.blob.%s/%s", m.AccountName, env.StorageEndpointSuffix, m.ContainerName)) | ||
| } | ||
|
|
||
| var clientErr error | ||
| var client *container.Client | ||
| // Try using shared key credentials first | ||
| if m.AccountKey != "" { | ||
| credential, newSharedKeyErr := azblob.NewSharedKeyCredential(m.AccountName, m.AccountKey) | ||
| if err != nil { | ||
| return nil, nil, fmt.Errorf("invalid shared key credentials with error: %w", newSharedKeyErr) | ||
| } | ||
| client, clientErr = container.NewClientWithSharedKeyCredential(URL.String(), credential, &options) | ||
| if clientErr != nil { | ||
| return nil, nil, fmt.Errorf("cannot init Blobstorage container client: %w", err) | ||
| } | ||
| } else { | ||
| // fallback to AAD | ||
| credential, tokenErr := settings.GetTokenCredential() | ||
| if err != nil { | ||
| return nil, nil, fmt.Errorf("invalid token credentials with error: %w", tokenErr) | ||
| } | ||
| client, clientErr = container.NewClient(URL.String(), credential, &options) | ||
| } | ||
| if clientErr != nil { | ||
| return nil, nil, fmt.Errorf("cannot init Blobstorage client: %w", clientErr) | ||
| } | ||
|
|
||
| createContainerOptions := container.CreateOptions{ | ||
| Access: &m.PublicAccessLevel, | ||
| Metadata: map[string]string{}, | ||
| } | ||
| timeoutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) | ||
| _, err = client.Create(timeoutCtx, &createContainerOptions) | ||
| cancel() | ||
| // Don't return error, container might already exist | ||
| log.Debugf("error creating container: %v", err) | ||
|
|
||
| return client, m, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| /* | ||
| Copyright 2021 The Dapr Authors | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package blobstorage | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
|
|
||
| azauth "github.com/dapr/components-contrib/internal/authentication/azure" | ||
| "github.com/dapr/kit/logger" | ||
| ) | ||
|
|
||
| type scenario struct { | ||
| metadata map[string]string | ||
| expectedFailureSubString string | ||
| } | ||
|
|
||
| func TestClientInitFailures(t *testing.T) { | ||
| log := logger.NewLogger("test") | ||
|
|
||
| scenarios := map[string]scenario{ | ||
| "missing accountName": { | ||
| metadata: createTestMetadata(false, true, true), | ||
| expectedFailureSubString: "missing or empty accountName field from metadata", | ||
| }, | ||
| "missing container": { | ||
| metadata: createTestMetadata(true, true, false), | ||
| expectedFailureSubString: "missing or empty containerName field from metadata", | ||
| }, | ||
| } | ||
|
|
||
| for name, s := range scenarios { | ||
| t.Run(name, func(t *testing.T) { | ||
| _, _, err := CreateContainerStorageClient(log, s.metadata) | ||
| assert.Contains(t, err.Error(), s.expectedFailureSubString) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func createTestMetadata(accountName bool, accountKey bool, container bool) map[string]string { | ||
| m := map[string]string{} | ||
| if accountName { | ||
| m[azauth.StorageAccountNameKeys[0]] = "account" | ||
| } | ||
| if accountKey { | ||
| m[azauth.StorageAccountKeyKeys[0]] = "key" | ||
| } | ||
| if container { | ||
| m[azauth.StorageContainerNameKeys[0]] = "test" | ||
| } | ||
| return m | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| /* | ||
| Copyright 2021 The Dapr Authors | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package blobstorage | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "strconv" | ||
|
|
||
| "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" | ||
|
|
||
| azauth "github.com/dapr/components-contrib/internal/authentication/azure" | ||
| mdutils "github.com/dapr/components-contrib/metadata" | ||
| ) | ||
|
|
||
| type BlobStorageMetadata struct { | ||
| AccountName string | ||
| AccountKey string | ||
| ContainerName string | ||
| RetryCount int32 `json:"retryCount,string"` | ||
| DecodeBase64 bool `json:"decodeBase64,string"` | ||
| PublicAccessLevel azblob.PublicAccessType | ||
| } | ||
|
|
||
| func parseMetadata(meta map[string]string) (*BlobStorageMetadata, error) { | ||
| m := BlobStorageMetadata{ | ||
| RetryCount: defaultBlobRetryCount, | ||
| } | ||
| mdutils.DecodeMetadata(meta, &m) | ||
|
|
||
| if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageAccountNameKeys...); ok && val != "" { | ||
| m.AccountName = val | ||
| } else { | ||
| return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageAccountNameKeys[0]) | ||
| } | ||
|
|
||
| if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageContainerNameKeys...); ok && val != "" { | ||
| m.ContainerName = val | ||
| } else { | ||
| return nil, fmt.Errorf("missing or empty %s field from metadata", azauth.StorageContainerNameKeys[0]) | ||
| } | ||
|
|
||
| if val, ok := mdutils.GetMetadataProperty(meta, azauth.StorageAccountKeyKeys...); ok && val != "" { | ||
| m.AccountKey = val | ||
| } | ||
|
|
||
| // per the Dapr documentation "none" is a valid value | ||
| if m.PublicAccessLevel == "none" { | ||
| m.PublicAccessLevel = "" | ||
| } | ||
| if m.PublicAccessLevel != "" && !isValidPublicAccessType(m.PublicAccessLevel) { | ||
| return nil, fmt.Errorf("invalid public access level: %s; allowed: %s", | ||
| m.PublicAccessLevel, azblob.PossiblePublicAccessTypeValues()) | ||
| } | ||
|
|
||
| // we need this key for backwards compatibility | ||
| if val, ok := meta["getBlobRetryCount"]; ok && val != "" { | ||
| // convert val from string to int32 | ||
| parseInt, err := strconv.ParseInt(val, 10, 32) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| m.RetryCount = int32(parseInt) | ||
| } | ||
|
|
||
| return &m, nil | ||
| } | ||
|
|
||
| func isValidPublicAccessType(accessType azblob.PublicAccessType) bool { | ||
| validTypes := azblob.PossiblePublicAccessTypeValues() | ||
| for _, item := range validTypes { | ||
| if item == accessType { | ||
| return true | ||
| } | ||
| } | ||
|
|
||
| return false | ||
| } |
Oops, something went wrong.