diff --git a/docs/content/storage-drivers/azure.md b/docs/content/storage-drivers/azure.md index 50f03a2bb9..9d273a6733 100644 --- a/docs/content/storage-drivers/azure.md +++ b/docs/content/storage-drivers/azure.md @@ -16,6 +16,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses [Mic | `realm` | no | Domain name suffix for the Storage Service API endpoint. For example realm for "Azure in China" would be `core.chinacloudapi.cn` and realm for "Azure Government" would be `core.usgovcloudapi.net`. By default, this is `core.windows.net`. | | `copy_status_poll_max_retry` | no | Max retry number for polling of copy operation status. Retries use a simple backoff algorithm where each retry number is multiplied by `copy_status_poll_delay`, and this number is used as the delay. Set to -1 to disable retries and abort if the copy does not complete immediately. Defaults to 5. | | `copy_status_poll_delay` | no | Time to wait between retries for polling of copy operation status. This time is multiplied by N on each retry, where N is the retry number. Defaults to 100ms | +| `disablekeepalives` | no | Disables HTTP keep-alives on the HTTP tranport when `true`. Each connection to the server will be used for a single HTTP request. The default is `false`. | ## Related information diff --git a/docs/content/storage-drivers/gcs.md b/docs/content/storage-drivers/gcs.md index f970b73e88..abe85a2056 100644 --- a/docs/content/storage-drivers/gcs.md +++ b/docs/content/storage-drivers/gcs.md @@ -14,6 +14,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Goog | `keyfile` | no | A private service account key file in JSON format used for [Service Account Authentication](https://cloud.google.com/storage/docs/authentication#service_accounts). | | `rootdirectory` | no | The root directory tree in which all registry files are stored. Defaults to the empty string (bucket root). If a prefix is used, the path `bucketname/` has to be pre-created before starting the registry. The prefix is applied to all Google Cloud Storage keys to allow you to segment data in your bucket if necessary.| | `chunksize` | no (default 5242880) | This is the chunk size used for uploading large blobs, must be a multiple of 256*1024. | +| `disablekeepalives` | no | Disables HTTP keep-alives on the HTTP tranport when `true`. Each connection to the server will be used for a single HTTP request. The default is `false`. | {{< hint type=note >}} Instead of a key file you can use [Google Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials). diff --git a/docs/content/storage-drivers/s3.md b/docs/content/storage-drivers/s3.md index f50cc3d20a..c39688c560 100644 --- a/docs/content/storage-drivers/s3.md +++ b/docs/content/storage-drivers/s3.md @@ -21,6 +21,7 @@ Amazon S3 or S3 compatible services for object storage. | `keyid` | no | Optional KMS key ID to use for encryption (encrypt must be true, or this parameter is ignored). The default is `none`. | | `secure` | no | Indicates whether to use HTTPS instead of HTTP. A boolean value. The default is `true`. | | `skipverify` | no | Skips TLS verification when the value is set to `true`. The default is `false`. | +| `disablekeepalives` | no | Disables HTTP keep-alives on the HTTP tranport when `true`. Each connection to the server will be used for a single HTTP request. The default is `false`. | | `v4auth` | no | Indicates whether the registry uses Version 4 of AWS's authentication. The default is `true`. | | `chunksize` | no | The S3 API requires multipart upload chunks to be at least 5MB. This value should be a number that is larger than 5 * 1024 * 1024.| | `multipartcopychunksize` | no | Default chunk size for all but the last S3 Multipart Upload part when copying stored objects. | @@ -32,6 +33,9 @@ Amazon S3 or S3 compatible services for object storage. | `usedualstack` | no | Use AWS dual-stack API endpoints. | | `accelerate` | no | Enable S3 Transfer Acceleration. | | `objectacl` | no | The S3 Canned ACL for objects. The default value is "private". | +| `usedualstack` | no | Use AWS S3 dual-stack endpoints, which support both IPv6 and IPv4, when set to `true`. The default is `false`. | +| `multipartcombinesmallpart` | no | Combines small pending uploads with the ready part when `true`. The default is `true`. | +| `accelerate` | no | Use AWS S3 Transfer Acceleration endpoints when set to `true`. The default is `false`. | | `loglevel` | no | The log level for the S3 client. The default value is `off`. | > **Note** You can provide empty strings for your access and secret keys to run the driver diff --git a/registry/storage/driver/azure/azure_auth.go b/registry/storage/driver/azure/azure_auth.go index 228bcdf294..b1abe36bd8 100644 --- a/registry/storage/driver/azure/azure_auth.go +++ b/registry/storage/driver/azure/azure_auth.go @@ -2,6 +2,7 @@ package azure import ( "context" + "net/http" "sync" "time" @@ -45,6 +46,14 @@ type azureClient struct { signer signer } +type azureTransporter struct { + roundTripper http.RoundTripper +} + +func (a *azureTransporter) Do(req *http.Request) (*http.Response, error) { + return a.roundTripper.RoundTrip(req) +} + func newAzureClient(params *Parameters) (*azureClient, error) { if params.AccountKey != "" { cred, err := azblob.NewSharedKeyCredential(params.AccountName, params.AccountKey) @@ -76,7 +85,29 @@ func newAzureClient(params *Parameters) (*azureClient, error) { return nil, err } - client, err := azblob.NewClient(params.ServiceURL, cred, nil) + httpTransportModified := false + httpTransport := http.DefaultTransport.(*http.Transport).Clone() + + if params.DisableKeepAlives { + httpTransport.DisableKeepAlives = true + httpTransportModified = true + } + + var httpRoundTripper http.RoundTripper = httpTransport + + var azClientOpts *azblob.ClientOptions + + if httpTransportModified { + azClientOpts = &azblob.ClientOptions{ + ClientOptions: azcore.ClientOptions{ + Transport: &azureTransporter{ + roundTripper: httpRoundTripper, + }, + }, + } + } + + client, err := azblob.NewClient(params.ServiceURL, cred, azClientOpts) if err != nil { return nil, err } diff --git a/registry/storage/driver/azure/parser.go b/registry/storage/driver/azure/parser.go index c463ac32af..2c684a8e0e 100644 --- a/registry/storage/driver/azure/parser.go +++ b/registry/storage/driver/azure/parser.go @@ -31,6 +31,7 @@ type Parameters struct { ServiceURL string `mapstructure:"serviceurl"` CopyStatusPollMaxRetry int `mapstructure:"copy_status_poll_max_retry"` CopyStatusPollDelay string `mapstructure:"copy_status_poll_delay"` + DisableKeepAlives bool `mapstructure:"disablekeepalives"` } func NewParameters(parameters map[string]interface{}) (*Parameters, error) { diff --git a/registry/storage/driver/gcs/gcs.go b/registry/storage/driver/gcs/gcs.go index d2cdb59673..8fdff0d8de 100644 --- a/registry/storage/driver/gcs/gcs.go +++ b/registry/storage/driver/gcs/gcs.go @@ -151,11 +151,41 @@ func FromParameters(ctx context.Context, parameters map[string]interface{}) (sto } } + disableKeepAlivesParam := false + disableKeepAlives := parameters["disablekeepalives"] + switch disableKeepAlives := disableKeepAlives.(type) { + case string: + b, err := strconv.ParseBool(disableKeepAlives) + if err != nil { + return nil, fmt.Errorf("the disablekeepalives parameter should be a boolean") + } + disableKeepAlivesParam = b + case bool: + disableKeepAlivesParam = disableKeepAlives + case nil: + // do nothing + default: + return nil, fmt.Errorf("the disablekeepalives parameter should be a boolean") + } + + httpTransportModified := false + httpTransport := http.DefaultTransport.(*http.Transport).Clone() + + if disableKeepAlivesParam { + httpTransport.DisableKeepAlives = true + httpTransportModified = true + } + var ts oauth2.TokenSource jwtConf := new(jwt.Config) var err error var gcs *storage.Client var options []option.ClientOption + if httpTransportModified { + options = append(options, option.WithHTTPClient(&http.Client{ + Transport: httpTransport, + })) + } if keyfile, ok := parameters["keyfile"]; ok { jsonKey, err := os.ReadFile(fmt.Sprint(keyfile)) if err != nil { diff --git a/registry/storage/driver/s3-aws/s3.go b/registry/storage/driver/s3-aws/s3.go index 60e205dbc6..9a44818d24 100644 --- a/registry/storage/driver/s3-aws/s3.go +++ b/registry/storage/driver/s3-aws/s3.go @@ -106,6 +106,7 @@ type DriverParameters struct { KeyID string Secure bool SkipVerify bool + DisableKeepAlives bool V4Auth bool ChunkSize int64 MultipartCopyChunkSize int64 @@ -290,6 +291,23 @@ func FromParameters(ctx context.Context, parameters map[string]interface{}) (*Dr return nil, fmt.Errorf("the skipVerify parameter should be a boolean") } + disableKeepAlivesBool := false + disableKeepAlives := parameters["disablekeepalives"] + switch disableKeepAlives := disableKeepAlives.(type) { + case string: + b, err := strconv.ParseBool(disableKeepAlives) + if err != nil { + return nil, fmt.Errorf("the disablekeepalives parameter should be a boolean") + } + disableKeepAlivesBool = b + case bool: + disableKeepAlivesBool = disableKeepAlives + case nil: + // do nothing + default: + return nil, fmt.Errorf("the disablekeepalives parameter should be a boolean") + } + v4Bool := true v4auth := parameters["v4auth"] switch v4auth := v4auth.(type) { @@ -433,6 +451,7 @@ func FromParameters(ctx context.Context, parameters map[string]interface{}) (*Dr fmt.Sprint(keyID), secureBool, skipVerifyBool, + disableKeepAlivesBool, v4Bool, chunkSize, multipartCopyChunkSize, @@ -539,11 +558,24 @@ func New(ctx context.Context, params DriverParameters) (*Driver, error) { awsConfig.UseDualStackEndpoint = endpoints.DualStackEndpointStateEnabled } + httpTransportModified := false + httpTransport := http.DefaultTransport.(*http.Transport).Clone() + if params.SkipVerify { - httpTransport := http.DefaultTransport.(*http.Transport).Clone() httpTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + httpTransportModified = true + } + + if params.DisableKeepAlives { + httpTransport.DisableKeepAlives = true + httpTransportModified = true + } + + var httpRoundTripper http.RoundTripper = httpTransport + + if httpTransportModified { awsConfig.WithHTTPClient(&http.Client{ - Transport: httpTransport, + Transport: httpRoundTripper, }) } diff --git a/registry/storage/driver/s3-aws/s3_test.go b/registry/storage/driver/s3-aws/s3_test.go index d798980112..0ecf975711 100644 --- a/registry/storage/driver/s3-aws/s3_test.go +++ b/registry/storage/driver/s3-aws/s3_test.go @@ -29,22 +29,23 @@ var ( func init() { var ( - accessKey = os.Getenv("AWS_ACCESS_KEY") - secretKey = os.Getenv("AWS_SECRET_KEY") - bucket = os.Getenv("S3_BUCKET") - encrypt = os.Getenv("S3_ENCRYPT") - keyID = os.Getenv("S3_KEY_ID") - secure = os.Getenv("S3_SECURE") - skipVerify = os.Getenv("S3_SKIP_VERIFY") - v4Auth = os.Getenv("S3_V4_AUTH") - region = os.Getenv("AWS_REGION") - objectACL = os.Getenv("S3_OBJECT_ACL") - regionEndpoint = os.Getenv("REGION_ENDPOINT") - forcePathStyle = os.Getenv("AWS_S3_FORCE_PATH_STYLE") - sessionToken = os.Getenv("AWS_SESSION_TOKEN") - useDualStack = os.Getenv("S3_USE_DUALSTACK") - accelerate = os.Getenv("S3_ACCELERATE") - logLevel = os.Getenv("S3_LOGLEVEL") + accessKey = os.Getenv("AWS_ACCESS_KEY") + secretKey = os.Getenv("AWS_SECRET_KEY") + bucket = os.Getenv("S3_BUCKET") + encrypt = os.Getenv("S3_ENCRYPT") + keyID = os.Getenv("S3_KEY_ID") + secure = os.Getenv("S3_SECURE") + skipVerify = os.Getenv("S3_SKIP_VERIFY") + disableKeepAlives = os.Getenv("S3_DISABLE_KEEP_ALIVES") + v4Auth = os.Getenv("S3_V4_AUTH") + region = os.Getenv("AWS_REGION") + objectACL = os.Getenv("S3_OBJECT_ACL") + regionEndpoint = os.Getenv("REGION_ENDPOINT") + forcePathStyle = os.Getenv("AWS_S3_FORCE_PATH_STYLE") + sessionToken = os.Getenv("AWS_SESSION_TOKEN") + useDualStack = os.Getenv("S3_USE_DUALSTACK") + accelerate = os.Getenv("S3_ACCELERATE") + logLevel = os.Getenv("S3_LOGLEVEL") ) var err error @@ -73,6 +74,14 @@ func init() { } } + disableKeepAlivesBool := false + if disableKeepAlives != "" { + disableKeepAlivesBool, err = strconv.ParseBool(disableKeepAlives) + if err != nil { + return nil, err + } + } + v4Bool := true if v4Auth != "" { v4Bool, err = strconv.ParseBool(v4Auth) @@ -112,6 +121,7 @@ func init() { keyID, secureBool, skipVerifyBool, + disableKeepAlivesBool, v4Bool, minChunkSize, defaultMultipartCopyChunkSize,