From d74a86be6bea708ae4d48fe5cad0c5313ad2c2da Mon Sep 17 00:00:00 2001 From: Josh Schneier Date: Wed, 24 Apr 2024 23:29:41 -0400 Subject: [PATCH] [s3] Add ``client_config`` configuration setting (#1386) --- docs/backends/amazon-S3.rst | 13 +++++++++++++ storages/backends/s3.py | 18 ++++++++++++++---- tests/test_s3.py | 13 +++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/docs/backends/amazon-S3.rst b/docs/backends/amazon-S3.rst index 30309b74..5dd7419b 100644 --- a/docs/backends/amazon-S3.rst +++ b/docs/backends/amazon-S3.rst @@ -242,12 +242,25 @@ Settings The signature versions are not backwards compatible so be careful about url endpoints if making this change for legacy projects. +``client_config`` or ``AWS_S3_CLIENT_CONFIG`` + + Default: ``None`` + + An instance of ``botocore.config.Config`` to do advanced configuration of the client such as + ``max_pool_connections``. See all options in the `Botocore docs`_. + + .. note:: + + Setting this overrides the settings for ``addressing_style``, ``signature_version`` and + ``proxies``. Include them as arguments to your ``botocore.config.Config`` class if you need them. + .. _AWS Signature Version 4: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html .. _S3 region list: https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region .. _list of canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Boto3 docs for uploading files: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_object .. _Boto3 docs for TransferConfig: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig .. _ManifestStaticFilesStorage: https://docs.djangoproject.com/en/3.1/ref/contrib/staticfiles/#manifeststaticfilesstorage +.. _Botocore docs: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html#botocore.config.Config .. _cloudfront-signed-url-header: diff --git a/storages/backends/s3.py b/storages/backends/s3.py index ef915164..8a2950a9 100644 --- a/storages/backends/s3.py +++ b/storages/backends/s3.py @@ -35,7 +35,7 @@ import boto3.session import s3transfer.constants from boto3.s3.transfer import TransferConfig - from botocore.client import Config + from botocore.config import Config from botocore.exceptions import ClientError from botocore.signers import CloudFrontSigner except ImportError as e: @@ -318,8 +318,17 @@ def __init__(self, **settings): self._bucket = None self._connections = threading.local() - if not self.config: - self.config = Config( + if self.config is not None: + warnings.warn( + "The 'config' class property is deprecated and will be " + "removed in a future version. Use AWS_S3_CLIENT_CONFIG " + "to customize any of the botocore.config.Config parameters.", + DeprecationWarning, + ) + self.client_config = self.config + + if self.client_config is None: + self.client_config = Config( s3={"addressing_style": self.addressing_style}, signature_version=self.signature_version, proxies=self.proxies, @@ -407,6 +416,7 @@ def get_default_settings(self): "default_acl": setting("AWS_DEFAULT_ACL", None), "use_threads": setting("AWS_S3_USE_THREADS", True), "transfer_config": setting("AWS_S3_TRANSFER_CONFIG", None), + "client_config": setting("AWS_S3_CLIENT_CONFIG", None), } def __getstate__(self): @@ -430,7 +440,7 @@ def connection(self): region_name=self.region_name, use_ssl=self.use_ssl, endpoint_url=self.endpoint_url, - config=self.config, + config=self.client_config, verify=self.verify, ) return self._connections.connection diff --git a/tests/test_s3.py b/tests/test_s3.py index 6bef07d7..b1b15954 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -11,6 +11,7 @@ import boto3 import boto3.s3.transfer +from botocore.config import Config as ClientConfig from botocore.exceptions import ClientError from django.conf import settings from django.core.exceptions import ImproperlyConfigured @@ -42,6 +43,18 @@ def test_s3_session(self): _ = storage.connection mock_session.assert_called_once_with(profile_name="test_profile") + def test_client_config(self): + with override_settings( + AWS_S3_CLIENT_CONFIG=ClientConfig(max_pool_connections=30) + ): + storage = s3.S3Storage() + with mock.patch("boto3.Session.resource") as mock_resource: + _ = storage.connection + mock_resource.assert_called_once() + self.assertEqual( + 30, mock_resource.call_args[1]["config"].max_pool_connections + ) + def test_pickle_with_bucket(self): """ Test that the storage can be pickled with a bucket attached