[s3] add setting to customize TransferConfig and deprecate AWS_S3_USE…

…_THREADS (#1280)
jschneier · Aug 26, 2023 · 7b625b8 · 7b625b8
1 parent 585536c
commit 7b625b8
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 12 deletions.
diff --git a/docs/backends/amazon-S3.rst b/docs/backends/amazon-S3.rst
@@ -134,6 +134,11 @@ searches for them:
   support the legacy ``s3`` (also known as ``v2``) version. You can check to see
   if your region is one of them in the `S3 region list`_.
 
+``AWS_S3_TRANSFER_CONFIG`` (optional, default is ``None``)
+
+  Set this to customize the transfer config options such as disabling threads for ``gevent`` compatibility;
+  See the `Boto3 docs for TransferConfig` for more info.
+
 .. note::
 
   The signature versions are not backwards compatible so be careful about url endpoints if making this change
@@ -143,6 +148,7 @@ searches for them:
 .. _S3 region list: http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
 .. _list of canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 .. _Boto3 docs for uploading files: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_object
+.. _Boto3 docs for TransferConfig: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig
 .. _ManifestStaticFilesStorage: https://docs.djangoproject.com/en/3.1/ref/contrib/staticfiles/#manifeststaticfilesstorage
 
 CloudFront

diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py
@@ -3,6 +3,7 @@
 import posixpath
 import tempfile
 import threading
+import warnings
 from datetime import datetime
 from datetime import timedelta
 from tempfile import SpooledTemporaryFile
@@ -145,7 +146,7 @@ def _get_file(self):
             )
             if 'r' in self._mode:
                 self._is_dirty = False
-                self.obj.download_fileobj(self._file, Config=self._storage._transfer_config)
+                self.obj.download_fileobj(self._file, Config=self._storage.transfer_config)
                 self._file.seek(0)
             if self._storage.gzip and self.obj.content_encoding == 'gzip':
                 self._file = self._decompress_file(mode=self._mode, file=self._file)
@@ -269,7 +270,16 @@ def __init__(self, **settings):
                 signature_version=self.signature_version,
                 proxies=self.proxies,
             )
-        self._transfer_config = TransferConfig(use_threads=self.use_threads)
+
+        if self.use_threads is False:
+            warnings.warn(
+                "The AWS_S3_USE_THREADS setting is deprecated. Use "
+                "AWS_S3_TRANSFER_CONFIG to customize any of the "
+                "boto.s3.transfer.TransferConfig parameters.", DeprecationWarning
+            )
+
+        if self.transfer_config is None:
+            self.transfer_config = TransferConfig(use_threads=self.use_threads)
 
     def get_cloudfront_signer(self, key_id, key):
         return _cloud_front_signer_from_pem(key_id, key)
@@ -330,6 +340,7 @@ def get_default_settings(self):
             'max_memory_size': setting('AWS_S3_MAX_MEMORY_SIZE', 0),
             'default_acl': setting('AWS_DEFAULT_ACL', None),
             'use_threads': setting('AWS_S3_USE_THREADS', True),
+            'transfer_config': setting('AWS_S3_TRANSFER_CONFIG', None),
         }
 
     def __getstate__(self):
@@ -419,7 +430,7 @@ def _save(self, name, content):
             params['ContentEncoding'] = 'gzip'
 
         obj = self.bucket.Object(name)
-        obj.upload_fileobj(content, ExtraArgs=params, Config=self._transfer_config)
+        obj.upload_fileobj(content, ExtraArgs=params, Config=self.transfer_config)
         return cleaned_name
 
     def delete(self, name):

diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py
@@ -7,6 +7,7 @@
 from unittest import skipIf
 from urllib.parse import urlparse
 
+import boto3.s3.transfer
 from botocore.exceptions import ClientError
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
@@ -96,7 +97,7 @@ def test_storage_save(self):
             ExtraArgs={
                 'ContentType': 'text/plain',
             },
-            Config=self.storage._transfer_config
+            Config=self.storage.transfer_config
         )
 
     def test_storage_save_non_seekable(self):
@@ -114,7 +115,7 @@ def test_storage_save_non_seekable(self):
             ExtraArgs={
                 'ContentType': 'text/plain',
             },
-            Config=self.storage._transfer_config
+            Config=self.storage.transfer_config
         )
 
     def test_storage_save_with_default_acl(self):
@@ -134,7 +135,7 @@ def test_storage_save_with_default_acl(self):
                 'ContentType': 'text/plain',
                 'ACL': 'private',
             },
-            Config=self.storage._transfer_config
+            Config=self.storage.transfer_config
         )
 
     def test_storage_object_parameters_not_overwritten_by_default(self):
@@ -155,7 +156,7 @@ def test_storage_object_parameters_not_overwritten_by_default(self):
                 'ContentType': 'text/plain',
                 'ACL': 'private',
             },
-            Config=self.storage._transfer_config
+            Config=self.storage.transfer_config
         )
 
     def test_content_type(self):
@@ -174,7 +175,7 @@ def test_content_type(self):
             ExtraArgs={
                 'ContentType': 'image/jpeg',
             },
-            Config=self.storage._transfer_config
+            Config=self.storage.transfer_config
         )
 
     def test_storage_save_gzipped(self):
@@ -191,7 +192,7 @@ def test_storage_save_gzipped(self):
                 'ContentType': 'application/octet-stream',
                 'ContentEncoding': 'gzip',
             },
-            Config=self.storage._transfer_config
+            Config=self.storage.transfer_config
         )
 
     def test_storage_save_gzipped_non_seekable(self):
@@ -208,7 +209,7 @@ def test_storage_save_gzipped_non_seekable(self):
                 'ContentType': 'application/octet-stream',
                 'ContentEncoding': 'gzip',
             },
-            Config=self.storage._transfer_config
+            Config=self.storage.transfer_config
         )
 
     def test_storage_save_gzip(self):
@@ -226,7 +227,7 @@ def test_storage_save_gzip(self):
                 'ContentType': 'text/css',
                 'ContentEncoding': 'gzip',
             },
-            Config=self.storage._transfer_config
+            Config=self.storage.transfer_config
         )
         args, kwargs = obj.upload_fileobj.call_args
         content = args[0]
@@ -254,7 +255,7 @@ def test_storage_save_gzip_twice(self):
                 'ContentType': 'text/css',
                 'ContentEncoding': 'gzip',
             },
-            Config=self.storage._transfer_config
+            Config=self.storage.transfer_config
         )
         args, kwargs = obj.upload_fileobj.call_args
         content = args[0]
@@ -767,6 +768,20 @@ def test_override_init_argument(self):
         storage = s3boto3.S3Boto3Storage(location='foo2')
         self.assertEqual(storage.location, 'foo2')
 
+    def test_use_threads_false(self):
+        with override_settings(AWS_S3_USE_THREADS=False):
+            storage = s3boto3.S3Boto3Storage()
+            self.assertFalse(storage.transfer_config.use_threads)
+
+    def test_transfer_config(self):
+        storage = s3boto3.S3Boto3Storage()
+        self.assertTrue(storage.transfer_config.use_threads)
+
+        transfer_config = boto3.s3.transfer.TransferConfig(use_threads=False)
+        with override_settings(AWS_S3_TRANSFER_CONFIG=transfer_config):
+            storage = s3boto3.S3Boto3Storage()
+            self.assertFalse(storage.transfer_config.use_threads)
+
 
 class S3StaticStorageTests(TestCase):
     def setUp(self):