From b66cef7cfc47013a8308ee15c7ff40c056c78208 Mon Sep 17 00:00:00 2001 From: Josh Schneier Date: Fri, 18 Dec 2020 00:46:55 -0500 Subject: [PATCH] Revert "S3: Workaround boto bug to fix collectstatic issue (#955)" This reverts commit 7771782552c4a80eda4f335303aab3185be17d0a. --- storages/backends/s3boto3.py | 7 +++--- storages/utils.py | 8 ------- tests/test_s3boto3.py | 46 ++++++++++++++++-------------------- 3 files changed, 23 insertions(+), 38 deletions(-) diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py index ab64787c9..a0e3c9a23 100644 --- a/storages/backends/s3boto3.py +++ b/storages/backends/s3boto3.py @@ -16,8 +16,8 @@ from storages.base import BaseStorage from storages.utils import ( - NonCloseableBufferedReader, check_location, get_available_overwrite_name, - lookup_env, safe_join, setting, to_bytes, + check_location, get_available_overwrite_name, lookup_env, safe_join, + setting, to_bytes, ) try: @@ -442,8 +442,7 @@ def _save(self, name, content): obj = self.bucket.Object(name) content.seek(0, os.SEEK_SET) - with NonCloseableBufferedReader(content) as reader: - obj.upload_fileobj(reader, ExtraArgs=params) + obj.upload_fileobj(content, ExtraArgs=params) return cleaned_name def delete(self, name): diff --git a/storages/utils.py b/storages/utils.py index bf961dd1b..dbd81e096 100644 --- a/storages/utils.py +++ b/storages/utils.py @@ -1,4 +1,3 @@ -import io import os import posixpath @@ -127,10 +126,3 @@ def get_available_overwrite_name(name, max_length): 'allows sufficient "max_length".' % name ) return os.path.join(dir_name, "{}{}".format(file_root, file_ext)) - - -# A buffered reader that does not actually close, workaround for -# https://github.com/boto/s3transfer/issues/80#issuecomment-562356142 -class NonCloseableBufferedReader(io.BufferedReader): - def close(self): - self.flush() diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py index 627a250c0..6d4585bf1 100644 --- a/tests/test_s3boto3.py +++ b/tests/test_s3boto3.py @@ -22,12 +22,6 @@ def setUp(self): self.storage._connections.connection = mock.MagicMock() -def assert_called_upload(mock_uploadobj, content, extra): - assert mock_uploadobj.call_count == 1 - assert mock_uploadobj.call_args[0][0].raw == content - assert mock_uploadobj.call_args[1] == extra - - class S3Boto3StorageTests(S3Boto3TestCase): def test_clean_name(self): @@ -113,12 +107,12 @@ def test_storage_save(self): self.storage.bucket.Object.assert_called_once_with(name) obj = self.storage.bucket.Object.return_value - extra = { - 'ExtraArgs': { + obj.upload_fileobj.assert_called_with( + content, + ExtraArgs={ 'ContentType': 'text/plain', } - } - assert_called_upload(obj.upload_fileobj, content, extra) + ) def test_storage_save_with_default_acl(self): """ @@ -131,13 +125,13 @@ def test_storage_save_with_default_acl(self): self.storage.bucket.Object.assert_called_once_with(name) obj = self.storage.bucket.Object.return_value - extra = { - 'ExtraArgs': { + obj.upload_fileobj.assert_called_with( + content, + ExtraArgs={ 'ContentType': 'text/plain', 'ACL': 'private', } - } - assert_called_upload(obj.upload_fileobj, content, extra) + ) def test_storage_object_parameters_not_overwritten_by_default(self): """ @@ -151,13 +145,13 @@ def test_storage_object_parameters_not_overwritten_by_default(self): self.storage.bucket.Object.assert_called_once_with(name) obj = self.storage.bucket.Object.return_value - extra = { - 'ExtraArgs': { + obj.upload_fileobj.assert_called_with( + content, + ExtraArgs={ 'ContentType': 'text/plain', 'ACL': 'private', } - } - assert_called_upload(obj.upload_fileobj, content, extra) + ) def test_content_type(self): """ @@ -170,12 +164,12 @@ def test_content_type(self): self.storage.bucket.Object.assert_called_once_with(name) obj = self.storage.bucket.Object.return_value - extra = { - 'ExtraArgs': { + obj.upload_fileobj.assert_called_with( + content, + ExtraArgs={ 'ContentType': 'image/jpeg', } - } - assert_called_upload(obj.upload_fileobj, content, extra) + ) def test_storage_save_gzipped(self): """ @@ -185,13 +179,13 @@ def test_storage_save_gzipped(self): content = ContentFile("I am gzip'd") self.storage.save(name, content) obj = self.storage.bucket.Object.return_value - extra = { - 'ExtraArgs': { + obj.upload_fileobj.assert_called_with( + content, + ExtraArgs={ 'ContentType': 'application/octet-stream', 'ContentEncoding': 'gzip', } - } - assert_called_upload(obj.upload_fileobj, content, extra) + ) def test_storage_save_gzip(self): """