From 1577eabc5da58c6f0850b12f1c4c2b824f1f4351 Mon Sep 17 00:00:00 2001 From: Nikolas Nyby Date: Thu, 16 Jul 2020 19:29:59 -0400 Subject: [PATCH 1/3] Fix "ValueError: seek of closed file" bug - closes #382 This is @mannpy's solution from: https://github.com/jschneier/django-storages/issues/382#issuecomment-592876060 --- storages/backends/s3boto3.py | 43 +++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py index 338052dbe..d4d3a3251 100644 --- a/storages/backends/s3boto3.py +++ b/storages/backends/s3boto3.py @@ -436,20 +436,37 @@ def _open(self, name, mode='rb'): return f def _save(self, name, content): - cleaned_name = self._clean_name(name) - name = self._normalize_name(cleaned_name) - params = self._get_write_parameters(name, content) - - if (self.gzip and - params['ContentType'] in self.gzip_content_types and - 'ContentEncoding' not in params): - content = self._compress_content(content) - params['ContentEncoding'] = 'gzip' - - obj = self.bucket.Object(name) + """ + We create a clone of the content file as when this is passed to + boto3 it wrongly closes the file upon upload where as the storage + backend expects it to still be open + """ + # Seek our content back to the start content.seek(0, os.SEEK_SET) - obj.upload_fileobj(content, ExtraArgs=params) - return cleaned_name + + # Create a temporary file that will write to disk after a specified + # size. This file will be automatically deleted when closed by + # boto3 or after exiting the `with` statement if the boto3 is fixed + with SpooledTemporaryFile() as content_autoclose: + # Write our original content into our copy that will be closed by boto3 + content_autoclose.write(content.read()) + + # Upload the object which will auto close the + # content_autoclose instance + cleaned_name = self._clean_name(name) + name = self._normalize_name(cleaned_name) + params = self._get_write_parameters(name, content) + + if (self.gzip and + params['ContentType'] in self.gzip_content_types and + 'ContentEncoding' not in params): + content = self._compress_content(content) + params['ContentEncoding'] = 'gzip' + + obj = self.bucket.Object(name) + content.seek(0, os.SEEK_SET) + obj.upload_fileobj(content, ExtraArgs=params) + return cleaned_name def delete(self, name): name = self._normalize_name(self._clean_name(name)) From 8e488e51d02a2b895f5dcd7a0c4376198a09017c Mon Sep 17 00:00:00 2001 From: nikolas Date: Fri, 31 Jul 2020 10:28:40 -0400 Subject: [PATCH 2/3] Update storages/backends/s3boto3.py Co-authored-by: Kyrylo Shpytsya --- storages/backends/s3boto3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py index d4d3a3251..3e485bac7 100644 --- a/storages/backends/s3boto3.py +++ b/storages/backends/s3boto3.py @@ -464,7 +464,7 @@ def _save(self, name, content): params['ContentEncoding'] = 'gzip' obj = self.bucket.Object(name) - content.seek(0, os.SEEK_SET) + content_autoclose.seek(0, os.SEEK_SET) obj.upload_fileobj(content, ExtraArgs=params) return cleaned_name From 7a202cf29644d3bb798f54b3a92cb119b6abcaee Mon Sep 17 00:00:00 2001 From: nikolas Date: Fri, 31 Jul 2020 10:28:57 -0400 Subject: [PATCH 3/3] Update storages/backends/s3boto3.py Co-authored-by: Kyrylo Shpytsya --- storages/backends/s3boto3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py index 3e485bac7..2a8f7407f 100644 --- a/storages/backends/s3boto3.py +++ b/storages/backends/s3boto3.py @@ -465,7 +465,7 @@ def _save(self, name, content): obj = self.bucket.Object(name) content_autoclose.seek(0, os.SEEK_SET) - obj.upload_fileobj(content, ExtraArgs=params) + obj.upload_fileobj(content_autoclose, ExtraArgs=params) return cleaned_name def delete(self, name):