diff --git a/CHANGES b/CHANGES index 0551d21..fd80092 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +0.2.1 (2019-06-20) +================== + +* Add boto3 file seek fix + 0.2.0 (2019-06-19) ================== -* Update to boto3 \ No newline at end of file +* Update to boto3 diff --git a/cacheds3storage/__init__.py b/cacheds3storage/__init__.py index d6ddbd8..7bda12e 100644 --- a/cacheds3storage/__init__.py +++ b/cacheds3storage/__init__.py @@ -1,5 +1,6 @@ +import os from django.core.files.storage import get_storage_class -from storages.backends.s3boto3 import S3Boto3Storage +from storages.backends.s3boto3 import S3Boto3Storage, SpooledTemporaryFile from django.conf import settings from dateutil import tz @@ -10,6 +11,29 @@ def __init__(self, *args, **kwargs): self.local_storage = get_storage_class( 'compressor.storage.CompressorFileStorage')() + # Fix for "ValueError: seek of closed file" problem with boto3 + # https://github.com/jschneier/django-storages/issues/382#issuecomment-377174808 + def _save_content(self, obj, content, parameters): + """ + We create a clone of the content file as when this is passed to boto3 it wrongly closes + the file upon upload where as the storage backend expects it to still be open + """ + # Seek our content back to the start + content.seek(0, os.SEEK_SET) + + # Create a temporary file that will write to disk after a specified size + content_autoclose = SpooledTemporaryFile() + + # Write our original content into our copy that will be closed by boto3 + content_autoclose.write(content.read()) + + # Upload the object which will auto close the content_autoclose instance + super(CachedS3BotoStorage, self)._save_content(obj, content_autoclose, parameters) + + # Cleanup if this is fixed upstream our duplicate should always close + if not content_autoclose.closed: + content_autoclose.close() + def save(self, name, content): name = super(CachedS3BotoStorage, self).save(name, content) self.local_storage._save(name, content) diff --git a/setup.py b/setup.py index 3dd29e5..d5fbad0 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="django-cacheds3storage", - version="0.2.0", + version="0.2.1", author="Anders Pearson", author_email="ctl-dev@columbia.edu", url="https://github.com/ccnmtl/django-cacheds3storage",