Skip to content

Commit

Permalink
Add fix for boto3 file seek error
Browse files Browse the repository at this point in the history
Fixes error:
```
  File
  "/var/www/mvsim/releases/blue/mvsim/ve/lib/python3.6/site-packages/django/core/files/base.py",
  line 76, in chunks
      self.seek(0)
      ValueError: seek of closed file
      Build step 'Execute shell' marked build as failure
```

jschneier/django-storages#382
  • Loading branch information
nikolas committed Jun 20, 2019
1 parent 582462e commit c4cba02
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 3 deletions.
7 changes: 6 additions & 1 deletion CHANGES
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
0.2.1 (2019-06-20)
==================

* Add boto3 file seek fix

0.2.0 (2019-06-19)
==================

* Update to boto3
* Update to boto3
26 changes: 25 additions & 1 deletion cacheds3storage/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
from django.core.files.storage import get_storage_class
from storages.backends.s3boto3 import S3Boto3Storage
from storages.backends.s3boto3 import S3Boto3Storage, SpooledTemporaryFile
from django.conf import settings
from dateutil import tz

Expand All @@ -10,6 +11,29 @@ def __init__(self, *args, **kwargs):
self.local_storage = get_storage_class(
'compressor.storage.CompressorFileStorage')()

# Fix for "ValueError: seek of closed file" problem with boto3
# https://github.com/jschneier/django-storages/issues/382#issuecomment-377174808
def _save_content(self, obj, content, parameters):
"""
We create a clone of the content file as when this is passed to boto3 it wrongly closes
the file upon upload where as the storage backend expects it to still be open
"""
# Seek our content back to the start
content.seek(0, os.SEEK_SET)

# Create a temporary file that will write to disk after a specified size
content_autoclose = SpooledTemporaryFile()

# Write our original content into our copy that will be closed by boto3
content_autoclose.write(content.read())

# Upload the object which will auto close the content_autoclose instance
super(CachedS3BotoStorage, self)._save_content(obj, content_autoclose, parameters)

# Cleanup if this is fixed upstream our duplicate should always close
if not content_autoclose.closed:
content_autoclose.close()

def save(self, name, content):
name = super(CachedS3BotoStorage, self).save(name, content)
self.local_storage._save(name, content)
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

setup(
name="django-cacheds3storage",
version="0.2.0",
version="0.2.1",
author="Anders Pearson",
author_email="ctl-dev@columbia.edu",
url="https://github.com/ccnmtl/django-cacheds3storage",
Expand Down

0 comments on commit c4cba02

Please sign in to comment.