Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

gcloud: Use DEFAULT_RETRY when uploading a file. #1156

Merged
merged 3 commits into from Aug 10, 2022
Merged

gcloud: Use DEFAULT_RETRY when uploading a file. #1156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fcb8732
gcloud: Use DEFAULT_RETRY when uploading a file.
Jul 15, 2022
8b21ca1
gcloud: Added missing retry arg to tests.
Aug 9, 2022
4c5b622
gcloud: Use DEFAULT_RETRY when deleting a file.
Aug 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 10 additions & 2 deletions storages/backends/gcloud.py
View file
Open in desktop
Expand Up @@ -24,6 +24,7 @@
from google.cloud.storage import Blob
from google.cloud.storage import Client
from google.cloud.storage.blob import _quote
from google.cloud.storage.retry import DEFAULT_RETRY
except ImportError:
raise ImproperlyConfigured("Could not load Google Cloud Storage bindings.\n"
"See https://github.com/GoogleCloudPlatform/gcloud-python")
Expand Down Expand Up @@ -93,6 +94,7 @@ def close(self):
blob_params = self._storage.get_object_parameters(self.name)
self.blob.upload_from_file(
self.file, rewind=True, content_type=self.mime_type,
retry=DEFAULT_RETRY,
predefined_acl=blob_params.get('acl', self._storage.default_acl))
self._file.close()
self._file = None
Expand Down Expand Up @@ -192,7 +194,13 @@ def _save(self, name, content):
for prop, val in blob_params.items():
setattr(file_object.blob, prop, val)

file_object.blob.upload_from_file(content, rewind=True, size=getattr(content, 'size', None), **upload_params)
file_object.blob.upload_from_file(
content,
rewind=True,
retry=DEFAULT_RETRY,
size=getattr(content, 'size', None),
**upload_params
)
return cleaned_name

def get_object_parameters(self, name):
Expand All @@ -215,7 +223,7 @@ def get_object_parameters(self, name):
def delete(self, name):
name = self._normalize_name(clean_name(name))
try:
self.bucket.delete_blob(name)
self.bucket.delete_blob(name, retry=DEFAULT_RETRY)
except NotFound:
pass

Expand Down
21 changes: 16 additions & 5 deletions tests/test_gcloud.py
View file
Open in desktop
Expand Up @@ -11,6 +11,7 @@
from django.utils import timezone
from google.cloud.exceptions import NotFound
from google.cloud.storage.blob import Blob
from google.cloud.storage.retry import DEFAULT_RETRY

from storages.backends import gcloud
from tests.utils import NonSeekableContentFile
Expand Down Expand Up @@ -97,6 +98,7 @@ def test_open_write(self, MockBlob):
MockBlob().upload_from_file.assert_called_with(
tmpfile, rewind=True,
content_type=mimetypes.guess_type(self.filename)[0],
retry=DEFAULT_RETRY,
predefined_acl='projectPrivate')

def test_save(self):
Expand All @@ -107,8 +109,13 @@ def test_save(self):

self.storage._client.bucket.assert_called_with(self.bucket_name)
self.storage._bucket.get_blob().upload_from_file.assert_called_with(
content, rewind=True, size=len(data), content_type=mimetypes.guess_type(self.filename)[0],
predefined_acl=None)
content,
rewind=True,
retry=DEFAULT_RETRY,
size=len(data),
content_type=mimetypes.guess_type(self.filename)[0],
predefined_acl=None
)

def test_save2(self):
data = 'This is some test ủⓝï℅ⅆℇ content.'
Expand All @@ -119,7 +126,7 @@ def test_save2(self):

self.storage._client.bucket.assert_called_with(self.bucket_name)
self.storage._bucket.get_blob().upload_from_file.assert_called_with(
content, rewind=True, size=len(data), content_type=mimetypes.guess_type(filename)[0],
content, rewind=True, retry=DEFAULT_RETRY, size=len(data), content_type=mimetypes.guess_type(filename)[0],
predefined_acl=None)

def test_save_with_default_acl(self):
Expand All @@ -136,14 +143,14 @@ def test_save_with_default_acl(self):

self.storage._client.bucket.assert_called_with(self.bucket_name)
self.storage._bucket.get_blob().upload_from_file.assert_called_with(
content, rewind=True, size=len(data), content_type=mimetypes.guess_type(filename)[0],
content, rewind=True, retry=DEFAULT_RETRY, size=len(data), content_type=mimetypes.guess_type(filename)[0],
predefined_acl='publicRead')

def test_delete(self):
self.storage.delete(self.filename)

self.storage._client.bucket.assert_called_with(self.bucket_name)
self.storage._bucket.delete_blob.assert_called_with(self.filename)
self.storage._bucket.delete_blob.assert_called_with(self.filename, retry=DEFAULT_RETRY)

def test_exists(self):
self.storage._bucket = mock.MagicMock()
Expand Down Expand Up @@ -420,6 +427,7 @@ def test_storage_save_gzipped(self):
obj.upload_from_file.assert_called_with(
mock.ANY,
rewind=True,
retry=DEFAULT_RETRY,
size=11,
predefined_acl=None,
content_type=None
Expand All @@ -436,6 +444,7 @@ def test_storage_save_gzipped_non_seekable(self):
obj.upload_from_file.assert_called_with(
mock.ANY,
rewind=True,
retry=DEFAULT_RETRY,
size=11,
predefined_acl=None,
content_type=None
Expand All @@ -455,6 +464,7 @@ def test_storage_save_gzip(self):
obj.upload_from_file.assert_called_with(
mock.ANY,
rewind=True,
retry=DEFAULT_RETRY,
size=None,
predefined_acl=None,
content_type='text/css',
Expand Down Expand Up @@ -484,6 +494,7 @@ def test_storage_save_gzip_twice(self):
obj.upload_from_file.assert_called_with(
mock.ANY,
rewind=True,
retry=DEFAULT_RETRY,
size=None,
predefined_acl=None,
content_type='text/css',
Expand Down