From fcb873221bf21c62680a92a0538b89397320d3ab Mon Sep 17 00:00:00 2001 From: Harry Karvonen Date: Fri, 15 Jul 2022 16:06:36 +0300 Subject: [PATCH 1/3] gcloud: Use DEFAULT_RETRY when uploading a file. --- storages/backends/gcloud.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/storages/backends/gcloud.py b/storages/backends/gcloud.py index b00311dd4..de80b2590 100644 --- a/storages/backends/gcloud.py +++ b/storages/backends/gcloud.py @@ -24,6 +24,7 @@ from google.cloud.storage import Blob from google.cloud.storage import Client from google.cloud.storage.blob import _quote + from google.cloud.storage.retry import DEFAULT_RETRY except ImportError: raise ImproperlyConfigured("Could not load Google Cloud Storage bindings.\n" "See https://github.com/GoogleCloudPlatform/gcloud-python") @@ -93,6 +94,7 @@ def close(self): blob_params = self._storage.get_object_parameters(self.name) self.blob.upload_from_file( self.file, rewind=True, content_type=self.mime_type, + retry=DEFAULT_RETRY, predefined_acl=blob_params.get('acl', self._storage.default_acl)) self._file.close() self._file = None @@ -192,7 +194,13 @@ def _save(self, name, content): for prop, val in blob_params.items(): setattr(file_object.blob, prop, val) - file_object.blob.upload_from_file(content, rewind=True, size=getattr(content, 'size', None), **upload_params) + file_object.blob.upload_from_file( + content, + rewind=True, + retry=DEFAULT_RETRY, + size=getattr(content, 'size', None), + **upload_params + ) return cleaned_name def get_object_parameters(self, name): From 8b21ca1d6a290a87fc60161e14311a9dda41e7de Mon Sep 17 00:00:00 2001 From: Harry Karvonen Date: Tue, 9 Aug 2022 08:47:42 +0300 Subject: [PATCH 2/3] gcloud: Added missing retry arg to tests. --- tests/test_gcloud.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_gcloud.py b/tests/test_gcloud.py index 14241efd3..42c83d337 100644 --- a/tests/test_gcloud.py +++ b/tests/test_gcloud.py @@ -11,6 +11,7 @@ from django.utils import timezone from google.cloud.exceptions import NotFound from google.cloud.storage.blob import Blob +from google.cloud.storage.retry import DEFAULT_RETRY from storages.backends import gcloud from tests.utils import NonSeekableContentFile @@ -97,6 +98,7 @@ def test_open_write(self, MockBlob): MockBlob().upload_from_file.assert_called_with( tmpfile, rewind=True, content_type=mimetypes.guess_type(self.filename)[0], + retry=DEFAULT_RETRY, predefined_acl='projectPrivate') def test_save(self): @@ -107,8 +109,13 @@ def test_save(self): self.storage._client.bucket.assert_called_with(self.bucket_name) self.storage._bucket.get_blob().upload_from_file.assert_called_with( - content, rewind=True, size=len(data), content_type=mimetypes.guess_type(self.filename)[0], - predefined_acl=None) + content, + rewind=True, + retry=DEFAULT_RETRY, + size=len(data), + content_type=mimetypes.guess_type(self.filename)[0], + predefined_acl=None + ) def test_save2(self): data = 'This is some test ủⓝï℅ⅆℇ content.' @@ -119,7 +126,7 @@ def test_save2(self): self.storage._client.bucket.assert_called_with(self.bucket_name) self.storage._bucket.get_blob().upload_from_file.assert_called_with( - content, rewind=True, size=len(data), content_type=mimetypes.guess_type(filename)[0], + content, rewind=True, retry=DEFAULT_RETRY, size=len(data), content_type=mimetypes.guess_type(filename)[0], predefined_acl=None) def test_save_with_default_acl(self): @@ -136,7 +143,7 @@ def test_save_with_default_acl(self): self.storage._client.bucket.assert_called_with(self.bucket_name) self.storage._bucket.get_blob().upload_from_file.assert_called_with( - content, rewind=True, size=len(data), content_type=mimetypes.guess_type(filename)[0], + content, rewind=True, retry=DEFAULT_RETRY, size=len(data), content_type=mimetypes.guess_type(filename)[0], predefined_acl='publicRead') def test_delete(self): @@ -420,6 +427,7 @@ def test_storage_save_gzipped(self): obj.upload_from_file.assert_called_with( mock.ANY, rewind=True, + retry=DEFAULT_RETRY, size=11, predefined_acl=None, content_type=None @@ -436,6 +444,7 @@ def test_storage_save_gzipped_non_seekable(self): obj.upload_from_file.assert_called_with( mock.ANY, rewind=True, + retry=DEFAULT_RETRY, size=11, predefined_acl=None, content_type=None @@ -455,6 +464,7 @@ def test_storage_save_gzip(self): obj.upload_from_file.assert_called_with( mock.ANY, rewind=True, + retry=DEFAULT_RETRY, size=None, predefined_acl=None, content_type='text/css', @@ -484,6 +494,7 @@ def test_storage_save_gzip_twice(self): obj.upload_from_file.assert_called_with( mock.ANY, rewind=True, + retry=DEFAULT_RETRY, size=None, predefined_acl=None, content_type='text/css', From 4c5b62287baf7c0e336bcc46607fede0bbbc8924 Mon Sep 17 00:00:00 2001 From: Harry Karvonen Date: Wed, 10 Aug 2022 08:34:42 +0300 Subject: [PATCH 3/3] gcloud: Use DEFAULT_RETRY when deleting a file. --- storages/backends/gcloud.py | 2 +- tests/test_gcloud.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storages/backends/gcloud.py b/storages/backends/gcloud.py index de80b2590..5db2b180c 100644 --- a/storages/backends/gcloud.py +++ b/storages/backends/gcloud.py @@ -223,7 +223,7 @@ def get_object_parameters(self, name): def delete(self, name): name = self._normalize_name(clean_name(name)) try: - self.bucket.delete_blob(name) + self.bucket.delete_blob(name, retry=DEFAULT_RETRY) except NotFound: pass diff --git a/tests/test_gcloud.py b/tests/test_gcloud.py index 42c83d337..2150228a3 100644 --- a/tests/test_gcloud.py +++ b/tests/test_gcloud.py @@ -150,7 +150,7 @@ def test_delete(self): self.storage.delete(self.filename) self.storage._client.bucket.assert_called_with(self.bucket_name) - self.storage._bucket.delete_blob.assert_called_with(self.filename) + self.storage._bucket.delete_blob.assert_called_with(self.filename, retry=DEFAULT_RETRY) def test_exists(self): self.storage._bucket = mock.MagicMock()