[gcloud] Use DEFAULT_RETRY when uploading/deleting files (#1156)

jschneier · Aug 10, 2022 · f18d68e · f18d68e
1 parent 785da47
commit f18d68e
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 7 deletions.
diff --git a/storages/backends/gcloud.py b/storages/backends/gcloud.py
@@ -24,6 +24,7 @@
     from google.cloud.storage import Blob
     from google.cloud.storage import Client
     from google.cloud.storage.blob import _quote
+    from google.cloud.storage.retry import DEFAULT_RETRY
 except ImportError:
     raise ImproperlyConfigured("Could not load Google Cloud Storage bindings.\n"
                                "See https://github.com/GoogleCloudPlatform/gcloud-python")
@@ -93,6 +94,7 @@ def close(self):
                 blob_params = self._storage.get_object_parameters(self.name)
                 self.blob.upload_from_file(
                     self.file, rewind=True, content_type=self.mime_type,
+                    retry=DEFAULT_RETRY,
                     predefined_acl=blob_params.get('acl', self._storage.default_acl))
             self._file.close()
             self._file = None
@@ -192,7 +194,13 @@ def _save(self, name, content):
         for prop, val in blob_params.items():
             setattr(file_object.blob, prop, val)
 
-        file_object.blob.upload_from_file(content, rewind=True, size=getattr(content, 'size', None), **upload_params)
+        file_object.blob.upload_from_file(
+            content,
+            rewind=True,
+            retry=DEFAULT_RETRY,
+            size=getattr(content, 'size', None),
+            **upload_params
+        )
         return cleaned_name
 
     def get_object_parameters(self, name):
@@ -215,7 +223,7 @@ def get_object_parameters(self, name):
     def delete(self, name):
         name = self._normalize_name(clean_name(name))
         try:
-            self.bucket.delete_blob(name)
+            self.bucket.delete_blob(name, retry=DEFAULT_RETRY)
         except NotFound:
             pass
 

diff --git a/tests/test_gcloud.py b/tests/test_gcloud.py
@@ -11,6 +11,7 @@
 from django.utils import timezone
 from google.cloud.exceptions import NotFound
 from google.cloud.storage.blob import Blob
+from google.cloud.storage.retry import DEFAULT_RETRY
 
 from storages.backends import gcloud
 from tests.utils import NonSeekableContentFile
@@ -97,6 +98,7 @@ def test_open_write(self, MockBlob):
         MockBlob().upload_from_file.assert_called_with(
             tmpfile, rewind=True,
             content_type=mimetypes.guess_type(self.filename)[0],
+            retry=DEFAULT_RETRY,
             predefined_acl='projectPrivate')
 
     def test_save(self):
@@ -107,8 +109,13 @@ def test_save(self):
 
         self.storage._client.bucket.assert_called_with(self.bucket_name)
         self.storage._bucket.get_blob().upload_from_file.assert_called_with(
-            content, rewind=True, size=len(data), content_type=mimetypes.guess_type(self.filename)[0],
-            predefined_acl=None)
+            content,
+            rewind=True,
+            retry=DEFAULT_RETRY,
+            size=len(data),
+            content_type=mimetypes.guess_type(self.filename)[0],
+            predefined_acl=None
+        )
 
     def test_save2(self):
         data = 'This is some test ủⓝï℅ⅆℇ content.'
@@ -119,7 +126,7 @@ def test_save2(self):
 
         self.storage._client.bucket.assert_called_with(self.bucket_name)
         self.storage._bucket.get_blob().upload_from_file.assert_called_with(
-            content, rewind=True, size=len(data), content_type=mimetypes.guess_type(filename)[0],
+            content, rewind=True, retry=DEFAULT_RETRY, size=len(data), content_type=mimetypes.guess_type(filename)[0],
             predefined_acl=None)
 
     def test_save_with_default_acl(self):
@@ -136,14 +143,14 @@ def test_save_with_default_acl(self):
 
         self.storage._client.bucket.assert_called_with(self.bucket_name)
         self.storage._bucket.get_blob().upload_from_file.assert_called_with(
-            content, rewind=True, size=len(data), content_type=mimetypes.guess_type(filename)[0],
+            content, rewind=True, retry=DEFAULT_RETRY, size=len(data), content_type=mimetypes.guess_type(filename)[0],
             predefined_acl='publicRead')
 
     def test_delete(self):
         self.storage.delete(self.filename)
 
         self.storage._client.bucket.assert_called_with(self.bucket_name)
-        self.storage._bucket.delete_blob.assert_called_with(self.filename)
+        self.storage._bucket.delete_blob.assert_called_with(self.filename, retry=DEFAULT_RETRY)
 
     def test_exists(self):
         self.storage._bucket = mock.MagicMock()
@@ -420,6 +427,7 @@ def test_storage_save_gzipped(self):
         obj.upload_from_file.assert_called_with(
             mock.ANY,
             rewind=True,
+            retry=DEFAULT_RETRY,
             size=11,
             predefined_acl=None,
             content_type=None
@@ -436,6 +444,7 @@ def test_storage_save_gzipped_non_seekable(self):
         obj.upload_from_file.assert_called_with(
             mock.ANY,
             rewind=True,
+            retry=DEFAULT_RETRY,
             size=11,
             predefined_acl=None,
             content_type=None
@@ -455,6 +464,7 @@ def test_storage_save_gzip(self):
         obj.upload_from_file.assert_called_with(
             mock.ANY,
             rewind=True,
+            retry=DEFAULT_RETRY,
             size=None,
             predefined_acl=None,
             content_type='text/css',
@@ -484,6 +494,7 @@ def test_storage_save_gzip_twice(self):
         obj.upload_from_file.assert_called_with(
             mock.ANY,
             rewind=True,
+            retry=DEFAULT_RETRY,
             size=None,
             predefined_acl=None,
             content_type='text/css',