GoogleCloud: Add GS_BLOB_CHUNK_SIZE setting (#757)

docs/backends/gcloud.rst

@@ -138,6 +138,15 @@ By default files with the same name will overwrite each other. Set this to ``Fal
 The maximum amount of memory a returned file can take up (in bytes) before being
 rolled over into a temporary file on disk. Default is 0: Do not roll over.
 
+``GS_BLOB_CHUNK_SIZE`` (optional: default is ``None``)
+
+The size of blob chunks that are sent via resumable upload. If this is not set then the generated request
+must fit in memory. Recommended if you are going to be uploading large files.
+
+.. note::
+
+   This must be a multiple of 256K (1024 * 256)
+
 ``GS_CACHE_CONTROL`` (optional: default is ``None``)
 
 Sets Cache-Control HTTP header for the file, more about HTTP caching can be found `here <https://developers.google.com/web/fundamentals/performance/optimizing-content-efficiency/http-caching#cache-control>`_

storages/backends/gcloud.py

@@ -30,7 +30,9 @@ def __init__(self, name, mode, storage):
         self._storage = storage
         self.blob = storage.bucket.get_blob(name)
         if not self.blob and 'w' in mode:
-            self.blob = Blob(self.name, storage.bucket)
+            self.blob = Blob(
+                self.name, storage.bucket,
+                chunk_size=setting('GS_BLOB_CHUNK_SIZE'))
         self._file = None
         self._is_dirty = False
 

tests/test_gcloud.py

@@ -86,7 +86,7 @@ def test_open_write(self, MockBlob):
         self.storage.default_acl = 'projectPrivate'
 
         f = self.storage.open(self.filename, 'wb')
-        MockBlob.assert_called_with(self.filename, self.storage._bucket)
+        MockBlob.assert_called_with(self.filename, self.storage._bucket, chunk_size=None)
 
         f.write(data)
         tmpfile = f._file