Azure: Support all blobstorage settings (#750)

docs/backends/azure.rst

@@ -76,8 +76,6 @@ Set the default storage (i.e: for media files) and the static storage
 
 The following settings are available:
 
-    is_emulated = setting('AZURE_EMULATED_MODE', False)
-
 ``AZURE_ACCOUNT_NAME``
 
     This setting is the Windows Azure Storage Account name, which in many cases
@@ -125,3 +123,39 @@ The following settings are available:
 ``AZURE_LOCATION``
 
     Default location for the uploaded files. This is a path that gets prepended to every file name.
+
+``AZURE_EMULATED_MODE``
+
+    Whether to use the emulator (i.e Azurite). Defaults to False.
+
+``AZURE_ENDPOINT_SUFFIX``
+
+    The host base component of the url, minus the account name. Defaults
+    to Azure (``core.windows.net``). Override this to use the China cloud
+    (``core.chinacloudapi.cn``).
+
+``AZURE_CUSTOM_DOMAIN``
+
+    The custom domain to use. This can be set in the Azure Portal. For
+    example, ``www.mydomain.com`` or ``mycdn.azureedge.net``.
+
+    It may contain a ``host:port`` when using the emulator
+    (``AZURE_EMULATED_MODE = True``).
+
+``AZURE_CONNECTION_STRING``
+
+    If specified, this will override all other parameters.
+    See http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/
+    for the connection string format.
+
+``AZURE_CUSTOM_CONNECTION_STRING``
+
+    This is similar to ``AZURE_CONNECTION_STRING``, but it's used
+    when generating the file's URL. A custom domain or CDN may be
+    specified here instead of within ``AZURE_CONNECTION_STRING``.
+    Defaults to ``AZURE_CONNECTION_STRING``'s value.
+
+``AZURE_TOKEN_CREDENTIAL``
+
+    A token credential used to authenticate HTTPS requests. The token value
+    should be updated before its expiration.

storages/backends/azure_storage.py

@@ -7,7 +7,7 @@
 
 from azure.common import AzureMissingResourceHttpError
 from azure.storage.blob import BlobPermissions, ContentSettings
-from azure.storage.common import CloudStorageAccount
+from azure.storage.blob.blockblobservice import BlockBlobService
 from django.core.exceptions import SuspiciousOperation
 from django.core.files.base import File
 from django.core.files.storage import Storage
@@ -66,8 +66,6 @@ def read(self, *args, **kwargs):
         return super(AzureStorageFile, self).read(*args, **kwargs)
 
     def write(self, content):
-        if len(content) > 100*1024*1024:
-            raise ValueError("Max chunk size is 100MB")
         if ('w' not in self._mode and
                 '+' not in self._mode and
                 'a' not in self._mode):
@@ -147,22 +145,52 @@ class AzureStorage(Storage):
     location = setting('AZURE_LOCATION', '')
     default_content_type = 'application/octet-stream'
     is_emulated = setting('AZURE_EMULATED_MODE', False)
+    endpoint_suffix = setting('AZURE_ENDPOINT_SUFFIX')
+    sas_token = setting('AZURE_SAS_TOKEN')
+    custom_domain = setting('AZURE_CUSTOM_DOMAIN')
+    connection_string = setting('AZURE_CONNECTION_STRING')
+    custom_connection_string = setting(
+        'AZURE_CUSTOM_CONNECTION_STRING', setting('AZURE_CONNECTION_STRING'))
+    token_credential = setting('AZURE_TOKEN_CREDENTIAL')
 
     def __init__(self):
         self._service = None
+        self._custom_service = None
 
-    @property
-    def service(self):
+    def _blob_service(self, custom_domain=None, connection_string=None):
         # This won't open a connection or anything,
         # it's akin to a client
+        return BlockBlobService(
+            account_name=self.account_name,
+            account_key=self.account_key,
+            sas_token=self.sas_token,
+            is_emulated=self.is_emulated,
+            protocol=self.azure_protocol,
+            custom_domain=custom_domain,
+            connection_string=connection_string,
+            token_credential=self.token_credential,
+            endpoint_suffix=self.endpoint_suffix)
+
+    @property
+    def service(self):
         if self._service is None:
-            account = CloudStorageAccount(
-                self.account_name,
-                self.account_key,
-                is_emulated=self.is_emulated)
-            self._service = account.create_block_blob_service()
+            custom_domain = None
+            if self.is_emulated:
+                custom_domain = self.custom_domain
+            self._service = self._blob_service(
+                custom_domain=custom_domain,
+                connection_string=self.connection_string)
         return self._service
 
+    @property
+    def custom_service(self):
+        """This is used to generate the URL"""
+        if self._custom_service is None:
+            self._custom_service = self._blob_service(
+                custom_domain=self.custom_domain,
+                connection_string=self.custom_connection_string)
+        return self._custom_service
+
     @property
     def azure_protocol(self):
         if self.azure_ssl:
@@ -256,13 +284,13 @@ def url(self, name, expire=None):
 
         make_blob_url_kwargs = {}
         if expire:
-            sas_token = self.service.generate_blob_shared_access_signature(
+            sas_token = self.custom_service.generate_blob_shared_access_signature(
                 self.azure_container, name, permission=BlobPermissions.READ, expiry=self._expire_at(expire))
             make_blob_url_kwargs['sas_token'] = sas_token
 
         if self.azure_protocol:
             make_blob_url_kwargs['protocol'] = self.azure_protocol
-        return self.service.make_blob_url(
+        return self.custom_service.make_blob_url(
             container_name=self.azure_container,
             blob_name=name,
             **make_blob_url_kwargs)

tests/integration/test_azure.py

@@ -64,6 +64,12 @@ def test_url(self):
         # has some query-string
         self.assertTrue("/test/my_file.txt?" in self.storage.url("my_file.txt"))
 
+    def test_url_custom_endpoint(self):
+        storage = azure_storage.AzureStorage()
+        storage.is_emulated = True
+        storage.custom_domain = 'foobar:123456'
+        self.assertTrue(storage.url("my_file.txt").startswith('https://foobar:123456/'))
+
     @override_settings(USE_TZ=True)
     def test_get_modified_time_tz(self):
         stream = io.BytesIO(b'Im a stream')

tests/test_azure.py

@@ -23,6 +23,7 @@ class AzureStorageTest(TestCase):
     def setUp(self, *args):
         self.storage = azure_storage.AzureStorage()
         self.storage._service = mock.MagicMock()
+        self.storage._custom_service = mock.MagicMock()
         self.storage.overwrite_files = True
         self.container_name = 'test'
         self.storage.azure_container = self.container_name
@@ -121,32 +122,158 @@ def test_get_available_invalid(self):
         self.assertRaises(ValueError, self.storage.get_available_name, "$$")
 
     def test_url(self):
-        self.storage._service.make_blob_url.return_value = 'ret_foo'
+        self.storage._custom_service.make_blob_url.return_value = 'ret_foo'
         self.assertEqual(self.storage.url('some blob'), 'ret_foo')
-        self.storage._service.make_blob_url.assert_called_once_with(
+        self.storage._custom_service.make_blob_url.assert_called_once_with(
             container_name=self.container_name,
             blob_name='some_blob',
             protocol='https')
 
     def test_url_expire(self):
         utc = pytz.timezone('UTC')
         fixed_time = utc.localize(datetime.datetime(2016, 11, 6, 4))
-        self.storage._service.generate_blob_shared_access_signature.return_value = 'foo_token'
-        self.storage._service.make_blob_url.return_value = 'ret_foo'
+        self.storage._custom_service.generate_blob_shared_access_signature.return_value = 'foo_token'
+        self.storage._custom_service.make_blob_url.return_value = 'ret_foo'
         with mock.patch('storages.backends.azure_storage.datetime') as d_mocked:
             d_mocked.utcnow.return_value = fixed_time
             self.assertEqual(self.storage.url('some blob', 100), 'ret_foo')
-            self.storage._service.generate_blob_shared_access_signature.assert_called_once_with(
+            self.storage._custom_service.generate_blob_shared_access_signature.assert_called_once_with(
                 self.container_name,
                 'some_blob',
                 permission=BlobPermissions.READ,
                 expiry=fixed_time + timedelta(seconds=100))
-            self.storage._service.make_blob_url.assert_called_once_with(
+            self.storage._custom_service.make_blob_url.assert_called_once_with(
                 container_name=self.container_name,
                 blob_name='some_blob',
                 sas_token='foo_token',
                 protocol='https')
 
+    def test_blob_service_default_params(self):
+        storage = azure_storage.AzureStorage()
+        with mock.patch(
+                'storages.backends.azure_storage.BlockBlobService',
+                autospec=True) as c_mocked:
+            self.assertIsNotNone(storage.service)
+            c_mocked.assert_called_once_with(
+                account_name=None,
+                account_key=None,
+                sas_token=None,
+                is_emulated=False,
+                protocol='https',
+                custom_domain=None,
+                connection_string=None,
+                token_credential=None,
+                endpoint_suffix=None)
+
+    def test_blob_service_params_no_emulator(self):
+        """Should ignore custom domain when emulator is not used"""
+        storage = azure_storage.AzureStorage()
+        storage.is_emulated = False
+        storage.custom_domain = 'foo_domain'
+        with mock.patch(
+                'storages.backends.azure_storage.BlockBlobService',
+                autospec=True) as c_mocked:
+            self.assertIsNotNone(storage.service)
+            c_mocked.assert_called_once_with(
+                account_name=None,
+                account_key=None,
+                sas_token=None,
+                is_emulated=False,
+                protocol='https',
+                custom_domain=None,
+                connection_string=None,
+                token_credential=None,
+                endpoint_suffix=None)
+
+    def test_blob_service_params(self):
+        storage = azure_storage.AzureStorage()
+        storage.is_emulated = True
+        storage.endpoint_suffix = 'foo_suffix'
+        storage.account_name = 'foo_name'
+        storage.account_key = 'foo_key'
+        storage.sas_token = 'foo_token'
+        storage.azure_ssl = True
+        storage.custom_domain = 'foo_domain'
+        storage.connection_string = 'foo_conn'
+        storage.token_credential = 'foo_cred'
+        with mock.patch(
+                'storages.backends.azure_storage.BlockBlobService',
+                autospec=True) as c_mocked:
+            self.assertIsNotNone(storage.service)
+            c_mocked.assert_called_once_with(
+                account_name='foo_name',
+                account_key='foo_key',
+                sas_token='foo_token',
+                is_emulated=True,
+                protocol='https',
+                custom_domain='foo_domain',
+                connection_string='foo_conn',
+                token_credential='foo_cred',
+                endpoint_suffix='foo_suffix')
+
+    def test_blob_custom_service_default_params(self):
+        storage = azure_storage.AzureStorage()
+        with mock.patch(
+                'storages.backends.azure_storage.BlockBlobService',
+                autospec=True) as c_mocked:
+            self.assertIsNotNone(storage.custom_service)
+            c_mocked.assert_called_once_with(
+                account_name=None,
+                account_key=None,
+                sas_token=None,
+                is_emulated=False,
+                protocol='https',
+                custom_domain=None,
+                connection_string=None,
+                token_credential=None,
+                endpoint_suffix=None)
+
+    def test_blob_custom_service_params_no_emulator(self):
+        """Should pass custom domain when emulator is not used"""
+        storage = azure_storage.AzureStorage()
+        storage.is_emulated = False
+        storage.custom_domain = 'foo_domain'
+        with mock.patch(
+                'storages.backends.azure_storage.BlockBlobService',
+                autospec=True) as c_mocked:
+            self.assertIsNotNone(storage.custom_service)
+            c_mocked.assert_called_once_with(
+                account_name=None,
+                account_key=None,
+                sas_token=None,
+                is_emulated=False,
+                protocol='https',
+                custom_domain='foo_domain',
+                connection_string=None,
+                token_credential=None,
+                endpoint_suffix=None)
+
+    def test_blob_custom_service_params(self):
+        storage = azure_storage.AzureStorage()
+        storage.is_emulated = True
+        storage.endpoint_suffix = 'foo_suffix'
+        storage.account_name = 'foo_name'
+        storage.account_key = 'foo_key'
+        storage.sas_token = 'foo_token'
+        storage.azure_ssl = True
+        storage.custom_domain = 'foo_domain'
+        storage.custom_connection_string = 'foo_conn'
+        storage.token_credential = 'foo_cred'
+        with mock.patch(
+                'storages.backends.azure_storage.BlockBlobService',
+                autospec=True) as c_mocked:
+            self.assertIsNotNone(storage.custom_service)
+            c_mocked.assert_called_once_with(
+                account_name='foo_name',
+                account_key='foo_key',
+                sas_token='foo_token',
+                is_emulated=True,
+                protocol='https',
+                custom_domain='foo_domain',
+                connection_string='foo_conn',
+                token_credential='foo_cred',
+                endpoint_suffix='foo_suffix')
+
     # From boto3
 
     def test_storage_save(self):