diff --git a/docs/backends/azure.rst b/docs/backends/azure.rst index b192c25a2..8683cdd16 100644 --- a/docs/backends/azure.rst +++ b/docs/backends/azure.rst @@ -76,8 +76,6 @@ Set the default storage (i.e: for media files) and the static storage The following settings are available: - is_emulated = setting('AZURE_EMULATED_MODE', False) - ``AZURE_ACCOUNT_NAME`` This setting is the Windows Azure Storage Account name, which in many cases @@ -125,3 +123,39 @@ The following settings are available: ``AZURE_LOCATION`` Default location for the uploaded files. This is a path that gets prepended to every file name. + +``AZURE_EMULATED_MODE`` + + Whether to use the emulator (i.e Azurite). Defaults to False. + +``AZURE_ENDPOINT_SUFFIX`` + + The host base component of the url, minus the account name. Defaults + to Azure (``core.windows.net``). Override this to use the China cloud + (``core.chinacloudapi.cn``). + +``AZURE_CUSTOM_DOMAIN`` + + The custom domain to use. This can be set in the Azure Portal. For + example, ``www.mydomain.com`` or ``mycdn.azureedge.net``. + + It may contain a ``host:port`` when using the emulator + (``AZURE_EMULATED_MODE = True``). + +``AZURE_CONNECTION_STRING`` + + If specified, this will override all other parameters. + See http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ + for the connection string format. + +``AZURE_CUSTOM_CONNECTION_STRING`` + + This is similar to ``AZURE_CONNECTION_STRING``, but it's used + when generating the file's URL. A custom domain or CDN may be + specified here instead of within ``AZURE_CONNECTION_STRING``. + Defaults to ``AZURE_CONNECTION_STRING``'s value. + +``AZURE_TOKEN_CREDENTIAL`` + + A token credential used to authenticate HTTPS requests. The token value + should be updated before its expiration. diff --git a/storages/backends/azure_storage.py b/storages/backends/azure_storage.py index 4f1890307..38827c020 100644 --- a/storages/backends/azure_storage.py +++ b/storages/backends/azure_storage.py @@ -7,7 +7,7 @@ from azure.common import AzureMissingResourceHttpError from azure.storage.blob import BlobPermissions, ContentSettings -from azure.storage.common import CloudStorageAccount +from azure.storage.blob.blockblobservice import BlockBlobService from django.core.exceptions import SuspiciousOperation from django.core.files.base import File from django.core.files.storage import Storage @@ -66,8 +66,6 @@ def read(self, *args, **kwargs): return super(AzureStorageFile, self).read(*args, **kwargs) def write(self, content): - if len(content) > 100*1024*1024: - raise ValueError("Max chunk size is 100MB") if ('w' not in self._mode and '+' not in self._mode and 'a' not in self._mode): @@ -147,22 +145,52 @@ class AzureStorage(Storage): location = setting('AZURE_LOCATION', '') default_content_type = 'application/octet-stream' is_emulated = setting('AZURE_EMULATED_MODE', False) + endpoint_suffix = setting('AZURE_ENDPOINT_SUFFIX') + sas_token = setting('AZURE_SAS_TOKEN') + custom_domain = setting('AZURE_CUSTOM_DOMAIN') + connection_string = setting('AZURE_CONNECTION_STRING') + custom_connection_string = setting( + 'AZURE_CUSTOM_CONNECTION_STRING', setting('AZURE_CONNECTION_STRING')) + token_credential = setting('AZURE_TOKEN_CREDENTIAL') def __init__(self): self._service = None + self._custom_service = None - @property - def service(self): + def _blob_service(self, custom_domain=None, connection_string=None): # This won't open a connection or anything, # it's akin to a client + return BlockBlobService( + account_name=self.account_name, + account_key=self.account_key, + sas_token=self.sas_token, + is_emulated=self.is_emulated, + protocol=self.azure_protocol, + custom_domain=custom_domain, + connection_string=connection_string, + token_credential=self.token_credential, + endpoint_suffix=self.endpoint_suffix) + + @property + def service(self): if self._service is None: - account = CloudStorageAccount( - self.account_name, - self.account_key, - is_emulated=self.is_emulated) - self._service = account.create_block_blob_service() + custom_domain = None + if self.is_emulated: + custom_domain = self.custom_domain + self._service = self._blob_service( + custom_domain=custom_domain, + connection_string=self.connection_string) return self._service + @property + def custom_service(self): + """This is used to generate the URL""" + if self._custom_service is None: + self._custom_service = self._blob_service( + custom_domain=self.custom_domain, + connection_string=self.custom_connection_string) + return self._custom_service + @property def azure_protocol(self): if self.azure_ssl: @@ -256,13 +284,13 @@ def url(self, name, expire=None): make_blob_url_kwargs = {} if expire: - sas_token = self.service.generate_blob_shared_access_signature( + sas_token = self.custom_service.generate_blob_shared_access_signature( self.azure_container, name, permission=BlobPermissions.READ, expiry=self._expire_at(expire)) make_blob_url_kwargs['sas_token'] = sas_token if self.azure_protocol: make_blob_url_kwargs['protocol'] = self.azure_protocol - return self.service.make_blob_url( + return self.custom_service.make_blob_url( container_name=self.azure_container, blob_name=name, **make_blob_url_kwargs) diff --git a/tests/integration/test_azure.py b/tests/integration/test_azure.py index edfc58e94..2748bd5e1 100644 --- a/tests/integration/test_azure.py +++ b/tests/integration/test_azure.py @@ -64,6 +64,12 @@ def test_url(self): # has some query-string self.assertTrue("/test/my_file.txt?" in self.storage.url("my_file.txt")) + def test_url_custom_endpoint(self): + storage = azure_storage.AzureStorage() + storage.is_emulated = True + storage.custom_domain = 'foobar:123456' + self.assertTrue(storage.url("my_file.txt").startswith('https://foobar:123456/')) + @override_settings(USE_TZ=True) def test_get_modified_time_tz(self): stream = io.BytesIO(b'Im a stream') diff --git a/tests/test_azure.py b/tests/test_azure.py index a79d9f632..62f2490f3 100644 --- a/tests/test_azure.py +++ b/tests/test_azure.py @@ -23,6 +23,7 @@ class AzureStorageTest(TestCase): def setUp(self, *args): self.storage = azure_storage.AzureStorage() self.storage._service = mock.MagicMock() + self.storage._custom_service = mock.MagicMock() self.storage.overwrite_files = True self.container_name = 'test' self.storage.azure_container = self.container_name @@ -121,9 +122,9 @@ def test_get_available_invalid(self): self.assertRaises(ValueError, self.storage.get_available_name, "$$") def test_url(self): - self.storage._service.make_blob_url.return_value = 'ret_foo' + self.storage._custom_service.make_blob_url.return_value = 'ret_foo' self.assertEqual(self.storage.url('some blob'), 'ret_foo') - self.storage._service.make_blob_url.assert_called_once_with( + self.storage._custom_service.make_blob_url.assert_called_once_with( container_name=self.container_name, blob_name='some_blob', protocol='https') @@ -131,22 +132,148 @@ def test_url(self): def test_url_expire(self): utc = pytz.timezone('UTC') fixed_time = utc.localize(datetime.datetime(2016, 11, 6, 4)) - self.storage._service.generate_blob_shared_access_signature.return_value = 'foo_token' - self.storage._service.make_blob_url.return_value = 'ret_foo' + self.storage._custom_service.generate_blob_shared_access_signature.return_value = 'foo_token' + self.storage._custom_service.make_blob_url.return_value = 'ret_foo' with mock.patch('storages.backends.azure_storage.datetime') as d_mocked: d_mocked.utcnow.return_value = fixed_time self.assertEqual(self.storage.url('some blob', 100), 'ret_foo') - self.storage._service.generate_blob_shared_access_signature.assert_called_once_with( + self.storage._custom_service.generate_blob_shared_access_signature.assert_called_once_with( self.container_name, 'some_blob', permission=BlobPermissions.READ, expiry=fixed_time + timedelta(seconds=100)) - self.storage._service.make_blob_url.assert_called_once_with( + self.storage._custom_service.make_blob_url.assert_called_once_with( container_name=self.container_name, blob_name='some_blob', sas_token='foo_token', protocol='https') + def test_blob_service_default_params(self): + storage = azure_storage.AzureStorage() + with mock.patch( + 'storages.backends.azure_storage.BlockBlobService', + autospec=True) as c_mocked: + self.assertIsNotNone(storage.service) + c_mocked.assert_called_once_with( + account_name=None, + account_key=None, + sas_token=None, + is_emulated=False, + protocol='https', + custom_domain=None, + connection_string=None, + token_credential=None, + endpoint_suffix=None) + + def test_blob_service_params_no_emulator(self): + """Should ignore custom domain when emulator is not used""" + storage = azure_storage.AzureStorage() + storage.is_emulated = False + storage.custom_domain = 'foo_domain' + with mock.patch( + 'storages.backends.azure_storage.BlockBlobService', + autospec=True) as c_mocked: + self.assertIsNotNone(storage.service) + c_mocked.assert_called_once_with( + account_name=None, + account_key=None, + sas_token=None, + is_emulated=False, + protocol='https', + custom_domain=None, + connection_string=None, + token_credential=None, + endpoint_suffix=None) + + def test_blob_service_params(self): + storage = azure_storage.AzureStorage() + storage.is_emulated = True + storage.endpoint_suffix = 'foo_suffix' + storage.account_name = 'foo_name' + storage.account_key = 'foo_key' + storage.sas_token = 'foo_token' + storage.azure_ssl = True + storage.custom_domain = 'foo_domain' + storage.connection_string = 'foo_conn' + storage.token_credential = 'foo_cred' + with mock.patch( + 'storages.backends.azure_storage.BlockBlobService', + autospec=True) as c_mocked: + self.assertIsNotNone(storage.service) + c_mocked.assert_called_once_with( + account_name='foo_name', + account_key='foo_key', + sas_token='foo_token', + is_emulated=True, + protocol='https', + custom_domain='foo_domain', + connection_string='foo_conn', + token_credential='foo_cred', + endpoint_suffix='foo_suffix') + + def test_blob_custom_service_default_params(self): + storage = azure_storage.AzureStorage() + with mock.patch( + 'storages.backends.azure_storage.BlockBlobService', + autospec=True) as c_mocked: + self.assertIsNotNone(storage.custom_service) + c_mocked.assert_called_once_with( + account_name=None, + account_key=None, + sas_token=None, + is_emulated=False, + protocol='https', + custom_domain=None, + connection_string=None, + token_credential=None, + endpoint_suffix=None) + + def test_blob_custom_service_params_no_emulator(self): + """Should pass custom domain when emulator is not used""" + storage = azure_storage.AzureStorage() + storage.is_emulated = False + storage.custom_domain = 'foo_domain' + with mock.patch( + 'storages.backends.azure_storage.BlockBlobService', + autospec=True) as c_mocked: + self.assertIsNotNone(storage.custom_service) + c_mocked.assert_called_once_with( + account_name=None, + account_key=None, + sas_token=None, + is_emulated=False, + protocol='https', + custom_domain='foo_domain', + connection_string=None, + token_credential=None, + endpoint_suffix=None) + + def test_blob_custom_service_params(self): + storage = azure_storage.AzureStorage() + storage.is_emulated = True + storage.endpoint_suffix = 'foo_suffix' + storage.account_name = 'foo_name' + storage.account_key = 'foo_key' + storage.sas_token = 'foo_token' + storage.azure_ssl = True + storage.custom_domain = 'foo_domain' + storage.custom_connection_string = 'foo_conn' + storage.token_credential = 'foo_cred' + with mock.patch( + 'storages.backends.azure_storage.BlockBlobService', + autospec=True) as c_mocked: + self.assertIsNotNone(storage.custom_service) + c_mocked.assert_called_once_with( + account_name='foo_name', + account_key='foo_key', + sas_token='foo_token', + is_emulated=True, + protocol='https', + custom_domain='foo_domain', + connection_string='foo_conn', + token_credential='foo_cred', + endpoint_suffix='foo_suffix') + # From boto3 def test_storage_save(self):