Update azure storage version

docs/backends/azure.rst

@@ -124,16 +124,6 @@ The following settings are available:
 
     Default location for the uploaded files. This is a path that gets prepended to every file name.
 
-``AZURE_EMULATED_MODE``
-
-    Whether to use the emulator (i.e Azurite). Defaults to False.
-
-``AZURE_ENDPOINT_SUFFIX``
-
-    The host base component of the url, minus the account name. Defaults
-    to Azure (``core.windows.net``). Override this to use the China cloud
-    (``core.chinacloudapi.cn``).
-
 ``AZURE_CUSTOM_DOMAIN``
 
     The custom domain to use. This can be set in the Azure Portal. For
@@ -148,13 +138,6 @@ The following settings are available:
     See http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/
     for the connection string format.
 
-``AZURE_CUSTOM_CONNECTION_STRING``
-
-    This is similar to ``AZURE_CONNECTION_STRING``, but it's used
-    when generating the file's URL. A custom domain or CDN may be
-    specified here instead of within ``AZURE_CONNECTION_STRING``.
-    Defaults to ``AZURE_CONNECTION_STRING``'s value.
-
 ``AZURE_TOKEN_CREDENTIAL``
 
     A token credential used to authenticate HTTPS requests. The token value

setup.cfg

@@ -37,7 +37,7 @@ packages =
 
 [options.extras_require]
 azure =
-    azure-storage-blob >=1.3.1,<12.0.0
+    azure-storage-blob >= 12.0.0
 boto3 =
     boto3 >= 1.4.4
 dropbox =

storages/backends/azure_storage.py

@@ -2,9 +2,11 @@
 from datetime import datetime, timedelta
 from tempfile import SpooledTemporaryFile
 
-from azure.common import AzureMissingResourceHttpError
-from azure.storage.blob import BlobPermissions, ContentSettings
-from azure.storage.blob.blockblobservice import BlockBlobService
+from azure.core.exceptions import ResourceNotFoundError
+from azure.storage.blob import (
+    BlobClient, BlobSasPermissions, ContainerClient, ContentSettings,
+    generate_blob_sas,
+)
 from django.core.exceptions import SuspiciousOperation
 from django.core.files.base import File
 from django.utils import timezone
@@ -40,12 +42,9 @@ def _get_file(self):
         if 'r' in self._mode or 'a' in self._mode:
             # I set max connection to 1 since spooledtempfile is
             # not seekable which is required if we use max_connections > 1
-            self._storage.service.get_blob_to_stream(
-                container_name=self._storage.azure_container,
-                blob_name=self._path,
-                stream=file,
-                max_connections=1,
-                timeout=self._storage.timeout)
+            download_stream = self._storage.client.download_blob(
+                self._path, timeout=self._storage.timeout)
+            download_stream.download_to_stream(file, max_concurrency=1)
         if 'r' in self._mode:
             file.seek(0)
 
@@ -122,8 +121,7 @@ def _get_valid_path(s):
 class AzureStorage(BaseStorage):
     def __init__(self, **settings):
         super().__init__(**settings)
-        self._service = None
-        self._custom_service = None
+        self._client = None
 
     def get_default_settings(self):
         return {
@@ -140,8 +138,6 @@ def get_default_settings(self):
             "location": setting('AZURE_LOCATION', ''),
             "default_content_type": 'application/octet-stream',
             "cache_control": setting("AZURE_CACHE_CONTROL"),
-            "is_emulated": setting('AZURE_EMULATED_MODE', False),
-            "endpoint_suffix": setting('AZURE_ENDPOINT_SUFFIX'),
             "sas_token": setting('AZURE_SAS_TOKEN'),
             "custom_domain": setting('AZURE_CUSTOM_DOMAIN'),
             "connection_string": setting('AZURE_CONNECTION_STRING'),
@@ -152,39 +148,35 @@ def get_default_settings(self):
             "token_credential": setting('AZURE_TOKEN_CREDENTIAL'),
         }
 
-    def _blob_service(self, custom_domain=None, connection_string=None):
-        # This won't open a connection or anything,
-        # it's akin to a client
-        return BlockBlobService(
-            account_name=self.account_name,
-            account_key=self.account_key,
-            sas_token=self.sas_token,
-            is_emulated=self.is_emulated,
-            protocol=self.azure_protocol,
-            custom_domain=custom_domain,
-            connection_string=connection_string,
-            token_credential=self.token_credential,
-            endpoint_suffix=self.endpoint_suffix)
-
-    @property
-    def service(self):
-        if self._service is None:
-            custom_domain = None
-            if self.is_emulated:
-                custom_domain = self.custom_domain
-            self._service = self._blob_service(
-                custom_domain=custom_domain,
-                connection_string=self.connection_string)
-        return self._service
+    def _container_client(self, custom_domain=None, connection_string=None):
+        if custom_domain is None:
+            account_domain = "blob.core.windows.net"
+        else:
+            account_domain = custom_domain
+        if connection_string is None:
+            connection_string = "{}://{}.{}".format(
+                self.azure_protocol,
+                self.account_name,
+                account_domain)
+        credential = None
+        if self.account_key:
+            credential = self.account_key
+        elif self.sas_token:
+            credential = self.sas_token
+        elif self.token_credential:
+            credential = self.token_credential
+        return ContainerClient(
+            connection_string,
+            self.azure_container,
+            credential=credential)
 
     @property
-    def custom_service(self):
-        """This is used to generate the URL"""
-        if self._custom_service is None:
-            self._custom_service = self._blob_service(
+    def client(self):
+        if self._client is None:
+            self._client = self._container_client(
                 custom_domain=self.custom_domain,
-                connection_string=self.custom_connection_string)
-        return self._custom_service
+                connection_string=self.connection_string)
+        return self._client
 
     @property
     def azure_protocol(self):
@@ -219,26 +211,25 @@ def get_available_name(self, name, max_length=_AZURE_NAME_MAX_LEN):
         return super().get_available_name(name, max_length)
 
     def exists(self, name):
-        return self.service.exists(
-            self.azure_container,
-            self._get_valid_path(name),
-            timeout=self.timeout)
+        blob_client = self.client.get_blob_client(self._get_valid_path(name))
+        try:
+            blob_client.get_blob_properties()
+            return True
+        except ResourceNotFoundError:
+            return False
 
     def delete(self, name):
         try:
-            self.service.delete_blob(
-                container_name=self.azure_container,
-                blob_name=self._get_valid_path(name),
+            self.client.delete_blob(
+                self._get_valid_path(name),
                 timeout=self.timeout)
-        except AzureMissingResourceHttpError:
+        except ResourceNotFoundError:
             pass
 
     def size(self, name):
-        properties = self.service.get_blob_properties(
-            self.azure_container,
-            self._get_valid_path(name),
-            timeout=self.timeout).properties
-        return properties.content_length
+        blob_client = self.client.get_blob_client(self._get_valid_path(name))
+        properties = blob_client.get_blob_properties(timeout=self.timeout)
+        return properties.size
 
     def _save(self, name, content):
         cleaned_name = clean_name(name)
@@ -250,13 +241,13 @@ def _save(self, name, content):
             content = content.file
 
         content.seek(0)
-        self.service.create_blob_from_stream(
-            container_name=self.azure_container,
-            blob_name=name,
-            stream=content,
+        self.client.upload_blob(
+            name,
+            content,
             content_settings=ContentSettings(**params),
-            max_connections=self.upload_max_conn,
-            timeout=self.timeout)
+            max_concurrency=self.upload_max_conn,
+            timeout=self.timeout,
+            overwrite=self.overwrite_files)
         return cleaned_name
 
     def _expire_at(self, expire):
@@ -269,17 +260,20 @@ def url(self, name, expire=None):
         if expire is None:
             expire = self.expiration_secs
 
-        make_blob_url_kwargs = {}
+        credential = None
         if expire:
-            sas_token = self.custom_service.generate_blob_shared_access_signature(
-                self.azure_container, name, permission=BlobPermissions.READ, expiry=self._expire_at(expire))
-            make_blob_url_kwargs['sas_token'] = sas_token
+            sas_token = generate_blob_sas(
+                self.account_name,
+                self.azure_container,
+                name,
+                account_key=self.account_key,
+                permission=BlobSasPermissions(read=True),
+                expiry=self._expire_at(expire))
+            credential = sas_token
 
-        return self.custom_service.make_blob_url(
-            container_name=self.azure_container,
-            blob_name=filepath_to_uri(name),
-            protocol=self.azure_protocol,
-            **make_blob_url_kwargs)
+        container_blob_url = self.client.get_blob_client(
+            filepath_to_uri(name)).url
+        return BlobClient.from_blob_url(container_blob_url, credential=credential).url
 
     def _get_content_settings_parameters(self, name, content=None):
         params = {}
@@ -311,10 +305,9 @@ def get_modified_time(self, name):
         Returns an (aware) datetime object containing the last modified time if
         USE_TZ is True, otherwise returns a naive datetime in the local timezone.
         """
-        properties = self.service.get_blob_properties(
-            self.azure_container,
+        properties = self.client.get_blob_properties(
             self._get_valid_path(name),
-            timeout=self.timeout).properties
+            timeout=self.timeout)
         if not setting('USE_TZ', False):
             return timezone.make_naive(properties.last_modified)
 
@@ -342,9 +335,8 @@ def list_all(self, path=''):
         # XXX make generator, add start, end
         return [
             blob.name
-            for blob in self.service.list_blobs(
-                self.azure_container,
-                prefix=path,
+            for blob in self.client.list_blobs(
+                name_starts_with=path,
                 timeout=self.timeout)]
 
     def listdir(self, path=''):

tests/integration/test_azure.py

@@ -15,12 +15,11 @@ class AzureStorageTest(TestCase):
 
     def setUp(self, *args):
         self.storage = azure_storage.AzureStorage()
-        self.storage.is_emulated = True
         self.storage.account_name = "XXX"
         self.storage.account_key = "KXXX"
         self.storage.azure_container = "test"
-        self.storage.service.delete_container(
-            self.storage.azure_container, fail_not_exist=False)
+        self.storage.client.delete_container(
+            fail_not_exist=False)
         self.storage.service.create_container(
             self.storage.azure_container, public_access=False, fail_on_exist=False)
 
@@ -71,7 +70,6 @@ def test_url_unsafe_chars(self):
 
     def test_url_custom_endpoint(self):
         storage = azure_storage.AzureStorage()
-        storage.is_emulated = True
         storage.custom_domain = 'foobar:123456'
         self.assertTrue(storage.url("my_file.txt").startswith('https://foobar:123456/'))
 
@@ -110,12 +108,8 @@ def test_open_read(self):
             fh.close()
 
         stream = io.BytesIO()
-        self.storage.service.get_blob_to_stream(
-            container_name=self.storage.azure_container,
-            blob_name='root/path/some file.txt',
-            stream=stream,
-            max_connections=1,
-            timeout=10)
+        download_stream = self.storage.client.download_blob('root/path/some file.txt', timeout=10)
+        download_stream.download_to_stream(stream, max_concurrency=1)
         stream.seek(0)
         self.assertEqual(stream.read(), b'Im a stream')
 
@@ -130,12 +124,8 @@ def test_open_write(self):
             fh.close()
 
         stream = io.BytesIO()
-        self.storage.service.get_blob_to_stream(
-            container_name=self.storage.azure_container,
-            blob_name=path,
-            stream=stream,
-            max_connections=1,
-            timeout=10)
+        download_stream = self.storage.client.download_blob(path, timeout=10)
+        download_stream.download_to_stream(stream, max_concurrency=1)
         stream.seek(0)
         self.assertEqual(stream.read(), b'foo')
 
@@ -148,12 +138,8 @@ def test_open_write(self):
             fh.close()
 
         stream = io.BytesIO()
-        self.storage.service.get_blob_to_stream(
-            container_name=self.storage.azure_container,
-            blob_name=path,
-            stream=stream,
-            max_connections=1,
-            timeout=10)
+        download_stream = self.storage.client.download_blob(path, timeout=10)
+        download_stream.download_to_stream(stream, max_concurrency=1)
         stream.seek(0)
         self.assertEqual(stream.read(), b'bar')
 
@@ -171,12 +157,8 @@ def test_open_read_write(self):
             fh.close()
 
         stream = io.BytesIO()
-        self.storage.service.get_blob_to_stream(
-            container_name=self.storage.azure_container,
-            blob_name='root/file.txt',
-            stream=stream,
-            max_connections=1,
-            timeout=10)
+        download_stream = self.storage.client.download_blob('root/file.txt', timeout=10)
+        download_stream.download_to_stream(stream, max_concurrency=1)
         stream.seek(0)
         self.assertEqual(stream.read(), b'Im a stream foo')
 
@@ -211,10 +193,8 @@ class Meta:
 class AzureStorageDjangoTest(TestCase):
 
     def setUp(self, *args):
-        default_storage.service.delete_container(
-            default_storage.azure_container, fail_not_exist=False)
-        default_storage.service.create_container(
-            default_storage.azure_container, public_access=False, fail_on_exist=False)
+        default_storage.service.delete_container()
+        default_storage.service.create_container()
 
     def test_is_azure(self):
         self.assertIsInstance(default_storage, azure_storage.AzureStorage)