Skip to content

Commit

Permalink
Merge pull request #161 from etianen/text-mode
Browse files Browse the repository at this point in the history 
Adding support for opening files in text mode
  • Loading branch information
@etianen
etianen committed Nov 4, 2023
2 parents 62c41bb + 9e57768 commit b239095
Show file tree
Hide file tree
Showing 5 changed files with 227 additions and 90 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.rst
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
django-s3-storage changelog
===========================

0.15.0
------

- Added support for opening files in text mode on Python 3.11+ (@etianen).

0.14.0
------

Expand Down
2 changes: 1 addition & 1 deletion django_s3_storage/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
"""


__version__ = (0, 14, 0)
__version__ = (0, 15, 0)
153 changes: 106 additions & 47 deletions django_s3_storage/storage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@
import shutil
from contextlib import closing
from datetime import timezone
from functools import wraps, partial
from io import TextIOBase
from functools import partial, wraps
from io import TextIOBase, TextIOWrapper
from tempfile import SpooledTemporaryFile
from threading import local
from urllib.parse import urljoin, urlsplit, urlunsplit

import boto3
from botocore.client import Config
from boto3.s3.transfer import TransferConfig
from botocore.client import Config
from botocore.exceptions import ClientError
from django.conf import settings
from django.contrib.staticfiles.storage import ManifestFilesMixin
Expand All @@ -41,6 +41,7 @@ def _do_wrap_errors(self, name, *args, **kwargs):
if code == "NoSuchKey":
err_cls = FileNotFoundError
raise err_cls(f"S3Storage error at {name!r}: {force_str(ex)}")

return _do_wrap_errors


Expand All @@ -63,6 +64,7 @@ def do_wrap_path_impl(self, name, *args, **kwargs):
# posix paths. We fix this by converting paths to system form, passing them to the default implementation, then
# converting them back to posix paths.
return _to_posix_path(func(self, _to_sys_path(name), *args, **kwargs))

return do_wrap_path_impl


Expand Down Expand Up @@ -108,18 +110,24 @@ def __init__(self, storage):
if storage.settings.AWS_ACCESS_KEY_ID:
connection_kwargs["aws_access_key_id"] = storage.settings.AWS_ACCESS_KEY_ID
if storage.settings.AWS_SECRET_ACCESS_KEY:
connection_kwargs["aws_secret_access_key"] = storage.settings.AWS_SECRET_ACCESS_KEY
connection_kwargs[
"aws_secret_access_key"
] = storage.settings.AWS_SECRET_ACCESS_KEY
if storage.settings.AWS_SESSION_TOKEN:
connection_kwargs["aws_session_token"] = storage.settings.AWS_SESSION_TOKEN
if storage.settings.AWS_S3_ENDPOINT_URL:
connection_kwargs["endpoint_url"] = storage.settings.AWS_S3_ENDPOINT_URL
self.session = boto3.session.Session()
self.s3_connection = self.session.client("s3", config=Config(
s3={"addressing_style": storage.settings.AWS_S3_ADDRESSING_STYLE},
signature_version=storage.settings.AWS_S3_SIGNATURE_VERSION,
max_pool_connections=storage.settings.AWS_S3_MAX_POOL_CONNECTIONS,
connect_timeout=storage.settings.AWS_S3_CONNECT_TIMEOUT
), **connection_kwargs)
self.s3_connection = self.session.client(
"s3",
config=Config(
s3={"addressing_style": storage.settings.AWS_S3_ADDRESSING_STYLE},
signature_version=storage.settings.AWS_S3_SIGNATURE_VERSION,
max_pool_connections=storage.settings.AWS_S3_MAX_POOL_CONNECTIONS,
connect_timeout=storage.settings.AWS_S3_CONNECT_TIMEOUT,
),
**connection_kwargs,
)


@deconstructible
Expand Down Expand Up @@ -155,7 +163,7 @@ class S3Storage(Storage):
"AWS_S3_FILE_OVERWRITE": False,
"AWS_S3_USE_THREADS": True,
"AWS_S3_MAX_POOL_CONNECTIONS": 10,
"AWS_S3_CONNECT_TIMEOUT": 60 # 60 seconds
"AWS_S3_CONNECT_TIMEOUT": 60, # 60 seconds
}

s3_settings_suffix = ""
Expand All @@ -178,16 +186,24 @@ def _setup(self):
setting_key,
self._kwargs.get(
setting_key.lower(),
getattr(settings, setting_key + self.s3_settings_suffix, setting_default_value),
getattr(
settings,
setting_key + self.s3_settings_suffix,
setting_default_value,
),
),
)
# Validate settings.
if not self.settings.AWS_S3_BUCKET_NAME:
raise ImproperlyConfigured(f"Setting AWS_S3_BUCKET_NAME{self.s3_settings_suffix} is required.")
raise ImproperlyConfigured(
f"Setting AWS_S3_BUCKET_NAME{self.s3_settings_suffix} is required."
)
# Create a thread-local connection manager.
self._connections = _Local(self)
# Set transfer config for S3 operations
self._transfer_config = TransferConfig(use_threads=self.settings.AWS_S3_USE_THREADS)
self._transfer_config = TransferConfig(
use_threads=self.settings.AWS_S3_USE_THREADS
)

@property
def s3_connection(self):
Expand All @@ -201,8 +217,8 @@ def __init__(self, **kwargs):
# Check for unknown kwargs.
for kwarg_key in kwargs.keys():
if (
kwarg_key.upper() not in self.default_auth_settings and
kwarg_key.upper() not in self.default_s3_settings
kwarg_key.upper() not in self.default_auth_settings
and kwarg_key.upper() not in self.default_s3_settings
):
raise ImproperlyConfigured(f"Unknown S3Storage parameter: {kwarg_key}")
# Set up the storage.
Expand All @@ -211,7 +227,9 @@ def __init__(self, **kwargs):
# Re-initialize the storage if an AWS setting changes.
setting_changed.connect(self._setting_changed_received)
# Register system checks.
checks.register(partial(self.__class__._system_checks, self), checks.Tags.security)
checks.register(
partial(self.__class__._system_checks, self), checks.Tags.security
)
# All done!
super().__init__()

Expand All @@ -237,7 +255,9 @@ def __reduce__(self):
def _get_key_name(self, name):
if name.startswith("/"):
name = name[1:]
return posixpath.normpath(posixpath.join(self.settings.AWS_S3_KEY_PREFIX, _to_posix_path(name)))
return posixpath.normpath(
posixpath.join(self.settings.AWS_S3_KEY_PREFIX, _to_posix_path(name))
)

def _object_params(self, name):
params = {
Expand All @@ -256,22 +276,29 @@ def _object_put_params(self, name):
),
"Metadata": {
key: _callable_setting(value, name)
for key, value
in self.settings.AWS_S3_METADATA.items()
for key, value in self.settings.AWS_S3_METADATA.items()
},
"StorageClass": "REDUCED_REDUNDANCY" if self.settings.AWS_S3_REDUCED_REDUNDANCY else "STANDARD",
"StorageClass": "REDUCED_REDUNDANCY"
if self.settings.AWS_S3_REDUCED_REDUNDANCY
else "STANDARD",
}
params.update(self._object_params(name))
# Set content disposition.
content_disposition = _callable_setting(self.settings.AWS_S3_CONTENT_DISPOSITION, name)
content_disposition = _callable_setting(
self.settings.AWS_S3_CONTENT_DISPOSITION, name
)
if content_disposition:
params["ContentDisposition"] = content_disposition
# Set content langauge.
content_langauge = _callable_setting(self.settings.AWS_S3_CONTENT_LANGUAGE, name)
content_langauge = _callable_setting(
self.settings.AWS_S3_CONTENT_LANGUAGE, name
)
if content_langauge:
params["ContentLanguage"] = content_langauge
# Set server-side encryption.
if self.settings.AWS_S3_ENCRYPT_KEY: # If this if False / None / empty then no encryption
if (
self.settings.AWS_S3_ENCRYPT_KEY
): # If this if False / None / empty then no encryption
if isinstance(self.settings.AWS_S3_ENCRYPT_KEY, str):
params["ServerSideEncryption"] = self.settings.AWS_S3_ENCRYPT_KEY
if self.settings.AWS_S3_KMS_ENCRYPTION_KEY_ID:
Expand All @@ -283,11 +310,11 @@ def _object_put_params(self, name):

def new_temporary_file(self):
"""Returns a new file to use when opening from or saving to S3"""
return SpooledTemporaryFile(max_size=1024*1024*10) # 10 MB.
return SpooledTemporaryFile(max_size=1024 * 1024 * 10) # 10 MB.

@_wrap_errors
def _open(self, name, mode="rb"):
if mode != "rb":
if mode not in ("rb", "rt", "r"):
raise ValueError("S3 files can only be opened in read-only mode")
# Load the key into a temporary file. It would be nice to stream the
# content, but S3 doesn't support seeking, which is sometimes needed.
Expand All @@ -298,6 +325,9 @@ def _open(self, name, mode="rb"):
# Un-gzip if required.
if obj.get("ContentEncoding") == "gzip":
content = gzip.GzipFile(name, "rb", fileobj=content)
# Decode text if required.
if "b" not in mode:
content = TextIOWrapper(content)
# All done!
return S3File(content, name, self)

Expand Down Expand Up @@ -325,7 +355,12 @@ def _save(self, name, content):
# Check if the content type is compressible.
content_type_family, content_type_subtype = content_type.lower().split("/")
content_type_subtype = content_type_subtype.split("+")[-1]
if content_type_family == "text" or content_type_subtype in ("xml", "json", "html", "javascript"):
if content_type_family == "text" or content_type_subtype in (
"xml",
"json",
"html",
"javascript",
):
# Compress the content.
temp_file = self.new_temporary_file()
temp_files.append(temp_file)
Expand All @@ -337,7 +372,9 @@ def _save(self, name, content):
temp_file.seek(0)
content = temp_file
put_params["ContentEncoding"] = "gzip"
put_params["Metadata"][_UNCOMPRESSED_SIZE_META_KEY] = f"{orig_size:d}"
put_params["Metadata"][
_UNCOMPRESSED_SIZE_META_KEY
] = f"{orig_size:d}"
else:
content.seek(0)
# Save the file.
Expand All @@ -346,8 +383,13 @@ def _save(self, name, content):
original_close = content.close
content.close = lambda: None
try:
self.s3_connection.upload_fileobj(content, put_params.pop('Bucket'), put_params.pop('Key'),
ExtraArgs=put_params, Config=self._transfer_config)
self.s3_connection.upload_fileobj(
content,
put_params.pop("Bucket"),
put_params.pop("Key"),
ExtraArgs=put_params,
Config=self._transfer_config,
)
finally:
# Restore the original close method.
content.close = original_close
Expand Down Expand Up @@ -385,8 +427,8 @@ def delete(self, name):
@_wrap_errors
def copy(self, src_name, dst_name):
self.s3_connection.copy_object(
CopySource=self._object_params(src_name),
**self._object_params(dst_name))
CopySource=self._object_params(src_name), **self._object_params(dst_name)
)

@_wrap_errors
def rename(self, src_name, dst_name):
Expand All @@ -402,7 +444,8 @@ def exists(self, name):
results = self.s3_connection.list_objects_v2(
Bucket=self.settings.AWS_S3_BUCKET_NAME,
MaxKeys=1,
Prefix=self._get_key_name(name) + "/", # Add the slash again, since _get_key_name removes it.
Prefix=self._get_key_name(name)
+ "/", # Add the slash again, since _get_key_name removes it.
)
except ClientError:
return False
Expand Down Expand Up @@ -449,7 +492,9 @@ def url(self, name, extra_params=None, client_method="get_object"):
# Use a public URL, if specified.
if self.settings.AWS_S3_PUBLIC_URL:
if extra_params or client_method != "get_object":
raise ValueError("Use of extra_params or client_method is not allowed with AWS_S3_PUBLIC_URL")
raise ValueError(
"Use of extra_params or client_method is not allowed with AWS_S3_PUBLIC_URL"
)
return urljoin(self.settings.AWS_S3_PUBLIC_URL, filepath_to_uri(name))
# Otherwise, generate the URL.
params = extra_params.copy() if extra_params else {}
Expand All @@ -462,8 +507,16 @@ def url(self, name, extra_params=None, client_method="get_object"):
# Strip off the query params if we're not interested in bucket auth.
if not self.settings.AWS_S3_BUCKET_AUTH:
if extra_params or client_method != "get_object":
raise ValueError("Use of extra_params or client_method is not allowed with AWS_S3_BUCKET_AUTH")
url = urlunsplit(urlsplit(url)[:3] + ("", "",))
raise ValueError(
"Use of extra_params or client_method is not allowed with AWS_S3_BUCKET_AUTH"
)
url = urlunsplit(
urlsplit(url)[:3]
+ (
"",
"",
)
)
# All done!
return url

Expand Down Expand Up @@ -501,8 +554,9 @@ def sync_meta_iter(self):
put_params["ContentEncoding"] = content_encoding
if content_encoding == "gzip":
try:
put_params["Metadata"][_UNCOMPRESSED_SIZE_META_KEY] = \
obj["Metadata"][_UNCOMPRESSED_SIZE_META_KEY]
put_params["Metadata"][_UNCOMPRESSED_SIZE_META_KEY] = obj[
"Metadata"
][_UNCOMPRESSED_SIZE_META_KEY]
except KeyError:
pass
# Update the metadata.
Expand All @@ -513,7 +567,7 @@ def sync_meta_iter(self):
"Key": self._get_key_name(name),
},
MetadataDirective="REPLACE",
**put_params
**put_params,
)
yield name

Expand All @@ -529,23 +583,28 @@ class StaticS3Storage(S3Storage):
"""

default_s3_settings = S3Storage.default_s3_settings.copy()
default_s3_settings.update({
"AWS_S3_BUCKET_AUTH": False,
})
default_s3_settings.update(
{
"AWS_S3_BUCKET_AUTH": False,
}
)

s3_settings_suffix = "_STATIC"


class ManifestStaticS3Storage(ManifestFilesMixin, StaticS3Storage):

default_s3_settings = StaticS3Storage.default_s3_settings.copy()
default_s3_settings.update({
"AWS_S3_MAX_AGE_SECONDS_CACHED": 60 * 60 * 24 * 365, # 1 year.
})
default_s3_settings.update(
{
"AWS_S3_MAX_AGE_SECONDS_CACHED": 60 * 60 * 24 * 365, # 1 year.
}
)

def post_process(self, *args, **kwargs):
initial_aws_s3_max_age_seconds = self.settings.AWS_S3_MAX_AGE_SECONDS
self.settings.AWS_S3_MAX_AGE_SECONDS = self.settings.AWS_S3_MAX_AGE_SECONDS_CACHED
self.settings.AWS_S3_MAX_AGE_SECONDS = (
self.settings.AWS_S3_MAX_AGE_SECONDS_CACHED
)
try:
yield from super().post_process(*args, **kwargs)
finally:
Expand Down
Empty file added pyproject.toml
View file
Empty file.

0 comments on commit b239095

Please sign in to comment.