Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix extra_args being passed to get_object call in MultipartDownloader #257

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Fix extra_args being passed to get_object call in MultipartDownloader #257

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 13 additions & 3 deletions s3transfer/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -543,6 +543,7 @@ def download_file(
key,
filename,
object_size,
extra_args,
callback,
)
parts_future = controller.submit(download_parts_handler)
Expand All @@ -563,7 +564,7 @@ def _process_future_results(self, futures):
future.result()

def _download_file_as_future(
self, bucket, key, filename, object_size, callback
self, bucket, key, filename, object_size, extra_args, callback
):
part_size = self._config.multipart_chunksize
num_parts = int(math.ceil(object_size / float(part_size)))
Expand All @@ -575,6 +576,7 @@ def _download_file_as_future(
filename,
part_size,
num_parts,
extra_args,
callback,
)
try:
Expand All @@ -593,7 +595,15 @@ def _calculate_range_param(self, part_size, part_index, num_parts):
return range_param

def _download_range(
self, bucket, key, filename, part_size, num_parts, callback, part_index
self,
bucket,
key,
filename,
part_size,
num_parts,
extra_args,
callback,
part_index,
):
try:
range_param = self._calculate_range_param(
Expand All @@ -606,7 +616,7 @@ def _download_range(
try:
logger.debug("Making get_object call.")
response = self._client.get_object(
Bucket=bucket, Key=key, Range=range_param
Bucket=bucket, Key=key, Range=range_param, **extra_args
)
streaming_body = StreamReaderProgress(
response['Body'], callback
Expand Down
24 changes: 24 additions & 0 deletions tests/unit/test_s3transfer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,30 @@ def test_download_file_fowards_extra_args(self):
SSECustomerKey='foo',
)

def test_mutlipart_download_file_fowards_extra_args(self):
extra_args = {
'SSECustomerKey': 'foo',
'SSECustomerAlgorithm': 'AES256',
}
osutil = InMemoryOSLayer({})
over_multipart_threshold = 100 * 1024 * 1024
transfer = S3Transfer(self.client, osutil=osutil)
self.client.head_object.return_value = {
'ContentLength': over_multipart_threshold
}
self.client.get_object.return_value = {'Body': BytesIO(b'foobar')}

transfer.download_file(
'bucket', 'key', 'filename', extra_args=extra_args
)

self.client.get_object.assert_called_with(
Bucket='bucket',
Key='key',
Range=self.client.get_object.call_args.kwargs['Range'],
**extra_args
)

def test_get_object_stream_is_retried_and_succeeds(self):
below_threshold = 20
osutil = InMemoryOSLayer({'smallfile': b'hello world'})
Expand Down