Skip to content

Commit

Permalink
Load Session from Zipfile
Browse files Browse the repository at this point in the history
When attempting to use botocore / boto3 in AWS EMR or PySpark, it's necessary to load from a zipfile. However, botocore does not support this.

```
+ PYTHONPATH=botocore.zip:boto3.zip
+ .venv/bin/python -c 'import boto3; client=boto3.client("rds")'
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/boto3.zip/boto3/__init__.py", line 91, in client
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/boto3.zip/boto3/session.py", line 258, in client
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 827, in create_client
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 700, in _get_internal_component
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 924, in get_component
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 163, in create_default_resolver
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/loaders.py", line 132, in _wrapper
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/loaders.py", line 424, in load_data
botocore.exceptions.DataNotFoundError: Unable to load data for: endpoints
```

This is a re-submission of #1969 [with permission from @gliptak](#1969 (comment)), the original author.
  • Loading branch information
kojiromike committed Apr 18, 2022
1 parent 7203964 commit 801847c
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 4 deletions.
2 changes: 0 additions & 2 deletions botocore/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,6 @@ def emit(self, record):
# individual case.
ScalarTypes = ('string', 'integer', 'boolean', 'timestamp', 'float', 'double')

BOTOCORE_ROOT = os.path.dirname(os.path.abspath(__file__))


# Used to specify anonymous (unsigned) request signature
class UNSIGNED(object):
Expand Down
5 changes: 3 additions & 2 deletions botocore/loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@
import logging
import os

from botocore import BOTOCORE_ROOT
import pkg_resources

from botocore.compat import OrderedDict, json
from botocore.exceptions import DataNotFoundError, UnknownServiceError
from botocore.utils import deep_merge
Expand Down Expand Up @@ -209,7 +210,7 @@ class Loader(object):
"""
FILE_LOADER_CLASS = JSONFileLoader
# The included models in botocore/data/ that we ship with botocore.
BUILTIN_DATA_PATH = os.path.join(BOTOCORE_ROOT, 'data')
BUILTIN_DATA_PATH = pkg_resources.resource_filename('botocore', 'data')
# For convenience we automatically add ~/.aws/models to the data path.
CUSTOMER_DATA_PATH = os.path.join(os.path.expanduser('~'),
'.aws', 'models')
Expand Down
20 changes: 20 additions & 0 deletions tests/functional/test_zip.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import shutil
import tempfile
import os
import nose
import sys
from tests import BaseSessionTest

class ZipTest(BaseSessionTest):
def test_load_from_zip(self):
fd, temp_path = tempfile.mkstemp(suffix='.zip')
root, ext = os.path.splitext(temp_path)
shutil.make_archive(root, 'zip', '../..')
sys.path.insert(0, temp_path)
super(ZipTest, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
's3', self.region)
sys.path.remove(temp_path)
os.close(fd)
os.remove(temp_path)

0 comments on commit 801847c

Please sign in to comment.