Skip to content

Commit

Permalink
Load Session from Zipfile
Browse files Browse the repository at this point in the history
When attempting to use botocore / boto3 in AWS EMR or PySpark, it's necessary to load from a zipfile. However, botocore does not support this.

```
+ PYTHONPATH=botocore.zip:boto3.zip
+ .venv/bin/python -c 'import boto3; client=boto3.client("rds")'
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/boto3.zip/boto3/__init__.py", line 91, in client
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/boto3.zip/boto3/session.py", line 258, in client
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 827, in create_client
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 700, in _get_internal_component
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 924, in get_component
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/session.py", line 163, in create_default_resolver
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/loaders.py", line 132, in _wrapper
  File "/private/var/folders/4f/2ps6fjrj2kn4klgq638844bc0000gq/T/tmp.TKx8uF60/botocore.zip/botocore/loaders.py", line 424, in load_data
botocore.exceptions.DataNotFoundError: Unable to load data for: endpoints
```

This is a re-submission of #1969 [with permission from @gliptak](#1969 (comment)), the original author.
  • Loading branch information
kojiromike committed May 18, 2022
1 parent 3dddafb commit 88fb613
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 5 deletions.
3 changes: 0 additions & 3 deletions botocore/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
# language governing permissions and limitations under the License.

import logging
import os
import re

__version__ = '1.26.2'
Expand Down Expand Up @@ -60,8 +59,6 @@ def emit(self, record):
# individual case.
ScalarTypes = ('string', 'integer', 'boolean', 'timestamp', 'float', 'double')

BOTOCORE_ROOT = os.path.dirname(os.path.abspath(__file__))


# Used to specify anonymous (unsigned) request signature
class UNSIGNED:
Expand Down
5 changes: 3 additions & 2 deletions botocore/loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,8 @@
import logging
import os

from botocore import BOTOCORE_ROOT
import pkg_resources

from botocore.compat import HAS_GZIP, OrderedDict, json
from botocore.exceptions import DataNotFoundError, UnknownServiceError
from botocore.utils import deep_merge
Expand Down Expand Up @@ -233,7 +234,7 @@ class Loader:

FILE_LOADER_CLASS = JSONFileLoader
# The included models in botocore/data/ that we ship with botocore.
BUILTIN_DATA_PATH = os.path.join(BOTOCORE_ROOT, 'data')
BUILTIN_DATA_PATH = pkg_resources.resource_filename('botocore', 'data')
# For convenience we automatically add ~/.aws/models to the data path.
CUSTOMER_DATA_PATH = os.path.join(
os.path.expanduser('~'), '.aws', 'models'
Expand Down
21 changes: 21 additions & 0 deletions tests/functional/test_zip.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import os
import shutil
import sys
import tempfile

from tests import BaseSessionTest


class ZipTest(BaseSessionTest):
def test_load_from_zip(self):
fd, temp_path = tempfile.mkstemp(suffix='.zip')
root, ext = os.path.splitext(temp_path)
shutil.make_archive(root, 'zip', '../..')
sys.path.insert(0, temp_path)
super(ZipTest, self).setUp()
self.region = 'us-west-2'
self.client = self.session.create_client(
's3', self.region)
sys.path.remove(temp_path)
os.close(fd)
os.remove(temp_path)

0 comments on commit 88fb613

Please sign in to comment.