MOTOR-689: Add async wrapper for pymongo.encryption.ClientEncryption

.evergreen/config.yml

@@ -328,12 +328,41 @@ functions:
           sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh
 
   "run tox":
+    - command: shell.exec
+      type: test
+      params:
+        silent: true
+        working_dir: "src"
+        script: |
+          if [ -n "${test_encryption}" ]; then
+            cat <<EOT > fle_creds.sh
+            export FLE_AWS_KEY="${fle_aws_key}"
+            export FLE_AWS_SECRET="${fle_aws_secret}"
+            export FLE_AZURE_CLIENTID="${fle_azure_clientid}"
+            export FLE_AZURE_TENANTID="${fle_azure_tenantid}"
+            export FLE_AZURE_CLIENTSECRET="${fle_azure_clientsecret}"
+            export FLE_GCP_EMAIL="${fle_gcp_email}"
+            export FLE_GCP_PRIVATEKEY="${fle_gcp_privatekey}"
+            # Needed for generating temporary aws credentials.
+            export AWS_ACCESS_KEY_ID="${fle_aws_key}"
+            export AWS_SECRET_ACCESS_KEY="${fle_aws_secret}"
+            export AWS_DEFAULT_REGION=us-east-1
+          EOT
+          fi
     - command: shell.exec
       type: test
       params:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
+          if [ -n "${test_encryption}" ]; then
+            # Disable xtrace (just in case it was accidentally set).
+            set +x
+            . ./fle_creds.sh
+            rm -f ./fle_creds.sh
+            export LIBMONGOCRYPT_URL="${libmongocrypt_url}"
+            export TEST_ENCRYPTION=1
+          fi
 
           PYTHON_BINARY="${PYTHON_BINARY}" \
           TOX_BINARY="${TOX_BINARY}" \
@@ -882,6 +911,16 @@ axes:
            INSTALL_TOX: true
            VIRTUALENV: "/opt/mongodbtoolchain/v2/bin/python3.6 -m virtualenv"
 
+  - id: encryption
+    display_name: "Encryption"
+    values:
+      - id: "encryption"
+        display_name: "Encryption"
+        tags: ["encryption_tag"]
+        variables:
+          test_encryption: true
+        batchtime: 10080  # 7 days
+
 buildvariants:
 
 # Test MongoDB 3.0 and Python only up to 3.6 on RHEL.

.evergreen/run-tox.sh

@@ -10,9 +10,12 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #       INSTALL_TOX             Whether to install tox in a virtualenv
 #       PYTHON_BINARY           Path to python
 #       VIRTUALENV              Path to virtualenv script
+#       TEST_ENCRYPTION         If non-empty, install pymongocrypt.
+
 
 AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
+TEST_ENCRYPTION=${TEST_ENCRYPTION:-}
 
 if [ "$AUTH" != "noauth" ]; then
     export DB_USER="bob"
@@ -43,5 +46,54 @@ if [ "${INSTALL_TOX}" = "true" ]; then
     TOX_BINARY=tox
 fi
 
+if [ -n "$TEST_ENCRYPTION" ]; then
+    createvirtualenv $PYTHON venv-encryption
+    trap "deactivate; rm -rf venv-encryption" EXIT HUP
+    PYTHON=python
+
+    if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+        $PYTHON -m pip install -U setuptools
+    fi
+
+    if [ -z "$LIBMONGOCRYPT_URL" ]; then
+        echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!"
+        exit 1
+    fi
+    curl -O "$LIBMONGOCRYPT_URL"
+    mkdir libmongocrypt
+    tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
+    ls -la libmongocrypt
+    ls -la libmongocrypt/nocrypto
+    # Use the nocrypto build to avoid dependency issues with older windows/python versions.
+    BASE=$(pwd)/libmongocrypt/nocrypto
+    if [ -f "${BASE}/lib/libmongocrypt.so" ]; then
+        export PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so
+    elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then
+        export PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib
+    elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then
+        PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll
+        # libmongocrypt's windows dll is not marked executable.
+        chmod +x $PYMONGOCRYPT_LIB
+        export PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB)
+    elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then
+        export PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so
+    else
+        echo "Cannot find libmongocrypt shared object file"
+        exit 1
+    fi
+
+    # TODO: Test with 'pip install pymongocrypt'
+    git clone --branch master https://github.com/mongodb/libmongocrypt.git libmongocrypt_git
+    python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt
+    python -m pip install ./libmongocrypt_git/bindings/python
+    python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
+    python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
+    # PATH is updated by PREPARE_SHELL for access to mongocryptd.
+
+    # Get access to the AWS temporary credentials:
+    # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
+    . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh
+fi
+
 # Run the tests, and store the results in Evergreen compatible XUnit XML
 ${TOX_BINARY} -e ${TOX_ENV} ${SETUP_ARGS} "$@"

.travis.yml

@@ -10,6 +10,7 @@ services: mongodb
 
 install:
   - pip install tornado
+  - pip install pymongo[encryption]
 
 script: "python setup.py test"
 

doc/api-asyncio/asyncio_motor_client_encryption.rst

@@ -0,0 +1,7 @@
+:class:`~motor.motor_asyncio.AsyncIOMotorClientEncryption`
+==========================================================
+
+.. currentmodule:: motor.motor_asyncio
+
+.. autoclass:: AsyncIOMotorClientEncryption
+  :members:

doc/api-asyncio/index.rst

@@ -8,6 +8,7 @@ Motor asyncio API
     asyncio_motor_database
     asyncio_motor_collection
     asyncio_motor_change_stream
+    asyncio_motor_client_encryption
     cursors
     asyncio_gridfs
     aiohttp

doc/api-tornado/index.rst

@@ -8,6 +8,7 @@ Motor Tornado API
     motor_database
     motor_collection
     motor_change_stream
+    motor_client_encryption
     cursors
     gridfs
     web

doc/api-tornado/motor_client_encryption.rst

@@ -0,0 +1,7 @@
+:class:`~motor.motor_tornado.MotorClientEncryption`
+===================================================
+
+.. currentmodule:: motor.motor_tornado
+
+.. autoclass:: MotorClientEncryption
+  :members:

motor/core.py

@@ -34,6 +34,7 @@
 from pymongo.cursor import Cursor, RawBatchCursor, _QUERY_OPTIONS
 from pymongo.database import Database
 from pymongo.driver_info import DriverInfo
+from pymongo.encryption import ClientEncryption
 
 from . import version as motor_version
 from .metaprogramming import (AsyncCommand,
@@ -140,7 +141,7 @@ def __init__(self, *args, **kwargs):
 
         :Parameters:
           - `io_loop` (optional): Special event loop
-            instance to use instead of default
+            instance to use instead of default.
         """
         if 'io_loop' in kwargs:
             io_loop = kwargs.pop('io_loop')
@@ -1795,3 +1796,49 @@ def __enter__(self):
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         pass
+
+class AgnosticClientEncryption(AgnosticBase):
+    """Explicit client-side field level encryption."""
+
+    __motor_class_name__ = 'MotorClientEncryption'
+    __delegate_class__ = ClientEncryption
+
+    create_data_key = AsyncCommand(doc=create_data_key_doc)
+    encrypt = AsyncCommand()
+    decrypt = AsyncCommand()
+    close = AsyncCommand(doc=close_doc)
+
+    def __init__(self, kms_providers, key_vault_namespace, key_vault_client, codec_options, io_loop=None):
+        """Explicit client-side field level encryption.
+
+        Takes the same constructor arguments as
+        :class:`pymongo.encryption.ClientEncryption`, as well as:
+
+        :Parameters:
+          - `io_loop` (optional): Special event loop
+            instance to use instead of default.
+        """
+        if io_loop:
+            self._framework.check_event_loop(io_loop)
+        else:
+            io_loop = self._framework.get_event_loop()
+        sync_client = key_vault_client.delegate
+        delegate = self.__delegate_class__(kms_providers, key_vault_namespace, sync_client, codec_options)
+        super().__init__(delegate)
+        self.io_loop = io_loop
+
+    def get_io_loop(self):
+        return self.io_loop
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.delegate:
+            await self.close()
+
+    def __enter__(self):
+        raise RuntimeError('Use this encryption module in "async with", not "with"')
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass

motor/docstrings.py

@@ -1268,3 +1268,32 @@ async def coro():
 .. _$expr: https://docs.mongodb.com/manual/reference/operator/query/expr/
 .. _$where: https://docs.mongodb.com/manual/reference/operator/query/where/
 """
+
+create_data_key_doc = """Create and insert a new data key into the key vault collection.
+
+Takes the same constructors as 
+:class:`pymongo.encryption.ClientEncryption.create_data_key`,
+with only the following slight difference using async syntax:
+
+:Parameters:
+  - `key_alt_names` (optional): An optional list of string alternate
+    names used to reference a key. If a key is created with alternate
+    names, then encryption may refer to the key by the unique alternate
+    name instead of by ``key_id``. The following example shows creating
+    and referring to a data key by alternate name::
+
+      await client_encryption.create_data_key("local", keyAltNames=["name1"])
+      # reference the key with the alternate name
+      await client_encryption.encrypt("457-55-5462", keyAltName="name1",
+                                algorithm=Algorithm.Random)
+"""
+
+close_doc = """Release resources.
+
+Note that using this class in a with-statement will automatically call
+:meth:`close`::
+
+    async with AsyncIOMotorClientEncryption(...) as client_encryption:
+        encrypted = await client_encryption.encrypt(value, ...)
+        decrypted = await client_encryption.decrypt(encrypted)
+"""

motor/motor_asyncio.py

@@ -18,7 +18,7 @@
 from .frameworks import asyncio as asyncio_framework
 from .metaprogramming import create_class_with_framework
 
-__all__ = ['AsyncIOMotorClient']
+__all__ = ['AsyncIOMotorClient','AsyncIOMotorClientEncryption']
 
 
 def create_asyncio_class(cls):
@@ -70,3 +70,7 @@ def create_asyncio_class(cls):
 
 AsyncIOMotorGridOutCursor = create_asyncio_class(
     motor_gridfs.AgnosticGridOutCursor)
+
+
+AsyncIOMotorClientEncryption = create_asyncio_class(
+    core.AgnosticClientEncryption)

motor/motor_tornado.py

@@ -18,7 +18,7 @@
 from .frameworks import tornado as tornado_framework
 from .metaprogramming import create_class_with_framework
 
-__all__ = ['MotorClient']
+__all__ = ['MotorClient', 'MotorClientEncryption']
 
 
 def create_motor_class(cls):
@@ -60,3 +60,6 @@ def create_motor_class(cls):
 
 
 MotorGridOutCursor = create_motor_class(motor_gridfs.AgnosticGridOutCursor)
+
+
+MotorClientEncryption = create_motor_class(core.AgnosticClientEncryption)

setup.py

@@ -33,6 +33,10 @@
 
 tests_require = ['mockupdb>=1.4.0']
 
+extras_require = {
+    'encryption': ['pymongocrypt>=1.1.0,<2.0.0'],
+}
+
 class test(Command):
     description = "run the tests"
 
@@ -147,4 +151,5 @@ def run(self):
       tests_require=tests_require,
       test_suite='test',
       zip_safe=False,
-      cmdclass={'test': test})
+      cmdclass={'test': test},
+      extras_require=extras_require)