MOTOR-689: Add async wrapper for pymongo.encryption.ClientEncryption

.evergreen/config.yml

@@ -334,6 +334,10 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
+          if [ "${CSFLE}" = "csfle" ]; then
+            # Disable xtrace (just in case it was accidentally set).
+            export TEST_ENCRYPTION=1
+          fi
 
           PYTHON_BINARY="${PYTHON_BINARY}" \
           TOX_BINARY="${TOX_BINARY}" \
@@ -730,6 +734,19 @@ axes:
            SSL: "nossl"
            AUTH: "noauth"
 
+  - id: csfle
+    display_name: "CSFLE"
+    values:
+      - id: csfle
+        display_name: CSFLE
+        variables:
+          CSFLE: "csfle"
+        batchtime: 10080  # 7 days
+      - id: nocsfle
+        display_name: NoCSFLE
+        variables:
+          CSFLE: "nocsfle"
+
   - id: tox-env
     display_name: "Tox Env"
     values:
@@ -886,27 +903,30 @@ buildvariants:
 
 # Test MongoDB 3.0 and Python only up to 3.6 on RHEL.
 - matrix_name: "test-mongodb-3.0-rhel"
-  display_name: "${os}-${tox-env}-${ssl}"
+  display_name: "${os}-${tox-env}-${ssl}-${csfle}"
   matrix_spec:
     os: "rhel"
     tox-env: ["tornado5-py36", "py3-pymongo-master"]
     ssl: "*"
+    csfle: "*"
   tasks:
      - ".3.0"
 
 # Main test matrix.
 - matrix_name: "main"
-  display_name: "${os}-${tox-env}-${ssl}"
+  display_name: "${os}-${tox-env}-${ssl}-${csfle}"
   matrix_spec:
     os: "ubuntu"
     tox-env: "*"
     ssl: "*"
+    csfle: "*"
   exclude_spec:
     # TODO: synchro needs PyMongo master's updated SSL test certs,
     # which may require Motor test suite changes.
     - os: "*"
       tox-env: ["synchro37"]
       ssl: "ssl"
+      csfle: "*"
   tasks:
      - ".latest"
      - ".4.4"
@@ -917,11 +937,12 @@ buildvariants:
      - ".3.2"
 
 - matrix_name: "test-win"
-  display_name: "${os}-${tox-env-win}-${ssl}"
+  display_name: "${os}-${tox-env-win}-${ssl}-${csfle}"
   matrix_spec:
     os: "win"
     tox-env-win: "*"
     ssl: "*"
+    csfle: "*"
   tasks:
      - ".latest"
      - ".4.4"
@@ -932,11 +953,12 @@ buildvariants:
      - ".3.2"
 
 - matrix_name: "test-macos"
-  display_name: "${os}-${tox-env-osx}-${ssl}"
+  display_name: "${os}-${tox-env-osx}-${ssl}-${csfle}"
   matrix_spec:
     os: "macos-1014"
     tox-env-osx: "*"
     ssl: "*"
+    csfle: "*"
   tasks:
      - ".latest"
 

.evergreen/run-tox.sh

@@ -10,9 +10,12 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #       INSTALL_TOX             Whether to install tox in a virtualenv
 #       PYTHON_BINARY           Path to python
 #       VIRTUALENV              Path to virtualenv script
+#       TEST_ENCRYPTION         If non-empty, install pymongocrypt.
+
 
 AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
+TEST_ENCRYPTION=${TEST_ENCRYPTION:-}
 
 if [ "$AUTH" != "noauth" ]; then
     export DB_USER="bob"
@@ -43,5 +46,17 @@ if [ "${INSTALL_TOX}" = "true" ]; then
     TOX_BINARY=tox
 fi
 
+# For createvirtualenv.
+. .evergreen/utils.sh
+
+if [ -n "$TEST_ENCRYPTION" ]; then
+    createvirtualenv $PYTHON_BINARY venv-encryption
+    trap "deactivate; rm -rf venv-encryption" EXIT HUP
+    PYTHON=python
+    python -m pip install pymongo[encryption]
+    python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
+    python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
+fi
+
 # Run the tests, and store the results in Evergreen compatible XUnit XML
 ${TOX_BINARY} -e ${TOX_ENV} ${SETUP_ARGS} "$@"

.evergreen/utils.sh

@@ -0,0 +1,70 @@
+#!/bin/bash -ex
+
+# Usage:
+# createvirtualenv /path/to/python /output/path/for/venv
+# * param1: Python binary to use for the virtualenv
+# * param2: Path to the virtualenv to create
+createvirtualenv () {
+    PYTHON=$1
+    VENVPATH=$2
+    if $PYTHON -m virtualenv --version; then
+        VIRTUALENV="$PYTHON -m virtualenv --never-download"
+    elif $PYTHON -m venv -h>/dev/null; then
+        VIRTUALENV="$PYTHON -m venv"
+    elif command -v virtualenv; then
+        VIRTUALENV="$(command -v virtualenv) -p $PYTHON --never-download"
+    else
+        echo "Cannot test without virtualenv"
+        exit 1
+    fi
+    $VIRTUALENV $VENVPATH
+    if [ "Windows_NT" = "$OS" ]; then
+        . $VENVPATH/Scripts/activate
+    else
+        . $VENVPATH/bin/activate
+    fi
+    # Upgrade to the latest versions of pip setuptools wheel so that
+    # pip can always download the latest cryptography+cffi wheels.
+    PYTHON_VERSION=$(python -c 'import sys;print("%s.%s" % sys.version_info[:2])')
+    if [[ $PYTHON_VERSION == "3.4" ]]; then
+        # pip 19.2 dropped support for Python 3.4.
+        python -m pip install --upgrade 'pip<19.2'
+    elif [[ $PYTHON_VERSION == "2.7" || $PYTHON_VERSION == "3.5" ]]; then
+        # pip 21 will drop support for Python 2.7 and 3.5.
+        python -m pip install --upgrade 'pip<21'
+    else
+        python -m pip install --upgrade pip
+    fi
+    python -m pip install --upgrade setuptools wheel
+}
+
+# Usage:
+# testinstall /path/to/python /path/to/.whl/or/.egg ["no-virtualenv"]
+# * param1: Python binary to test
+# * param2: Path to the wheel or egg file to install
+# * param3 (optional): If set to a non-empty string, don't create a virtualenv. Used in manylinux containers.
+testinstall () {
+    PYTHON=$1
+    RELEASE=$2
+    NO_VIRTUALENV=$3
+
+    if [ -z "$NO_VIRTUALENV" ]; then
+        createvirtualenv $PYTHON venvtestinstall
+        PYTHON=python
+    fi
+
+    if [[ $RELEASE == *.egg ]]; then
+        $PYTHON -m easy_install $RELEASE
+    else
+        $PYTHON -m pip install --upgrade $RELEASE
+    fi
+    cd tools
+    $PYTHON fail_if_no_c.py
+    $PYTHON -m pip uninstall -y pymongo
+    cd ..
+
+    if [ -z "$NO_VIRTUALENV" ]; then
+        deactivate
+        rm -rf venvtestinstall
+    fi
+}

.travis.yml

@@ -10,6 +10,7 @@ services: mongodb
 
 install:
   - pip install tornado
+  - pip install pymongo[encryption]
 
 script: "python setup.py test"
 

doc/api-asyncio/asyncio_motor_client_encryption.rst

@@ -0,0 +1,7 @@
+:class:`~motor.motor_asyncio.AsyncIOMotorClientEncryption`
+==========================================================
+
+.. currentmodule:: motor.motor_asyncio
+
+.. autoclass:: AsyncIOMotorClientEncryption
+  :members:

doc/api-asyncio/index.rst

@@ -8,6 +8,7 @@ Motor asyncio API
     asyncio_motor_database
     asyncio_motor_collection
     asyncio_motor_change_stream
+    asyncio_motor_client_encryption
     cursors
     asyncio_gridfs
     aiohttp

doc/api-tornado/index.rst

@@ -8,6 +8,7 @@ Motor Tornado API
     motor_database
     motor_collection
     motor_change_stream
+    motor_client_encryption
     cursors
     gridfs
     web

doc/api-tornado/motor_client_encryption.rst

@@ -0,0 +1,7 @@
+:class:`~motor.motor_tornado.MotorClientEncryption`
+===================================================
+
+.. currentmodule:: motor.motor_tornado
+
+.. autoclass:: MotorClientEncryption
+  :members:

motor/core.py

@@ -34,6 +34,7 @@
 from pymongo.cursor import Cursor, RawBatchCursor, _QUERY_OPTIONS
 from pymongo.database import Database
 from pymongo.driver_info import DriverInfo
+from pymongo.encryption import ClientEncryption
 
 from . import version as motor_version
 from .metaprogramming import (AsyncCommand,
@@ -140,7 +141,7 @@ def __init__(self, *args, **kwargs):
 
         :Parameters:
           - `io_loop` (optional): Special event loop
-            instance to use instead of default
+            instance to use instead of default.
         """
         if 'io_loop' in kwargs:
             io_loop = kwargs.pop('io_loop')
@@ -1795,3 +1796,49 @@ def __enter__(self):
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         pass
+
+class AgnosticClientEncryption(AgnosticBase):
+    """Explicit client-side field level encryption."""
+
+    __motor_class_name__ = 'MotorClientEncryption'
+    __delegate_class__ = ClientEncryption
+
+    create_data_key = AsyncCommand(doc=create_data_key_doc)
+    encrypt = AsyncCommand()
+    decrypt = AsyncCommand()
+    close = AsyncCommand(doc=close_doc)
+
+    def __init__(self, kms_providers, key_vault_namespace, key_vault_client, codec_options, io_loop=None):
+        """Explicit client-side field level encryption.
+
+        Takes the same constructor arguments as
+        :class:`pymongo.encryption.ClientEncryption`, as well as:
+
+        :Parameters:
+          - `io_loop` (optional): Special event loop
+            instance to use instead of default.
+        """
+        if io_loop:
+            self._framework.check_event_loop(io_loop)
+        else:
+            io_loop = self._framework.get_event_loop()
+        sync_client = key_vault_client.delegate
+        delegate = self.__delegate_class__(kms_providers, key_vault_namespace, sync_client, codec_options)
+        super().__init__(delegate)
+        self.io_loop = io_loop
+
+    def get_io_loop(self):
+        return self.io_loop
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        if self.delegate:
+            await self.close()
+
+    def __enter__(self):
+        raise RuntimeError('Use this encryption module in "async with", not "with"')
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass

motor/docstrings.py

@@ -1268,3 +1268,32 @@ async def coro():
 .. _$expr: https://docs.mongodb.com/manual/reference/operator/query/expr/
 .. _$where: https://docs.mongodb.com/manual/reference/operator/query/where/
 """
+
+create_data_key_doc = """Create and insert a new data key into the key vault collection.
+
+Takes the same constructors as 
+:class:`pymongo.encryption.ClientEncryption.create_data_key`,
+with only the following slight difference using async syntax:
+
+:Parameters:
+  - `key_alt_names` (optional): An optional list of string alternate
+    names used to reference a key. If a key is created with alternate
+    names, then encryption may refer to the key by the unique alternate
+    name instead of by ``key_id``. The following example shows creating
+    and referring to a data key by alternate name::
+
+      await client_encryption.create_data_key("local", keyAltNames=["name1"])
+      # reference the key with the alternate name
+      await client_encryption.encrypt("457-55-5462", keyAltName="name1",
+                                algorithm=Algorithm.Random)
+"""
+
+close_doc = """Release resources.
+
+Note that using this class in a with-statement will automatically call
+:meth:`close`::
+
+    async with AsyncIOMotorClientEncryption(...) as client_encryption:
+        encrypted = await client_encryption.encrypt(value, ...)
+        decrypted = await client_encryption.decrypt(encrypted)
+"""

motor/motor_asyncio.py

@@ -18,7 +18,7 @@
 from .frameworks import asyncio as asyncio_framework
 from .metaprogramming import create_class_with_framework
 
-__all__ = ['AsyncIOMotorClient']
+__all__ = ['AsyncIOMotorClient','AsyncIOMotorClientEncryption']
 
 
 def create_asyncio_class(cls):
@@ -70,3 +70,7 @@ def create_asyncio_class(cls):
 
 AsyncIOMotorGridOutCursor = create_asyncio_class(
     motor_gridfs.AgnosticGridOutCursor)
+
+
+AsyncIOMotorClientEncryption = create_asyncio_class(
+    core.AgnosticClientEncryption)

motor/motor_tornado.py

@@ -18,7 +18,7 @@
 from .frameworks import tornado as tornado_framework
 from .metaprogramming import create_class_with_framework
 
-__all__ = ['MotorClient']
+__all__ = ['MotorClient', 'MotorClientEncryption']
 
 
 def create_motor_class(cls):
@@ -60,3 +60,6 @@ def create_motor_class(cls):
 
 
 MotorGridOutCursor = create_motor_class(motor_gridfs.AgnosticGridOutCursor)
+
+
+MotorClientEncryption = create_motor_class(core.AgnosticClientEncryption)

setup.py

@@ -33,6 +33,10 @@
 
 tests_require = ['mockupdb>=1.4.0']
 
+extras_require = {
+    'encryption': ['pymongocrypt>=1.1.0,<2.0.0'],
+}
+
 class test(Command):
     description = "run the tests"
 
@@ -147,4 +151,5 @@ def run(self):
       tests_require=tests_require,
       test_suite='test',
       zip_safe=False,
-      cmdclass={'test': test})
+      cmdclass={'test': test},
+      extras_require=extras_require)