From 4d531d170d9092532ab9267718f3caa4933148aa Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Fri, 9 Jul 2021 13:24:09 -0700 Subject: [PATCH] PYTHON-2608 Test that KMS TLS connections verify peer certificates (#667) Use bash for all evergreen scripts. (cherry picked from commit 834500de569a37a6ce683614830aabbd3e2aa068) --- .evergreen/config.yml | 22 ++++++------ .evergreen/install-dependencies.sh | 2 +- .evergreen/run-mod-wsgi-tests.sh | 2 +- .evergreen/run-tests.sh | 9 +++++ test/test_encryption.py | 57 ++++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 13 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 1aafc80321..3026cee766 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -291,7 +291,7 @@ functions: DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} \ ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ - sh ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh + bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh # run-orchestration generates expansion file with the MONGODB_URI for the cluster - command: expansions.update params: @@ -309,7 +309,7 @@ functions: script: | set -o xtrace ${PREPARE_SHELL} - sh ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh + bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh - command: shell.exec type: setup params: @@ -317,7 +317,7 @@ functions: script: | set -o xtrace ${PREPARE_SHELL} - sh ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh + bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh "stop mongo-orchestration": - command: shell.exec @@ -325,7 +325,7 @@ functions: script: | set -o xtrace ${PREPARE_SHELL} - sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh + bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh "run mod_wsgi tests": - command: shell.exec @@ -335,7 +335,7 @@ functions: script: | set -o xtrace ${PREPARE_SHELL} - PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} sh ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh + PYTHON_BINARY=${PYTHON_BINARY} MOD_WSGI_VERSION=${MOD_WSGI_VERSION} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-mod-wsgi-tests.sh "run mockupdb tests": - command: shell.exec @@ -345,7 +345,7 @@ functions: script: | set -o xtrace ${PREPARE_SHELL} - PYTHON_BINARY=${PYTHON_BINARY} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} sh ${PROJECT_DIRECTORY}/.evergreen/run-mockupdb-tests.sh + PYTHON_BINARY=${PYTHON_BINARY} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-mockupdb-tests.sh "run cdecimal tests": - command: shell.exec @@ -365,7 +365,7 @@ functions: script: | set -o xtrace ${PREPARE_SHELL} - PYTHON_BINARY=${PYTHON_BINARY} sh ${PROJECT_DIRECTORY}/.evergreen/run-doctests.sh + PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/run-doctests.sh "run tests": - command: shell.exec @@ -434,7 +434,7 @@ functions: SSL=${SSL} \ DATA_LAKE=${DATA_LAKE} \ MONGODB_API_VERSION=${MONGODB_API_VERSION} \ - sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh "run enterprise auth tests": - command: shell.exec @@ -444,7 +444,7 @@ functions: working_dir: "src" script: | # DO NOT ECHO WITH XTRACE (which PREPARE_SHELL does) - PYTHON_BINARY=${PYTHON_BINARY} SASL_HOST=${sasl_host} SASL_PORT=${sasl_port} SASL_USER=${sasl_user} SASL_PASS=${sasl_pass} SASL_DB=${sasl_db} PRINCIPAL=${principal} GSSAPI_DB=${gssapi_db} KEYTAB_BASE64=${keytab_base64} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} sh ${PROJECT_DIRECTORY}/.evergreen/run-enterprise-auth-tests.sh + PYTHON_BINARY=${PYTHON_BINARY} SASL_HOST=${sasl_host} SASL_PORT=${sasl_port} SASL_USER=${sasl_user} SASL_PASS=${sasl_pass} SASL_DB=${sasl_db} PRINCIPAL=${principal} GSSAPI_DB=${gssapi_db} KEYTAB_BASE64=${keytab_base64} PROJECT_DIRECTORY=${PROJECT_DIRECTORY} bash ${PROJECT_DIRECTORY}/.evergreen/run-enterprise-auth-tests.sh "run atlas tests": - command: shell.exec @@ -714,7 +714,7 @@ functions: ${PREPARE_SHELL} file="${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh" # Don't use ${file} syntax here because evergreen treats it as an empty expansion. - [ -f "$file" ] && sh $file || echo "$file not available, skipping" + [ -f "$file" ] && bash $file || echo "$file not available, skipping" "run-ocsp-test": - command: shell.exec @@ -726,7 +726,7 @@ functions: PYTHON_BINARY=${PYTHON_BINARY} \ CA_FILE="$DRIVERS_TOOLS/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \ OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ - sh ${PROJECT_DIRECTORY}/.evergreen/run-ocsp-tests.sh + bash ${PROJECT_DIRECTORY}/.evergreen/run-ocsp-tests.sh run-valid-ocsp-server: - command: shell.exec diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index f28a957746..9f4bcdbb59 100644 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail diff --git a/.evergreen/run-mod-wsgi-tests.sh b/.evergreen/run-mod-wsgi-tests.sh index 5e8b7ca2ac..725023cc3a 100644 --- a/.evergreen/run-mod-wsgi-tests.sh +++ b/.evergreen/run-mod-wsgi-tests.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -o xtrace set -o errexit diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index a77061787c..e5172edf1c 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -164,6 +164,15 @@ if [ -n "$TEST_ENCRYPTION" ]; then # Get access to the AWS temporary credentials: # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh + + # Start the mock KMS servers. + if [ "$OS" != "Windows_NT" ]; then + pushd ${DRIVERS_TOOLS}/.evergreen/csfle + python -u lib/kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 & + python -u lib/kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 & + trap 'kill $(jobs -p)' EXIT HUP + popd + fi fi PYTHON_IMPL=$($PYTHON -c "import platform, sys; sys.stdout.write(platform.python_implementation())") diff --git a/test/test_encryption.py b/test/test_encryption.py index 7d5ba94083..2111b484f7 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -19,6 +19,7 @@ import os import traceback import socket +import ssl import sys import textwrap import uuid @@ -50,6 +51,7 @@ WriteError) from pymongo.mongo_client import MongoClient from pymongo.operations import InsertOne +from pymongo.ssl_support import _ssl from pymongo.write_concern import WriteConcern from test import unittest, IntegrationTest, PyMongoTestCase, client_context @@ -61,6 +63,7 @@ rs_or_single_client, wait_until) from test.utils_spec_runner import SpecRunner +from test.test_ssl import CA_PEM def get_client_opts(client): @@ -1630,5 +1633,59 @@ def test_bypassAutoEncryption(self): mongocryptd_client.admin.command('ping') +# https://github.com/mongodb/specifications/tree/master/source/client-side-encryption/tests#kms-tls-tests +class TestKmsTLSProse(EncryptionIntegrationTest): + @unittest.skipIf(sys.platform == 'win32', + "Can't test system ca certs on Windows") + @unittest.skipIf(ssl.OPENSSL_VERSION.lower().startswith('libressl') and + sys.platform == 'darwin' and not _ssl.IS_PYOPENSSL, + "LibreSSL on OSX doesn't support setting CA certificates " + "using SSL_CERT_FILE environment variable.") + @unittest.skipUnless(any(AWS_CREDS.values()), + 'AWS environment credentials are not set') + def setUp(self): + self.original_certs = os.environ.get('SSL_CERT_FILE') + def restore_certs(): + if self.original_certs is None: + os.environ.pop('SSL_CERT_FILE') + else: + os.environ['SSL_CERT_FILE'] = self.original_certs + # Tell OpenSSL where CA certificates live. + os.environ['SSL_CERT_FILE'] = CA_PEM + self.addCleanup(restore_certs) + + self.client_encrypted = ClientEncryption( + {'aws': AWS_CREDS}, 'keyvault.datakeys', self.client, OPTS) + self.addCleanup(self.client_encrypted.close) + + def test_invalid_kms_certificate_expired(self): + key = { + "region": "us-east-1", + "key": "arn:aws:kms:us-east-1:579766882180:key/" + "89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "endpoint": "mongodb://127.0.0.1:8000", + } + # Some examples: + # certificate verify failed: certificate has expired (_ssl.c:1129) + # amazon1-2018 Python 3.6: certificate verify failed (_ssl.c:852) + with self.assertRaisesRegex( + EncryptionError, 'expired|certificate verify failed'): + self.client_encrypted.create_data_key('aws', master_key=key) + + def test_invalid_hostname_in_kms_certificate(self): + key = { + "region": "us-east-1", + "key": "arn:aws:kms:us-east-1:579766882180:key/" + "89fcc2c4-08b0-4bd9-9f25-e30687b580d0", + "endpoint": "mongodb://127.0.0.1:8001", + } + # Some examples: + # certificate verify failed: IP address mismatch, certificate is not valid for '127.0.0.1'. (_ssl.c:1129)" + # hostname '127.0.0.1' doesn't match 'wronghost.com' + with self.assertRaisesRegex( + EncryptionError, 'IP address mismatch|wronghost'): + self.client_encrypted.create_data_key('aws', master_key=key) + + if __name__ == "__main__": unittest.main()