From b93b4067ea1ded1e33959920ae5ff4163fdd6939 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 24 Oct 2022 14:41:32 +0400 Subject: [PATCH 1/5] Fix x509 tests by using trustme This way the certificate won't be able to expire anymore and we won't have to store a blob in git. trustme only supports PEM, not DER, which is why test_x509_der has to re-export the certificate and key to DER. Additionally, since trustme does not support password-protected keys, the re-export was the perfect place to add a password, so test_x509_der also tests the password case, while it was test_x509_pem until now. The tests are still not end-to-end, we're just running the x509.py code without actually establishing a TLS connection. This could be fixed at a later point, but is considered out of scope here as that would be a new feature. --- dev-requirements.txt | 1 + tests/certs/test_cert.p12 | Bin 1941 -> 0 bytes tests/test_x509_adapter.py | 46 ++++++++++++++++++------------------- tox.ini | 1 + 4 files changed, 25 insertions(+), 23 deletions(-) delete mode 100644 tests/certs/test_cert.p12 diff --git a/dev-requirements.txt b/dev-requirements.txt index 0c7a35a5..202a17ac 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,3 +2,4 @@ pytest mock;python_version<"3.3" pyopenssl git+git://github.com/sigmavirus24/betamax +trustme diff --git a/tests/certs/test_cert.p12 b/tests/certs/test_cert.p12 deleted file mode 100644 index fddf5d65ebc178dcf70f7fbcf14a54323f5ab5b4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1941 zcmV;G2Wt2*f(MZT0Ru3C2Ui9ODuzgg_YDCD0ic2hNCbihL@9potn|NjWq~c zmzt?4SEWsL+D9|fh8@jN?+i0X1%*b7R3G@Ymv}cSI6SW=**8Uia?eKBQKU%hRI(GU zK}T*^w$7%SV~a{Sk%ctjY|A$-Q2pwO z#F-UE++JawQPEk0^HnX}D9qr)26+ZV6}vG)v}Dh{PE|zZJTb+>(?<>XMI4sOYefFx zacPH%al5pH`N}o>@1)SgDb&i_`!!tg9+HaoPr>g00e7o>V4{@uVh=zPY-!babriSMwL z%XabYXSylv#04=+XPlztfMig?3vHE*4gg@i@NQPvY9dkaWg<{YGeRu$VM^Qo-b_;RhX8mymf-|abxBns zZJSxH-m@$<$1)kiq=RZe@q@PB{kRw}kes?{o&YBC-iwPKiVEQFh%rNdDCoUMjb^*_ zUB!>6cy;Xluf5*@T-n%3Vn?ZG5zD-ACMbm`MJ5 z8+eg>dlO-cHM8qEzZV5~o-Z$qOCN_I6(ca6Ms!l+ZQV+pM}w9Org!PSCv0b7SE7J$ z3(TG`hT)rfQxVIQS5kJWLzJBWf$Ho6(gsY*CkRO9Gzjf~+YRZ4kFJ9prXQ5s8LXAb z&6yT&M0XQup@o^?^T0vdjpw-f$5v2$AE(e6ptP?A4!V%GE=(+?uYM=-E?C^@d^8cN z8YwiT3j&RSq5ESz1(T#2Y-t$CDbJS{9p(HYYxsbZF82~9?&46Sy5H)aHG{WxDP=$? zg=IlO2Xo9G&LNQUrwycq zGn#?Y)~Kk)xZEG?<(l-fC8J&kHhR&jJoDwl5Jm**r)3ZOik13t#cSz}wX*Y2R%SqZ zda5x1PqvMRm0lTqYW-Xi!b?VWt@TGF#Kw&zQxZf6mde>i*vRw`<1HN<9-U-QkX{mz zFG+SaFIAn4yk4np(EnEnPbZ14&H3u$twa1TDLB?EW?0yXw(|G$4r;Su3Ib_3H#Lbm zYD{yCO1ND-CLyJE2$rR3;3WoN8>+p2k z5=bP*?>gb$`C~&KQHSLjBwHQ+siBI&5np!H%#0=+JoQ!CAoON=T`OV#+@A}X)XiZN zD+W3vORY`needDdS9Bp$005cX)(@vV!yTD%eI|r@Tulk1ZeDdkOzPc|PtYWx1qoAu zi(J}7NfOj={DNQJ<&q9Wyl#|Q`xKWK%IXic09_4#bJT?w@E)KIT+cos{mR)DkmNoO zUmne6sR`?#HRQE)*`tiKZX4OKyZ~eFnyioK$G|AO2}^$G?k| bO;mB+&3k!%0s;sCkKB7X diff --git a/tests/test_x509_adapter.py b/tests/test_x509_adapter.py index bcc00b0f..728620c3 100644 --- a/tests/test_x509_adapter.py +++ b/tests/test_x509_adapter.py @@ -4,21 +4,23 @@ import pytest try: - from OpenSSL.crypto import load_pkcs12 + import OpenSSL except ImportError: PYOPENSSL_AVAILABLE = False else: PYOPENSSL_AVAILABLE = True from requests_toolbelt.adapters.x509 import X509Adapter + from cryptography import x509 from cryptography.hazmat.primitives.serialization import ( Encoding, PrivateFormat, - NoEncryption, - BestAvailableEncryption + BestAvailableEncryption, + load_pem_private_key, ) from requests_toolbelt import exceptions as exc from . import get_betamax +import trustme REQUESTS_SUPPORTS_SSL_CONTEXT = requests.__build__ >= 0x021200 @@ -27,27 +29,20 @@ class TestX509Adapter(unittest.TestCase): """Tests a simple requests.get() call using a .p12 cert. """ def setUp(self): - with open('./tests/certs/test_cert.p12', 'rb') as pkcs12_file: - self.pkcs12_data = pkcs12_file.read() - self.pkcs12_password_bytes = "test".encode('utf8') self.session = requests.Session() - @pytest.mark.xfail @pytest.mark.skipif(not REQUESTS_SUPPORTS_SSL_CONTEXT, - reason="Requires Requests v2.12.0 or later") + reason="Requires Requests v2.12.0 or later") @pytest.mark.skipif(not PYOPENSSL_AVAILABLE, - reason="Requires OpenSSL") + reason="Requires OpenSSL") def test_x509_pem(self): - p12 = load_pkcs12(self.pkcs12_data, self.pkcs12_password_bytes) - cert_bytes = p12.get_certificate().to_cryptography().public_bytes(Encoding.PEM) - pk_bytes = p12.get_privatekey().\ - to_cryptography_key().\ - private_bytes(Encoding.PEM, PrivateFormat.PKCS8, - BestAvailableEncryption(self.pkcs12_password_bytes)) + ca = trustme.CA() + cert = ca.issue_cert('pkiprojecttest01.dev.labs.internal') + cert_bytes = cert.cert_chain_pems[0].bytes() + pk_bytes = cert.private_key_pem.bytes() - adapter = X509Adapter(max_retries=3, cert_bytes=cert_bytes, - pk_bytes=pk_bytes, password=self.pkcs12_password_bytes) + adapter = X509Adapter(max_retries=3, cert_bytes=cert_bytes, pk_bytes=pk_bytes) self.session.mount('https://', adapter) recorder = get_betamax(self.session) with recorder.use_cassette('test_x509_adapter_pem'): @@ -56,16 +51,21 @@ def test_x509_pem(self): assert r.status_code == 200 assert r.text - @pytest.mark.xfail @pytest.mark.skipif(not REQUESTS_SUPPORTS_SSL_CONTEXT, reason="Requires Requests v2.12.0 or later") @pytest.mark.skipif(not PYOPENSSL_AVAILABLE, reason="Requires OpenSSL") - def test_x509_der(self): - p12 = load_pkcs12(self.pkcs12_data, self.pkcs12_password_bytes) - cert_bytes = p12.get_certificate().to_cryptography().public_bytes(Encoding.DER) - pk_bytes = p12.get_privatekey().to_cryptography_key().private_bytes(Encoding.DER, PrivateFormat.PKCS8, NoEncryption()) - adapter = X509Adapter(max_retries=3, cert_bytes=cert_bytes, pk_bytes=pk_bytes, encoding=Encoding.DER) + def test_x509_der_and_password(self): + ca = trustme.CA() + cert = ca.issue_cert('pkiprojecttest01.dev.labs.internal') + cert_bytes = x509.load_pem_x509_certificate( + cert.cert_chain_pems[0].bytes()).public_bytes(Encoding.DER) + pem_pk = load_pem_private_key(cert.private_key_pem.bytes(), password=None) + pk_bytes = pem_pk.private_bytes(Encoding.DER, PrivateFormat.PKCS8, + BestAvailableEncryption(self.pkcs12_password_bytes)) + + adapter = X509Adapter(max_retries=3, cert_bytes=cert_bytes, pk_bytes=pk_bytes, + password=self.pkcs12_password_bytes, encoding=Encoding.DER) self.session.mount('https://', adapter) recorder = get_betamax(self.session) with recorder.use_cassette('test_x509_adapter_der'): diff --git a/tox.ini b/tox.ini index be886a85..bcf47829 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ deps = pyopenssl ndg-httpsclient betamax>0.5.0 + trustme commands = py.test {posargs} From 06f105332b50233e1cc480d88d515fa934e6574c Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 24 Oct 2022 14:53:53 +0400 Subject: [PATCH 2/5] Fix noopenssl and Python 2.7 errors --- tests/test_x509_adapter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_x509_adapter.py b/tests/test_x509_adapter.py index 728620c3..203bf607 100644 --- a/tests/test_x509_adapter.py +++ b/tests/test_x509_adapter.py @@ -17,10 +17,10 @@ BestAvailableEncryption, load_pem_private_key, ) + import trustme from requests_toolbelt import exceptions as exc from . import get_betamax -import trustme REQUESTS_SUPPORTS_SSL_CONTEXT = requests.__build__ >= 0x021200 @@ -38,7 +38,7 @@ def setUp(self): reason="Requires OpenSSL") def test_x509_pem(self): ca = trustme.CA() - cert = ca.issue_cert('pkiprojecttest01.dev.labs.internal') + cert = ca.issue_cert(u'pkiprojecttest01.dev.labs.internal') cert_bytes = cert.cert_chain_pems[0].bytes() pk_bytes = cert.private_key_pem.bytes() @@ -57,7 +57,7 @@ def test_x509_pem(self): reason="Requires OpenSSL") def test_x509_der_and_password(self): ca = trustme.CA() - cert = ca.issue_cert('pkiprojecttest01.dev.labs.internal') + cert = ca.issue_cert(u'pkiprojecttest01.dev.labs.internal') cert_bytes = x509.load_pem_x509_certificate( cert.cert_chain_pems[0].bytes()).public_bytes(Encoding.DER) pem_pk = load_pem_private_key(cert.private_key_pem.bytes(), password=None) From 19b3990ff1840edd88c7cfe970034b0a20a13515 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 24 Oct 2022 15:15:59 +0400 Subject: [PATCH 3/5] Fix urllib3 warning with conditional import This will fix the warning for urllib3 1.26.x. To support urllib3 2.x we will need to vendor PyOpenSSLContext instead. --- requests_toolbelt/_compat.py | 12 ------------ requests_toolbelt/adapters/x509.py | 20 +++++++++++++++++++- tests/test_x509_adapter.py | 3 +++ tox.ini | 4 ++-- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/requests_toolbelt/_compat.py b/requests_toolbelt/_compat.py index 7927a382..21230ee6 100644 --- a/requests_toolbelt/_compat.py +++ b/requests_toolbelt/_compat.py @@ -49,18 +49,6 @@ except ImportError: from urllib3.contrib import appengine as gaecontrib -if requests.__build__ < 0x021200: - PyOpenSSLContext = None -else: - try: - from requests.packages.urllib3.contrib.pyopenssl \ - import PyOpenSSLContext - except ImportError: - try: - from urllib3.contrib.pyopenssl import PyOpenSSLContext - except ImportError: - PyOpenSSLContext = None - PY3 = sys.version_info > (3, 0) if PY3: diff --git a/requests_toolbelt/adapters/x509.py b/requests_toolbelt/adapters/x509.py index 21bfacb9..aff37706 100644 --- a/requests_toolbelt/adapters/x509.py +++ b/requests_toolbelt/adapters/x509.py @@ -18,7 +18,6 @@ from requests.adapters import HTTPAdapter import requests -from .._compat import PyOpenSSLContext from .. import exceptions as exc """ @@ -32,6 +31,9 @@ from _ssl import PROTOCOL_SSLv23 as PROTOCOL +PyOpenSSLContext = None + + class X509Adapter(HTTPAdapter): r"""Adapter for use with X.509 certificates. @@ -81,6 +83,7 @@ class X509Adapter(HTTPAdapter): """ def __init__(self, *args, **kwargs): + self._import_pyopensslcontext() self._check_version() cert_bytes = kwargs.pop('cert_bytes', None) pk_bytes = kwargs.pop('pk_bytes', None) @@ -118,6 +121,21 @@ def proxy_manager_for(self, *args, **kwargs): kwargs['ssl_context'] = self.ssl_context return super(X509Adapter, self).proxy_manager_for(*args, **kwargs) + def _import_pyopensslcontext(self): + global PyOpenSSLContext + + if requests.__build__ < 0x021200: + PyOpenSSLContext = None + else: + try: + from requests.packages.urllib3.contrib.pyopenssl \ + import PyOpenSSLContext + except ImportError: + try: + from urllib3.contrib.pyopenssl import PyOpenSSLContext + except ImportError: + PyOpenSSLContext = None + def _check_version(self): if PyOpenSSLContext is None: raise exc.VersionMismatchError( diff --git a/tests/test_x509_adapter.py b/tests/test_x509_adapter.py index 203bf607..84a10e2c 100644 --- a/tests/test_x509_adapter.py +++ b/tests/test_x509_adapter.py @@ -24,6 +24,9 @@ REQUESTS_SUPPORTS_SSL_CONTEXT = requests.__build__ >= 0x021200 +pytestmark = pytest.mark.filterwarnings( + "ignore:'urllib3.contrib.pyopenssl' module is deprecated:DeprecationWarning") + class TestX509Adapter(unittest.TestCase): """Tests a simple requests.get() call using a .p12 cert. diff --git a/tox.ini b/tox.ini index bcf47829..253c8bac 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ deps = betamax>0.5.0 trustme commands = - py.test {posargs} + pytest -W error::DeprecationWarning {posargs} [testenv:noopenssl] basepython = python3.7 @@ -31,7 +31,7 @@ deps = mock;python_version<"3.3" betamax>0.5.0 commands = - py.test {posargs} + pytest -W error::DeprecationWarning {posargs} [testenv:py27-flake8] basepython = python2.7 From 6d254251b4aaaaaf13868a73fe6167d38e253397 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 24 Oct 2022 15:30:02 +0400 Subject: [PATCH 4/5] Fix F822 flake8 error --- requests_toolbelt/_compat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/requests_toolbelt/_compat.py b/requests_toolbelt/_compat.py index 21230ee6..e813461e 100644 --- a/requests_toolbelt/_compat.py +++ b/requests_toolbelt/_compat.py @@ -308,5 +308,4 @@ def from_httplib(cls, message): # Python 2 'urlencode', 'gaecontrib', 'urljoin', - 'PyOpenSSLContext', ) From 0a3356de3bb38ef834eac756d0634410eff07313 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Mon, 24 Oct 2022 15:54:56 +0400 Subject: [PATCH 5/5] Fix tests by removing outdated test_compat.py There is no point in adapting that test since importing adapters.x509 already requires pyOpenSSL. However, the noopenssl tox configuration is still useful as we want to make sure the rest of the test suite does run without pyOpenSSL. --- tests/test_compat.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 tests/test_compat.py diff --git a/tests/test_compat.py b/tests/test_compat.py deleted file mode 100644 index dfe3016b..00000000 --- a/tests/test_compat.py +++ /dev/null @@ -1,14 +0,0 @@ -import pytest - -try: - import OpenSSL -except ImportError: - PYOPENSSL_AVAILABLE = False -else: - PYOPENSSL_AVAILABLE = True - -@pytest.mark.skipif(PYOPENSSL_AVAILABLE, - reason="Requires OpenSSL to be missing to test fallback") -def test_pyopensslcontext_is_none_when_package_missing(): - import requests_toolbelt._compat - assert requests_toolbelt._compat.PyOpenSSLContext is None