From 4ec225f1c73a6e95057921e54be9f2ec80e53b31 Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev Date: Fri, 6 Sep 2019 14:36:34 -0700 Subject: [PATCH 01/10] add support for password protected certificate files --- python2/httplib2/__init__.py | 44 ++++++++++++++++++++++++++++-------- python3/httplib2/__init__.py | 20 ++++++++++------ 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py index 2428e4ba..6f664f51 100644 --- a/python2/httplib2/__init__.py +++ b/python2/httplib2/__init__.py @@ -76,7 +76,7 @@ def _ssl_wrap_socket( - sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname + sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname, key_password ): if disable_validation: cert_reqs = ssl.CERT_NONE @@ -90,11 +90,16 @@ def _ssl_wrap_socket( context.verify_mode = cert_reqs context.check_hostname = cert_reqs != ssl.CERT_NONE if cert_file: - context.load_cert_chain(cert_file, key_file) + if key_password: + context.load_cert_chain(cert_file, key_file, key_password) + else: + context.load_cert_chain(cert_file, key_file) if ca_certs: context.load_verify_locations(ca_certs) return context.wrap_socket(sock, server_hostname=hostname) else: + if key_password: + raise NotSupportedOnThisPlatform("Certificate with password is not supported.") return ssl.wrap_socket( sock, keyfile=key_file, @@ -106,7 +111,7 @@ def _ssl_wrap_socket( def _ssl_wrap_socket_unsupported( - sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname + sock, key_file, cert_file, disable_validation, ca_certs, ssl_version, hostname, key_password ): if not disable_validation: raise CertificateValidationUnsupported( @@ -114,6 +119,8 @@ def _ssl_wrap_socket_unsupported( "the ssl module installed. To avoid this error, install " "the ssl module, or explicity disable validation." ) + if key_password: + raise NotSupportedOnThisPlatform("Certificate with password is not supported.") ssl_sock = socket.ssl(sock, key_file, cert_file) return httplib.FakeSocket(sock, ssl_sock) @@ -978,8 +985,13 @@ def iter(self, domain): class KeyCerts(Credentials): """Identical to Credentials except that name/password are mapped to key/cert.""" + def add(self, key, cert, domain, password): + self.credentials.append((domain.lower(), key, cert, password)) - pass + def iter(self, domain): + for (cdomain, key, cert, password) in self.credentials: + if cdomain == "" or domain == cdomain: + yield (key, cert, password) class AllHosts(object): @@ -1253,10 +1265,19 @@ def __init__( ca_certs=None, disable_ssl_certificate_validation=False, ssl_version=None, + key_password=None, ): - httplib.HTTPSConnection.__init__( - self, host, port=port, key_file=key_file, cert_file=cert_file, strict=strict - ) + if key_password: + httplib.HTTPSConnection.__init__(self, host, port=port, strict=strict) + self._context.load_cert_chain(cert_file, key_file, key_password) + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password + else: + httplib.HTTPSConnection.__init__( + self, host, port=port, key_file=key_file, cert_file=cert_file, strict=strict + ) + self.key_password = None self.timeout = timeout self.proxy_info = proxy_info if ca_certs is None: @@ -1366,6 +1387,7 @@ def connect(self): self.ca_certs, self.ssl_version, self.host, + self.key_password, ) if self.debuglevel > 0: print("connect: (%s, %s)" % (self.host, self.port)) @@ -1515,7 +1537,10 @@ def __init__( ca_certs=None, disable_ssl_certificate_validation=False, ssl_version=None, + key_password=None, ): + if key_password: + raise NotSupportedOnThisPlatform("Certificate with password is not supported.") httplib.HTTPSConnection.__init__( self, host, @@ -1680,10 +1705,10 @@ def add_credentials(self, name, password, domain=""): any time a request requires authentication.""" self.credentials.add(name, password, domain) - def add_certificate(self, key, cert, domain): + def add_certificate(self, key, cert, domain, password=None): """Add a key and cert that will be used any time a request requires authentication.""" - self.certificates.add(key, cert, domain) + self.certificates.add(key, cert, domain, password) def clear_credentials(self): """Remove all the names and passwords @@ -1958,6 +1983,7 @@ def request( ca_certs=self.ca_certs, disable_ssl_certificate_validation=self.disable_ssl_certificate_validation, ssl_version=self.ssl_version, + key_password = certs[0][2], ) else: conn = self.connections[conn_key] = connection_type( diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py index c6c25f8a..5269e915 100644 --- a/python3/httplib2/__init__.py +++ b/python3/httplib2/__init__.py @@ -175,7 +175,7 @@ class ProxiesUnavailableError(HttpLib2Error): def _build_ssl_context( disable_ssl_certificate_validation, ca_certs, cert_file=None, key_file=None, - maximum_version=None, minimum_version=None, + maximum_version=None, minimum_version=None, key_password=None, ): if not hasattr(ssl, "SSLContext"): raise RuntimeError("httplib2 requires Python 3.2+ for ssl.SSLContext") @@ -207,7 +207,7 @@ def _build_ssl_context( context.load_verify_locations(ca_certs) if cert_file: - context.load_cert_chain(cert_file, key_file) + context.load_cert_chain(cert_file, key_file, key_password) return context @@ -959,8 +959,13 @@ def iter(self, domain): class KeyCerts(Credentials): """Identical to Credentials except that name/password are mapped to key/cert.""" + def add(self, key, cert, domain, password): + self.credentials.append((domain.lower(), key, cert, password)) - pass + def iter(self, domain): + for (cdomain, key, cert, password) in self.credentials: + if cdomain == "" or domain == cdomain: + yield (key, cert, password) class AllHosts(object): @@ -1241,6 +1246,7 @@ def __init__( disable_ssl_certificate_validation=False, tls_maximum_version=None, tls_minimum_version=None, + key_password=None, ): self.disable_ssl_certificate_validation = disable_ssl_certificate_validation @@ -1253,12 +1259,11 @@ def __init__( context = _build_ssl_context( self.disable_ssl_certificate_validation, self.ca_certs, cert_file, key_file, maximum_version=tls_maximum_version, minimum_version=tls_minimum_version, + key_password=key_password, ) super(HTTPSConnectionWithTimeout, self).__init__( host, port=port, - key_file=key_file, - cert_file=cert_file, timeout=timeout, context=context, ) @@ -1503,10 +1508,10 @@ def add_credentials(self, name, password, domain=""): any time a request requires authentication.""" self.credentials.add(name, password, domain) - def add_certificate(self, key, cert, domain): + def add_certificate(self, key, cert, domain, password=None): """Add a key and cert that will be used any time a request requires authentication.""" - self.certificates.add(key, cert, domain) + self.certificates.add(key, cert, domain, password) def clear_credentials(self): """Remove all the names and passwords @@ -1778,6 +1783,7 @@ def request( disable_ssl_certificate_validation=self.disable_ssl_certificate_validation, tls_maximum_version=self.tls_maximum_version, tls_minimum_version=self.tls_minimum_version, + key_password=certs[0][2], ) else: conn = self.connections[conn_key] = connection_type( From 292af5e745dfd8e9c142f15ec188f58290e16269 Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev Date: Tue, 17 Sep 2019 11:08:48 -0700 Subject: [PATCH 02/10] added unit test --- python3/httplib2/__init__.py | 3 +++ tests/test_external.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py index 5269e915..b7aef3c0 100644 --- a/python3/httplib2/__init__.py +++ b/python3/httplib2/__init__.py @@ -1267,6 +1267,9 @@ def __init__( timeout=timeout, context=context, ) + self.key_file = key_file + self.cert_file = cert_file + self.key_password = key_password def connect(self): """Connect to a host on a given (SSL) port.""" diff --git a/tests/test_external.py b/tests/test_external.py index 0628d96a..11895f0b 100644 --- a/tests/test_external.py +++ b/tests/test_external.py @@ -61,6 +61,32 @@ def test_get_via_https_key_cert(): # Skip on 3.2 pass +def test_get_via_https_key_cert_password(): + # At this point I can only test + # that the key and cert files are passed in + # correctly to httplib. It would be nice to have + # a real https endpoint to test against. + http = httplib2.Http(timeout=2) + http.add_certificate("akeyfile", "acertfile", "bitworking.org", "apassword") + try: + http.request("https://bitworking.org", "GET") + except AttributeError: + assert http.connections["https:bitworking.org"].key_file == "akeyfile" + assert http.connections["https:bitworking.org"].cert_file == "acertfile" + assert http.connections["https:bitworking.org"].key_password == "apassword" + except IOError: + # Skip on 3.2 + pass + + try: + http.request("https://notthere.bitworking.org", "GET") + except httplib2.ServerNotFoundError: + assert http.connections["https:notthere.bitworking.org"].key_file is None + assert http.connections["https:notthere.bitworking.org"].cert_file is None + assert http.connections["https:notthere.bitworking.org"].key_password is None + except IOError: + # Skip on 3.2 + pass def test_ssl_invalid_ca_certs_path(): # Test that we get an ssl.SSLError when specifying a non-existent CA From 14d8af299b47495fb46a7c679c9a2df824242d22 Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev Date: Tue, 17 Sep 2019 12:12:38 -0700 Subject: [PATCH 03/10] added blank lines to fix lint error --- tests/test_external.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_external.py b/tests/test_external.py index 11895f0b..6da73bba 100644 --- a/tests/test_external.py +++ b/tests/test_external.py @@ -61,6 +61,7 @@ def test_get_via_https_key_cert(): # Skip on 3.2 pass + def test_get_via_https_key_cert_password(): # At this point I can only test # that the key and cert files are passed in @@ -88,6 +89,7 @@ def test_get_via_https_key_cert_password(): # Skip on 3.2 pass + def test_ssl_invalid_ca_certs_path(): # Test that we get an ssl.SSLError when specifying a non-existent CA # certs file. From 92c567b59050791bf98854ae22e95fb0d348b3a3 Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev Date: Wed, 18 Sep 2019 10:36:47 -0700 Subject: [PATCH 04/10] added unit test which loads the actual protected pem file --- tests/test_external.py | 17 ++++++++++++ tests/testdata/test_cert.pem | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 tests/testdata/test_cert.pem diff --git a/tests/test_external.py b/tests/test_external.py index 6da73bba..9ca4654a 100644 --- a/tests/test_external.py +++ b/tests/test_external.py @@ -90,6 +90,23 @@ def test_get_via_https_key_cert_password(): pass +def test_get_via_https_key_cert_password_with_pem(): + # At this point I can only test + # that the key and cert files are passed in + # correctly to httplib. It would be nice to have + # a real https endpoint to test against. + cert_filename = "tests/testdata/test_cert.pem" # password - 12345 + http = httplib2.Http(timeout=2) + http.add_certificate(cert_filename, cert_filename, "bitworking.org", "12345") + http.request("https://bitworking.org", "GET") + + # try invalid password + http = httplib2.Http(timeout=2) + http.add_certificate(cert_filename, cert_filename, "bitworking.org", "invalid") + with tests.assert_raises(ssl.SSLError): + http.request("https://bitworking.org", "GET") + + def test_ssl_invalid_ca_certs_path(): # Test that we get an ssl.SSLError when specifying a non-existent CA # certs file. diff --git a/tests/testdata/test_cert.pem b/tests/testdata/test_cert.pem new file mode 100644 index 00000000..94fdf736 --- /dev/null +++ b/tests/testdata/test_cert.pem @@ -0,0 +1,54 @@ +-----BEGIN CERTIFICATE----- +MIIECTCCAvGgAwIBAgIUXsxopvicqhbQMsg4zN3H5XcmTNswDQYJKoZIhvcNAQEL +BQAwgZMxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEYMBYGA1UEBwwPQ2FsaWZv +cm5pYSBDaXR5MREwDwYDVQQKDAhodHRwbGliMjERMA8GA1UECwwIaHR0cGxpYjIx +FTATBgNVBAMMDGh0dHBsaWIydGVzdDEgMB4GCSqGSIb3DQEJARYRaHR0cGxpYjJA +aHR0cGxpYjIwHhcNMTkwOTE4MTcxNzIxWhcNMTkxMDE4MTcxNzIxWjCBkzELMAkG +A1UEBhMCVVMxCzAJBgNVBAgMAkNBMRgwFgYDVQQHDA9DYWxpZm9ybmlhIENpdHkx +ETAPBgNVBAoMCGh0dHBsaWIyMREwDwYDVQQLDAhodHRwbGliMjEVMBMGA1UEAwwM +aHR0cGxpYjJ0ZXN0MSAwHgYJKoZIhvcNAQkBFhFodHRwbGliMkBodHRwbGliMjCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALrW/SVFlOPsAlKBbE/6vmAZ +cMRhbD5K0LoRrkCEp9FQnuJo5wBX0kv2A+JCiuhb60AQLWVTfSLhTuiVIFvWvE/z +CUFhve+qCi2eDAK7yvDQLHfnehT2FOaZ8DWnfYmrmx6kx+xMiK3XgGQeRhNHmOVk +PvX597/8OiDMB1mhS7GSBOFXcHoe+I2rswR7VTF0YO4L+6GDiApd8poXTnan1eZC +drJLG5FrwdwU57DAbUbI8BoPIFny//pICVFtR031FKXmhzklDhL/TT5RV7wa1jxF +Ists1GzDQFqDeK4acQxJD5gRuSa1XEtKWw9pcOIAIdAdIiHYdKWLgCC3jdpT1EEC +AwEAAaNTMFEwHQYDVR0OBBYEFALS7wgbswFYCCuNFjJuA8NfbKpSMB8GA1UdIwQY +MBaAFALS7wgbswFYCCuNFjJuA8NfbKpSMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBACU0PgSC01F9y29gEmHgFCDGZO+vcnRXEE7mcsD0+zi4F11C +xGFT+PQ68XGXkYxE6DiOzF8JhIu2jJBMu9CPDncoKJCWxu/kjm/7fOIkh4Y2VCSe +d2CPvCvYl78ni2YmW4RrFdYavSiI4M/CE0BaIgow9mn8j0Naj3l6UTuaKTKAlRtZ +pqMjFnOAzqOpZRoFw2UJns2JvWUFwN7lfMoBG0S0V6UxEKcD5LTiXDPCrLUsZUGH +ooGuk5epzynZOyl+eOqvm8Kzv0+TdniTgG6hKqwsai3OVvaplISchUSwZefranBK +dbUgpZkbzbS8Y3k0P0z9rqE7WzHoNX0YQOyKEwM= +-----END CERTIFICATE----- +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFHDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQItwJvPF46vkACAggA +MAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECAYxKTBIQWE/BIIEyK9Ty3nRHc0y +iB8FlUXH2ussnLiNnrigWbZ6wOSEnwpmk9qUI/7UXgl3kg/JpW29a47oytLOV9Ii +lnENu/yQRycoQSZveHv8/qquDqTyD4u7RRsxHJSYK2ltQtNxtsm9GidCULGoXb7j +wIewcFut7ZOI0jI7Z7MmEYObI2Vn0Tgp91T/+uCihQPF/qolPz5haB7F7XOL2nSb +mvWOZi0eq+n2gJKTLH/6A165lQFby2nulLNK7IlX8OeU5hSlGbvOOPFMkrCRp1oU +hwPzh41lV6uezxijRYiT9DFplTMdy6/S80MqdE9GOSFrkhOlrfXkIFPpoyi6fCDB +kmIqKXGzKMnvXl3ZowR0a8wyOQt51wOYhYWnCu38JU9iwcPWHSQnKf3Bu4kIJe7j +xvIzHv56f5doBZ9UI8s9aEGBrtLy3Iv0F27nzsR9QWFXPjWQIyyBlwOwYn2AqcMc +HYJOmRzbq0/VbuRFaLht7p0JqH2xb4szf4XodjtEV3XRVi0GL5fObnz/ogjcQ0D8 +zIQ7VNFMXN7NLeVrT9sdbnP9ljb8GfJe0RRVp/h0Xff+2q68QLO5dmDNbHV9B4as +j6EQRkwODu6lEcLMdjucFDbi8vJWsUE2ire5gZlByj2d8v1bcMhxxu5QoOPxU0UT +w96+HzsbdTttYPJDE1fFhITHNdR3dkeSxKeHs1cLf8kSBHI1IrzbUYPv7cEkGkNv +sPaaDi9SCE/3tXaF/7wbxLZ42MOUCWqG4KjlHKI3Gga3QlXDVO4zUWVvSSzITHE7 +EnYHOwr7RUVd9TgPgjKdkQtiF/9UlJ/yV41s/R6EbpwkrHL7EP/U/60MwZnYKIYr +MLbd3xdoVBA2w1BSv7rWo06o+dxOQGWg4i2/CGhbVsHoLmBCDyFSDrAzryT2EG0V +lntL+mW/7Ri7frbCXSW66v080oW5aNy/rzBeMZacIrPn8daUfBbtPRhx59LgBapf +4ZFhEVL09g10DMku68po6+6GSAfPVivzWWrsFoSI0QPT0tRaHqf7nwpvNa+Mu+Fq ++PRSPwnc8YVWFzGnCTVMFfbgd33BYesdsK+sNj0bwGsLjtVxi2L2wWVvKIQB0iwT +a5BAk67MXAFZCEn8XrEKwSjZYoVfrKFmifBBJi2Ca/7zJtkGxETqqTGdyb+fwlWU +HiF98QElpcIzjtoFJal/bFmbQtQSJRm2j6umpbpOmU4fzt2CzsqJNkvLhC00edfj +bfAK0MxdFFO2KfyVxaj77bw+oERfFfje6c23c1lxOp7M0D7XIevPKzEKNezhNpOh +cjOfBRgmbnUr/PtYB2lhFNnO/mscpBdCZYUlTRrVpm7zYLNxr1GZmn8l38L0EMVh +FjJRM2NDqW3e6E84ZXW7WN4xeaznGnw+NZnyi4IFe9oWJLKanR9eZzox5I0ZLTte +knsDTJDEV+nSgqj6Fda68cF71Jueo9HfdNV01muxdJ/w+LRAnYwintKn3YwTnQ+7 +s6grEIHsN9XDZk4OP9DMDWuiAKSr7JAFttc5B6xL5uuxbjK9/ZfslObSqM2Zu0ea +3qKVOZAMnTiXLbPup2ybwgNp2JyaTEDUKGLl76fBKgHK4BNicviDUXumDVtmN+KW +fz35Jam1Q4W0vTSwSuEzig== +-----END ENCRYPTED PRIVATE KEY----- From f9fd2b77d1eed1276d0b8c19a5dd63620db19b85 Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev Date: Tue, 24 Sep 2019 15:47:26 -0700 Subject: [PATCH 05/10] Add mock local server and add cert test using it --- tests/__init__.py | 63 +++++++++++++++++++++++++++++ tests/test_external.py | 19 ++++++++- tests/testdata/test_server_cert.pem | 52 ++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 tests/testdata/test_server_cert.pem diff --git a/tests/__init__.py b/tests/__init__.py index 28959d39..d84f7896 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -8,12 +8,14 @@ import gzip import hashlib import httplib2 +from http.server import BaseHTTPRequestHandler, HTTPServer import os import random import re import shutil import six import socket +import ssl import struct import sys import threading @@ -23,6 +25,12 @@ from six.moves import http_client, queue +SERVER_CERTFILE = 'tests/testdata/test_server_cert.pem' +CLIENT_CERTFILE = 'tests/testdata/test_cert.pem' +CLIENT_CERT_PASSWORD = '12345' +CLIENT_CERT_SERIAL = '5ECC68A6F89CAA16D032C838CCDDC7E577264CDB' + + @contextlib.contextmanager def assert_raises(exc_type): def _name(t): @@ -260,6 +268,61 @@ def getresponse(self): raise http_client.BadStatusLine("") +def _get_free_port(): + s = socket.socket(socket.AF_INET, type=socket.SOCK_STREAM) + s.bind(('localhost', 0)) + address, port = s.getsockname() + s.close() + return port + + +class _MockServerRequestHandler(BaseHTTPRequestHandler): + """Server request handler which always returns 200 and saves client cert info.""" + def do_GET(self): + # save client cert + self.server.last_client_cert = self.connection.getpeercert() + # Process an HTTP GET request and return a response with an HTTP 200 status. + self.send_response(200) + self.end_headers() + return + + +class MockHttpServer(): + """This creates local http server in a separate thread.""" + def __init__(self, handler=None, port=0, ssl=False): + self.handler = handler if handler else _MockServerRequestHandler + self.port = port if port else _get_free_port() + self.ssl = ssl + self.client_certfile = CLIENT_CERTFILE + self.certfile = SERVER_CERTFILE + + def __enter__(self): + self.server = HTTPServer(('localhost', self.port), self.handler) + + # wrap socket when SSL server requested + if self.ssl: + context = ssl.SSLContext(ssl.PROTOCOL_TLS) + # ask client to present own cert for mutual auth + context.verify_mode = ssl.CERT_OPTIONAL + if self.client_certfile: + # avoid verification failure by preloading matching client cert + context.load_verify_locations(self.client_certfile) + # load server cert + context.load_cert_chain(self.certfile) + self.server.socket = context.wrap_socket( + sock=self.server.socket, server_side=True) + + # Start running mock server in a separate thread. + # Daemon threads automatically shut down when the main process exits. + server_thread = threading.Thread(target=self.server.serve_forever) + server_thread.setDaemon(True) + server_thread.start() + return self + + def __exit__(self, type, value, traceback): + self.server.shutdown() + + @contextlib.contextmanager def server_socket(fun, request_count=1, timeout=5): gresult = [None] diff --git a/tests/test_external.py b/tests/test_external.py index 9ca4654a..ca2713bc 100644 --- a/tests/test_external.py +++ b/tests/test_external.py @@ -97,16 +97,31 @@ def test_get_via_https_key_cert_password_with_pem(): # a real https endpoint to test against. cert_filename = "tests/testdata/test_cert.pem" # password - 12345 http = httplib2.Http(timeout=2) - http.add_certificate(cert_filename, cert_filename, "bitworking.org", "12345") + http.add_certificate(tests.CLIENT_CERTFILE, tests.CLIENT_CERTFILE, + "bitworking.org", tests.CLIENT_CERT_PASSWORD) http.request("https://bitworking.org", "GET") # try invalid password http = httplib2.Http(timeout=2) - http.add_certificate(cert_filename, cert_filename, "bitworking.org", "invalid") + http.add_certificate(tests.CLIENT_CERTFILE, tests.CLIENT_CERTFILE, + "bitworking.org", "invalid") with tests.assert_raises(ssl.SSLError): http.request("https://bitworking.org", "GET") +def test_get_via_https_key_cert_password_with_pem_local_server(): + with tests.MockHttpServer(ssl=True) as server: + # load matching server cert to avoid verification failure + http = httplib2.Http(ca_certs=server.certfile) + # load client cert to be presented when server asks for it + http.add_certificate(tests.CLIENT_CERTFILE, tests.CLIENT_CERTFILE, + '', tests.CLIENT_CERT_PASSWORD) + url = 'https://localhost:{port}/'.format(port=server.port) + http.request(url, "GET") + # verify that client cert was presented with matching serial number + assert server.server.last_client_cert['serialNumber'] == tests.CLIENT_CERT_SERIAL + + def test_ssl_invalid_ca_certs_path(): # Test that we get an ssl.SSLError when specifying a non-existent CA # certs file. diff --git a/tests/testdata/test_server_cert.pem b/tests/testdata/test_server_cert.pem new file mode 100644 index 00000000..cf3d6287 --- /dev/null +++ b/tests/testdata/test_server_cert.pem @@ -0,0 +1,52 @@ +-----BEGIN CERTIFICATE----- +MIIEAzCCAuugAwIBAgIUTpOojxoGQvm3rv9wFXjeiJx1Hg4wDQYJKoZIhvcNAQEL +BQAwgZAxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEYMBYGA1UEBwwPQ2FsaWZv +cm5pYSBDaXR5MREwDwYDVQQKDAhodHRwbGliMjERMA8GA1UECwwIaHR0cGxpYjIx +EjAQBgNVBAMMCWxvY2FsaG9zdDEgMB4GCSqGSIb3DQEJARYRaHR0cGxpYjJAaHR0 +cGxpYjIwHhcNMTkwOTI0MTczNzE3WhcNMjkwOTIxMTczNzE3WjCBkDELMAkGA1UE +BhMCVVMxCzAJBgNVBAgMAkNBMRgwFgYDVQQHDA9DYWxpZm9ybmlhIENpdHkxETAP +BgNVBAoMCGh0dHBsaWIyMREwDwYDVQQLDAhodHRwbGliMjESMBAGA1UEAwwJbG9j +YWxob3N0MSAwHgYJKoZIhvcNAQkBFhFodHRwbGliMkBodHRwbGliMjCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAMd06x0H1gtV/ReIapewXWOXr+FI/mt8 +Hl73B6mgcZc/7CXiAQnAFmKdpoinSi8UETK71zQyI5wn6Hn5HNMZbhGxRp9/h5FG +5BCx5AQ1fZNRIgICM98MVOjdubc32mxRqOPtISO4Nxe3JxDgajfqB9tlXnV62ns5 +TMemxM2BJRS9cUGvO8RFU+VA+A04xPTZ2Z4ADbTbPVj+CfWYyc5ss1jqdK29+u8x +RlB3jUUorJ/xFK4+6TWVBL3FwGco0swN5CAamVsxo37PrJ0QQUwt2SSKNkGhBY5l +MSVvpTdgfIo02oIhxL/n+rGc4k7ZeM0HBJ7qpNaO9dikR1lcyqb0M2cCAwEAAaNT +MFEwHQYDVR0OBBYEFPdH4Ckg1zf0+HdS3mlLn7tEIO5IMB8GA1UdIwQYMBaAFPdH +4Ckg1zf0+HdS3mlLn7tEIO5IMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBAGq2S/ul1B0HLlA/Amc2psOX2PuK17XW2+HEB6Is/TRyMlC5uADjMpyu +3Wt6FpJe4cjCMdeQO/nasiLqgZKWVDKa7g2EW3zDBwLcytc+Ll59cIUcHWuc9Fli +uLNRo7bAW9V84W47v6flP86JO//8wdtWsp6a5RlB86ciNhxvQF5Eowavo1FTOwmE +kegZB99AEiCELOixOJPVmjwtJTlMLlvA94cbn5RTdIFpF3RIj/13lA737TpMt/ir +tzE9hZiy8MCWj9lP2wRUf8W1SIhTzyJjLuu6TGK3tLULWADOGu4Ev1gZYSDqrMgq +UHgWvCxvg8ADt2+m8cCPSsJRhJHR1Qo= +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDHdOsdB9YLVf0X +iGqXsF1jl6/hSP5rfB5e9wepoHGXP+wl4gEJwBZinaaIp0ovFBEyu9c0MiOcJ+h5 ++RzTGW4RsUaff4eRRuQQseQENX2TUSICAjPfDFTo3bm3N9psUajj7SEjuDcXtycQ +4Go36gfbZV51etp7OUzHpsTNgSUUvXFBrzvERVPlQPgNOMT02dmeAA202z1Y/gn1 +mMnObLNY6nStvfrvMUZQd41FKKyf8RSuPuk1lQS9xcBnKNLMDeQgGplbMaN+z6yd +EEFMLdkkijZBoQWOZTElb6U3YHyKNNqCIcS/5/qxnOJO2XjNBwSe6qTWjvXYpEdZ +XMqm9DNnAgMBAAECggEAZkiG6XRDR7zo9VB3mSJQQepfv1/3WJdl271CqLImjjGx +oBut/JoM4LWH2TwwxsO7rhC144ZyzHmKtkIRjg/Uai5G0TRNCSsZeZZAIAhuRUWt +8um+P/vK70AFJPf3guS9f2TzQaYOWpMJ7ZWn4tSZAuGQ9x3xPI+92ASll91Kbr9u +fQaUp/Pg2ALV1oF6mykuaeaswT2AxAHNNBHkeWktiqar1s34pnmIYb3hbQqxb9HV +WsbW0VBNnU1UvbPlhZP4W51n6+t6nOLIQSGTiYepHpCZmrYmyvAxM1NGq7mGWO9E +nlvHLPbMD3Tm3NrcccAGu++mFBcV37L4MlCaI3hwQQKBgQD6lxjFeNcuiovi9iDy +yLlfqCieK1QIS7FOHfKL0sn/qQhredNhYVBWWP4USaAXEMtWaEZ7d2SXaT4n6w7f +65b7WX2X5SMrwv8w45zhNXkX9yevscojsubH4xzHW6thauLM90ALjNS7tPtGkH+D +bqGmMz03b4t3qFCFFcO1KO1HFQKBgQDLwzqdq/OnnCBpDWi87j7+iaNYE6g4ZFxD +gZyrtPcrdjQhea0tuKPygEsTIhNpivqPUOhEm9OwE03AAqW3NRiIQN8oRLtcEKjl +PS82FC7qM7WVKWeZJnSB+RpsRxQXoVG0tpI34jnfcxZRNAJhf5Kz8nv9y/LH7tAD +D8tyFjrviwKBgQCDExW59QNZLM8O4H9LfwK3rlXQpglGbZFIsxFzYcaXG+tzjD2s +6iIDiHkeU4SRjA9QGysC2eib6kjAyIr3RVusDZtMIGbNNSoWgHhGtJmql3UCyZRa +J/HfDES5YpG6WxZW791oLTn5FSl6N4r7TJrxPEwA+y+QX1H+yuubjtTOIQKBgQDC +ZEd4gsJaJhW5g0Rn8jcA6Nh/v4kd+4kWEgIgwe2IdiV3xjhURTGLuZ9l6n1wlFlD +/uEIC02iTlg/lYb5SNtVqeX76c6BH5ex03RF+G1lm91hJ3YhYtGF6duubwUZIhrr +9715OQcTSR2CbMbUszuHFw/5aef9m7SxJxFljxW8zwKBgEaU2usiq9QDHZ+q8eol +3YZBRqp7igi2ljsHk83izasSa6j0rj2mxZPHdephRkar6VgMfRqmYr8UE9PXe08t +QP4tHypR+0n2+9GMmn1zm6QbLeQwkl4A2HDGush8pXPiMxF5IIyGVbajJyuppw8L +8G4XioXDK+R3Vk9m0sJA6BLh +-----END PRIVATE KEY----- From 01955b8fd322457ea506fe128618bfeaf6c73285 Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev Date: Tue, 24 Sep 2019 15:58:05 -0700 Subject: [PATCH 06/10] fix lint errors --- tests/__init__.py | 6 +++--- tests/test_http.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index d84f7896..b5712430 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -289,10 +289,10 @@ def do_GET(self): class MockHttpServer(): """This creates local http server in a separate thread.""" - def __init__(self, handler=None, port=0, ssl=False): + def __init__(self, handler=None, port=0, use_ssl=False): self.handler = handler if handler else _MockServerRequestHandler self.port = port if port else _get_free_port() - self.ssl = ssl + self.use_ssl = use_ssl self.client_certfile = CLIENT_CERTFILE self.certfile = SERVER_CERTFILE @@ -300,7 +300,7 @@ def __enter__(self): self.server = HTTPServer(('localhost', self.port), self.handler) # wrap socket when SSL server requested - if self.ssl: + if self.use_ssl: context = ssl.SSLContext(ssl.PROTOCOL_TLS) # ask client to present own cert for mutual auth context.verify_mode = ssl.CERT_OPTIONAL diff --git a/tests/test_http.py b/tests/test_http.py index 9bd9ee0e..ce0cd3af 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -8,6 +8,7 @@ import mock import os import pytest +import six from six.moves import http_client, urllib import socket import ssl @@ -70,10 +71,10 @@ def test_connection_refused_raises_exception(mock_socket_connect): mock_socket_connect.side_effect = _raise_connection_refused_exception http = httplib2.Http() http.force_exception_to_status_code = False - with tests.assert_raises(socket.error): + E = http_client.ResponseNotReady if six.PY2 else socket.error + with tests.assert_raises(E): http.request(DUMMY_URL) - @pytest.mark.skipif( os.environ.get("TRAVIS_PYTHON_VERSION") in ("2.7", "pypy"), reason="Fails on Travis py27/pypy, works elsewhere. " @@ -91,6 +92,7 @@ def test_connection_refused_returns_response(mock_socket_connect): b"connection refused" in content or b"actively refused" in content or b"socket is not connected" in content + or not content ) assert response.status == 400 From cb80a858457376ef79814d1858f48b2de201465e Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev Date: Tue, 24 Sep 2019 16:10:07 -0700 Subject: [PATCH 07/10] add missing file --- tests/test_external.py | 6 +++--- tests/test_http.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/test_external.py b/tests/test_external.py index ca2713bc..f3e0b6d1 100644 --- a/tests/test_external.py +++ b/tests/test_external.py @@ -95,7 +95,6 @@ def test_get_via_https_key_cert_password_with_pem(): # that the key and cert files are passed in # correctly to httplib. It would be nice to have # a real https endpoint to test against. - cert_filename = "tests/testdata/test_cert.pem" # password - 12345 http = httplib2.Http(timeout=2) http.add_certificate(tests.CLIENT_CERTFILE, tests.CLIENT_CERTFILE, "bitworking.org", tests.CLIENT_CERT_PASSWORD) @@ -110,14 +109,15 @@ def test_get_via_https_key_cert_password_with_pem(): def test_get_via_https_key_cert_password_with_pem_local_server(): - with tests.MockHttpServer(ssl=True) as server: + with tests.MockHttpServer(use_ssl=True) as server: # load matching server cert to avoid verification failure http = httplib2.Http(ca_certs=server.certfile) # load client cert to be presented when server asks for it http.add_certificate(tests.CLIENT_CERTFILE, tests.CLIENT_CERTFILE, '', tests.CLIENT_CERT_PASSWORD) url = 'https://localhost:{port}/'.format(port=server.port) - http.request(url, "GET") + response, content = http.request(url, "GET") + assert response.status == 200 # verify that client cert was presented with matching serial number assert server.server.last_client_cert['serialNumber'] == tests.CLIENT_CERT_SERIAL diff --git a/tests/test_http.py b/tests/test_http.py index ce0cd3af..9bd9ee0e 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -8,7 +8,6 @@ import mock import os import pytest -import six from six.moves import http_client, urllib import socket import ssl @@ -71,10 +70,10 @@ def test_connection_refused_raises_exception(mock_socket_connect): mock_socket_connect.side_effect = _raise_connection_refused_exception http = httplib2.Http() http.force_exception_to_status_code = False - E = http_client.ResponseNotReady if six.PY2 else socket.error - with tests.assert_raises(E): + with tests.assert_raises(socket.error): http.request(DUMMY_URL) + @pytest.mark.skipif( os.environ.get("TRAVIS_PYTHON_VERSION") in ("2.7", "pypy"), reason="Fails on Travis py27/pypy, works elsewhere. " @@ -92,7 +91,6 @@ def test_connection_refused_returns_response(mock_socket_connect): b"connection refused" in content or b"actively refused" in content or b"socket is not connected" in content - or not content ) assert response.status == 400 From d1ded21fb2b6070166aa4118662e29bb10b2b497 Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev Date: Wed, 25 Sep 2019 13:03:15 -0700 Subject: [PATCH 08/10] changed tests to not use external server --- tests/__init__.py | 9 +++---- tests/test_external.py | 54 +++++++++++++++++------------------------- 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index b5712430..14229bea 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -270,7 +270,7 @@ def getresponse(self): def _get_free_port(): s = socket.socket(socket.AF_INET, type=socket.SOCK_STREAM) - s.bind(('localhost', 0)) + s.bind(("localhost", 0)) address, port = s.getsockname() s.close() return port @@ -297,8 +297,7 @@ def __init__(self, handler=None, port=0, use_ssl=False): self.certfile = SERVER_CERTFILE def __enter__(self): - self.server = HTTPServer(('localhost', self.port), self.handler) - + self.server = HTTPServer(("localhost", self.port), self.handler) # wrap socket when SSL server requested if self.use_ssl: context = ssl.SSLContext(ssl.PROTOCOL_TLS) @@ -311,7 +310,9 @@ def __enter__(self): context.load_cert_chain(self.certfile) self.server.socket = context.wrap_socket( sock=self.server.socket, server_side=True) - + self.url = "https://localhost:{port}/".format(port=self.port) + else: + self.url = "http://localhost:{port}/".format(port=self.port) # Start running mock server in a separate thread. # Daemon threads automatically shut down when the main process exits. server_thread = threading.Thread(target=self.server.serve_forever) diff --git a/tests/test_external.py b/tests/test_external.py index f3e0b6d1..cbd7cdf9 100644 --- a/tests/test_external.py +++ b/tests/test_external.py @@ -68,58 +68,48 @@ def test_get_via_https_key_cert_password(): # correctly to httplib. It would be nice to have # a real https endpoint to test against. http = httplib2.Http(timeout=2) - http.add_certificate("akeyfile", "acertfile", "bitworking.org", "apassword") + http.add_certificate("akeyfile", "acertfile", "", "apassword") try: - http.request("https://bitworking.org", "GET") + with tests.MockHttpServer(use_ssl=True) as server: + http.request(server.url, "GET") except AttributeError: - assert http.connections["https:bitworking.org"].key_file == "akeyfile" - assert http.connections["https:bitworking.org"].cert_file == "acertfile" - assert http.connections["https:bitworking.org"].key_password == "apassword" + assert http.connections["https:localhost"].key_file == "akeyfile" + assert http.connections["https:localhost"].cert_file == "acertfile" + assert http.connections["https:localhost"].key_password == "apassword" except IOError: - # Skip on 3.2 + # Catch 'No such file or directory' since filenames are fake pass try: - http.request("https://notthere.bitworking.org", "GET") + http.request("https://notthere", "GET") except httplib2.ServerNotFoundError: - assert http.connections["https:notthere.bitworking.org"].key_file is None - assert http.connections["https:notthere.bitworking.org"].cert_file is None - assert http.connections["https:notthere.bitworking.org"].key_password is None + assert http.connections["https:notthere"].key_file is None + assert http.connections["https:notthere"].cert_file is None + assert http.connections["https:notthere"].key_password is None except IOError: - # Skip on 3.2 + # Catch 'No such file or directory' since filenames are fake pass def test_get_via_https_key_cert_password_with_pem(): - # At this point I can only test - # that the key and cert files are passed in - # correctly to httplib. It would be nice to have - # a real https endpoint to test against. - http = httplib2.Http(timeout=2) - http.add_certificate(tests.CLIENT_CERTFILE, tests.CLIENT_CERTFILE, - "bitworking.org", tests.CLIENT_CERT_PASSWORD) - http.request("https://bitworking.org", "GET") - - # try invalid password - http = httplib2.Http(timeout=2) - http.add_certificate(tests.CLIENT_CERTFILE, tests.CLIENT_CERTFILE, - "bitworking.org", "invalid") - with tests.assert_raises(ssl.SSLError): - http.request("https://bitworking.org", "GET") - - -def test_get_via_https_key_cert_password_with_pem_local_server(): with tests.MockHttpServer(use_ssl=True) as server: # load matching server cert to avoid verification failure http = httplib2.Http(ca_certs=server.certfile) # load client cert to be presented when server asks for it http.add_certificate(tests.CLIENT_CERTFILE, tests.CLIENT_CERTFILE, '', tests.CLIENT_CERT_PASSWORD) - url = 'https://localhost:{port}/'.format(port=server.port) - response, content = http.request(url, "GET") + response, content = http.request(server.url, "GET") assert response.status == 200 # verify that client cert was presented with matching serial number - assert server.server.last_client_cert['serialNumber'] == tests.CLIENT_CERT_SERIAL + assert server.server.last_client_cert["serialNumber"] == tests.CLIENT_CERT_SERIAL + + # try invalid password + http = httplib2.Http(ca_certs=server.certfile) + # load client cert to be presented when server asks for it + http.add_certificate(tests.CLIENT_CERTFILE, tests.CLIENT_CERTFILE, + "", "invalid") + with tests.assert_raises(ssl.SSLError): + http.request(server.url, "GET") def test_ssl_invalid_ca_certs_path(): From 2f0dbda5c033573254b410fb0c861664f606e0ec Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev <54158304+kkoshelev-g@users.noreply.github.com> Date: Tue, 15 Oct 2019 16:50:51 -0700 Subject: [PATCH 09/10] Delete settings.json --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 96d9064c..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "venv-36/bin/python3.6" -} \ No newline at end of file From d5fa9381a37a71289feabf2900559c62d7e36b1a Mon Sep 17 00:00:00 2001 From: Konstantin Koshelev Date: Wed, 16 Oct 2019 13:16:52 -0700 Subject: [PATCH 10/10] Fix lint error --- python2/httplib2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py index b1f1545a..d807bc1f 100644 --- a/python2/httplib2/__init__.py +++ b/python2/httplib2/__init__.py @@ -1983,7 +1983,7 @@ def request( ca_certs=self.ca_certs, disable_ssl_certificate_validation=self.disable_ssl_certificate_validation, ssl_version=self.ssl_version, - key_password = certs[0][2], + key_password=certs[0][2], ) else: conn = self.connections[conn_key] = connection_type(