From 4ff1d936befb0d5bea7d210b9188b1dd5fb806dc Mon Sep 17 00:00:00 2001 From: Jorge Lopez Silva Date: Mon, 17 Aug 2020 17:24:10 -0700 Subject: [PATCH] Integrate SSLTransport into urllib3. For connections that will attempt to use an HTTPS proxy with an HTTPS destination, we'll use the TLS in TLS support provided by SSL Transport. HTTPS proxy and HTTP destinations will continue using a single TLS session as expected. We'll still support the use of forwarding for HTTPS destinations with HTTPS proxies as long as the "use_forwarding_for_https" parameter is provided. Signed-off-by: Jorge Lopez Silva --- src/urllib3/connection.py | 52 +++++++++ src/urllib3/connectionpool.py | 17 ++- src/urllib3/poolmanager.py | 109 +++++++++++------- src/urllib3/util/proxy.py | 79 +++++++++++++ src/urllib3/util/ssl_.py | 18 ++- test/__init__.py | 28 ++++- test/test_poolmanager.py | 6 + test/test_proxymanager.py | 4 + test/test_ssl.py | 2 +- test/test_util.py | 65 ++++++++++- .../test_proxy_poolmanager.py | 81 ++++++++++--- 11 files changed, 397 insertions(+), 64 deletions(-) create mode 100644 src/urllib3/util/proxy.py diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 8c63f1fa8d..247214780c 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -9,6 +9,7 @@ from .packages import six from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection from .packages.six.moves.http_client import HTTPException # noqa: F401 +from .util.proxy import connection_requires_http_tunnel, generate_proxy_ssl_context try: # Compiled with SSL? import ssl @@ -112,6 +113,12 @@ def __init__(self, *args, **kw): #: The socket options provided by the user. If no options are #: provided, we use the default options. self.socket_options = kw.pop("socket_options", self.default_socket_options) + + # Proxy options provided by the user. + self.proxy = kw.pop("proxy", None) + self.proxy_config = kw.pop("proxy_config", None) + self.destination_scheme = kw.pop("destination_scheme", None) + _HTTPConnection.__init__(self, *args, **kw) @property @@ -328,8 +335,13 @@ def connect(self): # Add certificate verification conn = self._new_conn() hostname = self.host + tls_in_tls = False if self._is_using_tunnel(): + if self._connection_requires_tls_in_tls(): + conn = self._connect_tls_proxy(hostname, conn) + tls_in_tls = True + self.sock = conn # Calls self._set_hostport(), so self.host is @@ -389,6 +401,7 @@ def connect(self): ca_cert_data=self.ca_cert_data, server_hostname=server_hostname, ssl_context=context, + tls_in_tls=tls_in_tls, ) if self.assert_fingerprint: @@ -421,6 +434,45 @@ def connect(self): or self.assert_fingerprint is not None ) + def _connection_requires_tls_in_tls(self): + """ + Indicates if the current connection requires two TLS connections, one to + the proxy and one to the destination server. + """ + if not self.proxy or self.proxy.scheme != "https": + return False + + return connection_requires_http_tunnel( + self.proxy, self.proxy_config, self.destination_scheme + ) + + def _connect_tls_proxy(self, hostname, conn): + """ + Establish a TLS connection to the proxy using the provided SSL context. + """ + proxy_config = self.proxy_config + ssl_context = proxy_config.ssl_context + if not ssl_context: + ssl_context = generate_proxy_ssl_context( + self.ssl_version, + self.cert_reqs, + self.ca_certs, + self.ca_cert_dir, + self.ca_cert_data, + ) + + return ssl_wrap_socket( + sock=conn, + keyfile=self.key_file, + certfile=self.cert_file, + key_password=self.key_password, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + ca_cert_data=self.ca_cert_data, + server_hostname=hostname, + ssl_context=ssl_context, + ) + def _match_hostname(cert, asserted_hostname): try: diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 492590fb9e..b06a5c1108 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -39,6 +39,7 @@ from .response import HTTPResponse from .util.connection import is_connection_dropped +from .util.proxy import connection_requires_http_tunnel from .util.request import set_file_position from .util.response import assert_header_parsing from .util.retry import Retry @@ -181,6 +182,8 @@ def __init__( retries=None, _proxy=None, _proxy_headers=None, + _proxy_config=None, + _destination_scheme=None, **conn_kw ): ConnectionPool.__init__(self, host, port) @@ -202,6 +205,8 @@ def __init__( self.proxy = _proxy self.proxy_headers = _proxy_headers or {} + self.proxy_config = _proxy_config + self.destination_scheme = _destination_scheme # Fill the queue up so that doing get() on it will block properly for _ in xrange(maxsize): @@ -218,6 +223,10 @@ def __init__( # list. self.conn_kw.setdefault("socket_options", []) + self.conn_kw["proxy"] = self.proxy + self.conn_kw["proxy_config"] = self.proxy_config + self.conn_kw["destination_scheme"] = self.destination_scheme + def _new_conn(self): """ Return a fresh :class:`HTTPConnection`. @@ -637,7 +646,9 @@ def urlopen( # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. - if self.scheme == "http" or (self.proxy and self.proxy.scheme == "https"): + if not connection_requires_http_tunnel( + self.proxy, self.proxy_config, self.destination_scheme + ): headers = headers.copy() headers.update(self.proxy_headers) @@ -931,7 +942,9 @@ def _prepare_proxy(self, conn): improperly set Host: header to proxy's IP:port. """ - if self.proxy.scheme != "https": + if connection_requires_http_tunnel( + self.proxy, self.proxy_config, self.destination_scheme + ): conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) conn.connect() diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index ab183de85d..f733ca8c09 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -2,14 +2,12 @@ import collections import functools import logging -import warnings from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme from .exceptions import ( - HTTPWarning, LocationValueError, MaxRetryError, ProxySchemeUnknown, @@ -21,15 +19,13 @@ from .request import RequestMethods from .util.url import parse_url from .util.retry import Retry +from .util.proxy import connection_requires_http_tunnel +from .packages.six import PY3 +import urllib3.util.ssl_ as ssl_utils -__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] - - -class InvalidProxyConfigurationWarning(HTTPWarning): - """Raised when a user has an HTTPS proxy without enabling HTTPS proxies.""" - pass +__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] log = logging.getLogger(__name__) @@ -68,6 +64,8 @@ class InvalidProxyConfigurationWarning(HTTPWarning): "key_headers", # dict "key__proxy", # parsed proxy url "key__proxy_headers", # dict + "key__proxy_config", # class + "key__destination_scheme", # str "key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples "key__socks_options", # dict "key_assert_hostname", # bool or string @@ -79,6 +77,9 @@ class InvalidProxyConfigurationWarning(HTTPWarning): #: All custom key schemes should include the fields in this key at a minimum. PoolKey = collections.namedtuple("PoolKey", _key_fields) +_proxy_config_fields = ("ssl_context", "use_forwarding_for_https") +ProxyConfig = collections.namedtuple("ProxyConfig", _proxy_config_fields) + def _default_key_normalizer(key_class, request_context): """ @@ -170,6 +171,7 @@ class PoolManager(RequestMethods): """ proxy = None + proxy_config = None def __init__(self, num_pools=10, headers=None, **connection_pool_kw): RequestMethods.__init__(self, headers) @@ -326,14 +328,38 @@ def _merge_pool_kwargs(self, override): def _proxy_requires_url_absolute_form(self, parsed_url): """ Indicates if the proxy requires the complete destination URL in the - request. - - Normally this is only needed when not using an HTTP CONNECT tunnel. + request. Normally this is only needed when not using an HTTP CONNECT + tunnel. """ if self.proxy is None: return False - return parsed_url.scheme == "http" or self.proxy.scheme == "https" + return not connection_requires_http_tunnel( + self.proxy, self.proxy_config, parsed_url.scheme + ) + + def _validate_proxy_scheme_url_selection(self, url_scheme): + """ + Validates that were not attempting to do TLS in TLS connections on + Python2 or with unsupported SSL implementations. + """ + if self.proxy is None or url_scheme != "https": + return + + if self.proxy.scheme != "https": + return + + if not PY3 and not self.proxy_config.use_forwarding_for_https: + raise ProxySchemeUnsupported( + "Contacting HTTPS destinations through HTTPS proxies is not supported on py2." + ) + + if ( + ssl_utils.IS_PYOPENSSL or ssl_utils.IS_SECURETRANSPORT + ) and not self.proxy_config.use_forwarding_for_https: + raise ProxySchemeUnsupported( + "PyOpenSSL/SecureTransport don't support HTTPS proxies" + ) def urlopen(self, method, url, redirect=True, **kw): """ @@ -345,6 +371,8 @@ def urlopen(self, method, url, redirect=True, **kw): :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. """ u = parse_url(url) + self._validate_proxy_scheme_url_selection(u.scheme) + conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) kw["assert_same_host"] = False @@ -415,11 +443,13 @@ class ProxyManager(PoolManager): HTTPS/CONNECT case they are sent only once. Could be used for proxy authentication. - :param _allow_https_proxy_to_see_traffic: + :param proxy_ssl_context: + The proxy SSL context is used to establish the TLS connection to the + proxy when using HTTPS proxies. + + :param _use_forwarding_for_https: Allows forwarding of HTTPS requests to HTTPS proxies. The proxy will - have visibility of all the traffic sent. ONLY USE IF YOU KNOW WHAT - YOU'RE DOING. This flag might be removed at any time in any future - update. + have visibility of all the traffic sent which is HIGHLY INSECURE. Example: >>> proxy = urllib3.ProxyManager('http://localhost:3128/') @@ -440,7 +470,8 @@ def __init__( num_pools=10, headers=None, proxy_headers=None, - _allow_https_proxy_to_see_traffic=False, + proxy_ssl_context=None, + _use_forwarding_for_https=False, **connection_pool_kw ): @@ -461,15 +492,18 @@ def __init__( self.proxy = proxy self.proxy_headers = proxy_headers or {} + self.proxy_ssl_context = proxy_ssl_context + self.proxy_config = ProxyConfig(proxy_ssl_context, _use_forwarding_for_https) connection_pool_kw["_proxy"] = self.proxy connection_pool_kw["_proxy_headers"] = self.proxy_headers - - self.allow_insecure_proxy = _allow_https_proxy_to_see_traffic + connection_pool_kw["_proxy_config"] = self.proxy_config super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw) def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): + # Original destination is needed to identify if we need TLS within TLS. + self._set_destination_scheme(scheme) if scheme == "https": return super(ProxyManager, self).connection_from_host( host, port, scheme, pool_kwargs=pool_kwargs @@ -494,40 +528,29 @@ def _set_proxy_headers(self, url, headers=None): headers_.update(headers) return headers_ - def _validate_proxy_scheme_url_selection(self, url_scheme): - if ( - url_scheme == "https" - and self.proxy.scheme == "https" - and not self.allow_insecure_proxy - ): - warnings.warn( - "Your proxy configuration specified an HTTPS scheme for the proxy. " - "Are you sure you want to use HTTPS to contact the proxy? " - "This most likely indicates an error in your configuration." - "If you are sure you want use HTTPS to contact the proxy, enable " - "the _allow_https_proxy_to_see_traffic.", - InvalidProxyConfigurationWarning, - ) - - raise ProxySchemeUnsupported( - "Contacting HTTPS destinations through HTTPS proxies is not supported." - ) - def urlopen(self, method, url, redirect=True, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." u = parse_url(url) - self._validate_proxy_scheme_url_selection(u.scheme) + self._set_destination_scheme(u.scheme) - if u.scheme == "http" or self.proxy.scheme == "https": + if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme): # For connections using HTTP CONNECT, httplib sets the necessary - # headers on the CONNECT to the proxy. For HTTP or when talking - # HTTPS to the proxy, we'll definitely need to set 'Host' at the - # very least. + # headers on the CONNECT to the proxy. If we're not using CONNECT, + # we'll definitely need to set 'Host' at the very least. headers = kw.get("headers", self.headers) kw["headers"] = self._set_proxy_headers(url, headers) return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw) + def _set_destination_scheme(self, destination_scheme): + """ + Set the destination scheme for connection pools generated with this + ProxyManager. + + The destination scheme is needed for HTTPS proxy connections. + """ + self.connection_pool_kw["_destination_scheme"] = destination_scheme + def proxy_from_url(url, **kw): return ProxyManager(proxy_url=url, **kw) diff --git a/src/urllib3/util/proxy.py b/src/urllib3/util/proxy.py new file mode 100644 index 0000000000..49f7150907 --- /dev/null +++ b/src/urllib3/util/proxy.py @@ -0,0 +1,79 @@ +import os + +from .ssl_ import ( + resolve_cert_reqs, + resolve_ssl_version, + create_urllib3_context, +) + + +def connection_requires_http_tunnel( + proxy_url=None, proxy_config=None, destination_scheme=None +): + """ + Returns True if the connection requires an HTTP CONNECT through the proxy. + + :param destination_url: + :URL URL of the destination. + :param proxy_url: + :URL URL of the proxy. + :param proxy_config: + :class:`PoolManager.ProxyConfig` proxy configuration + """ + # If we're not using a proxy, no way to use a tunnel. + if proxy_url is None: + return False + + # HTTP destinations never require tunneling, we always forward. + if destination_scheme == "http": + return False + + # Support for forwarding with HTTPS proxies and HTTPS destinations. + if ( + proxy_url.scheme == "https" + and proxy_config + and proxy_config.use_forwarding_for_https + ): + return False + + # Otherwise always use a tunnel. + return True + + +def generate_proxy_ssl_context( + ssl_version, cert_reqs, ca_certs=None, ca_cert_dir=None, ca_cert_data=None +): + """ + Generates a default proxy ssl context if one hasn't been provided by the + user. + """ + ssl_context = create_urllib3_context( + ssl_version=resolve_ssl_version(ssl_version), + cert_reqs=resolve_cert_reqs(cert_reqs), + ) + if ( + not ca_certs + and not ca_cert_dir + and not ca_cert_data + and hasattr(ssl_context, "load_default_certs") + ): + ssl_context.load_default_certs() + + proxy_cert, proxy_key, proxy_pass = client_certificate_and_key_from_env() + + if proxy_cert: + ssl_context.load_cert_chain(proxy_cert, keyfile=proxy_key, password=proxy_pass) + + return ssl_context + + +def client_certificate_and_key_from_env(): + """ + Attempts to retrieve a client certificate and key from the environment + variables to use with the proxy. + """ + proxy_cert = os.environ.get("PROXY_SSLCERT") + proxy_key = os.environ.get("PROXY_SSLKEY") + proxy_pass = os.environ.get("PROXY_KEYPASSWD") + + return proxy_cert, proxy_key, proxy_pass diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index 9a8ccdaad2..7c067a64c1 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -14,6 +14,7 @@ SSLContext = None +SSLTransport = None HAS_SNI = False IS_PYOPENSSL = False IS_SECURETRANSPORT = False @@ -42,6 +43,7 @@ def _const_compare_digest_backport(a, b): import ssl from ssl import wrap_socket, CERT_REQUIRED from ssl import HAS_SNI # Has SNI? + from ..contrib.ssl import SSLTransport except ImportError: pass @@ -317,6 +319,7 @@ def ssl_wrap_socket( ca_cert_dir=None, key_password=None, ca_cert_data=None, + tls_in_tls=False, ): """ All arguments except for server_hostname, ssl_context, and ca_cert_dir have @@ -338,6 +341,8 @@ def ssl_wrap_socket( :param ca_cert_data: Optional string containing CA certificates in PEM format suitable for passing as the cadata parameter to SSLContext.load_verify_locations() + :param tls_in_tls: + Use SSLTransport of attempting to wrap the existing socket. """ context = ssl_context if context is None: @@ -388,7 +393,7 @@ def ssl_wrap_socket( server_hostname is not None and not is_ipaddress(server_hostname) ) or IS_SECURETRANSPORT: if HAS_SNI and server_hostname is not None: - return context.wrap_socket(sock, server_hostname=server_hostname) + return _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) warnings.warn( "An HTTPS request has been made, but the SNI (Server Name " @@ -401,7 +406,7 @@ def ssl_wrap_socket( SNIMissingWarning, ) - return context.wrap_socket(sock) + return _ssl_wrap_socket_impl(sock, context, tls_in_tls) def is_ipaddress(hostname): @@ -426,3 +431,12 @@ def _is_key_file_encrypted(key_file): return True return False + + +def _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname=None): + wrapped_sock = None + if tls_in_tls and SSLTransport: + wrapped_sock = SSLTransport(sock, context, server_hostname) + else: + wrapped_sock = context.wrap_socket(sock, server_hostname=server_hostname) + return wrapped_sock diff --git a/test/__init__.py b/test/__init__.py index 589473211d..a6d33b3c17 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -19,6 +19,8 @@ from urllib3.util import ssl_ from urllib3 import util +import urllib3.contrib.pyopenssl as pyopenssl + # We need a host that will not immediately close the connection with a TCP # Reset. if platform.system() == "Windows": @@ -153,6 +155,19 @@ def notBrotlipy(): ) +def onlySecureTransport(test): + """Runs this test when SecureTransport is in use.""" + + @six.wraps(test) + def wrapper(*args, **kwargs): + msg = "{name} only runs with SecureTransport".format(name=test.__name__) + if not ssl_.IS_SECURETRANSPORT: + pytest.skip(msg) + return test(*args, **kwargs) + + return wrapper + + def notSecureTransport(test): """Skips this test when SecureTransport is in use.""" @@ -167,7 +182,7 @@ def wrapper(*args, **kwargs): def notOpenSSL098(test): - """Skips this test for Python 3.5 macOS python.org distribution""" + """Skips this test for Python 3.5 , macOS python.org distribution""" @six.wraps(test) def wrapper(*args, **kwargs): @@ -277,6 +292,17 @@ def wrapper(*args, **kwargs): return wrapper +def withPyOpenSSL(test): + @six.wraps(test) + def wrapper(*args, **kwargs): + pyopenssl.inject_into_urllib3() + result = test(*args, **kwargs) + pyopenssl.extract_from_urllib3() + return result + + return wrapper + + class _ListHandler(logging.Handler): def __init__(self): super(_ListHandler, self).__init__() diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py index 0603236e97..6f42712902 100644 --- a/test/test_poolmanager.py +++ b/test/test_poolmanager.py @@ -366,3 +366,9 @@ def test_merge_pool_kwargs_invalid_key(self): p = PoolManager(strict=True) merged = p._merge_pool_kwargs({"invalid_key": None}) assert p.connection_pool_kw == merged + + def test_pool_manager_no_url_absolute_form(self): + """Valides we won't send a request with absolute form without a proxy""" + p = PoolManager(strict=True) + assert p._proxy_requires_url_absolute_form("http://example.com") is False + assert p._proxy_requires_url_absolute_form("https://example.com") is False diff --git a/test/test_proxymanager.py b/test/test_proxymanager.py index 2043580c8a..ac74414df4 100644 --- a/test/test_proxymanager.py +++ b/test/test_proxymanager.py @@ -62,6 +62,10 @@ def test_proxy_tunnel(self): assert p._proxy_requires_url_absolute_form(https_url) is False with ProxyManager("https://proxy:8080") as p: + assert p._proxy_requires_url_absolute_form(http_url) + assert p._proxy_requires_url_absolute_form(https_url) is False + + with ProxyManager("https://proxy:8080", _use_forwarding_for_https=True) as p: assert p._proxy_requires_url_absolute_form(http_url) assert p._proxy_requires_url_absolute_form(https_url) diff --git a/test/test_ssl.py b/test/test_ssl.py index f755938062..4026e83a5b 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -63,7 +63,7 @@ def test_context_sni_with_ip_address(monkeypatch, has_sni, server_hostname, uses if uses_sni: context.wrap_socket.assert_called_with(sock, server_hostname=server_hostname) else: - context.wrap_socket.assert_called_with(sock) + context.wrap_socket.assert_called_with(sock, server_hostname=None) @pytest.mark.parametrize( diff --git a/test/test_util.py b/test/test_util.py index 838c751518..eae64a57fb 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -4,6 +4,7 @@ import logging import io import ssl +import os import socket from itertools import chain @@ -28,10 +29,20 @@ SNIMissingWarning, UnrewindableBodyError, ) -from urllib3.util.connection import allowed_gai_family, _has_ipv6 +from urllib3.util.connection import ( + allowed_gai_family, + _has_ipv6, +) +from urllib3.util.proxy import ( + connection_requires_http_tunnel, + client_certificate_and_key_from_env, + generate_proxy_ssl_context, +) from urllib3.util import is_fp_closed, ssl_ from urllib3.packages import six +from urllib3.poolmanager import ProxyConfig + from . import clear_warnings from test import onlyPy3, onlyPy2, onlyBrotlipy, notBrotlipy @@ -781,7 +792,9 @@ def test_ssl_wrap_socket_with_no_sni_warns(self): sock=socket, server_hostname="www.google.com", ) - mock_context.wrap_socket.assert_called_once_with(socket) + mock_context.wrap_socket.assert_called_once_with( + socket, server_hostname=None + ) assert warn.call_count >= 1 warnings = [call[0][1] for call in warn.call_args_list] assert SNIMissingWarning in warnings @@ -838,3 +851,51 @@ def test_ip_family_ipv6_disabled(self): def test_assert_header_parsing_throws_typeerror_with_non_headers(self, headers): with pytest.raises(TypeError): assert_header_parsing(headers) + + def test_connection_requires_http_tunnel_no_proxy(self): + assert not connection_requires_http_tunnel( + proxy_url=None, proxy_config=None, destination_scheme=None + ) + + def test_connection_requires_http_tunnel_http_proxy(self): + proxy = parse_url("http://proxy:8080") + proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False) + destination_scheme = "http" + assert not connection_requires_http_tunnel( + proxy, proxy_config, destination_scheme + ) + + destination_scheme = "https" + assert connection_requires_http_tunnel(proxy, proxy_config, destination_scheme) + + def test_connection_requires_http_tunnel_https_proxy(self): + proxy = parse_url("https://proxy:8443") + proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False) + destination_scheme = "http" + assert not connection_requires_http_tunnel( + proxy, proxy_config, destination_scheme + ) + + def test_client_certificate_and_key_from_env(self): + test_cert = "cert" + test_key = "key" + test_pwd = "pwd" + + os.environ["PROXY_SSLCERT"] = test_cert + os.environ["PROXY_SSLKEY"] = test_key + os.environ["PROXY_SSLPWD"] = test_pwd + + test_cert, test_key, test_pwd == client_certificate_and_key_from_env() + + del os.environ["PROXY_SSLCERT"] + del os.environ["PROXY_SSLKEY"] + del os.environ["PROXY_SSLPWD"] + + cert, key, pwd = client_certificate_and_key_from_env() + assert cert is None + assert key is None + assert pwd is None + + def test_generate_proxy_ssl_context(self): + ssl_context = generate_proxy_ssl_context(ssl_version=None, cert_reqs=None) + ssl_context.verify_mode = ssl.CERT_REQUIRED diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index feface09e2..ec87d9ef0d 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -23,7 +23,15 @@ ) from urllib3.connectionpool import connection_from_url, VerifiedHTTPSConnection -from test import SHORT_TIMEOUT, LONG_TIMEOUT +from test import ( + SHORT_TIMEOUT, + LONG_TIMEOUT, + onlyPy3, + onlyPy2, + withPyOpenSSL, + onlySecureTransport, +) + # Retry failed tests pytestmark = pytest.mark.flaky @@ -38,7 +46,7 @@ def setup_class(cls): cls.https_url = "https://%s:%d" % (cls.https_host, cls.https_port) cls.https_url_alt = "https://%s:%d" % (cls.https_host_alt, cls.https_port) cls.proxy_url = "http://%s:%d" % (cls.proxy_host, cls.proxy_port) - cls.https_proxy_url = "https://%s:%d" % (cls.proxy_host, cls.https_proxy_port,) + cls.https_proxy_url = "https://%s:%d" % (cls.proxy_host, cls.https_proxy_port) # Generate another CA to test verification failure cls.certs_dir = tempfile.mkdtemp() @@ -60,7 +68,27 @@ def test_basic_proxy(self): r = http.request("GET", "%s/" % self.https_url) assert r.status == 200 + @onlyPy3 def test_https_proxy(self): + with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: + r = https.request("GET", "%s/" % self.http_url) + assert r.status == 200 + + r = https.request("GET", "%s/" % self.https_url) + assert r.status == 200 + + @onlyPy2 + def test_https_proxy_not_supported(self): + with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: + r = https.request("GET", "%s/" % self.http_url) + assert r.status == 200 + + with pytest.raises(ProxySchemeUnsupported): + https.request("GET", "%s/" % self.https_url) + + @withPyOpenSSL + @onlyPy3 + def test_https_proxy_pyopenssl_not_supported(self): with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: r = https.request("GET", "%s/" % self.http_url) assert r.status == 200 @@ -68,13 +96,24 @@ def test_https_proxy(self): with pytest.raises(ProxySchemeUnsupported): https.request("GET", "%s/" % self.https_url) + @onlySecureTransport + @onlyPy3 + def test_https_proxy_securetransport_not_supported(self): + with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: + r = https.request("GET", "%s/" % self.http_url) + assert r.status == 200 + + with pytest.raises(ProxySchemeUnsupported): + https.request("GET", "%s/" % self.https_url) + + def test_https_proxy_insecure(self): with proxy_from_url( - self.https_proxy_url, - ca_certs=DEFAULT_CA, - _allow_https_proxy_to_see_traffic=True, + self.https_proxy_url, ca_certs=DEFAULT_CA, _use_forwarding_for_https=True, ) as https: r = https.request("GET", "%s/" % self.http_url) - https.request("GET", "%s/" % self.https_url) + assert r.status == 200 + + r = https.request("GET", "%s/" % self.https_url) assert r.status == 200 def test_nagle_proxy(self): @@ -299,6 +338,7 @@ def test_headers(self): self.https_port, ) + @onlyPy3 def test_https_headers(self): with proxy_from_url( self.https_proxy_url, @@ -325,19 +365,34 @@ def test_https_headers(self): self.http_port, ) - with pytest.raises(ProxySchemeUnsupported): - http.request_encode_url("GET", "%s/headers" % self.https_url) - - r = http.request_encode_url( - "GET", "%s/headers" % self.http_url, headers={"Baz": "quux"} + r = http.request_encode_body( + "GET", "%s/headers" % self.https_url, headers={"Baz": "quux"} ) returned_headers = json.loads(r.data.decode()) assert returned_headers.get("Foo") is None assert returned_headers.get("Baz") == "quux" + assert returned_headers.get("Hickory") is None + assert returned_headers.get("Host") == "%s:%s" % ( + self.https_host, + self.https_port, + ) + + def test_https_headers_insecure(self): + with proxy_from_url( + self.https_proxy_url, + headers={"Foo": "bar"}, + proxy_headers={"Hickory": "dickory"}, + ca_certs=DEFAULT_CA, + _use_forwarding_for_https=True, + ) as http: + + r = http.request_encode_url("GET", "%s/headers" % self.https_url) + returned_headers = json.loads(r.data.decode()) + assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Hickory") == "dickory" assert returned_headers.get("Host") == "%s:%s" % ( - self.http_host, - self.http_port, + self.https_host, + self.https_port, ) def test_headerdict(self):