Integrate SSLTransport into urllib3.

For connections that will attempt to use an HTTPS proxy with an HTTPS
destination, we'll use the TLS in TLS support provided by SSL Transport.

HTTPS proxy and HTTP destinations will continue using a single TLS
session as expected.

HTTPS proxies and HTTPS destinations can continue to be insecure if
indicated by the allow_insecure_proxies parameter.

src/urllib3/connection.py

@@ -10,9 +10,10 @@
 from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection
 from .packages.six.moves.http_client import HTTPException  # noqa: F401
 
+hasSSL = False
 try:  # Compiled with SSL?
     import ssl
-
+    hasSSL = True
     BaseSSLError = ssl.SSLError
 except (ImportError, AttributeError):  # Platform-specific: No SSL.
     ssl = None
@@ -35,6 +36,7 @@ class ConnectionError(Exception):
     ConnectTimeoutError,
     SubjectAltNameWarning,
     SystemTimeWarning,
+    SSLError
 )
 from .packages.ssl_match_hostname import match_hostname, CertificateError
 
@@ -111,6 +113,12 @@ def __init__(self, *args, **kw):
         #: The socket options provided by the user. If no options are
         #: provided, we use the default options.
         self.socket_options = kw.pop("socket_options", self.default_socket_options)
+
+        # Proxy options provided by the user.
+        self.proxy = kw.pop("proxy", None)
+        self.proxy_config = kw.pop("proxy_config", None)
+        self.destination_scheme = kw.pop("destination_scheme", None)
+
         _HTTPConnection.__init__(self, *args, **kw)
 
     @property
@@ -309,8 +317,13 @@ def connect(self):
         # Add certificate verification
         conn = self._new_conn()
         hostname = self.host
+        tls_in_tls = False
 
         if self._is_using_tunnel():
+            if self._connection_requires_tls_in_tls():
+                conn = self._connect_tls_proxy(hostname, conn)
+                tls_in_tls = True
+
             self.sock = conn
 
             # Calls self._set_hostport(), so self.host is
@@ -370,6 +383,7 @@ def connect(self):
             ca_cert_data=self.ca_cert_data,
             server_hostname=server_hostname,
             ssl_context=context,
+            tls_in_tls=tls_in_tls,
         )
 
         if self.assert_fingerprint:
@@ -402,6 +416,60 @@ def connect(self):
             or self.assert_fingerprint is not None
         )
 
+    def _connection_requires_tls_in_tls(self):
+        """
+        Indicates if the current connection requires two TLS connections, one to
+        the proxy and one to the destination server.
+        """
+        if not self.proxy or not self.proxy_config:
+            return False
+
+        if self.destination_scheme == "http":
+            return False
+
+        if self.proxy.scheme != 'https' or self.proxy_config.allow_insecure_proxy:
+            return False
+
+        # Not supported in < py3 or ssl-less environments.
+        if not six.PY3 or not hasSSL:
+            raise SSLError(
+                "HTTPS proxies need python3 and SSL support to work properly"
+            )
+
+        return True
+
+
+    def _connect_tls_proxy(self, hostname, conn):
+        """
+        Establish a TLS connection to the proxy using the provided SSL context.
+        """
+        proxy_config = self.proxy_config
+        ssl_context = proxy_config.ssl_context
+        if not ssl_context:
+            ssl_context = create_urllib3_context(
+                ssl_version=resolve_ssl_version(self.ssl_version),
+                cert_reqs=resolve_cert_reqs(self.cert_reqs),
+            )
+            if (
+                not self.ca_certs
+                and not self.ca_cert_dir
+                and not self.ca_cert_data
+                and hasattr(ssl_context, "load_default_certs")
+            ):
+                ssl_context.load_default_certs()
+
+        return ssl_wrap_socket(
+            sock=conn,
+            keyfile=self.key_file,
+            certfile=self.cert_file,
+            key_password=self.key_password,
+            ca_certs=self.ca_certs,
+            ca_cert_dir=self.ca_cert_dir,
+            ca_cert_data=self.ca_cert_data,
+            server_hostname=hostname,
+            ssl_context=ssl_context,
+        )
+
 
 def _match_hostname(cert, asserted_hostname):
     try:

src/urllib3/connectionpool.py

@@ -38,7 +38,7 @@
 from .request import RequestMethods
 from .response import HTTPResponse
 
-from .util.connection import is_connection_dropped
+from .util.connection import is_connection_dropped, connection_requires_http_tunnel
 from .util.request import set_file_position
 from .util.response import assert_header_parsing
 from .util.retry import Retry
@@ -181,6 +181,8 @@ def __init__(
         retries=None,
         _proxy=None,
         _proxy_headers=None,
+        _proxy_config=None,
+        _destination_scheme=None,
         **conn_kw
     ):
         ConnectionPool.__init__(self, host, port)
@@ -202,6 +204,8 @@ def __init__(
 
         self.proxy = _proxy
         self.proxy_headers = _proxy_headers or {}
+        self.proxy_config = _proxy_config
+        self.destination_scheme = _destination_scheme
 
         # Fill the queue up so that doing get() on it will block properly
         for _ in xrange(maxsize):
@@ -218,6 +222,10 @@ def __init__(
             # list.
             self.conn_kw.setdefault("socket_options", [])
 
+            self.conn_kw["proxy"] = self.proxy
+            self.conn_kw["proxy_config"] = self.proxy_config
+            self.conn_kw["destination_scheme"] = self.destination_scheme
+
     def _new_conn(self):
         """
         Return a fresh :class:`HTTPConnection`.
@@ -637,7 +645,8 @@ def urlopen(
         # Merge the proxy headers. Only done when not using HTTP CONNECT. We
         # have to copy the headers dict so we can safely change it without those
         # changes being reflected in anyone else's copy.
-        if self.scheme == "http" or (self.proxy and self.proxy.scheme == "https"):
+        if not connection_requires_http_tunnel(self.proxy, self.proxy_config,
+                                               self.destination_scheme):
             headers = headers.copy()
             headers.update(self.proxy_headers)
 
@@ -929,7 +938,8 @@ def _prepare_proxy(self, conn):
         improperly set Host: header to proxy's IP:port.
         """
 
-        if self.proxy.scheme != "https":
+        if connection_requires_http_tunnel(self.proxy, self.proxy_config,
+                                           self.destination_scheme):
             conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
 
         conn.connect()

src/urllib3/poolmanager.py

@@ -2,14 +2,12 @@
 import collections
 import functools
 import logging
-import warnings
 
 from ._collections import RecentlyUsedContainer
 from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
 from .connectionpool import port_by_scheme
 
 from .exceptions import (
-    HTTPWarning,
     LocationValueError,
     MaxRetryError,
     ProxySchemeUnknown,
@@ -20,17 +18,13 @@
 from .request import RequestMethods
 from .util.url import parse_url
 from .util.retry import Retry
+from .util.connection import connection_requires_http_tunnel
+from .packages.six import PY3
 
 
 __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
 
 
-class InvalidProxyConfigurationWarning(HTTPWarning):
-    """Raised when a user has an HTTPS proxy without enabling HTTPS proxies."""
-
-    pass
-
-
 log = logging.getLogger(__name__)
 
 SSL_KEYWORDS = (
@@ -67,6 +61,8 @@ class InvalidProxyConfigurationWarning(HTTPWarning):
     "key_headers",  # dict
     "key__proxy",  # parsed proxy url
     "key__proxy_headers",  # dict
+    "key__proxy_config",  # class
+    "key__destination_scheme",  # str
     "key_socket_options",  # list of (level (int), optname (int), value (int or str)) tuples
     "key__socks_options",  # dict
     "key_assert_hostname",  # bool or string
@@ -78,6 +74,8 @@ class InvalidProxyConfigurationWarning(HTTPWarning):
 #: All custom key schemes should include the fields in this key at a minimum.
 PoolKey = collections.namedtuple("PoolKey", _key_fields)
 
+_proxy_config_fields = ("ssl_context", "allow_insecure_proxy")
+ProxyConfig = collections.namedtuple("ProxyConfig", _proxy_config_fields)
 
 def _default_key_normalizer(key_class, request_context):
     """
@@ -169,6 +167,7 @@ class PoolManager(RequestMethods):
     """
 
     proxy = None
+    proxy_config = None
 
     def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
         RequestMethods.__init__(self, headers)
@@ -323,14 +322,31 @@ def _merge_pool_kwargs(self, override):
     def _proxy_requires_url_absolute_form(self, parsed_url):
         """
         Indicates if the proxy requires the complete destination URL in the
-        request.
-
-        Normally this is only needed when not using an HTTP CONNECT tunnel.
+        request.  Normally this is only needed when not using an HTTP CONNECT
+        tunnel.
         """
         if self.proxy is None:
             return False
 
-        return parsed_url.scheme == "http" or self.proxy.scheme == "https"
+        return not connection_requires_http_tunnel(self.proxy,
+                                                   self.proxy_config,
+                                                   parsed_url.scheme)
+
+    def _validate_proxy_scheme_url_selection(self, url_scheme):
+        """
+        Validates that were not attempting to do TLS in TLS connections on
+        Python2.
+        """
+        if self.proxy is None or url_scheme != "https":
+            return
+
+        if self.proxy.scheme != "https":
+            return
+
+        if not PY3 and not self.proxy_config.allow_insecure_proxy:
+            raise ProxySchemeUnsupported(
+                "Contacting HTTPS destinations through HTTPS proxies is not supported on py2."
+            )
 
     def urlopen(self, method, url, redirect=True, **kw):
         """
@@ -342,6 +358,8 @@ def urlopen(self, method, url, redirect=True, **kw):
         :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
         """
         u = parse_url(url)
+        self._validate_proxy_scheme_url_selection(u.scheme)
+
         conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
 
         kw["assert_same_host"] = False
@@ -412,6 +430,10 @@ class ProxyManager(PoolManager):
         HTTPS/CONNECT case they are sent only once. Could be used for proxy
         authentication.
 
+    :param proxy_ssl_context:
+        The proxy SSL context is used to establish the TLS connection to the
+        proxy when using HTTPS proxies.
+
     :param _allow_https_proxy_to_see_traffic:
         Allows forwarding of HTTPS requests to HTTPS proxies. The proxy will
         have visibility of all the traffic sent. ONLY USE IF YOU KNOW WHAT
@@ -437,6 +459,7 @@ def __init__(
         num_pools=10,
         headers=None,
         proxy_headers=None,
+        proxy_ssl_context=None,
         _allow_https_proxy_to_see_traffic=False,
         **connection_pool_kw
     ):
@@ -458,15 +481,20 @@ def __init__(
 
         self.proxy = proxy
         self.proxy_headers = proxy_headers or {}
+        self.proxy_ssl_context = proxy_ssl_context
+        self.proxy_config = ProxyConfig(
+            proxy_ssl_context, _allow_https_proxy_to_see_traffic
+        )
 
         connection_pool_kw["_proxy"] = self.proxy
         connection_pool_kw["_proxy_headers"] = self.proxy_headers
-
-        self.allow_insecure_proxy = _allow_https_proxy_to_see_traffic
+        connection_pool_kw["_proxy_config"] = self.proxy_config
 
         super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
 
     def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
+        # Original destination is needed to identify if we need TLS within TLS.
+        self._set_destination_scheme(scheme)
         if scheme == "https":
             return super(ProxyManager, self).connection_from_host(
                 host, port, scheme, pool_kwargs=pool_kwargs
@@ -491,31 +519,12 @@ def _set_proxy_headers(self, url, headers=None):
             headers_.update(headers)
         return headers_
 
-    def _validate_proxy_scheme_url_selection(self, url_scheme):
-        if (
-            url_scheme == "https"
-            and self.proxy.scheme == "https"
-            and not self.allow_insecure_proxy
-        ):
-            warnings.warn(
-                "Your proxy configuration specified an HTTPS scheme for the proxy. "
-                "Are you sure you want to use HTTPS to contact the proxy? "
-                "This most likely indicates an error in your configuration."
-                "If you are sure you want use HTTPS to contact the proxy, enable "
-                "the _allow_https_proxy_to_see_traffic.",
-                InvalidProxyConfigurationWarning,
-            )
-
-            raise ProxySchemeUnsupported(
-                "Contacting HTTPS destinations through HTTPS proxies is not supported."
-            )
-
     def urlopen(self, method, url, redirect=True, **kw):
         "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
         u = parse_url(url)
-        self._validate_proxy_scheme_url_selection(u.scheme)
+        self._set_destination_scheme(u.scheme)
 
-        if u.scheme == "http" or self.proxy.scheme == "https":
+        if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
             # For connections using HTTP CONNECT, httplib sets the necessary
             # headers on the CONNECT to the proxy. For HTTP or when talking
             # HTTPS to the proxy, we'll definitely need to set 'Host' at the
@@ -525,6 +534,16 @@ def urlopen(self, method, url, redirect=True, **kw):
 
         return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)
 
+    def _set_destination_scheme(self, destination_scheme):
+        """
+        Set the destination scheme for connection pools generated with this
+        ProxyManager.
+
+        The destination scheme is needed for HTTPS proxy connections.
+        """
+        self.connection_pool_kw["_destination_scheme"] = destination_scheme
+
+
 
 def proxy_from_url(url, **kw):
     return ProxyManager(proxy_url=url, **kw)