Release 1.24.2 (#1564)

* Don't load system certificates by default when any other ``ca_certs``, ``ca_certs_dir`` or ``ssl_context`` parameters are specified.
* Remove Authorization header regardless of case when redirecting to cross-site. (Issue #1510)
* Add support for IPv6 addresses in subjectAltName section of certificates. (Issue #1269)

CHANGES.rst

@@ -1,12 +1,23 @@
 Changes
 =======
 
+1.24.2 (2019-04-17)
+-------------------
+
+* Don't load system certificates by default when any other ``ca_certs``, ``ca_certs_dir`` or
+  ``ssl_context`` parameters are specified.
+
+* Remove Authorization header regardless of case when redirecting to cross-site. (Issue #1510)
+
+* Add support for IPv6 addresses in subjectAltName section of certificates. (Issue #1269)
+
+
 1.24.1 (2018-11-02)
 -------------------
 
 * Remove quadratic behavior within ``GzipDecoder.decompress()`` (Issue #1467)
 
-* Restored functionality of `ciphers` parameter for `create_urllib3_context()`. (Issue #1462)
+* Restored functionality of ``ciphers`` parameter for ``create_urllib3_context()``. (Issue #1462)
 
 
 1.24 (2018-10-16)

CONTRIBUTORS.txt

@@ -272,5 +272,8 @@ In chronological order:
 * Justin Bramley <https://github.com/jbramleycl>
   * Add ability to handle multiple Content-Encodings
 
+* Katsuhiko YOSHIDA <https://github.com/kyoshidajp>
+  * Remove Authorization header regardless of case when redirecting to cross-site
+
 * [Your name or handle] <[email or website]>
   * [Brief summary of your changes]

dummyserver/certs/server.ipv6_san.crt

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICfTCCAeagAwIBAgIJAPcpn3/M5+piMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMTgxMjE5MDUyMjUyWhcNNDgxMjE4MDUyMjUyWjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+gQDXe3FqmCWvP8XPxqtT+0bfL1Tvzvebi46k0WIcUV8bP3vyYiSRXG9ALmyzZH4G
+HY9UVs4OEDkCMDOBSezB0y9ai/9doTNcaictdEBu8nfdXKoTtzrn+VX4UPrkH5hm
+7NQ1fTQuj1MR7yBCmYqN3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQABo3UwczAdBgNV
+HQ4EFgQUG+dK5Uos08QUwAWofDb3a8YcYlIwHwYDVR0jBBgwFoAUG+dK5Uos08QU
+wAWofDb3a8YcYlIwDwYDVR0TAQH/BAUwAwEB/zAgBgNVHREEGTAXggM6OjGHEAAA
+AAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQADgYEAjT767TDq6q4lOextf3tZ
+BjeuYDUy7bb1fDBAN5rBT1ywr7r0JE6/KOnsZx4jbevx3MllxNpx0gOM2bgwJlnG
+8tgwRB6pxDyln01WBj9b5ymK60jdkw7gg3yYpqEs5/VBQidFO3BmDqf5cGO8PU7p
+0VWdfJBP2UbwblNXdImI1zk=
+-----END CERTIFICATE-----

dummyserver/server.py

@@ -58,11 +58,16 @@
     'certfile': os.path.join(CERTS_PATH, 'server.ipv6addr.crt'),
     'keyfile': os.path.join(CERTS_PATH, 'server.ipv6addr.key'),
 }
+IPV6_SAN_CERTS = {
+    'certfile': os.path.join(CERTS_PATH, 'server.ipv6_san.crt'),
+    'keyfile': DEFAULT_CERTS['keyfile']
+}
 DEFAULT_CA = os.path.join(CERTS_PATH, 'cacert.pem')
 DEFAULT_CA_BAD = os.path.join(CERTS_PATH, 'client_bad.pem')
 NO_SAN_CA = os.path.join(CERTS_PATH, 'cacert.no_san.pem')
 DEFAULT_CA_DIR = os.path.join(CERTS_PATH, 'ca_path_test')
 IPV6_ADDR_CA = os.path.join(CERTS_PATH, 'server.ipv6addr.crt')
+IPV6_SAN_CA = os.path.join(CERTS_PATH, 'server.ipv6_san.crt')
 COMBINED_CERT_AND_KEY = os.path.join(CERTS_PATH, 'server.combined.pem')
 
 

src/urllib3/__init__.py

@@ -27,7 +27,7 @@
 
 __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
 __license__ = 'MIT'
-__version__ = '1.24.1'
+__version__ = '1.24.2'
 
 __all__ = (
     'HTTPConnectionPool',

src/urllib3/contrib/pyopenssl.py

@@ -184,6 +184,9 @@ def idna_encode(name):
         except idna.core.IDNAError:
             return None
 
+    if ':' in name:
+        return name
+
     name = idna_encode(name)
     if name is None:
         return None

src/urllib3/poolmanager.py

@@ -7,6 +7,7 @@
 from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
 from .connectionpool import port_by_scheme
 from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown
+from .packages import six
 from .packages.six.moves.urllib.parse import urljoin
 from .request import RequestMethods
 from .util.url import parse_url
@@ -342,8 +343,10 @@ def urlopen(self, method, url, redirect=True, **kw):
         # conn.is_same_host() which may use socket.gethostbyname() in the future.
         if (retries.remove_headers_on_redirect
                 and not conn.is_same_host(redirect_location)):
-            for header in retries.remove_headers_on_redirect:
-                kw['headers'].pop(header, None)
+            headers = list(six.iterkeys(kw['headers']))
+            for header in headers:
+                if header.lower() in retries.remove_headers_on_redirect:
+                    kw['headers'].pop(header, None)
 
         try:
             retries = retries.increment(method, url, response=response, _pool=conn)

src/urllib3/util/retry.py

@@ -179,7 +179,8 @@ def __init__(self, total=10, connect=None, read=None, redirect=None, status=None
         self.raise_on_status = raise_on_status
         self.history = history or tuple()
         self.respect_retry_after_header = respect_retry_after_header
-        self.remove_headers_on_redirect = remove_headers_on_redirect
+        self.remove_headers_on_redirect = frozenset([
+            h.lower() for h in remove_headers_on_redirect])
 
     def new(self, **kw):
         params = dict(

src/urllib3/util/ssl_.py

@@ -327,7 +327,10 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
             if e.errno == errno.ENOENT:
                 raise SSLError(e)
             raise
-    elif getattr(context, 'load_default_certs', None) is not None:
+
+    # Don't load system certs unless there were no CA certs or
+    # SSLContext object specified manually.
+    elif ssl_context is None and hasattr(context, 'load_default_certs'):
         # try to load OS default certs; works well on Windows (require Python3.4+)
         context.load_default_certs()
 

test/contrib/test_pyopenssl.py

@@ -31,7 +31,10 @@ def teardown_module():
         pass
 
 
-from ..with_dummyserver.test_https import TestHTTPS, TestHTTPS_TLSv1  # noqa: F401
+from ..with_dummyserver.test_https import (  # noqa: F401
+    TestHTTPS, TestHTTPS_TLSv1, TestHTTPS_IPv6Addr,
+    TestHTTPS_IPSAN, TestHTTPS_NoSAN, TestHTTPS_IPV6SAN
+)
 from ..with_dummyserver.test_socketlevel import (  # noqa: F401
     TestSNI, TestSocketClosing, TestClientCerts
 )

test/test_retry.py

@@ -253,9 +253,9 @@ def test_retry_method_not_in_whitelist(self):
     def test_retry_default_remove_headers_on_redirect(self):
         retry = Retry()
 
-        assert list(retry.remove_headers_on_redirect) == ['Authorization']
+        assert list(retry.remove_headers_on_redirect) == ['authorization']
 
     def test_retry_set_remove_headers_on_redirect(self):
-        retry = Retry(remove_headers_on_redirect=['X-API-Secret'])
+        retry = Retry(remove_headers_on_redirect=['x-api-secret'])
 
-        assert list(retry.remove_headers_on_redirect) == ['X-API-Secret']
+        assert list(retry.remove_headers_on_redirect) == ['x-api-secret']

test/test_ssl.py

@@ -88,3 +88,40 @@ def test_create_urllib3_context_set_ciphers(monkeypatch, ciphers, expected_ciphe
 
     assert context.set_ciphers.call_count == 1
     assert context.set_ciphers.call_args == mock.call(expected_ciphers)
+
+
+def test_wrap_socket_given_context_no_load_default_certs():
+    context = mock.create_autospec(ssl_.SSLContext)
+    context.load_default_certs = mock.Mock()
+
+    sock = mock.Mock()
+    ssl_.ssl_wrap_socket(sock, ssl_context=context)
+
+    context.load_default_certs.assert_not_called()
+
+
+def test_wrap_socket_given_ca_certs_no_load_default_certs(monkeypatch):
+    context = mock.create_autospec(ssl_.SSLContext)
+    context.load_default_certs = mock.Mock()
+    context.options = 0
+
+    monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context)
+
+    sock = mock.Mock()
+    ssl_.ssl_wrap_socket(sock, ca_certs="/tmp/fake-file")
+
+    context.load_default_certs.assert_not_called()
+    context.load_verify_locations.assert_called_with("/tmp/fake-file", None)
+
+
+def test_wrap_socket_default_loads_default_certs(monkeypatch):
+    context = mock.create_autospec(ssl_.SSLContext)
+    context.load_default_certs = mock.Mock()
+    context.options = 0
+
+    monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context)
+
+    sock = mock.Mock()
+    ssl_.ssl_wrap_socket(sock)
+
+    context.load_default_certs.assert_called_with()

test/with_dummyserver/test_https.py

@@ -17,7 +17,7 @@
                                 DEFAULT_CLIENT_NO_INTERMEDIATE_CERTS,
                                 NO_SAN_CERTS, NO_SAN_CA, DEFAULT_CA_DIR,
                                 IPV6_ADDR_CERTS, IPV6_ADDR_CA, HAS_IPV6,
-                                IP_SAN_CERTS)
+                                IP_SAN_CERTS, IPV6_SAN_CA, IPV6_SAN_CERTS)
 
 from test import (
     onlyPy279OrNewer,
@@ -625,5 +625,23 @@ def test_strip_square_brackets_before_validating(self):
         self.assertEqual(r.status, 200)
 
 
+class TestHTTPS_IPV6SAN(IPV6HTTPSDummyServerTestCase):
+    certs = IPV6_SAN_CERTS
+
+    def test_can_validate_ipv6_san(self):
+        """Ensure that urllib3 can validate SANs with IPv6 addresses in them."""
+        try:
+            import ipaddress  # noqa: F401
+        except ImportError:
+            pytest.skip("Only runs on systems with an ipaddress module")
+
+        https_pool = HTTPSConnectionPool('[::1]', self.port,
+                                         cert_reqs='CERT_REQUIRED',
+                                         ca_certs=IPV6_SAN_CA)
+        self.addCleanup(https_pool.close)
+        r = https_pool.request('GET', '/')
+        self.assertEqual(r.status, 200)
+
+
 if __name__ == '__main__':
     unittest.main()

test/with_dummyserver/test_poolmanager.py

@@ -123,6 +123,17 @@ def test_redirect_cross_host_remove_headers(self):
 
         self.assertNotIn('Authorization', data)
 
+        r = http.request('GET', '%s/redirect' % self.base_url,
+                         fields={'target': '%s/headers' % self.base_url_alt},
+                         headers={'authorization': 'foo'})
+
+        self.assertEqual(r.status, 200)
+
+        data = json.loads(r.data.decode('utf-8'))
+
+        self.assertNotIn('authorization', data)
+        self.assertNotIn('Authorization', data)
+
     def test_redirect_cross_host_no_remove_headers(self):
         http = PoolManager()
         self.addCleanup(http.clear)
@@ -155,6 +166,21 @@ def test_redirect_cross_host_set_removed_headers(self):
         self.assertNotIn('X-API-Secret', data)
         self.assertEqual(data['Authorization'], 'bar')
 
+        r = http.request('GET', '%s/redirect' % self.base_url,
+                         fields={'target': '%s/headers' % self.base_url_alt},
+                         headers={'x-api-secret': 'foo',
+                                  'authorization': 'bar'},
+                         retries=Retry(remove_headers_on_redirect=['X-API-Secret']))
+
+        self.assertEqual(r.status, 200)
+
+        data = json.loads(r.data.decode('utf-8'))
+
+        self.assertNotIn('x-api-secret', data)
+        self.assertNotIn('X-API-Secret', data)
+
+        self.assertEqual(data['Authorization'], 'bar')
+
     def test_raise_on_redirect(self):
         http = PoolManager()
         self.addCleanup(http.clear)