From 1efadf43dc63317cd9eaa3e0fdb9e05ab07254b1 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Wed, 17 Apr 2019 12:46:22 -0500 Subject: [PATCH] Release 1.24.2 (#1564) * Don't load system certificates by default when any other ``ca_certs``, ``ca_certs_dir`` or ``ssl_context`` parameters are specified. * Remove Authorization header regardless of case when redirecting to cross-site. (Issue #1510) * Add support for IPv6 addresses in subjectAltName section of certificates. (Issue #1269) --- CHANGES.rst | 13 +++++++- CONTRIBUTORS.txt | 3 ++ dummyserver/certs/server.ipv6_san.crt | 16 ++++++++++ dummyserver/server.py | 5 +++ src/urllib3/__init__.py | 2 +- src/urllib3/contrib/pyopenssl.py | 3 ++ src/urllib3/poolmanager.py | 7 +++-- src/urllib3/util/retry.py | 3 +- src/urllib3/util/ssl_.py | 5 ++- test/contrib/test_pyopenssl.py | 5 ++- test/test_retry.py | 6 ++-- test/test_ssl.py | 37 +++++++++++++++++++++++ test/with_dummyserver/test_https.py | 20 +++++++++++- test/with_dummyserver/test_poolmanager.py | 26 ++++++++++++++++ 14 files changed, 140 insertions(+), 11 deletions(-) create mode 100644 dummyserver/certs/server.ipv6_san.crt diff --git a/CHANGES.rst b/CHANGES.rst index ddc5d6ad78..1e264e281c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,12 +1,23 @@ Changes ======= +1.24.2 (2019-04-17) +------------------- + +* Don't load system certificates by default when any other ``ca_certs``, ``ca_certs_dir`` or + ``ssl_context`` parameters are specified. + +* Remove Authorization header regardless of case when redirecting to cross-site. (Issue #1510) + +* Add support for IPv6 addresses in subjectAltName section of certificates. (Issue #1269) + + 1.24.1 (2018-11-02) ------------------- * Remove quadratic behavior within ``GzipDecoder.decompress()`` (Issue #1467) -* Restored functionality of `ciphers` parameter for `create_urllib3_context()`. (Issue #1462) +* Restored functionality of ``ciphers`` parameter for ``create_urllib3_context()``. (Issue #1462) 1.24 (2018-10-16) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 71813f95f5..02b24fc7fc 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -272,5 +272,8 @@ In chronological order: * Justin Bramley * Add ability to handle multiple Content-Encodings +* Katsuhiko YOSHIDA + * Remove Authorization header regardless of case when redirecting to cross-site + * [Your name or handle] <[email or website]> * [Brief summary of your changes] diff --git a/dummyserver/certs/server.ipv6_san.crt b/dummyserver/certs/server.ipv6_san.crt new file mode 100644 index 0000000000..64acace329 --- /dev/null +++ b/dummyserver/certs/server.ipv6_san.crt @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICfTCCAeagAwIBAgIJAPcpn3/M5+piMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTgxMjE5MDUyMjUyWhcNNDgxMjE4MDUyMjUyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB +gQDXe3FqmCWvP8XPxqtT+0bfL1Tvzvebi46k0WIcUV8bP3vyYiSRXG9ALmyzZH4G +HY9UVs4OEDkCMDOBSezB0y9ai/9doTNcaictdEBu8nfdXKoTtzrn+VX4UPrkH5hm +7NQ1fTQuj1MR7yBCmYqN3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQABo3UwczAdBgNV +HQ4EFgQUG+dK5Uos08QUwAWofDb3a8YcYlIwHwYDVR0jBBgwFoAUG+dK5Uos08QU +wAWofDb3a8YcYlIwDwYDVR0TAQH/BAUwAwEB/zAgBgNVHREEGTAXggM6OjGHEAAA +AAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQADgYEAjT767TDq6q4lOextf3tZ +BjeuYDUy7bb1fDBAN5rBT1ywr7r0JE6/KOnsZx4jbevx3MllxNpx0gOM2bgwJlnG +8tgwRB6pxDyln01WBj9b5ymK60jdkw7gg3yYpqEs5/VBQidFO3BmDqf5cGO8PU7p +0VWdfJBP2UbwblNXdImI1zk= +-----END CERTIFICATE----- diff --git a/dummyserver/server.py b/dummyserver/server.py index c7da0e9823..b5d21592e0 100755 --- a/dummyserver/server.py +++ b/dummyserver/server.py @@ -58,11 +58,16 @@ 'certfile': os.path.join(CERTS_PATH, 'server.ipv6addr.crt'), 'keyfile': os.path.join(CERTS_PATH, 'server.ipv6addr.key'), } +IPV6_SAN_CERTS = { + 'certfile': os.path.join(CERTS_PATH, 'server.ipv6_san.crt'), + 'keyfile': DEFAULT_CERTS['keyfile'] +} DEFAULT_CA = os.path.join(CERTS_PATH, 'cacert.pem') DEFAULT_CA_BAD = os.path.join(CERTS_PATH, 'client_bad.pem') NO_SAN_CA = os.path.join(CERTS_PATH, 'cacert.no_san.pem') DEFAULT_CA_DIR = os.path.join(CERTS_PATH, 'ca_path_test') IPV6_ADDR_CA = os.path.join(CERTS_PATH, 'server.ipv6addr.crt') +IPV6_SAN_CA = os.path.join(CERTS_PATH, 'server.ipv6_san.crt') COMBINED_CERT_AND_KEY = os.path.join(CERTS_PATH, 'server.combined.pem') diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index 148a9c31a7..61915465f6 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -27,7 +27,7 @@ __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.24.1' +__version__ = '1.24.2' __all__ = ( 'HTTPConnectionPool', diff --git a/src/urllib3/contrib/pyopenssl.py b/src/urllib3/contrib/pyopenssl.py index 7c0e9465d9..5ab7803ac5 100644 --- a/src/urllib3/contrib/pyopenssl.py +++ b/src/urllib3/contrib/pyopenssl.py @@ -184,6 +184,9 @@ def idna_encode(name): except idna.core.IDNAError: return None + if ':' in name: + return name + name = idna_encode(name) if name is None: return None diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index fe5491cfda..32bd973021 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -7,6 +7,7 @@ from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from .packages import six from .packages.six.moves.urllib.parse import urljoin from .request import RequestMethods from .util.url import parse_url @@ -342,8 +343,10 @@ def urlopen(self, method, url, redirect=True, **kw): # conn.is_same_host() which may use socket.gethostbyname() in the future. if (retries.remove_headers_on_redirect and not conn.is_same_host(redirect_location)): - for header in retries.remove_headers_on_redirect: - kw['headers'].pop(header, None) + headers = list(six.iterkeys(kw['headers'])) + for header in headers: + if header.lower() in retries.remove_headers_on_redirect: + kw['headers'].pop(header, None) try: retries = retries.increment(method, url, response=response, _pool=conn) diff --git a/src/urllib3/util/retry.py b/src/urllib3/util/retry.py index e7d0abd610..02429ee8e4 100644 --- a/src/urllib3/util/retry.py +++ b/src/urllib3/util/retry.py @@ -179,7 +179,8 @@ def __init__(self, total=10, connect=None, read=None, redirect=None, status=None self.raise_on_status = raise_on_status self.history = history or tuple() self.respect_retry_after_header = respect_retry_after_header - self.remove_headers_on_redirect = remove_headers_on_redirect + self.remove_headers_on_redirect = frozenset([ + h.lower() for h in remove_headers_on_redirect]) def new(self, **kw): params = dict( diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index 64ea192a85..5ae4358273 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -327,7 +327,10 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if e.errno == errno.ENOENT: raise SSLError(e) raise - elif getattr(context, 'load_default_certs', None) is not None: + + # Don't load system certs unless there were no CA certs or + # SSLContext object specified manually. + elif ssl_context is None and hasattr(context, 'load_default_certs'): # try to load OS default certs; works well on Windows (require Python3.4+) context.load_default_certs() diff --git a/test/contrib/test_pyopenssl.py b/test/contrib/test_pyopenssl.py index 7fc296f31b..f08aa01893 100644 --- a/test/contrib/test_pyopenssl.py +++ b/test/contrib/test_pyopenssl.py @@ -31,7 +31,10 @@ def teardown_module(): pass -from ..with_dummyserver.test_https import TestHTTPS, TestHTTPS_TLSv1 # noqa: F401 +from ..with_dummyserver.test_https import ( # noqa: F401 + TestHTTPS, TestHTTPS_TLSv1, TestHTTPS_IPv6Addr, + TestHTTPS_IPSAN, TestHTTPS_NoSAN, TestHTTPS_IPV6SAN +) from ..with_dummyserver.test_socketlevel import ( # noqa: F401 TestSNI, TestSocketClosing, TestClientCerts ) diff --git a/test/test_retry.py b/test/test_retry.py index b4119b101b..7546c43fea 100644 --- a/test/test_retry.py +++ b/test/test_retry.py @@ -253,9 +253,9 @@ def test_retry_method_not_in_whitelist(self): def test_retry_default_remove_headers_on_redirect(self): retry = Retry() - assert list(retry.remove_headers_on_redirect) == ['Authorization'] + assert list(retry.remove_headers_on_redirect) == ['authorization'] def test_retry_set_remove_headers_on_redirect(self): - retry = Retry(remove_headers_on_redirect=['X-API-Secret']) + retry = Retry(remove_headers_on_redirect=['x-api-secret']) - assert list(retry.remove_headers_on_redirect) == ['X-API-Secret'] + assert list(retry.remove_headers_on_redirect) == ['x-api-secret'] diff --git a/test/test_ssl.py b/test/test_ssl.py index 47359717d2..6a46b4f3ea 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -88,3 +88,40 @@ def test_create_urllib3_context_set_ciphers(monkeypatch, ciphers, expected_ciphe assert context.set_ciphers.call_count == 1 assert context.set_ciphers.call_args == mock.call(expected_ciphers) + + +def test_wrap_socket_given_context_no_load_default_certs(): + context = mock.create_autospec(ssl_.SSLContext) + context.load_default_certs = mock.Mock() + + sock = mock.Mock() + ssl_.ssl_wrap_socket(sock, ssl_context=context) + + context.load_default_certs.assert_not_called() + + +def test_wrap_socket_given_ca_certs_no_load_default_certs(monkeypatch): + context = mock.create_autospec(ssl_.SSLContext) + context.load_default_certs = mock.Mock() + context.options = 0 + + monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) + + sock = mock.Mock() + ssl_.ssl_wrap_socket(sock, ca_certs="/tmp/fake-file") + + context.load_default_certs.assert_not_called() + context.load_verify_locations.assert_called_with("/tmp/fake-file", None) + + +def test_wrap_socket_default_loads_default_certs(monkeypatch): + context = mock.create_autospec(ssl_.SSLContext) + context.load_default_certs = mock.Mock() + context.options = 0 + + monkeypatch.setattr(ssl_, "SSLContext", lambda *_, **__: context) + + sock = mock.Mock() + ssl_.ssl_wrap_socket(sock) + + context.load_default_certs.assert_called_with() diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index 082ede96f8..acc149c307 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -17,7 +17,7 @@ DEFAULT_CLIENT_NO_INTERMEDIATE_CERTS, NO_SAN_CERTS, NO_SAN_CA, DEFAULT_CA_DIR, IPV6_ADDR_CERTS, IPV6_ADDR_CA, HAS_IPV6, - IP_SAN_CERTS) + IP_SAN_CERTS, IPV6_SAN_CA, IPV6_SAN_CERTS) from test import ( onlyPy279OrNewer, @@ -625,5 +625,23 @@ def test_strip_square_brackets_before_validating(self): self.assertEqual(r.status, 200) +class TestHTTPS_IPV6SAN(IPV6HTTPSDummyServerTestCase): + certs = IPV6_SAN_CERTS + + def test_can_validate_ipv6_san(self): + """Ensure that urllib3 can validate SANs with IPv6 addresses in them.""" + try: + import ipaddress # noqa: F401 + except ImportError: + pytest.skip("Only runs on systems with an ipaddress module") + + https_pool = HTTPSConnectionPool('[::1]', self.port, + cert_reqs='CERT_REQUIRED', + ca_certs=IPV6_SAN_CA) + self.addCleanup(https_pool.close) + r = https_pool.request('GET', '/') + self.assertEqual(r.status, 200) + + if __name__ == '__main__': unittest.main() diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index 2a13722c66..3c1eef8dac 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -123,6 +123,17 @@ def test_redirect_cross_host_remove_headers(self): self.assertNotIn('Authorization', data) + r = http.request('GET', '%s/redirect' % self.base_url, + fields={'target': '%s/headers' % self.base_url_alt}, + headers={'authorization': 'foo'}) + + self.assertEqual(r.status, 200) + + data = json.loads(r.data.decode('utf-8')) + + self.assertNotIn('authorization', data) + self.assertNotIn('Authorization', data) + def test_redirect_cross_host_no_remove_headers(self): http = PoolManager() self.addCleanup(http.clear) @@ -155,6 +166,21 @@ def test_redirect_cross_host_set_removed_headers(self): self.assertNotIn('X-API-Secret', data) self.assertEqual(data['Authorization'], 'bar') + r = http.request('GET', '%s/redirect' % self.base_url, + fields={'target': '%s/headers' % self.base_url_alt}, + headers={'x-api-secret': 'foo', + 'authorization': 'bar'}, + retries=Retry(remove_headers_on_redirect=['X-API-Secret'])) + + self.assertEqual(r.status, 200) + + data = json.loads(r.data.decode('utf-8')) + + self.assertNotIn('x-api-secret', data) + self.assertNotIn('X-API-Secret', data) + + self.assertEqual(data['Authorization'], 'bar') + def test_raise_on_redirect(self): http = PoolManager() self.addCleanup(http.clear)