Add support for TLS 1.3 to all HTTPSConnection implementations (#1496)

* Add tests for specific TLS/SSL versions

* Add change and update bindings

* SSLSocket.version() not available sometimes

* Add support for kTLSProtocolMaxSupported

* Try setProtocolVersionMax again if error

* Get ctypes.c_uint.value for SSLSocket.version()

* Opt-in TLS 1.3 on macOS 10.13

* Update tornado to 5.1.1

* Add documentation updates for TLSv1.3

* Add wbond/oscrypto license to contrib/securetransport

* Remove all TLS 1.3 ciphersuites from DEFAULT_CIPHERS

* Experiment showing cipher list per protocol

* Update test_https.py

* Update test_https.py

* Update test_https.py

* Update changelog wording to exclude pyOpenSSL

* minor rewording

* Add support for IPv6 in subjectAltName

* Don't use OP_ALL

* Update CHANGES.rst

* No PROTOCOL_TLSv1_3

* Remove DSS, rearrange SecureTransport ciphers

* Use ECDSA before RSA with ECDHE

* ReviReorder ciphers

* ECDHE

* Update test_https.py

* Turns out we don't need version detection

* Reorder per Hyneks post and favor ephemeral

* Refactor HTTPS unit tests

* Fix up tests

* Test locking pytest-httpbin

* Update requests.sh

* remove whitespace

CHANGES.rst

@@ -22,6 +22,14 @@ dev (master)
   ``brotlipy`` package is installed which can be requested with
   ``urllib3[brotli]`` extra. (Pull #1532)
 
+* Add TLSv1.3 support to CPython, pyOpenSSL, and SecureTransport ``SSLContext``
+  implementations. (Pull #1496)
+
+* Drop ciphers using DSS key exchange from default TLS cipher suites.
+  Improve default ciphers when using SecureTransport. (Pull #1496)
+
+* Add support for IPv6 addresses in subjectAltName section of certificates. (Issue #1269)
+
 * ... [Short description of non-trivial change.] (Issue #)
 
 

_travis/downstream/requests.sh

@@ -4,11 +4,14 @@ set -exo pipefail
 
 case "${1}" in
     install)
-        git clone --depth 1 https://github.com/requests/requests
+        git clone --depth 1 https://github.com/kennethreitz/requests
         cd requests
         git rev-parse HEAD
         python -m pip install --upgrade pipenv
         pipenv install --dev --skip-lock
+
+        # See: kennethreitz/requests/5004
+        python -m pip install pytest-httpbin==0.3.0
         ;;
     run)
         cd requests

_travis/install.sh

@@ -20,6 +20,9 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
     # The pip in older MacPython releases doesn't support a new enough TLS
     curl https://bootstrap.pypa.io/get-pip.py | sudo $PYTHON_EXE
     $PYTHON_EXE -m pip install virtualenv
+
+    # Enable TLS 1.3 on macOS
+    sudo defaults write /Library/Preferences/com.apple.networkd tcp_connect_enable_tls13 1
 else
     python -m pip install virtualenv
 fi

dev-requirements.txt

@@ -2,7 +2,7 @@ mock==2.0.0
 coverage~=4.5
 tox==2.9.1
 wheel==0.30.0
-tornado==5.0.2
+tornado==5.1.1
 PySocks==1.6.8
 pkginfo==1.4.2
 pytest-timeout==1.3.1

dummyserver/certs/server.ipv6_san.crt

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICfTCCAeagAwIBAgIJAPcpn3/M5+piMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
+BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
+aWRnaXRzIFB0eSBMdGQwHhcNMTgxMjE5MDUyMjUyWhcNNDgxMjE4MDUyMjUyWjBF
+MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
+ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+gQDXe3FqmCWvP8XPxqtT+0bfL1Tvzvebi46k0WIcUV8bP3vyYiSRXG9ALmyzZH4G
+HY9UVs4OEDkCMDOBSezB0y9ai/9doTNcaictdEBu8nfdXKoTtzrn+VX4UPrkH5hm
+7NQ1fTQuj1MR7yBCmYqN3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQABo3UwczAdBgNV
+HQ4EFgQUG+dK5Uos08QUwAWofDb3a8YcYlIwHwYDVR0jBBgwFoAUG+dK5Uos08QU
+wAWofDb3a8YcYlIwDwYDVR0TAQH/BAUwAwEB/zAgBgNVHREEGTAXggM6OjGHEAAA
+AAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQADgYEAjT767TDq6q4lOextf3tZ
+BjeuYDUy7bb1fDBAN5rBT1ywr7r0JE6/KOnsZx4jbevx3MllxNpx0gOM2bgwJlnG
+8tgwRB6pxDyln01WBj9b5ymK60jdkw7gg3yYpqEs5/VBQidFO3BmDqf5cGO8PU7p
+0VWdfJBP2UbwblNXdImI1zk=
+-----END CERTIFICATE-----

dummyserver/server.py

@@ -60,11 +60,16 @@
     'certfile': os.path.join(CERTS_PATH, 'server.ipv6addr.crt'),
     'keyfile': os.path.join(CERTS_PATH, 'server.ipv6addr.key'),
 }
+IPV6_SAN_CERTS = {
+    'certfile': os.path.join(CERTS_PATH, 'server.ipv6_san.crt'),
+    'keyfile': DEFAULT_CERTS['keyfile']
+}
 DEFAULT_CA = os.path.join(CERTS_PATH, 'cacert.pem')
 DEFAULT_CA_BAD = os.path.join(CERTS_PATH, 'client_bad.pem')
 NO_SAN_CA = os.path.join(CERTS_PATH, 'cacert.no_san.pem')
 DEFAULT_CA_DIR = os.path.join(CERTS_PATH, 'ca_path_test')
 IPV6_ADDR_CA = os.path.join(CERTS_PATH, 'server.ipv6addr.crt')
+IPV6_SAN_CA = os.path.join(CERTS_PATH, 'server.ipv6_san.crt')
 COMBINED_CERT_AND_KEY = os.path.join(CERTS_PATH, 'server.combined.pem')
 
 

src/urllib3/contrib/_securetransport/bindings.py

@@ -516,6 +516,8 @@ class SecurityConst(object):
     kTLSProtocol1 = 4
     kTLSProtocol11 = 7
     kTLSProtocol12 = 8
+    kTLSProtocol13 = 10
+    kTLSProtocolMaxSupported = 999
 
     kSSLClientSide = 1
     kSSLStreamType = 0
@@ -558,30 +560,27 @@ class SecurityConst(object):
     errSecInvalidTrustSettings = -25262
 
     # Cipher suites. We only pick the ones our default cipher string allows.
+    # Source: https://developer.apple.com/documentation/security/1550981-ssl_cipher_suite_values
     TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C
     TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030
     TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B
     TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F
-    TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3
+    TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA9
+    TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA8
     TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F
-    TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2
     TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E
     TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024
     TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028
     TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A
     TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014
     TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B
-    TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A
     TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039
-    TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038
     TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023
     TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027
     TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009
     TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013
     TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067
-    TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040
     TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033
-    TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032
     TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D
     TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C
     TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D
@@ -590,4 +589,5 @@ class SecurityConst(object):
     TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F
     TLS_AES_128_GCM_SHA256 = 0x1301
     TLS_AES_256_GCM_SHA384 = 0x1302
-    TLS_CHACHA20_POLY1305_SHA256 = 0x1303
+    TLS_AES_128_CCM_8_SHA256 = 0x1305
+    TLS_AES_128_CCM_SHA256 = 0x1304

src/urllib3/contrib/pyopenssl.py

@@ -70,27 +70,27 @@ class UnsupportedExtension(Exception):
 
 from .. import util
 
+
 __all__ = ['inject_into_urllib3', 'extract_from_urllib3']
 
 # SNI always works.
 HAS_SNI = True
 
 # Map from urllib3 to PyOpenSSL compatible parameter-values.
 _openssl_versions = {
-    ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD,
+    util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD,
     ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
 }
 
+if hasattr(ssl, 'PROTOCOL_SSLv3') and hasattr(OpenSSL.SSL, 'SSLv3_METHOD'):
+    _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD
+
 if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'):
     _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
 
 if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'):
     _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD
 
-try:
-    _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD})
-except AttributeError:
-    pass
 
 _stdlib_to_openssl_verify = {
     ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE,
@@ -186,6 +186,10 @@ def idna_encode(name):
         except idna.core.IDNAError:
             return None
 
+    # Don't send IPv6 addresses through the IDNA encoder.
+    if ':' in name:
+        return name
+
     name = idna_encode(name)
     if name is None:
         return None
@@ -288,6 +292,10 @@ def recv(self, *args, **kwargs):
                 raise timeout('The read operation timed out')
             else:
                 return self.recv(*args, **kwargs)
+
+        # TLS 1.3 post-handshake authentication
+        except OpenSSL.SSL.Error as e:
+            raise ssl.SSLError("read error: %r" % e)
         else:
             return data
 
@@ -310,6 +318,10 @@ def recv_into(self, *args, **kwargs):
             else:
                 return self.recv_into(*args, **kwargs)
 
+        # TLS 1.3 post-handshake authentication
+        except OpenSSL.SSL.Error as e:
+            raise ssl.SSLError("read error: %r" % e)
+
     def settimeout(self, timeout):
         return self.socket.settimeout(timeout)
 
@@ -362,6 +374,9 @@ def getpeercert(self, binary_form=False):
             'subjectAltName': get_subj_alt_name(x509)
         }
 
+    def version(self):
+        return self.connection.get_protocol_version_name()
+
     def _reuse(self):
         self._makefile_refs += 1
 

src/urllib3/contrib/securetransport.py

@@ -23,6 +23,31 @@
     urllib3.contrib.securetransport.inject_into_urllib3()
 
 Happy TLSing!
+
+This code is a bastardised version of the code found in Will Bond's oscrypto
+library. An enormous debt is owed to him for blazing this trail for us. For
+that reason, this code should be considered to be covered both by urllib3's
+license and by oscrypto's:
+
+    Copyright (c) 2015-2016 Will Bond <will@wbond.net>
+
+    Permission is hereby granted, free of charge, to any person obtaining a
+    copy of this software and associated documentation files (the "Software"),
+    to deal in the Software without restriction, including without limitation
+    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+    and/or sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
 """
 from __future__ import absolute_import
 
@@ -86,45 +111,43 @@
 # individual cipher suites. We need to do this because this is how
 # SecureTransport wants them.
 CIPHER_SUITES = [
-    SecurityConst.TLS_AES_256_GCM_SHA384,
-    SecurityConst.TLS_CHACHA20_POLY1305_SHA256,
-    SecurityConst.TLS_AES_128_GCM_SHA256,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
     SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_256_GCM_SHA384,
+    SecurityConst.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,
+    SecurityConst.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
     SecurityConst.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_128_GCM_SHA256,
     SecurityConst.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384,
-    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
-    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
-    SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA256,
-    SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,
-    SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
     SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384,
+    SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+    SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,
     SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+    SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,
+    SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA,
     SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA256,
     SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA,
-    SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA,
+    SecurityConst.TLS_AES_256_GCM_SHA384,
+    SecurityConst.TLS_AES_128_GCM_SHA256,
     SecurityConst.TLS_RSA_WITH_AES_256_GCM_SHA384,
     SecurityConst.TLS_RSA_WITH_AES_128_GCM_SHA256,
+    SecurityConst.TLS_AES_128_CCM_8_SHA256,
+    SecurityConst.TLS_AES_128_CCM_SHA256,
     SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA256,
     SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA256,
     SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA,
     SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA,
 ]
 
 # Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of
-# TLSv1 and a high of TLSv1.2. For everything else, we pin to that version.
+# TLSv1 and a high of TLSv1.3. For everything else, we pin to that version.
+# TLSv1 to 1.2 are supported on macOS 10.8+ and TLSv1.3 is macOS 10.13+
 _protocol_to_min_max = {
-    ssl.PROTOCOL_SSLv23: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12),
+    util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocolMaxSupported),
 }
 
 if hasattr(ssl, "PROTOCOL_SSLv2"):
@@ -147,8 +170,6 @@
     _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = (
         SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12
     )
-if hasattr(ssl, "PROTOCOL_TLS"):
-    _protocol_to_min_max[ssl.PROTOCOL_TLS] = _protocol_to_min_max[ssl.PROTOCOL_SSLv23]
 
 
 def inject_into_urllib3():
@@ -460,7 +481,14 @@ def handshake(self,
         # Set the minimum and maximum TLS versions.
         result = Security.SSLSetProtocolVersionMin(self.context, min_version)
         _assert_no_error(result)
+
+        # TLS 1.3 isn't necessarily enabled by the OS
+        # so we have to detect when we error out and try
+        # setting TLS 1.3 if it's allowed. kTLSProtocolMaxSupported
+        # was added in macOS 10.13 along with kTLSProtocol13.
         result = Security.SSLSetProtocolVersionMax(self.context, max_version)
+        if result != 0 and max_version == SecurityConst.kTLSProtocolMaxSupported:
+            result = Security.SSLSetProtocolVersionMax(self.context, SecurityConst.kTLSProtocol12)
         _assert_no_error(result)
 
         # If there's a trust DB, we need to use it. We do that by telling
@@ -669,6 +697,25 @@ def getpeercert(self, binary_form=False):
 
         return der_bytes
 
+    def version(self):
+        protocol = Security.SSLProtocol()
+        result = Security.SSLGetNegotiatedProtocolVersion(self.context, ctypes.byref(protocol))
+        _assert_no_error(result)
+        if protocol.value == SecurityConst.kTLSProtocol13:
+            return 'TLSv1.3'
+        elif protocol.value == SecurityConst.kTLSProtocol12:
+            return 'TLSv1.2'
+        elif protocol.value == SecurityConst.kTLSProtocol11:
+            return 'TLSv1.1'
+        elif protocol.value == SecurityConst.kTLSProtocol1:
+            return 'TLSv1'
+        elif protocol.value == SecurityConst.kSSLProtocol3:
+            return 'SSLv3'
+        elif protocol.value == SecurityConst.kSSLProtocol2:
+            return 'SSLv2'
+        else:
+            raise ssl.SSLError('Unknown TLS version: %r' % protocol)
+
     def _reuse(self):
         self._makefile_refs += 1
 

src/urllib3/util/__init__.py

@@ -12,6 +12,7 @@
     resolve_cert_reqs,
     resolve_ssl_version,
     ssl_wrap_socket,
+    PROTOCOL_TLS,
 )
 from .timeout import (
     current_time,
@@ -35,6 +36,7 @@
     'IS_PYOPENSSL',
     'IS_SECURETRANSPORT',
     'SSLContext',
+    'PROTOCOL_TLS',
     'Retry',
     'Timeout',
     'Url',