diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index a6993ea9e1..7cc1a23e9e 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,2 +1,3 @@ tidelift: pypi/urllib3 open_collective: urllib3 +custom: https://gitcoin.co/grants/65/urllib3 diff --git a/.github/ISSUE_TEMPLATE/01_feature_request.md b/.github/ISSUE_TEMPLATE/01_feature_request.md new file mode 100644 index 0000000000..90a531fefb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/01_feature_request.md @@ -0,0 +1,26 @@ +--- +name: 🎁 Feature Request +about: Suggest a new feature +--- + +### Context + +What are you trying to do? +How do you expect to do it? +Is it something you currently cannot do? +Is this related to an existing issue/problem? + +### Alternatives + +Can you achieve the same result doing it in an alternative way? +Is the alternative considerable? + +### Duplicate + +Has the feature been requested before? +If so, please provide a link to the issue. + +### Contribution + +Would you be willing to submit a PR? +_(Help can be provided if you need assistance submitting a PR)_ diff --git a/.github/ISSUE_TEMPLATE/02_bug_report.md b/.github/ISSUE_TEMPLATE/02_bug_report.md new file mode 100644 index 0000000000..4d4c37c105 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/02_bug_report.md @@ -0,0 +1,35 @@ +--- +name: 🐞 Bug Report +about: Something is broken +--- + +### Subject + +Describe the issue here. + +### Environment + +Describe your environment. +At least, paste here the output of: + +```python +import platform +import urllib3 + +print("OS", platform.platform()) +print("Python", platform.python_version()) +print("urllib3", urllib3.__version__) +``` + +### Steps to Reproduce + +A simple and isolated way to reproduce the issue. A code snippet would be great. + +### Expected Behavior + +What should happen. + +### Actual Behavior + +What happens instead. +You may attach logs, packet captures, etc. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..9b435ceda4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: 📚 Documentation + url: https://urllib3.readthedocs.io/en/latest/ + about: Make sure you read the relevant docs + - name: ❓ Ask on StackOverflow + url: https://stackoverflow.com/questions/tagged/urllib3 + about: Ask questions about usage in StackOverflow + - name: 💬 Ask the Community + url: https://discord.gg/CHEgCZN + about: Join urllib3's Discord server diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..e5765d5108 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,8 @@ + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6b6bd75ea..5ca94207b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,9 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Set up Python 3.8 - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: 3.8 - name: Install dependencies diff --git a/.travis.yml b/.travis.yml index 3c64246036..ca52bf2681 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,10 +32,6 @@ jobs: allow_failures: - python: nightly include: - # Lint & documentation. - - python: 3.8 - env: NOX_SESSION=docs - # Unit tests - python: 2.7 env: NOX_SESSION=test-2.7 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..0079d14e98 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ + +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at @sethmlarson or @shazow. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/README.rst b/README.rst index 90c1f0da00..5529a8f97a 100644 --- a/README.rst +++ b/README.rst @@ -6,11 +6,12 @@

- Build status on Travis - Build status on GitHub - Documentation Status + PyPI Version + Join our Discord Coverage Status - PyPI Version + Build Status on GitHub + Build Status on Travis + Documentation Status

urllib3 is a powerful, *user-friendly* HTTP client for Python. Much of the diff --git a/docs/conf.py b/docs/conf.py index 47470b5073..5d5dda25b6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -from datetime import date import os import sys +from datetime import date # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -30,7 +30,6 @@ def __getattr__(cls, name): import urllib3 - # -- General configuration ----------------------------------------------------- diff --git a/docs/contributing.rst b/docs/contributing.rst index 4102e01d74..05cac9ca0e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -13,7 +13,7 @@ If you wish to add a new feature or fix a bug: to start making your changes. #. Write a test which shows that the bug was fixed or that the feature works as expected. -#. Format your changes with black using command `$ nox -rs blacken` and lint your +#. Format your changes with black using command `$ nox -rs format` and lint your changes using command `nox -rs lint`. #. Send a pull request and bug the maintainer until it gets merged and published. :) Make sure to add yourself to ``CONTRIBUTORS.txt``. diff --git a/docs/images/banner.svg b/docs/images/banner.svg index 4659db6d72..86e6661e72 100644 --- a/docs/images/banner.svg +++ b/docs/images/banner.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/docs/reference/urllib3.util.rst b/docs/reference/urllib3.util.rst index 663efd6b68..cb51215e80 100644 --- a/docs/reference/urllib3.util.rst +++ b/docs/reference/urllib3.util.rst @@ -14,5 +14,4 @@ but can also be used independently. .. automodule:: urllib3.util :members: - :undoc-members: :show-inheritance: diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py index 1c8e9b6c86..c047094c4f 100644 --- a/dummyserver/handlers.py +++ b/dummyserver/handlers.py @@ -8,16 +8,15 @@ import sys import time import zlib - +from datetime import datetime, timedelta from io import BytesIO -from tornado.web import RequestHandler + from tornado import httputil -from datetime import datetime -from datetime import timedelta +from tornado.web import RequestHandler +from urllib3.packages.six import binary_type, ensure_str from urllib3.packages.six.moves.http_client import responses from urllib3.packages.six.moves.urllib.parse import urlsplit -from urllib3.packages.six import binary_type, ensure_str log = logging.getLogger(__name__) diff --git a/dummyserver/proxy.py b/dummyserver/proxy.py index 42f293104d..0cd8dedd26 100755 --- a/dummyserver/proxy.py +++ b/dummyserver/proxy.py @@ -25,16 +25,16 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -import sys import socket +import ssl +import sys import tornado.gen +import tornado.httpclient import tornado.httpserver import tornado.ioloop import tornado.iostream import tornado.web -import tornado.httpclient -import ssl __all__ = ["ProxyHandler", "run_proxy"] diff --git a/dummyserver/server.py b/dummyserver/server.py index f8b4dff5ce..9ecde97f35 100755 --- a/dummyserver/server.py +++ b/dummyserver/server.py @@ -7,24 +7,23 @@ import logging import os +import socket +import ssl import sys import threading -import socket import warnings -import ssl from datetime import datetime -from urllib3.exceptions import HTTPWarning -from urllib3.util import resolve_cert_reqs, resolve_ssl_version, ALPN_PROTOCOLS - -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization import tornado.httpserver import tornado.ioloop import tornado.netutil import tornado.web import trustme +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from urllib3.exceptions import HTTPWarning +from urllib3.util import ALPN_PROTOCOLS, resolve_cert_reqs, resolve_ssl_version log = logging.getLogger(__name__) diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py index 5a54922d4b..6a49e36cd2 100644 --- a/dummyserver/testcase.py +++ b/dummyserver/testcase.py @@ -4,17 +4,16 @@ import pytest from tornado import ioloop, web -from urllib3.connection import HTTPConnection - +from dummyserver.handlers import TestingApp +from dummyserver.proxy import ProxyHandler from dummyserver.server import ( - SocketServerThread, - run_tornado_app, - run_loop_in_thread, DEFAULT_CERTS, HAS_IPV6, + SocketServerThread, + run_loop_in_thread, + run_tornado_app, ) -from dummyserver.handlers import TestingApp -from dummyserver.proxy import ProxyHandler +from urllib3.connection import HTTPConnection def consume_socket(sock, chunks=65536): diff --git a/noxfile.py b/noxfile.py index 4b1cf3a11d..a8667583ae 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,7 +4,6 @@ import nox - # Whenever type-hints are completed on a file it should be added here so that # this file will continue to be checked by mypy. Errors from other files are # ignored. @@ -101,21 +100,24 @@ def app_engine(session): @nox.session() -def blacken(session): - """Run black code formatter.""" - session.install("black") +def format(session): + """Run code formatters.""" + session.install("black", "isort") session.run("black", *SOURCE_FILES) + session.run("isort", "--profile", "black", *SOURCE_FILES) lint(session) @nox.session def lint(session): - session.install("flake8", "black", "mypy") + session.install("flake8", "flake8-2020", "black", "isort", "mypy") session.run("flake8", "--version") session.run("black", "--version") + session.run("isort", "--version") session.run("mypy", "--version") session.run("black", "--check", *SOURCE_FILES) + session.run("isort", "--profile", "black", "--check", *SOURCE_FILES) session.run("flake8", *SOURCE_FILES) session.log("mypy --strict src/urllib3") diff --git a/setup.py b/setup.py index 8a96b9942b..d5030fbd79 100755 --- a/setup.py +++ b/setup.py @@ -1,11 +1,11 @@ #!/usr/bin/env python # This file is protected via CODEOWNERS -from setuptools import setup - +import codecs import os import re -import codecs + +from setuptools import setup base_path = os.path.dirname(__file__) diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py index ac3f9efa55..fe86b59d78 100644 --- a/src/urllib3/__init__.py +++ b/src/urllib3/__init__.py @@ -2,24 +2,22 @@ Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more """ from __future__ import absolute_import -import warnings -from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url +# Set default logging handler to avoid "No handler found" warnings. +import logging +import warnings +from logging import NullHandler from . import exceptions +from ._version import __version__ +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url from .filepost import encode_multipart_formdata from .poolmanager import PoolManager, ProxyManager, proxy_from_url from .response import HTTPResponse from .util.request import make_headers -from .util.url import get_host -from .util.timeout import Timeout from .util.retry import Retry -from ._version import __version__ - - -# Set default logging handler to avoid "No handler found" warnings. -import logging -from logging import NullHandler +from .util.timeout import Timeout +from .util.url import get_host __author__ = "Andrey Petrov (andrey.petrov@shazow.net)" __license__ = "MIT" diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py index 5c8f366e34..8a94ba5f74 100644 --- a/src/urllib3/_collections.py +++ b/src/urllib3/_collections.py @@ -17,9 +17,10 @@ def __exit__(self, exc_type, exc_value, traceback): from collections import OrderedDict -from .exceptions import InvalidHeader -from .packages.six import ensure_str, iterkeys, itervalues, PY3 +from .exceptions import InvalidHeader +from .packages import six +from .packages.six import iterkeys, itervalues __all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"] @@ -154,7 +155,7 @@ def __setitem__(self, key, val): def __getitem__(self, key): val = self._container[key.lower()] - return ", ".join([ensure_str(v, "ascii") for v in val[1:]]) + return ", ".join([six.ensure_str(v, "ascii") for v in val[1:]]) def __delitem__(self, key): del self._container[key.lower()] @@ -174,7 +175,7 @@ def __eq__(self, other): def __ne__(self, other): return not self.__eq__(other) - if not PY3: # Python 2 + if six.PY2: # Python 2 iterkeys = MutableMapping.iterkeys itervalues = MutableMapping.itervalues diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py index 81474178cc..f3336acff1 100644 --- a/src/urllib3/connection.py +++ b/src/urllib3/connection.py @@ -1,14 +1,18 @@ from __future__ import absolute_import -import re + import datetime import logging import os +import re import socket -from socket import error as SocketError, timeout as SocketTimeout import warnings +from socket import error as SocketError +from socket import timeout as SocketTimeout + from .packages import six from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection from .packages.six.moves.http_client import HTTPException # noqa: F401 +from .util.proxy import create_proxy_ssl_context try: # Compiled with SSL? import ssl @@ -39,28 +43,24 @@ class BrokenPipeError(Exception): pass +from ._collections import HTTPHeaderDict +from ._version import __version__ from .exceptions import ( - NewConnectionError, ConnectTimeoutError, + NewConnectionError, SubjectAltNameWarning, SystemTimeWarning, ) -from .packages.ssl_match_hostname import match_hostname, CertificateError - +from .packages.ssl_match_hostname import CertificateError, match_hostname +from .util import SUPPRESS_USER_AGENT, connection from .util.ssl_ import ( - resolve_cert_reqs, - resolve_ssl_version, assert_fingerprint, create_urllib3_context, + resolve_cert_reqs, + resolve_ssl_version, ssl_wrap_socket, ) - -from .util import connection, SUPPRESS_USER_AGENT - -from ._collections import HTTPHeaderDict -from ._version import __version__ - log = logging.getLogger(__name__) port_by_scheme = {"http": 80, "https": 443} @@ -117,6 +117,11 @@ def __init__(self, *args, **kw): #: The socket options provided by the user. If no options are #: provided, we use the default options. self.socket_options = kw.pop("socket_options", self.default_socket_options) + + # Proxy options provided by the user. + self.proxy = kw.pop("proxy", None) + self.proxy_config = kw.pop("proxy_config", None) + _HTTPConnection.__init__(self, *args, **kw) @property @@ -271,6 +276,7 @@ class HTTPSConnection(HTTPConnection): ca_cert_data = None ssl_version = None assert_fingerprint = None + tls_in_tls_required = False def __init__( self, @@ -335,8 +341,13 @@ def connect(self): # Add certificate verification conn = self._new_conn() hostname = self.host + tls_in_tls = False if self._is_using_tunnel(): + if self.tls_in_tls_required: + conn = self._connect_tls_proxy(hostname, conn) + tls_in_tls = True + self.sock = conn # Calls self._set_hostport(), so self.host is @@ -396,6 +407,7 @@ def connect(self): ca_cert_data=self.ca_cert_data, server_hostname=server_hostname, ssl_context=context, + tls_in_tls=tls_in_tls, ) if self.assert_fingerprint: @@ -428,6 +440,40 @@ def connect(self): or self.assert_fingerprint is not None ) + def _connect_tls_proxy(self, hostname, conn): + """ + Establish a TLS connection to the proxy using the provided SSL context. + """ + proxy_config = self.proxy_config + ssl_context = proxy_config.ssl_context + if ssl_context: + # If the user provided a proxy context, we assume CA and client + # certificates have already been set + return ssl_wrap_socket( + sock=conn, + server_hostname=hostname, + ssl_context=ssl_context, + ) + + ssl_context = create_proxy_ssl_context( + self.ssl_version, + self.cert_reqs, + self.ca_certs, + self.ca_cert_dir, + self.ca_cert_data, + ) + + # If no cert was provided, use only the default options for server + # certificate validation + return ssl_wrap_socket( + sock=conn, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + ca_cert_data=self.ca_cert_data, + server_hostname=hostname, + ssl_context=ssl_context, + ) + def _match_hostname(cert, asserted_hostname): try: diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py index 4547223c68..da1931eeb3 100644 --- a/src/urllib3/connectionpool.py +++ b/src/urllib3/connectionpool.py @@ -1,58 +1,53 @@ from __future__ import absolute_import + import errno import logging +import socket import sys import warnings +from socket import error as SocketError +from socket import timeout as SocketTimeout -from socket import error as SocketError, timeout as SocketTimeout -import socket - - +from .connection import ( + BaseSSLError, + BrokenPipeError, + DummyConnection, + HTTPConnection, + HTTPException, + HTTPSConnection, + VerifiedHTTPSConnection, + port_by_scheme, +) from .exceptions import ( ClosedPoolError, - ProtocolError, EmptyPoolError, HeaderParsingError, HostChangedError, + InsecureRequestWarning, LocationValueError, MaxRetryError, + NewConnectionError, + ProtocolError, ProxyError, ReadTimeoutError, SSLError, TimeoutError, - InsecureRequestWarning, - NewConnectionError, ) -from .packages.ssl_match_hostname import CertificateError from .packages import six from .packages.six.moves import queue -from .connection import ( - port_by_scheme, - DummyConnection, - HTTPConnection, - HTTPSConnection, - VerifiedHTTPSConnection, - HTTPException, - BaseSSLError, - BrokenPipeError, -) +from .packages.ssl_match_hostname import CertificateError from .request import RequestMethods from .response import HTTPResponse - from .util.connection import is_connection_dropped +from .util.proxy import connection_requires_http_tunnel +from .util.queue import LifoQueue from .util.request import set_file_position from .util.response import assert_header_parsing from .util.retry import Retry from .util.timeout import Timeout -from .util.url import ( - get_host, - parse_url, - Url, - _normalize_host as normalize_host, - _encode_target, -) -from .util.queue import LifoQueue - +from .util.url import Url, _encode_target +from .util.url import _normalize_host as normalize_host +from .util.url import get_host, parse_url xrange = six.moves.xrange @@ -182,6 +177,7 @@ def __init__( retries=None, _proxy=None, _proxy_headers=None, + _proxy_config=None, **conn_kw ): ConnectionPool.__init__(self, host, port) @@ -203,6 +199,7 @@ def __init__( self.proxy = _proxy self.proxy_headers = _proxy_headers or {} + self.proxy_config = _proxy_config # Fill the queue up so that doing get() on it will block properly for _ in xrange(maxsize): @@ -219,6 +216,9 @@ def __init__( # list. self.conn_kw.setdefault("socket_options", []) + self.conn_kw["proxy"] = self.proxy + self.conn_kw["proxy_config"] = self.proxy_config + def _new_conn(self): """ Return a fresh :class:`HTTPConnection`. @@ -621,6 +621,10 @@ def urlopen( Additional parameters are passed to :meth:`urllib3.response.HTTPResponse.from_httplib` """ + + parsed_url = parse_url(url) + destination_scheme = parsed_url.scheme + if headers is None: headers = self.headers @@ -638,7 +642,7 @@ def urlopen( if url.startswith("/"): url = six.ensure_str(_encode_target(url)) else: - url = six.ensure_str(parse_url(url).url) + url = six.ensure_str(parsed_url.url) conn = None @@ -653,10 +657,14 @@ def urlopen( # [1] release_this_conn = release_conn + http_tunnel_required = connection_requires_http_tunnel( + self.proxy, self.proxy_config, destination_scheme + ) + # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. - if self.scheme == "http" or (self.proxy and self.proxy.scheme == "https"): + if not http_tunnel_required: headers = headers.copy() headers.update(self.proxy_headers) @@ -682,7 +690,7 @@ def urlopen( is_new_proxy_conn = self.proxy is not None and not getattr( conn, "sock", None ) - if is_new_proxy_conn: + if is_new_proxy_conn and http_tunnel_required: self._prepare_proxy(conn) # Make the request on the httplib connection object. @@ -946,8 +954,10 @@ def _prepare_proxy(self, conn): improperly set Host: header to proxy's IP:port. """ - if self.proxy.scheme != "https": - conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) + conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) + + if self.proxy.scheme == "https": + conn.tls_in_tls_required = True conn.connect() diff --git a/src/urllib3/contrib/_securetransport/bindings.py b/src/urllib3/contrib/_securetransport/bindings.py index e879cb5970..11524d400b 100644 --- a/src/urllib3/contrib/_securetransport/bindings.py +++ b/src/urllib3/contrib/_securetransport/bindings.py @@ -32,21 +32,23 @@ from __future__ import absolute_import import platform -from ctypes.util import find_library from ctypes import ( - c_void_p, - c_int32, + CDLL, + CFUNCTYPE, + POINTER, + c_bool, + c_byte, c_char_p, + c_int32, + c_long, c_size_t, - c_byte, c_uint32, c_ulong, - c_long, - c_bool, + c_void_p, ) -from ctypes import CDLL, POINTER, CFUNCTYPE -from urllib3.packages.six import raise_from +from ctypes.util import find_library +from urllib3.packages.six import raise_from if platform.system() != "Darwin": raise ImportError("Only macOS is supported") diff --git a/src/urllib3/contrib/_securetransport/low_level.py b/src/urllib3/contrib/_securetransport/low_level.py index 41218a5d98..ed8120190c 100644 --- a/src/urllib3/contrib/_securetransport/low_level.py +++ b/src/urllib3/contrib/_securetransport/low_level.py @@ -10,14 +10,13 @@ import base64 import ctypes import itertools -import re import os +import re import ssl import struct import tempfile -from .bindings import Security, CoreFoundation, CFConst - +from .bindings import CFConst, CoreFoundation, Security # This regular expression is used to grab PEM data out of a PEM bundle. _PEM_CERTS_RE = re.compile( diff --git a/src/urllib3/contrib/appengine.py b/src/urllib3/contrib/appengine.py index d69340a497..aa64a0914c 100644 --- a/src/urllib3/contrib/appengine.py +++ b/src/urllib3/contrib/appengine.py @@ -39,24 +39,24 @@ """ from __future__ import absolute_import + import io import logging import warnings -from ..packages.six.moves.urllib.parse import urljoin from ..exceptions import ( HTTPError, HTTPWarning, MaxRetryError, ProtocolError, - TimeoutError, SSLError, + TimeoutError, ) - +from ..packages.six.moves.urllib.parse import urljoin from ..request import RequestMethods from ..response import HTTPResponse -from ..util.timeout import Timeout from ..util.retry import Retry +from ..util.timeout import Timeout from . import _appengine_environ try: diff --git a/src/urllib3/contrib/ntlmpool.py b/src/urllib3/contrib/ntlmpool.py index 1fd242a6e0..b2df45dcf6 100644 --- a/src/urllib3/contrib/ntlmpool.py +++ b/src/urllib3/contrib/ntlmpool.py @@ -6,12 +6,12 @@ from __future__ import absolute_import from logging import getLogger + from ntlm import ntlm from .. import HTTPSConnectionPool from ..packages.six.moves.http_client import HTTPSConnection - log = getLogger(__name__) diff --git a/src/urllib3/contrib/pyopenssl.py b/src/urllib3/contrib/pyopenssl.py index 54ee493638..0cabab1aed 100644 --- a/src/urllib3/contrib/pyopenssl.py +++ b/src/urllib3/contrib/pyopenssl.py @@ -60,8 +60,9 @@ class UnsupportedExtension(Exception): pass -from socket import timeout, error as SocketError from io import BytesIO +from socket import error as SocketError +from socket import timeout try: # Platform-specific: Python 2 from socket import _fileobject @@ -71,11 +72,10 @@ class UnsupportedExtension(Exception): import logging import ssl -from ..packages import six import sys from .. import util - +from ..packages import six __all__ = ["inject_into_urllib3", "extract_from_urllib3"] diff --git a/src/urllib3/contrib/securetransport.py b/src/urllib3/contrib/securetransport.py index 729220e122..ab092de67a 100644 --- a/src/urllib3/contrib/securetransport.py +++ b/src/urllib3/contrib/securetransport.py @@ -58,22 +58,23 @@ import errno import os.path import shutil -import six import socket import ssl import struct import threading import weakref +import six + from .. import util -from ._securetransport.bindings import Security, SecurityConst, CoreFoundation +from ._securetransport.bindings import CoreFoundation, Security, SecurityConst from ._securetransport.low_level import ( _assert_no_error, + _build_tls_unknown_ca_alert, _cert_array_from_pem, - _temporary_keychain, - _load_client_cert_chain, _create_cfstring_array, - _build_tls_unknown_ca_alert, + _load_client_cert_chain, + _temporary_keychain, ) try: # Platform-specific: Python 2 diff --git a/src/urllib3/contrib/socks.py b/src/urllib3/contrib/socks.py index 516e610bca..93df8325d5 100644 --- a/src/urllib3/contrib/socks.py +++ b/src/urllib3/contrib/socks.py @@ -44,6 +44,7 @@ import socks except ImportError: import warnings + from ..exceptions import DependencyWarning warnings.warn( @@ -56,7 +57,8 @@ ) raise -from socket import error as SocketError, timeout as SocketTimeout +from socket import error as SocketError +from socket import timeout as SocketTimeout from ..connection import HTTPConnection, HTTPSConnection from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool diff --git a/src/urllib3/exceptions.py b/src/urllib3/exceptions.py index 87d4ce77b9..d69958d5df 100644 --- a/src/urllib3/exceptions.py +++ b/src/urllib3/exceptions.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead # Base Exceptions diff --git a/src/urllib3/exceptions.pyi b/src/urllib3/exceptions.pyi index 1254abe7f6..ca528b09ad 100644 --- a/src/urllib3/exceptions.pyi +++ b/src/urllib3/exceptions.pyi @@ -1,4 +1,4 @@ -from typing import Any, Optional, Union, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional, Tuple, Union if TYPE_CHECKING: from urllib3.connectionpool import ConnectionPool diff --git a/src/urllib3/fields.py b/src/urllib3/fields.py index 29c0bd95be..9d630f491d 100644 --- a/src/urllib3/fields.py +++ b/src/urllib3/fields.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + import email.utils import mimetypes import re diff --git a/src/urllib3/filepost.py b/src/urllib3/filepost.py index b7b00992c6..36c9252c64 100644 --- a/src/urllib3/filepost.py +++ b/src/urllib3/filepost.py @@ -1,13 +1,13 @@ from __future__ import absolute_import + import binascii import codecs import os - from io import BytesIO +from .fields import RequestField from .packages import six from .packages.six import b -from .fields import RequestField writer = codecs.lookup("utf-8")[3] diff --git a/src/urllib3/packages/backports/makefile.py b/src/urllib3/packages/backports/makefile.py index a3156a69c0..b8fb2154b6 100644 --- a/src/urllib3/packages/backports/makefile.py +++ b/src/urllib3/packages/backports/makefile.py @@ -7,7 +7,6 @@ wants to create a "fake" socket object. """ import io - from socket import SocketIO diff --git a/src/urllib3/packages/ssl_match_hostname/__init__.py b/src/urllib3/packages/ssl_match_hostname/__init__.py index 72e68993ac..6b12fd90aa 100644 --- a/src/urllib3/packages/ssl_match_hostname/__init__.py +++ b/src/urllib3/packages/ssl_match_hostname/__init__.py @@ -10,7 +10,10 @@ except ImportError: try: # Backport of the function from a pypi module - from backports.ssl_match_hostname import CertificateError, match_hostname # type: ignore + from backports.ssl_match_hostname import ( # type: ignore + CertificateError, + match_hostname, + ) except ImportError: # Our vendored copy from ._implementation import CertificateError, match_hostname # type: ignore diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py index 70933fd0a2..3a31a285bf 100644 --- a/src/urllib3/poolmanager.py +++ b/src/urllib3/poolmanager.py @@ -1,15 +1,12 @@ from __future__ import absolute_import + import collections import functools import logging -import warnings from ._collections import RecentlyUsedContainer -from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool -from .connectionpool import port_by_scheme - +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme from .exceptions import ( - HTTPWarning, LocationValueError, MaxRetryError, ProxySchemeUnknown, @@ -19,19 +16,13 @@ from .packages import six from .packages.six.moves.urllib.parse import urljoin from .request import RequestMethods -from .util.url import parse_url +from .util.proxy import connection_requires_http_tunnel from .util.retry import Retry - +from .util.url import parse_url __all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] -class InvalidProxyConfigurationWarning(HTTPWarning): - """Raised when a user has an HTTPS proxy without enabling HTTPS proxies.""" - - pass - - log = logging.getLogger(__name__) SSL_KEYWORDS = ( @@ -68,6 +59,7 @@ class InvalidProxyConfigurationWarning(HTTPWarning): "key_headers", # dict "key__proxy", # parsed proxy url "key__proxy_headers", # dict + "key__proxy_config", # class "key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples "key__socks_options", # dict "key_assert_hostname", # bool or string @@ -79,6 +71,9 @@ class InvalidProxyConfigurationWarning(HTTPWarning): #: All custom key schemes should include the fields in this key at a minimum. PoolKey = collections.namedtuple("PoolKey", _key_fields) +_proxy_config_fields = ("ssl_context", "use_forwarding_for_https") +ProxyConfig = collections.namedtuple("ProxyConfig", _proxy_config_fields) + def _default_key_normalizer(key_class, request_context): """ @@ -170,6 +165,7 @@ class PoolManager(RequestMethods): """ proxy = None + proxy_config = None def __init__(self, num_pools=10, headers=None, **connection_pool_kw): RequestMethods.__init__(self, headers) @@ -326,14 +322,32 @@ def _merge_pool_kwargs(self, override): def _proxy_requires_url_absolute_form(self, parsed_url): """ Indicates if the proxy requires the complete destination URL in the - request. - - Normally this is only needed when not using an HTTP CONNECT tunnel. + request. Normally this is only needed when not using an HTTP CONNECT + tunnel. """ if self.proxy is None: return False - return parsed_url.scheme == "http" or self.proxy.scheme == "https" + return not connection_requires_http_tunnel( + self.proxy, self.proxy_config, parsed_url.scheme + ) + + def _validate_proxy_scheme_url_selection(self, url_scheme): + """ + Validates that were not attempting to do TLS in TLS connections on + Python2 or with unsupported SSL implementations. + """ + if self.proxy is None or url_scheme != "https": + return + + if self.proxy.scheme != "https": + return + + if six.PY2 and not self.proxy_config.use_forwarding_for_https: + raise ProxySchemeUnsupported( + "Contacting HTTPS destinations through HTTPS proxies " + "'via CONNECT tunnels' is not supported in Python 2" + ) def urlopen(self, method, url, redirect=True, **kw): """ @@ -345,6 +359,8 @@ def urlopen(self, method, url, redirect=True, **kw): :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. """ u = parse_url(url) + self._validate_proxy_scheme_url_selection(u.scheme) + conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) kw["assert_same_host"] = False @@ -415,11 +431,18 @@ class ProxyManager(PoolManager): HTTPS/CONNECT case they are sent only once. Could be used for proxy authentication. - :param _allow_https_proxy_to_see_traffic: - Allows forwarding of HTTPS requests to HTTPS proxies. The proxy will - have visibility of all the traffic sent. ONLY USE IF YOU KNOW WHAT - YOU'RE DOING. This flag might be removed at any time in any future - update. + :param proxy_ssl_context: + The proxy SSL context is used to establish the TLS connection to the + proxy when using HTTPS proxies. + + :param use_forwarding_for_https: + (Defaults to False) If set to True will forward requests to the HTTPS + proxy to be made on behalf of the client instead of creating a TLS + tunnel via the CONNECT method. **Enabling this flag means that request + and response headers and content will be visible from the HTTPS proxy** + whereas tunneling keeps request and response headers and content + private. IP address, target hostname, SNI, and port are always visible + to an HTTPS proxy even when this flag is disabled. Example: >>> proxy = urllib3.ProxyManager('http://localhost:3128/') @@ -440,7 +463,8 @@ def __init__( num_pools=10, headers=None, proxy_headers=None, - _allow_https_proxy_to_see_traffic=False, + proxy_ssl_context=None, + use_forwarding_for_https=False, **connection_pool_kw ): @@ -461,11 +485,12 @@ def __init__( self.proxy = proxy self.proxy_headers = proxy_headers or {} + self.proxy_ssl_context = proxy_ssl_context + self.proxy_config = ProxyConfig(proxy_ssl_context, use_forwarding_for_https) connection_pool_kw["_proxy"] = self.proxy connection_pool_kw["_proxy_headers"] = self.proxy_headers - - self.allow_insecure_proxy = _allow_https_proxy_to_see_traffic + connection_pool_kw["_proxy_config"] = self.proxy_config super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw) @@ -494,35 +519,13 @@ def _set_proxy_headers(self, url, headers=None): headers_.update(headers) return headers_ - def _validate_proxy_scheme_url_selection(self, url_scheme): - if ( - url_scheme == "https" - and self.proxy.scheme == "https" - and not self.allow_insecure_proxy - ): - warnings.warn( - "Your proxy configuration specified an HTTPS scheme for the proxy. " - "Are you sure you want to use HTTPS to contact the proxy? " - "This most likely indicates an error in your configuration." - "If you are sure you want use HTTPS to contact the proxy, enable " - "the _allow_https_proxy_to_see_traffic.", - InvalidProxyConfigurationWarning, - ) - - raise ProxySchemeUnsupported( - "Contacting HTTPS destinations through HTTPS proxies is not supported." - ) - def urlopen(self, method, url, redirect=True, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." u = parse_url(url) - self._validate_proxy_scheme_url_selection(u.scheme) - - if u.scheme == "http" or self.proxy.scheme == "https": + if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme): # For connections using HTTP CONNECT, httplib sets the necessary - # headers on the CONNECT to the proxy. For HTTP or when talking - # HTTPS to the proxy, we'll definitely need to set 'Host' at the - # very least. + # headers on the CONNECT to the proxy. If we're not using CONNECT, + # we'll definitely need to set 'Host' at the very least. headers = kw.get("headers", self.headers) kw["headers"] = self._set_proxy_headers(url, headers) diff --git a/src/urllib3/request.py b/src/urllib3/request.py index b058bf6e9f..398386a5b9 100644 --- a/src/urllib3/request.py +++ b/src/urllib3/request.py @@ -3,7 +3,6 @@ from .filepost import encode_multipart_formdata from .packages.six.moves.urllib.parse import urlencode - __all__ = ["RequestMethods"] diff --git a/src/urllib3/response.py b/src/urllib3/response.py index 149486ed57..38693f4fc6 100644 --- a/src/urllib3/response.py +++ b/src/urllib3/response.py @@ -1,10 +1,11 @@ from __future__ import absolute_import -from contextlib import contextmanager -import zlib + import io import logging -from socket import timeout as SocketTimeout +import zlib +from contextlib import contextmanager from socket import error as SocketError +from socket import timeout as SocketTimeout try: import brotli @@ -12,20 +13,20 @@ brotli = None from ._collections import HTTPHeaderDict +from .connection import BaseSSLError, HTTPException from .exceptions import ( BodyNotHttplibCompatible, - ProtocolError, DecodeError, - ReadTimeoutError, - ResponseNotChunked, + HTTPError, IncompleteRead, InvalidChunkLength, InvalidHeader, - HTTPError, + ProtocolError, + ReadTimeoutError, + ResponseNotChunked, SSLError, ) -from .packages.six import string_types as basestring, PY3 -from .connection import HTTPException, BaseSSLError +from .packages import six from .util.response import is_fp_closed, is_response_to_head log = logging.getLogger(__name__) @@ -233,7 +234,7 @@ def __init__( self.msg = msg self._request_url = request_url - if body and isinstance(body, (basestring, bytes)): + if body and isinstance(body, (six.string_types, bytes)): self._body = body self._pool = pool @@ -589,11 +590,11 @@ def from_httplib(ResponseCls, r, **response_kw): headers = r.msg if not isinstance(headers, HTTPHeaderDict): - if PY3: - headers = HTTPHeaderDict(headers.items()) - else: + if six.PY2: # Python 2.7 headers = HTTPHeaderDict.from_httplib(headers) + else: + headers = HTTPHeaderDict(headers.items()) # HTTPResponse objects in Python 3 don't have a .strict attribute strict = getattr(r, "strict", 0) diff --git a/src/urllib3/util/__init__.py b/src/urllib3/util/__init__.py index 24c16a2894..a230df293a 100644 --- a/src/urllib3/util/__init__.py +++ b/src/urllib3/util/__init__.py @@ -2,24 +2,23 @@ # For backwards compatibility, provide imports that used to be here. from .connection import is_connection_dropped -from .request import make_headers, SUPPRESS_USER_AGENT +from .request import SUPPRESS_USER_AGENT, make_headers from .response import is_fp_closed +from .retry import Retry from .ssl_ import ( - SSLContext, + ALPN_PROTOCOLS, HAS_SNI, IS_PYOPENSSL, IS_SECURETRANSPORT, + PROTOCOL_TLS, + SSLContext, assert_fingerprint, resolve_cert_reqs, resolve_ssl_version, ssl_wrap_socket, - PROTOCOL_TLS, - ALPN_PROTOCOLS, ) -from .timeout import current_time, Timeout - -from .retry import Retry -from .url import get_host, parse_url, split_first, Url +from .timeout import Timeout, current_time +from .url import Url, get_host, parse_url, split_first from .wait import wait_for_read, wait_for_write __all__ = ( diff --git a/src/urllib3/util/connection.py b/src/urllib3/util/connection.py index 9c89dc1426..cd57455748 100644 --- a/src/urllib3/util/connection.py +++ b/src/urllib3/util/connection.py @@ -1,7 +1,12 @@ from __future__ import absolute_import + import socket -from .wait import NoWayToWaitForSocketError, wait_for_read + +from urllib3.exceptions import LocationParseError + from ..contrib import _appengine_environ +from ..packages import six +from .wait import NoWayToWaitForSocketError, wait_for_read def is_connection_dropped(conn): # Platform-specific @@ -58,6 +63,13 @@ def create_connection( # The original create_connection function always returns all records. family = allowed_gai_family() + try: + host.encode("idna") + except UnicodeError: + return six.raise_from( + LocationParseError(u"'%s', label empty or too long" % host), None + ) + for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res sock = None diff --git a/src/urllib3/util/proxy.py b/src/urllib3/util/proxy.py new file mode 100644 index 0000000000..34f884d5b3 --- /dev/null +++ b/src/urllib3/util/proxy.py @@ -0,0 +1,56 @@ +from .ssl_ import create_urllib3_context, resolve_cert_reqs, resolve_ssl_version + + +def connection_requires_http_tunnel( + proxy_url=None, proxy_config=None, destination_scheme=None +): + """ + Returns True if the connection requires an HTTP CONNECT through the proxy. + + :param URL proxy_url: + URL of the proxy. + :param ProxyConfig proxy_config: + Proxy configuration from poolmanager.py + :param str destination_scheme: + The scheme of the destination. (i.e https, http, etc) + """ + # If we're not using a proxy, no way to use a tunnel. + if proxy_url is None: + return False + + # HTTP destinations never require tunneling, we always forward. + if destination_scheme == "http": + return False + + # Support for forwarding with HTTPS proxies and HTTPS destinations. + if ( + proxy_url.scheme == "https" + and proxy_config + and proxy_config.use_forwarding_for_https + ): + return False + + # Otherwise always use a tunnel. + return True + + +def create_proxy_ssl_context( + ssl_version, cert_reqs, ca_certs=None, ca_cert_dir=None, ca_cert_data=None +): + """ + Generates a default proxy ssl context if one hasn't been provided by the + user. + """ + ssl_context = create_urllib3_context( + ssl_version=resolve_ssl_version(ssl_version), + cert_reqs=resolve_cert_reqs(cert_reqs), + ) + if ( + not ca_certs + and not ca_cert_dir + and not ca_cert_data + and hasattr(ssl_context, "load_default_certs") + ): + ssl_context.load_default_certs() + + return ssl_context diff --git a/src/urllib3/util/queue.py b/src/urllib3/util/queue.py index d3d379a199..41784104ee 100644 --- a/src/urllib3/util/queue.py +++ b/src/urllib3/util/queue.py @@ -1,4 +1,5 @@ import collections + from ..packages import six from ..packages.six.moves import queue diff --git a/src/urllib3/util/request.py b/src/urllib3/util/request.py index 27b6c6cf92..60e37513b7 100644 --- a/src/urllib3/util/request.py +++ b/src/urllib3/util/request.py @@ -1,8 +1,9 @@ from __future__ import absolute_import + from base64 import b64encode -from ..packages.six import b, integer_types from ..exceptions import UnrewindableBodyError +from ..packages.six import b, integer_types # Use an invalid User-Agent to represent suppressing of default user agent. # See https://tools.ietf.org/html/rfc7231#section-5.5.3 and diff --git a/src/urllib3/util/response.py b/src/urllib3/util/response.py index 063d99155b..5ea609cced 100644 --- a/src/urllib3/util/response.py +++ b/src/urllib3/util/response.py @@ -1,8 +1,9 @@ from __future__ import absolute_import -from email.errors import StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect -from ..packages.six.moves import http_client as httplib + +from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect from ..exceptions import HeaderParsingError +from ..packages.six.moves import http_client as httplib def is_fp_closed(obj): diff --git a/src/urllib3/util/retry.py b/src/urllib3/util/retry.py index e5eda7a16d..ee51f922f8 100644 --- a/src/urllib3/util/retry.py +++ b/src/urllib3/util/retry.py @@ -1,23 +1,24 @@ from __future__ import absolute_import -import time + +import email import logging +import re +import time +import warnings from collections import namedtuple from itertools import takewhile -import email -import re from ..exceptions import ( ConnectTimeoutError, + InvalidHeader, MaxRetryError, ProtocolError, + ProxyError, ReadTimeoutError, ResponseError, - InvalidHeader, - ProxyError, ) from ..packages import six - log = logging.getLogger(__name__) @@ -27,6 +28,49 @@ ) +# TODO: In v2 we can remove this sentinel and metaclass with deprecated options. +_Default = object() + + +class _RetryMeta(type): + @property + def DEFAULT_METHOD_WHITELIST(cls): + warnings.warn( + "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and " + "will be removed in v2.0. Use 'Retry.DEFAULT_METHODS_ALLOWED' instead", + DeprecationWarning, + ) + return cls.DEFAULT_ALLOWED_METHODS + + @DEFAULT_METHOD_WHITELIST.setter + def DEFAULT_METHOD_WHITELIST(cls, value): + warnings.warn( + "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and " + "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead", + DeprecationWarning, + ) + cls.DEFAULT_ALLOWED_METHODS = value + + @property + def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls): + warnings.warn( + "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and " + "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead", + DeprecationWarning, + ) + return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT + + @DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter + def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value): + warnings.warn( + "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and " + "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead", + DeprecationWarning, + ) + cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value + + +@six.add_metaclass(_RetryMeta) class Retry(object): """Retry configuration. @@ -107,18 +151,23 @@ class Retry(object): If ``total`` is not set, it's a good idea to set this to 0 to account for unexpected edge cases and avoid infinite retry loops. - :param iterable method_whitelist: + :param iterable allowed_methods: Set of uppercased HTTP method verbs that we should retry on. By default, we only retry on methods which are considered to be idempotent (multiple requests with the same parameters end with the - same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`. + same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`. Set to a ``False`` value to retry on any verb. + .. warning:: + + Previously this parameter was named ``method_whitelist``, that + usage is deprecated in v1.26.0 and will be removed in v2.0. + :param iterable status_forcelist: A set of integer HTTP status codes that we should force a retry on. - A retry is initiated if the request method is in ``method_whitelist`` + A retry is initiated if the request method is in ``allowed_methods`` and the response status code is in ``status_forcelist``. By default, this is disabled with ``None``. @@ -159,13 +208,16 @@ class Retry(object): request. """ - DEFAULT_METHOD_WHITELIST = frozenset( + #: Default methods to be used for ``allowed_methods`` + DEFAULT_ALLOWED_METHODS = frozenset( ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] ) + #: Default status codes to be used for ``status_forcelist`` RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) - DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(["Authorization"]) + #: Default headers to be used for ``remove_headers_on_redirect`` + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) #: Maximum backoff time. BACKOFF_MAX = 120 @@ -178,16 +230,36 @@ def __init__( redirect=None, status=None, other=None, - method_whitelist=DEFAULT_METHOD_WHITELIST, + allowed_methods=_Default, status_forcelist=None, backoff_factor=0, raise_on_redirect=True, raise_on_status=True, history=None, respect_retry_after_header=True, - remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST, + remove_headers_on_redirect=_Default, + # TODO: Deprecated, remove in v2.0 + method_whitelist=_Default, ): + if method_whitelist is not _Default: + if allowed_methods is not _Default: + raise ValueError( + "Using both 'allowed_methods' and " + "'method_whitelist' together is not allowed. " + "Instead only use 'allowed_methods'" + ) + warnings.warn( + "Using 'method_whitelist' with Retry is deprecated and " + "will be removed in v2.0. Use 'allowed_methods' instead", + DeprecationWarning, + ) + allowed_methods = method_whitelist + if allowed_methods is _Default: + allowed_methods = self.DEFAULT_ALLOWED_METHODS + if remove_headers_on_redirect is _Default: + remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT + self.total = total self.connect = connect self.read = read @@ -200,7 +272,7 @@ def __init__( self.redirect = redirect self.status_forcelist = status_forcelist or set() - self.method_whitelist = method_whitelist + self.allowed_methods = allowed_methods self.backoff_factor = backoff_factor self.raise_on_redirect = raise_on_redirect self.raise_on_status = raise_on_status @@ -218,7 +290,6 @@ def new(self, **kw): redirect=self.redirect, status=self.status, other=self.other, - method_whitelist=self.method_whitelist, status_forcelist=self.status_forcelist, backoff_factor=self.backoff_factor, raise_on_redirect=self.raise_on_redirect, @@ -227,6 +298,23 @@ def new(self, **kw): remove_headers_on_redirect=self.remove_headers_on_redirect, respect_retry_after_header=self.respect_retry_after_header, ) + + # TODO: If already given in **kw we use what's given to us + # If not given we need to figure out what to pass. We decide + # based on whether our class has the 'method_whitelist' property + # and if so we pass the deprecated 'method_whitelist' otherwise + # we use 'allowed_methods'. Remove in v2.0 + if "method_whitelist" not in kw and "allowed_methods" not in kw: + if "method_whitelist" in self.__dict__: + warnings.warn( + "Using 'method_whitelist' with Retry is deprecated and " + "will be removed in v2.0. Use 'allowed_methods' instead", + DeprecationWarning, + ) + params["method_whitelist"] = self.allowed_methods + else: + params["allowed_methods"] = self.allowed_methods + params.update(kw) return type(self)(**params) @@ -340,15 +428,26 @@ def _is_read_error(self, err): def _is_method_retryable(self, method): """Checks if a given HTTP method should be retried upon, depending if - it is included on the method whitelist. + it is included in the allowed_methods """ - if self.method_whitelist and method.upper() not in self.method_whitelist: - return False + # TODO: For now favor if the Retry implementation sets its own method_whitelist + # property outside of our constructor to avoid breaking custom implementations. + if "method_whitelist" in self.__dict__: + warnings.warn( + "Using 'method_whitelist' with Retry is deprecated and " + "will be removed in v2.0. Use 'allowed_methods' instead", + DeprecationWarning, + ) + allowed_methods = self.method_whitelist + else: + allowed_methods = self.allowed_methods + if allowed_methods and method.upper() not in allowed_methods: + return False return True def is_retry(self, method, status_code, has_retry_after=False): - """Is this method/status code retryable? (Based on whitelists and control + """Is this method/status code retryable? (Based on allowlists and control variables such as the number of total retries to allow, whether to respect the Retry-After header, whether this header is present, and whether the returned status code is on the list of status codes to @@ -448,7 +547,7 @@ def increment( else: # Incrementing because of a server error like a 500 in - # status_forcelist and a the given method is in the whitelist + # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: @@ -483,6 +582,20 @@ def __repr__(self): "read={self.read}, redirect={self.redirect}, status={self.status})" ).format(cls=type(self), self=self) + def __getattr__(self, item): + if item == "method_whitelist": + # TODO: Remove this deprecated alias in v2.0 + warnings.warn( + "Using 'method_whitelist' with Retry is deprecated and " + "will be removed in v2.0. Use 'allowed_methods' instead", + DeprecationWarning, + ) + return self.allowed_methods + try: + return getattr(super(Retry, self), item) + except AttributeError: + return getattr(Retry, item) + # For backwards compatibility (equivalent to pre-v1.9): Retry.DEFAULT = Retry(3) diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py index 5d84409118..8773334067 100644 --- a/src/urllib3/util/ssl_.py +++ b/src/urllib3/util/ssl_.py @@ -1,18 +1,23 @@ from __future__ import absolute_import -import warnings + import hmac import os import sys - +import warnings from binascii import hexlify, unhexlify from hashlib import md5, sha1, sha256 -from .url import IPV4_RE, BRACELESS_IPV6_ADDRZ_RE -from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning +from ..exceptions import ( + InsecurePlatformWarning, + ProxySchemeUnsupported, + SNIMissingWarning, + SSLError, +) from ..packages import six - +from .url import BRACELESS_IPV6_ADDRZ_RE, IPV4_RE SSLContext = None +SSLTransport = None HAS_SNI = False IS_PYOPENSSL = False IS_SECURETRANSPORT = False @@ -39,8 +44,10 @@ def _const_compare_digest_backport(a, b): try: # Test for SSL features import ssl - from ssl import wrap_socket, CERT_REQUIRED from ssl import HAS_SNI # Has SNI? + from ssl import CERT_REQUIRED, wrap_socket + + from .ssltransport import SSLTransport except ImportError: pass @@ -58,12 +65,18 @@ def _const_compare_digest_backport(a, b): try: - from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION + from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3 except ImportError: OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 OP_NO_COMPRESSION = 0x20000 +try: # OP_NO_TICKET was added in Python 3.6 + from ssl import OP_NO_TICKET +except ImportError: + OP_NO_TICKET = 0x4000 + + # A secure default. # Sources for more information on TLS ciphers: # @@ -250,7 +263,7 @@ def create_urllib3_context( ``ssl.CERT_REQUIRED``. :param options: Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``, - ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``. + ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``. :param ciphers: Which cipher suites to allow the server to select. :returns: @@ -273,6 +286,11 @@ def create_urllib3_context( # Disable compression to prevent CRIME attacks for OpenSSL 1.0+ # (issue #309) options |= OP_NO_COMPRESSION + # TLSv1.2 only. Unless set explicitly, do not request tickets. + # This may save some bandwidth on wire, and although the ticket is encrypted, + # there is a risk associated with it being on wire, + # if the server is not rotating its ticketing keys properly. + options |= OP_NO_TICKET context.options |= options @@ -316,6 +334,7 @@ def ssl_wrap_socket( ca_cert_dir=None, key_password=None, ca_cert_data=None, + tls_in_tls=False, ): """ All arguments except for server_hostname, ssl_context, and ca_cert_dir have @@ -337,6 +356,8 @@ def ssl_wrap_socket( :param ca_cert_data: Optional string containing CA certificates in PEM format suitable for passing as the cadata parameter to SSLContext.load_verify_locations() + :param tls_in_tls: + Use SSLTransport to wrap the existing socket. """ context = ssl_context if context is None: @@ -394,9 +415,11 @@ def ssl_wrap_socket( ) if send_sni: - ssl_sock = context.wrap_socket(sock, server_hostname=server_hostname) + ssl_sock = _ssl_wrap_socket_impl( + sock, context, tls_in_tls, server_hostname=server_hostname + ) else: - ssl_sock = context.wrap_socket(sock) + ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls) return ssl_sock @@ -422,3 +445,20 @@ def _is_key_file_encrypted(key_file): return True return False + + +def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None): + if tls_in_tls: + if not SSLTransport: + # Import error, ssl is not available. + raise ProxySchemeUnsupported( + "TLS in TLS requires support for the 'ssl' module" + ) + + SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context) + return SSLTransport(sock, ssl_context, server_hostname) + + if server_hostname: + return ssl_context.wrap_socket(sock, server_hostname=server_hostname) + else: + return ssl_context.wrap_socket(sock) diff --git a/src/urllib3/contrib/ssl.py b/src/urllib3/util/ssltransport.py similarity index 87% rename from src/urllib3/contrib/ssl.py rename to src/urllib3/util/ssltransport.py index 69a9898861..d23e518de7 100644 --- a/src/urllib3/contrib/ssl.py +++ b/src/urllib3/util/ssltransport.py @@ -1,6 +1,9 @@ -import ssl -import socket import io +import socket +import ssl + +from urllib3.exceptions import ProxySchemeUnsupported +from urllib3.packages import six SSL_BLOCKSIZE = 16384 @@ -16,6 +19,28 @@ class SSLTransport: The class supports most of the socket API operations. """ + @staticmethod + def _validate_ssl_context_for_tls_in_tls(ssl_context): + """ + Raises a ProxySchemeUnsupported if the provided ssl_context can't be used + for TLS in TLS. + + The only requirement is that the ssl_context provides the 'wrap_bio' + methods. + """ + + if not hasattr(ssl_context, "wrap_bio"): + if six.PY2: + raise ProxySchemeUnsupported( + "TLS in TLS requires SSLContext.wrap_bio() which isn't " + "supported on Python 2" + ) + else: + raise ProxySchemeUnsupported( + "TLS in TLS requires SSLContext.wrap_bio() which isn't " + "available on non-native SSLContext" + ) + def __init__( self, socket, ssl_context, suppress_ragged_eofs=True, server_hostname=None ): diff --git a/src/urllib3/util/timeout.py b/src/urllib3/util/timeout.py index 2f287a3435..ff69593b05 100644 --- a/src/urllib3/util/timeout.py +++ b/src/urllib3/util/timeout.py @@ -1,9 +1,10 @@ from __future__ import absolute_import +import time + # The default socket timeout, used by httplib to indicate that no timeout was # specified by the user from socket import _GLOBAL_DEFAULT_TIMEOUT -import time from ..exceptions import TimeoutStateError diff --git a/src/urllib3/util/url.py b/src/urllib3/util/url.py index 793324e5fd..6ff238fe3c 100644 --- a/src/urllib3/util/url.py +++ b/src/urllib3/util/url.py @@ -1,11 +1,11 @@ from __future__ import absolute_import + import re from collections import namedtuple from ..exceptions import LocationParseError from ..packages import six - url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"] # We only want to normalize urls with an HTTP(S) scheme. diff --git a/src/urllib3/util/wait.py b/src/urllib3/util/wait.py index 53d10ee7e4..c280646c7b 100644 --- a/src/urllib3/util/wait.py +++ b/src/urllib3/util/wait.py @@ -1,7 +1,7 @@ import errno -from functools import partial import select import sys +from functools import partial try: from time import monotonic diff --git a/test/__init__.py b/test/__init__.py index 5ab3cac1bf..bafd0ab9c6 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,11 +1,11 @@ -import warnings -import sys import errno import logging -import socket -import ssl import os import platform +import socket +import ssl +import sys +import warnings import pytest @@ -14,10 +14,15 @@ except ImportError: brotli = None +from urllib3 import util from urllib3.exceptions import HTTPWarning from urllib3.packages import six from urllib3.util import ssl_ -from urllib3 import util + +try: + import urllib3.contrib.pyopenssl as pyopenssl +except ImportError: + pyopenssl = None # We need a host that will not immediately close the connection with a TCP # Reset. @@ -166,6 +171,19 @@ def notBrotlipy(): ) +def onlySecureTransport(test): + """Runs this test when SecureTransport is in use.""" + + @six.wraps(test) + def wrapper(*args, **kwargs): + msg = "{name} only runs with SecureTransport".format(name=test.__name__) + if not ssl_.IS_SECURETRANSPORT: + pytest.skip(msg) + return test(*args, **kwargs) + + return wrapper + + def notSecureTransport(test): """Skips this test when SecureTransport is in use.""" @@ -290,6 +308,21 @@ def wrapper(*args, **kwargs): return wrapper +def withPyOpenSSL(test): + @six.wraps(test) + def wrapper(*args, **kwargs): + if not pyopenssl: + pytest.skip("pyopenssl not available, skipping test.") + return test(*args, **kwargs) + + pyopenssl.inject_into_urllib3() + result = test(*args, **kwargs) + pyopenssl.extract_from_urllib3() + return result + + return wrapper + + class _ListHandler(logging.Handler): def __init__(self): super(_ListHandler, self).__init__() diff --git a/test/appengine/conftest.py b/test/appengine/conftest.py index 05bfefe74c..0b9d1f1fb0 100644 --- a/test/appengine/conftest.py +++ b/test/appengine/conftest.py @@ -28,7 +28,6 @@ import pytest import six - __all__ = [ "pytest_configure", "pytest_runtest_call", diff --git a/test/appengine/test_gae_manager.py b/test/appengine/test_gae_manager.py index 0221b29c74..3047f249d4 100644 --- a/test/appengine/test_gae_manager.py +++ b/test/appengine/test_gae_manager.py @@ -1,13 +1,13 @@ -import dummyserver.testcase +from test import SHORT_TIMEOUT +from test.with_dummyserver import test_connectionpool + import pytest -from urllib3.contrib import appengine +import dummyserver.testcase import urllib3.exceptions -import urllib3.util.url import urllib3.util.retry - -from test.with_dummyserver import test_connectionpool -from test import SHORT_TIMEOUT +import urllib3.util.url +from urllib3.contrib import appengine # This class is used so we can re-use the tests from the connection pool. diff --git a/test/appengine/test_urlfetch.py b/test/appengine/test_urlfetch.py index 2e727db0d4..74484ea405 100644 --- a/test/appengine/test_urlfetch.py +++ b/test/appengine/test_urlfetch.py @@ -3,10 +3,9 @@ Engine-patched version of httplib to make requests.""" import httplib +import pytest import StringIO - from mock import patch -import pytest from ..test_no_ssl import TestWithoutSSL diff --git a/test/benchmark.py b/test/benchmark.py index d480efae6a..67d141b252 100644 --- a/test/benchmark.py +++ b/test/benchmark.py @@ -13,7 +13,6 @@ sys.path.append("../") import urllib3 # noqa: E402 - # URLs to download. Doesn't matter as long as they're from the same host, so we # can take advantage of connection re-using. TO_DOWNLOAD = [ diff --git a/test/conftest.py b/test/conftest.py index 84e6c18e33..f55a83d5fc 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,18 +1,17 @@ import collections import contextlib -import threading import platform import sys +import threading import pytest import trustme -from tornado import web, ioloop - -from .tz_stub import stub_timezone_ctx +from tornado import ioloop, web from dummyserver.handlers import TestingApp -from dummyserver.server import run_tornado_app -from dummyserver.server import HAS_IPV6 +from dummyserver.server import HAS_IPV6, run_tornado_app + +from .tz_stub import stub_timezone_ctx # The Python 3.8+ default loop on Windows breaks Tornado diff --git a/test/contrib/test_pyopenssl.py b/test/contrib/test_pyopenssl.py index 7d1af31238..1a7f6f9714 100644 --- a/test/contrib/test_pyopenssl.py +++ b/test/contrib/test_pyopenssl.py @@ -5,9 +5,10 @@ import pytest try: - from urllib3.contrib.pyopenssl import _dnsname_to_stdlib, get_subj_alt_name from cryptography import x509 from OpenSSL.crypto import FILETYPE_PEM, load_certificate + + from urllib3.contrib.pyopenssl import _dnsname_to_stdlib, get_subj_alt_name except ImportError: pass @@ -33,19 +34,19 @@ def teardown_module(): from ..test_util import TestUtilSSL # noqa: E402, F401 from ..with_dummyserver.test_https import ( # noqa: E402, F401 TestHTTPS, + TestHTTPS_IPSAN, + TestHTTPS_IPv6Addr, + TestHTTPS_IPV6SAN, + TestHTTPS_NoSAN, TestHTTPS_TLSv1, TestHTTPS_TLSv1_1, TestHTTPS_TLSv1_2, TestHTTPS_TLSv1_3, - TestHTTPS_IPSAN, - TestHTTPS_IPv6Addr, - TestHTTPS_NoSAN, - TestHTTPS_IPV6SAN, ) from ..with_dummyserver.test_socketlevel import ( # noqa: E402, F401 + TestClientCerts, TestSNI, TestSocketClosing, - TestClientCerts, TestSSL, ) diff --git a/test/contrib/test_pyopenssl_dependencies.py b/test/contrib/test_pyopenssl_dependencies.py index bbb5833dab..d1498e9218 100644 --- a/test/contrib/test_pyopenssl_dependencies.py +++ b/test/contrib/test_pyopenssl_dependencies.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- import pytest - -from mock import patch, Mock +from mock import Mock, patch try: - from urllib3.contrib.pyopenssl import inject_into_urllib3, extract_from_urllib3 + from urllib3.contrib.pyopenssl import extract_from_urllib3, inject_into_urllib3 except ImportError: pass diff --git a/test/contrib/test_securetransport.py b/test/contrib/test_securetransport.py index 41519436cf..9a49a35521 100644 --- a/test/contrib/test_securetransport.py +++ b/test/contrib/test_securetransport.py @@ -40,9 +40,9 @@ def teardown_module(): TestHTTPS_TLSv1_2, ) from ..with_dummyserver.test_socketlevel import ( # noqa: E402, F401 + TestClientCerts, TestSNI, TestSocketClosing, - TestClientCerts, TestSSL, ) diff --git a/test/contrib/test_socks.py b/test/contrib/test_socks.py index d70eb7cff6..1966513c18 100644 --- a/test/contrib/test_socks.py +++ b/test/contrib/test_socks.py @@ -1,17 +1,17 @@ -import threading import socket +import threading +from test import SHORT_TIMEOUT -from urllib3.contrib import socks -from urllib3.exceptions import ConnectTimeoutError, NewConnectionError +import pytest -from dummyserver.server import DEFAULT_CERTS, DEFAULT_CA +from dummyserver.server import DEFAULT_CA, DEFAULT_CERTS from dummyserver.testcase import IPV4SocketDummyServerTestCase - -import pytest -from test import SHORT_TIMEOUT +from urllib3.contrib import socks +from urllib3.exceptions import ConnectTimeoutError, NewConnectionError try: import ssl + from urllib3.util import ssl_ as better_ssl HAS_SSL = True diff --git a/test/port_helpers.py b/test/port_helpers.py index d132cb0ac0..ae18ccae6d 100644 --- a/test/port_helpers.py +++ b/test/port_helpers.py @@ -3,7 +3,6 @@ import socket - # Don't use "localhost", since resolving it uses the DNS under recent # Windows versions (see issue #18792). HOST = "127.0.0.1" diff --git a/test/test_collections.py b/test/test_collections.py index e7be33fb7f..4b8624cb6c 100644 --- a/test/test_collections.py +++ b/test/test_collections.py @@ -1,6 +1,7 @@ -from urllib3._collections import HTTPHeaderDict, RecentlyUsedContainer as Container import pytest +from urllib3._collections import HTTPHeaderDict +from urllib3._collections import RecentlyUsedContainer as Container from urllib3.exceptions import InvalidHeader from urllib3.packages import six diff --git a/test/test_compatibility.py b/test/test_compatibility.py index 013699f822..58a9ab5c6f 100644 --- a/test/test_compatibility.py +++ b/test/test_compatibility.py @@ -3,8 +3,8 @@ import pytest from urllib3.connection import HTTPConnection -from urllib3.response import HTTPResponse from urllib3.packages.six.moves import http_cookiejar, urllib +from urllib3.response import HTTPResponse class TestVersionCompatibility(object): diff --git a/test/test_connection.py b/test/test_connection.py index a3cd29c976..821ce4c226 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -1,9 +1,9 @@ import datetime -import mock +import mock import pytest -from urllib3.connection import CertificateError, _match_hostname, RECENT_DATE +from urllib3.connection import RECENT_DATE, CertificateError, _match_hostname class TestConnection(object): diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index 193ba91df8..eec6bd27c8 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -1,21 +1,21 @@ from __future__ import absolute_import import ssl +from socket import error as SocketError +from ssl import SSLError as BaseSSLError +from test import SHORT_TIMEOUT + import pytest from mock import Mock +from dummyserver.server import DEFAULT_CA +from urllib3._collections import HTTPHeaderDict from urllib3.connectionpool import ( - connection_from_url, HTTPConnection, HTTPConnectionPool, HTTPSConnectionPool, + connection_from_url, ) -from urllib3.response import HTTPResponse -from urllib3.util.timeout import Timeout -from urllib3.packages.six.moves import http_client as httplib -from urllib3.packages.six.moves.http_client import HTTPException -from urllib3.packages.six.moves.queue import Empty -from urllib3.packages.ssl_match_hostname import CertificateError from urllib3.exceptions import ( ClosedPoolError, EmptyPoolError, @@ -26,14 +26,14 @@ SSLError, TimeoutError, ) -from urllib3._collections import HTTPHeaderDict -from .test_response import MockChunkedEncodingResponse, MockSock - -from socket import error as SocketError -from ssl import SSLError as BaseSSLError +from urllib3.packages.six.moves import http_client as httplib +from urllib3.packages.six.moves.http_client import HTTPException +from urllib3.packages.six.moves.queue import Empty +from urllib3.packages.ssl_match_hostname import CertificateError +from urllib3.response import HTTPResponse +from urllib3.util.timeout import Timeout -from dummyserver.server import DEFAULT_CA -from test import SHORT_TIMEOUT +from .test_response import MockChunkedEncodingResponse, MockSock class HTTPUnixConnection(HTTPConnection): diff --git a/test/test_exceptions.py b/test/test_exceptions.py index 50d1c7ab10..9fd0eb0fb0 100644 --- a/test/test_exceptions.py +++ b/test/test_exceptions.py @@ -2,18 +2,18 @@ import pytest +from urllib3.connectionpool import HTTPConnectionPool from urllib3.exceptions import ( - HTTPError, - MaxRetryError, - LocationParseError, ClosedPoolError, + ConnectTimeoutError, EmptyPoolError, + HeaderParsingError, HostChangedError, + HTTPError, + LocationParseError, + MaxRetryError, ReadTimeoutError, - ConnectTimeoutError, - HeaderParsingError, ) -from urllib3.connectionpool import HTTPConnectionPool class TestPickle(object): diff --git a/test/test_fields.py b/test/test_fields.py index e0d30db6b8..98ce17c1f4 100644 --- a/test/test_fields.py +++ b/test/test_fields.py @@ -1,6 +1,6 @@ import pytest -from urllib3.fields import format_header_param_rfc2231, guess_content_type, RequestField +from urllib3.fields import RequestField, format_header_param_rfc2231, guess_content_type from urllib3.packages.six import u diff --git a/test/test_filepost.py b/test/test_filepost.py index e884a44b18..5b0cfe1cb6 100644 --- a/test/test_filepost.py +++ b/test/test_filepost.py @@ -1,10 +1,9 @@ import pytest -from urllib3.filepost import encode_multipart_formdata, iter_fields from urllib3.fields import RequestField +from urllib3.filepost import encode_multipart_formdata, iter_fields from urllib3.packages.six import b, u - BOUNDARY = "!! test boundary !!" diff --git a/test/test_no_ssl.py b/test/test_no_ssl.py index 15612f4318..7cf6260e49 100644 --- a/test/test_no_ssl.py +++ b/test/test_no_ssl.py @@ -6,6 +6,7 @@ """ import sys + import pytest diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py index 0603236e97..c54ad7ed9a 100644 --- a/test/test_poolmanager.py +++ b/test/test_poolmanager.py @@ -1,12 +1,12 @@ import socket +from test import resolvesLocalhostFQDN import pytest -from urllib3.poolmanager import PoolKey, key_fn_by_scheme, PoolManager from urllib3 import connection_from_url from urllib3.exceptions import ClosedPoolError, LocationValueError +from urllib3.poolmanager import PoolKey, PoolManager, key_fn_by_scheme from urllib3.util import retry, timeout -from test import resolvesLocalhostFQDN class TestPoolManager(object): @@ -366,3 +366,9 @@ def test_merge_pool_kwargs_invalid_key(self): p = PoolManager(strict=True) merged = p._merge_pool_kwargs({"invalid_key": None}) assert p.connection_pool_kw == merged + + def test_pool_manager_no_url_absolute_form(self): + """Valides we won't send a request with absolute form without a proxy""" + p = PoolManager(strict=True) + assert p._proxy_requires_url_absolute_form("http://example.com") is False + assert p._proxy_requires_url_absolute_form("https://example.com") is False diff --git a/test/test_proxymanager.py b/test/test_proxymanager.py index 2043580c8a..7f1c396cce 100644 --- a/test/test_proxymanager.py +++ b/test/test_proxymanager.py @@ -1,14 +1,11 @@ import pytest -from .port_helpers import find_unused_port +from urllib3.exceptions import MaxRetryError, NewConnectionError, ProxyError from urllib3.poolmanager import ProxyManager -from urllib3.util.url import parse_url from urllib3.util.retry import Retry -from urllib3.exceptions import ( - MaxRetryError, - ProxyError, - NewConnectionError, -) +from urllib3.util.url import parse_url + +from .port_helpers import find_unused_port class TestProxyManager(object): @@ -62,6 +59,10 @@ def test_proxy_tunnel(self): assert p._proxy_requires_url_absolute_form(https_url) is False with ProxyManager("https://proxy:8080") as p: + assert p._proxy_requires_url_absolute_form(http_url) + assert p._proxy_requires_url_absolute_form(https_url) is False + + with ProxyManager("https://proxy:8080", use_forwarding_for_https=True) as p: assert p._proxy_requires_url_absolute_form(http_url) assert p._proxy_requires_url_absolute_form(https_url) diff --git a/test/test_queue_monkeypatch.py b/test/test_queue_monkeypatch.py index 4ebad62be0..f8420a0eb6 100644 --- a/test/test_queue_monkeypatch.py +++ b/test/test_queue_monkeypatch.py @@ -1,7 +1,6 @@ from __future__ import absolute_import import mock - import pytest from urllib3 import HTTPConnectionPool diff --git a/test/test_response.py b/test/test_response.py index b838622f27..03f2780c75 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -5,31 +5,28 @@ import socket import ssl import zlib +from base64 import b64decode +from io import BufferedReader, BytesIO, TextIOWrapper +from test import onlyBrotlipy -from io import BytesIO, BufferedReader, TextIOWrapper - -import pytest import mock +import pytest import six -from urllib3.response import HTTPResponse, brotli from urllib3.exceptions import ( DecodeError, - ResponseNotChunked, - ProtocolError, - InvalidHeader, - httplib_IncompleteRead, IncompleteRead, InvalidChunkLength, + InvalidHeader, + ProtocolError, + ResponseNotChunked, SSLError, + httplib_IncompleteRead, ) from urllib3.packages.six.moves import http_client as httplib -from urllib3.util.retry import Retry, RequestHistory +from urllib3.response import HTTPResponse, brotli from urllib3.util.response import is_fp_closed - -from test import onlyBrotlipy - -from base64 import b64decode +from urllib3.util.retry import RequestHistory, Retry # A known random (i.e, not-too-compressible) payload generated with: # "".join(random.choice(string.printable) for i in xrange(512)) diff --git a/test/test_retry.py b/test/test_retry.py index a29b03e2cd..cc36089796 100644 --- a/test/test_retry.py +++ b/test/test_retry.py @@ -1,10 +1,8 @@ +import warnings + import mock import pytest -from urllib3.response import HTTPResponse -from urllib3.packages import six -from urllib3.packages.six.moves import xrange -from urllib3.util.retry import Retry, RequestHistory from urllib3.exceptions import ( ConnectTimeoutError, InvalidHeader, @@ -13,6 +11,17 @@ ResponseError, SSLError, ) +from urllib3.packages import six +from urllib3.packages.six.moves import xrange +from urllib3.response import HTTPResponse +from urllib3.util.retry import RequestHistory, Retry + + +@pytest.fixture(scope="function", autouse=True) +def no_retry_deprecations(): + with warnings.catch_warnings(record=True) as w: + yield + assert len([str(x.message) for x in w if "Retry" in str(x.message)]) == 0 class TestRetry(object): @@ -196,14 +205,14 @@ def test_status_forcelist(self): retry = Retry(total=1, status_forcelist=["418"]) assert not retry.is_retry("GET", status_code=418) - def test_method_whitelist_with_status_forcelist(self): - # Falsey method_whitelist means to retry on any method. - retry = Retry(status_forcelist=[500], method_whitelist=None) + def test_allowed_methods_with_status_forcelist(self): + # Falsey allowed_methods means to retry on any method. + retry = Retry(status_forcelist=[500], allowed_methods=None) assert retry.is_retry("GET", status_code=500) assert retry.is_retry("POST", status_code=500) - # Criteria of method_whitelist and status_forcelist are ANDed. - retry = Retry(status_forcelist=[500], method_whitelist=["POST"]) + # Criteria of allowed_methods and status_forcelist are ANDed. + retry = Retry(status_forcelist=[500], allowed_methods=["POST"]) assert not retry.is_retry("GET", status_code=500) assert retry.is_retry("POST", status_code=500) @@ -251,7 +260,7 @@ def test_error_message(self): assert str(e.value.reason) == "conntimeout" def test_history(self): - retry = Retry(total=10, method_whitelist=frozenset(["GET", "POST"])) + retry = Retry(total=10, allowed_methods=frozenset(["GET", "POST"])) assert retry.history == tuple() connection_error = ConnectTimeoutError("conntimeout") retry = retry.increment("GET", "/test1", None, connection_error) diff --git a/test/test_retry_deprecated.py b/test/test_retry_deprecated.py new file mode 100644 index 0000000000..0c8de37661 --- /dev/null +++ b/test/test_retry_deprecated.py @@ -0,0 +1,471 @@ +# This is a copy-paste of test_retry.py with extra asserts about deprecated options. It will be removed for v2. +import warnings + +import mock +import pytest + +from urllib3.exceptions import ( + ConnectTimeoutError, + InvalidHeader, + MaxRetryError, + ReadTimeoutError, + ResponseError, + SSLError, +) +from urllib3.packages import six +from urllib3.packages.six.moves import xrange +from urllib3.response import HTTPResponse +from urllib3.util.retry import RequestHistory, Retry + + +# TODO: Remove this entire file once deprecated Retry options are removed in v2. +@pytest.fixture(scope="function") +def expect_retry_deprecation(): + with warnings.catch_warnings(record=True) as w: + yield + assert len([str(x.message) for x in w if "Retry" in str(x.message)]) > 0 + + +class TestRetry(object): + def test_string(self): + """ Retry string representation looks the way we expect """ + retry = Retry() + assert ( + str(retry) + == "Retry(total=10, connect=None, read=None, redirect=None, status=None)" + ) + for _ in range(3): + retry = retry.increment(method="GET") + assert ( + str(retry) + == "Retry(total=7, connect=None, read=None, redirect=None, status=None)" + ) + + def test_retry_both_specified(self): + """Total can win if it's lower than the connect value""" + error = ConnectTimeoutError() + retry = Retry(connect=3, total=2) + retry = retry.increment(error=error) + retry = retry.increment(error=error) + with pytest.raises(MaxRetryError) as e: + retry.increment(error=error) + assert e.value.reason == error + + def test_retry_higher_total_loses(self): + """ A lower connect timeout than the total is honored """ + error = ConnectTimeoutError() + retry = Retry(connect=2, total=3) + retry = retry.increment(error=error) + retry = retry.increment(error=error) + with pytest.raises(MaxRetryError): + retry.increment(error=error) + + def test_retry_higher_total_loses_vs_read(self): + """ A lower read timeout than the total is honored """ + error = ReadTimeoutError(None, "/", "read timed out") + retry = Retry(read=2, total=3) + retry = retry.increment(method="GET", error=error) + retry = retry.increment(method="GET", error=error) + with pytest.raises(MaxRetryError): + retry.increment(method="GET", error=error) + + def test_retry_total_none(self): + """ if Total is none, connect error should take precedence """ + error = ConnectTimeoutError() + retry = Retry(connect=2, total=None) + retry = retry.increment(error=error) + retry = retry.increment(error=error) + with pytest.raises(MaxRetryError) as e: + retry.increment(error=error) + assert e.value.reason == error + + error = ReadTimeoutError(None, "/", "read timed out") + retry = Retry(connect=2, total=None) + retry = retry.increment(method="GET", error=error) + retry = retry.increment(method="GET", error=error) + retry = retry.increment(method="GET", error=error) + assert not retry.is_exhausted() + + def test_retry_default(self): + """ If no value is specified, should retry connects 3 times """ + retry = Retry() + assert retry.total == 10 + assert retry.connect is None + assert retry.read is None + assert retry.redirect is None + assert retry.other is None + + error = ConnectTimeoutError() + retry = Retry(connect=1) + retry = retry.increment(error=error) + with pytest.raises(MaxRetryError): + retry.increment(error=error) + + retry = Retry(connect=1) + retry = retry.increment(error=error) + assert not retry.is_exhausted() + + assert Retry(0).raise_on_redirect + assert not Retry(False).raise_on_redirect + + def test_retry_other(self): + """ If an unexpected error is raised, should retry other times """ + other_error = SSLError() + retry = Retry(connect=1) + retry = retry.increment(error=other_error) + retry = retry.increment(error=other_error) + assert not retry.is_exhausted() + + retry = Retry(other=1) + retry = retry.increment(error=other_error) + with pytest.raises(MaxRetryError) as e: + retry.increment(error=other_error) + assert e.value.reason == other_error + + def test_retry_read_zero(self): + """ No second chances on read timeouts, by default """ + error = ReadTimeoutError(None, "/", "read timed out") + retry = Retry(read=0) + with pytest.raises(MaxRetryError) as e: + retry.increment(method="GET", error=error) + assert e.value.reason == error + + def test_status_counter(self): + resp = HTTPResponse(status=400) + retry = Retry(status=2) + retry = retry.increment(response=resp) + retry = retry.increment(response=resp) + with pytest.raises(MaxRetryError) as e: + retry.increment(response=resp) + assert str(e.value.reason) == ResponseError.SPECIFIC_ERROR.format( + status_code=400 + ) + + def test_backoff(self): + """ Backoff is computed correctly """ + max_backoff = Retry.BACKOFF_MAX + + retry = Retry(total=100, backoff_factor=0.2) + assert retry.get_backoff_time() == 0 # First request + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == 0 # First retry + + retry = retry.increment(method="GET") + assert retry.backoff_factor == 0.2 + assert retry.total == 98 + assert retry.get_backoff_time() == 0.4 # Start backoff + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == 0.8 + + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == 1.6 + + for _ in xrange(10): + retry = retry.increment(method="GET") + + assert retry.get_backoff_time() == max_backoff + + def test_zero_backoff(self): + retry = Retry() + assert retry.get_backoff_time() == 0 + retry = retry.increment(method="GET") + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == 0 + + def test_backoff_reset_after_redirect(self): + retry = Retry(total=100, redirect=5, backoff_factor=0.2) + retry = retry.increment(method="GET") + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == 0.4 + redirect_response = HTTPResponse(status=302, headers={"location": "test"}) + retry = retry.increment(method="GET", response=redirect_response) + assert retry.get_backoff_time() == 0 + retry = retry.increment(method="GET") + retry = retry.increment(method="GET") + assert retry.get_backoff_time() == 0.4 + + def test_sleep(self): + # sleep a very small amount of time so our code coverage is happy + retry = Retry(backoff_factor=0.0001) + retry = retry.increment(method="GET") + retry = retry.increment(method="GET") + retry.sleep() + + def test_status_forcelist(self): + retry = Retry(status_forcelist=xrange(500, 600)) + assert not retry.is_retry("GET", status_code=200) + assert not retry.is_retry("GET", status_code=400) + assert retry.is_retry("GET", status_code=500) + + retry = Retry(total=1, status_forcelist=[418]) + assert not retry.is_retry("GET", status_code=400) + assert retry.is_retry("GET", status_code=418) + + # String status codes are not matched. + retry = Retry(total=1, status_forcelist=["418"]) + assert not retry.is_retry("GET", status_code=418) + + def test_method_whitelist_with_status_forcelist(self, expect_retry_deprecation): + # Falsey method_whitelist means to retry on any method. + retry = Retry(status_forcelist=[500], method_whitelist=None) + assert retry.is_retry("GET", status_code=500) + assert retry.is_retry("POST", status_code=500) + + # Criteria of method_whitelist and status_forcelist are ANDed. + retry = Retry(status_forcelist=[500], method_whitelist=["POST"]) + assert not retry.is_retry("GET", status_code=500) + assert retry.is_retry("POST", status_code=500) + + def test_exhausted(self): + assert not Retry(0).is_exhausted() + assert Retry(-1).is_exhausted() + assert Retry(1).increment(method="GET").total == 0 + + @pytest.mark.parametrize("total", [-1, 0]) + def test_disabled(self, total): + with pytest.raises(MaxRetryError): + Retry(total).increment(method="GET") + + def test_error_message(self): + retry = Retry(total=0) + with pytest.raises(MaxRetryError) as e: + retry = retry.increment( + method="GET", error=ReadTimeoutError(None, "/", "read timed out") + ) + assert "Caused by redirect" not in str(e.value) + assert str(e.value.reason) == "None: read timed out" + + retry = Retry(total=1) + with pytest.raises(MaxRetryError) as e: + retry = retry.increment("POST", "/") + retry = retry.increment("POST", "/") + assert "Caused by redirect" not in str(e.value) + assert isinstance(e.value.reason, ResponseError) + assert str(e.value.reason) == ResponseError.GENERIC_ERROR + + retry = Retry(total=1) + response = HTTPResponse(status=500) + with pytest.raises(MaxRetryError) as e: + retry = retry.increment("POST", "/", response=response) + retry = retry.increment("POST", "/", response=response) + assert "Caused by redirect" not in str(e.value) + msg = ResponseError.SPECIFIC_ERROR.format(status_code=500) + assert str(e.value.reason) == msg + + retry = Retry(connect=1) + with pytest.raises(MaxRetryError) as e: + retry = retry.increment(error=ConnectTimeoutError("conntimeout")) + retry = retry.increment(error=ConnectTimeoutError("conntimeout")) + assert "Caused by redirect" not in str(e.value) + assert str(e.value.reason) == "conntimeout" + + def test_history(self, expect_retry_deprecation): + retry = Retry(total=10, method_whitelist=frozenset(["GET", "POST"])) + assert retry.history == tuple() + connection_error = ConnectTimeoutError("conntimeout") + retry = retry.increment("GET", "/test1", None, connection_error) + history = (RequestHistory("GET", "/test1", connection_error, None, None),) + assert retry.history == history + + read_error = ReadTimeoutError(None, "/test2", "read timed out") + retry = retry.increment("POST", "/test2", None, read_error) + history = ( + RequestHistory("GET", "/test1", connection_error, None, None), + RequestHistory("POST", "/test2", read_error, None, None), + ) + assert retry.history == history + + response = HTTPResponse(status=500) + retry = retry.increment("GET", "/test3", response, None) + history = ( + RequestHistory("GET", "/test1", connection_error, None, None), + RequestHistory("POST", "/test2", read_error, None, None), + RequestHistory("GET", "/test3", None, 500, None), + ) + assert retry.history == history + + def test_retry_method_not_in_whitelist(self): + error = ReadTimeoutError(None, "/", "read timed out") + retry = Retry() + with pytest.raises(ReadTimeoutError): + retry.increment(method="POST", error=error) + + def test_retry_default_remove_headers_on_redirect(self): + retry = Retry() + + assert list(retry.remove_headers_on_redirect) == ["authorization"] + + def test_retry_set_remove_headers_on_redirect(self): + retry = Retry(remove_headers_on_redirect=["X-API-Secret"]) + + assert list(retry.remove_headers_on_redirect) == ["x-api-secret"] + + @pytest.mark.parametrize("value", ["-1", "+1", "1.0", six.u("\xb2")]) # \xb2 = ^2 + def test_parse_retry_after_invalid(self, value): + retry = Retry() + with pytest.raises(InvalidHeader): + retry.parse_retry_after(value) + + @pytest.mark.parametrize( + "value, expected", [("0", 0), ("1000", 1000), ("\t42 ", 42)] + ) + def test_parse_retry_after(self, value, expected): + retry = Retry() + assert retry.parse_retry_after(value) == expected + + @pytest.mark.parametrize("respect_retry_after_header", [True, False]) + def test_respect_retry_after_header_propagated(self, respect_retry_after_header): + + retry = Retry(respect_retry_after_header=respect_retry_after_header) + new_retry = retry.new() + assert new_retry.respect_retry_after_header == respect_retry_after_header + + @pytest.mark.freeze_time("2019-06-03 11:00:00", tz_offset=0) + @pytest.mark.parametrize( + "retry_after_header,respect_retry_after_header,sleep_duration", + [ + ("3600", True, 3600), + ("3600", False, None), + # Will sleep due to header is 1 hour in future + ("Mon, 3 Jun 2019 12:00:00 UTC", True, 3600), + # Won't sleep due to not respecting header + ("Mon, 3 Jun 2019 12:00:00 UTC", False, None), + # Won't sleep due to current time reached + ("Mon, 3 Jun 2019 11:00:00 UTC", True, None), + # Won't sleep due to current time reached + not respecting header + ("Mon, 3 Jun 2019 11:00:00 UTC", False, None), + # Handle all the formats in RFC 7231 Section 7.1.1.1 + ("Mon, 03 Jun 2019 11:30:12 GMT", True, 1812), + ("Monday, 03-Jun-19 11:30:12 GMT", True, 1812), + # Assume that datetimes without a timezone are in UTC per RFC 7231 + ("Mon Jun 3 11:30:12 2019", True, 1812), + ], + ) + @pytest.mark.parametrize( + "stub_timezone", + [ + "UTC", + "Asia/Jerusalem", + None, + ], + indirect=True, + ) + @pytest.mark.usefixtures("stub_timezone") + def test_respect_retry_after_header_sleep( + self, retry_after_header, respect_retry_after_header, sleep_duration + ): + retry = Retry(respect_retry_after_header=respect_retry_after_header) + + with mock.patch("time.sleep") as sleep_mock: + # for the default behavior, it must be in RETRY_AFTER_STATUS_CODES + response = HTTPResponse( + status=503, headers={"Retry-After": retry_after_header} + ) + + retry.sleep(response) + + # The expected behavior is that we'll only sleep if respecting + # this header (since we won't have any backoff sleep attempts) + if respect_retry_after_header and sleep_duration is not None: + sleep_mock.assert_called_with(sleep_duration) + else: + sleep_mock.assert_not_called() + + +class TestRetryDeprecations(object): + def test_cls_get_default_method_whitelist(self, expect_retry_deprecation): + assert Retry.DEFAULT_ALLOWED_METHODS == Retry.DEFAULT_METHOD_WHITELIST + + def test_cls_get_default_redirect_headers_blacklist(self, expect_retry_deprecation): + assert ( + Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT + == Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST + ) + + def test_cls_set_default_method_whitelist(self, expect_retry_deprecation): + old_setting = Retry.DEFAULT_METHOD_WHITELIST + try: + Retry.DEFAULT_METHOD_WHITELIST = {"GET"} + retry = Retry() + assert retry.DEFAULT_ALLOWED_METHODS == {"GET"} + assert retry.DEFAULT_METHOD_WHITELIST == {"GET"} + assert retry.allowed_methods == {"GET"} + assert retry.method_whitelist == {"GET"} + + # Test that the default can be overridden both ways + retry = Retry(allowed_methods={"GET", "POST"}) + assert retry.DEFAULT_ALLOWED_METHODS == {"GET"} + assert retry.DEFAULT_METHOD_WHITELIST == {"GET"} + assert retry.allowed_methods == {"GET", "POST"} + assert retry.method_whitelist == {"GET", "POST"} + + retry = Retry(method_whitelist={"POST"}) + assert retry.DEFAULT_ALLOWED_METHODS == {"GET"} + assert retry.DEFAULT_METHOD_WHITELIST == {"GET"} + assert retry.allowed_methods == {"POST"} + assert retry.method_whitelist == {"POST"} + finally: + Retry.DEFAULT_METHOD_WHITELIST = old_setting + assert Retry.DEFAULT_ALLOWED_METHODS == old_setting + + def test_cls_set_default_redirect_headers_blacklist(self, expect_retry_deprecation): + old_setting = Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST + try: + Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST = {"test"} + retry = Retry() + assert retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT == {"test"} + assert retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST == {"test"} + assert retry.remove_headers_on_redirect == {"test"} + assert retry.remove_headers_on_redirect == {"test"} + + retry = Retry(remove_headers_on_redirect={"test2"}) + assert retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT == {"test"} + assert retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST == {"test"} + assert retry.remove_headers_on_redirect == {"test2"} + assert retry.remove_headers_on_redirect == {"test2"} + finally: + Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST = old_setting + assert Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST == old_setting + + @pytest.mark.parametrize( + "options", [(None, None), ({"GET"}, None), (None, {"GET"}), ({"GET"}, {"GET"})] + ) + def test_retry_allowed_methods_and_method_whitelist_error(self, options): + with pytest.raises(ValueError) as e: + Retry(allowed_methods=options[0], method_whitelist=options[1]) + assert str(e.value) == ( + "Using both 'allowed_methods' and 'method_whitelist' together " + "is not allowed. Instead only use 'allowed_methods'" + ) + + def test_retry_subclass_that_sets_method_whitelist(self, expect_retry_deprecation): + class SubclassRetry(Retry): + def __init__(self, **kwargs): + if "allowed_methods" in kwargs: + raise AssertionError( + "This subclass likely doesn't use 'allowed_methods'" + ) + + super(SubclassRetry, self).__init__(**kwargs) + + # Since we're setting 'method_whiteist' we get fallbacks + # within Retry.new() and Retry._is_method_retryable() + # to use 'method_whitelist' instead of 'allowed_methods' + self.method_whitelist = self.method_whitelist | {"POST"} + + retry = SubclassRetry() + assert retry.method_whitelist == Retry.DEFAULT_ALLOWED_METHODS | {"POST"} + assert retry.new(read=0).method_whitelist == retry.method_whitelist + assert retry._is_method_retryable("POST") + assert not retry._is_method_retryable("CONNECT") + + assert retry.new(method_whitelist={"GET"}).method_whitelist == {"GET", "POST"} + + # urllib3 doesn't do this during normal operation + # so we don't want users passing in 'allowed_methods' + # when their subclass doesn't support the option yet. + with pytest.raises(AssertionError) as e: + retry.new(allowed_methods={"GET"}) + assert str(e.value) == "This subclass likely doesn't use 'allowed_methods'" diff --git a/test/test_ssl.py b/test/test_ssl.py index f755938062..4a00d355e5 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -1,9 +1,10 @@ +from test import notPyPy2 + import mock import pytest -from urllib3.util import ssl_ -from urllib3.exceptions import SNIMissingWarning -from test import notPyPy2 +from urllib3.exceptions import SNIMissingWarning +from urllib3.util import ssl_ @pytest.mark.parametrize( diff --git a/test/contrib/test_ssltransport.py b/test/test_ssltransport.py similarity index 99% rename from test/contrib/test_ssltransport.py rename to test/test_ssltransport.py index d68afd4a90..fd6c52c29d 100644 --- a/test/contrib/test_ssltransport.py +++ b/test/test_ssltransport.py @@ -1,18 +1,14 @@ -from dummyserver.testcase import SocketDummyServerTestCase, consume_socket -from dummyserver.server import ( - DEFAULT_CERTS, - DEFAULT_CA, -) - -from urllib3.contrib.ssl import SSLTransport - +import platform import select -import pytest import socket import ssl import sys -import platform +import pytest + +from dummyserver.server import DEFAULT_CA, DEFAULT_CERTS +from dummyserver.testcase import SocketDummyServerTestCase, consume_socket +from urllib3.util.ssltransport import SSLTransport # consume_socket can iterate forever, we add timeouts to prevent halting. PER_TEST_TIMEOUT = 60 diff --git a/test/test_util.py b/test/test_util.py index ba3cc2f971..827df42726 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,41 +1,42 @@ # coding: utf-8 import hashlib -import warnings -import logging import io -import ssl +import logging import socket +import ssl +import warnings from itertools import chain +from test import notBrotlipy, onlyBrotlipy, onlyPy2, onlyPy3 -from mock import patch, Mock import pytest +from mock import Mock, patch from urllib3 import add_stderr_logger, disable_warnings, util -from urllib3.util.request import make_headers, rewind_body, _FAILEDTELL -from urllib3.util.response import assert_header_parsing -from urllib3.util.timeout import Timeout -from urllib3.util.url import get_host, parse_url, split_first, Url -from urllib3.util.ssl_ import ( - resolve_cert_reqs, - resolve_ssl_version, - ssl_wrap_socket, - _const_compare_digest_backport, -) from urllib3.exceptions import ( - LocationParseError, - TimeoutStateError, InsecureRequestWarning, + LocationParseError, SNIMissingWarning, + TimeoutStateError, UnrewindableBodyError, ) -from urllib3.util.connection import allowed_gai_family, _has_ipv6 -from urllib3.util import is_fp_closed from urllib3.packages import six +from urllib3.poolmanager import ProxyConfig +from urllib3.util import is_fp_closed +from urllib3.util.connection import _has_ipv6, allowed_gai_family, create_connection +from urllib3.util.proxy import connection_requires_http_tunnel, create_proxy_ssl_context +from urllib3.util.request import _FAILEDTELL, make_headers, rewind_body +from urllib3.util.response import assert_header_parsing +from urllib3.util.ssl_ import ( + _const_compare_digest_backport, + resolve_cert_reqs, + resolve_ssl_version, + ssl_wrap_socket, +) +from urllib3.util.timeout import Timeout +from urllib3.util.url import Url, get_host, parse_url, split_first from . import clear_warnings -from test import onlyPy3, onlyPy2, onlyBrotlipy, notBrotlipy - # This number represents a time in seconds, it doesn't mean anything in # isolation. Setting to a high-ish value to avoid conflicts with the smaller # numbers used for timeouts @@ -748,6 +749,34 @@ def test_assert_header_parsing_throws_typeerror_with_non_headers(self, headers): with pytest.raises(TypeError): assert_header_parsing(headers) + def test_connection_requires_http_tunnel_no_proxy(self): + assert not connection_requires_http_tunnel( + proxy_url=None, proxy_config=None, destination_scheme=None + ) + + def test_connection_requires_http_tunnel_http_proxy(self): + proxy = parse_url("http://proxy:8080") + proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False) + destination_scheme = "http" + assert not connection_requires_http_tunnel( + proxy, proxy_config, destination_scheme + ) + + destination_scheme = "https" + assert connection_requires_http_tunnel(proxy, proxy_config, destination_scheme) + + def test_connection_requires_http_tunnel_https_proxy(self): + proxy = parse_url("https://proxy:8443") + proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False) + destination_scheme = "http" + assert not connection_requires_http_tunnel( + proxy, proxy_config, destination_scheme + ) + + def test_create_proxy_ssl_context(self): + ssl_context = create_proxy_ssl_context(ssl_version=None, cert_reqs=None) + ssl_context.verify_mode = ssl.CERT_REQUIRED + @onlyPy3 def test_assert_header_parsing_no_error_on_multipart(self): from http import client @@ -762,6 +791,29 @@ def test_assert_header_parsing_no_error_on_multipart(self): header_msg.seek(0) assert_header_parsing(client.parse_headers(header_msg)) + @pytest.mark.parametrize("host", [".localhost", "...", "t" * 64]) + def test_create_connection_with_invalid_idna_labels(self, host): + with pytest.raises(LocationParseError) as ctx: + create_connection((host, 80)) + assert str(ctx.value) == "Failed to parse: '%s', label empty or too long" % host + + @pytest.mark.parametrize( + "host", + [ + "a.example.com", + "localhost.", + "[dead::beef]", + "[dead::beef%en5]", + "[dead::beef%en5.]", + ], + ) + @patch("socket.getaddrinfo") + @patch("socket.socket") + def test_create_connection_with_valid_idna_labels(self, socket, getaddrinfo, host): + getaddrinfo.return_value = [(None, None, None, None, None)] + socket.return_value = Mock() + create_connection((host, 80)) + class TestUtilSSL(object): """Test utils that use an SSL backend.""" diff --git a/test/test_wait.py b/test/test_wait.py index edfe7687bc..38dad79dee 100644 --- a/test/test_wait.py +++ b/test/test_wait.py @@ -6,20 +6,22 @@ from time import monotonic except ImportError: from time import time as monotonic + import time import pytest -from .socketpair_helper import socketpair from urllib3.util.wait import ( + _have_working_poll, + poll_wait_for_socket, + select_wait_for_socket, wait_for_read, - wait_for_write, wait_for_socket, - select_wait_for_socket, - poll_wait_for_socket, - _have_working_poll, + wait_for_write, ) +from .socketpair_helper import socketpair + @pytest.fixture def spair(): diff --git a/test/tz_stub.py b/test/tz_stub.py index 5b1a8c7e18..c48f5df024 100644 --- a/test/tz_stub.py +++ b/test/tz_stub.py @@ -1,7 +1,8 @@ -from contextlib import contextmanager -import time import datetime import os +import time +from contextlib import contextmanager + import pytest from dateutil import tz diff --git a/test/with_dummyserver/test_chunked_transfer.py b/test/with_dummyserver/test_chunked_transfer.py index 0c6793f615..9e1a8a56ff 100644 --- a/test/with_dummyserver/test_chunked_transfer.py +++ b/test/with_dummyserver/test_chunked_transfer.py @@ -2,14 +2,14 @@ import pytest -from urllib3 import HTTPConnectionPool -from urllib3.util.retry import Retry -from urllib3.util import SUPPRESS_USER_AGENT from dummyserver.testcase import ( + ConnectionMarker, SocketDummyServerTestCase, consume_socket, - ConnectionMarker, ) +from urllib3 import HTTPConnectionPool +from urllib3.util import SUPPRESS_USER_AGENT +from urllib3.util.retry import Retry # Retry failed tests pytestmark = pytest.mark.flaky diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 18ce0c3edf..95616fce00 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -5,34 +5,33 @@ import sys import time import warnings -import pytest +from test import LONG_TIMEOUT, SHORT_TIMEOUT +from threading import Event import mock +import pytest -from .. import TARPIT_HOST, VALID_SOURCE_ADDRESSES, INVALID_SOURCE_ADDRESSES -from ..port_helpers import find_unused_port -from urllib3 import encode_multipart_formdata, HTTPConnectionPool +from dummyserver.server import HAS_IPV6_AND_DNS, NoIPv6Warning +from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase +from urllib3 import HTTPConnectionPool, encode_multipart_formdata from urllib3.connection import _get_default_user_agent from urllib3.exceptions import ( ConnectTimeoutError, - EmptyPoolError, DecodeError, + EmptyPoolError, MaxRetryError, - ReadTimeoutError, NewConnectionError, + ReadTimeoutError, UnrewindableBodyError, ) from urllib3.packages.six import b, u from urllib3.packages.six.moves.urllib.parse import urlencode from urllib3.util import SUPPRESS_USER_AGENT -from urllib3.util.retry import Retry, RequestHistory +from urllib3.util.retry import RequestHistory, Retry from urllib3.util.timeout import Timeout -from test import SHORT_TIMEOUT, LONG_TIMEOUT -from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase -from dummyserver.server import NoIPv6Warning, HAS_IPV6_AND_DNS - -from threading import Event +from .. import INVALID_SOURCE_ADDRESSES, TARPIT_HOST, VALID_SOURCE_ADDRESSES +from ..port_helpers import find_unused_port pytestmark = pytest.mark.flaky diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py index fc3966a76f..3f3f8ec1a0 100644 --- a/test/with_dummyserver/test_https.py +++ b/test/with_dummyserver/test_https.py @@ -2,53 +2,53 @@ import json import logging import os.path +import shutil import ssl import sys -import shutil import tempfile import warnings - -import mock -import pytest -import trustme - -from dummyserver.testcase import HTTPSDummyServerTestCase -from dummyserver.server import ( - encrypt_key_pem, - DEFAULT_CA, - DEFAULT_CA_KEY, - DEFAULT_CERTS, -) - from test import ( - onlyPy279OrNewer, - notSecureTransport, + LONG_TIMEOUT, + SHORT_TIMEOUT, + TARPIT_HOST, notOpenSSL098, + notSecureTransport, + onlyPy279OrNewer, requires_network, requires_ssl_context_keyfile_password, requiresTLSv1, requiresTLSv1_1, requiresTLSv1_2, requiresTLSv1_3, - TARPIT_HOST, - SHORT_TIMEOUT, - LONG_TIMEOUT, resolvesLocalhostFQDN, ) + +import mock +import pytest +import trustme + +import urllib3.util as util +from dummyserver.server import ( + DEFAULT_CA, + DEFAULT_CA_KEY, + DEFAULT_CERTS, + encrypt_key_pem, +) +from dummyserver.testcase import HTTPSDummyServerTestCase from urllib3 import HTTPSConnectionPool -from urllib3.connection import VerifiedHTTPSConnection, RECENT_DATE +from urllib3.connection import RECENT_DATE, VerifiedHTTPSConnection from urllib3.exceptions import ( - SSLError, ConnectTimeoutError, - InsecureRequestWarning, - SystemTimeWarning, InsecurePlatformWarning, + InsecureRequestWarning, MaxRetryError, ProtocolError, + SSLError, + SystemTimeWarning, ) from urllib3.packages import six from urllib3.util.timeout import Timeout -import urllib3.util as util + from .. import has_alpn # Retry failed tests diff --git a/test/with_dummyserver/test_no_ssl.py b/test/with_dummyserver/test_no_ssl.py index 7f4d350a4b..43e79b70b6 100644 --- a/test/with_dummyserver/test_no_ssl.py +++ b/test/with_dummyserver/test_no_ssl.py @@ -4,11 +4,11 @@ Note: Import urllib3 inside the test functions to get the importblocker to work """ import pytest -from ..test_no_ssl import TestWithoutSSL +import urllib3 from dummyserver.testcase import HTTPDummyServerTestCase, HTTPSDummyServerTestCase -import urllib3 +from ..test_no_ssl import TestWithoutSSL # Retry failed tests pytestmark = pytest.mark.flaky diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py index 9d2303e889..d877cc99ac 100644 --- a/test/with_dummyserver/test_poolmanager.py +++ b/test/with_dummyserver/test_poolmanager.py @@ -1,16 +1,15 @@ import json +from test import LONG_TIMEOUT import pytest from dummyserver.server import HAS_IPV6 from dummyserver.testcase import HTTPDummyServerTestCase, IPv6HTTPDummyServerTestCase -from urllib3.poolmanager import PoolManager from urllib3.connectionpool import port_by_scheme from urllib3.exceptions import MaxRetryError, URLSchemeUnknown +from urllib3.poolmanager import PoolManager from urllib3.util.retry import Retry -from test import LONG_TIMEOUT - # Retry failed tests pytestmark = pytest.mark.flaky diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py index 2125f14480..67cee77a58 100644 --- a/test/with_dummyserver/test_proxy_poolmanager.py +++ b/test/with_dummyserver/test_proxy_poolmanager.py @@ -3,27 +3,33 @@ import shutil import socket import tempfile - +from test import ( + LONG_TIMEOUT, + SHORT_TIMEOUT, + onlyPy2, + onlyPy3, + onlySecureTransport, + withPyOpenSSL, +) import pytest import trustme -from dummyserver.testcase import HTTPDummyProxyTestCase, IPv6HTTPDummyProxyTestCase from dummyserver.server import DEFAULT_CA, HAS_IPV6, get_unreachable_address -from .. import TARPIT_HOST, requires_network - +from dummyserver.testcase import HTTPDummyProxyTestCase, IPv6HTTPDummyProxyTestCase from urllib3._collections import HTTPHeaderDict -from urllib3.poolmanager import proxy_from_url, ProxyManager +from urllib3.connectionpool import VerifiedHTTPSConnection, connection_from_url from urllib3.exceptions import ( + ConnectTimeoutError, MaxRetryError, - SSLError, ProxyError, - ConnectTimeoutError, ProxySchemeUnsupported, + SSLError, ) -from urllib3.connectionpool import connection_from_url, VerifiedHTTPSConnection +from urllib3.poolmanager import ProxyManager, proxy_from_url +from urllib3.util.ssl_ import create_urllib3_context -from test import SHORT_TIMEOUT, LONG_TIMEOUT +from .. import TARPIT_HOST, requires_network # Retry failed tests pytestmark = pytest.mark.flaky @@ -63,21 +69,75 @@ def test_basic_proxy(self): r = http.request("GET", "%s/" % self.https_url) assert r.status == 200 + @onlyPy3 def test_https_proxy(self): + with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: + r = https.request("GET", "%s/" % self.https_url) + assert r.status == 200 + + r = https.request("GET", "%s/" % self.http_url) + assert r.status == 200 + + @onlyPy3 + def test_https_proxy_with_proxy_ssl_context(self): + proxy_ssl_context = create_urllib3_context() + proxy_ssl_context.load_verify_locations(DEFAULT_CA) + with proxy_from_url( + self.https_proxy_url, + proxy_ssl_context=proxy_ssl_context, + ca_certs=DEFAULT_CA, + ) as https: + r = https.request("GET", "%s/" % self.https_url) + assert r.status == 200 + + r = https.request("GET", "%s/" % self.http_url) + assert r.status == 200 + + @onlyPy2 + def test_https_proxy_not_supported(self): + with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: + r = https.request("GET", "%s/" % self.http_url) + assert r.status == 200 + + with pytest.raises(ProxySchemeUnsupported) as excinfo: + https.request("GET", "%s/" % self.https_url) + + assert "is not supported in Python 2" in str(excinfo.value) + + @withPyOpenSSL + @onlyPy3 + def test_https_proxy_pyopenssl_not_supported(self): + with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: + r = https.request("GET", "%s/" % self.http_url) + assert r.status == 200 + + with pytest.raises(ProxySchemeUnsupported) as excinfo: + https.request("GET", "%s/" % self.https_url) + + assert "isn't available on non-native SSLContext" in str(excinfo.value) + + @onlySecureTransport + @onlyPy3 + def test_https_proxy_securetransport_not_supported(self): with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https: r = https.request("GET", "%s/" % self.http_url) assert r.status == 200 - with pytest.raises(ProxySchemeUnsupported): + with pytest.raises(ProxySchemeUnsupported) as excinfo: https.request("GET", "%s/" % self.https_url) + assert "isn't available on non-native SSLContext" in str(excinfo.value) + + def test_https_proxy_forwarding_for_https(self): with proxy_from_url( self.https_proxy_url, ca_certs=DEFAULT_CA, - _allow_https_proxy_to_see_traffic=True, + use_forwarding_for_https=True, ) as https: r = https.request("GET", "%s/" % self.http_url) - https.request("GET", "%s/" % self.https_url) + assert r.status == 200 + + r = https.request("GET", "%s/" % self.https_url) assert r.status == 200 def test_nagle_proxy(self): @@ -302,6 +362,7 @@ def test_headers(self): self.https_port, ) + @onlyPy3 def test_https_headers(self): with proxy_from_url( self.https_proxy_url, @@ -328,19 +389,34 @@ def test_https_headers(self): self.http_port, ) - with pytest.raises(ProxySchemeUnsupported): - http.request_encode_url("GET", "%s/headers" % self.https_url) - - r = http.request_encode_url( - "GET", "%s/headers" % self.http_url, headers={"Baz": "quux"} + r = http.request_encode_body( + "GET", "%s/headers" % self.https_url, headers={"Baz": "quux"} ) returned_headers = json.loads(r.data.decode()) assert returned_headers.get("Foo") is None assert returned_headers.get("Baz") == "quux" + assert returned_headers.get("Hickory") is None + assert returned_headers.get("Host") == "%s:%s" % ( + self.https_host, + self.https_port, + ) + + def test_https_headers_forwarding_for_https(self): + with proxy_from_url( + self.https_proxy_url, + headers={"Foo": "bar"}, + proxy_headers={"Hickory": "dickory"}, + ca_certs=DEFAULT_CA, + use_forwarding_for_https=True, + ) as http: + + r = http.request_encode_url("GET", "%s/headers" % self.https_url) + returned_headers = json.loads(r.data.decode()) + assert returned_headers.get("Foo") == "bar" assert returned_headers.get("Hickory") == "dickory" assert returned_headers.get("Host") == "%s:%s" % ( - self.http_host, - self.http_port, + self.https_host, + self.https_port, ) def test_headerdict(self): diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index 3f68923a0f..0ae4c238d2 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1,33 +1,29 @@ # TODO: Break this module up into pieces. Maybe group by functionality tested # rather than the socket level-ness of it. -from urllib3 import HTTPConnectionPool, HTTPSConnectionPool -from urllib3.connection import HTTPConnection -from urllib3.poolmanager import proxy_from_url -from urllib3.connection import _get_default_user_agent +from dummyserver.server import ( + DEFAULT_CA, + DEFAULT_CERTS, + encrypt_key_pem, + get_unreachable_address, +) +from dummyserver.testcase import SocketDummyServerTestCase, consume_socket +from urllib3 import HTTPConnectionPool, HTTPSConnectionPool, util +from urllib3._collections import HTTPHeaderDict +from urllib3.connection import HTTPConnection, _get_default_user_agent from urllib3.exceptions import ( MaxRetryError, + ProtocolError, ProxyError, ReadTimeoutError, SSLError, - ProtocolError, ) from urllib3.packages.six.moves import http_client as httplib -from urllib3 import util -from urllib3.util import ssl_wrap_socket -from urllib3.util import ssl_ -from urllib3.util.timeout import Timeout +from urllib3.poolmanager import proxy_from_url +from urllib3.util import ssl_, ssl_wrap_socket from urllib3.util.retry import Retry -from urllib3._collections import HTTPHeaderDict - -from dummyserver.testcase import SocketDummyServerTestCase, consume_socket -from dummyserver.server import ( - DEFAULT_CERTS, - DEFAULT_CA, - get_unreachable_address, - encrypt_key_pem, -) +from urllib3.util.timeout import Timeout -from .. import onlyPy3, LogRecorder, has_alpn +from .. import LogRecorder, has_alpn, onlyPy3 try: from mimetools import Message as MimeToolMessage @@ -37,29 +33,28 @@ class MimeToolMessage(object): pass -from collections import OrderedDict -import os.path -from threading import Event import os +import os.path import select -import socket import shutil +import socket import ssl import tempfile -import mock - -import pytest -import trustme - +from collections import OrderedDict from test import ( - requires_ssl_context_keyfile_password, - SHORT_TIMEOUT, LONG_TIMEOUT, + SHORT_TIMEOUT, notPyPy2, notSecureTransport, notWindows, + requires_ssl_context_keyfile_password, resolvesLocalhostFQDN, ) +from threading import Event + +import mock +import pytest +import trustme # Retry failed tests pytestmark = pytest.mark.flaky