urllib3 is a powerful, *user-friendly* HTTP client for Python. Much of the
From 6c429006c73da330b3c7e5b349a9a4b0eaf5d609 Mon Sep 17 00:00:00 2001
From: Andrey Petrov
Date: Tue, 22 Sep 2020 15:42:38 -0400
Subject: [PATCH 02/14] Add Contributor Covenant as Code of Conduct
---
CODE_OF_CONDUCT.md | 128 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 128 insertions(+)
create mode 100644 CODE_OF_CONDUCT.md
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000000..0079d14e98
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,128 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+ and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+ overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+ advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+ address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at @sethmlarson or @shazow.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.
From fc89ea40881e78028571b723b233e3a8d7d7af75 Mon Sep 17 00:00:00 2001
From: Seth Michael Larson
Date: Wed, 23 Sep 2020 10:14:52 -0500
Subject: [PATCH 03/14] Update the Discord invite
---
README.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index bb4a52a2e4..5529a8f97a 100644
--- a/README.rst
+++ b/README.rst
@@ -7,7 +7,7 @@
-
+
From d3fc8468b3b27e6fab4cc366b1b6d299ee8321f2 Mon Sep 17 00:00:00 2001
From: Seth Michael Larson
Date: Thu, 24 Sep 2020 08:20:20 -0500
Subject: [PATCH 04/14] Update color scheme of banner to match logo
---
docs/images/banner.svg | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/images/banner.svg b/docs/images/banner.svg
index 4659db6d72..86e6661e72 100644
--- a/docs/images/banner.svg
+++ b/docs/images/banner.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
From 2a5c028972bc019130b4ec59abc8e797e08d3de3 Mon Sep 17 00:00:00 2001
From: Andrey Petrov
Date: Fri, 25 Sep 2020 11:15:16 -0400
Subject: [PATCH 05/14] FUNDING.yml: Add Gitcoin
---
.github/FUNDING.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
index a6993ea9e1..7cc1a23e9e 100644
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -1,2 +1,3 @@
tidelift: pypi/urllib3
open_collective: urllib3
+custom: https://gitcoin.co/grants/65/urllib3
From ed628df10b2810b9bf1e02aca0adb23ea4f4d977 Mon Sep 17 00:00:00 2001
From: PleasantMachine9 <65126927+PleasantMachine9@users.noreply.github.com>
Date: Fri, 25 Sep 2020 17:25:58 +0000
Subject: [PATCH 06/14] Disable the use of session tickets on TLSv1.2 by
default
Since currently session resumption is not supported by urllib3
there is no reason to request tickets from the server.
It takes up extra bytes in transit (~200 bytes), and raises some
minor security concerns.
See: https://blog.filippo.io/we-need-to-talk-about-session-tickets
---
src/urllib3/util/ssl_.py | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py
index 5d84409118..70982a58c1 100644
--- a/src/urllib3/util/ssl_.py
+++ b/src/urllib3/util/ssl_.py
@@ -64,6 +64,12 @@ def _const_compare_digest_backport(a, b):
OP_NO_COMPRESSION = 0x20000
+try: # OP_NO_TICKET was added in Python 3.6
+ from ssl import OP_NO_TICKET
+except ImportError:
+ OP_NO_TICKET = 0x4000
+
+
# A secure default.
# Sources for more information on TLS ciphers:
#
@@ -250,7 +256,7 @@ def create_urllib3_context(
``ssl.CERT_REQUIRED``.
:param options:
Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
- ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``.
+ ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``.
:param ciphers:
Which cipher suites to allow the server to select.
:returns:
@@ -273,6 +279,11 @@ def create_urllib3_context(
# Disable compression to prevent CRIME attacks for OpenSSL 1.0+
# (issue #309)
options |= OP_NO_COMPRESSION
+ # TLSv1.2 only. Unless set explicitly, do not request tickets.
+ # This may save some bandwidth on wire, and although the ticket is encrypted,
+ # there is a risk associated with it being on wire,
+ # if the server is not rotating its ticketing keys properly.
+ options |= OP_NO_TICKET
context.options |= options
From 6d38f171c4921043e1ff633e2a3e9f7ea382e1d5 Mon Sep 17 00:00:00 2001
From: hodbn
Date: Fri, 25 Sep 2020 21:26:16 +0300
Subject: [PATCH 07/14] Add PR and issue templates (#1978)
* Add issue templates
* Add SO and Discord to issue templates
* Add a PR template
* Fix Discord invite URL in PR contact link
* Title caps bug report template
* Simplify PR template
* Simplify bug report template
* Switch Bug Report and Feature Request issue templates order
* Update PULL_REQUEST_TEMPLATE.md
* Update 01_feature_request.md
* Update 02_bug_report.md
* Update config.yml
Co-authored-by: Seth Michael Larson
---
.github/ISSUE_TEMPLATE/01_feature_request.md | 26 +++++++++++++++
.github/ISSUE_TEMPLATE/02_bug_report.md | 35 ++++++++++++++++++++
.github/ISSUE_TEMPLATE/config.yml | 11 ++++++
.github/PULL_REQUEST_TEMPLATE.md | 8 +++++
4 files changed, 80 insertions(+)
create mode 100644 .github/ISSUE_TEMPLATE/01_feature_request.md
create mode 100644 .github/ISSUE_TEMPLATE/02_bug_report.md
create mode 100644 .github/ISSUE_TEMPLATE/config.yml
create mode 100644 .github/PULL_REQUEST_TEMPLATE.md
diff --git a/.github/ISSUE_TEMPLATE/01_feature_request.md b/.github/ISSUE_TEMPLATE/01_feature_request.md
new file mode 100644
index 0000000000..90a531fefb
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/01_feature_request.md
@@ -0,0 +1,26 @@
+---
+name: 🎁 Feature Request
+about: Suggest a new feature
+---
+
+### Context
+
+What are you trying to do?
+How do you expect to do it?
+Is it something you currently cannot do?
+Is this related to an existing issue/problem?
+
+### Alternatives
+
+Can you achieve the same result doing it in an alternative way?
+Is the alternative considerable?
+
+### Duplicate
+
+Has the feature been requested before?
+If so, please provide a link to the issue.
+
+### Contribution
+
+Would you be willing to submit a PR?
+_(Help can be provided if you need assistance submitting a PR)_
diff --git a/.github/ISSUE_TEMPLATE/02_bug_report.md b/.github/ISSUE_TEMPLATE/02_bug_report.md
new file mode 100644
index 0000000000..4d4c37c105
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/02_bug_report.md
@@ -0,0 +1,35 @@
+---
+name: 🐞 Bug Report
+about: Something is broken
+---
+
+### Subject
+
+Describe the issue here.
+
+### Environment
+
+Describe your environment.
+At least, paste here the output of:
+
+```python
+import platform
+import urllib3
+
+print("OS", platform.platform())
+print("Python", platform.python_version())
+print("urllib3", urllib3.__version__)
+```
+
+### Steps to Reproduce
+
+A simple and isolated way to reproduce the issue. A code snippet would be great.
+
+### Expected Behavior
+
+What should happen.
+
+### Actual Behavior
+
+What happens instead.
+You may attach logs, packet captures, etc.
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 0000000000..9b435ceda4
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+ - name: 📚 Documentation
+ url: https://urllib3.readthedocs.io/en/latest/
+ about: Make sure you read the relevant docs
+ - name: ❓ Ask on StackOverflow
+ url: https://stackoverflow.com/questions/tagged/urllib3
+ about: Ask questions about usage in StackOverflow
+ - name: 💬 Ask the Community
+ url: https://discord.gg/CHEgCZN
+ about: Join urllib3's Discord server
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 0000000000..e5765d5108
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,8 @@
+
From 382ab32f23795c44faae83b4e8b18a16fb605a0a Mon Sep 17 00:00:00 2001
From: Seth Michael Larson
Date: Mon, 28 Sep 2020 12:44:55 -0500
Subject: [PATCH 08/14] Rename Retry options and defaults
---
docs/reference/urllib3.util.rst | 1 -
src/urllib3/util/retry.py | 141 +++++++++-
test/test_retry.py | 20 +-
test/test_retry_deprecated.py | 470 ++++++++++++++++++++++++++++++++
4 files changed, 611 insertions(+), 21 deletions(-)
create mode 100644 test/test_retry_deprecated.py
diff --git a/docs/reference/urllib3.util.rst b/docs/reference/urllib3.util.rst
index 663efd6b68..cb51215e80 100644
--- a/docs/reference/urllib3.util.rst
+++ b/docs/reference/urllib3.util.rst
@@ -14,5 +14,4 @@ but can also be used independently.
.. automodule:: urllib3.util
:members:
- :undoc-members:
:show-inheritance:
diff --git a/src/urllib3/util/retry.py b/src/urllib3/util/retry.py
index e5eda7a16d..9d9f4a3c3d 100644
--- a/src/urllib3/util/retry.py
+++ b/src/urllib3/util/retry.py
@@ -5,6 +5,7 @@
from itertools import takewhile
import email
import re
+import warnings
from ..exceptions import (
ConnectTimeoutError,
@@ -27,6 +28,49 @@
)
+# TODO: In v2 we can remove this sentinel and metaclass with deprecated options.
+_Default = object()
+
+
+class _RetryMeta(type):
+ @property
+ def DEFAULT_METHOD_WHITELIST(cls):
+ warnings.warn(
+ "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
+ "will be removed in v2.0. Use 'Retry.DEFAULT_METHODS_ALLOWED' instead",
+ DeprecationWarning,
+ )
+ return cls.DEFAULT_ALLOWED_METHODS
+
+ @DEFAULT_METHOD_WHITELIST.setter
+ def DEFAULT_METHOD_WHITELIST(cls, value):
+ warnings.warn(
+ "Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
+ "will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
+ DeprecationWarning,
+ )
+ cls.DEFAULT_ALLOWED_METHODS = value
+
+ @property
+ def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls):
+ warnings.warn(
+ "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
+ "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
+ DeprecationWarning,
+ )
+ return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
+
+ @DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter
+ def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value):
+ warnings.warn(
+ "Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
+ "will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
+ DeprecationWarning,
+ )
+ cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value
+
+
+@six.add_metaclass(_RetryMeta)
class Retry(object):
"""Retry configuration.
@@ -107,18 +151,23 @@ class Retry(object):
If ``total`` is not set, it's a good idea to set this to 0 to account
for unexpected edge cases and avoid infinite retry loops.
- :param iterable method_whitelist:
+ :param iterable allowed_methods:
Set of uppercased HTTP method verbs that we should retry on.
By default, we only retry on methods which are considered to be
idempotent (multiple requests with the same parameters end with the
- same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`.
+ same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
Set to a ``False`` value to retry on any verb.
+ .. warning::
+
+ Previously this parameter was named ``method_whitelist``, that
+ usage is deprecated in v1.26.0 and will be removed in v2.0.
+
:param iterable status_forcelist:
A set of integer HTTP status codes that we should force a retry on.
- A retry is initiated if the request method is in ``method_whitelist``
+ A retry is initiated if the request method is in ``allowed_methods``
and the response status code is in ``status_forcelist``.
By default, this is disabled with ``None``.
@@ -159,13 +208,16 @@ class Retry(object):
request.
"""
- DEFAULT_METHOD_WHITELIST = frozenset(
+ #: Default methods to be used for ``allowed_methods``
+ DEFAULT_ALLOWED_METHODS = frozenset(
["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
)
+ #: Default status codes to be used for ``status_forcelist``
RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503])
- DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(["Authorization"])
+ #: Default headers to be used for ``remove_headers_on_redirect``
+ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])
#: Maximum backoff time.
BACKOFF_MAX = 120
@@ -178,16 +230,36 @@ def __init__(
redirect=None,
status=None,
other=None,
- method_whitelist=DEFAULT_METHOD_WHITELIST,
+ allowed_methods=_Default,
status_forcelist=None,
backoff_factor=0,
raise_on_redirect=True,
raise_on_status=True,
history=None,
respect_retry_after_header=True,
- remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST,
+ remove_headers_on_redirect=_Default,
+ # TODO: Deprecated, remove in v2.0
+ method_whitelist=_Default,
):
+ if method_whitelist is not _Default:
+ if allowed_methods is not _Default:
+ raise ValueError(
+ "Using both 'allowed_methods' and "
+ "'method_whitelist' together is not allowed. "
+ "Instead only use 'allowed_methods'"
+ )
+ warnings.warn(
+ "Using 'method_whitelist' with Retry is deprecated and "
+ "will be removed in v2.0. Use 'allowed_methods' instead",
+ DeprecationWarning,
+ )
+ allowed_methods = method_whitelist
+ if allowed_methods is _Default:
+ allowed_methods = self.DEFAULT_ALLOWED_METHODS
+ if remove_headers_on_redirect is _Default:
+ remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
+
self.total = total
self.connect = connect
self.read = read
@@ -200,7 +272,7 @@ def __init__(
self.redirect = redirect
self.status_forcelist = status_forcelist or set()
- self.method_whitelist = method_whitelist
+ self.allowed_methods = allowed_methods
self.backoff_factor = backoff_factor
self.raise_on_redirect = raise_on_redirect
self.raise_on_status = raise_on_status
@@ -218,7 +290,6 @@ def new(self, **kw):
redirect=self.redirect,
status=self.status,
other=self.other,
- method_whitelist=self.method_whitelist,
status_forcelist=self.status_forcelist,
backoff_factor=self.backoff_factor,
raise_on_redirect=self.raise_on_redirect,
@@ -227,6 +298,23 @@ def new(self, **kw):
remove_headers_on_redirect=self.remove_headers_on_redirect,
respect_retry_after_header=self.respect_retry_after_header,
)
+
+ # TODO: If already given in **kw we use what's given to us
+ # If not given we need to figure out what to pass. We decide
+ # based on whether our class has the 'method_whitelist' property
+ # and if so we pass the deprecated 'method_whitelist' otherwise
+ # we use 'allowed_methods'. Remove in v2.0
+ if "method_whitelist" not in kw and "allowed_methods" not in kw:
+ if "method_whitelist" in self.__dict__:
+ warnings.warn(
+ "Using 'method_whitelist' with Retry is deprecated and "
+ "will be removed in v2.0. Use 'allowed_methods' instead",
+ DeprecationWarning,
+ )
+ params["method_whitelist"] = self.allowed_methods
+ else:
+ params["allowed_methods"] = self.allowed_methods
+
params.update(kw)
return type(self)(**params)
@@ -340,15 +428,26 @@ def _is_read_error(self, err):
def _is_method_retryable(self, method):
"""Checks if a given HTTP method should be retried upon, depending if
- it is included on the method whitelist.
+ it is included in the allowed_methods
"""
- if self.method_whitelist and method.upper() not in self.method_whitelist:
- return False
+ # TODO: For now favor if the Retry implementation sets its own method_whitelist
+ # property outside of our constructor to avoid breaking custom implementations.
+ if "method_whitelist" in self.__dict__:
+ warnings.warn(
+ "Using 'method_whitelist' with Retry is deprecated and "
+ "will be removed in v2.0. Use 'allowed_methods' instead",
+ DeprecationWarning,
+ )
+ allowed_methods = self.method_whitelist
+ else:
+ allowed_methods = self.allowed_methods
+ if allowed_methods and method.upper() not in allowed_methods:
+ return False
return True
def is_retry(self, method, status_code, has_retry_after=False):
- """Is this method/status code retryable? (Based on whitelists and control
+ """Is this method/status code retryable? (Based on allowlists and control
variables such as the number of total retries to allow, whether to
respect the Retry-After header, whether this header is present, and
whether the returned status code is on the list of status codes to
@@ -448,7 +547,7 @@ def increment(
else:
# Incrementing because of a server error like a 500 in
- # status_forcelist and a the given method is in the whitelist
+ # status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
@@ -483,6 +582,20 @@ def __repr__(self):
"read={self.read}, redirect={self.redirect}, status={self.status})"
).format(cls=type(self), self=self)
+ def __getattr__(self, item):
+ if item == "method_whitelist":
+ # TODO: Remove this deprecated alias in v2.0
+ warnings.warn(
+ "Using 'method_whitelist' with Retry is deprecated and "
+ "will be removed in v2.0. Use 'allowed_methods' instead",
+ DeprecationWarning,
+ )
+ return self.allowed_methods
+ try:
+ return getattr(super(Retry, self), item)
+ except AttributeError:
+ return getattr(Retry, item)
+
# For backwards compatibility (equivalent to pre-v1.9):
Retry.DEFAULT = Retry(3)
diff --git a/test/test_retry.py b/test/test_retry.py
index a29b03e2cd..0ca79dd3a2 100644
--- a/test/test_retry.py
+++ b/test/test_retry.py
@@ -1,5 +1,6 @@
import mock
import pytest
+import warnings
from urllib3.response import HTTPResponse
from urllib3.packages import six
@@ -15,6 +16,13 @@
)
+@pytest.fixture(scope="function", autouse=True)
+def no_retry_deprecations():
+ with warnings.catch_warnings(record=True) as w:
+ yield
+ assert len([str(x.message) for x in w if "Retry" in str(x.message)]) == 0
+
+
class TestRetry(object):
def test_string(self):
""" Retry string representation looks the way we expect """
@@ -196,14 +204,14 @@ def test_status_forcelist(self):
retry = Retry(total=1, status_forcelist=["418"])
assert not retry.is_retry("GET", status_code=418)
- def test_method_whitelist_with_status_forcelist(self):
- # Falsey method_whitelist means to retry on any method.
- retry = Retry(status_forcelist=[500], method_whitelist=None)
+ def test_allowed_methods_with_status_forcelist(self):
+ # Falsey allowed_methods means to retry on any method.
+ retry = Retry(status_forcelist=[500], allowed_methods=None)
assert retry.is_retry("GET", status_code=500)
assert retry.is_retry("POST", status_code=500)
- # Criteria of method_whitelist and status_forcelist are ANDed.
- retry = Retry(status_forcelist=[500], method_whitelist=["POST"])
+ # Criteria of allowed_methods and status_forcelist are ANDed.
+ retry = Retry(status_forcelist=[500], allowed_methods=["POST"])
assert not retry.is_retry("GET", status_code=500)
assert retry.is_retry("POST", status_code=500)
@@ -251,7 +259,7 @@ def test_error_message(self):
assert str(e.value.reason) == "conntimeout"
def test_history(self):
- retry = Retry(total=10, method_whitelist=frozenset(["GET", "POST"]))
+ retry = Retry(total=10, allowed_methods=frozenset(["GET", "POST"]))
assert retry.history == tuple()
connection_error = ConnectTimeoutError("conntimeout")
retry = retry.increment("GET", "/test1", None, connection_error)
diff --git a/test/test_retry_deprecated.py b/test/test_retry_deprecated.py
new file mode 100644
index 0000000000..73b5ef0f71
--- /dev/null
+++ b/test/test_retry_deprecated.py
@@ -0,0 +1,470 @@
+# This is a copy-paste of test_retry.py with extra asserts about deprecated options. It will be removed for v2.
+import mock
+import pytest
+import warnings
+
+from urllib3.response import HTTPResponse
+from urllib3.packages import six
+from urllib3.packages.six.moves import xrange
+from urllib3.util.retry import Retry, RequestHistory
+from urllib3.exceptions import (
+ ConnectTimeoutError,
+ InvalidHeader,
+ MaxRetryError,
+ ReadTimeoutError,
+ ResponseError,
+ SSLError,
+)
+
+
+# TODO: Remove this entire file once deprecated Retry options are removed in v2.
+@pytest.fixture(scope="function")
+def expect_retry_deprecation():
+ with warnings.catch_warnings(record=True) as w:
+ yield
+ assert len([str(x.message) for x in w if "Retry" in str(x.message)]) > 0
+
+
+class TestRetry(object):
+ def test_string(self):
+ """ Retry string representation looks the way we expect """
+ retry = Retry()
+ assert (
+ str(retry)
+ == "Retry(total=10, connect=None, read=None, redirect=None, status=None)"
+ )
+ for _ in range(3):
+ retry = retry.increment(method="GET")
+ assert (
+ str(retry)
+ == "Retry(total=7, connect=None, read=None, redirect=None, status=None)"
+ )
+
+ def test_retry_both_specified(self):
+ """Total can win if it's lower than the connect value"""
+ error = ConnectTimeoutError()
+ retry = Retry(connect=3, total=2)
+ retry = retry.increment(error=error)
+ retry = retry.increment(error=error)
+ with pytest.raises(MaxRetryError) as e:
+ retry.increment(error=error)
+ assert e.value.reason == error
+
+ def test_retry_higher_total_loses(self):
+ """ A lower connect timeout than the total is honored """
+ error = ConnectTimeoutError()
+ retry = Retry(connect=2, total=3)
+ retry = retry.increment(error=error)
+ retry = retry.increment(error=error)
+ with pytest.raises(MaxRetryError):
+ retry.increment(error=error)
+
+ def test_retry_higher_total_loses_vs_read(self):
+ """ A lower read timeout than the total is honored """
+ error = ReadTimeoutError(None, "/", "read timed out")
+ retry = Retry(read=2, total=3)
+ retry = retry.increment(method="GET", error=error)
+ retry = retry.increment(method="GET", error=error)
+ with pytest.raises(MaxRetryError):
+ retry.increment(method="GET", error=error)
+
+ def test_retry_total_none(self):
+ """ if Total is none, connect error should take precedence """
+ error = ConnectTimeoutError()
+ retry = Retry(connect=2, total=None)
+ retry = retry.increment(error=error)
+ retry = retry.increment(error=error)
+ with pytest.raises(MaxRetryError) as e:
+ retry.increment(error=error)
+ assert e.value.reason == error
+
+ error = ReadTimeoutError(None, "/", "read timed out")
+ retry = Retry(connect=2, total=None)
+ retry = retry.increment(method="GET", error=error)
+ retry = retry.increment(method="GET", error=error)
+ retry = retry.increment(method="GET", error=error)
+ assert not retry.is_exhausted()
+
+ def test_retry_default(self):
+ """ If no value is specified, should retry connects 3 times """
+ retry = Retry()
+ assert retry.total == 10
+ assert retry.connect is None
+ assert retry.read is None
+ assert retry.redirect is None
+ assert retry.other is None
+
+ error = ConnectTimeoutError()
+ retry = Retry(connect=1)
+ retry = retry.increment(error=error)
+ with pytest.raises(MaxRetryError):
+ retry.increment(error=error)
+
+ retry = Retry(connect=1)
+ retry = retry.increment(error=error)
+ assert not retry.is_exhausted()
+
+ assert Retry(0).raise_on_redirect
+ assert not Retry(False).raise_on_redirect
+
+ def test_retry_other(self):
+ """ If an unexpected error is raised, should retry other times """
+ other_error = SSLError()
+ retry = Retry(connect=1)
+ retry = retry.increment(error=other_error)
+ retry = retry.increment(error=other_error)
+ assert not retry.is_exhausted()
+
+ retry = Retry(other=1)
+ retry = retry.increment(error=other_error)
+ with pytest.raises(MaxRetryError) as e:
+ retry.increment(error=other_error)
+ assert e.value.reason == other_error
+
+ def test_retry_read_zero(self):
+ """ No second chances on read timeouts, by default """
+ error = ReadTimeoutError(None, "/", "read timed out")
+ retry = Retry(read=0)
+ with pytest.raises(MaxRetryError) as e:
+ retry.increment(method="GET", error=error)
+ assert e.value.reason == error
+
+ def test_status_counter(self):
+ resp = HTTPResponse(status=400)
+ retry = Retry(status=2)
+ retry = retry.increment(response=resp)
+ retry = retry.increment(response=resp)
+ with pytest.raises(MaxRetryError) as e:
+ retry.increment(response=resp)
+ assert str(e.value.reason) == ResponseError.SPECIFIC_ERROR.format(
+ status_code=400
+ )
+
+ def test_backoff(self):
+ """ Backoff is computed correctly """
+ max_backoff = Retry.BACKOFF_MAX
+
+ retry = Retry(total=100, backoff_factor=0.2)
+ assert retry.get_backoff_time() == 0 # First request
+
+ retry = retry.increment(method="GET")
+ assert retry.get_backoff_time() == 0 # First retry
+
+ retry = retry.increment(method="GET")
+ assert retry.backoff_factor == 0.2
+ assert retry.total == 98
+ assert retry.get_backoff_time() == 0.4 # Start backoff
+
+ retry = retry.increment(method="GET")
+ assert retry.get_backoff_time() == 0.8
+
+ retry = retry.increment(method="GET")
+ assert retry.get_backoff_time() == 1.6
+
+ for _ in xrange(10):
+ retry = retry.increment(method="GET")
+
+ assert retry.get_backoff_time() == max_backoff
+
+ def test_zero_backoff(self):
+ retry = Retry()
+ assert retry.get_backoff_time() == 0
+ retry = retry.increment(method="GET")
+ retry = retry.increment(method="GET")
+ assert retry.get_backoff_time() == 0
+
+ def test_backoff_reset_after_redirect(self):
+ retry = Retry(total=100, redirect=5, backoff_factor=0.2)
+ retry = retry.increment(method="GET")
+ retry = retry.increment(method="GET")
+ assert retry.get_backoff_time() == 0.4
+ redirect_response = HTTPResponse(status=302, headers={"location": "test"})
+ retry = retry.increment(method="GET", response=redirect_response)
+ assert retry.get_backoff_time() == 0
+ retry = retry.increment(method="GET")
+ retry = retry.increment(method="GET")
+ assert retry.get_backoff_time() == 0.4
+
+ def test_sleep(self):
+ # sleep a very small amount of time so our code coverage is happy
+ retry = Retry(backoff_factor=0.0001)
+ retry = retry.increment(method="GET")
+ retry = retry.increment(method="GET")
+ retry.sleep()
+
+ def test_status_forcelist(self):
+ retry = Retry(status_forcelist=xrange(500, 600))
+ assert not retry.is_retry("GET", status_code=200)
+ assert not retry.is_retry("GET", status_code=400)
+ assert retry.is_retry("GET", status_code=500)
+
+ retry = Retry(total=1, status_forcelist=[418])
+ assert not retry.is_retry("GET", status_code=400)
+ assert retry.is_retry("GET", status_code=418)
+
+ # String status codes are not matched.
+ retry = Retry(total=1, status_forcelist=["418"])
+ assert not retry.is_retry("GET", status_code=418)
+
+ def test_method_whitelist_with_status_forcelist(self, expect_retry_deprecation):
+ # Falsey method_whitelist means to retry on any method.
+ retry = Retry(status_forcelist=[500], method_whitelist=None)
+ assert retry.is_retry("GET", status_code=500)
+ assert retry.is_retry("POST", status_code=500)
+
+ # Criteria of method_whitelist and status_forcelist are ANDed.
+ retry = Retry(status_forcelist=[500], method_whitelist=["POST"])
+ assert not retry.is_retry("GET", status_code=500)
+ assert retry.is_retry("POST", status_code=500)
+
+ def test_exhausted(self):
+ assert not Retry(0).is_exhausted()
+ assert Retry(-1).is_exhausted()
+ assert Retry(1).increment(method="GET").total == 0
+
+ @pytest.mark.parametrize("total", [-1, 0])
+ def test_disabled(self, total):
+ with pytest.raises(MaxRetryError):
+ Retry(total).increment(method="GET")
+
+ def test_error_message(self):
+ retry = Retry(total=0)
+ with pytest.raises(MaxRetryError) as e:
+ retry = retry.increment(
+ method="GET", error=ReadTimeoutError(None, "/", "read timed out")
+ )
+ assert "Caused by redirect" not in str(e.value)
+ assert str(e.value.reason) == "None: read timed out"
+
+ retry = Retry(total=1)
+ with pytest.raises(MaxRetryError) as e:
+ retry = retry.increment("POST", "/")
+ retry = retry.increment("POST", "/")
+ assert "Caused by redirect" not in str(e.value)
+ assert isinstance(e.value.reason, ResponseError)
+ assert str(e.value.reason) == ResponseError.GENERIC_ERROR
+
+ retry = Retry(total=1)
+ response = HTTPResponse(status=500)
+ with pytest.raises(MaxRetryError) as e:
+ retry = retry.increment("POST", "/", response=response)
+ retry = retry.increment("POST", "/", response=response)
+ assert "Caused by redirect" not in str(e.value)
+ msg = ResponseError.SPECIFIC_ERROR.format(status_code=500)
+ assert str(e.value.reason) == msg
+
+ retry = Retry(connect=1)
+ with pytest.raises(MaxRetryError) as e:
+ retry = retry.increment(error=ConnectTimeoutError("conntimeout"))
+ retry = retry.increment(error=ConnectTimeoutError("conntimeout"))
+ assert "Caused by redirect" not in str(e.value)
+ assert str(e.value.reason) == "conntimeout"
+
+ def test_history(self, expect_retry_deprecation):
+ retry = Retry(total=10, method_whitelist=frozenset(["GET", "POST"]))
+ assert retry.history == tuple()
+ connection_error = ConnectTimeoutError("conntimeout")
+ retry = retry.increment("GET", "/test1", None, connection_error)
+ history = (RequestHistory("GET", "/test1", connection_error, None, None),)
+ assert retry.history == history
+
+ read_error = ReadTimeoutError(None, "/test2", "read timed out")
+ retry = retry.increment("POST", "/test2", None, read_error)
+ history = (
+ RequestHistory("GET", "/test1", connection_error, None, None),
+ RequestHistory("POST", "/test2", read_error, None, None),
+ )
+ assert retry.history == history
+
+ response = HTTPResponse(status=500)
+ retry = retry.increment("GET", "/test3", response, None)
+ history = (
+ RequestHistory("GET", "/test1", connection_error, None, None),
+ RequestHistory("POST", "/test2", read_error, None, None),
+ RequestHistory("GET", "/test3", None, 500, None),
+ )
+ assert retry.history == history
+
+ def test_retry_method_not_in_whitelist(self):
+ error = ReadTimeoutError(None, "/", "read timed out")
+ retry = Retry()
+ with pytest.raises(ReadTimeoutError):
+ retry.increment(method="POST", error=error)
+
+ def test_retry_default_remove_headers_on_redirect(self):
+ retry = Retry()
+
+ assert list(retry.remove_headers_on_redirect) == ["authorization"]
+
+ def test_retry_set_remove_headers_on_redirect(self):
+ retry = Retry(remove_headers_on_redirect=["X-API-Secret"])
+
+ assert list(retry.remove_headers_on_redirect) == ["x-api-secret"]
+
+ @pytest.mark.parametrize("value", ["-1", "+1", "1.0", six.u("\xb2")]) # \xb2 = ^2
+ def test_parse_retry_after_invalid(self, value):
+ retry = Retry()
+ with pytest.raises(InvalidHeader):
+ retry.parse_retry_after(value)
+
+ @pytest.mark.parametrize(
+ "value, expected", [("0", 0), ("1000", 1000), ("\t42 ", 42)]
+ )
+ def test_parse_retry_after(self, value, expected):
+ retry = Retry()
+ assert retry.parse_retry_after(value) == expected
+
+ @pytest.mark.parametrize("respect_retry_after_header", [True, False])
+ def test_respect_retry_after_header_propagated(self, respect_retry_after_header):
+
+ retry = Retry(respect_retry_after_header=respect_retry_after_header)
+ new_retry = retry.new()
+ assert new_retry.respect_retry_after_header == respect_retry_after_header
+
+ @pytest.mark.freeze_time("2019-06-03 11:00:00", tz_offset=0)
+ @pytest.mark.parametrize(
+ "retry_after_header,respect_retry_after_header,sleep_duration",
+ [
+ ("3600", True, 3600),
+ ("3600", False, None),
+ # Will sleep due to header is 1 hour in future
+ ("Mon, 3 Jun 2019 12:00:00 UTC", True, 3600),
+ # Won't sleep due to not respecting header
+ ("Mon, 3 Jun 2019 12:00:00 UTC", False, None),
+ # Won't sleep due to current time reached
+ ("Mon, 3 Jun 2019 11:00:00 UTC", True, None),
+ # Won't sleep due to current time reached + not respecting header
+ ("Mon, 3 Jun 2019 11:00:00 UTC", False, None),
+ # Handle all the formats in RFC 7231 Section 7.1.1.1
+ ("Mon, 03 Jun 2019 11:30:12 GMT", True, 1812),
+ ("Monday, 03-Jun-19 11:30:12 GMT", True, 1812),
+ # Assume that datetimes without a timezone are in UTC per RFC 7231
+ ("Mon Jun 3 11:30:12 2019", True, 1812),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "stub_timezone",
+ [
+ "UTC",
+ "Asia/Jerusalem",
+ None,
+ ],
+ indirect=True,
+ )
+ @pytest.mark.usefixtures("stub_timezone")
+ def test_respect_retry_after_header_sleep(
+ self, retry_after_header, respect_retry_after_header, sleep_duration
+ ):
+ retry = Retry(respect_retry_after_header=respect_retry_after_header)
+
+ with mock.patch("time.sleep") as sleep_mock:
+ # for the default behavior, it must be in RETRY_AFTER_STATUS_CODES
+ response = HTTPResponse(
+ status=503, headers={"Retry-After": retry_after_header}
+ )
+
+ retry.sleep(response)
+
+ # The expected behavior is that we'll only sleep if respecting
+ # this header (since we won't have any backoff sleep attempts)
+ if respect_retry_after_header and sleep_duration is not None:
+ sleep_mock.assert_called_with(sleep_duration)
+ else:
+ sleep_mock.assert_not_called()
+
+
+class TestRetryDeprecations(object):
+ def test_cls_get_default_method_whitelist(self, expect_retry_deprecation):
+ assert Retry.DEFAULT_ALLOWED_METHODS == Retry.DEFAULT_METHOD_WHITELIST
+
+ def test_cls_get_default_redirect_headers_blacklist(self, expect_retry_deprecation):
+ assert (
+ Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
+ == Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST
+ )
+
+ def test_cls_set_default_method_whitelist(self, expect_retry_deprecation):
+ old_setting = Retry.DEFAULT_METHOD_WHITELIST
+ try:
+ Retry.DEFAULT_METHOD_WHITELIST = {"GET"}
+ retry = Retry()
+ assert retry.DEFAULT_ALLOWED_METHODS == {"GET"}
+ assert retry.DEFAULT_METHOD_WHITELIST == {"GET"}
+ assert retry.allowed_methods == {"GET"}
+ assert retry.method_whitelist == {"GET"}
+
+ # Test that the default can be overridden both ways
+ retry = Retry(allowed_methods={"GET", "POST"})
+ assert retry.DEFAULT_ALLOWED_METHODS == {"GET"}
+ assert retry.DEFAULT_METHOD_WHITELIST == {"GET"}
+ assert retry.allowed_methods == {"GET", "POST"}
+ assert retry.method_whitelist == {"GET", "POST"}
+
+ retry = Retry(method_whitelist={"POST"})
+ assert retry.DEFAULT_ALLOWED_METHODS == {"GET"}
+ assert retry.DEFAULT_METHOD_WHITELIST == {"GET"}
+ assert retry.allowed_methods == {"POST"}
+ assert retry.method_whitelist == {"POST"}
+ finally:
+ Retry.DEFAULT_METHOD_WHITELIST = old_setting
+ assert Retry.DEFAULT_ALLOWED_METHODS == old_setting
+
+ def test_cls_set_default_redirect_headers_blacklist(self, expect_retry_deprecation):
+ old_setting = Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST
+ try:
+ Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST = {"test"}
+ retry = Retry()
+ assert retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT == {"test"}
+ assert retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST == {"test"}
+ assert retry.remove_headers_on_redirect == {"test"}
+ assert retry.remove_headers_on_redirect == {"test"}
+
+ retry = Retry(remove_headers_on_redirect={"test2"})
+ assert retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT == {"test"}
+ assert retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST == {"test"}
+ assert retry.remove_headers_on_redirect == {"test2"}
+ assert retry.remove_headers_on_redirect == {"test2"}
+ finally:
+ Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST = old_setting
+ assert Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST == old_setting
+
+ @pytest.mark.parametrize(
+ "options", [(None, None), ({"GET"}, None), (None, {"GET"}), ({"GET"}, {"GET"})]
+ )
+ def test_retry_allowed_methods_and_method_whitelist_error(self, options):
+ with pytest.raises(ValueError) as e:
+ Retry(allowed_methods=options[0], method_whitelist=options[1])
+ assert str(e.value) == (
+ "Using both 'allowed_methods' and 'method_whitelist' together "
+ "is not allowed. Instead only use 'allowed_methods'"
+ )
+
+ def test_retry_subclass_that_sets_method_whitelist(self, expect_retry_deprecation):
+ class SubclassRetry(Retry):
+ def __init__(self, **kwargs):
+ if "allowed_methods" in kwargs:
+ raise AssertionError(
+ "This subclass likely doesn't use 'allowed_methods'"
+ )
+
+ super(SubclassRetry, self).__init__(**kwargs)
+
+ # Since we're setting 'method_whiteist' we get fallbacks
+ # within Retry.new() and Retry._is_method_retryable()
+ # to use 'method_whitelist' instead of 'allowed_methods'
+ self.method_whitelist = self.method_whitelist | {"POST"}
+
+ retry = SubclassRetry()
+ assert retry.method_whitelist == Retry.DEFAULT_ALLOWED_METHODS | {"POST"}
+ assert retry.new(read=0).method_whitelist == retry.method_whitelist
+ assert retry._is_method_retryable("POST")
+ assert not retry._is_method_retryable("CONNECT")
+
+ assert retry.new(method_whitelist={"GET"}).method_whitelist == {"GET", "POST"}
+
+ # urllib3 doesn't do this during normal operation
+ # so we don't want users passing in 'allowed_methods'
+ # when their subclass doesn't support the option yet.
+ with pytest.raises(AssertionError) as e:
+ retry.new(allowed_methods={"GET"})
+ assert str(e.value) == "This subclass likely doesn't use 'allowed_methods'"
From 691679f738ae098ce024aeae90cfdd8013b8e1e7 Mon Sep 17 00:00:00 2001
From: Jorge
Date: Mon, 28 Sep 2020 13:11:27 -0700
Subject: [PATCH 09/14] Integrate TLS-in-TLS support into urllib3 (#1923)
---
src/urllib3/connection.py | 47 ++++++++
src/urllib3/connectionpool.py | 26 ++++-
src/urllib3/poolmanager.py | 96 +++++++++--------
src/urllib3/util/proxy.py | 60 +++++++++++
src/urllib3/util/ssl_.py | 35 +++++-
.../{contrib/ssl.py => util/ssltransport.py} | 25 +++++
test/__init__.py | 33 ++++++
test/test_poolmanager.py | 6 ++
test/test_proxymanager.py | 4 +
test/{contrib => }/test_ssltransport.py | 2 +-
test/test_util.py | 36 ++++++-
.../test_proxy_poolmanager.py | 101 ++++++++++++++++--
12 files changed, 405 insertions(+), 66 deletions(-)
create mode 100644 src/urllib3/util/proxy.py
rename src/urllib3/{contrib/ssl.py => util/ssltransport.py} (87%)
rename test/{contrib => }/test_ssltransport.py (99%)
diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py
index 81474178cc..f9e84fdbfd 100644
--- a/src/urllib3/connection.py
+++ b/src/urllib3/connection.py
@@ -9,6 +9,7 @@
from .packages import six
from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection
from .packages.six.moves.http_client import HTTPException # noqa: F401
+from .util.proxy import create_proxy_ssl_context
try: # Compiled with SSL?
import ssl
@@ -117,6 +118,11 @@ def __init__(self, *args, **kw):
#: The socket options provided by the user. If no options are
#: provided, we use the default options.
self.socket_options = kw.pop("socket_options", self.default_socket_options)
+
+ # Proxy options provided by the user.
+ self.proxy = kw.pop("proxy", None)
+ self.proxy_config = kw.pop("proxy_config", None)
+
_HTTPConnection.__init__(self, *args, **kw)
@property
@@ -271,6 +277,7 @@ class HTTPSConnection(HTTPConnection):
ca_cert_data = None
ssl_version = None
assert_fingerprint = None
+ tls_in_tls_required = False
def __init__(
self,
@@ -335,8 +342,13 @@ def connect(self):
# Add certificate verification
conn = self._new_conn()
hostname = self.host
+ tls_in_tls = False
if self._is_using_tunnel():
+ if self.tls_in_tls_required:
+ conn = self._connect_tls_proxy(hostname, conn)
+ tls_in_tls = True
+
self.sock = conn
# Calls self._set_hostport(), so self.host is
@@ -396,6 +408,7 @@ def connect(self):
ca_cert_data=self.ca_cert_data,
server_hostname=server_hostname,
ssl_context=context,
+ tls_in_tls=tls_in_tls,
)
if self.assert_fingerprint:
@@ -428,6 +441,40 @@ def connect(self):
or self.assert_fingerprint is not None
)
+ def _connect_tls_proxy(self, hostname, conn):
+ """
+ Establish a TLS connection to the proxy using the provided SSL context.
+ """
+ proxy_config = self.proxy_config
+ ssl_context = proxy_config.ssl_context
+ if ssl_context:
+ # If the user provided a proxy context, we assume CA and client
+ # certificates have already been set
+ return ssl_wrap_socket(
+ sock=conn,
+ server_hostname=hostname,
+ ssl_context=ssl_context,
+ )
+
+ ssl_context = create_proxy_ssl_context(
+ self.ssl_version,
+ self.cert_reqs,
+ self.ca_certs,
+ self.ca_cert_dir,
+ self.ca_cert_data,
+ )
+
+ # If no cert was provided, use only the default options for server
+ # certificate validation
+ return ssl_wrap_socket(
+ sock=conn,
+ ca_certs=self.ca_certs,
+ ca_cert_dir=self.ca_cert_dir,
+ ca_cert_data=self.ca_cert_data,
+ server_hostname=hostname,
+ ssl_context=ssl_context,
+ )
+
def _match_hostname(cert, asserted_hostname):
try:
diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py
index 4547223c68..c48a0b846d 100644
--- a/src/urllib3/connectionpool.py
+++ b/src/urllib3/connectionpool.py
@@ -40,6 +40,7 @@
from .response import HTTPResponse
from .util.connection import is_connection_dropped
+from .util.proxy import connection_requires_http_tunnel
from .util.request import set_file_position
from .util.response import assert_header_parsing
from .util.retry import Retry
@@ -182,6 +183,7 @@ def __init__(
retries=None,
_proxy=None,
_proxy_headers=None,
+ _proxy_config=None,
**conn_kw
):
ConnectionPool.__init__(self, host, port)
@@ -203,6 +205,7 @@ def __init__(
self.proxy = _proxy
self.proxy_headers = _proxy_headers or {}
+ self.proxy_config = _proxy_config
# Fill the queue up so that doing get() on it will block properly
for _ in xrange(maxsize):
@@ -219,6 +222,9 @@ def __init__(
# list.
self.conn_kw.setdefault("socket_options", [])
+ self.conn_kw["proxy"] = self.proxy
+ self.conn_kw["proxy_config"] = self.proxy_config
+
def _new_conn(self):
"""
Return a fresh :class:`HTTPConnection`.
@@ -621,6 +627,10 @@ def urlopen(
Additional parameters are passed to
:meth:`urllib3.response.HTTPResponse.from_httplib`
"""
+
+ parsed_url = parse_url(url)
+ destination_scheme = parsed_url.scheme
+
if headers is None:
headers = self.headers
@@ -638,7 +648,7 @@ def urlopen(
if url.startswith("/"):
url = six.ensure_str(_encode_target(url))
else:
- url = six.ensure_str(parse_url(url).url)
+ url = six.ensure_str(parsed_url.url)
conn = None
@@ -653,10 +663,14 @@ def urlopen(
# [1]
release_this_conn = release_conn
+ http_tunnel_required = connection_requires_http_tunnel(
+ self.proxy, self.proxy_config, destination_scheme
+ )
+
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
- if self.scheme == "http" or (self.proxy and self.proxy.scheme == "https"):
+ if not http_tunnel_required:
headers = headers.copy()
headers.update(self.proxy_headers)
@@ -682,7 +696,7 @@ def urlopen(
is_new_proxy_conn = self.proxy is not None and not getattr(
conn, "sock", None
)
- if is_new_proxy_conn:
+ if is_new_proxy_conn and http_tunnel_required:
self._prepare_proxy(conn)
# Make the request on the httplib connection object.
@@ -946,8 +960,10 @@ def _prepare_proxy(self, conn):
improperly set Host: header to proxy's IP:port.
"""
- if self.proxy.scheme != "https":
- conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
+ conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
+
+ if self.proxy.scheme == "https":
+ conn.tls_in_tls_required = True
conn.connect()
diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py
index 70933fd0a2..c8ae62c062 100644
--- a/src/urllib3/poolmanager.py
+++ b/src/urllib3/poolmanager.py
@@ -2,14 +2,12 @@
import collections
import functools
import logging
-import warnings
from ._collections import RecentlyUsedContainer
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from .connectionpool import port_by_scheme
from .exceptions import (
- HTTPWarning,
LocationValueError,
MaxRetryError,
ProxySchemeUnknown,
@@ -21,17 +19,13 @@
from .request import RequestMethods
from .util.url import parse_url
from .util.retry import Retry
+from .util.proxy import connection_requires_http_tunnel
+from .packages.six import PY3
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
-class InvalidProxyConfigurationWarning(HTTPWarning):
- """Raised when a user has an HTTPS proxy without enabling HTTPS proxies."""
-
- pass
-
-
log = logging.getLogger(__name__)
SSL_KEYWORDS = (
@@ -68,6 +62,7 @@ class InvalidProxyConfigurationWarning(HTTPWarning):
"key_headers", # dict
"key__proxy", # parsed proxy url
"key__proxy_headers", # dict
+ "key__proxy_config", # class
"key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples
"key__socks_options", # dict
"key_assert_hostname", # bool or string
@@ -79,6 +74,9 @@ class InvalidProxyConfigurationWarning(HTTPWarning):
#: All custom key schemes should include the fields in this key at a minimum.
PoolKey = collections.namedtuple("PoolKey", _key_fields)
+_proxy_config_fields = ("ssl_context", "use_forwarding_for_https")
+ProxyConfig = collections.namedtuple("ProxyConfig", _proxy_config_fields)
+
def _default_key_normalizer(key_class, request_context):
"""
@@ -170,6 +168,7 @@ class PoolManager(RequestMethods):
"""
proxy = None
+ proxy_config = None
def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
RequestMethods.__init__(self, headers)
@@ -326,14 +325,32 @@ def _merge_pool_kwargs(self, override):
def _proxy_requires_url_absolute_form(self, parsed_url):
"""
Indicates if the proxy requires the complete destination URL in the
- request.
-
- Normally this is only needed when not using an HTTP CONNECT tunnel.
+ request. Normally this is only needed when not using an HTTP CONNECT
+ tunnel.
"""
if self.proxy is None:
return False
- return parsed_url.scheme == "http" or self.proxy.scheme == "https"
+ return not connection_requires_http_tunnel(
+ self.proxy, self.proxy_config, parsed_url.scheme
+ )
+
+ def _validate_proxy_scheme_url_selection(self, url_scheme):
+ """
+ Validates that were not attempting to do TLS in TLS connections on
+ Python2 or with unsupported SSL implementations.
+ """
+ if self.proxy is None or url_scheme != "https":
+ return
+
+ if self.proxy.scheme != "https":
+ return
+
+ if not PY3 and not self.proxy_config.use_forwarding_for_https:
+ raise ProxySchemeUnsupported(
+ "Contacting HTTPS destinations through HTTPS proxies "
+ "'via CONNECT tunnels' is not supported in Python 2"
+ )
def urlopen(self, method, url, redirect=True, **kw):
"""
@@ -345,6 +362,8 @@ def urlopen(self, method, url, redirect=True, **kw):
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
u = parse_url(url)
+ self._validate_proxy_scheme_url_selection(u.scheme)
+
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
kw["assert_same_host"] = False
@@ -415,11 +434,18 @@ class ProxyManager(PoolManager):
HTTPS/CONNECT case they are sent only once. Could be used for proxy
authentication.
- :param _allow_https_proxy_to_see_traffic:
- Allows forwarding of HTTPS requests to HTTPS proxies. The proxy will
- have visibility of all the traffic sent. ONLY USE IF YOU KNOW WHAT
- YOU'RE DOING. This flag might be removed at any time in any future
- update.
+ :param proxy_ssl_context:
+ The proxy SSL context is used to establish the TLS connection to the
+ proxy when using HTTPS proxies.
+
+ :param use_forwarding_for_https:
+ (Defaults to False) If set to True will forward requests to the HTTPS
+ proxy to be made on behalf of the client instead of creating a TLS
+ tunnel via the CONNECT method. **Enabling this flag means that request
+ and response headers and content will be visible from the HTTPS proxy**
+ whereas tunneling keeps request and response headers and content
+ private. IP address, target hostname, SNI, and port are always visible
+ to an HTTPS proxy even when this flag is disabled.
Example:
>>> proxy = urllib3.ProxyManager('http://localhost:3128/')
@@ -440,7 +466,8 @@ def __init__(
num_pools=10,
headers=None,
proxy_headers=None,
- _allow_https_proxy_to_see_traffic=False,
+ proxy_ssl_context=None,
+ use_forwarding_for_https=False,
**connection_pool_kw
):
@@ -461,11 +488,12 @@ def __init__(
self.proxy = proxy
self.proxy_headers = proxy_headers or {}
+ self.proxy_ssl_context = proxy_ssl_context
+ self.proxy_config = ProxyConfig(proxy_ssl_context, use_forwarding_for_https)
connection_pool_kw["_proxy"] = self.proxy
connection_pool_kw["_proxy_headers"] = self.proxy_headers
-
- self.allow_insecure_proxy = _allow_https_proxy_to_see_traffic
+ connection_pool_kw["_proxy_config"] = self.proxy_config
super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
@@ -494,35 +522,13 @@ def _set_proxy_headers(self, url, headers=None):
headers_.update(headers)
return headers_
- def _validate_proxy_scheme_url_selection(self, url_scheme):
- if (
- url_scheme == "https"
- and self.proxy.scheme == "https"
- and not self.allow_insecure_proxy
- ):
- warnings.warn(
- "Your proxy configuration specified an HTTPS scheme for the proxy. "
- "Are you sure you want to use HTTPS to contact the proxy? "
- "This most likely indicates an error in your configuration."
- "If you are sure you want use HTTPS to contact the proxy, enable "
- "the _allow_https_proxy_to_see_traffic.",
- InvalidProxyConfigurationWarning,
- )
-
- raise ProxySchemeUnsupported(
- "Contacting HTTPS destinations through HTTPS proxies is not supported."
- )
-
def urlopen(self, method, url, redirect=True, **kw):
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
u = parse_url(url)
- self._validate_proxy_scheme_url_selection(u.scheme)
-
- if u.scheme == "http" or self.proxy.scheme == "https":
+ if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
# For connections using HTTP CONNECT, httplib sets the necessary
- # headers on the CONNECT to the proxy. For HTTP or when talking
- # HTTPS to the proxy, we'll definitely need to set 'Host' at the
- # very least.
+ # headers on the CONNECT to the proxy. If we're not using CONNECT,
+ # we'll definitely need to set 'Host' at the very least.
headers = kw.get("headers", self.headers)
kw["headers"] = self._set_proxy_headers(url, headers)
diff --git a/src/urllib3/util/proxy.py b/src/urllib3/util/proxy.py
new file mode 100644
index 0000000000..68c6af33c7
--- /dev/null
+++ b/src/urllib3/util/proxy.py
@@ -0,0 +1,60 @@
+from .ssl_ import (
+ resolve_cert_reqs,
+ resolve_ssl_version,
+ create_urllib3_context,
+)
+
+
+def connection_requires_http_tunnel(
+ proxy_url=None, proxy_config=None, destination_scheme=None
+):
+ """
+ Returns True if the connection requires an HTTP CONNECT through the proxy.
+
+ :param URL proxy_url:
+ URL of the proxy.
+ :param ProxyConfig proxy_config:
+ Proxy configuration from poolmanager.py
+ :param str destination_scheme:
+ The scheme of the destination. (i.e https, http, etc)
+ """
+ # If we're not using a proxy, no way to use a tunnel.
+ if proxy_url is None:
+ return False
+
+ # HTTP destinations never require tunneling, we always forward.
+ if destination_scheme == "http":
+ return False
+
+ # Support for forwarding with HTTPS proxies and HTTPS destinations.
+ if (
+ proxy_url.scheme == "https"
+ and proxy_config
+ and proxy_config.use_forwarding_for_https
+ ):
+ return False
+
+ # Otherwise always use a tunnel.
+ return True
+
+
+def create_proxy_ssl_context(
+ ssl_version, cert_reqs, ca_certs=None, ca_cert_dir=None, ca_cert_data=None
+):
+ """
+ Generates a default proxy ssl context if one hasn't been provided by the
+ user.
+ """
+ ssl_context = create_urllib3_context(
+ ssl_version=resolve_ssl_version(ssl_version),
+ cert_reqs=resolve_cert_reqs(cert_reqs),
+ )
+ if (
+ not ca_certs
+ and not ca_cert_dir
+ and not ca_cert_data
+ and hasattr(ssl_context, "load_default_certs")
+ ):
+ ssl_context.load_default_certs()
+
+ return ssl_context
diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py
index 70982a58c1..6ae7f4707a 100644
--- a/src/urllib3/util/ssl_.py
+++ b/src/urllib3/util/ssl_.py
@@ -8,11 +8,17 @@
from hashlib import md5, sha1, sha256
from .url import IPV4_RE, BRACELESS_IPV6_ADDRZ_RE
-from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning
+from ..exceptions import (
+ SSLError,
+ InsecurePlatformWarning,
+ SNIMissingWarning,
+ ProxySchemeUnsupported,
+)
from ..packages import six
SSLContext = None
+SSLTransport = None
HAS_SNI = False
IS_PYOPENSSL = False
IS_SECURETRANSPORT = False
@@ -41,6 +47,7 @@ def _const_compare_digest_backport(a, b):
import ssl
from ssl import wrap_socket, CERT_REQUIRED
from ssl import HAS_SNI # Has SNI?
+ from .ssltransport import SSLTransport
except ImportError:
pass
@@ -327,6 +334,7 @@ def ssl_wrap_socket(
ca_cert_dir=None,
key_password=None,
ca_cert_data=None,
+ tls_in_tls=False,
):
"""
All arguments except for server_hostname, ssl_context, and ca_cert_dir have
@@ -348,6 +356,8 @@ def ssl_wrap_socket(
:param ca_cert_data:
Optional string containing CA certificates in PEM format suitable for
passing as the cadata parameter to SSLContext.load_verify_locations()
+ :param tls_in_tls:
+ Use SSLTransport to wrap the existing socket.
"""
context = ssl_context
if context is None:
@@ -405,9 +415,11 @@ def ssl_wrap_socket(
)
if send_sni:
- ssl_sock = context.wrap_socket(sock, server_hostname=server_hostname)
+ ssl_sock = _ssl_wrap_socket_impl(
+ sock, context, tls_in_tls, server_hostname=server_hostname
+ )
else:
- ssl_sock = context.wrap_socket(sock)
+ ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls)
return ssl_sock
@@ -433,3 +445,20 @@ def _is_key_file_encrypted(key_file):
return True
return False
+
+
+def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None):
+ if tls_in_tls:
+ if not SSLTransport:
+ # Import error, ssl is not available.
+ raise ProxySchemeUnsupported(
+ "TLS in TLS requires support for the 'ssl' module"
+ )
+
+ SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context)
+ return SSLTransport(sock, ssl_context, server_hostname)
+
+ if server_hostname:
+ return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
+ else:
+ return ssl_context.wrap_socket(sock)
diff --git a/src/urllib3/contrib/ssl.py b/src/urllib3/util/ssltransport.py
similarity index 87%
rename from src/urllib3/contrib/ssl.py
rename to src/urllib3/util/ssltransport.py
index 69a9898861..c38e2b8fed 100644
--- a/src/urllib3/contrib/ssl.py
+++ b/src/urllib3/util/ssltransport.py
@@ -2,6 +2,9 @@
import socket
import io
+from urllib3.exceptions import ProxySchemeUnsupported
+from urllib3.packages.six import PY3
+
SSL_BLOCKSIZE = 16384
@@ -16,6 +19,28 @@ class SSLTransport:
The class supports most of the socket API operations.
"""
+ @staticmethod
+ def _validate_ssl_context_for_tls_in_tls(ssl_context):
+ """
+ Raises a ProxySchemeUnsupported if the provided ssl_context can't be used
+ for TLS in TLS.
+
+ The only requirement is that the ssl_context provides the 'wrap_bio'
+ methods.
+ """
+
+ if not hasattr(ssl_context, "wrap_bio"):
+ if not PY3:
+ raise ProxySchemeUnsupported(
+ "TLS in TLS requires SSLContext.wrap_bio() which isn't "
+ "supported on Python 2"
+ )
+ else:
+ raise ProxySchemeUnsupported(
+ "TLS in TLS requires SSLContext.wrap_bio() which isn't "
+ "available on non-native SSLContext"
+ )
+
def __init__(
self, socket, ssl_context, suppress_ragged_eofs=True, server_hostname=None
):
diff --git a/test/__init__.py b/test/__init__.py
index 5ab3cac1bf..faf2fd6bef 100644
--- a/test/__init__.py
+++ b/test/__init__.py
@@ -19,6 +19,11 @@
from urllib3.util import ssl_
from urllib3 import util
+try:
+ import urllib3.contrib.pyopenssl as pyopenssl
+except ImportError:
+ pyopenssl = None
+
# We need a host that will not immediately close the connection with a TCP
# Reset.
if platform.system() == "Windows":
@@ -166,6 +171,19 @@ def notBrotlipy():
)
+def onlySecureTransport(test):
+ """Runs this test when SecureTransport is in use."""
+
+ @six.wraps(test)
+ def wrapper(*args, **kwargs):
+ msg = "{name} only runs with SecureTransport".format(name=test.__name__)
+ if not ssl_.IS_SECURETRANSPORT:
+ pytest.skip(msg)
+ return test(*args, **kwargs)
+
+ return wrapper
+
+
def notSecureTransport(test):
"""Skips this test when SecureTransport is in use."""
@@ -290,6 +308,21 @@ def wrapper(*args, **kwargs):
return wrapper
+def withPyOpenSSL(test):
+ @six.wraps(test)
+ def wrapper(*args, **kwargs):
+ if not pyopenssl:
+ pytest.skip("pyopenssl not available, skipping test.")
+ return test(*args, **kwargs)
+
+ pyopenssl.inject_into_urllib3()
+ result = test(*args, **kwargs)
+ pyopenssl.extract_from_urllib3()
+ return result
+
+ return wrapper
+
+
class _ListHandler(logging.Handler):
def __init__(self):
super(_ListHandler, self).__init__()
diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py
index 0603236e97..6f42712902 100644
--- a/test/test_poolmanager.py
+++ b/test/test_poolmanager.py
@@ -366,3 +366,9 @@ def test_merge_pool_kwargs_invalid_key(self):
p = PoolManager(strict=True)
merged = p._merge_pool_kwargs({"invalid_key": None})
assert p.connection_pool_kw == merged
+
+ def test_pool_manager_no_url_absolute_form(self):
+ """Valides we won't send a request with absolute form without a proxy"""
+ p = PoolManager(strict=True)
+ assert p._proxy_requires_url_absolute_form("http://example.com") is False
+ assert p._proxy_requires_url_absolute_form("https://example.com") is False
diff --git a/test/test_proxymanager.py b/test/test_proxymanager.py
index 2043580c8a..fe8f72a298 100644
--- a/test/test_proxymanager.py
+++ b/test/test_proxymanager.py
@@ -62,6 +62,10 @@ def test_proxy_tunnel(self):
assert p._proxy_requires_url_absolute_form(https_url) is False
with ProxyManager("https://proxy:8080") as p:
+ assert p._proxy_requires_url_absolute_form(http_url)
+ assert p._proxy_requires_url_absolute_form(https_url) is False
+
+ with ProxyManager("https://proxy:8080", use_forwarding_for_https=True) as p:
assert p._proxy_requires_url_absolute_form(http_url)
assert p._proxy_requires_url_absolute_form(https_url)
diff --git a/test/contrib/test_ssltransport.py b/test/test_ssltransport.py
similarity index 99%
rename from test/contrib/test_ssltransport.py
rename to test/test_ssltransport.py
index d68afd4a90..2b8ac4272f 100644
--- a/test/contrib/test_ssltransport.py
+++ b/test/test_ssltransport.py
@@ -4,7 +4,7 @@
DEFAULT_CA,
)
-from urllib3.contrib.ssl import SSLTransport
+from urllib3.util.ssltransport import SSLTransport
import select
import pytest
diff --git a/test/test_util.py b/test/test_util.py
index ba3cc2f971..45b6ea1fc5 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -28,10 +28,16 @@
SNIMissingWarning,
UnrewindableBodyError,
)
-from urllib3.util.connection import allowed_gai_family, _has_ipv6
+from urllib3.util.proxy import (
+ connection_requires_http_tunnel,
+ create_proxy_ssl_context,
+)
from urllib3.util import is_fp_closed
+from urllib3.util.connection import allowed_gai_family, _has_ipv6
from urllib3.packages import six
+from urllib3.poolmanager import ProxyConfig
+
from . import clear_warnings
from test import onlyPy3, onlyPy2, onlyBrotlipy, notBrotlipy
@@ -748,6 +754,34 @@ def test_assert_header_parsing_throws_typeerror_with_non_headers(self, headers):
with pytest.raises(TypeError):
assert_header_parsing(headers)
+ def test_connection_requires_http_tunnel_no_proxy(self):
+ assert not connection_requires_http_tunnel(
+ proxy_url=None, proxy_config=None, destination_scheme=None
+ )
+
+ def test_connection_requires_http_tunnel_http_proxy(self):
+ proxy = parse_url("http://proxy:8080")
+ proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False)
+ destination_scheme = "http"
+ assert not connection_requires_http_tunnel(
+ proxy, proxy_config, destination_scheme
+ )
+
+ destination_scheme = "https"
+ assert connection_requires_http_tunnel(proxy, proxy_config, destination_scheme)
+
+ def test_connection_requires_http_tunnel_https_proxy(self):
+ proxy = parse_url("https://proxy:8443")
+ proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False)
+ destination_scheme = "http"
+ assert not connection_requires_http_tunnel(
+ proxy, proxy_config, destination_scheme
+ )
+
+ def test_create_proxy_ssl_context(self):
+ ssl_context = create_proxy_ssl_context(ssl_version=None, cert_reqs=None)
+ ssl_context.verify_mode = ssl.CERT_REQUIRED
+
@onlyPy3
def test_assert_header_parsing_no_error_on_multipart(self):
from http import client
diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py
index 2125f14480..40f1a2597a 100644
--- a/test/with_dummyserver/test_proxy_poolmanager.py
+++ b/test/with_dummyserver/test_proxy_poolmanager.py
@@ -22,8 +22,17 @@
ProxySchemeUnsupported,
)
from urllib3.connectionpool import connection_from_url, VerifiedHTTPSConnection
+from urllib3.util.ssl_ import create_urllib3_context
+
+from test import (
+ SHORT_TIMEOUT,
+ LONG_TIMEOUT,
+ onlyPy3,
+ onlyPy2,
+ withPyOpenSSL,
+ onlySecureTransport,
+)
-from test import SHORT_TIMEOUT, LONG_TIMEOUT
# Retry failed tests
pytestmark = pytest.mark.flaky
@@ -63,21 +72,75 @@ def test_basic_proxy(self):
r = http.request("GET", "%s/" % self.https_url)
assert r.status == 200
+ @onlyPy3
def test_https_proxy(self):
+ with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https:
+ r = https.request("GET", "%s/" % self.https_url)
+ assert r.status == 200
+
+ r = https.request("GET", "%s/" % self.http_url)
+ assert r.status == 200
+
+ @onlyPy3
+ def test_https_proxy_with_proxy_ssl_context(self):
+ proxy_ssl_context = create_urllib3_context()
+ proxy_ssl_context.load_verify_locations(DEFAULT_CA)
+ with proxy_from_url(
+ self.https_proxy_url,
+ proxy_ssl_context=proxy_ssl_context,
+ ca_certs=DEFAULT_CA,
+ ) as https:
+ r = https.request("GET", "%s/" % self.https_url)
+ assert r.status == 200
+
+ r = https.request("GET", "%s/" % self.http_url)
+ assert r.status == 200
+
+ @onlyPy2
+ def test_https_proxy_not_supported(self):
+ with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https:
+ r = https.request("GET", "%s/" % self.http_url)
+ assert r.status == 200
+
+ with pytest.raises(ProxySchemeUnsupported) as excinfo:
+ https.request("GET", "%s/" % self.https_url)
+
+ assert "is not supported in Python 2" in str(excinfo.value)
+
+ @withPyOpenSSL
+ @onlyPy3
+ def test_https_proxy_pyopenssl_not_supported(self):
+ with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https:
+ r = https.request("GET", "%s/" % self.http_url)
+ assert r.status == 200
+
+ with pytest.raises(ProxySchemeUnsupported) as excinfo:
+ https.request("GET", "%s/" % self.https_url)
+
+ assert "isn't available on non-native SSLContext" in str(excinfo.value)
+
+ @onlySecureTransport
+ @onlyPy3
+ def test_https_proxy_securetransport_not_supported(self):
with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https:
r = https.request("GET", "%s/" % self.http_url)
assert r.status == 200
- with pytest.raises(ProxySchemeUnsupported):
+ with pytest.raises(ProxySchemeUnsupported) as excinfo:
https.request("GET", "%s/" % self.https_url)
+ assert "isn't available on non-native SSLContext" in str(excinfo.value)
+
+ def test_https_proxy_forwarding_for_https(self):
with proxy_from_url(
self.https_proxy_url,
ca_certs=DEFAULT_CA,
- _allow_https_proxy_to_see_traffic=True,
+ use_forwarding_for_https=True,
) as https:
r = https.request("GET", "%s/" % self.http_url)
- https.request("GET", "%s/" % self.https_url)
+ assert r.status == 200
+
+ r = https.request("GET", "%s/" % self.https_url)
assert r.status == 200
def test_nagle_proxy(self):
@@ -302,6 +365,7 @@ def test_headers(self):
self.https_port,
)
+ @onlyPy3
def test_https_headers(self):
with proxy_from_url(
self.https_proxy_url,
@@ -328,19 +392,34 @@ def test_https_headers(self):
self.http_port,
)
- with pytest.raises(ProxySchemeUnsupported):
- http.request_encode_url("GET", "%s/headers" % self.https_url)
-
- r = http.request_encode_url(
- "GET", "%s/headers" % self.http_url, headers={"Baz": "quux"}
+ r = http.request_encode_body(
+ "GET", "%s/headers" % self.https_url, headers={"Baz": "quux"}
)
returned_headers = json.loads(r.data.decode())
assert returned_headers.get("Foo") is None
assert returned_headers.get("Baz") == "quux"
+ assert returned_headers.get("Hickory") is None
+ assert returned_headers.get("Host") == "%s:%s" % (
+ self.https_host,
+ self.https_port,
+ )
+
+ def test_https_headers_forwarding_for_https(self):
+ with proxy_from_url(
+ self.https_proxy_url,
+ headers={"Foo": "bar"},
+ proxy_headers={"Hickory": "dickory"},
+ ca_certs=DEFAULT_CA,
+ use_forwarding_for_https=True,
+ ) as http:
+
+ r = http.request_encode_url("GET", "%s/headers" % self.https_url)
+ returned_headers = json.loads(r.data.decode())
+ assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Hickory") == "dickory"
assert returned_headers.get("Host") == "%s:%s" % (
- self.http_host,
- self.http_port,
+ self.https_host,
+ self.https_port,
)
def test_headerdict(self):
From f76e1cd63660447803fca342b11c62be602139ad Mon Sep 17 00:00:00 2001
From: Quentin Pradet
Date: Tue, 29 Sep 2020 00:47:11 +0400
Subject: [PATCH 10/14] Enforce flake8-2020
We don't know if we'll ever have a Python 4 or if six.PY3 will include
Python 4+ or if six will still be of any use by then, but I'd like
avoiding having to think about it in reviews. :)
---
noxfile.py | 2 +-
src/urllib3/_collections.py | 7 ++++---
src/urllib3/poolmanager.py | 3 +--
src/urllib3/response.py | 10 +++++-----
src/urllib3/util/ssltransport.py | 4 ++--
5 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/noxfile.py b/noxfile.py
index 4b1cf3a11d..e9f98a9783 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -111,7 +111,7 @@ def blacken(session):
@nox.session
def lint(session):
- session.install("flake8", "black", "mypy")
+ session.install("flake8", "flake8-2020", "black", "mypy")
session.run("flake8", "--version")
session.run("black", "--version")
session.run("mypy", "--version")
diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py
index 5c8f366e34..0b77482b49 100644
--- a/src/urllib3/_collections.py
+++ b/src/urllib3/_collections.py
@@ -18,7 +18,8 @@ def __exit__(self, exc_type, exc_value, traceback):
from collections import OrderedDict
from .exceptions import InvalidHeader
-from .packages.six import ensure_str, iterkeys, itervalues, PY3
+from .packages import six
+from .packages.six import iterkeys, itervalues
__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
@@ -154,7 +155,7 @@ def __setitem__(self, key, val):
def __getitem__(self, key):
val = self._container[key.lower()]
- return ", ".join([ensure_str(v, "ascii") for v in val[1:]])
+ return ", ".join([six.ensure_str(v, "ascii") for v in val[1:]])
def __delitem__(self, key):
del self._container[key.lower()]
@@ -174,7 +175,7 @@ def __eq__(self, other):
def __ne__(self, other):
return not self.__eq__(other)
- if not PY3: # Python 2
+ if six.PY2: # Python 2
iterkeys = MutableMapping.iterkeys
itervalues = MutableMapping.itervalues
diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py
index c8ae62c062..ee22a8e88a 100644
--- a/src/urllib3/poolmanager.py
+++ b/src/urllib3/poolmanager.py
@@ -20,7 +20,6 @@
from .util.url import parse_url
from .util.retry import Retry
from .util.proxy import connection_requires_http_tunnel
-from .packages.six import PY3
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
@@ -346,7 +345,7 @@ def _validate_proxy_scheme_url_selection(self, url_scheme):
if self.proxy.scheme != "https":
return
- if not PY3 and not self.proxy_config.use_forwarding_for_https:
+ if six.PY2 and not self.proxy_config.use_forwarding_for_https:
raise ProxySchemeUnsupported(
"Contacting HTTPS destinations through HTTPS proxies "
"'via CONNECT tunnels' is not supported in Python 2"
diff --git a/src/urllib3/response.py b/src/urllib3/response.py
index 149486ed57..accdf1fd6d 100644
--- a/src/urllib3/response.py
+++ b/src/urllib3/response.py
@@ -24,7 +24,7 @@
HTTPError,
SSLError,
)
-from .packages.six import string_types as basestring, PY3
+from .packages import six
from .connection import HTTPException, BaseSSLError
from .util.response import is_fp_closed, is_response_to_head
@@ -233,7 +233,7 @@ def __init__(
self.msg = msg
self._request_url = request_url
- if body and isinstance(body, (basestring, bytes)):
+ if body and isinstance(body, (six.string_types, bytes)):
self._body = body
self._pool = pool
@@ -589,11 +589,11 @@ def from_httplib(ResponseCls, r, **response_kw):
headers = r.msg
if not isinstance(headers, HTTPHeaderDict):
- if PY3:
- headers = HTTPHeaderDict(headers.items())
- else:
+ if six.PY2:
# Python 2.7
headers = HTTPHeaderDict.from_httplib(headers)
+ else:
+ headers = HTTPHeaderDict(headers.items())
# HTTPResponse objects in Python 3 don't have a .strict attribute
strict = getattr(r, "strict", 0)
diff --git a/src/urllib3/util/ssltransport.py b/src/urllib3/util/ssltransport.py
index c38e2b8fed..8cd1053417 100644
--- a/src/urllib3/util/ssltransport.py
+++ b/src/urllib3/util/ssltransport.py
@@ -3,7 +3,7 @@
import io
from urllib3.exceptions import ProxySchemeUnsupported
-from urllib3.packages.six import PY3
+from urllib3.packages import six
SSL_BLOCKSIZE = 16384
@@ -30,7 +30,7 @@ def _validate_ssl_context_for_tls_in_tls(ssl_context):
"""
if not hasattr(ssl_context, "wrap_bio"):
- if not PY3:
+ if six.PY2:
raise ProxySchemeUnsupported(
"TLS in TLS requires SSLContext.wrap_bio() which isn't "
"supported on Python 2"
From 6e2c387f5b6f52779de0a16afa06922ea75d3ff5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=B0=AD=E4=B9=9D=E9=BC=8E?= <109224573@qq.com>
Date: Tue, 29 Sep 2020 18:11:43 +0800
Subject: [PATCH 11/14] Use GitHub Actions checkout v2 (#2006)
---
.github/workflows/ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b6b6bd75ea..5ca94207b1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,9 +12,9 @@ jobs:
steps:
- name: Checkout Repository
- uses: actions/checkout@v1
+ uses: actions/checkout@v2
- name: Set up Python 3.8
- uses: actions/setup-python@v1
+ uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Install dependencies
From 7230507245b836621984dc47d23590ba38f2dc4d Mon Sep 17 00:00:00 2001
From: Quentin Pradet
Date: Tue, 29 Sep 2020 08:58:51 +0400
Subject: [PATCH 12/14] Stop testing docs in Travis
Let's rely on the ReadTheDocs pull request check instead. This does mean
that `nox -rs docs` won't get exercised by continuous integration, which
is an acceptable compromise.
---
.travis.yml | 4 ----
1 file changed, 4 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 3c64246036..ca52bf2681 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,10 +32,6 @@ jobs:
allow_failures:
- python: nightly
include:
- # Lint & documentation.
- - python: 3.8
- env: NOX_SESSION=docs
-
# Unit tests
- python: 2.7
env: NOX_SESSION=test-2.7
From 9bc44598d90a39a6de665d37dc593905bd8563f8 Mon Sep 17 00:00:00 2001
From: Hasan Ramezani
Date: Wed, 30 Sep 2020 21:42:43 +0200
Subject: [PATCH 13/14] Raise error on invalid labels in create_connection()
---
src/urllib3/util/connection.py | 11 +++++++++++
test/test_util.py | 24 ++++++++++++++++++++++++
2 files changed, 35 insertions(+)
diff --git a/src/urllib3/util/connection.py b/src/urllib3/util/connection.py
index 9c89dc1426..75bb4dd8db 100644
--- a/src/urllib3/util/connection.py
+++ b/src/urllib3/util/connection.py
@@ -1,7 +1,11 @@
from __future__ import absolute_import
import socket
+
+from urllib3.exceptions import LocationParseError
+
from .wait import NoWayToWaitForSocketError, wait_for_read
from ..contrib import _appengine_environ
+from ..packages import six
def is_connection_dropped(conn): # Platform-specific
@@ -58,6 +62,13 @@ def create_connection(
# The original create_connection function always returns all records.
family = allowed_gai_family()
+ try:
+ host.encode("idna")
+ except UnicodeError:
+ return six.raise_from(
+ LocationParseError(u"'%s', label empty or too long" % host), None
+ )
+
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
sock = None
diff --git a/test/test_util.py b/test/test_util.py
index 45b6ea1fc5..3dc6989ff7 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -11,6 +11,7 @@
import pytest
from urllib3 import add_stderr_logger, disable_warnings, util
+from urllib3.util.connection import create_connection
from urllib3.util.request import make_headers, rewind_body, _FAILEDTELL
from urllib3.util.response import assert_header_parsing
from urllib3.util.timeout import Timeout
@@ -796,6 +797,29 @@ def test_assert_header_parsing_no_error_on_multipart(self):
header_msg.seek(0)
assert_header_parsing(client.parse_headers(header_msg))
+ @pytest.mark.parametrize("host", [".localhost", "...", "t" * 64])
+ def test_create_connection_with_invalid_idna_labels(self, host):
+ with pytest.raises(LocationParseError) as ctx:
+ create_connection((host, 80))
+ assert str(ctx.value) == "Failed to parse: '%s', label empty or too long" % host
+
+ @pytest.mark.parametrize(
+ "host",
+ [
+ "a.example.com",
+ "localhost.",
+ "[dead::beef]",
+ "[dead::beef%en5]",
+ "[dead::beef%en5.]",
+ ],
+ )
+ @patch("socket.getaddrinfo")
+ @patch("socket.socket")
+ def test_create_connection_with_valid_idna_labels(self, socket, getaddrinfo, host):
+ getaddrinfo.return_value = [(None, None, None, None, None)]
+ socket.return_value = Mock()
+ create_connection((host, 80))
+
class TestUtilSSL(object):
"""Test utils that use an SSL backend."""
From dabe77d7d3f69c1b961826dabac28eda86699024 Mon Sep 17 00:00:00 2001
From: Quentin Pradet
Date: Thu, 1 Oct 2020 16:45:14 +0400
Subject: [PATCH 14/14] Sort imports with 'isort'
---
docs/conf.py | 3 +-
docs/contributing.rst | 2 +-
dummyserver/handlers.py | 9 ++-
dummyserver/proxy.py | 6 +-
dummyserver/server.py | 13 ++---
dummyserver/testcase.py | 13 ++---
noxfile.py | 12 ++--
setup.py | 6 +-
src/urllib3/__init__.py | 18 +++---
src/urllib3/_collections.py | 2 +-
src/urllib3/connection.py | 25 ++++-----
src/urllib3/connectionpool.py | 50 ++++++++---------
.../contrib/_securetransport/bindings.py | 18 +++---
.../contrib/_securetransport/low_level.py | 5 +-
src/urllib3/contrib/appengine.py | 8 +--
src/urllib3/contrib/ntlmpool.py | 2 +-
src/urllib3/contrib/pyopenssl.py | 6 +-
src/urllib3/contrib/securetransport.py | 9 +--
src/urllib3/contrib/socks.py | 4 +-
src/urllib3/exceptions.py | 1 +
src/urllib3/exceptions.pyi | 2 +-
src/urllib3/fields.py | 1 +
src/urllib3/filepost.py | 4 +-
src/urllib3/packages/backports/makefile.py | 1 -
.../packages/ssl_match_hostname/__init__.py | 5 +-
src/urllib3/poolmanager.py | 10 ++--
src/urllib3/request.py | 1 -
src/urllib3/response.py | 17 +++---
src/urllib3/util/__init__.py | 15 +++--
src/urllib3/util/connection.py | 3 +-
src/urllib3/util/proxy.py | 6 +-
src/urllib3/util/queue.py | 1 +
src/urllib3/util/request.py | 3 +-
src/urllib3/util/response.py | 5 +-
src/urllib3/util/retry.py | 14 ++---
src/urllib3/util/ssl_.py | 16 +++---
src/urllib3/util/ssltransport.py | 4 +-
src/urllib3/util/timeout.py | 3 +-
src/urllib3/util/url.py | 2 +-
src/urllib3/util/wait.py | 2 +-
test/__init__.py | 10 ++--
test/appengine/conftest.py | 1 -
test/appengine/test_gae_manager.py | 12 ++--
test/appengine/test_urlfetch.py | 3 +-
test/benchmark.py | 1 -
test/conftest.py | 11 ++--
test/contrib/test_pyopenssl.py | 13 +++--
test/contrib/test_pyopenssl_dependencies.py | 5 +-
test/contrib/test_securetransport.py | 2 +-
test/contrib/test_socks.py | 14 ++---
test/port_helpers.py | 1 -
test/test_collections.py | 3 +-
test/test_compatibility.py | 2 +-
test/test_connection.py | 4 +-
test/test_connectionpool.py | 28 +++++-----
test/test_exceptions.py | 12 ++--
test/test_fields.py | 2 +-
test/test_filepost.py | 3 +-
test/test_no_ssl.py | 1 +
test/test_poolmanager.py | 4 +-
test/test_proxymanager.py | 11 ++--
test/test_queue_monkeypatch.py | 1 -
test/test_response.py | 23 ++++----
test/test_retry.py | 11 ++--
test/test_retry_deprecated.py | 11 ++--
test/test_ssl.py | 7 ++-
test/test_ssltransport.py | 16 ++----
test/test_util.py | 46 +++++++---------
test/test_wait.py | 12 ++--
test/tz_stub.py | 5 +-
.../with_dummyserver/test_chunked_transfer.py | 8 +--
test/with_dummyserver/test_connectionpool.py | 23 ++++----
test/with_dummyserver/test_https.py | 48 ++++++++--------
test/with_dummyserver/test_no_ssl.py | 4 +-
test/with_dummyserver/test_poolmanager.py | 5 +-
.../test_proxy_poolmanager.py | 31 +++++------
test/with_dummyserver/test_socketlevel.py | 55 +++++++++----------
77 files changed, 373 insertions(+), 403 deletions(-)
diff --git a/docs/conf.py b/docs/conf.py
index 47470b5073..5d5dda25b6 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
-from datetime import date
import os
import sys
+from datetime import date
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
@@ -30,7 +30,6 @@ def __getattr__(cls, name):
import urllib3
-
# -- General configuration -----------------------------------------------------
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 4102e01d74..05cac9ca0e 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -13,7 +13,7 @@ If you wish to add a new feature or fix a bug:
to start making your changes.
#. Write a test which shows that the bug was fixed or that the feature works
as expected.
-#. Format your changes with black using command `$ nox -rs blacken` and lint your
+#. Format your changes with black using command `$ nox -rs format` and lint your
changes using command `nox -rs lint`.
#. Send a pull request and bug the maintainer until it gets merged and published.
:) Make sure to add yourself to ``CONTRIBUTORS.txt``.
diff --git a/dummyserver/handlers.py b/dummyserver/handlers.py
index 1c8e9b6c86..c047094c4f 100644
--- a/dummyserver/handlers.py
+++ b/dummyserver/handlers.py
@@ -8,16 +8,15 @@
import sys
import time
import zlib
-
+from datetime import datetime, timedelta
from io import BytesIO
-from tornado.web import RequestHandler
+
from tornado import httputil
-from datetime import datetime
-from datetime import timedelta
+from tornado.web import RequestHandler
+from urllib3.packages.six import binary_type, ensure_str
from urllib3.packages.six.moves.http_client import responses
from urllib3.packages.six.moves.urllib.parse import urlsplit
-from urllib3.packages.six import binary_type, ensure_str
log = logging.getLogger(__name__)
diff --git a/dummyserver/proxy.py b/dummyserver/proxy.py
index 42f293104d..0cd8dedd26 100755
--- a/dummyserver/proxy.py
+++ b/dummyserver/proxy.py
@@ -25,16 +25,16 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
-import sys
import socket
+import ssl
+import sys
import tornado.gen
+import tornado.httpclient
import tornado.httpserver
import tornado.ioloop
import tornado.iostream
import tornado.web
-import tornado.httpclient
-import ssl
__all__ = ["ProxyHandler", "run_proxy"]
diff --git a/dummyserver/server.py b/dummyserver/server.py
index f8b4dff5ce..9ecde97f35 100755
--- a/dummyserver/server.py
+++ b/dummyserver/server.py
@@ -7,24 +7,23 @@
import logging
import os
+import socket
+import ssl
import sys
import threading
-import socket
import warnings
-import ssl
from datetime import datetime
-from urllib3.exceptions import HTTPWarning
-from urllib3.util import resolve_cert_reqs, resolve_ssl_version, ALPN_PROTOCOLS
-
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import serialization
import tornado.httpserver
import tornado.ioloop
import tornado.netutil
import tornado.web
import trustme
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from urllib3.exceptions import HTTPWarning
+from urllib3.util import ALPN_PROTOCOLS, resolve_cert_reqs, resolve_ssl_version
log = logging.getLogger(__name__)
diff --git a/dummyserver/testcase.py b/dummyserver/testcase.py
index 5a54922d4b..6a49e36cd2 100644
--- a/dummyserver/testcase.py
+++ b/dummyserver/testcase.py
@@ -4,17 +4,16 @@
import pytest
from tornado import ioloop, web
-from urllib3.connection import HTTPConnection
-
+from dummyserver.handlers import TestingApp
+from dummyserver.proxy import ProxyHandler
from dummyserver.server import (
- SocketServerThread,
- run_tornado_app,
- run_loop_in_thread,
DEFAULT_CERTS,
HAS_IPV6,
+ SocketServerThread,
+ run_loop_in_thread,
+ run_tornado_app,
)
-from dummyserver.handlers import TestingApp
-from dummyserver.proxy import ProxyHandler
+from urllib3.connection import HTTPConnection
def consume_socket(sock, chunks=65536):
diff --git a/noxfile.py b/noxfile.py
index e9f98a9783..a8667583ae 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -4,7 +4,6 @@
import nox
-
# Whenever type-hints are completed on a file it should be added here so that
# this file will continue to be checked by mypy. Errors from other files are
# ignored.
@@ -101,21 +100,24 @@ def app_engine(session):
@nox.session()
-def blacken(session):
- """Run black code formatter."""
- session.install("black")
+def format(session):
+ """Run code formatters."""
+ session.install("black", "isort")
session.run("black", *SOURCE_FILES)
+ session.run("isort", "--profile", "black", *SOURCE_FILES)
lint(session)
@nox.session
def lint(session):
- session.install("flake8", "flake8-2020", "black", "mypy")
+ session.install("flake8", "flake8-2020", "black", "isort", "mypy")
session.run("flake8", "--version")
session.run("black", "--version")
+ session.run("isort", "--version")
session.run("mypy", "--version")
session.run("black", "--check", *SOURCE_FILES)
+ session.run("isort", "--profile", "black", "--check", *SOURCE_FILES)
session.run("flake8", *SOURCE_FILES)
session.log("mypy --strict src/urllib3")
diff --git a/setup.py b/setup.py
index 8a96b9942b..d5030fbd79 100755
--- a/setup.py
+++ b/setup.py
@@ -1,11 +1,11 @@
#!/usr/bin/env python
# This file is protected via CODEOWNERS
-from setuptools import setup
-
+import codecs
import os
import re
-import codecs
+
+from setuptools import setup
base_path = os.path.dirname(__file__)
diff --git a/src/urllib3/__init__.py b/src/urllib3/__init__.py
index ac3f9efa55..fe86b59d78 100644
--- a/src/urllib3/__init__.py
+++ b/src/urllib3/__init__.py
@@ -2,24 +2,22 @@
Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more
"""
from __future__ import absolute_import
-import warnings
-from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
+# Set default logging handler to avoid "No handler found" warnings.
+import logging
+import warnings
+from logging import NullHandler
from . import exceptions
+from ._version import __version__
+from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
from .filepost import encode_multipart_formdata
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
from .response import HTTPResponse
from .util.request import make_headers
-from .util.url import get_host
-from .util.timeout import Timeout
from .util.retry import Retry
-from ._version import __version__
-
-
-# Set default logging handler to avoid "No handler found" warnings.
-import logging
-from logging import NullHandler
+from .util.timeout import Timeout
+from .util.url import get_host
__author__ = "Andrey Petrov (andrey.petrov@shazow.net)"
__license__ = "MIT"
diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py
index 0b77482b49..8a94ba5f74 100644
--- a/src/urllib3/_collections.py
+++ b/src/urllib3/_collections.py
@@ -17,11 +17,11 @@ def __exit__(self, exc_type, exc_value, traceback):
from collections import OrderedDict
+
from .exceptions import InvalidHeader
from .packages import six
from .packages.six import iterkeys, itervalues
-
__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py
index f9e84fdbfd..f3336acff1 100644
--- a/src/urllib3/connection.py
+++ b/src/urllib3/connection.py
@@ -1,11 +1,14 @@
from __future__ import absolute_import
-import re
+
import datetime
import logging
import os
+import re
import socket
-from socket import error as SocketError, timeout as SocketTimeout
import warnings
+from socket import error as SocketError
+from socket import timeout as SocketTimeout
+
from .packages import six
from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection
from .packages.six.moves.http_client import HTTPException # noqa: F401
@@ -40,28 +43,24 @@ class BrokenPipeError(Exception):
pass
+from ._collections import HTTPHeaderDict
+from ._version import __version__
from .exceptions import (
- NewConnectionError,
ConnectTimeoutError,
+ NewConnectionError,
SubjectAltNameWarning,
SystemTimeWarning,
)
-from .packages.ssl_match_hostname import match_hostname, CertificateError
-
+from .packages.ssl_match_hostname import CertificateError, match_hostname
+from .util import SUPPRESS_USER_AGENT, connection
from .util.ssl_ import (
- resolve_cert_reqs,
- resolve_ssl_version,
assert_fingerprint,
create_urllib3_context,
+ resolve_cert_reqs,
+ resolve_ssl_version,
ssl_wrap_socket,
)
-
-from .util import connection, SUPPRESS_USER_AGENT
-
-from ._collections import HTTPHeaderDict
-from ._version import __version__
-
log = logging.getLogger(__name__)
port_by_scheme = {"http": 80, "https": 443}
diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py
index c48a0b846d..da1931eeb3 100644
--- a/src/urllib3/connectionpool.py
+++ b/src/urllib3/connectionpool.py
@@ -1,59 +1,53 @@
from __future__ import absolute_import
+
import errno
import logging
+import socket
import sys
import warnings
+from socket import error as SocketError
+from socket import timeout as SocketTimeout
-from socket import error as SocketError, timeout as SocketTimeout
-import socket
-
-
+from .connection import (
+ BaseSSLError,
+ BrokenPipeError,
+ DummyConnection,
+ HTTPConnection,
+ HTTPException,
+ HTTPSConnection,
+ VerifiedHTTPSConnection,
+ port_by_scheme,
+)
from .exceptions import (
ClosedPoolError,
- ProtocolError,
EmptyPoolError,
HeaderParsingError,
HostChangedError,
+ InsecureRequestWarning,
LocationValueError,
MaxRetryError,
+ NewConnectionError,
+ ProtocolError,
ProxyError,
ReadTimeoutError,
SSLError,
TimeoutError,
- InsecureRequestWarning,
- NewConnectionError,
)
-from .packages.ssl_match_hostname import CertificateError
from .packages import six
from .packages.six.moves import queue
-from .connection import (
- port_by_scheme,
- DummyConnection,
- HTTPConnection,
- HTTPSConnection,
- VerifiedHTTPSConnection,
- HTTPException,
- BaseSSLError,
- BrokenPipeError,
-)
+from .packages.ssl_match_hostname import CertificateError
from .request import RequestMethods
from .response import HTTPResponse
-
from .util.connection import is_connection_dropped
from .util.proxy import connection_requires_http_tunnel
+from .util.queue import LifoQueue
from .util.request import set_file_position
from .util.response import assert_header_parsing
from .util.retry import Retry
from .util.timeout import Timeout
-from .util.url import (
- get_host,
- parse_url,
- Url,
- _normalize_host as normalize_host,
- _encode_target,
-)
-from .util.queue import LifoQueue
-
+from .util.url import Url, _encode_target
+from .util.url import _normalize_host as normalize_host
+from .util.url import get_host, parse_url
xrange = six.moves.xrange
diff --git a/src/urllib3/contrib/_securetransport/bindings.py b/src/urllib3/contrib/_securetransport/bindings.py
index e879cb5970..11524d400b 100644
--- a/src/urllib3/contrib/_securetransport/bindings.py
+++ b/src/urllib3/contrib/_securetransport/bindings.py
@@ -32,21 +32,23 @@
from __future__ import absolute_import
import platform
-from ctypes.util import find_library
from ctypes import (
- c_void_p,
- c_int32,
+ CDLL,
+ CFUNCTYPE,
+ POINTER,
+ c_bool,
+ c_byte,
c_char_p,
+ c_int32,
+ c_long,
c_size_t,
- c_byte,
c_uint32,
c_ulong,
- c_long,
- c_bool,
+ c_void_p,
)
-from ctypes import CDLL, POINTER, CFUNCTYPE
-from urllib3.packages.six import raise_from
+from ctypes.util import find_library
+from urllib3.packages.six import raise_from
if platform.system() != "Darwin":
raise ImportError("Only macOS is supported")
diff --git a/src/urllib3/contrib/_securetransport/low_level.py b/src/urllib3/contrib/_securetransport/low_level.py
index 3d466a52ed..bb302fc601 100644
--- a/src/urllib3/contrib/_securetransport/low_level.py
+++ b/src/urllib3/contrib/_securetransport/low_level.py
@@ -10,13 +10,12 @@
import base64
import ctypes
import itertools
-import re
import os
+import re
import ssl
import tempfile
-from .bindings import Security, CoreFoundation, CFConst
-
+from .bindings import CFConst, CoreFoundation, Security
# This regular expression is used to grab PEM data out of a PEM bundle.
_PEM_CERTS_RE = re.compile(
diff --git a/src/urllib3/contrib/appengine.py b/src/urllib3/contrib/appengine.py
index d69340a497..aa64a0914c 100644
--- a/src/urllib3/contrib/appengine.py
+++ b/src/urllib3/contrib/appengine.py
@@ -39,24 +39,24 @@
"""
from __future__ import absolute_import
+
import io
import logging
import warnings
-from ..packages.six.moves.urllib.parse import urljoin
from ..exceptions import (
HTTPError,
HTTPWarning,
MaxRetryError,
ProtocolError,
- TimeoutError,
SSLError,
+ TimeoutError,
)
-
+from ..packages.six.moves.urllib.parse import urljoin
from ..request import RequestMethods
from ..response import HTTPResponse
-from ..util.timeout import Timeout
from ..util.retry import Retry
+from ..util.timeout import Timeout
from . import _appengine_environ
try:
diff --git a/src/urllib3/contrib/ntlmpool.py b/src/urllib3/contrib/ntlmpool.py
index 1fd242a6e0..b2df45dcf6 100644
--- a/src/urllib3/contrib/ntlmpool.py
+++ b/src/urllib3/contrib/ntlmpool.py
@@ -6,12 +6,12 @@
from __future__ import absolute_import
from logging import getLogger
+
from ntlm import ntlm
from .. import HTTPSConnectionPool
from ..packages.six.moves.http_client import HTTPSConnection
-
log = getLogger(__name__)
diff --git a/src/urllib3/contrib/pyopenssl.py b/src/urllib3/contrib/pyopenssl.py
index 54ee493638..0cabab1aed 100644
--- a/src/urllib3/contrib/pyopenssl.py
+++ b/src/urllib3/contrib/pyopenssl.py
@@ -60,8 +60,9 @@ class UnsupportedExtension(Exception):
pass
-from socket import timeout, error as SocketError
from io import BytesIO
+from socket import error as SocketError
+from socket import timeout
try: # Platform-specific: Python 2
from socket import _fileobject
@@ -71,11 +72,10 @@ class UnsupportedExtension(Exception):
import logging
import ssl
-from ..packages import six
import sys
from .. import util
-
+from ..packages import six
__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
diff --git a/src/urllib3/contrib/securetransport.py b/src/urllib3/contrib/securetransport.py
index 3fffb2fe88..866f00d46f 100644
--- a/src/urllib3/contrib/securetransport.py
+++ b/src/urllib3/contrib/securetransport.py
@@ -58,20 +58,21 @@
import errno
import os.path
import shutil
-import six
import socket
import ssl
import threading
import weakref
+import six
+
from .. import util
-from ._securetransport.bindings import Security, SecurityConst, CoreFoundation
+from ._securetransport.bindings import CoreFoundation, Security, SecurityConst
from ._securetransport.low_level import (
_assert_no_error,
_cert_array_from_pem,
- _temporary_keychain,
- _load_client_cert_chain,
_create_cfstring_array,
+ _load_client_cert_chain,
+ _temporary_keychain,
)
try: # Platform-specific: Python 2
diff --git a/src/urllib3/contrib/socks.py b/src/urllib3/contrib/socks.py
index 516e610bca..93df8325d5 100644
--- a/src/urllib3/contrib/socks.py
+++ b/src/urllib3/contrib/socks.py
@@ -44,6 +44,7 @@
import socks
except ImportError:
import warnings
+
from ..exceptions import DependencyWarning
warnings.warn(
@@ -56,7 +57,8 @@
)
raise
-from socket import error as SocketError, timeout as SocketTimeout
+from socket import error as SocketError
+from socket import timeout as SocketTimeout
from ..connection import HTTPConnection, HTTPSConnection
from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool
diff --git a/src/urllib3/exceptions.py b/src/urllib3/exceptions.py
index 87d4ce77b9..d69958d5df 100644
--- a/src/urllib3/exceptions.py
+++ b/src/urllib3/exceptions.py
@@ -1,4 +1,5 @@
from __future__ import absolute_import
+
from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead
# Base Exceptions
diff --git a/src/urllib3/exceptions.pyi b/src/urllib3/exceptions.pyi
index 1254abe7f6..ca528b09ad 100644
--- a/src/urllib3/exceptions.pyi
+++ b/src/urllib3/exceptions.pyi
@@ -1,4 +1,4 @@
-from typing import Any, Optional, Union, Tuple, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
if TYPE_CHECKING:
from urllib3.connectionpool import ConnectionPool
diff --git a/src/urllib3/fields.py b/src/urllib3/fields.py
index 29c0bd95be..9d630f491d 100644
--- a/src/urllib3/fields.py
+++ b/src/urllib3/fields.py
@@ -1,4 +1,5 @@
from __future__ import absolute_import
+
import email.utils
import mimetypes
import re
diff --git a/src/urllib3/filepost.py b/src/urllib3/filepost.py
index b7b00992c6..36c9252c64 100644
--- a/src/urllib3/filepost.py
+++ b/src/urllib3/filepost.py
@@ -1,13 +1,13 @@
from __future__ import absolute_import
+
import binascii
import codecs
import os
-
from io import BytesIO
+from .fields import RequestField
from .packages import six
from .packages.six import b
-from .fields import RequestField
writer = codecs.lookup("utf-8")[3]
diff --git a/src/urllib3/packages/backports/makefile.py b/src/urllib3/packages/backports/makefile.py
index a3156a69c0..b8fb2154b6 100644
--- a/src/urllib3/packages/backports/makefile.py
+++ b/src/urllib3/packages/backports/makefile.py
@@ -7,7 +7,6 @@
wants to create a "fake" socket object.
"""
import io
-
from socket import SocketIO
diff --git a/src/urllib3/packages/ssl_match_hostname/__init__.py b/src/urllib3/packages/ssl_match_hostname/__init__.py
index 72e68993ac..6b12fd90aa 100644
--- a/src/urllib3/packages/ssl_match_hostname/__init__.py
+++ b/src/urllib3/packages/ssl_match_hostname/__init__.py
@@ -10,7 +10,10 @@
except ImportError:
try:
# Backport of the function from a pypi module
- from backports.ssl_match_hostname import CertificateError, match_hostname # type: ignore
+ from backports.ssl_match_hostname import ( # type: ignore
+ CertificateError,
+ match_hostname,
+ )
except ImportError:
# Our vendored copy
from ._implementation import CertificateError, match_hostname # type: ignore
diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py
index ee22a8e88a..3a31a285bf 100644
--- a/src/urllib3/poolmanager.py
+++ b/src/urllib3/poolmanager.py
@@ -1,12 +1,11 @@
from __future__ import absolute_import
+
import collections
import functools
import logging
from ._collections import RecentlyUsedContainer
-from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
-from .connectionpool import port_by_scheme
-
+from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
from .exceptions import (
LocationValueError,
MaxRetryError,
@@ -17,10 +16,9 @@
from .packages import six
from .packages.six.moves.urllib.parse import urljoin
from .request import RequestMethods
-from .util.url import parse_url
-from .util.retry import Retry
from .util.proxy import connection_requires_http_tunnel
-
+from .util.retry import Retry
+from .util.url import parse_url
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
diff --git a/src/urllib3/request.py b/src/urllib3/request.py
index b058bf6e9f..398386a5b9 100644
--- a/src/urllib3/request.py
+++ b/src/urllib3/request.py
@@ -3,7 +3,6 @@
from .filepost import encode_multipart_formdata
from .packages.six.moves.urllib.parse import urlencode
-
__all__ = ["RequestMethods"]
diff --git a/src/urllib3/response.py b/src/urllib3/response.py
index accdf1fd6d..38693f4fc6 100644
--- a/src/urllib3/response.py
+++ b/src/urllib3/response.py
@@ -1,10 +1,11 @@
from __future__ import absolute_import
-from contextlib import contextmanager
-import zlib
+
import io
import logging
-from socket import timeout as SocketTimeout
+import zlib
+from contextlib import contextmanager
from socket import error as SocketError
+from socket import timeout as SocketTimeout
try:
import brotli
@@ -12,20 +13,20 @@
brotli = None
from ._collections import HTTPHeaderDict
+from .connection import BaseSSLError, HTTPException
from .exceptions import (
BodyNotHttplibCompatible,
- ProtocolError,
DecodeError,
- ReadTimeoutError,
- ResponseNotChunked,
+ HTTPError,
IncompleteRead,
InvalidChunkLength,
InvalidHeader,
- HTTPError,
+ ProtocolError,
+ ReadTimeoutError,
+ ResponseNotChunked,
SSLError,
)
from .packages import six
-from .connection import HTTPException, BaseSSLError
from .util.response import is_fp_closed, is_response_to_head
log = logging.getLogger(__name__)
diff --git a/src/urllib3/util/__init__.py b/src/urllib3/util/__init__.py
index 24c16a2894..a230df293a 100644
--- a/src/urllib3/util/__init__.py
+++ b/src/urllib3/util/__init__.py
@@ -2,24 +2,23 @@
# For backwards compatibility, provide imports that used to be here.
from .connection import is_connection_dropped
-from .request import make_headers, SUPPRESS_USER_AGENT
+from .request import SUPPRESS_USER_AGENT, make_headers
from .response import is_fp_closed
+from .retry import Retry
from .ssl_ import (
- SSLContext,
+ ALPN_PROTOCOLS,
HAS_SNI,
IS_PYOPENSSL,
IS_SECURETRANSPORT,
+ PROTOCOL_TLS,
+ SSLContext,
assert_fingerprint,
resolve_cert_reqs,
resolve_ssl_version,
ssl_wrap_socket,
- PROTOCOL_TLS,
- ALPN_PROTOCOLS,
)
-from .timeout import current_time, Timeout
-
-from .retry import Retry
-from .url import get_host, parse_url, split_first, Url
+from .timeout import Timeout, current_time
+from .url import Url, get_host, parse_url, split_first
from .wait import wait_for_read, wait_for_write
__all__ = (
diff --git a/src/urllib3/util/connection.py b/src/urllib3/util/connection.py
index 75bb4dd8db..cd57455748 100644
--- a/src/urllib3/util/connection.py
+++ b/src/urllib3/util/connection.py
@@ -1,11 +1,12 @@
from __future__ import absolute_import
+
import socket
from urllib3.exceptions import LocationParseError
-from .wait import NoWayToWaitForSocketError, wait_for_read
from ..contrib import _appengine_environ
from ..packages import six
+from .wait import NoWayToWaitForSocketError, wait_for_read
def is_connection_dropped(conn): # Platform-specific
diff --git a/src/urllib3/util/proxy.py b/src/urllib3/util/proxy.py
index 68c6af33c7..34f884d5b3 100644
--- a/src/urllib3/util/proxy.py
+++ b/src/urllib3/util/proxy.py
@@ -1,8 +1,4 @@
-from .ssl_ import (
- resolve_cert_reqs,
- resolve_ssl_version,
- create_urllib3_context,
-)
+from .ssl_ import create_urllib3_context, resolve_cert_reqs, resolve_ssl_version
def connection_requires_http_tunnel(
diff --git a/src/urllib3/util/queue.py b/src/urllib3/util/queue.py
index d3d379a199..41784104ee 100644
--- a/src/urllib3/util/queue.py
+++ b/src/urllib3/util/queue.py
@@ -1,4 +1,5 @@
import collections
+
from ..packages import six
from ..packages.six.moves import queue
diff --git a/src/urllib3/util/request.py b/src/urllib3/util/request.py
index 27b6c6cf92..60e37513b7 100644
--- a/src/urllib3/util/request.py
+++ b/src/urllib3/util/request.py
@@ -1,8 +1,9 @@
from __future__ import absolute_import
+
from base64 import b64encode
-from ..packages.six import b, integer_types
from ..exceptions import UnrewindableBodyError
+from ..packages.six import b, integer_types
# Use an invalid User-Agent to represent suppressing of default user agent.
# See https://tools.ietf.org/html/rfc7231#section-5.5.3 and
diff --git a/src/urllib3/util/response.py b/src/urllib3/util/response.py
index 063d99155b..5ea609cced 100644
--- a/src/urllib3/util/response.py
+++ b/src/urllib3/util/response.py
@@ -1,8 +1,9 @@
from __future__ import absolute_import
-from email.errors import StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect
-from ..packages.six.moves import http_client as httplib
+
+from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect
from ..exceptions import HeaderParsingError
+from ..packages.six.moves import http_client as httplib
def is_fp_closed(obj):
diff --git a/src/urllib3/util/retry.py b/src/urllib3/util/retry.py
index 9d9f4a3c3d..ee51f922f8 100644
--- a/src/urllib3/util/retry.py
+++ b/src/urllib3/util/retry.py
@@ -1,24 +1,24 @@
from __future__ import absolute_import
-import time
-import logging
-from collections import namedtuple
-from itertools import takewhile
+
import email
+import logging
import re
+import time
import warnings
+from collections import namedtuple
+from itertools import takewhile
from ..exceptions import (
ConnectTimeoutError,
+ InvalidHeader,
MaxRetryError,
ProtocolError,
+ ProxyError,
ReadTimeoutError,
ResponseError,
- InvalidHeader,
- ProxyError,
)
from ..packages import six
-
log = logging.getLogger(__name__)
diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py
index 6ae7f4707a..8773334067 100644
--- a/src/urllib3/util/ssl_.py
+++ b/src/urllib3/util/ssl_.py
@@ -1,21 +1,20 @@
from __future__ import absolute_import
-import warnings
+
import hmac
import os
import sys
-
+import warnings
from binascii import hexlify, unhexlify
from hashlib import md5, sha1, sha256
-from .url import IPV4_RE, BRACELESS_IPV6_ADDRZ_RE
from ..exceptions import (
- SSLError,
InsecurePlatformWarning,
- SNIMissingWarning,
ProxySchemeUnsupported,
+ SNIMissingWarning,
+ SSLError,
)
from ..packages import six
-
+from .url import BRACELESS_IPV6_ADDRZ_RE, IPV4_RE
SSLContext = None
SSLTransport = None
@@ -45,8 +44,9 @@ def _const_compare_digest_backport(a, b):
try: # Test for SSL features
import ssl
- from ssl import wrap_socket, CERT_REQUIRED
from ssl import HAS_SNI # Has SNI?
+ from ssl import CERT_REQUIRED, wrap_socket
+
from .ssltransport import SSLTransport
except ImportError:
pass
@@ -65,7 +65,7 @@ def _const_compare_digest_backport(a, b):
try:
- from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION
+ from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3
except ImportError:
OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
OP_NO_COMPRESSION = 0x20000
diff --git a/src/urllib3/util/ssltransport.py b/src/urllib3/util/ssltransport.py
index 8cd1053417..d23e518de7 100644
--- a/src/urllib3/util/ssltransport.py
+++ b/src/urllib3/util/ssltransport.py
@@ -1,6 +1,6 @@
-import ssl
-import socket
import io
+import socket
+import ssl
from urllib3.exceptions import ProxySchemeUnsupported
from urllib3.packages import six
diff --git a/src/urllib3/util/timeout.py b/src/urllib3/util/timeout.py
index 2f287a3435..ff69593b05 100644
--- a/src/urllib3/util/timeout.py
+++ b/src/urllib3/util/timeout.py
@@ -1,9 +1,10 @@
from __future__ import absolute_import
+import time
+
# The default socket timeout, used by httplib to indicate that no timeout was
# specified by the user
from socket import _GLOBAL_DEFAULT_TIMEOUT
-import time
from ..exceptions import TimeoutStateError
diff --git a/src/urllib3/util/url.py b/src/urllib3/util/url.py
index 793324e5fd..6ff238fe3c 100644
--- a/src/urllib3/util/url.py
+++ b/src/urllib3/util/url.py
@@ -1,11 +1,11 @@
from __future__ import absolute_import
+
import re
from collections import namedtuple
from ..exceptions import LocationParseError
from ..packages import six
-
url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"]
# We only want to normalize urls with an HTTP(S) scheme.
diff --git a/src/urllib3/util/wait.py b/src/urllib3/util/wait.py
index 53d10ee7e4..c280646c7b 100644
--- a/src/urllib3/util/wait.py
+++ b/src/urllib3/util/wait.py
@@ -1,7 +1,7 @@
import errno
-from functools import partial
import select
import sys
+from functools import partial
try:
from time import monotonic
diff --git a/test/__init__.py b/test/__init__.py
index faf2fd6bef..bafd0ab9c6 100644
--- a/test/__init__.py
+++ b/test/__init__.py
@@ -1,11 +1,11 @@
-import warnings
-import sys
import errno
import logging
-import socket
-import ssl
import os
import platform
+import socket
+import ssl
+import sys
+import warnings
import pytest
@@ -14,10 +14,10 @@
except ImportError:
brotli = None
+from urllib3 import util
from urllib3.exceptions import HTTPWarning
from urllib3.packages import six
from urllib3.util import ssl_
-from urllib3 import util
try:
import urllib3.contrib.pyopenssl as pyopenssl
diff --git a/test/appengine/conftest.py b/test/appengine/conftest.py
index 05bfefe74c..0b9d1f1fb0 100644
--- a/test/appengine/conftest.py
+++ b/test/appengine/conftest.py
@@ -28,7 +28,6 @@
import pytest
import six
-
__all__ = [
"pytest_configure",
"pytest_runtest_call",
diff --git a/test/appengine/test_gae_manager.py b/test/appengine/test_gae_manager.py
index 0221b29c74..3047f249d4 100644
--- a/test/appengine/test_gae_manager.py
+++ b/test/appengine/test_gae_manager.py
@@ -1,13 +1,13 @@
-import dummyserver.testcase
+from test import SHORT_TIMEOUT
+from test.with_dummyserver import test_connectionpool
+
import pytest
-from urllib3.contrib import appengine
+import dummyserver.testcase
import urllib3.exceptions
-import urllib3.util.url
import urllib3.util.retry
-
-from test.with_dummyserver import test_connectionpool
-from test import SHORT_TIMEOUT
+import urllib3.util.url
+from urllib3.contrib import appengine
# This class is used so we can re-use the tests from the connection pool.
diff --git a/test/appengine/test_urlfetch.py b/test/appengine/test_urlfetch.py
index 2e727db0d4..74484ea405 100644
--- a/test/appengine/test_urlfetch.py
+++ b/test/appengine/test_urlfetch.py
@@ -3,10 +3,9 @@
Engine-patched version of httplib to make requests."""
import httplib
+import pytest
import StringIO
-
from mock import patch
-import pytest
from ..test_no_ssl import TestWithoutSSL
diff --git a/test/benchmark.py b/test/benchmark.py
index d480efae6a..67d141b252 100644
--- a/test/benchmark.py
+++ b/test/benchmark.py
@@ -13,7 +13,6 @@
sys.path.append("../")
import urllib3 # noqa: E402
-
# URLs to download. Doesn't matter as long as they're from the same host, so we
# can take advantage of connection re-using.
TO_DOWNLOAD = [
diff --git a/test/conftest.py b/test/conftest.py
index 84e6c18e33..f55a83d5fc 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -1,18 +1,17 @@
import collections
import contextlib
-import threading
import platform
import sys
+import threading
import pytest
import trustme
-from tornado import web, ioloop
-
-from .tz_stub import stub_timezone_ctx
+from tornado import ioloop, web
from dummyserver.handlers import TestingApp
-from dummyserver.server import run_tornado_app
-from dummyserver.server import HAS_IPV6
+from dummyserver.server import HAS_IPV6, run_tornado_app
+
+from .tz_stub import stub_timezone_ctx
# The Python 3.8+ default loop on Windows breaks Tornado
diff --git a/test/contrib/test_pyopenssl.py b/test/contrib/test_pyopenssl.py
index 7d1af31238..1a7f6f9714 100644
--- a/test/contrib/test_pyopenssl.py
+++ b/test/contrib/test_pyopenssl.py
@@ -5,9 +5,10 @@
import pytest
try:
- from urllib3.contrib.pyopenssl import _dnsname_to_stdlib, get_subj_alt_name
from cryptography import x509
from OpenSSL.crypto import FILETYPE_PEM, load_certificate
+
+ from urllib3.contrib.pyopenssl import _dnsname_to_stdlib, get_subj_alt_name
except ImportError:
pass
@@ -33,19 +34,19 @@ def teardown_module():
from ..test_util import TestUtilSSL # noqa: E402, F401
from ..with_dummyserver.test_https import ( # noqa: E402, F401
TestHTTPS,
+ TestHTTPS_IPSAN,
+ TestHTTPS_IPv6Addr,
+ TestHTTPS_IPV6SAN,
+ TestHTTPS_NoSAN,
TestHTTPS_TLSv1,
TestHTTPS_TLSv1_1,
TestHTTPS_TLSv1_2,
TestHTTPS_TLSv1_3,
- TestHTTPS_IPSAN,
- TestHTTPS_IPv6Addr,
- TestHTTPS_NoSAN,
- TestHTTPS_IPV6SAN,
)
from ..with_dummyserver.test_socketlevel import ( # noqa: E402, F401
+ TestClientCerts,
TestSNI,
TestSocketClosing,
- TestClientCerts,
TestSSL,
)
diff --git a/test/contrib/test_pyopenssl_dependencies.py b/test/contrib/test_pyopenssl_dependencies.py
index bbb5833dab..d1498e9218 100644
--- a/test/contrib/test_pyopenssl_dependencies.py
+++ b/test/contrib/test_pyopenssl_dependencies.py
@@ -1,10 +1,9 @@
# -*- coding: utf-8 -*-
import pytest
-
-from mock import patch, Mock
+from mock import Mock, patch
try:
- from urllib3.contrib.pyopenssl import inject_into_urllib3, extract_from_urllib3
+ from urllib3.contrib.pyopenssl import extract_from_urllib3, inject_into_urllib3
except ImportError:
pass
diff --git a/test/contrib/test_securetransport.py b/test/contrib/test_securetransport.py
index 41519436cf..9a49a35521 100644
--- a/test/contrib/test_securetransport.py
+++ b/test/contrib/test_securetransport.py
@@ -40,9 +40,9 @@ def teardown_module():
TestHTTPS_TLSv1_2,
)
from ..with_dummyserver.test_socketlevel import ( # noqa: E402, F401
+ TestClientCerts,
TestSNI,
TestSocketClosing,
- TestClientCerts,
TestSSL,
)
diff --git a/test/contrib/test_socks.py b/test/contrib/test_socks.py
index d70eb7cff6..1966513c18 100644
--- a/test/contrib/test_socks.py
+++ b/test/contrib/test_socks.py
@@ -1,17 +1,17 @@
-import threading
import socket
+import threading
+from test import SHORT_TIMEOUT
-from urllib3.contrib import socks
-from urllib3.exceptions import ConnectTimeoutError, NewConnectionError
+import pytest
-from dummyserver.server import DEFAULT_CERTS, DEFAULT_CA
+from dummyserver.server import DEFAULT_CA, DEFAULT_CERTS
from dummyserver.testcase import IPV4SocketDummyServerTestCase
-
-import pytest
-from test import SHORT_TIMEOUT
+from urllib3.contrib import socks
+from urllib3.exceptions import ConnectTimeoutError, NewConnectionError
try:
import ssl
+
from urllib3.util import ssl_ as better_ssl
HAS_SSL = True
diff --git a/test/port_helpers.py b/test/port_helpers.py
index d132cb0ac0..ae18ccae6d 100644
--- a/test/port_helpers.py
+++ b/test/port_helpers.py
@@ -3,7 +3,6 @@
import socket
-
# Don't use "localhost", since resolving it uses the DNS under recent
# Windows versions (see issue #18792).
HOST = "127.0.0.1"
diff --git a/test/test_collections.py b/test/test_collections.py
index e7be33fb7f..4b8624cb6c 100644
--- a/test/test_collections.py
+++ b/test/test_collections.py
@@ -1,6 +1,7 @@
-from urllib3._collections import HTTPHeaderDict, RecentlyUsedContainer as Container
import pytest
+from urllib3._collections import HTTPHeaderDict
+from urllib3._collections import RecentlyUsedContainer as Container
from urllib3.exceptions import InvalidHeader
from urllib3.packages import six
diff --git a/test/test_compatibility.py b/test/test_compatibility.py
index 013699f822..58a9ab5c6f 100644
--- a/test/test_compatibility.py
+++ b/test/test_compatibility.py
@@ -3,8 +3,8 @@
import pytest
from urllib3.connection import HTTPConnection
-from urllib3.response import HTTPResponse
from urllib3.packages.six.moves import http_cookiejar, urllib
+from urllib3.response import HTTPResponse
class TestVersionCompatibility(object):
diff --git a/test/test_connection.py b/test/test_connection.py
index a3cd29c976..821ce4c226 100644
--- a/test/test_connection.py
+++ b/test/test_connection.py
@@ -1,9 +1,9 @@
import datetime
-import mock
+import mock
import pytest
-from urllib3.connection import CertificateError, _match_hostname, RECENT_DATE
+from urllib3.connection import RECENT_DATE, CertificateError, _match_hostname
class TestConnection(object):
diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py
index 193ba91df8..eec6bd27c8 100644
--- a/test/test_connectionpool.py
+++ b/test/test_connectionpool.py
@@ -1,21 +1,21 @@
from __future__ import absolute_import
import ssl
+from socket import error as SocketError
+from ssl import SSLError as BaseSSLError
+from test import SHORT_TIMEOUT
+
import pytest
from mock import Mock
+from dummyserver.server import DEFAULT_CA
+from urllib3._collections import HTTPHeaderDict
from urllib3.connectionpool import (
- connection_from_url,
HTTPConnection,
HTTPConnectionPool,
HTTPSConnectionPool,
+ connection_from_url,
)
-from urllib3.response import HTTPResponse
-from urllib3.util.timeout import Timeout
-from urllib3.packages.six.moves import http_client as httplib
-from urllib3.packages.six.moves.http_client import HTTPException
-from urllib3.packages.six.moves.queue import Empty
-from urllib3.packages.ssl_match_hostname import CertificateError
from urllib3.exceptions import (
ClosedPoolError,
EmptyPoolError,
@@ -26,14 +26,14 @@
SSLError,
TimeoutError,
)
-from urllib3._collections import HTTPHeaderDict
-from .test_response import MockChunkedEncodingResponse, MockSock
-
-from socket import error as SocketError
-from ssl import SSLError as BaseSSLError
+from urllib3.packages.six.moves import http_client as httplib
+from urllib3.packages.six.moves.http_client import HTTPException
+from urllib3.packages.six.moves.queue import Empty
+from urllib3.packages.ssl_match_hostname import CertificateError
+from urllib3.response import HTTPResponse
+from urllib3.util.timeout import Timeout
-from dummyserver.server import DEFAULT_CA
-from test import SHORT_TIMEOUT
+from .test_response import MockChunkedEncodingResponse, MockSock
class HTTPUnixConnection(HTTPConnection):
diff --git a/test/test_exceptions.py b/test/test_exceptions.py
index 50d1c7ab10..9fd0eb0fb0 100644
--- a/test/test_exceptions.py
+++ b/test/test_exceptions.py
@@ -2,18 +2,18 @@
import pytest
+from urllib3.connectionpool import HTTPConnectionPool
from urllib3.exceptions import (
- HTTPError,
- MaxRetryError,
- LocationParseError,
ClosedPoolError,
+ ConnectTimeoutError,
EmptyPoolError,
+ HeaderParsingError,
HostChangedError,
+ HTTPError,
+ LocationParseError,
+ MaxRetryError,
ReadTimeoutError,
- ConnectTimeoutError,
- HeaderParsingError,
)
-from urllib3.connectionpool import HTTPConnectionPool
class TestPickle(object):
diff --git a/test/test_fields.py b/test/test_fields.py
index e0d30db6b8..98ce17c1f4 100644
--- a/test/test_fields.py
+++ b/test/test_fields.py
@@ -1,6 +1,6 @@
import pytest
-from urllib3.fields import format_header_param_rfc2231, guess_content_type, RequestField
+from urllib3.fields import RequestField, format_header_param_rfc2231, guess_content_type
from urllib3.packages.six import u
diff --git a/test/test_filepost.py b/test/test_filepost.py
index e884a44b18..5b0cfe1cb6 100644
--- a/test/test_filepost.py
+++ b/test/test_filepost.py
@@ -1,10 +1,9 @@
import pytest
-from urllib3.filepost import encode_multipart_formdata, iter_fields
from urllib3.fields import RequestField
+from urllib3.filepost import encode_multipart_formdata, iter_fields
from urllib3.packages.six import b, u
-
BOUNDARY = "!! test boundary !!"
diff --git a/test/test_no_ssl.py b/test/test_no_ssl.py
index 15612f4318..7cf6260e49 100644
--- a/test/test_no_ssl.py
+++ b/test/test_no_ssl.py
@@ -6,6 +6,7 @@
"""
import sys
+
import pytest
diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py
index 6f42712902..c54ad7ed9a 100644
--- a/test/test_poolmanager.py
+++ b/test/test_poolmanager.py
@@ -1,12 +1,12 @@
import socket
+from test import resolvesLocalhostFQDN
import pytest
-from urllib3.poolmanager import PoolKey, key_fn_by_scheme, PoolManager
from urllib3 import connection_from_url
from urllib3.exceptions import ClosedPoolError, LocationValueError
+from urllib3.poolmanager import PoolKey, PoolManager, key_fn_by_scheme
from urllib3.util import retry, timeout
-from test import resolvesLocalhostFQDN
class TestPoolManager(object):
diff --git a/test/test_proxymanager.py b/test/test_proxymanager.py
index fe8f72a298..7f1c396cce 100644
--- a/test/test_proxymanager.py
+++ b/test/test_proxymanager.py
@@ -1,14 +1,11 @@
import pytest
-from .port_helpers import find_unused_port
+from urllib3.exceptions import MaxRetryError, NewConnectionError, ProxyError
from urllib3.poolmanager import ProxyManager
-from urllib3.util.url import parse_url
from urllib3.util.retry import Retry
-from urllib3.exceptions import (
- MaxRetryError,
- ProxyError,
- NewConnectionError,
-)
+from urllib3.util.url import parse_url
+
+from .port_helpers import find_unused_port
class TestProxyManager(object):
diff --git a/test/test_queue_monkeypatch.py b/test/test_queue_monkeypatch.py
index 4ebad62be0..f8420a0eb6 100644
--- a/test/test_queue_monkeypatch.py
+++ b/test/test_queue_monkeypatch.py
@@ -1,7 +1,6 @@
from __future__ import absolute_import
import mock
-
import pytest
from urllib3 import HTTPConnectionPool
diff --git a/test/test_response.py b/test/test_response.py
index b838622f27..03f2780c75 100644
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -5,31 +5,28 @@
import socket
import ssl
import zlib
+from base64 import b64decode
+from io import BufferedReader, BytesIO, TextIOWrapper
+from test import onlyBrotlipy
-from io import BytesIO, BufferedReader, TextIOWrapper
-
-import pytest
import mock
+import pytest
import six
-from urllib3.response import HTTPResponse, brotli
from urllib3.exceptions import (
DecodeError,
- ResponseNotChunked,
- ProtocolError,
- InvalidHeader,
- httplib_IncompleteRead,
IncompleteRead,
InvalidChunkLength,
+ InvalidHeader,
+ ProtocolError,
+ ResponseNotChunked,
SSLError,
+ httplib_IncompleteRead,
)
from urllib3.packages.six.moves import http_client as httplib
-from urllib3.util.retry import Retry, RequestHistory
+from urllib3.response import HTTPResponse, brotli
from urllib3.util.response import is_fp_closed
-
-from test import onlyBrotlipy
-
-from base64 import b64decode
+from urllib3.util.retry import RequestHistory, Retry
# A known random (i.e, not-too-compressible) payload generated with:
# "".join(random.choice(string.printable) for i in xrange(512))
diff --git a/test/test_retry.py b/test/test_retry.py
index 0ca79dd3a2..cc36089796 100644
--- a/test/test_retry.py
+++ b/test/test_retry.py
@@ -1,11 +1,8 @@
+import warnings
+
import mock
import pytest
-import warnings
-from urllib3.response import HTTPResponse
-from urllib3.packages import six
-from urllib3.packages.six.moves import xrange
-from urllib3.util.retry import Retry, RequestHistory
from urllib3.exceptions import (
ConnectTimeoutError,
InvalidHeader,
@@ -14,6 +11,10 @@
ResponseError,
SSLError,
)
+from urllib3.packages import six
+from urllib3.packages.six.moves import xrange
+from urllib3.response import HTTPResponse
+from urllib3.util.retry import RequestHistory, Retry
@pytest.fixture(scope="function", autouse=True)
diff --git a/test/test_retry_deprecated.py b/test/test_retry_deprecated.py
index 73b5ef0f71..0c8de37661 100644
--- a/test/test_retry_deprecated.py
+++ b/test/test_retry_deprecated.py
@@ -1,12 +1,9 @@
# This is a copy-paste of test_retry.py with extra asserts about deprecated options. It will be removed for v2.
+import warnings
+
import mock
import pytest
-import warnings
-from urllib3.response import HTTPResponse
-from urllib3.packages import six
-from urllib3.packages.six.moves import xrange
-from urllib3.util.retry import Retry, RequestHistory
from urllib3.exceptions import (
ConnectTimeoutError,
InvalidHeader,
@@ -15,6 +12,10 @@
ResponseError,
SSLError,
)
+from urllib3.packages import six
+from urllib3.packages.six.moves import xrange
+from urllib3.response import HTTPResponse
+from urllib3.util.retry import RequestHistory, Retry
# TODO: Remove this entire file once deprecated Retry options are removed in v2.
diff --git a/test/test_ssl.py b/test/test_ssl.py
index f755938062..4a00d355e5 100644
--- a/test/test_ssl.py
+++ b/test/test_ssl.py
@@ -1,9 +1,10 @@
+from test import notPyPy2
+
import mock
import pytest
-from urllib3.util import ssl_
-from urllib3.exceptions import SNIMissingWarning
-from test import notPyPy2
+from urllib3.exceptions import SNIMissingWarning
+from urllib3.util import ssl_
@pytest.mark.parametrize(
diff --git a/test/test_ssltransport.py b/test/test_ssltransport.py
index 2b8ac4272f..fd6c52c29d 100644
--- a/test/test_ssltransport.py
+++ b/test/test_ssltransport.py
@@ -1,18 +1,14 @@
-from dummyserver.testcase import SocketDummyServerTestCase, consume_socket
-from dummyserver.server import (
- DEFAULT_CERTS,
- DEFAULT_CA,
-)
-
-from urllib3.util.ssltransport import SSLTransport
-
+import platform
import select
-import pytest
import socket
import ssl
import sys
-import platform
+import pytest
+
+from dummyserver.server import DEFAULT_CA, DEFAULT_CERTS
+from dummyserver.testcase import SocketDummyServerTestCase, consume_socket
+from urllib3.util.ssltransport import SSLTransport
# consume_socket can iterate forever, we add timeouts to prevent halting.
PER_TEST_TIMEOUT = 60
diff --git a/test/test_util.py b/test/test_util.py
index 3dc6989ff7..827df42726 100644
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -1,48 +1,42 @@
# coding: utf-8
import hashlib
-import warnings
-import logging
import io
-import ssl
+import logging
import socket
+import ssl
+import warnings
from itertools import chain
+from test import notBrotlipy, onlyBrotlipy, onlyPy2, onlyPy3
-from mock import patch, Mock
import pytest
+from mock import Mock, patch
from urllib3 import add_stderr_logger, disable_warnings, util
-from urllib3.util.connection import create_connection
-from urllib3.util.request import make_headers, rewind_body, _FAILEDTELL
-from urllib3.util.response import assert_header_parsing
-from urllib3.util.timeout import Timeout
-from urllib3.util.url import get_host, parse_url, split_first, Url
-from urllib3.util.ssl_ import (
- resolve_cert_reqs,
- resolve_ssl_version,
- ssl_wrap_socket,
- _const_compare_digest_backport,
-)
from urllib3.exceptions import (
- LocationParseError,
- TimeoutStateError,
InsecureRequestWarning,
+ LocationParseError,
SNIMissingWarning,
+ TimeoutStateError,
UnrewindableBodyError,
)
-from urllib3.util.proxy import (
- connection_requires_http_tunnel,
- create_proxy_ssl_context,
-)
-from urllib3.util import is_fp_closed
-from urllib3.util.connection import allowed_gai_family, _has_ipv6
from urllib3.packages import six
-
from urllib3.poolmanager import ProxyConfig
+from urllib3.util import is_fp_closed
+from urllib3.util.connection import _has_ipv6, allowed_gai_family, create_connection
+from urllib3.util.proxy import connection_requires_http_tunnel, create_proxy_ssl_context
+from urllib3.util.request import _FAILEDTELL, make_headers, rewind_body
+from urllib3.util.response import assert_header_parsing
+from urllib3.util.ssl_ import (
+ _const_compare_digest_backport,
+ resolve_cert_reqs,
+ resolve_ssl_version,
+ ssl_wrap_socket,
+)
+from urllib3.util.timeout import Timeout
+from urllib3.util.url import Url, get_host, parse_url, split_first
from . import clear_warnings
-from test import onlyPy3, onlyPy2, onlyBrotlipy, notBrotlipy
-
# This number represents a time in seconds, it doesn't mean anything in
# isolation. Setting to a high-ish value to avoid conflicts with the smaller
# numbers used for timeouts
diff --git a/test/test_wait.py b/test/test_wait.py
index edfe7687bc..38dad79dee 100644
--- a/test/test_wait.py
+++ b/test/test_wait.py
@@ -6,20 +6,22 @@
from time import monotonic
except ImportError:
from time import time as monotonic
+
import time
import pytest
-from .socketpair_helper import socketpair
from urllib3.util.wait import (
+ _have_working_poll,
+ poll_wait_for_socket,
+ select_wait_for_socket,
wait_for_read,
- wait_for_write,
wait_for_socket,
- select_wait_for_socket,
- poll_wait_for_socket,
- _have_working_poll,
+ wait_for_write,
)
+from .socketpair_helper import socketpair
+
@pytest.fixture
def spair():
diff --git a/test/tz_stub.py b/test/tz_stub.py
index 5b1a8c7e18..c48f5df024 100644
--- a/test/tz_stub.py
+++ b/test/tz_stub.py
@@ -1,7 +1,8 @@
-from contextlib import contextmanager
-import time
import datetime
import os
+import time
+from contextlib import contextmanager
+
import pytest
from dateutil import tz
diff --git a/test/with_dummyserver/test_chunked_transfer.py b/test/with_dummyserver/test_chunked_transfer.py
index 0c6793f615..9e1a8a56ff 100644
--- a/test/with_dummyserver/test_chunked_transfer.py
+++ b/test/with_dummyserver/test_chunked_transfer.py
@@ -2,14 +2,14 @@
import pytest
-from urllib3 import HTTPConnectionPool
-from urllib3.util.retry import Retry
-from urllib3.util import SUPPRESS_USER_AGENT
from dummyserver.testcase import (
+ ConnectionMarker,
SocketDummyServerTestCase,
consume_socket,
- ConnectionMarker,
)
+from urllib3 import HTTPConnectionPool
+from urllib3.util import SUPPRESS_USER_AGENT
+from urllib3.util.retry import Retry
# Retry failed tests
pytestmark = pytest.mark.flaky
diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py
index 18ce0c3edf..95616fce00 100644
--- a/test/with_dummyserver/test_connectionpool.py
+++ b/test/with_dummyserver/test_connectionpool.py
@@ -5,34 +5,33 @@
import sys
import time
import warnings
-import pytest
+from test import LONG_TIMEOUT, SHORT_TIMEOUT
+from threading import Event
import mock
+import pytest
-from .. import TARPIT_HOST, VALID_SOURCE_ADDRESSES, INVALID_SOURCE_ADDRESSES
-from ..port_helpers import find_unused_port
-from urllib3 import encode_multipart_formdata, HTTPConnectionPool
+from dummyserver.server import HAS_IPV6_AND_DNS, NoIPv6Warning
+from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase
+from urllib3 import HTTPConnectionPool, encode_multipart_formdata
from urllib3.connection import _get_default_user_agent
from urllib3.exceptions import (
ConnectTimeoutError,
- EmptyPoolError,
DecodeError,
+ EmptyPoolError,
MaxRetryError,
- ReadTimeoutError,
NewConnectionError,
+ ReadTimeoutError,
UnrewindableBodyError,
)
from urllib3.packages.six import b, u
from urllib3.packages.six.moves.urllib.parse import urlencode
from urllib3.util import SUPPRESS_USER_AGENT
-from urllib3.util.retry import Retry, RequestHistory
+from urllib3.util.retry import RequestHistory, Retry
from urllib3.util.timeout import Timeout
-from test import SHORT_TIMEOUT, LONG_TIMEOUT
-from dummyserver.testcase import HTTPDummyServerTestCase, SocketDummyServerTestCase
-from dummyserver.server import NoIPv6Warning, HAS_IPV6_AND_DNS
-
-from threading import Event
+from .. import INVALID_SOURCE_ADDRESSES, TARPIT_HOST, VALID_SOURCE_ADDRESSES
+from ..port_helpers import find_unused_port
pytestmark = pytest.mark.flaky
diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py
index 5eb241de6b..4c587d3432 100644
--- a/test/with_dummyserver/test_https.py
+++ b/test/with_dummyserver/test_https.py
@@ -2,53 +2,53 @@
import json
import logging
import os.path
+import shutil
import ssl
import sys
-import shutil
import tempfile
import warnings
-
-import mock
-import pytest
-import trustme
-
-from dummyserver.testcase import HTTPSDummyServerTestCase
-from dummyserver.server import (
- encrypt_key_pem,
- DEFAULT_CA,
- DEFAULT_CA_KEY,
- DEFAULT_CERTS,
-)
-
from test import (
- onlyPy279OrNewer,
- notSecureTransport,
+ LONG_TIMEOUT,
+ SHORT_TIMEOUT,
+ TARPIT_HOST,
notOpenSSL098,
+ notSecureTransport,
+ onlyPy279OrNewer,
requires_network,
requires_ssl_context_keyfile_password,
requiresTLSv1,
requiresTLSv1_1,
requiresTLSv1_2,
requiresTLSv1_3,
- TARPIT_HOST,
- SHORT_TIMEOUT,
- LONG_TIMEOUT,
resolvesLocalhostFQDN,
)
+
+import mock
+import pytest
+import trustme
+
+import urllib3.util as util
+from dummyserver.server import (
+ DEFAULT_CA,
+ DEFAULT_CA_KEY,
+ DEFAULT_CERTS,
+ encrypt_key_pem,
+)
+from dummyserver.testcase import HTTPSDummyServerTestCase
from urllib3 import HTTPSConnectionPool
-from urllib3.connection import VerifiedHTTPSConnection, RECENT_DATE
+from urllib3.connection import RECENT_DATE, VerifiedHTTPSConnection
from urllib3.exceptions import (
- SSLError,
ConnectTimeoutError,
- InsecureRequestWarning,
- SystemTimeWarning,
InsecurePlatformWarning,
+ InsecureRequestWarning,
MaxRetryError,
ProtocolError,
+ SSLError,
+ SystemTimeWarning,
)
from urllib3.packages import six
from urllib3.util.timeout import Timeout
-import urllib3.util as util
+
from .. import has_alpn
# Retry failed tests
diff --git a/test/with_dummyserver/test_no_ssl.py b/test/with_dummyserver/test_no_ssl.py
index 7f4d350a4b..43e79b70b6 100644
--- a/test/with_dummyserver/test_no_ssl.py
+++ b/test/with_dummyserver/test_no_ssl.py
@@ -4,11 +4,11 @@
Note: Import urllib3 inside the test functions to get the importblocker to work
"""
import pytest
-from ..test_no_ssl import TestWithoutSSL
+import urllib3
from dummyserver.testcase import HTTPDummyServerTestCase, HTTPSDummyServerTestCase
-import urllib3
+from ..test_no_ssl import TestWithoutSSL
# Retry failed tests
pytestmark = pytest.mark.flaky
diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py
index 9d2303e889..d877cc99ac 100644
--- a/test/with_dummyserver/test_poolmanager.py
+++ b/test/with_dummyserver/test_poolmanager.py
@@ -1,16 +1,15 @@
import json
+from test import LONG_TIMEOUT
import pytest
from dummyserver.server import HAS_IPV6
from dummyserver.testcase import HTTPDummyServerTestCase, IPv6HTTPDummyServerTestCase
-from urllib3.poolmanager import PoolManager
from urllib3.connectionpool import port_by_scheme
from urllib3.exceptions import MaxRetryError, URLSchemeUnknown
+from urllib3.poolmanager import PoolManager
from urllib3.util.retry import Retry
-from test import LONG_TIMEOUT
-
# Retry failed tests
pytestmark = pytest.mark.flaky
diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py
index 40f1a2597a..67cee77a58 100644
--- a/test/with_dummyserver/test_proxy_poolmanager.py
+++ b/test/with_dummyserver/test_proxy_poolmanager.py
@@ -3,36 +3,33 @@
import shutil
import socket
import tempfile
-
+from test import (
+ LONG_TIMEOUT,
+ SHORT_TIMEOUT,
+ onlyPy2,
+ onlyPy3,
+ onlySecureTransport,
+ withPyOpenSSL,
+)
import pytest
import trustme
-from dummyserver.testcase import HTTPDummyProxyTestCase, IPv6HTTPDummyProxyTestCase
from dummyserver.server import DEFAULT_CA, HAS_IPV6, get_unreachable_address
-from .. import TARPIT_HOST, requires_network
-
+from dummyserver.testcase import HTTPDummyProxyTestCase, IPv6HTTPDummyProxyTestCase
from urllib3._collections import HTTPHeaderDict
-from urllib3.poolmanager import proxy_from_url, ProxyManager
+from urllib3.connectionpool import VerifiedHTTPSConnection, connection_from_url
from urllib3.exceptions import (
+ ConnectTimeoutError,
MaxRetryError,
- SSLError,
ProxyError,
- ConnectTimeoutError,
ProxySchemeUnsupported,
+ SSLError,
)
-from urllib3.connectionpool import connection_from_url, VerifiedHTTPSConnection
+from urllib3.poolmanager import ProxyManager, proxy_from_url
from urllib3.util.ssl_ import create_urllib3_context
-from test import (
- SHORT_TIMEOUT,
- LONG_TIMEOUT,
- onlyPy3,
- onlyPy2,
- withPyOpenSSL,
- onlySecureTransport,
-)
-
+from .. import TARPIT_HOST, requires_network
# Retry failed tests
pytestmark = pytest.mark.flaky
diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py
index 72725aa9b5..4e2c292da9 100644
--- a/test/with_dummyserver/test_socketlevel.py
+++ b/test/with_dummyserver/test_socketlevel.py
@@ -1,33 +1,29 @@
# TODO: Break this module up into pieces. Maybe group by functionality tested
# rather than the socket level-ness of it.
-from urllib3 import HTTPConnectionPool, HTTPSConnectionPool
-from urllib3.connection import HTTPConnection
-from urllib3.poolmanager import proxy_from_url
-from urllib3.connection import _get_default_user_agent
+from dummyserver.server import (
+ DEFAULT_CA,
+ DEFAULT_CERTS,
+ encrypt_key_pem,
+ get_unreachable_address,
+)
+from dummyserver.testcase import SocketDummyServerTestCase, consume_socket
+from urllib3 import HTTPConnectionPool, HTTPSConnectionPool, util
+from urllib3._collections import HTTPHeaderDict
+from urllib3.connection import HTTPConnection, _get_default_user_agent
from urllib3.exceptions import (
MaxRetryError,
+ ProtocolError,
ProxyError,
ReadTimeoutError,
SSLError,
- ProtocolError,
)
from urllib3.packages.six.moves import http_client as httplib
-from urllib3 import util
-from urllib3.util import ssl_wrap_socket
-from urllib3.util import ssl_
-from urllib3.util.timeout import Timeout
+from urllib3.poolmanager import proxy_from_url
+from urllib3.util import ssl_, ssl_wrap_socket
from urllib3.util.retry import Retry
-from urllib3._collections import HTTPHeaderDict
-
-from dummyserver.testcase import SocketDummyServerTestCase, consume_socket
-from dummyserver.server import (
- DEFAULT_CERTS,
- DEFAULT_CA,
- get_unreachable_address,
- encrypt_key_pem,
-)
+from urllib3.util.timeout import Timeout
-from .. import onlyPy3, LogRecorder, has_alpn
+from .. import LogRecorder, has_alpn, onlyPy3
try:
from mimetools import Message as MimeToolMessage
@@ -37,29 +33,28 @@ class MimeToolMessage(object):
pass
-from collections import OrderedDict
-import os.path
-from threading import Event
import os
+import os.path
import select
-import socket
import shutil
+import socket
import ssl
import tempfile
-import mock
-
-import pytest
-import trustme
-
+from collections import OrderedDict
from test import (
- requires_ssl_context_keyfile_password,
- SHORT_TIMEOUT,
LONG_TIMEOUT,
+ SHORT_TIMEOUT,
notPyPy2,
notSecureTransport,
notWindows,
+ requires_ssl_context_keyfile_password,
resolvesLocalhostFQDN,
)
+from threading import Event
+
+import mock
+import pytest
+import trustme
# Retry failed tests
pytestmark = pytest.mark.flaky