Backport Retry-After date parsing fixes

dev-requirements.txt

@@ -7,10 +7,13 @@ PySocks==1.7.1
 win-inet-pton==1.1.0
 pytest==4.6.9
 pytest-timeout==1.3.4
+pytest-freezegun==0.4.2
 flaky==3.6.1
 trustme==0.5.3
 cryptography==2.8
 gcp-devrel-py-tools==0.0.15
 
 # https://github.com/GoogleCloudPlatform/python-repo-tools/issues/23
 pylint<2.0;python_version<="2.7"
+
+python-dateutil==2.8.1

dummyserver/handlers.py

@@ -316,7 +316,7 @@ def redirect_after(self, request):
         date = request.params.get("date")
         if date:
             retry_after = str(
-                httputil.format_timestamp(datetime.fromtimestamp(float(date)))
+                httputil.format_timestamp(datetime.utcfromtimestamp(float(date)))
             )
         else:
             retry_after = "1"

src/urllib3/util/retry.py

@@ -252,10 +252,17 @@ def parse_retry_after(self, retry_after):
         if re.match(r"^\s*[0-9]+\s*$", retry_after):
             seconds = int(retry_after)
         else:
-            retry_date_tuple = email.utils.parsedate(retry_after)
+            retry_date_tuple = email.utils.parsedate_tz(retry_after)
             if retry_date_tuple is None:
                 raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
-            retry_date = time.mktime(retry_date_tuple)
+            if retry_date_tuple[9] is None:  # Python 2
+                # Assume UTC if no timezone was specified
+                # On Python2.7, parsedate_tz returns None for a timezone offset
+                # instead of 0 if no timezone is given, where mktime_tz treats
+                # a None timezone offset as local time.
+                retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:]
+
+            retry_date = email.utils.mktime_tz(retry_date_tuple)
             seconds = retry_date - time.time()
 
         if seconds < 0:

test/conftest.py

@@ -8,6 +8,8 @@
 import trustme
 from tornado import web, ioloop
 
+from .tz_stub import stub_timezone_ctx
+
 from dummyserver.handlers import TestingApp
 from dummyserver.server import run_tornado_app
 from dummyserver.server import HAS_IPV6
@@ -96,3 +98,12 @@ def ipv6_san_server(tmp_path_factory):
 
     with run_server_in_thread("https", "::1", tmpdir, ca, server_cert) as cfg:
         yield cfg
+
+
+@pytest.yield_fixture
+def stub_timezone(request):
+    """
+    A pytest fixture that runs the test with a stub timezone.
+    """
+    with stub_timezone_ctx(request.param):
+        yield

test/test_retry.py

@@ -1,7 +1,5 @@
-import datetime
 import mock
 import pytest
-import time
 
 from urllib3.response import HTTPResponse
 from urllib3.packages import six
@@ -297,6 +295,7 @@ def test_respect_retry_after_header_propagated(self, respect_retry_after_header)
         new_retry = retry.new()
         assert new_retry.respect_retry_after_header == respect_retry_after_header
 
+    @pytest.mark.freeze_time("2019-06-03 11:00:00", tz_offset=0)
     @pytest.mark.parametrize(
         "retry_after_header,respect_retry_after_header,sleep_duration",
         [
@@ -310,24 +309,29 @@ def test_respect_retry_after_header_propagated(self, respect_retry_after_header)
             ("Mon, 3 Jun 2019 11:00:00 UTC", True, None),
             # Won't sleep due to current time reached + not respecting header
             ("Mon, 3 Jun 2019 11:00:00 UTC", False, None),
+            # Handle all the formats in RFC 7231 Section 7.1.1.1
+            ("Mon, 03 Jun 2019 11:30:12 GMT", True, 1812),
+            ("Monday, 03-Jun-19 11:30:12 GMT", True, 1812),
+            # Assume that datetimes without a timezone are in UTC per RFC 7231
+            ("Mon Jun  3 11:30:12 2019", True, 1812),
         ],
     )
+    @pytest.mark.parametrize(
+        "stub_timezone",
+        [
+            "UTC",
+            "Asia/Jerusalem",
+            None,
+        ],
+        indirect=True,
+    )
+    @pytest.mark.usefixtures("stub_timezone")
     def test_respect_retry_after_header_sleep(
         self, retry_after_header, respect_retry_after_header, sleep_duration
     ):
         retry = Retry(respect_retry_after_header=respect_retry_after_header)
 
-        # Date header syntax can specify an absolute date; compare this to the
-        # time in the parametrized inputs above.
-        current_time = mock.MagicMock(
-            return_value=time.mktime(
-                datetime.datetime(year=2019, month=6, day=3, hour=11).timetuple()
-            )
-        )
-
-        with mock.patch("time.sleep") as sleep_mock, mock.patch(
-            "time.time", current_time
-        ):
+        with mock.patch("time.sleep") as sleep_mock:
             # for the default behavior, it must be in RETRY_AFTER_STATUS_CODES
             response = HTTPResponse(
                 status=503, headers={"Retry-After": retry_after_header}

test/tz_stub.py

@@ -0,0 +1,39 @@
+from contextlib import contextmanager
+import time
+import datetime
+import os
+import pytest
+from dateutil import tz
+
+
+@contextmanager
+def stub_timezone_ctx(tzname):
+    """
+    Switch to a locally-known timezone specified by `tzname`.
+    On exit, restore the previous timezone.
+    If `tzname` is `None`, do nothing.
+    """
+    if tzname is None:
+        yield
+        return
+
+    # Only supported on Unix
+    if not hasattr(time, "tzset"):
+        pytest.skip("Timezone patching is not supported")
+
+    # Make sure the new timezone exists, at least in dateutil
+    new_tz = tz.gettz(tzname)
+    if new_tz is None:
+        raise ValueError("Invalid timezone specified: %r" % (tzname,))
+
+    # Get the current timezone
+    local_tz = tz.tzlocal()
+    if local_tz is None:
+        raise EnvironmentError("Cannot determine current timezone")
+    old_tzname = datetime.datetime.now(local_tz).tzname()
+
+    os.environ["TZ"] = tzname
+    time.tzset()
+    yield
+    os.environ["TZ"] = old_tzname
+    time.tzset()