⚡ improve download performance for a large quantity of data (#116)

and also reduce amt of call to _maybe_next_cycle in http/1.1 state
machine handler

CHANGES.rst

@@ -1,3 +1,9 @@
+2.7.909 (2024-05-17)
+====================
+
+- Improve (large) data download performance by increasing the default blocksize.
+- Improve HTTP/1.1 performance by reducing the amount of time we want to infer "if next cycle" should be triggered.
+
 2.7.908 (2024-05-16)
 ====================
 

src/urllib3/_constant.py

@@ -219,7 +219,9 @@ def __new__(
 
 # Default value for `blocksize` - a new parameter introduced to
 # http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
-DEFAULT_BLOCKSIZE: int = 16384
+# The maximum TCP packet size is 65535 octets. But Python seems to buffer packets, so
+# passing a "very-high" value does improve responsiveness.
+DEFAULT_BLOCKSIZE: int = 65535 * 2
 
 # Mozilla TLS recommendations for ciphers
 # General-purpose servers with a variety of clients, recommended for almost all systems.

src/urllib3/_version.py

@@ -1,4 +1,4 @@
 # This file is protected via CODEOWNERS
 from __future__ import annotations
 
-__version__ = "2.7.908"
+__version__ = "2.7.909"

src/urllib3/contrib/hface/protocols/http1/_h11.py

@@ -14,14 +14,17 @@
 
 from __future__ import annotations
 
+from functools import lru_cache
+
 import h11
 
 from ..._stream_matrix import StreamMatrix
-from ..._typing import HeadersType, HeaderType
+from ..._typing import HeadersType
 from ...events import ConnectionTerminated, DataReceived, Event, HeadersReceived
 from .._protocols import HTTP1Protocol
 
 
+@lru_cache(maxsize=64)
 def capitalize_header_name(name: bytes) -> bytes:
     """
     Take a header name and capitalize it.
@@ -90,11 +93,9 @@ def headers_from_response(
 
     Generates from pseudo (colon) headers from a response line.
     """
-    headers: list[HeaderType] = [
+    return [
         (b":status", str(response.status_code).encode("ascii"))
-    ]
-    headers.extend(response.headers)
-    return headers
+    ] + response.headers.raw_items()
 
 
 class HTTP1ProtocolHyperImpl(HTTP1Protocol):
@@ -215,7 +216,6 @@ def _h11_submit(self, h11_event: h11.Event) -> None:
     def _h11_data_received(self, data: bytes) -> None:
         self._connection.receive_data(data)
         self._fetch_events()
-        self._maybe_start_next_cycle()
 
     def _fetch_events(self) -> None:
         a = self._events.append
@@ -248,8 +248,9 @@ def _fetch_events(self) -> None:
     def _headers_from_h11_response(
         self, h11_event: h11.Response | h11.InformationalResponse
     ) -> Event:
-        headers = headers_from_response(h11_event)
-        return HeadersReceived(self._current_stream_id, headers)
+        return HeadersReceived(
+            self._current_stream_id, headers_from_response(h11_event)
+        )
 
     def _data_from_h11(self, h11_event: h11.Data) -> Event:
         return DataReceived(self._current_stream_id, h11_event.data)

test/with_traefik/asynchronous/test_send_data.py

@@ -133,7 +133,6 @@ async def track(
         ) -> None:
             nonlocal progress_track
             progress_track.append((total_sent, content_length, is_completed, any_error))
-            print(progress_track[-1])
 
         async with AsyncHTTPSConnectionPool(
             self.host, self.https_port, ca_certs=self.ca_authority
@@ -144,7 +143,7 @@ async def track(
         assert progress_track[-1][-2] is True
         assert progress_track[0][1] == 16800 * 3
         assert progress_track[-1][0] == 16800 * 3
-        assert progress_track[0][0] < 16800 * 3
+        assert progress_track[0][0] <= 16800 * 3
 
     async def test_upload_track_progress_no_content_length(self) -> None:
         progress_track = []
@@ -169,4 +168,4 @@ async def track(
         assert progress_track[-1][-2] is True
         assert progress_track[0][1] is None
         assert progress_track[-1][0] == 16800 * 3
-        assert progress_track[0][0] < 16800 * 3
+        assert progress_track[0][0] <= 16800 * 3

test/with_traefik/test_send_data.py

@@ -130,7 +130,6 @@ def track(
         ) -> None:
             nonlocal progress_track
             progress_track.append((total_sent, content_length, is_completed, any_error))
-            print(progress_track[-1])
 
         with HTTPSConnectionPool(
             self.host, self.https_port, ca_certs=self.ca_authority
@@ -141,7 +140,7 @@ def track(
         assert progress_track[-1][-2] is True
         assert progress_track[0][1] == 16800 * 3
         assert progress_track[-1][0] == 16800 * 3
-        assert progress_track[0][0] < 16800 * 3
+        assert progress_track[0][0] <= 16800 * 3
 
     def test_upload_track_progress_no_content_length(self) -> None:
         progress_track = []
@@ -166,4 +165,4 @@ def track(
         assert progress_track[-1][-2] is True
         assert progress_track[0][1] is None
         assert progress_track[-1][0] == 16800 * 3
-        assert progress_track[0][0] < 16800 * 3
+        assert progress_track[0][0] <= 16800 * 3