Allow configuring max_incomplete_event_size for h11 implementation (#…

…1514)

* Solves #1234

* Update docs/settings.md

Co-authored-by: Marcelo Trylesinski <marcelotryle@gmail.com>

* Update uvicorn/main.py

Co-authored-by: Marcelo Trylesinski <marcelotryle@gmail.com>

* Fix lint

* Fixing tests

* Tweak tests a bit

* Remove event_loop fixture

Co-authored-by: Marcelo Trylesinski <marcelotryle@gmail.com>

docs/deployment.md

@@ -120,6 +120,9 @@ Options:
   --app-dir TEXT                  Look for APP in the specified directory, by
                                   adding this to the PYTHONPATH. Defaults to
                                   the current working directory.  [default: .]
+  --h11-max-incomplete-event-size INTEGER
+                                  For h11, the maximum number of bytes to
+                                  buffer of an incomplete event.
   --factory                       Treat APP as an application factory, i.e. a
                                   () -> <ASGI app> callable.
   --help                          Show this message and exit.

docs/index.md

@@ -187,6 +187,9 @@ Options:
   --app-dir TEXT                  Look for APP in the specified directory, by
                                   adding this to the PYTHONPATH. Defaults to
                                   the current working directory.  [default: .]
+  --h11-max-incomplete-event-size INTEGER
+                                  For h11, the maximum number of bytes to
+                                  buffer of an incomplete event.
   --factory                       Treat APP as an application factory, i.e. a
                                   () -> <ASGI app> callable.
   --help                          Show this message and exit.

docs/settings.md

@@ -65,6 +65,7 @@ For more nuanced control over which file modifications trigger reloads, install
 * `--ws-ping-interval <float>` - Set the WebSockets ping interval, in seconds. Please note that this can be used only with the default `websockets` protocol.
 * `--ws-ping-timeout <float>` - Set the WebSockets ping timeout, in seconds. Please note that this can be used only with the default `websockets` protocol.
 * `--lifespan <str>` - Set the Lifespan protocol implementation. **Options:** *'auto', 'on', 'off'.* **Default:** *'auto'*.
+* `--h11-max-incomplete-event-size <int>` - Set the maximum number of bytes to buffer of an incomplete event. Only available for `h11` HTTP protocol implementation. **Default:** *'16384'* (16 KB).
 
 ## Application Interface
 

tests/protocols/test_http.py

@@ -85,6 +85,17 @@
     ]
 )
 
+GET_REQUEST_HUGE_HEADERS = [
+    b"".join(
+        [
+            b"GET / HTTP/1.1\r\n",
+            b"Host: example.org\r\n",
+            b"Cookie: " + b"x" * 32 * 1024,
+        ]
+    ),
+    b"".join([b"x" * 32 * 1024 + b"\r\n", b"\r\n", b"\r\n"]),
+]
+
 
 class MockTransport:
     def __init__(self, sockname=None, peername=None, sslcontext=False):
@@ -801,3 +812,75 @@ def send_fragmented_req(path):
     assert bad_response != response[: len(bad_response)]
     server.should_exit = True
     t.join()
+
+
+@pytest.mark.anyio
+async def test_huge_headers_h11protocol_failure():
+    app = Response("Hello, world", media_type="text/plain")
+
+    protocol = get_connected_protocol(app, H11Protocol)
+    # Huge headers make h11 fail in it's default config
+    # h11 sends back a 400 in this case
+    protocol.data_received(GET_REQUEST_HUGE_HEADERS[0])
+    assert b"HTTP/1.1 400 Bad Request" in protocol.transport.buffer
+    assert b"Connection: close" in protocol.transport.buffer
+    assert b"Invalid HTTP request received." in protocol.transport.buffer
+
+
+@pytest.mark.anyio
+@pytest.mark.skipif(HttpToolsProtocol is None, reason="httptools is not installed")
+async def test_huge_headers_httptools_will_pass():
+    app = Response("Hello, world", media_type="text/plain")
+
+    protocol = get_connected_protocol(app, HttpToolsProtocol)
+    # Huge headers make h11 fail in it's default config
+    # httptools protocol will always pass
+    protocol.data_received(GET_REQUEST_HUGE_HEADERS[0])
+    protocol.data_received(GET_REQUEST_HUGE_HEADERS[1])
+    await protocol.loop.run_one()
+    assert b"HTTP/1.1 200 OK" in protocol.transport.buffer
+    assert b"Hello, world" in protocol.transport.buffer
+
+
+@pytest.mark.anyio
+async def test_huge_headers_h11protocol_failure_with_setting():
+    app = Response("Hello, world", media_type="text/plain")
+
+    protocol = get_connected_protocol(
+        app, H11Protocol, h11_max_incomplete_event_size=20 * 1024
+    )
+    # Huge headers make h11 fail in it's default config
+    # h11 sends back a 400 in this case
+    protocol.data_received(GET_REQUEST_HUGE_HEADERS[0])
+    assert b"HTTP/1.1 400 Bad Request" in protocol.transport.buffer
+    assert b"Connection: close" in protocol.transport.buffer
+    assert b"Invalid HTTP request received." in protocol.transport.buffer
+
+
+@pytest.mark.anyio
+@pytest.mark.skipif(HttpToolsProtocol is None, reason="httptools is not installed")
+async def test_huge_headers_httptools():
+    app = Response("Hello, world", media_type="text/plain")
+
+    protocol = get_connected_protocol(app, HttpToolsProtocol)
+    # Huge headers make h11 fail in it's default config
+    # httptools protocol will always pass
+    protocol.data_received(GET_REQUEST_HUGE_HEADERS[0])
+    protocol.data_received(GET_REQUEST_HUGE_HEADERS[1])
+    await protocol.loop.run_one()
+    assert b"HTTP/1.1 200 OK" in protocol.transport.buffer
+    assert b"Hello, world" in protocol.transport.buffer
+
+
+@pytest.mark.anyio
+async def test_huge_headers_h11_max_incomplete():
+    app = Response("Hello, world", media_type="text/plain")
+
+    protocol = get_connected_protocol(
+        app, H11Protocol, h11_max_incomplete_event_size=64 * 1024
+    )
+    protocol.data_received(GET_REQUEST_HUGE_HEADERS[0])
+    protocol.data_received(GET_REQUEST_HUGE_HEADERS[1])
+    await protocol.loop.run_one()
+    assert b"HTTP/1.1 200 OK" in protocol.transport.buffer
+    assert b"Hello, world" in protocol.transport.buffer

tests/test_cli.py

@@ -122,6 +122,21 @@ def test_cli_incomplete_app_parameter() -> None:
     assert result.exit_code == 1
 
 
+def test_cli_event_size() -> None:
+    runner = CliRunner()
+
+    with mock.patch.object(main, "run") as mock_run:
+        result = runner.invoke(
+            cli,
+            ["tests.test_cli:App", "--h11-max-incomplete-event-size", str(32 * 1024)],
+        )
+
+    assert result.output == ""
+    assert result.exit_code == 0
+    mock_run.assert_called_once()
+    assert mock_run.call_args[1]["h11_max_incomplete_event_size"] == 32768
+
+
 @pytest.fixture()
 def load_env_h11_protocol():
     old_environ = dict(os.environ)

uvicorn/config.py

@@ -10,6 +10,8 @@
 from pathlib import Path
 from typing import Awaitable, Callable, Dict, List, Optional, Tuple, Type, Union
 
+from h11._connection import DEFAULT_MAX_INCOMPLETE_EVENT_SIZE
+
 from uvicorn._logging import TRACE_LOG_LEVEL
 
 if sys.version_info < (3, 8):  # pragma: py-gte-38
@@ -240,6 +242,7 @@ def __init__(
         ssl_ciphers: str = "TLSv1",
         headers: Optional[List[Tuple[str, str]]] = None,
         factory: bool = False,
+        h11_max_incomplete_event_size: int = DEFAULT_MAX_INCOMPLETE_EVENT_SIZE,
     ):
         self.app = app
         self.host = host
@@ -283,6 +286,7 @@ def __init__(
         self.headers: List[Tuple[str, str]] = headers or []
         self.encoded_headers: List[Tuple[bytes, bytes]] = []
         self.factory = factory
+        self.h11_max_incomplete_event_size = h11_max_incomplete_event_size
 
         self.loaded = False
         self.configure_logging()

uvicorn/main.py

@@ -7,6 +7,7 @@
 
 import click
 from asgiref.typing import ASGIApplication
+from h11._connection import DEFAULT_MAX_INCOMPLETE_EVENT_SIZE
 
 import uvicorn
 from uvicorn.config import (
@@ -339,6 +340,13 @@ def print_version(ctx: click.Context, param: click.Parameter, value: bool) -> No
     help="Look for APP in the specified directory, by adding this to the PYTHONPATH."
     " Defaults to the current working directory.",
 )
+@click.option(
+    "--h11-max-incomplete-event-size",
+    "h11_max_incomplete_event_size",
+    type=int,
+    default=None,
+    help="For h11, the maximum number of bytes to buffer of an incomplete event.",
+)
 @click.option(
     "--factory",
     is_flag=True,
@@ -391,6 +399,7 @@ def main(
     headers: typing.List[str],
     use_colors: bool,
     app_dir: str,
+    h11_max_incomplete_event_size: int,
     factory: bool,
 ) -> None:
     run(
@@ -439,6 +448,7 @@ def main(
         use_colors=use_colors,
         factory=factory,
         app_dir=app_dir,
+        h11_max_incomplete_event_size=h11_max_incomplete_event_size,
     )
 
 
@@ -489,6 +499,7 @@ def run(
     use_colors: typing.Optional[bool] = None,
     app_dir: typing.Optional[str] = None,
     factory: bool = False,
+    h11_max_incomplete_event_size: int = DEFAULT_MAX_INCOMPLETE_EVENT_SIZE,
 ) -> None:
     if app_dir is not None:
         sys.path.insert(0, app_dir)
@@ -538,6 +549,7 @@ def run(
         headers=headers,
         use_colors=use_colors,
         factory=factory,
+        h11_max_incomplete_event_size=h11_max_incomplete_event_size,
     )
     server = Server(config=config)
 

uvicorn/protocols/http/h11_impl.py

@@ -77,7 +77,7 @@ def __init__(
         self.logger = logging.getLogger("uvicorn.error")
         self.access_logger = logging.getLogger("uvicorn.access")
         self.access_log = self.access_logger.hasHandlers()
-        self.conn = h11.Connection(h11.SERVER)
+        self.conn = h11.Connection(h11.SERVER, config.h11_max_incomplete_event_size)
         self.ws_protocol_class = config.ws_protocol_class
         self.root_path = config.root_path
         self.limit_concurrency = config.limit_concurrency