From f1c2d0e04283f0945e923fa3a01c008213e43a0c Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Sun, 12 Dec 2021 09:07:25 +0200 Subject: [PATCH 01/59] WIP to begin http3 --- sanic/app.py | 9 ++ sanic/http/__init__.py | 5 + sanic/http/constants.py | 25 +++++ sanic/{http.py => http/http1.py} | 21 +--- sanic/http/http3.py | 136 ++++++++++++++++++++++++ sanic/server/protocols/http_protocol.py | 76 ++++++++++--- sanic/server/runners.py | 44 +++++++- 7 files changed, 280 insertions(+), 36 deletions(-) create mode 100644 sanic/http/__init__.py create mode 100644 sanic/http/constants.py rename sanic/{http.py => http/http1.py} (96%) create mode 100644 sanic/http/http3.py diff --git a/sanic/app.py b/sanic/app.py index f02301657f..a0eca9c2a1 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -35,6 +35,7 @@ Dict, Iterable, List, + Literal, Optional, Set, Tuple, @@ -68,6 +69,7 @@ ) from sanic.handlers import ErrorHandler from sanic.http import Stage +from sanic.http.constants import HTTP from sanic.log import LOGGING_CONFIG_DEFAULTS, Colors, error_logger, logger from sanic.mixins.listeners import ListenerEvent from sanic.models.futures import ( @@ -1050,6 +1052,7 @@ def run( fast: bool = False, verbosity: int = 0, motd_display: Optional[Dict[str, str]] = None, + version: HTTP = HTTP.VERSION_1, ) -> None: """ Run the HTTP Server and listen until keyboard interrupt or term @@ -1156,6 +1159,7 @@ def run( protocol=protocol, backlog=backlog, register_sys_signals=register_sys_signals, + version=version, ) try: @@ -1383,6 +1387,7 @@ def _helper( backlog: int = 100, register_sys_signals: bool = True, run_async: bool = False, + version: Union[HTTP, Literal[1], Literal[3]] = HTTP.VERSION_1, ): """Helper function used by `run` and `create_server`.""" if self.config.PROXIES_COUNT and self.config.PROXIES_COUNT < 0: @@ -1392,6 +1397,9 @@ def _helper( "#proxy-configuration" ) + if isinstance(version, int): + version = HTTP(version) + self.debug = debug self.state.host = host self.state.port = port @@ -1425,6 +1433,7 @@ def _helper( "loop": loop, "register_sys_signals": register_sys_signals, "backlog": backlog, + "version": version, } self.motd(serve_location) diff --git a/sanic/http/__init__.py b/sanic/http/__init__.py new file mode 100644 index 0000000000..8a96102926 --- /dev/null +++ b/sanic/http/__init__.py @@ -0,0 +1,5 @@ +from .constants import Stage +from .http1 import Http + + +__all__ = ("Http", "Stage") diff --git a/sanic/http/constants.py b/sanic/http/constants.py new file mode 100644 index 0000000000..3589071243 --- /dev/null +++ b/sanic/http/constants.py @@ -0,0 +1,25 @@ +from enum import Enum + + +class Stage(Enum): + """ + Enum for representing the stage of the request/response cycle + + | ``IDLE`` Waiting for request + | ``REQUEST`` Request headers being received + | ``HANDLER`` Headers done, handler running + | ``RESPONSE`` Response headers sent, body in progress + | ``FAILED`` Unrecoverable state (error while sending response) + | + """ + + IDLE = 0 # Waiting for request + REQUEST = 1 # Request headers being received + HANDLER = 3 # Headers done, handler running + RESPONSE = 4 # Response headers sent, body in progress + FAILED = 100 # Unrecoverable state (error while sending response) + + +class HTTP(Enum): + VERSION_1 = 1 + VERSION_3 = 3 diff --git a/sanic/http.py b/sanic/http/http1.py similarity index 96% rename from sanic/http.py rename to sanic/http/http1.py index 86f23fe3e3..c00929bed7 100644 --- a/sanic/http.py +++ b/sanic/http/http1.py @@ -8,7 +8,6 @@ from sanic.response import BaseHTTPResponse from asyncio import CancelledError, sleep -from enum import Enum from sanic.compat import Header from sanic.exceptions import ( @@ -20,29 +19,11 @@ ) from sanic.headers import format_http1_response from sanic.helpers import has_message_body +from sanic.http.constants import Stage from sanic.log import access_logger, error_logger, logger from sanic.touchup import TouchUpMeta -class Stage(Enum): - """ - Enum for representing the stage of the request/response cycle - - | ``IDLE`` Waiting for request - | ``REQUEST`` Request headers being received - | ``HANDLER`` Headers done, handler running - | ``RESPONSE`` Response headers sent, body in progress - | ``FAILED`` Unrecoverable state (error while sending response) - | - """ - - IDLE = 0 # Waiting for request - REQUEST = 1 # Request headers being received - HANDLER = 3 # Headers done, handler running - RESPONSE = 4 # Response headers sent, body in progress - FAILED = 100 # Unrecoverable state (error while sending response) - - HTTP_CONTINUE = b"HTTP/1.1 100 Continue\r\n\r\n" diff --git a/sanic/http/http3.py b/sanic/http/http3.py new file mode 100644 index 0000000000..cc82a51d9e --- /dev/null +++ b/sanic/http/http3.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Callable, Dict, Optional, Union + +from aioquic.h0.connection import H0_ALPN, H0Connection +from aioquic.h3.connection import H3_ALPN, H3Connection +from aioquic.h3.events import H3Event + +# from aioquic.h3.events import ( +# DatagramReceived, +# DataReceived, +# H3Event, +# HeadersReceived, +# WebTransportStreamDataReceived, +# ) +from aioquic.quic.configuration import QuicConfiguration + +# from aioquic.quic.events import ( +# DatagramFrameReceived, +# ProtocolNegotiated, +# QuicEvent, +# ) +from aioquic.tls import SessionTicket + + +if TYPE_CHECKING: + from sanic.request import Request + +# from sanic.compat import Header +from sanic.log import logger +from sanic.response import BaseHTTPResponse + + +HttpConnection = Union[H0Connection, H3Connection] + + +async def handler(request: Request): + logger.info(f"Request received: {request}") + response = await request.app.handle_request(request) + logger.info(f"Build response: {response=}") + + +class Transport: + ... + + +class Http3: + def __init__( + self, + connection: HttpConnection, + transmit: Callable[[], None], + ) -> None: + self.request_body = None + self.connection = connection + self.transmit = transmit + + def http_event_received(self, event: H3Event) -> None: + print("[http_event_received]:", event) + # if isinstance(event, HeadersReceived): + # method, path, *rem = event.headers + # headers = Header(((k.decode(), v.decode()) for k, v in rem)) + # method = method[1].decode() + # path = path[1] + # scheme = headers.pop(":scheme") + # authority = headers.pop(":authority") + # print(f"{headers=}") + # print(f"{method=}") + # print(f"{path=}") + # print(f"{scheme=}") + # print(f"{authority=}") + # if authority: + # headers["host"] = authority + + # request = Request( + # path, headers, "3", method, Transport(), app, b"" + # ) + # request.stream = Stream( + # connection=self._http, transmit=self.transmit + # ) + # print(f"{request=}") + + # asyncio.ensure_future(handler(request)) + + async def send(self, data: bytes, end_stream: bool) -> None: + print(f"[send]: {data=} {end_stream=}") + print(self.response.headers) + # self.connection.send_headers( + # stream_id=0, + # headers=[ + # (b":status", str(self.response.status).encode()), + # *( + # (k.encode(), v.encode()) + # for k, v in self.response.headers.items() + # ), + # ], + # ) + # self.connection.send_data( + # stream_id=0, + # data=data, + # end_stream=end_stream, + # ) + # self.transmit() + + def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: + print(f"[respond]: {response=}") + self.response, response.stream = response, self + return response + + +class SessionTicketStore: + """ + Simple in-memory store for session tickets. + """ + + def __init__(self) -> None: + self.tickets: Dict[bytes, SessionTicket] = {} + + def add(self, ticket: SessionTicket) -> None: + self.tickets[ticket.ticket] = ticket + + def pop(self, label: bytes) -> Optional[SessionTicket]: + return self.tickets.pop(label, None) + + +def get_config(): + config = QuicConfiguration( + alpn_protocols=H3_ALPN + H0_ALPN + ["siduck"], + is_client=False, + max_datagram_frame_size=65536, + ) + config.load_cert_chain("./cert.pem", "./key.pem", password="qqqqqqqq") + return config + + +def get_ticket_store(): + return SessionTicketStore() diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index 409f5e4b2f..551603e0fb 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -2,6 +2,9 @@ from typing import TYPE_CHECKING, Optional +from aioquic.h3.connection import H3_ALPN, H3Connection + +from sanic.http.http3 import Http3 from sanic.touchup.meta import TouchUpMeta @@ -11,6 +14,10 @@ from asyncio import CancelledError from time import monotonic as current_time +from aioquic.asyncio import QuicConnectionProtocol +from aioquic.h3.events import H3Event +from aioquic.quic.events import ProtocolNegotiated, QuicEvent + from sanic.exceptions import RequestTimeout, ServiceUnavailable from sanic.http import Http, Stage from sanic.log import error_logger, logger @@ -19,12 +26,35 @@ from sanic.server.protocols.base_protocol import SanicProtocol -class HttpProtocol(SanicProtocol, metaclass=TouchUpMeta): +class HttpProtocolMixin: + def _setup_connection(self, *args, **kwargs): + self._http = self.HTTP_CLASS(self, *args, **kwargs) + self._time = current_time() + try: + self.check_timeouts() + except AttributeError: + ... + + def _setup(self): + self.request: Optional[Request] = None + self.access_log = self.app.config.ACCESS_LOG + self.request_handler = self.app.handle_request + self.error_handler = self.app.error_handler + self.request_timeout = self.app.config.REQUEST_TIMEOUT + self.response_timeout = self.app.config.RESPONSE_TIMEOUT + self.keep_alive_timeout = self.app.config.KEEP_ALIVE_TIMEOUT + self.request_max_size = self.app.config.REQUEST_MAX_SIZE + self.request_class = self.app.request_class or Request + + +class HttpProtocol(HttpProtocolMixin, SanicProtocol, metaclass=TouchUpMeta): """ This class provides implements the HTTP 1.1 protocol on top of our Sanic Server transport """ + HTTP_CLASS = Http + __touchup__ = ( "send", "connection_task", @@ -70,25 +100,12 @@ def __init__( unix=unix, ) self.url = None - self.request: Optional[Request] = None - self.access_log = self.app.config.ACCESS_LOG - self.request_handler = self.app.handle_request - self.error_handler = self.app.error_handler - self.request_timeout = self.app.config.REQUEST_TIMEOUT - self.response_timeout = self.app.config.RESPONSE_TIMEOUT - self.keep_alive_timeout = self.app.config.KEEP_ALIVE_TIMEOUT - self.request_max_size = self.app.config.REQUEST_MAX_SIZE - self.request_class = self.app.request_class or Request self.state = state if state else {} + self._setup() if "requests_count" not in self.state: self.state["requests_count"] = 0 self._exception = None - def _setup_connection(self): - self._http = Http(self) - self._time = current_time() - self.check_timeouts() - async def connection_task(self): # no cov """ Run a HTTP connection. @@ -236,3 +253,32 @@ def data_received(self, data: bytes): self._data_received.set() except Exception: error_logger.exception("protocol.data_received") + + +class Http3Protocol(HttpProtocolMixin, QuicConnectionProtocol): + HTTP_CLASS = Http3 + + def __init__(self, *args, app: Sanic, **kwargs) -> None: + self.app = app + super().__init__(*args, **kwargs) + self._setup() + self._connection = None + + def quic_event_received(self, event: QuicEvent) -> None: + print("[quic_event_received]:", event) + if isinstance(event, ProtocolNegotiated): + self._setup_connection(transmit=self.transmit) + if event.alpn_protocol in H3_ALPN: + self._connection = H3Connection( + self._quic, enable_webtransport=True + ) + # elif event.alpn_protocol in H0_ALPN: + # self._http = H0Connection(self._quic) + # elif isinstance(event, DatagramFrameReceived): + # if event.data == b"quack": + # self._quic.send_datagram_frame(b"quack-ack") + + # pass event to the HTTP layer + if self._connection is not None: + for http_event in self._connection.handle_event(event): + self._http.http_event_received(http_event) diff --git a/sanic/server/runners.py b/sanic/server/runners.py index 94a2932827..c5fbd2f25f 100644 --- a/sanic/server/runners.py +++ b/sanic/server/runners.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Dict, Optional, Type, Union from sanic.config import Config +from sanic.http.constants import HTTP from sanic.server.events import trigger_events @@ -19,11 +20,14 @@ from signal import SIG_IGN, SIGINT, SIGTERM, Signals from signal import signal as signal_func +from aioquic.asyncio import serve as quic_serve + from sanic.compat import OS_IS_WINDOWS, ctrlc_workaround_for_windows +from sanic.http.http3 import get_config, get_ticket_store from sanic.log import error_logger, logger from sanic.models.server_types import Signal from sanic.server.async_server import AsyncioServer -from sanic.server.protocols.http_protocol import HttpProtocol +from sanic.server.protocols.http_protocol import Http3Protocol, HttpProtocol from sanic.server.socket import ( bind_socket, bind_unix_socket, @@ -49,6 +53,7 @@ def serve( signal=Signal(), state=None, asyncio_server_kwargs=None, + version=HTTP.VERSION_1, ): """Start asynchronous HTTP Server on an individual process. @@ -85,6 +90,9 @@ def serve( app.asgi = False + if version is HTTP.VERSION_3: + return serve_http_3(host, port, app, loop) + connections = connections if connections is not None else set() protocol_kwargs = _build_protocol_kwargs(protocol, app.config) server = partial( @@ -185,6 +193,40 @@ def serve( remove_unix_socket(unix) +def serve_http_3(host, port, app, loop): + protocol = partial(Http3Protocol, app=app) + ticket_store = get_ticket_store() + config = get_config() + coro = quic_serve( + host, + port, + configuration=config, + create_protocol=protocol, + session_ticket_fetcher=ticket_store.pop, + session_ticket_handler=ticket_store.add, + ) + server = AsyncioServer(app, loop, coro, []) + loop.run_until_complete(server.startup()) + loop.run_until_complete(server.before_start()) + loop.run_until_complete(server) + loop.run_until_complete(server.after_start()) + + pid = os.getpid() + try: + logger.info("Starting worker [%s]", pid) + loop.run_forever() + except KeyboardInterrupt: + pass + finally: + logger.info("Stopping worker [%s]", pid) + + loop.run_until_complete(server.before_stop()) + + # DO close connections here + + loop.run_until_complete(server.after_stop()) + + def serve_single(server_settings): main_start = server_settings.pop("main_start", None) main_stop = server_settings.pop("main_stop", None) From a937977ca2052921901e0a1640be78fcbd38b00d Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 27 Dec 2021 23:22:37 +0200 Subject: [PATCH 02/59] WIP --- sanic/app.py | 3 +- sanic/cli/app.py | 3 + sanic/cli/arguments.py | 24 +++ sanic/config.py | 8 +- sanic/constants.py | 2 + sanic/http/http3.py | 190 +++++++++++++++-------- sanic/request.py | 4 + sanic/response.py | 4 +- sanic/server/protocols/http_protocol.py | 6 +- sanic/server/runners.py | 31 +++- sanic/tls.py | 196 ------------------------ 11 files changed, 205 insertions(+), 266 deletions(-) delete mode 100644 sanic/tls.py diff --git a/sanic/app.py b/sanic/app.py index cf80912dd5..55c4920ec0 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -76,6 +76,7 @@ from sanic.helpers import _default from sanic.http import Stage from sanic.http.constants import HTTP +from sanic.http.tls import process_to_context from sanic.log import ( LOGGING_CONFIG_DEFAULTS, Colors, @@ -104,7 +105,6 @@ from sanic.server.protocols.websocket_protocol import WebSocketProtocol from sanic.server.websockets.impl import ConnectionClosed from sanic.signals import Signal, SignalRouter -from sanic.tls import process_to_context from sanic.touchup import TouchUp, TouchUpMeta @@ -1216,6 +1216,7 @@ def run( finally: self.is_running = False logger.info("Server Stopped") + print("END OF RUN") def stop(self): """ diff --git a/sanic/cli/app.py b/sanic/cli/app.py index 3001b6e1fa..9c5e55d256 100644 --- a/sanic/cli/app.py +++ b/sanic/cli/app.py @@ -11,6 +11,7 @@ from sanic.app import Sanic from sanic.application.logo import get_logo from sanic.cli.arguments import Group +from sanic.http.constants import HTTP from sanic.log import error_logger from sanic.simple import create_simple_server @@ -160,6 +161,7 @@ def _build_run_kwargs(self): elif len(ssl) == 1 and ssl[0] is not None: # Use only one cert, no TLSSelector. ssl = ssl[0] + version = HTTP(self.args.http) kwargs = { "access_log": self.args.access_log, "debug": self.args.debug, @@ -172,6 +174,7 @@ def _build_run_kwargs(self): "unix": self.args.unix, "verbosity": self.args.verbosity or 0, "workers": self.args.workers, + "version": version, } if self.args.auto_reload: diff --git a/sanic/cli/arguments.py b/sanic/cli/arguments.py index 20644bdc45..05e228f457 100644 --- a/sanic/cli/arguments.py +++ b/sanic/cli/arguments.py @@ -83,6 +83,30 @@ def attach(self): ) +class HTTPVersionGroup(Group): + name = "HTTP version" + + def attach(self): + group = self.container.add_mutually_exclusive_group() + group.add_argument( + "--http", + dest="http", + type=int, + default=1, + help=( + "Which HTTP version to use: HTTP/1.1 or HTTP/3. Value should " + "be either 1 or 3 [default 1]" + ), + ) + group.add_argument( + "-3", + dest="http", + action="store_const", + const=3, + help=("Run Sanic server using HTTP/3"), + ) + + class SocketGroup(Group): name = "Socket binding" diff --git a/sanic/config.py b/sanic/config.py index 30c8627f64..c99d9abe86 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -26,6 +26,9 @@ "GRACEFUL_SHUTDOWN_TIMEOUT": 15.0, # 15 sec "KEEP_ALIVE_TIMEOUT": 5, # 5 seconds "KEEP_ALIVE": True, + "LOCAL_TLS_KEY": _default, + "LOCAL_TLS_CERT": _default, + "LOCALHOST": "localhost", "MOTD": True, "MOTD_DISPLAY": {}, "NOISY_EXCEPTIONS": False, @@ -68,9 +71,12 @@ class Config(dict, metaclass=DescriptorMeta): GRACEFUL_SHUTDOWN_TIMEOUT: float KEEP_ALIVE_TIMEOUT: int KEEP_ALIVE: bool - NOISY_EXCEPTIONS: bool + LOCAL_TLS_KEY: Union[Path, str, Default] + LOCAL_TLS_CERT: Union[Path, str, Default] + LOCALHOST: str MOTD: bool MOTD_DISPLAY: Dict[str, str] + NOISY_EXCEPTIONS: bool PROXIES_COUNT: Optional[int] REAL_IP_HEADER: Optional[str] REGISTER: bool diff --git a/sanic/constants.py b/sanic/constants.py index 80f1d2a9bf..0bc68f2666 100644 --- a/sanic/constants.py +++ b/sanic/constants.py @@ -26,3 +26,5 @@ def __str__(self) -> str: HTTP_METHODS = tuple(HTTPMethod.__members__.values()) DEFAULT_HTTP_CONTENT_TYPE = "application/octet-stream" +DEFAULT_LOCAL_TLS_KEY = "key.pem" +DEFAULT_LOCAL_TLS_CERT = "cert.pem" diff --git a/sanic/http/http3.py b/sanic/http/http3.py index cc82a51d9e..ce0d079768 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -1,18 +1,21 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Callable, Dict, Optional, Union +import asyncio + +from abc import ABC +from ast import Mod +from ssl import SSLContext +from typing import TYPE_CHECKING, Callable, Dict, Optional, Type, Union from aioquic.h0.connection import H0_ALPN, H0Connection from aioquic.h3.connection import H3_ALPN, H3Connection -from aioquic.h3.events import H3Event - -# from aioquic.h3.events import ( -# DatagramReceived, -# DataReceived, -# H3Event, -# HeadersReceived, -# WebTransportStreamDataReceived, -# ) +from aioquic.h3.events import ( + DatagramReceived, + DataReceived, + H3Event, + HeadersReceived, + WebTransportStreamDataReceived, +) from aioquic.quic.configuration import QuicConfiguration # from aioquic.quic.events import ( @@ -22,90 +25,145 @@ # ) from aioquic.tls import SessionTicket +from sanic.compat import Header +from sanic.exceptions import SanicException +from sanic.http.tls import CertSimple + if TYPE_CHECKING: + from sanic import Sanic from sanic.request import Request + from sanic.response import BaseHTTPResponse + from sanic.server.protocols.http_protocol import Http3Protocol # from sanic.compat import Header +# from sanic.application.state import Mode from sanic.log import logger -from sanic.response import BaseHTTPResponse HttpConnection = Union[H0Connection, H3Connection] -async def handler(request: Request): - logger.info(f"Request received: {request}") - response = await request.app.handle_request(request) - logger.info(f"Build response: {response=}") +class Transport: + ... -class Transport: +class Receiver(ABC): + def __init__(self, transmit, protocol, request) -> None: + self.transmit = transmit + self.protocol = protocol + self.request = request + + +class HTTPReceiver(Receiver): + async def respond(self): + logger.info(f"Request received: {self.request}") + await self.protocol.app.handle_request(self.request) + + +class WebsocketReceiver(Receiver): + ... + + +class WebTransportReceiver(Receiver): ... class Http3: + HANDLER_PROPERTY_MAPPING = { + DataReceived: "stream_id", + HeadersReceived: "stream_id", + DatagramReceived: "flow_id", + WebTransportStreamDataReceived: "session_id", + } + def __init__( self, - connection: HttpConnection, + protocol: Http3Protocol, transmit: Callable[[], None], ) -> None: self.request_body = None - self.connection = connection + self.request: Optional[Request] = None + self.protocol = protocol self.transmit = transmit + self.receivers: Dict[int, Receiver] = {} def http_event_received(self, event: H3Event) -> None: print("[http_event_received]:", event) - # if isinstance(event, HeadersReceived): - # method, path, *rem = event.headers - # headers = Header(((k.decode(), v.decode()) for k, v in rem)) - # method = method[1].decode() - # path = path[1] - # scheme = headers.pop(":scheme") - # authority = headers.pop(":authority") - # print(f"{headers=}") - # print(f"{method=}") - # print(f"{path=}") - # print(f"{scheme=}") - # print(f"{authority=}") - # if authority: - # headers["host"] = authority - - # request = Request( - # path, headers, "3", method, Transport(), app, b"" - # ) - # request.stream = Stream( - # connection=self._http, transmit=self.transmit - # ) - # print(f"{request=}") + receiver = self.get_or_make_receiver(event) + print(f"{receiver=}") # asyncio.ensure_future(handler(request)) - async def send(self, data: bytes, end_stream: bool) -> None: - print(f"[send]: {data=} {end_stream=}") - print(self.response.headers) - # self.connection.send_headers( - # stream_id=0, - # headers=[ - # (b":status", str(self.response.status).encode()), - # *( - # (k.encode(), v.encode()) - # for k, v in self.response.headers.items() - # ), - # ], - # ) - # self.connection.send_data( - # stream_id=0, - # data=data, - # end_stream=end_stream, - # ) - # self.transmit() - - def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: + def get_or_make_receiver(self, event: H3Event) -> Receiver: + if ( + isinstance(event, HeadersReceived) + and event.stream_id not in self.receivers + ): + self.request = self._make_request(event) + receiver = HTTPReceiver(self.transmit, self.protocol, self.request) + self.receivers[event.stream_id] = receiver + asyncio.ensure_future(receiver.respond()) + else: + ident = getattr(event, self.HANDLER_PROPERTY_MAPPING[type(event)]) + return self.receivers[ident] + + def _make_request(self, event: HeadersReceived) -> Request: + method, path, *rem = event.headers + headers = Header(((k.decode(), v.decode()) for k, v in rem)) + method = method[1].decode() + path = path[1] + scheme = headers.pop(":scheme") + authority = headers.pop(":authority") + print(f"{headers=}") + print(f"{method=}") + print(f"{path=}") + print(f"{scheme=}") + print(f"{authority=}") + if authority: + headers["host"] = authority + + request = self.protocol.request_class( + path, headers, "3", method, Transport(), self.protocol.app, b"" + ) + request.stream = self + print(f"{request=}") + return request + + async def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: print(f"[respond]: {response=}") + response.headers.update({"foo": "bar"}) self.response, response.stream = response, self + + # Need more appropriate place to send these + self.protocol.connection.send_headers( + stream_id=0, + headers=[ + (b":status", str(self.response.status).encode()), + *( + (k.encode(), v.encode()) + for k, v in self.response.headers.items() + ), + ], + ) + # TEMP + await self.drain(response) + return response + async def drain(self, response: BaseHTTPResponse) -> None: + await self.send(response.body, False) + + async def send(self, data: bytes, end_stream: bool) -> None: + print(f"[send]: {data=} {end_stream=}") + print(self.response.headers) + self.protocol.connection.send_data( + stream_id=0, + data=data, + end_stream=end_stream, + ) + self.transmit() + class SessionTicketStore: """ @@ -122,13 +180,19 @@ def pop(self, label: bytes) -> Optional[SessionTicket]: return self.tickets.pop(label, None) -def get_config(): +def get_config(app: Sanic, ssl: SSLContext): + if not isinstance(ssl, CertSimple): + raise SanicException("SSLContext is not CertSimple") + config = QuicConfiguration( alpn_protocols=H3_ALPN + H0_ALPN + ["siduck"], is_client=False, max_datagram_frame_size=65536, ) - config.load_cert_chain("./cert.pem", "./key.pem", password="qqqqqqqq") + # TODO: + # - add password kwarg, read from config.TLS_CERT_PASSWORD + config.load_cert_chain(ssl.sanic["cert"], ssl.sanic["key"]) + return config diff --git a/sanic/request.py b/sanic/request.py index 97ab998211..ee535ac2f1 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -1,5 +1,6 @@ from __future__ import annotations +from inspect import isawaitable from typing import ( TYPE_CHECKING, Any, @@ -205,6 +206,9 @@ async def respond( # Connect the response if isinstance(response, BaseHTTPResponse) and self.stream: response = self.stream.respond(response) + + if isawaitable(response): + response = await response # Run response middleware try: response = await self.app._run_response_middleware( diff --git a/sanic/response.py b/sanic/response.py index 8525d381bb..cd3bd952b5 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -25,6 +25,7 @@ from sanic.exceptions import SanicException, ServerError from sanic.helpers import has_message_body, remove_entity_headers from sanic.http import Http +from sanic.http.http3 import Http3 from sanic.models.protocol_types import HTMLProtocol, Range @@ -56,7 +57,7 @@ def __init__(self): self.asgi: bool = False self.body: Optional[bytes] = None self.content_type: Optional[str] = None - self.stream: Optional[Union[Http, ASGIApp]] = None + self.stream: Optional[Union[Http, ASGIApp, Http3]] = None self.status: int = None self.headers = Header({}) self._cookies: Optional[CookieJar] = None @@ -121,6 +122,7 @@ async def send( :param data: str or bytes to be written :param end_stream: whether to close the stream after this block """ + print(f">>> BaseHTTPResponse: {data=} {end_stream=} {self.body=}") if data is None and end_stream is None: end_stream = True if self.stream is None: diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index 551603e0fb..df2ed6997e 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -262,7 +262,7 @@ def __init__(self, *args, app: Sanic, **kwargs) -> None: self.app = app super().__init__(*args, **kwargs) self._setup() - self._connection = None + self._connection: Optional[H3Connection] = None def quic_event_received(self, event: QuicEvent) -> None: print("[quic_event_received]:", event) @@ -282,3 +282,7 @@ def quic_event_received(self, event: QuicEvent) -> None: if self._connection is not None: for http_event in self._connection.handle_event(event): self._http.http_event_received(http_event) + + @property + def connection(self) -> Optional[H3Connection]: + return self._connection diff --git a/sanic/server/runners.py b/sanic/server/runners.py index aed22ffe6f..b5a9d9c23c 100644 --- a/sanic/server/runners.py +++ b/sanic/server/runners.py @@ -7,6 +7,7 @@ from sanic.config import Config from sanic.http.constants import HTTP +from sanic.http.tls import get_ssl_context from sanic.server.events import trigger_events @@ -94,7 +95,7 @@ def serve( app.asgi = False if version is HTTP.VERSION_3: - return serve_http_3(host, port, app, loop) + return serve_http_3(host, port, app, loop, ssl) connections = connections if connections is not None else set() protocol_kwargs = _build_protocol_kwargs(protocol, app.config) @@ -200,10 +201,19 @@ def serve( remove_unix_socket(unix) -def serve_http_3(host, port, app, loop): +def serve_http_3( + host, + port, + app, + loop, + ssl, + register_sys_signals: bool = True, + run_multiple: bool = False, +): protocol = partial(Http3Protocol, app=app) ticket_store = get_ticket_store() - config = get_config() + ssl_context = get_ssl_context(app, ssl) + config = get_config(app, ssl_context) coro = quic_serve( host, port, @@ -214,6 +224,21 @@ def serve_http_3(host, port, app, loop): ) server = AsyncioServer(app, loop, coro, []) loop.run_until_complete(server.startup()) + + # TODO: Cleanup the non-DRY code block + # Ignore SIGINT when run_multiple + if run_multiple: + signal_func(SIGINT, SIG_IGN) + os.environ["SANIC_WORKER_PROCESS"] = "true" + + # Register signals for graceful termination + if register_sys_signals: + if OS_IS_WINDOWS: + ctrlc_workaround_for_windows(app) + else: + for _signal in [SIGTERM] if run_multiple else [SIGINT, SIGTERM]: + loop.add_signal_handler(_signal, app.stop) + loop.run_until_complete(server.before_start()) loop.run_until_complete(server) loop.run_until_complete(server.after_start()) diff --git a/sanic/tls.py b/sanic/tls.py deleted file mode 100644 index be30f4a263..0000000000 --- a/sanic/tls.py +++ /dev/null @@ -1,196 +0,0 @@ -import os -import ssl - -from typing import Iterable, Optional, Union - -from sanic.log import logger - - -# Only allow secure ciphers, notably leaving out AES-CBC mode -# OpenSSL chooses ECDSA or RSA depending on the cert in use -CIPHERS_TLS12 = [ - "ECDHE-ECDSA-CHACHA20-POLY1305", - "ECDHE-ECDSA-AES256-GCM-SHA384", - "ECDHE-ECDSA-AES128-GCM-SHA256", - "ECDHE-RSA-CHACHA20-POLY1305", - "ECDHE-RSA-AES256-GCM-SHA384", - "ECDHE-RSA-AES128-GCM-SHA256", -] - - -def create_context( - certfile: Optional[str] = None, - keyfile: Optional[str] = None, - password: Optional[str] = None, -) -> ssl.SSLContext: - """Create a context with secure crypto and HTTP/1.1 in protocols.""" - context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH) - context.minimum_version = ssl.TLSVersion.TLSv1_2 - context.set_ciphers(":".join(CIPHERS_TLS12)) - context.set_alpn_protocols(["http/1.1"]) - context.sni_callback = server_name_callback - if certfile and keyfile: - context.load_cert_chain(certfile, keyfile, password) - return context - - -def shorthand_to_ctx( - ctxdef: Union[None, ssl.SSLContext, dict, str] -) -> Optional[ssl.SSLContext]: - """Convert an ssl argument shorthand to an SSLContext object.""" - if ctxdef is None or isinstance(ctxdef, ssl.SSLContext): - return ctxdef - if isinstance(ctxdef, str): - return load_cert_dir(ctxdef) - if isinstance(ctxdef, dict): - return CertSimple(**ctxdef) - raise ValueError( - f"Invalid ssl argument {type(ctxdef)}." - " Expecting a list of certdirs, a dict or an SSLContext." - ) - - -def process_to_context( - ssldef: Union[None, ssl.SSLContext, dict, str, list, tuple] -) -> Optional[ssl.SSLContext]: - """Process app.run ssl argument from easy formats to full SSLContext.""" - return ( - CertSelector(map(shorthand_to_ctx, ssldef)) - if isinstance(ssldef, (list, tuple)) - else shorthand_to_ctx(ssldef) - ) - - -def load_cert_dir(p: str) -> ssl.SSLContext: - if os.path.isfile(p): - raise ValueError(f"Certificate folder expected but {p} is a file.") - keyfile = os.path.join(p, "privkey.pem") - certfile = os.path.join(p, "fullchain.pem") - if not os.access(keyfile, os.R_OK): - raise ValueError( - f"Certificate not found or permission denied {keyfile}" - ) - if not os.access(certfile, os.R_OK): - raise ValueError( - f"Certificate not found or permission denied {certfile}" - ) - return CertSimple(certfile, keyfile) - - -class CertSimple(ssl.SSLContext): - """A wrapper for creating SSLContext with a sanic attribute.""" - - def __new__(cls, cert, key, **kw): - # try common aliases, rename to cert/key - certfile = kw["cert"] = kw.pop("certificate", None) or cert - keyfile = kw["key"] = kw.pop("keyfile", None) or key - password = kw.pop("password", None) - if not certfile or not keyfile: - raise ValueError("SSL dict needs filenames for cert and key.") - subject = {} - if "names" not in kw: - cert = ssl._ssl._test_decode_cert(certfile) # type: ignore - kw["names"] = [ - name - for t, name in cert["subjectAltName"] - if t in ["DNS", "IP Address"] - ] - subject = {k: v for item in cert["subject"] for k, v in item} - self = create_context(certfile, keyfile, password) - self.__class__ = cls - self.sanic = {**subject, **kw} - return self - - def __init__(self, cert, key, **kw): - pass # Do not call super().__init__ because it is already initialized - - -class CertSelector(ssl.SSLContext): - """Automatically select SSL certificate based on the hostname that the - client is trying to access, via SSL SNI. Paths to certificate folders - with privkey.pem and fullchain.pem in them should be provided, and - will be matched in the order given whenever there is a new connection. - """ - - def __new__(cls, ctxs): - return super().__new__(cls) - - def __init__(self, ctxs: Iterable[Optional[ssl.SSLContext]]): - super().__init__() - self.sni_callback = selector_sni_callback # type: ignore - self.sanic_select = [] - self.sanic_fallback = None - all_names = [] - for i, ctx in enumerate(ctxs): - if not ctx: - continue - names = dict(getattr(ctx, "sanic", {})).get("names", []) - all_names += names - self.sanic_select.append(ctx) - if i == 0: - self.sanic_fallback = ctx - if not all_names: - raise ValueError( - "No certificates with SubjectAlternativeNames found." - ) - logger.info(f"Certificate vhosts: {', '.join(all_names)}") - - -def find_cert(self: CertSelector, server_name: str): - """Find the first certificate that matches the given SNI. - - :raises ssl.CertificateError: No matching certificate found. - :return: A matching ssl.SSLContext object if found.""" - if not server_name: - if self.sanic_fallback: - return self.sanic_fallback - raise ValueError( - "The client provided no SNI to match for certificate." - ) - for ctx in self.sanic_select: - if match_hostname(ctx, server_name): - return ctx - if self.sanic_fallback: - return self.sanic_fallback - raise ValueError(f"No certificate found matching hostname {server_name!r}") - - -def match_hostname( - ctx: Union[ssl.SSLContext, CertSelector], hostname: str -) -> bool: - """Match names from CertSelector against a received hostname.""" - # Local certs are considered trusted, so this can be less pedantic - # and thus faster than the deprecated ssl.match_hostname function is. - names = dict(getattr(ctx, "sanic", {})).get("names", []) - hostname = hostname.lower() - for name in names: - if name.startswith("*."): - if hostname.split(".", 1)[-1] == name[2:]: - return True - elif name == hostname: - return True - return False - - -def selector_sni_callback( - sslobj: ssl.SSLObject, server_name: str, ctx: CertSelector -) -> Optional[int]: - """Select a certificate matching the SNI.""" - # Call server_name_callback to store the SNI on sslobj - server_name_callback(sslobj, server_name, ctx) - # Find a new context matching the hostname - try: - sslobj.context = find_cert(ctx, server_name) - except ValueError as e: - logger.warning(f"Rejecting TLS connection: {e}") - # This would show ERR_SSL_UNRECOGNIZED_NAME_ALERT on client side if - # asyncio/uvloop did proper SSL shutdown. They don't. - return ssl.ALERT_DESCRIPTION_UNRECOGNIZED_NAME - return None # mypy complains without explicit return - - -def server_name_callback( - sslobj: ssl.SSLObject, server_name: str, ctx: ssl.SSLContext -) -> None: - """Store the received SNI as sslobj.sanic_server_name.""" - sslobj.sanic_server_name = server_name # type: ignore From 6a65222f1e2a8cd534886792ceefabb416ee83bc Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 27 Dec 2021 23:22:42 +0200 Subject: [PATCH 03/59] WIP --- sanic/http/tls.py | 313 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 sanic/http/tls.py diff --git a/sanic/http/tls.py b/sanic/http/tls.py new file mode 100644 index 0000000000..844fe435cb --- /dev/null +++ b/sanic/http/tls.py @@ -0,0 +1,313 @@ +from __future__ import annotations + +import os +import ssl +import subprocess + +from contextlib import suppress +from inspect import currentframe, getframeinfo +from pathlib import Path +from ssl import SSLContext +from tempfile import mkdtemp +from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union + +from sanic.application.state import Mode +from sanic.constants import DEFAULT_LOCAL_TLS_CERT, DEFAULT_LOCAL_TLS_KEY +from sanic.exceptions import SanicException +from sanic.helpers import Default, _default +from sanic.log import logger + + +if TYPE_CHECKING: + from sanic import Sanic + + +# Only allow secure ciphers, notably leaving out AES-CBC mode +# OpenSSL chooses ECDSA or RSA depending on the cert in use +CIPHERS_TLS12 = [ + "ECDHE-ECDSA-CHACHA20-POLY1305", + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-CHACHA20-POLY1305", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES128-GCM-SHA256", +] + + +def create_context( + certfile: Optional[str] = None, + keyfile: Optional[str] = None, + password: Optional[str] = None, +) -> ssl.SSLContext: + """Create a context with secure crypto and HTTP/1.1 in protocols.""" + context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.set_ciphers(":".join(CIPHERS_TLS12)) + context.set_alpn_protocols(["http/1.1"]) + context.sni_callback = server_name_callback + if certfile and keyfile: + context.load_cert_chain(certfile, keyfile, password) + return context + + +def shorthand_to_ctx( + ctxdef: Union[None, ssl.SSLContext, dict, str] +) -> Optional[ssl.SSLContext]: + """Convert an ssl argument shorthand to an SSLContext object.""" + if ctxdef is None or isinstance(ctxdef, ssl.SSLContext): + return ctxdef + if isinstance(ctxdef, str): + return load_cert_dir(ctxdef) + if isinstance(ctxdef, dict): + return CertSimple(**ctxdef) + raise ValueError( + f"Invalid ssl argument {type(ctxdef)}." + " Expecting a list of certdirs, a dict or an SSLContext." + ) + + +def process_to_context( + ssldef: Union[None, ssl.SSLContext, dict, str, list, tuple] +) -> Optional[ssl.SSLContext]: + """Process app.run ssl argument from easy formats to full SSLContext.""" + return ( + CertSelector(map(shorthand_to_ctx, ssldef)) + if isinstance(ssldef, (list, tuple)) + else shorthand_to_ctx(ssldef) + ) + + +def load_cert_dir(p: str) -> ssl.SSLContext: + if os.path.isfile(p): + raise ValueError(f"Certificate folder expected but {p} is a file.") + keyfile = os.path.join(p, "privkey.pem") + certfile = os.path.join(p, "fullchain.pem") + if not os.access(keyfile, os.R_OK): + raise ValueError( + f"Certificate not found or permission denied {keyfile}" + ) + if not os.access(certfile, os.R_OK): + raise ValueError( + f"Certificate not found or permission denied {certfile}" + ) + return CertSimple(certfile, keyfile) + + +class CertSimple(ssl.SSLContext): + """A wrapper for creating SSLContext with a sanic attribute.""" + + sanic: Dict[str, Any] + + def __new__(cls, cert, key, **kw): + # try common aliases, rename to cert/key + certfile = kw["cert"] = kw.pop("certificate", None) or cert + keyfile = kw["key"] = kw.pop("keyfile", None) or key + password = kw.pop("password", None) + if not certfile or not keyfile: + raise ValueError("SSL dict needs filenames for cert and key.") + subject = {} + if "names" not in kw: + cert = ssl._ssl._test_decode_cert(certfile) # type: ignore + kw["names"] = [ + name + for t, name in cert["subjectAltName"] + if t in ["DNS", "IP Address"] + ] + subject = {k: v for item in cert["subject"] for k, v in item} + self = create_context(certfile, keyfile, password) + self.__class__ = cls + self.sanic = {**subject, **kw} + return self + + def __init__(self, cert, key, **kw): + pass # Do not call super().__init__ because it is already initialized + + +class CertSelector(ssl.SSLContext): + """Automatically select SSL certificate based on the hostname that the + client is trying to access, via SSL SNI. Paths to certificate folders + with privkey.pem and fullchain.pem in them should be provided, and + will be matched in the order given whenever there is a new connection. + """ + + def __new__(cls, ctxs): + return super().__new__(cls) + + def __init__(self, ctxs: Iterable[Optional[ssl.SSLContext]]): + super().__init__() + self.sni_callback = selector_sni_callback # type: ignore + self.sanic_select = [] + self.sanic_fallback = None + all_names = [] + for i, ctx in enumerate(ctxs): + if not ctx: + continue + names = dict(getattr(ctx, "sanic", {})).get("names", []) + all_names += names + self.sanic_select.append(ctx) + if i == 0: + self.sanic_fallback = ctx + if not all_names: + raise ValueError( + "No certificates with SubjectAlternativeNames found." + ) + logger.info(f"Certificate vhosts: {', '.join(all_names)}") + + +def find_cert(self: CertSelector, server_name: str): + """Find the first certificate that matches the given SNI. + + :raises ssl.CertificateError: No matching certificate found. + :return: A matching ssl.SSLContext object if found.""" + if not server_name: + if self.sanic_fallback: + return self.sanic_fallback + raise ValueError( + "The client provided no SNI to match for certificate." + ) + for ctx in self.sanic_select: + if match_hostname(ctx, server_name): + return ctx + if self.sanic_fallback: + return self.sanic_fallback + raise ValueError(f"No certificate found matching hostname {server_name!r}") + + +def match_hostname( + ctx: Union[ssl.SSLContext, CertSelector], hostname: str +) -> bool: + """Match names from CertSelector against a received hostname.""" + # Local certs are considered trusted, so this can be less pedantic + # and thus faster than the deprecated ssl.match_hostname function is. + names = dict(getattr(ctx, "sanic", {})).get("names", []) + hostname = hostname.lower() + for name in names: + if name.startswith("*."): + if hostname.split(".", 1)[-1] == name[2:]: + return True + elif name == hostname: + return True + return False + + +def selector_sni_callback( + sslobj: ssl.SSLObject, server_name: str, ctx: CertSelector +) -> Optional[int]: + """Select a certificate matching the SNI.""" + # Call server_name_callback to store the SNI on sslobj + server_name_callback(sslobj, server_name, ctx) + # Find a new context matching the hostname + try: + sslobj.context = find_cert(ctx, server_name) + except ValueError as e: + logger.warning(f"Rejecting TLS connection: {e}") + # This would show ERR_SSL_UNRECOGNIZED_NAME_ALERT on client side if + # asyncio/uvloop did proper SSL shutdown. They don't. + return ssl.ALERT_DESCRIPTION_UNRECOGNIZED_NAME + return None # mypy complains without explicit return + + +def server_name_callback( + sslobj: ssl.SSLObject, server_name: str, ctx: ssl.SSLContext +) -> None: + """Store the received SNI as sslobj.sanic_server_name.""" + sslobj.sanic_server_name = server_name # type: ignore + + +def _make_path(maybe_path: Union[Path, str], tmpdir: Optional[Path]) -> Path: + if isinstance(maybe_path, Path): + return maybe_path + else: + path = Path(maybe_path) + if not path.exists(): + if not tmpdir: + raise RuntimeError("Reached an unknown state. No tmpdir.") + return tmpdir / maybe_path + + return path + + +def get_ssl_context(app: Sanic, ssl: Optional[SSLContext]) -> SSLContext: + if ssl: + return ssl + + if app.state.mode is Mode.PRODUCTION: + raise SanicException( + "Cannot run Sanic as an HTTP/3 server in PRODUCTION mode " + "without passing a TLS certificate. If you are developing " + "locally, please enable DEVELOPMENT mode and Sanic will " + "generate a localhost TLS certificate. For more information " + "please see: ___." + ) + + try: + tmpdir = None + if isinstance(app.config.LOCAL_TLS_KEY, Default) or isinstance( + app.config.LOCAL_TLS_CERT, Default + ): + tmpdir = Path(mkdtemp()) + + key = ( + DEFAULT_LOCAL_TLS_KEY + if isinstance(app.config.LOCAL_TLS_KEY, Default) + else app.config.LOCAL_TLS_KEY + ) + cert = ( + DEFAULT_LOCAL_TLS_CERT + if isinstance(app.config.LOCAL_TLS_CERT, Default) + else app.config.LOCAL_TLS_CERT + ) + + key_path = _make_path(key, tmpdir) + cert_path = _make_path(cert, tmpdir) + + if not cert_path.exists(): + generate_local_certificate( + key_path, cert_path, app.config.LOCALHOST + ) + finally: + + @app.main_process_stop + async def cleanup(*_): + if tmpdir: + with suppress(FileNotFoundError): + key_path.unlink() + cert_path.unlink() + tmpdir.rmdir() + + return CertSimple(cert_path, key_path) + + +def generate_local_certificate( + key_path: Path, cert_path: Path, localhost: str +): + check_mkcert() + + cmd = [ + "mkcert", + "-key-file", + str(key_path), + "-cert-file", + str(cert_path), + localhost, + ] + subprocess.run(cmd, check=True) + + +def check_mkcert(): + try: + subprocess.run( + ["mkcert", "-help"], + check=True, + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + except Exception as e: + raise SanicException( + "Sanic uses mkcert to generate local TLS certificates. Since you " + "did not supply a certificate, Sanic is attempting to generate " + "one for you, but cannot proceed since mkcert does not appear to " + "be installed. Please install mkcert or supply TLS certificates " + "to proceed. Installation instructions can be found here: " + "https://github.com/FiloSottile/mkcert" + ) from e From c8524067a57f0839884d4a1d8b2c85154a9a30eb Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Sun, 16 Jan 2022 21:54:52 +0200 Subject: [PATCH 04/59] Add aioquic and HTTP auto --- sanic/http/constants.py | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/sanic/http/constants.py b/sanic/http/constants.py index 3589071243..6b914c1a93 100644 --- a/sanic/http/constants.py +++ b/sanic/http/constants.py @@ -21,5 +21,6 @@ class Stage(Enum): class HTTP(Enum): + AUTO = 0 VERSION_1 = 1 VERSION_3 = 3 diff --git a/setup.py b/setup.py index ea6f285a80..5dc874c31e 100644 --- a/setup.py +++ b/setup.py @@ -148,6 +148,7 @@ def open_local(paths, mode="r", encoding="utf8"): "docs": docs_require, "all": all_require, "ext": ["sanic-ext"], + "http3": ["aioquic"], } setup_kwargs["install_requires"] = requirements From 65459fdeb674903f7a015a9c1e25cfc74bc11251 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 17 Jan 2022 20:52:25 +0200 Subject: [PATCH 05/59] WIP --- sanic/cli/app.py | 7 ++++++- sanic/cli/arguments.py | 24 ++++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/sanic/cli/app.py b/sanic/cli/app.py index 83410bb3aa..19847deef7 100644 --- a/sanic/cli/app.py +++ b/sanic/cli/app.py @@ -59,10 +59,13 @@ def __init__(self) -> None: os.environ.get("SANIC_RELOADER_PROCESS", "") != "true" ) self.args: List[Any] = [] + self.groups: List[Group] = [] def attach(self): for group in Group._registry: - group.create(self.parser).attach() + instance = group.create(self.parser) + instance.attach() + self.groups.append(instance) def run(self): # This is to provide backwards compat -v to display version @@ -142,6 +145,8 @@ def _get_app(self): return app def _build_run_kwargs(self): + for group in self.groups: + group.prepare(self.args) ssl: Union[None, dict, str, list] = [] if self.args.tlshost: ssl.append(None) diff --git a/sanic/cli/arguments.py b/sanic/cli/arguments.py index b56132cc82..f443dbda7d 100644 --- a/sanic/cli/arguments.py +++ b/sanic/cli/arguments.py @@ -1,6 +1,7 @@ from __future__ import annotations from argparse import ArgumentParser, _ArgumentGroup +from os import getpid from typing import List, Optional, Type, Union from sanic_routing import __version__ as __routing_version__ # type: ignore @@ -38,6 +39,9 @@ def add_bool_arguments(self, *args, **kwargs): "--no-" + args[0][2:], *args[1:], action="store_false", **kwargs ) + def prepare(self, args) -> None: + ... + class GeneralGroup(Group): name = None @@ -91,21 +95,33 @@ def attach(self): group.add_argument( "--http", dest="http", + action="append", type=int, - default=1, + default=0, help=( - "Which HTTP version to use: HTTP/1.1 or HTTP/3. Value should " - "be either 1 or 3 [default 1]" + "Which HTTP version to use: HTTP/1.1 or HTTP/3. Value should\n" + "be either 0, 1, or 3, where '0' means use whatever versions\n" + "are available [default 0]" ), ) + group.add_argument( + "-1", + dest="http", + action="append_const", + const=1, + help=("Run Sanic server using HTTP/1.1"), + ) group.add_argument( "-3", dest="http", - action="store_const", + action="append_const", const=3, help=("Run Sanic server using HTTP/3"), ) + def prepare(self, args): + print(args.http) + class SocketGroup(Group): name = "Socket binding" From cab745379100bda12927778e6d6b5e90c2e5b7c7 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Wed, 19 Jan 2022 21:57:08 +0200 Subject: [PATCH 06/59] Add multi-serve for http3 and http1 --- sanic/cli/app.py | 9 +++++---- sanic/cli/arguments.py | 20 +++++++++++--------- sanic/http/constants.py | 5 ++--- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/sanic/cli/app.py b/sanic/cli/app.py index 19847deef7..179fdbf2b9 100644 --- a/sanic/cli/app.py +++ b/sanic/cli/app.py @@ -11,7 +11,6 @@ from sanic.app import Sanic from sanic.application.logo import get_logo from sanic.cli.arguments import Group -from sanic.http.constants import HTTP from sanic.log import error_logger from sanic.simple import create_simple_server @@ -78,9 +77,13 @@ def run(self): try: app = self._get_app() kwargs = self._build_run_kwargs() - app.run(**kwargs) except ValueError: error_logger.exception("Failed to run app") + else: + for http_version in self.args.http: + app.prepare(**kwargs, version=http_version) + + Sanic.serve() def _precheck(self): # # Custom TLS mismatch handling for better diagnostics @@ -159,7 +162,6 @@ def _build_run_kwargs(self): elif len(ssl) == 1 and ssl[0] is not None: # Use only one cert, no TLSSelector. ssl = ssl[0] - version = HTTP(self.args.http) kwargs = { "access_log": self.args.access_log, "debug": self.args.debug, @@ -172,7 +174,6 @@ def _build_run_kwargs(self): "unix": self.args.unix, "verbosity": self.args.verbosity or 0, "workers": self.args.workers, - "version": version, } for maybe_arg in ("auto_reload", "dev"): diff --git a/sanic/cli/arguments.py b/sanic/cli/arguments.py index f443dbda7d..74ef7d8597 100644 --- a/sanic/cli/arguments.py +++ b/sanic/cli/arguments.py @@ -1,12 +1,12 @@ from __future__ import annotations from argparse import ArgumentParser, _ArgumentGroup -from os import getpid from typing import List, Optional, Type, Union from sanic_routing import __version__ as __routing_version__ # type: ignore from sanic import __version__ +from sanic.http.constants import HTTP class Group: @@ -91,27 +91,27 @@ class HTTPVersionGroup(Group): name = "HTTP version" def attach(self): - group = self.container.add_mutually_exclusive_group() - group.add_argument( + http_values = [http.value for http in HTTP.__members__.values()] + + self.container.add_argument( "--http", dest="http", action="append", + choices=http_values, type=int, - default=0, help=( "Which HTTP version to use: HTTP/1.1 or HTTP/3. Value should\n" - "be either 0, 1, or 3, where '0' means use whatever versions\n" - "are available [default 0]" + "be either 1, or 3. [default 1]" ), ) - group.add_argument( + self.container.add_argument( "-1", dest="http", action="append_const", const=1, help=("Run Sanic server using HTTP/1.1"), ) - group.add_argument( + self.container.add_argument( "-3", dest="http", action="append_const", @@ -120,7 +120,9 @@ def attach(self): ) def prepare(self, args): - print(args.http) + if not args.http: + args.http = [1] + args.http = tuple(sorted(set(map(HTTP, args.http)), reverse=True)) class SocketGroup(Group): diff --git a/sanic/http/constants.py b/sanic/http/constants.py index 6b914c1a93..62d8f46f53 100644 --- a/sanic/http/constants.py +++ b/sanic/http/constants.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import Enum, IntEnum class Stage(Enum): @@ -20,7 +20,6 @@ class Stage(Enum): FAILED = 100 # Unrecoverable state (error while sending response) -class HTTP(Enum): - AUTO = 0 +class HTTP(IntEnum): VERSION_1 = 1 VERSION_3 = 3 From 1bb80ba6f34e4fa41feaa0e290c1a7d8cf41ecb7 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Thu, 20 Jan 2022 13:46:42 +0200 Subject: [PATCH 07/59] Add spinner on startup delay --- sanic/application/spinner.py | 88 ++++++++++++++++++++++++++++++++ sanic/cli/app.py | 1 + sanic/cli/arguments.py | 2 - sanic/http/constants.py | 6 +++ sanic/http/tls.py | 38 ++++++++++---- sanic/mixins/runner.py | 97 ++++++++++++++++++++++++++++++++---- 6 files changed, 210 insertions(+), 22 deletions(-) create mode 100644 sanic/application/spinner.py diff --git a/sanic/application/spinner.py b/sanic/application/spinner.py new file mode 100644 index 0000000000..c1e35338f2 --- /dev/null +++ b/sanic/application/spinner.py @@ -0,0 +1,88 @@ +import os +import sys +import time + +from contextlib import contextmanager +from curses.ascii import SP +from queue import Queue +from threading import Thread + + +if os.name == "nt": + import ctypes + import msvcrt + + class _CursorInfo(ctypes.Structure): + _fields_ = [("size", ctypes.c_int), ("visible", ctypes.c_byte)] + + +class Spinner: + def __init__(self, message: str) -> None: + self.message = message + self.queue: Queue[int] = Queue() + self.spinner = self.cursor() + self.thread = Thread(target=self.run) + + def start(self): + self.queue.put(1) + self.thread.start() + self.hide() + + def run(self): + while self.queue.get(): + output = f"\r{self.message} [{next(self.spinner)}]" + sys.stdout.write(output) + sys.stdout.flush() + time.sleep(0.1) + self.queue.put(1) + + def stop(self): + self.queue.put(0) + self.thread.join() + self.show() + + @staticmethod + def cursor(): + while True: + for cursor in "|/-\\": + yield cursor + + @staticmethod + def hide(): + if os.name == "nt": + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo( + handle, ctypes.byref(ci) + ) + ci.visible = False + ctypes.windll.kernel32.SetConsoleCursorInfo( + handle, ctypes.byref(ci) + ) + elif os.name == "posix": + sys.stdout.write("\033[?25l") + sys.stdout.flush() + + @staticmethod + def show(): + if os.name == "nt": + ci = _CursorInfo() + handle = ctypes.windll.kernel32.GetStdHandle(-11) + ctypes.windll.kernel32.GetConsoleCursorInfo( + handle, ctypes.byref(ci) + ) + ci.visible = True + ctypes.windll.kernel32.SetConsoleCursorInfo( + handle, ctypes.byref(ci) + ) + elif os.name == "posix": + sys.stdout.write("\033[?25h") + sys.stdout.flush() + + +@contextmanager +def loading(message: str = "Loading"): + spinner = Spinner(message) + spinner.start() + yield + spinner.stop() diff --git a/sanic/cli/app.py b/sanic/cli/app.py index 179fdbf2b9..45916a3f22 100644 --- a/sanic/cli/app.py +++ b/sanic/cli/app.py @@ -143,6 +143,7 @@ def _get_app(self): " Example File: project/sanic_server.py -> app\n" " Example Module: project.sanic_server.app" ) + sys.exit(1) else: raise e return app diff --git a/sanic/cli/arguments.py b/sanic/cli/arguments.py index 74ef7d8597..8cfa131c97 100644 --- a/sanic/cli/arguments.py +++ b/sanic/cli/arguments.py @@ -134,7 +134,6 @@ def attach(self): "--host", dest="host", type=str, - default="127.0.0.1", help="Host address [default 127.0.0.1]", ) self.container.add_argument( @@ -142,7 +141,6 @@ def attach(self): "--port", dest="port", type=int, - default=8000, help="Port to serve on [default 8000]", ) self.container.add_argument( diff --git a/sanic/http/constants.py b/sanic/http/constants.py index 62d8f46f53..c9e37cf349 100644 --- a/sanic/http/constants.py +++ b/sanic/http/constants.py @@ -23,3 +23,9 @@ class Stage(Enum): class HTTP(IntEnum): VERSION_1 = 1 VERSION_3 = 3 + + def display(self) -> str: + value = str(self.value) + if value == 1: + value = "1.1" + return f"HTTP/{value}" diff --git a/sanic/http/tls.py b/sanic/http/tls.py index 232e1b6248..7429740c32 100644 --- a/sanic/http/tls.py +++ b/sanic/http/tls.py @@ -3,15 +3,16 @@ import os import ssl import subprocess +import sys from contextlib import suppress -from inspect import currentframe, getframeinfo from pathlib import Path from ssl import SSLContext from tempfile import mkdtemp from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union from sanic.application.constants import Mode +from sanic.application.spinner import loading from sanic.constants import DEFAULT_LOCAL_TLS_CERT, DEFAULT_LOCAL_TLS_KEY from sanic.exceptions import SanicException from sanic.helpers import Default @@ -283,15 +284,32 @@ def generate_local_certificate( ): check_mkcert() - cmd = [ - "mkcert", - "-key-file", - str(key_path), - "-cert-file", - str(cert_path), - localhost, - ] - subprocess.run(cmd, check=True) + if not key_path.parent.exists() or not cert_path.parent.exists(): + raise SanicException( + f"Cannot generate certificate at [{key_path}, {cert_path}]. One " + "or more of the directories does not exist." + ) + + message = "Generating TLS certificate" + with loading(message): + cmd = [ + "mkcert", + "-key-file", + str(key_path), + "-cert-file", + str(cert_path), + localhost, + ] + resp = subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + sys.stdout.write("\r" + " " * (len(message) + 4)) + sys.stdout.flush() + sys.stdout.write(resp.stdout) def check_mkcert(): diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py index 39aa42fe09..1a8cfbbda4 100644 --- a/sanic/mixins/runner.py +++ b/sanic/mixins/runner.py @@ -26,8 +26,10 @@ Literal, Optional, Set, + Tuple, Type, Union, + cast, ) from sanic import reloader_helpers @@ -38,7 +40,7 @@ from sanic.compat import OS_IS_WINDOWS from sanic.helpers import _default from sanic.http.constants import HTTP -from sanic.http.tls import process_to_context +from sanic.http.tls import get_ssl_context, process_to_context from sanic.log import Colors, error_logger, logger from sanic.models.handler_types import ListenerType from sanic.server import Signal as ServerSignal @@ -181,6 +183,11 @@ def prepare( verbosity: int = 0, motd_display: Optional[Dict[str, str]] = None, ) -> None: + if version == 3 and self.state.server_info: + raise RuntimeError( + "Serving multiple HTTP/3 instances is not supported." + ) + if dev: debug = True auto_reload = True @@ -222,7 +229,7 @@ def prepare( return if sock is None: - host, port = host or "127.0.0.1", port or 8000 + host, port = self.get_address(host, port, version) if protocol is None: protocol = ( @@ -327,7 +334,7 @@ async def create_server( """ if sock is None: - host, port = host or "127.0.0.1", port or 8000 + host, port = host, port = self.get_address(host, port) if protocol is None: protocol = ( @@ -411,13 +418,19 @@ def _helper( "#proxy-configuration" ) + if not self.state.is_debug: + self.state.mode = Mode.DEBUG if debug else Mode.PRODUCTION + if isinstance(version, int): version = HTTP(version) ssl = process_to_context(ssl) - - if not self.state.is_debug: - self.state.mode = Mode.DEBUG if debug else Mode.PRODUCTION + if version is HTTP.VERSION_3: + # TODO: + # - Add API option to allow localhost TLS also on HTTP/1.1 + if TYPE_CHECKING: + self = cast(Sanic, self) + ssl = get_ssl_context(self, ssl) self.state.host = host or "" self.state.port = port or 0 @@ -441,7 +454,7 @@ def _helper( "backlog": backlog, } - self.motd(self.serve_location) + self.motd(server_settings=server_settings) if sys.stdout.isatty() and not self.state.is_debug: error_logger.warning( @@ -467,7 +480,16 @@ def _helper( return server_settings - def motd(self, serve_location): + def motd( + self, + serve_location: str = "", + server_settings: Optional[Dict[str, Any]] = None, + ): + if serve_location: + # TODO: Deprecation warning + ... + else: + serve_location = self.get_server_location(server_settings) if self.config.MOTD: mode = [f"{self.state.mode},"] if self.state.fast: @@ -480,9 +502,16 @@ def motd(self, serve_location): else: mode.append(f"w/ {self.state.workers} workers") + server = ", ".join( + ( + self.state.server, + server_settings["version"].display(), # type: ignore + ) + ) + display = { "mode": " ".join(mode), - "server": self.state.server, + "server": server, "python": platform.python_version(), "platform": platform.platform(), } @@ -506,7 +535,9 @@ def motd(self, serve_location): module_name = package_name.replace("-", "_") try: module = import_module(module_name) - packages.append(f"{package_name}=={module.__version__}") + packages.append( + f"{package_name}=={module.__version__}" # type: ignore + ) except ImportError: ... @@ -526,6 +557,10 @@ def motd(self, serve_location): @property def serve_location(self) -> str: + # TODO: + # - Will show only the primary server information. The state needs to + # reflect only the first server_info. + # - Deprecate this property in favor of getter serve_location = "" proto = "http" if self.state.ssl is not None: @@ -545,6 +580,48 @@ def serve_location(self) -> str: return serve_location + @staticmethod + def get_server_location( + server_settings: Optional[Dict[str, Any]] = None + ) -> str: + # TODO: + # - Update server_settings to an obj + serve_location = "" + proto = "http" + if not server_settings: + return serve_location + + if server_settings["ssl"] is not None: + proto = "https" + if server_settings["unix"]: + serve_location = f'{server_settings["unix"]} {proto}://...' + elif server_settings["sock"]: + serve_location = ( + f'{server_settings["sock"].getsockname()} {proto}://...' + ) + elif server_settings["host"] and server_settings["port"]: + # colon(:) is legal for a host only in an ipv6 address + display_host = ( + f'[{server_settings["host"]}]' + if ":" in server_settings["host"] + else server_settings["host"] + ) + serve_location = ( + f'{proto}://{display_host}:{server_settings["port"]}' + ) + + return serve_location + + @staticmethod + def get_address( + host: Optional[str], + port: Optional[int], + version: HTTPVersion = HTTP.VERSION_1, + ) -> Tuple[str, int]: + host = host or "127.0.0.1" + port = port or (8443 if version == 3 else 8000) + return host, port + @classmethod def should_auto_reload(cls) -> bool: return any(app.state.auto_reload for app in cls._app_registry.values()) From 6e3e5f16622292e25d47ba2c306f82a2ffad43ef Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Thu, 17 Feb 2022 09:43:52 +0200 Subject: [PATCH 08/59] Better exception --- sanic/mixins/runner.py | 4 +++- sanic/server/runners.py | 43 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py index 1a8cfbbda4..6f288ff10b 100644 --- a/sanic/mixins/runner.py +++ b/sanic/mixins/runner.py @@ -185,7 +185,9 @@ def prepare( ) -> None: if version == 3 and self.state.server_info: raise RuntimeError( - "Serving multiple HTTP/3 instances is not supported." + "Serving HTTP/3 instances as a secondary server is " + "not supported. There can only be a single HTTP/3 worker " + "and it must be prepared first." ) if dev: diff --git a/sanic/server/runners.py b/sanic/server/runners.py index 980f834892..86779472bf 100644 --- a/sanic/server/runners.py +++ b/sanic/server/runners.py @@ -95,8 +95,47 @@ def serve( app.asgi = False if version is HTTP.VERSION_3: - return serve_http_3(host, port, app, loop, ssl) + return _serve_http_3(host, port, app, loop, ssl) + return _serve_http_1( + host, + port, + app, + ssl, + sock, + unix, + reuse_port, + loop, + protocol, + backlog, + register_sys_signals, + run_multiple, + run_async, + connections, + signal, + state, + asyncio_server_kwargs, + ) + +def _serve_http_1( + host, + port, + app, + ssl, + sock, + unix, + reuse_port, + loop, + protocol, + backlog, + register_sys_signals, + run_multiple, + run_async, + connections, + signal, + state, + asyncio_server_kwargs, +): connections = connections if connections is not None else set() protocol_kwargs = _build_protocol_kwargs(protocol, app.config) server = partial( @@ -201,7 +240,7 @@ def serve( remove_unix_socket(unix) -def serve_http_3( +def _serve_http_3( host, port, app, From 64d44192cc0615980f4f5307b0631c83db4176e7 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Thu, 17 Feb 2022 12:10:18 +0200 Subject: [PATCH 09/59] Add alt-svc header touchup --- sanic/http/http1.py | 6 ++++++ sanic/touchup/schemes/__init__.py | 1 + sanic/touchup/schemes/base.py | 26 ++++++++++++++++++++++---- sanic/touchup/schemes/ode.py | 22 ++++++---------------- sanic/touchup/service.py | 6 ++---- 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/sanic/http/http1.py b/sanic/http/http1.py index 96232fb5a2..111357cbae 100644 --- a/sanic/http/http1.py +++ b/sanic/http/http1.py @@ -333,6 +333,12 @@ async def http1_response_header( self.response_func = self.head_response_ignored headers["connection"] = "keep-alive" if self.keep_alive else "close" + + # This header may be removed or modified by the AltSvcCheck Touchup + # service. At server start, we either remove this header from ever + # being assigned, or we change the value as required. + headers["alt-svc"] = "" + ret = format_http1_response(status, res.processed_headers) if data: ret += data diff --git a/sanic/touchup/schemes/__init__.py b/sanic/touchup/schemes/__init__.py index 87057a5fce..dd4145abad 100644 --- a/sanic/touchup/schemes/__init__.py +++ b/sanic/touchup/schemes/__init__.py @@ -1,3 +1,4 @@ +from .altsvc import AltSvcCheck # noqa from .base import BaseScheme from .ode import OptionalDispatchEvent # noqa diff --git a/sanic/touchup/schemes/base.py b/sanic/touchup/schemes/base.py index d16619b2f8..9e32c32371 100644 --- a/sanic/touchup/schemes/base.py +++ b/sanic/touchup/schemes/base.py @@ -1,5 +1,8 @@ from abc import ABC, abstractmethod -from typing import Set, Type +from ast import NodeTransformer, parse +from inspect import getsource +from textwrap import dedent +from typing import Any, Dict, List, Set, Type class BaseScheme(ABC): @@ -10,11 +13,26 @@ def __init__(self, app) -> None: self.app = app @abstractmethod - def run(self, method, module_globals) -> None: + def visitors(self) -> List[NodeTransformer]: ... def __init_subclass__(cls): BaseScheme._registry.add(cls) - def __call__(self, method, module_globals): - return self.run(method, module_globals) + def __call__(self): + return self.visitors() + + @classmethod + def build(cls, method, module_globals, app): + raw_source = getsource(method) + src = dedent(raw_source) + node = parse(src) + + for scheme in cls._registry: + for visitor in scheme(app)(): + node = visitor.visit(node) + + compiled_src = compile(node, method.__name__, "exec") + exec_locals: Dict[str, Any] = {} + exec(compiled_src, module_globals, exec_locals) # nosec + return exec_locals[method.__name__] diff --git a/sanic/touchup/schemes/ode.py b/sanic/touchup/schemes/ode.py index 7c6ed3d7b8..c9b78c8bb3 100644 --- a/sanic/touchup/schemes/ode.py +++ b/sanic/touchup/schemes/ode.py @@ -1,7 +1,5 @@ -from ast import Attribute, Await, Dict, Expr, NodeTransformer, parse -from inspect import getsource -from textwrap import dedent -from typing import Any +from ast import Attribute, Await, Expr, NodeTransformer +from typing import Any, List from sanic.log import logger @@ -20,18 +18,10 @@ def __init__(self, app) -> None: signal.name for signal in app.signal_router.routes ] - def run(self, method, module_globals): - raw_source = getsource(method) - src = dedent(raw_source) - tree = parse(src) - node = RemoveDispatch( - self._registered_events, self.app.state.verbosity - ).visit(tree) - compiled_src = compile(node, method.__name__, "exec") - exec_locals: Dict[str, Any] = {} - exec(compiled_src, module_globals, exec_locals) # nosec - - return exec_locals[method.__name__] + def visitors(self) -> List[NodeTransformer]: + return [ + RemoveDispatch(self._registered_events, self.app.state.verbosity) + ] def _sync_events(self): all_events = set() diff --git a/sanic/touchup/service.py b/sanic/touchup/service.py index 95792dca10..b1b996fb5b 100644 --- a/sanic/touchup/service.py +++ b/sanic/touchup/service.py @@ -21,10 +21,8 @@ def run(cls, app): module = getmodule(target) module_globals = dict(getmembers(module)) - - for scheme in BaseScheme._registry: - modified = scheme(app)(method, module_globals) - setattr(target, method_name, modified) + modified = BaseScheme.build(method, module_globals, app) + setattr(target, method_name, modified) target.__touched__ = True From 97158d8b6470954224098c12ad38d3ccfa04c03e Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Thu, 17 Feb 2022 12:11:47 +0200 Subject: [PATCH 10/59] Add altsvs --- sanic/touchup/schemes/altsvc.py | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 sanic/touchup/schemes/altsvc.py diff --git a/sanic/touchup/schemes/altsvc.py b/sanic/touchup/schemes/altsvc.py new file mode 100644 index 0000000000..05e7269bbe --- /dev/null +++ b/sanic/touchup/schemes/altsvc.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from ast import Assign, Constant, NodeTransformer, Subscript +from typing import TYPE_CHECKING, Any, List + +from sanic.http.constants import HTTP + +from .base import BaseScheme + + +if TYPE_CHECKING: + from sanic import Sanic + + +class AltSvcCheck(BaseScheme): + ident = "ALTSVC" + + def visitors(self) -> List[NodeTransformer]: + return [RemoveAltSvc(self.app, self.app.state.verbosity)] + + +class RemoveAltSvc(NodeTransformer): + def __init__(self, app: Sanic, verbosity: int = 0) -> None: + self._app = app + self._verbosity = verbosity + self._versions = { + info.settings["version"] for info in app.state.server_info + } + + def visit_Assign(self, node: Assign) -> Any: + if any(self._matches(target) for target in node.targets): + if self._should_remove(): + return None + assert isinstance(node.value, Constant) + node.value.value = self.value() + return node + + def _should_remove(self) -> bool: + return len(self._versions) == 1 + + @staticmethod + def _matches(node) -> bool: + return ( + isinstance(node, Subscript) + and isinstance(node.slice, Constant) + and node.slice.value == "alt-svc" + ) + + def value(self): + values = [] + for info in self._app.state.server_info: + port = info.settings["port"] + version = info.settings["version"] + if version is HTTP.VERSION_3: + values.append(f'h3=":{port}"') + return ", ".join(values) From 13ee4c473823f478e2f2479bb0266ae3ffad0cc0 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 21 Feb 2022 22:37:10 +0200 Subject: [PATCH 11/59] Allow for TLS certs to be created on HTTP/1.1 dev servers --- sanic/cli/app.py | 2 ++ sanic/cli/arguments.py | 11 ++++++++++- sanic/http/tls.py | 2 +- sanic/mixins/runner.py | 9 ++++++--- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/sanic/cli/app.py b/sanic/cli/app.py index 45916a3f22..a0b08b24be 100644 --- a/sanic/cli/app.py +++ b/sanic/cli/app.py @@ -175,6 +175,7 @@ def _build_run_kwargs(self): "unix": self.args.unix, "verbosity": self.args.verbosity or 0, "workers": self.args.workers, + "auto_cert": self.args.auto_cert, } for maybe_arg in ("auto_reload", "dev"): @@ -184,4 +185,5 @@ def _build_run_kwargs(self): if self.args.path: kwargs["auto_reload"] = True kwargs["reload_dir"] = self.args.path + return kwargs diff --git a/sanic/cli/arguments.py b/sanic/cli/arguments.py index 8cfa131c97..e6662a5737 100644 --- a/sanic/cli/arguments.py +++ b/sanic/cli/arguments.py @@ -249,7 +249,16 @@ def attach(self): "--dev", dest="dev", action="store_true", - help=("debug + auto reload."), + help=("debug + auto reload"), + ) + self.container.add_argument( + "--auto-cert", + dest="auto_cert", + action="store_true", + help=( + "Create a temporary TLS certificate for local development " + "(requires mkcert)" + ), ) diff --git a/sanic/http/tls.py b/sanic/http/tls.py index 7429740c32..387f669aac 100644 --- a/sanic/http/tls.py +++ b/sanic/http/tls.py @@ -234,7 +234,7 @@ def get_ssl_context(app: Sanic, ssl: Optional[SSLContext]) -> SSLContext: if app.state.mode is Mode.PRODUCTION: raise SanicException( - "Cannot run Sanic as an HTTP/3 server in PRODUCTION mode " + "Cannot run Sanic as an HTTPS server in PRODUCTION mode " "without passing a TLS certificate. If you are developing " "locally, please enable DEVELOPMENT mode and Sanic will " "generate a localhost TLS certificate. For more information " diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py index 6f288ff10b..3b25d4ea46 100644 --- a/sanic/mixins/runner.py +++ b/sanic/mixins/runner.py @@ -95,6 +95,7 @@ def run( fast: bool = False, verbosity: int = 0, motd_display: Optional[Dict[str, str]] = None, + auto_cert: bool = False, ) -> None: """ Run the HTTP Server and listen until keyboard interrupt or term @@ -154,6 +155,7 @@ def run( fast=fast, verbosity=verbosity, motd_display=motd_display, + auto_cert=auto_cert, ) self.__class__.serve(primary=self) # type: ignore @@ -182,6 +184,7 @@ def prepare( fast: bool = False, verbosity: int = 0, motd_display: Optional[Dict[str, str]] = None, + auto_cert: bool = False, ) -> None: if version == 3 and self.state.server_info: raise RuntimeError( @@ -267,6 +270,7 @@ def prepare( protocol=protocol, backlog=backlog, register_sys_signals=register_sys_signals, + auto_cert=auto_cert, ) self.state.server_info.append( ApplicationServerInfo(settings=server_settings) @@ -411,6 +415,7 @@ def _helper( backlog: int = 100, register_sys_signals: bool = True, run_async: bool = False, + auto_cert: bool = False, ) -> Dict[str, Any]: """Helper function used by `run` and `create_server`.""" if self.config.PROXIES_COUNT and self.config.PROXIES_COUNT < 0: @@ -427,9 +432,7 @@ def _helper( version = HTTP(version) ssl = process_to_context(ssl) - if version is HTTP.VERSION_3: - # TODO: - # - Add API option to allow localhost TLS also on HTTP/1.1 + if version is HTTP.VERSION_3 or auto_cert: if TYPE_CHECKING: self = cast(Sanic, self) ssl = get_ssl_context(self, ssl) From 80cc6a273f0adb322b79747db433ca804786bd00 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 21 Feb 2022 22:59:22 +0200 Subject: [PATCH 12/59] Add deprecation notice --- sanic/mixins/runner.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py index 3b25d4ea46..cffee16509 100644 --- a/sanic/mixins/runner.py +++ b/sanic/mixins/runner.py @@ -41,7 +41,7 @@ from sanic.helpers import _default from sanic.http.constants import HTTP from sanic.http.tls import get_ssl_context, process_to_context -from sanic.log import Colors, error_logger, logger +from sanic.log import Colors, deprecation, error_logger, logger from sanic.models.handler_types import ListenerType from sanic.server import Signal as ServerSignal from sanic.server import try_use_uvloop @@ -491,8 +491,11 @@ def motd( server_settings: Optional[Dict[str, Any]] = None, ): if serve_location: - # TODO: Deprecation warning - ... + deprecation( + "Specifying a serve_location in the MOTD is deprecated and " + "will be removed.", + 22.9, + ) else: serve_location = self.get_server_location(server_settings) if self.config.MOTD: From bb1bf6671cd7a06ec10ed15df1366833995544a8 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 22 Feb 2022 00:06:58 +0200 Subject: [PATCH 13/59] Cleanup TODOs --- sanic/mixins/runner.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py index cffee16509..c6b81c81fe 100644 --- a/sanic/mixins/runner.py +++ b/sanic/mixins/runner.py @@ -565,35 +565,13 @@ def motd( @property def serve_location(self) -> str: - # TODO: - # - Will show only the primary server information. The state needs to - # reflect only the first server_info. - # - Deprecate this property in favor of getter - serve_location = "" - proto = "http" - if self.state.ssl is not None: - proto = "https" - if self.state.unix: - serve_location = f"{self.state.unix} {proto}://..." - elif self.state.sock: - serve_location = f"{self.state.sock.getsockname()} {proto}://..." - elif self.state.host and self.state.port: - # colon(:) is legal for a host only in an ipv6 address - display_host = ( - f"[{self.state.host}]" - if ":" in self.state.host - else self.state.host - ) - serve_location = f"{proto}://{display_host}:{self.state.port}" - - return serve_location + server_settings = self.state.server_info[0].settings + return self.get_server_location(server_settings) @staticmethod def get_server_location( server_settings: Optional[Dict[str, Any]] = None ) -> str: - # TODO: - # - Update server_settings to an obj serve_location = "" proto = "http" if not server_settings: From e4477517bc621d063ea2eda79b817575ddb4e635 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 22 Feb 2022 09:20:56 +0200 Subject: [PATCH 14/59] Cleanup merge conflicts --- sanic/cli/app.py | 1 + sanic/cli/arguments.py | 28 +- sanic/http/constants.py | 4 + sanic/mixins/runner.py | 780 ++++++++++++++++++++++++++++++++ sanic/touchup/schemes/altsvc.py | 56 +++ 5 files changed, 857 insertions(+), 12 deletions(-) create mode 100644 sanic/mixins/runner.py create mode 100644 sanic/touchup/schemes/altsvc.py diff --git a/sanic/cli/app.py b/sanic/cli/app.py index 9c5e55d256..2d12c2c7d0 100644 --- a/sanic/cli/app.py +++ b/sanic/cli/app.py @@ -175,6 +175,7 @@ def _build_run_kwargs(self): "verbosity": self.args.verbosity or 0, "workers": self.args.workers, "version": version, + "auto_tls": self.args.auto_tls, } if self.args.auto_reload: diff --git a/sanic/cli/arguments.py b/sanic/cli/arguments.py index 05e228f457..e68b535c86 100644 --- a/sanic/cli/arguments.py +++ b/sanic/cli/arguments.py @@ -206,18 +206,6 @@ def attach(self): action="store_true", help="Run the server in debug mode", ) - self.container.add_argument( - "-d", - "--dev", - dest="debug", - action="store_true", - help=( - "Currently is an alias for --debug. But starting in v22.3, \n" - "--debug will no longer automatically trigger auto_restart. \n" - "However, --dev will continue, effectively making it the \n" - "same as debug + auto_reload." - ), - ) self.container.add_argument( "-r", "--reload", @@ -236,6 +224,22 @@ def attach(self): action="append", help="Extra directories to watch and reload on changes", ) + self.container.add_argument( + "-d", + "--dev", + dest="dev", + action="store_true", + help=("debug + auto reload"), + ) + self.container.add_argument( + "--auto-tls", + dest="auto_tls", + action="store_true", + help=( + "Create a temporary TLS certificate for local development " + "(requires mkcert)" + ), + ) class OutputGroup(Group): diff --git a/sanic/http/constants.py b/sanic/http/constants.py index 3589071243..359e9ec0e9 100644 --- a/sanic/http/constants.py +++ b/sanic/http/constants.py @@ -23,3 +23,7 @@ class Stage(Enum): class HTTP(Enum): VERSION_1 = 1 VERSION_3 = 3 + + def display(self) -> str: + value = 1.1 if self.value == 1 else self.value + return f"HTTP/{value}" diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py new file mode 100644 index 0000000000..91547c142c --- /dev/null +++ b/sanic/mixins/runner.py @@ -0,0 +1,780 @@ +from __future__ import annotations + +import os +import platform +import sys + +from asyncio import ( + AbstractEventLoop, + CancelledError, + Protocol, + all_tasks, + get_event_loop, + get_running_loop, +) +from contextlib import suppress +from functools import partial +from importlib import import_module +from pathlib import Path +from socket import socket +from ssl import SSLContext +from typing import ( + TYPE_CHECKING, + Any, + Dict, + List, + Literal, + Optional, + Set, + Tuple, + Type, + Union, + cast, +) + +from sanic import reloader_helpers +from sanic.application.logo import get_logo +from sanic.application.motd import MOTD +from sanic.application.state import ApplicationServerInfo, Mode, ServerStage +from sanic.base.meta import SanicMeta +from sanic.compat import OS_IS_WINDOWS +from sanic.helpers import _default +from sanic.http.constants import HTTP +from sanic.http.tls import get_ssl_context, process_to_context +from sanic.log import Colors, deprecation, error_logger, logger +from sanic.models.handler_types import ListenerType +from sanic.server import Signal as ServerSignal +from sanic.server import try_use_uvloop +from sanic.server.async_server import AsyncioServer +from sanic.server.protocols.http_protocol import HttpProtocol +from sanic.server.protocols.websocket_protocol import WebSocketProtocol +from sanic.server.runners import serve, serve_multiple, serve_single + + +if TYPE_CHECKING: # no cov + from sanic import Sanic + from sanic.application.state import ApplicationState + from sanic.config import Config + +SANIC_PACKAGES = ("sanic-routing", "sanic-testing", "sanic-ext") +HTTPVersion = Union[HTTP, Literal[1], Literal[3]] + + +class RunnerMixin(metaclass=SanicMeta): + _app_registry: Dict[str, Sanic] + config: Config + listeners: Dict[str, List[ListenerType[Any]]] + state: ApplicationState + websocket_enabled: bool + + def make_coffee(self, *args, **kwargs): + self.state.coffee = True + self.run(*args, **kwargs) + + def run( + self, + host: Optional[str] = None, + port: Optional[int] = None, + *, + dev: bool = False, + debug: bool = False, + auto_reload: Optional[bool] = None, + version: HTTPVersion = HTTP.VERSION_1, + ssl: Union[None, SSLContext, dict, str, list, tuple] = None, + sock: Optional[socket] = None, + workers: int = 1, + protocol: Optional[Type[Protocol]] = None, + backlog: int = 100, + register_sys_signals: bool = True, + access_log: Optional[bool] = None, + unix: Optional[str] = None, + loop: AbstractEventLoop = None, + reload_dir: Optional[Union[List[str], str]] = None, + noisy_exceptions: Optional[bool] = None, + motd: bool = True, + fast: bool = False, + verbosity: int = 0, + motd_display: Optional[Dict[str, str]] = None, + auto_tls: bool = False, + ) -> None: + """ + Run the HTTP Server and listen until keyboard interrupt or term + signal. On termination, drain connections before closing. + + :param host: Address to host on + :type host: str + :param port: Port to host on + :type port: int + :param debug: Enables debug output (slows server) + :type debug: bool + :param auto_reload: Reload app whenever its source code is changed. + Enabled by default in debug mode. + :type auto_relaod: bool + :param ssl: SSLContext, or location of certificate and key + for SSL encryption of worker(s) + :type ssl: str, dict, SSLContext or list + :param sock: Socket for the server to accept connections from + :type sock: socket + :param workers: Number of processes received before it is respected + :type workers: int + :param protocol: Subclass of asyncio Protocol class + :type protocol: type[Protocol] + :param backlog: a number of unaccepted connections that the system + will allow before refusing new connections + :type backlog: int + :param register_sys_signals: Register SIG* events + :type register_sys_signals: bool + :param access_log: Enables writing access logs (slows server) + :type access_log: bool + :param unix: Unix socket to listen on instead of TCP port + :type unix: str + :param noisy_exceptions: Log exceptions that are normally considered + to be quiet/silent + :type noisy_exceptions: bool + :return: Nothing + """ + self.prepare( + host=host, + port=port, + dev=dev, + debug=debug, + auto_reload=auto_reload, + version=version, + ssl=ssl, + sock=sock, + workers=workers, + protocol=protocol, + backlog=backlog, + register_sys_signals=register_sys_signals, + access_log=access_log, + unix=unix, + loop=loop, + reload_dir=reload_dir, + noisy_exceptions=noisy_exceptions, + motd=motd, + fast=fast, + verbosity=verbosity, + motd_display=motd_display, + auto_tls=auto_tls, + ) + + self.__class__.serve(primary=self) # type: ignore + + def prepare( + self, + host: Optional[str] = None, + port: Optional[int] = None, + *, + dev: bool = False, + debug: bool = False, + auto_reload: Optional[bool] = None, + version: HTTPVersion = HTTP.VERSION_1, + ssl: Union[None, SSLContext, dict, str, list, tuple] = None, + sock: Optional[socket] = None, + workers: int = 1, + protocol: Optional[Type[Protocol]] = None, + backlog: int = 100, + register_sys_signals: bool = True, + access_log: Optional[bool] = None, + unix: Optional[str] = None, + loop: AbstractEventLoop = None, + reload_dir: Optional[Union[List[str], str]] = None, + noisy_exceptions: Optional[bool] = None, + motd: bool = True, + fast: bool = False, + verbosity: int = 0, + motd_display: Optional[Dict[str, str]] = None, + auto_tls: bool = False, + ) -> None: + if version == 3 and self.state.server_info: + raise RuntimeError( + "Serving HTTP/3 instances as a secondary server is " + "not supported. There can only be a single HTTP/3 worker " + "and it must be the first instance prepared." + ) + + if dev: + debug = True + auto_reload = True + + self.state.verbosity = verbosity + if not self.state.auto_reload: + self.state.auto_reload = bool(auto_reload) + + if fast and workers != 1: + raise RuntimeError("You cannot use both fast=True and workers=X") + + if motd_display: + self.config.MOTD_DISPLAY.update(motd_display) + + if reload_dir: + if isinstance(reload_dir, str): + reload_dir = [reload_dir] + + for directory in reload_dir: + direc = Path(directory) + if not direc.is_dir(): + logger.warning( + f"Directory {directory} could not be located" + ) + self.state.reload_dirs.add(Path(directory)) + + if loop is not None: + raise TypeError( + "loop is not a valid argument. To use an existing loop, " + "change to create_server().\nSee more: " + "https://sanic.readthedocs.io/en/latest/sanic/deploying.html" + "#asynchronous-support" + ) + + if ( + self.__class__.should_auto_reload() + and os.environ.get("SANIC_SERVER_RUNNING") != "true" + ): # no cov + return + + if sock is None: + host, port = self.get_address(host, port, version, auto_tls) + + if protocol is None: + protocol = ( + WebSocketProtocol if self.websocket_enabled else HttpProtocol + ) + + # Set explicitly passed configuration values + for attribute, value in { + "ACCESS_LOG": access_log, + "AUTO_RELOAD": auto_reload, + "MOTD": motd, + "NOISY_EXCEPTIONS": noisy_exceptions, + }.items(): + if value is not None: + setattr(self.config, attribute, value) + + if fast: + self.state.fast = True + try: + workers = len(os.sched_getaffinity(0)) + except AttributeError: # no cov + workers = os.cpu_count() or 1 + + server_settings = self._helper( + host=host, + port=port, + debug=debug, + version=version, + ssl=ssl, + sock=sock, + unix=unix, + workers=workers, + protocol=protocol, + backlog=backlog, + register_sys_signals=register_sys_signals, + auto_tls=auto_tls, + ) + self.state.server_info.append( + ApplicationServerInfo(settings=server_settings) + ) + + if self.config.USE_UVLOOP is True or ( + self.config.USE_UVLOOP is _default and not OS_IS_WINDOWS + ): + try_use_uvloop() + + async def create_server( + self, + host: Optional[str] = None, + port: Optional[int] = None, + *, + debug: bool = False, + ssl: Union[None, SSLContext, dict, str, list, tuple] = None, + sock: Optional[socket] = None, + protocol: Type[Protocol] = None, + backlog: int = 100, + access_log: Optional[bool] = None, + unix: Optional[str] = None, + return_asyncio_server: bool = False, + asyncio_server_kwargs: Dict[str, Any] = None, + noisy_exceptions: Optional[bool] = None, + ) -> Optional[AsyncioServer]: + """ + Asynchronous version of :func:`run`. + + This method will take care of the operations necessary to invoke + the *before_start* events via :func:`trigger_events` method invocation + before starting the *sanic* app in Async mode. + + .. note:: + This does not support multiprocessing and is not the preferred + way to run a :class:`Sanic` application. + + :param host: Address to host on + :type host: str + :param port: Port to host on + :type port: int + :param debug: Enables debug output (slows server) + :type debug: bool + :param ssl: SSLContext, or location of certificate and key + for SSL encryption of worker(s) + :type ssl: SSLContext or dict + :param sock: Socket for the server to accept connections from + :type sock: socket + :param protocol: Subclass of asyncio Protocol class + :type protocol: type[Protocol] + :param backlog: a number of unaccepted connections that the system + will allow before refusing new connections + :type backlog: int + :param access_log: Enables writing access logs (slows server) + :type access_log: bool + :param return_asyncio_server: flag that defines whether there's a need + to return asyncio.Server or + start it serving right away + :type return_asyncio_server: bool + :param asyncio_server_kwargs: key-value arguments for + asyncio/uvloop create_server method + :type asyncio_server_kwargs: dict + :param noisy_exceptions: Log exceptions that are normally considered + to be quiet/silent + :type noisy_exceptions: bool + :return: AsyncioServer if return_asyncio_server is true, else Nothing + """ + + if sock is None: + host, port = host, port = self.get_address(host, port) + + if protocol is None: + protocol = ( + WebSocketProtocol if self.websocket_enabled else HttpProtocol + ) + + # Set explicitly passed configuration values + for attribute, value in { + "ACCESS_LOG": access_log, + "NOISY_EXCEPTIONS": noisy_exceptions, + }.items(): + if value is not None: + setattr(self.config, attribute, value) + + server_settings = self._helper( + host=host, + port=port, + debug=debug, + ssl=ssl, + sock=sock, + unix=unix, + loop=get_event_loop(), + protocol=protocol, + backlog=backlog, + run_async=return_asyncio_server, + ) + + if self.config.USE_UVLOOP is not _default: + error_logger.warning( + "You are trying to change the uvloop configuration, but " + "this is only effective when using the run(...) method. " + "When using the create_server(...) method Sanic will use " + "the already existing loop." + ) + + main_start = server_settings.pop("main_start", None) + main_stop = server_settings.pop("main_stop", None) + if main_start or main_stop: + logger.warning( + "Listener events for the main process are not available " + "with create_server()" + ) + + return await serve( + asyncio_server_kwargs=asyncio_server_kwargs, **server_settings + ) + + def stop(self): + """ + This kills the Sanic + """ + if self.state.stage is not ServerStage.STOPPED: + self.shutdown_tasks(timeout=0) + for task in all_tasks(): + with suppress(AttributeError): + if task.get_name() == "RunServer": + task.cancel() + get_event_loop().stop() + + def _helper( + self, + host: Optional[str] = None, + port: Optional[int] = None, + debug: bool = False, + version: HTTPVersion = HTTP.VERSION_1, + ssl: Union[None, SSLContext, dict, str, list, tuple] = None, + sock: Optional[socket] = None, + unix: Optional[str] = None, + workers: int = 1, + loop: AbstractEventLoop = None, + protocol: Type[Protocol] = HttpProtocol, + backlog: int = 100, + register_sys_signals: bool = True, + run_async: bool = False, + auto_tls: bool = False, + ) -> Dict[str, Any]: + """Helper function used by `run` and `create_server`.""" + if self.config.PROXIES_COUNT and self.config.PROXIES_COUNT < 0: + raise ValueError( + "PROXIES_COUNT cannot be negative. " + "https://sanic.readthedocs.io/en/latest/sanic/config.html" + "#proxy-configuration" + ) + + if not self.state.is_debug: + self.state.mode = Mode.DEBUG if debug else Mode.PRODUCTION + + if isinstance(version, int): + version = HTTP(version) + + ssl = process_to_context(ssl) + if version is HTTP.VERSION_3 or auto_tls: + if TYPE_CHECKING: # no cov + self = cast(Sanic, self) + ssl = get_ssl_context(self, ssl) + + self.state.host = host or "" + self.state.port = port or 0 + self.state.workers = workers + self.state.ssl = ssl + self.state.unix = unix + self.state.sock = sock + + server_settings = { + "protocol": protocol, + "host": host, + "port": port, + "version": version, + "sock": sock, + "unix": unix, + "ssl": ssl, + "app": self, + "signal": ServerSignal(), + "loop": loop, + "register_sys_signals": register_sys_signals, + "backlog": backlog, + } + + self.motd(server_settings=server_settings) + + if sys.stdout.isatty() and not self.state.is_debug: + error_logger.warning( + f"{Colors.YELLOW}Sanic is running in PRODUCTION mode. " + "Consider using '--debug' or '--dev' while actively " + f"developing your application.{Colors.END}" + ) + + # Register start/stop events + for event_name, settings_name, reverse in ( + ("main_process_start", "main_start", False), + ("main_process_stop", "main_stop", True), + ): + listeners = self.listeners[event_name].copy() + if reverse: + listeners.reverse() + # Prepend sanic to the arguments when listeners are triggered + listeners = [partial(listener, self) for listener in listeners] + server_settings[settings_name] = listeners # type: ignore + + if run_async: + server_settings["run_async"] = True + + return server_settings + + def motd( + self, + serve_location: str = "", + server_settings: Optional[Dict[str, Any]] = None, + ): + if serve_location: + deprecation( + "Specifying a serve_location in the MOTD is deprecated and " + "will be removed.", + 22.9, + ) + else: + serve_location = self.get_server_location(server_settings) + if self.config.MOTD: + mode = [f"{self.state.mode},"] + if self.state.fast: + mode.append("goin' fast") + if self.state.asgi: + mode.append("ASGI") + else: + if self.state.workers == 1: + mode.append("single worker") + else: + mode.append(f"w/ {self.state.workers} workers") + + server = ", ".join( + ( + self.state.server, + server_settings["version"].display(), # type: ignore + ) + ) + + display = { + "mode": " ".join(mode), + "server": server, + "python": platform.python_version(), + "platform": platform.platform(), + } + extra = {} + if self.config.AUTO_RELOAD: + reload_display = "enabled" + if self.state.reload_dirs: + reload_display += ", ".join( + [ + "", + *( + str(path.absolute()) + for path in self.state.reload_dirs + ), + ] + ) + display["auto-reload"] = reload_display + + packages = [] + for package_name in SANIC_PACKAGES: + module_name = package_name.replace("-", "_") + try: + module = import_module(module_name) + packages.append( + f"{package_name}=={module.__version__}" # type: ignore + ) + except ImportError: + ... + + if packages: + display["packages"] = ", ".join(packages) + + if self.config.MOTD_DISPLAY: + extra.update(self.config.MOTD_DISPLAY) + + logo = ( + get_logo(coffee=self.state.coffee) + if self.config.LOGO == "" or self.config.LOGO is True + else self.config.LOGO + ) + + MOTD.output(logo, serve_location, display, extra) + + @property + def serve_location(self) -> str: + server_settings = self.state.server_info[0].settings + return self.get_server_location(server_settings) + + @staticmethod + def get_server_location( + server_settings: Optional[Dict[str, Any]] = None + ) -> str: + serve_location = "" + proto = "http" + if not server_settings: + return serve_location + + if server_settings["ssl"] is not None: + proto = "https" + if server_settings["unix"]: + serve_location = f'{server_settings["unix"]} {proto}://...' + elif server_settings["sock"]: + serve_location = ( + f'{server_settings["sock"].getsockname()} {proto}://...' + ) + elif server_settings["host"] and server_settings["port"]: + # colon(:) is legal for a host only in an ipv6 address + display_host = ( + f'[{server_settings["host"]}]' + if ":" in server_settings["host"] + else server_settings["host"] + ) + serve_location = ( + f'{proto}://{display_host}:{server_settings["port"]}' + ) + + return serve_location + + @staticmethod + def get_address( + host: Optional[str], + port: Optional[int], + version: HTTPVersion = HTTP.VERSION_1, + auto_tls: bool = False, + ) -> Tuple[str, int]: + host = host or "127.0.0.1" + port = port or (8443 if (version == 3 or auto_tls) else 8000) + return host, port + + @classmethod + def should_auto_reload(cls) -> bool: + return any(app.state.auto_reload for app in cls._app_registry.values()) + + @classmethod + def serve(cls, primary: Optional[Sanic] = None) -> None: + apps = list(cls._app_registry.values()) + + if not primary: + try: + primary = apps[0] + except IndexError: + raise RuntimeError("Did not find any applications.") + + # We want to run auto_reload if ANY of the applications have it enabled + if ( + cls.should_auto_reload() + and os.environ.get("SANIC_SERVER_RUNNING") != "true" + ): + reload_dirs: Set[Path] = primary.state.reload_dirs.union( + *(app.state.reload_dirs for app in apps) + ) + return reloader_helpers.watchdog(1.0, reload_dirs) + + # This exists primarily for unit testing + if not primary.state.server_info: # no cov + for app in apps: + app.state.server_info.clear() + return + + primary_server_info = primary.state.server_info[0] + primary.before_server_start(partial(primary._start_servers, apps=apps)) + + try: + primary_server_info.stage = ServerStage.SERVING + + if primary.state.workers > 1 and os.name != "posix": # no cov + logger.warn( + f"Multiprocessing is currently not supported on {os.name}," + " using workers=1 instead" + ) + primary.state.workers = 1 + if primary.state.workers == 1: + serve_single(primary_server_info.settings) + elif primary.state.workers == 0: + raise RuntimeError("Cannot serve with no workers") + else: + serve_multiple( + primary_server_info.settings, primary.state.workers + ) + except BaseException: + error_logger.exception( + "Experienced exception while trying to serve" + ) + raise + finally: + primary_server_info.stage = ServerStage.STOPPED + logger.info("Server Stopped") + for app in apps: + app.state.server_info.clear() + app.router.reset() + app.signal_router.reset() + + async def _start_servers( + self, + primary: Sanic, + _, + apps: List[Sanic], + ) -> None: + for app in apps: + if ( + app.name is not primary.name + and app.state.workers != primary.state.workers + and app.state.server_info + ): + message = ( + f"The primary application {repr(primary)} is running " + f"with {primary.state.workers} worker(s). All " + "application instances will run with the same number. " + f"You requested {repr(app)} to run with " + f"{app.state.workers} worker(s), which will be ignored " + "in favor of the primary application." + ) + if sys.stdout.isatty(): + message = "".join( + [ + Colors.YELLOW, + message, + Colors.END, + ] + ) + error_logger.warning(message, exc_info=True) + for server_info in app.state.server_info: + if server_info.stage is not ServerStage.SERVING: + app.state.primary = False + handlers = [ + *server_info.settings.pop("main_start", []), + *server_info.settings.pop("main_stop", []), + ] + if handlers: + error_logger.warning( + f"Sanic found {len(handlers)} listener(s) on " + "secondary applications attached to the main " + "process. These will be ignored since main " + "process listeners can only be attached to your " + "primary application: " + f"{repr(primary)}" + ) + + if not server_info.settings["loop"]: + server_info.settings["loop"] = get_running_loop() + + try: + server_info.server = await serve( + **server_info.settings, + run_async=True, + reuse_port=bool(primary.state.workers - 1), + ) + except OSError as e: # no cov + first_message = ( + "An OSError was detected on startup. " + "The encountered error was: " + ) + second_message = str(e) + if sys.stdout.isatty(): + message_parts = [ + Colors.YELLOW, + first_message, + Colors.RED, + second_message, + Colors.END, + ] + else: + message_parts = [first_message, second_message] + message = "".join(message_parts) + error_logger.warning(message, exc_info=True) + continue + primary.add_task( + self._run_server(app, server_info), name="RunServer" + ) + + async def _run_server( + self, + app: RunnerMixin, + server_info: ApplicationServerInfo, + ) -> None: + + try: + # We should never get to this point without a server + # This is primarily to keep mypy happy + if not server_info.server: # no cov + raise RuntimeError("Could not locate AsyncioServer") + if app.state.stage is ServerStage.STOPPED: + server_info.stage = ServerStage.SERVING + await server_info.server.startup() + await server_info.server.before_start() + await server_info.server.after_start() + await server_info.server.serve_forever() + except CancelledError: + # We should never get to this point without a server + # This is primarily to keep mypy happy + if not server_info.server: # no cov + raise RuntimeError("Could not locate AsyncioServer") + await server_info.server.before_stop() + await server_info.server.close() + await server_info.server.after_stop() + finally: + server_info.stage = ServerStage.STOPPED + server_info.server = None diff --git a/sanic/touchup/schemes/altsvc.py b/sanic/touchup/schemes/altsvc.py new file mode 100644 index 0000000000..fc4b90d0aa --- /dev/null +++ b/sanic/touchup/schemes/altsvc.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from ast import Assign, Constant, NodeTransformer, Subscript +from typing import TYPE_CHECKING, Any, List + +from sanic.http.constants import HTTP + +from .base import BaseScheme + + +if TYPE_CHECKING: # no cov + from sanic import Sanic + + +class AltSvcCheck(BaseScheme): + ident = "ALTSVC" + + def visitors(self) -> List[NodeTransformer]: + return [RemoveAltSvc(self.app, self.app.state.verbosity)] + + +class RemoveAltSvc(NodeTransformer): + def __init__(self, app: Sanic, verbosity: int = 0) -> None: + self._app = app + self._verbosity = verbosity + self._versions = { + info.settings["version"] for info in app.state.server_info + } + + def visit_Assign(self, node: Assign) -> Any: + if any(self._matches(target) for target in node.targets): + if self._should_remove(): + return None + assert isinstance(node.value, Constant) + node.value.value = self.value() + return node + + def _should_remove(self) -> bool: + return len(self._versions) == 1 + + @staticmethod + def _matches(node) -> bool: + return ( + isinstance(node, Subscript) + and isinstance(node.slice, Constant) + and node.slice.value == "alt-svc" + ) + + def value(self): + values = [] + for info in self._app.state.server_info: + port = info.settings["port"] + version = info.settings["version"] + if version is HTTP.VERSION_3: + values.append(f'h3=":{port}"') + return ", ".join(values) From 964e0b09f46e3b57ebdf406671daee6c02039f63 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 22 Feb 2022 09:23:59 +0200 Subject: [PATCH 15/59] Cleanup merge conflicts --- sanic/cli/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sanic/cli/app.py b/sanic/cli/app.py index 0ce93adce4..e3ddabd94a 100644 --- a/sanic/cli/app.py +++ b/sanic/cli/app.py @@ -175,7 +175,6 @@ def _build_run_kwargs(self): "unix": self.args.unix, "verbosity": self.args.verbosity or 0, "workers": self.args.workers, - "version": version, "auto_tls": self.args.auto_tls, } From d26d79c18258e9e7a76003e0d1409e42f1df7451 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Wed, 23 Feb 2022 10:01:12 +0200 Subject: [PATCH 16/59] Add TLS password to config --- sanic/config.py | 2 ++ sanic/http/http3.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/sanic/config.py b/sanic/config.py index f213f5c94d..d0245c1d98 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -41,6 +41,7 @@ "REQUEST_MAX_SIZE": 100000000, # 100 megabytes "REQUEST_TIMEOUT": 60, # 60 seconds "RESPONSE_TIMEOUT": 60, # 60 seconds + "TLS_CERT_PASSWORD": "", "USE_UVLOOP": _default, "WEBSOCKET_MAX_SIZE": 2**20, # 1 megabyte "WEBSOCKET_PING_INTERVAL": 20, @@ -87,6 +88,7 @@ class Config(dict, metaclass=DescriptorMeta): REQUEST_TIMEOUT: int RESPONSE_TIMEOUT: int SERVER_NAME: str + TLS_CERT_PASSWORD: str USE_UVLOOP: Union[Default, bool] WEBSOCKET_MAX_SIZE: int WEBSOCKET_PING_INTERVAL: int diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 3d1b2b5671..b954566f37 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -188,9 +188,10 @@ def get_config(app: Sanic, ssl: SSLContext): is_client=False, max_datagram_frame_size=65536, ) - # TODO: - # - add password kwarg, read from config.TLS_CERT_PASSWORD - config.load_cert_chain(ssl.sanic["cert"], ssl.sanic["key"]) + password = app.config.TLS_CERT_PASSWORD or None + config.load_cert_chain( + ssl.sanic["cert"], ssl.sanic["key"], password=password + ) return config From 06035c8d51e034e43460fe02d63e0c56e59f9af7 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Wed, 23 Feb 2022 14:42:11 +0200 Subject: [PATCH 17/59] Move HTTP streaming events to receiver --- sanic/http/http3.py | 164 ++++++++++++++++-------- sanic/log.py | 3 +- sanic/request.py | 14 +- sanic/server/protocols/http_protocol.py | 6 +- 4 files changed, 125 insertions(+), 62 deletions(-) diff --git a/sanic/http/http3.py b/sanic/http/http3.py index b954566f37..be639d599d 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -2,9 +2,9 @@ import asyncio -from abc import ABC +from abc import ABC, abstractmethod from ssl import SSLContext -from typing import TYPE_CHECKING, Callable, Dict, Optional, Union +from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Union from aioquic.h0.connection import H0_ALPN, H0Connection from aioquic.h3.connection import H3_ALPN, H3Connection @@ -32,12 +32,14 @@ if TYPE_CHECKING: from sanic import Sanic from sanic.request import Request - from sanic.response import BaseHTTPResponse + from sanic.response import BaseHTTPResponse, HTTPResponse from sanic.server.protocols.http_protocol import Http3Protocol # from sanic.compat import Header +from sanic.http.constants import Stage + # from sanic.application.state import Mode -from sanic.log import logger +from sanic.log import Colors, logger HttpConnection = Union[H0Connection, H3Connection] @@ -48,24 +50,95 @@ class Transport: class Receiver(ABC): - def __init__(self, transmit, protocol, request) -> None: + def __init__(self, transmit, protocol, request: Request) -> None: self.transmit = transmit self.protocol = protocol self.request = request + self.queue: asyncio.Queue[Tuple[bytes, bool]] = asyncio.Queue() + + @abstractmethod + async def run(self): + ... class HTTPReceiver(Receiver): - async def respond(self): + stage: Stage + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.request_body = None + self.stage = Stage.IDLE + self.headers_sent = False + + async def run(self): + self.stage = Stage.HANDLER + logger.info(f"Request received: {self.request}") - await self.protocol.app.handle_request(self.request) + await self.protocol.request_handler(self.request) + + self.stage = Stage.RESPONSE + + while self.stage is Stage.RESPONSE: + if not self.headers_sent: + self.send_headers() + data, end_stream = await self.queue.get() + + # Chunked + size = len(data) + if end_stream: + data = ( + b"%x\r\n%b\r\n0\r\n\r\n" % (size, data) + if size + else b"0\r\n\r\n" + ) + elif size: + data = b"%x\r\n%b\r\n" % (size, data) + + self.protocol.connection.send_data( + stream_id=self.request.stream_id, + data=data, + end_stream=end_stream, + ) + self.transmit() + + if end_stream: + self.stage = Stage.IDLE + + def send_headers(self) -> None: + response = self.request.stream.response + self.protocol.connection.send_headers( + stream_id=self.request.stream_id, + headers=[ + (b":status", str(response.status).encode()), + *( + (k.encode(), v.encode()) + for k, v in response.headers.items() + ), + ], + ) + self.headers_sent = True + + async def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: + print(f"{Colors.BLUE}[respond]: {Colors.GREEN}{response=}{Colors.END}") + self.response, response.stream = response, self + + return response + + async def send(self, data: bytes, end_stream: bool) -> None: + print( + f"{Colors.BLUE}[send]: {Colors.GREEN}{data=} {end_stream=}{Colors.END}" + ) + self.queue.put_nowait((data, end_stream)) class WebsocketReceiver(Receiver): - ... + async def run(self): + ... class WebTransportReceiver(Receiver): - ... + async def run(self): + ... class Http3: @@ -81,37 +154,48 @@ def __init__( protocol: Http3Protocol, transmit: Callable[[], None], ) -> None: - self.request_body = None - self.request: Optional[Request] = None + # self.request_body = None + # self.request: Optional[Request] = None self.protocol = protocol self.transmit = transmit self.receivers: Dict[int, Receiver] = {} def http_event_received(self, event: H3Event) -> None: - print("[http_event_received]:", event) - receiver = self.get_or_make_receiver(event) + print( + f"{Colors.BLUE}[http_event_received]: " + f"{Colors.YELLOW}{event}{Colors.END}" + ) + receiver, created_new = self.get_or_make_receiver(event) print(f"{receiver=}") - # asyncio.ensure_future(handler(request)) + if isinstance(event, HeadersReceived) and created_new: + asyncio.ensure_future(receiver.run()) + else: + print(f"{Colors.RED}DOING NOTHING{Colors.END}") - def get_or_make_receiver(self, event: H3Event) -> Receiver: + def get_or_make_receiver(self, event: H3Event) -> Tuple[Receiver, bool]: if ( isinstance(event, HeadersReceived) and event.stream_id not in self.receivers ): - self.request = self._make_request(event) - receiver = HTTPReceiver(self.transmit, self.protocol, self.request) + request = self._make_request(event) + receiver = HTTPReceiver(self.transmit, self.protocol, request) + request.stream = receiver + self.receivers[event.stream_id] = receiver - asyncio.ensure_future(receiver.respond()) + return receiver, True else: ident = getattr(event, self.HANDLER_PROPERTY_MAPPING[type(event)]) - return self.receivers[ident] + return self.receivers[ident], False + + def get_receiver_by_stream_id(self, stream_id: int) -> Receiver: + return self.receivers[stream_id] def _make_request(self, event: HeadersReceived) -> Request: - method, path, *rem = event.headers + method_header, path_header, *rem = event.headers headers = Header(((k.decode(), v.decode()) for k, v in rem)) - method = method[1].decode() - path = path[1] + method = method_header[1].decode() + path = path_header[1] scheme = headers.pop(":scheme") authority = headers.pop(":authority") print(f"{headers=}") @@ -125,44 +209,10 @@ def _make_request(self, event: HeadersReceived) -> Request: request = self.protocol.request_class( path, headers, "3", method, Transport(), self.protocol.app, b"" ) - request.stream = self + request._stream_id = event.stream_id print(f"{request=}") return request - async def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: - print(f"[respond]: {response=}") - response.headers.update({"foo": "bar"}) - self.response, response.stream = response, self - - # Need more appropriate place to send these - self.protocol.connection.send_headers( - stream_id=0, - headers=[ - (b":status", str(self.response.status).encode()), - *( - (k.encode(), v.encode()) - for k, v in self.response.headers.items() - ), - ], - ) - # TEMP - await self.drain(response) - - return response - - async def drain(self, response: BaseHTTPResponse) -> None: - await self.send(response.body, False) - - async def send(self, data: bytes, end_stream: bool) -> None: - print(f"[send]: {data=} {end_stream=}") - print(self.response.headers) - self.protocol.connection.send_data( - stream_id=0, - data=data, - end_stream=end_stream, - ) - self.transmit() - class SessionTicketStore: """ diff --git a/sanic/log.py b/sanic/log.py index 4b3b960c4d..d5f991e6c0 100644 --- a/sanic/log.py +++ b/sanic/log.py @@ -62,7 +62,8 @@ class Colors(str, Enum): # no cov BLUE = "\033[01;34m" GREEN = "\033[01;32m" YELLOW = "\033[01;33m" - RED = "\033[01;31m" + RED = "\033[01;34m" + PURPLE = "\033[01;35m" logger = logging.getLogger("sanic.root") # no cov diff --git a/sanic/request.py b/sanic/request.py index a1ce91c3e3..ef91135649 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -13,8 +13,9 @@ Union, ) -from sanic_routing.route import Route # type: ignore +from sanic_routing.route import Route +from sanic.http.http3 import HTTPReceiver # type: ignore from sanic.models.http_types import Credentials @@ -91,6 +92,7 @@ class Request: "_protocol", "_remote_addr", "_socket", + "_stream_id", "_match_info", "_name", "app", @@ -127,6 +129,7 @@ def __init__( transport: TransportProtocol, app: Sanic, head: bytes = b"", + stream_id: int = 0, ): self.raw_url = url_bytes @@ -134,6 +137,7 @@ def __init__( self._parsed_url = parse_url(url_bytes) self._id: Optional[Union[uuid.UUID, str, int]] = None self._name: Optional[str] = None + self._stream_id = stream_id self.app = app self.headers = Header(headers) @@ -162,7 +166,9 @@ def __init__( self.request_middleware_started = False self._cookies: Optional[Dict[str, str]] = None self._match_info: Dict[str, Any] = {} - self.stream: Optional[Http] = None + # TODO: + # - Create an ABC (called Stream) for Http and HTTPReceiver to subclass + self.stream: Optional[Union[Http, HTTPReceiver]] = None self.route: Optional[Route] = None self._protocol = None self.responded: bool = False @@ -175,6 +181,10 @@ def __repr__(self): def generate_id(*_): return uuid.uuid4() + @property + def stream_id(self): + return self._stream_id + def reset_response(self): try: if ( diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index c215208acd..e522f3dcfb 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -20,7 +20,7 @@ from sanic.exceptions import RequestTimeout, ServiceUnavailable from sanic.http import Http, Stage -from sanic.log import error_logger, logger +from sanic.log import Colors, error_logger, logger from sanic.models.server_types import ConnInfo from sanic.request import Request from sanic.server.protocols.base_protocol import SanicProtocol @@ -265,7 +265,9 @@ def __init__(self, *args, app: Sanic, **kwargs) -> None: self._connection: Optional[H3Connection] = None def quic_event_received(self, event: QuicEvent) -> None: - print("[quic_event_received]:", event) + print( + f"{Colors.BLUE}[quic_event_received]: {Colors.PURPLE}{event}{Colors.END}" + ) if isinstance(event, ProtocolNegotiated): self._setup_connection(transmit=self.transmit) if event.alpn_protocol in H3_ALPN: From 2262c692ba412f784ef75d3a998bfaa68c56737c Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Thu, 24 Feb 2022 09:59:47 +0200 Subject: [PATCH 18/59] Streaming send --- sanic/app.py | 2 + sanic/http/http3.py | 96 +++++++++++++++++++++++++++++---------------- sanic/log.py | 4 +- sanic/response.py | 2 +- 4 files changed, 67 insertions(+), 37 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index 625962a7ad..05b994430d 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -920,6 +920,7 @@ async def handle_request(self, request: Request): # no cov if isawaitable(response): response = await response + print(f"{response=}", request.responded) if request.responded: if response is not None: error_logger.error( @@ -945,6 +946,7 @@ async def handle_request(self, request: Request): # no cov "response": response, }, ) + ... await response.send(end_stream=True) elif isinstance(response, ResponseStream): resp = await response(request) diff --git a/sanic/http/http3.py b/sanic/http/http3.py index be639d599d..34895c3e4e 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -4,6 +4,7 @@ from abc import ABC, abstractmethod from ssl import SSLContext +from sys import exc_info from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Union from aioquic.h0.connection import H0_ALPN, H0Connection @@ -39,7 +40,7 @@ from sanic.http.constants import Stage # from sanic.application.state import Mode -from sanic.log import Colors, logger +from sanic.log import Colors, error_logger, logger HttpConnection = Union[H0Connection, H3Connection] @@ -54,7 +55,6 @@ def __init__(self, transmit, protocol, request: Request) -> None: self.transmit = transmit self.protocol = protocol self.request = request - self.queue: asyncio.Queue[Tuple[bytes, bool]] = asyncio.Queue() @abstractmethod async def run(self): @@ -69,42 +69,23 @@ def __init__(self, *args, **kwargs) -> None: self.request_body = None self.stage = Stage.IDLE self.headers_sent = False + self.response = None async def run(self): self.stage = Stage.HANDLER - logger.info(f"Request received: {self.request}") - await self.protocol.request_handler(self.request) - - self.stage = Stage.RESPONSE - - while self.stage is Stage.RESPONSE: - if not self.headers_sent: - self.send_headers() - data, end_stream = await self.queue.get() - - # Chunked - size = len(data) - if end_stream: - data = ( - b"%x\r\n%b\r\n0\r\n\r\n" % (size, data) - if size - else b"0\r\n\r\n" - ) - elif size: - data = b"%x\r\n%b\r\n" % (size, data) - - self.protocol.connection.send_data( - stream_id=self.request.stream_id, - data=data, - end_stream=end_stream, - ) - self.transmit() - - if end_stream: - self.stage = Stage.IDLE + try: + logger.info(f">>> Request received: {self.request}") + await self.protocol.request_handler(self.request) + except Exception: + # TODO: + # - Handler errors + raise + else: + self.stage = Stage.RESPONSE def send_headers(self) -> None: + print(f"{Colors.RED}SEND HEADERS{Colors.END}") response = self.request.stream.response self.protocol.connection.send_headers( stream_id=self.request.stream_id, @@ -117,9 +98,22 @@ def send_headers(self) -> None: ], ) self.headers_sent = True + self.stage = Stage.RESPONSE + + if self.response.body: + self._send(self.response.body, False) - async def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: + def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: print(f"{Colors.BLUE}[respond]: {Colors.GREEN}{response=}{Colors.END}") + + if self.stage is not Stage.HANDLER: + self.stage = Stage.FAILED + raise RuntimeError("Response already started") + + # Disconnect any earlier but unused response object + if self.response is not None: + self.response.stream = None + self.response, response.stream = response, self return response @@ -128,7 +122,37 @@ async def send(self, data: bytes, end_stream: bool) -> None: print( f"{Colors.BLUE}[send]: {Colors.GREEN}{data=} {end_stream=}{Colors.END}" ) - self.queue.put_nowait((data, end_stream)) + self._send(data, end_stream) + + def _send(self, data: bytes, end_stream: bool) -> None: + if not self.headers_sent: + self.send_headers() + if self.stage is not Stage.RESPONSE: + raise Exception(f"not ready to send: {self.stage}") + + print(f"{data=}") + + # Chunked + size = len(data) + if end_stream: + data = ( + b"%x\r\n%b\r\n0\r\n\r\n" % (size, data) + if size + else b"0\r\n\r\n" + ) + elif size: + data = b"%x\r\n%b\r\n" % (size, data) + + print(f"{Colors.RED}TRANSMITTING{Colors.END}") + self.protocol.connection.send_data( + stream_id=self.request.stream_id, + data=data, + end_stream=end_stream, + ) + self.transmit() + + if end_stream: + self.stage = Stage.IDLE class WebsocketReceiver(Receiver): @@ -170,6 +194,10 @@ def http_event_received(self, event: H3Event) -> None: if isinstance(event, HeadersReceived) and created_new: asyncio.ensure_future(receiver.run()) + elif isinstance(event, DataReceived): + # event.stream_ended + # TEMP + receiver.request.body = event.data else: print(f"{Colors.RED}DOING NOTHING{Colors.END}") diff --git a/sanic/log.py b/sanic/log.py index d5f991e6c0..97308ecedf 100644 --- a/sanic/log.py +++ b/sanic/log.py @@ -59,10 +59,10 @@ class Colors(str, Enum): # no cov END = "\033[0m" - BLUE = "\033[01;34m" + RED = "\033[01;31m" GREEN = "\033[01;32m" YELLOW = "\033[01;33m" - RED = "\033[01;34m" + BLUE = "\033[01;34m" PURPLE = "\033[01;35m" diff --git a/sanic/response.py b/sanic/response.py index cd3bd952b5..cddd7b64d5 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -122,7 +122,7 @@ async def send( :param data: str or bytes to be written :param end_stream: whether to close the stream after this block """ - print(f">>> BaseHTTPResponse: {data=} {end_stream=} {self.body=}") + print(f">>> BaseHTTPResponse: {data=} {end_stream=}") if data is None and end_stream is None: end_stream = True if self.stream is None: From 517d7d53abdf7895f9f6438b451e86541edfccda Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Thu, 24 Feb 2022 11:39:46 +0200 Subject: [PATCH 19/59] Setup response headers --- sanic/http/http3.py | 100 ++++++++++++++++++++++++++++---------------- sanic/request.py | 2 +- sanic/response.py | 9 ++-- 3 files changed, 72 insertions(+), 39 deletions(-) diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 34895c3e4e..254d7bc45a 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -4,8 +4,7 @@ from abc import ABC, abstractmethod from ssl import SSLContext -from sys import exc_info -from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union from aioquic.h0.connection import H0_ALPN, H0Connection from aioquic.h3.connection import H3_ALPN, H3Connection @@ -17,30 +16,22 @@ WebTransportStreamDataReceived, ) from aioquic.quic.configuration import QuicConfiguration - -# from aioquic.quic.events import ( -# DatagramFrameReceived, -# ProtocolNegotiated, -# QuicEvent, -# ) from aioquic.tls import SessionTicket from sanic.compat import Header from sanic.exceptions import SanicException +from sanic.helpers import has_message_body from sanic.http.tls import CertSimple if TYPE_CHECKING: from sanic import Sanic from sanic.request import Request - from sanic.response import BaseHTTPResponse, HTTPResponse + from sanic.response import BaseHTTPResponse from sanic.server.protocols.http_protocol import Http3Protocol -# from sanic.compat import Header from sanic.http.constants import Stage - -# from sanic.application.state import Mode -from sanic.log import Colors, error_logger, logger +from sanic.log import Colors, logger HttpConnection = Union[H0Connection, H3Connection] @@ -51,6 +42,8 @@ class Transport: class Receiver(ABC): + future: asyncio.Future + def __init__(self, transmit, protocol, request: Request) -> None: self.transmit = transmit self.protocol = protocol @@ -69,10 +62,11 @@ def __init__(self, *args, **kwargs) -> None: self.request_body = None self.stage = Stage.IDLE self.headers_sent = False - self.response = None + self.response: Optional[BaseHTTPResponse] = None async def run(self): self.stage = Stage.HANDLER + self.head_only = self.request.method.upper() == "HEAD" try: logger.info(f">>> Request received: {self.request}") @@ -82,26 +76,57 @@ async def run(self): # - Handler errors raise else: - self.stage = Stage.RESPONSE + self.stage = Stage.IDLE + + def _prepare_headers( + self, response: BaseHTTPResponse + ) -> List[Tuple[bytes, bytes]]: + size = len(response.body) if response.body else 0 + headers = response.headers + status = response.status + + if not has_message_body(status) and ( + size + or "content-length" in headers + or "transfer-encoding" in headers + ): + headers.pop("content-length", None) + headers.pop("transfer-encoding", None) + logger.warning( + f"Message body set in response on {self.request.path}. " + f"A {status} response may only have headers, no body." + ) + elif "content-length" not in headers: + if size: + headers["content-length"] = size + else: + headers["transfer-encoding"] = "chunked" + + headers = [ + (b":status", str(response.status).encode()), + *response.processed_headers, + ] + return headers def send_headers(self) -> None: print(f"{Colors.RED}SEND HEADERS{Colors.END}") - response = self.request.stream.response + if not self.response: + raise Exception("no response") + + response = self.response + headers = self._prepare_headers(response) + self.protocol.connection.send_headers( stream_id=self.request.stream_id, - headers=[ - (b":status", str(response.status).encode()), - *( - (k.encode(), v.encode()) - for k, v in response.headers.items() - ), - ], + headers=headers, ) self.headers_sent = True self.stage = Stage.RESPONSE - if self.response.body: + if self.response.body and not self.head_only: self._send(self.response.body, False) + elif self.head_only: + self.future.cancel() def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: print(f"{Colors.BLUE}[respond]: {Colors.GREEN}{response=}{Colors.END}") @@ -120,7 +145,8 @@ def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: async def send(self, data: bytes, end_stream: bool) -> None: print( - f"{Colors.BLUE}[send]: {Colors.GREEN}{data=} {end_stream=}{Colors.END}" + f"{Colors.BLUE}[send]: {Colors.GREEN}{data=} " + f"{end_stream=}{Colors.END}" ) self._send(data, end_stream) @@ -133,15 +159,19 @@ def _send(self, data: bytes, end_stream: bool) -> None: print(f"{data=}") # Chunked - size = len(data) - if end_stream: - data = ( - b"%x\r\n%b\r\n0\r\n\r\n" % (size, data) - if size - else b"0\r\n\r\n" - ) - elif size: - data = b"%x\r\n%b\r\n" % (size, data) + if ( + self.response + and self.response.headers.get("transfer-encoding") == "chunked" + ): + size = len(data) + if end_stream: + data = ( + b"%x\r\n%b\r\n0\r\n\r\n" % (size, data) + if size + else b"0\r\n\r\n" + ) + elif size: + data = b"%x\r\n%b\r\n" % (size, data) print(f"{Colors.RED}TRANSMITTING{Colors.END}") self.protocol.connection.send_data( @@ -193,7 +223,7 @@ def http_event_received(self, event: H3Event) -> None: print(f"{receiver=}") if isinstance(event, HeadersReceived) and created_new: - asyncio.ensure_future(receiver.run()) + receiver.future = asyncio.ensure_future(receiver.run()) elif isinstance(event, DataReceived): # event.stream_ended # TEMP diff --git a/sanic/request.py b/sanic/request.py index ef91135649..d565b9113f 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -226,7 +226,7 @@ async def respond( response = self.stream.respond(response) if isawaitable(response): - response = await response + response = await response # type: ignore # Run response middleware try: response = await self.app._run_response_middleware( diff --git a/sanic/response.py b/sanic/response.py index cddd7b64d5..2bb6397d93 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -25,12 +25,12 @@ from sanic.exceptions import SanicException, ServerError from sanic.helpers import has_message_body, remove_entity_headers from sanic.http import Http -from sanic.http.http3 import Http3 from sanic.models.protocol_types import HTMLProtocol, Range if TYPE_CHECKING: from sanic.asgi import ASGIApp + from sanic.http.http3 import HTTPReceiver from sanic.request import Request else: Request = TypeVar("Request") @@ -57,7 +57,7 @@ def __init__(self): self.asgi: bool = False self.body: Optional[bytes] = None self.content_type: Optional[str] = None - self.stream: Optional[Union[Http, ASGIApp, Http3]] = None + self.stream: Optional[Union[Http, ASGIApp, HTTPReceiver]] = None self.status: int = None self.headers = Header({}) self._cookies: Optional[CookieJar] = None @@ -141,7 +141,10 @@ async def send( if hasattr(data, "encode") else data or b"" ) - await self.stream.send(data, end_stream=end_stream) + await self.stream.send( + data, # type: ignore + end_stream=end_stream or False, + ) class HTTPResponse(BaseHTTPResponse): From 35bbdfea6c711e298c056fbc3832149b47d648cf Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Wed, 11 May 2022 08:55:19 +0300 Subject: [PATCH 20/59] Logging --- sanic/http/http3.py | 4 +++- sanic/log.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 254d7bc45a..9e36cc481c 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -109,7 +109,9 @@ def _prepare_headers( return headers def send_headers(self) -> None: - print(f"{Colors.RED}SEND HEADERS{Colors.END}") + logger.debug( + f"{Colors.RED}SEND HEADERS{Colors.END}", extra={"verbosity": 2} + ) if not self.response: raise Exception("no response") diff --git a/sanic/log.py b/sanic/log.py index 97308ecedf..e624c88234 100644 --- a/sanic/log.py +++ b/sanic/log.py @@ -66,17 +66,30 @@ class Colors(str, Enum): # no cov PURPLE = "\033[01;35m" +class VerbosityFilter(logging.Filter): + verbosity: int = 0 + + def filter(self, record: logging.LogRecord) -> bool: + verbosity = getattr(record, "verbosity", 0) + return verbosity <= self.verbosity + + +_verbosity_filter = VerbosityFilter() + logger = logging.getLogger("sanic.root") # no cov +logger.addFilter(_verbosity_filter) """ General Sanic logger """ error_logger = logging.getLogger("sanic.error") # no cov +error_logger.addFilter(_verbosity_filter) """ Logger used by Sanic for error logging """ access_logger = logging.getLogger("sanic.access") # no cov +access_logger.addFilter(_verbosity_filter) """ Logger used by Sanic for access logging """ From 199d6ea99a30dffadaac58f77cc22d57e87c943c Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Wed, 11 May 2022 09:23:04 +0300 Subject: [PATCH 21/59] Move verbosity filtering to logger --- sanic/app.py | 5 +++-- sanic/application/state.py | 5 ++++- sanic/asgi.py | 43 ++++++++++++++++++------------------ sanic/log.py | 18 +++++++++++++-- sanic/touchup/schemes/ode.py | 13 +++++------ tests/test_logging.py | 39 ++++++++++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 33 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index 4dbb1b2e5d..b87d1df6c7 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -1573,8 +1573,9 @@ async def _server_event( "shutdown", ): raise SanicException(f"Invalid server event: {event}") - if self.state.verbosity >= 1: - logger.debug(f"Triggering server events: {event}") + logger.debug( + f"Triggering server events: {event}", extra={"verbosity": 1} + ) reverse = concern == "shutdown" if loop is None: loop = self.loop diff --git a/sanic/application/state.py b/sanic/application/state.py index 724ddcb5a8..5975c2a6f2 100644 --- a/sanic/application/state.py +++ b/sanic/application/state.py @@ -9,7 +9,7 @@ from ssl import SSLContext from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union -from sanic.log import logger +from sanic.log import VerbosityFilter, logger from sanic.server.async_server import AsyncioServer @@ -91,6 +91,9 @@ def set_mode(self, value: Union[str, Mode]): if getattr(self.app, "configure_logging", False) and self.app.debug: logger.setLevel(logging.DEBUG) + def set_verbosity(self, value: int): + VerbosityFilter.verbosity = value + @property def is_debug(self): return self.mode is Mode.DEBUG diff --git a/sanic/asgi.py b/sanic/asgi.py index 2614016866..560abc66cc 100644 --- a/sanic/asgi.py +++ b/sanic/asgi.py @@ -25,27 +25,28 @@ class Lifespan: def __init__(self, asgi_app: ASGIApp) -> None: self.asgi_app = asgi_app - if self.asgi_app.sanic_app.state.verbosity > 0: - if ( - "server.init.before" - in self.asgi_app.sanic_app.signal_router.name_index - ): - logger.debug( - 'You have set a listener for "before_server_start" ' - "in ASGI mode. " - "It will be executed as early as possible, but not before " - "the ASGI server is started." - ) - if ( - "server.shutdown.after" - in self.asgi_app.sanic_app.signal_router.name_index - ): - logger.debug( - 'You have set a listener for "after_server_stop" ' - "in ASGI mode. " - "It will be executed as late as possible, but not after " - "the ASGI server is stopped." - ) + if ( + "server.init.before" + in self.asgi_app.sanic_app.signal_router.name_index + ): + logger.debug( + 'You have set a listener for "before_server_start" ' + "in ASGI mode. " + "It will be executed as early as possible, but not before " + "the ASGI server is started.", + extra={"verbosity": 1}, + ) + if ( + "server.shutdown.after" + in self.asgi_app.sanic_app.signal_router.name_index + ): + logger.debug( + 'You have set a listener for "after_server_stop" ' + "in ASGI mode. " + "It will be executed as late as possible, but not after " + "the ASGI server is stopped.", + extra={"verbosity": 1}, + ) async def startup(self) -> None: """ diff --git a/sanic/log.py b/sanic/log.py index 4b3b960c4d..e624c88234 100644 --- a/sanic/log.py +++ b/sanic/log.py @@ -59,23 +59,37 @@ class Colors(str, Enum): # no cov END = "\033[0m" - BLUE = "\033[01;34m" + RED = "\033[01;31m" GREEN = "\033[01;32m" YELLOW = "\033[01;33m" - RED = "\033[01;31m" + BLUE = "\033[01;34m" + PURPLE = "\033[01;35m" + + +class VerbosityFilter(logging.Filter): + verbosity: int = 0 + + def filter(self, record: logging.LogRecord) -> bool: + verbosity = getattr(record, "verbosity", 0) + return verbosity <= self.verbosity + +_verbosity_filter = VerbosityFilter() logger = logging.getLogger("sanic.root") # no cov +logger.addFilter(_verbosity_filter) """ General Sanic logger """ error_logger = logging.getLogger("sanic.error") # no cov +error_logger.addFilter(_verbosity_filter) """ Logger used by Sanic for error logging """ access_logger = logging.getLogger("sanic.access") # no cov +access_logger.addFilter(_verbosity_filter) """ Logger used by Sanic for access logging """ diff --git a/sanic/touchup/schemes/ode.py b/sanic/touchup/schemes/ode.py index 7c6ed3d7b8..6303ed17cf 100644 --- a/sanic/touchup/schemes/ode.py +++ b/sanic/touchup/schemes/ode.py @@ -24,9 +24,7 @@ def run(self, method, module_globals): raw_source = getsource(method) src = dedent(raw_source) tree = parse(src) - node = RemoveDispatch( - self._registered_events, self.app.state.verbosity - ).visit(tree) + node = RemoveDispatch(self._registered_events).visit(tree) compiled_src = compile(node, method.__name__, "exec") exec_locals: Dict[str, Any] = {} exec(compiled_src, module_globals, exec_locals) # nosec @@ -64,9 +62,8 @@ async def noop(**_): # no cov class RemoveDispatch(NodeTransformer): - def __init__(self, registered_events, verbosity: int = 0) -> None: + def __init__(self, registered_events) -> None: self._registered_events = registered_events - self._verbosity = verbosity def visit_Expr(self, node: Expr) -> Any: call = node.value @@ -83,8 +80,10 @@ def visit_Expr(self, node: Expr) -> Any: if hasattr(event, "s"): event_name = getattr(event, "value", event.s) if self._not_registered(event_name): - if self._verbosity >= 2: - logger.debug(f"Disabling event: {event_name}") + logger.debug( + f"Disabling event: {event_name}", + extra={"verbosity": 2}, + ) return None return node diff --git a/tests/test_logging.py b/tests/test_logging.py index c475b00bc2..274b407c54 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -209,3 +209,42 @@ async def handler(request): "request": f"GET {request.scheme}://{request.host}/", }, ) + + +@pytest.mark.parametrize( + "app_verbosity,log_verbosity,exists", + ( + (0, 0, True), + (0, 1, False), + (0, 2, False), + (1, 0, True), + (1, 1, True), + (1, 2, False), + (2, 0, True), + (2, 1, True), + (2, 2, True), + ), +) +def test_verbosity(app, caplog, app_verbosity, log_verbosity, exists): + rand_string = str(uuid.uuid4()) + + @app.get("/") + def log_info(request): + logger.info("DEFAULT") + logger.info(rand_string, extra={"verbosity": log_verbosity}) + return text("hello") + + with caplog.at_level(logging.INFO): + _ = app.test_client.get( + "/", server_kwargs={"verbosity": app_verbosity} + ) + + record = ("sanic.root", logging.INFO, rand_string) + + if exists: + assert record in caplog.record_tuples + else: + assert record not in caplog.record_tuples + + if app_verbosity == 0: + assert ("sanic.root", logging.INFO, "DEFAULT") in caplog.record_tuples From e98a631a20163a945a30687e8ac8642f7d4775a8 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Wed, 11 May 2022 09:49:04 +0300 Subject: [PATCH 22/59] Fix verbosity test on ASGI --- tests/test_asgi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 3687f5766f..58d1773c14 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -10,6 +10,7 @@ from sanic.application.state import Mode from sanic.asgi import MockTransport from sanic.exceptions import Forbidden, InvalidUsage, ServiceUnavailable +from sanic.log import VerbosityFilter from sanic.request import Request from sanic.response import json, text from sanic.server.websockets.connection import WebSocketConnection @@ -221,6 +222,7 @@ def install_signal_handlers(self): assert after_server_stop app.state.mode = Mode.DEBUG + app.state.verbosity = 0 with caplog.at_level(logging.DEBUG): server.run() From 4d2afeda7733f44a712e37d5c2ffd1c51ebc58a7 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Wed, 11 May 2022 09:53:39 +0300 Subject: [PATCH 23/59] Add Sanic color --- sanic/log.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sanic/log.py b/sanic/log.py index e624c88234..5911e8329d 100644 --- a/sanic/log.py +++ b/sanic/log.py @@ -59,11 +59,12 @@ class Colors(str, Enum): # no cov END = "\033[0m" - RED = "\033[01;31m" - GREEN = "\033[01;32m" - YELLOW = "\033[01;33m" BLUE = "\033[01;34m" + GREEN = "\033[01;32m" PURPLE = "\033[01;35m" + RED = "\033[01;31m" + SANIC = "\033[38;2;255;13;104m" + YELLOW = "\033[01;33m" class VerbosityFilter(logging.Filter): From 425182cb93892bc6535673f8741c9956bf6476b0 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Thu, 12 May 2022 09:42:58 +0300 Subject: [PATCH 24/59] WIP --- sanic/app.py | 1 - sanic/http/http3.py | 130 +++++++++++++++++------- sanic/log.py | 1 + sanic/request.py | 31 +++--- sanic/response.py | 5 +- sanic/server/protocols/http_protocol.py | 9 +- sanic/server/runners.py | 4 +- 7 files changed, 123 insertions(+), 58 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index de1c648d79..309799bf46 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -924,7 +924,6 @@ async def handle_request(self, request: Request): # no cov if isawaitable(response): response = await response - print(f"{response=}", request.responded) if request.responded: if response is not None: error_logger.error( diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 9e36cc481c..95b8b2af9d 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -4,7 +4,16 @@ from abc import ABC, abstractmethod from ssl import SSLContext -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + List, + Optional, + Tuple, + Union, + cast, +) from aioquic.h0.connection import H0_ALPN, H0Connection from aioquic.h3.connection import H3_ALPN, H3Connection @@ -19,7 +28,7 @@ from aioquic.tls import SessionTicket from sanic.compat import Header -from sanic.exceptions import SanicException +from sanic.exceptions import PayloadTooLarge, SanicException from sanic.helpers import has_message_body from sanic.http.tls import CertSimple @@ -63,20 +72,56 @@ def __init__(self, *args, **kwargs) -> None: self.stage = Stage.IDLE self.headers_sent = False self.response: Optional[BaseHTTPResponse] = None + self.request_max_size = self.protocol.request_max_size + self.request_bytes = 0 - async def run(self): + async def run(self, exception: Optional[Exception] = None): self.stage = Stage.HANDLER self.head_only = self.request.method.upper() == "HEAD" - try: - logger.info(f">>> Request received: {self.request}") - await self.protocol.request_handler(self.request) - except Exception: - # TODO: - # - Handler errors - raise + if exception: + logger.info( + f"{Colors.BLUE}[exception]: " + f"{Colors.RED}{exception}{Colors.END}", + exc_info=True, + extra={"verbosity": 1}, + ) + await self.error_response(exception) else: - self.stage = Stage.IDLE + try: + logger.info( + f"{Colors.BLUE}[request]:{Colors.END} {self.request}", + extra={"verbosity": 1}, + ) + await self.protocol.request_handler(self.request) + except Exception as e: + # This should largely be handled within the request handler. + # But, just in case... + await self.run(e) + self.stage = Stage.IDLE + + async def error_response(self, exception: Exception) -> None: + """ + Handle response when exception encountered + """ + # Disconnect after an error if in any other state than handler + # if self.stage is not Stage.HANDLER: + # self.keep_alive = False + + # TODO: + # - Do we need this? + # Request failure? Respond but then disconnect + if self.stage is Stage.REQUEST: + self.stage = Stage.HANDLER + + # From request and handler states we can respond, otherwise be silent + if self.stage is Stage.HANDLER: + app = self.protocol.app + + # if self.request is None: + # self.create_empty_request() + + await app.handle_exception(self.request, exception) def _prepare_headers( self, response: BaseHTTPResponse @@ -110,7 +155,8 @@ def _prepare_headers( def send_headers(self) -> None: logger.debug( - f"{Colors.RED}SEND HEADERS{Colors.END}", extra={"verbosity": 2} + f"{Colors.BLUE}[send]: {Colors.GREEN}HEADERS{Colors.END}", + extra={"verbosity": 2}, ) if not self.response: raise Exception("no response") @@ -131,7 +177,10 @@ def send_headers(self) -> None: self.future.cancel() def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: - print(f"{Colors.BLUE}[respond]: {Colors.GREEN}{response=}{Colors.END}") + logger.debug( + f"{Colors.BLUE}[respond]:{Colors.END} {response}", + extra={"verbosity": 2}, + ) if self.stage is not Stage.HANDLER: self.stage = Stage.FAILED @@ -145,10 +194,18 @@ def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: return response + def receive_body(self, data: bytes) -> None: + self.request_bytes += len(data) + if self.request_bytes > self.request_max_size: + raise PayloadTooLarge("Request body exceeds the size limit") + + self.request.body = data + async def send(self, data: bytes, end_stream: bool) -> None: - print( + logger.debug( f"{Colors.BLUE}[send]: {Colors.GREEN}{data=} " - f"{end_stream=}{Colors.END}" + f"{end_stream=}{Colors.END}", + extra={"verbosity": 2}, ) self._send(data, end_stream) @@ -158,8 +215,6 @@ def _send(self, data: bytes, end_stream: bool) -> None: if self.stage is not Stage.RESPONSE: raise Exception(f"not ready to send: {self.stage}") - print(f"{data=}") - # Chunked if ( self.response @@ -175,7 +230,10 @@ def _send(self, data: bytes, end_stream: bool) -> None: elif size: data = b"%x\r\n%b\r\n" % (size, data) - print(f"{Colors.RED}TRANSMITTING{Colors.END}") + logger.debug( + f"{Colors.BLUE}[transmitting]{Colors.END}", + extra={"verbosity": 2}, + ) self.protocol.connection.send_data( stream_id=self.request.stream_id, data=data, @@ -210,28 +268,32 @@ def __init__( protocol: Http3Protocol, transmit: Callable[[], None], ) -> None: - # self.request_body = None - # self.request: Optional[Request] = None self.protocol = protocol self.transmit = transmit self.receivers: Dict[int, Receiver] = {} def http_event_received(self, event: H3Event) -> None: - print( + logger.debug( f"{Colors.BLUE}[http_event_received]: " - f"{Colors.YELLOW}{event}{Colors.END}" + f"{Colors.YELLOW}{event}{Colors.END}", + extra={"verbosity": 2}, ) receiver, created_new = self.get_or_make_receiver(event) - print(f"{receiver=}") + receiver = cast(HTTPReceiver, receiver) if isinstance(event, HeadersReceived) and created_new: receiver.future = asyncio.ensure_future(receiver.run()) elif isinstance(event, DataReceived): - # event.stream_ended - # TEMP - receiver.request.body = event.data + try: + receiver.receive_body(event.data) + except Exception as e: + receiver.future.cancel() + receiver.future = asyncio.ensure_future(receiver.run(e)) else: - print(f"{Colors.RED}DOING NOTHING{Colors.END}") + logger.debug( + f"{Colors.RED}DOING NOTHING{Colors.END}", + extra={"verbosity": 2}, + ) def get_or_make_receiver(self, event: H3Event) -> Tuple[Receiver, bool]: if ( @@ -257,12 +319,9 @@ def _make_request(self, event: HeadersReceived) -> Request: method = method_header[1].decode() path = path_header[1] scheme = headers.pop(":scheme") + authority = headers.pop(":authority") - print(f"{headers=}") - print(f"{method=}") - print(f"{path=}") - print(f"{scheme=}") - print(f"{authority=}") + if authority: headers["host"] = authority @@ -270,7 +329,8 @@ def _make_request(self, event: HeadersReceived) -> Request: path, headers, "3", method, Transport(), self.protocol.app, b"" ) request._stream_id = event.stream_id - print(f"{request=}") + request._scheme = scheme + return request @@ -304,7 +364,3 @@ def get_config(app: Sanic, ssl: SSLContext): ) return config - - -def get_ticket_store(): - return SessionTicketStore() diff --git a/sanic/log.py b/sanic/log.py index 8920b5bcd1..5911e8329d 100644 --- a/sanic/log.py +++ b/sanic/log.py @@ -59,6 +59,7 @@ class Colors(str, Enum): # no cov END = "\033[0m" + BLUE = "\033[01;34m" GREEN = "\033[01;32m" PURPLE = "\033[01;35m" RED = "\033[01;31m" diff --git a/sanic/request.py b/sanic/request.py index 209fd6b05e..cf047adddf 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -93,6 +93,7 @@ class Request: "_port", "_protocol", "_remote_addr", + "_scheme", "_socket", "_stream_id", "_match_info", @@ -725,23 +726,25 @@ def scheme(self) -> str: :return: http|https|ws|wss or arbitrary value given by the headers. :rtype: str """ - if "//" in self.app.config.get("SERVER_NAME", ""): - return self.app.config.SERVER_NAME.split("//")[0] - if "proto" in self.forwarded: - return str(self.forwarded["proto"]) + if not hasattr(self, "_scheme"): + if "//" in self.app.config.get("SERVER_NAME", ""): + return self.app.config.SERVER_NAME.split("//")[0] + if "proto" in self.forwarded: + return str(self.forwarded["proto"]) - if ( - self.app.websocket_enabled - and self.headers.getone("upgrade", "").lower() == "websocket" - ): - scheme = "ws" - else: - scheme = "http" + if ( + self.app.websocket_enabled + and self.headers.getone("upgrade", "").lower() == "websocket" + ): + scheme = "ws" + else: + scheme = "http" - if self.transport.get_extra_info("sslcontext"): - scheme += "s" + if self.transport.get_extra_info("sslcontext"): + scheme += "s" + self._scheme = scheme - return scheme + return self._scheme @property def host(self) -> str: diff --git a/sanic/response.py b/sanic/response.py index 5b89133064..acf7e79a17 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -72,6 +72,10 @@ def __init__(self): self.headers = Header({}) self._cookies: Optional[CookieJar] = None + def __repr__(self): + class_name = self.__class__.__name__ + return f"<{class_name}: {self.status} {self.content_type}>" + def _encode_body(self, data: Optional[AnyStr]): if data is None: return b"" @@ -132,7 +136,6 @@ async def send( :param data: str or bytes to be written :param end_stream: whether to close the stream after this block """ - print(f">>> BaseHTTPResponse: {data=} {end_stream=}") if data is None and end_stream is None: end_stream = True if self.stream is None: diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index e522f3dcfb..0fb78bdca6 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -15,7 +15,8 @@ from time import monotonic as current_time from aioquic.asyncio import QuicConnectionProtocol -from aioquic.h3.events import H3Event + +# from aioquic.h3.events import H3Event from aioquic.quic.events import ProtocolNegotiated, QuicEvent from sanic.exceptions import RequestTimeout, ServiceUnavailable @@ -265,8 +266,10 @@ def __init__(self, *args, app: Sanic, **kwargs) -> None: self._connection: Optional[H3Connection] = None def quic_event_received(self, event: QuicEvent) -> None: - print( - f"{Colors.BLUE}[quic_event_received]: {Colors.PURPLE}{event}{Colors.END}" + logger.debug( + f"{Colors.BLUE}[quic_event_received]: " + f"{Colors.PURPLE}{event}{Colors.END}", + extra={"verbosity": 2}, ) if isinstance(event, ProtocolNegotiated): self._setup_connection(transmit=self.transmit) diff --git a/sanic/server/runners.py b/sanic/server/runners.py index 86779472bf..86d2e14fca 100644 --- a/sanic/server/runners.py +++ b/sanic/server/runners.py @@ -27,7 +27,7 @@ from sanic.application.ext import setup_ext from sanic.compat import OS_IS_WINDOWS, ctrlc_workaround_for_windows -from sanic.http.http3 import get_config, get_ticket_store +from sanic.http.http3 import SessionTicketStore, get_config from sanic.log import error_logger, logger from sanic.models.server_types import Signal from sanic.server.async_server import AsyncioServer @@ -250,7 +250,7 @@ def _serve_http_3( run_multiple: bool = False, ): protocol = partial(Http3Protocol, app=app) - ticket_store = get_ticket_store() + ticket_store = SessionTicketStore() ssl_context = get_ssl_context(app, ssl) config = get_config(app, ssl_context) coro = quic_serve( From 9045719330609199496ef1c7f57e6af9ccaf24c5 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 24 May 2022 13:45:34 +0300 Subject: [PATCH 25/59] Finish flow --- sanic/app.py | 5 ++++- sanic/config.py | 4 ++-- sanic/http/http3.py | 18 +++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index 20516b3b8c..9450779ab8 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -64,6 +64,7 @@ URLBuildError, ) from sanic.handlers import ErrorHandler +from sanic.helpers import _default from sanic.http import Stage from sanic.log import ( LOGGING_CONFIG_DEFAULTS, @@ -1533,8 +1534,10 @@ async def _startup(self): if hasattr(self, "_ext"): self.ext._display() - if self.state.is_debug: + if self.state.is_debug and self.config.TOUCHUP is not True: self.config.TOUCHUP = False + elif self.config.TOUCHUP is _default: + self.config.TOUCHUP = True # Setup routers self.signalize(self.config.TOUCHUP) diff --git a/sanic/config.py b/sanic/config.py index 9b80f5a5c9..77a2869b0c 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -42,7 +42,7 @@ "REQUEST_TIMEOUT": 60, # 60 seconds "RESPONSE_TIMEOUT": 60, # 60 seconds "TLS_CERT_PASSWORD": "", - "TOUCHUP": True, + "TOUCHUP": _default, "USE_UVLOOP": _default, "WEBSOCKET_MAX_SIZE": 2**20, # 1 megabyte "WEBSOCKET_PING_INTERVAL": 20, @@ -90,7 +90,7 @@ class Config(dict, metaclass=DescriptorMeta): RESPONSE_TIMEOUT: int SERVER_NAME: str TLS_CERT_PASSWORD: str - TOUCHUP: bool + TOUCHUP: Union[Default, bool] USE_UVLOOP: Union[Default, bool] WEBSOCKET_MAX_SIZE: int WEBSOCKET_PING_INTERVAL: int diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 95b8b2af9d..38768ffe65 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -314,13 +314,17 @@ def get_receiver_by_stream_id(self, stream_id: int) -> Receiver: return self.receivers[stream_id] def _make_request(self, event: HeadersReceived) -> Request: - method_header, path_header, *rem = event.headers - headers = Header(((k.decode(), v.decode()) for k, v in rem)) - method = method_header[1].decode() - path = path_header[1] - scheme = headers.pop(":scheme") - - authority = headers.pop(":authority") + # TODO: + # Cleanup + # method_header, path_header, *rem = event.headers + headers = Header(((k.decode(), v.decode()) for k, v in event.headers)) + # method = method_header[1].decode() + # path = path_header[1] + method = headers[":method"] + path = headers[":path"].encode() + scheme = headers.pop(":scheme", "") + + authority = headers.pop(":authority", "") if authority: headers["host"] = authority From dd96f1a6443bdbefe810265fedb2f824fe65419a Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Thu, 16 Jun 2022 10:56:01 +0300 Subject: [PATCH 26/59] Get regular tests running again --- sanic/http/http3.py | 6 +++++- sanic/mixins/runner.py | 13 ++++++++----- tests/test_cli.py | 3 +-- tests/test_http.py | 2 +- tests/test_logging.py | 4 ++-- tests/test_motd.py | 2 +- tox.ini | 2 +- 7 files changed, 19 insertions(+), 13 deletions(-) diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 38768ffe65..a0d022dcdb 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -30,7 +30,7 @@ from sanic.compat import Header from sanic.exceptions import PayloadTooLarge, SanicException from sanic.helpers import has_message_body -from sanic.http.tls import CertSimple +from sanic.http.tls import CertSelector, CertSimple if TYPE_CHECKING: @@ -354,6 +354,10 @@ def pop(self, label: bytes) -> Optional[SessionTicket]: def get_config(app: Sanic, ssl: SSLContext): + # TODO: + # - proper selection needed if servince with multiple certs + if isinstance(ssl, CertSelector): + ssl = ssl.sanic_select[0] if not isinstance(ssl, CertSimple): raise SanicException("SSLContext is not CertSimple") diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py index 9bf4690c7e..43b6b1fd33 100644 --- a/sanic/mixins/runner.py +++ b/sanic/mixins/runner.py @@ -512,12 +512,15 @@ def motd( else: mode.append(f"w/ {self.state.workers} workers") - server = ", ".join( - ( - self.state.server, - server_settings["version"].display(), # type: ignore + if server_settings: + server = ", ".join( + ( + self.state.server, + server_settings["version"].display(), # type: ignore + ) ) - ) + else: + server = "" display = { "mode": " ".join(mode), diff --git a/tests/test_cli.py b/tests/test_cli.py index f77e345355..02fd07ef80 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -147,8 +147,7 @@ def test_tls_wrong_options(cmd): assert not out lines = err.decode().split("\n") - errmsg = lines[6] - assert errmsg == "TLS certificates must be specified by either of:" + assert "TLS certificates must be specified by either of:" in lines @pytest.mark.parametrize( diff --git a/tests/test_http.py b/tests/test_http.py index 653857a12c..ec5357b09d 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -115,7 +115,7 @@ def test_full_message(client): """ ) response = client.recv() - assert len(response) == 140 + assert len(response) == 151 assert b"200 OK" in response diff --git a/tests/test_logging.py b/tests/test_logging.py index 274b407c54..180c10b29d 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -163,7 +163,7 @@ def test_logging_modified_root_logger_config(): def test_access_log_client_ip_remote_addr(monkeypatch): access = Mock() - monkeypatch.setattr(sanic.http, "access_logger", access) + monkeypatch.setattr(sanic.http.http1, "access_logger", access) app = Sanic("test_logging") app.config.PROXIES_COUNT = 2 @@ -190,7 +190,7 @@ async def handler(request): def test_access_log_client_ip_reqip(monkeypatch): access = Mock() - monkeypatch.setattr(sanic.http, "access_logger", access) + monkeypatch.setattr(sanic.http.http1, "access_logger", access) app = Sanic("test_logging") diff --git a/tests/test_motd.py b/tests/test_motd.py index f3f95a2591..83c7e4bf8c 100644 --- a/tests/test_motd.py +++ b/tests/test_motd.py @@ -53,7 +53,7 @@ def test_motd_with_expected_info(app, run_startup): assert logs[1][2] == f"Sanic v{__version__}" assert logs[3][2] == "mode: debug, single worker" - assert logs[4][2] == "server: sanic" + assert logs[4][2] == "server: sanic, HTTP/1.1" assert logs[5][2] == f"python: {platform.python_version()}" assert logs[6][2] == f"platform: {platform.platform()}" diff --git a/tox.ini b/tox.ini index 0e5f80a2e9..70c2228819 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ usedevelop = true setenv = {py37,py38,py39,py310,pyNightly}-no-ext: SANIC_NO_UJSON=1 {py37,py38,py39,py310,pyNightly}-no-ext: SANIC_NO_UVLOOP=1 -extras = test +extras = test, http3 deps = httpx==0.23 allowlist_externals = From 15d654cbca8cc06be00e8394bb0739b88900a00e Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Thu, 16 Jun 2022 11:40:43 +0300 Subject: [PATCH 27/59] Add trustme certs (#2468) * Add trustme certs * WIP * Do not allow trustme local cert for HTTP/3 --- sanic/config.py | 9 + sanic/constants.py | 9 + sanic/http/http1.py | 3 +- sanic/http/http3.py | 20 +- sanic/http/stream.py | 19 ++ sanic/http/tls/__init__.py | 5 + sanic/http/{tls.py => tls/context.py} | 242 ++++++--------------- sanic/http/tls/creators.py | 269 ++++++++++++++++++++++++ sanic/request.py | 12 +- sanic/server/protocols/http_protocol.py | 2 + tests/http3/test_server.py | 24 +++ 11 files changed, 419 insertions(+), 195 deletions(-) create mode 100644 sanic/http/stream.py create mode 100644 sanic/http/tls/__init__.py rename sanic/http/{tls.py => tls/context.py} (63%) create mode 100644 sanic/http/tls/creators.py create mode 100644 tests/http3/test_server.py diff --git a/sanic/config.py b/sanic/config.py index 77a2869b0c..115c0bf95b 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Optional, Sequence, Union +from sanic.constants import LocalCertCreator from sanic.errorpages import DEFAULT_FORMAT, check_error_format from sanic.helpers import Default, _default from sanic.http import Http @@ -26,6 +27,7 @@ "GRACEFUL_SHUTDOWN_TIMEOUT": 15.0, # 15 sec "KEEP_ALIVE_TIMEOUT": 5, # 5 seconds "KEEP_ALIVE": True, + "LOCAL_CERT_CREATOR": LocalCertCreator.AUTO, "LOCAL_TLS_KEY": _default, "LOCAL_TLS_CERT": _default, "LOCALHOST": "localhost", @@ -73,6 +75,7 @@ class Config(dict, metaclass=DescriptorMeta): GRACEFUL_SHUTDOWN_TIMEOUT: float KEEP_ALIVE_TIMEOUT: int KEEP_ALIVE: bool + LOCAL_CERT_CREATOR: Union[str, LocalCertCreator] LOCAL_TLS_KEY: Union[Path, str, Default] LOCAL_TLS_CERT: Union[Path, str, Default] LOCALHOST: str @@ -172,6 +175,12 @@ def _post_set(self, attr, value) -> None: "be supported starting in v22.6.", 22.6, ) + elif attr == "LOCAL_CERT_CREATOR" and not isinstance( + self.LOCAL_CERT_CREATOR, LocalCertCreator + ): + self.LOCAL_CERT_CREATOR = LocalCertCreator[ + self.LOCAL_CERT_CREATOR.upper() + ] @property def LOGO(self): diff --git a/sanic/constants.py b/sanic/constants.py index 0bc68f2666..52ec50ef00 100644 --- a/sanic/constants.py +++ b/sanic/constants.py @@ -24,6 +24,15 @@ def __str__(self) -> str: DELETE = auto() +class LocalCertCreator(str, Enum): + def _generate_next_value_(name, start, count, last_values): + return name.upper() + + AUTO = auto() + TRUSTME = auto() + MKCERT = auto() + + HTTP_METHODS = tuple(HTTPMethod.__members__.values()) DEFAULT_HTTP_CONTENT_TYPE = "application/octet-stream" DEFAULT_LOCAL_TLS_KEY = "key.pem" diff --git a/sanic/http/http1.py b/sanic/http/http1.py index 0b3a60fd9e..87cabde1c5 100644 --- a/sanic/http/http1.py +++ b/sanic/http/http1.py @@ -20,6 +20,7 @@ from sanic.headers import format_http1_response from sanic.helpers import has_message_body from sanic.http.constants import Stage +from sanic.http.stream import Stream from sanic.log import access_logger, error_logger, logger from sanic.touchup import TouchUpMeta @@ -27,7 +28,7 @@ HTTP_CONTINUE = b"HTTP/1.1 100 Continue\r\n\r\n" -class Http(metaclass=TouchUpMeta): +class Http(Stream, metaclass=TouchUpMeta): """ Internal helper for managing the HTTP request/response cycle diff --git a/sanic/http/http3.py b/sanic/http/http3.py index a0d022dcdb..e5ca518f02 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -3,7 +3,6 @@ import asyncio from abc import ABC, abstractmethod -from ssl import SSLContext from typing import ( TYPE_CHECKING, Callable, @@ -28,9 +27,11 @@ from aioquic.tls import SessionTicket from sanic.compat import Header +from sanic.constants import LocalCertCreator from sanic.exceptions import PayloadTooLarge, SanicException from sanic.helpers import has_message_body -from sanic.http.tls import CertSelector, CertSimple +from sanic.http.stream import Stream +from sanic.http.tls.context import CertSelector, CertSimple, SanicSSLContext if TYPE_CHECKING: @@ -63,7 +64,7 @@ async def run(self): ... -class HTTPReceiver(Receiver): +class HTTPReceiver(Receiver, Stream): stage: Stage def __init__(self, *args, **kwargs) -> None: @@ -353,11 +354,11 @@ def pop(self, label: bytes) -> Optional[SessionTicket]: return self.tickets.pop(label, None) -def get_config(app: Sanic, ssl: SSLContext): +def get_config(app: Sanic, ssl: Union[SanicSSLContext, CertSelector]): # TODO: # - proper selection needed if servince with multiple certs if isinstance(ssl, CertSelector): - ssl = ssl.sanic_select[0] + ssl = cast(SanicSSLContext, ssl.sanic_select[0]) if not isinstance(ssl, CertSimple): raise SanicException("SSLContext is not CertSimple") @@ -367,6 +368,15 @@ def get_config(app: Sanic, ssl: SSLContext): max_datagram_frame_size=65536, ) password = app.config.TLS_CERT_PASSWORD or None + + if app.config.LOCAL_CERT_CREATOR is LocalCertCreator.TRUSTME: + raise SanicException( + "Sorry, you cannot currently use trustme as a local certificate " + "generator for an HTTP/3 server. This is not yet supported. You " + "should be able to use mkcert instead. For more information, see: " + "https://github.com/aiortc/aioquic/issues/295." + ) + config.load_cert_chain( ssl.sanic["cert"], ssl.sanic["key"], password=password ) diff --git a/sanic/http/stream.py b/sanic/http/stream.py new file mode 100644 index 0000000000..e71338b1a6 --- /dev/null +++ b/sanic/http/stream.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional, Tuple + +from sanic.http.constants import Stage + + +if TYPE_CHECKING: + from sanic.response import BaseHTTPResponse + + +class Stream: + stage: Stage + response: Optional[BaseHTTPResponse] + __touchup__: Tuple[str, ...] = tuple() + __slots__ = () + + def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: + raise NotImplementedError("Not implemented") diff --git a/sanic/http/tls/__init__.py b/sanic/http/tls/__init__.py new file mode 100644 index 0000000000..b12fe529f8 --- /dev/null +++ b/sanic/http/tls/__init__.py @@ -0,0 +1,5 @@ +from .context import process_to_context +from .creators import get_ssl_context + + +__all__ = ("get_ssl_context", "process_to_context") diff --git a/sanic/http/tls.py b/sanic/http/tls/context.py similarity index 63% rename from sanic/http/tls.py rename to sanic/http/tls/context.py index 387f669aac..f77fa56051 100644 --- a/sanic/http/tls.py +++ b/sanic/http/tls/context.py @@ -2,25 +2,10 @@ import os import ssl -import subprocess -import sys - -from contextlib import suppress -from pathlib import Path -from ssl import SSLContext -from tempfile import mkdtemp -from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union - -from sanic.application.constants import Mode -from sanic.application.spinner import loading -from sanic.constants import DEFAULT_LOCAL_TLS_CERT, DEFAULT_LOCAL_TLS_KEY -from sanic.exceptions import SanicException -from sanic.helpers import Default -from sanic.log import logger +from typing import Any, Dict, Iterable, Optional, Union -if TYPE_CHECKING: - from sanic import Sanic +from sanic.log import logger # Only allow secure ciphers, notably leaving out AES-CBC mode @@ -94,67 +79,6 @@ def load_cert_dir(p: str) -> ssl.SSLContext: return CertSimple(certfile, keyfile) -class CertSimple(ssl.SSLContext): - """A wrapper for creating SSLContext with a sanic attribute.""" - - sanic: Dict[str, Any] - - def __new__(cls, cert, key, **kw): - # try common aliases, rename to cert/key - certfile = kw["cert"] = kw.pop("certificate", None) or cert - keyfile = kw["key"] = kw.pop("keyfile", None) or key - password = kw.pop("password", None) - if not certfile or not keyfile: - raise ValueError("SSL dict needs filenames for cert and key.") - subject = {} - if "names" not in kw: - cert = ssl._ssl._test_decode_cert(certfile) # type: ignore - kw["names"] = [ - name - for t, name in cert["subjectAltName"] - if t in ["DNS", "IP Address"] - ] - subject = {k: v for item in cert["subject"] for k, v in item} - self = create_context(certfile, keyfile, password) - self.__class__ = cls - self.sanic = {**subject, **kw} - return self - - def __init__(self, cert, key, **kw): - pass # Do not call super().__init__ because it is already initialized - - -class CertSelector(ssl.SSLContext): - """Automatically select SSL certificate based on the hostname that the - client is trying to access, via SSL SNI. Paths to certificate folders - with privkey.pem and fullchain.pem in them should be provided, and - will be matched in the order given whenever there is a new connection. - """ - - def __new__(cls, ctxs): - return super().__new__(cls) - - def __init__(self, ctxs: Iterable[Optional[ssl.SSLContext]]): - super().__init__() - self.sni_callback = selector_sni_callback # type: ignore - self.sanic_select = [] - self.sanic_fallback = None - all_names = [] - for i, ctx in enumerate(ctxs): - if not ctx: - continue - names = dict(getattr(ctx, "sanic", {})).get("names", []) - all_names += names - self.sanic_select.append(ctx) - if i == 0: - self.sanic_fallback = ctx - if not all_names: - raise ValueError( - "No certificates with SubjectAlternativeNames found." - ) - logger.info(f"Certificate vhosts: {', '.join(all_names)}") - - def find_cert(self: CertSelector, server_name: str): """Find the first certificate that matches the given SNI. @@ -215,117 +139,71 @@ def server_name_callback( sslobj.sanic_server_name = server_name # type: ignore -def _make_path(maybe_path: Union[Path, str], tmpdir: Optional[Path]) -> Path: - if isinstance(maybe_path, Path): - return maybe_path - else: - path = Path(maybe_path) - if not path.exists(): - if not tmpdir: - raise RuntimeError("Reached an unknown state. No tmpdir.") - return tmpdir / maybe_path +class SanicSSLContext(ssl.SSLContext): + sanic: Dict[str, os.PathLike] - return path + @classmethod + def create_from_ssl_context(cls, context: ssl.SSLContext): + context.__class__ = cls + return context -def get_ssl_context(app: Sanic, ssl: Optional[SSLContext]) -> SSLContext: - if ssl: - return ssl - - if app.state.mode is Mode.PRODUCTION: - raise SanicException( - "Cannot run Sanic as an HTTPS server in PRODUCTION mode " - "without passing a TLS certificate. If you are developing " - "locally, please enable DEVELOPMENT mode and Sanic will " - "generate a localhost TLS certificate. For more information " - "please see: ___." - ) - - try: - tmpdir = None - if isinstance(app.config.LOCAL_TLS_KEY, Default) or isinstance( - app.config.LOCAL_TLS_CERT, Default - ): - tmpdir = Path(mkdtemp()) - - key = ( - DEFAULT_LOCAL_TLS_KEY - if isinstance(app.config.LOCAL_TLS_KEY, Default) - else app.config.LOCAL_TLS_KEY - ) - cert = ( - DEFAULT_LOCAL_TLS_CERT - if isinstance(app.config.LOCAL_TLS_CERT, Default) - else app.config.LOCAL_TLS_CERT - ) - - key_path = _make_path(key, tmpdir) - cert_path = _make_path(cert, tmpdir) - - if not cert_path.exists(): - generate_local_certificate( - key_path, cert_path, app.config.LOCALHOST - ) - finally: +class CertSimple(SanicSSLContext): + """A wrapper for creating SSLContext with a sanic attribute.""" - @app.main_process_stop - async def cleanup(*_): - if tmpdir: - with suppress(FileNotFoundError): - key_path.unlink() - cert_path.unlink() - tmpdir.rmdir() + sanic: Dict[str, Any] - return CertSimple(cert_path, key_path) + def __new__(cls, cert, key, **kw): + # try common aliases, rename to cert/key + certfile = kw["cert"] = kw.pop("certificate", None) or cert + keyfile = kw["key"] = kw.pop("keyfile", None) or key + password = kw.pop("password", None) + if not certfile or not keyfile: + raise ValueError("SSL dict needs filenames for cert and key.") + subject = {} + if "names" not in kw: + cert = ssl._ssl._test_decode_cert(certfile) # type: ignore + kw["names"] = [ + name + for t, name in cert["subjectAltName"] + if t in ["DNS", "IP Address"] + ] + subject = {k: v for item in cert["subject"] for k, v in item} + self = create_context(certfile, keyfile, password) + self.__class__ = cls + self.sanic = {**subject, **kw} + return self + def __init__(self, cert, key, **kw): + pass # Do not call super().__init__ because it is already initialized -def generate_local_certificate( - key_path: Path, cert_path: Path, localhost: str -): - check_mkcert() - if not key_path.parent.exists() or not cert_path.parent.exists(): - raise SanicException( - f"Cannot generate certificate at [{key_path}, {cert_path}]. One " - "or more of the directories does not exist." - ) - - message = "Generating TLS certificate" - with loading(message): - cmd = [ - "mkcert", - "-key-file", - str(key_path), - "-cert-file", - str(cert_path), - localhost, - ] - resp = subprocess.run( - cmd, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - sys.stdout.write("\r" + " " * (len(message) + 4)) - sys.stdout.flush() - sys.stdout.write(resp.stdout) +class CertSelector(ssl.SSLContext): + """Automatically select SSL certificate based on the hostname that the + client is trying to access, via SSL SNI. Paths to certificate folders + with privkey.pem and fullchain.pem in them should be provided, and + will be matched in the order given whenever there is a new connection. + """ + def __new__(cls, ctxs): + return super().__new__(cls) -def check_mkcert(): - try: - subprocess.run( - ["mkcert", "-help"], - check=True, - stderr=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - ) - except Exception as e: - raise SanicException( - "Sanic uses mkcert to generate local TLS certificates. Since you " - "did not supply a certificate, Sanic is attempting to generate " - "one for you, but cannot proceed since mkcert does not appear to " - "be installed. Please install mkcert or supply TLS certificates " - "to proceed. Installation instructions can be found here: " - "https://github.com/FiloSottile/mkcert" - ) from e + def __init__(self, ctxs: Iterable[Optional[ssl.SSLContext]]): + super().__init__() + self.sni_callback = selector_sni_callback # type: ignore + self.sanic_select = [] + self.sanic_fallback = None + all_names = [] + for i, ctx in enumerate(ctxs): + if not ctx: + continue + names = dict(getattr(ctx, "sanic", {})).get("names", []) + all_names += names + self.sanic_select.append(ctx) + if i == 0: + self.sanic_fallback = ctx + if not all_names: + raise ValueError( + "No certificates with SubjectAlternativeNames found." + ) + logger.info(f"Certificate vhosts: {', '.join(all_names)}") diff --git a/sanic/http/tls/creators.py b/sanic/http/tls/creators.py new file mode 100644 index 0000000000..90e0f7f6b3 --- /dev/null +++ b/sanic/http/tls/creators.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import ssl +import subprocess +import sys + +from abc import ABC, abstractmethod +from contextlib import suppress +from pathlib import Path +from tempfile import mkdtemp +from typing import TYPE_CHECKING, Optional, Tuple, Type, Union, cast + +from sanic.application.constants import Mode +from sanic.application.spinner import loading +from sanic.constants import ( + DEFAULT_LOCAL_TLS_CERT, + DEFAULT_LOCAL_TLS_KEY, + LocalCertCreator, +) +from sanic.exceptions import SanicException +from sanic.helpers import Default +from sanic.http.tls.context import CertSimple, SanicSSLContext + + +try: + import trustme + + TRUSTME_INSTALLED = True +except (ImportError, ModuleNotFoundError): + TRUSTME_INSTALLED = False + +if TYPE_CHECKING: + from sanic import Sanic + + +# Only allow secure ciphers, notably leaving out AES-CBC mode +# OpenSSL chooses ECDSA or RSA depending on the cert in use +CIPHERS_TLS12 = [ + "ECDHE-ECDSA-CHACHA20-POLY1305", + "ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-CHACHA20-POLY1305", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES128-GCM-SHA256", +] + + +def _make_path(maybe_path: Union[Path, str], tmpdir: Optional[Path]) -> Path: + if isinstance(maybe_path, Path): + return maybe_path + else: + path = Path(maybe_path) + if not path.exists(): + if not tmpdir: + raise RuntimeError("Reached an unknown state. No tmpdir.") + return tmpdir / maybe_path + + return path + + +def get_ssl_context( + app: Sanic, ssl: Optional[ssl.SSLContext] +) -> ssl.SSLContext: + if ssl: + return ssl + + if app.state.mode is Mode.PRODUCTION: + raise SanicException( + "Cannot run Sanic as an HTTPS server in PRODUCTION mode " + "without passing a TLS certificate. If you are developing " + "locally, please enable DEVELOPMENT mode and Sanic will " + "generate a localhost TLS certificate. For more information " + "please see: ___." + ) + + creator = CertCreator.select( + app, + cast(LocalCertCreator, app.config.LOCAL_CERT_CREATOR), + app.config.LOCAL_TLS_KEY, + app.config.LOCAL_TLS_CERT, + ) + context = creator.generate_cert(app.config.LOCALHOST) + return context + + +class CertCreator(ABC): + def __init__(self, app, key, cert) -> None: + self.app = app + self.key = key + self.cert = cert + self.tmpdir = None + + if isinstance(self.key, Default) or isinstance(self.cert, Default): + self.tmpdir = Path(mkdtemp()) + + key = ( + DEFAULT_LOCAL_TLS_KEY + if isinstance(self.key, Default) + else self.key + ) + cert = ( + DEFAULT_LOCAL_TLS_CERT + if isinstance(self.cert, Default) + else self.cert + ) + + self.key_path = _make_path(key, self.tmpdir) + self.cert_path = _make_path(cert, self.tmpdir) + + @abstractmethod + def check_supported(self) -> None: + ... + + @abstractmethod + def generate_cert(self, localhost: str) -> ssl.SSLContext: + ... + + @classmethod + def select( + cls, + app: Sanic, + cert_creator: LocalCertCreator, + local_tls_key, + local_tls_cert, + ) -> CertCreator: + + creator: Optional[CertCreator] = None + + cert_creator_options: Tuple[ + Tuple[Type[CertCreator], LocalCertCreator], ... + ] = ( + (MkcertCreator, LocalCertCreator.MKCERT), + (TrustmeCreator, LocalCertCreator.TRUSTME), + ) + for creator_class, local_creator in cert_creator_options: + creator = cls._try_select( + app, + creator, + creator_class, + local_creator, + cert_creator, + local_tls_key, + local_tls_cert, + ) + + if not creator: + raise SanicException("...") + + return creator + + @staticmethod + def _try_select( + app: Sanic, + creator: Optional[CertCreator], + creator_class: Type[CertCreator], + creator_requirement: LocalCertCreator, + creator_requested: LocalCertCreator, + local_tls_key, + local_tls_cert, + ): + if creator or ( + creator_requested is not LocalCertCreator.AUTO + and creator_requested is not creator_requirement + ): + return creator + + instance = creator_class(app, local_tls_key, local_tls_cert) + try: + instance.check_supported() + except SanicException: + if creator_requested is creator_requirement: + raise + + return instance + + +class MkcertCreator(CertCreator): + def check_supported(self) -> None: + try: + subprocess.run( + ["mkcert", "-help"], + check=True, + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + except Exception as e: + raise SanicException( + "Sanic is attempting to use mkcert to generate local TLS " + "certificates since you did not supply a certificate, but " + "one is required. Sanic cannot proceed since mkcert does not " + "appear to be installed. Alternatively, you can use trustme. " + "Please install mkcert, trustme, or supply TLS certificates " + "to proceed. Installation instructions can be found here: " + "https://github.com/FiloSottile/mkcert.\n" + "Find out more information about your options here: " + "_____" + ) from e + + def generate_cert(self, localhost: str) -> ssl.SSLContext: + try: + if not self.cert_path.exists(): + message = "Generating TLS certificate" + with loading(message): + cmd = [ + "mkcert", + "-key-file", + str(self.key_path), + "-cert-file", + str(self.cert_path), + localhost, + ] + resp = subprocess.run( + cmd, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + sys.stdout.write("\r" + " " * (len(message) + 4)) + sys.stdout.flush() + sys.stdout.write(resp.stdout) + finally: + + @self.app.main_process_stop + async def cleanup(*_): + if self.tmpdir: + with suppress(FileNotFoundError): + self.key_path.unlink() + self.cert_path.unlink() + self.tmpdir.rmdir() + + return CertSimple(self.cert_path, self.key_path) + + +class TrustmeCreator(CertCreator): + def check_supported(self) -> None: + if not TRUSTME_INSTALLED: + raise SanicException( + "Sanic is attempting to use trustme to generate local TLS " + "certificates since you did not supply a certificate, but " + "one is required. Sanic cannot proceed since trustme does not " + "appear to be installed. Alternatively, you can use mkcert. " + "Please install mkcert, trustme, or supply TLS certificates " + "to proceed. Installation instructions can be found here: " + "https://github.com/python-trio/trustme.\n" + "Find out more information about your options here: " + "_____" + ) + + def generate_cert(self, localhost: str) -> ssl.SSLContext: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sanic_context = SanicSSLContext.create_from_ssl_context(context) + sanic_context.sanic = { + "cert": self.cert_path.absolute(), + "key": self.key_path.absolute(), + } + ca = trustme.CA() + server_cert = ca.issue_cert(localhost) + server_cert.configure_cert(sanic_context) + ca.configure_trust(context) + + ca.cert_pem.write_to_path(str(self.cert_path.absolute())) + server_cert.private_key_and_cert_chain_pem.write_to_path( + str(self.key_path.absolute()) + ) + + sanic_context.verify_mode = False + + return context diff --git a/sanic/request.py b/sanic/request.py index 8d7cfbe802..0d5ead9e61 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -15,7 +15,7 @@ from sanic_routing.route import Route -from sanic.http.http3 import HTTPReceiver # type: ignore +from sanic.http.stream import Stream from sanic.models.asgi import ASGIScope from sanic.models.http_types import Credentials @@ -48,7 +48,7 @@ parse_host, parse_xforwarded, ) -from sanic.http import Http, Stage +from sanic.http import Stage from sanic.log import error_logger, logger from sanic.models.protocol_types import TransportProtocol from sanic.response import BaseHTTPResponse, HTTPResponse @@ -169,14 +169,12 @@ def __init__( Tuple[bool, bool, str, str], List[Tuple[str, str]] ] = defaultdict(list) self.request_middleware_started = False + self.responded: bool = False + self.route: Optional[Route] = None + self.stream: Optional[Stream] = None self._cookies: Optional[Dict[str, str]] = None self._match_info: Dict[str, Any] = {} - # TODO: - # - Create an ABC (called Stream) for Http and HTTPReceiver to subclass - self.stream: Optional[Union[Http, HTTPReceiver]] = None - self.route: Optional[Route] = None self._protocol = None - self.responded: bool = False def __repr__(self): class_name = self.__class__.__name__ diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index 2d53e00577..c8fcf5e3e3 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -30,6 +30,8 @@ class HttpProtocolMixin: + __slots__ = () + def _setup_connection(self, *args, **kwargs): self._http = self.HTTP_CLASS(self, *args, **kwargs) self._time = current_time() diff --git a/tests/http3/test_server.py b/tests/http3/test_server.py new file mode 100644 index 0000000000..3912aa4c10 --- /dev/null +++ b/tests/http3/test_server.py @@ -0,0 +1,24 @@ +from asyncio import Event +from pathlib import Path + +from sanic import Sanic + + +parent_dir = Path(__file__).parent.parent +localhost_dir = str(parent_dir / "certs/localhost") +sanic_dir = str(parent_dir / "certs/sanic.example") + + +def test_server_starts(app: Sanic): + ev = Event() + + @app.after_server_start + def shutdown(*_): + ev.set() + app.stop() + + print(localhost_dir) + print(sanic_dir) + app.run(version=3, ssl=[localhost_dir, sanic_dir]) + + assert ev.is_set() From 191d5c5433a3b0c2f428def48b384e228059fad9 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Sun, 19 Jun 2022 23:03:53 +0300 Subject: [PATCH 28/59] Add some unit tests --- sanic/http/http3.py | 15 +-- tests/http3/test_http_receiver.py | 164 ++++++++++++++++++++++++++++++ tests/http3/test_server.py | 88 ++++++++++++++-- 3 files changed, 254 insertions(+), 13 deletions(-) create mode 100644 tests/http3/test_http_receiver.py diff --git a/sanic/http/http3.py b/sanic/http/http3.py index e5ca518f02..3439c572fa 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -34,7 +34,7 @@ from sanic.http.tls.context import CertSelector, CertSimple, SanicSSLContext -if TYPE_CHECKING: +if TYPE_CHECKING: # noqa from sanic import Sanic from sanic.request import Request from sanic.response import BaseHTTPResponse @@ -60,7 +60,7 @@ def __init__(self, transmit, protocol, request: Request) -> None: self.request = request @abstractmethod - async def run(self): + async def run(self): # no cov ... @@ -95,7 +95,7 @@ async def run(self, exception: Optional[Exception] = None): extra={"verbosity": 1}, ) await self.protocol.request_handler(self.request) - except Exception as e: + except Exception as e: # no cov # This should largely be handled within the request handler. # But, just in case... await self.run(e) @@ -200,7 +200,7 @@ def receive_body(self, data: bytes) -> None: if self.request_bytes > self.request_max_size: raise PayloadTooLarge("Request body exceeds the size limit") - self.request.body = data + self.request.body += data async def send(self, data: bytes, end_stream: bool) -> None: logger.debug( @@ -246,12 +246,12 @@ def _send(self, data: bytes, end_stream: bool) -> None: self.stage = Stage.IDLE -class WebsocketReceiver(Receiver): +class WebsocketReceiver(Receiver): # noqa async def run(self): ... -class WebTransportReceiver(Receiver): +class WebTransportReceiver(Receiver): # noqa async def run(self): ... @@ -356,7 +356,8 @@ def pop(self, label: bytes) -> Optional[SessionTicket]: def get_config(app: Sanic, ssl: Union[SanicSSLContext, CertSelector]): # TODO: - # - proper selection needed if servince with multiple certs + # - proper selection needed if servince with multiple certs insted of + # just taking the first if isinstance(ssl, CertSelector): ssl = cast(SanicSSLContext, ssl.sanic_select[0]) if not isinstance(ssl, CertSimple): diff --git a/tests/http3/test_http_receiver.py b/tests/http3/test_http_receiver.py new file mode 100644 index 0000000000..0cfeab047b --- /dev/null +++ b/tests/http3/test_http_receiver.py @@ -0,0 +1,164 @@ +from unittest.mock import AsyncMock, Mock + +import pytest + +from aioquic.h3.events import DataReceived, HeadersReceived +from aioquic.quic.configuration import QuicConfiguration +from aioquic.quic.connection import QuicConnection +from aioquic.quic.events import ProtocolNegotiated + +from sanic import Request, Sanic +from sanic.compat import Header +from sanic.config import DEFAULT_CONFIG +from sanic.exceptions import PayloadTooLarge +from sanic.http.constants import Stage +from sanic.http.http3 import Http3, HTTPReceiver +from sanic.response import empty +from sanic.server.protocols.http_protocol import Http3Protocol + + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture(autouse=True) +async def setup(app: Sanic): + @app.get("/") + async def handler(*_): + return empty() + + app.router.finalize() + app.signal_router.finalize() + app.signal_router.allow_fail_builtin = False + + +@pytest.fixture +def http_request(app): + return Request(b"/", Header({}), "3", "GET", Mock(), app) + + +def generate_protocol(app): + connection = QuicConnection(configuration=QuicConfiguration()) + connection._ack_delay = 0 + connection._loss = Mock() + connection._loss.spaces = [] + connection._loss.get_loss_detection_time = lambda: None + connection.datagrams_to_send = Mock(return_value=[]) # type: ignore + return Http3Protocol( + connection, + app=app, + stream_handler=None, + ) + + +def generate_http_receiver(app, http_request) -> HTTPReceiver: + protocol = generate_protocol(app) + receiver = HTTPReceiver( + protocol.transmit, + protocol, + http_request, + ) + http_request.stream = receiver + return receiver + + +def test_http_receiver_init(app: Sanic, http_request: Request): + receiver = generate_http_receiver(app, http_request) + assert receiver.request_body is None + assert receiver.stage is Stage.IDLE + assert receiver.headers_sent is False + assert receiver.response is None + assert receiver.request_max_size == DEFAULT_CONFIG["REQUEST_MAX_SIZE"] + assert receiver.request_bytes == 0 + + +async def test_http_receiver_run_request(app: Sanic, http_request: Request): + handler = AsyncMock() + + class mock_handle(Sanic): + handle_request = handler + + app.__class__ = mock_handle + receiver = generate_http_receiver(app, http_request) + receiver.protocol.quic_event_received( + ProtocolNegotiated(alpn_protocol="h3") + ) + await receiver.run() + handler.assert_awaited_once_with(receiver.request) + + +async def test_http_receiver_run_exception(app: Sanic, http_request: Request): + handler = AsyncMock() + + class mock_handle(Sanic): + handle_exception = handler + + app.__class__ = mock_handle + receiver = generate_http_receiver(app, http_request) + receiver.protocol.quic_event_received( + ProtocolNegotiated(alpn_protocol="h3") + ) + exception = Exception("Oof") + await receiver.run(exception) + handler.assert_awaited_once_with(receiver.request, exception) + + handler.reset_mock() + receiver.stage = Stage.REQUEST + await receiver.run(exception) + handler.assert_awaited_once_with(receiver.request, exception) + + +def test_http_receiver_respond(app: Sanic, http_request: Request): + receiver = generate_http_receiver(app, http_request) + response = empty() + + receiver.stage = Stage.RESPONSE + with pytest.raises(RuntimeError, match="Response already started"): + receiver.respond(response) + + receiver.stage = Stage.HANDLER + resp = receiver.respond(response) + + assert resp is response + assert response.stream is receiver + + +def test_http_receiver_receive_body(app: Sanic, http_request: Request): + receiver = generate_http_receiver(app, http_request) + receiver.request_max_size = 4 + + receiver.receive_body(b"..") + assert receiver.request.body == b".." + + receiver.receive_body(b"..") + assert receiver.request.body == b"...." + + with pytest.raises( + PayloadTooLarge, match="Request body exceeds the size limit" + ): + receiver.receive_body(b"..") + + +def test_http3_events(app): + protocol = generate_protocol(app) + http3 = Http3(protocol, protocol.transmit) + http3.http_event_received( + HeadersReceived( + [ + (b":method", b"GET"), + (b":path", b"/location"), + (b":scheme", b"https"), + (b":authority", b"localhost:8443"), + (b"foo", b"bar"), + ], + 1, + False, + ) + ) + http3.http_event_received(DataReceived(b"foobar", 1, False)) + receiver = http3.receivers[1] + + assert len(http3.receivers) == 1 + assert receiver.request.path == "/location" + assert receiver.request.method == "GET" + assert receiver.request.headers["foo"] == "bar" + assert receiver.request.body == b"foobar" diff --git a/tests/http3/test_server.py b/tests/http3/test_server.py index 3912aa4c10..40c09c00e4 100644 --- a/tests/http3/test_server.py +++ b/tests/http3/test_server.py @@ -1,15 +1,20 @@ +import logging + from asyncio import Event from pathlib import Path +import pytest + from sanic import Sanic +from sanic.http.constants import HTTP parent_dir = Path(__file__).parent.parent -localhost_dir = str(parent_dir / "certs/localhost") -sanic_dir = str(parent_dir / "certs/sanic.example") +localhost_dir = parent_dir / "certs/localhost" -def test_server_starts(app: Sanic): +@pytest.mark.parametrize("version", (3, HTTP.VERSION_3)) +def test_server_starts_http3(app: Sanic, version, caplog): ev = Event() @app.after_server_start @@ -17,8 +22,79 @@ def shutdown(*_): ev.set() app.stop() - print(localhost_dir) - print(sanic_dir) - app.run(version=3, ssl=[localhost_dir, sanic_dir]) + with caplog.at_level(logging.INFO): + app.run( + version=version, + ssl={ + "cert": localhost_dir / "fullchain.pem", + "key": localhost_dir / "privkey.pem", + }, + ) assert ev.is_set() + assert ( + "sanic.root", + logging.INFO, + "server: sanic, HTTP/3", + ) in caplog.record_tuples + + +def test_server_starts_http1_and_http3(app: Sanic, caplog): + @app.after_server_start + def shutdown(*_): + app.stop() + + app.prepare( + version=3, + ssl={ + "cert": localhost_dir / "fullchain.pem", + "key": localhost_dir / "privkey.pem", + }, + ) + app.prepare( + version=1, + ssl={ + "cert": localhost_dir / "fullchain.pem", + "key": localhost_dir / "privkey.pem", + }, + ) + with caplog.at_level(logging.INFO): + Sanic.serve() + + assert ( + "sanic.root", + logging.INFO, + "server: sanic, HTTP/1.1", + ) in caplog.record_tuples + assert ( + "sanic.root", + logging.INFO, + "server: sanic, HTTP/3", + ) in caplog.record_tuples + + +def test_server_starts_http1_and_http3_bad_order(app: Sanic, caplog): + @app.after_server_start + def shutdown(*_): + app.stop() + + app.prepare( + version=1, + ssl={ + "cert": localhost_dir / "fullchain.pem", + "key": localhost_dir / "privkey.pem", + }, + ) + message = ( + "Serving HTTP/3 instances as a secondary server is not supported. " + "There can only be a single HTTP/3 worker and it must be the first " + "instance prepared." + ) + with pytest.raises(RuntimeError, match=message): + app.prepare( + version=3, + ssl={ + "cert": localhost_dir / "fullchain.pem", + "key": localhost_dir / "privkey.pem", + }, + ) From 872b58f26f4e76bc2631db087e059d3492df52df Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 01:12:38 +0300 Subject: [PATCH 29/59] Add some unit tests --- .coveragerc | 1 + sanic/app.py | 2 +- sanic/application/ext.py | 2 +- sanic/application/spinner.py | 6 +-- sanic/application/state.py | 2 +- sanic/asgi.py | 2 +- sanic/blueprint_group.py | 2 +- sanic/blueprints.py | 2 +- sanic/http/http1.py | 2 +- sanic/http/http3.py | 2 +- sanic/mixins/runner.py | 4 +- sanic/request.py | 2 +- sanic/server/events.py | 2 +- sanic/server/protocols/base_protocol.py | 2 +- sanic/server/protocols/http_protocol.py | 2 +- sanic/server/protocols/websocket_protocol.py | 2 +- sanic/server/websockets/frame.py | 4 +- sanic/touchup/schemes/altsvc.py | 2 +- sanic/views.py | 2 +- tests/http3/test_http_receiver.py | 1 + tests/test_http.py | 50 ++------------------ tests/test_unix_socket.py | 16 +++---- 22 files changed, 37 insertions(+), 75 deletions(-) diff --git a/.coveragerc b/.coveragerc index 228560650c..1a042c34e9 100644 --- a/.coveragerc +++ b/.coveragerc @@ -20,6 +20,7 @@ exclude_lines = noqa NOQA pragma: no cover + TYPE_CHECKING omit = site-packages sanic/__main__.py diff --git a/sanic/app.py b/sanic/app.py index b55782732c..a069fd5900 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -93,7 +93,7 @@ from sanic.touchup import TouchUp, TouchUpMeta -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: try: from sanic_ext import Extend # type: ignore from sanic_ext.extensions.base import Extension # type: ignore diff --git a/sanic/application/ext.py b/sanic/application/ext.py index deb7c5d4c2..eac1e3179a 100644 --- a/sanic/application/ext.py +++ b/sanic/application/ext.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic import Sanic try: diff --git a/sanic/application/spinner.py b/sanic/application/spinner.py index c1e35338f2..139f0db44e 100644 --- a/sanic/application/spinner.py +++ b/sanic/application/spinner.py @@ -8,7 +8,7 @@ from threading import Thread -if os.name == "nt": +if os.name == "nt": # noqa import ctypes import msvcrt @@ -16,7 +16,7 @@ class _CursorInfo(ctypes.Structure): _fields_ = [("size", ctypes.c_int), ("visible", ctypes.c_byte)] -class Spinner: +class Spinner: # noqa def __init__(self, message: str) -> None: self.message = message self.queue: Queue[int] = Queue() @@ -81,7 +81,7 @@ def show(): @contextmanager -def loading(message: str = "Loading"): +def loading(message: str = "Loading"): # noqa spinner = Spinner(message) spinner.start() yield diff --git a/sanic/application/state.py b/sanic/application/state.py index 74bfa5b8b0..f308f2c6ec 100644 --- a/sanic/application/state.py +++ b/sanic/application/state.py @@ -13,7 +13,7 @@ from sanic.server.async_server import AsyncioServer -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic import Sanic diff --git a/sanic/asgi.py b/sanic/asgi.py index 3dbd95a702..10357ae876 100644 --- a/sanic/asgi.py +++ b/sanic/asgi.py @@ -17,7 +17,7 @@ from sanic.server.websockets.connection import WebSocketConnection -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic import Sanic diff --git a/sanic/blueprint_group.py b/sanic/blueprint_group.py index b16d8c58e6..a9b514106b 100644 --- a/sanic/blueprint_group.py +++ b/sanic/blueprint_group.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, List, Optional, Union -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic.blueprints import Blueprint diff --git a/sanic/blueprints.py b/sanic/blueprints.py index df4501ddcc..bee9c437b0 100644 --- a/sanic/blueprints.py +++ b/sanic/blueprints.py @@ -36,7 +36,7 @@ ) -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic import Sanic diff --git a/sanic/http/http1.py b/sanic/http/http1.py index 7e0122e7a9..4d5fe6f4d7 100644 --- a/sanic/http/http1.py +++ b/sanic/http/http1.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Optional -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic.request import Request from sanic.response import BaseHTTPResponse diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 3439c572fa..e141762afc 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -34,7 +34,7 @@ from sanic.http.tls.context import CertSelector, CertSimple, SanicSSLContext -if TYPE_CHECKING: # noqa +if TYPE_CHECKING: from sanic import Sanic from sanic.request import Request from sanic.response import BaseHTTPResponse diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py index 44ca6a4d53..0257183b91 100644 --- a/sanic/mixins/runner.py +++ b/sanic/mixins/runner.py @@ -52,7 +52,7 @@ from sanic.server.runners import serve, serve_multiple, serve_single -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic import Sanic from sanic.application.state import ApplicationState from sanic.config import Config @@ -434,7 +434,7 @@ def _helper( ssl = process_to_context(ssl) if version is HTTP.VERSION_3 or auto_tls: - if TYPE_CHECKING: # no cov + if TYPE_CHECKING: self = cast(Sanic, self) ssl = get_ssl_context(self, ssl) diff --git a/sanic/request.py b/sanic/request.py index a96dc37f96..2e30ae149a 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -21,7 +21,7 @@ from sanic.models.http_types import Credentials -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic.server import ConnInfo from sanic.app import Sanic diff --git a/sanic/server/events.py b/sanic/server/events.py index 41a89aea1d..ae93c78e9c 100644 --- a/sanic/server/events.py +++ b/sanic/server/events.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Callable, Iterable, Optional -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic import Sanic diff --git a/sanic/server/protocols/base_protocol.py b/sanic/server/protocols/base_protocol.py index 3a2716698f..63d4bfb5b7 100644 --- a/sanic/server/protocols/base_protocol.py +++ b/sanic/server/protocols/base_protocol.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Optional -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic.app import Sanic import asyncio diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index c8fcf5e3e3..0b3d55b8fb 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -8,7 +8,7 @@ from sanic.touchup.meta import TouchUpMeta -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic.app import Sanic import sys diff --git a/sanic/server/protocols/websocket_protocol.py b/sanic/server/protocols/websocket_protocol.py index 866f52460f..e55ebdd610 100644 --- a/sanic/server/protocols/websocket_protocol.py +++ b/sanic/server/protocols/websocket_protocol.py @@ -11,7 +11,7 @@ from ..websockets.impl import WebsocketImplProtocol -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from websockets import http11 diff --git a/sanic/server/websockets/frame.py b/sanic/server/websockets/frame.py index e4972516a3..b31e93c115 100644 --- a/sanic/server/websockets/frame.py +++ b/sanic/server/websockets/frame.py @@ -9,7 +9,7 @@ from sanic.exceptions import ServerError -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from .impl import WebsocketImplProtocol UTF8Decoder = codecs.getincrementaldecoder("utf-8") @@ -37,7 +37,7 @@ class WebsocketFrameAssembler: "get_id", "put_id", ) - if TYPE_CHECKING: # no cov + if TYPE_CHECKING: protocol: "WebsocketImplProtocol" read_mutex: asyncio.Lock write_mutex: asyncio.Lock diff --git a/sanic/touchup/schemes/altsvc.py b/sanic/touchup/schemes/altsvc.py index fc4b90d0aa..05e7269bbe 100644 --- a/sanic/touchup/schemes/altsvc.py +++ b/sanic/touchup/schemes/altsvc.py @@ -8,7 +8,7 @@ from .base import BaseScheme -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic import Sanic diff --git a/sanic/views.py b/sanic/views.py index 23cd110d22..627f20680e 100644 --- a/sanic/views.py +++ b/sanic/views.py @@ -13,7 +13,7 @@ from sanic.models.handler_types import RouteHandler -if TYPE_CHECKING: # no cov +if TYPE_CHECKING: from sanic import Sanic from sanic.blueprints import Blueprint diff --git a/tests/http3/test_http_receiver.py b/tests/http3/test_http_receiver.py index 0cfeab047b..458ab5d1ea 100644 --- a/tests/http3/test_http_receiver.py +++ b/tests/http3/test_http_receiver.py @@ -158,6 +158,7 @@ def test_http3_events(app): receiver = http3.receivers[1] assert len(http3.receivers) == 1 + assert receiver.request.stream_id == 1 assert receiver.request.path == "/location" assert receiver.request.method == "GET" assert receiver.request.headers["foo"] == "bar" diff --git a/tests/test_http.py b/tests/test_http.py index ec5357b09d..4d8051c63a 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1,9 +1,7 @@ -import asyncio import json as stdjson from collections import namedtuple -from textwrap import dedent -from typing import AnyStr +from pathlib import Path import pytest @@ -12,51 +10,13 @@ from sanic import json, text from sanic.app import Sanic +from .client import RawClient -PORT = 1234 +parent_dir = Path(__file__).parent +localhost_dir = parent_dir / "certs/localhost" -class RawClient: - CRLF = b"\r\n" - - def __init__(self, host: str, port: int): - self.reader = None - self.writer = None - self.host = host - self.port = port - - async def connect(self): - self.reader, self.writer = await asyncio.open_connection( - self.host, self.port - ) - - async def close(self): - self.writer.close() - await self.writer.wait_closed() - - async def send(self, message: AnyStr): - if isinstance(message, str): - msg = self._clean(message).encode("utf-8") - else: - msg = message - await self._send(msg) - - async def _send(self, message: bytes): - if not self.writer: - raise Exception("No open write stream") - self.writer.write(message) - - async def recv(self, nbytes: int = -1) -> bytes: - if not self.reader: - raise Exception("No open read stream") - return await self.reader.read(nbytes) - - def _clean(self, message: str) -> str: - return ( - dedent(message) - .lstrip("\n") - .replace("\n", self.CRLF.decode("utf-8")) - ) +PORT = 1234 @pytest.fixture diff --git a/tests/test_unix_socket.py b/tests/test_unix_socket.py index 12b286b400..aa4cd68560 100644 --- a/tests/test_unix_socket.py +++ b/tests/test_unix_socket.py @@ -53,7 +53,7 @@ def test_unix_socket_creation(caplog): assert os.path.exists(SOCKPATH) ino = os.stat(SOCKPATH).st_ino - app = Sanic(name=__name__) + app = Sanic(name="test") @app.listener("after_server_start") def running(app, loop): @@ -74,7 +74,7 @@ def running(app, loop): @pytest.mark.parametrize("path", (".", "no-such-directory/sanictest.sock")) def test_invalid_paths(path): - app = Sanic(name=__name__) + app = Sanic(name="test") with pytest.raises((FileExistsError, FileNotFoundError)): app.run(unix=path) @@ -84,7 +84,7 @@ def test_dont_replace_file(): with open(SOCKPATH, "w") as f: f.write("File, not socket") - app = Sanic(name=__name__) + app = Sanic(name="test") @app.listener("after_server_start") def stop(app, loop): @@ -101,7 +101,7 @@ def test_dont_follow_symlink(): sock.bind(SOCKPATH2) os.symlink(SOCKPATH2, SOCKPATH) - app = Sanic(name=__name__) + app = Sanic(name="test") @app.listener("after_server_start") def stop(app, loop): @@ -112,7 +112,7 @@ def stop(app, loop): def test_socket_deleted_while_running(): - app = Sanic(name=__name__) + app = Sanic(name="test") @app.listener("after_server_start") async def hack(app, loop): @@ -123,7 +123,7 @@ async def hack(app, loop): def test_socket_replaced_with_file(): - app = Sanic(name=__name__) + app = Sanic(name="test") @app.listener("after_server_start") async def hack(app, loop): @@ -136,7 +136,7 @@ async def hack(app, loop): def test_unix_connection(): - app = Sanic(name=__name__) + app = Sanic(name="test") @app.get("/") def handler(request): @@ -159,7 +159,7 @@ async def client(app, loop): app.run(host="myhost.invalid", unix=SOCKPATH) -app_multi = Sanic(name=__name__) +app_multi = Sanic(name="test") def handler(request): From 1222d11731d04437def8bf5b918688af53a9fea9 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 01:15:09 +0300 Subject: [PATCH 30/59] Change relative import of test client --- tests/test_http.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_http.py b/tests/test_http.py index 4d8051c63a..1e385449c5 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -9,8 +9,7 @@ from sanic import json, text from sanic.app import Sanic - -from .client import RawClient +from tests.client import RawClient parent_dir = Path(__file__).parent From d1ddbcac052b99c60d5fd0af82ae9a23dcee5827 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 01:49:07 +0300 Subject: [PATCH 31/59] TLS creators tests --- tests/test_tls.py | 123 +++++++++++++++++++++++++++++----------------- 1 file changed, 77 insertions(+), 46 deletions(-) diff --git a/tests/test_tls.py b/tests/test_tls.py index b0674be49e..0e0e5d7a10 100644 --- a/tests/test_tls.py +++ b/tests/test_tls.py @@ -1,18 +1,20 @@ import logging import os import ssl -import uuid +import subprocess from contextlib import contextmanager +from pathlib import Path +from unittest.mock import Mock, patch from urllib.parse import urlparse import pytest -from sanic_testing.testing import HOST, PORT, SanicTestClient +from sanic_testing.testing import HOST, PORT from sanic import Sanic -from sanic.compat import OS_IS_WINDOWS -from sanic.log import logger +from sanic.helpers import _default +from sanic.http.tls.creators import MkcertCreator from sanic.response import text @@ -28,7 +30,8 @@ @contextmanager def replace_server_name(hostname): - """Temporarily replace the server name sent with all TLS requests with a fake hostname.""" + """Temporarily replace the server name sent with all TLS requests with + a fake hostname.""" def hack_wrap_bio( self, @@ -69,8 +72,7 @@ async def handler(request): app.add_route(handler, path) - port = app.test_client.port - request, response = app.test_client.get( + request, _ = app.test_client.get( f"https://{HOST}:{PORT}" + path + f"?{query}", server_kwargs={"ssl": context}, ) @@ -100,7 +102,7 @@ async def handler(request): app.add_route(handler, path) - request, response = app.test_client.get( + request, _ = app.test_client.get( f"https://{HOST}:{PORT}" + path + f"?{query}", server_kwargs={"ssl": ssl_dict}, ) @@ -116,22 +118,22 @@ async def handler(request): def test_cert_sni_single(app): @app.get("/sni") - async def handler(request): + async def handler1(request): return text(request.conn_info.server_name) @app.get("/commonname") - async def handler(request): + async def handler2(request): return text(request.conn_info.cert.get("commonName")) port = app.test_client.port - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://localhost:{port}/sni", server_kwargs={"ssl": localhost_dir}, ) assert response.status == 200 assert response.text == "localhost" - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://localhost:{port}/commonname", server_kwargs={"ssl": localhost_dir}, ) @@ -143,16 +145,16 @@ def test_cert_sni_list(app): ssl_list = [sanic_dir, localhost_dir] @app.get("/sni") - async def handler(request): + async def handler1(request): return text(request.conn_info.server_name) @app.get("/commonname") - async def handler(request): + async def handler2(request): return text(request.conn_info.cert.get("commonName")) # This test should match the localhost cert port = app.test_client.port - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://localhost:{port}/sni", server_kwargs={"ssl": ssl_list}, ) @@ -168,14 +170,14 @@ async def handler(request): # This part should use the sanic.example cert because it matches with replace_server_name("www.sanic.example"): - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://127.0.0.1:{port}/sni", server_kwargs={"ssl": ssl_list}, ) assert response.status == 200 assert response.text == "www.sanic.example" - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://127.0.0.1:{port}/commonname", server_kwargs={"ssl": ssl_list}, ) @@ -184,14 +186,14 @@ async def handler(request): # This part should use the sanic.example cert, that being the first listed with replace_server_name("invalid.test"): - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://127.0.0.1:{port}/sni", server_kwargs={"ssl": ssl_list}, ) assert response.status == 200 assert response.text == "invalid.test" - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://127.0.0.1:{port}/commonname", server_kwargs={"ssl": ssl_list}, ) @@ -200,7 +202,8 @@ async def handler(request): def test_missing_sni(app): - """The sanic cert does not list 127.0.0.1 and httpx does not send IP as SNI anyway.""" + """The sanic cert does not list 127.0.0.1 and httpx does not send + IP as SNI anyway.""" ssl_list = [None, sanic_dir] @app.get("/sni") @@ -209,7 +212,7 @@ async def handler(request): port = app.test_client.port with pytest.raises(Exception) as exc: - request, response = app.test_client.get( + app.test_client.get( f"https://127.0.0.1:{port}/sni", server_kwargs={"ssl": ssl_list}, ) @@ -217,7 +220,8 @@ async def handler(request): def test_no_matching_cert(app): - """The sanic cert does not list 127.0.0.1 and httpx does not send IP as SNI anyway.""" + """The sanic cert does not list 127.0.0.1 and httpx does not send + IP as SNI anyway.""" ssl_list = [None, sanic_dir] @app.get("/sni") @@ -227,7 +231,7 @@ async def handler(request): port = app.test_client.port with replace_server_name("invalid.test"): with pytest.raises(Exception) as exc: - request, response = app.test_client.get( + app.test_client.get( f"https://127.0.0.1:{port}/sni", server_kwargs={"ssl": ssl_list}, ) @@ -244,7 +248,7 @@ async def handler(request): port = app.test_client.port with replace_server_name("foo.sanic.test"): - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://127.0.0.1:{port}/sni", server_kwargs={"ssl": ssl_list}, ) @@ -253,14 +257,14 @@ async def handler(request): with replace_server_name("sanic.test"): with pytest.raises(Exception) as exc: - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://127.0.0.1:{port}/sni", server_kwargs={"ssl": ssl_list}, ) assert "Request and response object expected" in str(exc.value) with replace_server_name("sub.foo.sanic.test"): with pytest.raises(Exception) as exc: - request, response = app.test_client.get( + _, response = app.test_client.get( f"https://127.0.0.1:{port}/sni", server_kwargs={"ssl": ssl_list}, ) @@ -275,9 +279,7 @@ async def handler(request): ssl_dict = {"cert": None, "key": None} with pytest.raises(ValueError) as excinfo: - request, response = app.test_client.get( - "/test", server_kwargs={"ssl": ssl_dict} - ) + app.test_client.get("/test", server_kwargs={"ssl": ssl_dict}) assert str(excinfo.value) == "SSL dict needs filenames for cert and key." @@ -288,9 +290,7 @@ async def handler(request): return text("ssl test") with pytest.raises(ValueError) as excinfo: - request, response = app.test_client.get( - "/test", server_kwargs={"ssl": False} - ) + app.test_client.get("/test", server_kwargs={"ssl": False}) assert "Invalid ssl argument" in str(excinfo.value) @@ -303,9 +303,7 @@ async def handler(request): ssl_list = [sanic_cert] with pytest.raises(ValueError) as excinfo: - request, response = app.test_client.get( - "/test", server_kwargs={"ssl": ssl_list} - ) + app.test_client.get("/test", server_kwargs={"ssl": ssl_list}) assert "folder expected" in str(excinfo.value) assert sanic_cert in str(excinfo.value) @@ -319,9 +317,7 @@ async def handler(request): ssl_list = [invalid_dir] with pytest.raises(ValueError) as excinfo: - request, response = app.test_client.get( - "/test", server_kwargs={"ssl": ssl_list} - ) + app.test_client.get("/test", server_kwargs={"ssl": ssl_list}) assert "not found" in str(excinfo.value) assert invalid_dir + "/privkey.pem" in str(excinfo.value) @@ -336,9 +332,7 @@ async def handler(request): ssl_list = [invalid2] with pytest.raises(ValueError) as excinfo: - request, response = app.test_client.get( - "/test", server_kwargs={"ssl": ssl_list} - ) + app.test_client.get("/test", server_kwargs={"ssl": ssl_list}) assert "not found" in str(excinfo.value) assert invalid2 + "/fullchain.pem" in str(excinfo.value) @@ -352,15 +346,13 @@ async def handler(request): ssl_list = [None] with pytest.raises(ValueError) as excinfo: - request, response = app.test_client.get( - "/test", server_kwargs={"ssl": ssl_list} - ) + app.test_client.get("/test", server_kwargs={"ssl": ssl_list}) assert "No certificates" in str(excinfo.value) def test_logger_vhosts(caplog): - app = Sanic(name=__name__) + app = Sanic(name="test_logger_vhosts") @app.after_server_start def stop(*args): @@ -374,5 +366,44 @@ def stop(*args): ][0] assert logmsg == ( - "Certificate vhosts: localhost, 127.0.0.1, 0:0:0:0:0:0:0:1, sanic.example, www.sanic.example, *.sanic.test, 2001:DB8:0:0:0:0:0:541C" + "Certificate vhosts: localhost, 127.0.0.1, 0:0:0:0:0:0:0:1, " + "sanic.example, www.sanic.example, *.sanic.test, " + "2001:DB8:0:0:0:0:0:541C" ) + + +def test_mk_cert_creator_default(app: Sanic): + cert_creator = MkcertCreator(app, _default, _default) + assert isinstance(cert_creator.tmpdir, Path) + assert cert_creator.tmpdir.exists() + + +def test_mk_cert_creator_is_supported(app): + cert_creator = MkcertCreator(app, _default, _default) + with patch("subprocess.run") as run: + cert_creator.check_supported() + run.assert_called_once_with( + ["mkcert", "-help"], + check=True, + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + + +def test_mk_cert_creator_generate_cert_default(app): + cert_creator = MkcertCreator(app, _default, _default) + with patch("subprocess.run") as run: + with patch("sanic.http.tls.creators.CertSimple"): + retval = Mock() + retval.stdout = "foo" + run.return_value = retval + cert_creator.generate_cert("localhost") + run.assert_called_once() + + +def test_mk_cert_creator_generate_cert_localhost(app): + cert_creator = MkcertCreator(app, localhost_key, localhost_cert) + with patch("subprocess.run") as run: + with patch("sanic.http.tls.creators.CertSimple"): + cert_creator.generate_cert("localhost") + run.assert_not_called() From c2792eb83b3ac6b31bc30a504e6d9a33de51933b Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 02:00:32 +0300 Subject: [PATCH 32/59] Update tests --- tests/.benchmarks/__init__.py | 0 tests/__init__.py | 0 tests/__pycache__/__init__.py | 0 tests/benchmark/__init__.py | 0 tests/benchmark/__pycache__/__init__.py | 0 tests/certs/__init__.py | 0 tests/certs/invalid.certmissing/__init__.py | 0 tests/certs/localhost/__init__.py | 0 tests/certs/sanic.example/__init__.py | 0 tests/client.py | 47 ++++++++++++++++ tests/fake/__init__.py | 0 tests/fake/__pycache__/__init__.py | 0 tests/http3/__init__.py | 0 tests/http3/__pycache__/__init__.py | 0 tests/performance/__init__.py | 0 tests/performance/aiohttp/__init__.py | 0 tests/performance/bottle/__init__.py | 0 tests/performance/falcon/__init__.py | 0 tests/performance/golang/__init__.py | 0 tests/performance/kyoukai/__init__.py | 0 tests/performance/sanic/__init__.py | 0 tests/performance/tornado/__init__.py | 0 tests/performance/wheezy/__init__.py | 0 tests/static/__init__.py | 0 tests/static/__pycache__/__init__.py | 0 tests/static/bp/__init__.py | 0 tests/static/nested/__init__.py | 0 tests/static/nested/dir/__init__.py | 0 tests/test_http_alt_svc.py | 61 +++++++++++++++++++++ 29 files changed, 108 insertions(+) create mode 100644 tests/.benchmarks/__init__.py create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/__init__.py create mode 100644 tests/benchmark/__init__.py create mode 100644 tests/benchmark/__pycache__/__init__.py create mode 100644 tests/certs/__init__.py create mode 100644 tests/certs/invalid.certmissing/__init__.py create mode 100644 tests/certs/localhost/__init__.py create mode 100644 tests/certs/sanic.example/__init__.py create mode 100644 tests/client.py create mode 100644 tests/fake/__init__.py create mode 100644 tests/fake/__pycache__/__init__.py create mode 100644 tests/http3/__init__.py create mode 100644 tests/http3/__pycache__/__init__.py create mode 100644 tests/performance/__init__.py create mode 100644 tests/performance/aiohttp/__init__.py create mode 100644 tests/performance/bottle/__init__.py create mode 100644 tests/performance/falcon/__init__.py create mode 100644 tests/performance/golang/__init__.py create mode 100644 tests/performance/kyoukai/__init__.py create mode 100644 tests/performance/sanic/__init__.py create mode 100644 tests/performance/tornado/__init__.py create mode 100644 tests/performance/wheezy/__init__.py create mode 100644 tests/static/__init__.py create mode 100644 tests/static/__pycache__/__init__.py create mode 100644 tests/static/bp/__init__.py create mode 100644 tests/static/nested/__init__.py create mode 100644 tests/static/nested/dir/__init__.py create mode 100644 tests/test_http_alt_svc.py diff --git a/tests/.benchmarks/__init__.py b/tests/.benchmarks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/__pycache__/__init__.py b/tests/__pycache__/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/benchmark/__init__.py b/tests/benchmark/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/benchmark/__pycache__/__init__.py b/tests/benchmark/__pycache__/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/certs/__init__.py b/tests/certs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/certs/invalid.certmissing/__init__.py b/tests/certs/invalid.certmissing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/certs/localhost/__init__.py b/tests/certs/localhost/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/certs/sanic.example/__init__.py b/tests/certs/sanic.example/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/client.py b/tests/client.py new file mode 100644 index 0000000000..4c0b29a0fd --- /dev/null +++ b/tests/client.py @@ -0,0 +1,47 @@ +import asyncio + +from textwrap import dedent +from typing import AnyStr + + +class RawClient: + CRLF = b"\r\n" + + def __init__(self, host: str, port: int): + self.reader = None + self.writer = None + self.host = host + self.port = port + + async def connect(self): + self.reader, self.writer = await asyncio.open_connection( + self.host, self.port + ) + + async def close(self): + self.writer.close() + await self.writer.wait_closed() + + async def send(self, message: AnyStr): + if isinstance(message, str): + msg = self._clean(message).encode("utf-8") + else: + msg = message + await self._send(msg) + + async def _send(self, message: bytes): + if not self.writer: + raise Exception("No open write stream") + self.writer.write(message) + + async def recv(self, nbytes: int = -1) -> bytes: + if not self.reader: + raise Exception("No open read stream") + return await self.reader.read(nbytes) + + def _clean(self, message: str) -> str: + return ( + dedent(message) + .lstrip("\n") + .replace("\n", self.CRLF.decode("utf-8")) + ) diff --git a/tests/fake/__init__.py b/tests/fake/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fake/__pycache__/__init__.py b/tests/fake/__pycache__/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/http3/__init__.py b/tests/http3/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/http3/__pycache__/__init__.py b/tests/http3/__pycache__/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/performance/aiohttp/__init__.py b/tests/performance/aiohttp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/performance/bottle/__init__.py b/tests/performance/bottle/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/performance/falcon/__init__.py b/tests/performance/falcon/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/performance/golang/__init__.py b/tests/performance/golang/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/performance/kyoukai/__init__.py b/tests/performance/kyoukai/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/performance/sanic/__init__.py b/tests/performance/sanic/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/performance/tornado/__init__.py b/tests/performance/tornado/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/performance/wheezy/__init__.py b/tests/performance/wheezy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/static/__init__.py b/tests/static/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/static/__pycache__/__init__.py b/tests/static/__pycache__/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/static/bp/__init__.py b/tests/static/bp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/static/nested/__init__.py b/tests/static/nested/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/static/nested/dir/__init__.py b/tests/static/nested/dir/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_http_alt_svc.py b/tests/test_http_alt_svc.py new file mode 100644 index 0000000000..a923d4ca8e --- /dev/null +++ b/tests/test_http_alt_svc.py @@ -0,0 +1,61 @@ +from pathlib import Path + +from sanic.app import Sanic +from sanic.response import empty +from tests.client import RawClient + + +parent_dir = Path(__file__).parent +localhost_dir = parent_dir / "certs/localhost" + +PORT = 12344 + + +def test_http1_response_has_alt_svc(): + Sanic._app_registry.clear() + app = Sanic("TestAltSvc") + app.config.TOUCHUP = True + response = b"" + + @app.get("/") + async def handler(*_): + return empty() + + @app.after_server_start + async def do_request(*_): + nonlocal response + + app.router.reset() + app.router.finalize() + + client = RawClient(app.state.host, app.state.port) + await client.connect() + await client.send( + """ + GET / HTTP/1.1 + host: localhost:7777 + + """ + ) + response = await client.recv() + await client.close() + + @app.after_server_start + def shutdown(*_): + app.stop() + + app.prepare( + version=3, + ssl={ + "cert": localhost_dir / "fullchain.pem", + "key": localhost_dir / "privkey.pem", + }, + port=PORT, + ) + app.prepare( + version=1, + port=PORT, + ) + Sanic.serve() + + assert f'alt-svc: h3=":{PORT}"\r\n'.encode() in response From 9572091d092d4a9be8287d75c2a987bb109ab8c7 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 02:11:37 +0300 Subject: [PATCH 33/59] clenup --- tests/__pycache__/__init__.py | 0 tests/benchmark/__pycache__/__init__.py | 0 tests/fake/__pycache__/__init__.py | 0 tests/http3/__pycache__/__init__.py | 0 tests/static/__pycache__/__init__.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/__pycache__/__init__.py delete mode 100644 tests/benchmark/__pycache__/__init__.py delete mode 100644 tests/fake/__pycache__/__init__.py delete mode 100644 tests/http3/__pycache__/__init__.py delete mode 100644 tests/static/__pycache__/__init__.py diff --git a/tests/__pycache__/__init__.py b/tests/__pycache__/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/benchmark/__pycache__/__init__.py b/tests/benchmark/__pycache__/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/fake/__pycache__/__init__.py b/tests/fake/__pycache__/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/http3/__pycache__/__init__.py b/tests/http3/__pycache__/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/static/__pycache__/__init__.py b/tests/static/__pycache__/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From 1c9a0370f57ac4109860c6da77df1f6e769929ea Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 14:36:42 +0300 Subject: [PATCH 34/59] Add TLS creator tests --- tests/test_tls.py | 91 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/tests/test_tls.py b/tests/test_tls.py index 0e0e5d7a10..49f341ca42 100644 --- a/tests/test_tls.py +++ b/tests/test_tls.py @@ -12,9 +12,13 @@ from sanic_testing.testing import HOST, PORT +import sanic.http.tls.creators + from sanic import Sanic +from sanic.exceptions import SanicException from sanic.helpers import _default -from sanic.http.tls.creators import MkcertCreator +from sanic.http.tls.context import SanicSSLContext +from sanic.http.tls.creators import MkcertCreator, TrustmeCreator from sanic.response import text @@ -28,6 +32,31 @@ sanic_key = os.path.join(sanic_dir, "privkey.pem") +@pytest.fixture +def server_cert(): + return Mock() + + +@pytest.fixture +def issue_cert(server_cert): + mock = Mock(return_value=server_cert) + return mock + + +@pytest.fixture +def ca(issue_cert): + ca = Mock() + ca.issue_cert = issue_cert + return ca + + +@pytest.fixture +def trustme(ca): + module = Mock() + module.CA = Mock(return_value=ca) + return module + + @contextmanager def replace_server_name(hostname): """Temporarily replace the server name sent with all TLS requests with @@ -390,6 +419,16 @@ def test_mk_cert_creator_is_supported(app): ) +def test_mk_cert_creator_is_not_supported(app): + cert_creator = MkcertCreator(app, _default, _default) + with patch("subprocess.run") as run: + run.side_effect = Exception("") + with pytest.raises( + SanicException, match="Sanic is attempting to use mkcert" + ): + cert_creator.check_supported() + + def test_mk_cert_creator_generate_cert_default(app): cert_creator = MkcertCreator(app, _default, _default) with patch("subprocess.run") as run: @@ -407,3 +446,53 @@ def test_mk_cert_creator_generate_cert_localhost(app): with patch("sanic.http.tls.creators.CertSimple"): cert_creator.generate_cert("localhost") run.assert_not_called() + + +def test_trustme_creator_default(app: Sanic): + cert_creator = TrustmeCreator(app, _default, _default) + assert isinstance(cert_creator.tmpdir, Path) + assert cert_creator.tmpdir.exists() + + +def test_trustme_creator_is_supported(app, monkeypatch): + monkeypatch.setattr(sanic.http.tls.creators, "TRUSTME_INSTALLED", True) + cert_creator = TrustmeCreator(app, _default, _default) + cert_creator.check_supported() + + +def test_trustme_creator_is_not_supported(app, monkeypatch): + monkeypatch.setattr(sanic.http.tls.creators, "TRUSTME_INSTALLED", False) + cert_creator = TrustmeCreator(app, _default, _default) + with pytest.raises( + SanicException, match="Sanic is attempting to use trustme" + ): + cert_creator.check_supported() + + +def test_trustme_creator_generate_cert_default( + app, monkeypatch, trustme, issue_cert, server_cert, ca +): + monkeypatch.setattr(sanic.http.tls.creators, "trustme", trustme) + cert_creator = TrustmeCreator(app, _default, _default) + cert = cert_creator.generate_cert("localhost") + + assert isinstance(cert, SanicSSLContext) + trustme.CA.assert_called_once_with() + issue_cert.assert_called_once_with("localhost") + server_cert.configure_cert.assert_called_once() + ca.configure_trust.assert_called_once() + ca.cert_pem.write_to_path.assert_called_once_with(str(cert.sanic["cert"])) + write_to_path = server_cert.private_key_and_cert_chain_pem.write_to_path + write_to_path.assert_called_once_with(str(cert.sanic["key"])) + + +def test_trustme_creator_generate_cert_localhost( + app, monkeypatch, trustme, server_cert, ca +): + monkeypatch.setattr(sanic.http.tls.creators, "trustme", trustme) + cert_creator = TrustmeCreator(app, localhost_key, localhost_cert) + cert_creator.generate_cert("localhost") + + ca.cert_pem.write_to_path.assert_called_once_with(localhost_cert) + write_to_path = server_cert.private_key_and_cert_chain_pem.write_to_path + write_to_path.assert_called_once_with(localhost_key) From 000a1662066a66fc1aa466fb1523258712914919 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 15:12:20 +0300 Subject: [PATCH 35/59] Add TLS creator selection test --- sanic/http/tls/creators.py | 10 +++- tests/test_tls.py | 114 ++++++++++++++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/sanic/http/tls/creators.py b/sanic/http/tls/creators.py index 90e0f7f6b3..b4c981dc02 100644 --- a/sanic/http/tls/creators.py +++ b/sanic/http/tls/creators.py @@ -142,9 +142,15 @@ def select( local_tls_key, local_tls_cert, ) + if creator: + break if not creator: - raise SanicException("...") + raise SanicException( + "Sanic could not find package to create a TLS certificate. " + "You must have either mkcert or trustme installed. See " + "_____ for more details." + ) return creator @@ -170,6 +176,8 @@ def _try_select( except SanicException: if creator_requested is creator_requirement: raise + else: + return None return instance diff --git a/tests/test_tls.py b/tests/test_tls.py index 49f341ca42..b8115b436c 100644 --- a/tests/test_tls.py +++ b/tests/test_tls.py @@ -15,10 +15,16 @@ import sanic.http.tls.creators from sanic import Sanic +from sanic.application.constants import Mode +from sanic.constants import LocalCertCreator from sanic.exceptions import SanicException from sanic.helpers import _default from sanic.http.tls.context import SanicSSLContext -from sanic.http.tls.creators import MkcertCreator, TrustmeCreator +from sanic.http.tls.creators import ( + MkcertCreator, + TrustmeCreator, + get_ssl_context, +) from sanic.response import text @@ -57,6 +63,34 @@ def trustme(ca): return module +@pytest.fixture +def MockMkcertCreator(): + class Creator(MkcertCreator): + SUPPORTED = True + + def check_supported(self): + if not self.SUPPORTED: + raise SanicException("Nope") + + generate_cert = Mock() + + return Creator + + +@pytest.fixture +def MockTrustmeCreator(): + class Creator(TrustmeCreator): + SUPPORTED = True + + def check_supported(self): + if not self.SUPPORTED: + raise SanicException("Nope") + + generate_cert = Mock() + + return Creator + + @contextmanager def replace_server_name(hostname): """Temporarily replace the server name sent with all TLS requests with @@ -496,3 +530,81 @@ def test_trustme_creator_generate_cert_localhost( ca.cert_pem.write_to_path.assert_called_once_with(localhost_cert) write_to_path = server_cert.private_key_and_cert_chain_pem.write_to_path write_to_path.assert_called_once_with(localhost_key) + + +def test_get_ssl_context_with_ssl_context(app): + mock_context = Mock() + context = get_ssl_context(app, mock_context) + assert context is mock_context + + +def test_get_ssl_context_in_production(app): + app.state.mode = Mode.PRODUCTION + with pytest.raises( + SanicException, + match="Cannot run Sanic as an HTTPS server in PRODUCTION mode", + ): + get_ssl_context(app, None) + + +@pytest.mark.parametrize( + "requirement,mk_supported,trustme_supported,mk_called,trustme_called,err", + ( + (LocalCertCreator.AUTO, True, False, True, False, None), + (LocalCertCreator.AUTO, True, True, True, False, None), + (LocalCertCreator.AUTO, False, True, False, True, None), + ( + LocalCertCreator.AUTO, + False, + False, + False, + False, + "Sanic could not find package to create a TLS certificate", + ), + (LocalCertCreator.MKCERT, True, False, True, False, None), + (LocalCertCreator.MKCERT, True, True, True, False, None), + (LocalCertCreator.MKCERT, False, True, False, False, "Nope"), + (LocalCertCreator.MKCERT, False, False, False, False, "Nope"), + (LocalCertCreator.TRUSTME, True, False, False, False, "Nope"), + (LocalCertCreator.TRUSTME, True, True, False, True, None), + (LocalCertCreator.TRUSTME, False, True, False, True, None), + (LocalCertCreator.TRUSTME, False, False, False, False, "Nope"), + ), +) +def test_get_ssl_context_only_mkcert( + app, + monkeypatch, + MockMkcertCreator, + MockTrustmeCreator, + requirement, + mk_supported, + trustme_supported, + mk_called, + trustme_called, + err, +): + app.state.mode = Mode.DEBUG + app.config.LOCAL_CERT_CREATOR = requirement + monkeypatch.setattr( + sanic.http.tls.creators, "MkcertCreator", MockMkcertCreator + ) + monkeypatch.setattr( + sanic.http.tls.creators, "TrustmeCreator", MockTrustmeCreator + ) + MockMkcertCreator.SUPPORTED = mk_supported + MockTrustmeCreator.SUPPORTED = trustme_supported + + if err: + with pytest.raises(SanicException, match=err): + get_ssl_context(app, None) + else: + get_ssl_context(app, None) + + if mk_called: + MockMkcertCreator.generate_cert.assert_called_once_with("localhost") + else: + MockMkcertCreator.generate_cert.assert_not_called() + if trustme_called: + MockTrustmeCreator.generate_cert.assert_called_once_with("localhost") + else: + MockTrustmeCreator.generate_cert.assert_not_called() From ec33f8a8397d64baa3fe517df10a4c5dfedcd923 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 15:28:39 +0300 Subject: [PATCH 36/59] Remove unneeded files --- tests/performance/__init__.py | 0 tests/performance/aiohttp/__init__.py | 0 tests/performance/bottle/__init__.py | 0 tests/performance/falcon/__init__.py | 0 tests/performance/golang/__init__.py | 0 tests/performance/kyoukai/__init__.py | 0 tests/performance/sanic/__init__.py | 0 tests/performance/tornado/__init__.py | 0 tests/performance/wheezy/__init__.py | 0 9 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/performance/__init__.py delete mode 100644 tests/performance/aiohttp/__init__.py delete mode 100644 tests/performance/bottle/__init__.py delete mode 100644 tests/performance/falcon/__init__.py delete mode 100644 tests/performance/golang/__init__.py delete mode 100644 tests/performance/kyoukai/__init__.py delete mode 100644 tests/performance/sanic/__init__.py delete mode 100644 tests/performance/tornado/__init__.py delete mode 100644 tests/performance/wheezy/__init__.py diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/performance/aiohttp/__init__.py b/tests/performance/aiohttp/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/performance/bottle/__init__.py b/tests/performance/bottle/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/performance/falcon/__init__.py b/tests/performance/falcon/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/performance/golang/__init__.py b/tests/performance/golang/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/performance/kyoukai/__init__.py b/tests/performance/kyoukai/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/performance/sanic/__init__.py b/tests/performance/sanic/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/performance/tornado/__init__.py b/tests/performance/tornado/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/performance/wheezy/__init__.py b/tests/performance/wheezy/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From f1cfc34339c9d61d0cc36c95ab3243b1e7bc01cb Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 15:39:19 +0300 Subject: [PATCH 37/59] Cleanup config tests --- sanic/config.py | 26 +++++++++++++------------- tests/test_config.py | 37 ++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/sanic/config.py b/sanic/config.py index 115c0bf95b..fd63ca5479 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -168,19 +168,19 @@ def _post_set(self, attr, value) -> None: "REQUEST_MAX_SIZE", ): self._configure_header_size() - elif attr == "LOGO": - self._LOGO = value - deprecation( - "Setting the config.LOGO is deprecated and will no longer " - "be supported starting in v22.6.", - 22.6, - ) - elif attr == "LOCAL_CERT_CREATOR" and not isinstance( - self.LOCAL_CERT_CREATOR, LocalCertCreator - ): - self.LOCAL_CERT_CREATOR = LocalCertCreator[ - self.LOCAL_CERT_CREATOR.upper() - ] + if attr == "LOGO": + self._LOGO = value + deprecation( + "Setting the config.LOGO is deprecated and will no longer " + "be supported starting in v22.6.", + 22.6, + ) + elif attr == "LOCAL_CERT_CREATOR" and not isinstance( + self.LOCAL_CERT_CREATOR, LocalCertCreator + ): + self.LOCAL_CERT_CREATOR = LocalCertCreator[ + self.LOCAL_CERT_CREATOR.upper() + ] @property def LOGO(self): diff --git a/tests/test_config.py b/tests/test_config.py index d8a7bd85b3..7a2dea4500 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,5 @@ import logging +import os from contextlib import contextmanager from os import environ @@ -13,6 +14,7 @@ from sanic import Sanic from sanic.config import DEFAULT_CONFIG, Config +from sanic.constants import LocalCertCreator from sanic.exceptions import PyFileError @@ -49,7 +51,7 @@ def test_load_from_object(app: Sanic): def test_load_from_object_string(app: Sanic): - app.config.load("test_config.ConfigTest") + app.config.load("tests.test_config.ConfigTest") assert "CONFIG_VALUE" in app.config assert app.config.CONFIG_VALUE == "should be used" assert "not_for_config" not in app.config @@ -71,14 +73,14 @@ def test_load_from_object_string_exception(app: Sanic): def test_auto_env_prefix(): environ["SANIC_TEST_ANSWER"] = "42" - app = Sanic(name=__name__) + app = Sanic(name="Test") assert app.config.TEST_ANSWER == 42 del environ["SANIC_TEST_ANSWER"] def test_auto_bool_env_prefix(): environ["SANIC_TEST_ANSWER"] = "True" - app = Sanic(name=__name__) + app = Sanic(name="Test") assert app.config.TEST_ANSWER is True del environ["SANIC_TEST_ANSWER"] @@ -86,28 +88,28 @@ def test_auto_bool_env_prefix(): @pytest.mark.parametrize("env_prefix", [None, ""]) def test_empty_load_env_prefix(env_prefix): environ["SANIC_TEST_ANSWER"] = "42" - app = Sanic(name=__name__, env_prefix=env_prefix) + app = Sanic(name="Test", env_prefix=env_prefix) assert getattr(app.config, "TEST_ANSWER", None) is None del environ["SANIC_TEST_ANSWER"] def test_env_prefix(): environ["MYAPP_TEST_ANSWER"] = "42" - app = Sanic(name=__name__, env_prefix="MYAPP_") + app = Sanic(name="Test", env_prefix="MYAPP_") assert app.config.TEST_ANSWER == 42 del environ["MYAPP_TEST_ANSWER"] def test_env_prefix_float_values(): environ["MYAPP_TEST_ROI"] = "2.3" - app = Sanic(name=__name__, env_prefix="MYAPP_") + app = Sanic(name="Test", env_prefix="MYAPP_") assert app.config.TEST_ROI == 2.3 del environ["MYAPP_TEST_ROI"] def test_env_prefix_string_value(): environ["MYAPP_TEST_TOKEN"] = "somerandomtesttoken" - app = Sanic(name=__name__, env_prefix="MYAPP_") + app = Sanic(name="Test", env_prefix="MYAPP_") assert app.config.TEST_TOKEN == "somerandomtesttoken" del environ["MYAPP_TEST_TOKEN"] @@ -116,7 +118,7 @@ def test_env_w_custom_converter(): environ["SANIC_TEST_ANSWER"] = "42" config = Config(converters=[UltimateAnswer]) - app = Sanic(name=__name__, config=config) + app = Sanic(name="Test", config=config) assert isinstance(app.config.TEST_ANSWER, UltimateAnswer) assert app.config.TEST_ANSWER.answer == 42 del environ["SANIC_TEST_ANSWER"] @@ -125,7 +127,7 @@ def test_env_w_custom_converter(): def test_env_lowercase(): with pytest.warns(None) as record: environ["SANIC_test_answer"] = "42" - app = Sanic(name=__name__) + app = Sanic(name="Test") assert app.config.test_answer == 42 assert str(record[0].message) == ( "[DEPRECATION v22.9] Lowercase environment variables will not be " @@ -435,3 +437,20 @@ def test_negative_proxy_count(app: Sanic): ) with pytest.raises(ValueError, match=message): app.prepare() + + +@pytest.mark.parametrize( + "passed,expected", + ( + ("auto", LocalCertCreator.AUTO), + ("mkcert", LocalCertCreator.MKCERT), + ("trustme", LocalCertCreator.TRUSTME), + ("AUTO", LocalCertCreator.AUTO), + ("MKCERT", LocalCertCreator.MKCERT), + ("TRUSTME", LocalCertCreator.TRUSTME), + ), +) +def test_convert_local_cert_creator(passed, expected): + os.environ["SANIC_LOCAL_CERT_CREATOR"] = passed + app = Sanic("Test") + assert app.config.LOCAL_CERT_CREATOR is expected From 53bf127db886abc39734fbf4c5817393d516a0b7 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 16:05:40 +0300 Subject: [PATCH 38/59] Add some typeing --- sanic/http/http1.py | 20 +------------------- sanic/http/http3.py | 39 ++++++++++++++++----------------------- sanic/http/stream.py | 23 +++++++++++++++++++++++ 3 files changed, 40 insertions(+), 42 deletions(-) diff --git a/sanic/http/http1.py b/sanic/http/http1.py index 4d5fe6f4d7..8b0e6cbe8b 100644 --- a/sanic/http/http1.py +++ b/sanic/http/http1.py @@ -49,7 +49,7 @@ class Http(Stream, metaclass=TouchUpMeta): HEADER_CEILING = 16_384 HEADER_MAX_SIZE = 0 - + __version__ = "1.1" __touchup__ = ( "http1_request_header", "http1_response_header", @@ -427,24 +427,6 @@ async def error_response(self, exception: Exception) -> None: await app.handle_exception(self.request, exception) - def create_empty_request(self) -> None: - """ - Current error handling code needs a request object that won't exist - if an error occurred during before a request was received. Create a - bogus response for error handling use. - """ - - # FIXME: Avoid this by refactoring error handling and response code - self.request = self.protocol.request_class( - url_bytes=self.url.encode() if self.url else b"*", - headers=Header({}), - version="1.1", - method="NONE", - transport=self.protocol.transport, - app=self.protocol.app, - ) - self.request.stream = self - def log_response(self) -> None: """ Helper method provided to enable the logging of responses in case if diff --git a/sanic/http/http3.py b/sanic/http/http3.py index e141762afc..7b0807255c 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -66,6 +66,9 @@ async def run(self): # no cov class HTTPReceiver(Receiver, Stream): stage: Stage + request: Request + + __version__ = "3" def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -105,24 +108,13 @@ async def error_response(self, exception: Exception) -> None: """ Handle response when exception encountered """ - # Disconnect after an error if in any other state than handler - # if self.stage is not Stage.HANDLER: - # self.keep_alive = False - - # TODO: - # - Do we need this? - # Request failure? Respond but then disconnect - if self.stage is Stage.REQUEST: - self.stage = Stage.HANDLER - # From request and handler states we can respond, otherwise be silent - if self.stage is Stage.HANDLER: - app = self.protocol.app + app = self.protocol.app - # if self.request is None: - # self.create_empty_request() + if self.request is None: + self.create_empty_request() - await app.handle_exception(self.request, exception) + await app.handle_exception(self.request, exception) def _prepare_headers( self, response: BaseHTTPResponse @@ -315,23 +307,24 @@ def get_receiver_by_stream_id(self, stream_id: int) -> Receiver: return self.receivers[stream_id] def _make_request(self, event: HeadersReceived) -> Request: - # TODO: - # Cleanup - # method_header, path_header, *rem = event.headers headers = Header(((k.decode(), v.decode()) for k, v in event.headers)) - # method = method_header[1].decode() - # path = path_header[1] method = headers[":method"] - path = headers[":path"].encode() + path = headers[":path"] scheme = headers.pop(":scheme", "") - authority = headers.pop(":authority", "") + self.url = path if authority: headers["host"] = authority request = self.protocol.request_class( - path, headers, "3", method, Transport(), self.protocol.app, b"" + path.encode(), + headers, + "3", + method, + Transport(), + self.protocol.app, + b"", ) request._stream_id = event.stream_id request._scheme = scheme diff --git a/sanic/http/stream.py b/sanic/http/stream.py index e71338b1a6..b2d034391e 100644 --- a/sanic/http/stream.py +++ b/sanic/http/stream.py @@ -2,18 +2,41 @@ from typing import TYPE_CHECKING, Optional, Tuple +from sanic.compat import Header from sanic.http.constants import Stage if TYPE_CHECKING: from sanic.response import BaseHTTPResponse + from sanic.server.protocols.http_protocol import HttpProtocol class Stream: stage: Stage response: Optional[BaseHTTPResponse] + protocol: HttpProtocol + url: Optional[str] + __version__ = "unknown" __touchup__: Tuple[str, ...] = tuple() __slots__ = () def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: raise NotImplementedError("Not implemented") + + def create_empty_request(self) -> None: + """ + Current error handling code needs a request object that won't exist + if an error occurred during before a request was received. Create a + bogus response for error handling use. + """ + + # FIXME: Avoid this by refactoring error handling and response code + self.request = self.protocol.request_class( + url_bytes=self.url.encode() if self.url else b"*", + headers=Header({}), + version=self.__class__.__version__, + method="NONE", + transport=self.protocol.transport, + app=self.protocol.app, + ) + self.request.stream = self From d228ae3ab9aa2c2913c48f85e8e639c51b1dce7a Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 16:11:25 +0300 Subject: [PATCH 39/59] Remove unnecessary coverage --- sanic/http/tls/creators.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sanic/http/tls/creators.py b/sanic/http/tls/creators.py index b4c981dc02..bc0369d766 100644 --- a/sanic/http/tls/creators.py +++ b/sanic/http/tls/creators.py @@ -25,7 +25,7 @@ try: import trustme - TRUSTME_INSTALLED = True + TRUSTME_INSTALLED = True # noqa except (ImportError, ModuleNotFoundError): TRUSTME_INSTALLED = False @@ -108,11 +108,11 @@ def __init__(self, app, key, cert) -> None: self.cert_path = _make_path(cert, self.tmpdir) @abstractmethod - def check_supported(self) -> None: + def check_supported(self) -> None: # no cov ... @abstractmethod - def generate_cert(self, localhost: str) -> ssl.SSLContext: + def generate_cert(self, localhost: str) -> ssl.SSLContext: # no cov ... @classmethod @@ -230,7 +230,7 @@ def generate_cert(self, localhost: str) -> ssl.SSLContext: finally: @self.app.main_process_stop - async def cleanup(*_): + async def cleanup(*_): # no cov if self.tmpdir: with suppress(FileNotFoundError): self.key_path.unlink() From a8c65fc7da1ea8498ab55eb7eb213b6d5538725b Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 16:33:21 +0300 Subject: [PATCH 40/59] Reduce duplicate code in server runners --- sanic/http/http3.py | 5 +- sanic/http/tls/creators.py | 2 - sanic/server/runners.py | 111 +++++++++++++++++++------------------ 3 files changed, 60 insertions(+), 58 deletions(-) diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 7b0807255c..52f0a123b4 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -3,6 +3,7 @@ import asyncio from abc import ABC, abstractmethod +from ssl import SSLContext from typing import ( TYPE_CHECKING, Callable, @@ -347,7 +348,9 @@ def pop(self, label: bytes) -> Optional[SessionTicket]: return self.tickets.pop(label, None) -def get_config(app: Sanic, ssl: Union[SanicSSLContext, CertSelector]): +def get_config( + app: Sanic, ssl: Union[SanicSSLContext, CertSelector, SSLContext] +): # TODO: # - proper selection needed if servince with multiple certs insted of # just taking the first diff --git a/sanic/http/tls/creators.py b/sanic/http/tls/creators.py index bc0369d766..808e3565c0 100644 --- a/sanic/http/tls/creators.py +++ b/sanic/http/tls/creators.py @@ -272,6 +272,4 @@ def generate_cert(self, localhost: str) -> ssl.SSLContext: str(self.key_path.absolute()) ) - sanic_context.verify_mode = False - return context diff --git a/sanic/server/runners.py b/sanic/server/runners.py index 86d2e14fca..81c8b64a44 100644 --- a/sanic/server/runners.py +++ b/sanic/server/runners.py @@ -117,6 +117,45 @@ def serve( ) +def _setup_system_signals( + app: Sanic, + run_multiple: bool, + register_sys_signals: bool, + loop: asyncio.AbstractEventLoop, +) -> None: + # Ignore SIGINT when run_multiple + if run_multiple: + signal_func(SIGINT, SIG_IGN) + os.environ["SANIC_WORKER_PROCESS"] = "true" + + # Register signals for graceful termination + if register_sys_signals: + if OS_IS_WINDOWS: + ctrlc_workaround_for_windows(app) + else: + for _signal in [SIGTERM] if run_multiple else [SIGINT, SIGTERM]: + loop.add_signal_handler(_signal, app.stop) + + +def _run_server_forever(loop, before_stop, after_stop, cleanup, unix): + pid = os.getpid() + try: + logger.info("Starting worker [%s]", pid) + loop.run_forever() + except KeyboardInterrupt: + pass + finally: + logger.info("Stopping worker [%s]", pid) + + loop.run_until_complete(before_stop()) + + if cleanup: + cleanup() + + loop.run_until_complete(after_stop()) + remove_unix_socket(unix) + + def _serve_http_1( host, port, @@ -183,30 +222,7 @@ def _serve_http_1( error_logger.exception("Unable to start server", exc_info=True) return - # Ignore SIGINT when run_multiple - if run_multiple: - signal_func(SIGINT, SIG_IGN) - os.environ["SANIC_WORKER_PROCESS"] = "true" - - # Register signals for graceful termination - if register_sys_signals: - if OS_IS_WINDOWS: - ctrlc_workaround_for_windows(app) - else: - for _signal in [SIGTERM] if run_multiple else [SIGINT, SIGTERM]: - loop.add_signal_handler(_signal, app.stop) - - loop.run_until_complete(app._server_event("init", "after")) - pid = os.getpid() - try: - logger.info("Starting worker [%s]", pid) - loop.run_forever() - finally: - logger.info("Stopping worker [%s]", pid) - - # Run the on_stop function if provided - loop.run_until_complete(app._server_event("shutdown", "before")) - + def _cleanup(): # Wait for event loop to finish and all connections to drain http_server.close() loop.run_until_complete(http_server.wait_closed()) @@ -236,8 +252,16 @@ def _serve_http_1( conn.websocket.fail_connection(code=1001) else: conn.abort() - loop.run_until_complete(app._server_event("shutdown", "after")) - remove_unix_socket(unix) + + _setup_system_signals(app, run_multiple, register_sys_signals, loop) + loop.run_until_complete(app._server_event("init", "after")) + _run_server_forever( + loop, + partial(app._server_event, "shutdown", "before"), + partial(app._server_event, "shutdown", "after"), + _cleanup, + unix, + ) def _serve_http_3( @@ -263,39 +287,16 @@ def _serve_http_3( ) server = AsyncioServer(app, loop, coro, []) loop.run_until_complete(server.startup()) - - # TODO: Cleanup the non-DRY code block - # Ignore SIGINT when run_multiple - if run_multiple: - signal_func(SIGINT, SIG_IGN) - os.environ["SANIC_WORKER_PROCESS"] = "true" - - # Register signals for graceful termination - if register_sys_signals: - if OS_IS_WINDOWS: - ctrlc_workaround_for_windows(app) - else: - for _signal in [SIGTERM] if run_multiple else [SIGINT, SIGTERM]: - loop.add_signal_handler(_signal, app.stop) - loop.run_until_complete(server.before_start()) loop.run_until_complete(server) + _setup_system_signals(app, run_multiple, register_sys_signals, loop) loop.run_until_complete(server.after_start()) - pid = os.getpid() - try: - logger.info("Starting worker [%s]", pid) - loop.run_forever() - except KeyboardInterrupt: - pass - finally: - logger.info("Stopping worker [%s]", pid) - - loop.run_until_complete(server.before_stop()) - - # DO close connections here - - loop.run_until_complete(server.after_stop()) + # TODO: Create connection cleanup and graceful shutdown + cleanup = None + _run_server_forever( + loop, server.before_stop, server.after_stop, cleanup, None + ) def serve_single(server_settings): From 84fd7e0e66e990339bc432cdf942477a7fe70e6f Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 16:37:00 +0300 Subject: [PATCH 41/59] Remove unnecessary inits --- tests/.benchmarks/__init__.py | 0 tests/benchmark/__init__.py | 0 tests/certs/__init__.py | 0 tests/fake/__init__.py | 0 tests/static/__init__.py | 0 tests/static/bp/__init__.py | 0 tests/static/nested/__init__.py | 0 tests/static/nested/dir/__init__.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/.benchmarks/__init__.py delete mode 100644 tests/benchmark/__init__.py delete mode 100644 tests/certs/__init__.py delete mode 100644 tests/fake/__init__.py delete mode 100644 tests/static/__init__.py delete mode 100644 tests/static/bp/__init__.py delete mode 100644 tests/static/nested/__init__.py delete mode 100644 tests/static/nested/dir/__init__.py diff --git a/tests/.benchmarks/__init__.py b/tests/.benchmarks/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/benchmark/__init__.py b/tests/benchmark/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/certs/__init__.py b/tests/certs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/fake/__init__.py b/tests/fake/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/static/__init__.py b/tests/static/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/static/bp/__init__.py b/tests/static/bp/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/static/nested/__init__.py b/tests/static/nested/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/static/nested/dir/__init__.py b/tests/static/nested/dir/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From b64c5b6221d1aa804d51312d6a63ce1c75715018 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 16:37:42 +0300 Subject: [PATCH 42/59] Remove unnecessary inits --- tests/certs/invalid.certmissing/__init__.py | 0 tests/certs/localhost/__init__.py | 0 tests/certs/sanic.example/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/certs/invalid.certmissing/__init__.py delete mode 100644 tests/certs/localhost/__init__.py delete mode 100644 tests/certs/sanic.example/__init__.py diff --git a/tests/certs/invalid.certmissing/__init__.py b/tests/certs/invalid.certmissing/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/certs/localhost/__init__.py b/tests/certs/localhost/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/certs/sanic.example/__init__.py b/tests/certs/sanic.example/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From bb68e28b7f1c2b2dea7688267b624f01f11dd6e9 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 16:54:15 +0300 Subject: [PATCH 43/59] Fix some typing and linting issues --- sanic/application/spinner.py | 4 +--- sanic/http/stream.py | 5 ++++- sanic/http/tls/creators.py | 2 +- sanic/request.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/sanic/application/spinner.py b/sanic/application/spinner.py index 139f0db44e..e89513ea42 100644 --- a/sanic/application/spinner.py +++ b/sanic/application/spinner.py @@ -3,14 +3,12 @@ import time from contextlib import contextmanager -from curses.ascii import SP from queue import Queue from threading import Thread if os.name == "nt": # noqa - import ctypes - import msvcrt + import ctypes # noqa class _CursorInfo(ctypes.Structure): _fields_ = [("size", ctypes.c_int), ("visible", ctypes.c_byte)] diff --git a/sanic/http/stream.py b/sanic/http/stream.py index b2d034391e..c75f6423be 100644 --- a/sanic/http/stream.py +++ b/sanic/http/stream.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple, Union from sanic.compat import Header from sanic.http.constants import Stage @@ -16,6 +16,9 @@ class Stream: response: Optional[BaseHTTPResponse] protocol: HttpProtocol url: Optional[str] + request_body: Optional[bytes] + request_max_size: Union[int, float] + __version__ = "unknown" __touchup__: Tuple[str, ...] = tuple() __slots__ = () diff --git a/sanic/http/tls/creators.py b/sanic/http/tls/creators.py index 808e3565c0..4eb0d2be65 100644 --- a/sanic/http/tls/creators.py +++ b/sanic/http/tls/creators.py @@ -23,7 +23,7 @@ try: - import trustme + import trustme # type: ignore TRUSTME_INSTALLED = True # noqa except (ImportError, ModuleNotFoundError): diff --git a/sanic/request.py b/sanic/request.py index 2e30ae149a..1f38121639 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -14,7 +14,7 @@ Union, ) -from sanic_routing.route import Route +from sanic_routing.route import Route # type: ignore from sanic.http.stream import Stream from sanic.models.asgi import ASGIScope From ed840a2ddf3aec950b6c2b2b4cd65a670ed1501a Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 21:51:08 +0300 Subject: [PATCH 44/59] Expand test coverage --- sanic/http/http1.py | 18 +++++++++++++ sanic/http/http3.py | 8 +----- sanic/http/stream.py | 20 --------------- tests/http3/test_http_receiver.py | 42 ++++++++++++++++++++++++++++++- 4 files changed, 60 insertions(+), 28 deletions(-) diff --git a/sanic/http/http1.py b/sanic/http/http1.py index 8b0e6cbe8b..2ebc37d98a 100644 --- a/sanic/http/http1.py +++ b/sanic/http/http1.py @@ -427,6 +427,24 @@ async def error_response(self, exception: Exception) -> None: await app.handle_exception(self.request, exception) + def create_empty_request(self) -> None: + """ + Current error handling code needs a request object that won't exist + if an error occurred during before a request was received. Create a + bogus response for error handling use. + """ + + # FIXME: Avoid this by refactoring error handling and response code + self.request = self.protocol.request_class( + url_bytes=self.url.encode() if self.url else b"*", + headers=Header({}), + version="1.1", + method="NONE", + transport=self.protocol.transport, + app=self.protocol.app, + ) + self.request.stream = self + def log_response(self) -> None: """ Helper method provided to enable the logging of responses in case if diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 52f0a123b4..db1a5d71c5 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -69,8 +69,6 @@ class HTTPReceiver(Receiver, Stream): stage: Stage request: Request - __version__ = "3" - def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.request_body = None @@ -112,9 +110,6 @@ async def error_response(self, exception: Exception) -> None: # From request and handler states we can respond, otherwise be silent app = self.protocol.app - if self.request is None: - self.create_empty_request() - await app.handle_exception(self.request, exception) def _prepare_headers( @@ -153,7 +148,7 @@ def send_headers(self) -> None: extra={"verbosity": 2}, ) if not self.response: - raise Exception("no response") + raise RuntimeError("no response") response = self.response headers = self._prepare_headers(response) @@ -313,7 +308,6 @@ def _make_request(self, event: HeadersReceived) -> Request: path = headers[":path"] scheme = headers.pop(":scheme", "") authority = headers.pop(":authority", "") - self.url = path if authority: headers["host"] = authority diff --git a/sanic/http/stream.py b/sanic/http/stream.py index c75f6423be..4660ad22c6 100644 --- a/sanic/http/stream.py +++ b/sanic/http/stream.py @@ -2,7 +2,6 @@ from typing import TYPE_CHECKING, Optional, Tuple, Union -from sanic.compat import Header from sanic.http.constants import Stage @@ -19,27 +18,8 @@ class Stream: request_body: Optional[bytes] request_max_size: Union[int, float] - __version__ = "unknown" __touchup__: Tuple[str, ...] = tuple() __slots__ = () def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: raise NotImplementedError("Not implemented") - - def create_empty_request(self) -> None: - """ - Current error handling code needs a request object that won't exist - if an error occurred during before a request was received. Create a - bogus response for error handling use. - """ - - # FIXME: Avoid this by refactoring error handling and response code - self.request = self.protocol.request_class( - url_bytes=self.url.encode() if self.url else b"*", - headers=Header({}), - version=self.__class__.__version__, - method="NONE", - transport=self.protocol.transport, - app=self.protocol.app, - ) - self.request.stream = self diff --git a/tests/http3/test_http_receiver.py b/tests/http3/test_http_receiver.py index 458ab5d1ea..a427bb7a6f 100644 --- a/tests/http3/test_http_receiver.py +++ b/tests/http3/test_http_receiver.py @@ -2,6 +2,7 @@ import pytest +from aioquic.h3.connection import H3Connection from aioquic.h3.events import DataReceived, HeadersReceived from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.connection import QuicConnection @@ -13,7 +14,7 @@ from sanic.exceptions import PayloadTooLarge from sanic.http.constants import Stage from sanic.http.http3 import Http3, HTTPReceiver -from sanic.response import empty +from sanic.response import empty, json from sanic.server.protocols.http_protocol import Http3Protocol @@ -116,8 +117,10 @@ def test_http_receiver_respond(app: Sanic, http_request: Request): receiver.respond(response) receiver.stage = Stage.HANDLER + receiver.response = Mock() resp = receiver.respond(response) + assert receiver.response is resp assert resp is response assert response.stream is receiver @@ -163,3 +166,40 @@ def test_http3_events(app): assert receiver.request.method == "GET" assert receiver.request.headers["foo"] == "bar" assert receiver.request.body == b"foobar" + + +async def test_send_headers(app: Sanic, http_request: Request): + send_headers_mock = Mock() + existing_send_headers = H3Connection.send_headers + receiver = generate_http_receiver(app, http_request) + receiver.protocol.quic_event_received( + ProtocolNegotiated(alpn_protocol="h3") + ) + + def send_headers(*args, **kwargs): + send_headers_mock(*args, **kwargs) + return existing_send_headers( + receiver.protocol.connection, *args, **kwargs + ) + + receiver.protocol.connection.send_headers = send_headers + receiver.head_only = False + response = json({}, status=201, headers={"foo": "bar"}) + + with pytest.raises(RuntimeError, match="no response"): + receiver.send_headers() + + receiver.response = response + receiver.send_headers() + + assert receiver.headers_sent + assert receiver.stage is Stage.RESPONSE + send_headers_mock.assert_called_once_with( + stream_id=0, + headers=[ + (b":status", b"201"), + (b"foo", b"bar"), + (b"content-length", b"2"), + (b"content-type", b"application/json"), + ], + ) From 3d37a926e5fd835db07117ac3c7a30493882a1bc Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 22:20:49 +0300 Subject: [PATCH 45/59] Cleanup existing tests --- tests/conftest.py | 1 + tests/test_asgi.py | 2 +- tests/test_config.py | 1 + tests/test_custom_request.py | 2 +- tests/test_exceptions.py | 8 ++++---- tests/test_logging.py | 2 +- tests/test_request_stream.py | 2 +- tests/test_requests.py | 8 ++++---- tests/test_versioning.py | 24 ++++++++++++------------ tox.ini | 2 +- 10 files changed, 27 insertions(+), 25 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fe4ba47d62..b10ed736c7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -150,6 +150,7 @@ def app(request): yield app for target, method_name in TouchUp._registry: setattr(target, method_name, CACHE[method_name]) + Sanic._app_registry.clear() @pytest.fixture(scope="function") diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 61d36fa732..4c473047a0 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -417,7 +417,7 @@ async def test_request_class_custom(): class MyCustomRequest(Request): pass - app = Sanic(name=__name__, request_class=MyCustomRequest) + app = Sanic(name="Test", request_class=MyCustomRequest) @app.get("/custom") def custom_request(request): diff --git a/tests/test_config.py b/tests/test_config.py index 7a2dea4500..764d7940b5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -454,3 +454,4 @@ def test_convert_local_cert_creator(passed, expected): os.environ["SANIC_LOCAL_CERT_CREATOR"] = passed app = Sanic("Test") assert app.config.LOCAL_CERT_CREATOR is expected + del os.environ["SANIC_LOCAL_CERT_CREATOR"] diff --git a/tests/test_custom_request.py b/tests/test_custom_request.py index 758feebaa6..84ab705a81 100644 --- a/tests/test_custom_request.py +++ b/tests/test_custom_request.py @@ -17,7 +17,7 @@ async def receive_body(self): def test_custom_request(): - app = Sanic(name=__name__, request_class=CustomRequest) + app = Sanic(name="Test", request_class=CustomRequest) @app.route("/post", methods=["POST"]) async def post_handler(request): diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 5ab8678679..12008ee9d1 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -259,7 +259,7 @@ def tempest(_): def test_exception_in_ws_logged(caplog): - app = Sanic(__name__) + app = Sanic("Test") @app.websocket("/feed") async def feed(request, ws): @@ -279,7 +279,7 @@ async def feed(request, ws): @pytest.mark.parametrize("debug", (True, False)) def test_contextual_exception_context(debug): - app = Sanic(__name__) + app = Sanic("Test") class TeapotError(SanicException): status_code = 418 @@ -314,7 +314,7 @@ def fail(): @pytest.mark.parametrize("debug", (True, False)) def test_contextual_exception_extra(debug): - app = Sanic(__name__) + app = Sanic("Test") class TeapotError(SanicException): status_code = 418 @@ -361,7 +361,7 @@ def fail(): @pytest.mark.parametrize("override", (True, False)) def test_contextual_exception_functional_message(override): - app = Sanic(__name__) + app = Sanic("Test") class TeapotError(SanicException): status_code = 418 diff --git a/tests/test_logging.py b/tests/test_logging.py index 180c10b29d..18d456660f 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -136,7 +136,7 @@ async def conn_lost(request): async def test_logger(caplog): rand_string = str(uuid.uuid4()) - app = Sanic(name=__name__) + app = Sanic(name="Test") @app.get("/") def log_info(request): diff --git a/tests/test_request_stream.py b/tests/test_request_stream.py index 37cbc04b54..8bf1fa5705 100644 --- a/tests/test_request_stream.py +++ b/tests/test_request_stream.py @@ -552,7 +552,7 @@ async def handler(request): def test_streaming_echo(): """2-way streaming chat between server and client.""" - app = Sanic(name=__name__) + app = Sanic(name="Test") @app.post("/echo", stream=True) async def handler(request): diff --git a/tests/test_requests.py b/tests/test_requests.py index 4d7fb0aa13..9a984bb46e 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -2050,7 +2050,7 @@ async def post(request): def test_endpoint_basic(): - app = Sanic(name=__name__) + app = Sanic(name="Test") @app.route("/") def my_unique_handler(request): @@ -2058,12 +2058,12 @@ def my_unique_handler(request): request, response = app.test_client.get("/") - assert request.endpoint == "test_requests.my_unique_handler" + assert request.endpoint == "Test.my_unique_handler" @pytest.mark.asyncio async def test_endpoint_basic_asgi(): - app = Sanic(name=__name__) + app = Sanic(name="Test") @app.route("/") def my_unique_handler(request): @@ -2071,7 +2071,7 @@ def my_unique_handler(request): request, response = await app.asgi_client.get("/") - assert request.endpoint == "test_requests.my_unique_handler" + assert request.endpoint == "Test.my_unique_handler" def test_endpoint_named_app(): diff --git a/tests/test_versioning.py b/tests/test_versioning.py index 396629b301..26f4ca4c22 100644 --- a/tests/test_versioning.py +++ b/tests/test_versioning.py @@ -19,7 +19,7 @@ def test_route(app, handler): def test_bp(app, handler): - bp = Blueprint(__name__, version=1) + bp = Blueprint("Test", version=1) bp.route("/")(handler) app.blueprint(bp) @@ -28,7 +28,7 @@ def test_bp(app, handler): def test_bp_use_route(app, handler): - bp = Blueprint(__name__, version=1) + bp = Blueprint("Test", version=1) bp.route("/", version=1.1)(handler) app.blueprint(bp) @@ -37,7 +37,7 @@ def test_bp_use_route(app, handler): def test_bp_group(app, handler): - bp = Blueprint(__name__) + bp = Blueprint("Test") bp.route("/")(handler) group = Blueprint.group(bp, version=1) app.blueprint(group) @@ -47,7 +47,7 @@ def test_bp_group(app, handler): def test_bp_group_use_bp(app, handler): - bp = Blueprint(__name__, version=1.1) + bp = Blueprint("Test", version=1.1) bp.route("/")(handler) group = Blueprint.group(bp, version=1) app.blueprint(group) @@ -57,7 +57,7 @@ def test_bp_group_use_bp(app, handler): def test_bp_group_use_registration(app, handler): - bp = Blueprint(__name__, version=1.1) + bp = Blueprint("Test", version=1.1) bp.route("/")(handler) group = Blueprint.group(bp, version=1) app.blueprint(group, version=1.2) @@ -67,7 +67,7 @@ def test_bp_group_use_registration(app, handler): def test_bp_group_use_route(app, handler): - bp = Blueprint(__name__, version=1.1) + bp = Blueprint("Test", version=1.1) bp.route("/", version=1.3)(handler) group = Blueprint.group(bp, version=1) app.blueprint(group, version=1.2) @@ -84,7 +84,7 @@ def test_version_prefix_route(app, handler): def test_version_prefix_bp(app, handler): - bp = Blueprint(__name__, version=1, version_prefix="/api/v") + bp = Blueprint("Test", version=1, version_prefix="/api/v") bp.route("/")(handler) app.blueprint(bp) @@ -93,7 +93,7 @@ def test_version_prefix_bp(app, handler): def test_version_prefix_bp_use_route(app, handler): - bp = Blueprint(__name__, version=1, version_prefix="/ignore/v") + bp = Blueprint("Test", version=1, version_prefix="/ignore/v") bp.route("/", version=1.1, version_prefix="/api/v")(handler) app.blueprint(bp) @@ -102,7 +102,7 @@ def test_version_prefix_bp_use_route(app, handler): def test_version_prefix_bp_group(app, handler): - bp = Blueprint(__name__) + bp = Blueprint("Test") bp.route("/")(handler) group = Blueprint.group(bp, version=1, version_prefix="/api/v") app.blueprint(group) @@ -112,7 +112,7 @@ def test_version_prefix_bp_group(app, handler): def test_version_prefix_bp_group_use_bp(app, handler): - bp = Blueprint(__name__, version=1.1, version_prefix="/api/v") + bp = Blueprint("Test", version=1.1, version_prefix="/api/v") bp.route("/")(handler) group = Blueprint.group(bp, version=1, version_prefix="/ignore/v") app.blueprint(group) @@ -122,7 +122,7 @@ def test_version_prefix_bp_group_use_bp(app, handler): def test_version_prefix_bp_group_use_registration(app, handler): - bp = Blueprint(__name__, version=1.1, version_prefix="/alsoignore/v") + bp = Blueprint("Test", version=1.1, version_prefix="/alsoignore/v") bp.route("/")(handler) group = Blueprint.group(bp, version=1, version_prefix="/ignore/v") app.blueprint(group, version=1.2, version_prefix="/api/v") @@ -132,7 +132,7 @@ def test_version_prefix_bp_group_use_registration(app, handler): def test_version_prefix_bp_group_use_route(app, handler): - bp = Blueprint(__name__, version=1.1, version_prefix="/alsoignore/v") + bp = Blueprint("Test", version=1.1, version_prefix="/alsoignore/v") bp.route("/", version=1.3, version_prefix="/api/v")(handler) group = Blueprint.group(bp, version=1, version_prefix="/ignore/v") app.blueprint(group, version=1.2, version_prefix="/stillignoring/v") diff --git a/tox.ini b/tox.ini index 70c2228819..2a044f9ea3 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,7 @@ commands = [testenv:docs] platform = linux|linux2|darwin allowlist_externals = make -extras = docs +extras = docs, http3 commands = make docs-test From 9111f93a468fe0a60ea11bf34b01d9fe494d4a10 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 20 Jun 2022 23:31:37 +0300 Subject: [PATCH 46/59] Increase testing coverage --- sanic/http/http3.py | 33 +++++++++-------- sanic/http/stream.py | 4 ++- sanic/http/tls/creators.py | 6 ++-- tests/http3/test_http_receiver.py | 38 ++++++++++++++++++++ tests/http3/test_session_ticket_store.py | 46 ++++++++++++++++++++++++ tests/test_tls.py | 28 +++++++++++++++ 6 files changed, 135 insertions(+), 20 deletions(-) create mode 100644 tests/http3/test_session_ticket_store.py diff --git a/sanic/http/http3.py b/sanic/http/http3.py index db1a5d71c5..60e90a003c 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -83,7 +83,7 @@ async def run(self, exception: Optional[Exception] = None): self.head_only = self.request.method.upper() == "HEAD" if exception: - logger.info( + logger.info( # no cov f"{Colors.BLUE}[exception]: " f"{Colors.RED}{exception}{Colors.END}", exc_info=True, @@ -92,7 +92,7 @@ async def run(self, exception: Optional[Exception] = None): await self.error_response(exception) else: try: - logger.info( + logger.info( # no cov f"{Colors.BLUE}[request]:{Colors.END} {self.request}", extra={"verbosity": 1}, ) @@ -126,7 +126,7 @@ def _prepare_headers( ): headers.pop("content-length", None) headers.pop("transfer-encoding", None) - logger.warning( + logger.warning( # no cov f"Message body set in response on {self.request.path}. " f"A {status} response may only have headers, no body." ) @@ -143,7 +143,7 @@ def _prepare_headers( return headers def send_headers(self) -> None: - logger.debug( + logger.debug( # no cov f"{Colors.BLUE}[send]: {Colors.GREEN}HEADERS{Colors.END}", extra={"verbosity": 2}, ) @@ -166,7 +166,7 @@ def send_headers(self) -> None: self.future.cancel() def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: - logger.debug( + logger.debug( # no cov f"{Colors.BLUE}[respond]:{Colors.END} {response}", extra={"verbosity": 2}, ) @@ -191,7 +191,7 @@ def receive_body(self, data: bytes) -> None: self.request.body += data async def send(self, data: bytes, end_stream: bool) -> None: - logger.debug( + logger.debug( # no cov f"{Colors.BLUE}[send]: {Colors.GREEN}{data=} " f"{end_stream=}{Colors.END}", extra={"verbosity": 2}, @@ -219,7 +219,7 @@ def _send(self, data: bytes, end_stream: bool) -> None: elif size: data = b"%x\r\n%b\r\n" % (size, data) - logger.debug( + logger.debug( # no cov f"{Colors.BLUE}[transmitting]{Colors.END}", extra={"verbosity": 2}, ) @@ -262,7 +262,7 @@ def __init__( self.receivers: Dict[int, Receiver] = {} def http_event_received(self, event: H3Event) -> None: - logger.debug( + logger.debug( # no cov f"{Colors.BLUE}[http_event_received]: " f"{Colors.YELLOW}{event}{Colors.END}", extra={"verbosity": 2}, @@ -279,7 +279,7 @@ def http_event_received(self, event: H3Event) -> None: receiver.future.cancel() receiver.future = asyncio.ensure_future(receiver.run(e)) else: - logger.debug( + logger.debug( # no cov f"{Colors.RED}DOING NOTHING{Colors.END}", extra={"verbosity": 2}, ) @@ -350,6 +350,13 @@ def get_config( # just taking the first if isinstance(ssl, CertSelector): ssl = cast(SanicSSLContext, ssl.sanic_select[0]) + if app.config.LOCAL_CERT_CREATOR is LocalCertCreator.TRUSTME: + raise SanicException( + "Sorry, you cannot currently use trustme as a local certificate " + "generator for an HTTP/3 server. This is not yet supported. You " + "should be able to use mkcert instead. For more information, see: " + "https://github.com/aiortc/aioquic/issues/295." + ) if not isinstance(ssl, CertSimple): raise SanicException("SSLContext is not CertSimple") @@ -360,14 +367,6 @@ def get_config( ) password = app.config.TLS_CERT_PASSWORD or None - if app.config.LOCAL_CERT_CREATOR is LocalCertCreator.TRUSTME: - raise SanicException( - "Sorry, you cannot currently use trustme as a local certificate " - "generator for an HTTP/3 server. This is not yet supported. You " - "should be able to use mkcert instead. For more information, see: " - "https://github.com/aiortc/aioquic/issues/295." - ) - config.load_cert_chain( ssl.sanic["cert"], ssl.sanic["key"], password=password ) diff --git a/sanic/http/stream.py b/sanic/http/stream.py index 4660ad22c6..9b413195bd 100644 --- a/sanic/http/stream.py +++ b/sanic/http/stream.py @@ -21,5 +21,7 @@ class Stream: __touchup__: Tuple[str, ...] = tuple() __slots__ = () - def respond(self, response: BaseHTTPResponse) -> BaseHTTPResponse: + def respond( + self, response: BaseHTTPResponse + ) -> BaseHTTPResponse: # no cov raise NotImplementedError("Not implemented") diff --git a/sanic/http/tls/creators.py b/sanic/http/tls/creators.py index 4eb0d2be65..f8fbf6cffd 100644 --- a/sanic/http/tls/creators.py +++ b/sanic/http/tls/creators.py @@ -27,6 +27,7 @@ TRUSTME_INSTALLED = True # noqa except (ImportError, ModuleNotFoundError): + trustme = None # type: ignore TRUSTME_INSTALLED = False if TYPE_CHECKING: @@ -185,7 +186,7 @@ def _try_select( class MkcertCreator(CertCreator): def check_supported(self) -> None: try: - subprocess.run( + subprocess.run( # nosec B603 B607 ["mkcert", "-help"], check=True, stderr=subprocess.DEVNULL, @@ -208,6 +209,7 @@ def generate_cert(self, localhost: str) -> ssl.SSLContext: try: if not self.cert_path.exists(): message = "Generating TLS certificate" + # TODO: Validate input for security with loading(message): cmd = [ "mkcert", @@ -217,7 +219,7 @@ def generate_cert(self, localhost: str) -> ssl.SSLContext: str(self.cert_path), localhost, ] - resp = subprocess.run( + resp = subprocess.run( # nosec B603 cmd, check=True, stdout=subprocess.PIPE, diff --git a/tests/http3/test_http_receiver.py b/tests/http3/test_http_receiver.py index a427bb7a6f..a8589fb2d0 100644 --- a/tests/http3/test_http_receiver.py +++ b/tests/http3/test_http_receiver.py @@ -203,3 +203,41 @@ def send_headers(*args, **kwargs): (b"content-type", b"application/json"), ], ) + + +def test_multiple_streams(app): + protocol = generate_protocol(app) + http3 = Http3(protocol, protocol.transmit) + http3.http_event_received( + HeadersReceived( + [ + (b":method", b"GET"), + (b":path", b"/location"), + (b":scheme", b"https"), + (b":authority", b"localhost:8443"), + (b"foo", b"bar"), + ], + 1, + False, + ) + ) + http3.http_event_received( + HeadersReceived( + [ + (b":method", b"GET"), + (b":path", b"/location"), + (b":scheme", b"https"), + (b":authority", b"localhost:8443"), + (b"foo", b"bar"), + ], + 2, + False, + ) + ) + + receiver1 = http3.get_receiver_by_stream_id(1) + receiver2 = http3.get_receiver_by_stream_id(2) + assert len(http3.receivers) == 2 + assert isinstance(receiver1, HTTPReceiver) + assert isinstance(receiver2, HTTPReceiver) + assert receiver1 is not receiver2 diff --git a/tests/http3/test_session_ticket_store.py b/tests/http3/test_session_ticket_store.py new file mode 100644 index 0000000000..c7fa29c486 --- /dev/null +++ b/tests/http3/test_session_ticket_store.py @@ -0,0 +1,46 @@ +from datetime import datetime + +from aioquic.tls import CipherSuite, SessionTicket + +from sanic.http.http3 import SessionTicketStore + + +def _generate_ticket(label): + return SessionTicket( + 1, + CipherSuite.AES_128_GCM_SHA256, + datetime.now(), + datetime.now(), + label, + label.decode(), + label, + None, + [], + ) + + +def test_session_ticket_store(): + store = SessionTicketStore() + + assert len(store.tickets) == 0 + + ticket1 = _generate_ticket(b"foo") + store.add(ticket1) + + assert len(store.tickets) == 1 + + ticket2 = _generate_ticket(b"bar") + store.add(ticket2) + + assert len(store.tickets) == 2 + assert len(store.tickets) == 2 + + popped2 = store.pop(ticket2.ticket) + + assert len(store.tickets) == 1 + assert popped2 is ticket2 + + popped1 = store.pop(ticket1.ticket) + + assert len(store.tickets) == 0 + assert popped1 is ticket1 diff --git a/tests/test_tls.py b/tests/test_tls.py index b8115b436c..cc8eb3f92f 100644 --- a/tests/test_tls.py +++ b/tests/test_tls.py @@ -608,3 +608,31 @@ def test_get_ssl_context_only_mkcert( MockTrustmeCreator.generate_cert.assert_called_once_with("localhost") else: MockTrustmeCreator.generate_cert.assert_not_called() + + +def test_no_http3_with_trustme( + app, + monkeypatch, + MockTrustmeCreator, +): + monkeypatch.setattr( + sanic.http.tls.creators, "TrustmeCreator", MockTrustmeCreator + ) + MockTrustmeCreator.SUPPORTED = True + app.config.LOCAL_CERT_CREATOR = "TRUSTME" + with pytest.raises( + SanicException, + match=( + "Sorry, you cannot currently use trustme as a local certificate " + "generator for an HTTP/3 server" + ), + ): + app.run(version=3, debug=True) + + +def test_sanic_ssl_context_create(): + context = ssl.SSLContext() + sanic_context = SanicSSLContext.create_from_ssl_context(context) + + assert sanic_context is context + assert isinstance(sanic_context, SanicSSLContext) From 689c830d54a4715b23f4f4b8caabd65e43c7ff20 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 21 Jun 2022 12:18:11 +0300 Subject: [PATCH 47/59] Add connection info and transport details --- sanic/cli/arguments.py | 2 +- sanic/http/http3.py | 36 ++++++++++++++++---- sanic/models/protocol_types.py | 11 ++----- sanic/models/server_types.py | 15 +++++++-- sanic/request.py | 4 +++ sanic/server/protocols/http_protocol.py | 6 ++++ tests/http3/test_http_receiver.py | 44 +++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 19 deletions(-) diff --git a/sanic/cli/arguments.py b/sanic/cli/arguments.py index 33348cec7e..209deaa6f3 100644 --- a/sanic/cli/arguments.py +++ b/sanic/cli/arguments.py @@ -253,7 +253,7 @@ def attach(self): action="store_true", help=( "Create a temporary TLS certificate for local development " - "(requires mkcert)" + "(requires mkcert or trustme)" ), ) diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 60e90a003c..29dc3e903b 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -6,6 +6,7 @@ from ssl import SSLContext from typing import ( TYPE_CHECKING, + Any, Callable, Dict, List, @@ -31,8 +32,12 @@ from sanic.constants import LocalCertCreator from sanic.exceptions import PayloadTooLarge, SanicException from sanic.helpers import has_message_body +from sanic.http.constants import Stage from sanic.http.stream import Stream from sanic.http.tls.context import CertSelector, CertSimple, SanicSSLContext +from sanic.log import Colors, logger +from sanic.models.protocol_types import TransportProtocol +from sanic.models.server_types import ConnInfo if TYPE_CHECKING: @@ -41,15 +46,30 @@ from sanic.response import BaseHTTPResponse from sanic.server.protocols.http_protocol import Http3Protocol -from sanic.http.constants import Stage -from sanic.log import Colors, logger - HttpConnection = Union[H0Connection, H3Connection] -class Transport: - ... +class HTTP3Transport(TransportProtocol): + __slots__ = ("_protocol",) + + def __init__(self, protocol: Http3Protocol): + self._protocol = protocol + + def get_protocol(self) -> Http3Protocol: + return self._protocol + + def get_extra_info(self, info: str, default: Any = None) -> Any: + if ( + info in ("socket", "sockname", "peername") + and self._protocol._transport + ): + return self._protocol._transport.get_extra_info(info, default) + elif info == "network_paths": + return self._protocol._quic._network_paths + elif info == "ssl_context": + return self._protocol.app.state.ssl + return default class Receiver(ABC): @@ -192,7 +212,7 @@ def receive_body(self, data: bytes) -> None: async def send(self, data: bytes, end_stream: bool) -> None: logger.debug( # no cov - f"{Colors.BLUE}[send]: {Colors.GREEN}{data=} " + f"{Colors.BLUE}[send]: {Colors.GREEN}data={data.decode()} " f"{end_stream=}{Colors.END}", extra={"verbosity": 2}, ) @@ -312,15 +332,17 @@ def _make_request(self, event: HeadersReceived) -> Request: if authority: headers["host"] = authority + transport = HTTP3Transport(self.protocol) request = self.protocol.request_class( path.encode(), headers, "3", method, - Transport(), + transport, self.protocol.app, b"", ) + request.conn_info = ConnInfo(transport) request._stream_id = event.stream_id request._scheme = scheme diff --git a/sanic/models/protocol_types.py b/sanic/models/protocol_types.py index 14bc275cbf..fc94567061 100644 --- a/sanic/models/protocol_types.py +++ b/sanic/models/protocol_types.py @@ -1,12 +1,12 @@ import sys +from asyncio import BaseTransport from typing import Any, AnyStr, TypeVar, Union from sanic.models.asgi import ASGIScope if sys.version_info < (3, 8): - from asyncio import BaseTransport # from sanic.models.asgi import MockTransport MockTransport = TypeVar("MockTransport") @@ -18,14 +18,9 @@ # Protocol is a 3.8+ feature from typing import Protocol - class TransportProtocol(Protocol): + class TransportProtocol(BaseTransport): scope: ASGIScope - - def get_protocol(self): - ... - - def get_extra_info(self, info: str) -> Union[str, bool, None]: - ... + __slots__ = () class HTMLProtocol(Protocol): def __html__(self) -> AnyStr: diff --git a/sanic/models/server_types.py b/sanic/models/server_types.py index ba9f2918d9..da88a8ff95 100644 --- a/sanic/models/server_types.py +++ b/sanic/models/server_types.py @@ -1,8 +1,8 @@ from __future__ import annotations -from ssl import SSLObject +from ssl import SSLContext, SSLObject from types import SimpleNamespace -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from sanic.models.protocol_types import TransportProtocol @@ -28,6 +28,7 @@ class ConnInfo: "sockname", "ssl", "cert", + "network_paths", ) def __init__(self, transport: TransportProtocol, unix=None): @@ -40,17 +41,22 @@ def __init__(self, transport: TransportProtocol, unix=None): self.ssl = False self.server_name = "" self.cert: Dict[str, Any] = {} + self.network_paths: List[Any] = [] sslobj: Optional[SSLObject] = transport.get_extra_info( "ssl_object" ) # type: ignore + sslctx: Optional[SSLContext] = transport.get_extra_info( + "ssl_context" + ) # type: ignore if sslobj: self.ssl = True self.server_name = getattr(sslobj, "sanic_server_name", None) or "" self.cert = dict(getattr(sslobj.context, "sanic", {})) + if sslctx and not self.cert: + self.cert = dict(getattr(sslctx, "sanic", {})) if isinstance(addr, str): # UNIX socket self.server = unix or addr return - # IPv4 (ip, port) or IPv6 (ip, port, flowinfo, scopeid) if isinstance(addr, tuple): self.server = addr[0] if len(addr) == 2 else f"[{addr[0]}]" @@ -59,6 +65,9 @@ def __init__(self, transport: TransportProtocol, unix=None): if addr[1] != (443 if self.ssl else 80): self.server = f"{self.server}:{addr[1]}" self.peername = addr = transport.get_extra_info("peername") + self.network_paths = transport.get_extra_info( # type: ignore + "network_paths" + ) if isinstance(addr, tuple): self.client = addr[0] if len(addr) == 2 else f"[{addr[0]}]" diff --git a/sanic/request.py b/sanic/request.py index 1f38121639..76ff9c0511 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -681,6 +681,10 @@ def path(self) -> str: """ return self._parsed_url.path.decode("utf-8") + @property + def network_paths(self): + return self.conn_info.network_paths + # Proxy properties (using SERVER_NAME/forwarded/request/transport info) @property diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index 0b3d55b8fb..cf0e1eaab2 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -51,6 +51,12 @@ def _setup(self): self.request_max_size = self.app.config.REQUEST_MAX_SIZE self.request_class = self.app.request_class or Request + @property + def http(self): + if not hasattr(self, "_http"): + return None + return self._http + class HttpProtocol(HttpProtocolMixin, SanicProtocol, metaclass=TouchUpMeta): """ diff --git a/tests/http3/test_http_receiver.py b/tests/http3/test_http_receiver.py index a8589fb2d0..1a36377405 100644 --- a/tests/http3/test_http_receiver.py +++ b/tests/http3/test_http_receiver.py @@ -14,6 +14,7 @@ from sanic.exceptions import PayloadTooLarge from sanic.http.constants import Stage from sanic.http.http3 import Http3, HTTPReceiver +from sanic.models.server_types import ConnInfo from sanic.response import empty, json from sanic.server.protocols.http_protocol import Http3Protocol @@ -241,3 +242,46 @@ def test_multiple_streams(app): assert isinstance(receiver1, HTTPReceiver) assert isinstance(receiver2, HTTPReceiver) assert receiver1 is not receiver2 + + +def test_request_stream_id(app): + protocol = generate_protocol(app) + http3 = Http3(protocol, protocol.transmit) + http3.http_event_received( + HeadersReceived( + [ + (b":method", b"GET"), + (b":path", b"/location"), + (b":scheme", b"https"), + (b":authority", b"localhost:8443"), + (b"foo", b"bar"), + ], + 1, + False, + ) + ) + receiver = http3.get_receiver_by_stream_id(1) + + assert isinstance(receiver.request, Request) + assert receiver.request.stream_id == 1 + + +def test_request_conn_info(app): + protocol = generate_protocol(app) + http3 = Http3(protocol, protocol.transmit) + http3.http_event_received( + HeadersReceived( + [ + (b":method", b"GET"), + (b":path", b"/location"), + (b":scheme", b"https"), + (b":authority", b"localhost:8443"), + (b"foo", b"bar"), + ], + 1, + False, + ) + ) + receiver = http3.get_receiver_by_stream_id(1) + + assert isinstance(receiver.request.conn_info, ConnInfo) From ceee998a81aa5ba9667a3fc5fbb6a8b4b0734a8d Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 21 Jun 2022 13:16:46 +0300 Subject: [PATCH 48/59] Fix 3.7 and 3.8 tests --- sanic/http/http3.py | 2 +- sanic/mixins/runner.py | 10 ++++++++-- sanic/models/asgi.py | 11 +++++++---- sanic/models/protocol_types.py | 22 +++++++++++----------- tests/asyncmock.py | 14 ++++++++++++++ tests/http3/test_http_receiver.py | 7 ++++++- tests/test_http_alt_svc.py | 5 +++++ tests/test_multi_serve.py | 2 +- tests/test_response.py | 5 +++-- tests/test_tasks.py | 2 +- tests/test_websockets.py | 2 +- 11 files changed, 58 insertions(+), 24 deletions(-) diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 29dc3e903b..6d60198d39 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -213,7 +213,7 @@ def receive_body(self, data: bytes) -> None: async def send(self, data: bytes, end_stream: bool) -> None: logger.debug( # no cov f"{Colors.BLUE}[send]: {Colors.GREEN}data={data.decode()} " - f"{end_stream=}{Colors.END}", + f"end_stream={end_stream}{Colors.END}", extra={"verbosity": 2}, ) self._send(data, end_stream) diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py index 0257183b91..ee787776e6 100644 --- a/sanic/mixins/runner.py +++ b/sanic/mixins/runner.py @@ -2,6 +2,7 @@ import os import platform +import sys from asyncio import ( AbstractEventLoop, @@ -23,7 +24,6 @@ Any, Dict, List, - Literal, Optional, Set, Tuple, @@ -58,7 +58,13 @@ from sanic.config import Config SANIC_PACKAGES = ("sanic-routing", "sanic-testing", "sanic-ext") -HTTPVersion = Union[HTTP, Literal[1], Literal[3]] + +if sys.version_info < (3, 8): + HTTPVersion = Union[HTTP, int] +else: + from typing import Literal + + HTTPVersion = Union[HTTP, Literal[1], Literal[3]] class RunnerMixin(metaclass=SanicMeta): diff --git a/sanic/models/asgi.py b/sanic/models/asgi.py index 2b0ee0ed47..562085337a 100644 --- a/sanic/models/asgi.py +++ b/sanic/models/asgi.py @@ -4,6 +4,7 @@ from typing import Any, Awaitable, Callable, MutableMapping, Optional, Union from sanic.exceptions import BadRequest +from sanic.models.protocol_types import TransportProtocol from sanic.server.websockets.connection import WebSocketConnection @@ -56,7 +57,7 @@ async def drain(self) -> None: await self._not_paused.wait() -class MockTransport: # no cov +class MockTransport(TransportProtocol): # no cov _protocol: Optional[MockProtocol] def __init__( @@ -68,17 +69,19 @@ def __init__( self._protocol = None self.loop = None - def get_protocol(self) -> MockProtocol: + def get_protocol(self) -> MockProtocol: # type: ignore if not self._protocol: self._protocol = MockProtocol(self, self.loop) return self._protocol - def get_extra_info(self, info: str) -> Union[str, bool, None]: + def get_extra_info( + self, info: str, default=None + ) -> Union[str, bool, None]: if info == "peername": return self.scope.get("client") elif info == "sslcontext": return self.scope.get("scheme") in ["https", "wss"] - return None + return default def get_websocket_connection(self) -> WebSocketConnection: try: diff --git a/sanic/models/protocol_types.py b/sanic/models/protocol_types.py index fc94567061..24b4361dae 100644 --- a/sanic/models/protocol_types.py +++ b/sanic/models/protocol_types.py @@ -1,27 +1,22 @@ +from __future__ import annotations + import sys from asyncio import BaseTransport -from typing import Any, AnyStr, TypeVar, Union - -from sanic.models.asgi import ASGIScope +from typing import TYPE_CHECKING, Any, AnyStr -if sys.version_info < (3, 8): +if TYPE_CHECKING: + from sanic.models.asgi import ASGIScope - # from sanic.models.asgi import MockTransport - MockTransport = TypeVar("MockTransport") - TransportProtocol = Union[MockTransport, BaseTransport] +if sys.version_info < (3, 8): Range = Any HTMLProtocol = Any else: # Protocol is a 3.8+ feature from typing import Protocol - class TransportProtocol(BaseTransport): - scope: ASGIScope - __slots__ = () - class HTMLProtocol(Protocol): def __html__(self) -> AnyStr: ... @@ -41,3 +36,8 @@ def size(self) -> int: def total(self) -> int: ... + + +class TransportProtocol(BaseTransport): + scope: ASGIScope + __slots__ = () diff --git a/tests/asyncmock.py b/tests/asyncmock.py index eec1764664..835012fd8f 100644 --- a/tests/asyncmock.py +++ b/tests/asyncmock.py @@ -25,6 +25,10 @@ async def dummy(): def __await__(self): return self().__await__() + def reset_mock(self, *args, **kwargs): + super().reset_mock(*args, **kwargs) + self.await_count = 0 + def assert_awaited_once(self): if not self.await_count == 1: msg = ( @@ -32,3 +36,13 @@ def assert_awaited_once(self): f" Awaited {self.await_count} times." ) raise AssertionError(msg) + + def assert_awaited_once_with(self, *args, **kwargs): + if not self.await_count == 1: + msg = ( + f"Expected to have been awaited once." + f" Awaited {self.await_count} times." + ) + raise AssertionError(msg) + self.assert_awaited_once() + return self.assert_called_with(*args, **kwargs) diff --git a/tests/http3/test_http_receiver.py b/tests/http3/test_http_receiver.py index 1a36377405..3a0338658c 100644 --- a/tests/http3/test_http_receiver.py +++ b/tests/http3/test_http_receiver.py @@ -1,4 +1,4 @@ -from unittest.mock import AsyncMock, Mock +from unittest.mock import Mock import pytest @@ -19,6 +19,11 @@ from sanic.server.protocols.http_protocol import Http3Protocol +try: + from unittest.mock import AsyncMock +except ImportError: + from tests.asyncmock import AsyncMock # type: ignore + pytestmark = pytest.mark.asyncio diff --git a/tests/test_http_alt_svc.py b/tests/test_http_alt_svc.py index a923d4ca8e..62f2b02e51 100644 --- a/tests/test_http_alt_svc.py +++ b/tests/test_http_alt_svc.py @@ -1,5 +1,9 @@ +import sys + from pathlib import Path +import pytest + from sanic.app import Sanic from sanic.response import empty from tests.client import RawClient @@ -11,6 +15,7 @@ PORT = 12344 +@pytest.mark.skipif(sys.version_info < (3, 9), reason="Not supported in 3.7") def test_http1_response_has_alt_svc(): Sanic._app_registry.clear() app = Sanic("TestAltSvc") diff --git a/tests/test_multi_serve.py b/tests/test_multi_serve.py index dde72b5c26..be7960e6b4 100644 --- a/tests/test_multi_serve.py +++ b/tests/test_multi_serve.py @@ -14,7 +14,7 @@ try: from unittest.mock import AsyncMock except ImportError: - from asyncmock import AsyncMock # type: ignore + from tests.asyncmock import AsyncMock # type: ignore @pytest.fixture diff --git a/tests/test_response.py b/tests/test_response.py index e1c19d2ab9..a25a1318e4 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -101,11 +101,12 @@ async def test(request: Request): return json({"ok": True}, headers={"CONTENT-TYPE": "application/json"}) request, response = app.test_client.get("/") - assert dict(response.headers) == { + for key, value in { "connection": "keep-alive", "content-length": "11", "content-type": "application/json", - } + }.items(): + assert response.headers[key] == value def test_response_content_length(app): diff --git a/tests/test_tasks.py b/tests/test_tasks.py index 8396a9f964..1d5283197d 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -13,7 +13,7 @@ try: from unittest.mock import AsyncMock except ImportError: - from asyncmock import AsyncMock # type: ignore + from tests.asyncmock import AsyncMock # type: ignore pytestmark = pytest.mark.asyncio diff --git a/tests/test_websockets.py b/tests/test_websockets.py index 329eff455f..dd8413b981 100644 --- a/tests/test_websockets.py +++ b/tests/test_websockets.py @@ -14,7 +14,7 @@ try: from unittest.mock import AsyncMock except ImportError: - from asyncmock import AsyncMock # type: ignore + from tests.asyncmock import AsyncMock # type: ignore @pytest.mark.asyncio From 44b6b725502f38cf6e9f24b5d18c04063858ef72 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 21 Jun 2022 14:23:24 +0300 Subject: [PATCH 49/59] Fix 3.7 --- tests/http3/test_server.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/http3/test_server.py b/tests/http3/test_server.py index 40c09c00e4..bed2446a25 100644 --- a/tests/http3/test_server.py +++ b/tests/http3/test_server.py @@ -1,4 +1,5 @@ import logging +import sys from asyncio import Event from pathlib import Path @@ -6,6 +7,7 @@ import pytest from sanic import Sanic +from sanic.compat import UVLOOP_INSTALLED from sanic.http.constants import HTTP @@ -14,6 +16,10 @@ @pytest.mark.parametrize("version", (3, HTTP.VERSION_3)) +@pytest.mark.skipif( + sys.version_info < (3, 8) and not UVLOOP_INSTALLED, + reason="In 3.7 w/o uvloop the port is not always released", +) def test_server_starts_http3(app: Sanic, version, caplog): ev = Event() @@ -39,6 +45,10 @@ def shutdown(*_): ) in caplog.record_tuples +@pytest.mark.skipif( + sys.version_info < (3, 8) and not UVLOOP_INSTALLED, + reason="In 3.7 w/o uvloop the port is not always released", +) def test_server_starts_http1_and_http3(app: Sanic, caplog): @app.after_server_start def shutdown(*_): @@ -73,6 +83,10 @@ def shutdown(*_): ) in caplog.record_tuples +@pytest.mark.skipif( + sys.version_info < (3, 8) and not UVLOOP_INSTALLED, + reason="In 3.7 w/o uvloop the port is not always released", +) def test_server_starts_http1_and_http3_bad_order(app: Sanic, caplog): @app.after_server_start def shutdown(*_): From 089a8d208f6a366f83b73a71d55462e2223c3857 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Sun, 26 Jun 2022 11:01:48 +0300 Subject: [PATCH 50/59] Better typing and error message --- pyproject.toml | 6 ++++++ sanic/http/http3.py | 5 +++-- sanic/http/tls/creators.py | 7 ++++--- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 578c40c62f..8b63d86daa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,3 +16,9 @@ lines_after_imports = 2 lines_between_types = 1 multi_line_output = 3 profile = "black" + +[[tool.mypy.overrides]] +module = [ + "trustme.*", +] +ignore_missing_imports = true diff --git a/sanic/http/http3.py b/sanic/http/http3.py index 6d60198d39..09ecad5b9f 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -30,7 +30,7 @@ from sanic.compat import Header from sanic.constants import LocalCertCreator -from sanic.exceptions import PayloadTooLarge, SanicException +from sanic.exceptions import PayloadTooLarge, SanicException, ServerError from sanic.helpers import has_message_body from sanic.http.constants import Stage from sanic.http.stream import Stream @@ -222,7 +222,7 @@ def _send(self, data: bytes, end_stream: bool) -> None: if not self.headers_sent: self.send_headers() if self.stage is not Stage.RESPONSE: - raise Exception(f"not ready to send: {self.stage}") + raise ServerError(f"not ready to send: {self.stage}") # Chunked if ( @@ -299,6 +299,7 @@ def http_event_received(self, event: H3Event) -> None: receiver.future.cancel() receiver.future = asyncio.ensure_future(receiver.run(e)) else: + ... # Intentionally here to help out Touchup logger.debug( # no cov f"{Colors.RED}DOING NOTHING{Colors.END}", extra={"verbosity": 2}, diff --git a/sanic/http/tls/creators.py b/sanic/http/tls/creators.py index f8fbf6cffd..2043cfd681 100644 --- a/sanic/http/tls/creators.py +++ b/sanic/http/tls/creators.py @@ -8,6 +8,7 @@ from contextlib import suppress from pathlib import Path from tempfile import mkdtemp +from types import ModuleType from typing import TYPE_CHECKING, Optional, Tuple, Type, Union, cast from sanic.application.constants import Mode @@ -23,11 +24,11 @@ try: - import trustme # type: ignore + import trustme - TRUSTME_INSTALLED = True # noqa + TRUSTME_INSTALLED = True except (ImportError, ModuleNotFoundError): - trustme = None # type: ignore + trustme = ModuleType("trustme") TRUSTME_INSTALLED = False if TYPE_CHECKING: From 2f244c1ca575ba26006c652f05f2ca442248b1e9 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Sun, 26 Jun 2022 11:09:03 +0300 Subject: [PATCH 51/59] Remove unnecessary __version__ --- sanic/http/http1.py | 1 - sanic/models/asgi.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sanic/http/http1.py b/sanic/http/http1.py index 2ebc37d98a..1f7870ee64 100644 --- a/sanic/http/http1.py +++ b/sanic/http/http1.py @@ -49,7 +49,6 @@ class Http(Stream, metaclass=TouchUpMeta): HEADER_CEILING = 16_384 HEADER_MAX_SIZE = 0 - __version__ = "1.1" __touchup__ = ( "http1_request_header", "http1_response_header", diff --git a/sanic/models/asgi.py b/sanic/models/asgi.py index 562085337a..df6ab3d2ee 100644 --- a/sanic/models/asgi.py +++ b/sanic/models/asgi.py @@ -76,7 +76,7 @@ def get_protocol(self) -> MockProtocol: # type: ignore def get_extra_info( self, info: str, default=None - ) -> Union[str, bool, None]: + ) -> Optional[Union[str, bool]]: if info == "peername": return self.scope.get("client") elif info == "sslcontext": From ea005325202a8e1d437677684c329529209677cb Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Sun, 26 Jun 2022 11:34:13 +0300 Subject: [PATCH 52/59] Only allow stream_id on HTTP/3 --- sanic/request.py | 7 ++++++- sanic/server/protocols/http_protocol.py | 24 ++++++++++++++++-------- tests/test_request.py | 12 ++++++++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/sanic/request.py b/sanic/request.py index 76ff9c0511..c3ef504814 100644 --- a/sanic/request.py +++ b/sanic/request.py @@ -14,8 +14,9 @@ Union, ) -from sanic_routing.route import Route # type: ignore +from sanic_routing.route import Route +from sanic.http.constants import HTTP # type: ignore from sanic.http.stream import Stream from sanic.models.asgi import ASGIScope from sanic.models.http_types import Credentials @@ -196,6 +197,10 @@ def generate_id(*_): @property def stream_id(self): + if self.protocol.version is not HTTP.VERSION_3: + raise ServerError( + "Stream ID is only a property of a HTTP/3 request" + ) return self._stream_id def reset_response(self): diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index cf0e1eaab2..b3d7625b06 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -4,6 +4,7 @@ from aioquic.h3.connection import H3_ALPN, H3Connection +from sanic.http.constants import HTTP from sanic.http.http3 import Http3 from sanic.touchup.meta import TouchUpMeta @@ -17,9 +18,11 @@ from time import monotonic as current_time from aioquic.asyncio import QuicConnectionProtocol - -# from aioquic.h3.events import H3Event -from aioquic.quic.events import ProtocolNegotiated, QuicEvent +from aioquic.quic.events import ( + DatagramFrameReceived, + ProtocolNegotiated, + QuicEvent, +) from sanic.exceptions import RequestTimeout, ServiceUnavailable from sanic.http import Http, Stage @@ -31,6 +34,7 @@ class HttpProtocolMixin: __slots__ = () + __version__: HTTP def _setup_connection(self, *args, **kwargs): self._http = self.HTTP_CLASS(self, *args, **kwargs) @@ -57,6 +61,10 @@ def http(self): return None return self._http + @property + def version(self): + return self.__class__.__version__ + class HttpProtocol(HttpProtocolMixin, SanicProtocol, metaclass=TouchUpMeta): """ @@ -70,6 +78,7 @@ class HttpProtocol(HttpProtocolMixin, SanicProtocol, metaclass=TouchUpMeta): "send", "connection_task", ) + __version__ = HTTP.VERSION_1 __slots__ = ( # request params "request", @@ -271,6 +280,7 @@ def data_received(self, data: bytes): class Http3Protocol(HttpProtocolMixin, QuicConnectionProtocol): HTTP_CLASS = Http3 + __version__ = HTTP.VERSION_3 def __init__(self, *args, app: Sanic, **kwargs) -> None: self.app = app @@ -290,11 +300,9 @@ def quic_event_received(self, event: QuicEvent) -> None: self._connection = H3Connection( self._quic, enable_webtransport=True ) - # elif event.alpn_protocol in H0_ALPN: - # self._http = H0Connection(self._quic) - # elif isinstance(event, DatagramFrameReceived): - # if event.data == b"quack": - # self._quic.send_datagram_frame(b"quack-ack") + elif isinstance(event, DatagramFrameReceived): + if event.data == b"quack": + self._quic.send_datagram_frame(b"quack-ack") # pass event to the HTTP layer if self._connection is not None: diff --git a/tests/test_request.py b/tests/test_request.py index cb68325f46..e01cc1e724 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -231,3 +231,15 @@ async def get(request): _, resp = app.test_client.get("/") assert resp.json["same"] + + +def test_request_stream_id(app): + @app.get("/") + async def get(request): + try: + request.stream_id + except Exception as e: + return response.text(str(e)) + + _, resp = app.test_client.get("/") + assert resp.text == "Stream ID is only a property of a HTTP/3 request" From 93ca605a19bdf51d05609dea5e9090ab90781b3a Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Sun, 26 Jun 2022 11:43:56 +0300 Subject: [PATCH 53/59] Cleanup sanic-routing imports --- pyproject.toml | 1 + sanic/app.py | 7 ++----- sanic/blueprints.py | 4 ++-- sanic/cli/arguments.py | 2 +- sanic/mixins/routes.py | 2 +- sanic/router.py | 10 ++++------ sanic/signals.py | 6 +++--- 7 files changed, 14 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8b63d86daa..7c5e29600d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,5 +20,6 @@ profile = "black" [[tool.mypy.overrides]] module = [ "trustme.*", + "sanic_routing.*", ] ignore_missing_imports = true diff --git a/sanic/app.py b/sanic/app.py index a069fd5900..a58c5fe69a 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -43,11 +43,8 @@ from urllib.parse import urlencode, urlunparse from warnings import filterwarnings -from sanic_routing.exceptions import ( # type: ignore - FinalizationError, - NotFound, -) -from sanic_routing.route import Route # type: ignore +from sanic_routing.exceptions import FinalizationError, NotFound +from sanic_routing.route import Route from sanic.application.ext import setup_ext from sanic.application.state import ApplicationState, Mode, ServerStage diff --git a/sanic/blueprints.py b/sanic/blueprints.py index bee9c437b0..2b9e52e09e 100644 --- a/sanic/blueprints.py +++ b/sanic/blueprints.py @@ -21,8 +21,8 @@ Union, ) -from sanic_routing.exceptions import NotFound # type: ignore -from sanic_routing.route import Route # type: ignore +from sanic_routing.exceptions import NotFound +from sanic_routing.route import Route from sanic.base.root import BaseSanic from sanic.blueprint_group import BlueprintGroup diff --git a/sanic/cli/arguments.py b/sanic/cli/arguments.py index 209deaa6f3..cde125fb98 100644 --- a/sanic/cli/arguments.py +++ b/sanic/cli/arguments.py @@ -3,7 +3,7 @@ from argparse import ArgumentParser, _ArgumentGroup from typing import List, Optional, Type, Union -from sanic_routing import __version__ as __routing_version__ # type: ignore +from sanic_routing import __version__ as __routing_version__ from sanic import __version__ from sanic.http.constants import HTTP diff --git a/sanic/mixins/routes.py b/sanic/mixins/routes.py index ca390abe8e..5704c600db 100644 --- a/sanic/mixins/routes.py +++ b/sanic/mixins/routes.py @@ -21,7 +21,7 @@ ) from urllib.parse import unquote -from sanic_routing.route import Route # type: ignore +from sanic_routing.route import Route from sanic.base.meta import SanicMeta from sanic.compat import stat_async diff --git a/sanic/router.py b/sanic/router.py index 01c268b9a9..ec4d852f5d 100644 --- a/sanic/router.py +++ b/sanic/router.py @@ -5,12 +5,10 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from uuid import UUID -from sanic_routing import BaseRouter # type: ignore -from sanic_routing.exceptions import NoMethod # type: ignore -from sanic_routing.exceptions import ( - NotFound as RoutingNotFound, # type: ignore -) -from sanic_routing.route import Route # type: ignore +from sanic_routing import BaseRouter +from sanic_routing.exceptions import NoMethod +from sanic_routing.exceptions import NotFound as RoutingNotFound +from sanic_routing.route import Route from sanic.constants import HTTP_METHODS from sanic.errorpages import check_error_format diff --git a/sanic/signals.py b/sanic/signals.py index d62a117c52..80c6300b73 100644 --- a/sanic/signals.py +++ b/sanic/signals.py @@ -6,9 +6,9 @@ from inspect import isawaitable from typing import Any, Dict, List, Optional, Tuple, Union, cast -from sanic_routing import BaseRouter, Route, RouteGroup # type: ignore -from sanic_routing.exceptions import NotFound # type: ignore -from sanic_routing.utils import path_to_parts # type: ignore +from sanic_routing import BaseRouter, Route, RouteGroup +from sanic_routing.exceptions import NotFound +from sanic_routing.utils import path_to_parts from sanic.exceptions import InvalidSignal from sanic.log import error_logger, logger From 96042fc32c116a8ffa76cd16710349c182c99611 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Sun, 26 Jun 2022 12:56:52 +0300 Subject: [PATCH 54/59] Setup mock test for HTTP/3 send headers --- tests/http3/test_http_receiver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/http3/test_http_receiver.py b/tests/http3/test_http_receiver.py index 3a0338658c..2784e87b30 100644 --- a/tests/http3/test_http_receiver.py +++ b/tests/http3/test_http_receiver.py @@ -182,6 +182,8 @@ async def test_send_headers(app: Sanic, http_request: Request): ProtocolNegotiated(alpn_protocol="h3") ) + http_request._protocol = receiver.protocol + def send_headers(*args, **kwargs): send_headers_mock(*args, **kwargs) return existing_send_headers( From 095586f8870e8eab8af0ffc2f754abe607a823fc Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 27 Jun 2022 11:35:15 +0300 Subject: [PATCH 55/59] Cleanup tests --- sanic/app.py | 40 +++----------- sanic/config.py | 17 +----- sanic/handlers.py | 95 +++----------------------------- sanic/mixins/routes.py | 15 +---- sanic/mixins/runner.py | 6 +- sanic/response.py | 38 +------------ sanic/server/async_server.py | 10 ---- tests/test_app.py | 48 ---------------- tests/test_config.py | 9 --- tests/test_exceptions_handler.py | 23 ++------ tests/test_motd.py | 29 ---------- tests/test_request_cancel.py | 16 +++--- tests/test_response.py | 33 ++++------- 13 files changed, 45 insertions(+), 334 deletions(-) diff --git a/sanic/app.py b/sanic/app.py index a58c5fe69a..359cc64fa2 100644 --- a/sanic/app.py +++ b/sanic/app.py @@ -169,7 +169,6 @@ def __init__( strict_slashes: bool = False, log_config: Optional[Dict[str, Any]] = None, configure_logging: bool = True, - register: Optional[bool] = None, dumps: Optional[Callable[..., AnyStr]] = None, ) -> None: super().__init__(name=name) @@ -218,20 +217,9 @@ def __init__( # Register alternative method names self.go_fast = self.run - - if register is not None: - deprecation( - "The register argument is deprecated and will stop working " - "in v22.6. After v22.6 all apps will be added to the Sanic " - "app registry.", - 22.6, - ) - self.config.REGISTER = register - if self.config.REGISTER: - self.__class__.register_app(self) - self.router.ctx.app = self self.signal_router.ctx.app = self + self.__class__.register_app(self) if dumps: BaseHTTPResponse._dumps = dumps # type: ignore @@ -736,37 +724,24 @@ async def handle_exception( "has at least partially been sent." ) - # ----------------- deprecated ----------------- handler = self.error_handler._lookup( exception, request.name if request else None ) if handler: - deprecation( + logger.warning( "An error occurred while handling the request after at " "least some part of the response was sent to the client. " - "Therefore, the response from your custom exception " - f"handler {handler.__name__} will not be sent to the " - "client. Beginning in v22.6, Sanic will stop executing " - "custom exception handlers in this scenario. Exception " - "handlers should only be used to generate the exception " - "responses. If you would like to perform any other " - "action on a raised exception, please consider using a " + "The response from your custom exception handler " + f"{handler.__name__} will not be sent to the client." + "Exception handlers should only be used to generate the " + "exception responses. If you would like to perform any " + "other action on a raised exception, consider using a " "signal handler like " '`@app.signal("http.lifecycle.exception")`\n' "For further information, please see the docs: " "https://sanicframework.org/en/guide/advanced/" "signals.html", - 22.6, ) - try: - response = self.error_handler.response(request, exception) - if isawaitable(response): - response = await response - except BaseException as e: - logger.error("An error occurred in the exception handler.") - error_logger.exception(e) - # ---------------------------------------------- - return # -------------------------------------------- # @@ -1556,7 +1531,6 @@ async def _startup(self): if self.state.primary: # TODO: # - Raise warning if secondary apps have error handler config - ErrorHandler.finalize(self.error_handler, config=self.config) if self.config.TOUCHUP: TouchUp.run(self) diff --git a/sanic/config.py b/sanic/config.py index fd63ca5479..cfb55f73bf 100644 --- a/sanic/config.py +++ b/sanic/config.py @@ -36,7 +36,6 @@ "NOISY_EXCEPTIONS": False, "PROXIES_COUNT": None, "REAL_IP_HEADER": None, - "REGISTER": True, "REQUEST_BUFFER_SIZE": 65536, # 64 KiB "REQUEST_MAX_HEADER_SIZE": 8192, # 8 KiB, but cannot exceed 16384 "REQUEST_ID_HEADER": "X-Request-ID", @@ -84,7 +83,6 @@ class Config(dict, metaclass=DescriptorMeta): NOISY_EXCEPTIONS: bool PROXIES_COUNT: Optional[int] REAL_IP_HEADER: Optional[str] - REGISTER: bool REQUEST_BUFFER_SIZE: int REQUEST_MAX_HEADER_SIZE: int REQUEST_ID_HEADER: str @@ -111,7 +109,6 @@ def __init__( super().__init__({**DEFAULT_CONFIG, **defaults}) self._converters = [str, str_to_bool, float, int] - self._LOGO = "" if converters: for converter in converters: @@ -168,24 +165,14 @@ def _post_set(self, attr, value) -> None: "REQUEST_MAX_SIZE", ): self._configure_header_size() - if attr == "LOGO": - self._LOGO = value - deprecation( - "Setting the config.LOGO is deprecated and will no longer " - "be supported starting in v22.6.", - 22.6, - ) - elif attr == "LOCAL_CERT_CREATOR" and not isinstance( + + if attr == "LOCAL_CERT_CREATOR" and not isinstance( self.LOCAL_CERT_CREATOR, LocalCertCreator ): self.LOCAL_CERT_CREATOR = LocalCertCreator[ self.LOCAL_CERT_CREATOR.upper() ] - @property - def LOGO(self): - return self._LOGO - @property def FALLBACK_ERROR_FORMAT(self) -> str: if self._FALLBACK_ERROR_FORMAT is _default: diff --git a/sanic/handlers.py b/sanic/handlers.py index 69b2b3f237..13bcff94ab 100644 --- a/sanic/handlers.py +++ b/sanic/handlers.py @@ -1,21 +1,13 @@ from __future__ import annotations -from typing import Dict, List, Optional, Tuple, Type, Union - -from sanic.config import Config -from sanic.errorpages import ( - DEFAULT_FORMAT, - BaseRenderer, - TextRenderer, - exception_response, -) +from typing import Dict, List, Optional, Tuple, Type + +from sanic.errorpages import BaseRenderer, TextRenderer, exception_response from sanic.exceptions import ( HeaderNotFound, InvalidRangeType, RangeNotSatisfiable, - SanicException, ) -from sanic.helpers import Default, _default from sanic.log import deprecation, error_logger from sanic.models.handler_types import RouteHandler from sanic.response import text @@ -36,91 +28,22 @@ class ErrorHandler: def __init__( self, - fallback: Union[str, Default] = _default, base: Type[BaseRenderer] = TextRenderer, ): self.cached_handlers: Dict[ Tuple[Type[BaseException], Optional[str]], Optional[RouteHandler] ] = {} self.debug = False - self._fallback = fallback self.base = base - if fallback is not _default: - self._warn_fallback_deprecation() - - @property - def fallback(self): # no cov - # This is for backwards compat and can be removed in v22.6 - if self._fallback is _default: - return DEFAULT_FORMAT - return self._fallback - - @fallback.setter - def fallback(self, value: str): # no cov - self._warn_fallback_deprecation() - if not isinstance(value, str): - raise SanicException( - f"Cannot set error handler fallback to: value={value}" - ) - self._fallback = value - - @staticmethod - def _warn_fallback_deprecation(): + @classmethod + def finalize(cls, *args, **kwargs): deprecation( - "Setting the ErrorHandler fallback value directly is " - "deprecated and no longer supported. This feature will " - "be removed in v22.6. Instead, use " - "app.config.FALLBACK_ERROR_FORMAT.", - 22.6, + "ErrorHandler.finalize is deprecated and no longer needed. " + "Please remove update your code to remove it. ", + 22.12, ) - @classmethod - def _get_fallback_value(cls, error_handler: ErrorHandler, config: Config): - if error_handler._fallback is not _default: - if config._FALLBACK_ERROR_FORMAT == error_handler._fallback: - return error_handler.fallback - - error_logger.warning( - "Conflicting error fallback values were found in the " - "error handler and in the app.config while handling an " - "exception. Using the value from app.config." - ) - return config.FALLBACK_ERROR_FORMAT - - @classmethod - def finalize( - cls, - error_handler: ErrorHandler, - config: Config, - fallback: Optional[str] = None, - ): - if fallback: - deprecation( - "Setting the ErrorHandler fallback value via finalize() " - "is deprecated and no longer supported. This feature will " - "be removed in v22.6. Instead, use " - "app.config.FALLBACK_ERROR_FORMAT.", - 22.6, - ) - - if not fallback: - fallback = config.FALLBACK_ERROR_FORMAT - - if fallback != DEFAULT_FORMAT: - if error_handler._fallback is not _default: - error_logger.warning( - f"Setting the fallback value to {fallback}. This changes " - "the current non-default value " - f"'{error_handler._fallback}'." - ) - error_handler._fallback = fallback - - if not isinstance(error_handler, cls): - error_logger.warning( - f"Error handler is non-conforming: {type(error_handler)}" - ) - def _full_lookup(self, exception, route_name: Optional[str] = None): return self.lookup(exception, route_name) @@ -237,7 +160,7 @@ def default(self, request, exception): :return: """ self.log(request, exception) - fallback = ErrorHandler._get_fallback_value(self, request.app.config) + fallback = request.app.config.FALLBACK_ERROR_FORMAT return exception_response( request, exception, diff --git a/sanic/mixins/routes.py b/sanic/mixins/routes.py index 5704c600db..425b29e0ba 100644 --- a/sanic/mixins/routes.py +++ b/sanic/mixins/routes.py @@ -34,7 +34,7 @@ RangeNotSatisfiable, ) from sanic.handlers import ContentRangeHandler -from sanic.log import deprecation, error_logger +from sanic.log import error_logger from sanic.models.futures import FutureRoute, FutureStatic from sanic.models.handler_types import RouteHandler from sanic.response import HTTPResponse, file, file_stream @@ -1025,17 +1025,6 @@ def visit_Return(self, node: Return) -> Any: nonlocal types with suppress(AttributeError): - if node.value.func.id == "stream": # type: ignore - deprecation( - "The sanic.response.stream method has been " - "deprecated and will be removed in v22.6. Please " - "upgrade your application to use the new style " - "streaming pattern. See " - "https://sanicframework.org/en/guide/advanced/" - "streaming.html#response-streaming for more " - "information.", - 22.6, - ) checks = [node.value.func.id] # type: ignore if node.value.keywords: # type: ignore checks += [ @@ -1066,7 +1055,7 @@ def _build_route_context(self, raw): raise AttributeError( "Cannot use restricted route context: " f"{restricted_arguments}. This limitation is only in place " - "until v22.3 when the restricted names will no longer be in" + "until v22.9 when the restricted names will no longer be in" "conflict. See https://github.com/sanic-org/sanic/issues/2303 " "for more information." ) diff --git a/sanic/mixins/runner.py b/sanic/mixins/runner.py index ee787776e6..3d2b327d18 100644 --- a/sanic/mixins/runner.py +++ b/sanic/mixins/runner.py @@ -565,11 +565,7 @@ def motd( if self.config.MOTD_DISPLAY: extra.update(self.config.MOTD_DISPLAY) - logo = ( - get_logo(coffee=self.state.coffee) - if self.config.LOGO == "" or self.config.LOGO is True - else self.config.LOGO - ) + logo = get_logo(coffee=self.state.coffee) MOTD.output(logo, serve_location, display, extra) diff --git a/sanic/response.py b/sanic/response.py index e894142d36..9dc20580a0 100644 --- a/sanic/response.py +++ b/sanic/response.py @@ -426,8 +426,7 @@ def redirect( class ResponseStream: """ ResponseStream is a compat layer to bridge the gap after the deprecation - of StreamingHTTPResponse. In v22.6 it will be removed when: - - stream is removed + of StreamingHTTPResponse. It will be removed when: - file_stream is moved to new style streaming - file and file_stream are combined into a single API """ @@ -555,38 +554,3 @@ async def _streaming_fn(response): headers=headers, content_type=mime_type, ) - - -def stream( - streaming_fn: Callable[ - [Union[BaseHTTPResponse, ResponseStream]], Coroutine[Any, Any, None] - ], - status: int = 200, - headers: Optional[Dict[str, str]] = None, - content_type: str = "text/plain; charset=utf-8", -) -> ResponseStream: - """Accepts a coroutine `streaming_fn` which can be used to - write chunks to a streaming response. Returns a `ResponseStream`. - - Example usage:: - - @app.route("/") - async def index(request): - async def streaming_fn(response): - await response.write('foo') - await response.write('bar') - - return stream(streaming_fn, content_type='text/plain') - - :param streaming_fn: A coroutine accepts a response and - writes content to that response. - :param status: HTTP status. - :param content_type: Specific content_type. - :param headers: Custom Headers. - """ - return ResponseStream( - streaming_fn, - headers=headers, - content_type=content_type, - status=status, - ) diff --git a/sanic/server/async_server.py b/sanic/server/async_server.py index c13af4646e..e2cf4fa195 100644 --- a/sanic/server/async_server.py +++ b/sanic/server/async_server.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING from sanic.exceptions import SanicException -from sanic.log import deprecation if TYPE_CHECKING: @@ -35,15 +34,6 @@ def __init__( self.serve_coro = serve_coro self.server = None - @property - def init(self): - deprecation( - "AsyncioServer.init has been deprecated and will be removed " - "in v22.6. Use Sanic.state.is_started instead.", - 22.6, - ) - return self.app.state.is_started - def startup(self): """ Trigger "before_server_start" events diff --git a/tests/test_app.py b/tests/test_app.py index 1c8f705b5d..6c6f9e9a73 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -4,7 +4,6 @@ from collections import Counter from inspect import isawaitable -from os import environ from unittest.mock import Mock, patch import pytest @@ -113,19 +112,6 @@ def test_create_server_main_convenience(app, caplog): ) in caplog.record_tuples -def test_create_server_init(app, caplog): - loop = asyncio.get_event_loop() - asyncio_srv_coro = app.create_server(return_asyncio_server=True) - server = loop.run_until_complete(asyncio_srv_coro) - - message = ( - "AsyncioServer.init has been deprecated and will be removed in v22.6. " - "Use Sanic.state.is_started instead." - ) - with pytest.warns(DeprecationWarning, match=message): - server.init - - def test_app_loop_not_running(app): with pytest.raises(SanicException) as excinfo: app.loop @@ -385,40 +371,6 @@ def test_get_app_default_ambiguous(): Sanic.get_app() -def test_app_no_registry(): - Sanic("no-register", register=False) - with pytest.raises( - SanicException, match='Sanic app name "no-register" not found.' - ): - Sanic.get_app("no-register") - - -def test_app_no_registry_deprecation_message(): - with pytest.warns(DeprecationWarning) as records: - Sanic("no-register", register=False) - Sanic("yes-register", register=True) - - message = ( - "[DEPRECATION v22.6] The register argument is deprecated and will " - "stop working in v22.6. After v22.6 all apps will be added to the " - "Sanic app registry." - ) - - assert len(records) == 2 - for record in records: - assert record.message.args[0] == message - - -def test_app_no_registry_env(): - environ["SANIC_REGISTER"] = "False" - Sanic("no-register") - with pytest.raises( - SanicException, match='Sanic app name "no-register" not found.' - ): - Sanic.get_app("no-register") - del environ["SANIC_REGISTER"] - - def test_app_set_attribute_warning(app): message = ( "Setting variables on Sanic instances is not allowed. You should " diff --git a/tests/test_config.py b/tests/test_config.py index 764d7940b5..f52d8472c8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -371,15 +371,6 @@ def test_update_from_lowercase_key(app: Sanic): assert "test_setting_value" not in app.config -def test_deprecation_notice_when_setting_logo(app: Sanic): - message = ( - "Setting the config.LOGO is deprecated and will no longer be " - "supported starting in v22.6." - ) - with pytest.warns(DeprecationWarning, match=message): - app.config.LOGO = "My Custom Logo" - - def test_config_set_methods(app: Sanic, monkeypatch: MonkeyPatch): post_set = Mock() monkeypatch.setattr(Config, "_post_set", post_set) diff --git a/tests/test_exceptions_handler.py b/tests/test_exceptions_handler.py index 78f79388f7..47b8378c94 100644 --- a/tests/test_exceptions_handler.py +++ b/tests/test_exceptions_handler.py @@ -13,13 +13,7 @@ from sanic.exceptions import BadRequest, Forbidden, NotFound, ServerError from sanic.handlers import ErrorHandler from sanic.request import Request -from sanic.response import stream, text - - -async def sample_streaming_fn(response): - await response.write("foo,") - await asyncio.sleep(0.001) - await response.write("bar") +from sanic.response import text class ErrorWithRequestCtx(ServerError): @@ -81,10 +75,10 @@ def handler_exception(request, exception): @exception_handler_app.exception(Forbidden) async def async_handler_exception(request, exception): - return stream( - sample_streaming_fn, - content_type="text/csv", - ) + response = await request.respond(content_type="text/csv") + await response.send("foo,") + await asyncio.sleep(0.001) + await response.send("bar") @exception_handler_app.middleware async def some_request_middleware(request): @@ -183,7 +177,7 @@ def import_error_handler(): class ModuleNotFoundError(ImportError): pass - handler = ErrorHandler("auto") + handler = ErrorHandler() handler.add(ImportError, import_error_handler) handler.add(CustomError, custom_error_handler) handler.add(ServerError, server_error_handler) @@ -257,11 +251,6 @@ async def handler2(request: Request): await request.respond() raise ServerError("Exception") - with caplog.at_level(logging.WARNING): - _, response = app.test_client.get("/1") - assert "some text" in response.text - - # Change to assert warning not in the records in the future version. message_in_records( caplog.records, ( diff --git a/tests/test_motd.py b/tests/test_motd.py index 83c7e4bf8c..51b838b12d 100644 --- a/tests/test_motd.py +++ b/tests/test_motd.py @@ -19,35 +19,6 @@ def test_logo_base(app, run_startup): assert logs[0][2] == BASE_LOGO -def test_logo_false(app, run_startup): - app.config.LOGO = False - - logs = run_startup(app) - - banner, port = logs[1][2].rsplit(":", 1) - assert logs[0][1] == logging.INFO - assert banner == "Goin' Fast @ http://127.0.0.1" - assert int(port) > 0 - - -def test_logo_true(app, run_startup): - app.config.LOGO = True - - logs = run_startup(app) - - assert logs[0][1] == logging.DEBUG - assert logs[0][2] == BASE_LOGO - - -def test_logo_custom(app, run_startup): - app.config.LOGO = "My Custom Logo" - - logs = run_startup(app) - - assert logs[0][1] == logging.DEBUG - assert logs[0][2] == "My Custom Logo" - - def test_motd_with_expected_info(app, run_startup): logs = run_startup(app) diff --git a/tests/test_request_cancel.py b/tests/test_request_cancel.py index 8b5de18dc9..4680949be0 100644 --- a/tests/test_request_cancel.py +++ b/tests/test_request_cancel.py @@ -3,7 +3,7 @@ import pytest -from sanic.response import stream, text +from sanic.response import text @pytest.mark.asyncio @@ -43,18 +43,16 @@ async def test_stream_request_cancel_when_conn_lost(app): async def post(request, id): assert isinstance(request.stream, asyncio.Queue) - async def streaming(response): - while True: - body = await request.stream.get() - if body is None: - break - await response.write(body.decode("utf-8")) + response = await request.respond() await asyncio.sleep(1.0) # at this point client is already disconnected app.ctx.still_serving_cancelled_request = True - - return stream(streaming) + while True: + body = await request.stream.get() + if body is None: + break + await response.send(body.decode("utf-8")) # schedule client call loop = asyncio.get_event_loop() diff --git a/tests/test_response.py b/tests/test_response.py index a25a1318e4..632806cd93 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -27,7 +27,6 @@ file_stream, json, raw, - stream, text, ) @@ -49,10 +48,13 @@ async def hello_route(request: Request): assert b"Internal Server Error" in response.body -async def sample_streaming_fn(response): - await response.write("foo,") +async def sample_streaming_fn(request, response=None): + if not response: + response = await request.respond(content_type="text/csv") + await response.send("foo,") await asyncio.sleep(0.001) - await response.write("bar") + await response.send("bar") + await response.eof() def test_method_not_allowed(): @@ -217,10 +219,7 @@ def test_no_content(json_app): def streaming_app(app): @app.route("/") async def test(request: Request): - return stream( - sample_streaming_fn, - content_type="text/csv", - ) + await sample_streaming_fn(request) return app @@ -229,11 +228,11 @@ async def test(request: Request): def non_chunked_streaming_app(app): @app.route("/") async def test(request: Request): - return stream( - sample_streaming_fn, + response = await request.respond( headers={"Content-Length": "7"}, content_type="text/csv", ) + await sample_streaming_fn(request, response) return app @@ -283,18 +282,6 @@ def test_non_chunked_streaming_returns_correct_content( assert response.text == "foo,bar" -def test_stream_response_with_cookies_legacy(app): - @app.route("/") - async def test(request: Request): - response = stream(sample_streaming_fn, content_type="text/csv") - response.cookies["test"] = "modified" - response.cookies["test"] = "pass" - return response - - request, response = app.test_client.get("/") - assert response.cookies["test"] == "pass" - - def test_stream_response_with_cookies(app): @app.route("/") async def test(request: Request): @@ -317,7 +304,7 @@ async def test(request: Request): def test_stream_response_without_cookies(app): @app.route("/") async def test(request: Request): - return stream(sample_streaming_fn, content_type="text/csv") + await sample_streaming_fn(request) request, response = app.test_client.get("/") assert response.cookies == {} From 11532084441b37ab6cb97fbffcc765e668d5cdae Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Mon, 27 Jun 2022 11:39:28 +0300 Subject: [PATCH 56/59] Add back removed assert --- tests/test_exceptions_handler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_exceptions_handler.py b/tests/test_exceptions_handler.py index 47b8378c94..c505cead9f 100644 --- a/tests/test_exceptions_handler.py +++ b/tests/test_exceptions_handler.py @@ -251,6 +251,10 @@ async def handler2(request: Request): await request.respond() raise ServerError("Exception") + with caplog.at_level(logging.WARNING): + _, response = app.test_client.get("/1") + assert "some text" in response.text + message_in_records( caplog.records, ( From 8ce57bce7d7aa74a40decfdab8f99f44386700d8 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 28 Jun 2022 09:34:53 +0300 Subject: [PATCH 57/59] Add release notes to changelog --- docs/sanic/changelog.rst | 1 + docs/sanic/releases/22/22.6.md | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 docs/sanic/releases/22/22.6.md diff --git a/docs/sanic/changelog.rst b/docs/sanic/changelog.rst index d810075abe..4e8b363c1a 100644 --- a/docs/sanic/changelog.rst +++ b/docs/sanic/changelog.rst @@ -1,6 +1,7 @@ 📜 Changelog ============ +.. mdinclude:: ./releases/22/22.6.md .. mdinclude:: ./releases/22/22.3.md .. mdinclude:: ./releases/21/21.12.md .. mdinclude:: ./releases/21/21.9.md diff --git a/docs/sanic/releases/22/22.6.md b/docs/sanic/releases/22/22.6.md new file mode 100644 index 0000000000..1c8b3f7b58 --- /dev/null +++ b/docs/sanic/releases/22/22.6.md @@ -0,0 +1,40 @@ +## Version 22.6.0 + +### Features +- [#2378](https://github.com/sanic-org/sanic/pull/2378) Introduce HTTP/3 and autogeneration of TLS certificates in `DEBUG` mode + - 👶 *EARLY RELEASE FEATURE*: Serving Sanic over HTTP/3 is an early release feature. It does not yet fully cover the HTTP/3 spec, but instead aims for feature parity with Sanic's existing HTTP/1.1 server. Websockets, WebTransport, push responses are examples of some features not yet implemented. + - 📦 *EXTRA REQUIREMENT*: Not all HTTP clients are capable of interfacing with HTTP/3 servers. You may need to install a [HTTP/3 capable client](https://curl.se/docs/http3.html). + - 📦 *EXTRA REQUIREMENT*: In order to use TLS autogeneration, you must install either [mkcert](https://github.com/FiloSottile/mkcert) or [trustme](https://github.com/python-trio/trustme). +- [#2416](https://github.com/sanic-org/sanic/pull/2416) Add message to `task.cancel` +- [#2420](https://github.com/sanic-org/sanic/pull/2420) Add exception aliases for more consistent naming with standard HTTP response types (`BadRequest`, `MethodNotAllowed`, `RangeNotSatisfiable`) +- [#2432](https://github.com/sanic-org/sanic/pull/2432) Expose ASGI `scope` as a property on the `Request` object +- [#2438](https://github.com/sanic-org/sanic/pull/2438) Easier access to websocket class for annotation: `from sanic import Websocket` +- [#2439](https://github.com/sanic-org/sanic/pull/2439) New API for reading form values with options: `Request.get_form` +- [#2447](https://github.com/sanic-org/sanic/pull/2447), [#2486](https://github.com/sanic-org/sanic/pull/2486) Improved API to support setting cache control headers +- [#2453](https://github.com/sanic-org/sanic/pull/2453) Move verbosity filtering to logger +- [#2475](https://github.com/sanic-org/sanic/pull/2475) Expose getter for current request using `Request.get_current()` + +### Bugfixes +- [#2448](https://github.com/sanic-org/sanic/pull/2448) Fix to allow running with `pythonw.exe` or places where there is no `sys.stdout` +- [#2451](https://github.com/sanic-org/sanic/pull/2451) Trigger `http.lifecycle.request` signal in ASGI mode +- [#2455](https://github.com/sanic-org/sanic/pull/2455) Resolve typing of stacked route definitions +- [#2463](https://github.com/sanic-org/sanic/pull/2463) Properly catch websocket CancelledError in websocket handler in Python 3.7 + +### Deprecations and Removals +- [#2487](https://github.com/sanic-org/sanic/pull/2487) v22.6 deprecations and changes + 1. Optional application registry + 1. Execution of custom handlers after some part of response was sent + 1. Configuring fallback handlers on the `ErrorHandler` + 1. Custom `LOGO` setting + 1. `sanic.response.stream` + 1. `AsyncioServer.init` + +### Developer infrastructure +- [#2449](https://github.com/sanic-org/sanic/pull/2449) Clean up `black` and `isort` config +- [#2479](https://github.com/sanic-org/sanic/pull/2479) Fix some flappy tests + +### Improved Documentation +- [#2461](https://github.com/sanic-org/sanic/pull/2461) Update example to match current application naming standards +- [#2466](https://github.com/sanic-org/sanic/pull/2466) Better type annotation for `Extend` +- [#2485](https://github.com/sanic-org/sanic/pull/2485) Improved help messages in CLI + From 436461c0683d24c686736f66b2e141c369f814a7 Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 28 Jun 2022 13:50:59 +0300 Subject: [PATCH 58/59] Bump version --- CHANGELOG.rst | 14 ++++---- docs/index.rst | 2 +- docs/sanic/releases/21/21.12.md | 6 ++-- docs/sanic/releases/22/22.6.md | 4 ++- sanic/__version__.py | 2 +- sanic/http/http3.py | 45 ++++++++++++++----------- sanic/server/protocols/http_protocol.py | 26 ++++++++------ sanic/server/runners.py | 15 +++++++-- 8 files changed, 71 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d8a76c6cc0..f3f6ad881e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -313,8 +313,10 @@ Version 21.3.0 `#2074 `_ Performance adjustments in ``handle_request_`` -Version 20.12.3 ---------------- +Version 20.12.3 🔷 +------------------ + +`Current LTS version` **Bugfixes** @@ -348,8 +350,8 @@ Version 19.12.5 `#2027 `_ Remove old chardet requirement, add in hard multidict requirement -Version 20.12.0 ---------------- +Version 20.12.0 🔹 +----------------- **Features** @@ -357,8 +359,8 @@ Version 20.12.0 `#1993 `_ Add disable app registry -Version 20.12.0 ---------------- +Version 20.12.0 🔹 +----------------- **Features** diff --git a/docs/index.rst b/docs/index.rst index 34e0e00631..c4588c66a5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,7 +9,7 @@ API === .. toctree:: - :maxdepth: 2 + :maxdepth: 3 👥 User Guide sanic/api_reference diff --git a/docs/sanic/releases/21/21.12.md b/docs/sanic/releases/21/21.12.md index f8f0d95456..1ec1670a06 100644 --- a/docs/sanic/releases/21/21.12.md +++ b/docs/sanic/releases/21/21.12.md @@ -1,10 +1,12 @@ -## Version 21.12.1 +## Version 21.12.1 🔷 + +_Current LTS version_ - [#2349](https://github.com/sanic-org/sanic/pull/2349) Only display MOTD on startup - [#2354](https://github.com/sanic-org/sanic/pull/2354) Ignore name argument in Python 3.7 - [#2355](https://github.com/sanic-org/sanic/pull/2355) Add config.update support for all config values -## Version 21.12.0 +## Version 21.12.0 🔹 ### Features - [#2260](https://github.com/sanic-org/sanic/pull/2260) Allow early Blueprint registrations to still apply later added objects diff --git a/docs/sanic/releases/22/22.6.md b/docs/sanic/releases/22/22.6.md index 1c8b3f7b58..8265411812 100644 --- a/docs/sanic/releases/22/22.6.md +++ b/docs/sanic/releases/22/22.6.md @@ -1,4 +1,6 @@ -## Version 22.6.0 +## Version 22.6.0 🔶 + +_Current version_ ### Features - [#2378](https://github.com/sanic-org/sanic/pull/2378) Introduce HTTP/3 and autogeneration of TLS certificates in `DEBUG` mode diff --git a/sanic/__version__.py b/sanic/__version__.py index ead0aa2fd4..ac0f77e8f9 100644 --- a/sanic/__version__.py +++ b/sanic/__version__.py @@ -1 +1 @@ -__version__ = "22.3.2" +__version__ = "22.6.0" diff --git a/sanic/http/http3.py b/sanic/http/http3.py index f48fa7ee50..e29e29adf8 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -16,18 +16,6 @@ cast, ) -from aioquic.h0.connection import H0_ALPN, H0Connection -from aioquic.h3.connection import H3_ALPN, H3Connection -from aioquic.h3.events import ( - DatagramReceived, - DataReceived, - H3Event, - HeadersReceived, - WebTransportStreamDataReceived, -) -from aioquic.quic.configuration import QuicConfiguration -from aioquic.tls import SessionTicket - from sanic.compat import Header from sanic.constants import LocalCertCreator from sanic.exceptions import PayloadTooLarge, SanicException, ServerError @@ -40,14 +28,30 @@ from sanic.models.server_types import ConnInfo +try: + from aioquic.h0.connection import H0_ALPN, H0Connection + from aioquic.h3.connection import H3_ALPN, H3Connection + from aioquic.h3.events import ( + DatagramReceived, + DataReceived, + H3Event, + HeadersReceived, + WebTransportStreamDataReceived, + ) + from aioquic.quic.configuration import QuicConfiguration + from aioquic.tls import SessionTicket + + HTTP3_AVAILABLE = True +except ModuleNotFoundError: + HTTP3_AVAILABLE = False + if TYPE_CHECKING: from sanic import Sanic from sanic.request import Request from sanic.response import BaseHTTPResponse from sanic.server.protocols.http_protocol import Http3Protocol - -HttpConnection = Union[H0Connection, H3Connection] + HttpConnection = Union[H0Connection, H3Connection] class HTTP3Transport(TransportProtocol): @@ -269,12 +273,13 @@ class Http3: Internal helper for managing the HTTP/3 request/response cycle """ - HANDLER_PROPERTY_MAPPING = { - DataReceived: "stream_id", - HeadersReceived: "stream_id", - DatagramReceived: "flow_id", - WebTransportStreamDataReceived: "session_id", - } + if HTTP3_AVAILABLE: + HANDLER_PROPERTY_MAPPING = { + DataReceived: "stream_id", + HeadersReceived: "stream_id", + DatagramReceived: "flow_id", + WebTransportStreamDataReceived: "session_id", + } def __init__( self, diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index b3d7625b06..a4be09862f 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING, Optional -from aioquic.h3.connection import H3_ALPN, H3Connection - from sanic.http.constants import HTTP from sanic.http.http3 import Http3 from sanic.touchup.meta import TouchUpMeta @@ -17,13 +15,6 @@ from asyncio import CancelledError from time import monotonic as current_time -from aioquic.asyncio import QuicConnectionProtocol -from aioquic.quic.events import ( - DatagramFrameReceived, - ProtocolNegotiated, - QuicEvent, -) - from sanic.exceptions import RequestTimeout, ServiceUnavailable from sanic.http import Http, Stage from sanic.log import Colors, error_logger, logger @@ -32,6 +23,21 @@ from sanic.server.protocols.base_protocol import SanicProtocol +ConnectionProtocol = type("ConnectionProtocol", (), {}) +try: + from aioquic.asyncio import QuicConnectionProtocol + from aioquic.h3.connection import H3_ALPN, H3Connection + from aioquic.quic.events import ( + DatagramFrameReceived, + ProtocolNegotiated, + QuicEvent, + ) + + ConnectionProtocol = QuicConnectionProtocol +except ModuleNotFoundError: + ... + + class HttpProtocolMixin: __slots__ = () __version__: HTTP @@ -278,7 +284,7 @@ def data_received(self, data: bytes): error_logger.exception("protocol.data_received") -class Http3Protocol(HttpProtocolMixin, QuicConnectionProtocol): +class Http3Protocol(HttpProtocolMixin, ConnectionProtocol): # type: ignore HTTP_CLASS = Http3 __version__ = HTTP.VERSION_3 diff --git a/sanic/server/runners.py b/sanic/server/runners.py index 81c8b64a44..dc99a9efb8 100644 --- a/sanic/server/runners.py +++ b/sanic/server/runners.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Dict, Optional, Type, Union from sanic.config import Config +from sanic.exceptions import ServerError from sanic.http.constants import HTTP from sanic.http.tls import get_ssl_context from sanic.server.events import trigger_events @@ -23,8 +24,6 @@ from signal import SIG_IGN, SIGINT, SIGTERM, Signals from signal import signal as signal_func -from aioquic.asyncio import serve as quic_serve - from sanic.application.ext import setup_ext from sanic.compat import OS_IS_WINDOWS, ctrlc_workaround_for_windows from sanic.http.http3 import SessionTicketStore, get_config @@ -39,6 +38,14 @@ ) +try: + from aioquic.asyncio import serve as quic_serve + + HTTP3_AVAILABLE = True +except ModuleNotFoundError: + HTTP3_AVAILABLE = False + + def serve( host, port, @@ -273,6 +280,10 @@ def _serve_http_3( register_sys_signals: bool = True, run_multiple: bool = False, ): + if not HTTP3_AVAILABLE: + raise ServerError( + "Cannot run HTTP/3 server without aioquic installed. " + ) protocol = partial(Http3Protocol, app=app) ticket_store = SessionTicketStore() ssl_context = get_ssl_context(app, ssl) From 0e5df3e7cca81f7c7acf2a639cbfb834eed3b9fe Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Tue, 28 Jun 2022 14:12:44 +0300 Subject: [PATCH 59/59] Add nocov --- sanic/http/http3.py | 2 +- sanic/server/protocols/http_protocol.py | 2 +- sanic/server/runners.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sanic/http/http3.py b/sanic/http/http3.py index e29e29adf8..18919ba5ff 100644 --- a/sanic/http/http3.py +++ b/sanic/http/http3.py @@ -42,7 +42,7 @@ from aioquic.tls import SessionTicket HTTP3_AVAILABLE = True -except ModuleNotFoundError: +except ModuleNotFoundError: # no cov HTTP3_AVAILABLE = False if TYPE_CHECKING: diff --git a/sanic/server/protocols/http_protocol.py b/sanic/server/protocols/http_protocol.py index a4be09862f..616e2303aa 100644 --- a/sanic/server/protocols/http_protocol.py +++ b/sanic/server/protocols/http_protocol.py @@ -34,7 +34,7 @@ ) ConnectionProtocol = QuicConnectionProtocol -except ModuleNotFoundError: +except ModuleNotFoundError: # no cov ... diff --git a/sanic/server/runners.py b/sanic/server/runners.py index dc99a9efb8..a1b86e81a5 100644 --- a/sanic/server/runners.py +++ b/sanic/server/runners.py @@ -42,7 +42,7 @@ from aioquic.asyncio import serve as quic_serve HTTP3_AVAILABLE = True -except ModuleNotFoundError: +except ModuleNotFoundError: # no cov HTTP3_AVAILABLE = False