Upgrade to Tornado 6

pyproject.toml

@@ -49,6 +49,7 @@ no_implicit_reexport = false
 module = [
     "ahocorasick.*",
     "aioapns.*",
+    "asgiref.*",
     "bitfield.*",
     "bmemcached.*",
     "bson.*",
@@ -61,6 +62,7 @@ module = [
     "disposable_email_domains.*",
     "django.*",
     "django_auth_ldap.*",
+    "django_bmemcached.*",
     "django_cte.*",
     "django_otp.*",
     "django_scim.*",
@@ -91,7 +93,6 @@ module = [
     "sourcemap.*",
     "talon_core.*",
     "tlds.*",
-    "tornado.*",
     "twitter.*",
     "two_factor.*",
     "zxcvbn.*",

requirements/common.in

@@ -84,7 +84,7 @@ redis
 sourcemap
 
 # Tornado used for server->client push system
-tornado==4.*  # https://github.com/zulip/zulip/issues/8913
+tornado
 
 # Fast JSON parser
 orjson

requirements/dev.txt

@@ -46,9 +46,9 @@ arrow==1.2.2 \
     --hash=sha256:05caf1fd3d9a11a1135b2b6f09887421153b94558e5ef4d090b567b47173ac2b \
     --hash=sha256:d622c46ca681b5b3e3574fcb60a04e5cc81b9625112d5fb2b44220c36c892177
     # via gitlint-core
-asgiref==3.5.0 \
-    --hash=sha256:2f8abc20f7248433085eda803936d98992f1343ddb022065779f37c5da0181d0 \
-    --hash=sha256:88d59c13d634dcffe0510be048210188edd79aeccb6a6c9028cdad6f31d730a9
+asgiref==3.5.1 \
+    --hash=sha256:45a429524fba18aba9d512498b19d220c4d628e75b40cf5c627524dbaebc5cc1 \
+    --hash=sha256:fddeea3c53fa99d0cdb613c3941cc6e52d822491fc2753fba25768fb5bf4e865
     # via django
 async-generator==1.10 \
     --hash=sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b \
@@ -1929,12 +1929,48 @@ tomli==2.0.1 \
     #   black
     #   mypy
     #   pep517
-tornado==4.5.3 \
-    --hash=sha256:5ef073ac6180038ccf99411fe05ae9aafb675952a2c8db60592d5daf8401f803 \
-    --hash=sha256:6d14e47eab0e15799cf3cdcc86b0b98279da68522caace2bd7ce644287685f0a \
-    --hash=sha256:92b7ca81e18ba9ec3031a7ee73d4577ac21d41a0c9b775a9182f43301c3b5f8e \
-    --hash=sha256:ab587996fe6fb9ce65abfda440f9b61e4f9f2cf921967723540679176915e4c3 \
-    --hash=sha256:b36298e9f63f18cad97378db2222c0e0ca6a55f6304e605515e05a25483ed51a
+tornado==6.1 \
+    --hash=sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb \
+    --hash=sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c \
+    --hash=sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288 \
+    --hash=sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95 \
+    --hash=sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558 \
+    --hash=sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe \
+    --hash=sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791 \
+    --hash=sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d \
+    --hash=sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326 \
+    --hash=sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b \
+    --hash=sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4 \
+    --hash=sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c \
+    --hash=sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910 \
+    --hash=sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5 \
+    --hash=sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c \
+    --hash=sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0 \
+    --hash=sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675 \
+    --hash=sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd \
+    --hash=sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f \
+    --hash=sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c \
+    --hash=sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea \
+    --hash=sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6 \
+    --hash=sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05 \
+    --hash=sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd \
+    --hash=sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575 \
+    --hash=sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a \
+    --hash=sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37 \
+    --hash=sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795 \
+    --hash=sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f \
+    --hash=sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32 \
+    --hash=sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c \
+    --hash=sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01 \
+    --hash=sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4 \
+    --hash=sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2 \
+    --hash=sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921 \
+    --hash=sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085 \
+    --hash=sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df \
+    --hash=sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102 \
+    --hash=sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5 \
+    --hash=sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68 \
+    --hash=sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5
     # via
     #   -r requirements/common.in
     #   snakeviz

requirements/prod.txt

@@ -38,9 +38,9 @@ argon2-cffi-bindings==21.2.0 \
     --hash=sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e \
     --hash=sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351
     # via argon2-cffi
-asgiref==3.5.0 \
-    --hash=sha256:2f8abc20f7248433085eda803936d98992f1343ddb022065779f37c5da0181d0 \
-    --hash=sha256:88d59c13d634dcffe0510be048210188edd79aeccb6a6c9028cdad6f31d730a9
+asgiref==3.5.1 \
+    --hash=sha256:45a429524fba18aba9d512498b19d220c4d628e75b40cf5c627524dbaebc5cc1 \
+    --hash=sha256:fddeea3c53fa99d0cdb613c3941cc6e52d822491fc2753fba25768fb5bf4e865
     # via django
 async-timeout==4.0.2 \
     --hash=sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15 \
@@ -1281,12 +1281,48 @@ tlds==2022032300 \
     --hash=sha256:40ad49151d06de78c647101c6c7c79b68bf33651bbdd15f134eab46b5f458037 \
     --hash=sha256:ecc7d864bfb3eb4e6d03ceba545e25b50483877f86e6d5b757ae51923260a954
     # via -r requirements/common.in
-tornado==4.5.3 \
-    --hash=sha256:5ef073ac6180038ccf99411fe05ae9aafb675952a2c8db60592d5daf8401f803 \
-    --hash=sha256:6d14e47eab0e15799cf3cdcc86b0b98279da68522caace2bd7ce644287685f0a \
-    --hash=sha256:92b7ca81e18ba9ec3031a7ee73d4577ac21d41a0c9b775a9182f43301c3b5f8e \
-    --hash=sha256:ab587996fe6fb9ce65abfda440f9b61e4f9f2cf921967723540679176915e4c3 \
-    --hash=sha256:b36298e9f63f18cad97378db2222c0e0ca6a55f6304e605515e05a25483ed51a
+tornado==6.1 \
+    --hash=sha256:0a00ff4561e2929a2c37ce706cb8233b7907e0cdc22eab98888aca5dd3775feb \
+    --hash=sha256:0d321a39c36e5f2c4ff12b4ed58d41390460f798422c4504e09eb5678e09998c \
+    --hash=sha256:1e8225a1070cd8eec59a996c43229fe8f95689cb16e552d130b9793cb570a288 \
+    --hash=sha256:20241b3cb4f425e971cb0a8e4ffc9b0a861530ae3c52f2b0434e6c1b57e9fd95 \
+    --hash=sha256:25ad220258349a12ae87ede08a7b04aca51237721f63b1808d39bdb4b2164558 \
+    --hash=sha256:33892118b165401f291070100d6d09359ca74addda679b60390b09f8ef325ffe \
+    --hash=sha256:33c6e81d7bd55b468d2e793517c909b139960b6c790a60b7991b9b6b76fb9791 \
+    --hash=sha256:3447475585bae2e77ecb832fc0300c3695516a47d46cefa0528181a34c5b9d3d \
+    --hash=sha256:34ca2dac9e4d7afb0bed4677512e36a52f09caa6fded70b4e3e1c89dbd92c326 \
+    --hash=sha256:3e63498f680547ed24d2c71e6497f24bca791aca2fe116dbc2bd0ac7f191691b \
+    --hash=sha256:548430be2740e327b3fe0201abe471f314741efcb0067ec4f2d7dcfb4825f3e4 \
+    --hash=sha256:6196a5c39286cc37c024cd78834fb9345e464525d8991c21e908cc046d1cc02c \
+    --hash=sha256:61b32d06ae8a036a6607805e6720ef00a3c98207038444ba7fd3d169cd998910 \
+    --hash=sha256:6286efab1ed6e74b7028327365cf7346b1d777d63ab30e21a0f4d5b275fc17d5 \
+    --hash=sha256:65d98939f1a2e74b58839f8c4dab3b6b3c1ce84972ae712be02845e65391ac7c \
+    --hash=sha256:66324e4e1beede9ac79e60f88de548da58b1f8ab4b2f1354d8375774f997e6c0 \
+    --hash=sha256:6c77c9937962577a6a76917845d06af6ab9197702a42e1346d8ae2e76b5e3675 \
+    --hash=sha256:70dec29e8ac485dbf57481baee40781c63e381bebea080991893cd297742b8fd \
+    --hash=sha256:7250a3fa399f08ec9cb3f7b1b987955d17e044f1ade821b32e5f435130250d7f \
+    --hash=sha256:748290bf9112b581c525e6e6d3820621ff020ed95af6f17fedef416b27ed564c \
+    --hash=sha256:7da13da6f985aab7f6f28debab00c67ff9cbacd588e8477034c0652ac141feea \
+    --hash=sha256:8f959b26f2634a091bb42241c3ed8d3cedb506e7c27b8dd5c7b9f745318ddbb6 \
+    --hash=sha256:9de9e5188a782be6b1ce866e8a51bc76a0fbaa0e16613823fc38e4fc2556ad05 \
+    --hash=sha256:a48900ecea1cbb71b8c71c620dee15b62f85f7c14189bdeee54966fbd9a0c5bd \
+    --hash=sha256:b87936fd2c317b6ee08a5741ea06b9d11a6074ef4cc42e031bc6403f82a32575 \
+    --hash=sha256:c77da1263aa361938476f04c4b6c8916001b90b2c2fdd92d8d535e1af48fba5a \
+    --hash=sha256:cb5ec8eead331e3bb4ce8066cf06d2dfef1bfb1b2a73082dfe8a161301b76e37 \
+    --hash=sha256:cc0ee35043162abbf717b7df924597ade8e5395e7b66d18270116f8745ceb795 \
+    --hash=sha256:d14d30e7f46a0476efb0deb5b61343b1526f73ebb5ed84f23dc794bdb88f9d9f \
+    --hash=sha256:d371e811d6b156d82aa5f9a4e08b58debf97c302a35714f6f45e35139c332e32 \
+    --hash=sha256:d3d20ea5782ba63ed13bc2b8c291a053c8d807a8fa927d941bd718468f7b950c \
+    --hash=sha256:d3f7594930c423fd9f5d1a76bee85a2c36fd8b4b16921cae7e965f22575e9c01 \
+    --hash=sha256:dcef026f608f678c118779cd6591c8af6e9b4155c44e0d1bc0c87c036fb8c8c4 \
+    --hash=sha256:e0791ac58d91ac58f694d8d2957884df8e4e2f6687cdf367ef7eb7497f79eaa2 \
+    --hash=sha256:e385b637ac3acaae8022e7e47dfa7b83d3620e432e3ecb9a3f7f58f150e50921 \
+    --hash=sha256:e519d64089b0876c7b467274468709dadf11e41d65f63bba207e04217f47c085 \
+    --hash=sha256:e7229e60ac41a1202444497ddde70a48d33909e484f96eb0da9baf8dc68541df \
+    --hash=sha256:ed3ad863b1b40cd1d4bd21e7498329ccaece75db5a5bf58cd3c9f130843e7102 \
+    --hash=sha256:f0ba29bafd8e7e22920567ce0d232c26d4d47c8b5cf4ed7b562b5db39fa199c5 \
+    --hash=sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68 \
+    --hash=sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5
     # via -r requirements/common.in
 traitlets==5.1.1 \
     --hash=sha256:059f456c5a7c1c82b98c2e8c799f39c9b8128f6d0d46941ee118daace9eb70c7 \

tools/run-dev.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
 import argparse
+import asyncio
 import os
 import pwd
 import signal
 import subprocess
 import sys
-from typing import Any, Callable, Generator, List, Sequence
+from typing import List, Sequence
 from urllib.parse import urlunparse
 
 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
@@ -16,8 +17,8 @@
 
 sanity_check.check_venv(__file__)
 
-from tornado import gen, httpclient, httputil, web
-from tornado.ioloop import IOLoop
+from tornado import httpclient, httputil, web
+from tornado.platform.asyncio import AsyncIOMainLoop
 
 from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok
 
@@ -84,8 +85,10 @@
     settings_module = "zproject.test_settings"
     # Don't auto-reload when running Puppeteer tests
     runserver_args = ["--noreload"]
+    tornado_autoreload = []
 else:
     settings_module = "zproject.settings"
+    tornado_autoreload = ["-m", "tornado.autoreload"]
 
 manage_args = [f"--settings={settings_module}"]
 os.environ["DJANGO_SETTINGS_MODULE"] = settings_module
@@ -134,6 +137,8 @@ def server_processes() -> List[List[str]]:
         [
             "env",
             "PYTHONUNBUFFERED=1",
+            "python3",
+            *tornado_autoreload,
             "./manage.py",
             "runtornado",
             *manage_args,
@@ -193,22 +198,7 @@ def transform_url(protocol: str, path: str, query: str, target_port: int, target
     return newpath
 
 
-@gen.engine
-def fetch_request(
-    url: str, callback: Any, **kwargs: Any
-) -> "Generator[Callable[..., Any], Any, None]":
-    # use large timeouts to handle polling requests
-    req = httpclient.HTTPRequest(
-        url,
-        connect_timeout=240.0,
-        request_timeout=240.0,
-        decompress_response=False,
-        **kwargs,
-    )
-    client = httpclient.AsyncHTTPClient()
-    # wait for response
-    response = yield gen.Task(client.fetch, req)
-    callback(response)
+client: httpclient.AsyncHTTPClient
 
 
 class BaseHandler(web.RequestHandler):
@@ -248,24 +238,9 @@ def options(self) -> None:
     def delete(self) -> None:
         pass
 
-    def handle_response(self, response: Any) -> None:
-        if response.error and not isinstance(response.error, httpclient.HTTPError):
-            self.set_status(500)
-            self.write("Internal server error:\n" + str(response.error))
-        else:
-            self.set_status(response.code, response.reason)
-            self._headers = httputil.HTTPHeaders()  # clear tornado default header
-
-            for header, v in response.headers.get_all():
-                # some header appear multiple times, eg 'Set-Cookie'
-                if header.lower() != "transfer-encoding":
-                    self.add_header(header, v)
-            if response.body:
-                self.write(response.body)
-        self.finish()
-
-    @web.asynchronous
-    def prepare(self) -> None:
+    async def prepare(self) -> None:
+        assert self.request.method is not None
+        assert self.request.remote_ip is not None
         if "X-REAL-IP" not in self.request.headers:
             self.request.headers["X-REAL-IP"] = self.request.remote_ip
         if "X-FORWARDED_PORT" not in self.request.headers:
@@ -278,22 +253,35 @@ def prepare(self) -> None:
             self.target_host,
         )
         try:
-            fetch_request(
+            request = httpclient.HTTPRequest(
                 url=url,
-                callback=self.handle_response,
                 method=self.request.method,
                 headers=self._add_request_headers(["upgrade-insecure-requests"]),
                 follow_redirects=False,
                 body=getattr(self.request, "body"),
                 allow_nonstandard_methods=True,
+                # use large timeouts to handle polling requests
+                connect_timeout=240.0,
+                request_timeout=240.0,
+                # https://github.com/tornadoweb/tornado/issues/2743
+                decompress_response=False,
             )
-        except httpclient.HTTPError as e:
-            if hasattr(e, "response") and e.response:
-                self.handle_response(e.response)
-            else:
-                self.set_status(500)
-                self.write("Internal server error:\n" + str(e))
-                self.finish()
+            response = await client.fetch(request, raise_error=False)
+
+            self.set_status(response.code, response.reason)
+            self._headers = httputil.HTTPHeaders()  # clear tornado default header
+
+            for header, v in response.headers.get_all():
+                # some header appear multiple times, eg 'Set-Cookie'
+                if header.lower() != "transfer-encoding":
+                    self.add_header(header, v)
+            if response.body:
+                self.write(response.body)
+            self.finish()
+        except (ConnectionError, httpclient.HTTPError) as e:
+            self.set_status(500)
+            self.write("Internal server error:\n" + str(e))
+            self.finish()
 
 
 class WebPackHandler(BaseHandler):
@@ -310,27 +298,21 @@ class TornadoHandler(BaseHandler):
 
 class Application(web.Application):
     def __init__(self, enable_logging: bool = False) -> None:
-        handlers = [
-            (r"/json/events.*", TornadoHandler),
-            (r"/api/v1/events.*", TornadoHandler),
-            (r"/webpack.*", WebPackHandler),
-            (r"/.*", DjangoHandler),
-        ]
-        super().__init__(handlers, enable_logging=enable_logging)
-
-    def log_request(self, handler: BaseHandler) -> None:
+        super().__init__(
+            [
+                (r"/json/events.*", TornadoHandler),
+                (r"/api/v1/events.*", TornadoHandler),
+                (r"/webpack.*", WebPackHandler),
+                (r"/.*", DjangoHandler),
+            ],
+            enable_logging=enable_logging,
+        )
+
+    def log_request(self, handler: web.RequestHandler) -> None:
         if self.settings["enable_logging"]:
             super().log_request(handler)
 
 
-def shutdown_handler(*args: Any, **kwargs: Any) -> None:
-    io_loop = IOLoop.instance()
-    if io_loop._callbacks:
-        io_loop.call_later(1, shutdown_handler)
-    else:
-        io_loop.stop()
-
-
 def print_listeners() -> None:
     # Since we can't import settings from here, we duplicate some
     # EXTERNAL_HOST logic from dev_settings.py.
@@ -360,7 +342,12 @@ def print_listeners() -> None:
 
 children = []
 
-try:
+
+async def serve() -> None:
+    global client
+
+    AsyncIOMainLoop().install()
+
     if options.test:
         do_one_time_webpack_compile()
     else:
@@ -369,6 +356,7 @@ def print_listeners() -> None:
     for cmd in server_processes():
         children.append(subprocess.Popen(cmd))
 
+    client = httpclient.AsyncHTTPClient()
     app = Application(enable_logging=options.enable_tornado_logging)
     try:
         app.listen(proxy_port, address=options.interface)
@@ -379,10 +367,14 @@ def print_listeners() -> None:
 
     print_listeners()
 
-    ioloop = IOLoop.instance()
+
+loop = asyncio.new_event_loop()
+
+try:
+    loop.run_until_complete(serve())
     for s in (signal.SIGINT, signal.SIGTERM):
-        signal.signal(s, shutdown_handler)
-    ioloop.start()
+        loop.add_signal_handler(s, loop.stop)
+    loop.run_forever()
 finally:
     for child in children:
         child.terminate()