ENH Add Pyodide support

.github/workflows/ci-cd.yml

@@ -269,6 +269,82 @@ jobs:
             steps.python-install.outputs.python-version
           }}
 
+  test-pyodide:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+
+    name: Test pyodide
+    needs: gen_llhttp
+    runs-on: ubuntu-22.04
+    env:
+      PYODIDE_VERSION: 0.25.0a1
+      # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION.
+      # The appropriate versions can be found in the Pyodide repodata.json
+      # "info" field, or in Makefile.envs:
+      # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2
+      PYTHON_VERSION: 3.11.4
+      EMSCRIPTEN_VERSION: 3.1.45
+      NODE_VERSION: 18
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Setup Python ${{ env.PYTHON_VERSION }}
+      id: python-install
+      uses: actions/setup-python@v4
+      with:
+        allow-prereleases: true
+        python-version: ${{ env.PYTHON_VERSION }}
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"    # - name: Cache
+    - name: Cache PyPI
+      uses: actions/cache@v3.3.2
+      with:
+        key: pip-ci-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ matrix.no-extensions }}-${{ hashFiles('requirements/*.txt') }}
+        path: ${{ steps.pip-cache.outputs.dir }}
+        restore-keys: |
+            pip-ci-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ matrix.no-extensions }}-
+    - name: Update pip, wheel, setuptools, build, twine
+      run: |
+        python -m pip install -U pip wheel setuptools build twine
+    - name: Install dependencies
+      run: |
+        python -m pip install -r requirements/test.in -c requirements/test.txt
+    - name: Restore llhttp generated files
+      if: ${{ matrix.no-extensions == '' }}
+      uses: actions/download-artifact@v3
+      with:
+        name: llhttp
+        path: vendor/llhttp/build/
+    - name: Cythonize
+      if: ${{ matrix.no-extensions == '' }}
+      run: |
+        make cythonize
+    - uses: mymindstorm/setup-emsdk@v12
+      with:
+        version: ${{ env.EMSCRIPTEN_VERSION }}
+        actions-cache-folder: emsdk-cache
+    - name: Install pyodide-build
+      run: pip install "pydantic<2" pyodide-build==$PYODIDE_VERSION
+    - name: Build
+      run: |
+        CFLAGS=-g2 LDFLAGS=-g2 pyodide build
+
+    - uses: pyodide/pyodide-actions/download-pyodide@v1
+      with:
+        version: ${{ env.PYODIDE_VERSION }}
+        to: pyodide-dist
+
+    - uses: pyodide/pyodide-actions/install-browser@v1
+
+    - name: Test
+      run: |
+        pip install pytest-pyodide
+        pytest tests/test_pyodide.py --rt chrome --dist-dir ./pyodide-dist
+
   check:  # This job does nothing and is only used for the branch protection
     if: always()
 

.pre-commit-config.yaml

@@ -29,6 +29,9 @@ repos:
   rev: v1.5.0
   hooks:
   - id: yesqa
+    additional_dependencies:
+      - flake8-docstrings==1.6.0
+      - flake8-requirements==1.7.8
 - repo: https://github.com/PyCQA/isort
   rev: '5.12.0'
   hooks:

CHANGES/7803.feature

@@ -0,0 +1 @@
+Added basic support for using aiohttp in Pyodide.

CONTRIBUTORS.txt

@@ -141,6 +141,7 @@ Hans Adema
 Harmon Y.
 Harry Liu
 Hiroshi Ogawa
+Hood Chatham
 Hrishikesh Paranjape
 Hu Bo
 Hugh Young

aiohttp/client.py

@@ -68,17 +68,26 @@
     ClientRequest,
     ClientResponse,
     Fingerprint,
+    PyodideClientRequest,
+    PyodideClientResponse,
     RequestInfo,
 )
 from .client_ws import (
     DEFAULT_WS_CLIENT_TIMEOUT,
     ClientWebSocketResponse,
     ClientWSTimeout,
 )
-from .connector import BaseConnector, NamedPipeConnector, TCPConnector, UnixConnector
+from .connector import (
+    BaseConnector,
+    NamedPipeConnector,
+    PyodideConnector,
+    TCPConnector,
+    UnixConnector,
+)
 from .cookiejar import CookieJar
 from .helpers import (
     _SENTINEL,
+    IS_PYODIDE,
     BasicAuth,
     TimeoutHandle,
     ceil_timeout,
@@ -211,8 +220,8 @@ def __init__(
         skip_auto_headers: Optional[Iterable[str]] = None,
         auth: Optional[BasicAuth] = None,
         json_serialize: JSONEncoder = json.dumps,
-        request_class: Type[ClientRequest] = ClientRequest,
-        response_class: Type[ClientResponse] = ClientResponse,
+        request_class: Optional[Type[ClientRequest]] = None,
+        response_class: Optional[Type[ClientResponse]] = None,
         ws_response_class: Type[ClientWebSocketResponse] = ClientWebSocketResponse,
         version: HttpVersion = http.HttpVersion11,
         cookie_jar: Optional[AbstractCookieJar] = None,
@@ -241,7 +250,7 @@ def __init__(
         loop = asyncio.get_running_loop()
 
         if connector is None:
-            connector = TCPConnector()
+            connector = PyodideConnector() if IS_PYODIDE else TCPConnector()
 
         # Initialize these three attrs before raising any exception,
         # they are used in __del__
@@ -291,6 +300,11 @@ def __init__(
         else:
             self._skip_auto_headers = frozenset()
 
+        if request_class is None:
+            request_class = PyodideClientRequest if IS_PYODIDE else ClientRequest
+        if response_class is None:
+            response_class = PyodideClientResponse if IS_PYODIDE else ClientResponse
+
         self._request_class = request_class
         self._response_class = response_class
         self._ws_response_class = ws_response_class

aiohttp/client_reqrep.py

@@ -44,9 +44,13 @@
 from .formdata import FormData
 from .hdrs import CONTENT_TYPE
 from .helpers import (
+    IS_PYODIDE,
     BaseTimerContext,
     BasicAuth,
     HeadersMixin,
+    JsArrayBuffer,
+    JsRequest,
+    JsResponse,
     TimerNoop,
     basicauth_from_netrc,
     is_expected_content_type,
@@ -86,7 +90,7 @@
 
 if TYPE_CHECKING:
     from .client import ClientSession
-    from .connector import Connection
+    from .connector import Connection, PyodideConnection
     from .tracing import Trace
 
 
@@ -689,6 +693,73 @@ async def _on_headers_request_sent(
             await trace.send_request_headers(method, url, headers)
 
 
+def _make_js_request(
+    path: str, *, method: str, headers: CIMultiDict[str], signal: Any, body: Any
+) -> JsRequest:
+    from js import Headers, Request  # type:ignore[import-not-found] # noqa: I900
+    from pyodide.ffi import to_js  # type:ignore[import-not-found] # noqa: I900
+
+    if method.lower() in ["get", "head"]:
+        body = None
+    # TODO: to_js does an unnecessary copy.
+    elif isinstance(body, payload.Payload):
+        body = to_js(body._value)
+    elif isinstance(body, (bytes, bytearray)):
+        body = to_js(body)
+    else:
+        # What else can happen here? Maybe body could be a list of
+        # bytes? In that case we should turn it into a Blob.
+        raise NotImplementedError("OOPS")
+
+    return cast(
+        JsRequest,
+        Request.new(
+            path,
+            method=method,
+            headers=Headers.new(headers.items()),
+            body=body,
+            signal=signal,
+        ),
+    )
+
+
+class PyodideClientRequest(ClientRequest):
+    async def send(
+        self, conn: "PyodideConnection"  # type:ignore[override]
+    ) -> "ClientResponse":
+        if not IS_PYODIDE:
+            raise RuntimeError("PyodideClientRequest only works in Pyodide")
+
+        protocol = conn.protocol
+        assert protocol is not None
+
+        request = _make_js_request(
+            str(self.url),
+            method=self.method,
+            headers=self.headers,
+            body=self.body,
+            signal=protocol.abortcontroller.signal,
+        )
+        response_future = protocol.fetch_handler(request)
+        response_class = self.response_class
+        assert response_class is not None
+        assert issubclass(response_class, PyodideClientResponse)
+        self.response = response_class(
+            self.method,
+            self.original_url,
+            writer=None,  # type:ignore[arg-type]
+            continue100=self._continue,
+            timer=self._timer,
+            request_info=self.request_info,
+            traces=self._traces,
+            loop=self.loop,
+            session=self._session,
+            response_future=response_future,
+        )
+        self.response.version = self.version
+        return self.response
+
+
 class ClientResponse(HeadersMixin):
     # Some of these attributes are None when created,
     # but will be set by the start() method.
@@ -1123,3 +1194,57 @@ async def __aexit__(
         # if state is broken
         self.release()
         await self.wait_for_close()
+
+
+class PyodideClientResponse(ClientResponse):
+    def __init__(
+        self,
+        method: str,
+        url: URL,
+        *,
+        writer: "asyncio.Task[None]",
+        continue100: Optional["asyncio.Future[bool]"],
+        timer: Optional[BaseTimerContext],
+        request_info: RequestInfo,
+        traces: List["Trace"],
+        loop: asyncio.AbstractEventLoop,
+        session: "ClientSession",
+        response_future: "asyncio.Future[JsResponse]",
+    ):
+        if not IS_PYODIDE:
+            raise RuntimeError("PyodideClientResponse only works in Pyodide")
+        self.response_future = response_future
+        super().__init__(
+            method,
+            url,
+            writer=writer,
+            continue100=continue100,
+            timer=timer,
+            request_info=request_info,
+            traces=traces,
+            loop=loop,
+            session=session,
+        )
+
+    async def start(self, connection: "Connection") -> "ClientResponse":
+        from .streams import DataQueue
+
+        self._connection = connection
+        self._protocol = connection.protocol
+        jsresp = await self.response_future
+        self.status = jsresp.status
+        self.reason = jsresp.statusText
+        # This is not quite correct in handling of repeated headers
+        self._headers = cast(CIMultiDictProxy[str], CIMultiDict(jsresp.headers))
+        self._raw_headers = tuple(
+            (e[0].encode(), e[1].encode()) for e in jsresp.headers
+        )
+        self.content = DataQueue(self._loop)  # type:ignore[assignment]
+
+        def done_callback(fut: "asyncio.Future[JsArrayBuffer]") -> None:
+            data = fut.result().to_bytes()
+            self.content.feed_data(data, len(data))
+            self.content.feed_eof()
+
+        jsresp.arrayBuffer().add_done_callback(done_callback)
+        return self