Add Pyodide/Emscripten client support via FetchConnector (continuation of #7803)

.github/workflows/ci-cd.yml

@@ -372,6 +372,66 @@ jobs:
         # Currently only Android supports colored output. See https://github.com/python/cpython/issues/150932 for iOS.
         CIBW_TEST_COMMAND: python -m pytest ${{ matrix.config.platform == 'android' && '--color=yes' || '' }}
 
+  test-pyodide:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+
+    name: Test (pyodide, ${{ matrix.pyver }})
+    runs-on: ubuntu-latest
+    needs: gen_llhttp
+    env:
+      # Must be a version supported by the pinned cibuildwheel below.
+      PYODIDE_VERSION: 0.29.3
+    strategy:
+      matrix:
+        pyver: ["cp313"]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v7
+      with:
+        submodules: true
+    - name: Setup Python ${{ matrix.pyver }}
+      id: python-install
+      # important: do not use system python
+      env:
+        UV_PYTHON_PREFERENCE: only-managed
+      uses: astral-sh/setup-uv@v8.2.0
+      with:
+        python-version: ${{ matrix.pyver }}
+        activate-environment: true
+        enable-cache: true
+    - name: Install build tooling and cython
+      run: |
+        uv pip install -U pip wheel setuptools build twine -r requirements/cython.in -c requirements/cython.txt
+    - name: Restore llhttp generated files
+      uses: actions/download-artifact@v8
+      with:
+        name: llhttp
+        path: vendor/llhttp/build/
+    - name: Cythonize
+      run: |
+        make cythonize
+    - name: Install cibuildwheel
+      run: uv pip install cibuildwheel==3.4.1
+    - name: Build wheel
+      # See the comment in test-mobile about unsetting GITHUB_ACTIONS.
+      run: env -u GITHUB_ACTIONS cibuildwheel --output-dir dist
+      env:
+        CIBW_BUILD: ${{ matrix.pyver }}-*
+        CIBW_PLATFORM: pyodide
+        CIBW_PYODIDE_VERSION: ${{ env.PYODIDE_VERSION }}
+    - name: Download Pyodide distribution
+      uses: pyodide/pyodide-actions/download-pyodide@v2
+      with:
+        version: ${{ env.PYODIDE_VERSION }}
+        to: pyodide-dist
+    - name: Install test dependencies
+      # aiohttp itself is needed on the host because tests/conftest.py
+      # imports it; the pure-Python build is sufficient.
+      run: AIOHTTP_NO_EXTENSIONS=1 uv pip install -e . pytest-pyodide==0.59.2 pytest pytest-aiohttp pytest-timeout coverage
+    - name: Run tests in Pyodide (Node.js)
+      run: python -m pytest tests/test_pyodide.py --rt node --dist-dir=./pyodide-dist
+
   autobahn:
     permissions:
       contents: read # to fetch code (actions/checkout)
@@ -556,6 +616,7 @@ jobs:
     - lint
     - test
     - test-mobile
+    - test-pyodide
     - autobahn
 
     runs-on: ubuntu-latest

.mypy.ini

@@ -36,3 +36,16 @@ ignore_missing_imports = True
 
 [mypy-gunicorn.*]
 ignore_missing_imports = True
+
+# JavaScript FFI modules, only available inside Emscripten/Pyodide runtimes.
+[mypy-js]
+ignore_missing_imports = True
+
+[mypy-pyodide.*]
+ignore_missing_imports = True
+
+# The run_in_pyodide decorator comes from pytest.importorskip() because
+# pytest-pyodide is only installed for the dedicated Pyodide CI job.
+[mypy-test_pyodide]
+disallow_any_decorated = False
+disallow_untyped_decorators = False

CHANGES/7803.feature.rst

@@ -0,0 +1,5 @@
+Added client support for running under Pyodide / Emscripten (WebAssembly in
+browsers and Node.js).  :class:`~aiohttp.ClientSession` now automatically
+uses the new :class:`~aiohttp.FetchConnector`, which dispatches requests
+through the JavaScript ``fetch()`` API instead of raw sockets -- by
+:user:`hoodmane` and :user:`bendichter`.

CONTRIBUTORS.txt

@@ -62,6 +62,7 @@ Austin Scola
 Bai Haoran
 Ben Bader
 Ben Beasley
+Ben Dichter
 Ben Greiner
 Ben Kallus
 Ben Timby

aiohttp/__init__.py

@@ -87,6 +87,7 @@
     get_payload,
     payload_type,
 )
+from .pyodide import FetchConnector
 from .resolver import AsyncResolver, DefaultResolver, ThreadedResolver
 from .streams import EMPTY_PAYLOAD, DataQueue, EofStream, StreamReader
 from .tracing import (
@@ -139,6 +140,7 @@
     "ClientWSTimeout",
     "ConnectionTimeoutError",
     "ContentTypeError",
+    "FetchConnector",
     "Fingerprint",
     "InvalidURL",
     "InvalidUrlClientError",

aiohttp/_llhttp_wasm_shim.c

@@ -0,0 +1,35 @@
+/* llhttp's api.c references external wasm_on_* callbacks whenever __wasm__
+   is defined; they are normally provided by JavaScript in llhttp's own
+   WebAssembly bundle.  When llhttp is instead linked into a Python extension
+   compiled with Emscripten (Pyodide) nothing provides those symbols, which
+   makes the extension fail to load.  aiohttp installs its own callbacks via
+   llhttp_init() and never uses the wasm_settings/llhttp_alloc() path, so
+   no-op definitions are sufficient to satisfy the linker. */
+#ifdef __wasm__
+
+#include "llhttp.h"
+
+int wasm_on_message_begin(llhttp_t *p) { return 0; }
+
+int wasm_on_url(llhttp_t *p, const char *at, size_t length) { return 0; }
+
+int wasm_on_status(llhttp_t *p, const char *at, size_t length) { return 0; }
+
+int wasm_on_header_field(llhttp_t *p, const char *at, size_t length) {
+  return 0;
+}
+
+int wasm_on_header_value(llhttp_t *p, const char *at, size_t length) {
+  return 0;
+}
+
+int wasm_on_headers_complete(llhttp_t *p, int status_code, uint8_t upgrade,
+                             int should_keep_alive) {
+  return 0;
+}
+
+int wasm_on_body(llhttp_t *p, const char *at, size_t length) { return 0; }
+
+int wasm_on_message_complete(llhttp_t *p) { return 0; }
+
+#endif /* __wasm__ */

aiohttp/client.py

@@ -350,7 +350,13 @@ def __init__(
             )
 
         if connector is None:
-            connector = TCPConnector(ssl_shutdown_timeout=ssl_shutdown_timeout)
+            if sys.platform == "emscripten":
+                # WebAssembly has no sockets; requests go through fetch().
+                from .pyodide import FetchConnector
+
+                connector = FetchConnector()
+            else:
+                connector = TCPConnector(ssl_shutdown_timeout=ssl_shutdown_timeout)
         # Initialize these three attrs before raising any exception,
         # they are used in __del__
         self._connector = connector