diff --git a/.codecov.yml b/.codecov.yml index fb11a3f13f9..b5dd4ab4711 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -10,6 +10,11 @@ comment: coverage: range: "95..100" +ignore: + # Runs only inside the Pyodide runtime (test-pyodide CI job), where + # coverage is not collected. + - tests/test_pyodide.py + component_management: individual_components: - component_id: project diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index c018dba6177..c07a66cc14f 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -372,6 +372,66 @@ jobs: # Currently only Android supports colored output. See https://github.com/python/cpython/issues/150932 for iOS. CIBW_TEST_COMMAND: python -m pytest ${{ matrix.config.platform == 'android' && '--color=yes' || '' }} + test-pyodide: + permissions: + contents: read # to fetch code (actions/checkout) + + name: Test (pyodide, ${{ matrix.pyver }}) + runs-on: ubuntu-latest + needs: gen_llhttp + env: + # Must be a version supported by the pinned cibuildwheel below. + PYODIDE_VERSION: 0.29.3 + strategy: + matrix: + pyver: ["cp313"] + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + submodules: true + - name: Setup Python ${{ matrix.pyver }} + id: python-install + # important: do not use system python + env: + UV_PYTHON_PREFERENCE: only-managed + uses: astral-sh/setup-uv@v8.2.0 + with: + python-version: ${{ matrix.pyver }} + activate-environment: true + enable-cache: true + - name: Install build tooling and cython + run: | + uv pip install -U pip wheel setuptools build twine -r requirements/cython.in -c requirements/cython.txt + - name: Restore llhttp generated files + uses: actions/download-artifact@v8 + with: + name: llhttp + path: vendor/llhttp/build/ + - name: Cythonize + run: | + make cythonize + - name: Install cibuildwheel + run: uv pip install cibuildwheel==3.4.1 + - name: Build wheel + # See the comment in test-mobile about unsetting GITHUB_ACTIONS. + run: env -u GITHUB_ACTIONS cibuildwheel --output-dir dist + env: + CIBW_BUILD: ${{ matrix.pyver }}-* + CIBW_PLATFORM: pyodide + CIBW_PYODIDE_VERSION: ${{ env.PYODIDE_VERSION }} + - name: Download Pyodide distribution + uses: pyodide/pyodide-actions/download-pyodide@v2 + with: + version: ${{ env.PYODIDE_VERSION }} + to: pyodide-dist + - name: Install test dependencies + # aiohttp itself is needed on the host because tests/conftest.py + # imports it; the pure-Python build is sufficient. + run: AIOHTTP_NO_EXTENSIONS=1 uv pip install -e . pytest-pyodide==0.59.2 pytest pytest-aiohttp pytest-timeout coverage + - name: Run tests in Pyodide (Node.js) + run: python -m pytest tests/test_pyodide.py --rt node --dist-dir=./pyodide-dist + autobahn: permissions: contents: read # to fetch code (actions/checkout) @@ -556,6 +616,7 @@ jobs: - lint - test - test-mobile + - test-pyodide - autobahn runs-on: ubuntu-latest diff --git a/.mypy.ini b/.mypy.ini index 4d5c2eaaf09..c1cb586173e 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -36,3 +36,16 @@ ignore_missing_imports = True [mypy-gunicorn.*] ignore_missing_imports = True + +# JavaScript FFI modules, only available inside Emscripten/Pyodide runtimes. +[mypy-js] +ignore_missing_imports = True + +[mypy-pyodide.*] +ignore_missing_imports = True + +# The run_in_pyodide decorator comes from pytest.importorskip() because +# pytest-pyodide is only installed for the dedicated Pyodide CI job. +[mypy-test_pyodide] +disallow_any_decorated = False +disallow_untyped_decorators = False diff --git a/CHANGES/7803.feature.rst b/CHANGES/7803.feature.rst new file mode 100644 index 00000000000..71fda314327 --- /dev/null +++ b/CHANGES/7803.feature.rst @@ -0,0 +1,5 @@ +Added client support for running under Pyodide / Emscripten (WebAssembly in +browsers and Node.js). :class:`~aiohttp.ClientSession` now automatically +uses the new :class:`~aiohttp.FetchConnector`, which dispatches requests +through the JavaScript ``fetch()`` API instead of raw sockets -- by +:user:`hoodmane` and :user:`bendichter`. diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 65c20d023e7..7aa21ab76b0 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -62,6 +62,7 @@ Austin Scola Bai Haoran Ben Bader Ben Beasley +Ben Dichter Ben Greiner Ben Kallus Ben Timby diff --git a/aiohttp/__init__.py b/aiohttp/__init__.py index 64bc3ced0a1..59294c0ebcb 100644 --- a/aiohttp/__init__.py +++ b/aiohttp/__init__.py @@ -87,6 +87,7 @@ get_payload, payload_type, ) +from .pyodide import FetchConnector from .resolver import AsyncResolver, DefaultResolver, ThreadedResolver from .streams import EMPTY_PAYLOAD, DataQueue, EofStream, StreamReader from .tracing import ( @@ -139,6 +140,7 @@ "ClientWSTimeout", "ConnectionTimeoutError", "ContentTypeError", + "FetchConnector", "Fingerprint", "InvalidURL", "InvalidUrlClientError", diff --git a/aiohttp/_llhttp_wasm_shim.c b/aiohttp/_llhttp_wasm_shim.c new file mode 100644 index 00000000000..f23107df6f2 --- /dev/null +++ b/aiohttp/_llhttp_wasm_shim.c @@ -0,0 +1,35 @@ +/* llhttp's api.c references external wasm_on_* callbacks whenever __wasm__ + is defined; they are normally provided by JavaScript in llhttp's own + WebAssembly bundle. When llhttp is instead linked into a Python extension + compiled with Emscripten (Pyodide) nothing provides those symbols, which + makes the extension fail to load. aiohttp installs its own callbacks via + llhttp_init() and never uses the wasm_settings/llhttp_alloc() path, so + no-op definitions are sufficient to satisfy the linker. */ +#ifdef __wasm__ + +#include "llhttp.h" + +int wasm_on_message_begin(llhttp_t *p) { return 0; } + +int wasm_on_url(llhttp_t *p, const char *at, size_t length) { return 0; } + +int wasm_on_status(llhttp_t *p, const char *at, size_t length) { return 0; } + +int wasm_on_header_field(llhttp_t *p, const char *at, size_t length) { + return 0; +} + +int wasm_on_header_value(llhttp_t *p, const char *at, size_t length) { + return 0; +} + +int wasm_on_headers_complete(llhttp_t *p, int status_code, uint8_t upgrade, + int should_keep_alive) { + return 0; +} + +int wasm_on_body(llhttp_t *p, const char *at, size_t length) { return 0; } + +int wasm_on_message_complete(llhttp_t *p) { return 0; } + +#endif /* __wasm__ */ diff --git a/aiohttp/client.py b/aiohttp/client.py index 809fa63ed24..e53121a9cdc 100644 --- a/aiohttp/client.py +++ b/aiohttp/client.py @@ -350,7 +350,13 @@ def __init__( ) if connector is None: - connector = TCPConnector(ssl_shutdown_timeout=ssl_shutdown_timeout) + if sys.platform == "emscripten": + # WebAssembly has no sockets; requests go through fetch(). + from .pyodide import FetchConnector + + connector = FetchConnector() + else: + connector = TCPConnector(ssl_shutdown_timeout=ssl_shutdown_timeout) # Initialize these three attrs before raising any exception, # they are used in __del__ self._connector = connector diff --git a/aiohttp/pyodide.py b/aiohttp/pyodide.py new file mode 100644 index 00000000000..7733baed84b --- /dev/null +++ b/aiohttp/pyodide.py @@ -0,0 +1,285 @@ +"""Client support for Pyodide / Emscripten platforms (browsers, Node.js). + +WebAssembly runtimes have no raw sockets, so the regular +:class:`~aiohttp.TCPConnector` cannot work there. This module provides +:class:`FetchConnector`, a connector that dispatches each request through +the JavaScript ``fetch()`` API instead of a TCP connection. It is picked +automatically by :class:`~aiohttp.ClientSession` when running under +Emscripten, so most code can simply use aiohttp as usual. + +The connector speaks regular HTTP/1.1 with the rest of the client stack: +the serialized request is decoded with :class:`~aiohttp.http.HttpRequestParser` +and the ``fetch()`` result is re-serialized into HTTP/1.1 response bytes fed +through the standard :class:`~aiohttp.client_proto.ResponseHandler`. This +keeps ``ClientSession`` itself completely unaware of the platform. + +Limitations imposed by ``fetch()``: + +* No proxies, no ``CONNECT``, no connection upgrades (WebSockets). +* Redirects are followed transparently by the browser; aiohttp only sees + the final response and cannot report redirect history. +* The trust store, cookies (in browsers) and CORS policy are managed by the + JavaScript runtime, not by aiohttp; ``ssl`` arguments are ignored. +* ``Expect: 100-continue`` is answered locally instead of by the server. +""" + +import asyncio +import sys +from collections.abc import Callable, Iterable, Mapping +from typing import TYPE_CHECKING, Any, Final + +from . import hdrs +from .base_protocol import BaseProtocol +from .client_exceptions import ClientConnectionError +from .client_proto import ResponseHandler +from .connector import BaseConnector +from .helpers import EMPTY_BODY_STATUS_CODES, set_exception, set_result +from .http import HttpRequestParser +from .http_parser import RawRequestMessage +from .streams import StreamReader + +if TYPE_CHECKING: + from .client import ClientTimeout + from .client_reqrep import ClientRequest + from .tracing import Trace + +__all__ = ("FetchConnector",) + +IS_EMSCRIPTEN: Final = sys.platform == "emscripten" + +# Connection management and body framing are handled by fetch() itself +# (browsers forbid many of these outright), and the body handed to fetch() +# is already unframed and decoded, so these request headers must not be +# forwarded. +_UNSENDABLE_REQUEST_HEADERS: Final = frozenset( + ( + hdrs.ACCEPT_ENCODING.lower(), + hdrs.CONNECTION.lower(), + hdrs.CONTENT_ENCODING.lower(), + hdrs.CONTENT_LENGTH.lower(), + hdrs.EXPECT.lower(), + hdrs.HOST.lower(), + hdrs.KEEP_ALIVE.lower(), + hdrs.PROXY_AUTHENTICATE.lower(), + hdrs.PROXY_AUTHORIZATION.lower(), + hdrs.TE.lower(), + hdrs.TRAILER.lower(), + hdrs.TRANSFER_ENCODING.lower(), + hdrs.UPGRADE.lower(), + ) +) + +# fetch() exposes the response body already decompressed and unframed, so +# the original framing headers would contradict the bytes we feed to the +# response parser. Set-Cookie is skipped here because the iterator folds +# repeated values into one comma-joined string; it is recovered separately +# via Headers.getSetCookie() where available. +_UNUSABLE_RESPONSE_HEADERS: Final = frozenset( + ( + hdrs.CONNECTION.lower(), + hdrs.CONTENT_ENCODING.lower(), + hdrs.CONTENT_LENGTH.lower(), + hdrs.KEEP_ALIVE.lower(), + hdrs.SET_COOKIE.lower(), + hdrs.TRANSFER_ENCODING.lower(), + ) +) + + +class _RequestSinkProtocol(BaseProtocol): + """Owner protocol for the request parser; flow control is a no-op.""" + + def pause_reading(self) -> None: + self._reading_paused = True + + def resume_reading(self, resume_parser: bool = True) -> None: + self._reading_paused = False + + +class _FetchTransport(asyncio.Transport): + """Fake transport that hands written request bytes back to the protocol.""" + + def __init__(self, protocol: "FetchClientProtocol") -> None: + super().__init__() + self._protocol = protocol + self._closing = False + + def write(self, data: "bytes | bytearray | memoryview[Any]") -> None: + self._protocol._request_bytes_received(bytes(data)) + + def writelines( + self, list_of_data: "Iterable[bytes | bytearray | memoryview[Any]]" + ) -> None: + self.write(b"".join(bytes(data) for data in list_of_data)) + + def is_closing(self) -> bool: + return self._closing + + def close(self) -> None: + if not self._closing: + self._closing = True + self._protocol._transport_closed() + + def abort(self) -> None: + self.close() + + +class FetchClientProtocol(ResponseHandler): + """A ResponseHandler that round-trips one request through ``fetch()``. + + The request bytes written by ``ClientRequest`` are decoded with + ``HttpRequestParser``; once the request body is complete it is sent with + ``fetch()`` and the JavaScript response is re-serialized into HTTP/1.1 + bytes for the regular response parser. Each protocol instance serves + exactly one request (``FetchConnector`` never pools connections). + """ + + def __init__( + self, + loop: asyncio.AbstractEventLoop, + request: "ClientRequest", + *, + fetch: Callable[..., Any], + fetch_options: Mapping[str, Any], + ) -> None: + super().__init__(loop) + self._request = request + self._js_fetch = fetch + self._fetch_options = fetch_options + self._fetch_task: asyncio.Task[None] | None = None + self._abort_controller: Any = None + sink = _RequestSinkProtocol(loop) + self._request_parser = HttpRequestParser(sink, loop, 2**16) + self.connection_made(_FetchTransport(self)) + + def _request_bytes_received(self, data: bytes) -> None: + messages, _, _ = self._request_parser.feed_data(data) + if messages and self._fetch_task is None: + message, payload = messages[0] + self._fetch_task = self._loop.create_task( + self._fetch_and_respond(message, payload) + ) + + def _transport_closed(self) -> None: + if self._abort_controller is not None: + self._abort_controller.abort() + self._abort_controller = None + if self._fetch_task is not None and not self._fetch_task.done(): + self._fetch_task.cancel() + if not self._connection_lost_called: + self._loop.call_soon(self.connection_lost, None) + + def _make_fetch_arguments( + self, message: RawRequestMessage, body: bytes + ) -> dict[str, Any]: + headers = [ + (name, value) + for name, value in message.headers.items() + if name.lower() not in _UNSENDABLE_REQUEST_HEADERS + ] + options: dict[str, Any] = {"method": message.method, "headers": headers} + if body: + options["body"] = body + options.update(self._fetch_options) + if IS_EMSCRIPTEN: # pragma: no cover + from js import AbortController # noqa: I900 + from pyodide.ffi import to_js # noqa: I900 + + self._abort_controller = AbortController.new() + options["signal"] = self._abort_controller.signal + # An array of [name, value] arrays is a valid HeadersInit. + options["headers"] = to_js(options["headers"]) + if body: + options["body"] = to_js(body) + return options + + async def _fetch_and_respond( + self, message: RawRequestMessage, payload: StreamReader + ) -> None: + try: + body = await payload.read() + options = self._make_fetch_arguments(message, body) + jsresp = await self._js_fetch(str(self._request.url), **options) + self.data_received(await self._serialize_response(message, jsresp)) + except asyncio.CancelledError: + raise + except Exception as exc: + set_exception( + self, + ClientConnectionError(f"fetch() failed: {exc!r}"), + exc, + ) + + async def _serialize_response( + self, message: RawRequestMessage, jsresp: Any + ) -> bytes: + body: bytes = (await jsresp.arrayBuffer()).to_bytes() + status = jsresp.status + lines = [f"HTTP/1.1 {status} {jsresp.statusText}".rstrip()] + lines.extend( + f"{name}: {value}" + for name, value in jsresp.headers + if name.lower() not in _UNUSABLE_RESPONSE_HEADERS + ) + if hasattr(jsresp.headers, "getSetCookie"): + lines.extend(f"Set-Cookie: {c}" for c in jsresp.headers.getSetCookie()) + if status not in EMPTY_BODY_STATUS_CODES and message.method != hdrs.METH_HEAD: + lines.append(f"Content-Length: {len(body)}") + lines.append("Connection: close") + head = "\r\n".join(lines).encode("latin-1", "backslashreplace") + return head + b"\r\n\r\n" + body + + +class FetchConnector(BaseConnector): + """Connector that performs requests via the JavaScript ``fetch()`` API. + + Used as the default connector when running under Emscripten (Pyodide). + + fetch - The fetch implementation to use. Defaults to ``js.fetch``; + mainly useful for testing or wrapping fetch with custom behavior. + fetch_options - Extra options merged into the ``fetch()`` init argument, + e.g. ``{"credentials": "include", "cache": "no-store"}``. + limit - The total number of simultaneous in-flight requests. + limit_per_host - Number of simultaneous requests to one host. + """ + + def __init__( + self, + *, + fetch: Callable[..., Any] | None = None, + fetch_options: Mapping[str, Any] | None = None, + limit: int = 100, + limit_per_host: int = 0, + ) -> None: + if fetch is None: + if not IS_EMSCRIPTEN: + raise RuntimeError( + "FetchConnector requires the JavaScript fetch() API and only " + "works under Emscripten/Pyodide (or with an explicit fetch=)" + ) + from js import fetch as js_fetch # noqa: I900 # pragma: no cover + + fetch = js_fetch # pragma: no cover + super().__init__(force_close=True, limit=limit, limit_per_host=limit_per_host) + self._fetch = fetch + self._fetch_options = dict(fetch_options or {}) + + async def _create_connection( + self, req: "ClientRequest", traces: list["Trace"], timeout: "ClientTimeout" + ) -> ResponseHandler: + if req.proxy is not None: + raise ClientConnectionError( + "Proxies are not supported by fetch(); " + "the JavaScript runtime manages the network path" + ) + if req.method == hdrs.METH_CONNECT or hdrs.UPGRADE in req.headers: + raise ClientConnectionError( + "Connection upgrades (e.g. WebSockets) are not supported by " + "fetch(); use the JavaScript WebSocket API instead" + ) + if req._continue is not None: + # There is no way to wait for a real 100 Continue over fetch(). + set_result(req._continue, True) + return FetchClientProtocol( + self._loop, req, fetch=self._fetch, fetch_options=self._fetch_options + ) diff --git a/docs/client_reference.rst b/docs/client_reference.rst index 95e0fb03717..fefff0a64a2 100644 --- a/docs/client_reference.rst +++ b/docs/client_reference.rst @@ -1053,6 +1053,10 @@ There are standard connectors: *HTTPS* schemes supported). 2. :class:`UnixConnector` for connecting via UNIX socket (it's used mostly for testing purposes). +3. :class:`FetchConnector` for dispatching requests through the JavaScript + ``fetch()`` API when running under Pyodide / Emscripten (WebAssembly), + where raw sockets are unavailable. Selected automatically on that + platform. All connector classes should be derived from :class:`BaseConnector`. @@ -1396,6 +1400,49 @@ is controlled by *force_close* constructor's parameter). Path to *UNIX socket*, read-only :class:`str` property. +.. class:: FetchConnector(*, fetch=None, fetch_options=None, \ + limit=100, limit_per_host=0) + :canonical: aiohttp.pyodide.FetchConnector + + JavaScript ``fetch()`` connector. + + WebAssembly runtimes (Pyodide in browsers or Node.js) have no raw + sockets, so :class:`TCPConnector` cannot be used there. This connector + performs each request through the JavaScript ``fetch()`` API instead. + When aiohttp runs under Emscripten, :class:`ClientSession` uses it as + the default connector, so it rarely needs to be created explicitly -- + do so only to customize its parameters:: + + conn = aiohttp.FetchConnector(fetch_options={"credentials": "include"}) + session = aiohttp.ClientSession(connector=conn) + + :class:`FetchConnector` is inherited from :class:`BaseConnector`. + + Because the JavaScript runtime manages the network, some aiohttp + features do not apply: proxies and connection upgrades (WebSockets) + raise :exc:`ClientConnectionError`, redirects are followed + transparently by ``fetch()`` before aiohttp sees the final response, + ``ssl`` arguments are ignored (the runtime's trust store applies), and + in browsers, cookies and CORS are enforced by the browser itself. + + :param fetch: the ``fetch()`` implementation to use. Defaults to the + global JavaScript ``fetch``; mainly useful for testing or wrapping + ``fetch()`` with custom behavior. + + :param dict fetch_options: extra options merged into the ``fetch()`` + *init* argument, e.g. ``{"credentials": "include"}``. See the + `fetch documentation + `_ + for the available options. + + :param int limit: total number of simultaneous in-flight requests. + + :param int limit_per_host: limit of simultaneous requests to the same + endpoint (``0`` for no limit). + + .. versionadded:: 4.0 + + .. class:: Connection :canonical: aiohttp.connector.Connection diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index ede9812a2e2..37b8b3ee2fc 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -124,6 +124,7 @@ DoS downstreams Dup elasticsearch +Emscripten encodings env environ @@ -180,6 +181,7 @@ iterables javascript Jinja jitter +js json keepalive keepalived @@ -272,6 +274,7 @@ py pydantic pyenv pyflakes +Pyodide pypa PyPI pyright @@ -401,6 +404,7 @@ waituntil wakeup wakeups webapp +WebAssembly websocket websocket’s websockets diff --git a/setup.py b/setup.py index 0933d969b00..0665b0181e7 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,14 @@ "define_macros": [("LLHTTP_STRICT_MODE", 0)], "include_dirs": ["vendor/llhttp/build"], } + # When targeting Emscripten (Pyodide), llhttp's api.c expects + # JavaScript-provided wasm_on_* callbacks; provide no-op stubs so the + # extension has no undefined symbols (aiohttp uses llhttp_init() with + # its own callbacks instead). + if sys.platform == "emscripten" or "emscripten" in os.environ.get( + "_PYTHON_HOST_PLATFORM", "" + ): + llhttp_sources.append("aiohttp/_llhttp_wasm_shim.c") cython_trace_macros = [("CYTHON_TRACE", 1)] if CYTHON_TRACING else [] if cython_trace_macros: diff --git a/tests/test_fetch_connector.py b/tests/test_fetch_connector.py new file mode 100644 index 00000000000..a9c921ac0bb --- /dev/null +++ b/tests/test_fetch_connector.py @@ -0,0 +1,365 @@ +"""Tests for aiohttp.pyodide.FetchConnector using a stubbed fetch(). + +These tests exercise the whole fetch-based client pipeline (request +serialization, request parsing, response synthesis) without requiring an +Emscripten runtime: the JavaScript ``fetch()`` entry point is replaced with +an in-process stub. Integration tests running under a real Pyodide runtime +live in ``tests/test_pyodide.py``. +""" + +import asyncio +import json +import sys +from typing import Any, NoReturn, Union +from unittest import mock + +import pytest +from yarl import URL + +import aiohttp +from aiohttp.pyodide import FetchClientProtocol, FetchConnector, _FetchTransport + + +class StubArrayBuffer: + """Mimics a JsProxy of an ArrayBuffer.""" + + def __init__(self, data: bytes) -> None: + self._data = data + + def to_bytes(self) -> bytes: + return self._data + + +class StubHeaders(list[tuple[str, str]]): + """Mimics iteration over a JavaScript Headers object.""" + + def __init__( + self, + entries: list[tuple[str, str]], + set_cookies: tuple[str, ...] = (), + ) -> None: + super().__init__(entries) + self._set_cookies = list(set_cookies) + + def getSetCookie(self) -> list[str]: + return self._set_cookies + + +class StubJsResponse: + """Mimics a JsProxy of a fetch() Response.""" + + def __init__( + self, + status: int = 200, + statusText: str = "OK", + headers: Union[StubHeaders, None] = None, + body: bytes = b"", + ) -> None: + self.status = status + self.statusText = statusText + self.headers = headers if headers is not None else StubHeaders([]) + self._body = body + + async def arrayBuffer(self) -> StubArrayBuffer: + return StubArrayBuffer(self._body) + + +class StubFetch: + """Records calls and replies with a canned response.""" + + def __init__(self, response: Union[StubJsResponse, None] = None) -> None: + self.response = response if response is not None else StubJsResponse() + self.calls: list[tuple[str, dict[str, Any]]] = [] + + async def __call__(self, url: str, **options: Any) -> StubJsResponse: + self.calls.append((url, options)) + return self.response + + @property + def last_options(self) -> dict[str, Any]: + return self.calls[-1][1] + + @property + def last_headers(self) -> dict[str, str]: + return {k.lower(): v for k, v in self.last_options["headers"]} + + +async def test_get_request() -> None: + fetch = StubFetch( + StubJsResponse( + headers=StubHeaders([("content-type", "text/html; charset=utf-8")]), + body=b"

hi

", + ) + ) + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + async with session.get("http://example.com/page?q=1") as resp: + assert resp.status == 200 + assert resp.reason == "OK" + assert resp.headers["Content-Type"] == "text/html; charset=utf-8" + assert await resp.text() == "

hi

" + + url, options = fetch.calls[0] + assert url == "http://example.com/page?q=1" + assert options["method"] == "GET" + assert "body" not in options + + +async def test_request_headers_forwarded_and_filtered() -> None: + fetch = StubFetch() + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + await session.get("http://example.com/", headers={"X-Custom": "yes"}) + + headers = fetch.last_headers + assert headers["x-custom"] == "yes" + assert "aiohttp" in headers["user-agent"] + # fetch() manages connection lifetime, framing and content negotiation. + for forbidden in ("host", "connection", "content-length", "accept-encoding"): + assert forbidden not in headers + + +async def test_post_bytes_body() -> None: + fetch = StubFetch() + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + await session.post("http://example.com/", data=b"some-bytes") + + assert bytes(fetch.last_options["body"]) == b"some-bytes" + assert fetch.last_options["method"] == "POST" + + +async def test_post_json_body() -> None: + fetch = StubFetch() + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + await session.post("http://example.com/", json={"x": 1}) + + assert json.loads(bytes(fetch.last_options["body"])) == {"x": 1} + assert fetch.last_headers["content-type"] == "application/json" + + +async def test_chunked_body_is_unframed() -> None: + """A chunked request body must be de-chunked before it reaches fetch().""" + + async def gen() -> Any: + yield b"chunk1-" + yield b"chunk2" + + fetch = StubFetch() + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + await session.post("http://example.com/", data=gen()) + + assert bytes(fetch.last_options["body"]) == b"chunk1-chunk2" + assert "transfer-encoding" not in fetch.last_headers + + +async def test_response_framing_headers_stripped() -> None: + """fetch() bodies arrive decoded; stale framing headers must not leak.""" + fetch = StubFetch( + StubJsResponse( + headers=StubHeaders( + [ + ("content-type", "text/plain"), + # Values describe the on-the-wire (compressed) form and + # would contradict the decoded body fetch() hands over. + ("content-encoding", "gzip"), + ("content-length", "999999"), + ] + ), + body=b"decoded", + ) + ) + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + async with session.get("http://example.com/") as resp: + assert await resp.read() == b"decoded" + assert "Content-Encoding" not in resp.headers + assert resp.headers["Content-Length"] == "7" + + +async def test_set_cookie_headers_recovered() -> None: + """Repeated Set-Cookie headers come from getSetCookie(), not iteration.""" + fetch = StubFetch( + StubJsResponse( + headers=StubHeaders( + [("set-cookie", "a=1; Path=/, b=2; Path=/")], + set_cookies=("a=1; Path=/", "b=2; Path=/"), + ), + body=b"ok", + ) + ) + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + async with session.get("http://example.com/") as resp: + assert list(resp.headers.getall("Set-Cookie")) == [ + "a=1; Path=/", + "b=2; Path=/", + ] + cookies = session.cookie_jar.filter_cookies(resp.url) + assert cookies["a"].value == "1" + assert cookies["b"].value == "2" + + +async def test_no_content_response() -> None: + fetch = StubFetch(StubJsResponse(status=204, statusText="No Content")) + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + async with session.get("http://example.com/") as resp: + assert resp.status == 204 + assert await resp.read() == b"" + + +async def test_head_request() -> None: + fetch = StubFetch( + StubJsResponse(headers=StubHeaders([("content-type", "text/plain")])) + ) + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + async with session.head("http://example.com/") as resp: + assert resp.status == 200 + assert await resp.read() == b"" + assert fetch.last_options["method"] == "HEAD" + + +async def test_fetch_failure_raises_client_error() -> None: + async def failing_fetch(url: str, **options: Any) -> NoReturn: + raise TypeError("Failed to fetch") + + async with aiohttp.ClientSession( + connector=FetchConnector(fetch=failing_fetch) + ) as session: + with pytest.raises(aiohttp.ClientConnectionError, match="Failed to fetch"): + await session.get("http://example.com/") + + +async def test_proxy_rejected() -> None: + async with aiohttp.ClientSession( + connector=FetchConnector(fetch=StubFetch()) + ) as session: + with pytest.raises(aiohttp.ClientConnectionError, match="[Pp]roxies"): + await session.get("http://example.com/", proxy="http://proxy.example:8080") + + +async def test_websocket_rejected() -> None: + async with aiohttp.ClientSession( + connector=FetchConnector(fetch=StubFetch()) + ) as session: + with pytest.raises(aiohttp.ClientConnectionError, match="WebSocket"): + await session.ws_connect("http://example.com/ws") + + +async def test_expect100_answered_locally() -> None: + fetch = StubFetch() + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + async with session.post( + "http://example.com/", data=b"abc", expect100=True + ) as resp: + assert resp.status == 200 + + assert bytes(fetch.last_options["body"]) == b"abc" + + +async def test_fetch_options_merged() -> None: + fetch = StubFetch() + connector = FetchConnector(fetch=fetch, fetch_options={"credentials": "include"}) + async with aiohttp.ClientSession(connector=connector) as session: + await session.get("http://example.com/") + + assert fetch.last_options["credentials"] == "include" + + +async def test_total_timeout_cancels_fetch() -> None: + cancelled = asyncio.Event() + + async def hanging_fetch(url: str, **options: Any) -> NoReturn: + try: + while True: + await asyncio.sleep(3600) + except asyncio.CancelledError: + cancelled.set() + raise + + async with aiohttp.ClientSession( + connector=FetchConnector(fetch=hanging_fetch), + timeout=aiohttp.ClientTimeout(total=0.05), + ) as session: + with pytest.raises(asyncio.TimeoutError): + await session.get("http://example.com/") + await asyncio.wait_for(cancelled.wait(), 1) + + +async def test_concurrent_requests() -> None: + fetch = StubFetch(StubJsResponse(body=b"payload")) + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + responses = await asyncio.gather( + *(session.get("http://example.com/") for _ in range(10)) + ) + for resp in responses: + assert await resp.read() == b"payload" + resp.release() + + assert len(fetch.calls) == 10 + + +async def test_large_body_pauses_request_stream() -> None: + """A body larger than the request parser's buffer exercises flow control.""" + big = b"x" * 200_000 + fetch = StubFetch() + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + await session.post("http://example.com/", data=big) + + assert bytes(fetch.last_options["body"]) == big + + +async def test_response_headers_without_get_set_cookie() -> None: + """Runtimes without Headers.getSetCookie() still work (without cookies).""" + fetch = StubFetch( + StubJsResponse(headers=[("content-type", "text/plain")], body=b"ok") # type: ignore[arg-type] + ) + async with aiohttp.ClientSession(connector=FetchConnector(fetch=fetch)) as session: + async with session.get("http://example.com/") as resp: + assert await resp.read() == b"ok" + + +async def test_transport_close_aborts_fetch() -> None: + """Closing the connection aborts the in-flight fetch().""" + request = mock.Mock() + request.url = URL("http://example.com/") + protocol = FetchClientProtocol( + asyncio.get_running_loop(), request, fetch=StubFetch(), fetch_options={} + ) + controller = mock.Mock() + protocol._abort_controller = controller + protocol._fetch_task = asyncio.ensure_future(asyncio.sleep(3600)) + transport = protocol.transport + assert transport is not None + assert not transport.is_closing() + transport.abort() + transport.close() # Second close is a no-op. + assert transport.is_closing() + assert controller.abort.called + await asyncio.sleep(0) + assert protocol._fetch_task.cancelled() + + +def test_transport_writelines() -> None: + written: list[bytes] = [] + protocol = mock.Mock() + protocol._request_bytes_received = written.append + transport = _FetchTransport(protocol) + transport.writelines([b"a", bytearray(b"b"), memoryview(b"c")]) + assert written == [b"abc"] + + +@pytest.mark.skipif(sys.platform == "emscripten", reason="fetch() exists here") +def test_requires_fetch_outside_emscripten() -> None: + with pytest.raises(RuntimeError, match="Emscripten"): + FetchConnector() + + +@pytest.mark.skipif(sys.platform == "emscripten", reason="patches the platform") +async def test_default_connector_selected_by_platform( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Under Emscripten, ClientSession defaults to FetchConnector. + + Off-platform the constructor raises because the fetch() API is missing, + which is enough to prove the selection logic without a WebAssembly + runtime; real construction is covered in tests/test_pyodide.py. + """ + monkeypatch.setattr(sys, "platform", "emscripten") + with pytest.raises(RuntimeError, match="Emscripten"): + aiohttp.ClientSession() diff --git a/tests/test_pyodide.py b/tests/test_pyodide.py new file mode 100644 index 00000000000..80a84872387 --- /dev/null +++ b/tests/test_pyodide.py @@ -0,0 +1,224 @@ +"""Integration tests running aiohttp inside a real Pyodide runtime. + +These tests require a Pyodide distribution and the pytest-pyodide plugin; +they are run from the dedicated ``test-pyodide`` CI job:: + + pytest tests/test_pyodide.py --rt node --dist-dir=./pyodide-dist + +The aiohttp wheel under test must be placed in ``dist/`` first (the CI job +builds it with cibuildwheel). Unit tests for the fetch()-based connector +that do not need a WebAssembly runtime live in +``tests/test_fetch_connector.py``. +""" + +import shutil +import threading +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +import pytest + +pytest_pyodide = pytest.importorskip("pytest_pyodide") +run_in_pyodide = pytest_pyodide.run_in_pyodide + +# Wheels for aiohttp's runtime dependencies that are part of the Pyodide +# distribution; loaded into the runtime before the aiohttp wheel itself. +DEPENDENCIES = [ + "aiohappyeyeballs", + "aiosignal", + "frozenlist", + "multidict", + "propcache", + "yarl", +] + + +class _EchoHandler(BaseHTTPRequestHandler): + """A tiny HTTP server exercising the client from the outside.""" + + def _read_body(self) -> bytes: + length = int(self.headers.get("Content-Length", 0)) + return self.rfile.read(length) if length else b"" + + def _respond( + self, + body: bytes, + content_type: str = "text/plain", + status: int = 200, + extra_headers: tuple[tuple[str, str], ...] = (), + ) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + for name, value in extra_headers: + self.send_header(name, value) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + if self.path == "/json": + self._respond(b'{"hello": "world"}', "application/json") + elif self.path == "/cookies": + self._respond( + b"ok", + extra_headers=( + ("Set-Cookie", "first=1; Path=/"), + ("Set-Cookie", "second=2; Path=/"), + ), + ) + elif self.path == "/redirect": + self.send_response(302) + self.send_header("Location", "/json") + self.send_header("Content-Length", "0") + self.end_headers() + else: + self._respond(b"not found", status=404) + + def do_POST(self) -> None: + body = self._read_body() + # Strip CR/LF before echoing the request's content type back in a + # header (response splitting; also keeps CodeQL happy). + content_type = self.headers.get("Content-Type", "application/octet-stream") + content_type = content_type.replace("\r", "").replace("\n", "") + self._respond( + body, + content_type, + extra_headers=(("X-Request-Method", "POST"),), + ) + + def do_PUT(self) -> None: + self._respond(self._read_body(), extra_headers=(("X-Request-Method", "PUT"),)) + + def log_message(self, format: str, *args: object) -> None: + pass # Silence per-request stderr noise. + + +@pytest.fixture(scope="module") +def echo_server_url() -> Iterator[str]: + server = ThreadingHTTPServer(("127.0.0.1", 0), _EchoHandler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + thread.join() + server.server_close() + + +@pytest.fixture +def selenium_with_aiohttp( + selenium_standalone: Any, request: pytest.FixtureRequest +) -> Iterator[Any]: + """Load the locally built aiohttp wheel and its dependencies.""" + wheels = sorted(Path("dist").glob("aiohttp-*.whl")) + if not wheels: + pytest.fail("no aiohttp wheel in dist/; build one with cibuildwheel first") + wheel = wheels[-1] + dist_dir = Path(request.config.option.dist_dir) + dist_wheel = dist_dir / wheel.name + shutil.copyfile(wheel, dist_wheel) + try: + selenium_standalone.load_package(DEPENDENCIES) + selenium_standalone.load_package(wheel.name) + yield selenium_standalone + finally: + dist_wheel.unlink() + + +@run_in_pyodide +async def _get_json(selenium: Any, base_url: str) -> None: + import aiohttp + from aiohttp.pyodide import FetchConnector + + async with aiohttp.ClientSession() as session: + assert isinstance(session.connector, FetchConnector) + async with session.get(base_url + "/json") as resp: + assert resp.status == 200 + assert resp.headers["Content-Type"] == "application/json" + assert await resp.json() == {"hello": "world"} + + +@run_in_pyodide +async def _post_bodies(selenium: Any, base_url: str) -> None: + from collections.abc import AsyncIterator + + import aiohttp + + async with aiohttp.ClientSession() as session: + async with session.post(base_url + "/echo", data=b"raw-bytes") as resp: + assert await resp.read() == b"raw-bytes" + assert resp.headers["X-Request-Method"] == "POST" + + async with session.post(base_url + "/echo", json={"a": [1, 2]}) as resp: + assert await resp.json() == {"a": [1, 2]} + + async def gen() -> AsyncIterator[bytes]: + yield b"chunk1-" + yield b"chunk2" + + async with session.post(base_url + "/echo", data=gen()) as resp: + assert await resp.read() == b"chunk1-chunk2" + + async with session.put(base_url + "/echo", data=b"put-data") as resp: + assert await resp.read() == b"put-data" + assert resp.headers["X-Request-Method"] == "PUT" + + +@run_in_pyodide +async def _redirects_cookies_errors(selenium: Any, base_url: str) -> None: + import asyncio + + import aiohttp + + async with aiohttp.ClientSession() as session: + # fetch() follows the redirect transparently. + async with session.get(base_url + "/redirect") as resp: + assert resp.status == 200 + assert await resp.json() == {"hello": "world"} + + async with session.get(base_url + "/cookies") as resp: + assert list(resp.headers.getall("Set-Cookie")) == [ + "first=1; Path=/", + "second=2; Path=/", + ] + + async with session.get(base_url + "/missing") as resp: + assert resp.status == 404 + try: + await session.get(base_url + "/missing", raise_for_status=True) + except aiohttp.ClientResponseError as e: + assert e.status == 404 + else: + raise AssertionError("expected ClientResponseError") + + results = await asyncio.gather( + *(session.get(base_url + "/json") for _ in range(5)) + ) + for r in results: + assert await r.json() == {"hello": "world"} + r.release() + + try: + await session.get("http://127.0.0.1:2/") + except aiohttp.ClientConnectionError: + pass + else: + raise AssertionError("expected ClientConnectionError") + + +def test_get_json(selenium_with_aiohttp: Any, echo_server_url: str) -> None: + _get_json(selenium_with_aiohttp, echo_server_url) + + +def test_post_bodies(selenium_with_aiohttp: Any, echo_server_url: str) -> None: + _post_bodies(selenium_with_aiohttp, echo_server_url) + + +def test_redirects_cookies_errors( + selenium_with_aiohttp: Any, echo_server_url: str +) -> None: + _redirects_cookies_errors(selenium_with_aiohttp, echo_server_url)