From bfe5feb7a56b78c70fbe28f3644fbb0c1c61b0ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 May 2026 23:11:32 +0000 Subject: [PATCH 1/3] Bump cryptography from 47.0.0 to 48.0.0 (#12455) Bumps [cryptography](https://github.com/pyca/cryptography) from 47.0.0 to 48.0.0.
Changelog

Sourced from cryptography's changelog.

48.0.0 - 2026-05-04


* **BACKWARDS INCOMPATIBLE:** Support for Python 3.8 has been removed.
  ``cryptography`` now requires Python 3.9 or later.
* **BACKWARDS INCOMPATIBLE:** Loading an X.509 CRL whose inner
  ``TBSCertList.signature`` algorithm does not match the outer
``signatureAlgorithm`` now raises ``ValueError``. Previously, such CRLs
were parsed successfully and only rejected during signature validation.
* Added support for :doc:`/hazmat/primitives/asymmetric/mlkem` and
  :doc:`/hazmat/primitives/asymmetric/mldsa` when using OpenSSL 3.5.0 or
later, in addition to the existing AWS-LC and BoringSSL support. This
means
  post-quantum algorithms are now available to users of our wheels.

    

  • Note: Going forward, we do not guarantee that all
functionality
    
in cryptography will be available when building against
    
OpenSSL. See :doc:/statements/state-of-openssl for more
information.
    • 



.. _v47-0-0:

Commits

Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/constraints.txt | 2 +- requirements/dev.txt | 2 +- requirements/lint.txt | 26 ++++++++++++++++++++--- requirements/test-common.txt | 41 ++++++++++++++++++++++++++++++++++-- requirements/test-ft.txt | 2 +- requirements/test.txt | 2 +- 6 files changed, 66 insertions(+), 9 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 00cb4310cfd..d9f3bf4051e 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -70,7 +70,7 @@ coverage==7.13.5 # via # -r requirements/test-common.in # pytest-cov -cryptography==47.0.0 +cryptography==48.0.0 # via trustme cython==3.2.4 # via -r requirements/cython.in diff --git a/requirements/dev.txt b/requirements/dev.txt index f8f43bc4e0c..7bbccc6e139 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -70,7 +70,7 @@ coverage==7.13.5 # via # -r requirements/test-common.in # pytest-cov -cryptography==47.0.0 +cryptography==48.0.0 # via trustme distlib==0.4.0 # via virtualenv diff --git a/requirements/lint.txt b/requirements/lint.txt index b95a311dc74..b8eb0c7d394 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -17,7 +17,13 @@ annotated-types==0.7.0 ast-serialize==0.3.0 # via mypy async-timeout==5.0.1 - # via valkey + # via + # aiohttp + # valkey +attrs==26.1.0 + # via aiohttp +backports-asyncio-runner==1.2.0 + # via pytest-asyncio backports-zstd==1.3.0 ; implementation_name == "cpython" and python_version < "3.14" # via -r requirements/lint.in blockbuster==1.5.26 @@ -31,7 +37,7 @@ cfgv==3.5.0 # via pre-commit click==8.3.3 # via slotscheck -cryptography==47.0.0 +cryptography==48.0.0 # via trustme distlib==0.4.0 # via virtualenv @@ -45,10 +51,16 @@ forbiddenfruit==0.1.4 # via blockbuster freezegun==1.5.5 # via -r requirements/lint.in +frozenlist==1.8.0 + # via + # aiohttp + # aiosignal identify==2.6.19 # via pre-commit idna==3.11 - # via trustme + # via + # trustme + # yarl iniconfig==2.3.0 # via pytest isal==1.7.2 @@ -59,6 +71,10 @@ markdown-it-py==4.2.0 # via rich mdurl==0.1.2 # via markdown-it-py +multidict==6.7.1 + # via + # aiohttp + # yarl mypy==2.0.0 ; implementation_name == "cpython" # via -r requirements/lint.in mypy-extensions==1.1.0 @@ -133,12 +149,14 @@ trustme==1.2.1 # via -r requirements/lint.in typing-extensions==4.15.0 # via + # aiosignal # cryptography # exceptiongroup # multidict # mypy # pydantic # pydantic-core + # pytest-asyncio # python-on-whales # typing-inspection # virtualenv @@ -150,5 +168,7 @@ valkey==6.1.1 # via -r requirements/lint.in virtualenv==21.3.1 # via pre-commit +yarl==1.23.0 + # via aiohttp zlib-ng==1.0.0 # via -r requirements/lint.in diff --git a/requirements/test-common.txt b/requirements/test-common.txt index 17925250939..de9d62a88a3 100644 --- a/requirements/test-common.txt +++ b/requirements/test-common.txt @@ -4,10 +4,22 @@ # # pip-compile --allow-unsafe --output-file=requirements/test-common.txt --strip-extras requirements/test-common.in # +aiohappyeyeballs==2.6.1 + # via aiohttp +aiohttp==3.13.5 + # via pytest-aiohttp +aiosignal==1.4.0 + # via aiohttp annotated-types==0.7.0 # via pydantic ast-serialize==0.3.0 # via mypy +async-timeout==5.0.1 + # via aiohttp +attrs==26.1.0 + # via aiohttp +backports-asyncio-runner==1.2.0 + # via pytest-asyncio blockbuster==1.5.26 # via -r requirements/test-common.in cffi==2.0.0 @@ -20,7 +32,7 @@ coverage==7.13.5 # via # -r requirements/test-common.in # pytest-cov -cryptography==47.0.0 +cryptography==48.0.0 # via trustme exceptiongroup==1.3.1 # via pytest @@ -30,8 +42,14 @@ forbiddenfruit==0.1.4 # via blockbuster freezegun==1.5.5 # via -r requirements/test-common.in +frozenlist==1.8.0 + # via + # aiohttp + # aiosignal idna==3.11 - # via trustme + # via + # trustme + # yarl iniconfig==2.3.0 # via pytest isal==1.8.0 ; python_version < "3.14" @@ -42,6 +60,10 @@ markdown-it-py==4.2.0 # via rich mdurl==0.1.2 # via markdown-it-py +multidict==6.7.1 + # via + # aiohttp + # yarl mypy==2.0.0 ; implementation_name == "cpython" # via -r requirements/test-common.in mypy-extensions==1.1.0 @@ -56,6 +78,10 @@ pluggy==1.6.0 # via # pytest # pytest-cov +propcache==0.5.2 + # via + # aiohttp + # yarl proxy-py==2.4.10 # via -r requirements/test-common.in pycparser==3.0 @@ -71,10 +97,16 @@ pygments==2.20.0 pytest==9.0.3 # via # -r requirements/test-common.in + # pytest-aiohttp + # pytest-asyncio # pytest-codspeed # pytest-cov # pytest-mock # pytest-xdist +pytest-aiohttp==1.1.0 + # via -r requirements/test-common.in +pytest-asyncio==1.3.0 + # via pytest-aiohttp pytest-codspeed==4.5.0 # via -r requirements/test-common.in pytest-cov==7.1.0 @@ -102,16 +134,21 @@ trustme==1.2.1 ; platform_machine != "i686" # via -r requirements/test-common.in typing-extensions==4.15.0 # via + # aiosignal # cryptography # exceptiongroup + # multidict # mypy # pydantic # pydantic-core + # pytest-asyncio # python-on-whales # typing-inspection typing-inspection==0.4.2 # via pydantic wait-for-it==2.3.0 # via -r requirements/test-common.in +yarl==1.23.0 + # via aiohttp zlib-ng==1.0.0 # via -r requirements/test-common.in diff --git a/requirements/test-ft.txt b/requirements/test-ft.txt index db20ecf5538..9d153f5aa1f 100644 --- a/requirements/test-ft.txt +++ b/requirements/test-ft.txt @@ -45,7 +45,7 @@ coverage==7.13.5 # via # -r requirements/test-common.in # pytest-cov -cryptography==47.0.0 +cryptography==48.0.0 # via trustme exceptiongroup==1.3.1 # via pytest diff --git a/requirements/test.txt b/requirements/test.txt index 9d7b155a462..e330060e3e7 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -45,7 +45,7 @@ coverage==7.13.5 # via # -r requirements/test-common.in # pytest-cov -cryptography==47.0.0 +cryptography==48.0.0 # via trustme exceptiongroup==1.3.1 # via pytest From 76d3a7f12aa088346fafb1cfd82f9c591010d336 Mon Sep 17 00:00:00 2001 From: Andrew Karelin <36686667+AndrewKarelin@users.noreply.github.com> Date: Mon, 11 May 2026 04:26:56 +0500 Subject: [PATCH 2/3] Don't re-await main_task in run_app finally when it's already done (#12493) --- CHANGES/12493.bugfix | 3 +++ aiohttp/web.py | 15 ++++++++++++--- tests/test_run_app.py | 23 +++++++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 CHANGES/12493.bugfix diff --git a/CHANGES/12493.bugfix b/CHANGES/12493.bugfix new file mode 100644 index 00000000000..7a68daec0ba --- /dev/null +++ b/CHANGES/12493.bugfix @@ -0,0 +1,3 @@ +Fixed :func:`aiohttp.web.run_app` losing inner traceback frames when an +exception is raised during application startup (e.g. inside +``cleanup_ctx`` or ``on_startup``). Regression since 3.10.6. diff --git a/aiohttp/web.py b/aiohttp/web.py index 15cfcc99b98..ae2f098c554 100644 --- a/aiohttp/web.py +++ b/aiohttp/web.py @@ -488,9 +488,18 @@ def run_app( pass finally: try: - main_task.cancel() - with suppress(asyncio.CancelledError): - loop.run_until_complete(main_task) + # Skip when ``main_task`` is already done (e.g. raised during startup). + # Re-running ``loop.run_until_complete`` on a finished task calls + # ``Future.result`` again, which does + # ``raise self._exception.with_traceback(self._exception_tb)`` and + # resets ``exc.__traceback__`` to the originally saved tb — by then + # shallow — clobbering the deep traceback the caller would otherwise + # see (frames from ``cleanup_ctx`` / ``on_startup`` and the user code + # that actually raised). + if not main_task.done(): + main_task.cancel() + with suppress(asyncio.CancelledError): + loop.run_until_complete(main_task) finally: _cancel_tasks(asyncio.all_tasks(loop), loop) loop.run_until_complete(loop.shutdown_asyncgens()) diff --git a/tests/test_run_app.py b/tests/test_run_app.py index 5518440bec0..130ee8862dd 100644 --- a/tests/test_run_app.py +++ b/tests/test_run_app.py @@ -9,6 +9,7 @@ import subprocess import sys import time +import traceback from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator from typing import Any, NoReturn from unittest import mock @@ -121,6 +122,28 @@ def test_run_app_close_loop(patched_loop: asyncio.AbstractEventLoop) -> None: assert patched_loop.is_closed() +def test_run_app_preserves_startup_traceback( + patched_loop: asyncio.AbstractEventLoop, +) -> None: + # Regression: when an exception is raised during startup (here in a + # cleanup_ctx async generator), the user code frame must remain in the + # traceback that propagates out of run_app. Previously the second + # loop.run_until_complete(main_task) in run_app's finally clobbered it. + + async def failing_ctx(_app: web.Application) -> AsyncIterator[None]: + raise RuntimeError("boom from failing_ctx") + yield # type: ignore[unreachable] # required to make this an async generator + + app = web.Application() + app.cleanup_ctx.append(failing_ctx) + + with pytest.raises(RuntimeError, match="boom from failing_ctx") as exc_info: + web.run_app(app, print=None, loop=patched_loop) + + frames = [f.name for f in traceback.extract_tb(exc_info.tb)] + assert "failing_ctx" in frames, frames + + mock_unix_server_single = [ mock.call(mock.ANY, "/tmp/testsock1.sock", ssl=None, backlog=128), ] From e8f43710c510d90e192e4e28b45dd7ae1d2db7c7 Mon Sep 17 00:00:00 2001 From: Sam Bull Date: Mon, 11 May 2026 00:45:49 +0100 Subject: [PATCH 3/3] Change headers to a dict that parses comma-separated values (#7679) Co-authored-by: Rodrigo Nogueira --- CHANGES/12296.bugfix.rst | 5 + aiohttp/_http_parser.pyx | 14 ++- aiohttp/client_exceptions.py | 5 +- aiohttp/client_reqrep.py | 19 ++- aiohttp/helpers.py | 79 ++++++++++++- aiohttp/http_parser.py | 29 ++--- aiohttp/multipart.py | 19 ++- aiohttp/test_utils.py | 7 +- aiohttp/web_request.py | 72 ++++-------- tests/test_benchmarks_web_urldispatcher.py | 5 +- tests/test_client_exceptions.py | 10 +- tests/test_client_functional.py | 6 - tests/test_client_request.py | 4 +- tests/test_client_response.py | 117 ++++++++++--------- tests/test_http_parser.py | 112 +++++++++++++++--- tests/test_multipart.py | 130 ++++++++++----------- tests/test_test_utils.py | 5 +- tests/test_web_functional.py | 5 +- tests/test_web_request.py | 12 +- tests/test_web_response.py | 4 +- 20 files changed, 403 insertions(+), 256 deletions(-) create mode 100644 CHANGES/12296.bugfix.rst diff --git a/CHANGES/12296.bugfix.rst b/CHANGES/12296.bugfix.rst new file mode 100644 index 00000000000..ea790514286 --- /dev/null +++ b/CHANGES/12296.bugfix.rst @@ -0,0 +1,5 @@ +Normalized parsing of list-style ``Connection`` and ``Transfer-Encoding`` +headers so repeated field lines and comma-joined values are handled +consistently in the HTTP parser, without changing ``CIMultiDict`` +storage semantics. +-- by :user:`rodrigobnogueira`. diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index ac942ec1076..b6dd21fe83a 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -13,12 +13,13 @@ from cpython.mem cimport PyMem_Free, PyMem_Malloc from libc.limits cimport ULLONG_MAX from libc.string cimport memcpy -from multidict import CIMultiDict as _CIMultiDict, CIMultiDictProxy as _CIMultiDictProxy +from multidict import CIMultiDict as _CIMultiDict from yarl import URL as _URL from aiohttp import hdrs from aiohttp.helpers import DEBUG, set_exception +from .helpers import HeadersDictProxy as _HeadersDictProxy from .http_exceptions import ( BadHttpMessage, BadHttpMethod, @@ -61,7 +62,7 @@ __all__ = ('HttpRequestParser', 'HttpResponseParser', cdef object URL = _URL cdef object URL_build = URL.build cdef object CIMultiDict = _CIMultiDict -cdef object CIMultiDictProxy = _CIMultiDictProxy +cdef object HeadersDictProxy = _HeadersDictProxy cdef object HttpVersion = _HttpVersion cdef object HttpVersion10 = _HttpVersion10 cdef object HttpVersion11 = _HttpVersion11 @@ -76,6 +77,7 @@ cdef tuple EMPTY_FEED_DATA_RESULT = ((), False, b"") # In lax mode (response parser default), the check is skipped entirely # since real-world servers (e.g. Google APIs, Werkzeug) commonly send # duplicate headers like Content-Type or Server. +# https://www.rfc-editor.org/rfc/rfc9110.html#section-5.5-6 cdef frozenset SINGLETON_HEADERS = frozenset({ hdrs.CONTENT_LENGTH, hdrs.CONTENT_LOCATION, @@ -129,7 +131,7 @@ cdef class RawRequestMessage: cdef readonly str method cdef readonly str path cdef readonly object version # HttpVersion - cdef readonly object headers # CIMultiDict + cdef readonly object headers # HeadersDictProxy cdef readonly object raw_headers # tuple cdef readonly object should_close cdef readonly object compression @@ -229,7 +231,7 @@ cdef class RawResponseMessage: cdef readonly object version # HttpVersion cdef readonly int code cdef readonly str reason - cdef readonly object headers # CIMultiDict + cdef readonly object headers # HeadersDictProxy cdef readonly object raw_headers # tuple cdef readonly object should_close cdef readonly object compression @@ -316,7 +318,7 @@ cdef class HttpParser: bytearray _buf str _path str _reason - list _headers + object _headers set _seen_singletons list _raw_headers bint _upgraded @@ -463,7 +465,7 @@ cdef class HttpParser: chunked = self._cparser.flags & cparser.F_CHUNKED raw_headers = tuple(self._raw_headers) - headers = CIMultiDictProxy(CIMultiDict(self._headers)) + headers = HeadersDictProxy(CIMultiDict(self._headers)) if self._cparser.type == cparser.HTTP_REQUEST: if http_version == HttpVersion11 and hdrs.HOST not in headers: diff --git a/aiohttp/client_exceptions.py b/aiohttp/client_exceptions.py index 3b66c7fdcd2..054613dac17 100644 --- a/aiohttp/client_exceptions.py +++ b/aiohttp/client_exceptions.py @@ -1,10 +1,9 @@ """HTTP related errors.""" import asyncio +from collections.abc import Mapping from typing import TYPE_CHECKING, Union -from multidict import MultiMapping - from .typedefs import StrOrURL try: @@ -73,7 +72,7 @@ def __init__( *, status: int | None = None, message: str = "", - headers: MultiMapping[str] | None = None, + headers: Mapping[str, str] | None = None, ) -> None: self.request_info = request_info if status is not None: diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py index 25160f198bc..815f9c2dd61 100644 --- a/aiohttp/client_reqrep.py +++ b/aiohttp/client_reqrep.py @@ -38,6 +38,7 @@ _SENTINEL, BaseTimerContext, BasicAuth, + HeadersDictProxy, HeadersMixin, TimerNoop, frozen_dataclass_decorator, @@ -193,7 +194,7 @@ class ClientResponse(HeadersMixin): content: StreamReader = None # type: ignore[assignment] # Payload stream _body: bytes | None = None - _headers: CIMultiDictProxy[str] = None # type: ignore[assignment] + _headers: HeadersDictProxy = None # type: ignore[assignment] _history: tuple["ClientResponse", ...] = () _raw_headers: RawHeaders = None # type: ignore[assignment] @@ -324,7 +325,7 @@ def host(self) -> str: return self._url.host @reify - def headers(self) -> "CIMultiDictProxy[str]": + def headers(self) -> HeadersDictProxy: return self._headers @reify @@ -393,14 +394,8 @@ def history(self) -> tuple["ClientResponse", ...]: @reify def links(self) -> "MultiDictProxy[MultiDictProxy[str | URL]]": - links_str = ", ".join(self.headers.getall("link", [])) - - if not links_str: - return MultiDictProxy(MultiDict()) - links: MultiDict[MultiDictProxy[str | URL]] = MultiDict() - - for val in re.split(r",(?=\s*<)", links_str): + for val in self.headers.getall("link"): match = re.match(r"\s*<(.*)>(.*)", val) if match is None: # Malformed link continue @@ -462,14 +457,14 @@ async def start(self, connection: "Connection") -> "ClientResponse": self.reason = message.reason # headers - self._headers = message.headers # type is CIMultiDictProxy - self._raw_headers = message.raw_headers # type is Tuple[bytes, bytes] + self._headers = message.headers + self._raw_headers = message.raw_headers # payload self.content = payload # cookies - if cookie_hdrs := self.headers.getall(hdrs.SET_COOKIE, ()): + if cookie_hdrs := self.headers._md.getall(hdrs.SET_COOKIE, ()): # Store raw cookie headers for CookieJar self._raw_cookie_headers = tuple(cookie_hdrs) return self diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py index 9f27e3860a8..3e5677a46bd 100644 --- a/aiohttp/helpers.py +++ b/aiohttp/helpers.py @@ -44,7 +44,7 @@ from urllib.parse import quote from urllib.request import getproxies, proxy_bypass -from multidict import CIMultiDict, MultiDict, MultiDictProxy, MultiMapping +from multidict import CIMultiDict, MultiDict, MultiDictProxy from propcache.api import under_cached_property as reify from yarl import URL @@ -71,6 +71,37 @@ # https://github.com/python/cpython/blob/1857a40807daeae3a1bf5efb682de9c9ae6df845/Lib/asyncio/selector_events.py#L766 DEFAULT_CHUNK_SIZE = 2**18 # 256 KiB COOKIE_MAX_LENGTH = 4096 +_QUOTED_PAIR_SUB = re.compile(r"\\(.)") +_QUOTED_STRING = r'"(?:[^"\\]|\\.)*"' +_ESCAPED_COMMENT = r"(?:[^()\\]|\\.)*" +# Matches one element in a comma-separated header list. +# Group 1: content of a top-level quoted-string (quotes stripped). +# Group 2: an unquoted element (may contain parameter quoted-strings / comments). +_LIST_ELEMENT_RE = re.compile( + rf""" + [ \t]* + (?: + "( (?:[^"\\]|\\.)* )" # group 1: top-level quoted-string + | ( # group 2: unquoted element + (?: + (?<=[^\s]=) {_QUOTED_STRING} # parameter quoted value + | (?<=\s) \( {_ESCAPED_COMMENT} \) # comment + | [^,] # any non-comma character + )+? + ) + ) + [ \t]* (?:,|\Z) + """, + re.VERBOSE, +) +# Finds parameter quoted-strings and comments inside an unquoted element for unescaping. +_PROTECTED_RE = re.compile( + rf""" + (?<=[^\s]=) {_QUOTED_STRING} # parameter quoted-string + | (?<=\s) \( {_ESCAPED_COMMENT} \) # comment + """, + re.VERBOSE, +) _T = TypeVar("_T") _S = TypeVar("_S") @@ -753,10 +784,54 @@ def ceil_timeout( return async_timeout.timeout_at(when) +class HeadersDictProxy(Mapping[str, str]): + def __init__(self, md: CIMultiDict[str]): + self._md = md + + def getall(self, key: str) -> tuple[str, ...]: + val = self.get(key, "") + unescape = _QUOTED_PAIR_SUB.sub + values = [] + for m in _LIST_ELEMENT_RE.finditer(val): + qs = m.group(1) + if qs is not None: + values.append(unescape(r"\1", qs)) + else: + raw = m.group(2).strip() + if raw: + values.append( + _PROTECTED_RE.sub(lambda p: unescape(r"\1", p.group()), raw) + ) + return tuple(values) + + def __eq__(self, other: object) -> bool: + return self._md.__eq__(other) + + def __getitem__(self, key: str) -> str: + return ", ".join(self._md.getall(key)) + + def __iter__(self) -> Iterator[str]: + # We need to deduplicate keys from MultiDict + # But, we also need to retain ordering + seen = set() + for k in self._md.__iter__(): + if k in seen: + continue + seen.add(k) + yield k + + def __len__(self) -> int: + return len(set(self._md.keys())) + + def __repr__(self) -> str: + body = ", ".join(f"'{k}': {v!r}" for k, v in self.items()) + return f"<{self.__class__.__name__}({body})>" + + class HeadersMixin: """Mixin for handling headers.""" - _headers: MultiMapping[str] + _headers: Mapping[str, str] _content_type: str | None = None _content_dict: dict[str, str] | None = None _stored_content_type: str | None | _SENTINEL = sentinel diff --git a/aiohttp/http_parser.py b/aiohttp/http_parser.py index 60c26e58577..468218f6c51 100644 --- a/aiohttp/http_parser.py +++ b/aiohttp/http_parser.py @@ -17,7 +17,7 @@ TypeVar, ) -from multidict import CIMultiDict, CIMultiDictProxy, istr +from multidict import CIMultiDict, istr from yarl import URL from . import hdrs @@ -37,6 +37,7 @@ EMPTY_BODY_STATUS_CODES, NO_EXTENSIONS, BaseTimerContext, + HeadersDictProxy, set_exception, ) from .http_exceptions import ( @@ -66,6 +67,8 @@ "RawResponseMessage", ) +_T = TypeVar("_T") + _SEP = Literal[b"\r\n", b"\n"] ASCIISET: Final[set[str]] = set(string.printable) @@ -112,7 +115,7 @@ class RawRequestMessage(NamedTuple): method: str path: str version: HttpVersion - headers: CIMultiDictProxy[str] + headers: HeadersDictProxy raw_headers: RawHeaders should_close: bool compression: str | None @@ -125,7 +128,7 @@ class RawResponseMessage(NamedTuple): version: HttpVersion code: int reason: str - headers: CIMultiDictProxy[str] + headers: HeadersDictProxy raw_headers: RawHeaders should_close: bool compression: str | None @@ -161,9 +164,7 @@ def __init__(self, max_field_size: int = 8190, lax: bool = False) -> None: self.max_field_size = max_field_size self._lax = lax - def parse_headers( - self, lines: list[bytes] - ) -> tuple["CIMultiDictProxy[str]", RawHeaders]: + def parse_headers(self, lines: list[bytes]) -> tuple[HeadersDictProxy, RawHeaders]: headers: CIMultiDict[str] = CIMultiDict() # note: "raw" does not mean inclusion of OWS before/after the field value raw_headers = [] @@ -237,10 +238,10 @@ def parse_headers( headers.add(name, value) raw_headers.append((bname, bvalue)) - return (CIMultiDictProxy(headers), tuple(raw_headers)) + return (HeadersDictProxy(headers), tuple(raw_headers)) -def _is_supported_upgrade(headers: CIMultiDictProxy[str]) -> bool: +def _is_supported_upgrade(headers: HeadersDictProxy) -> bool: """Check if the upgrade header is supported.""" u = headers.get(hdrs.UPGRADE, "") # .lower() can transform non-ascii characters. @@ -544,9 +545,7 @@ def get_content_length() -> int | None: def parse_headers( self, lines: list[bytes] - ) -> tuple[ - "CIMultiDictProxy[str]", RawHeaders, bool | None, str | None, bool, bool - ]: + ) -> tuple[HeadersDictProxy, RawHeaders, bool | None, str | None, bool, bool]: """Parses RFC 5322 headers from a stream. Line continuations are supported. Returns list of header name @@ -560,12 +559,14 @@ def parse_headers( # keep-alive and protocol switching # RFC 9110 section 7.6.1 defines Connection as a comma-separated list. - conn_values = headers.getall(hdrs.CONNECTION, ()) + # We use a simple comma split here rather than getall() for performance, + # as the target tokens (close, keep-alive, upgrade) are simple ASCII + # values that never contain commas. + conn_values = headers.get(hdrs.CONNECTION) if conn_values: conn_tokens = { token.lower() - for conn_value in conn_values - for token in (part.strip(" \t") for part in conn_value.split(",")) + for token in (part.strip(" \t") for part in conn_values.split(",")) if token and token.isascii() } diff --git a/aiohttp/multipart.py b/aiohttp/multipart.py index 9d5e5d27b84..f5ff30804d1 100644 --- a/aiohttp/multipart.py +++ b/aiohttp/multipart.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, Union, cast from urllib.parse import parse_qsl, unquote, urlencode -from multidict import CIMultiDict, CIMultiDictProxy +from multidict import CIMultiDict from .abc import AbstractStreamWriter from .compression_utils import ZLibCompressor, ZLibDecompressor @@ -22,7 +22,14 @@ CONTENT_TRANSFER_ENCODING, CONTENT_TYPE, ) -from .helpers import CHAR, DEFAULT_CHUNK_SIZE, TOKEN, parse_mimetype, reify +from .helpers import ( + CHAR, + DEFAULT_CHUNK_SIZE, + TOKEN, + HeadersDictProxy, + parse_mimetype, + reify, +) from .http import HeadersParser from .http_exceptions import BadHttpMessage from .log import internal_logger @@ -258,7 +265,7 @@ class BodyPartReader: def __init__( self, boundary: bytes, - headers: "CIMultiDictProxy[str]", + headers: HeadersDictProxy, content: StreamReader, *, subtype: str = "mixed", @@ -765,7 +772,7 @@ async def fetch_next_part( def _get_part_reader( self, - headers: "CIMultiDictProxy[str]", + headers: HeadersDictProxy, ) -> Union["MultipartReader", BodyPartReader]: """Dispatches the response by the `Content-Type` header. @@ -852,7 +859,7 @@ async def _read_boundary(self) -> None: else: raise ValueError(f"Invalid boundary {chunk!r}, expected {self._boundary!r}") - async def _read_headers(self) -> "CIMultiDictProxy[str]": + async def _read_headers(self) -> HeadersDictProxy: lines = [] while True: chunk = await self._content.readline(max_line_length=self._max_field_size) @@ -863,7 +870,7 @@ async def _read_headers(self) -> "CIMultiDictProxy[str]": if len(lines) > self._max_headers: raise BadHttpMessage("Too many headers received") parser = HeadersParser(max_field_size=self._max_field_size) - headers, raw_headers = parser.parse_headers(lines) + headers, _ = parser.parse_headers(lines) return headers async def _maybe_release_last_part(self) -> None: diff --git a/aiohttp/test_utils.py b/aiohttp/test_utils.py index 1179cadb37a..6bbe590b18c 100644 --- a/aiohttp/test_utils.py +++ b/aiohttp/test_utils.py @@ -12,7 +12,7 @@ from unittest import IsolatedAsyncioTestCase, mock from aiosignal import Signal -from multidict import CIMultiDict, CIMultiDictProxy +from multidict import CIMultiDict from yarl import URL import aiohttp @@ -27,6 +27,7 @@ from .abc import AbstractCookieJar, AbstractStreamWriter from .client_reqrep import ClientResponse from .client_ws import ClientWebSocketResponse +from .helpers import HeadersDictProxy from .http import HttpVersion, RawRequestMessage from .streams import EMPTY_PAYLOAD, StreamReader from .typedefs import LooseHeaders, StrOrURL @@ -597,12 +598,12 @@ def make_mocked_request( closing = True if headers: - headers = CIMultiDictProxy(CIMultiDict(headers)) + headers = HeadersDictProxy(CIMultiDict(headers)) raw_hdrs = tuple( (k.encode("utf-8"), v.encode("utf-8")) for k, v in headers.items() ) else: - headers = CIMultiDictProxy(CIMultiDict()) + headers = HeadersDictProxy(CIMultiDict()) raw_hdrs = () chunked = "chunked" in headers.get(hdrs.TRANSFER_ENCODING, "").lower() diff --git a/aiohttp/web_request.py b/aiohttp/web_request.py index 2034e0b3f83..ee85c2af6f2 100644 --- a/aiohttp/web_request.py +++ b/aiohttp/web_request.py @@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar, cast, overload from urllib.parse import parse_qsl -from multidict import CIMultiDict, CIMultiDictProxy, MultiDict, MultiDictProxy +from multidict import CIMultiDict, MultiDict, MultiDictProxy from yarl import URL from . import hdrs @@ -26,6 +26,7 @@ LIST_QUOTED_ETAG_RE, ChainMapProxy, ETag, + HeadersDictProxy, HeadersMixin, RequestKey, frozen_dataclass_decorator, @@ -76,7 +77,7 @@ class FileField: filename: str file: io.BufferedReader content_type: str - headers: CIMultiDictProxy[str] + headers: HeadersDictProxy _TCHAR: Final[str] = string.digits + string.ascii_letters + r"!#$%&'*+.^_`|~-" @@ -90,16 +91,10 @@ class FileField: # qdtext includes 0x5C to escape 0x5D ('\]') # qdtext excludes obs-text (because obsoleted, and encoding not specified) -_QUOTED_PAIR: Final[str] = r"\\[\t !-~]" - -_QUOTED_STRING: Final[str] = rf'"(?:{_QUOTED_PAIR}|{_QDTEXT})*"' - # This does not have a ReDOS/performance concern as long as it used with re.match(). -_FORWARDED_PAIR: Final[str] = rf"({_TOKEN})=({_TOKEN}|{_QUOTED_STRING})(:\d{{1,4}})?" - -_QUOTED_PAIR_REPLACE_RE: Final[Pattern[str]] = re.compile(r"\\([\t !-~])") -# same pattern as _QUOTED_PAIR but contains a capture group - +_FORWARDED_PAIR: Final[str] = ( + rf'[ \t]*({_TOKEN})=({_TOKEN}|".*")(:\d{{1,4}})?[ \t]*(?:\Z|;)' +) _FORWARDED_PAIR_RE: Final[Pattern[str]] = re.compile(_FORWARDED_PAIR) ############################################################ @@ -139,7 +134,7 @@ def __init__( self._payload_writer = payload_writer self._payload = payload - self._headers: CIMultiDictProxy[str] = message.headers + self._headers: HeadersDictProxy = message.headers self._method = message.method self._version = message.version self._cache: dict[str, Any] = {} @@ -203,10 +198,11 @@ def clone( dct["path"] = str(new_url) if headers is not sentinel: # a copy semantic - new_headers = CIMultiDictProxy(CIMultiDict(headers)) + new_headers = HeadersDictProxy(CIMultiDict(headers)) dct["headers"] = new_headers dct["raw_headers"] = tuple( - (k.encode("utf-8"), v.encode("utf-8")) for k, v in new_headers.items() + (k.encode("utf-8"), v.encode("utf-8")) + for k, v in new_headers._md.items() ) message = self._message._replace(**dct) @@ -314,44 +310,26 @@ def forwarded(self) -> tuple[Mapping[str, str], ...]: Returns a tuple containing one or more immutable dicts """ elems = [] - for field_value in self._message.headers.getall(hdrs.FORWARDED, ()): - length = len(field_value) + for field_value in self._message.headers.getall(hdrs.FORWARDED): pos = 0 - need_separator = False elem: dict[str, str] = {} elems.append(types.MappingProxyType(elem)) - while 0 <= pos < length: + while 0 <= pos < len(field_value): match = _FORWARDED_PAIR_RE.match(field_value, pos) if match is not None: # got a valid forwarded-pair - if need_separator: - # bad syntax here, skip to next comma - pos = field_value.find(",", pos) - else: - name, value, port = match.groups() - if value[0] == '"': - # quoted string: remove quotes and unescape - value = _QUOTED_PAIR_REPLACE_RE.sub(r"\1", value[1:-1]) - if port: - value += port - elem[name.lower()] = value - pos += len(match.group(0)) - need_separator = True - elif field_value[pos] == ",": # next forwarded-element - need_separator = False - elem = {} - elems.append(types.MappingProxyType(elem)) - pos += 1 - elif field_value[pos] == ";": # next forwarded-pair - need_separator = False - pos += 1 - elif field_value[pos] in " \t": - # Allow whitespace even between forwarded-pairs, though - # RFC 7239 doesn't. This simplifies code and is in line - # with Postel's law. - pos += 1 + name, value, port = match.groups() + if value[0] == value[-1] == '"': + value = value[1:-1] + if port: + value += port + elem[name.lower()] = value + pos += len(match.group(0)) + elif not field_value[pos : field_value.find(";", pos)].strip(" \t"): + # Empty value + pos = field_value.find(";", pos) + 1 else: - # bad syntax here, skip to next comma - pos = field_value.find(",", pos) + # bad syntax here, skip to next field value + break return tuple(elems) @reify @@ -467,7 +445,7 @@ def query_string(self) -> str: return self._rel_url.query_string @reify - def headers(self) -> CIMultiDictProxy[str]: + def headers(self) -> HeadersDictProxy: """A case-insensitive multidict proxy with all headers.""" return self._headers diff --git a/tests/test_benchmarks_web_urldispatcher.py b/tests/test_benchmarks_web_urldispatcher.py index 5992d55a21a..4fe8bcae929 100644 --- a/tests/test_benchmarks_web_urldispatcher.py +++ b/tests/test_benchmarks_web_urldispatcher.py @@ -10,12 +10,13 @@ from unittest import mock import pytest -from multidict import CIMultiDict, CIMultiDictProxy +from multidict import CIMultiDict from pytest_codspeed import BenchmarkFixture from yarl import URL import aiohttp from aiohttp import web +from aiohttp.helpers import HeadersDictProxy from aiohttp.http import HttpVersion, RawRequestMessage @@ -38,7 +39,7 @@ def _mock_request(method: str, path: str) -> web.Request: method, path, HttpVersion(1, 1), - CIMultiDictProxy(CIMultiDict()), + HeadersDictProxy(CIMultiDict()), (), False, None, diff --git a/tests/test_client_exceptions.py b/tests/test_client_exceptions.py index a7bf4db05ef..b95325fb152 100644 --- a/tests/test_client_exceptions.py +++ b/tests/test_client_exceptions.py @@ -6,6 +6,7 @@ from yarl import URL from aiohttp import client, client_reqrep +from aiohttp.helpers import HeadersDictProxy class TestClientResponseError: @@ -43,7 +44,7 @@ def test_pickle(self) -> None: history=(), status=400, message="Something wrong", - headers=CIMultiDict(foo="bar"), + headers=HeadersDictProxy(CIMultiDict(foo="bar")), ) err.foo = "bar" # type: ignore[attr-defined] for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -66,11 +67,12 @@ def test_repr(self) -> None: history=(), status=400, message="Something wrong", - headers=CIMultiDict(), + headers=HeadersDictProxy(CIMultiDict()), ) assert repr(err) == ( "ClientResponseError(%r, (), status=400, " - "message='Something wrong', headers=)" % (self.request_info,) + "message='Something wrong', headers=)" + % (self.request_info,) ) def test_str(self) -> None: @@ -79,7 +81,7 @@ def test_str(self) -> None: history=(), status=400, message="Something wrong", - headers=CIMultiDict(), + headers=HeadersDictProxy(CIMultiDict()), ) assert str(err) == ("400, message='Something wrong', url='http://example.com'") diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py index 8c4a9eb34a3..c98f3b2a168 100644 --- a/tests/test_client_functional.py +++ b/tests/test_client_functional.py @@ -5548,12 +5548,6 @@ async def handler(request: web.Request) -> web.Response: # and .www.amazon.it domains resp = await session.get(f"http://www.amazon.it:{port}/") - # Check headers - cookie_headers = resp.headers.getall("Set-Cookie") - assert ( - len(cookie_headers) == 12 - ), f"Expected 12 headers, got {len(cookie_headers)}" - # Check parsed cookies - SimpleCookie only keeps the last # cookie with each name. So we expect 10 unique cookie names # (not 12) diff --git a/tests/test_client_request.py b/tests/test_client_request.py index eb71690f518..e028433d61d 100644 --- a/tests/test_client_request.py +++ b/tests/test_client_request.py @@ -27,7 +27,7 @@ from aiohttp.compression_utils import ZLibBackend from aiohttp.connector import Connection from aiohttp.hdrs import METH_DELETE -from aiohttp.helpers import TimerNoop +from aiohttp.helpers import HeadersDictProxy, TimerNoop from aiohttp.http import HttpVersion10, HttpVersion11, StreamWriter from aiohttp.multipart import MultipartWriter @@ -1631,7 +1631,7 @@ async def start(self, connection: Connection) -> ClientResponse: conn = connection self.status = 123 self.reason = "Test OK" - self._headers = CIMultiDictProxy(CIMultiDict()) + self._headers = HeadersDictProxy(CIMultiDict()) self.cookies = SimpleCookie() return self diff --git a/tests/test_client_response.py b/tests/test_client_response.py index edca472b4ee..dfc1a1f3fdb 100644 --- a/tests/test_client_response.py +++ b/tests/test_client_response.py @@ -8,15 +8,15 @@ from unittest import mock import pytest -from multidict import CIMultiDict, CIMultiDictProxy +from multidict import CIMultiDict from pytest_mock import MockerFixture from yarl import URL import aiohttp -from aiohttp import ClientSession, hdrs, http +from aiohttp import ClientSession, http from aiohttp.client_reqrep import ClientResponse from aiohttp.connector import Connection -from aiohttp.helpers import TimerNoop +from aiohttp.helpers import HeadersDictProxy, TimerNoop from aiohttp.multipart import BadContentDispositionHeader from aiohttp.tracing import Trace @@ -406,7 +406,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": "application/json;charset=cp1251"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect @@ -438,7 +438,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": # lie about the encoding h = {"Content-Type": "application/json;charset=utf-8"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect with pytest.raises(UnicodeDecodeError): @@ -472,7 +472,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": "text/html; charset=\udc81gutf-8\udc81\udc8d"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect @@ -504,7 +504,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": "application/json"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect with mock.patch.object(response, "get_encoding") as m: @@ -538,7 +538,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": content_type} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect @@ -565,7 +565,7 @@ async def test_get_encoding_body_none(session: ClientSession) -> None: ) h = {"Content-Type": "text/html"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = AssertionError @@ -599,7 +599,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": "application/json;charset=cp1251"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect @@ -630,7 +630,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": "application/json;charset=cp1251"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect @@ -661,7 +661,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": "application/this.is-1_content+subtype+json;charset=cp1251"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect @@ -692,7 +692,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": "custom/type;charset=cp1251"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect @@ -716,7 +716,7 @@ async def test_json_custom_loader(session: ClientSession) -> None: original_url=url, ) h = {"Content-Type": "application/json;charset=cp1251"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) response._body = b"data" def custom(content: str) -> str: @@ -741,7 +741,7 @@ async def test_json_invalid_content_type(session: ClientSession) -> None: original_url=url, ) h = {"Content-Type": "data/octet-stream"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) response._body = b"" response.status = 500 @@ -767,7 +767,7 @@ async def test_json_no_content(session: ClientSession) -> None: original_url=url, ) h = {"Content-Type": "application/json"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) response._body = b"" with pytest.raises(JSONDecodeError): @@ -796,7 +796,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": "application/json;charset=utf8"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect with mock.patch.object(response, "get_encoding") as m: @@ -824,7 +824,7 @@ def test_get_encoding_unknown( ) h = {"Content-Type": "application/json"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert response.get_encoding() == "utf-8" @@ -925,7 +925,7 @@ def test_content_type() -> None: original_url=url, ) h = {"Content-Type": "application/json;charset=cp1251"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert "application/json" == response.content_type @@ -944,7 +944,7 @@ def test_content_type_no_header() -> None: request_headers=CIMultiDict[str](), original_url=url, ) - response._headers = CIMultiDictProxy(CIMultiDict({})) + response._headers = HeadersDictProxy(CIMultiDict()) assert "application/octet-stream" == response.content_type @@ -964,7 +964,7 @@ def test_charset() -> None: original_url=url, ) h = {"Content-Type": "application/json;charset=cp1251"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert "cp1251" == response.charset @@ -983,7 +983,7 @@ def test_charset_no_header() -> None: request_headers=CIMultiDict[str](), original_url=url, ) - response._headers = CIMultiDictProxy(CIMultiDict({})) + response._headers = HeadersDictProxy(CIMultiDict()) assert response.charset is None @@ -1003,7 +1003,7 @@ def test_charset_no_charset() -> None: original_url=url, ) h = {"Content-Type": "application/json"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert response.charset is None @@ -1023,7 +1023,7 @@ def test_content_disposition_full() -> None: original_url=url, ) h = {"Content-Disposition": 'attachment; filename="archive.tar.gz"; foo=bar'} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert response.content_disposition is not None assert "attachment" == response.content_disposition.type @@ -1048,7 +1048,7 @@ def test_content_disposition_no_parameters() -> None: original_url=url, ) h = {"Content-Disposition": "attachment"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert response.content_disposition is not None assert "attachment" == response.content_disposition.type @@ -1078,7 +1078,7 @@ def test_content_disposition_empty_parts(content_disposition: str) -> None: original_url=url, ) h = {"Content-Disposition": content_disposition} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) with pytest.warns(BadContentDispositionHeader): assert response.content_disposition is not None @@ -1100,7 +1100,7 @@ def test_content_disposition_no_header() -> None: request_headers=CIMultiDict[str](), original_url=url, ) - response._headers = CIMultiDictProxy(CIMultiDict({})) + response._headers = HeadersDictProxy(CIMultiDict()) assert response.content_disposition is None @@ -1119,7 +1119,7 @@ def test_default_encoding_is_utf8() -> None: request_headers=CIMultiDict[str](), original_url=url, ) - response._headers = CIMultiDictProxy(CIMultiDict({})) + response._headers = HeadersDictProxy(CIMultiDict()) response._body = b"" assert response.get_encoding() == "utf-8" @@ -1227,7 +1227,7 @@ def test_redirect_history_in_exception() -> None: original_url=hist_url, ) - hist_response._headers = CIMultiDictProxy(CIMultiDict(hist_headers)) + hist_response._headers = HeadersDictProxy(CIMultiDict(hist_headers)) hist_response.status = 301 hist_response.reason = "REDIRECT" @@ -1263,7 +1263,7 @@ def side_effect(*args: object, **kwargs: object) -> "asyncio.Future[bytes]": return fut h = {"Content-Type": "application/json;charset=cp1251"} - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) content = response.content = mock.Mock() content.read.side_effect = side_effect @@ -1343,7 +1343,7 @@ def test_response_links_comma_separated( ), ), ) - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert response.links == { "next": {"url": URL("http://example.com/page/1.html"), "rel": "next"}, "home": {"url": URL("http://example.com/"), "rel": "home"}, @@ -1370,7 +1370,7 @@ def test_response_links_multiple_headers( ("Link", "; rel=next"), ("Link", "; rel=home"), ) - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert response.links == { "next": {"url": URL("http://example.com/page/1.html"), "rel": "next"}, "home": {"url": URL("http://example.com/"), "rel": "home"}, @@ -1394,7 +1394,7 @@ def test_response_links_no_rel( original_url=url, ) h = (("Link", ""),) - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert response.links == { "http://example.com/": {"url": URL("http://example.com/")} } @@ -1417,7 +1417,7 @@ def test_response_links_quoted( original_url=url, ) h = (("Link", '; rel="home-page"'),) - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert response.links == { "home-page": {"url": URL("http://example.com/"), "rel": "home-page"} } @@ -1440,7 +1440,7 @@ def test_response_links_relative( original_url=url, ) h = (("Link", "; rel=rel"),) - response._headers = CIMultiDictProxy(CIMultiDict(h)) + response._headers = HeadersDictProxy(CIMultiDict(h)) assert response.links == { "rel": {"url": URL("http://def-cl-resp.org/relative/path"), "rel": "rel"} } @@ -1462,7 +1462,7 @@ def test_response_links_empty( request_headers=CIMultiDict[str](), original_url=url, ) - response._headers = CIMultiDictProxy(CIMultiDict()) + response._headers = HeadersDictProxy(CIMultiDict()) assert response.links == {} @@ -1529,7 +1529,7 @@ def test_response_duplicate_cookie_names( ("Set-Cookie", "user-pref=light; Domain=api.example.com; Path=/"), ] ) - response._headers = CIMultiDictProxy(headers) + response._headers = HeadersDictProxy(CIMultiDict(headers)) # Set raw cookie headers as done in ClientResponse.start() response._raw_cookie_headers = tuple(headers.getall("Set-Cookie", [])) @@ -1540,9 +1540,7 @@ def test_response_duplicate_cookie_names( assert response.cookies["user-pref"].value == "light" # Last one wins -def test_response_raw_cookie_headers_preserved( - event_loop: asyncio.AbstractEventLoop, session: ClientSession -) -> None: +async def test_response_raw_cookie_headers_preserved(session: ClientSession) -> None: """Test that raw Set-Cookie headers are preserved in _raw_cookie_headers.""" url = URL("http://example.com") response = ClientResponse( @@ -1552,34 +1550,49 @@ def test_response_raw_cookie_headers_preserved( continue100=None, timer=TimerNoop(), traces=[], - loop=event_loop, + loop=asyncio.get_running_loop(), session=session, request_headers=CIMultiDict[str](), original_url=url, ) # Set headers with multiple cookies - cookie_headers = [ + cookie_headers = ( "session-id=123; Domain=.example.com; Path=/; Secure", "session-id=456; Domain=.www.example.com; Path=/", "tracking=xyz; Domain=.example.com; Path=/; HttpOnly", - ] + ) + md = CIMultiDict[str]() + for c in cookie_headers: + md.add("Set-Cookie", c) + raw_hdrs = tuple((k.encode(), v.encode()) for k, v in md.items()) - headers: CIMultiDict[str] = CIMultiDict() - for cookie_hdr in cookie_headers: - headers.add("Set-Cookie", cookie_hdr) + message = http.RawResponseMessage( + version=http.HttpVersion11, + code=200, + reason="OK", + headers=HeadersDictProxy(md), + raw_headers=raw_hdrs, + should_close=False, + compression=None, + upgrade=False, + chunked=False, + ) + payload = mock.create_autospec(aiohttp.StreamReader, spec_set=True, instance=True) - response._headers = CIMultiDictProxy(headers) + connection = mock.create_autospec(Connection, spec_set=True, instance=True) + connection.protocol = aiohttp.DataQueue(asyncio.get_running_loop()) + connection.protocol.feed_data((message, payload)) - # Set raw cookie headers as done in ClientResponse.start() - response._raw_cookie_headers = tuple(response.headers.getall(hdrs.SET_COOKIE, [])) + await response.start(connection) # Verify raw headers are preserved - assert response._raw_cookie_headers == tuple(cookie_headers) - assert len(response._raw_cookie_headers) == 3 + assert response._raw_cookie_headers == cookie_headers # But SimpleCookie only has unique names - assert len(response.cookies) == 2 # 'session-id' and 'tracking' + assert len(response.cookies) == 2 + assert "session-id" in response.cookies + assert "tracking" in response.cookies def test_response_cookies_setter_updates_raw_headers( diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 97790951960..fa71d9aa2ed 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -19,7 +19,7 @@ from aiohttp import http_exceptions, streams from aiohttp.base_protocol import BaseProtocol from aiohttp.client_proto import ResponseHandler -from aiohttp.helpers import DEFAULT_CHUNK_SIZE, NO_EXTENSIONS +from aiohttp.helpers import DEFAULT_CHUNK_SIZE, NO_EXTENSIONS, HeadersDictProxy from aiohttp.http_parser import ( DeflateBuffer, HeadersParser, @@ -263,6 +263,72 @@ def test_bad_header_name( parser.feed_data(text) +@pytest.mark.parametrize( + ("hdr_vals", "expected"), + ( + ( + ('"http://example.com/a.html,foo", apples',), + ("http://example.com/a.html,foo", "apples"), + ), + (("bananas, apples",), ("bananas", "apples")), + (("bananas", "apples"), ("bananas", "apples")), + ( + ('"http://example.com/a.html,foo", "apples"',), + ("http://example.com/a.html,foo", "apples"), + ), + ( + ('"Sat, 04 May 1996", "Wed, 14 Sep 2005"',), + ("Sat, 04 May 1996", "Wed, 14 Sep 2005"), + ), + (("foo,bar,baz",), ("foo", "bar", "baz")), + (('"applebanna, this',), ('"applebanna', "this")), + (('fooo", "bar"',), ('fooo"', "bar")), + ((" spam , eggs ",), ("spam", "eggs")), + ((" spam ", " eggs "), ("spam", "eggs")), + ((r'spam"foo\"bar"',), (r'spam"foo\"bar"',)), + # https://www.rfc-editor.org/rfc/rfc9110.html#name-recipient-requirements + (("foo, ,bar,",), ("foo", "bar")), + ((", , ",), ()), + (("",), ()), + # Escaped characters in quoted strings + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.6.4-3 + ((r'"foo\"bar"',), ('foo"bar',)), + ((r'"foo\\\"bar"',), (r"foo\"bar",)), + ((r'"foo\\", bar',), ("foo\\", "bar")), + # Comments: https://www.rfc-editor.org/rfc/rfc9110.html#section-5.6.5 + ((r"foo (bar\"spam\)eggs,\\baz)",), (r'foo (bar"spam)eggs,\baz)',)), + # Not a comment (requires whitespace) + (("foo(bar,spam)",), ("foo(bar", "spam)")), + # Parameters: https://www.rfc-editor.org/rfc/rfc9110.html#section-5.6.6 + (("text/html;charset=utf-8",), ("text/html;charset=utf-8",)), + (('Text/HTML; Charset="utf-8"',), ('Text/HTML; Charset="utf-8"',)), + ( + ("text/plain; q=0.5, text/html, text/x-dvi; q=0.8; format=flowed, */*",), + ( + "text/plain; q=0.5", + "text/html", + "text/x-dvi; q=0.8; format=flowed", + "*/*", + ), + ), + ((r'foo; bar="spam,\"eggs", baz',), ('foo; bar="spam,"eggs"', "baz")), + ((r'foo;bar="spam\\\",eggs",baz',), ('foo;bar="spam\\",eggs"', "baz")), + # Not valid parameters + (('foo; bar ="spam,eggs"',), ('foo; bar ="spam', 'eggs"')), + (('foo;bar= "spam,eggs"',), ('foo;bar= "spam', 'eggs"')), + ), +) +def test_list_headers( + parser: HttpRequestParser, hdr_vals: tuple[str], expected: tuple[str, ...] +) -> None: + headers = "\r\n".join(f"Foo: {v}" for v in hdr_vals) + text = f"POST / HTTP/1.1\r\nHost: a\r\n{headers}\r\n\r\n".encode() + messages, upgrade, tail = parser.feed_data(text) + msg = messages[0][0] + + assert msg.headers.getall("Foo") == expected + + @pytest.mark.parametrize( "hdr", ( @@ -597,11 +663,10 @@ def test_parse_headers_multi(parser: HttpRequestParser) -> None: assert len(messages) == 1 msg = messages[0][0] - assert list(msg.headers.items()) == [ + assert tuple(msg.headers.items()) == ( ("Host", "a"), - ("Set-Cookie", "c1=cookie1"), - ("Set-Cookie", "c2=cookie2"), - ] + ("Set-Cookie", "c1=cookie1, c2=cookie2"), + ) assert msg.raw_headers == ( (b"Host", b"a"), (b"Set-Cookie", b"c1=cookie1"), @@ -732,6 +797,15 @@ def test_request_te_chunked123(parser: HttpRequestParser) -> None: parser.feed_data(text) +def test_request_te_empty_list_invalid(parser: HttpRequestParser) -> None: + text = b"GET /test HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: , \t ,\r\n\r\n" + with pytest.raises( + http_exceptions.BadHttpMessage, + match="Request has invalid `Transfer-Encoding`", + ): + parser.feed_data(text) + + async def test_request_te_last_chunked(parser: HttpRequestParser) -> None: text = b"GET /test HTTP/1.1\r\nHost: a\r\nTransfer-Encoding: not, chunked\r\n\r\n1\r\nT\r\n3\r\nest\r\n0\r\n\r\n" messages, upgrade, tail = parser.feed_data(text) @@ -1769,6 +1843,16 @@ async def test_http_response_parser_notchunked( assert await messages[0][1].read() == b"1\r\nT\r\n3\r\nest\r\n0\r\n\r\n" +async def test_http_response_parser_empty_list_te_not_chunked( + response: HttpResponseParser, +) -> None: + text = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: , \t ,\r\n\r\nbody" + messages, upgrade, tail = response.feed_data(text) + response.feed_eof() + + assert await messages[0][1].read() == b"body" + + async def test_http_response_parser_last_chunked( response: HttpResponseParser, ) -> None: @@ -1952,11 +2036,7 @@ def test_parse_content_length_payload_multiple(response: HttpResponseParser) -> assert msg.version == HttpVersion(major=1, minor=1) assert msg.code == 200 assert msg.reason == "OK" - assert msg.headers == CIMultiDict( - [ - ("Content-Length", "5"), - ] - ) + assert msg.headers == HeadersDictProxy(CIMultiDict([("Content-Length", "5")])) assert msg.raw_headers == ((b"content-length", b"5"),) assert not msg.should_close assert msg.compression is None @@ -1992,11 +2072,7 @@ def test_parse_content_length_than_chunked_payload( assert msg.version == HttpVersion(major=1, minor=1) assert msg.code == 200 assert msg.reason == "OK" - assert msg.headers == CIMultiDict( - [ - ("Content-Length", "5"), - ] - ) + assert msg.headers == HeadersDictProxy(CIMultiDict([("Content-Length", "5")])) assert msg.raw_headers == ((b"content-length", b"5"),) assert not msg.should_close assert msg.compression is None @@ -2038,10 +2114,8 @@ def test_parse_chunked_payload_empty_body_than_another_chunked( assert msg.version == HttpVersion(major=1, minor=1) assert msg.code == code assert msg.reason == "OK" - assert msg.headers == CIMultiDict( - [ - ("Transfer-Encoding", "chunked"), - ] + assert msg.headers == HeadersDictProxy( + CIMultiDict([("Transfer-Encoding", "chunked")]) ) assert msg.raw_headers == ((b"transfer-encoding", b"chunked"),) assert not msg.should_close diff --git a/tests/test_multipart.py b/tests/test_multipart.py index 83046ccc034..aec0bcae922 100644 --- a/tests/test_multipart.py +++ b/tests/test_multipart.py @@ -8,7 +8,7 @@ from unittest import mock import pytest -from multidict import CIMultiDict, CIMultiDictProxy +from multidict import CIMultiDict import aiohttp from aiohttp import payload @@ -20,7 +20,7 @@ CONTENT_TRANSFER_ENCODING, CONTENT_TYPE, ) -from aiohttp.helpers import DEFAULT_CHUNK_SIZE, parse_mimetype +from aiohttp.helpers import DEFAULT_CHUNK_SIZE, HeadersDictProxy, parse_mimetype from aiohttp.multipart import ( BodyPartReader, BodyPartReaderPayload, @@ -106,7 +106,7 @@ def __exit__( class Response: - def __init__(self, headers: CIMultiDictProxy[str], content: Stream) -> None: + def __init__(self, headers: HeadersDictProxy, content: Stream) -> None: self.headers = headers self.content = content @@ -161,7 +161,7 @@ async def test_release_when_stream_at_eof(self) -> None: class TestPartReader: async def test_next(self) -> None: with Stream(b"Hello, world!\r\n--:") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.next() assert b"Hello, world!" == result @@ -169,7 +169,7 @@ async def test_next(self) -> None: async def test_next_next(self) -> None: with Stream(b"Hello, world!\r\n--:") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.next() assert b"Hello, world!" == result @@ -179,7 +179,7 @@ async def test_next_next(self) -> None: async def test_read(self) -> None: with Stream(b"Hello, world!\r\n--:") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.read() assert b"Hello, world!" == result @@ -187,7 +187,7 @@ async def test_read(self) -> None: async def test_read_chunk_at_eof(self) -> None: with Stream(b"--:") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) obj._at_eof = True result = await obj.read_chunk() @@ -195,7 +195,7 @@ async def test_read_chunk_at_eof(self) -> None: async def test_read_chunk_without_content_length(self) -> None: with Stream(b"Hello, world!\r\n--:") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) c1 = await obj.read_chunk(8) c2 = await obj.read_chunk(8) @@ -219,7 +219,7 @@ def prepare(data: bytes) -> bytes: prepare(b""), ], ): - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) c1 = await obj.read_chunk(8) assert c1 == b"Hello, " @@ -230,7 +230,7 @@ def prepare(data: bytes) -> bytes: async def test_read_all_at_once(self) -> None: with Stream(b"Hello, World!\r\n--:--\r\n") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.read_chunk() assert b"Hello, World!" == result @@ -240,7 +240,7 @@ async def test_read_all_at_once(self) -> None: async def test_read_incomplete_body_chunked(self) -> None: with Stream(b"Hello, World!\r\n-") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = b"" with pytest.raises(ValueError): @@ -251,7 +251,7 @@ async def test_read_incomplete_body_chunked(self) -> None: async def test_read_with_content_length_malformed_crlf(self) -> None: # Content-Length is correct but data after content is not \r\n content = b"Hello" - h = CIMultiDictProxy(CIMultiDict({"CONTENT-LENGTH": str(len(content))})) + h = HeadersDictProxy(CIMultiDict({"CONTENT-LENGTH": str(len(content))})) # Malformed: "XX" instead of "\r\n" after content with Stream(content + b"XX--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) @@ -274,7 +274,7 @@ def prepare(data: bytes) -> bytes: prepare(b""), ], ): - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) c1 = await obj.read_chunk(12) assert c1 == b"Hello, World" @@ -285,7 +285,7 @@ def prepare(data: bytes) -> bytes: async def test_multi_read_chunk(self) -> None: with Stream(b"Hello,\r\n--:\r\n\r\nworld!\r\n--:--") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.read_chunk(8) assert b"Hello," == result @@ -296,7 +296,7 @@ async def test_multi_read_chunk(self) -> None: async def test_read_chunk_properly_counts_read_bytes(self) -> None: expected = b"." * 10 size = len(expected) - h = CIMultiDictProxy(CIMultiDict({"CONTENT-LENGTH": str(size)})) + h = HeadersDictProxy(CIMultiDict({"CONTENT-LENGTH": str(size)})) with StreamWithShortenRead(expected + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = bytearray() @@ -311,7 +311,7 @@ async def test_read_chunk_properly_counts_read_bytes(self) -> None: async def test_read_does_not_read_boundary(self) -> None: with Stream(b"Hello, world!\r\n--:") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.read() assert b"Hello, world!" == result @@ -319,7 +319,7 @@ async def test_read_does_not_read_boundary(self) -> None: async def test_multiread(self) -> None: with Stream(b"Hello,\r\n--:\r\n\r\nworld!\r\n--:--") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.read() assert b"Hello," == result @@ -329,7 +329,7 @@ async def test_multiread(self) -> None: async def test_read_multiline(self) -> None: with Stream(b"Hello\n,\r\nworld!\r\n--:--") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.read() assert b"Hello\n,\r\nworld!" == result @@ -338,7 +338,7 @@ async def test_read_multiline(self) -> None: assert obj.at_eof() async def test_read_respects_content_length(self) -> None: - h = CIMultiDictProxy(CIMultiDict({"CONTENT-LENGTH": "100500"})) + h = HeadersDictProxy(CIMultiDict({"CONTENT-LENGTH": "100500"})) with Stream(b"." * 100500 + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.read() @@ -346,7 +346,7 @@ async def test_read_respects_content_length(self) -> None: assert obj.at_eof() async def test_read_with_content_encoding_gzip(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "gzip"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_ENCODING: "gzip"})) with Stream( b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x0b\xc9\xccMU" b"(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00" @@ -361,7 +361,7 @@ async def test_read_with_content_encoding_deflate(self) -> None: content = b"A" * 1_000_000 # Large enough to exceed max_length. compressed = ZLibBackend.compress(content, wbits=-ZLibBackend.MAX_WBITS) - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) with Stream(compressed + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.read(decode=True) @@ -374,14 +374,14 @@ async def test_read_with_content_encoding_identity(self) -> None: b"(\xc9W\x08J\xcdI\xacP\x04\x00$\xfb\x9eV\x0e\x00\x00\x00" b"\r\n" ) - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "identity"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_ENCODING: "identity"})) with Stream(thing + b"--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.read(decode=True) assert thing[:-2] == result async def test_read_with_content_encoding_unknown(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "snappy"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_ENCODING: "snappy"})) with Stream(b"\x0e4Time to Relax!\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) with pytest.raises(RuntimeError): @@ -391,7 +391,7 @@ async def test_read_decode_compressed_exceeds_max_size(self) -> None: # Compressed data is small, but decompresses beyond client_max_size. original = b"A" * 1024 compressed = gzip.compress(original) - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "gzip"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_ENCODING: "gzip"})) with Stream(compressed + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader( BOUNDARY, @@ -404,14 +404,14 @@ async def test_read_decode_compressed_exceeds_max_size(self) -> None: await obj.read(decode=True) async def test_read_with_content_transfer_encoding_base64(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: "base64"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: "base64"})) with Stream(b"VGltZSB0byBSZWxheCE=\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.read(decode=True) assert b"Time to Relax!" == result async def test_decode_with_content_transfer_encoding_base64(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: "base64"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: "base64"})) with Stream(b"VG\r\r\nltZSB0byBSZ\r\nWxheCE=\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = b"" @@ -421,7 +421,7 @@ async def test_decode_with_content_transfer_encoding_base64(self) -> None: assert b"Time to Relax!" == result async def test_decode_iter_with_content_transfer_encoding_base64(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: "base64"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: "base64"})) with Stream(b"VG\r\r\nltZSB0byBSZ\r\nWxheCE=\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = b"" @@ -432,7 +432,7 @@ async def test_decode_iter_with_content_transfer_encoding_base64(self) -> None: assert b"Time to Relax!" == result async def test_decode_with_content_encoding_deflate(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) data = b"\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00" with Stream(data + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) @@ -441,7 +441,7 @@ async def test_decode_with_content_encoding_deflate(self) -> None: assert b"Time to Relax!" == result async def test_decode_with_content_encoding_identity(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "identity"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_ENCODING: "identity"})) data = b"Time to Relax!" with Stream(data + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) @@ -450,7 +450,7 @@ async def test_decode_with_content_encoding_identity(self) -> None: assert data == result async def test_decode_with_content_encoding_unknown(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "snappy"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_ENCODING: "snappy"})) data = b"Time to Relax!" with Stream(data + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) @@ -459,7 +459,7 @@ async def test_decode_with_content_encoding_unknown(self) -> None: obj.decode(chunk) async def test_read_with_content_transfer_encoding_quoted_printable(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TRANSFER_ENCODING: "quoted-printable"}) ) with Stream( @@ -481,14 +481,14 @@ async def test_read_with_content_transfer_encoding_binary( b"\xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82," b" \xd0\xbc\xd0\xb8\xd1\x80!" ) - h = CIMultiDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: encoding})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: encoding})) with Stream(data + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.read(decode=True) assert data == result async def test_read_with_content_transfer_encoding_unknown(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: "unknown"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TRANSFER_ENCODING: "unknown"})) with Stream(b"\x0e4Time to Relax!\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) with pytest.raises(RuntimeError): @@ -496,34 +496,34 @@ async def test_read_with_content_transfer_encoding_unknown(self) -> None: async def test_read_text(self) -> None: with Stream(b"Hello, world!\r\n--:--") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.text() assert "Hello, world!" == result async def test_read_text_default_encoding(self) -> None: with Stream("Привет, Мир!\r\n--:--".encode()) as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.text() assert "Привет, Мир!" == result async def test_read_text_encoding(self) -> None: with Stream("Привет, Мир!\r\n--:--".encode("cp1251")) as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.text(encoding="cp1251") assert "Привет, Мир!" == result async def test_read_text_guess_encoding(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain;charset=cp1251"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain;charset=cp1251"})) with Stream("Привет, Мир!\r\n--:--".encode("cp1251")) as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.text() assert "Привет, Мир!" == result async def test_read_text_compressed(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_ENCODING: "deflate", CONTENT_TYPE: "text/plain"}) ) with Stream(b"\x0b\xc9\xccMU(\xc9W\x08J\xcdI\xacP\x04\x00\r\n--:--") as stream: @@ -532,7 +532,7 @@ async def test_read_text_compressed(self) -> None: assert "Time to Relax!" == result async def test_read_text_while_closed(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain"})) with Stream(b"") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) obj._at_eof = True @@ -540,21 +540,21 @@ async def test_read_text_while_closed(self) -> None: assert "" == result async def test_read_json(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TYPE: "application/json"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TYPE: "application/json"})) with Stream(b'{"test": "passed"}\r\n--:--') as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.json() assert {"test": "passed"} == result async def test_read_json_encoding(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TYPE: "application/json"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TYPE: "application/json"})) with Stream('{"тест": "пассед"}\r\n--:--'.encode("cp1251")) as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) result = await obj.json(encoding="cp1251") assert {"тест": "пассед"} == result async def test_read_json_guess_encoding(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: "application/json; charset=cp1251"}) ) with Stream('{"тест": "пассед"}\r\n--:--'.encode("cp1251")) as stream: @@ -563,7 +563,7 @@ async def test_read_json_guess_encoding(self) -> None: assert {"тест": "пассед"} == result async def test_read_json_compressed(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_ENCODING: "deflate", CONTENT_TYPE: "application/json"}) ) with Stream(b"\xabV*I-.Q\xb2RP*H,.NMQ\xaa\x05\x00\r\n--:--") as stream: @@ -572,7 +572,7 @@ async def test_read_json_compressed(self) -> None: assert {"test": "passed"} == result async def test_read_json_while_closed(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TYPE: "application/json"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TYPE: "application/json"})) with Stream(b"") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) obj._at_eof = True @@ -580,7 +580,7 @@ async def test_read_json_while_closed(self) -> None: assert result is None async def test_read_form(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: "application/x-www-form-urlencoded"}) ) with Stream(b"foo=bar&foo=baz&boo=\r\n--:--") as stream: @@ -589,7 +589,7 @@ async def test_read_form(self) -> None: assert [("foo", "bar"), ("foo", "baz"), ("boo", "")] == result async def test_read_form_invalid_utf8(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: "application/x-www-form-urlencoded"}) ) with Stream(b"\xff\r\n--:--") as stream: @@ -600,7 +600,7 @@ async def test_read_form_invalid_utf8(self) -> None: await obj.form() async def test_read_form_encoding(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: "application/x-www-form-urlencoded"}) ) with Stream("foo=bar&foo=baz&boo=\r\n--:--".encode("cp1251")) as stream: @@ -609,7 +609,7 @@ async def test_read_form_encoding(self) -> None: assert [("foo", "bar"), ("foo", "baz"), ("boo", "")] == result async def test_read_form_guess_encoding(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict( {CONTENT_TYPE: "application/x-www-form-urlencoded; charset=utf-8"} ) @@ -620,7 +620,7 @@ async def test_read_form_guess_encoding(self) -> None: assert [("foo", "bar"), ("foo", "baz"), ("boo", "")] == result async def test_read_form_while_closed(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: "application/x-www-form-urlencoded"}) ) with Stream(b"") as stream: @@ -631,7 +631,7 @@ async def test_read_form_while_closed(self) -> None: async def test_readline(self) -> None: with Stream(b"Hello\n,\r\nworld!\r\n--:--") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) result = await obj.readline() assert b"Hello\n" == result @@ -645,14 +645,14 @@ async def test_readline(self) -> None: async def test_release(self) -> None: with Stream(b"Hello,\r\n--:\r\n\r\nworld!\r\n--:--") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) await obj.release() assert obj.at_eof() assert b"--:\r\n\r\nworld!\r\n--:--" == stream.content.read() async def test_release_respects_content_length(self) -> None: - h = CIMultiDictProxy(CIMultiDict({"CONTENT-LENGTH": "100500"})) + h = HeadersDictProxy(CIMultiDict({"CONTENT-LENGTH": "100500"})) with Stream(b"." * 100500 + b"\r\n--:--") as stream: obj = aiohttp.BodyPartReader(BOUNDARY, h, stream) await obj.release() @@ -660,14 +660,14 @@ async def test_release_respects_content_length(self) -> None: async def test_release_release(self) -> None: with Stream(b"Hello,\r\n--:\r\n\r\nworld!\r\n--:--") as stream: - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) await obj.release() await obj.release() assert b"--:\r\n\r\nworld!\r\n--:--" == stream.content.read() async def test_filename(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_DISPOSITION: "attachment; filename=foo.html"}) ) part = aiohttp.BodyPartReader(BOUNDARY, h, mock.Mock()) @@ -681,7 +681,7 @@ async def test_reading_long_part(self) -> None: ) stream.feed_data(b"0" * size + b"\r\n--:--") stream.feed_eof() - d = CIMultiDictProxy[str](CIMultiDict()) + d = HeadersDictProxy(CIMultiDict()) obj = aiohttp.BodyPartReader(BOUNDARY, d, stream) data = await obj.read() assert len(data) == size @@ -689,7 +689,7 @@ async def test_reading_long_part(self) -> None: class TestMultipartReader: def test_from_response(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: 'multipart/related;boundary=":"'}) ) with Stream(b"--:\r\n\r\nhello\r\n--:--") as stream: @@ -699,7 +699,7 @@ def test_from_response(self) -> None: assert isinstance(res.stream, aiohttp.MultipartReader) def test_bad_boundary(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: "multipart/related;boundary=" + "a" * 80}) ) with Stream(b"") as stream: @@ -708,7 +708,7 @@ def test_bad_boundary(self) -> None: aiohttp.MultipartReader.from_response(resp) # type: ignore[arg-type] def test_dispatch(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain"})) with Stream(b"--:\r\n\r\necho\r\n--:--") as stream: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, @@ -718,7 +718,7 @@ def test_dispatch(self) -> None: assert isinstance(res, reader.part_reader_cls) def test_dispatch_bodypart(self) -> None: - h = CIMultiDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain"})) with Stream(b"--:\r\n\r\necho\r\n--:--") as stream: reader = aiohttp.MultipartReader( {CONTENT_TYPE: 'multipart/related;boundary=":"'}, @@ -728,7 +728,7 @@ def test_dispatch_bodypart(self) -> None: assert isinstance(res, reader.part_reader_cls) def test_dispatch_multipart(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: "multipart/related;boundary=--:--"}) ) with Stream( @@ -752,7 +752,7 @@ def test_dispatch_custom_multipart_reader(self) -> None: class CustomReader(aiohttp.MultipartReader): pass - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: "multipart/related;boundary=--:--"}) ) with Stream( @@ -774,7 +774,7 @@ class CustomReader(aiohttp.MultipartReader): assert isinstance(res, CustomReader) async def test_emit_next(self) -> None: - h = CIMultiDictProxy( + h = HeadersDictProxy( CIMultiDict({CONTENT_TYPE: 'multipart/related;boundary=":"'}) ) with Stream(b"--:\r\n\r\necho\r\n--:--") as stream: @@ -1618,7 +1618,7 @@ async def check(reader: aiohttp.MultipartReader) -> None: async def test_async_for_bodypart() -> None: - h = CIMultiDictProxy[str](CIMultiDict()) + h = HeadersDictProxy(CIMultiDict()) with Stream(b"foobarbaz\r\n--:--") as stream: part = aiohttp.BodyPartReader(boundary=b"--:", headers=h, content=stream) async for data in part: @@ -1729,7 +1729,7 @@ async def test_multipart_writer_reusability_with_io_payloads( async def test_body_part_reader_payload_as_bytes() -> None: """Test that BodyPartReaderPayload.as_bytes raises TypeError.""" # Create a mock BodyPartReader - headers = CIMultiDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain"})) + headers = HeadersDictProxy(CIMultiDict({CONTENT_TYPE: "text/plain"})) protocol = mock.Mock(_reading_paused=False) stream = StreamReader(protocol, 2**16, loop=asyncio.get_event_loop()) body_part = BodyPartReader(BOUNDARY, headers, stream) @@ -1756,7 +1756,7 @@ async def write(inp: bytes) -> None: nonlocal output output += inp - h = CIMultiDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) + h = HeadersDictProxy(CIMultiDict({CONTENT_ENCODING: "deflate"})) if sys.version_info >= (3, 12): writer = mock.create_autospec( AbstractStreamWriter, write=write, spec_set=True, instance=True diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index 13ae3ee97ac..10e20d0b58a 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -7,12 +7,13 @@ from unittest import mock import pytest -from multidict import CIMultiDict, CIMultiDictProxy +from multidict import CIMultiDict from pytest_aiohttp import AiohttpClient from yarl import URL import aiohttp from aiohttp import web +from aiohttp.helpers import HeadersDictProxy from aiohttp.test_utils import ( REUSE_ADDRESS, AioHTTPTestCase, @@ -170,7 +171,7 @@ def test_make_mocked_request(headers: Mapping[str, str]) -> None: assert req.method == "GET" assert req.path == "/" assert isinstance(req, web.Request) - assert isinstance(req.headers, CIMultiDictProxy) + assert isinstance(req.headers, HeadersDictProxy) def test_make_mocked_request_sslcontext() -> None: diff --git a/tests/test_web_functional.py b/tests/test_web_functional.py index 442aeade528..823b3e0f14f 100644 --- a/tests/test_web_functional.py +++ b/tests/test_web_functional.py @@ -9,7 +9,7 @@ from unittest import mock import pytest -from multidict import CIMultiDictProxy, MultiDict +from multidict import MultiDict from pytest_aiohttp import AiohttpClient, AiohttpServer from pytest_mock import MockerFixture from yarl import URL @@ -27,6 +27,7 @@ from aiohttp.abc import AbstractResolver, ResolveResult from aiohttp.compression_utils import ZLibBackend, ZLibCompressObjProtocol from aiohttp.hdrs import CONTENT_LENGTH, CONTENT_TYPE, TRANSFER_ENCODING +from aiohttp.helpers import HeadersDictProxy from aiohttp.typedefs import Handler, Middleware from aiohttp.web_protocol import RequestHandler @@ -2194,7 +2195,7 @@ async def handler(request: web.Request) -> web.Response: async def test_request_headers_type(aiohttp_client: AiohttpClient) -> None: async def handler(request: web.Request) -> web.Response: - assert isinstance(request.headers, CIMultiDictProxy) + assert isinstance(request.headers, HeadersDictProxy) return web.Response() app = web.Application() diff --git a/tests/test_web_request.py b/tests/test_web_request.py index d2bb290e2f5..7c0ab5d9be4 100644 --- a/tests/test_web_request.py +++ b/tests/test_web_request.py @@ -11,13 +11,13 @@ from unittest import mock import pytest -from multidict import CIMultiDict, CIMultiDictProxy, MultiDict +from multidict import CIMultiDict, MultiDict from pytest_aiohttp import AiohttpClient from yarl import URL from aiohttp import ETag, HttpVersion, web from aiohttp.base_protocol import BaseProtocol -from aiohttp.helpers import DEFAULT_CHUNK_SIZE +from aiohttp.helpers import DEFAULT_CHUNK_SIZE, HeadersDictProxy from aiohttp.http_exceptions import BadHttpMessage, LineTooLong from aiohttp.http_parser import RawRequestMessage from aiohttp.streams import StreamReader @@ -35,7 +35,7 @@ def test_base_ctor() -> None: "GET", "/path/to?a=1&b=2", HttpVersion(1, 1), - CIMultiDictProxy(CIMultiDict()), + HeadersDictProxy(CIMultiDict()), (), False, None, @@ -731,14 +731,12 @@ def test_multiple_forwarded_headers_bad_syntax() -> None: headers = CIMultiDict[str]() headers.add("Forwarded", "for=_1;by=_2") headers.add("Forwarded", "invalid value") - headers.add("Forwarded", "") headers.add("Forwarded", "for=_3;by=_4") req = make_mocked_request("GET", "/", headers=headers) - assert len(req.forwarded) == 4 + assert len(req.forwarded) == 3 assert req.forwarded[0]["for"] == "_1" assert "for" not in req.forwarded[1] - assert "for" not in req.forwarded[2] - assert req.forwarded[3]["by"] == "_4" + assert req.forwarded[2]["by"] == "_4" def test_multiple_forwarded_headers_injection() -> None: diff --git a/tests/test_web_response.py b/tests/test_web_response.py index daa9de46bc9..3dee8d3a1b1 100644 --- a/tests/test_web_response.py +++ b/tests/test_web_response.py @@ -16,7 +16,7 @@ from aiohttp import HttpVersion, HttpVersion10, HttpVersion11, hdrs, web from aiohttp.abc import AbstractStreamWriter -from aiohttp.helpers import ETag +from aiohttp.helpers import ETag, HeadersDictProxy from aiohttp.http_writer import StreamWriter, _serialize_headers from aiohttp.multipart import BodyPartReader, MultipartWriter from aiohttp.payload import BytesPayload, StringPayload @@ -1176,7 +1176,7 @@ def read(self, size: int = -1) -> bytes: (io.BytesIO(b"test"), "test"), (io.BufferedReader(io.BytesIO(b"test")), "test"), (async_iter(), None), - (BodyPartReader(b"x", CIMultiDictProxy(CIMultiDict()), mock.Mock()), None), + (BodyPartReader(b"x", HeadersDictProxy(CIMultiDict()), mock.Mock()), None), ( mpwriter, "--x\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: 4\r\n\r\ntest",