From 0cde6cd67b0c7eaa2b54b678eb8ae6f87ec6ef50 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Fri, 7 Aug 2026 14:16:15 +0000 Subject: [PATCH 1/5] optimize client and connection reuse across Sessions --- pyproject.toml | 2 +- src/crawlee/errors.py | 11 ++ src/crawlee/http_clients/_impit.py | 199 ++++++++++++++++++----- src/crawlee/sessions/_cookies.py | 44 +++++ tests/unit/http_clients/test_impit.py | 223 ++++++++++++++++++++++++++ tests/unit/server.py | 31 ++++ tests/unit/sessions/test_cookies.py | 85 ++++++++++ uv.lock | 92 +++++------ 8 files changed, 602 insertions(+), 85 deletions(-) create mode 100644 tests/unit/http_clients/test_impit.py diff --git a/pyproject.toml b/pyproject.toml index e20b1afdda..51372dc136 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "async-timeout>=5.0.1", "cachetools>=5.5.0", "colorama>=0.4.0", - "impit>=0.8.0", + "impit>=0.13.2", "more-itertools>=10.2.0", "protego>=0.5.0", "psutil>=6.0.0", diff --git a/src/crawlee/errors.py b/src/crawlee/errors.py index 539bcf7711..b7f292d4a0 100644 --- a/src/crawlee/errors.py +++ b/src/crawlee/errors.py @@ -18,6 +18,7 @@ 'RequestHandlerError', 'ServiceConflictError', 'SessionError', + 'TooManyRedirectsError', 'UserDefinedErrorHandlerError', ] @@ -57,6 +58,16 @@ class ProxyError(SessionError): """Raised when a proxy is being blocked or malfunctions.""" +@docs_group('Errors') +class TooManyRedirectsError(Exception): + """Raised when a request exceeds the maximum number of redirects allowed by the HTTP client.""" + + def __init__(self, url: str, max_redirects: int) -> None: + super().__init__(f'Exceeded the limit of {max_redirects} redirects while requesting {url}.') + self.url = url + self.max_redirects = max_redirects + + @docs_group('Errors') class HttpStatusCodeError(Exception): """Raised when the response status code indicates an error.""" diff --git a/src/crawlee/http_clients/_impit.py b/src/crawlee/http_clients/_impit.py index 11e8c81ada..a7b792bd05 100644 --- a/src/crawlee/http_clients/_impit.py +++ b/src/crawlee/http_clients/_impit.py @@ -2,25 +2,25 @@ import asyncio from contextlib import asynccontextmanager +from http import HTTPStatus from logging import getLogger -from typing import TYPE_CHECKING, Any, TypedDict +from typing import TYPE_CHECKING, Any -from cachetools import LRUCache from impit import AsyncClient, Browser, HTTPError, Response, TimeoutException, TransportError from impit import ProxyError as ImpitProxyError from typing_extensions import override +from yarl import URL from crawlee._types import HttpHeaders from crawlee._utils.blocked import ROTATE_PROXY_ERRORS from crawlee._utils.docs import docs_group from crawlee._utils.urls import validate_http_url -from crawlee.errors import ProxyError +from crawlee.errors import ProxyError, TooManyRedirectsError from crawlee.http_clients import HttpClient, HttpCrawlingResult, HttpResponse if TYPE_CHECKING: from collections.abc import AsyncGenerator, AsyncIterator from datetime import timedelta - from http.cookiejar import CookieJar from crawlee import Request from crawlee._types import HttpMethod, HttpPayload @@ -30,12 +30,48 @@ logger = getLogger(__name__) +_REDIRECT_STATUS_CODES = frozenset( + { + HTTPStatus.MOVED_PERMANENTLY, + HTTPStatus.FOUND, + HTTPStatus.SEE_OTHER, + HTTPStatus.TEMPORARY_REDIRECT, + HTTPStatus.PERMANENT_REDIRECT, + } +) -class _ClientCacheEntry(TypedDict): - """Type definition for client cache entries.""" +# Status codes that redirect a `POST` as a `GET`. `HTTPStatus.SEE_OTHER` does so for any method but `GET` and `HEAD`. +_MOVED_STATUS_CODES = frozenset({HTTPStatus.MOVED_PERMANENTLY, HTTPStatus.FOUND}) - client: AsyncClient - cookie_jar: CookieJar | None +_HTTP_SCHEMES = frozenset({'http', 'https'}) + +# Headers scoped to a single origin, dropped as soon as a redirect leaves it. +_CROSS_ORIGIN_HEADERS = frozenset({'authorization', 'cookie', 'proxy-authorization'}) + +# Headers describing a request body, dropped when a redirect turns the request into a bodyless `GET`. +_REQUEST_BODY_HEADERS = frozenset({'content-encoding', 'content-language', 'content-location', 'content-type'}) + + +def _is_cross_origin(url: URL, next_url: URL) -> bool: + """Check whether a redirect from `url` to `next_url` leaves the origin. + + Origins are compared as strings, because `yarl` considers an explicitly written default port different from + an omitted one. + """ + return str(url.origin()) != str(next_url.origin()) + + +def _redirect_method(status_code: int, method: str) -> str: + """Resolve the method of a redirected request, following the `HTTP-redirect fetch` algorithm. + + See https://fetch.spec.whatwg.org/#http-redirect-fetch. + """ + if (status_code in _MOVED_STATUS_CODES and method == 'POST') or ( + status_code == HTTPStatus.SEE_OTHER and method not in {'GET', 'HEAD'} + ): + return 'GET' + + return method class _ImpitResponse: @@ -96,6 +132,8 @@ def __init__( http3: bool = False, verify: bool = True, browser: Browser | None = 'firefox', + follow_redirects: bool = True, + max_redirects: int = 20, **async_client_kwargs: Any, ) -> None: """Initialize a new instance. @@ -105,6 +143,8 @@ def __init__( http3: Whether to enable HTTP/3 support. verify: SSL certificates used to verify the identity of requested hosts. browser: Browser to impersonate. + follow_redirects: Whether to follow HTTP redirects. + max_redirects: Maximum number of redirects to follow before raising `TooManyRedirectsError`. async_client_kwargs: Additional keyword arguments for `impit.AsyncClient`. """ super().__init__( @@ -113,10 +153,12 @@ def __init__( self._http3 = http3 self._verify = verify self._browser = browser + self._follow_redirects = follow_redirects + self._max_redirects = max_redirects self._async_client_kwargs = async_client_kwargs - self._client_by_proxy_url = LRUCache[str | None, _ClientCacheEntry](maxsize=10) + self._client_by_proxy_url = dict[str | None, AsyncClient]() @override async def crawl( @@ -128,15 +170,15 @@ async def crawl( statistics: Statistics | None = None, timeout: timedelta | None = None, ) -> HttpCrawlingResult: - client = self._get_client(proxy_info.url if proxy_info else None, session.cookies.jar if session else None) - try: - response = await client.request( - url=request.url, + response = await self._request_with_redirects( method=request.method, - content=request.payload, - headers=dict(request.headers) if request.headers else None, - timeout=timeout.total_seconds() if timeout else None, + url=request.url, + headers=dict(request.headers) if request.headers else {}, + payload=request.payload, + session=session, + proxy_info=proxy_info, + timeout=timeout, ) except TimeoutException as exc: raise asyncio.TimeoutError from exc @@ -169,15 +211,15 @@ async def send_request( if isinstance(headers, dict) or headers is None: headers = HttpHeaders(headers or {}) - client = self._get_client(proxy_info.url if proxy_info else None, session.cookies.jar if session else None) - try: - response = await client.request( + response = await self._request_with_redirects( method=method, url=url, - content=payload, - headers=dict(headers) if headers else None, - timeout=timeout.total_seconds() if timeout else None, + headers=dict(headers), + payload=payload, + session=session, + proxy_info=proxy_info, + timeout=timeout, ) except TimeoutException as exc: raise asyncio.TimeoutError from exc @@ -203,15 +245,18 @@ async def stream( ) -> AsyncGenerator[HttpResponse]: validate_http_url(url) - client = self._get_client(proxy_info.url if proxy_info else None, session.cookies.jar if session else None) + if isinstance(headers, dict) or headers is None: + headers = HttpHeaders(headers or {}) try: - response = await client.request( + response = await self._request_with_redirects( method=method, url=url, - content=payload, - headers=dict(headers) if headers else None, - timeout=timeout.total_seconds() if timeout else None, + headers=dict(headers), + payload=payload, + session=session, + proxy_info=proxy_info, + timeout=timeout, stream=True, ) except TimeoutException as exc: @@ -222,34 +267,112 @@ async def stream( finally: response.close() - def _get_client(self, proxy_url: str | None, cookie_jar: CookieJar | None) -> AsyncClient: + async def _request_with_redirects( + self, + *, + method: str, + url: str, + headers: dict[str, str], + payload: HttpPayload | None, + session: Session | None, + proxy_info: ProxyInfo | None, + timeout: timedelta | None, + stream: bool = False, + ) -> Response: + """Perform a request, following redirects one hop at a time. + + Redirects are resolved here instead of by `impit`, so that cookies of the given session are attached to and + collected from every single hop. A client is therefore never bound to a session and can be shared by all + of them. + + Args: + method: The HTTP method to use. + url: The URL to send the request to. + headers: The headers to include in the request. + payload: The data to be sent as the request body. + session: The session whose cookies are sent and updated. + proxy_info: The information about the proxy to be used. + timeout: Maximum time allowed to process the request. + stream: Whether the body of the final response should be streamed. + + Raises: + TooManyRedirectsError: If the number of redirects exceeds `max_redirects`. + + Returns: + The final response of the redirect chain. + """ + client = self._get_client(proxy_info.url if proxy_info else None) + current_url = URL(url) + content = payload + + for _ in range(self._max_redirects + 1): + # Header names are normalized to lowercase by `HttpHeaders`, so this replaces any `Cookie` header + # passed by the caller instead of adding a second one. + if session and (cookie_string := session.cookies.get_cookie_string(str(current_url))): + headers['cookie'] = cookie_string + + response = await client.request( + method=method, + url=str(current_url), + content=content, + headers=headers or None, + timeout=timeout.total_seconds() if timeout else None, + stream=stream, + ) + + if session and self._persist_cookies_per_session: + session.cookies.extract_cookie_from_header(str(current_url), response.headers.get_list('set-cookie')) + + if not self._follow_redirects or response.status_code not in _REDIRECT_STATUS_CODES: + return response + + location = response.headers.get('location') + if not location: + return response + + next_url = current_url.join(URL(location)) + if next_url.scheme not in _HTTP_SCHEMES: + return response + + next_method = _redirect_method(response.status_code, method) + if next_method != method: + method = next_method + content = None + headers = {key: value for key, value in headers.items() if key.lower() not in _REQUEST_BODY_HEADERS} + + if _is_cross_origin(current_url, next_url): + headers = {key: value for key, value in headers.items() if key.lower() not in _CROSS_ORIGIN_HEADERS} + + if stream: + response.close() + + current_url = next_url + + raise TooManyRedirectsError(url, self._max_redirects) + + def _get_client(self, proxy_url: str | None) -> AsyncClient: """Retrieve or create an HTTP client for the given proxy URL. If a client for the specified proxy URL does not exist, create and store a new one. """ - cached_data = self._client_by_proxy_url.get(proxy_url) - if cached_data: - client = cached_data['client'] - client_cookie_jar = cached_data['cookie_jar'] - if client_cookie_jar is cookie_jar: - # If the cookie jar matches, return the existing client. - return client + if client := self._client_by_proxy_url.get(proxy_url): + return client # Prepare a default kwargs for the new client. kwargs: dict[str, Any] = { 'proxy': proxy_url, 'http3': self._http3, 'verify': self._verify, - 'follow_redirects': True, 'browser': self._browser, } # Update the default kwargs with any additional user-provided kwargs. kwargs.update(self._async_client_kwargs) - client = AsyncClient(**kwargs, cookie_jar=cookie_jar) + # Redirects are followed hop by hop by this client. + client = AsyncClient(**kwargs, follow_redirects=False) - self._client_by_proxy_url[proxy_url] = _ClientCacheEntry(client=client, cookie_jar=cookie_jar) + self._client_by_proxy_url[proxy_url] = client return client diff --git a/src/crawlee/sessions/_cookies.py b/src/crawlee/sessions/_cookies.py index c0ea252f7b..fa406924b3 100644 --- a/src/crawlee/sessions/_cookies.py +++ b/src/crawlee/sessions/_cookies.py @@ -1,8 +1,10 @@ from __future__ import annotations from copy import deepcopy +from email.message import Message from http.cookiejar import Cookie, CookieJar from typing import TYPE_CHECKING, Any, Literal +from urllib.request import Request as UrlRequest from typing_extensions import NotRequired, Required, TypedDict @@ -13,6 +15,18 @@ from typing import TypeGuard +class _SetCookieResponse: + """Minimal response adapter exposing `Set-Cookie` headers to `CookieJar.extract_cookies`.""" + + def __init__(self, set_cookie_headers: list[str]) -> None: + self._message = Message() + for header in set_cookie_headers: + self._message['Set-Cookie'] = header + + def info(self) -> Message: + return self._message + + @docs_group('Session management') class CookieParam(TypedDict, total=False): """Dictionary representation of cookies for `SessionCookies.set` method.""" @@ -217,6 +231,36 @@ def set_cookies(self, cookie_dicts: list[CookieParam]) -> None: self.set(**cookie_dict) self._jar.clear_expired_cookies() + def get_cookie_string(self, url: str) -> str: + """Build the value of the `Cookie` header for the given URL. + + Only cookies matching the domain, path and security requirements of the URL are included. + + Args: + url: The URL the header is built for. + + Returns: + The `Cookie` header value, or an empty string if no stored cookie matches the URL. + """ + # `UrlRequest` is only used as a carrier of the URL and headers for the jar, it never opens a connection. + url_request = UrlRequest(url) # noqa: S310 + self._jar.add_cookie_header(url_request) + return url_request.get_header('Cookie', '') + + def extract_cookie_from_header(self, url: str, set_cookie_headers: list[str]) -> None: + """Store cookies from the raw `Set-Cookie` headers of a response. + + Attributes omitted from a header, such as domain and path, are derived from the URL. It must therefore be + the URL that produced the given headers, not the URL the request started from. + + Args: + url: The URL of the response carrying the headers. + set_cookie_headers: Raw values of the `Set-Cookie` response headers. + """ + response = _SetCookieResponse(set_cookie_headers) + self._jar.extract_cookies(response, UrlRequest(url)) # noqa: S310 # ty: ignore[invalid-argument-type] + self._jar.clear_expired_cookies() + def get_cookies_as_playwright_format(self) -> list[PlaywrightCookieParam]: """Get cookies in playwright format.""" return [self._to_playwright(cookie) for cookie in self.get_cookies_as_dicts()] diff --git a/tests/unit/http_clients/test_impit.py b/tests/unit/http_clients/test_impit.py new file mode 100644 index 0000000000..61fe6e2b47 --- /dev/null +++ b/tests/unit/http_clients/test_impit.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +import pytest + +from crawlee.errors import TooManyRedirectsError +from crawlee.http_clients import ImpitHttpClient +from crawlee.sessions import Session + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + + from yarl import URL + + from crawlee._types import HttpMethod + from crawlee.http_clients import HttpResponse + + +@pytest.fixture +async def http_client() -> AsyncGenerator[ImpitHttpClient]: + client = ImpitHttpClient() + async with client: + yield client + + +async def read_json(response: HttpResponse) -> dict: + """Read the body of an HTTP response and decode it as JSON.""" + return json.loads((await response.read()).decode()) + + +async def test_sessions_share_one_client(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that requests of different sessions are served by a single underlying client.""" + for _ in range(3): + await http_client.send_request(str(server_url / 'cookies'), session=Session()) + + assert len(http_client._client_by_proxy_url) == 1 + + +async def test_cookies_are_kept_apart_per_session(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that sessions sharing a client don't see cookies of each other.""" + first_session = Session() + second_session = Session() + + await http_client.send_request(str((server_url / 'set_cookies').with_query(a='1')), session=first_session) + await http_client.send_request(str((server_url / 'set_cookies').with_query(b='2')), session=second_session) + + assert {item['name'] for item in first_session.cookies.get_cookies_as_dicts()} == {'a'} + assert {item['name'] for item in second_session.cookies.get_cookies_as_dicts()} == {'b'} + + first_response = await http_client.send_request(str(server_url / 'cookies'), session=first_session) + second_response = await http_client.send_request(str(server_url / 'cookies'), session=second_session) + + assert (await read_json(first_response))['cookies'] == {'a': '1'} + assert (await read_json(second_response))['cookies'] == {'b': '2'} + + +async def test_cookies_are_collected_on_every_hop(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that a cookie set by a redirecting response is sent on the following hop.""" + session = Session() + + response = await http_client.send_request( + str((server_url / 'set_cookies').with_query(a='1')), + session=session, + ) + + assert (await read_json(response))['cookies'] == {'a': '1'} + + +async def test_cookies_are_not_stored_without_persistence(server_url: URL) -> None: + """Test that `persist_cookies_per_session` keeps the session jar untouched.""" + session = Session() + + async with ImpitHttpClient(persist_cookies_per_session=False) as client: + await client.send_request(str((server_url / 'set_cookies').with_query(a='1')), session=session) + + assert session.cookies.get_cookies_as_dicts() == [] + + +@pytest.mark.parametrize( + ('status_code', 'method', 'expected_method', 'expected_body'), + [ + pytest.param(301, 'POST', 'GET', '', id='301-post'), + pytest.param(301, 'PUT', 'PUT', 'payload', id='301-put'), + pytest.param(302, 'POST', 'GET', '', id='302-post'), + pytest.param(302, 'PUT', 'PUT', 'payload', id='302-put'), + pytest.param(303, 'POST', 'GET', '', id='303-post'), + pytest.param(303, 'PUT', 'GET', '', id='303-put'), + pytest.param(307, 'POST', 'POST', 'payload', id='307-post'), + pytest.param(308, 'PUT', 'PUT', 'payload', id='308-put'), + ], +) +async def test_redirect_method_follows_fetch_algorithm( + http_client: ImpitHttpClient, + server_url: URL, + *, + status_code: int, + method: HttpMethod, + expected_method: str, + expected_body: str, +) -> None: + """Test that the method and the body of a redirected request follow the WHATWG Fetch algorithm.""" + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'method'), status=status_code) + + response = await http_client.send_request( + str(redirect_url), + method=method, + payload=b'payload', + headers={'content-type': 'application/octet-stream'}, + ) + echo = await read_json(response) + + assert echo['method'] == expected_method + assert echo['body'] == expected_body + + +async def test_body_headers_dropped_when_method_changes(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that headers describing the request body are dropped once a redirect turns the request into a `GET`.""" + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'headers'), status=302) + + response = await http_client.send_request( + str(redirect_url), + method='POST', + payload=b'payload', + headers={'content-type': 'application/json', 'content-language': 'uk', 'x-custom': 'kept'}, + ) + headers = await read_json(response) + + assert 'content-type' not in headers + assert 'content-language' not in headers + assert headers['x-custom'] == 'kept' + + +async def test_authorization_kept_within_origin(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that credentials survive a redirect that stays on the same origin.""" + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'headers'), status=302) + + response = await http_client.send_request(str(redirect_url), headers={'authorization': 'Bearer token'}) + headers = await read_json(response) + + assert headers['authorization'] == 'Bearer token' + + +async def test_authorization_dropped_across_origins( + http_client: ImpitHttpClient, + server_url: URL, + redirect_server_url: URL, +) -> None: + """Test that credentials are dropped as soon as a redirect leaves the origin.""" + redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'headers'), status=302) + + response = await http_client.send_request( + str(redirect_url), + headers={'authorization': 'Bearer token', 'x-custom': 'kept'}, + ) + headers = await read_json(response) + + assert 'authorization' not in headers + assert headers['x-custom'] == 'kept' + + +async def test_explicit_cookie_header_kept_within_origin(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that a `Cookie` header set by the caller survives a redirect within the origin.""" + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'cookies'), status=302) + + response = await http_client.send_request(str(redirect_url), headers={'cookie': 'manual=value'}) + + assert (await read_json(response))['cookies'] == {'manual': 'value'} + + +async def test_session_cookies_replace_explicit_cookie_header(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that cookies of a session take over the `Cookie` header instead of being sent next to it.""" + session = Session() + session.cookies.set('from_jar', '1', domain=server_url.host or '') + + response = await http_client.send_request( + str(server_url / 'cookies'), + session=session, + headers={'cookie': 'manual=value'}, + ) + + assert (await read_json(response))['cookies'] == {'from_jar': '1'} + + +async def test_explicit_cookie_header_dropped_across_origins( + http_client: ImpitHttpClient, + server_url: URL, + redirect_server_url: URL, +) -> None: + """Test that a `Cookie` header set by the caller is dropped once a redirect leaves the origin. + + Cookies stored in a session jar are not affected, as they are matched by domain on every hop. Ports give no + isolation of their own, see RFC 6265, section 8.5. + """ + redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'cookies'), status=302) + + response = await http_client.send_request(str(redirect_url), headers={'cookie': 'manual=value'}) + + assert (await read_json(response))['cookies'] == {} + + +async def test_too_many_redirects(server_url: URL) -> None: + """Test that an endless redirect chain is cut off by `max_redirects`.""" + async with ImpitHttpClient(max_redirects=2) as client: + with pytest.raises(TooManyRedirectsError) as exc_info: + await client.send_request(str(server_url / 'redirect_loop')) + + assert exc_info.value.max_redirects == 2 + + +async def test_stream_follows_redirects(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that streamed requests follow redirects and carry session cookies along.""" + session = Session() + stream_url = (server_url / 'set_cookies').with_query(a='1') + + async with http_client.stream(str(stream_url), session=session) as response: + content = b'' + async for chunk in response.read_stream(): + content += chunk + + assert json.loads(content.decode())['cookies'] == {'a': '1'} + assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'a'} diff --git a/tests/unit/server.py b/tests/unit/server.py index 1bccab6308..f12a553287 100644 --- a/tests/unit/server.py +++ b/tests/unit/server.py @@ -126,6 +126,8 @@ async def app(scope: dict[str, Any], receive: Receive, send: Send) -> None: 'get': get_echo, 'post': post_echo, 'redirect': redirect_to_url, + 'redirect_loop': redirect_loop, + 'method': echo_method, 'json': hello_world_json, 'xml': hello_world_xml, 'robots.txt': robots_txt, @@ -270,6 +272,35 @@ async def echo_headers(scope: dict[str, Any], _receive: Receive, send: Send) -> await send_json_response(send, headers) +async def echo_method(scope: dict[str, Any], receive: Receive, send: Send) -> None: + """Echo back the method and the body of the request.""" + body = b'' + more_body = True + + while more_body: + message = await receive() + if message['type'] == 'http.request': + body += message.get('body', b'') + more_body = message.get('more_body', False) + + await send_json_response(send, {'method': scope['method'], 'body': body.decode()}) + + +async def redirect_loop(scope: dict[str, Any], _receive: Receive, send: Send) -> None: + """Handle requests that endlessly redirect back to the same endpoint.""" + await send( + { + 'type': 'http.response.start', + 'status': 302, + 'headers': [ + [b'content-type', b'text/plain; charset=utf-8'], + [b'location', str(scope['path']).encode()], + ], + } + ) + await send({'type': 'http.response.body', 'body': b'Redirecting...'}) + + async def start_enqueue_endpoint(_scope: dict[str, Any], _receive: Receive, send: Send) -> None: """Handle requests for the main page with links.""" await send_html_response( diff --git a/tests/unit/sessions/test_cookies.py b/tests/unit/sessions/test_cookies.py index 0c6ab4b965..0c5cd5db43 100644 --- a/tests/unit/sessions/test_cookies.py +++ b/tests/unit/sessions/test_cookies.py @@ -146,3 +146,88 @@ def test_store_multidomain_cookies() -> None: assert check_cookies['test.io'] == ('a', '1') assert check_cookies['notest.io'] == ('a', '2') + + +def test_extract_cookie_from_header() -> None: + """Test that cookies are parsed from raw `Set-Cookie` headers with attributes taken from the URL.""" + session_cookies = SessionCookies() + session_cookies.extract_cookie_from_header( + 'https://example.com/login', + [ + 'sid=abc123; Path=/; Secure; HttpOnly; SameSite=Lax', + 'theme=dark; Domain=example.com; Path=/settings', + ], + ) + cookies = {item['name']: item for item in session_cookies.get_cookies_as_dicts()} + + assert cookies['sid']['domain'] == 'example.com' + assert cookies['sid']['path'] == '/' + assert cookies['sid']['secure'] + assert cookies['sid']['http_only'] + assert cookies['sid']['same_site'] == 'Lax' + + assert cookies['theme']['domain'] == '.example.com' + assert cookies['theme']['path'] == '/settings' + assert not cookies['theme']['secure'] + + +def test_extract_cookie_from_header_uses_url_for_defaults() -> None: + """Test that the domain of a cookie without the `Domain` attribute comes from the URL of the response.""" + session_cookies = SessionCookies() + session_cookies.extract_cookie_from_header('https://first.example.com/', ['a=1; Path=/']) + session_cookies.extract_cookie_from_header('https://second.example.com/', ['b=2; Path=/']) + domains = {item['name']: item.get('domain') for item in session_cookies.get_cookies_as_dicts()} + + assert domains == {'a': 'first.example.com', 'b': 'second.example.com'} + + +def test_extract_cookie_from_header_with_empty_headers() -> None: + """Test that a response without `Set-Cookie` headers leaves the jar untouched.""" + session_cookies = SessionCookies() + session_cookies.set('existing', 'value', domain='example.com') + session_cookies.extract_cookie_from_header('https://example.com/', []) + + assert len(session_cookies) == 1 + + +def test_get_cookie_string() -> None: + """Test that the `Cookie` header contains all cookies matching the URL.""" + session_cookies = SessionCookies() + session_cookies.set('a', '1', domain='example.com') + session_cookies.set('b', '2', domain='example.com') + + cookie_string = session_cookies.get_cookie_string('https://example.com/page') + + assert set(cookie_string.split('; ')) == {'a=1', 'b=2'} + + +def test_get_cookie_string_respects_domain_path_and_secure() -> None: + """Test that cookies not matching the URL are left out of the `Cookie` header.""" + session_cookies = SessionCookies() + session_cookies.set('same_domain', '1', domain='example.com') + session_cookies.set('other_domain', '2', domain='other.com') + session_cookies.set('other_path', '3', domain='example.com', path='/admin') + session_cookies.set('secure_only', '4', domain='example.com', secure=True) + + assert session_cookies.get_cookie_string('http://example.com/page') == 'same_domain=1' + assert set(session_cookies.get_cookie_string('https://example.com/admin').split('; ')) == { + 'same_domain=1', + 'other_path=3', + 'secure_only=4', + } + + +def test_get_cookie_string_without_matching_cookies() -> None: + """Test that an empty string is returned when no cookie matches the URL.""" + session_cookies = SessionCookies() + session_cookies.set('a', '1', domain='example.com') + + assert session_cookies.get_cookie_string('https://other.com/') == '' + + +def test_get_cookie_string_round_trip() -> None: + """Test that cookies extracted from a response are sent back to the same URL.""" + session_cookies = SessionCookies() + session_cookies.extract_cookie_from_header('https://example.com/login', ['sid=abc123; Path=/']) + + assert session_cookies.get_cookie_string('https://example.com/dashboard') == 'sid=abc123' diff --git a/uv.lock b/uv.lock index 2719ff3791..a1c0742f75 100644 --- a/uv.lock +++ b/uv.lock @@ -946,7 +946,7 @@ requires-dist = [ { name = "curl-cffi", marker = "extra == 'curl-impersonate'", specifier = ">=0.9.0" }, { name = "html5lib", marker = "extra == 'beautifulsoup'", specifier = ">=1.0" }, { name = "httpx", extras = ["brotli", "http2", "zstd"], marker = "extra == 'httpx'", specifier = ">=0.27.0" }, - { name = "impit", specifier = ">=0.8.0" }, + { name = "impit", specifier = ">=0.13.2" }, { name = "inquirer", marker = "extra == 'cli'", specifier = ">=3.3.0" }, { name = "jaro-winkler", marker = "extra == 'adaptive-crawler'", specifier = ">=2.0.3" }, { name = "lxml", extras = ["html-clean"], marker = "extra == 'pydantic-ai'", specifier = ">=5.2.0" }, @@ -1745,51 +1745,51 @@ wheels = [ [[package]] name = "impit" -version = "0.13.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/49/916205ecf0649269627e8954e9fd13ba3688b82455c90ff0e3abf2389ddb/impit-0.13.1.tar.gz", hash = "sha256:ad47c4be3760d4e4e10dc27c0dc9e4c7129fa05b4ee69d55e56f788c0ab42832", size = 159671, upload-time = "2026-06-25T08:56:30.894Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/89/688691da90f9abf6e0f425aa7b2e81d65ee0f5d99233aadd6a93ec3d6673/impit-0.13.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:641bac14302ec4102934c9263b70072c26531fe22fd20d5edc45ac5a41fb3f4b", size = 4006813, upload-time = "2026-06-25T08:55:25.425Z" }, - { url = "https://files.pythonhosted.org/packages/17/24/4392640b4c9f91e76929a2333c1a0d9ce9bda1b243da1816b57117c3177a/impit-0.13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4eebd2a41ffab78dc72cf487d2d7205183ee34db841e97ae6d1a896fbc153485", size = 3867081, upload-time = "2026-06-25T08:55:27.149Z" }, - { url = "https://files.pythonhosted.org/packages/28/f8/ed3230b3acdea73d9b90a6fb7ff5c0df69e862f03c58a0e42ce79c194e52/impit-0.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03cc0f9dd5fe9d45dacb77932ecca74b151bd188e24c982629bcac396789ebd7", size = 4234837, upload-time = "2026-06-25T08:55:28.478Z" }, - { url = "https://files.pythonhosted.org/packages/8c/e2/6e651dae576f85336b21b989ddfc214f16bb16ab054d38261f0c79a8f9ed/impit-0.13.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:659eb6e9cbbe9484b2ca63837df9d5c69deefa1f30b4a9a19d57d9bf3f271911", size = 4096430, upload-time = "2026-06-25T08:55:29.77Z" }, - { url = "https://files.pythonhosted.org/packages/30/a3/b0a0dffa5599007e841c7ed87f80f0f3ea894708d221a6d18e1ee28b0d69/impit-0.13.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:df3adf6ef77e468ce45062f330f09a1f5c9e534cbdca63d6b6e7d3318cfbccc4", size = 4305849, upload-time = "2026-06-25T08:55:31.57Z" }, - { url = "https://files.pythonhosted.org/packages/28/f7/8773af6fa13714a107459fe300f2d71db243623129054a95c3d9cbf49583/impit-0.13.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b27794af0886fed5b4f75c707276ca2c0a184ae6ba6bc1ab23a4b1e1eee2b06f", size = 4468023, upload-time = "2026-06-25T08:55:33.432Z" }, - { url = "https://files.pythonhosted.org/packages/5e/7e/8e149d4410b2aabf7c77f5046f65514d55870bdeafdd3125437748ceac84/impit-0.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:1aa976c1fda3f8e8341ba266e5716206e4a52f99de552880fe6c26565936951e", size = 3921503, upload-time = "2026-06-25T08:55:35.428Z" }, - { url = "https://files.pythonhosted.org/packages/96/ab/96c06866ddd1033f6c5c6903184ce8a76119953a562791af680974d306ed/impit-0.13.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f6decb19b0eca3820aedb52040d523d7f596011b6b71a4cb3268202f7082636e", size = 4006590, upload-time = "2026-06-25T08:55:37.047Z" }, - { url = "https://files.pythonhosted.org/packages/ab/93/e0f918b916f4179c3536d1692d1320b74ed9629a5c4d05da855bc77c73d7/impit-0.13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:07f995f6b00ea3c877c801c1aaca0693d369a41eb5cc8495a3da88d30e67ba6f", size = 3867310, upload-time = "2026-06-25T08:55:38.603Z" }, - { url = "https://files.pythonhosted.org/packages/97/49/f6296236d730363b1d2865df9aedfeb85366a7e327f6784130af852a1c0f/impit-0.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ca3625ecceb94cc06cbeb3eb30169da1b430854ae26b0edd5f465eca66f00c", size = 4234606, upload-time = "2026-06-25T08:55:40.079Z" }, - { url = "https://files.pythonhosted.org/packages/44/36/d913d6078eeaa9500a01917373b7a4fc8055d5f8dbbc1f7c248282779ffc/impit-0.13.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5b80d6967448dbf06cb84c608c835c601f73f6f1cfe57bc49d02fb59692ce47e", size = 4096268, upload-time = "2026-06-25T08:55:41.758Z" }, - { url = "https://files.pythonhosted.org/packages/29/c3/01ff30c22df660014c7f7db2c0791b6aa6ffa05a7f122a9e1ec6e93b6818/impit-0.13.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a3275851d6155ba2a631c2024703879b0cc6a0d6ac7fa949ffb6eae319cfe8cc", size = 4305658, upload-time = "2026-06-25T08:55:43.141Z" }, - { url = "https://files.pythonhosted.org/packages/e3/f6/b3c43d14a54b3edccb4485063a7465bb256d734e917286327e796702d93d/impit-0.13.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6360bc416b41022eb6ee1a8d64b435d9a8acfd92e6261883188595ae221b4833", size = 4467719, upload-time = "2026-06-25T08:55:44.513Z" }, - { url = "https://files.pythonhosted.org/packages/ae/57/2371060df14e2cc08b373b5280492039b30ab2f0a25225b3b3943d8f9d7f/impit-0.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:931b272a25a87bc5553896bd203ecf648e1837f5300cc45dc1145be699ae3cd7", size = 3921452, upload-time = "2026-06-25T08:55:45.898Z" }, - { url = "https://files.pythonhosted.org/packages/ca/1d/fe7303a7bd2212c862acb30c8f52893751c154559ed59e994844da886246/impit-0.13.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:5f6400a3f4196d0d1d072b56bfde76ea8b5145b07c6e0103a3f6b7393609c608", size = 4006193, upload-time = "2026-06-25T08:55:47.291Z" }, - { url = "https://files.pythonhosted.org/packages/44/9d/1d8e04ccd73953cd89f05161863578542991b52e7570c5998fb2e83230d9/impit-0.13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:57e651edbeaf9023a5b0721f29a3a3cb8ced9ebafeab87dcb3fe23ef9f07786d", size = 3867591, upload-time = "2026-06-25T08:55:48.866Z" }, - { url = "https://files.pythonhosted.org/packages/74/82/7377f8ce20e2c1a9e06e640be88269cdea6e63b18a3fdf79592f7aa3d513/impit-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:866e686f51c2b2fe358ddcaf609b3d73521e66f10c1c6fc4131f7ee4f9494951", size = 4231176, upload-time = "2026-06-25T08:55:50.457Z" }, - { url = "https://files.pythonhosted.org/packages/37/41/9081ef526d1f0c6edb926dd4c49b4331e5fba290f51a5076d226ac616aca/impit-0.13.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:09a1c3cab2fc6e71592c0a3943c4199e7b083191ccef038bb09f33d8d32065c8", size = 4094077, upload-time = "2026-06-25T08:55:51.889Z" }, - { url = "https://files.pythonhosted.org/packages/5d/21/26aed5ea6c4a4af81aa4c6ced69507440c683fc57fd1e1dc6def64154604/impit-0.13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:abd759e41ac9ccfc7def35cfff02fcae22f1ce9a1ba9e65573d400ee1541aa15", size = 4303899, upload-time = "2026-06-25T08:55:53.541Z" }, - { url = "https://files.pythonhosted.org/packages/54/38/b091e6bfed5cc4bfa2dd950d2b2448102a0ef1e3655fda6f29f4c70c27c7/impit-0.13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:de2f77cff033244804cfd59557b6f510790728d0d067b6093dcb80c7289f5ae1", size = 4465412, upload-time = "2026-06-25T08:55:55.067Z" }, - { url = "https://files.pythonhosted.org/packages/13/f8/138bfdd2861adda2d8bc4289752c48c45374bfdd5c3995eb510c03b9605e/impit-0.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:585d666608841de008a0a85a53da8e3d2d4324b2c32a2a0371ed22ddf6234970", size = 3919717, upload-time = "2026-06-25T08:55:56.577Z" }, - { url = "https://files.pythonhosted.org/packages/64/e5/55d2219cd316de87ec98736ac34a3c76b181e962e3946bfc1a91d1913a46/impit-0.13.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3529e1bc181c1107e47c3d4821e97c0c597b8c71735ee371a9526e3818e2ee72", size = 4006142, upload-time = "2026-06-25T08:55:58.075Z" }, - { url = "https://files.pythonhosted.org/packages/08/8e/0af35cfeb1d7852028b02bec25adef02f70dcc49b1fab4f75e7b6fe2d94a/impit-0.13.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5271202c572d2e4fc1b1a6155237dab028c42ada154a07b5fb304f65fee8581b", size = 3867425, upload-time = "2026-06-25T08:55:59.568Z" }, - { url = "https://files.pythonhosted.org/packages/a8/d5/49993e1db1bd6653a1a17e999eef6ac9243ff3724d287cacf449bec53b1e/impit-0.13.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10eedaa7900e7b7b620b6bde2a6f30ad4dc44dc908f57792ca248d32653243a8", size = 4231133, upload-time = "2026-06-25T08:56:01.313Z" }, - { url = "https://files.pythonhosted.org/packages/bd/92/8861af93f6a1ef02d2ff90446e13c9ecdef05f8185eee00f24fe988bff22/impit-0.13.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7e9e11a3bfb2467d92ffc49be5bdf3b18fc63cb3721c5bea3d5bdfec97c64137", size = 4094091, upload-time = "2026-06-25T08:56:02.777Z" }, - { url = "https://files.pythonhosted.org/packages/87/cf/2cff1e60319089d7b3fdfa711ccb924974bea8b0283e54b71837c093e0a5/impit-0.13.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3e2e2e3a4482c2977dd2c0b9934e40f0a62086555706815fb619cf99caa19721", size = 4304029, upload-time = "2026-06-25T08:56:04.352Z" }, - { url = "https://files.pythonhosted.org/packages/29/6a/86e3b4283dd2d07a1149868a11562437789c493be230b2c16249d4bbab7a/impit-0.13.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1a209fd691068f8d76217c58f5ae4e122de3d98f7896d96c0847cc8c8d639a96", size = 4465429, upload-time = "2026-06-25T08:56:05.749Z" }, - { url = "https://files.pythonhosted.org/packages/8d/da/a7193cc49d5fe0a8e9eff6a0cf6411e36c116234a4cf2c590c5bc0fd73df/impit-0.13.1-cp313-cp313-win_amd64.whl", hash = "sha256:794a16fd12ccdf75a294d59211b7835f5828025808c8336c9bafe86174cdb203", size = 3919877, upload-time = "2026-06-25T08:56:07.22Z" }, - { url = "https://files.pythonhosted.org/packages/3c/48/38e5cd614b59d38ca7165f0ede4c85c6a21e8342a4546cc163c2db9bb4bd/impit-0.13.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:6096c9605ac5603b0966e7900d09e9b90c768d9c96a48b7e7aa3290698b11523", size = 4006807, upload-time = "2026-06-25T08:56:08.732Z" }, - { url = "https://files.pythonhosted.org/packages/6c/d1/22194bb577ef615a23bf63beb88a75aab507bac03797dcb66a25e69265e6/impit-0.13.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e475c80399cff9e88718b6c8ab5687e0bc91118f1388162c29b34c25d2b9b1dc", size = 3868213, upload-time = "2026-06-25T08:56:10.178Z" }, - { url = "https://files.pythonhosted.org/packages/fe/d1/fe93c04c85042c07d4e9be7e5762c7fe5047021f2053355058f90349af1b/impit-0.13.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4aa3a0a2203f10e455d62628443100ed6c3f01fd08af1498e6ee7f433cba18e", size = 4232100, upload-time = "2026-06-25T08:56:11.956Z" }, - { url = "https://files.pythonhosted.org/packages/10/59/b95ebe285560797587523de719db63ef9bed84fc757e535bc2624934d8f1/impit-0.13.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:6d6615a397fa4fdd538d2e6d0b21af7d4f757bc24eaab8bc84483239dc269da7", size = 4095235, upload-time = "2026-06-25T08:56:14.017Z" }, - { url = "https://files.pythonhosted.org/packages/5e/7c/93fa966b74f1ce700ec7098822f739326add361e194ab8f6cbb62ae8e036/impit-0.13.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:930b42beb39e4d95c28fc9ddadf6cc4e699b3c01aaf9741fb0e8c6691630875d", size = 4304402, upload-time = "2026-06-25T08:56:15.997Z" }, - { url = "https://files.pythonhosted.org/packages/f7/97/1fe6c14a1ccc6aac5ff0b27ab86337e57c6334050e8f5733746977ffb1ac/impit-0.13.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a83e74ab95ec78070d3d3ebf2ece8c5e50b1535845ccea9f2ac7999b4b14dbf2", size = 4466571, upload-time = "2026-06-25T08:56:17.974Z" }, - { url = "https://files.pythonhosted.org/packages/72/03/3bba97808c241f1c6a6c53a4e48ea20e20172e79170fd8f66a9bf6953487/impit-0.13.1-cp314-cp314-win_amd64.whl", hash = "sha256:7054d5c65018b1400ab57eb1685dd5b985f03be2148be99c43633d40e8288714", size = 3920249, upload-time = "2026-06-25T08:56:19.342Z" }, - { url = "https://files.pythonhosted.org/packages/16/bd/7e2e3cc22aacc7a4257f29d7c58d0facda4abd200412e7b85f5e1ae2954e/impit-0.13.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:b8aff2c2a9d011cf608f60d82c7bcc45fc7b321f9bd6d0283ed8642ce95d9add", size = 4007087, upload-time = "2026-06-25T08:56:20.897Z" }, - { url = "https://files.pythonhosted.org/packages/4e/99/fea2db4ef00d9f7cc5abd5d68517fc468469cb7fb6093746f32edabf1c92/impit-0.13.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2cff1617a2530e820809a72c6f131c18343b7b9d34e6a429a737320e5bc5ce5e", size = 3867849, upload-time = "2026-06-25T08:56:22.582Z" }, - { url = "https://files.pythonhosted.org/packages/77/dc/c31a332549ab21c8ffb53bfdd2e0591a64c59f832e7b928ec51dad65418b/impit-0.13.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f1c90228bbb8b98abb0d339c94bf3bd16617e13325527bcb94afdad2537c87d", size = 4232744, upload-time = "2026-06-25T08:56:23.954Z" }, - { url = "https://files.pythonhosted.org/packages/e5/19/fc8f5b3121cee3d1d2ee227075ec8d69f2c0d323612062e593f6e877841c/impit-0.13.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:3c47c3a91fe9e7ac091536751b1dcad4b8060956f1a72c1b0c5ce772a0fb0c8d", size = 4094777, upload-time = "2026-06-25T08:56:26.024Z" }, - { url = "https://files.pythonhosted.org/packages/9c/54/224d6c44b2c7fcde51355ca60cb27acc20c787d2a0e2c1154aff35c9551f/impit-0.13.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2d5223f8ec2d0fa633c7b9a3776c5cdc11d6a59b8ffc98ab1e09d6bb38517e0", size = 4304423, upload-time = "2026-06-25T08:56:27.535Z" }, - { url = "https://files.pythonhosted.org/packages/90/ab/d378b7e44e0ef525dee96e65e1cab55b709bb567bf2a7fc397e2f37e30ea/impit-0.13.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e0bfa83d16f6e6b2c8475a261d4b8970c6e5819c993d1db41b8bc99dc92e08fc", size = 4467056, upload-time = "2026-06-25T08:56:29.033Z" }, +version = "0.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/6c/199de798f0d8a0d2b4f723001630eb85071f3ca8e895ceb25b305eac2eac/impit-0.13.2.tar.gz", hash = "sha256:5d71d0e2b55c9c267f99edf36e0fe6a6dff83f992f06d7abbb80095c342deeb1", size = 163512, upload-time = "2026-08-05T10:38:35.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/95/c51ab2d1a0a30dc507a883a803b4d2254d60becdeda3cbbaaf8a9b2b2d38/impit-0.13.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a17b7d87cff4dd602306147aa0acf717aafd128ecebdfdda9dc84545f948fb6c", size = 4011136, upload-time = "2026-08-05T10:37:17.067Z" }, + { url = "https://files.pythonhosted.org/packages/fb/40/e089714584c964d3d7996b0b9e14d130c1c8db12886f35670db8691dfe6f/impit-0.13.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2ff09608aa2fa15680787ecf8ff8c145bb2d41201c9f3412b6f8c2a846fc2ad", size = 3872332, upload-time = "2026-08-05T10:37:19.796Z" }, + { url = "https://files.pythonhosted.org/packages/8b/93/7674b319dea6b38e0838520ece61d95d9c18725157241cdeeb4de422d19f/impit-0.13.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9688ce348d8fbb5fdeaaa4a51fda4456ae8ec4bdef61daa09a9492b904d77fc", size = 4235365, upload-time = "2026-08-05T10:37:21.606Z" }, + { url = "https://files.pythonhosted.org/packages/c2/74/6dd6dd237af0e43f3dee8215897592f1e6c4d91d1452b23c405ef1fe3e19/impit-0.13.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3be04890a65876c58bb5008f7d63e8b1ba9600a186f32865073b2bc565da64e0", size = 4093093, upload-time = "2026-08-05T10:37:23.442Z" }, + { url = "https://files.pythonhosted.org/packages/a9/5b/bbc185892835d1a1d70b041853f22ab30540e1125f5e9a80a9334cfc86ac/impit-0.13.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8da4bd265b83ec38d7d3deafe889288fc8462b03e740c7a6d4654e9b51526cb1", size = 4306777, upload-time = "2026-08-05T10:37:25.598Z" }, + { url = "https://files.pythonhosted.org/packages/47/9c/c952ffe59be1e503635effb87cbc99ddc63709ce3097abc1dcf1d37429b5/impit-0.13.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5d3eb43b882d3e91fb5d0cdf8439889f5892e262aaa75fb3ec3d391ff0f9986a", size = 4464757, upload-time = "2026-08-05T10:37:27.255Z" }, + { url = "https://files.pythonhosted.org/packages/67/06/0ba3aa867072ee40948e0fd8259adccf91a7eac767ec7cd12f0217fa444b/impit-0.13.2-cp310-cp310-win_amd64.whl", hash = "sha256:d3c9214ba9e1bc6835c5facf28d7ee444ad89eb2b6bc86d71975b5b153b42e80", size = 3924979, upload-time = "2026-08-05T10:37:29.861Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a0/19883f06cf55b74a2363b0180d50a6a28aef1b686070b32fbcf65ce387c2/impit-0.13.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:57a1e2d13b6397b33ee1eff2b74cc58c1b7768062c1ad2648c247565b2ddbe14", size = 4010732, upload-time = "2026-08-05T10:37:32.354Z" }, + { url = "https://files.pythonhosted.org/packages/08/20/33985c1cf2e0d99de4443e1c006920cb694145fba897d6ad4bc7e0cb7439/impit-0.13.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a34f2f9acee413b53184107fde4dd840e39b6080b93da86beda68a4507bcc190", size = 3872014, upload-time = "2026-08-05T10:37:34.742Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a2/8fe3b8693ad25eef2f2e107b14ce6380f76f31ec9a4e00d59661e20c5c18/impit-0.13.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbef4379b0aa17e47cff4866562bbdafff9097eb1c873a91ed4abb7be4209286", size = 4235120, upload-time = "2026-08-05T10:37:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/1a/48/cbdc6d856955032412d0f2696913116a81080da6d79e34b189f810cb45eb/impit-0.13.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:ed8a27f13a21c2922e4d2c6842d30c95d757164c6574f0ae11f9073d810c4c8d", size = 4092893, upload-time = "2026-08-05T10:37:38.208Z" }, + { url = "https://files.pythonhosted.org/packages/a7/45/a06f1b2443a4f703666109ce6ba8bdf677e349b008de42dbc99b85eb8d71/impit-0.13.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c5d03de3ea716187ccd8fb08e70a214b05b03f0c71dba0af1fe4b4b45e2e0007", size = 4306478, upload-time = "2026-08-05T10:37:40.006Z" }, + { url = "https://files.pythonhosted.org/packages/eb/76/99d872ebd4c8e10fa9634c007370be6c2649a27bf5bd3532f657c8abed66/impit-0.13.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5247a3f5fdb2084fab3522538d492e6e0faf069a391ceba87d5f7f2f0c4f9929", size = 4463944, upload-time = "2026-08-05T10:37:41.705Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5f/dc0ad11ca4e0b3641505eec177abc4b47b1de77a9751bb0f436aa252f49b/impit-0.13.2-cp311-cp311-win_amd64.whl", hash = "sha256:790406c9855263e45926711c0a4442c301caae3c907261b64480292b3a5f9484", size = 3924400, upload-time = "2026-08-05T10:37:43.626Z" }, + { url = "https://files.pythonhosted.org/packages/fb/85/7c9ccb59aa71a14f765bee77d831e46dd85d0418222c79603e8d9624b61f/impit-0.13.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:32935bceda2751fbcce8b221892ac2d628f3f960b4753de0ab9d01a0de39de5b", size = 4010069, upload-time = "2026-08-05T10:37:45.29Z" }, + { url = "https://files.pythonhosted.org/packages/e5/6d/702ed813b22d4468e5d6cf1e7aa715ea24d663cd5bfeca017c4fe782ff8c/impit-0.13.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5b368e61718fab972f67ea591a79629e3baa46b8269f6bef27c7d1434b11c3af", size = 3872117, upload-time = "2026-08-05T10:37:47.075Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fd/af40bc51fc21e2e11bae695d6836ccb37ebe3ec5d710a1e87c2f1b225764/impit-0.13.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3ac01f91f0085638f89cec234fa517178a6f881229b74a32fb709a45d9bc158", size = 4232424, upload-time = "2026-08-05T10:37:48.989Z" }, + { url = "https://files.pythonhosted.org/packages/44/9e/9d6954f3c2837616f57e37fb6117defe5badb2e9570bd536623ca3b94f91/impit-0.13.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:08d7007bc81530ca36d4ff462356540cde9c145099b1eb9efdf0ae7ae7c15330", size = 4087989, upload-time = "2026-08-05T10:37:50.849Z" }, + { url = "https://files.pythonhosted.org/packages/92/49/604a093015df191485b3a9d59b45dde83ca358d24eb03964f5d1766cc63b/impit-0.13.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f8bbff2090e7e9dd3af610ce70a0a0cf4b2413673cb02ef991dfa7722438e84", size = 4305232, upload-time = "2026-08-05T10:37:52.545Z" }, + { url = "https://files.pythonhosted.org/packages/53/82/8ed3995577c04f7e9fd206f3e07ccec6352c30122eab9f84b9ba4856e892/impit-0.13.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a663fbd28c2a84d8f7e5e113e032fc4fdd5b03563641cae0fdd2700be0e563e4", size = 4458770, upload-time = "2026-08-05T10:37:54.472Z" }, + { url = "https://files.pythonhosted.org/packages/8b/86/f8aa61e5c077c25a7281539faf69abd428541e88a9ba2d9c3c8427896b82/impit-0.13.2-cp312-cp312-win_amd64.whl", hash = "sha256:36a99b06aaaebe895ba8558681b438a71bd732fb13baf5a5ddb6752f976a292a", size = 3921025, upload-time = "2026-08-05T10:37:56.304Z" }, + { url = "https://files.pythonhosted.org/packages/28/ea/a14ebd839737b6ac58ba4f0d0a49aab876e5c3c648756d17eebb7e61c6c2/impit-0.13.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:2f5188631b8d05bce79508bafeab51b9242e0d24fc874bbf27c52dd32b8d38e2", size = 4010013, upload-time = "2026-08-05T10:37:57.965Z" }, + { url = "https://files.pythonhosted.org/packages/1d/e6/ddfb727994634068089e89339eca2aef14066437161e0f7ca6cb7285c4d0/impit-0.13.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49545b63ca58a9e56cf27453c391f87b8317633eb3fb3fe33a38a520870cb775", size = 3872057, upload-time = "2026-08-05T10:37:59.883Z" }, + { url = "https://files.pythonhosted.org/packages/cd/ce/64733d23af6b9558b7123ac56b4a776937d162e04653028608c81367e79b/impit-0.13.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b5f864300c7a8857096d9378b6f0dda87f72dfdaa32a2c3cffbd6e03eea31ee", size = 4232429, upload-time = "2026-08-05T10:38:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/40/d4/dd0cadfb119c8e1608e9f5af1c2b95ef2e7b6bf6d4cc9a0b22c6d532c9ab/impit-0.13.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b1e479822d81f7d76dd02b0dfaa2f71bab33305328592747fc0700d5f43c1ae8", size = 4088211, upload-time = "2026-08-05T10:38:03.605Z" }, + { url = "https://files.pythonhosted.org/packages/d0/90/0b178fbe84f71357e419605836ddbd755c0b59f942ef4b0ab55272ff5e81/impit-0.13.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:33b56a677215092edfd0c18f2f2bfb71f5f866c497c78c2d8f66918d971b1b22", size = 4305269, upload-time = "2026-08-05T10:38:05.402Z" }, + { url = "https://files.pythonhosted.org/packages/28/99/02f45e910eb30751254447bcffcb94338953be8f1eeb8cf54515ac4d63f8/impit-0.13.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1c4c12b3c716be07ce979a46effccb11aafb8f13ab9bc1ae91f82f87977657de", size = 4458743, upload-time = "2026-08-05T10:38:07.319Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7f/d52f7835f7bdd64b3c892a2d65933ccea2828004ec6a28ab48219b6988e7/impit-0.13.2-cp313-cp313-win_amd64.whl", hash = "sha256:f841a190805fc8b2a57814ecb78e894d8c55f33e44f8a846b168c940bdc88542", size = 3920889, upload-time = "2026-08-05T10:38:10.306Z" }, + { url = "https://files.pythonhosted.org/packages/66/90/5116c7107fbdd96a970a87c8f9cf0d70c234fba067b6149d5ec6f8da02ca/impit-0.13.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:6527d96a0a06ab6972809cfa03c9017837a2e7cbe271ba9ba16327b96b90fd0a", size = 4011141, upload-time = "2026-08-05T10:38:12.141Z" }, + { url = "https://files.pythonhosted.org/packages/89/4b/12bbec53445e6a5a86dcf9ff498d4bdbe32d3b6254faec0f8f6e076c5633/impit-0.13.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:be59d8b7da8e18de9338845310fef4866a4e7f4de51c6fab851906b854cf035e", size = 3872815, upload-time = "2026-08-05T10:38:13.948Z" }, + { url = "https://files.pythonhosted.org/packages/ed/96/3b2d93667de98cbde8e91e050a68eafa5be84a4d2ecf2cd576d9a85089db/impit-0.13.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adf37bef86dc3b38300003a8511937f80c46dbfde1bfb8f3f262878d250c122", size = 4233363, upload-time = "2026-08-05T10:38:15.738Z" }, + { url = "https://files.pythonhosted.org/packages/2f/c4/2533a020bbead5c51278abbea651e58630a34d9a401ec1f8a9c8b1b9f237/impit-0.13.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:4ad5eb2b4b9c8e922b47d5d0ceac1096e9a380be888dc77bb832dc1874f83205", size = 4090845, upload-time = "2026-08-05T10:38:17.696Z" }, + { url = "https://files.pythonhosted.org/packages/37/e5/2cd3baeb875bdb3defdc668050983c193a84735faaeb85313c895156f6db/impit-0.13.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1bc1fd28748a03e4a9ce68fb0f3ce0bf96b5beee437124e224510539af434a7a", size = 4305906, upload-time = "2026-08-05T10:38:19.874Z" }, + { url = "https://files.pythonhosted.org/packages/74/d7/a310b99f8ee337445d755bbe4002de9e81035b2f7a3aeffa58d5b2a54f0a/impit-0.13.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e2ab7da918e0c58fdf2fa337ecd3e77aac9a7e4009dafe2c3cc8df0f7d9ef34e", size = 4460145, upload-time = "2026-08-05T10:38:21.678Z" }, + { url = "https://files.pythonhosted.org/packages/f8/41/37d4ae1ef69d41e8175a38588b8ba3599ff01df1b2c3e0ca576f563d83aa/impit-0.13.2-cp314-cp314-win_amd64.whl", hash = "sha256:36a58d382ad7f78cf176fff147cef02a41a4629c0245f48bc3237fe23a9b1893", size = 3921654, upload-time = "2026-08-05T10:38:23.483Z" }, + { url = "https://files.pythonhosted.org/packages/72/d8/90f408d55d0541e46a8d94ced9edfe58d1e177d2e7732cf9e48ada1008ba/impit-0.13.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c53d86bde02e776b45a1de2cbafda84c92e8224f2ba45d03627330a5ce289bd6", size = 4011066, upload-time = "2026-08-05T10:38:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9b/87e95e02cdf6c8e970155ece8053d38d9dcb98ccecb107a84f2f4d8b8554/impit-0.13.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:402ff84bc97f0188960bd5e1ca6cd070c14ffc835d6e6ddb90bc3978d85e6d32", size = 3872705, upload-time = "2026-08-05T10:38:26.802Z" }, + { url = "https://files.pythonhosted.org/packages/2c/11/a51ed20241cbdade2e457ddb58e58ea1a92e69a4e36909c84c8efdff4bbb/impit-0.13.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c378244f3dea168b70a03f0d9e4d4ad9ba270d64a1018094108090bc7a674e42", size = 4234172, upload-time = "2026-08-05T10:38:28.48Z" }, + { url = "https://files.pythonhosted.org/packages/26/bd/0af84e73eb0f9cf451051003f90641f986446817abfb48838a83453d8852/impit-0.13.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:79cbb81d2a91ee1f9017088f428c536b9a6b405f0513701d809e6293ca62aed6", size = 4089234, upload-time = "2026-08-05T10:38:30.276Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f9/20200622e5e2c1d409b5e4fd791ead9c579c71a6203c85a7460c25f3824a/impit-0.13.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:468daae39f0c122a6b40ee4b6de354eccde81e4119de1be672082bc866936c50", size = 4305456, upload-time = "2026-08-05T10:38:32.151Z" }, + { url = "https://files.pythonhosted.org/packages/37/cc/463615f964606a42ef362fc1afb8e510efcb98a6ad99afcd46b4151250ec/impit-0.13.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:430dd19b61bab7dbe01c0aa6d6e0296ec145424fe39e4321f87e45fa0e13187f", size = 4461102, upload-time = "2026-08-05T10:38:33.938Z" }, ] [[package]] From 36b96dd65af93d0fdd3769b124bae3f99878f67f Mon Sep 17 00:00:00 2001 From: Max Bohomolov <34358312+Mantisus@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:03:03 +0300 Subject: [PATCH 2/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/crawlee/http_clients/_impit.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/crawlee/http_clients/_impit.py b/src/crawlee/http_clients/_impit.py index a7b792bd05..a1a73002f1 100644 --- a/src/crawlee/http_clients/_impit.py +++ b/src/crawlee/http_clients/_impit.py @@ -48,8 +48,9 @@ # Headers scoped to a single origin, dropped as soon as a redirect leaves it. _CROSS_ORIGIN_HEADERS = frozenset({'authorization', 'cookie', 'proxy-authorization'}) -# Headers describing a request body, dropped when a redirect turns the request into a bodyless `GET`. -_REQUEST_BODY_HEADERS = frozenset({'content-encoding', 'content-language', 'content-location', 'content-type'}) +_REQUEST_BODY_HEADERS = frozenset( + {'content-encoding', 'content-language', 'content-location', 'content-type', 'content-length'} +) def _is_cross_origin(url: URL, next_url: URL) -> bool: From 8f8c3cc11d5e06393efe2d9449a832731cfc68a1 Mon Sep 17 00:00:00 2001 From: Max Bohomolov <34358312+Mantisus@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:03:29 +0300 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: Vlada Dusek --- src/crawlee/http_clients/_impit.py | 6 +++--- src/crawlee/sessions/_cookies.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crawlee/http_clients/_impit.py b/src/crawlee/http_clients/_impit.py index a1a73002f1..d3c60dd26d 100644 --- a/src/crawlee/http_clients/_impit.py +++ b/src/crawlee/http_clients/_impit.py @@ -339,10 +339,10 @@ async def _request_with_redirects( if next_method != method: method = next_method content = None - headers = {key: value for key, value in headers.items() if key.lower() not in _REQUEST_BODY_HEADERS} + headers = {key: value for key, value in headers.items() if key not in _REQUEST_BODY_HEADERS} if _is_cross_origin(current_url, next_url): - headers = {key: value for key, value in headers.items() if key.lower() not in _CROSS_ORIGIN_HEADERS} + headers = {key: value for key, value in headers.items() if key not in _CROSS_ORIGIN_HEADERS} if stream: response.close() @@ -370,7 +370,7 @@ def _get_client(self, proxy_url: str | None) -> AsyncClient: # Update the default kwargs with any additional user-provided kwargs. kwargs.update(self._async_client_kwargs) - # Redirects are followed hop by hop by this client. + # Redirects are followed hop by hop by `_request_with_redirects`. client = AsyncClient(**kwargs, follow_redirects=False) self._client_by_proxy_url[proxy_url] = client diff --git a/src/crawlee/sessions/_cookies.py b/src/crawlee/sessions/_cookies.py index fa406924b3..46dceac0a9 100644 --- a/src/crawlee/sessions/_cookies.py +++ b/src/crawlee/sessions/_cookies.py @@ -247,7 +247,7 @@ def get_cookie_string(self, url: str) -> str: self._jar.add_cookie_header(url_request) return url_request.get_header('Cookie', '') - def extract_cookie_from_header(self, url: str, set_cookie_headers: list[str]) -> None: + def extract_cookies_from_headers(self, url: str, set_cookie_headers: list[str]) -> None: """Store cookies from the raw `Set-Cookie` headers of a response. Attributes omitted from a header, such as domain and path, are derived from the URL. It must therefore be From d1cbb18519751392c1a2d2f018287fcaac38c479 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Mon, 10 Aug 2026 14:17:20 +0000 Subject: [PATCH 4/5] use update `extract_cookies_from_headers` method --- src/crawlee/http_clients/_impit.py | 2 +- tests/unit/sessions/test_cookies.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/crawlee/http_clients/_impit.py b/src/crawlee/http_clients/_impit.py index d3c60dd26d..3cebc127fc 100644 --- a/src/crawlee/http_clients/_impit.py +++ b/src/crawlee/http_clients/_impit.py @@ -322,7 +322,7 @@ async def _request_with_redirects( ) if session and self._persist_cookies_per_session: - session.cookies.extract_cookie_from_header(str(current_url), response.headers.get_list('set-cookie')) + session.cookies.extract_cookies_from_headers(str(current_url), response.headers.get_list('set-cookie')) if not self._follow_redirects or response.status_code not in _REDIRECT_STATUS_CODES: return response diff --git a/tests/unit/sessions/test_cookies.py b/tests/unit/sessions/test_cookies.py index 0c5cd5db43..f1043c7416 100644 --- a/tests/unit/sessions/test_cookies.py +++ b/tests/unit/sessions/test_cookies.py @@ -148,10 +148,10 @@ def test_store_multidomain_cookies() -> None: assert check_cookies['notest.io'] == ('a', '2') -def test_extract_cookie_from_header() -> None: +def test_extract_cookies_from_headers() -> None: """Test that cookies are parsed from raw `Set-Cookie` headers with attributes taken from the URL.""" session_cookies = SessionCookies() - session_cookies.extract_cookie_from_header( + session_cookies.extract_cookies_from_headers( 'https://example.com/login', [ 'sid=abc123; Path=/; Secure; HttpOnly; SameSite=Lax', @@ -171,21 +171,21 @@ def test_extract_cookie_from_header() -> None: assert not cookies['theme']['secure'] -def test_extract_cookie_from_header_uses_url_for_defaults() -> None: +def test_extract_cookies_from_headers_uses_url_for_defaults() -> None: """Test that the domain of a cookie without the `Domain` attribute comes from the URL of the response.""" session_cookies = SessionCookies() - session_cookies.extract_cookie_from_header('https://first.example.com/', ['a=1; Path=/']) - session_cookies.extract_cookie_from_header('https://second.example.com/', ['b=2; Path=/']) + session_cookies.extract_cookies_from_headers('https://first.example.com/', ['a=1; Path=/']) + session_cookies.extract_cookies_from_headers('https://second.example.com/', ['b=2; Path=/']) domains = {item['name']: item.get('domain') for item in session_cookies.get_cookies_as_dicts()} assert domains == {'a': 'first.example.com', 'b': 'second.example.com'} -def test_extract_cookie_from_header_with_empty_headers() -> None: +def test_extract_cookies_from_headers_with_empty_headers() -> None: """Test that a response without `Set-Cookie` headers leaves the jar untouched.""" session_cookies = SessionCookies() session_cookies.set('existing', 'value', domain='example.com') - session_cookies.extract_cookie_from_header('https://example.com/', []) + session_cookies.extract_cookies_from_headers('https://example.com/', []) assert len(session_cookies) == 1 @@ -228,6 +228,6 @@ def test_get_cookie_string_without_matching_cookies() -> None: def test_get_cookie_string_round_trip() -> None: """Test that cookies extracted from a response are sent back to the same URL.""" session_cookies = SessionCookies() - session_cookies.extract_cookie_from_header('https://example.com/login', ['sid=abc123; Path=/']) + session_cookies.extract_cookies_from_headers('https://example.com/login', ['sid=abc123; Path=/']) assert session_cookies.get_cookie_string('https://example.com/dashboard') == 'sid=abc123' From 81bc9293a74432d28f5d035a2115ec708e74227c Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Mon, 10 Aug 2026 15:00:22 +0000 Subject: [PATCH 5/5] fix --- src/crawlee/errors.py | 11 ---- src/crawlee/http_clients/_impit.py | 39 +++++++++----- tests/unit/http_clients/test_impit.py | 75 ++++++++++++++++++--------- tests/unit/server.py | 36 +++++++------ 4 files changed, 97 insertions(+), 64 deletions(-) diff --git a/src/crawlee/errors.py b/src/crawlee/errors.py index b7f292d4a0..539bcf7711 100644 --- a/src/crawlee/errors.py +++ b/src/crawlee/errors.py @@ -18,7 +18,6 @@ 'RequestHandlerError', 'ServiceConflictError', 'SessionError', - 'TooManyRedirectsError', 'UserDefinedErrorHandlerError', ] @@ -58,16 +57,6 @@ class ProxyError(SessionError): """Raised when a proxy is being blocked or malfunctions.""" -@docs_group('Errors') -class TooManyRedirectsError(Exception): - """Raised when a request exceeds the maximum number of redirects allowed by the HTTP client.""" - - def __init__(self, url: str, max_redirects: int) -> None: - super().__init__(f'Exceeded the limit of {max_redirects} redirects while requesting {url}.') - self.url = url - self.max_redirects = max_redirects - - @docs_group('Errors') class HttpStatusCodeError(Exception): """Raised when the response status code indicates an error.""" diff --git a/src/crawlee/http_clients/_impit.py b/src/crawlee/http_clients/_impit.py index 3cebc127fc..202908156e 100644 --- a/src/crawlee/http_clients/_impit.py +++ b/src/crawlee/http_clients/_impit.py @@ -4,9 +4,10 @@ from contextlib import asynccontextmanager from http import HTTPStatus from logging import getLogger +from time import monotonic from typing import TYPE_CHECKING, Any -from impit import AsyncClient, Browser, HTTPError, Response, TimeoutException, TransportError +from impit import AsyncClient, Browser, HTTPError, Response, TimeoutException, TooManyRedirects, TransportError from impit import ProxyError as ImpitProxyError from typing_extensions import override from yarl import URL @@ -15,7 +16,7 @@ from crawlee._utils.blocked import ROTATE_PROXY_ERRORS from crawlee._utils.docs import docs_group from crawlee._utils.urls import validate_http_url -from crawlee.errors import ProxyError, TooManyRedirectsError +from crawlee.errors import ProxyError from crawlee.http_clients import HttpClient, HttpCrawlingResult, HttpResponse if TYPE_CHECKING: @@ -145,7 +146,7 @@ def __init__( verify: SSL certificates used to verify the identity of requested hosts. browser: Browser to impersonate. follow_redirects: Whether to follow HTTP redirects. - max_redirects: Maximum number of redirects to follow before raising `TooManyRedirectsError`. + max_redirects: Maximum number of redirects to follow before raising `impit.TooManyRedirects`. async_client_kwargs: Additional keyword arguments for `impit.AsyncClient`. """ super().__init__( @@ -297,27 +298,39 @@ async def _request_with_redirects( stream: Whether the body of the final response should be streamed. Raises: - TooManyRedirectsError: If the number of redirects exceeds `max_redirects`. + TooManyRedirects: If the number of redirects exceeds `max_redirects`. Returns: The final response of the redirect chain. """ client = self._get_client(proxy_info.url if proxy_info else None) - current_url = URL(url) + # `encoded=True` keeps the URL byte for byte and safe `%2F` in query. + current_url = URL(url, encoded=True) content = payload + # The timeout bounds the whole chain rather than each hop, which is how `impit` treats it as well. + deadline = monotonic() + timeout.total_seconds() if timeout else None for _ in range(self._max_redirects + 1): - # Header names are normalized to lowercase by `HttpHeaders`, so this replaces any `Cookie` header - # passed by the caller instead of adding a second one. - if session and (cookie_string := session.cookies.get_cookie_string(str(current_url))): - headers['cookie'] = cookie_string + remaining = deadline - monotonic() if deadline is not None else None + if remaining is not None and remaining <= 0: + raise asyncio.TimeoutError + + # Rebuilt per hop, so that the `Cookie` header never reaches a URL its cookies do not match. A header + # passed by the caller wins over the session, which is how `impit` treats its own cookie jar. + request_headers = dict(headers) + if ( + session + and 'cookie' not in request_headers + and (cookie_string := session.cookies.get_cookie_string(str(current_url))) + ): + request_headers['cookie'] = cookie_string response = await client.request( method=method, url=str(current_url), content=content, - headers=headers or None, - timeout=timeout.total_seconds() if timeout else None, + headers=request_headers or None, + timeout=remaining, stream=stream, ) @@ -331,7 +344,7 @@ async def _request_with_redirects( if not location: return response - next_url = current_url.join(URL(location)) + next_url = current_url.join(URL(location, encoded=True)) if next_url.scheme not in _HTTP_SCHEMES: return response @@ -349,7 +362,7 @@ async def _request_with_redirects( current_url = next_url - raise TooManyRedirectsError(url, self._max_redirects) + raise TooManyRedirects(f'Exceeded the limit of {self._max_redirects} redirects while requesting {url}.') def _get_client(self, proxy_url: str | None) -> AsyncClient: """Retrieve or create an HTTP client for the given proxy URL. diff --git a/tests/unit/http_clients/test_impit.py b/tests/unit/http_clients/test_impit.py index 61fe6e2b47..657d94f5db 100644 --- a/tests/unit/http_clients/test_impit.py +++ b/tests/unit/http_clients/test_impit.py @@ -4,10 +4,11 @@ from typing import TYPE_CHECKING import pytest +from impit import TooManyRedirects -from crawlee.errors import TooManyRedirectsError +from crawlee import Request from crawlee.http_clients import ImpitHttpClient -from crawlee.sessions import Session +from crawlee.sessions import CookieParam, Session if TYPE_CHECKING: from collections.abc import AsyncGenerator @@ -38,7 +39,7 @@ async def test_sessions_share_one_client(http_client: ImpitHttpClient, server_ur assert len(http_client._client_by_proxy_url) == 1 -async def test_cookies_are_kept_apart_per_session(http_client: ImpitHttpClient, server_url: URL) -> None: +async def test_cookies_isolated_per_session(http_client: ImpitHttpClient, server_url: URL) -> None: """Test that sessions sharing a client don't see cookies of each other.""" first_session = Session() second_session = Session() @@ -56,7 +57,7 @@ async def test_cookies_are_kept_apart_per_session(http_client: ImpitHttpClient, assert (await read_json(second_response))['cookies'] == {'b': '2'} -async def test_cookies_are_collected_on_every_hop(http_client: ImpitHttpClient, server_url: URL) -> None: +async def test_cookies_collected_on_redirect(http_client: ImpitHttpClient, server_url: URL) -> None: """Test that a cookie set by a redirecting response is sent on the following hop.""" session = Session() @@ -68,7 +69,7 @@ async def test_cookies_are_collected_on_every_hop(http_client: ImpitHttpClient, assert (await read_json(response))['cookies'] == {'a': '1'} -async def test_cookies_are_not_stored_without_persistence(server_url: URL) -> None: +async def test_cookies_not_persisted(server_url: URL) -> None: """Test that `persist_cookies_per_session` keeps the session jar untouched.""" session = Session() @@ -91,7 +92,7 @@ async def test_cookies_are_not_stored_without_persistence(server_url: URL) -> No pytest.param(308, 'PUT', 'PUT', 'payload', id='308-put'), ], ) -async def test_redirect_method_follows_fetch_algorithm( +async def test_redirect_method( http_client: ImpitHttpClient, server_url: URL, *, @@ -115,7 +116,7 @@ async def test_redirect_method_follows_fetch_algorithm( assert echo['body'] == expected_body -async def test_body_headers_dropped_when_method_changes(http_client: ImpitHttpClient, server_url: URL) -> None: +async def test_body_headers_dropped(http_client: ImpitHttpClient, server_url: URL) -> None: """Test that headers describing the request body are dropped once a redirect turns the request into a `GET`.""" redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'headers'), status=302) @@ -132,7 +133,7 @@ async def test_body_headers_dropped_when_method_changes(http_client: ImpitHttpCl assert headers['x-custom'] == 'kept' -async def test_authorization_kept_within_origin(http_client: ImpitHttpClient, server_url: URL) -> None: +async def test_auth_kept_same_origin(http_client: ImpitHttpClient, server_url: URL) -> None: """Test that credentials survive a redirect that stays on the same origin.""" redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'headers'), status=302) @@ -142,7 +143,7 @@ async def test_authorization_kept_within_origin(http_client: ImpitHttpClient, se assert headers['authorization'] == 'Bearer token' -async def test_authorization_dropped_across_origins( +async def test_auth_dropped_cross_origin( http_client: ImpitHttpClient, server_url: URL, redirect_server_url: URL, @@ -160,7 +161,7 @@ async def test_authorization_dropped_across_origins( assert headers['x-custom'] == 'kept' -async def test_explicit_cookie_header_kept_within_origin(http_client: ImpitHttpClient, server_url: URL) -> None: +async def test_cookie_header_kept_same_origin(http_client: ImpitHttpClient, server_url: URL) -> None: """Test that a `Cookie` header set by the caller survives a redirect within the origin.""" redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'cookies'), status=302) @@ -169,10 +170,22 @@ async def test_explicit_cookie_header_kept_within_origin(http_client: ImpitHttpC assert (await read_json(response))['cookies'] == {'manual': 'value'} -async def test_session_cookies_replace_explicit_cookie_header(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that cookies of a session take over the `Cookie` header instead of being sent next to it.""" - session = Session() - session.cookies.set('from_jar', '1', domain=server_url.host or '') +async def test_cookie_header_rebuilt_per_hop(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that the `Cookie` header of one hop does not reach a hop whose URL the cookie does not match.""" + session = Session( + cookies=[CookieParam(name='scoped', value='value', domain=server_url.host or '', path='/redirect')] + ) + + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'cookies'), status=302) + response = await http_client.send_request(str(redirect_url), session=session) + + assert (await read_json(response))['cookies'] == {} + assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'scoped'} + + +async def test_cookie_header_wins_over_session(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that a `Cookie` header passed by the caller replaces the cookies of the session, as `impit` does.""" + session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=server_url.host or '')]) response = await http_client.send_request( str(server_url / 'cookies'), @@ -180,19 +193,15 @@ async def test_session_cookies_replace_explicit_cookie_header(http_client: Impit headers={'cookie': 'manual=value'}, ) - assert (await read_json(response))['cookies'] == {'from_jar': '1'} + assert (await read_json(response))['cookies'] == {'manual': 'value'} -async def test_explicit_cookie_header_dropped_across_origins( +async def test_cookie_header_dropped_cross_origin( http_client: ImpitHttpClient, server_url: URL, redirect_server_url: URL, ) -> None: - """Test that a `Cookie` header set by the caller is dropped once a redirect leaves the origin. - - Cookies stored in a session jar are not affected, as they are matched by domain on every hop. Ports give no - isolation of their own, see RFC 6265, section 8.5. - """ + """Test that a `Cookie` header set by the caller is dropped once a redirect leaves the origin.""" redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'cookies'), status=302) response = await http_client.send_request(str(redirect_url), headers={'cookie': 'manual=value'}) @@ -203,11 +212,9 @@ async def test_explicit_cookie_header_dropped_across_origins( async def test_too_many_redirects(server_url: URL) -> None: """Test that an endless redirect chain is cut off by `max_redirects`.""" async with ImpitHttpClient(max_redirects=2) as client: - with pytest.raises(TooManyRedirectsError) as exc_info: + with pytest.raises(TooManyRedirects, match='limit of 2 redirects'): await client.send_request(str(server_url / 'redirect_loop')) - assert exc_info.value.max_redirects == 2 - async def test_stream_follows_redirects(http_client: ImpitHttpClient, server_url: URL) -> None: """Test that streamed requests follow redirects and carry session cookies along.""" @@ -221,3 +228,23 @@ async def test_stream_follows_redirects(http_client: ImpitHttpClient, server_url assert json.loads(content.decode())['cookies'] == {'a': '1'} assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'a'} + + +async def test_crawl_keeps_cookies_and_encoding(http_client: ImpitHttpClient, server_url: URL) -> None: + """Test that `crawl` carries session cookies through a redirect and sends signed URLs without re-encoding.""" + session = Session(cookies=[CookieParam(name='preset', value='value', domain=server_url.host or '')]) + + signed_query = 'X-Amz-Credential=AKIA%2F20240101%2Fus-east-1&X-Amz-Date=2024-01-01T00%3A00%3A00Z' + target_url = f'{server_url / "cookies"}?{signed_query}' + + direct_request = Request.from_url(target_url) + direct_result = await http_client.crawl(direct_request, session=session) + + assert json.loads((await direct_result.http_response.read()).decode())['cookies'] == {'preset': 'value'} + assert direct_request.loaded_url == target_url + + redirected_request = Request.from_url(str((server_url / 'redirect').with_query(url=target_url, status=302))) + redirected_result = await http_client.crawl(redirected_request, session=session) + + assert json.loads((await redirected_result.http_response.read()).decode())['cookies'] == {'preset': 'value'} + assert redirected_request.loaded_url == target_url diff --git a/tests/unit/server.py b/tests/unit/server.py index f12a553287..9029d48912 100644 --- a/tests/unit/server.py +++ b/tests/unit/server.py @@ -69,6 +69,24 @@ def get_cookies_from_headers(headers: dict[str, Any]) -> dict[str, str]: return cookies +async def read_body(receive: Receive) -> bytes: + """Read the whole body of a request, stopping early if the client disconnects.""" + body = b'' + more_body = True + + while more_body: + message = await receive() + + if message['type'] == 'http.disconnect': + break + + if message['type'] == 'http.request': + body += message.get('body', b'') + more_body = message.get('more_body', False) + + return body + + async def send_json_response(send: Send, data: Any, status: int = 200) -> None: """Send a JSON response to the client.""" await send( @@ -214,16 +232,9 @@ async def post_echo(scope: dict[str, Any], receive: Receive, send: Send) -> None headers = get_headers_dict(scope) # Read the request body - body = b'' form = {} json_data = None - more_body = True - - while more_body: - message = await receive() - if message['type'] == 'http.request': - body += message.get('body', b'') - more_body = message.get('more_body', False) + body = await read_body(receive) # Parse body based on content type content_type = headers.get('content-type', '').lower() @@ -274,14 +285,7 @@ async def echo_headers(scope: dict[str, Any], _receive: Receive, send: Send) -> async def echo_method(scope: dict[str, Any], receive: Receive, send: Send) -> None: """Echo back the method and the body of the request.""" - body = b'' - more_body = True - - while more_body: - message = await receive() - if message['type'] == 'http.request': - body += message.get('body', b'') - more_body = message.get('more_body', False) + body = await read_body(receive) await send_json_response(send, {'method': scope['method'], 'body': body.decode()})