fix(curl-impersonate): treat a refused CONNECT tunnel as a proxy error

pyproject.toml

@@ -68,8 +68,7 @@ cli = [
     "rich>=13.9.0",
     "typer>=0.26.0",
 ]
-# TODO: Remove the upper bound, see #2108.
-curl-impersonate = ["curl-cffi>=0.9.0,<0.16.0"]
+curl-impersonate = ["curl-cffi>=0.9.0"]
 # TODO: Remove the upper bounds, see #2108.
 httpx = ["httpx[brotli,http2,zstd]>=0.27.0", "apify_fingerprint_datapoints>=0.0.2,<0.14.0", "browserforge>=1.2.3"]
 parsel = ["parsel>=1.10.0"]

src/crawlee/http_clients/_curl_impersonate.py

@@ -39,6 +39,13 @@
     from crawlee.statistics import Statistics
 
 
+_CURL_PROXY_ERRORS = [
+    # Raised as a plain `ConnectionError` by `curl_cffi`, which maps only some tunnel failures to `CurlProxyError`.
+    'CONNECT tunnel failed',
+]
+"""Content of `libcurl` proxy errors that `curl_cffi` does not report as `CurlProxyError`."""
+
+
 class _EmptyCookies(CurlCookies):
     @override
     def get_cookies_for_curl(self, request: CurlRequest) -> list[CurlMorsel]:
@@ -48,6 +55,11 @@ def get_cookies_for_curl(self, request: CurlRequest) -> list[CurlMorsel]:
     def update_cookies_from_curl(self, morsels: list[CurlMorsel]) -> None:
         return None
 
+    @override
+    def update_cookies_from_curl_changes(self, changes: list[bytes]) -> None:
+        # The write path `curl_cffi` uses since 0.16.0; the one above is kept for older versions.
+        return None
+
 
 class _AsyncSession(AsyncSession):
     @override
@@ -347,7 +359,7 @@ def _is_proxy_error(error: CurlRequestError) -> bool:
         Check if the error message contains known proxy-related error keywords or if it is an instance
         of `CurlProxyError`.
         """
-        if any(needle in str(error) for needle in ROTATE_PROXY_ERRORS):
+        if any(needle in str(error) for needle in (*ROTATE_PROXY_ERRORS, *_CURL_PROXY_ERRORS)):
             return True
 
         if isinstance(error, CurlProxyError):  # noqa: SIM103

tests/unit/http_clients/test_curl_impersonate.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from crawlee.http_clients import CurlImpersonateHttpClient
+from crawlee.sessions import Session
+
+if TYPE_CHECKING:
+    from yarl import URL
+
+
+async def test_internal_session_cookie_jar_stays_empty(server_url: URL) -> None:
+    """Test that response cookies never accumulate in the jar of the internally cached `curl_cffi` session."""
+    session = Session()
+    client = CurlImpersonateHttpClient()
+
+    async with client:
+        for index in range(5):
+            url = server_url.with_path('set_cookies').extend_query(**{f'c{index}': index})
+            await client.send_request(str(url), session=session)
+
+        curl_session = client._client_by_proxy_url[None]
+        assert list(curl_session._cookies.jar) == []
+
+    # The cookies are read from the `curl` handle, so they still reach the crawlee session.
+    assert {cookie['name'] for cookie in session.cookies} == {f'c{index}' for index in range(5)}