diff --git a/pyproject.toml b/pyproject.toml index 0470908c60..e4c1b2a413 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,8 +56,7 @@ adaptive-crawler = [ "jaro-winkler>=2.0.3", "playwright>=1.27.0", "scikit-learn>=1.6.0", - # TODO: Remove the upper bound, see #2108. - "apify_fingerprint_datapoints>=0.0.3,<0.14.0", + "apify_fingerprint_datapoints>=0.0.3", "browserforge>=1.2.4" ] pydantic-ai = ["pydantic-ai-slim[openai]>=2.1.0", "parsel>=1.10.0", "lxml[html_clean]>=5.2.0"] @@ -70,10 +69,9 @@ cli = [ ] # TODO: Remove the upper bound, see #2108. curl-impersonate = ["curl-cffi>=0.9.0,<0.16.0"] -# TODO: Remove the upper bounds, see #2108. -httpx = ["httpx[brotli,http2,zstd]>=0.27.0", "apify_fingerprint_datapoints>=0.0.2,<0.14.0", "browserforge>=1.2.3"] +httpx = ["httpx[brotli,http2,zstd]>=0.27.0", "apify_fingerprint_datapoints>=0.0.2", "browserforge>=1.2.3"] parsel = ["parsel>=1.10.0"] -playwright = ["playwright>=1.27.0", "apify_fingerprint_datapoints>=0.0.2,<0.14.0", "browserforge>=1.2.3"] +playwright = ["playwright>=1.27.0", "apify_fingerprint_datapoints>=0.0.2", "browserforge>=1.2.3"] otel = [ "opentelemetry-api>=1.34.1", "opentelemetry-distro[otlp]>=0.54", @@ -90,8 +88,7 @@ sql_postgres = [ stagehand = [ "stagehand>=3.19.5", "playwright>=1.27.0", - # TODO: Remove the upper bound, see #2108. - "apify_fingerprint_datapoints>=0.0.2,<0.14.0", + "apify_fingerprint_datapoints>=0.0.2", "browserforge>=1.2.3", ] sql_sqlite = [ diff --git a/src/crawlee/fingerprint_suite/_browserforge_adapter.py b/src/crawlee/fingerprint_suite/_browserforge_adapter.py index 3f37c17949..7c6033aa07 100644 --- a/src/crawlee/fingerprint_suite/_browserforge_adapter.py +++ b/src/crawlee/fingerprint_suite/_browserforge_adapter.py @@ -3,12 +3,12 @@ import random from collections.abc import Iterable from copy import deepcopy -from functools import reduce +from functools import cache, reduce from operator import or_ from typing import TYPE_CHECKING, Any, Literal import apify_fingerprint_datapoints -from browserforge.bayesian_network import extract_json +from browserforge.bayesian_network import extract_json, get_possible_values from browserforge.fingerprints import Fingerprint as bf_Fingerprint from browserforge.fingerprints import FingerprintGenerator as bf_FingerprintGenerator from browserforge.fingerprints import Screen @@ -87,6 +87,17 @@ def generate( # headers without `sec-...` headers are valid. max_attempts += 50 + # `browserforge` takes `user_agent` only as a hint - it derives browser, operating system and device from it, + # but samples the generated user agent freely. Keep the allow list to enforce it below. + allowed_user_agents = frozenset([user_agent] if isinstance(user_agent, str) else user_agent or []) + + if allowed_user_agents: + # Pinning narrows the sampling to one browser version, but not to one operating system version. + max_attempts += 50 + + # Header satisfying everything but the allow list, used as a last resort below. + fallback_header: dict[str, str] | None = None + # Use browserforge to generate headers until it satisfies our additional requirements. for _attempt in range(max_attempts): generated_header: dict[str, str] = super().generate( @@ -114,7 +125,24 @@ def generate( # Accept chromium header only with all sec headers. continue + if allowed_user_agents and generated_header['User-Agent'] not in allowed_user_agents: + pinned_user_agent = _pick_interchangeable_user_agent( + generated_header['User-Agent'], allowed_user_agents + ) + if pinned_user_agent is None: + # Nothing interchangeable in the allow list, so no better header can be generated. + return generated_header + + fallback_header = generated_header + user_agent = [pinned_user_agent] + continue + return generated_header + + if fallback_header is not None: + # The allow list is only a preference, so a header satisfying everything else beats failing. + return fallback_header + raise RuntimeError('Failed to generate header.') def _contains_all_sec_headers(self, headers: dict[str, str]) -> bool: @@ -250,6 +278,44 @@ def generate(self, browser_type: SupportedBrowserType = 'chrome') -> dict[str, s return self._generator.generate(browser=[browser_type]) +def _pick_interchangeable_user_agent(generated_user_agent: str, allowed_user_agents: frozenset[str]) -> str | None: + """Pick a user agent from the allow list interchangeable with the generated one, `None` if there is none.""" + generated_traits = _get_user_agent_traits(generated_user_agent) + if not all(generated_traits): + return None + + candidates = [ + allowed_user_agent + for allowed_user_agent in allowed_user_agents + if _get_user_agent_traits(allowed_user_agent) == generated_traits + ] + return random.choice(candidates) if candidates else None + + +# Unbounded - only a few hundred user agents exist, and the default 128 entries would thrash on the allow list scan. +@cache +def _get_user_agent_traits(user_agent: str) -> tuple[frozenset[str], frozenset[str], frozenset[str]]: + """Get browser names, operating systems and devices the header network links to the `user_agent`. + + `browserforge` derives the browser name and version, the operating system and the device from its `user_agent` + argument. Only the name is compared here, because the caller constrains the browser by name, so an allowed user + agent differing just in the browser version is a valid substitute. + """ + possible_values: dict[str, Any] = {} + # The header network holds the user agent under a different node name for each HTTP version. + for node_name in ('user-agent', 'User-Agent'): + possible_values.update( + get_possible_values(bf_HeaderGenerator.header_generator_network, {node_name: (user_agent,)}) + ) + + return ( + # `*BROWSER` values are `{name}/{version}`, keep just the name. + frozenset(browser.split('/', maxsplit=1)[0] for browser in possible_values.get('*BROWSER', ())), + frozenset(possible_values.get('*OPERATING_SYSTEM', ())), + frozenset(possible_values.get('*DEVICE', ())), + ) + + def get_available_header_network() -> dict: """Get header network that contains possible header values.""" return extract_json(apify_fingerprint_datapoints.get_header_network()) diff --git a/tests/unit/fingerprint_suite/test_adapters.py b/tests/unit/fingerprint_suite/test_adapters.py index 34d95ba487..0f80bc0a7f 100644 --- a/tests/unit/fingerprint_suite/test_adapters.py +++ b/tests/unit/fingerprint_suite/test_adapters.py @@ -35,6 +35,30 @@ def test_fingerprint_generator_some_options_stress_test() -> None: assert fingerprint.screen.availWidth > 500 +def test_fingerprint_generator_respects_screen_options_without_strict() -> None: + """Test that screen constraints are respected without `strict`, where `browserforge` silently drops them.""" + min_width = 300 + max_width = 600 + min_height = 500 + max_height = 1200 + + fingerprint_generator = DefaultFingerprintGenerator( + header_options=HeaderGeneratorOptions(browsers=['firefox'], operating_systems=['android']), + screen_options=ScreenOptions( + min_width=min_width, + max_width=max_width, + min_height=min_height, + max_height=max_height, + ), + ) + + for _ in range(20): + fingerprint = fingerprint_generator.generate() + + assert min_width <= fingerprint.screen.width <= max_width + assert min_height <= fingerprint.screen.height <= max_height + + def test_fingerprint_generator_all_options() -> None: """Test that header generator can work with all the options. Some most basic checks of fingerprint. diff --git a/uv.lock b/uv.lock index 0310c9e821..fc4a0b2c44 100644 --- a/uv.lock +++ b/uv.lock @@ -98,11 +98,11 @@ wheels = [ [[package]] name = "apify-fingerprint-datapoints" -version = "0.13.0" +version = "0.14.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f9/f1/b74f95767581372ab849c8b13e384b62f60d034584892c60c4a3442d9312/apify_fingerprint_datapoints-0.13.0.tar.gz", hash = "sha256:263141c19e9bc90a821e6b4e2b845925f17e0b8fbd53a897fc71546bd50df7f1", size = 934827, upload-time = "2026-05-04T09:08:45.036Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/d4/17a1e60b45ffe06de88ab184e6f7e4509f86239cb619f3b2dcbf97a5c648/apify_fingerprint_datapoints-0.14.0.tar.gz", hash = "sha256:96ee062787740530c0dc10231e6370e227e2bb2009677a3e175942476021dc8a", size = 823427, upload-time = "2026-08-01T01:00:55.142Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/58/8402442bf6af5a3a8068fe5431c42ea4f73c1eb18f621f9bf7c5de80caf5/apify_fingerprint_datapoints-0.13.0-py3-none-any.whl", hash = "sha256:0213d42297be19e8035202b41fb2e840a1e5d79874c99c882a5027a7d0b1a0eb", size = 761652, upload-time = "2026-05-04T09:08:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/c7/b2/c5347a8568da062a7421ec7c746345f6ef62e8d45ef6ea9028c7b74c0ffa/apify_fingerprint_datapoints-0.14.0-py3-none-any.whl", hash = "sha256:b18626d8440e104aa7b34abade1af211d307fad7fdb66b423bff90cf4fbcc65d", size = 650335, upload-time = "2026-08-01T01:00:53.803Z" }, ] [[package]] @@ -927,10 +927,10 @@ dev = [ requires-dist = [ { name = "aiomysql", marker = "extra == 'sql-mysql'", specifier = ">=0.3.2" }, { name = "aiosqlite", marker = "extra == 'sql-sqlite'", specifier = ">=0.21.0" }, - { name = "apify-fingerprint-datapoints", marker = "extra == 'adaptive-crawler'", specifier = ">=0.0.3,<0.14.0" }, - { name = "apify-fingerprint-datapoints", marker = "extra == 'httpx'", specifier = ">=0.0.2,<0.14.0" }, - { name = "apify-fingerprint-datapoints", marker = "extra == 'playwright'", specifier = ">=0.0.2,<0.14.0" }, - { name = "apify-fingerprint-datapoints", marker = "extra == 'stagehand'", specifier = ">=0.0.2,<0.14.0" }, + { name = "apify-fingerprint-datapoints", marker = "extra == 'adaptive-crawler'", specifier = ">=0.0.3" }, + { name = "apify-fingerprint-datapoints", marker = "extra == 'httpx'", specifier = ">=0.0.2" }, + { name = "apify-fingerprint-datapoints", marker = "extra == 'playwright'", specifier = ">=0.0.2" }, + { name = "apify-fingerprint-datapoints", marker = "extra == 'stagehand'", specifier = ">=0.0.2" }, { name = "async-timeout", specifier = ">=5.0.1" }, { name = "asyncpg", marker = "extra == 'sql-postgres'", specifier = ">=0.24.0" }, { name = "beautifulsoup4", extras = ["lxml"], marker = "extra == 'beautifulsoup'", specifier = ">=4.12.0" },