From 0a78f071c0e58a289a808f4391048a9cc292fe9e Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Sat, 8 Aug 2026 11:09:05 +0800 Subject: [PATCH 1/2] fix(config): make body visibility timeout configurable --- crawl4ai/async_configs.py | 6 ++++++ crawl4ai/async_crawler_strategy.py | 2 +- tests/test_config_defaults.py | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 27320fd4b..084060f9f 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -244,6 +244,7 @@ class UntrustedConfigError(ValueError): "fetch_ssl_certificate", # timing / waiting "wait_until", "page_timeout", "wait_for", "wait_for_timeout", + "body_visibility_timeout", "wait_for_images", "delay_before_return_html", "mean_delay", "max_range", # scrolling / rendering "ignore_body_visibility", "scan_full_page", "scroll_delay", @@ -1463,6 +1464,8 @@ class CrawlerRunConfig(): Default: False. ignore_body_visibility (bool): If True, ignore whether the body is visible before proceeding. Default: True. + body_visibility_timeout (int): Maximum time in ms to wait for the body to become visible. + Default: 30000. scan_full_page (bool): If True, scroll through the entire page to load all content. Default: False. scroll_delay (float): Delay in seconds between scroll steps if scan_full_page is True. @@ -1640,6 +1643,7 @@ def __init__( c4a_script: Union[str, List[str]] = None, js_only: bool = False, ignore_body_visibility: bool = True, + body_visibility_timeout: int = 30000, scan_full_page: bool = False, scroll_delay: float = 0.2, max_scroll_steps: Optional[int] = None, @@ -1770,6 +1774,7 @@ def __init__( self.c4a_script = c4a_script self.js_only = js_only self.ignore_body_visibility = ignore_body_visibility + self.body_visibility_timeout = body_visibility_timeout self.scan_full_page = scan_full_page self.scroll_delay = scroll_delay self.max_scroll_steps = max_scroll_steps @@ -2137,6 +2142,7 @@ def to_dict(self): "js_code_before_wait": self.js_code_before_wait, "js_only": self.js_only, "ignore_body_visibility": self.ignore_body_visibility, + "body_visibility_timeout": self.body_visibility_timeout, "scan_full_page": self.scan_full_page, "scroll_delay": self.scroll_delay, "max_scroll_steps": self.max_scroll_steps, diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index 265c376e9..9202124ab 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -823,7 +823,7 @@ async def handle_request_failed_capture(request): style.opacity !== '0'; return isVisible; }""", - timeout=30000, + timeout=config.body_visibility_timeout, ) if not is_visible and not config.ignore_body_visibility: diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py index 700886aa9..93a9484ca 100644 --- a/tests/test_config_defaults.py +++ b/tests/test_config_defaults.py @@ -226,13 +226,17 @@ def test_dump_load_survives_reset(self): assert loaded.headless is False def test_crawler_run_config_dump_load(self): - CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True) + assert CrawlerRunConfig().body_visibility_timeout == 30000 + CrawlerRunConfig.set_defaults( + verbose=False, scan_full_page=True, body_visibility_timeout=2000 + ) cfg = CrawlerRunConfig() data = cfg.dump() CrawlerRunConfig.reset_defaults() loaded = CrawlerRunConfig.load(data) assert loaded.verbose is False assert loaded.scan_full_page is True + assert loaded.body_visibility_timeout == 2000 def test_to_dict_includes_user_default_values(self): BrowserConfig.set_defaults(headless=False) From 64bf678e13844e95cbd55a57c255fd627fb1b4f5 Mon Sep 17 00:00:00 2001 From: nightcityblade Date: Tue, 11 Aug 2026 23:56:27 +0800 Subject: [PATCH 2/2] fix(config): validate body visibility timeout --- crawl4ai/async_configs.py | 8 ++++- docs/md_v2/api/parameters.md | 1 + docs/md_v2/complete-sdk-reference.md | 1 + tests/test_config_defaults.py | 45 ++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 084060f9f..3efb0d380 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -300,7 +300,7 @@ def _cap_timeout(v): return min(int(v), _MAX_TIMEOUT_MS) if type_name == "CrawlerRunConfig": - for f in ("page_timeout", "wait_for_timeout"): + for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"): if f in params: params[f] = _cap_timeout(params[f]) if isinstance(params.get("max_scroll_steps"), int): @@ -1774,6 +1774,12 @@ def __init__( self.c4a_script = c4a_script self.js_only = js_only self.ignore_body_visibility = ignore_body_visibility + if ( + not isinstance(body_visibility_timeout, (int, float)) + or isinstance(body_visibility_timeout, bool) + or body_visibility_timeout <= 0 + ): + raise ValueError("body_visibility_timeout must be a positive number") self.body_visibility_timeout = body_visibility_timeout self.scan_full_page = scan_full_page self.scroll_delay = scroll_delay diff --git a/docs/md_v2/api/parameters.md b/docs/md_v2/api/parameters.md index 568e14c30..f9f759a58 100644 --- a/docs/md_v2/api/parameters.md +++ b/docs/md_v2/api/parameters.md @@ -159,6 +159,7 @@ Use these for controlling whether you read or write from a local content cache. | **`c4a_script`** | `str or list[str]` (None) | C4A script that compiles to JavaScript. Alternative to writing raw JS. | | **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. | | **`ignore_body_visibility`** | `bool` (True) | Skip checking if `` is visible. Usually best to keep `True`. | +| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `` to become visible. Must be positive. | | **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). | | **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. | | **`max_scroll_steps`** | `int or None` (None) | Maximum number of scroll steps during full page scan. If None, scrolls until entire page is loaded. | diff --git a/docs/md_v2/complete-sdk-reference.md b/docs/md_v2/complete-sdk-reference.md index aa0517b2d..1f3299075 100644 --- a/docs/md_v2/complete-sdk-reference.md +++ b/docs/md_v2/complete-sdk-reference.md @@ -1791,6 +1791,7 @@ run_cfg = CrawlerRunConfig( | **`js_code_before_wait`** | `str or list[str]` (None) | JavaScript to run **before** `wait_for`. Use for triggering loading that `wait_for` then checks. | | **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. | | **`ignore_body_visibility`** | `bool` (True) | Skip checking if `` is visible. Usually best to keep `True`. | +| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `` to become visible. Must be positive. | | **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). | | **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. | | **`process_iframes`** | `bool` (False) | Inlines iframe content for single-page extraction. | diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py index 93a9484ca..3a7fba641 100644 --- a/tests/test_config_defaults.py +++ b/tests/test_config_defaults.py @@ -1,7 +1,12 @@ """Tests for BrowserConfig.set_defaults / CrawlerRunConfig.set_defaults.""" +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + import pytest + from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig +from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy @pytest.fixture(autouse=True) @@ -238,6 +243,46 @@ def test_crawler_run_config_dump_load(self): assert loaded.scan_full_page is True assert loaded.body_visibility_timeout == 2000 + @pytest.mark.parametrize("timeout", [None, 0, -1, "1000", True]) + def test_body_visibility_timeout_must_be_positive_number(self, timeout): + with pytest.raises(ValueError, match="must be a positive number"): + CrawlerRunConfig(body_visibility_timeout=timeout) + + def test_untrusted_body_visibility_timeout_is_clamped(self): + from crawl4ai.async_configs import Provenance + + config = CrawlerRunConfig.load( + {"body_visibility_timeout": 500_000}, provenance=Provenance.UNTRUSTED + ) + assert config.body_visibility_timeout == 60_000 + + @pytest.mark.asyncio + async def test_body_visibility_timeout_reaches_wait(self): + page = MagicMock() + page.evaluate = AsyncMock() + page.set_content = AsyncMock() + page.wait_for_selector = AsyncMock() + page.content = AsyncMock(return_value="visible") + + strategy = AsyncPlaywrightCrawlerStrategy.__new__( + AsyncPlaywrightCrawlerStrategy + ) + strategy.browser_config = SimpleNamespace( + use_persistent_context=False, accept_downloads=False, text_mode=True + ) + strategy.browser_manager = SimpleNamespace( + get_page=AsyncMock(return_value=(page, MagicMock())) + ) + strategy.execute_hook = AsyncMock() + strategy.csp_compliant_wait = AsyncMock(return_value=True) + + config = CrawlerRunConfig( + session_id="body-timeout-test", body_visibility_timeout=1234 + ) + await strategy._crawl_web("raw:visible", config) + + assert strategy.csp_compliant_wait.await_args.kwargs["timeout"] == 1234 + def test_to_dict_includes_user_default_values(self): BrowserConfig.set_defaults(headless=False) cfg = BrowserConfig()