diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 8216d19bc..6d6cd4bc8 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -408,6 +408,7 @@ async def arun( _block_reason = "" _done = False crawl_result = None + _is_pdf_response = False _crawl_stats = { "attempts": 0, "retries": 0, @@ -458,6 +459,13 @@ async def arun( async_response = await self.crawler_strategy.crawl( url, config=config) + _is_pdf_response = any( + key.lower() == "content-type" + and "application/pdf" in str(value).lower() + for key, value in ( + async_response.response_headers or {} + ).items() + ) html = sanitize_input_encode(async_response.html) screenshot_data = async_response.screenshot @@ -505,7 +513,7 @@ async def arun( # Check if blocked (skip for raw: URLs — # caller-provided content, anti-bot N/A) - if _is_raw_url: + if _is_raw_url or _is_pdf_response: _blocked = False _block_reason = "" else: @@ -625,7 +633,12 @@ async def arun( # empty by design, and is_blocked() would misread "0 bytes # html" as a block. _has_download = bool(getattr(crawl_result, "downloaded_files", None)) - if not _fallback_succeeded and not _is_raw_url and not _has_download: + if ( + not _fallback_succeeded + and not _is_raw_url + and not _has_download + and not _is_pdf_response + ): _blocked, _block_reason = is_blocked( crawl_result.status_code, crawl_result.html or "") if _blocked: @@ -1246,4 +1259,4 @@ async def amap_domain( config or DomainMapperConfig(**kwargs) if kwargs else DomainMapperConfig() ) - return await self._domain_mapper.scan(domain, mapper_config) \ No newline at end of file + return await self._domain_mapper.scan(domain, mapper_config) diff --git a/tests/test_issue_2135_pdf_antibot.py b/tests/test_issue_2135_pdf_antibot.py new file mode 100644 index 000000000..784025ddd --- /dev/null +++ b/tests/test_issue_2135_pdf_antibot.py @@ -0,0 +1,30 @@ +import pytest + +from crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig, CrawlResult +from crawl4ai.processors.pdf import PDFCrawlerStrategy + + +@pytest.mark.asyncio +async def test_pdf_response_skips_antibot_retries_and_fallback(): + fallback_calls = [] + + async def process_html(url, html, **kwargs): + return CrawlResult(url=url, html=html, success=True, status_code=200) + + async def fallback(url): + fallback_calls.append(url) + return "fallback" + + crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy()) + crawler.aprocess_html = process_html + result = await crawler.arun( + "https://example.com/document.pdf", + config=CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + max_retries=1, + fallback_fetch_function=fallback, + ), + ) + assert result.success + assert result.crawl_stats["attempts"] == 1 + assert fallback_calls == []