diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index 9f498f0b13..4fe2ac62ef 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -48,7 +48,7 @@ from typing_extensions import Unpack - from crawlee._types import UseStateFunction + from crawlee._types import SkippedReason, UseStateFunction from crawlee.crawlers._basic._basic_crawler import _BasicCrawlerOptions @@ -221,6 +221,13 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon static_crawler.post_navigation_hook(adaptive_post_navigation_hook_static) playwright_crawler.post_navigation_hook(adaptive_post_navigation_hook_pw) + # The sub crawlers check links against robots.txt in `enqueue_links`. They share the robots.txt cache of this + # crawler, so that each file is fetched once, and report the links they skip to its `on_skipped_request`. + for sub_crawler in (static_crawler, playwright_crawler): + sub_crawler._robots_txt_file_cache = self._robots_txt_file_cache # noqa: SLF001 + sub_crawler._robots_txt_lock = self._robots_txt_lock # noqa: SLF001 + sub_crawler.on_skipped_request(self._report_skipped_request) + self._additional_context_managers = [ *self._additional_context_managers, self.rendering_type_predictor, @@ -285,6 +292,10 @@ def with_parsel_static_parser( **kwargs, ) + async def _report_skipped_request(self, url: str, reason: SkippedReason) -> None: + if self._on_skipped_request: + await self._on_skipped_request(url, reason) + async def _crawl_one( self, rendering_type: RenderingType, diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 76a2f40369..7b822a7456 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -15,6 +15,7 @@ from typing_extensions import override from crawlee import ConcurrencySettings, Request +from crawlee._utils.robots import RobotsTxtFile from crawlee.crawlers import ( AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawlingContext, @@ -43,6 +44,8 @@ from yarl import URL + from crawlee._types import SkippedReason + _H1_TEXT = 'Static' _H2_TEXT = 'Only in browser' @@ -695,6 +698,54 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: mocked_browser_handler.assert_called_once_with() +@pytest.mark.parametrize( + 'rendering_type', + [ + pytest.param('static', id='static'), + pytest.param('client only', id='browser'), + ], +) +async def test_on_skipped_request(server_url: URL, rendering_type: RenderingType) -> None: + """Links that a sub crawler skips in `enqueue_links` because of robots.txt reach the callback of the crawler.""" + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=_SimpleRenderingTypePredictor( + rendering_types=cycle([rendering_type]), detection_probability_recommendation=cycle([0]) + ), + respect_robots_txt_file=True, + ) + skip = Mock() + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + await context.enqueue_links() + + @crawler.on_skipped_request + async def skipped_hook(url: str, _reason: SkippedReason) -> None: + skip(url) + + await crawler.run([str(server_url / 'start_enqueue')]) + + skip.assert_has_calls([call(str(server_url / f'page_{index}')) for index in range(1, 5)], any_order=True) + + +async def test_fetches_robots_txt_once(server_url: URL, monkeypatch: pytest.MonkeyPatch) -> None: + """The crawler and its sub crawlers fetch the robots.txt file of a domain once, detection runs included.""" + find = AsyncMock(wraps=RobotsTxtFile.find) + monkeypatch.setattr(RobotsTxtFile, 'find', find) + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=_SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([1])), + respect_robots_txt_file=True, + ) + + @crawler.router.default_handler + async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: + await context.enqueue_links() + + await crawler.run([str(server_url / 'start_enqueue')]) + + find.assert_awaited_once() + + async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[str]) -> None: """Test that `context.query_selector_one` works regardless of the crawl type for BeautifulSoup variant.