Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@

from typing_extensions import Unpack

from crawlee._types import UseStateFunction
from crawlee._types import SkippedReason, UseStateFunction
from crawlee.crawlers._basic._basic_crawler import _BasicCrawlerOptions


Expand Down Expand Up @@ -221,6 +221,13 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon
static_crawler.post_navigation_hook(adaptive_post_navigation_hook_static)
playwright_crawler.post_navigation_hook(adaptive_post_navigation_hook_pw)

# The sub crawlers check links against robots.txt in `enqueue_links`. They share the robots.txt cache of this
# crawler, so that each file is fetched once, and report the links they skip to its `on_skipped_request`.
for sub_crawler in (static_crawler, playwright_crawler):
sub_crawler._robots_txt_file_cache = self._robots_txt_file_cache # noqa: SLF001
sub_crawler._robots_txt_lock = self._robots_txt_lock # noqa: SLF001
sub_crawler.on_skipped_request(self._report_skipped_request)

self._additional_context_managers = [
*self._additional_context_managers,
self.rendering_type_predictor,
Expand Down Expand Up @@ -285,6 +292,10 @@ def with_parsel_static_parser(
**kwargs,
)

async def _report_skipped_request(self, url: str, reason: SkippedReason) -> None:
if self._on_skipped_request:
await self._on_skipped_request(url, reason)

async def _crawl_one(
self,
rendering_type: RenderingType,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from typing_extensions import override

from crawlee import ConcurrencySettings, Request
from crawlee._utils.robots import RobotsTxtFile
from crawlee.crawlers import (
AdaptivePlaywrightCrawler,
AdaptivePlaywrightCrawlingContext,
Expand Down Expand Up @@ -43,6 +44,8 @@

from yarl import URL

from crawlee._types import SkippedReason


_H1_TEXT = 'Static'
_H2_TEXT = 'Only in browser'
Expand Down Expand Up @@ -695,6 +698,54 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
mocked_browser_handler.assert_called_once_with()


@pytest.mark.parametrize(
'rendering_type',
[
pytest.param('static', id='static'),
pytest.param('client only', id='browser'),
],
)
async def test_on_skipped_request(server_url: URL, rendering_type: RenderingType) -> None:
"""Links that a sub crawler skips in `enqueue_links` because of robots.txt reach the callback of the crawler."""
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
rendering_type_predictor=_SimpleRenderingTypePredictor(
rendering_types=cycle([rendering_type]), detection_probability_recommendation=cycle([0])
),
respect_robots_txt_file=True,
)
skip = Mock()

@crawler.router.default_handler
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
await context.enqueue_links()

@crawler.on_skipped_request
async def skipped_hook(url: str, _reason: SkippedReason) -> None:
skip(url)

await crawler.run([str(server_url / 'start_enqueue')])

skip.assert_has_calls([call(str(server_url / f'page_{index}')) for index in range(1, 5)], any_order=True)


async def test_fetches_robots_txt_once(server_url: URL, monkeypatch: pytest.MonkeyPatch) -> None:
"""The crawler and its sub crawlers fetch the robots.txt file of a domain once, detection runs included."""
find = AsyncMock(wraps=RobotsTxtFile.find)
monkeypatch.setattr(RobotsTxtFile, 'find', find)
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
rendering_type_predictor=_SimpleRenderingTypePredictor(detection_probability_recommendation=cycle([1])),
respect_robots_txt_file=True,
)

@crawler.router.default_handler
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
await context.enqueue_links()

await crawler.run([str(server_url / 'start_enqueue')])

find.assert_awaited_once()


async def test_adaptive_context_query_selector_beautiful_soup(test_urls: list[str]) -> None:
"""Test that `context.query_selector_one` works regardless of the crawl type for BeautifulSoup variant.

Expand Down
Loading