From bde89523d5e305723e20be0d060d65f165def890 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 12 Nov 2026 12:26:51 +0000 Subject: [PATCH 1/2] remove `use_session_pool` and always process requests with a session --- docs/03_concepts/02_requests.mdx | 2 +- docs/03_concepts/03_crawling_context.mdx | 2 +- docs/03_concepts/15_cookie_management.mdx | 6 -- .../cookie_management/playwright_cookies.py | 3 - .../cookie_management/read_write_cookies.py | 3 - .../cookie_management/retry_pinned_session.py | 3 - .../retry_restore_cookies.py | 5 +- .../cookie_management/retry_single_session.py | 3 - .../proxy_management/session_bs_example.py | 2 +- .../proxy_management/session_pw_example.py | 2 +- .../session_management/multi_sessions_http.py | 16 ++--- .../session_management/one_session_http.py | 5 +- .../session_management/sm_basic.py | 17 +++-- .../session_management/sm_beautifulsoup.py | 15 ++--- .../session_management/sm_http.py | 17 +++-- .../session_management/sm_parsel.py | 15 ++--- .../session_management/sm_playwright.py | 15 ++--- .../code_examples/login_crawler/http_login.py | 4 -- .../login_crawler/playwright_login.py | 4 -- .../scrapy_migration/crawlee_post.py | 4 -- docs/06_upgrading/upgrading_to_v2.md | 21 ++++++ src/crawlee/_types.py | 2 +- .../_abstract_http/_abstract_http_crawler.py | 2 +- src/crawlee/crawlers/_basic/_basic_crawler.py | 67 ++++++++----------- .../_playwright/_playwright_crawler.py | 7 +- .../test_adaptive_playwright_crawler.py | 5 +- .../crawlers/_basic/test_basic_crawler.py | 2 +- .../crawlers/_http/test_http_crawler.py | 25 +++---- .../_playwright/test_playwright_crawler.py | 14 ++-- .../crawlers/_basic/test_basic_crawler.py | 43 ++++++++---- .../unit/crawlers/_http/test_http_crawler.py | 1 - 31 files changed, 145 insertions(+), 187 deletions(-) diff --git a/docs/03_concepts/02_requests.mdx b/docs/03_concepts/02_requests.mdx index b6efcc80da..85dd6c6210 100644 --- a/docs/03_concepts/02_requests.mdx +++ b/docs/03_concepts/02_requests.mdx @@ -72,7 +72,7 @@ The crawler's `max_crawl_depth` option limits how deep the crawl goes. Requests ## Binding to a session -Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't available anymore, the request fails with a `RequestCollisionError` and isn't retried. For details on sessions, see [Session management](./session-management). +Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't available anymore, the request fails with a `RequestCollisionError` and isn't retried. In the failed request handler, `context.session` is then a retired session with the bound id, not taken from the pool. For details on sessions, see [Session management](./session-management). ## Request lifecycle diff --git a/docs/03_concepts/03_crawling_context.mdx b/docs/03_concepts/03_crawling_context.mdx index b58f6dd3d0..b69162f112 100644 --- a/docs/03_concepts/03_crawling_context.mdx +++ b/docs/03_concepts/03_crawling_context.mdx @@ -30,7 +30,7 @@ For details on the crawler-specific attributes, see [HTTP crawlers](./http-crawl The `request` attribute is the `Request` being processed. It carries the URL, the label used for routing, user data and the retry count. For details, see [Requests](./requests). -The `session` attribute is the `Session` assigned to the request, or `None` when the crawler runs without a session pool. You can call `mark_bad()` or `retire()` on it from the handler when a page shows signs of blocking. For details, see [Session management](./session-management). +The `session` attribute is the `Session` assigned to the request. You can call `mark_bad()` or `retire()` on it from the handler when a page shows signs of blocking. For details, see [Session management](./session-management). The `proxy_info` attribute is a `ProxyInfo` describing the proxy used for the request, or `None` when no proxy is configured. For details, see [Proxy management](./proxy-management). diff --git a/docs/03_concepts/15_cookie_management.mdx b/docs/03_concepts/15_cookie_management.mdx index 973fc1a600..6e127da64c 100644 --- a/docs/03_concepts/15_cookie_management.mdx +++ b/docs/03_concepts/15_cookie_management.mdx @@ -31,12 +31,6 @@ Access the jar through `context.session.cook {ReadWriteCookies} -:::note - -`context.session` is `None` only when the crawler runs without a session pool (`use_session_pool` set to `False`). With the default pool it's always set. The examples guard with `if context.session is None: return` to cover that case. - -::: - A cookie has a `name`, a `value`, and optional parameters such as `domain`. If a cookie has no `domain`, it applies to any domain, so a login cookie set that way is sent to every host you crawl. Set `domain` on anything sensitive. For the full set of parameters, see `CookieParam`. To dump the whole jar, use `get_cookies_as_dicts()`, and to load several cookies at once, use `set_cookies()`. ## Seeding cookies on new sessions diff --git a/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py index 26a7444937..7e87c74e18 100644 --- a/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py +++ b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py @@ -13,9 +13,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: PlaywrightCrawlingContext) -> None: - if context.session is None: - return - # The browser receives the `visited` cookie during the first navigation. # It's synced back onto the session after the handler returns, so on the # second request it's already on `context.session`. diff --git a/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py index 899ef0c3c0..57d85d4095 100644 --- a/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py +++ b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py @@ -8,9 +8,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if context.session is None: - return - # Set a cookie on the session. It's sent with every following # request that runs on this session. context.session.cookies.set( diff --git a/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py index 919e7e5c20..0f4dc2fe0c 100644 --- a/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py +++ b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py @@ -9,9 +9,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if context.session is None: - return - if context.request.user_data.get('prepared'): # Cookies from the setup step are on this pinned session. context.log.info( diff --git a/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py index 44cd965c46..dc565c4154 100644 --- a/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py +++ b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py @@ -12,9 +12,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if context.session is None: - return - state = await context.use_state(default_value={}) if not state.get('cookies'): # First pass: establish cookies once and store them for every session, @@ -30,7 +27,7 @@ async def apply_cookies(context: BasicCrawlingContext) -> None: # Runs before navigation. Apply the shared cookies to whichever session # handles the request, so every session ends up with them. state = await context.use_state(default_value={}) - if context.session and (cookies := state.get('cookies')): + if cookies := state.get('cookies'): context.session.cookies.set_cookies(cast('list[CookieParam]', cookies)) await crawler.run(['https://httpbingo.org/cookies']) diff --git a/docs/03_concepts/code_examples/cookie_management/retry_single_session.py b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py index 1eb8f650ed..a5390a321b 100644 --- a/docs/03_concepts/code_examples/cookie_management/retry_single_session.py +++ b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py @@ -23,9 +23,6 @@ async def main() -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if context.session is None: - return - # First attempt: establish cookies, then raise to trigger a retry. if not context.session.cookies: await context.send_request('https://httpbingo.org/cookies/set?logged_in=1') diff --git a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py index 1243b0e488..4c9e615e06 100644 --- a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py +++ b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py @@ -14,7 +14,7 @@ async def main() -> None: ) crawler = BeautifulSoupCrawler( proxy_configuration=proxy_configuration, - use_session_pool=True, + save_response_cookies=True, ) # ... diff --git a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py index 68309bda59..02205a1ad6 100644 --- a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py +++ b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py @@ -14,7 +14,7 @@ async def main() -> None: ) crawler = PlaywrightCrawler( proxy_configuration=proxy_configuration, - use_session_pool=True, + save_response_cookies=True, ) # ... diff --git a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py index 98b2028597..e6275d0a61 100644 --- a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py +++ b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py @@ -45,21 +45,17 @@ async def basic_handler(context: HttpCrawlingContext) -> None: # Initialize the session and bind the next request to this session if needed @crawler.router.handler(label='session_init') async def session_init(context: HttpCrawlingContext) -> None: - next_requests = [] - if context.session: - context.log.info('Init session %s', context.session.id) - next_request = Request.from_url( - 'https://a.placeholder.com', session_id=context.session.id - ) - next_requests.append(next_request) - - await context.add_requests(next_requests) + context.log.info('Init session %s', context.session.id) + next_request = Request.from_url( + 'https://a.placeholder.com', session_id=context.session.id + ) + await context.add_requests([next_request]) # Handle errors when a session is blocked and no longer available in the pool # when attempting to execute requests bound to it @crawler.failed_request_handler async def error_processing(context: BasicCrawlingContext, error: Exception) -> None: - if isinstance(error, RequestCollisionError) and context.session: + if isinstance(error, RequestCollisionError): context.log.error( 'Request %s failed, because the bound session is unavailable', context.request.url, diff --git a/docs/03_concepts/code_examples/session_management/one_session_http.py b/docs/03_concepts/code_examples/session_management/one_session_http.py index 6bb67a240d..b5c6bac0b1 100644 --- a/docs/03_concepts/code_examples/session_management/one_session_http.py +++ b/docs/03_concepts/code_examples/session_management/one_session_http.py @@ -39,13 +39,12 @@ async def basic_handler(context: HttpCrawlingContext) -> None: # Handler for session initialization (authentication, initial cookies, etc.) @crawler.router.handler(label='session_init') async def session_init(context: HttpCrawlingContext) -> None: - if context.session: - context.log.info('Init session %s', context.session.id) + context.log.info('Init session %s', context.session.id) # Monitor if our session gets blocked and explicitly stop the crawler @crawler.error_handler async def error_processing(context: BasicCrawlingContext, error: Exception) -> None: - if isinstance(error, SessionError) and context.session: + if isinstance(error, SessionError): context.log.info('Session %s blocked', context.session.id) crawler.stop() diff --git a/docs/03_concepts/code_examples/session_management/sm_basic.py b/docs/03_concepts/code_examples/session_management/sm_basic.py index 30e1d7ae92..3371c6a0c4 100644 --- a/docs/03_concepts/code_examples/session_management/sm_basic.py +++ b/docs/03_concepts/code_examples/session_management/sm_basic.py @@ -16,8 +16,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = BasicCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -33,13 +31,14 @@ async def default_handler(context: BasicCrawlingContext) -> None: page_content = (await response.read()).decode() title_match = re.search(r'(.*?)', page_content) - if context.session and (title := title_match.group(1) if title_match else None): - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + title = title_match.group(1) if title_match else None + + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py index a54fd8425f..db8028213f 100644 --- a/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py +++ b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py @@ -15,8 +15,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = BeautifulSoupCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -27,13 +25,12 @@ async def main() -> None: async def default_handler(context: BeautifulSoupCrawlingContext) -> None: title = context.soup.title.get_text() if context.soup.title else None - if context.session: - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/03_concepts/code_examples/session_management/sm_http.py b/docs/03_concepts/code_examples/session_management/sm_http.py index 9497594d3b..18c8585152 100644 --- a/docs/03_concepts/code_examples/session_management/sm_http.py +++ b/docs/03_concepts/code_examples/session_management/sm_http.py @@ -16,8 +16,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = HttpCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -29,13 +27,14 @@ async def default_handler(context: HttpCrawlingContext) -> None: page_content = (await context.http_response.read()).decode() title_match = re.search(r'(.*?)', page_content) - if context.session and (title := title_match.group(1) if title_match else None): - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + title = title_match.group(1) if title_match else None + + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/03_concepts/code_examples/session_management/sm_parsel.py b/docs/03_concepts/code_examples/session_management/sm_parsel.py index 66752a63c3..508a88a4c6 100644 --- a/docs/03_concepts/code_examples/session_management/sm_parsel.py +++ b/docs/03_concepts/code_examples/session_management/sm_parsel.py @@ -15,8 +15,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = ParselCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -27,13 +25,12 @@ async def main() -> None: async def default_handler(context: ParselCrawlingContext) -> None: title = context.selector.css('title::text').get() - if context.session: - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/03_concepts/code_examples/session_management/sm_playwright.py b/docs/03_concepts/code_examples/session_management/sm_playwright.py index 46a4c4f096..7e848fbcad 100644 --- a/docs/03_concepts/code_examples/session_management/sm_playwright.py +++ b/docs/03_concepts/code_examples/session_management/sm_playwright.py @@ -15,8 +15,6 @@ async def main() -> None: # to manage concurrent sessions and proxy rotation crawler = PlaywrightCrawler( proxy_configuration=proxy_configuration, - # Activates the Session pool (default is true). - use_session_pool=True, # Overrides default Session pool configuration. session_pool=SessionPool(max_pool_size=100), ) @@ -27,13 +25,12 @@ async def main() -> None: async def default_handler(context: PlaywrightCrawlingContext) -> None: title = await context.page.title() - if context.session: - if title == 'Blocked': - context.session.retire() - elif title == 'Not sure if blocked, might also be a connection error': - context.session.mark_bad() - else: - context.session.mark_good() # BasicCrawler handles this automatically. + if title == 'Blocked': + context.session.retire() + elif title == 'Not sure if blocked, might also be a connection error': + context.session.mark_bad() + else: + context.session.mark_good() # BasicCrawler handles this automatically. await crawler.run(['https://crawlee.dev/']) diff --git a/docs/04_guides/code_examples/login_crawler/http_login.py b/docs/04_guides/code_examples/login_crawler/http_login.py index 82848ce834..c5fe5a4cea 100644 --- a/docs/04_guides/code_examples/login_crawler/http_login.py +++ b/docs/04_guides/code_examples/login_crawler/http_login.py @@ -41,10 +41,6 @@ async def request_handler(context: HttpCrawlingContext) -> None: async def login_handler(context: HttpCrawlingContext) -> None: context.log.info('Processing login at %s ...', context.request.url) - # Verify that a session is available before proceeding - if not context.session: - raise RuntimeError('Session not found') - # Parse the API response containing authentication tokens and user data data = json.loads(await context.http_response.read()) diff --git a/docs/04_guides/code_examples/login_crawler/playwright_login.py b/docs/04_guides/code_examples/login_crawler/playwright_login.py index 650bdd917a..ad4a3cfe72 100644 --- a/docs/04_guides/code_examples/login_crawler/playwright_login.py +++ b/docs/04_guides/code_examples/login_crawler/playwright_login.py @@ -44,10 +44,6 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def login_handler(context: PlaywrightCrawlingContext) -> None: context.log.info('Processing login %s ...', context.request.url) - # Check if the session is available - if not context.session: - raise RuntimeError('Session not found') - # Entering data into the form, `delay` to simulate human typing # Without this, the data will be entered instantly await context.page.type('#userName', 'crawlee_test', delay=100) diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py index b97a77552e..a830c552ac 100644 --- a/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py +++ b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py @@ -8,10 +8,6 @@ async def main() -> None: @crawler.router.default_handler async def login_page(context: ParselCrawlingContext) -> None: - # The CSRF token is tied to the session cookie issued for this GET. - if not context.session: - raise RuntimeError('Session not found') - # highlight-start # Like Scrapy's `FormRequest.from_response`, the helper keeps the hidden # `csrf_token` field, encodes the data and sets the `Content-Type` header. diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index ee57b83421..0275f57883 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -580,6 +580,27 @@ The `ThrottlingRequestManager` warning about capping a delay now calls it a "req Log records hold the message template and its arguments separately. `record.msg` is the template, such as `Retrying request to %s due to: %s. %s`, and `record.args` holds the values. To read the final text, use `record.getMessage()`, which formatters and handlers already do. +## `use_session_pool` is removed + +The `use_session_pool` option is removed from the crawlers. Every crawler processes its requests with sessions from its `SessionPool`, the same as Crawlee JS v4. To configure the pool, pass your own `SessionPool` as `session_pool`. + +`context.session` is now a `Session` instead of `Session | None`, so you can drop the `if context.session` checks. A request bound by `session_id` to a session that's no longer available still fails with a `RequestCollisionError`, and its failed request handler gets a retired session with the bound id instead of `None`. + +Without the session pool, a 401, 403 or 429 response failed the request without a retry. Now it retires the session, and the request is retried with another one, up to `max_session_rotations` times. With `proxy_urls`, a session keeps its proxy, so the proxy changes when the session does, not on every request. + +To keep the response cookies out of the sessions, pass `save_response_cookies=False`. To retire every session after its first request, set `max_usage_count` to 1: + +```python +# v1 +crawler = ParselCrawler(use_session_pool=False) + +# v2 +crawler = ParselCrawler( + save_response_cookies=False, + session_pool=SessionPool(create_session_settings={'max_usage_count': 1}), +) +``` + ## Session retirement is permanent In v1, `Session.retire()` only added `max_error_score` to the error score, so a later `mark_good()` could lower it again and make the retired session usable. In v2, a separate `retired` flag tracks retirement, exposed as the `Session.retired` property, and a retired session is never usable again. Create a new session to replace it. diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 85f90a3cfc..36d9e6f252 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -656,7 +656,7 @@ class BasicCrawlingContext: request: Request """Request object for the current page being processed.""" - session: Session | None + session: Session """Session object for the current page being processed.""" proxy_info: ProxyInfo | None diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py index 2559eff3a2..f13a8b6b48 100644 --- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py +++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py @@ -309,7 +309,7 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera """ # Without saving, the session cookies are still sent, but the response cookies go to a copy. cookies = None - if context.session is not None and not self._save_response_cookies: + if not self._save_response_cookies: cookies = SessionCookies(context.session.cookies) async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout: diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 9296285425..3f4a5bba69 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -70,7 +70,7 @@ from crawlee.http_clients import ImpitHttpClient from crawlee.request_loaders import MinIntervalSignal, RateLimitedSignal from crawlee.router import Router -from crawlee.sessions import SessionPool +from crawlee.sessions import Session, SessionPool from crawlee.statistics import Statistics, StatisticsState from crawlee.storages import Dataset, KeyValueStore, RequestQueue @@ -101,7 +101,7 @@ from crawlee.http_clients import HttpClient, HttpResponse from crawlee.proxy_configuration import ProxyConfiguration, ProxyInfo from crawlee.request_loaders import RequestManager - from crawlee.sessions import Session, SessionCookies + from crawlee.sessions import SessionCookies from crawlee.statistics import FinalStatistics from crawlee.storage_backends import StorageBackend from crawlee.storage_backends.models import DatasetItemsListPage @@ -167,9 +167,6 @@ class _BasicCrawlerOptions(TypedDict): If not set, crawling continues without depth restrictions. """ - use_session_pool: NotRequired[bool] - """Enable the use of a session pool for managing sessions during crawling.""" - retry_on_blocked: NotRequired[bool] """If True, the crawler attempts to bypass bot protections automatically.""" @@ -302,7 +299,6 @@ def __init__( max_requests_per_crawl: int | None = None, max_session_rotations: int = 10, max_crawl_depth: int | None = None, - use_session_pool: bool = True, retry_on_blocked: bool = True, additional_http_error_status_codes: Iterable[int] | None = None, ignore_http_error_status_codes: Iterable[int] | None = None, @@ -350,7 +346,6 @@ def __init__( this depth. The crawl depth starts at 0 for initial requests and increases with each subsequent level of links. Requests at the maximum depth will still be processed, but no new links will be enqueued from those requests. If not set, crawling continues without depth restrictions. - use_session_pool: Enable the use of a session pool for managing sessions during crawling. retry_on_blocked: If True, the crawler attempts to bypass bot protections automatically. additional_http_error_status_codes: Additional HTTP status codes to treat as errors, triggering automatic retries when encountered. @@ -465,7 +460,6 @@ def __init__( ) # Retry and session settings - self._use_session_pool = use_session_pool self._retry_on_blocked = retry_on_blocked # Logging setup @@ -662,11 +656,8 @@ def _stop_if_max_requests_count_exceeded(self) -> None: reason=f'The crawler has reached its limit of {self._max_requests_per_crawl} requests per crawl. ' ) - async def _get_session(self) -> Session | None: - """If session pool is being used, try to take a session from it.""" - if not self._use_session_pool: - return None - + async def _get_session(self) -> Session: + """Take a session from the session pool.""" return await wait_for( self._session_pool.get_session, timeout=self._internal_timeout, @@ -676,11 +667,8 @@ async def _get_session(self) -> Session | None: logger=self._logger, ) - async def _get_session_by_id(self, session_id: str | None) -> Session | None: - """If session pool is being used, try to take a session by id from it.""" - if not self._use_session_pool or not session_id: - return None - + async def _get_session_by_id(self, session_id: str) -> Session | None: + """Take the session with the given id from the session pool.""" return await wait_for( partial(self._session_pool.get_session_by_id, session_id), timeout=self._internal_timeout, @@ -690,13 +678,13 @@ async def _get_session_by_id(self, session_id: str | None) -> Session | None: logger=self._logger, ) - async def _get_proxy_info(self, request: Request, session: Session | None) -> ProxyInfo | None: + async def _get_proxy_info(self, request: Request, session: Session) -> ProxyInfo | None: """Retrieve a new ProxyInfo object based on crawler configuration and the current request and session.""" if not self._proxy_configuration: return None return await self._proxy_configuration.new_proxy_info( - session_id=session.id if session else None, + session_id=session.id, request=request, ) @@ -798,9 +786,8 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta self._statistics.reset() await self._statistics.reset_store() - if self._use_session_pool: - self._session_pool.reset() - await self._session_pool.reset_store() + self._session_pool.reset() + await self._session_pool.reset_store() if requests is not None: await self.add_requests(requests) @@ -945,7 +932,7 @@ async def _run_crawler(self) -> None: for cm in ( owned_concurrency_system, self._statistics, - self._session_pool if self._use_session_pool else None, + self._session_pool, self._http_client, self._crawler_state_rec_task, *self._additional_context_managers, @@ -1322,7 +1309,7 @@ async def _handle_error_handler_replacement( if context.request.session_rotation_count is not None: new_request.session_rotation_count = context.request.session_rotation_count - if retire_session and context.session: + if retire_session: context.session.retire() await request_manager.add_request(new_request) @@ -1386,8 +1373,7 @@ async def _handle_request_error(self, context: TCrawlingContext | BasicCrawlingC context.request.state = RequestState.ERROR raise - if context.session: - context.session.mark_bad() + context.session.mark_bad() async def _handle_failed_request(self, context: TCrawlingContext | BasicCrawlingContext, error: Exception) -> None: self._logger.error( @@ -1445,7 +1431,7 @@ def _get_only_inner_most_exception(self, error: BaseException) -> BaseException: def _prepare_send_request_function( self, - session: Session | None, + session: Session, proxy_info: ProxyInfo | None, register_deferred_cleanup: Callable[[DeferredCleanupCallback], None], ) -> SendRequestFunction: @@ -1630,8 +1616,14 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 self._logger.debug('Processing request', extra=request_log_extra(request)) + bound_session_missing = False if request.session_id: session = await self._get_session_by_id(request.session_id) + if session is None: + bound_session_missing = True + # The bound session isn't available, so `_check_request_collision` fails the request. A retired + # stand-in with the bound id gives the failed request handler a session. + session = Session(id=request.session_id, retired=True) else: session = await self._get_session() proxy_info = await self._get_proxy_info(request, session) @@ -1659,7 +1651,7 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 request.state = RequestState.REQUEST_HANDLER try: - self._check_request_collision(request, session) + self._check_request_collision(request, bound_session_missing=bound_session_missing) await self._run_request_handler(context=context) except TimeoutError as e: raise RequestHandlerError(e, context) from e @@ -1670,7 +1662,7 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 await self._mark_request_as_handled(request) - if session and session.is_usable: + if session.is_usable: session.mark_good() self._statistics.record_request_processing_finish(request.unique_key) @@ -1698,9 +1690,6 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002 await self._handle_request_error(primary_error.crawling_context, primary_error.wrapped_exception) except SessionError as session_error: - if not session: - raise RuntimeError('SessionError raised in a crawling context without a session') from session_error - request.state = RequestState.ERROR_HANDLER if self._should_retry_request(context, session_error): @@ -1902,33 +1891,33 @@ def _record_rate_limit_status_code( ) return False - def _raise_for_session_blocked_status_code(self, session: Session | None, status_code: int) -> None: + def _raise_for_session_blocked_status_code(self, session: Session, status_code: int) -> None: """Raise an exception if the given status code indicates the session is blocked. Args: - session: The session used for the request. If `None`, no check is performed. + session: The session used for the request. status_code: The HTTP status code to check. Raises: SessionError: If the status code indicates the session is blocked. """ - if session is not None and session.is_blocked_status_code( + if session.is_blocked_status_code( status_code=status_code, ignore_http_error_status_codes=self._ignore_http_error_status_codes, ): raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}') - def _check_request_collision(self, request: Request, session: Session | None) -> None: + def _check_request_collision(self, request: Request, *, bound_session_missing: bool) -> None: """Raise an exception if a request cannot access required resources. Args: request: The `Request` that might require specific resources (like a session). - session: The `Session` that was retrieved for the request, or `None` if not available. + bound_session_missing: Whether the `Session` bound to the `Request` isn't available in the session pool. Raises: RequestCollisionError: If the `Session` referenced by the `Request` is not available. """ - if self._use_session_pool and request.session_id and not session: + if bound_session_missing: raise RequestCollisionError( f'The Session (id: {request.session_id}) bound to the Request is no longer available in SessionPool' ) diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 2803ee3c18..72c0e5aafe 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -372,9 +372,8 @@ async def _navigate( context.register_deferred_cleanup(lambda: context.page.__aexit__(None, None, None)) - if context.session: - session_cookies = context.session.cookies.get_cookies_as_playwright_format() - await self._update_cookies(context.page, session_cookies) + session_cookies = context.session.cookies.get_cookies_as_playwright_format() + await self._update_cookies(context.page, session_cookies) if context.request.method != 'GET': # Call the notification only once @@ -592,7 +591,7 @@ async def _execute_post_navigation_hooks(self, context: TPostNavContext) -> Asyn async def _create_crawling_context(self, context: TPostNavContext) -> AsyncGenerator[TCrawlingContext, None]: yield self._build_crawling_context(context) - if context.session and self._save_response_cookies: + if self._save_response_cookies: pw_cookies = await self._get_cookies(context.page) context.session.cookies.set_cookies_from_playwright_format(pw_cookies) diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 813fe6e50e..59e0ff36ac 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -897,9 +897,8 @@ async def test_change_context_state_after_handling( async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: nonlocal used_session_id - if context.session is not None: - used_session_id = context.session.id - context.session.user_data['session_state'] = True + used_session_id = context.session.id + context.session.user_data['session_state'] = True try: # `page` is only available in the browser sub crawler, static crawling raises here. diff --git a/tests/integration/crawlers/_basic/test_basic_crawler.py b/tests/integration/crawlers/_basic/test_basic_crawler.py index eb08a4643d..285c2e0a28 100644 --- a/tests/integration/crawlers/_basic/test_basic_crawler.py +++ b/tests/integration/crawlers/_basic/test_basic_crawler.py @@ -83,7 +83,7 @@ async def test_session_rotation(server_url: URL) -> None: @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - session_ids.append(context.session.id if context.session else None) + session_ids.append(context.session.id) raise SessionError('Test error') await crawler.run([str(server_url)]) diff --git a/tests/integration/crawlers/_http/test_http_crawler.py b/tests/integration/crawlers/_http/test_http_crawler.py index 79b310b6f4..673c7aa427 100644 --- a/tests/integration/crawlers/_http/test_http_crawler.py +++ b/tests/integration/crawlers/_http/test_http_crawler.py @@ -121,22 +121,17 @@ async def test_handles_client_errors( @pytest.mark.parametrize( - ('ignore_http_error_status_codes', 'use_session_pool', 'expected_session_rotate', 'expected_number_error'), + ('ignore_http_error_status_codes', 'expected_session_rotate', 'expected_number_error'), [ # change session and retry for no block 4xx statuses - pytest.param([], True, 4, 1, id='default_behavior'), - # error without retry for all 4xx statuses - pytest.param([], False, 0, 1, id='default_behavior_without_session_pool'), - # take as successful status codes from the `ignore_http_error_status_codes` list with Session Pool - pytest.param([403], True, 0, 0, id='ignore_error_status_codes'), - # take as successful status codes from the `ignore_http_error_status_codes` list without Session Pool - pytest.param([403], False, 0, 0, id='ignore_error_status_codes_without_session_pool'), + pytest.param([], 4, 1, id='default_behavior'), + # take as successful status codes from the `ignore_http_error_status_codes` list + pytest.param([403], 0, 0, id='ignore_error_status_codes'), ], ) async def test_handles_session_block_errors( *, ignore_http_error_status_codes: list[int], - use_session_pool: bool, expected_session_rotate: int, expected_number_error: int, mock_request_handler: AsyncMock, @@ -147,7 +142,6 @@ async def test_handles_session_block_errors( ignore_http_error_status_codes=ignore_http_error_status_codes, max_request_retries=3, max_session_rotations=5, - use_session_pool=use_session_pool, ) await crawler.add_requests([str(server_url / 'status/403')]) @@ -185,7 +179,7 @@ async def test_stores_cookies(http_client: HttpClient, server_url: URL) -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: visit(context.request.url) - track_session_usage(context.session.id if context.session else None) + track_session_usage(context.session.id) await crawler.run( [ @@ -441,9 +435,6 @@ async def test_isolation_cookies(http_client: HttpClient, server_url: URL) -> No @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - if not context.session: - return - sessions_ids.append(context.session.id) if context.request.unique_key not in {'1', '2'}: @@ -496,7 +487,7 @@ async def test_store_complex_cookies(server_url: URL) -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: visit(context.request.url) - track_session_usage(context.session.id if context.session else None) + track_session_usage(context.session.id) await crawler.run([str(server_url / 'set_complex_cookies')]) @@ -597,7 +588,7 @@ async def test_save_response_cookies_disabled(server_url: URL) -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: - session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else [] + session_cookies = context.session.cookies.get_cookies_as_dicts() handler_call( sent_cookies=json.loads(await context.http_response.read())['cookies'], session_cookies={cookie['name'] for cookie in session_cookies}, @@ -617,7 +608,7 @@ async def test_context_send_request_saves_cookies(server_url: URL) -> None: @crawler.router.default_handler async def handler(context: HttpCrawlingContext) -> None: await context.send_request(str(server_url.with_path('set_cookies').extend_query(b=2))) - session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else [] + session_cookies = context.session.cookies.get_cookies_as_dicts() handler_call(session_cookies={cookie['name'] for cookie in session_cookies}) await crawler.run([str(server_url / 'cookies')]) diff --git a/tests/integration/crawlers/_playwright/test_playwright_crawler.py b/tests/integration/crawlers/_playwright/test_playwright_crawler.py index 25750d60f8..b4372d501c 100644 --- a/tests/integration/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/integration/crawlers/_playwright/test_playwright_crawler.py @@ -446,9 +446,6 @@ async def test_isolation_cookies(*, use_incognito_pages: bool, server_url: URL) @crawler.router.default_handler async def handler(context: PlaywrightCrawlingContext) -> None: - if not context.session: - return - sessions_ids.append(context.session.id) sessions[context.session.id] = context.session response_data = json.loads(await context.response.text()) @@ -530,8 +527,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: # Simulate cookies installed from an external source in the browser await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}]) - if context.session: - session_ids.append(context.session.id) + session_ids.append(context.session.id) await crawler.run([str(server_url)]) @@ -555,7 +551,7 @@ async def test_save_response_cookies_disabled(server_url: URL) -> None: async def request_handler(context: PlaywrightCrawlingContext) -> None: await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}]) handler_call( - session_id=context.session.id if context.session else None, + session_id=context.session.id, page_cookies={cookie['name'] for cookie in await context.page.context.cookies()}, ) @@ -658,9 +654,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: cookies = await context.page.context.cookies() playwright_cookies.extend(cookies) - if context.session: - context.session.cookies.set_cookies_from_playwright_format(cookies) - session_cookies.extend(context.session.cookies.get_cookies_as_dicts()) + context.session.cookies.set_cookies_from_playwright_format(cookies) + session_cookies.extend(context.session.cookies.get_cookies_as_dicts()) await crawler.run([str(server_url / 'set_complex_cookies')]) @@ -1521,5 +1516,4 @@ async def handler(context: PlaywrightCrawlingContext) -> None: assert handled[0].request.retry_count == 0 assert (handled[0].request.session_rotation_count or 0) == 0 session = handled[0].session - assert session is not None assert session.error_score == 0 diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index ddde8c4bc3..0a96197512 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -625,14 +625,16 @@ async def test_send_request_forwards_arguments_to_http_client() -> None: http_client = AsyncMock(spec=HttpClient) response = Mock(spec=HttpResponse) http_client.send_request.return_value = response - crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1) + crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1) timeout = timedelta(seconds=12) cookies = SessionCookies({'name': 'value'}) result = None + session = None @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - nonlocal result + nonlocal result, session + session = context.session result = await context.send_request( 'https://example.com', method='POST', @@ -646,12 +648,13 @@ async def handler(context: BasicCrawlingContext) -> None: await crawler.run(['https://a.placeholder.com']) assert result is response + assert session is not None http_client.send_request.assert_awaited_once_with( url='https://example.com', method='POST', payload=b'Hello, world!', headers={'Content-Type': 'text/plain'}, - session=None, + session=session, cookies=cookies, proxy_info=None, timeout=timeout, @@ -663,7 +666,7 @@ async def test_send_request_default_timeout() -> None: """Test that `context.send_request` without a `timeout` leaves the timeout to the default of the HTTP client.""" http_client = AsyncMock(spec=HttpClient) http_client.send_request.return_value = Mock(spec=HttpResponse) - crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1) + crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1) @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: @@ -2634,10 +2637,10 @@ async def test_session_retire_in_user_handler(*, retire: bool) -> None: @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - if context.session: - sessions.append(context.session.id) + sessions.append(context.session.id) - context.session.retire() if retire else None + if retire: + context.session.retire() await context.add_requests(['https://b.placeholder.com']) @@ -2658,8 +2661,7 @@ async def test_bound_session_to_request() -> None: @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - if context.session: - used_sessions.append(context.session.id) + used_sessions.append(context.session.id) requests = [ Request.from_url('https://a.placeholder.com', session_id=check_session.id, always_enqueue=True) @@ -2690,8 +2692,7 @@ def create_session() -> Session: @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: - if context.session: - used_sessions.append(context.session.id) + used_sessions.append(context.session.id) requests = [ Request.from_url('https://a.placeholder.com', session_id=str(session_id), use_extended_unique_key=True) @@ -2732,6 +2733,21 @@ async def error_req_hook(context: BasicCrawlingContext, error: Exception) -> Non assert error_handler_mock.call_count == 1 +async def test_missing_bound_session_gives_retired_stand_in() -> None: + """Test that a request bound to a missing session reaches the failed request handler with a retired session.""" + failed_request_handler = AsyncMock() + crawler = BasicCrawler(request_handler=AsyncMock()) + crawler.failed_request_handler(failed_request_handler) + + await crawler.run([Request.from_url('https://a.placeholder.com', session_id='missing')]) + + failed_request_handler.assert_awaited_once() + context, error = failed_request_handler.call_args.args + assert isinstance(error, RequestCollisionError) + assert context.session.id == 'missing' + assert context.session.retired + + async def test_handles_session_error_in_failed_request_handler() -> None: crawler = BasicCrawler(max_session_rotations=1) handler_requests = set() @@ -3167,7 +3183,7 @@ async def error_handler(context: BasicCrawlingContext, error: Exception) -> Requ if isinstance(context.request.user_data['request_state'], list): context.request.user_data['request_state'].append('error') - if context.session and isinstance(context.session.user_data['session_state'], list): + if isinstance(context.session.user_data['session_state'], list): context.session.user_data['session_state'].append('error') context.session.cookies['error'] = 'yes' @@ -3175,7 +3191,7 @@ async def error_handler(context: BasicCrawlingContext, error: Exception) -> Requ async def handler(context: BasicCrawlingContext) -> None: if isinstance(context.request.user_data['request_state'], list): context.request.user_data['request_state'].append(f'modified_{context.request.retry_count}') - if context.session and isinstance(context.session.user_data['session_state'], list): + if isinstance(context.session.user_data['session_state'], list): context.session.user_data['session_state'].append(f'modified_{context.request.retry_count}') context.session.cookies[f'modified_{context.request.retry_count}'] = 'yes' @@ -3545,7 +3561,6 @@ async def test_handler_raised_throttled_error_is_deferred(monkeypatch: pytest.Mo assert context.request.retry_count == 0 assert stats.requests_finished == 1 assert stats.requests_failed == 0 - assert context.session is not None assert context.session.error_score == 0 assert throttler._domain_states['throttled.placeholder.com'].consecutive_429_count == 1 diff --git a/tests/unit/crawlers/_http/test_http_crawler.py b/tests/unit/crawlers/_http/test_http_crawler.py index b9076a0b88..0ef3c94208 100644 --- a/tests/unit/crawlers/_http/test_http_crawler.py +++ b/tests/unit/crawlers/_http/test_http_crawler.py @@ -127,7 +127,6 @@ async def handler(context: HttpCrawlingContext) -> None: assert request.retry_count == 0 assert (request.session_rotation_count or 0) == 0 session = handled[0].session - assert session is not None assert session.error_score == 0 assert session.is_usable From 9ac1328f137d4c176866284e48abab20252e282f Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Sun, 15 Nov 2026 22:22:11 +0000 Subject: [PATCH 2/2] enter and exit only the objects a crawler creates --- docs/06_upgrading/upgrading_to_v2.md | 39 +++- src/crawlee/_utils/context.py | 33 ++- src/crawlee/autoscaling/_types.py | 10 +- .../_adaptive_playwright_crawler.py | 23 +- .../_rendering_type_predictor.py | 20 +- src/crawlee/crawlers/_basic/_basic_crawler.py | 110 ++++++---- .../_playwright/_playwright_crawler.py | 77 +++---- .../crawlers/_pydantic_ai/_base_extractor.py | 39 +++- .../_pydantic_ai/_pydantic_ai_crawler.py | 20 +- src/crawlee/crawlers/_pydantic_ai/_types.py | 6 +- .../crawlers/_stagehand/_stagehand_crawler.py | 4 +- src/crawlee/http_clients/_base.py | 10 +- src/crawlee/sessions/_session_pool.py | 27 ++- src/crawlee/statistics/_statistics.py | 50 +++-- .../test_adaptive_playwright_crawler.py | 22 +- .../crawlers/_http/test_http_crawler.py | 51 +++-- .../_playwright/test_playwright_crawler.py | 31 ++- .../_pydantic_ai/test_pydantic_ai_crawler.py | 3 +- tests/unit/_statistics/test_enter_exit.py | 101 +++++++++ .../test_adaptive_playwright_crawler.py | 22 ++ .../_adaptive_playwright/test_predictor.py | 63 ++++++ .../crawlers/_basic/test_basic_crawler.py | 197 ++++++++++++++++-- .../unit/crawlers/_http/test_http_crawler.py | 16 +- .../_playwright/test_playwright_crawler.py | 21 ++ .../_pydantic_ai/test_direct_extractor.py | 11 + .../_pydantic_ai/test_pydantic_ai_crawler.py | 47 ++++- .../_stagehand/test_stagehand_crawler.py | 16 +- tests/unit/http_clients/test_http_clients.py | 22 ++ tests/unit/sessions/test_session_pool.py | 65 ++++++ 29 files changed, 951 insertions(+), 205 deletions(-) create mode 100644 tests/unit/_statistics/test_enter_exit.py diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md index 0275f57883..e00403b4f9 100644 --- a/docs/06_upgrading/upgrading_to_v2.md +++ b/docs/06_upgrading/upgrading_to_v2.md @@ -489,6 +489,35 @@ class MyHttpClient(HttpClient): `SitemapRequestLoader.fetch_next_request` skips a URL while its request is still in progress, for example when another sitemap lists it too. In v1, it returned a request for the URL again, and `get_total_count` counted the URL twice. +## Crawlers manage only what they create + +A crawler now enters and exits only the objects it creates, the same as Crawlee JS. An object you pass is yours to enter and exit. The `Statistics` and the event managers are the exceptions, since the crawler enters them on every run. You can enter an event manager yourself as well. In v1, the crawler entered a passed object that wasn't active yet and exited it when its run finished, even while another crawler still used it. + +What you do with an object you pass depends on its type: + +- `SessionPool`, `BrowserPool`, an HTTP client and the `rendering_type_predictor` of `AdaptivePlaywrightCrawler`: enter it with `async with` before you run the crawler, or `run()` raises a `RuntimeError`. Keep it open until every crawler using it has finished. +- The `extractor` of `PydanticAiCrawler`: enter it the same way. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`. +- `Statistics`: don't enter it yourself, or `run()` raises a `RuntimeError`. In v1, the crawler left entered statistics alone. Crawlers that run one after another can share an instance, including another crawler's statistics. The instance adds up their runs, so the `max_requests_per_crawl` of the second crawler counts the requests of the first one too. When a crawler runs again, it resets the instance, including what the other crawlers counted. Two crawlers can't run with the same instance at the same time. + +For example: + +```python +# v1 +crawler = ParselCrawler(session_pool=SessionPool(max_pool_size=10)) +await crawler.run(urls) + +# v2 +session_pool = SessionPool(max_pool_size=10) +crawler = ParselCrawler(session_pool=session_pool) + +async with session_pool: + await crawler.run(urls) +``` + +When a crawler runs again, it resets only a session pool it created. A `SessionPool` you pass keeps its sessions. In v1, a repeated `run()` reset a passed pool too. + +`RenderingTypePredictor` has a new `active` property, which `initialize()` and `clear()` of the base class keep up to date. If your predictor overrides them, call `super()`. + ## Resetting statistics and the session pool `Statistics.reset()` is now synchronous and resets only the in-memory statistics, the same as Crawlee JS. In v1, it also cleared the persisted record. To clear the record too, call the new `reset_store()` once the statistics are exited: @@ -593,12 +622,14 @@ To keep the response cookies out of the sessions, pass `save_response_cookies=Fa ```python # v1 crawler = ParselCrawler(use_session_pool=False) +await crawler.run(urls) # v2 -crawler = ParselCrawler( - save_response_cookies=False, - session_pool=SessionPool(create_session_settings={'max_usage_count': 1}), -) +session_pool = SessionPool(create_session_settings={'max_usage_count': 1}) +crawler = ParselCrawler(save_response_cookies=False, session_pool=session_pool) + +async with session_pool: + await crawler.run(urls) ``` ## Session retirement is permanent diff --git a/src/crawlee/_utils/context.py b/src/crawlee/_utils/context.py index 9faa994376..ce5e519c47 100644 --- a/src/crawlee/_utils/context.py +++ b/src/crawlee/_utils/context.py @@ -3,11 +3,42 @@ import inspect from collections.abc import Callable from functools import wraps -from typing import Any, TypeVar, cast +from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast + +if TYPE_CHECKING: + from types import TracebackType T = TypeVar('T', bound=Callable[..., Any]) +class Collaborator(Protocol): + """An async context manager that reports through `active` whether it's entered.""" + + @property + def active(self) -> bool: + """Whether the object is entered and not exited yet.""" + + async def __aenter__(self) -> Any: + """Prepare the object for use. + + Raises: + RuntimeError: If the object is already active. + """ + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + exc_traceback: TracebackType | None, + /, + ) -> Any: + """Release what the object holds. + + Raises: + RuntimeError: If the object is not active. + """ + + def ensure_context(method: T) -> T: """Ensure the (async) context manager is initialized before executing the method. diff --git a/src/crawlee/autoscaling/_types.py b/src/crawlee/autoscaling/_types.py index 678a4f3590..ac872acf3c 100644 --- a/src/crawlee/autoscaling/_types.py +++ b/src/crawlee/autoscaling/_types.py @@ -7,6 +7,7 @@ from pydantic import Field from pydantic.dataclasses import dataclass as pydantic_dataclass +from crawlee._utils.context import Collaborator from crawlee._utils.docs import docs_group @@ -104,12 +105,13 @@ class ConcurrencyConsumer: @docs_group('Autoscaling') -class ConcurrencySystemInterface(Protocol): +class ConcurrencySystemInterface(Collaborator, Protocol): """The contract between an `AutoscaledPool` and the component that governs its concurrency. The system answers whether there is free compute for one more task and tracks the budget that tasks are booked against. `ConcurrencySystem` is the default implementation. Anything matching this contract can take its place, - without depending on its internals. + without depending on its internals. An implementation is an async context manager, and `AutoscaledPool.run` refuses + to run on one that isn't entered. Every allocation method is told which `ConcurrencyConsumer` is asking, so an implementation may allocate per consumer. `ConcurrencySystem` does not, and serves whoever asks first, which can starve a consumer that joins @@ -127,10 +129,6 @@ def desired_concurrency(self) -> int: def current_concurrency(self) -> int: """The number of parallel tasks currently booked against this system.""" - @property - def active(self) -> bool: - """Whether the system is ready to be booked against. `AutoscaledPool.run` refuses to run when it is not.""" - def has_capacity_for_task(self, consumer: ConcurrencyConsumer) -> bool: """Report whether one more task may start right now on behalf of `consumer`. diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py index 8f2e74ab2f..7af0e76e9d 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py @@ -134,7 +134,8 @@ def __init__( Args: rendering_type_predictor: Object that implements RenderingTypePredictor and is capable of predicting which - rendering method should be used. If None, then `DefaultRenderingTypePredictor` is used. + rendering method should be used. If None, then `DefaultRenderingTypePredictor` is used. Keep a + predictor passed in open with `async with` until every crawler using it has finished. result_checker: Function that evaluates whether crawling result is valid or not. result_comparator: Function that compares two crawling results and decides whether they are equivalent. static_parser: Implementation of `AbstractHttpParser`. Parser that will be used for static crawling. @@ -143,11 +144,10 @@ def __init__( save_response_cookies: Whether to save the response cookies to the session, in both the static and the browser sub crawler. statistics: A custom `Statistics[AdaptivePlaywrightCrawlerStatisticState]` instance, allowing the use of - non-default configuration. + non-default configuration. The crawler enters it on every run, so don't enter it yourself. kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`. """ # Adaptive crawling related. - self.rendering_type_predictor = rendering_type_predictor or DefaultRenderingTypePredictor() self.result_checker = result_checker or (lambda _: True) self.result_comparator = result_comparator or create_default_comparator(result_checker) @@ -163,6 +163,8 @@ def __init__( super().__init__(statistics=adaptive_statistics, **kwargs) + self.rendering_type_predictor = self._own_or_borrow(rendering_type_predictor, DefaultRenderingTypePredictor) + # Sub crawlers related. playwright_crawler_specific_kwargs = playwright_crawler_specific_kwargs or _PlaywrightCrawlerAdditionalOptions() @@ -228,13 +230,14 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon static_crawler.post_navigation_hook(adaptive_post_navigation_hook_static) playwright_crawler.post_navigation_hook(adaptive_post_navigation_hook_pw) - self._additional_context_managers = [ - *self._additional_context_managers, - self.rendering_type_predictor, - static_crawler.statistics, - playwright_crawler.statistics, - playwright_crawler._browser_pool, # noqa: SLF001 # Intentional access to private member. - ] + # The sub crawlers never run themselves, so this crawler enters and exits their statistics. + self._owned_context_managers.extend([static_crawler.statistics, playwright_crawler.statistics]) + + # The default pool is the one the browser sub crawler built. + self._own_or_borrow( + playwright_crawler_specific_kwargs.get('browser_pool'), + lambda: playwright_crawler._browser_pool, # noqa: SLF001 # Intentional access to private member. + ) # Sub crawler pipeline related self._pw_context_pipeline = playwright_crawler._context_pipeline # noqa:SLF001 # Intentional access to private member. diff --git a/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py b/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py index de80baf56f..dd3e979ba1 100644 --- a/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py +++ b/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py @@ -68,6 +68,11 @@ def __init__(self) -> None: # Flag to indicate the state. self._active = False + @property + def active(self) -> bool: + """Whether the predictor is initialized and not cleared since.""" + return self._active + @abstractmethod def predict(self, request: Request) -> RenderingTypePrediction: """Get `RenderingTypePrediction` based on the input request. @@ -164,15 +169,20 @@ async def initialize(self) -> None: await super().initialize() if not self._state.is_initialized: - await self._state.initialize() + try: + await self._state.initialize() + except BaseException: + await super().clear() + raise @override async def clear(self) -> None: """Clear the predictor state.""" - await super().clear() - - if self._state.is_initialized: - await self._state.teardown() + try: + if self._state.is_initialized: + await self._state.teardown() + finally: + await super().clear() @override def predict(self, request: Request) -> RenderingTypePrediction: diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 3f4a5bba69..76314667fa 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -42,6 +42,7 @@ SendRequestFunction, SkippedReason, ) +from crawlee._utils.context import Collaborator from crawlee._utils.docs import docs_group from crawlee._utils.file import atomic_write, export_csv_to_stream, export_json_to_stream from crawlee._utils.globs import check_url_patterns @@ -111,6 +112,7 @@ TRequestIterator = TypeVar('TRequestIterator', str, Request) TParams = ParamSpec('TParams') T = TypeVar('T') +TCollaborator = TypeVar('TCollaborator', bound=Collaborator) ErrorHandler = Callable[[TCrawlingContext, Exception], Awaitable[Request | None]] FailedRequestHandler = Callable[[TCrawlingContext, Exception], Awaitable[None]] @@ -133,13 +135,15 @@ class _BasicCrawlerOptions(TypedDict): """Manager of requests that should be processed by the crawler.""" session_pool: NotRequired[SessionPool] - """A custom `SessionPool` instance, allowing the use of non-default configuration.""" + """A custom `SessionPool` instance, allowing the use of non-default configuration. Keep it open with `async with` + until every crawler using it has finished.""" proxy_configuration: NotRequired[ProxyConfiguration] """HTTP proxy configuration used when making requests.""" http_client: NotRequired[HttpClient] - """HTTP client used by `BasicCrawlingContext.send_request` method.""" + """HTTP client used by `BasicCrawlingContext.send_request` method. Keep it open with `async with` until every + crawler using it has finished.""" max_request_retries: NotRequired[int] """Specifies the maximum number of retries allowed for a request if its processing fails. @@ -175,7 +179,8 @@ class _BasicCrawlerOptions(TypedDict): concurrency_system: NotRequired[ConcurrencySystemInterface] """A concurrency system to use instead of the default one, e.g. one shared between crawlers to cap their combined - concurrency. Cannot be combined with `concurrency_settings`.""" + concurrency. Keep it open with `async with` until every crawler using it has finished. Cannot be combined with + `concurrency_settings`.""" request_handler_timeout: NotRequired[timedelta] """Maximum duration allowed for a single request handler to run.""" @@ -205,8 +210,9 @@ class _BasicCrawlerOptions(TypedDict): """HTTP status codes that are typically considered errors but should be treated as successful responses.""" _additional_context_managers: NotRequired[Sequence[AbstractAsyncContextManager]] - """Additional context managers used throughout the crawler lifecycle. Intended for use by - subclasses rather than direct instantiation of `BasicCrawler`.""" + """Additional context managers the crawler owns, entered and exited on every run. A subclass lists the defaults it + builds here, so that they aren't taken for objects passed in. Intended for use by subclasses rather than direct + instantiation of `BasicCrawler`.""" respect_robots_txt_file: NotRequired[bool] """If set to `True`, the crawler will automatically try to fetch the robots.txt file for each domain, @@ -238,7 +244,8 @@ class _BasicCrawlerOptionsGeneric(TypedDict, Generic[TCrawlingContext, TStatisti subclasses rather than direct instantiation of `BasicCrawler`.""" statistics: NotRequired[Statistics[TStatisticsState]] - """A custom `Statistics` instance, allowing the use of non-default configuration.""" + """A custom `Statistics` instance, allowing the use of non-default configuration. The crawler enters it on every + run, so don't enter it yourself. Crawlers can share it only when they run one after another.""" class BasicCrawlerOptions( @@ -326,9 +333,11 @@ def __init__( event_manager: The event manager for managing events for the crawler and all its components. storage_backend: The storage backend for managing storages for the crawler and all its components. request_manager: Manager of requests that should be processed by the crawler. - session_pool: A custom `SessionPool` instance, allowing the use of non-default configuration. + session_pool: A custom `SessionPool` instance, allowing the use of non-default configuration. Keep it open + with `async with` until every crawler using it has finished. proxy_configuration: HTTP proxy configuration used when making requests. - http_client: HTTP client used by `BasicCrawlingContext.send_request` method. + http_client: HTTP client used by `BasicCrawlingContext.send_request` method. Keep it open with `async with` + until every crawler using it has finished. request_handler: A callable responsible for handling requests. max_request_retries: Specifies the maximum number of retries allowed for a request if its processing fails. This includes retries due to navigation errors or errors thrown from user-supplied functions @@ -357,7 +366,9 @@ def __init__( crawlers to cap their combined concurrency. Keep it open with `async with` until every crawler using it has finished. Cannot be combined with `concurrency_settings`. request_handler_timeout: Maximum duration allowed for a single request handler to run. - statistics: A custom `Statistics` instance, allowing the use of non-default configuration. + statistics: A custom `Statistics` instance, allowing the use of non-default configuration. The crawler + enters it on every run, so don't enter it yourself. Crawlers can share it only when they run one after + another. abort_on_error: If True, the crawler stops immediately when any request handler error occurs. keep_alive: If True, it will keep crawler alive even if there are no requests in queue. Use `crawler.stop()` to exit the crawler. @@ -378,7 +389,8 @@ def __init__( it, the ID follows the order in which crawlers are created. _context_pipeline: Enables extending the request lifecycle and modifying the crawling context. Intended for use by subclasses rather than direct instantiation of `BasicCrawler`. - _additional_context_managers: Additional context managers used throughout the crawler lifecycle. + _additional_context_managers: Additional context managers the crawler owns, entered and exited on every + run. A subclass lists the defaults it builds here, so that they aren't taken for objects passed in. Intended for use by subclasses rather than direct instantiation of `BasicCrawler`. """ self._id = str(BasicCrawler.__next_id) if id is None else id @@ -404,9 +416,14 @@ def __init__( config = self._service_locator.get_configuration() + # The crawler enters and exits what it owns on every run, and only checks that the collaborators passed in are + # active. + self._owned_context_managers: list[AbstractAsyncContextManager] = [*(_additional_context_managers or [])] + self._borrowed_collaborators: list[Collaborator] = [] + # Core components self._request_manager = request_manager - self._session_pool = session_pool or SessionPool() + self._session_pool = self._own_or_borrow(session_pool, SessionPool) self._proxy_configuration = proxy_configuration self._additional_http_error_status_codes = ( @@ -416,7 +433,7 @@ def __init__( set(ignore_http_error_status_codes) if ignore_http_error_status_codes else set() ) - self._http_client = http_client or ImpitHttpClient() + self._http_client = self._own_or_borrow(http_client, ImpitHttpClient) # Request router setup self._router: Router[TCrawlingContext] | None = None @@ -497,9 +514,6 @@ async def persist_state_factory() -> KeyValueStore: ), ) - # Additional context managers to enter and exit - self._additional_context_managers = _additional_context_managers or [] - # Internal, not explicitly configurable components self._robots_txt_file_cache: LRUCache[str, RobotsTxtFile] = LRUCache(maxsize=1000) self._robots_txt_lock = asyncio.Lock() @@ -513,6 +527,8 @@ async def persist_state_factory() -> KeyValueStore: self._concurrency_settings = concurrency_settings self._borrowed_concurrency_system = concurrency_system + if concurrency_system is not None: + self._borrowed_collaborators.append(concurrency_system) # The default system is built per run, so that a run is not scaled on the load and concurrency of the previous # one. A supplied system is known up front. @@ -531,6 +547,21 @@ async def persist_state_factory() -> KeyValueStore: # The last run started on this instance, which stays readable after it finishes. self._last_run: CrawlerRun[TStatisticsState] | None = None + def _own_or_borrow(self, passed: TCollaborator | None, build_default: Callable[[], TCollaborator]) -> TCollaborator: + """Return the collaborator passed in, or build the default one, and record which of them the crawler owns. + + A collaborator passed in that a subclass listed in `_additional_context_managers` counts as the crawler's own. + """ + if passed is None: + default = build_default() + self._owned_context_managers.append(default) + return default + + if passed not in self._owned_context_managers: + self._borrowed_collaborators.append(passed) + + return passed + @property def log(self) -> logging.Logger: """The logger used by the crawler.""" @@ -771,6 +802,8 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta requests: The requests to be enqueued before the crawler starts. Raises: + RuntimeError: If the crawler is already running, an object passed to it, such as a `SessionPool`, isn't + entered, or its statistics are already active. PersistentRateLimitError: If a domain throttled by a `ThrottlingRequestManager` has rate-limited every request for longer than the manager's `max_domain_stall` and no other requests are left. """ @@ -779,6 +812,19 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta 'This crawler instance is already running, you can add more requests to it via `crawler.add_requests()`' ) + for collaborator in self._borrowed_collaborators: + if not collaborator.active: + raise RuntimeError( + f'The {type(collaborator).__name__} passed to the crawler is not active. Enter it with ' + '`async with` before running the crawler.' + ) + + if self._statistics.active: + raise RuntimeError( + "The statistics of the crawler are already active. The crawler enters them itself, so don't enter " + 'them yourself. Crawlers can share statistics only when they run one after another.' + ) + self._last_run = self._create_run() try: @@ -786,8 +832,10 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta self._statistics.reset() await self._statistics.reset_store() - self._session_pool.reset() - await self._session_pool.reset_store() + # A session pool passed in belongs to the caller, who may still use it in another crawler. + if self._session_pool in self._owned_context_managers: + self._session_pool.reset() + await self._session_pool.reset_store() if requests is not None: await self.add_requests(requests) @@ -922,24 +970,16 @@ async def _run_crawler(self) -> None: # A system supplied by the caller is only borrowed, and its lifecycle belongs to whoever created it. The one # the crawler builds for this run is entered and exited below. - owned_concurrency_system = self._concurrency_system if self._borrowed_concurrency_system is None else None - - # Collect the context managers to be entered. Context managers that are already active are excluded, - # as they were likely entered by the caller, who will also be responsible for exiting them. - contexts_to_enter.extend( - [ - cm - for cm in ( - owned_concurrency_system, - self._statistics, - self._session_pool, - self._http_client, - self._crawler_state_rec_task, - *self._additional_context_managers, - ) - if cm and getattr(cm, 'active', False) is False - ] - ) + if self._borrowed_concurrency_system is None: + contexts_to_enter.append(self._concurrency_system) + + # The statistics count only while they're entered, so the crawler enters them on every run, even when they + # were passed in. + contexts_to_enter.append(self._statistics) + contexts_to_enter.extend(self._owned_context_managers) + + # The status messages of the recurring task read the statistics, so it starts after them and stops before them. + contexts_to_enter.append(self._crawler_state_rec_task) async with AsyncExitStack() as exit_stack: for context in contexts_to_enter: diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 72c0e5aafe..e3e19c0472 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -129,7 +129,8 @@ def __init__( """Initialize a new instance. Args: - browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages. + browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages. Keep it + open with `async with` until every crawler using it has finished. user_data_dir: Path to a user data directory, which stores browser session data like cookies and local storage. browser_type: The type of browser to launch: @@ -162,49 +163,48 @@ def __init__( self._shared_navigation_timeouts: dict[int, SharedTimeout] = {} self._save_response_cookies = save_response_cookies - if browser_pool: - # Raise an exception if browser_pool is provided together with other browser-related arguments. - if any( - param not in [None, 'default'] - for param in ( - user_data_dir, - use_incognito_pages, - headless, - browser_type, - browser_launch_options, - browser_new_context_options, - fingerprint_generator, - ) - ): - raise ValueError( - 'You cannot provide `headless`, `browser_type`, `browser_launch_options`, ' - '`browser_new_context_options`, `use_incognito_pages`, `user_data_dir` or ' - '`fingerprint_generator` arguments when `browser_pool` is provided.' - ) + if browser_pool and any( + param not in [None, 'default'] + for param in ( + user_data_dir, + use_incognito_pages, + headless, + browser_type, + browser_launch_options, + browser_new_context_options, + fingerprint_generator, + ) + ): + raise ValueError( + 'You cannot provide `headless`, `browser_type`, `browser_launch_options`, ' + '`browser_new_context_options`, `use_incognito_pages`, `user_data_dir` or ' + '`fingerprint_generator` arguments when `browser_pool` is provided.' + ) - # If browser_pool is not provided, create a new instance of BrowserPool with specified arguments. - else: - if fingerprint_generator == 'default': - generator_browser_type: list[Literal['chrome', 'firefox', 'safari', 'edge']] | None = ( - [fingerprint_browser_type_from_playwright_browser_type(browser_type)] if browser_type else None - ) + # Before `super().__init__()`, so that a failure here doesn't take up a crawler id. + generator: FingerprintGenerator | None = None + if fingerprint_generator != 'default': + generator = fingerprint_generator + elif browser_pool is None: + generator_browser_type: list[Literal['chrome', 'firefox', 'safari', 'edge']] | None = ( + [fingerprint_browser_type_from_playwright_browser_type(browser_type)] if browser_type else None + ) - fingerprint_generator = DefaultFingerprintGenerator( - header_options=HeaderGeneratorOptions(browsers=generator_browser_type) - ) + generator = DefaultFingerprintGenerator( + header_options=HeaderGeneratorOptions(browsers=generator_browser_type) + ) - browser_pool = BrowserPool.with_default_plugin( + def build_browser_pool() -> BrowserPool: + return BrowserPool.with_default_plugin( headless=headless, browser_type=browser_type, user_data_dir=user_data_dir, browser_launch_options=browser_launch_options, browser_new_context_options=browser_new_context_options, use_incognito_pages=use_incognito_pages, - fingerprint_generator=fingerprint_generator, + fingerprint_generator=generator, ) - self._browser_pool = browser_pool - # Compose the context pipeline with the Playwright-specific context enhancer. kwargs['_context_pipeline'] = ( ContextPipeline() @@ -216,11 +216,13 @@ def __init__( .compose(self._handle_blocked_request_by_content) .compose(self._create_crawling_context) ) - kwargs['_additional_context_managers'] = [self._browser_pool] self._pre_navigation_hooks: list[Callable[[TPreNavContext], Awaitable[None]]] = [] self._post_navigation_hooks: list[Callable[[TPostNavContext], Awaitable[None]]] = [] - kwargs['http_client'] = PlaywrightHttpClient() if not kwargs.get('http_client') else kwargs['http_client'] + if kwargs.get('http_client') is None: + http_client = PlaywrightHttpClient() + kwargs['http_client'] = http_client + kwargs['_additional_context_managers'] = [*kwargs.get('_additional_context_managers', []), http_client] # Lower the default concurrency for browser crawlers, unless the caller configured it themselves. if kwargs.get('concurrency_settings') is None and kwargs.get('concurrency_system') is None: @@ -231,6 +233,8 @@ def __init__( super().__init__(**kwargs) + self._browser_pool = self._own_or_borrow(browser_pool, build_browser_pool) + async def _manage_shared_navigation_timeout( self, context: BasicCrawlingContext ) -> AsyncGenerator[BasicCrawlingContext, None]: @@ -639,7 +643,8 @@ class _PlaywrightCrawlerAdditionalOptions(TypedDict): """ browser_pool: NotRequired[BrowserPool] - """A `BrowserPool` instance to be used for launching the browsers and getting pages.""" + """A `BrowserPool` instance to be used for launching the browsers and getting pages. Keep it open with `async with` + until every crawler using it has finished.""" browser_type: NotRequired[BrowserType] """The type of browser to launch: diff --git a/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py b/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py index cc24f91d6b..82dcc03c6c 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py +++ b/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py @@ -12,6 +12,9 @@ from ._types import PydanticAiUsageStats if TYPE_CHECKING: + from types import TracebackType + from typing import Self + from pydantic_ai.models import Model from pydantic_ai.usage import UsageLimits @@ -27,7 +30,7 @@ class BasePydanticAiHtmlExtractor(ABC): An HTML extractor turns a page into a validated Pydantic model with the help of an LLM. This abstract base implements the parts the built-in extractors share: resolving the model, composing the task instructions with - the distiller's prompt notes, and accumulating token usage. + the distiller's prompt notes, and accumulating token usage. It also provides a default `async with` lifecycle. The public interface is the `PydanticAiHtmlExtractor` protocol. The concrete extractors are `PydanticAiDirectExtractor` and `PydanticAiSelectorExtractor`. @@ -56,6 +59,40 @@ def __init__( self._base_instructions = self._compose_instructions(instructions, distiller) self._usage_limits = usage_limits self._ai_usage = PydanticAiUsageStats() + self._active = False + + @property + def active(self) -> bool: + """Whether the extractor is entered and not exited yet.""" + return self._active + + async def __aenter__(self) -> Self: + """Prepare the extractor for extraction. + + Raises: + RuntimeError: If the extractor is already active. + """ + if self._active: + raise RuntimeError(f'The {type(self).__name__} is already active.') + + self._active = True + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + exc_traceback: TracebackType | None, + ) -> None: + """Release what the extractor holds. + + Raises: + RuntimeError: If the extractor is not active. + """ + if not self._active: + raise RuntimeError(f'The {type(self).__name__} is not active.') + + self._active = False @property def ai_usage(self) -> PydanticAiUsageStats: diff --git a/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py b/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py index 55b61e3e65..5c156c49c8 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py +++ b/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py @@ -1,10 +1,10 @@ from __future__ import annotations import warnings -from contextlib import AbstractAsyncContextManager from typing import TYPE_CHECKING from parsel import Selector +from pydantic_ai.models import infer_model from crawlee._utils.docs import docs_group from crawlee.crawlers import AbstractHttpCrawler, HttpCrawlerOptions @@ -90,7 +90,8 @@ def __init__( `model` nor `extractor` is given. Provide at most one of `model` or `extractor`. extractor: A pre-configured `PydanticAiHtmlExtractor`, for full control over the distiller, instructions, caching, usage limits, and model fallback. Pass a `PydanticAiSelectorExtractor` here for - cached-selector extraction. Provide at most one of `model` or `extractor`. + cached-selector extraction. Keep it open with `async with` until every crawler using it has finished. + Provide at most one of `model` or `extractor`. kwargs: Additional keyword arguments to pass to the underlying `AbstractHttpCrawler`. """ # Call the notification only once. @@ -103,10 +104,10 @@ def __init__( if model is not None and extractor is not None: raise ValueError('Provide at most one of `model` or `extractor`.') + # Before `super().__init__()`, so that a failure here doesn't take up a crawler id. + default_model = model if model is not None else _DEFAULT_AI_MODEL if extractor is None: - extractor = PydanticAiDirectExtractor(model if model is not None else _DEFAULT_AI_MODEL) - - self._extractor = extractor + default_model = infer_model(default_model) async def final_step( context: ParsedHttpCrawlingContext[Selector], @@ -121,18 +122,13 @@ async def final_step( kwargs['_context_pipeline'] = self._create_static_content_crawler_pipeline().compose(final_step) - # If the extractor is an async context manager, add it to the crawler's additional context managers so it's - # properly entered and exited around the crawl. - if isinstance(extractor, AbstractAsyncContextManager): - kwargs['_additional_context_managers'] = [ - *kwargs.get('_additional_context_managers', []), - extractor, - ] super().__init__( parser=ParselParser(), **kwargs, ) + self._extractor = self._own_or_borrow(extractor, lambda: PydanticAiDirectExtractor(default_model)) + @property def extractor(self) -> PydanticAiHtmlExtractor: """The extractor used to turn pages into structured data.""" diff --git a/src/crawlee/crawlers/_pydantic_ai/_types.py b/src/crawlee/crawlers/_pydantic_ai/_types.py index 4ddad30eda..5dccf5d6a1 100644 --- a/src/crawlee/crawlers/_pydantic_ai/_types.py +++ b/src/crawlee/crawlers/_pydantic_ai/_types.py @@ -5,6 +5,7 @@ from pydantic import BaseModel +from crawlee._utils.context import Collaborator from crawlee._utils.docs import docs_group if TYPE_CHECKING: @@ -52,12 +53,13 @@ def add(self, usage: RunUsage) -> None: @docs_group('Other') -class PydanticAiHtmlExtractor(Protocol): +class PydanticAiHtmlExtractor(Collaborator, Protocol): """Interface for HTML extractors. An extractor turns an HTML page into a validated Pydantic model using an LLM. The input format (cleaned HTML, skeleton, Markdown, ...) is decided by the `PydanticAiHtmlDistiller` an implementation composes. The model and base - instructions are set at construction. Each `extract` call runs one extraction. The built-in extractors are + instructions are set at construction. Each `extract` call runs one extraction. An extractor is an async context + manager, and `PydanticAiCrawler` refuses to run with one that isn't entered. The built-in extractors are `PydanticAiDirectExtractor` and `PydanticAiSelectorExtractor`. """ diff --git a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py index df71060797..167f6533cf 100644 --- a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py +++ b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py @@ -109,7 +109,8 @@ def __init__( stagehand_options: Stagehand-specific configuration (model, API key, env, etc.). Cannot be specified if `browser_pool` is provided. browser_pool: A pre-configured `BrowserPool`. All plugins must be instances of `StagehandBrowserPlugin`. - If omitted, a pool is created automatically from the other browser arguments. + If omitted, a pool is created automatically from the other browser arguments. Keep a pool passed in + open with `async with` until every crawler using it has finished. user_data_dir: Path to a user data directory, which stores browser session data like cookies and local storage. Cannot be specified if `browser_pool` is provided. headless: Whether to run the browser in headless mode. Defaults to the value from Crawlee's @@ -158,6 +159,7 @@ def __init__( ) ] ) + kwargs['_additional_context_managers'] = [*kwargs.get('_additional_context_managers', []), browser_pool] super().__init__( browser_pool=browser_pool, diff --git a/src/crawlee/http_clients/_base.py b/src/crawlee/http_clients/_base.py index e69322a003..65498c2418 100644 --- a/src/crawlee/http_clients/_base.py +++ b/src/crawlee/http_clients/_base.py @@ -389,15 +389,17 @@ async def __aenter__(self) -> HttpClient: return self async def __aexit__( - self, exc_type: BaseException | None, exc_value: BaseException | None, traceback: TracebackType | None + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_traceback: TracebackType | None ) -> None: """Deinitialize the client and clean up resources when exiting the context manager. Raises: - RuntimeError: If the context manager is already active. + RuntimeError: If the context manager is not active. """ if not self._active: raise RuntimeError(f'The {self.__class__.__name__} is not active.') - await self.cleanup() - self._active = False + try: + await self.cleanup() + finally: + self._active = False diff --git a/src/crawlee/sessions/_session_pool.py b/src/crawlee/sessions/_session_pool.py index f206c64f58..a855224d6a 100644 --- a/src/crawlee/sessions/_session_pool.py +++ b/src/crawlee/sessions/_session_pool.py @@ -118,12 +118,20 @@ async def __aenter__(self) -> SessionPool: self._active = True - state = await self._state.initialize() - state.max_pool_size = self._max_pool_size - self._remove_retired_sessions() - - if not state.sessions: - await self._fill_sessions_to_max() + try: + state = await self._state.initialize() + state.max_pool_size = self._max_pool_size + self._remove_retired_sessions() + + if not state.sessions: + await self._fill_sessions_to_max() + except BaseException: + try: + if self._state.is_initialized: + await self._state.teardown() + finally: + self._active = False + raise return self @@ -141,9 +149,10 @@ async def __aexit__( if not self._active: raise RuntimeError(f'The {self.__class__.__name__} is not active.') - await self._state.teardown() - - self._active = False + try: + await self._state.teardown() + finally: + self._active = False @overload def get_state(self, *, as_dict: Literal[True]) -> dict: ... diff --git a/src/crawlee/statistics/_statistics.py b/src/crawlee/statistics/_statistics.py index 74ff13e5ba..3c684a4da1 100644 --- a/src/crawlee/statistics/_statistics.py +++ b/src/crawlee/statistics/_statistics.py @@ -172,22 +172,27 @@ async def __aenter__(self) -> Self: if self._active: raise RuntimeError(f'The {self.__class__.__name__} is already active.') - await self._state.initialize() - - # The timestamps belong to the previous run, restored or kept in memory. Fold that run into the accumulated - # runtime and clear them, so that the downtime since then (e.g. a migration or resurrection) doesn't count - # towards the runtime, and a clock that is behind the previous host's doesn't make `crawler_runtime` treat the - # new run as already checkpointed. - self.state.accumulated_runtime = self.state.crawler_runtime - self.state.crawler_last_started_at = None - self.state.crawler_finished_at = None - self.state.stats_persisted_at = None - - # Start periodic logging and let it print the restored state before the new run starts being measured. - self._periodic_logger.start() - await asyncio.sleep(0.01) self._active = True + try: + await self._state.initialize() + + # The timestamps belong to the previous run, restored or kept in memory. Fold that run into the accumulated + # runtime and clear them, so that the downtime since then (e.g. a migration or resurrection) doesn't count + # towards the runtime, and a clock that is behind the previous host's doesn't make `crawler_runtime` treat + # the new run as already checkpointed. + self.state.accumulated_runtime = self.state.crawler_runtime + self.state.crawler_last_started_at = None + self.state.crawler_finished_at = None + self.state.stats_persisted_at = None + + # Start periodic logging and let it print the restored state before the new run starts being measured. + self._periodic_logger.start() + await asyncio.sleep(0.01) + except BaseException: + await self._stop_capturing() + raise + self.state.crawler_last_started_at = datetime.now(UTC) self.state.crawler_started_at = self.state.crawler_started_at or self.state.crawler_last_started_at return self @@ -206,11 +211,10 @@ async def __aexit__( if not self._active: raise RuntimeError(f'The {self.__class__.__name__} is not active.') - # Stop logging; the finish timestamp freezes the measured run before the final persist. - await self._periodic_logger.stop() + # Before the first await, so that a cancelled exit doesn't count the time until the next run as runtime. self.state.crawler_finished_at = datetime.now(UTC) - self._active = False - await self._state.teardown() + + await self._stop_capturing() @property def state(self) -> TStatisticsState: @@ -316,6 +320,16 @@ async def reset_store(self) -> None: """ await self._state.reset_store() + async def _stop_capturing(self) -> None: + try: + await self._periodic_logger.stop() + finally: + try: + if self._state.is_initialized: + await self._state.teardown() + finally: + self._active = False + def _log(self) -> None: stats = self.calculate() if self._statistics_log_format == 'table': diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 59e0ff36ac..11068041f9 100644 --- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -29,6 +29,7 @@ AdaptivePlaywrightCrawlerStatisticState, ) from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawling_context import AdaptiveContextError +from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor from crawlee.errors import RequestThrottledError from crawlee.http_clients import HttpClient, HttpResponse from crawlee.sessions import SessionPool @@ -123,6 +124,12 @@ def predict(self, request: Request) -> RenderingTypePrediction: def store_result(self, request: Request, rendering_type: RenderingType) -> None: pass + @property + @override + def active(self) -> bool: + # Lets the tests pass the predictor without entering it. + return True + @dataclass(frozen=True) class TestInput: @@ -573,6 +580,17 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: assert stored_results == [saved_data] +async def test_passed_predictor_left_open(test_urls: list[str]) -> None: + """A rendering type predictor passed to the crawler stays active after the run, for the caller to exit.""" + async with DefaultRenderingTypePredictor() as predictor: + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(rendering_type_predictor=predictor) + crawler.router.default_handler(AsyncMock()) + + await crawler.run(test_urls[:1]) + + assert predictor.active + + async def test_adaptive_playwright_crawler_statistics_in_init() -> None: """Tests that adaptive crawler uses created AdaptivePlaywrightCrawlerStatistics from inputted Statistics.""" persistence_enabled = True @@ -988,7 +1006,7 @@ async def test_throttled_static_429_is_deferred(monkeypatch: pytest.MonkeyPatch) throttler = await open_throttler_held_until_reclaim(monkeypatch) record_pacing_signal = Mock(wraps=throttler.record_pacing_signal) monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.side_effect = [ Mock( spec=HttpResponse, @@ -1038,7 +1056,7 @@ async def test_static_handler_raised_throttled_error_is_deferred(monkeypatch: py throttler = await open_throttler_held_until_reclaim(monkeypatch) record_pacing_signal = Mock(wraps=throttler.record_pacing_signal) monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.return_value = Mock( spec=HttpResponse, status_code=200, diff --git a/tests/integration/crawlers/_http/test_http_crawler.py b/tests/integration/crawlers/_http/test_http_crawler.py index 673c7aa427..d96eae7498 100644 --- a/tests/integration/crawlers/_http/test_http_crawler.py +++ b/tests/integration/crawlers/_http/test_http_crawler.py @@ -421,13 +421,14 @@ async def test_isolation_cookies(http_client: HttpClient, server_url: URL) -> No sessions_cookies: dict[str, dict[str, str]] = {} response_cookies: dict[str, dict[str, str]] = {} + session_pool = SessionPool( + max_pool_size=1, + create_session_settings={ + 'max_error_score': 50, + }, + ) crawler = HttpCrawler( - session_pool=SessionPool( - max_pool_size=1, - create_session_settings={ - 'max_error_score': 50, - }, - ), + session_pool=session_pool, http_client=http_client, max_request_retries=10, concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1), @@ -449,16 +450,19 @@ async def handler(context: HttpCrawlingContext) -> None: if context.request.user_data.get('retire_session'): context.session.retire() - await crawler.run( - [ - # The first request sets the cookie in the session - str(server_url.with_path('set_cookies').extend_query(a=1)), - # With the second request, we check the cookies in the session and set retire - Request.from_url(str(server_url.with_path('/cookies')), unique_key='1', user_data={'retire_session': True}), - # The third request is made with a new session to make sure it does not use another session's cookies - Request.from_url(str(server_url.with_path('/cookies')), unique_key='2'), - ] - ) + async with session_pool: + await crawler.run( + [ + # The first request sets the cookie in the session + str(server_url.with_path('set_cookies').extend_query(a=1)), + # With the second request, we check the cookies in the session and set retire + Request.from_url( + str(server_url.with_path('/cookies')), unique_key='1', user_data={'retire_session': True} + ), + # The third request is made with a new session to make sure it does not use another session's cookies + Request.from_url(str(server_url.with_path('/cookies')), unique_key='2'), + ] + ) assert len(sessions_cookies) == 2 assert len(response_cookies) == 2 @@ -580,10 +584,8 @@ async def handler(context: HttpCrawlingContext) -> None: async def test_save_response_cookies_disabled(server_url: URL) -> None: """Test that `save_response_cookies=False` keeps the response cookies out of the session, but sends its cookies.""" preset_cookie = CookieParam(name='preset', value='1', domain=server_url.host or '') - crawler = HttpCrawler( - session_pool=SessionPool(max_pool_size=1, create_session_settings={'cookies': [preset_cookie]}), - save_response_cookies=False, - ) + session_pool = SessionPool(max_pool_size=1, create_session_settings={'cookies': [preset_cookie]}) + crawler = HttpCrawler(session_pool=session_pool, save_response_cookies=False) handler_call = Mock() @crawler.router.default_handler @@ -595,14 +597,16 @@ async def handler(context: HttpCrawlingContext) -> None: ) # The cookie set by the redirecting response is still sent to the redirect target. - await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) + async with session_pool: + await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) handler_call.assert_called_once_with(sent_cookies={'preset': '1', 'a': '1'}, session_cookies={'preset'}) async def test_context_send_request_saves_cookies(server_url: URL) -> None: """Test that `context.send_request` stores response cookies in the session with `save_response_cookies=False`.""" - crawler = HttpCrawler(session_pool=SessionPool(max_pool_size=1), save_response_cookies=False) + session_pool = SessionPool(max_pool_size=1) + crawler = HttpCrawler(session_pool=session_pool, save_response_cookies=False) handler_call = Mock() @crawler.router.default_handler @@ -611,7 +615,8 @@ async def handler(context: HttpCrawlingContext) -> None: session_cookies = context.session.cookies.get_cookies_as_dicts() handler_call(session_cookies={cookie['name'] for cookie in session_cookies}) - await crawler.run([str(server_url / 'cookies')]) + async with session_pool: + await crawler.run([str(server_url / 'cookies')]) handler_call.assert_called_once_with(session_cookies={'b'}) diff --git a/tests/integration/crawlers/_playwright/test_playwright_crawler.py b/tests/integration/crawlers/_playwright/test_playwright_crawler.py index b4372d501c..8a2ee0bc93 100644 --- a/tests/integration/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/integration/crawlers/_playwright/test_playwright_crawler.py @@ -434,12 +434,15 @@ async def test_isolation_cookies(*, use_incognito_pages: bool, server_url: URL) sessions: dict[str, Session] = {} response_cookies: dict[str, dict[str, str]] = {} - async with BrowserPool( - plugins=[PlaywrightBrowserPlugin(use_incognito_pages=use_incognito_pages)], - browser_inactive_threshold=timedelta(minutes=10), - ) as browser_pool: + async with ( + BrowserPool( + plugins=[PlaywrightBrowserPlugin(use_incognito_pages=use_incognito_pages)], + browser_inactive_threshold=timedelta(minutes=10), + ) as browser_pool, + SessionPool(max_pool_size=1) as session_pool, + ): crawler = PlaywrightCrawler( - session_pool=SessionPool(max_pool_size=1), + session_pool=session_pool, concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1), browser_pool=browser_pool, ) @@ -604,7 +607,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: async def test_send_request_skips_cookies_deleted_by_page(server_url: URL) -> None: """Test that `context.send_request` doesn't send a session cookie that the page deleted in the handler.""" - crawler = PlaywrightCrawler(session_pool=SessionPool(max_pool_size=1)) + session_pool = SessionPool(max_pool_size=1) + crawler = PlaywrightCrawler(session_pool=session_pool) sent_cookies: list[dict[str, str]] = [] @crawler.router.default_handler @@ -617,7 +621,8 @@ async def second_handler(context: PlaywrightCrawlingContext) -> None: response = await context.send_request(str(server_url / 'cookies')) sent_cookies.append(json.loads(await response.read())['cookies']) - await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) + async with session_pool: + await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))]) assert sent_cookies == [{}] @@ -983,7 +988,8 @@ async def test_send_request_with_client(server_url: URL) -> None: """Check that the persist context works with fingerprints.""" check_data: dict[str, Any] = {} - crawler = PlaywrightCrawler(http_client=ImpitHttpClient(headers={'user-agent': 'My User-Agent'})) + http_client = ImpitHttpClient(headers={'user-agent': 'My User-Agent'}) + crawler = PlaywrightCrawler(http_client=http_client) @crawler.router.default_handler async def request_handler(context: PlaywrightCrawlingContext) -> None: @@ -992,7 +998,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: send_request_response = await context.send_request(str(server_url / 'user-agent')) check_data['send_request'] = dict(json.loads(await send_request_response.read())) - await crawler.run([str(server_url / 'user-agent')]) + async with http_client: + await crawler.run([str(server_url / 'user-agent')]) assert check_data['default'].get('user-agent') is not None assert check_data['send_request']['user-agent'] == 'My User-Agent' @@ -1489,9 +1496,10 @@ async def test_throttled_429_is_deferred(monkeypatch: pytest.MonkeyPatch) -> Non record_pacing_signal = Mock(wraps=throttler.record_pacing_signal) monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal) failed_request_handler = AsyncMock() + session_pool = SessionPool(max_pool_size=1) crawler = PlaywrightCrawler( request_manager=throttler, - session_pool=SessionPool(max_pool_size=1), + session_pool=session_pool, max_request_retries=0, max_session_rotations=0, ) @@ -1508,7 +1516,8 @@ async def fulfill_with_next_status(context: PlaywrightPreNavCrawlingContext) -> async def handler(context: PlaywrightCrawlingContext) -> None: handled.append(context) - await crawler.run([url]) + async with session_pool: + await crawler.run([url]) assert [context.response.status for context in handled] == [200] failed_request_handler.assert_not_called() diff --git a/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py b/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py index 905878c17f..37ade376b3 100644 --- a/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py +++ b/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py @@ -71,7 +71,8 @@ async def request_handler(context: PydanticAiCrawlingContext) -> None: await context.extract(_Article, scope='article', additional_instructions='hint') with patch.object(extractor, 'extract', extract_mock): - await crawler.run([Request.from_url(str(server_url), label='detail')]) + async with extractor: + await crawler.run([Request.from_url(str(server_url), label='detail')]) first_call, second_call, third_call = extract_mock.call_args_list diff --git a/tests/unit/_statistics/test_enter_exit.py b/tests/unit/_statistics/test_enter_exit.py new file mode 100644 index 0000000000..2324af6941 --- /dev/null +++ b/tests/unit/_statistics/test_enter_exit.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, Mock + +import pytest + +from crawlee.statistics import Statistics + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('initialize failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_enter_frees_stats(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """Statistics whose entry fails or is cancelled stay inactive and can be entered again.""" + statistics = Statistics.with_default_state() + monkeypatch.setattr(statistics._state, 'initialize', AsyncMock(side_effect=error)) + + with pytest.raises(type(error)): + async with statistics: + pass + + assert not statistics.active + monkeypatch.undo() + + async with statistics: + assert statistics.active + + +async def test_failed_exit_frees_stats(monkeypatch: pytest.MonkeyPatch) -> None: + """Statistics whose exit fails are no longer active, so they can be entered again.""" + statistics = Statistics.with_default_state() + + with pytest.raises(RuntimeError, match='teardown failed'): + async with statistics: + monkeypatch.setattr(statistics._state, 'teardown', AsyncMock(side_effect=RuntimeError('teardown failed'))) + + assert not statistics.active + monkeypatch.undo() + + async with statistics: + assert statistics.active + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """Statistics can't be entered again until their exit has torn the state down.""" + statistics = Statistics.with_default_state() + teardown_started = asyncio.Event() + release_teardown = asyncio.Event() + + async def wait_for_release() -> None: + teardown_started.set() + await release_teardown.wait() + + await statistics.__aenter__() + monkeypatch.setattr(statistics._state, 'teardown', AsyncMock(side_effect=wait_for_release)) + exit_task = asyncio.create_task(statistics.__aexit__(None, None, None)) + await teardown_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await statistics.__aenter__() + + release_teardown.set() + await exit_task + + +async def test_failed_start_tears_down_stats(monkeypatch: pytest.MonkeyPatch) -> None: + """Statistics whose entry fails after loading the state tear the state down and can be entered again.""" + statistics = Statistics.with_default_state() + teardown = AsyncMock(wraps=statistics._state.teardown) + monkeypatch.setattr(statistics._state, 'teardown', teardown) + monkeypatch.setattr(statistics._periodic_logger, 'start', Mock(side_effect=RuntimeError('start failed'))) + + with pytest.raises(RuntimeError, match='start failed'): + async with statistics: + pass + + assert not statistics.active + teardown.assert_awaited_once() + monkeypatch.undo() + + async with statistics: + assert statistics.active + + +async def test_cancelled_exit_finishes_run(monkeypatch: pytest.MonkeyPatch) -> None: + """Statistics whose exit is cancelled still record when the run finished and tear their state down.""" + statistics = Statistics.with_default_state() + + with pytest.raises(asyncio.CancelledError): + async with statistics: + monkeypatch.setattr(statistics._periodic_logger, 'stop', AsyncMock(side_effect=asyncio.CancelledError())) + + assert statistics.state.crawler_finished_at is not None + assert statistics._state.is_initialized is False + monkeypatch.undo() + await statistics._periodic_logger.stop() diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 5295f15712..adf46db9a1 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -7,7 +7,9 @@ import pytest from crawlee.autoscaling import ConcurrencySystem +from crawlee.browsers import BrowserPool from crawlee.crawlers import AdaptivePlaywrightCrawler +from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor async def test_accepts_concurrency_system() -> None: @@ -18,6 +20,26 @@ async def test_accepts_concurrency_system() -> None: assert crawler.concurrency_system is concurrency_system +async def test_run_needs_entered_predictor() -> None: + """A rendering type predictor passed to the crawler has to be entered before the crawler runs.""" + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + rendering_type_predictor=DefaultRenderingTypePredictor() + ) + + with pytest.raises(RuntimeError, match='DefaultRenderingTypePredictor passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + +async def test_run_needs_entered_browser_pool() -> None: + """A browser pool passed to the Playwright sub crawler has to be entered before the crawler runs.""" + crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( + playwright_crawler_specific_kwargs={'browser_pool': BrowserPool()} + ) + + with pytest.raises(RuntimeError, match='BrowserPool passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + @pytest.mark.parametrize( 'optional_module_name', [ diff --git a/tests/unit/crawlers/_adaptive_playwright/test_predictor.py b/tests/unit/crawlers/_adaptive_playwright/test_predictor.py index 681307cc9e..72d01f7515 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_predictor.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_predictor.py @@ -1,5 +1,8 @@ from __future__ import annotations +import asyncio +from unittest.mock import AsyncMock + import pytest from crawlee import Request @@ -217,3 +220,63 @@ def test_url_similarity(url_1: str, url_2: str, expected_rounded_similarity: flo round(calculate_url_similarity(url_1=get_url_components(url_1), url_2=get_url_components(url_2)), 2) == expected_rounded_similarity ) + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('initialize failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_enter_frees_predictor(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """A predictor whose entry fails or is cancelled stays inactive and can be entered again.""" + predictor = DefaultRenderingTypePredictor() + monkeypatch.setattr(predictor._state, 'initialize', AsyncMock(side_effect=error)) + + with pytest.raises(type(error)): + async with predictor: + pass + + assert not predictor.active + monkeypatch.undo() + + async with predictor: + assert predictor.active + + +async def test_failed_exit_frees_predictor(monkeypatch: pytest.MonkeyPatch) -> None: + """A predictor whose exit fails is no longer active, so it can be entered again.""" + predictor = DefaultRenderingTypePredictor() + + with pytest.raises(RuntimeError, match='teardown failed'): + async with predictor: + monkeypatch.setattr(predictor._state, 'teardown', AsyncMock(side_effect=RuntimeError('teardown failed'))) + + assert not predictor.active + monkeypatch.undo() + + async with predictor: + assert predictor.active + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A predictor can't be entered again until its exit has torn the state down.""" + predictor = DefaultRenderingTypePredictor() + teardown_started = asyncio.Event() + release_teardown = asyncio.Event() + + async def wait_for_release() -> None: + teardown_started.set() + await release_teardown.wait() + + await predictor.__aenter__() + monkeypatch.setattr(predictor._state, 'teardown', AsyncMock(side_effect=wait_for_release)) + exit_task = asyncio.create_task(predictor.__aexit__(None, None, None)) + await teardown_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await predictor.__aenter__() + + release_teardown.set() + await exit_task diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 0a96197512..e9f9fc85ac 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -44,7 +44,7 @@ EventMigratingData, LocalEventManager, ) -from crawlee.http_clients import HttpClient, HttpResponse +from crawlee.http_clients import HttpClient, HttpResponse, ImpitHttpClient from crawlee.request_loaders import ( FinishedStatus, MinIntervalSignal, @@ -622,7 +622,7 @@ async def failed_request_handler(context: BasicCrawlingContext, error: Exception async def test_send_request_forwards_arguments_to_http_client() -> None: """Test that `context.send_request` passes all its arguments to the HTTP client.""" - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) response = Mock(spec=HttpResponse) http_client.send_request.return_value = response crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1) @@ -664,7 +664,7 @@ async def handler(context: BasicCrawlingContext) -> None: async def test_send_request_default_timeout() -> None: """Test that `context.send_request` without a `timeout` leaves the timeout to the default of the HTTP client.""" - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.return_value = Mock(spec=HttpResponse) crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1) @@ -686,7 +686,7 @@ async def handler(context: BasicCrawlingContext) -> None: ) async def test_send_request_response_closed_after_request(*, handler_fails: bool) -> None: """Test that a response of `context.send_request` stays open in the handler and is closed after the request.""" - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) response = Mock(spec=HttpResponse) http_client.send_request.return_value = response crawler = BasicCrawler(http_client=http_client, max_request_retries=0, max_requests_per_crawl=1) @@ -1475,6 +1475,164 @@ async def test_repeated_run_starts_with_fresh_statistics() -> None: assert crawler.statistics.state.requests_finished == 1 +async def test_crawlers_share_stats_in_turn() -> None: + """Statistics passed to two crawlers that run one after the other add up the requests and the runtime of both.""" + statistics = Statistics.with_default_state() + first_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + second_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + + await first_crawler.run(['https://a.placeholder.com']) + started_at = statistics.state.crawler_started_at + first_runtime = statistics.state.crawler_runtime + stats = await second_crawler.run(['https://b.placeholder.com']) + + assert stats.requests_finished == 2 + assert statistics.state.crawler_started_at == started_at + assert statistics.state.accumulated_runtime == first_runtime + + +async def test_shared_stats_concurrent_raises() -> None: + """Two crawlers can't run with the same statistics at the same time, since the first one has them entered.""" + statistics = Statistics.with_default_state() + handler_started = asyncio.Event() + release_handler = asyncio.Event() + + async def blocking_handler(_context: BasicCrawlingContext) -> None: + handler_started.set() + await release_handler.wait() + + first_crawler = BasicCrawler(request_handler=blocking_handler, statistics=statistics) + second_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + + first_run = asyncio.create_task(first_crawler.run(['https://a.placeholder.com'])) + await handler_started.wait() + + with pytest.raises(RuntimeError, match='statistics of the crawler are already active'): + await second_crawler.run(['https://b.placeholder.com']) + + release_handler.set() + stats = await first_run + + assert stats.requests_finished == 1 + + +async def test_shared_stats_started_together(monkeypatch: pytest.MonkeyPatch) -> None: + """A crawler started while another crawler is still entering the shared statistics raises.""" + statistics = Statistics.with_default_state() + entry_started = asyncio.Event() + release_entry = asyncio.Event() + initialize = statistics._state.initialize + + async def hold_first_initialize() -> StatisticsState: + # Only the first entry waits, so that an unblocked second entry finishes instead of hanging the test. + if not entry_started.is_set(): + entry_started.set() + await release_entry.wait() + return await initialize() + + monkeypatch.setattr(statistics._state, 'initialize', AsyncMock(side_effect=hold_first_initialize)) + first_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + second_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + + first_run = asyncio.create_task(first_crawler.run(['https://a.placeholder.com'])) + await entry_started.wait() + + # The first entry makes the statistics active before its first await, so the second crawler stops in `run()`. + with pytest.raises(RuntimeError, match='statistics of the crawler are already active'): + await second_crawler.run(['https://b.placeholder.com']) + + release_entry.set() + stats = await first_run + + assert stats.requests_finished == 1 + assert not statistics.active + + +async def test_run_after_failed_exit(monkeypatch: pytest.MonkeyPatch) -> None: + """A failed exit of an object the crawler built doesn't keep it active, so the next run works.""" + monkeypatch.setattr(ImpitHttpClient, 'cleanup', AsyncMock(side_effect=[RuntimeError('cleanup failed'), None])) + handler = AsyncMock() + crawler = BasicCrawler(request_handler=handler) + + with pytest.raises(RuntimeError, match='cleanup failed'): + await crawler.run(['https://a.placeholder.com']) + + await crawler.run(['https://b.placeholder.com']) + + assert handler.await_count == 2 + + +async def test_run_rejects_entered_stats() -> None: + """Statistics the caller entered make `run()` raise, since the crawler enters them itself.""" + async with Statistics.with_default_state() as statistics: + crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics) + + with pytest.raises(RuntimeError, match='statistics of the crawler are already active'): + await crawler.run(['https://a.placeholder.com']) + + +async def test_repeated_run_keeps_sessions() -> None: + """A repeated `run()` keeps the sessions of a persisted session pool passed to the crawler, and leaves it open.""" + async with SessionPool(max_pool_size=1, persistence_enabled=True) as session_pool: + session_ids = set(session_pool.get_state(as_dict=False).sessions) + crawler = BasicCrawler(request_handler=AsyncMock(), session_pool=session_pool) + + await crawler.run(['https://a.placeholder.com']) + await crawler.run(['https://b.placeholder.com']) + + assert set(session_pool.get_state(as_dict=False).sessions) == session_ids + assert session_pool.active + + +async def test_repeated_run_resets_own_sessions() -> None: + """A repeated `run()` replaces the sessions of the session pool the crawler built.""" + crawler = BasicCrawler() + session_ids = list[set[str]]() + + @crawler.router.default_handler + async def handler(_context: BasicCrawlingContext) -> None: + session_ids.append(set(crawler._session_pool.get_state(as_dict=False).sessions)) + + await crawler.run(['https://a.placeholder.com']) + await crawler.run(['https://b.placeholder.com']) + + first_ids, second_ids = session_ids + assert first_ids.isdisjoint(second_ids) + + +async def test_run_needs_entered_session_pool() -> None: + """A session pool passed to the crawler has to be entered before the crawler runs.""" + session_pool = SessionPool() + crawler = BasicCrawler(request_handler=AsyncMock(), session_pool=session_pool) + + with pytest.raises(RuntimeError, match='SessionPool passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + async with session_pool: + stats = await crawler.run(['https://a.placeholder.com']) + + assert stats.requests_finished == 1 + + +async def test_run_needs_entered_http_client() -> None: + """An HTTP client passed to the crawler has to be entered before the crawler runs.""" + http_client = ImpitHttpClient() + crawler = BasicCrawler(request_handler=AsyncMock(), http_client=http_client) + + with pytest.raises(RuntimeError, match='ImpitHttpClient passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + +async def test_passed_http_client_left_open() -> None: + """The crawler doesn't close an HTTP client passed to it.""" + http_client = ImpitHttpClient() + crawler = BasicCrawler(request_handler=AsyncMock(), http_client=http_client) + + async with http_client: + await crawler.run(['https://a.placeholder.com']) + assert http_client.active + + def test_crawler_log() -> None: crawler = BasicCrawler() assert isinstance(crawler.log, logging.Logger) @@ -2632,7 +2790,8 @@ async def signal_idle_reached(*args: object, **kwargs: object) -> bool: ], ) async def test_session_retire_in_user_handler(*, retire: bool) -> None: - crawler = BasicCrawler(session_pool=SessionPool(max_pool_size=1)) + session_pool = SessionPool(max_pool_size=1) + crawler = BasicCrawler(session_pool=session_pool) sessions = list[str]() @crawler.router.default_handler @@ -2644,7 +2803,8 @@ async def handler(context: BasicCrawlingContext) -> None: await context.add_requests(['https://b.placeholder.com']) - await crawler.run(['https://a.placeholder.com']) + async with session_pool: + await crawler.run(['https://a.placeholder.com']) # The session should differ if `retire` was called and match otherwise since pool size == 1 if retire: @@ -2688,7 +2848,8 @@ def create_session() -> Session: check_sessions = [str(session_id) for session_id in range(10)] used_sessions = list[str]() - crawler = BasicCrawler(session_pool=SessionPool(create_session_function=create_session_function())) + session_pool = SessionPool(create_session_function=create_session_function()) + crawler = BasicCrawler(session_pool=session_pool) @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: @@ -2699,7 +2860,8 @@ async def handler(context: BasicCrawlingContext) -> None: for session_id in range(10) ] - await crawler.run(requests) + async with session_pool: + await crawler.run(requests) assert len(used_sessions) == 10 assert set(used_sessions) == set(check_sessions) @@ -3139,20 +3301,23 @@ async def test_crawler_intermediate_statistics() -> None: crawler = BasicCrawler() check_time = timedelta(seconds=0.1) + handler_started = asyncio.Event() + @crawler.router.default_handler async def handler(_: BasicCrawlingContext) -> None: + handler_started.set() await asyncio.sleep(check_time.total_seconds() * 5) - # Start crawler and wait until statistics are initialized. Use a generous timeout so that crawler startup - # has enough time to reach the active state on slow CI runners under xdist load. + # The handler starts only once the statistics are entered and measure the run. Use a generous timeout so that + # crawler startup has enough time on slow CI runners under xdist load. crawler_task = asyncio.create_task(crawler.run(['https://a.placeholder.com'])) - assert await poll_until_condition(lambda: crawler.statistics.active, timeout=30) - active_seen_at = datetime.now(UTC) + await asyncio.wait_for(handler_started.wait(), timeout=30) + started_seen_at = datetime.now(UTC) # Wait some time and check that runtime is updated. The runtime is measured on the wall clock, which on Windows # ticks at ~15.6 ms independently of the `asyncio.sleep` clock, so bound it by a wall-clock interval inside the run. await asyncio.sleep(check_time.total_seconds()) - min_runtime = datetime.now(UTC) - active_seen_at + min_runtime = datetime.now(UTC) - started_seen_at crawler.statistics.calculate() assert min_runtime > timedelta(0) assert crawler.statistics.state.crawler_runtime >= min_runtime @@ -3544,16 +3709,18 @@ async def test_handler_raised_throttled_error_is_deferred(monkeypatch: pytest.Mo """A `RequestThrottledError` from the handler starts a backoff without costing a retry or session reputation.""" throttler = await open_throttler_held_until_reclaim(monkeypatch) failed_request_handler = AsyncMock() + session_pool = SessionPool(max_pool_size=1) crawler = BasicCrawler( request_manager=throttler, - session_pool=SessionPool(max_pool_size=1), + session_pool=session_pool, max_request_retries=0, ) crawler.failed_request_handler(failed_request_handler) handler = AsyncMock(side_effect=[RequestThrottledError, None]) crawler.router.default_handler(handler) - stats = await crawler.run([THROTTLED_URL]) + async with session_pool: + stats = await crawler.run([THROTTLED_URL]) assert handler.await_count == 2 failed_request_handler.assert_not_called() diff --git a/tests/unit/crawlers/_http/test_http_crawler.py b/tests/unit/crawlers/_http/test_http_crawler.py index 0ef3c94208..d9acc12140 100644 --- a/tests/unit/crawlers/_http/test_http_crawler.py +++ b/tests/unit/crawlers/_http/test_http_crawler.py @@ -51,7 +51,7 @@ async def test_navigation_response_closed_after_request(*, status_code: int, exp """Test that the response of the navigation request stays open in the handler and is closed after the request.""" url = 'https://a.placeholder.com' response = mock_response(status_code, url) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.return_value = response crawler = HttpCrawler(http_client=http_client, max_request_retries=0) handler_call = Mock() @@ -71,7 +71,7 @@ async def test_body_read_bound_by_navigation_timeout() -> None: url = 'https://a.placeholder.com' response = mock_response(200, url) response.read.side_effect = asyncio.Event().wait - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) http_client.send_request.return_value = response crawler = HttpCrawler( http_client=http_client, @@ -100,14 +100,15 @@ async def test_body_read_bound_by_navigation_timeout() -> None: async def test_throttled_429_is_deferred(crawler_kwargs: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None: """A throttled 429 is retried later without spending a retry, a session rotation or session reputation.""" throttler = await open_throttler_held_until_reclaim(monkeypatch) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) url = 'https://throttled.placeholder.com/page' http_client.send_request.side_effect = [mock_response(429, url), mock_response(200, url)] failed_request_handler = AsyncMock() + session_pool = SessionPool(max_pool_size=1) crawler = HttpCrawler( http_client=http_client, request_manager=throttler, - session_pool=SessionPool(max_pool_size=1), + session_pool=session_pool, max_request_retries=0, max_session_rotations=0, **crawler_kwargs, @@ -119,7 +120,8 @@ async def test_throttled_429_is_deferred(crawler_kwargs: dict[str, Any], monkeyp async def handler(context: HttpCrawlingContext) -> None: handled.append(context) - await crawler.run([url]) + async with session_pool: + await crawler.run([url]) assert [context.http_response.status_code for context in handled] == [200] failed_request_handler.assert_not_called() @@ -134,7 +136,7 @@ async def handler(context: HttpCrawlingContext) -> None: async def test_tandem_429_deferred(monkeypatch: pytest.MonkeyPatch) -> None: """A 429 reaches a throttler wrapped in a `RequestManagerTandem`, so the request is deferred without a retry.""" throttler = await open_throttler_held_until_reclaim(monkeypatch) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) url = 'https://throttled.placeholder.com/page' http_client.send_request.side_effect = [mock_response(429, url), mock_response(200, url)] failed_request_handler = AsyncMock() @@ -163,7 +165,7 @@ async def test_429_reported_once(monkeypatch: pytest.MonkeyPatch) -> None: throttler = await open_throttler_held_until_reclaim(monkeypatch) record_pacing_signal = Mock(wraps=throttler.record_pacing_signal) monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal) - http_client = AsyncMock(spec=HttpClient) + http_client = AsyncMock(spec=HttpClient, active=True) url = 'https://throttled.placeholder.com/page' http_client.send_request.side_effect = [mock_response(429, url), mock_response(200, url)] crawler = HttpCrawler(http_client=http_client, request_manager=throttler) diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 8a4b500788..a243f5bb6e 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -9,8 +9,10 @@ service_locator, ) from crawlee.autoscaling import ConcurrencySystem +from crawlee.browsers import BrowserPool from crawlee.configuration import Configuration from crawlee.crawlers import ( + BasicCrawler, PlaywrightCrawler, ) @@ -34,6 +36,25 @@ async def test_accepts_concurrency_system() -> None: assert crawler.concurrency_system is concurrency_system +async def test_run_needs_entered_browser_pool() -> None: + """A browser pool passed to the crawler has to be entered before the crawler runs.""" + crawler = PlaywrightCrawler(browser_pool=BrowserPool()) + + with pytest.raises(RuntimeError, match='BrowserPool passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + +def test_unsupported_browser_type_takes_no_id() -> None: + """A crawler with a browser type its default fingerprint generator doesn't support doesn't use up a crawler id.""" + first_crawler = BasicCrawler() + + with pytest.raises(ValueError, match='Unsupported browser type'): + PlaywrightCrawler(browser_type='edge') # ty: ignore[invalid-argument-type] + + second_crawler = BasicCrawler() + assert int(second_crawler._id) == int(first_crawler._id) + 1 + + def test_import_error_handled() -> None: blocked = { mod_name: None for mod_name in sys.modules if mod_name == 'playwright' or mod_name.startswith('playwright.') diff --git a/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py b/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py index 759a277b18..74ffa04260 100644 --- a/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py +++ b/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py @@ -159,3 +159,14 @@ async def test_raise_for_invalid_output() -> None: # The extractor's usage stats reflect the 3 failed attempts (1 initial + 2 retries). assert extractor.ai_usage.requests == 3 + + +async def test_enter_sets_active() -> None: + """Entering the extractor makes it active until it's exited.""" + extractor = PydanticAiDirectExtractor(TestModel()) + assert extractor.active is False + + async with extractor: + assert extractor.active is True + + assert extractor.active is False diff --git a/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py b/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py index 7cd2aaad4d..356dd3e79f 100644 --- a/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py +++ b/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py @@ -1,14 +1,17 @@ from __future__ import annotations import sys -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest +from pydantic_ai.exceptions import UserError from pydantic_ai.models.test import TestModel from crawlee.crawlers import ( + BasicCrawler, PydanticAiCrawler, PydanticAiDirectExtractor, + PydanticAiSelectorExtractor, ) @@ -42,6 +45,37 @@ def test_exposes_extractor_and_usage() -> None: assert crawler.ai_usage is extractor.ai_usage +@pytest.mark.parametrize( + 'extractor_class', + [ + pytest.param(PydanticAiDirectExtractor, id='direct'), + pytest.param(PydanticAiSelectorExtractor, id='selector'), + ], +) +async def test_run_needs_entered_extractor( + extractor_class: type[PydanticAiDirectExtractor | PydanticAiSelectorExtractor], +) -> None: + """An extractor passed to the crawler has to be entered before the crawler runs.""" + crawler = PydanticAiCrawler(extractor=extractor_class(TestModel())) + + with pytest.raises(RuntimeError, match=f'{extractor_class.__name__} passed to the crawler is not active'): + await crawler.run(['https://a.placeholder.com']) + + +async def test_owns_default_extractor(monkeypatch: pytest.MonkeyPatch) -> None: + """The crawler enters and exits the default extractor it builds.""" + extractor_enter = AsyncMock() + extractor_exit = AsyncMock() + monkeypatch.setattr(PydanticAiDirectExtractor, '__aenter__', extractor_enter) + monkeypatch.setattr(PydanticAiDirectExtractor, '__aexit__', extractor_exit) + crawler = PydanticAiCrawler(model=TestModel()) + + await crawler.run() + + extractor_enter.assert_awaited_once() + extractor_exit.assert_awaited_once() + + def test_import_error_handled() -> None: # The `ai` extra is optional, so accessing the crawler without `pydantic_ai` installed must raise a clear error. blocked = {name: None for name in sys.modules if name == 'pydantic_ai' or name.startswith('pydantic_ai.')} @@ -51,3 +85,14 @@ def test_import_error_handled() -> None: sys.modules.pop(name, None) with pytest.raises(ImportError): from crawlee.crawlers._pydantic_ai import PydanticAiCrawler # noqa: F401 PLC0415 + + +def test_unknown_model_takes_no_id() -> None: + """A crawler with an unknown model doesn't use up a crawler id.""" + first_crawler = BasicCrawler() + + with pytest.raises(UserError, match='Unknown model'): + PydanticAiCrawler(model='no-such-provider:model') + + second_crawler = BasicCrawler() + assert int(second_crawler._id) == int(first_crawler._id) + 1 diff --git a/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py b/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py index 2604ca53da..b5fc104bef 100644 --- a/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py +++ b/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import pytest @@ -41,6 +41,20 @@ def test_init_with_valid_browser_pool() -> None: StagehandCrawler(browser_pool=pool) +async def test_owns_default_browser_pool(monkeypatch: pytest.MonkeyPatch) -> None: + """The crawler enters and exits the browser pool it builds itself.""" + pool_enter = AsyncMock() + pool_exit = AsyncMock() + monkeypatch.setattr(BrowserPool, '__aenter__', pool_enter) + monkeypatch.setattr(BrowserPool, '__aexit__', pool_exit) + crawler = StagehandCrawler() + + await crawler.run() + + pool_enter.assert_awaited_once() + pool_exit.assert_awaited_once() + + def test_forwards_save_response_cookies() -> None: """`save_response_cookies` reaches the `PlaywrightCrawler` that saves the cookies of the page.""" crawler = StagehandCrawler(save_response_cookies=False) diff --git a/tests/unit/http_clients/test_http_clients.py b/tests/unit/http_clients/test_http_clients.py index adde6d710d..cb365ba98f 100644 --- a/tests/unit/http_clients/test_http_clients.py +++ b/tests/unit/http_clients/test_http_clients.py @@ -312,3 +312,25 @@ async def test_ignore_tls_errors_reaches_fetch( await client.send_request(url, proxy_info=proxy_info, ignore_tls_errors=request_flag) assert fetch.call_args.kwargs['ignore_tls_errors'] is expected_flag + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A client can't be entered again until its exit has cleaned it up.""" + http_client = ImpitHttpClient() + cleanup_started = asyncio.Event() + release_cleanup = asyncio.Event() + + async def wait_for_release() -> None: + cleanup_started.set() + await release_cleanup.wait() + + await http_client.__aenter__() + monkeypatch.setattr(http_client, 'cleanup', AsyncMock(side_effect=wait_for_release)) + exit_task = asyncio.create_task(http_client.__aexit__(None, None, None)) + await cleanup_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await http_client.__aenter__() + + release_cleanup.set() + await exit_task diff --git a/tests/unit/sessions/test_session_pool.py b/tests/unit/sessions/test_session_pool.py index 404612af69..8c29723737 100644 --- a/tests/unit/sessions/test_session_pool.py +++ b/tests/unit/sessions/test_session_pool.py @@ -1,8 +1,10 @@ from __future__ import annotations +import asyncio import logging from datetime import UTC, datetime from typing import TYPE_CHECKING +from unittest.mock import AsyncMock import pytest @@ -275,3 +277,66 @@ async def test_methods_raise_error_when_not_active() -> None: async with session_pool: assert session_pool.active is True + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(RuntimeError('fill failed'), id='failed'), + pytest.param(asyncio.CancelledError(), id='cancelled'), + ], +) +async def test_failed_enter_frees_pool(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """A pool whose entry fails or is cancelled tears its state down, stays inactive and can be entered again.""" + session_pool = SessionPool(max_pool_size=MAX_POOL_SIZE, persistence_enabled=False) + teardown = AsyncMock(wraps=session_pool._state.teardown) + monkeypatch.setattr(session_pool._state, 'teardown', teardown) + monkeypatch.setattr(session_pool, '_fill_sessions_to_max', AsyncMock(side_effect=error)) + + with pytest.raises(type(error)): + async with session_pool: + pass + + assert not session_pool.active + teardown.assert_awaited_once() + monkeypatch.undo() + + async with session_pool: + assert session_pool.active + + +async def test_failed_exit_frees_pool(monkeypatch: pytest.MonkeyPatch) -> None: + """A pool whose exit fails is no longer active, so it can be entered again.""" + session_pool = SessionPool(max_pool_size=MAX_POOL_SIZE, persistence_enabled=False) + + with pytest.raises(RuntimeError, match='teardown failed'): + async with session_pool: + monkeypatch.setattr(session_pool._state, 'teardown', AsyncMock(side_effect=RuntimeError('teardown failed'))) + + assert not session_pool.active + monkeypatch.undo() + + async with session_pool: + assert session_pool.active + + +async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A pool can't be entered again until its exit has torn the state down.""" + session_pool = SessionPool(max_pool_size=MAX_POOL_SIZE, persistence_enabled=False) + teardown_started = asyncio.Event() + release_teardown = asyncio.Event() + + async def wait_for_release() -> None: + teardown_started.set() + await release_teardown.wait() + + await session_pool.__aenter__() + monkeypatch.setattr(session_pool._state, 'teardown', AsyncMock(side_effect=wait_for_release)) + exit_task = asyncio.create_task(session_pool.__aexit__(None, None, None)) + await teardown_started.wait() + + with pytest.raises(RuntimeError, match='already active'): + await session_pool.__aenter__() + + release_teardown.set() + await exit_task