diff --git a/docs/03_concepts/02_requests.mdx b/docs/03_concepts/02_requests.mdx
index b6efcc80da..85dd6c6210 100644
--- a/docs/03_concepts/02_requests.mdx
+++ b/docs/03_concepts/02_requests.mdx
@@ -72,7 +72,7 @@ The crawler's `max_crawl_depth` option limits how deep the crawl goes. Requests
## Binding to a session
-Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't available anymore, the request fails with a `RequestCollisionError` and isn't retried. For details on sessions, see [Session management](./session-management).
+Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't available anymore, the request fails with a `RequestCollisionError` and isn't retried. In the failed request handler, `context.session` is then a retired session with the bound id, not taken from the pool. For details on sessions, see [Session management](./session-management).
## Request lifecycle
diff --git a/docs/03_concepts/03_crawling_context.mdx b/docs/03_concepts/03_crawling_context.mdx
index b58f6dd3d0..b69162f112 100644
--- a/docs/03_concepts/03_crawling_context.mdx
+++ b/docs/03_concepts/03_crawling_context.mdx
@@ -30,7 +30,7 @@ For details on the crawler-specific attributes, see [HTTP crawlers](./http-crawl
The `request` attribute is the `Request` being processed. It carries the URL, the label used for routing, user data and the retry count. For details, see [Requests](./requests).
-The `session` attribute is the `Session` assigned to the request, or `None` when the crawler runs without a session pool. You can call `mark_bad()` or `retire()` on it from the handler when a page shows signs of blocking. For details, see [Session management](./session-management).
+The `session` attribute is the `Session` assigned to the request. You can call `mark_bad()` or `retire()` on it from the handler when a page shows signs of blocking. For details, see [Session management](./session-management).
The `proxy_info` attribute is a `ProxyInfo` describing the proxy used for the request, or `None` when no proxy is configured. For details, see [Proxy management](./proxy-management).
diff --git a/docs/03_concepts/15_cookie_management.mdx b/docs/03_concepts/15_cookie_management.mdx
index 973fc1a600..6e127da64c 100644
--- a/docs/03_concepts/15_cookie_management.mdx
+++ b/docs/03_concepts/15_cookie_management.mdx
@@ -31,12 +31,6 @@ Access the jar through `context.session.cook
{ReadWriteCookies}
-:::note
-
-`context.session` is `None` only when the crawler runs without a session pool (`use_session_pool` set to `False`). With the default pool it's always set. The examples guard with `if context.session is None: return` to cover that case.
-
-:::
-
A cookie has a `name`, a `value`, and optional parameters such as `domain`. If a cookie has no `domain`, it applies to any domain, so a login cookie set that way is sent to every host you crawl. Set `domain` on anything sensitive. For the full set of parameters, see `CookieParam`. To dump the whole jar, use `get_cookies_as_dicts()`, and to load several cookies at once, use `set_cookies()`.
## Seeding cookies on new sessions
diff --git a/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py
index 26a7444937..7e87c74e18 100644
--- a/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py
+++ b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py
@@ -13,9 +13,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: PlaywrightCrawlingContext) -> None:
- if context.session is None:
- return
-
# The browser receives the `visited` cookie during the first navigation.
# It's synced back onto the session after the handler returns, so on the
# second request it's already on `context.session`.
diff --git a/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py
index 899ef0c3c0..57d85d4095 100644
--- a/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py
+++ b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py
@@ -8,9 +8,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if context.session is None:
- return
-
# Set a cookie on the session. It's sent with every following
# request that runs on this session.
context.session.cookies.set(
diff --git a/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py
index 919e7e5c20..0f4dc2fe0c 100644
--- a/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py
+++ b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py
@@ -9,9 +9,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if context.session is None:
- return
-
if context.request.user_data.get('prepared'):
# Cookies from the setup step are on this pinned session.
context.log.info(
diff --git a/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py
index 44cd965c46..dc565c4154 100644
--- a/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py
+++ b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py
@@ -12,9 +12,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if context.session is None:
- return
-
state = await context.use_state(default_value={})
if not state.get('cookies'):
# First pass: establish cookies once and store them for every session,
@@ -30,7 +27,7 @@ async def apply_cookies(context: BasicCrawlingContext) -> None:
# Runs before navigation. Apply the shared cookies to whichever session
# handles the request, so every session ends up with them.
state = await context.use_state(default_value={})
- if context.session and (cookies := state.get('cookies')):
+ if cookies := state.get('cookies'):
context.session.cookies.set_cookies(cast('list[CookieParam]', cookies))
await crawler.run(['https://httpbingo.org/cookies'])
diff --git a/docs/03_concepts/code_examples/cookie_management/retry_single_session.py b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py
index 1eb8f650ed..a5390a321b 100644
--- a/docs/03_concepts/code_examples/cookie_management/retry_single_session.py
+++ b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py
@@ -23,9 +23,6 @@ async def main() -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if context.session is None:
- return
-
# First attempt: establish cookies, then raise to trigger a retry.
if not context.session.cookies:
await context.send_request('https://httpbingo.org/cookies/set?logged_in=1')
diff --git a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py
index 1243b0e488..4c9e615e06 100644
--- a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py
+++ b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py
@@ -14,7 +14,7 @@ async def main() -> None:
)
crawler = BeautifulSoupCrawler(
proxy_configuration=proxy_configuration,
- use_session_pool=True,
+ save_response_cookies=True,
)
# ...
diff --git a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py
index 68309bda59..02205a1ad6 100644
--- a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py
+++ b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py
@@ -14,7 +14,7 @@ async def main() -> None:
)
crawler = PlaywrightCrawler(
proxy_configuration=proxy_configuration,
- use_session_pool=True,
+ save_response_cookies=True,
)
# ...
diff --git a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py
index 98b2028597..e6275d0a61 100644
--- a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py
+++ b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py
@@ -45,21 +45,17 @@ async def basic_handler(context: HttpCrawlingContext) -> None:
# Initialize the session and bind the next request to this session if needed
@crawler.router.handler(label='session_init')
async def session_init(context: HttpCrawlingContext) -> None:
- next_requests = []
- if context.session:
- context.log.info('Init session %s', context.session.id)
- next_request = Request.from_url(
- 'https://a.placeholder.com', session_id=context.session.id
- )
- next_requests.append(next_request)
-
- await context.add_requests(next_requests)
+ context.log.info('Init session %s', context.session.id)
+ next_request = Request.from_url(
+ 'https://a.placeholder.com', session_id=context.session.id
+ )
+ await context.add_requests([next_request])
# Handle errors when a session is blocked and no longer available in the pool
# when attempting to execute requests bound to it
@crawler.failed_request_handler
async def error_processing(context: BasicCrawlingContext, error: Exception) -> None:
- if isinstance(error, RequestCollisionError) and context.session:
+ if isinstance(error, RequestCollisionError):
context.log.error(
'Request %s failed, because the bound session is unavailable',
context.request.url,
diff --git a/docs/03_concepts/code_examples/session_management/one_session_http.py b/docs/03_concepts/code_examples/session_management/one_session_http.py
index 6bb67a240d..b5c6bac0b1 100644
--- a/docs/03_concepts/code_examples/session_management/one_session_http.py
+++ b/docs/03_concepts/code_examples/session_management/one_session_http.py
@@ -39,13 +39,12 @@ async def basic_handler(context: HttpCrawlingContext) -> None:
# Handler for session initialization (authentication, initial cookies, etc.)
@crawler.router.handler(label='session_init')
async def session_init(context: HttpCrawlingContext) -> None:
- if context.session:
- context.log.info('Init session %s', context.session.id)
+ context.log.info('Init session %s', context.session.id)
# Monitor if our session gets blocked and explicitly stop the crawler
@crawler.error_handler
async def error_processing(context: BasicCrawlingContext, error: Exception) -> None:
- if isinstance(error, SessionError) and context.session:
+ if isinstance(error, SessionError):
context.log.info('Session %s blocked', context.session.id)
crawler.stop()
diff --git a/docs/03_concepts/code_examples/session_management/sm_basic.py b/docs/03_concepts/code_examples/session_management/sm_basic.py
index 30e1d7ae92..3371c6a0c4 100644
--- a/docs/03_concepts/code_examples/session_management/sm_basic.py
+++ b/docs/03_concepts/code_examples/session_management/sm_basic.py
@@ -16,8 +16,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = BasicCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -33,13 +31,14 @@ async def default_handler(context: BasicCrawlingContext) -> None:
page_content = (await response.read()).decode()
title_match = re.search(r'(.*?)', page_content)
- if context.session and (title := title_match.group(1) if title_match else None):
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ title = title_match.group(1) if title_match else None
+
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py
index a54fd8425f..db8028213f 100644
--- a/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py
+++ b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py
@@ -15,8 +15,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = BeautifulSoupCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -27,13 +25,12 @@ async def main() -> None:
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
title = context.soup.title.get_text() if context.soup.title else None
- if context.session:
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/03_concepts/code_examples/session_management/sm_http.py b/docs/03_concepts/code_examples/session_management/sm_http.py
index 9497594d3b..18c8585152 100644
--- a/docs/03_concepts/code_examples/session_management/sm_http.py
+++ b/docs/03_concepts/code_examples/session_management/sm_http.py
@@ -16,8 +16,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = HttpCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -29,13 +27,14 @@ async def default_handler(context: HttpCrawlingContext) -> None:
page_content = (await context.http_response.read()).decode()
title_match = re.search(r'(.*?)', page_content)
- if context.session and (title := title_match.group(1) if title_match else None):
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ title = title_match.group(1) if title_match else None
+
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/03_concepts/code_examples/session_management/sm_parsel.py b/docs/03_concepts/code_examples/session_management/sm_parsel.py
index 66752a63c3..508a88a4c6 100644
--- a/docs/03_concepts/code_examples/session_management/sm_parsel.py
+++ b/docs/03_concepts/code_examples/session_management/sm_parsel.py
@@ -15,8 +15,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = ParselCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -27,13 +25,12 @@ async def main() -> None:
async def default_handler(context: ParselCrawlingContext) -> None:
title = context.selector.css('title::text').get()
- if context.session:
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/03_concepts/code_examples/session_management/sm_playwright.py b/docs/03_concepts/code_examples/session_management/sm_playwright.py
index 46a4c4f096..7e848fbcad 100644
--- a/docs/03_concepts/code_examples/session_management/sm_playwright.py
+++ b/docs/03_concepts/code_examples/session_management/sm_playwright.py
@@ -15,8 +15,6 @@ async def main() -> None:
# to manage concurrent sessions and proxy rotation
crawler = PlaywrightCrawler(
proxy_configuration=proxy_configuration,
- # Activates the Session pool (default is true).
- use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
@@ -27,13 +25,12 @@ async def main() -> None:
async def default_handler(context: PlaywrightCrawlingContext) -> None:
title = await context.page.title()
- if context.session:
- if title == 'Blocked':
- context.session.retire()
- elif title == 'Not sure if blocked, might also be a connection error':
- context.session.mark_bad()
- else:
- context.session.mark_good() # BasicCrawler handles this automatically.
+ if title == 'Blocked':
+ context.session.retire()
+ elif title == 'Not sure if blocked, might also be a connection error':
+ context.session.mark_bad()
+ else:
+ context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
diff --git a/docs/04_guides/code_examples/login_crawler/http_login.py b/docs/04_guides/code_examples/login_crawler/http_login.py
index 82848ce834..c5fe5a4cea 100644
--- a/docs/04_guides/code_examples/login_crawler/http_login.py
+++ b/docs/04_guides/code_examples/login_crawler/http_login.py
@@ -41,10 +41,6 @@ async def request_handler(context: HttpCrawlingContext) -> None:
async def login_handler(context: HttpCrawlingContext) -> None:
context.log.info('Processing login at %s ...', context.request.url)
- # Verify that a session is available before proceeding
- if not context.session:
- raise RuntimeError('Session not found')
-
# Parse the API response containing authentication tokens and user data
data = json.loads(await context.http_response.read())
diff --git a/docs/04_guides/code_examples/login_crawler/playwright_login.py b/docs/04_guides/code_examples/login_crawler/playwright_login.py
index 650bdd917a..ad4a3cfe72 100644
--- a/docs/04_guides/code_examples/login_crawler/playwright_login.py
+++ b/docs/04_guides/code_examples/login_crawler/playwright_login.py
@@ -44,10 +44,6 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
async def login_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info('Processing login %s ...', context.request.url)
- # Check if the session is available
- if not context.session:
- raise RuntimeError('Session not found')
-
# Entering data into the form, `delay` to simulate human typing
# Without this, the data will be entered instantly
await context.page.type('#userName', 'crawlee_test', delay=100)
diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py
index b97a77552e..a830c552ac 100644
--- a/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py
+++ b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py
@@ -8,10 +8,6 @@ async def main() -> None:
@crawler.router.default_handler
async def login_page(context: ParselCrawlingContext) -> None:
- # The CSRF token is tied to the session cookie issued for this GET.
- if not context.session:
- raise RuntimeError('Session not found')
-
# highlight-start
# Like Scrapy's `FormRequest.from_response`, the helper keeps the hidden
# `csrf_token` field, encodes the data and sets the `Content-Type` header.
diff --git a/docs/06_upgrading/upgrading_to_v2.md b/docs/06_upgrading/upgrading_to_v2.md
index ee57b83421..e00403b4f9 100644
--- a/docs/06_upgrading/upgrading_to_v2.md
+++ b/docs/06_upgrading/upgrading_to_v2.md
@@ -489,6 +489,35 @@ class MyHttpClient(HttpClient):
`SitemapRequestLoader.fetch_next_request` skips a URL while its request is still in progress, for example when another sitemap lists it too. In v1, it returned a request for the URL again, and `get_total_count` counted the URL twice.
+## Crawlers manage only what they create
+
+A crawler now enters and exits only the objects it creates, the same as Crawlee JS. An object you pass is yours to enter and exit. The `Statistics` and the event managers are the exceptions, since the crawler enters them on every run. You can enter an event manager yourself as well. In v1, the crawler entered a passed object that wasn't active yet and exited it when its run finished, even while another crawler still used it.
+
+What you do with an object you pass depends on its type:
+
+- `SessionPool`, `BrowserPool`, an HTTP client and the `rendering_type_predictor` of `AdaptivePlaywrightCrawler`: enter it with `async with` before you run the crawler, or `run()` raises a `RuntimeError`. Keep it open until every crawler using it has finished.
+- The `extractor` of `PydanticAiCrawler`: enter it the same way. `PydanticAiDirectExtractor` is now an async context manager, like `PydanticAiSelectorExtractor`. A selector extractor enters and exits its `fallback` itself, so don't enter the fallback yourself, and give each selector extractor its own. If your subclass of `BasePydanticAiHtmlExtractor` overrides `__aenter__` or `__aexit__`, call `super()`. A custom extractor that implements the `PydanticAiHtmlExtractor` protocol directly needs the new `active` property, `__aenter__` and `__aexit__`.
+- `Statistics`: don't enter it yourself, or `run()` raises a `RuntimeError`. In v1, the crawler left entered statistics alone. Crawlers that run one after another can share an instance, including another crawler's statistics. The instance adds up their runs, so the `max_requests_per_crawl` of the second crawler counts the requests of the first one too. When a crawler runs again, it resets the instance, including what the other crawlers counted. Two crawlers can't run with the same instance at the same time.
+
+For example:
+
+```python
+# v1
+crawler = ParselCrawler(session_pool=SessionPool(max_pool_size=10))
+await crawler.run(urls)
+
+# v2
+session_pool = SessionPool(max_pool_size=10)
+crawler = ParselCrawler(session_pool=session_pool)
+
+async with session_pool:
+ await crawler.run(urls)
+```
+
+When a crawler runs again, it resets only a session pool it created. A `SessionPool` you pass keeps its sessions. In v1, a repeated `run()` reset a passed pool too.
+
+`RenderingTypePredictor` has a new `active` property, which `initialize()` and `clear()` of the base class keep up to date. If your predictor overrides them, call `super()`.
+
## Resetting statistics and the session pool
`Statistics.reset()` is now synchronous and resets only the in-memory statistics, the same as Crawlee JS. In v1, it also cleared the persisted record. To clear the record too, call the new `reset_store()` once the statistics are exited:
@@ -580,6 +609,29 @@ The `ThrottlingRequestManager` warning about capping a delay now calls it a "req
Log records hold the message template and its arguments separately. `record.msg` is the template, such as `Retrying request to %s due to: %s. %s`, and `record.args` holds the values. To read the final text, use `record.getMessage()`, which formatters and handlers already do.
+## `use_session_pool` is removed
+
+The `use_session_pool` option is removed from the crawlers. Every crawler processes its requests with sessions from its `SessionPool`, the same as Crawlee JS v4. To configure the pool, pass your own `SessionPool` as `session_pool`.
+
+`context.session` is now a `Session` instead of `Session | None`, so you can drop the `if context.session` checks. A request bound by `session_id` to a session that's no longer available still fails with a `RequestCollisionError`, and its failed request handler gets a retired session with the bound id instead of `None`.
+
+Without the session pool, a 401, 403 or 429 response failed the request without a retry. Now it retires the session, and the request is retried with another one, up to `max_session_rotations` times. With `proxy_urls`, a session keeps its proxy, so the proxy changes when the session does, not on every request.
+
+To keep the response cookies out of the sessions, pass `save_response_cookies=False`. To retire every session after its first request, set `max_usage_count` to 1:
+
+```python
+# v1
+crawler = ParselCrawler(use_session_pool=False)
+await crawler.run(urls)
+
+# v2
+session_pool = SessionPool(create_session_settings={'max_usage_count': 1})
+crawler = ParselCrawler(save_response_cookies=False, session_pool=session_pool)
+
+async with session_pool:
+ await crawler.run(urls)
+```
+
## Session retirement is permanent
In v1, `Session.retire()` only added `max_error_score` to the error score, so a later `mark_good()` could lower it again and make the retired session usable. In v2, a separate `retired` flag tracks retirement, exposed as the `Session.retired` property, and a retired session is never usable again. Create a new session to replace it.
diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py
index 85f90a3cfc..36d9e6f252 100644
--- a/src/crawlee/_types.py
+++ b/src/crawlee/_types.py
@@ -656,7 +656,7 @@ class BasicCrawlingContext:
request: Request
"""Request object for the current page being processed."""
- session: Session | None
+ session: Session
"""Session object for the current page being processed."""
proxy_info: ProxyInfo | None
diff --git a/src/crawlee/_utils/context.py b/src/crawlee/_utils/context.py
index 9faa994376..ce5e519c47 100644
--- a/src/crawlee/_utils/context.py
+++ b/src/crawlee/_utils/context.py
@@ -3,11 +3,42 @@
import inspect
from collections.abc import Callable
from functools import wraps
-from typing import Any, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Protocol, TypeVar, cast
+
+if TYPE_CHECKING:
+ from types import TracebackType
T = TypeVar('T', bound=Callable[..., Any])
+class Collaborator(Protocol):
+ """An async context manager that reports through `active` whether it's entered."""
+
+ @property
+ def active(self) -> bool:
+ """Whether the object is entered and not exited yet."""
+
+ async def __aenter__(self) -> Any:
+ """Prepare the object for use.
+
+ Raises:
+ RuntimeError: If the object is already active.
+ """
+
+ async def __aexit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_value: BaseException | None,
+ exc_traceback: TracebackType | None,
+ /,
+ ) -> Any:
+ """Release what the object holds.
+
+ Raises:
+ RuntimeError: If the object is not active.
+ """
+
+
def ensure_context(method: T) -> T:
"""Ensure the (async) context manager is initialized before executing the method.
diff --git a/src/crawlee/autoscaling/_types.py b/src/crawlee/autoscaling/_types.py
index 678a4f3590..ac872acf3c 100644
--- a/src/crawlee/autoscaling/_types.py
+++ b/src/crawlee/autoscaling/_types.py
@@ -7,6 +7,7 @@
from pydantic import Field
from pydantic.dataclasses import dataclass as pydantic_dataclass
+from crawlee._utils.context import Collaborator
from crawlee._utils.docs import docs_group
@@ -104,12 +105,13 @@ class ConcurrencyConsumer:
@docs_group('Autoscaling')
-class ConcurrencySystemInterface(Protocol):
+class ConcurrencySystemInterface(Collaborator, Protocol):
"""The contract between an `AutoscaledPool` and the component that governs its concurrency.
The system answers whether there is free compute for one more task and tracks the budget that tasks are booked
against. `ConcurrencySystem` is the default implementation. Anything matching this contract can take its place,
- without depending on its internals.
+ without depending on its internals. An implementation is an async context manager, and `AutoscaledPool.run` refuses
+ to run on one that isn't entered.
Every allocation method is told which `ConcurrencyConsumer` is asking, so an implementation may allocate per
consumer. `ConcurrencySystem` does not, and serves whoever asks first, which can starve a consumer that joins
@@ -127,10 +129,6 @@ def desired_concurrency(self) -> int:
def current_concurrency(self) -> int:
"""The number of parallel tasks currently booked against this system."""
- @property
- def active(self) -> bool:
- """Whether the system is ready to be booked against. `AutoscaledPool.run` refuses to run when it is not."""
-
def has_capacity_for_task(self, consumer: ConcurrencyConsumer) -> bool:
"""Report whether one more task may start right now on behalf of `consumer`.
diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py
index b2a949efdc..b4c0a0e22a 100644
--- a/src/crawlee/browsers/_browser_pool.py
+++ b/src/crawlee/browsers/_browser_pool.py
@@ -214,12 +214,29 @@ async def __aenter__(self) -> BrowserPool:
self._close_inactive_browsers_task.start()
timeout = self._operation_timeout.total_seconds()
+ entered_plugins: list[BrowserPlugin] = []
try:
for plugin in self._plugins:
await asyncio.wait_for(plugin.__aenter__(), timeout)
+ entered_plugins.append(plugin)
except TimeoutError:
logger.warning('Initializing of the browser plugin %s timed out, will be skipped.', plugin)
+ except BaseException:
+ try:
+ await self._stop_recurring_tasks()
+ finally:
+ try:
+ plugin_exits = (entered_plugin.__aexit__(None, None, None) for entered_plugin in entered_plugins)
+ plugin_results = await asyncio.gather(*plugin_exits, return_exceptions=True)
+ finally:
+ self._active = False
+ for result in plugin_results:
+ if isinstance(result, BaseException):
+ logger.warning(
+ 'Exiting a browser plugin failed while rolling back the pool entry.', exc_info=result
+ )
+ raise
return self
@@ -237,18 +254,39 @@ async def __aexit__(
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')
- await self._identify_inactive_browsers_task.stop()
- await self._close_inactive_browsers_task.stop()
+ # Every browser and plugin is closed even when another one fails, since an inactive pool can't be exited again.
+ failures: list[BaseException] = []
+ try:
+ try:
+ await self._stop_recurring_tasks()
- for browser in self._active_browsers + self._inactive_browsers:
- await browser.close(force=True)
- self._active_browsers.clear()
- self._inactive_browsers.clear()
+ browser_closes = (
+ browser.close(force=True) for browser in self._active_browsers + self._inactive_browsers
+ )
+ browser_results = await asyncio.gather(*browser_closes, return_exceptions=True)
+ failures.extend(result for result in browser_results if isinstance(result, BaseException))
+ finally:
+ self._active_browsers.clear()
+ self._inactive_browsers.clear()
- for plugin in self._plugins:
- await plugin.__aexit__(exc_type, exc_value, exc_traceback)
+ try:
+ # Plugins left inactive by a timed-out entry have nothing to exit.
+ active_plugins = [plugin for plugin in self._plugins if plugin.active]
+ plugin_exits = (plugin.__aexit__(exc_type, exc_value, exc_traceback) for plugin in active_plugins)
+ plugin_results = await asyncio.gather(*plugin_exits, return_exceptions=True)
+ failures.extend(result for result in plugin_results if isinstance(result, BaseException))
+ finally:
+ self._active = False
+ except BaseException:
+ # A cancellation above is raised instead, so the failures are only logged.
+ for failure in failures:
+ logger.warning('Closing the browser pool failed.', exc_info=failure)
+ raise
- self._active = False
+ for failure in failures[1:]:
+ logger.warning('Closing the browser pool failed.', exc_info=failure)
+ if failures:
+ raise failures[0]
@ensure_context
async def new_page(
@@ -392,6 +430,12 @@ async def _close_inactive_browsers(self) -> None:
await browser.close()
self._inactive_browsers.remove(browser)
+ async def _stop_recurring_tasks(self) -> None:
+ try:
+ await self._identify_inactive_browsers_task.stop()
+ finally:
+ await self._close_inactive_browsers_task.stop()
+
async def _execute_hooks(self, hooks: list[Callable[..., Awaitable[None]]], *args: Any) -> None:
"""Execute the provided hooks with the given arguments."""
for hook in hooks:
diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py
index 5cc53e7be8..475505220f 100644
--- a/src/crawlee/browsers/_playwright_browser_plugin.py
+++ b/src/crawlee/browsers/_playwright_browser_plugin.py
@@ -155,7 +155,21 @@ async def __aenter__(self) -> PlaywrightBrowserPlugin:
raise RuntimeError(f'The {self.__class__.__name__} is already active.')
self._active = True
- self._playwright = await self._playwright_context_manager.__aenter__()
+
+ try:
+ self._playwright = await self._playwright_context_manager.__aenter__()
+ except BaseException:
+ # The Playwright driver may already be running, so it's stopped too.
+ half_started_context_manager = self._playwright_context_manager
+ self._playwright_context_manager = async_playwright()
+ try:
+ await half_started_context_manager.__aexit__(None, None, None)
+ except Exception:
+ logger.warning('Stopping Playwright failed while rolling back the plugin entry.', exc_info=True)
+ finally:
+ self._active = False
+ raise
+
return self
@override
@@ -168,9 +182,11 @@ async def __aexit__(
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')
- await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback)
- self._playwright_context_manager = async_playwright()
- self._active = False
+ try:
+ await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback)
+ finally:
+ self._playwright_context_manager = async_playwright()
+ self._active = False
@override
@ensure_context
diff --git a/src/crawlee/browsers/_stagehand_browser_plugin.py b/src/crawlee/browsers/_stagehand_browser_plugin.py
index a94a973bee..1d431acd15 100644
--- a/src/crawlee/browsers/_stagehand_browser_plugin.py
+++ b/src/crawlee/browsers/_stagehand_browser_plugin.py
@@ -163,19 +163,33 @@ async def __aenter__(self) -> StagehandBrowserPlugin:
raise RuntimeError(f'The {self.__class__.__name__} is already active.')
self._active = True
- self._playwright = await self._playwright_context_manager.__aenter__()
- if self._stagehand_options.env == 'LOCAL':
- if 'executable_path' not in self._browser_launch_options:
- chrome_path = self._playwright.chromium.executable_path
- self._browser_launch_options['executable_path'] = chrome_path
- logger.debug('Resolved Chromium path from Playwright: %s', chrome_path)
-
- self._stagehand_init_params['local_chrome_path'] = self._browser_launch_options['executable_path']
-
- client = AsyncStagehand(**self._stagehand_init_params)
- await client.__aenter__()
- self._stagehand_client = client
+ try:
+ self._playwright = await self._playwright_context_manager.__aenter__()
+
+ if self._stagehand_options.env == 'LOCAL':
+ if 'executable_path' not in self._browser_launch_options:
+ chrome_path = self._playwright.chromium.executable_path
+ self._browser_launch_options['executable_path'] = chrome_path
+ logger.debug('Resolved Chromium path from Playwright: %s', chrome_path)
+
+ self._stagehand_init_params['local_chrome_path'] = self._browser_launch_options['executable_path']
+
+ client = AsyncStagehand(**self._stagehand_init_params)
+ await client.__aenter__()
+ self._stagehand_client = client
+ except BaseException:
+ # The Playwright driver may already be running, so it's stopped too.
+ self._playwright = None
+ playwright_context_manager = self._playwright_context_manager
+ self._playwright_context_manager = async_playwright()
+ try:
+ await playwright_context_manager.__aexit__(None, None, None)
+ except Exception:
+ logger.warning('Stopping Playwright failed while rolling back the plugin entry.', exc_info=True)
+ finally:
+ self._active = False
+ raise
return self
@@ -189,14 +203,17 @@ async def __aexit__(
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')
- if self._stagehand_client is not None:
- await self._stagehand_client.__aexit__(exc_type, exc_value, exc_traceback)
- self._stagehand_client = None
-
- await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback)
- self._playwright_context_manager = async_playwright()
- self._playwright = None
- self._active = False
+ try:
+ if self._stagehand_client is not None:
+ await self._stagehand_client.__aexit__(exc_type, exc_value, exc_traceback)
+ finally:
+ try:
+ await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback)
+ finally:
+ self._stagehand_client = None
+ self._playwright = None
+ self._playwright_context_manager = async_playwright()
+ self._active = False
@override
@ensure_context
diff --git a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
index 2559eff3a2..f13a8b6b48 100644
--- a/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
+++ b/src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py
@@ -309,7 +309,7 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera
"""
# Without saving, the session cookies are still sent, but the response cookies go to a copy.
cookies = None
- if context.session is not None and not self._save_response_cookies:
+ if not self._save_response_cookies:
cookies = SessionCookies(context.session.cookies)
async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout:
diff --git a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py
index 8f2e74ab2f..7af0e76e9d 100644
--- a/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py
+++ b/src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py
@@ -134,7 +134,8 @@ def __init__(
Args:
rendering_type_predictor: Object that implements RenderingTypePredictor and is capable of predicting which
- rendering method should be used. If None, then `DefaultRenderingTypePredictor` is used.
+ rendering method should be used. If None, then `DefaultRenderingTypePredictor` is used. Keep a
+ predictor passed in open with `async with` until every crawler using it has finished.
result_checker: Function that evaluates whether crawling result is valid or not.
result_comparator: Function that compares two crawling results and decides whether they are equivalent.
static_parser: Implementation of `AbstractHttpParser`. Parser that will be used for static crawling.
@@ -143,11 +144,10 @@ def __init__(
save_response_cookies: Whether to save the response cookies to the session, in both the static and the
browser sub crawler.
statistics: A custom `Statistics[AdaptivePlaywrightCrawlerStatisticState]` instance, allowing the use of
- non-default configuration.
+ non-default configuration. The crawler enters it on every run, so don't enter it yourself.
kwargs: Additional keyword arguments to pass to the underlying `BasicCrawler`.
"""
# Adaptive crawling related.
- self.rendering_type_predictor = rendering_type_predictor or DefaultRenderingTypePredictor()
self.result_checker = result_checker or (lambda _: True)
self.result_comparator = result_comparator or create_default_comparator(result_checker)
@@ -163,6 +163,8 @@ def __init__(
super().__init__(statistics=adaptive_statistics, **kwargs)
+ self.rendering_type_predictor = self._own_or_borrow(rendering_type_predictor, DefaultRenderingTypePredictor)
+
# Sub crawlers related.
playwright_crawler_specific_kwargs = playwright_crawler_specific_kwargs or _PlaywrightCrawlerAdditionalOptions()
@@ -228,13 +230,14 @@ async def adaptive_post_navigation_hook_pw(context: PlaywrightPostNavCrawlingCon
static_crawler.post_navigation_hook(adaptive_post_navigation_hook_static)
playwright_crawler.post_navigation_hook(adaptive_post_navigation_hook_pw)
- self._additional_context_managers = [
- *self._additional_context_managers,
- self.rendering_type_predictor,
- static_crawler.statistics,
- playwright_crawler.statistics,
- playwright_crawler._browser_pool, # noqa: SLF001 # Intentional access to private member.
- ]
+ # The sub crawlers never run themselves, so this crawler enters and exits their statistics.
+ self._owned_context_managers.extend([static_crawler.statistics, playwright_crawler.statistics])
+
+ # The default pool is the one the browser sub crawler built.
+ self._own_or_borrow(
+ playwright_crawler_specific_kwargs.get('browser_pool'),
+ lambda: playwright_crawler._browser_pool, # noqa: SLF001 # Intentional access to private member.
+ )
# Sub crawler pipeline related
self._pw_context_pipeline = playwright_crawler._context_pipeline # noqa:SLF001 # Intentional access to private member.
diff --git a/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py b/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py
index de80baf56f..dd3e979ba1 100644
--- a/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py
+++ b/src/crawlee/crawlers/_adaptive_playwright/_rendering_type_predictor.py
@@ -68,6 +68,11 @@ def __init__(self) -> None:
# Flag to indicate the state.
self._active = False
+ @property
+ def active(self) -> bool:
+ """Whether the predictor is initialized and not cleared since."""
+ return self._active
+
@abstractmethod
def predict(self, request: Request) -> RenderingTypePrediction:
"""Get `RenderingTypePrediction` based on the input request.
@@ -164,15 +169,20 @@ async def initialize(self) -> None:
await super().initialize()
if not self._state.is_initialized:
- await self._state.initialize()
+ try:
+ await self._state.initialize()
+ except BaseException:
+ await super().clear()
+ raise
@override
async def clear(self) -> None:
"""Clear the predictor state."""
- await super().clear()
-
- if self._state.is_initialized:
- await self._state.teardown()
+ try:
+ if self._state.is_initialized:
+ await self._state.teardown()
+ finally:
+ await super().clear()
@override
def predict(self, request: Request) -> RenderingTypePrediction:
diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py
index 9296285425..76314667fa 100644
--- a/src/crawlee/crawlers/_basic/_basic_crawler.py
+++ b/src/crawlee/crawlers/_basic/_basic_crawler.py
@@ -42,6 +42,7 @@
SendRequestFunction,
SkippedReason,
)
+from crawlee._utils.context import Collaborator
from crawlee._utils.docs import docs_group
from crawlee._utils.file import atomic_write, export_csv_to_stream, export_json_to_stream
from crawlee._utils.globs import check_url_patterns
@@ -70,7 +71,7 @@
from crawlee.http_clients import ImpitHttpClient
from crawlee.request_loaders import MinIntervalSignal, RateLimitedSignal
from crawlee.router import Router
-from crawlee.sessions import SessionPool
+from crawlee.sessions import Session, SessionPool
from crawlee.statistics import Statistics, StatisticsState
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
@@ -101,7 +102,7 @@
from crawlee.http_clients import HttpClient, HttpResponse
from crawlee.proxy_configuration import ProxyConfiguration, ProxyInfo
from crawlee.request_loaders import RequestManager
- from crawlee.sessions import Session, SessionCookies
+ from crawlee.sessions import SessionCookies
from crawlee.statistics import FinalStatistics
from crawlee.storage_backends import StorageBackend
from crawlee.storage_backends.models import DatasetItemsListPage
@@ -111,6 +112,7 @@
TRequestIterator = TypeVar('TRequestIterator', str, Request)
TParams = ParamSpec('TParams')
T = TypeVar('T')
+TCollaborator = TypeVar('TCollaborator', bound=Collaborator)
ErrorHandler = Callable[[TCrawlingContext, Exception], Awaitable[Request | None]]
FailedRequestHandler = Callable[[TCrawlingContext, Exception], Awaitable[None]]
@@ -133,13 +135,15 @@ class _BasicCrawlerOptions(TypedDict):
"""Manager of requests that should be processed by the crawler."""
session_pool: NotRequired[SessionPool]
- """A custom `SessionPool` instance, allowing the use of non-default configuration."""
+ """A custom `SessionPool` instance, allowing the use of non-default configuration. Keep it open with `async with`
+ until every crawler using it has finished."""
proxy_configuration: NotRequired[ProxyConfiguration]
"""HTTP proxy configuration used when making requests."""
http_client: NotRequired[HttpClient]
- """HTTP client used by `BasicCrawlingContext.send_request` method."""
+ """HTTP client used by `BasicCrawlingContext.send_request` method. Keep it open with `async with` until every
+ crawler using it has finished."""
max_request_retries: NotRequired[int]
"""Specifies the maximum number of retries allowed for a request if its processing fails.
@@ -167,9 +171,6 @@ class _BasicCrawlerOptions(TypedDict):
If not set, crawling continues without depth restrictions.
"""
- use_session_pool: NotRequired[bool]
- """Enable the use of a session pool for managing sessions during crawling."""
-
retry_on_blocked: NotRequired[bool]
"""If True, the crawler attempts to bypass bot protections automatically."""
@@ -178,7 +179,8 @@ class _BasicCrawlerOptions(TypedDict):
concurrency_system: NotRequired[ConcurrencySystemInterface]
"""A concurrency system to use instead of the default one, e.g. one shared between crawlers to cap their combined
- concurrency. Cannot be combined with `concurrency_settings`."""
+ concurrency. Keep it open with `async with` until every crawler using it has finished. Cannot be combined with
+ `concurrency_settings`."""
request_handler_timeout: NotRequired[timedelta]
"""Maximum duration allowed for a single request handler to run."""
@@ -208,8 +210,9 @@ class _BasicCrawlerOptions(TypedDict):
"""HTTP status codes that are typically considered errors but should be treated as successful responses."""
_additional_context_managers: NotRequired[Sequence[AbstractAsyncContextManager]]
- """Additional context managers used throughout the crawler lifecycle. Intended for use by
- subclasses rather than direct instantiation of `BasicCrawler`."""
+ """Additional context managers the crawler owns, entered and exited on every run. A subclass lists the defaults it
+ builds here, so that they aren't taken for objects passed in. Intended for use by subclasses rather than direct
+ instantiation of `BasicCrawler`."""
respect_robots_txt_file: NotRequired[bool]
"""If set to `True`, the crawler will automatically try to fetch the robots.txt file for each domain,
@@ -241,7 +244,8 @@ class _BasicCrawlerOptionsGeneric(TypedDict, Generic[TCrawlingContext, TStatisti
subclasses rather than direct instantiation of `BasicCrawler`."""
statistics: NotRequired[Statistics[TStatisticsState]]
- """A custom `Statistics` instance, allowing the use of non-default configuration."""
+ """A custom `Statistics` instance, allowing the use of non-default configuration. The crawler enters it on every
+ run, so don't enter it yourself. Crawlers can share it only when they run one after another."""
class BasicCrawlerOptions(
@@ -302,7 +306,6 @@ def __init__(
max_requests_per_crawl: int | None = None,
max_session_rotations: int = 10,
max_crawl_depth: int | None = None,
- use_session_pool: bool = True,
retry_on_blocked: bool = True,
additional_http_error_status_codes: Iterable[int] | None = None,
ignore_http_error_status_codes: Iterable[int] | None = None,
@@ -330,9 +333,11 @@ def __init__(
event_manager: The event manager for managing events for the crawler and all its components.
storage_backend: The storage backend for managing storages for the crawler and all its components.
request_manager: Manager of requests that should be processed by the crawler.
- session_pool: A custom `SessionPool` instance, allowing the use of non-default configuration.
+ session_pool: A custom `SessionPool` instance, allowing the use of non-default configuration. Keep it open
+ with `async with` until every crawler using it has finished.
proxy_configuration: HTTP proxy configuration used when making requests.
- http_client: HTTP client used by `BasicCrawlingContext.send_request` method.
+ http_client: HTTP client used by `BasicCrawlingContext.send_request` method. Keep it open with `async with`
+ until every crawler using it has finished.
request_handler: A callable responsible for handling requests.
max_request_retries: Specifies the maximum number of retries allowed for a request if its processing fails.
This includes retries due to navigation errors or errors thrown from user-supplied functions
@@ -350,7 +355,6 @@ def __init__(
this depth. The crawl depth starts at 0 for initial requests and increases with each subsequent level
of links. Requests at the maximum depth will still be processed, but no new links will be enqueued
from those requests. If not set, crawling continues without depth restrictions.
- use_session_pool: Enable the use of a session pool for managing sessions during crawling.
retry_on_blocked: If True, the crawler attempts to bypass bot protections automatically.
additional_http_error_status_codes: Additional HTTP status codes to treat as errors,
triggering automatic retries when encountered.
@@ -362,7 +366,9 @@ def __init__(
crawlers to cap their combined concurrency. Keep it open with `async with` until every crawler using it
has finished. Cannot be combined with `concurrency_settings`.
request_handler_timeout: Maximum duration allowed for a single request handler to run.
- statistics: A custom `Statistics` instance, allowing the use of non-default configuration.
+ statistics: A custom `Statistics` instance, allowing the use of non-default configuration. The crawler
+ enters it on every run, so don't enter it yourself. Crawlers can share it only when they run one after
+ another.
abort_on_error: If True, the crawler stops immediately when any request handler error occurs.
keep_alive: If True, it will keep crawler alive even if there are no requests in queue.
Use `crawler.stop()` to exit the crawler.
@@ -383,7 +389,8 @@ def __init__(
it, the ID follows the order in which crawlers are created.
_context_pipeline: Enables extending the request lifecycle and modifying the crawling context.
Intended for use by subclasses rather than direct instantiation of `BasicCrawler`.
- _additional_context_managers: Additional context managers used throughout the crawler lifecycle.
+ _additional_context_managers: Additional context managers the crawler owns, entered and exited on every
+ run. A subclass lists the defaults it builds here, so that they aren't taken for objects passed in.
Intended for use by subclasses rather than direct instantiation of `BasicCrawler`.
"""
self._id = str(BasicCrawler.__next_id) if id is None else id
@@ -409,9 +416,14 @@ def __init__(
config = self._service_locator.get_configuration()
+ # The crawler enters and exits what it owns on every run, and only checks that the collaborators passed in are
+ # active.
+ self._owned_context_managers: list[AbstractAsyncContextManager] = [*(_additional_context_managers or [])]
+ self._borrowed_collaborators: list[Collaborator] = []
+
# Core components
self._request_manager = request_manager
- self._session_pool = session_pool or SessionPool()
+ self._session_pool = self._own_or_borrow(session_pool, SessionPool)
self._proxy_configuration = proxy_configuration
self._additional_http_error_status_codes = (
@@ -421,7 +433,7 @@ def __init__(
set(ignore_http_error_status_codes) if ignore_http_error_status_codes else set()
)
- self._http_client = http_client or ImpitHttpClient()
+ self._http_client = self._own_or_borrow(http_client, ImpitHttpClient)
# Request router setup
self._router: Router[TCrawlingContext] | None = None
@@ -465,7 +477,6 @@ def __init__(
)
# Retry and session settings
- self._use_session_pool = use_session_pool
self._retry_on_blocked = retry_on_blocked
# Logging setup
@@ -503,9 +514,6 @@ async def persist_state_factory() -> KeyValueStore:
),
)
- # Additional context managers to enter and exit
- self._additional_context_managers = _additional_context_managers or []
-
# Internal, not explicitly configurable components
self._robots_txt_file_cache: LRUCache[str, RobotsTxtFile] = LRUCache(maxsize=1000)
self._robots_txt_lock = asyncio.Lock()
@@ -519,6 +527,8 @@ async def persist_state_factory() -> KeyValueStore:
self._concurrency_settings = concurrency_settings
self._borrowed_concurrency_system = concurrency_system
+ if concurrency_system is not None:
+ self._borrowed_collaborators.append(concurrency_system)
# The default system is built per run, so that a run is not scaled on the load and concurrency of the previous
# one. A supplied system is known up front.
@@ -537,6 +547,21 @@ async def persist_state_factory() -> KeyValueStore:
# The last run started on this instance, which stays readable after it finishes.
self._last_run: CrawlerRun[TStatisticsState] | None = None
+ def _own_or_borrow(self, passed: TCollaborator | None, build_default: Callable[[], TCollaborator]) -> TCollaborator:
+ """Return the collaborator passed in, or build the default one, and record which of them the crawler owns.
+
+ A collaborator passed in that a subclass listed in `_additional_context_managers` counts as the crawler's own.
+ """
+ if passed is None:
+ default = build_default()
+ self._owned_context_managers.append(default)
+ return default
+
+ if passed not in self._owned_context_managers:
+ self._borrowed_collaborators.append(passed)
+
+ return passed
+
@property
def log(self) -> logging.Logger:
"""The logger used by the crawler."""
@@ -662,11 +687,8 @@ def _stop_if_max_requests_count_exceeded(self) -> None:
reason=f'The crawler has reached its limit of {self._max_requests_per_crawl} requests per crawl. '
)
- async def _get_session(self) -> Session | None:
- """If session pool is being used, try to take a session from it."""
- if not self._use_session_pool:
- return None
-
+ async def _get_session(self) -> Session:
+ """Take a session from the session pool."""
return await wait_for(
self._session_pool.get_session,
timeout=self._internal_timeout,
@@ -676,11 +698,8 @@ async def _get_session(self) -> Session | None:
logger=self._logger,
)
- async def _get_session_by_id(self, session_id: str | None) -> Session | None:
- """If session pool is being used, try to take a session by id from it."""
- if not self._use_session_pool or not session_id:
- return None
-
+ async def _get_session_by_id(self, session_id: str) -> Session | None:
+ """Take the session with the given id from the session pool."""
return await wait_for(
partial(self._session_pool.get_session_by_id, session_id),
timeout=self._internal_timeout,
@@ -690,13 +709,13 @@ async def _get_session_by_id(self, session_id: str | None) -> Session | None:
logger=self._logger,
)
- async def _get_proxy_info(self, request: Request, session: Session | None) -> ProxyInfo | None:
+ async def _get_proxy_info(self, request: Request, session: Session) -> ProxyInfo | None:
"""Retrieve a new ProxyInfo object based on crawler configuration and the current request and session."""
if not self._proxy_configuration:
return None
return await self._proxy_configuration.new_proxy_info(
- session_id=session.id if session else None,
+ session_id=session.id,
request=request,
)
@@ -783,6 +802,8 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta
requests: The requests to be enqueued before the crawler starts.
Raises:
+ RuntimeError: If the crawler is already running, an object passed to it, such as a `SessionPool`, isn't
+ entered, or its statistics are already active.
PersistentRateLimitError: If a domain throttled by a `ThrottlingRequestManager` has rate-limited every
request for longer than the manager's `max_domain_stall` and no other requests are left.
"""
@@ -791,6 +812,19 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta
'This crawler instance is already running, you can add more requests to it via `crawler.add_requests()`'
)
+ for collaborator in self._borrowed_collaborators:
+ if not collaborator.active:
+ raise RuntimeError(
+ f'The {type(collaborator).__name__} passed to the crawler is not active. Enter it with '
+ '`async with` before running the crawler.'
+ )
+
+ if self._statistics.active:
+ raise RuntimeError(
+ "The statistics of the crawler are already active. The crawler enters them itself, so don't enter "
+ 'them yourself. Crawlers can share statistics only when they run one after another.'
+ )
+
self._last_run = self._create_run()
try:
@@ -798,7 +832,8 @@ async def run(self, requests: Sequence[str | Request] | None = None) -> FinalSta
self._statistics.reset()
await self._statistics.reset_store()
- if self._use_session_pool:
+ # A session pool passed in belongs to the caller, who may still use it in another crawler.
+ if self._session_pool in self._owned_context_managers:
self._session_pool.reset()
await self._session_pool.reset_store()
@@ -935,24 +970,16 @@ async def _run_crawler(self) -> None:
# A system supplied by the caller is only borrowed, and its lifecycle belongs to whoever created it. The one
# the crawler builds for this run is entered and exited below.
- owned_concurrency_system = self._concurrency_system if self._borrowed_concurrency_system is None else None
-
- # Collect the context managers to be entered. Context managers that are already active are excluded,
- # as they were likely entered by the caller, who will also be responsible for exiting them.
- contexts_to_enter.extend(
- [
- cm
- for cm in (
- owned_concurrency_system,
- self._statistics,
- self._session_pool if self._use_session_pool else None,
- self._http_client,
- self._crawler_state_rec_task,
- *self._additional_context_managers,
- )
- if cm and getattr(cm, 'active', False) is False
- ]
- )
+ if self._borrowed_concurrency_system is None:
+ contexts_to_enter.append(self._concurrency_system)
+
+ # The statistics count only while they're entered, so the crawler enters them on every run, even when they
+ # were passed in.
+ contexts_to_enter.append(self._statistics)
+ contexts_to_enter.extend(self._owned_context_managers)
+
+ # The status messages of the recurring task read the statistics, so it starts after them and stops before them.
+ contexts_to_enter.append(self._crawler_state_rec_task)
async with AsyncExitStack() as exit_stack:
for context in contexts_to_enter:
@@ -1322,7 +1349,7 @@ async def _handle_error_handler_replacement(
if context.request.session_rotation_count is not None:
new_request.session_rotation_count = context.request.session_rotation_count
- if retire_session and context.session:
+ if retire_session:
context.session.retire()
await request_manager.add_request(new_request)
@@ -1386,8 +1413,7 @@ async def _handle_request_error(self, context: TCrawlingContext | BasicCrawlingC
context.request.state = RequestState.ERROR
raise
- if context.session:
- context.session.mark_bad()
+ context.session.mark_bad()
async def _handle_failed_request(self, context: TCrawlingContext | BasicCrawlingContext, error: Exception) -> None:
self._logger.error(
@@ -1445,7 +1471,7 @@ def _get_only_inner_most_exception(self, error: BaseException) -> BaseException:
def _prepare_send_request_function(
self,
- session: Session | None,
+ session: Session,
proxy_info: ProxyInfo | None,
register_deferred_cleanup: Callable[[DeferredCleanupCallback], None],
) -> SendRequestFunction:
@@ -1630,8 +1656,14 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002
self._logger.debug('Processing request', extra=request_log_extra(request))
+ bound_session_missing = False
if request.session_id:
session = await self._get_session_by_id(request.session_id)
+ if session is None:
+ bound_session_missing = True
+ # The bound session isn't available, so `_check_request_collision` fails the request. A retired
+ # stand-in with the bound id gives the failed request handler a session.
+ session = Session(id=request.session_id, retired=True)
else:
session = await self._get_session()
proxy_info = await self._get_proxy_info(request, session)
@@ -1659,7 +1691,7 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002
request.state = RequestState.REQUEST_HANDLER
try:
- self._check_request_collision(request, session)
+ self._check_request_collision(request, bound_session_missing=bound_session_missing)
await self._run_request_handler(context=context)
except TimeoutError as e:
raise RequestHandlerError(e, context) from e
@@ -1670,7 +1702,7 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002
await self._mark_request_as_handled(request)
- if session and session.is_usable:
+ if session.is_usable:
session.mark_good()
self._statistics.record_request_processing_finish(request.unique_key)
@@ -1698,9 +1730,6 @@ async def __run_task_function(self, task_id: str) -> None: # noqa: ARG002
await self._handle_request_error(primary_error.crawling_context, primary_error.wrapped_exception)
except SessionError as session_error:
- if not session:
- raise RuntimeError('SessionError raised in a crawling context without a session') from session_error
-
request.state = RequestState.ERROR_HANDLER
if self._should_retry_request(context, session_error):
@@ -1902,33 +1931,33 @@ def _record_rate_limit_status_code(
)
return False
- def _raise_for_session_blocked_status_code(self, session: Session | None, status_code: int) -> None:
+ def _raise_for_session_blocked_status_code(self, session: Session, status_code: int) -> None:
"""Raise an exception if the given status code indicates the session is blocked.
Args:
- session: The session used for the request. If `None`, no check is performed.
+ session: The session used for the request.
status_code: The HTTP status code to check.
Raises:
SessionError: If the status code indicates the session is blocked.
"""
- if session is not None and session.is_blocked_status_code(
+ if session.is_blocked_status_code(
status_code=status_code,
ignore_http_error_status_codes=self._ignore_http_error_status_codes,
):
raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}')
- def _check_request_collision(self, request: Request, session: Session | None) -> None:
+ def _check_request_collision(self, request: Request, *, bound_session_missing: bool) -> None:
"""Raise an exception if a request cannot access required resources.
Args:
request: The `Request` that might require specific resources (like a session).
- session: The `Session` that was retrieved for the request, or `None` if not available.
+ bound_session_missing: Whether the `Session` bound to the `Request` isn't available in the session pool.
Raises:
RequestCollisionError: If the `Session` referenced by the `Request` is not available.
"""
- if self._use_session_pool and request.session_id and not session:
+ if bound_session_missing:
raise RequestCollisionError(
f'The Session (id: {request.session_id}) bound to the Request is no longer available in SessionPool'
)
diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py
index 2803ee3c18..e3e19c0472 100644
--- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py
+++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py
@@ -129,7 +129,8 @@ def __init__(
"""Initialize a new instance.
Args:
- browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages.
+ browser_pool: A `BrowserPool` instance to be used for launching the browsers and getting pages. Keep it
+ open with `async with` until every crawler using it has finished.
user_data_dir: Path to a user data directory, which stores browser session data like cookies
and local storage.
browser_type: The type of browser to launch:
@@ -162,49 +163,48 @@ def __init__(
self._shared_navigation_timeouts: dict[int, SharedTimeout] = {}
self._save_response_cookies = save_response_cookies
- if browser_pool:
- # Raise an exception if browser_pool is provided together with other browser-related arguments.
- if any(
- param not in [None, 'default']
- for param in (
- user_data_dir,
- use_incognito_pages,
- headless,
- browser_type,
- browser_launch_options,
- browser_new_context_options,
- fingerprint_generator,
- )
- ):
- raise ValueError(
- 'You cannot provide `headless`, `browser_type`, `browser_launch_options`, '
- '`browser_new_context_options`, `use_incognito_pages`, `user_data_dir` or '
- '`fingerprint_generator` arguments when `browser_pool` is provided.'
- )
+ if browser_pool and any(
+ param not in [None, 'default']
+ for param in (
+ user_data_dir,
+ use_incognito_pages,
+ headless,
+ browser_type,
+ browser_launch_options,
+ browser_new_context_options,
+ fingerprint_generator,
+ )
+ ):
+ raise ValueError(
+ 'You cannot provide `headless`, `browser_type`, `browser_launch_options`, '
+ '`browser_new_context_options`, `use_incognito_pages`, `user_data_dir` or '
+ '`fingerprint_generator` arguments when `browser_pool` is provided.'
+ )
- # If browser_pool is not provided, create a new instance of BrowserPool with specified arguments.
- else:
- if fingerprint_generator == 'default':
- generator_browser_type: list[Literal['chrome', 'firefox', 'safari', 'edge']] | None = (
- [fingerprint_browser_type_from_playwright_browser_type(browser_type)] if browser_type else None
- )
+ # Before `super().__init__()`, so that a failure here doesn't take up a crawler id.
+ generator: FingerprintGenerator | None = None
+ if fingerprint_generator != 'default':
+ generator = fingerprint_generator
+ elif browser_pool is None:
+ generator_browser_type: list[Literal['chrome', 'firefox', 'safari', 'edge']] | None = (
+ [fingerprint_browser_type_from_playwright_browser_type(browser_type)] if browser_type else None
+ )
- fingerprint_generator = DefaultFingerprintGenerator(
- header_options=HeaderGeneratorOptions(browsers=generator_browser_type)
- )
+ generator = DefaultFingerprintGenerator(
+ header_options=HeaderGeneratorOptions(browsers=generator_browser_type)
+ )
- browser_pool = BrowserPool.with_default_plugin(
+ def build_browser_pool() -> BrowserPool:
+ return BrowserPool.with_default_plugin(
headless=headless,
browser_type=browser_type,
user_data_dir=user_data_dir,
browser_launch_options=browser_launch_options,
browser_new_context_options=browser_new_context_options,
use_incognito_pages=use_incognito_pages,
- fingerprint_generator=fingerprint_generator,
+ fingerprint_generator=generator,
)
- self._browser_pool = browser_pool
-
# Compose the context pipeline with the Playwright-specific context enhancer.
kwargs['_context_pipeline'] = (
ContextPipeline()
@@ -216,11 +216,13 @@ def __init__(
.compose(self._handle_blocked_request_by_content)
.compose(self._create_crawling_context)
)
- kwargs['_additional_context_managers'] = [self._browser_pool]
self._pre_navigation_hooks: list[Callable[[TPreNavContext], Awaitable[None]]] = []
self._post_navigation_hooks: list[Callable[[TPostNavContext], Awaitable[None]]] = []
- kwargs['http_client'] = PlaywrightHttpClient() if not kwargs.get('http_client') else kwargs['http_client']
+ if kwargs.get('http_client') is None:
+ http_client = PlaywrightHttpClient()
+ kwargs['http_client'] = http_client
+ kwargs['_additional_context_managers'] = [*kwargs.get('_additional_context_managers', []), http_client]
# Lower the default concurrency for browser crawlers, unless the caller configured it themselves.
if kwargs.get('concurrency_settings') is None and kwargs.get('concurrency_system') is None:
@@ -231,6 +233,8 @@ def __init__(
super().__init__(**kwargs)
+ self._browser_pool = self._own_or_borrow(browser_pool, build_browser_pool)
+
async def _manage_shared_navigation_timeout(
self, context: BasicCrawlingContext
) -> AsyncGenerator[BasicCrawlingContext, None]:
@@ -372,9 +376,8 @@ async def _navigate(
context.register_deferred_cleanup(lambda: context.page.__aexit__(None, None, None))
- if context.session:
- session_cookies = context.session.cookies.get_cookies_as_playwright_format()
- await self._update_cookies(context.page, session_cookies)
+ session_cookies = context.session.cookies.get_cookies_as_playwright_format()
+ await self._update_cookies(context.page, session_cookies)
if context.request.method != 'GET':
# Call the notification only once
@@ -592,7 +595,7 @@ async def _execute_post_navigation_hooks(self, context: TPostNavContext) -> Asyn
async def _create_crawling_context(self, context: TPostNavContext) -> AsyncGenerator[TCrawlingContext, None]:
yield self._build_crawling_context(context)
- if context.session and self._save_response_cookies:
+ if self._save_response_cookies:
pw_cookies = await self._get_cookies(context.page)
context.session.cookies.set_cookies_from_playwright_format(pw_cookies)
@@ -640,7 +643,8 @@ class _PlaywrightCrawlerAdditionalOptions(TypedDict):
"""
browser_pool: NotRequired[BrowserPool]
- """A `BrowserPool` instance to be used for launching the browsers and getting pages."""
+ """A `BrowserPool` instance to be used for launching the browsers and getting pages. Keep it open with `async with`
+ until every crawler using it has finished."""
browser_type: NotRequired[BrowserType]
"""The type of browser to launch:
diff --git a/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py b/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py
index cc24f91d6b..82dcc03c6c 100644
--- a/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py
+++ b/src/crawlee/crawlers/_pydantic_ai/_base_extractor.py
@@ -12,6 +12,9 @@
from ._types import PydanticAiUsageStats
if TYPE_CHECKING:
+ from types import TracebackType
+ from typing import Self
+
from pydantic_ai.models import Model
from pydantic_ai.usage import UsageLimits
@@ -27,7 +30,7 @@ class BasePydanticAiHtmlExtractor(ABC):
An HTML extractor turns a page into a validated Pydantic model with the help of an LLM. This abstract base
implements the parts the built-in extractors share: resolving the model, composing the task instructions with
- the distiller's prompt notes, and accumulating token usage.
+ the distiller's prompt notes, and accumulating token usage. It also provides a default `async with` lifecycle.
The public interface is the `PydanticAiHtmlExtractor` protocol. The concrete extractors are
`PydanticAiDirectExtractor` and `PydanticAiSelectorExtractor`.
@@ -56,6 +59,40 @@ def __init__(
self._base_instructions = self._compose_instructions(instructions, distiller)
self._usage_limits = usage_limits
self._ai_usage = PydanticAiUsageStats()
+ self._active = False
+
+ @property
+ def active(self) -> bool:
+ """Whether the extractor is entered and not exited yet."""
+ return self._active
+
+ async def __aenter__(self) -> Self:
+ """Prepare the extractor for extraction.
+
+ Raises:
+ RuntimeError: If the extractor is already active.
+ """
+ if self._active:
+ raise RuntimeError(f'The {type(self).__name__} is already active.')
+
+ self._active = True
+ return self
+
+ async def __aexit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_value: BaseException | None,
+ exc_traceback: TracebackType | None,
+ ) -> None:
+ """Release what the extractor holds.
+
+ Raises:
+ RuntimeError: If the extractor is not active.
+ """
+ if not self._active:
+ raise RuntimeError(f'The {type(self).__name__} is not active.')
+
+ self._active = False
@property
def ai_usage(self) -> PydanticAiUsageStats:
diff --git a/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py b/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py
index 55b61e3e65..5c156c49c8 100644
--- a/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py
+++ b/src/crawlee/crawlers/_pydantic_ai/_pydantic_ai_crawler.py
@@ -1,10 +1,10 @@
from __future__ import annotations
import warnings
-from contextlib import AbstractAsyncContextManager
from typing import TYPE_CHECKING
from parsel import Selector
+from pydantic_ai.models import infer_model
from crawlee._utils.docs import docs_group
from crawlee.crawlers import AbstractHttpCrawler, HttpCrawlerOptions
@@ -90,7 +90,8 @@ def __init__(
`model` nor `extractor` is given. Provide at most one of `model` or `extractor`.
extractor: A pre-configured `PydanticAiHtmlExtractor`, for full control over the distiller, instructions,
caching, usage limits, and model fallback. Pass a `PydanticAiSelectorExtractor` here for
- cached-selector extraction. Provide at most one of `model` or `extractor`.
+ cached-selector extraction. Keep it open with `async with` until every crawler using it has finished.
+ Provide at most one of `model` or `extractor`.
kwargs: Additional keyword arguments to pass to the underlying `AbstractHttpCrawler`.
"""
# Call the notification only once.
@@ -103,10 +104,10 @@ def __init__(
if model is not None and extractor is not None:
raise ValueError('Provide at most one of `model` or `extractor`.')
+ # Before `super().__init__()`, so that a failure here doesn't take up a crawler id.
+ default_model = model if model is not None else _DEFAULT_AI_MODEL
if extractor is None:
- extractor = PydanticAiDirectExtractor(model if model is not None else _DEFAULT_AI_MODEL)
-
- self._extractor = extractor
+ default_model = infer_model(default_model)
async def final_step(
context: ParsedHttpCrawlingContext[Selector],
@@ -121,18 +122,13 @@ async def final_step(
kwargs['_context_pipeline'] = self._create_static_content_crawler_pipeline().compose(final_step)
- # If the extractor is an async context manager, add it to the crawler's additional context managers so it's
- # properly entered and exited around the crawl.
- if isinstance(extractor, AbstractAsyncContextManager):
- kwargs['_additional_context_managers'] = [
- *kwargs.get('_additional_context_managers', []),
- extractor,
- ]
super().__init__(
parser=ParselParser(),
**kwargs,
)
+ self._extractor = self._own_or_borrow(extractor, lambda: PydanticAiDirectExtractor(default_model))
+
@property
def extractor(self) -> PydanticAiHtmlExtractor:
"""The extractor used to turn pages into structured data."""
diff --git a/src/crawlee/crawlers/_pydantic_ai/_types.py b/src/crawlee/crawlers/_pydantic_ai/_types.py
index 4ddad30eda..5dccf5d6a1 100644
--- a/src/crawlee/crawlers/_pydantic_ai/_types.py
+++ b/src/crawlee/crawlers/_pydantic_ai/_types.py
@@ -5,6 +5,7 @@
from pydantic import BaseModel
+from crawlee._utils.context import Collaborator
from crawlee._utils.docs import docs_group
if TYPE_CHECKING:
@@ -52,12 +53,13 @@ def add(self, usage: RunUsage) -> None:
@docs_group('Other')
-class PydanticAiHtmlExtractor(Protocol):
+class PydanticAiHtmlExtractor(Collaborator, Protocol):
"""Interface for HTML extractors.
An extractor turns an HTML page into a validated Pydantic model using an LLM. The input format (cleaned HTML,
skeleton, Markdown, ...) is decided by the `PydanticAiHtmlDistiller` an implementation composes. The model and base
- instructions are set at construction. Each `extract` call runs one extraction. The built-in extractors are
+ instructions are set at construction. Each `extract` call runs one extraction. An extractor is an async context
+ manager, and `PydanticAiCrawler` refuses to run with one that isn't entered. The built-in extractors are
`PydanticAiDirectExtractor` and `PydanticAiSelectorExtractor`.
"""
diff --git a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py
index df71060797..167f6533cf 100644
--- a/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py
+++ b/src/crawlee/crawlers/_stagehand/_stagehand_crawler.py
@@ -109,7 +109,8 @@ def __init__(
stagehand_options: Stagehand-specific configuration (model, API key, env, etc.).
Cannot be specified if `browser_pool` is provided.
browser_pool: A pre-configured `BrowserPool`. All plugins must be instances of `StagehandBrowserPlugin`.
- If omitted, a pool is created automatically from the other browser arguments.
+ If omitted, a pool is created automatically from the other browser arguments. Keep a pool passed in
+ open with `async with` until every crawler using it has finished.
user_data_dir: Path to a user data directory, which stores browser session data like cookies and
local storage. Cannot be specified if `browser_pool` is provided.
headless: Whether to run the browser in headless mode. Defaults to the value from Crawlee's
@@ -158,6 +159,7 @@ def __init__(
)
]
)
+ kwargs['_additional_context_managers'] = [*kwargs.get('_additional_context_managers', []), browser_pool]
super().__init__(
browser_pool=browser_pool,
diff --git a/src/crawlee/http_clients/_base.py b/src/crawlee/http_clients/_base.py
index e69322a003..65498c2418 100644
--- a/src/crawlee/http_clients/_base.py
+++ b/src/crawlee/http_clients/_base.py
@@ -389,15 +389,17 @@ async def __aenter__(self) -> HttpClient:
return self
async def __aexit__(
- self, exc_type: BaseException | None, exc_value: BaseException | None, traceback: TracebackType | None
+ self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_traceback: TracebackType | None
) -> None:
"""Deinitialize the client and clean up resources when exiting the context manager.
Raises:
- RuntimeError: If the context manager is already active.
+ RuntimeError: If the context manager is not active.
"""
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')
- await self.cleanup()
- self._active = False
+ try:
+ await self.cleanup()
+ finally:
+ self._active = False
diff --git a/src/crawlee/sessions/_session_pool.py b/src/crawlee/sessions/_session_pool.py
index f206c64f58..a855224d6a 100644
--- a/src/crawlee/sessions/_session_pool.py
+++ b/src/crawlee/sessions/_session_pool.py
@@ -118,12 +118,20 @@ async def __aenter__(self) -> SessionPool:
self._active = True
- state = await self._state.initialize()
- state.max_pool_size = self._max_pool_size
- self._remove_retired_sessions()
-
- if not state.sessions:
- await self._fill_sessions_to_max()
+ try:
+ state = await self._state.initialize()
+ state.max_pool_size = self._max_pool_size
+ self._remove_retired_sessions()
+
+ if not state.sessions:
+ await self._fill_sessions_to_max()
+ except BaseException:
+ try:
+ if self._state.is_initialized:
+ await self._state.teardown()
+ finally:
+ self._active = False
+ raise
return self
@@ -141,9 +149,10 @@ async def __aexit__(
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')
- await self._state.teardown()
-
- self._active = False
+ try:
+ await self._state.teardown()
+ finally:
+ self._active = False
@overload
def get_state(self, *, as_dict: Literal[True]) -> dict: ...
diff --git a/src/crawlee/statistics/_statistics.py b/src/crawlee/statistics/_statistics.py
index 74ff13e5ba..3c684a4da1 100644
--- a/src/crawlee/statistics/_statistics.py
+++ b/src/crawlee/statistics/_statistics.py
@@ -172,22 +172,27 @@ async def __aenter__(self) -> Self:
if self._active:
raise RuntimeError(f'The {self.__class__.__name__} is already active.')
- await self._state.initialize()
-
- # The timestamps belong to the previous run, restored or kept in memory. Fold that run into the accumulated
- # runtime and clear them, so that the downtime since then (e.g. a migration or resurrection) doesn't count
- # towards the runtime, and a clock that is behind the previous host's doesn't make `crawler_runtime` treat the
- # new run as already checkpointed.
- self.state.accumulated_runtime = self.state.crawler_runtime
- self.state.crawler_last_started_at = None
- self.state.crawler_finished_at = None
- self.state.stats_persisted_at = None
-
- # Start periodic logging and let it print the restored state before the new run starts being measured.
- self._periodic_logger.start()
- await asyncio.sleep(0.01)
self._active = True
+ try:
+ await self._state.initialize()
+
+ # The timestamps belong to the previous run, restored or kept in memory. Fold that run into the accumulated
+ # runtime and clear them, so that the downtime since then (e.g. a migration or resurrection) doesn't count
+ # towards the runtime, and a clock that is behind the previous host's doesn't make `crawler_runtime` treat
+ # the new run as already checkpointed.
+ self.state.accumulated_runtime = self.state.crawler_runtime
+ self.state.crawler_last_started_at = None
+ self.state.crawler_finished_at = None
+ self.state.stats_persisted_at = None
+
+ # Start periodic logging and let it print the restored state before the new run starts being measured.
+ self._periodic_logger.start()
+ await asyncio.sleep(0.01)
+ except BaseException:
+ await self._stop_capturing()
+ raise
+
self.state.crawler_last_started_at = datetime.now(UTC)
self.state.crawler_started_at = self.state.crawler_started_at or self.state.crawler_last_started_at
return self
@@ -206,11 +211,10 @@ async def __aexit__(
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')
- # Stop logging; the finish timestamp freezes the measured run before the final persist.
- await self._periodic_logger.stop()
+ # Before the first await, so that a cancelled exit doesn't count the time until the next run as runtime.
self.state.crawler_finished_at = datetime.now(UTC)
- self._active = False
- await self._state.teardown()
+
+ await self._stop_capturing()
@property
def state(self) -> TStatisticsState:
@@ -316,6 +320,16 @@ async def reset_store(self) -> None:
"""
await self._state.reset_store()
+ async def _stop_capturing(self) -> None:
+ try:
+ await self._periodic_logger.stop()
+ finally:
+ try:
+ if self._state.is_initialized:
+ await self._state.teardown()
+ finally:
+ self._active = False
+
def _log(self) -> None:
stats = self.calculate()
if self._statistics_log_format == 'table':
diff --git a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
index 813fe6e50e..11068041f9 100644
--- a/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
+++ b/tests/integration/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
@@ -29,6 +29,7 @@
AdaptivePlaywrightCrawlerStatisticState,
)
from crawlee.crawlers._adaptive_playwright._adaptive_playwright_crawling_context import AdaptiveContextError
+from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor
from crawlee.errors import RequestThrottledError
from crawlee.http_clients import HttpClient, HttpResponse
from crawlee.sessions import SessionPool
@@ -123,6 +124,12 @@ def predict(self, request: Request) -> RenderingTypePrediction:
def store_result(self, request: Request, rendering_type: RenderingType) -> None:
pass
+ @property
+ @override
+ def active(self) -> bool:
+ # Lets the tests pass the predictor without entering it.
+ return True
+
@dataclass(frozen=True)
class TestInput:
@@ -573,6 +580,17 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
assert stored_results == [saved_data]
+async def test_passed_predictor_left_open(test_urls: list[str]) -> None:
+ """A rendering type predictor passed to the crawler stays active after the run, for the caller to exit."""
+ async with DefaultRenderingTypePredictor() as predictor:
+ crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(rendering_type_predictor=predictor)
+ crawler.router.default_handler(AsyncMock())
+
+ await crawler.run(test_urls[:1])
+
+ assert predictor.active
+
+
async def test_adaptive_playwright_crawler_statistics_in_init() -> None:
"""Tests that adaptive crawler uses created AdaptivePlaywrightCrawlerStatistics from inputted Statistics."""
persistence_enabled = True
@@ -897,9 +915,8 @@ async def test_change_context_state_after_handling(
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
nonlocal used_session_id
- if context.session is not None:
- used_session_id = context.session.id
- context.session.user_data['session_state'] = True
+ used_session_id = context.session.id
+ context.session.user_data['session_state'] = True
try:
# `page` is only available in the browser sub crawler, static crawling raises here.
@@ -989,7 +1006,7 @@ async def test_throttled_static_429_is_deferred(monkeypatch: pytest.MonkeyPatch)
throttler = await open_throttler_held_until_reclaim(monkeypatch)
record_pacing_signal = Mock(wraps=throttler.record_pacing_signal)
monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal)
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
http_client.send_request.side_effect = [
Mock(
spec=HttpResponse,
@@ -1039,7 +1056,7 @@ async def test_static_handler_raised_throttled_error_is_deferred(monkeypatch: py
throttler = await open_throttler_held_until_reclaim(monkeypatch)
record_pacing_signal = Mock(wraps=throttler.record_pacing_signal)
monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal)
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
http_client.send_request.return_value = Mock(
spec=HttpResponse,
status_code=200,
diff --git a/tests/integration/crawlers/_basic/test_basic_crawler.py b/tests/integration/crawlers/_basic/test_basic_crawler.py
index eb08a4643d..285c2e0a28 100644
--- a/tests/integration/crawlers/_basic/test_basic_crawler.py
+++ b/tests/integration/crawlers/_basic/test_basic_crawler.py
@@ -83,7 +83,7 @@ async def test_session_rotation(server_url: URL) -> None:
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- session_ids.append(context.session.id if context.session else None)
+ session_ids.append(context.session.id)
raise SessionError('Test error')
await crawler.run([str(server_url)])
diff --git a/tests/integration/crawlers/_http/test_http_crawler.py b/tests/integration/crawlers/_http/test_http_crawler.py
index 79b310b6f4..d96eae7498 100644
--- a/tests/integration/crawlers/_http/test_http_crawler.py
+++ b/tests/integration/crawlers/_http/test_http_crawler.py
@@ -121,22 +121,17 @@ async def test_handles_client_errors(
@pytest.mark.parametrize(
- ('ignore_http_error_status_codes', 'use_session_pool', 'expected_session_rotate', 'expected_number_error'),
+ ('ignore_http_error_status_codes', 'expected_session_rotate', 'expected_number_error'),
[
# change session and retry for no block 4xx statuses
- pytest.param([], True, 4, 1, id='default_behavior'),
- # error without retry for all 4xx statuses
- pytest.param([], False, 0, 1, id='default_behavior_without_session_pool'),
- # take as successful status codes from the `ignore_http_error_status_codes` list with Session Pool
- pytest.param([403], True, 0, 0, id='ignore_error_status_codes'),
- # take as successful status codes from the `ignore_http_error_status_codes` list without Session Pool
- pytest.param([403], False, 0, 0, id='ignore_error_status_codes_without_session_pool'),
+ pytest.param([], 4, 1, id='default_behavior'),
+ # take as successful status codes from the `ignore_http_error_status_codes` list
+ pytest.param([403], 0, 0, id='ignore_error_status_codes'),
],
)
async def test_handles_session_block_errors(
*,
ignore_http_error_status_codes: list[int],
- use_session_pool: bool,
expected_session_rotate: int,
expected_number_error: int,
mock_request_handler: AsyncMock,
@@ -147,7 +142,6 @@ async def test_handles_session_block_errors(
ignore_http_error_status_codes=ignore_http_error_status_codes,
max_request_retries=3,
max_session_rotations=5,
- use_session_pool=use_session_pool,
)
await crawler.add_requests([str(server_url / 'status/403')])
@@ -185,7 +179,7 @@ async def test_stores_cookies(http_client: HttpClient, server_url: URL) -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
visit(context.request.url)
- track_session_usage(context.session.id if context.session else None)
+ track_session_usage(context.session.id)
await crawler.run(
[
@@ -427,13 +421,14 @@ async def test_isolation_cookies(http_client: HttpClient, server_url: URL) -> No
sessions_cookies: dict[str, dict[str, str]] = {}
response_cookies: dict[str, dict[str, str]] = {}
+ session_pool = SessionPool(
+ max_pool_size=1,
+ create_session_settings={
+ 'max_error_score': 50,
+ },
+ )
crawler = HttpCrawler(
- session_pool=SessionPool(
- max_pool_size=1,
- create_session_settings={
- 'max_error_score': 50,
- },
- ),
+ session_pool=session_pool,
http_client=http_client,
max_request_retries=10,
concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1),
@@ -441,9 +436,6 @@ async def test_isolation_cookies(http_client: HttpClient, server_url: URL) -> No
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- if not context.session:
- return
-
sessions_ids.append(context.session.id)
if context.request.unique_key not in {'1', '2'}:
@@ -458,16 +450,19 @@ async def handler(context: HttpCrawlingContext) -> None:
if context.request.user_data.get('retire_session'):
context.session.retire()
- await crawler.run(
- [
- # The first request sets the cookie in the session
- str(server_url.with_path('set_cookies').extend_query(a=1)),
- # With the second request, we check the cookies in the session and set retire
- Request.from_url(str(server_url.with_path('/cookies')), unique_key='1', user_data={'retire_session': True}),
- # The third request is made with a new session to make sure it does not use another session's cookies
- Request.from_url(str(server_url.with_path('/cookies')), unique_key='2'),
- ]
- )
+ async with session_pool:
+ await crawler.run(
+ [
+ # The first request sets the cookie in the session
+ str(server_url.with_path('set_cookies').extend_query(a=1)),
+ # With the second request, we check the cookies in the session and set retire
+ Request.from_url(
+ str(server_url.with_path('/cookies')), unique_key='1', user_data={'retire_session': True}
+ ),
+ # The third request is made with a new session to make sure it does not use another session's cookies
+ Request.from_url(str(server_url.with_path('/cookies')), unique_key='2'),
+ ]
+ )
assert len(sessions_cookies) == 2
assert len(response_cookies) == 2
@@ -496,7 +491,7 @@ async def test_store_complex_cookies(server_url: URL) -> None:
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
visit(context.request.url)
- track_session_usage(context.session.id if context.session else None)
+ track_session_usage(context.session.id)
await crawler.run([str(server_url / 'set_complex_cookies')])
@@ -589,38 +584,39 @@ async def handler(context: HttpCrawlingContext) -> None:
async def test_save_response_cookies_disabled(server_url: URL) -> None:
"""Test that `save_response_cookies=False` keeps the response cookies out of the session, but sends its cookies."""
preset_cookie = CookieParam(name='preset', value='1', domain=server_url.host or '')
- crawler = HttpCrawler(
- session_pool=SessionPool(max_pool_size=1, create_session_settings={'cookies': [preset_cookie]}),
- save_response_cookies=False,
- )
+ session_pool = SessionPool(max_pool_size=1, create_session_settings={'cookies': [preset_cookie]})
+ crawler = HttpCrawler(session_pool=session_pool, save_response_cookies=False)
handler_call = Mock()
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
- session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else []
+ session_cookies = context.session.cookies.get_cookies_as_dicts()
handler_call(
sent_cookies=json.loads(await context.http_response.read())['cookies'],
session_cookies={cookie['name'] for cookie in session_cookies},
)
# The cookie set by the redirecting response is still sent to the redirect target.
- await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))])
+ async with session_pool:
+ await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))])
handler_call.assert_called_once_with(sent_cookies={'preset': '1', 'a': '1'}, session_cookies={'preset'})
async def test_context_send_request_saves_cookies(server_url: URL) -> None:
"""Test that `context.send_request` stores response cookies in the session with `save_response_cookies=False`."""
- crawler = HttpCrawler(session_pool=SessionPool(max_pool_size=1), save_response_cookies=False)
+ session_pool = SessionPool(max_pool_size=1)
+ crawler = HttpCrawler(session_pool=session_pool, save_response_cookies=False)
handler_call = Mock()
@crawler.router.default_handler
async def handler(context: HttpCrawlingContext) -> None:
await context.send_request(str(server_url.with_path('set_cookies').extend_query(b=2)))
- session_cookies = context.session.cookies.get_cookies_as_dicts() if context.session else []
+ session_cookies = context.session.cookies.get_cookies_as_dicts()
handler_call(session_cookies={cookie['name'] for cookie in session_cookies})
- await crawler.run([str(server_url / 'cookies')])
+ async with session_pool:
+ await crawler.run([str(server_url / 'cookies')])
handler_call.assert_called_once_with(session_cookies={'b'})
diff --git a/tests/integration/crawlers/_playwright/test_playwright_crawler.py b/tests/integration/crawlers/_playwright/test_playwright_crawler.py
index 25750d60f8..8a2ee0bc93 100644
--- a/tests/integration/crawlers/_playwright/test_playwright_crawler.py
+++ b/tests/integration/crawlers/_playwright/test_playwright_crawler.py
@@ -434,21 +434,21 @@ async def test_isolation_cookies(*, use_incognito_pages: bool, server_url: URL)
sessions: dict[str, Session] = {}
response_cookies: dict[str, dict[str, str]] = {}
- async with BrowserPool(
- plugins=[PlaywrightBrowserPlugin(use_incognito_pages=use_incognito_pages)],
- browser_inactive_threshold=timedelta(minutes=10),
- ) as browser_pool:
+ async with (
+ BrowserPool(
+ plugins=[PlaywrightBrowserPlugin(use_incognito_pages=use_incognito_pages)],
+ browser_inactive_threshold=timedelta(minutes=10),
+ ) as browser_pool,
+ SessionPool(max_pool_size=1) as session_pool,
+ ):
crawler = PlaywrightCrawler(
- session_pool=SessionPool(max_pool_size=1),
+ session_pool=session_pool,
concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1),
browser_pool=browser_pool,
)
@crawler.router.default_handler
async def handler(context: PlaywrightCrawlingContext) -> None:
- if not context.session:
- return
-
sessions_ids.append(context.session.id)
sessions[context.session.id] = context.session
response_data = json.loads(await context.response.text())
@@ -530,8 +530,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
# Simulate cookies installed from an external source in the browser
await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}])
- if context.session:
- session_ids.append(context.session.id)
+ session_ids.append(context.session.id)
await crawler.run([str(server_url)])
@@ -555,7 +554,7 @@ async def test_save_response_cookies_disabled(server_url: URL) -> None:
async def request_handler(context: PlaywrightCrawlingContext) -> None:
await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}])
handler_call(
- session_id=context.session.id if context.session else None,
+ session_id=context.session.id,
page_cookies={cookie['name'] for cookie in await context.page.context.cookies()},
)
@@ -608,7 +607,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
async def test_send_request_skips_cookies_deleted_by_page(server_url: URL) -> None:
"""Test that `context.send_request` doesn't send a session cookie that the page deleted in the handler."""
- crawler = PlaywrightCrawler(session_pool=SessionPool(max_pool_size=1))
+ session_pool = SessionPool(max_pool_size=1)
+ crawler = PlaywrightCrawler(session_pool=session_pool)
sent_cookies: list[dict[str, str]] = []
@crawler.router.default_handler
@@ -621,7 +621,8 @@ async def second_handler(context: PlaywrightCrawlingContext) -> None:
response = await context.send_request(str(server_url / 'cookies'))
sent_cookies.append(json.loads(await response.read())['cookies'])
- await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))])
+ async with session_pool:
+ await crawler.run([str(server_url.with_path('set_cookies').extend_query(a=1))])
assert sent_cookies == [{}]
@@ -658,9 +659,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
cookies = await context.page.context.cookies()
playwright_cookies.extend(cookies)
- if context.session:
- context.session.cookies.set_cookies_from_playwright_format(cookies)
- session_cookies.extend(context.session.cookies.get_cookies_as_dicts())
+ context.session.cookies.set_cookies_from_playwright_format(cookies)
+ session_cookies.extend(context.session.cookies.get_cookies_as_dicts())
await crawler.run([str(server_url / 'set_complex_cookies')])
@@ -988,7 +988,8 @@ async def test_send_request_with_client(server_url: URL) -> None:
"""Check that the persist context works with fingerprints."""
check_data: dict[str, Any] = {}
- crawler = PlaywrightCrawler(http_client=ImpitHttpClient(headers={'user-agent': 'My User-Agent'}))
+ http_client = ImpitHttpClient(headers={'user-agent': 'My User-Agent'})
+ crawler = PlaywrightCrawler(http_client=http_client)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
@@ -997,7 +998,8 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
send_request_response = await context.send_request(str(server_url / 'user-agent'))
check_data['send_request'] = dict(json.loads(await send_request_response.read()))
- await crawler.run([str(server_url / 'user-agent')])
+ async with http_client:
+ await crawler.run([str(server_url / 'user-agent')])
assert check_data['default'].get('user-agent') is not None
assert check_data['send_request']['user-agent'] == 'My User-Agent'
@@ -1494,9 +1496,10 @@ async def test_throttled_429_is_deferred(monkeypatch: pytest.MonkeyPatch) -> Non
record_pacing_signal = Mock(wraps=throttler.record_pacing_signal)
monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal)
failed_request_handler = AsyncMock()
+ session_pool = SessionPool(max_pool_size=1)
crawler = PlaywrightCrawler(
request_manager=throttler,
- session_pool=SessionPool(max_pool_size=1),
+ session_pool=session_pool,
max_request_retries=0,
max_session_rotations=0,
)
@@ -1513,7 +1516,8 @@ async def fulfill_with_next_status(context: PlaywrightPreNavCrawlingContext) ->
async def handler(context: PlaywrightCrawlingContext) -> None:
handled.append(context)
- await crawler.run([url])
+ async with session_pool:
+ await crawler.run([url])
assert [context.response.status for context in handled] == [200]
failed_request_handler.assert_not_called()
@@ -1521,5 +1525,4 @@ async def handler(context: PlaywrightCrawlingContext) -> None:
assert handled[0].request.retry_count == 0
assert (handled[0].request.session_rotation_count or 0) == 0
session = handled[0].session
- assert session is not None
assert session.error_score == 0
diff --git a/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py b/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py
index 905878c17f..37ade376b3 100644
--- a/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py
+++ b/tests/integration/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py
@@ -71,7 +71,8 @@ async def request_handler(context: PydanticAiCrawlingContext) -> None:
await context.extract(_Article, scope='article', additional_instructions='hint')
with patch.object(extractor, 'extract', extract_mock):
- await crawler.run([Request.from_url(str(server_url), label='detail')])
+ async with extractor:
+ await crawler.run([Request.from_url(str(server_url), label='detail')])
first_call, second_call, third_call = extract_mock.call_args_list
diff --git a/tests/unit/_statistics/test_enter_exit.py b/tests/unit/_statistics/test_enter_exit.py
new file mode 100644
index 0000000000..2324af6941
--- /dev/null
+++ b/tests/unit/_statistics/test_enter_exit.py
@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from crawlee.statistics import Statistics
+
+
+@pytest.mark.parametrize(
+ 'error',
+ [
+ pytest.param(RuntimeError('initialize failed'), id='failed'),
+ pytest.param(asyncio.CancelledError(), id='cancelled'),
+ ],
+)
+async def test_failed_enter_frees_stats(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None:
+ """Statistics whose entry fails or is cancelled stay inactive and can be entered again."""
+ statistics = Statistics.with_default_state()
+ monkeypatch.setattr(statistics._state, 'initialize', AsyncMock(side_effect=error))
+
+ with pytest.raises(type(error)):
+ async with statistics:
+ pass
+
+ assert not statistics.active
+ monkeypatch.undo()
+
+ async with statistics:
+ assert statistics.active
+
+
+async def test_failed_exit_frees_stats(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Statistics whose exit fails are no longer active, so they can be entered again."""
+ statistics = Statistics.with_default_state()
+
+ with pytest.raises(RuntimeError, match='teardown failed'):
+ async with statistics:
+ monkeypatch.setattr(statistics._state, 'teardown', AsyncMock(side_effect=RuntimeError('teardown failed')))
+
+ assert not statistics.active
+ monkeypatch.undo()
+
+ async with statistics:
+ assert statistics.active
+
+
+async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Statistics can't be entered again until their exit has torn the state down."""
+ statistics = Statistics.with_default_state()
+ teardown_started = asyncio.Event()
+ release_teardown = asyncio.Event()
+
+ async def wait_for_release() -> None:
+ teardown_started.set()
+ await release_teardown.wait()
+
+ await statistics.__aenter__()
+ monkeypatch.setattr(statistics._state, 'teardown', AsyncMock(side_effect=wait_for_release))
+ exit_task = asyncio.create_task(statistics.__aexit__(None, None, None))
+ await teardown_started.wait()
+
+ with pytest.raises(RuntimeError, match='already active'):
+ await statistics.__aenter__()
+
+ release_teardown.set()
+ await exit_task
+
+
+async def test_failed_start_tears_down_stats(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Statistics whose entry fails after loading the state tear the state down and can be entered again."""
+ statistics = Statistics.with_default_state()
+ teardown = AsyncMock(wraps=statistics._state.teardown)
+ monkeypatch.setattr(statistics._state, 'teardown', teardown)
+ monkeypatch.setattr(statistics._periodic_logger, 'start', Mock(side_effect=RuntimeError('start failed')))
+
+ with pytest.raises(RuntimeError, match='start failed'):
+ async with statistics:
+ pass
+
+ assert not statistics.active
+ teardown.assert_awaited_once()
+ monkeypatch.undo()
+
+ async with statistics:
+ assert statistics.active
+
+
+async def test_cancelled_exit_finishes_run(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Statistics whose exit is cancelled still record when the run finished and tear their state down."""
+ statistics = Statistics.with_default_state()
+
+ with pytest.raises(asyncio.CancelledError):
+ async with statistics:
+ monkeypatch.setattr(statistics._periodic_logger, 'stop', AsyncMock(side_effect=asyncio.CancelledError()))
+
+ assert statistics.state.crawler_finished_at is not None
+ assert statistics._state.is_initialized is False
+ monkeypatch.undo()
+ await statistics._periodic_logger.stop()
diff --git a/tests/unit/browsers/test_browser_pool.py b/tests/unit/browsers/test_browser_pool.py
new file mode 100644
index 0000000000..b5724e5685
--- /dev/null
+++ b/tests/unit/browsers/test_browser_pool.py
@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import timedelta
+from unittest.mock import AsyncMock
+
+import pytest
+
+from crawlee.browsers import BrowserPool
+
+
+async def test_timed_out_plugin_is_not_exited() -> None:
+ """A pool whose plugin entry timed out exits cleanly without exiting that plugin."""
+ entered_plugin = AsyncMock()
+ slow_plugin = AsyncMock()
+ slow_plugin.__aenter__.side_effect = asyncio.Event().wait
+ slow_plugin.active = False
+ browser_pool = BrowserPool(plugins=[entered_plugin, slow_plugin], operation_timeout=timedelta(milliseconds=10))
+
+ async with browser_pool:
+ pass
+
+ assert not browser_pool.active
+ entered_plugin.__aexit__.assert_awaited_once()
+ slow_plugin.__aexit__.assert_not_awaited()
+
+
+@pytest.mark.parametrize(
+ 'error',
+ [
+ pytest.param(RuntimeError('plugin failed'), id='failed'),
+ pytest.param(asyncio.CancelledError(), id='cancelled'),
+ ],
+)
+async def test_failed_enter_frees_pool(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None:
+ """A pool whose entry fails or is cancelled stops its tasks and entered plugins and can be entered again."""
+ entered_plugin = AsyncMock()
+ failing_plugin = AsyncMock()
+ failing_plugin.__aenter__.side_effect = error
+ browser_pool = BrowserPool(plugins=[entered_plugin, failing_plugin])
+ identify_stop = AsyncMock(wraps=browser_pool._identify_inactive_browsers_task.stop)
+ close_stop = AsyncMock(wraps=browser_pool._close_inactive_browsers_task.stop)
+ monkeypatch.setattr(browser_pool._identify_inactive_browsers_task, 'stop', identify_stop)
+ monkeypatch.setattr(browser_pool._close_inactive_browsers_task, 'stop', close_stop)
+
+ with pytest.raises(type(error)):
+ async with browser_pool:
+ pass
+
+ assert not browser_pool.active
+ identify_stop.assert_awaited_once()
+ close_stop.assert_awaited_once()
+ entered_plugin.__aexit__.assert_awaited_once()
+ failing_plugin.__aexit__.assert_not_awaited()
+
+ failing_plugin.__aenter__.side_effect = None
+ async with browser_pool:
+ assert browser_pool.active
+
+
+async def test_failed_rollback_keeps_enter_error(caplog: pytest.LogCaptureFixture) -> None:
+ """A plugin that fails to exit while the pool rolls back a failed entry is logged and keeps the entry error."""
+ entered_plugin = AsyncMock()
+ entered_plugin.__aexit__.side_effect = RuntimeError('exit failed')
+ failing_plugin = AsyncMock()
+ failing_plugin.__aenter__.side_effect = RuntimeError('enter failed')
+ browser_pool = BrowserPool(plugins=[entered_plugin, failing_plugin])
+
+ with pytest.raises(RuntimeError, match='enter failed'):
+ async with browser_pool:
+ pass
+
+ assert not browser_pool.active
+ entered_plugin.__aexit__.assert_awaited_once()
+ assert 'Exiting a browser plugin failed while rolling back the pool entry.' in caplog.text
+
+
+async def test_cancelled_rollback_exits_plugins(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A pool whose entry rollback is cancelled while stopping its tasks still exits the plugins it entered."""
+ entered_plugin = AsyncMock()
+ failing_plugin = AsyncMock()
+ failing_plugin.__aenter__.side_effect = RuntimeError('enter failed')
+ browser_pool = BrowserPool(plugins=[entered_plugin, failing_plugin])
+ monkeypatch.setattr(
+ browser_pool._identify_inactive_browsers_task, 'stop', AsyncMock(side_effect=asyncio.CancelledError())
+ )
+
+ with pytest.raises(asyncio.CancelledError):
+ async with browser_pool:
+ pass
+
+ assert not browser_pool.active
+ entered_plugin.__aexit__.assert_awaited_once()
+ monkeypatch.undo()
+ await browser_pool._identify_inactive_browsers_task.stop()
+
+
+async def test_failed_exit_frees_pool() -> None:
+ """A pool whose plugin fails to exit is no longer active and can be entered again."""
+ plugin = AsyncMock()
+ plugin.__aexit__.side_effect = RuntimeError('plugin failed')
+ browser_pool = BrowserPool(plugins=[plugin])
+
+ with pytest.raises(RuntimeError, match='plugin failed'):
+ async with browser_pool:
+ pass
+
+ assert not browser_pool.active
+
+ plugin.__aexit__.side_effect = None
+ async with browser_pool:
+ assert browser_pool.active
+
+
+async def test_failed_browser_close_exits_plugins() -> None:
+ """A pool whose browser fails to close still closes its other browsers and exits its plugins."""
+ plugin = AsyncMock()
+ failing_browser = AsyncMock()
+ failing_browser.close.side_effect = RuntimeError('close failed')
+ other_browser = AsyncMock()
+ browser_pool = BrowserPool(plugins=[plugin])
+
+ with pytest.raises(RuntimeError, match='close failed'):
+ async with browser_pool:
+ browser_pool._active_browsers.extend([failing_browser, other_browser])
+
+ assert not browser_pool.active
+ other_browser.close.assert_awaited_once()
+ plugin.__aexit__.assert_awaited_once()
+
+
+async def test_plugin_exit_error_keeps_cancellation(caplog: pytest.LogCaptureFixture) -> None:
+ """A plugin that fails to exit is logged and doesn't replace a cancellation of the pool's exit."""
+ failing_plugin = AsyncMock()
+ failing_plugin.__aexit__.side_effect = RuntimeError('exit failed')
+ close_started = asyncio.Event()
+
+ async def close_until_cancelled(**_: object) -> None:
+ close_started.set()
+ await asyncio.Event().wait()
+
+ browser = AsyncMock()
+ browser.close.side_effect = close_until_cancelled
+ browser_pool = BrowserPool(plugins=[failing_plugin])
+ await browser_pool.__aenter__()
+ browser_pool._active_browsers.append(browser)
+
+ exit_task = asyncio.create_task(browser_pool.__aexit__(None, None, None))
+ await close_started.wait()
+ exit_task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await exit_task
+
+ failing_plugin.__aexit__.assert_awaited_once()
+ assert 'exit failed' in caplog.text
+
+
+async def test_cancelled_plugin_exit_logs_close_error(caplog: pytest.LogCaptureFixture) -> None:
+ """A browser that fails to close is logged when the pool's exit is cancelled while exiting its plugins."""
+ exit_started = asyncio.Event()
+
+ async def exit_until_cancelled(*_: object) -> None:
+ exit_started.set()
+ await asyncio.Event().wait()
+
+ plugin = AsyncMock()
+ plugin.__aexit__.side_effect = exit_until_cancelled
+ browser = AsyncMock()
+ browser.close.side_effect = RuntimeError('close failed')
+ browser_pool = BrowserPool(plugins=[plugin])
+ await browser_pool.__aenter__()
+ browser_pool._active_browsers.append(browser)
+
+ exit_task = asyncio.create_task(browser_pool.__aexit__(None, None, None))
+ await exit_started.wait()
+ exit_task.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await exit_task
+
+ assert not browser_pool.active
+ assert 'close failed' in caplog.text
+
+
+async def test_cancelled_task_stop_stops_the_other(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A pool whose first task stop is cancelled still stops the other task and exits its plugins."""
+ plugin = AsyncMock()
+ browser_pool = BrowserPool(plugins=[plugin])
+ close_stop = AsyncMock(wraps=browser_pool._close_inactive_browsers_task.stop)
+ await browser_pool.__aenter__()
+ monkeypatch.setattr(
+ browser_pool._identify_inactive_browsers_task, 'stop', AsyncMock(side_effect=asyncio.CancelledError())
+ )
+ monkeypatch.setattr(browser_pool._close_inactive_browsers_task, 'stop', close_stop)
+
+ with pytest.raises(asyncio.CancelledError):
+ await browser_pool.__aexit__(None, None, None)
+
+ close_stop.assert_awaited_once()
+ plugin.__aexit__.assert_awaited_once()
+ monkeypatch.undo()
+ await browser_pool._identify_inactive_browsers_task.stop()
+
+
+async def test_failed_plugin_exit_exits_others() -> None:
+ """A pool whose plugin fails to exit still exits its other plugins and raises the failure."""
+ failing_plugin = AsyncMock()
+ failing_plugin.__aexit__.side_effect = RuntimeError('exit failed')
+ other_plugin = AsyncMock()
+ browser_pool = BrowserPool(plugins=[failing_plugin, other_plugin])
+
+ with pytest.raises(RuntimeError, match='exit failed'):
+ async with browser_pool:
+ pass
+
+ assert not browser_pool.active
+ other_plugin.__aexit__.assert_awaited_once()
+
+
+async def test_extra_exit_failures_are_logged(caplog: pytest.LogCaptureFixture) -> None:
+ """A pool whose plugins fail to exit raises the first failure and logs the others."""
+ first_plugin = AsyncMock()
+ first_plugin.__aexit__.side_effect = RuntimeError('first failed')
+ second_plugin = AsyncMock()
+ second_plugin.__aexit__.side_effect = RuntimeError('second failed')
+ browser_pool = BrowserPool(plugins=[first_plugin, second_plugin])
+
+ with pytest.raises(RuntimeError, match='first failed'):
+ async with browser_pool:
+ pass
+
+ assert 'Closing the browser pool failed.' in caplog.text
+ assert 'second failed' in caplog.text
diff --git a/tests/unit/browsers/test_playwright_browser_plugin.py b/tests/unit/browsers/test_playwright_browser_plugin.py
index f45b4449a3..a9cd7e724c 100644
--- a/tests/unit/browsers/test_playwright_browser_plugin.py
+++ b/tests/unit/browsers/test_playwright_browser_plugin.py
@@ -1,5 +1,10 @@
from __future__ import annotations
+import asyncio
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
from crawlee.browsers import PlaywrightBrowserPlugin
@@ -29,3 +34,66 @@ async def test_executable_path_with_chrome_browser_type() -> None:
assert plugin.browser_type == 'chromium'
assert plugin.browser_launch_options['executable_path'] == '/path/to/chrome'
assert plugin.browser_launch_options['channel'] == 'chrome'
+
+
+@pytest.mark.parametrize(
+ 'error',
+ [
+ pytest.param(RuntimeError('playwright failed'), id='failed'),
+ pytest.param(asyncio.CancelledError(), id='cancelled'),
+ ],
+)
+async def test_failed_enter_frees_plugin(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None:
+ """A plugin whose entry fails or is cancelled stops Playwright, stays inactive and can be entered again."""
+ plugin = PlaywrightBrowserPlugin()
+ failing_context_manager = AsyncMock()
+ failing_context_manager.__aenter__.side_effect = error
+ monkeypatch.setattr(plugin, '_playwright_context_manager', failing_context_manager)
+ monkeypatch.setattr('crawlee.browsers._playwright_browser_plugin.async_playwright', Mock(return_value=AsyncMock()))
+
+ with pytest.raises(type(error)):
+ async with plugin:
+ pass
+
+ assert not plugin.active
+ failing_context_manager.__aexit__.assert_awaited_once()
+
+ async with plugin:
+ assert plugin.active
+
+
+async def test_failed_rollback_keeps_enter_error(
+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """A Playwright that fails to stop during an entry rollback is logged and the entry error is kept."""
+ plugin = PlaywrightBrowserPlugin()
+ failing_context_manager = AsyncMock()
+ failing_context_manager.__aenter__.side_effect = RuntimeError('start failed')
+ failing_context_manager.__aexit__.side_effect = RuntimeError('stop failed')
+ monkeypatch.setattr(plugin, '_playwright_context_manager', failing_context_manager)
+ monkeypatch.setattr('crawlee.browsers._playwright_browser_plugin.async_playwright', Mock(return_value=AsyncMock()))
+
+ with pytest.raises(RuntimeError, match='start failed'):
+ async with plugin:
+ pass
+
+ assert not plugin.active
+ assert 'Stopping Playwright failed while rolling back the plugin entry.' in caplog.text
+
+
+async def test_failed_exit_frees_plugin(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A plugin whose exit fails is no longer active and can be entered again."""
+ plugin = PlaywrightBrowserPlugin()
+ failing_context_manager = AsyncMock()
+ failing_context_manager.__aexit__.side_effect = RuntimeError('stop failed')
+ monkeypatch.setattr(plugin, '_playwright_context_manager', failing_context_manager)
+ monkeypatch.setattr('crawlee.browsers._playwright_browser_plugin.async_playwright', Mock(return_value=AsyncMock()))
+
+ with pytest.raises(RuntimeError, match='stop failed'):
+ async with plugin:
+ pass
+
+ assert not plugin.active
+
+ async with plugin:
+ assert plugin.active
diff --git a/tests/unit/browsers/test_stagehand_browser_plugin.py b/tests/unit/browsers/test_stagehand_browser_plugin.py
index eedcd4d0a5..5fb9a9d9a3 100644
--- a/tests/unit/browsers/test_stagehand_browser_plugin.py
+++ b/tests/unit/browsers/test_stagehand_browser_plugin.py
@@ -1,11 +1,11 @@
from __future__ import annotations
-from typing import TYPE_CHECKING
+import asyncio
+from unittest.mock import AsyncMock, Mock
-from crawlee.browsers import StagehandBrowserPlugin, StagehandOptions
+import pytest
-if TYPE_CHECKING:
- import pytest
+from crawlee.browsers import StagehandBrowserPlugin, StagehandOptions
def test_initial_state() -> None:
@@ -71,3 +71,108 @@ def test_stagehand_options_defaults_when_not_provided() -> None:
assert isinstance(plugin.stagehand_options, StagehandOptions)
assert plugin.stagehand_options == StagehandOptions()
+
+
+@pytest.mark.parametrize(
+ 'error',
+ [
+ pytest.param(RuntimeError('stagehand failed'), id='failed'),
+ pytest.param(asyncio.CancelledError(), id='cancelled'),
+ ],
+)
+async def test_failed_enter_stops_playwright(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None:
+ """A plugin whose Stagehand client fails to start stops Playwright and can be entered again."""
+ plugin = StagehandBrowserPlugin()
+ playwright_context_manager = AsyncMock()
+ monkeypatch.setattr(plugin, '_playwright_context_manager', playwright_context_manager)
+ monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.async_playwright', Mock(return_value=AsyncMock()))
+ stagehand_client = AsyncMock()
+ stagehand_client.__aenter__.side_effect = error
+ monkeypatch.setattr(
+ 'crawlee.browsers._stagehand_browser_plugin.AsyncStagehand', Mock(return_value=stagehand_client)
+ )
+
+ with pytest.raises(type(error)):
+ async with plugin:
+ pass
+
+ assert not plugin.active
+ playwright_context_manager.__aexit__.assert_awaited_once()
+
+ stagehand_client.__aenter__.side_effect = None
+ async with plugin:
+ assert plugin.active
+
+
+async def test_failed_exit_stops_playwright(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A plugin whose Stagehand client fails to close still stops Playwright and can be entered again."""
+ plugin = StagehandBrowserPlugin()
+ playwright_context_manager = AsyncMock()
+ monkeypatch.setattr(plugin, '_playwright_context_manager', playwright_context_manager)
+ monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.async_playwright', Mock(return_value=AsyncMock()))
+ stagehand_client = AsyncMock()
+ stagehand_client.__aexit__.side_effect = RuntimeError('close failed')
+ monkeypatch.setattr(
+ 'crawlee.browsers._stagehand_browser_plugin.AsyncStagehand', Mock(return_value=stagehand_client)
+ )
+
+ with pytest.raises(RuntimeError, match='close failed'):
+ async with plugin:
+ pass
+
+ assert not plugin.active
+ playwright_context_manager.__aexit__.assert_awaited_once()
+
+ stagehand_client.__aexit__.side_effect = None
+ async with plugin:
+ assert plugin.active
+
+
+@pytest.mark.parametrize(
+ 'error',
+ [
+ pytest.param(RuntimeError('playwright failed'), id='failed'),
+ pytest.param(asyncio.CancelledError(), id='cancelled'),
+ ],
+)
+async def test_failed_playwright_start_frees_plugin(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None:
+ """A plugin whose Playwright fails to start stops it, stays inactive and can be entered again."""
+ plugin = StagehandBrowserPlugin()
+ failing_context_manager = AsyncMock()
+ failing_context_manager.__aenter__.side_effect = error
+ monkeypatch.setattr(plugin, '_playwright_context_manager', failing_context_manager)
+ monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.async_playwright', Mock(return_value=AsyncMock()))
+ monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.AsyncStagehand', Mock(return_value=AsyncMock()))
+
+ with pytest.raises(type(error)):
+ async with plugin:
+ pass
+
+ assert not plugin.active
+ failing_context_manager.__aexit__.assert_awaited_once()
+
+ async with plugin:
+ assert plugin.active
+
+
+async def test_failed_rollback_keeps_enter_error(
+ monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """A Playwright that fails to stop during an entry rollback is logged and the entry error is kept."""
+ plugin = StagehandBrowserPlugin()
+ playwright_context_manager = AsyncMock()
+ playwright_context_manager.__aexit__.side_effect = RuntimeError('stop failed')
+ monkeypatch.setattr(plugin, '_playwright_context_manager', playwright_context_manager)
+ monkeypatch.setattr('crawlee.browsers._stagehand_browser_plugin.async_playwright', Mock(return_value=AsyncMock()))
+ stagehand_client = AsyncMock()
+ stagehand_client.__aenter__.side_effect = RuntimeError('stagehand failed')
+ monkeypatch.setattr(
+ 'crawlee.browsers._stagehand_browser_plugin.AsyncStagehand', Mock(return_value=stagehand_client)
+ )
+
+ with pytest.raises(RuntimeError, match='stagehand failed'):
+ async with plugin:
+ pass
+
+ assert not plugin.active
+ assert 'Stopping Playwright failed while rolling back the plugin entry.' in caplog.text
diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
index 5295f15712..adf46db9a1 100644
--- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
+++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py
@@ -7,7 +7,9 @@
import pytest
from crawlee.autoscaling import ConcurrencySystem
+from crawlee.browsers import BrowserPool
from crawlee.crawlers import AdaptivePlaywrightCrawler
+from crawlee.crawlers._adaptive_playwright._rendering_type_predictor import DefaultRenderingTypePredictor
async def test_accepts_concurrency_system() -> None:
@@ -18,6 +20,26 @@ async def test_accepts_concurrency_system() -> None:
assert crawler.concurrency_system is concurrency_system
+async def test_run_needs_entered_predictor() -> None:
+ """A rendering type predictor passed to the crawler has to be entered before the crawler runs."""
+ crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
+ rendering_type_predictor=DefaultRenderingTypePredictor()
+ )
+
+ with pytest.raises(RuntimeError, match='DefaultRenderingTypePredictor passed to the crawler is not active'):
+ await crawler.run(['https://a.placeholder.com'])
+
+
+async def test_run_needs_entered_browser_pool() -> None:
+ """A browser pool passed to the Playwright sub crawler has to be entered before the crawler runs."""
+ crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
+ playwright_crawler_specific_kwargs={'browser_pool': BrowserPool()}
+ )
+
+ with pytest.raises(RuntimeError, match='BrowserPool passed to the crawler is not active'):
+ await crawler.run(['https://a.placeholder.com'])
+
+
@pytest.mark.parametrize(
'optional_module_name',
[
diff --git a/tests/unit/crawlers/_adaptive_playwright/test_predictor.py b/tests/unit/crawlers/_adaptive_playwright/test_predictor.py
index 681307cc9e..72d01f7515 100644
--- a/tests/unit/crawlers/_adaptive_playwright/test_predictor.py
+++ b/tests/unit/crawlers/_adaptive_playwright/test_predictor.py
@@ -1,5 +1,8 @@
from __future__ import annotations
+import asyncio
+from unittest.mock import AsyncMock
+
import pytest
from crawlee import Request
@@ -217,3 +220,63 @@ def test_url_similarity(url_1: str, url_2: str, expected_rounded_similarity: flo
round(calculate_url_similarity(url_1=get_url_components(url_1), url_2=get_url_components(url_2)), 2)
== expected_rounded_similarity
)
+
+
+@pytest.mark.parametrize(
+ 'error',
+ [
+ pytest.param(RuntimeError('initialize failed'), id='failed'),
+ pytest.param(asyncio.CancelledError(), id='cancelled'),
+ ],
+)
+async def test_failed_enter_frees_predictor(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None:
+ """A predictor whose entry fails or is cancelled stays inactive and can be entered again."""
+ predictor = DefaultRenderingTypePredictor()
+ monkeypatch.setattr(predictor._state, 'initialize', AsyncMock(side_effect=error))
+
+ with pytest.raises(type(error)):
+ async with predictor:
+ pass
+
+ assert not predictor.active
+ monkeypatch.undo()
+
+ async with predictor:
+ assert predictor.active
+
+
+async def test_failed_exit_frees_predictor(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A predictor whose exit fails is no longer active, so it can be entered again."""
+ predictor = DefaultRenderingTypePredictor()
+
+ with pytest.raises(RuntimeError, match='teardown failed'):
+ async with predictor:
+ monkeypatch.setattr(predictor._state, 'teardown', AsyncMock(side_effect=RuntimeError('teardown failed')))
+
+ assert not predictor.active
+ monkeypatch.undo()
+
+ async with predictor:
+ assert predictor.active
+
+
+async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A predictor can't be entered again until its exit has torn the state down."""
+ predictor = DefaultRenderingTypePredictor()
+ teardown_started = asyncio.Event()
+ release_teardown = asyncio.Event()
+
+ async def wait_for_release() -> None:
+ teardown_started.set()
+ await release_teardown.wait()
+
+ await predictor.__aenter__()
+ monkeypatch.setattr(predictor._state, 'teardown', AsyncMock(side_effect=wait_for_release))
+ exit_task = asyncio.create_task(predictor.__aexit__(None, None, None))
+ await teardown_started.wait()
+
+ with pytest.raises(RuntimeError, match='already active'):
+ await predictor.__aenter__()
+
+ release_teardown.set()
+ await exit_task
diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py
index ddde8c4bc3..e9f9fc85ac 100644
--- a/tests/unit/crawlers/_basic/test_basic_crawler.py
+++ b/tests/unit/crawlers/_basic/test_basic_crawler.py
@@ -44,7 +44,7 @@
EventMigratingData,
LocalEventManager,
)
-from crawlee.http_clients import HttpClient, HttpResponse
+from crawlee.http_clients import HttpClient, HttpResponse, ImpitHttpClient
from crawlee.request_loaders import (
FinishedStatus,
MinIntervalSignal,
@@ -622,17 +622,19 @@ async def failed_request_handler(context: BasicCrawlingContext, error: Exception
async def test_send_request_forwards_arguments_to_http_client() -> None:
"""Test that `context.send_request` passes all its arguments to the HTTP client."""
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
response = Mock(spec=HttpResponse)
http_client.send_request.return_value = response
- crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1)
+ crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1)
timeout = timedelta(seconds=12)
cookies = SessionCookies({'name': 'value'})
result = None
+ session = None
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- nonlocal result
+ nonlocal result, session
+ session = context.session
result = await context.send_request(
'https://example.com',
method='POST',
@@ -646,12 +648,13 @@ async def handler(context: BasicCrawlingContext) -> None:
await crawler.run(['https://a.placeholder.com'])
assert result is response
+ assert session is not None
http_client.send_request.assert_awaited_once_with(
url='https://example.com',
method='POST',
payload=b'Hello, world!',
headers={'Content-Type': 'text/plain'},
- session=None,
+ session=session,
cookies=cookies,
proxy_info=None,
timeout=timeout,
@@ -661,9 +664,9 @@ async def handler(context: BasicCrawlingContext) -> None:
async def test_send_request_default_timeout() -> None:
"""Test that `context.send_request` without a `timeout` leaves the timeout to the default of the HTTP client."""
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
http_client.send_request.return_value = Mock(spec=HttpResponse)
- crawler = BasicCrawler(http_client=http_client, use_session_pool=False, max_requests_per_crawl=1)
+ crawler = BasicCrawler(http_client=http_client, max_requests_per_crawl=1)
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
@@ -683,7 +686,7 @@ async def handler(context: BasicCrawlingContext) -> None:
)
async def test_send_request_response_closed_after_request(*, handler_fails: bool) -> None:
"""Test that a response of `context.send_request` stays open in the handler and is closed after the request."""
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
response = Mock(spec=HttpResponse)
http_client.send_request.return_value = response
crawler = BasicCrawler(http_client=http_client, max_request_retries=0, max_requests_per_crawl=1)
@@ -1472,6 +1475,164 @@ async def test_repeated_run_starts_with_fresh_statistics() -> None:
assert crawler.statistics.state.requests_finished == 1
+async def test_crawlers_share_stats_in_turn() -> None:
+ """Statistics passed to two crawlers that run one after the other add up the requests and the runtime of both."""
+ statistics = Statistics.with_default_state()
+ first_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics)
+ second_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics)
+
+ await first_crawler.run(['https://a.placeholder.com'])
+ started_at = statistics.state.crawler_started_at
+ first_runtime = statistics.state.crawler_runtime
+ stats = await second_crawler.run(['https://b.placeholder.com'])
+
+ assert stats.requests_finished == 2
+ assert statistics.state.crawler_started_at == started_at
+ assert statistics.state.accumulated_runtime == first_runtime
+
+
+async def test_shared_stats_concurrent_raises() -> None:
+ """Two crawlers can't run with the same statistics at the same time, since the first one has them entered."""
+ statistics = Statistics.with_default_state()
+ handler_started = asyncio.Event()
+ release_handler = asyncio.Event()
+
+ async def blocking_handler(_context: BasicCrawlingContext) -> None:
+ handler_started.set()
+ await release_handler.wait()
+
+ first_crawler = BasicCrawler(request_handler=blocking_handler, statistics=statistics)
+ second_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics)
+
+ first_run = asyncio.create_task(first_crawler.run(['https://a.placeholder.com']))
+ await handler_started.wait()
+
+ with pytest.raises(RuntimeError, match='statistics of the crawler are already active'):
+ await second_crawler.run(['https://b.placeholder.com'])
+
+ release_handler.set()
+ stats = await first_run
+
+ assert stats.requests_finished == 1
+
+
+async def test_shared_stats_started_together(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A crawler started while another crawler is still entering the shared statistics raises."""
+ statistics = Statistics.with_default_state()
+ entry_started = asyncio.Event()
+ release_entry = asyncio.Event()
+ initialize = statistics._state.initialize
+
+ async def hold_first_initialize() -> StatisticsState:
+ # Only the first entry waits, so that an unblocked second entry finishes instead of hanging the test.
+ if not entry_started.is_set():
+ entry_started.set()
+ await release_entry.wait()
+ return await initialize()
+
+ monkeypatch.setattr(statistics._state, 'initialize', AsyncMock(side_effect=hold_first_initialize))
+ first_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics)
+ second_crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics)
+
+ first_run = asyncio.create_task(first_crawler.run(['https://a.placeholder.com']))
+ await entry_started.wait()
+
+ # The first entry makes the statistics active before its first await, so the second crawler stops in `run()`.
+ with pytest.raises(RuntimeError, match='statistics of the crawler are already active'):
+ await second_crawler.run(['https://b.placeholder.com'])
+
+ release_entry.set()
+ stats = await first_run
+
+ assert stats.requests_finished == 1
+ assert not statistics.active
+
+
+async def test_run_after_failed_exit(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A failed exit of an object the crawler built doesn't keep it active, so the next run works."""
+ monkeypatch.setattr(ImpitHttpClient, 'cleanup', AsyncMock(side_effect=[RuntimeError('cleanup failed'), None]))
+ handler = AsyncMock()
+ crawler = BasicCrawler(request_handler=handler)
+
+ with pytest.raises(RuntimeError, match='cleanup failed'):
+ await crawler.run(['https://a.placeholder.com'])
+
+ await crawler.run(['https://b.placeholder.com'])
+
+ assert handler.await_count == 2
+
+
+async def test_run_rejects_entered_stats() -> None:
+ """Statistics the caller entered make `run()` raise, since the crawler enters them itself."""
+ async with Statistics.with_default_state() as statistics:
+ crawler = BasicCrawler(request_handler=AsyncMock(), statistics=statistics)
+
+ with pytest.raises(RuntimeError, match='statistics of the crawler are already active'):
+ await crawler.run(['https://a.placeholder.com'])
+
+
+async def test_repeated_run_keeps_sessions() -> None:
+ """A repeated `run()` keeps the sessions of a persisted session pool passed to the crawler, and leaves it open."""
+ async with SessionPool(max_pool_size=1, persistence_enabled=True) as session_pool:
+ session_ids = set(session_pool.get_state(as_dict=False).sessions)
+ crawler = BasicCrawler(request_handler=AsyncMock(), session_pool=session_pool)
+
+ await crawler.run(['https://a.placeholder.com'])
+ await crawler.run(['https://b.placeholder.com'])
+
+ assert set(session_pool.get_state(as_dict=False).sessions) == session_ids
+ assert session_pool.active
+
+
+async def test_repeated_run_resets_own_sessions() -> None:
+ """A repeated `run()` replaces the sessions of the session pool the crawler built."""
+ crawler = BasicCrawler()
+ session_ids = list[set[str]]()
+
+ @crawler.router.default_handler
+ async def handler(_context: BasicCrawlingContext) -> None:
+ session_ids.append(set(crawler._session_pool.get_state(as_dict=False).sessions))
+
+ await crawler.run(['https://a.placeholder.com'])
+ await crawler.run(['https://b.placeholder.com'])
+
+ first_ids, second_ids = session_ids
+ assert first_ids.isdisjoint(second_ids)
+
+
+async def test_run_needs_entered_session_pool() -> None:
+ """A session pool passed to the crawler has to be entered before the crawler runs."""
+ session_pool = SessionPool()
+ crawler = BasicCrawler(request_handler=AsyncMock(), session_pool=session_pool)
+
+ with pytest.raises(RuntimeError, match='SessionPool passed to the crawler is not active'):
+ await crawler.run(['https://a.placeholder.com'])
+
+ async with session_pool:
+ stats = await crawler.run(['https://a.placeholder.com'])
+
+ assert stats.requests_finished == 1
+
+
+async def test_run_needs_entered_http_client() -> None:
+ """An HTTP client passed to the crawler has to be entered before the crawler runs."""
+ http_client = ImpitHttpClient()
+ crawler = BasicCrawler(request_handler=AsyncMock(), http_client=http_client)
+
+ with pytest.raises(RuntimeError, match='ImpitHttpClient passed to the crawler is not active'):
+ await crawler.run(['https://a.placeholder.com'])
+
+
+async def test_passed_http_client_left_open() -> None:
+ """The crawler doesn't close an HTTP client passed to it."""
+ http_client = ImpitHttpClient()
+ crawler = BasicCrawler(request_handler=AsyncMock(), http_client=http_client)
+
+ async with http_client:
+ await crawler.run(['https://a.placeholder.com'])
+ assert http_client.active
+
+
def test_crawler_log() -> None:
crawler = BasicCrawler()
assert isinstance(crawler.log, logging.Logger)
@@ -2629,19 +2790,21 @@ async def signal_idle_reached(*args: object, **kwargs: object) -> bool:
],
)
async def test_session_retire_in_user_handler(*, retire: bool) -> None:
- crawler = BasicCrawler(session_pool=SessionPool(max_pool_size=1))
+ session_pool = SessionPool(max_pool_size=1)
+ crawler = BasicCrawler(session_pool=session_pool)
sessions = list[str]()
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- if context.session:
- sessions.append(context.session.id)
+ sessions.append(context.session.id)
- context.session.retire() if retire else None
+ if retire:
+ context.session.retire()
await context.add_requests(['https://b.placeholder.com'])
- await crawler.run(['https://a.placeholder.com'])
+ async with session_pool:
+ await crawler.run(['https://a.placeholder.com'])
# The session should differ if `retire` was called and match otherwise since pool size == 1
if retire:
@@ -2658,8 +2821,7 @@ async def test_bound_session_to_request() -> None:
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- if context.session:
- used_sessions.append(context.session.id)
+ used_sessions.append(context.session.id)
requests = [
Request.from_url('https://a.placeholder.com', session_id=check_session.id, always_enqueue=True)
@@ -2686,19 +2848,20 @@ def create_session() -> Session:
check_sessions = [str(session_id) for session_id in range(10)]
used_sessions = list[str]()
- crawler = BasicCrawler(session_pool=SessionPool(create_session_function=create_session_function()))
+ session_pool = SessionPool(create_session_function=create_session_function())
+ crawler = BasicCrawler(session_pool=session_pool)
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
- if context.session:
- used_sessions.append(context.session.id)
+ used_sessions.append(context.session.id)
requests = [
Request.from_url('https://a.placeholder.com', session_id=str(session_id), use_extended_unique_key=True)
for session_id in range(10)
]
- await crawler.run(requests)
+ async with session_pool:
+ await crawler.run(requests)
assert len(used_sessions) == 10
assert set(used_sessions) == set(check_sessions)
@@ -2732,6 +2895,21 @@ async def error_req_hook(context: BasicCrawlingContext, error: Exception) -> Non
assert error_handler_mock.call_count == 1
+async def test_missing_bound_session_gives_retired_stand_in() -> None:
+ """Test that a request bound to a missing session reaches the failed request handler with a retired session."""
+ failed_request_handler = AsyncMock()
+ crawler = BasicCrawler(request_handler=AsyncMock())
+ crawler.failed_request_handler(failed_request_handler)
+
+ await crawler.run([Request.from_url('https://a.placeholder.com', session_id='missing')])
+
+ failed_request_handler.assert_awaited_once()
+ context, error = failed_request_handler.call_args.args
+ assert isinstance(error, RequestCollisionError)
+ assert context.session.id == 'missing'
+ assert context.session.retired
+
+
async def test_handles_session_error_in_failed_request_handler() -> None:
crawler = BasicCrawler(max_session_rotations=1)
handler_requests = set()
@@ -3123,20 +3301,23 @@ async def test_crawler_intermediate_statistics() -> None:
crawler = BasicCrawler()
check_time = timedelta(seconds=0.1)
+ handler_started = asyncio.Event()
+
@crawler.router.default_handler
async def handler(_: BasicCrawlingContext) -> None:
+ handler_started.set()
await asyncio.sleep(check_time.total_seconds() * 5)
- # Start crawler and wait until statistics are initialized. Use a generous timeout so that crawler startup
- # has enough time to reach the active state on slow CI runners under xdist load.
+ # The handler starts only once the statistics are entered and measure the run. Use a generous timeout so that
+ # crawler startup has enough time on slow CI runners under xdist load.
crawler_task = asyncio.create_task(crawler.run(['https://a.placeholder.com']))
- assert await poll_until_condition(lambda: crawler.statistics.active, timeout=30)
- active_seen_at = datetime.now(UTC)
+ await asyncio.wait_for(handler_started.wait(), timeout=30)
+ started_seen_at = datetime.now(UTC)
# Wait some time and check that runtime is updated. The runtime is measured on the wall clock, which on Windows
# ticks at ~15.6 ms independently of the `asyncio.sleep` clock, so bound it by a wall-clock interval inside the run.
await asyncio.sleep(check_time.total_seconds())
- min_runtime = datetime.now(UTC) - active_seen_at
+ min_runtime = datetime.now(UTC) - started_seen_at
crawler.statistics.calculate()
assert min_runtime > timedelta(0)
assert crawler.statistics.state.crawler_runtime >= min_runtime
@@ -3167,7 +3348,7 @@ async def error_handler(context: BasicCrawlingContext, error: Exception) -> Requ
if isinstance(context.request.user_data['request_state'], list):
context.request.user_data['request_state'].append('error')
- if context.session and isinstance(context.session.user_data['session_state'], list):
+ if isinstance(context.session.user_data['session_state'], list):
context.session.user_data['session_state'].append('error')
context.session.cookies['error'] = 'yes'
@@ -3175,7 +3356,7 @@ async def error_handler(context: BasicCrawlingContext, error: Exception) -> Requ
async def handler(context: BasicCrawlingContext) -> None:
if isinstance(context.request.user_data['request_state'], list):
context.request.user_data['request_state'].append(f'modified_{context.request.retry_count}')
- if context.session and isinstance(context.session.user_data['session_state'], list):
+ if isinstance(context.session.user_data['session_state'], list):
context.session.user_data['session_state'].append(f'modified_{context.request.retry_count}')
context.session.cookies[f'modified_{context.request.retry_count}'] = 'yes'
@@ -3528,16 +3709,18 @@ async def test_handler_raised_throttled_error_is_deferred(monkeypatch: pytest.Mo
"""A `RequestThrottledError` from the handler starts a backoff without costing a retry or session reputation."""
throttler = await open_throttler_held_until_reclaim(monkeypatch)
failed_request_handler = AsyncMock()
+ session_pool = SessionPool(max_pool_size=1)
crawler = BasicCrawler(
request_manager=throttler,
- session_pool=SessionPool(max_pool_size=1),
+ session_pool=session_pool,
max_request_retries=0,
)
crawler.failed_request_handler(failed_request_handler)
handler = AsyncMock(side_effect=[RequestThrottledError, None])
crawler.router.default_handler(handler)
- stats = await crawler.run([THROTTLED_URL])
+ async with session_pool:
+ stats = await crawler.run([THROTTLED_URL])
assert handler.await_count == 2
failed_request_handler.assert_not_called()
@@ -3545,7 +3728,6 @@ async def test_handler_raised_throttled_error_is_deferred(monkeypatch: pytest.Mo
assert context.request.retry_count == 0
assert stats.requests_finished == 1
assert stats.requests_failed == 0
- assert context.session is not None
assert context.session.error_score == 0
assert throttler._domain_states['throttled.placeholder.com'].consecutive_429_count == 1
diff --git a/tests/unit/crawlers/_http/test_http_crawler.py b/tests/unit/crawlers/_http/test_http_crawler.py
index b9076a0b88..d9acc12140 100644
--- a/tests/unit/crawlers/_http/test_http_crawler.py
+++ b/tests/unit/crawlers/_http/test_http_crawler.py
@@ -51,7 +51,7 @@ async def test_navigation_response_closed_after_request(*, status_code: int, exp
"""Test that the response of the navigation request stays open in the handler and is closed after the request."""
url = 'https://a.placeholder.com'
response = mock_response(status_code, url)
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
http_client.send_request.return_value = response
crawler = HttpCrawler(http_client=http_client, max_request_retries=0)
handler_call = Mock()
@@ -71,7 +71,7 @@ async def test_body_read_bound_by_navigation_timeout() -> None:
url = 'https://a.placeholder.com'
response = mock_response(200, url)
response.read.side_effect = asyncio.Event().wait
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
http_client.send_request.return_value = response
crawler = HttpCrawler(
http_client=http_client,
@@ -100,14 +100,15 @@ async def test_body_read_bound_by_navigation_timeout() -> None:
async def test_throttled_429_is_deferred(crawler_kwargs: dict[str, Any], monkeypatch: pytest.MonkeyPatch) -> None:
"""A throttled 429 is retried later without spending a retry, a session rotation or session reputation."""
throttler = await open_throttler_held_until_reclaim(monkeypatch)
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
url = 'https://throttled.placeholder.com/page'
http_client.send_request.side_effect = [mock_response(429, url), mock_response(200, url)]
failed_request_handler = AsyncMock()
+ session_pool = SessionPool(max_pool_size=1)
crawler = HttpCrawler(
http_client=http_client,
request_manager=throttler,
- session_pool=SessionPool(max_pool_size=1),
+ session_pool=session_pool,
max_request_retries=0,
max_session_rotations=0,
**crawler_kwargs,
@@ -119,7 +120,8 @@ async def test_throttled_429_is_deferred(crawler_kwargs: dict[str, Any], monkeyp
async def handler(context: HttpCrawlingContext) -> None:
handled.append(context)
- await crawler.run([url])
+ async with session_pool:
+ await crawler.run([url])
assert [context.http_response.status_code for context in handled] == [200]
failed_request_handler.assert_not_called()
@@ -127,7 +129,6 @@ async def handler(context: HttpCrawlingContext) -> None:
assert request.retry_count == 0
assert (request.session_rotation_count or 0) == 0
session = handled[0].session
- assert session is not None
assert session.error_score == 0
assert session.is_usable
@@ -135,7 +136,7 @@ async def handler(context: HttpCrawlingContext) -> None:
async def test_tandem_429_deferred(monkeypatch: pytest.MonkeyPatch) -> None:
"""A 429 reaches a throttler wrapped in a `RequestManagerTandem`, so the request is deferred without a retry."""
throttler = await open_throttler_held_until_reclaim(monkeypatch)
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
url = 'https://throttled.placeholder.com/page'
http_client.send_request.side_effect = [mock_response(429, url), mock_response(200, url)]
failed_request_handler = AsyncMock()
@@ -164,7 +165,7 @@ async def test_429_reported_once(monkeypatch: pytest.MonkeyPatch) -> None:
throttler = await open_throttler_held_until_reclaim(monkeypatch)
record_pacing_signal = Mock(wraps=throttler.record_pacing_signal)
monkeypatch.setattr(throttler, 'record_pacing_signal', record_pacing_signal)
- http_client = AsyncMock(spec=HttpClient)
+ http_client = AsyncMock(spec=HttpClient, active=True)
url = 'https://throttled.placeholder.com/page'
http_client.send_request.side_effect = [mock_response(429, url), mock_response(200, url)]
crawler = HttpCrawler(http_client=http_client, request_manager=throttler)
diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py
index 8a4b500788..a243f5bb6e 100644
--- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py
+++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py
@@ -9,8 +9,10 @@
service_locator,
)
from crawlee.autoscaling import ConcurrencySystem
+from crawlee.browsers import BrowserPool
from crawlee.configuration import Configuration
from crawlee.crawlers import (
+ BasicCrawler,
PlaywrightCrawler,
)
@@ -34,6 +36,25 @@ async def test_accepts_concurrency_system() -> None:
assert crawler.concurrency_system is concurrency_system
+async def test_run_needs_entered_browser_pool() -> None:
+ """A browser pool passed to the crawler has to be entered before the crawler runs."""
+ crawler = PlaywrightCrawler(browser_pool=BrowserPool())
+
+ with pytest.raises(RuntimeError, match='BrowserPool passed to the crawler is not active'):
+ await crawler.run(['https://a.placeholder.com'])
+
+
+def test_unsupported_browser_type_takes_no_id() -> None:
+ """A crawler with a browser type its default fingerprint generator doesn't support doesn't use up a crawler id."""
+ first_crawler = BasicCrawler()
+
+ with pytest.raises(ValueError, match='Unsupported browser type'):
+ PlaywrightCrawler(browser_type='edge') # ty: ignore[invalid-argument-type]
+
+ second_crawler = BasicCrawler()
+ assert int(second_crawler._id) == int(first_crawler._id) + 1
+
+
def test_import_error_handled() -> None:
blocked = {
mod_name: None for mod_name in sys.modules if mod_name == 'playwright' or mod_name.startswith('playwright.')
diff --git a/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py b/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py
index 759a277b18..74ffa04260 100644
--- a/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py
+++ b/tests/unit/crawlers/_pydantic_ai/test_direct_extractor.py
@@ -159,3 +159,14 @@ async def test_raise_for_invalid_output() -> None:
# The extractor's usage stats reflect the 3 failed attempts (1 initial + 2 retries).
assert extractor.ai_usage.requests == 3
+
+
+async def test_enter_sets_active() -> None:
+ """Entering the extractor makes it active until it's exited."""
+ extractor = PydanticAiDirectExtractor(TestModel())
+ assert extractor.active is False
+
+ async with extractor:
+ assert extractor.active is True
+
+ assert extractor.active is False
diff --git a/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py b/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py
index 7cd2aaad4d..356dd3e79f 100644
--- a/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py
+++ b/tests/unit/crawlers/_pydantic_ai/test_pydantic_ai_crawler.py
@@ -1,14 +1,17 @@
from __future__ import annotations
import sys
-from unittest.mock import patch
+from unittest.mock import AsyncMock, patch
import pytest
+from pydantic_ai.exceptions import UserError
from pydantic_ai.models.test import TestModel
from crawlee.crawlers import (
+ BasicCrawler,
PydanticAiCrawler,
PydanticAiDirectExtractor,
+ PydanticAiSelectorExtractor,
)
@@ -42,6 +45,37 @@ def test_exposes_extractor_and_usage() -> None:
assert crawler.ai_usage is extractor.ai_usage
+@pytest.mark.parametrize(
+ 'extractor_class',
+ [
+ pytest.param(PydanticAiDirectExtractor, id='direct'),
+ pytest.param(PydanticAiSelectorExtractor, id='selector'),
+ ],
+)
+async def test_run_needs_entered_extractor(
+ extractor_class: type[PydanticAiDirectExtractor | PydanticAiSelectorExtractor],
+) -> None:
+ """An extractor passed to the crawler has to be entered before the crawler runs."""
+ crawler = PydanticAiCrawler(extractor=extractor_class(TestModel()))
+
+ with pytest.raises(RuntimeError, match=f'{extractor_class.__name__} passed to the crawler is not active'):
+ await crawler.run(['https://a.placeholder.com'])
+
+
+async def test_owns_default_extractor(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The crawler enters and exits the default extractor it builds."""
+ extractor_enter = AsyncMock()
+ extractor_exit = AsyncMock()
+ monkeypatch.setattr(PydanticAiDirectExtractor, '__aenter__', extractor_enter)
+ monkeypatch.setattr(PydanticAiDirectExtractor, '__aexit__', extractor_exit)
+ crawler = PydanticAiCrawler(model=TestModel())
+
+ await crawler.run()
+
+ extractor_enter.assert_awaited_once()
+ extractor_exit.assert_awaited_once()
+
+
def test_import_error_handled() -> None:
# The `ai` extra is optional, so accessing the crawler without `pydantic_ai` installed must raise a clear error.
blocked = {name: None for name in sys.modules if name == 'pydantic_ai' or name.startswith('pydantic_ai.')}
@@ -51,3 +85,14 @@ def test_import_error_handled() -> None:
sys.modules.pop(name, None)
with pytest.raises(ImportError):
from crawlee.crawlers._pydantic_ai import PydanticAiCrawler # noqa: F401 PLC0415
+
+
+def test_unknown_model_takes_no_id() -> None:
+ """A crawler with an unknown model doesn't use up a crawler id."""
+ first_crawler = BasicCrawler()
+
+ with pytest.raises(UserError, match='Unknown model'):
+ PydanticAiCrawler(model='no-such-provider:model')
+
+ second_crawler = BasicCrawler()
+ assert int(second_crawler._id) == int(first_crawler._id) + 1
diff --git a/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py b/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py
index 2604ca53da..b5fc104bef 100644
--- a/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py
+++ b/tests/unit/crawlers/_stagehand/test_stagehand_crawler.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import sys
-from unittest.mock import patch
+from unittest.mock import AsyncMock, patch
import pytest
@@ -41,6 +41,20 @@ def test_init_with_valid_browser_pool() -> None:
StagehandCrawler(browser_pool=pool)
+async def test_owns_default_browser_pool(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The crawler enters and exits the browser pool it builds itself."""
+ pool_enter = AsyncMock()
+ pool_exit = AsyncMock()
+ monkeypatch.setattr(BrowserPool, '__aenter__', pool_enter)
+ monkeypatch.setattr(BrowserPool, '__aexit__', pool_exit)
+ crawler = StagehandCrawler()
+
+ await crawler.run()
+
+ pool_enter.assert_awaited_once()
+ pool_exit.assert_awaited_once()
+
+
def test_forwards_save_response_cookies() -> None:
"""`save_response_cookies` reaches the `PlaywrightCrawler` that saves the cookies of the page."""
crawler = StagehandCrawler(save_response_cookies=False)
diff --git a/tests/unit/http_clients/test_http_clients.py b/tests/unit/http_clients/test_http_clients.py
index adde6d710d..cb365ba98f 100644
--- a/tests/unit/http_clients/test_http_clients.py
+++ b/tests/unit/http_clients/test_http_clients.py
@@ -312,3 +312,25 @@ async def test_ignore_tls_errors_reaches_fetch(
await client.send_request(url, proxy_info=proxy_info, ignore_tls_errors=request_flag)
assert fetch.call_args.kwargs['ignore_tls_errors'] is expected_flag
+
+
+async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A client can't be entered again until its exit has cleaned it up."""
+ http_client = ImpitHttpClient()
+ cleanup_started = asyncio.Event()
+ release_cleanup = asyncio.Event()
+
+ async def wait_for_release() -> None:
+ cleanup_started.set()
+ await release_cleanup.wait()
+
+ await http_client.__aenter__()
+ monkeypatch.setattr(http_client, 'cleanup', AsyncMock(side_effect=wait_for_release))
+ exit_task = asyncio.create_task(http_client.__aexit__(None, None, None))
+ await cleanup_started.wait()
+
+ with pytest.raises(RuntimeError, match='already active'):
+ await http_client.__aenter__()
+
+ release_cleanup.set()
+ await exit_task
diff --git a/tests/unit/sessions/test_session_pool.py b/tests/unit/sessions/test_session_pool.py
index 404612af69..8c29723737 100644
--- a/tests/unit/sessions/test_session_pool.py
+++ b/tests/unit/sessions/test_session_pool.py
@@ -1,8 +1,10 @@
from __future__ import annotations
+import asyncio
import logging
from datetime import UTC, datetime
from typing import TYPE_CHECKING
+from unittest.mock import AsyncMock
import pytest
@@ -275,3 +277,66 @@ async def test_methods_raise_error_when_not_active() -> None:
async with session_pool:
assert session_pool.active is True
+
+
+@pytest.mark.parametrize(
+ 'error',
+ [
+ pytest.param(RuntimeError('fill failed'), id='failed'),
+ pytest.param(asyncio.CancelledError(), id='cancelled'),
+ ],
+)
+async def test_failed_enter_frees_pool(monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None:
+ """A pool whose entry fails or is cancelled tears its state down, stays inactive and can be entered again."""
+ session_pool = SessionPool(max_pool_size=MAX_POOL_SIZE, persistence_enabled=False)
+ teardown = AsyncMock(wraps=session_pool._state.teardown)
+ monkeypatch.setattr(session_pool._state, 'teardown', teardown)
+ monkeypatch.setattr(session_pool, '_fill_sessions_to_max', AsyncMock(side_effect=error))
+
+ with pytest.raises(type(error)):
+ async with session_pool:
+ pass
+
+ assert not session_pool.active
+ teardown.assert_awaited_once()
+ monkeypatch.undo()
+
+ async with session_pool:
+ assert session_pool.active
+
+
+async def test_failed_exit_frees_pool(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A pool whose exit fails is no longer active, so it can be entered again."""
+ session_pool = SessionPool(max_pool_size=MAX_POOL_SIZE, persistence_enabled=False)
+
+ with pytest.raises(RuntimeError, match='teardown failed'):
+ async with session_pool:
+ monkeypatch.setattr(session_pool._state, 'teardown', AsyncMock(side_effect=RuntimeError('teardown failed')))
+
+ assert not session_pool.active
+ monkeypatch.undo()
+
+ async with session_pool:
+ assert session_pool.active
+
+
+async def test_enter_during_exit_raises(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A pool can't be entered again until its exit has torn the state down."""
+ session_pool = SessionPool(max_pool_size=MAX_POOL_SIZE, persistence_enabled=False)
+ teardown_started = asyncio.Event()
+ release_teardown = asyncio.Event()
+
+ async def wait_for_release() -> None:
+ teardown_started.set()
+ await release_teardown.wait()
+
+ await session_pool.__aenter__()
+ monkeypatch.setattr(session_pool._state, 'teardown', AsyncMock(side_effect=wait_for_release))
+ exit_task = asyncio.create_task(session_pool.__aexit__(None, None, None))
+ await teardown_started.wait()
+
+ with pytest.raises(RuntimeError, match='already active'):
+ await session_pool.__aenter__()
+
+ release_teardown.set()
+ await exit_task