diff --git a/src/crawlee/browsers/_browser_controller.py b/src/crawlee/browsers/_browser_controller.py index 7489dc25fb..f1bcf59145 100644 --- a/src/crawlee/browsers/_browser_controller.py +++ b/src/crawlee/browsers/_browser_controller.py @@ -100,3 +100,13 @@ async def close(self, *, force: bool = False) -> None: Raises: ValueError: If there are still open pages when trying to close the browser. """ + + def register_page_closed(self, page: Page) -> None: # noqa: B027 + """Stop tracking the page as open. + + Called from the page's `close` event, and by the `BrowserPool` when closing the page never finishes, in which + case that event never arrives. It can be called more than once for the same page. + + Args: + page: The page that is no longer open. + """ diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index a4c549d35d..bca61c7679 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -45,6 +45,9 @@ class BrowserPool: _GENERATED_PAGE_ID_LENGTH = 8 """The length of the newly generated page ID.""" + _PAGE_CLOSE_TIMEOUT = timedelta(seconds=5) + """The time limit for closing a page, after which the pool releases the page anyway.""" + def __init__( self, plugins: Sequence[BrowserPlugin] | None = None, @@ -398,21 +401,38 @@ async def _execute_hooks(self, hooks: list[Callable[..., Awaitable[None]]], *arg await hook(*args) def _override_page_close(self, crawlee_page: CrawleePage, browser_controller: BrowserController) -> None: - """Override the page's close method to execute pre and post close hooks.""" - if self._pre_page_close_hooks or self._post_page_close_hooks: - original_close = crawlee_page.page.close - - async def close_with_hooks(*args: Any, **kwargs: Any) -> None: - try: - await self._execute_hooks(self._pre_page_close_hooks, crawlee_page, browser_controller) - finally: - await original_close(*args, **kwargs) - await self._execute_hooks(self._post_page_close_hooks, crawlee_page.id, browser_controller) - - # `Page.close` is a regular method, so type checkers reject reassigning it directly. - # Install the hook-wrapped version through an untyped reference to the page. - page: Any = crawlee_page.page - page.close = close_with_hooks + """Override the page's close method to execute pre and post close hooks and bound the close itself. + + A browser can acknowledge the close and then never destroy the page, so neither the close call nor the page's + `close` event ever finishes. When the close does not finish within `_PAGE_CLOSE_TIMEOUT`, the page is released + from its browser anyway, and the browser is retired, so it gets no new pages and is closed once its remaining + pages are closed. The hooks themselves run without a time limit. + """ + original_close = crawlee_page.page.close + + async def close_page(*args: Any, **kwargs: Any) -> None: + timeout = self._PAGE_CLOSE_TIMEOUT.total_seconds() + try: + await asyncio.wait_for(original_close(*args, **kwargs), timeout) + except asyncio.TimeoutError: + logger.warning( + f'Closing the page {crawlee_page.id} did not finish within {timeout} seconds, releasing it from ' + 'the pool anyway.' + ) + browser_controller.register_page_closed(crawlee_page.page) + self._retire_browser(browser_controller) + + async def close_with_hooks(*args: Any, **kwargs: Any) -> None: + try: + await self._execute_hooks(self._pre_page_close_hooks, crawlee_page, browser_controller) + finally: + await close_page(*args, **kwargs) + await self._execute_hooks(self._post_page_close_hooks, crawlee_page.id, browser_controller) + + # `Page.close` is a regular method, so type checkers reject reassigning it directly. + # Install the wrapped version through an untyped reference to the page. + page: Any = crawlee_page.page + page.close = close_with_hooks def pre_launch_hook( self, hook: Callable[[str, BrowserPlugin], Awaitable[None]] diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index ae40fee43e..8c80c62730 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -197,7 +197,7 @@ async def new_page( page = await self._browser_context.new_page() # Handle page close event - page.on(event='close', f=self._on_page_close) + page.on(event='close', f=self.register_page_closed) # Update internal state self._pages.append(page) @@ -225,9 +225,10 @@ async def close(self, *, force: bool = False) -> None: await self._browser_context.close() await self._browser.close() - def _on_page_close(self, page: Page) -> None: - """Handle actions after a page is closed.""" - self._pages.remove(page) + @override + def register_page_closed(self, page: Page) -> None: + if page in self._pages: + self._pages.remove(page) def _filter_context_options(self, options: dict[str, Any]) -> dict[str, Any]: """Filter browser context options based on the current mode (incognito vs persistent). diff --git a/src/crawlee/browsers/_stagehand_browser_controller.py b/src/crawlee/browsers/_stagehand_browser_controller.py index ff60327775..9add4f8306 100644 --- a/src/crawlee/browsers/_stagehand_browser_controller.py +++ b/src/crawlee/browsers/_stagehand_browser_controller.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: from collections.abc import Mapping - from playwright.async_api import Browser, BrowserContext, Playwright + from playwright.async_api import Browser, BrowserContext, Page, Playwright from stagehand import AsyncSession, AsyncStagehand from crawlee.browsers._stagehand_types import StagehandOptions @@ -162,7 +162,7 @@ async def new_page( raw_page = await self._browser_context.new_page() page = StagehandPage(raw_page, self._session) - raw_page.on('close', lambda _: self._on_page_close(page)) + raw_page.on('close', lambda _: self.register_page_closed(page)) self._pages.append(page) self._last_page_opened_at = datetime.now(timezone.utc) @@ -195,8 +195,9 @@ async def close(self, *, force: bool = False) -> None: self._session = None self._browser_context = None - def _on_page_close(self, page: StagehandPage) -> None: - if page in self._pages: + @override + def register_page_closed(self, page: Page) -> None: + if isinstance(page, StagehandPage) and page in self._pages: self._pages.remove(page) def _get_extra_http_headers( diff --git a/tests/integration/browsers/test_browser_pool.py b/tests/integration/browsers/test_browser_pool.py index 0aba86b599..b5c062c86c 100644 --- a/tests/integration/browsers/test_browser_pool.py +++ b/tests/integration/browsers/test_browser_pool.py @@ -3,9 +3,10 @@ import asyncio from datetime import timedelta from typing import TYPE_CHECKING -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock import pytest +from playwright.async_api import Page from crawlee.browsers import BrowserPool, PlaywrightBrowserController, PlaywrightBrowserPlugin from crawlee.browsers._browser_controller import BrowserController @@ -15,7 +16,7 @@ from collections.abc import Mapping from typing import Any - from playwright.async_api import BrowserContext, Page + from playwright.async_api import BrowserContext from yarl import URL from crawlee.browsers._browser_plugin import BrowserPlugin @@ -364,6 +365,55 @@ async def hook(page_id: str, controller: BrowserController) -> None: assert isinstance(controller, BrowserController) +async def test_page_close_that_never_finishes_releases_page(monkeypatch: pytest.MonkeyPatch) -> None: + """A page whose close never finishes is released from its browser, and the browser is retired.""" + monkeypatch.setattr(BrowserPool, '_PAGE_CLOSE_TIMEOUT', timedelta(milliseconds=100)) + monkeypatch.setattr(Page, 'close', AsyncMock(side_effect=asyncio.Event().wait)) + + async with BrowserPool() as browser_pool: + test_page = await browser_pool.new_page() + controller = browser_pool.active_browsers[0] + + await asyncio.wait_for(test_page.page.close(), timeout=10) + + assert controller.pages_count == 0 + assert controller in browser_pool.inactive_browsers + + # The page's own `close` event can still arrive later, when the browser is closed. It must be a no-op. + controller.register_page_closed(test_page.page) + assert controller.pages_count == 0 + + +async def test_slow_page_close_hooks_are_not_cancelled(monkeypatch: pytest.MonkeyPatch) -> None: + """Page close hooks that outlast the close time limit finish, and the browser is not retired.""" + monkeypatch.setattr(BrowserPool, '_PAGE_CLOSE_TIMEOUT', timedelta(milliseconds=100)) + pre_hook_finished = Mock() + post_hook_finished = Mock() + + async with BrowserPool() as browser_pool: + + @browser_pool.pre_page_close_hook + async def pre_hook(_crawlee_page: CrawleePage, _controller: BrowserController) -> None: + await asyncio.sleep(0.3) + pre_hook_finished() + + @browser_pool.post_page_close_hook + async def post_hook(_page_id: str, _controller: BrowserController) -> None: + await asyncio.sleep(0.3) + post_hook_finished() + + test_page = await browser_pool.new_page() + controller = browser_pool.active_browsers[0] + + await asyncio.wait_for(test_page.page.close(), timeout=10) + + pre_hook_finished.assert_called_once() + post_hook_finished.assert_called_once() + assert test_page.page.is_closed() + assert controller.pages_count == 0 + assert controller in browser_pool.active_browsers + + async def test_hooks_execution_order() -> None: call_order: list[str] = []