From 9448688e7c47922bf819a4e56dd8704c67427c38 Mon Sep 17 00:00:00 2001 From: Prashanth Lande Date: Fri, 13 Nov 2026 11:39:35 +0530 Subject: [PATCH 1/2] fix(browsers): Release a page whose close never finishes --- src/crawlee/browsers/_browser_controller.py | 10 +++ src/crawlee/browsers/_browser_pool.py | 56 ++++++++++++++--- .../_playwright_browser_controller.py | 9 +-- .../browsers/_stagehand_browser_controller.py | 9 +-- .../integration/browsers/test_browser_pool.py | 62 ++++++++++++++++++- 5 files changed, 128 insertions(+), 18 deletions(-) diff --git a/src/crawlee/browsers/_browser_controller.py b/src/crawlee/browsers/_browser_controller.py index 7489dc25fb..f1bcf59145 100644 --- a/src/crawlee/browsers/_browser_controller.py +++ b/src/crawlee/browsers/_browser_controller.py @@ -100,3 +100,13 @@ async def close(self, *, force: bool = False) -> None: Raises: ValueError: If there are still open pages when trying to close the browser. """ + + def register_page_closed(self, page: Page) -> None: # noqa: B027 + """Stop tracking the page as open. + + Called from the page's `close` event, and by the `BrowserPool` when closing the page never finishes, in which + case that event never arrives. It can be called more than once for the same page. + + Args: + page: The page that is no longer open. + """ diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index a4c549d35d..e590c66d11 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -45,6 +45,12 @@ class BrowserPool: _GENERATED_PAGE_ID_LENGTH = 8 """The length of the newly generated page ID.""" + _PAGE_CLOSE_TIMEOUT = timedelta(seconds=5) + """The time limit for the page close hooks and for the page close itself. + + Closing a page can take up to twice this long before the pool releases the page anyway. + """ + def __init__( self, plugins: Sequence[BrowserPlugin] | None = None, @@ -398,21 +404,53 @@ async def _execute_hooks(self, hooks: list[Callable[..., Awaitable[None]]], *arg await hook(*args) def _override_page_close(self, crawlee_page: CrawleePage, browser_controller: BrowserController) -> None: - """Override the page's close method to execute pre and post close hooks.""" - if self._pre_page_close_hooks or self._post_page_close_hooks: - original_close = crawlee_page.page.close + """Override the page's close method to execute pre and post close hooks within a time limit. + + A browser can acknowledge the close and then never destroy the page, so neither the close call nor the page's + `close` event ever finishes. The page is then released from its browser anyway, and the browser is retired, so + it gets no new pages and is closed once its remaining pages are closed. + """ + original_close = crawlee_page.page.close + + async def close_with_hooks(*args: Any, **kwargs: Any) -> None: + timeout = self._PAGE_CLOSE_TIMEOUT.total_seconds() - async def close_with_hooks(*args: Any, **kwargs: Any) -> None: + # Tracked separately from the whole sequence finishing, so a slow post page close hook does not retire a + # browser that closed its page. + page_closed = False + + async def close_sequence() -> None: + nonlocal page_closed try: await self._execute_hooks(self._pre_page_close_hooks, crawlee_page, browser_controller) finally: - await original_close(*args, **kwargs) + # Bounded on its own, because a timed out pre page close hook leaves the close to run here, inside + # the already cancelled sequence. + await asyncio.wait_for(original_close(*args, **kwargs), timeout) + page_closed = True await self._execute_hooks(self._post_page_close_hooks, crawlee_page.id, browser_controller) - # `Page.close` is a regular method, so type checkers reject reassigning it directly. - # Install the hook-wrapped version through an untyped reference to the page. - page: Any = crawlee_page.page - page.close = close_with_hooks + try: + await asyncio.wait_for(close_sequence(), timeout) + except asyncio.TimeoutError: + if page_closed: + logger.warning( + f'The page close hooks for the page {crawlee_page.id} did not finish within {timeout} seconds ' + 'and were cancelled.' + ) + else: + logger.warning( + f'Closing the page {crawlee_page.id} did not finish within {timeout} seconds, releasing it ' + 'from the pool anyway.' + ) + self._retire_browser(browser_controller) + finally: + browser_controller.register_page_closed(crawlee_page.page) + + # `Page.close` is a regular method, so type checkers reject reassigning it directly. + # Install the wrapped version through an untyped reference to the page. + page: Any = crawlee_page.page + page.close = close_with_hooks def pre_launch_hook( self, hook: Callable[[str, BrowserPlugin], Awaitable[None]] diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index ae40fee43e..8c80c62730 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -197,7 +197,7 @@ async def new_page( page = await self._browser_context.new_page() # Handle page close event - page.on(event='close', f=self._on_page_close) + page.on(event='close', f=self.register_page_closed) # Update internal state self._pages.append(page) @@ -225,9 +225,10 @@ async def close(self, *, force: bool = False) -> None: await self._browser_context.close() await self._browser.close() - def _on_page_close(self, page: Page) -> None: - """Handle actions after a page is closed.""" - self._pages.remove(page) + @override + def register_page_closed(self, page: Page) -> None: + if page in self._pages: + self._pages.remove(page) def _filter_context_options(self, options: dict[str, Any]) -> dict[str, Any]: """Filter browser context options based on the current mode (incognito vs persistent). diff --git a/src/crawlee/browsers/_stagehand_browser_controller.py b/src/crawlee/browsers/_stagehand_browser_controller.py index ff60327775..9add4f8306 100644 --- a/src/crawlee/browsers/_stagehand_browser_controller.py +++ b/src/crawlee/browsers/_stagehand_browser_controller.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: from collections.abc import Mapping - from playwright.async_api import Browser, BrowserContext, Playwright + from playwright.async_api import Browser, BrowserContext, Page, Playwright from stagehand import AsyncSession, AsyncStagehand from crawlee.browsers._stagehand_types import StagehandOptions @@ -162,7 +162,7 @@ async def new_page( raw_page = await self._browser_context.new_page() page = StagehandPage(raw_page, self._session) - raw_page.on('close', lambda _: self._on_page_close(page)) + raw_page.on('close', lambda _: self.register_page_closed(page)) self._pages.append(page) self._last_page_opened_at = datetime.now(timezone.utc) @@ -195,8 +195,9 @@ async def close(self, *, force: bool = False) -> None: self._session = None self._browser_context = None - def _on_page_close(self, page: StagehandPage) -> None: - if page in self._pages: + @override + def register_page_closed(self, page: Page) -> None: + if isinstance(page, StagehandPage) and page in self._pages: self._pages.remove(page) def _get_extra_http_headers( diff --git a/tests/integration/browsers/test_browser_pool.py b/tests/integration/browsers/test_browser_pool.py index 0aba86b599..98bfea1907 100644 --- a/tests/integration/browsers/test_browser_pool.py +++ b/tests/integration/browsers/test_browser_pool.py @@ -1,11 +1,13 @@ from __future__ import annotations import asyncio +import logging from datetime import timedelta from typing import TYPE_CHECKING from unittest.mock import AsyncMock import pytest +from playwright.async_api import Page from crawlee.browsers import BrowserPool, PlaywrightBrowserController, PlaywrightBrowserPlugin from crawlee.browsers._browser_controller import BrowserController @@ -15,7 +17,7 @@ from collections.abc import Mapping from typing import Any - from playwright.async_api import BrowserContext, Page + from playwright.async_api import BrowserContext from yarl import URL from crawlee.browsers._browser_plugin import BrowserPlugin @@ -364,6 +366,64 @@ async def hook(page_id: str, controller: BrowserController) -> None: assert isinstance(controller, BrowserController) +@pytest.mark.parametrize( + 'pre_page_close_hook_hangs', + [ + pytest.param(False, id='close never finishes'), + pytest.param(True, id='pre page close hook and close never finish'), + ], +) +async def test_page_close_that_never_finishes_releases_page( + monkeypatch: pytest.MonkeyPatch, *, pre_page_close_hook_hangs: bool +) -> None: + """A page whose close never finishes is released from its browser, and the browser is retired.""" + monkeypatch.setattr(BrowserPool, '_PAGE_CLOSE_TIMEOUT', timedelta(milliseconds=100)) + monkeypatch.setattr(Page, 'close', AsyncMock(side_effect=asyncio.Event().wait)) + + async with BrowserPool() as browser_pool: + if pre_page_close_hook_hangs: + + @browser_pool.pre_page_close_hook + async def hook(_crawlee_page: CrawleePage, _controller: BrowserController) -> None: + await asyncio.Event().wait() + + test_page = await browser_pool.new_page() + controller = browser_pool.active_browsers[0] + + await asyncio.wait_for(test_page.page.close(), timeout=10) + + assert controller.pages_count == 0 + assert controller in browser_pool.inactive_browsers + + # The page's own `close` event can still arrive later, when the browser is closed. It must be a no-op. + controller.register_page_closed(test_page.page) + assert controller.pages_count == 0 + + +async def test_slow_post_page_close_hook_does_not_retire_browser( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A post page close hook that outlasts the close time limit does not retire a browser that closed its page.""" + monkeypatch.setattr(BrowserPool, '_PAGE_CLOSE_TIMEOUT', timedelta(milliseconds=100)) + + async with BrowserPool() as browser_pool: + + @browser_pool.post_page_close_hook + async def hook(_page_id: str, _controller: BrowserController) -> None: + await asyncio.Event().wait() + + test_page = await browser_pool.new_page() + controller = browser_pool.active_browsers[0] + + with caplog.at_level(logging.WARNING): + await asyncio.wait_for(test_page.page.close(), timeout=10) + + assert f'The page close hooks for the page {test_page.id} did not finish' in caplog.text + assert test_page.page.is_closed() + assert controller.pages_count == 0 + assert controller in browser_pool.active_browsers + + async def test_hooks_execution_order() -> None: call_order: list[str] = [] From 52a1acf6506f80f66d586ecf2e511e45c010e344 Mon Sep 17 00:00:00 2001 From: Prashanth Lande Date: Mon, 16 Nov 2026 14:43:08 +0530 Subject: [PATCH 2/2] fix(browsers): limit the timeout to the page close and leave hooks unbounded --- src/crawlee/browsers/_browser_pool.py | 56 +++++++------------ .../integration/browsers/test_browser_pool.py | 44 ++++++--------- 2 files changed, 36 insertions(+), 64 deletions(-) diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index e590c66d11..bca61c7679 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -46,10 +46,7 @@ class BrowserPool: """The length of the newly generated page ID.""" _PAGE_CLOSE_TIMEOUT = timedelta(seconds=5) - """The time limit for the page close hooks and for the page close itself. - - Closing a page can take up to twice this long before the pool releases the page anyway. - """ + """The time limit for closing a page, after which the pool releases the page anyway.""" def __init__( self, @@ -404,48 +401,33 @@ async def _execute_hooks(self, hooks: list[Callable[..., Awaitable[None]]], *arg await hook(*args) def _override_page_close(self, crawlee_page: CrawleePage, browser_controller: BrowserController) -> None: - """Override the page's close method to execute pre and post close hooks within a time limit. + """Override the page's close method to execute pre and post close hooks and bound the close itself. A browser can acknowledge the close and then never destroy the page, so neither the close call nor the page's - `close` event ever finishes. The page is then released from its browser anyway, and the browser is retired, so - it gets no new pages and is closed once its remaining pages are closed. + `close` event ever finishes. When the close does not finish within `_PAGE_CLOSE_TIMEOUT`, the page is released + from its browser anyway, and the browser is retired, so it gets no new pages and is closed once its remaining + pages are closed. The hooks themselves run without a time limit. """ original_close = crawlee_page.page.close - async def close_with_hooks(*args: Any, **kwargs: Any) -> None: + async def close_page(*args: Any, **kwargs: Any) -> None: timeout = self._PAGE_CLOSE_TIMEOUT.total_seconds() - - # Tracked separately from the whole sequence finishing, so a slow post page close hook does not retire a - # browser that closed its page. - page_closed = False - - async def close_sequence() -> None: - nonlocal page_closed - try: - await self._execute_hooks(self._pre_page_close_hooks, crawlee_page, browser_controller) - finally: - # Bounded on its own, because a timed out pre page close hook leaves the close to run here, inside - # the already cancelled sequence. - await asyncio.wait_for(original_close(*args, **kwargs), timeout) - page_closed = True - await self._execute_hooks(self._post_page_close_hooks, crawlee_page.id, browser_controller) - try: - await asyncio.wait_for(close_sequence(), timeout) + await asyncio.wait_for(original_close(*args, **kwargs), timeout) except asyncio.TimeoutError: - if page_closed: - logger.warning( - f'The page close hooks for the page {crawlee_page.id} did not finish within {timeout} seconds ' - 'and were cancelled.' - ) - else: - logger.warning( - f'Closing the page {crawlee_page.id} did not finish within {timeout} seconds, releasing it ' - 'from the pool anyway.' - ) - self._retire_browser(browser_controller) - finally: + logger.warning( + f'Closing the page {crawlee_page.id} did not finish within {timeout} seconds, releasing it from ' + 'the pool anyway.' + ) browser_controller.register_page_closed(crawlee_page.page) + self._retire_browser(browser_controller) + + async def close_with_hooks(*args: Any, **kwargs: Any) -> None: + try: + await self._execute_hooks(self._pre_page_close_hooks, crawlee_page, browser_controller) + finally: + await close_page(*args, **kwargs) + await self._execute_hooks(self._post_page_close_hooks, crawlee_page.id, browser_controller) # `Page.close` is a regular method, so type checkers reject reassigning it directly. # Install the wrapped version through an untyped reference to the page. diff --git a/tests/integration/browsers/test_browser_pool.py b/tests/integration/browsers/test_browser_pool.py index 98bfea1907..b5c062c86c 100644 --- a/tests/integration/browsers/test_browser_pool.py +++ b/tests/integration/browsers/test_browser_pool.py @@ -1,10 +1,9 @@ from __future__ import annotations import asyncio -import logging from datetime import timedelta from typing import TYPE_CHECKING -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock import pytest from playwright.async_api import Page @@ -366,27 +365,12 @@ async def hook(page_id: str, controller: BrowserController) -> None: assert isinstance(controller, BrowserController) -@pytest.mark.parametrize( - 'pre_page_close_hook_hangs', - [ - pytest.param(False, id='close never finishes'), - pytest.param(True, id='pre page close hook and close never finish'), - ], -) -async def test_page_close_that_never_finishes_releases_page( - monkeypatch: pytest.MonkeyPatch, *, pre_page_close_hook_hangs: bool -) -> None: +async def test_page_close_that_never_finishes_releases_page(monkeypatch: pytest.MonkeyPatch) -> None: """A page whose close never finishes is released from its browser, and the browser is retired.""" monkeypatch.setattr(BrowserPool, '_PAGE_CLOSE_TIMEOUT', timedelta(milliseconds=100)) monkeypatch.setattr(Page, 'close', AsyncMock(side_effect=asyncio.Event().wait)) async with BrowserPool() as browser_pool: - if pre_page_close_hook_hangs: - - @browser_pool.pre_page_close_hook - async def hook(_crawlee_page: CrawleePage, _controller: BrowserController) -> None: - await asyncio.Event().wait() - test_page = await browser_pool.new_page() controller = browser_pool.active_browsers[0] @@ -400,25 +384,31 @@ async def hook(_crawlee_page: CrawleePage, _controller: BrowserController) -> No assert controller.pages_count == 0 -async def test_slow_post_page_close_hook_does_not_retire_browser( - monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture -) -> None: - """A post page close hook that outlasts the close time limit does not retire a browser that closed its page.""" +async def test_slow_page_close_hooks_are_not_cancelled(monkeypatch: pytest.MonkeyPatch) -> None: + """Page close hooks that outlast the close time limit finish, and the browser is not retired.""" monkeypatch.setattr(BrowserPool, '_PAGE_CLOSE_TIMEOUT', timedelta(milliseconds=100)) + pre_hook_finished = Mock() + post_hook_finished = Mock() async with BrowserPool() as browser_pool: + @browser_pool.pre_page_close_hook + async def pre_hook(_crawlee_page: CrawleePage, _controller: BrowserController) -> None: + await asyncio.sleep(0.3) + pre_hook_finished() + @browser_pool.post_page_close_hook - async def hook(_page_id: str, _controller: BrowserController) -> None: - await asyncio.Event().wait() + async def post_hook(_page_id: str, _controller: BrowserController) -> None: + await asyncio.sleep(0.3) + post_hook_finished() test_page = await browser_pool.new_page() controller = browser_pool.active_browsers[0] - with caplog.at_level(logging.WARNING): - await asyncio.wait_for(test_page.page.close(), timeout=10) + await asyncio.wait_for(test_page.page.close(), timeout=10) - assert f'The page close hooks for the page {test_page.id} did not finish' in caplog.text + pre_hook_finished.assert_called_once() + post_hook_finished.assert_called_once() assert test_page.page.is_closed() assert controller.pages_count == 0 assert controller in browser_pool.active_browsers