Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/crawlee/browsers/_browser_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,3 +100,13 @@ async def close(self, *, force: bool = False) -> None:
Raises:
ValueError: If there are still open pages when trying to close the browser.
"""

def register_page_closed(self, page: Page) -> None: # noqa: B027
"""Stop tracking the page as open.

Called from the page's `close` event, and by the `BrowserPool` when closing the page never finishes, in which
case that event never arrives. It can be called more than once for the same page.

Args:
page: The page that is no longer open.
"""
50 changes: 35 additions & 15 deletions src/crawlee/browsers/_browser_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ class BrowserPool:
_GENERATED_PAGE_ID_LENGTH = 8
"""The length of the newly generated page ID."""

_PAGE_CLOSE_TIMEOUT = timedelta(seconds=5)
"""The time limit for closing a page, after which the pool releases the page anyway."""

def __init__(
self,
plugins: Sequence[BrowserPlugin] | None = None,
Expand Down Expand Up @@ -398,21 +401,38 @@ async def _execute_hooks(self, hooks: list[Callable[..., Awaitable[None]]], *arg
await hook(*args)

def _override_page_close(self, crawlee_page: CrawleePage, browser_controller: BrowserController) -> None:
"""Override the page's close method to execute pre and post close hooks."""
if self._pre_page_close_hooks or self._post_page_close_hooks:
original_close = crawlee_page.page.close

async def close_with_hooks(*args: Any, **kwargs: Any) -> None:
try:
await self._execute_hooks(self._pre_page_close_hooks, crawlee_page, browser_controller)
finally:
await original_close(*args, **kwargs)
await self._execute_hooks(self._post_page_close_hooks, crawlee_page.id, browser_controller)

# `Page.close` is a regular method, so type checkers reject reassigning it directly.
# Install the hook-wrapped version through an untyped reference to the page.
page: Any = crawlee_page.page
page.close = close_with_hooks
"""Override the page's close method to execute pre and post close hooks and bound the close itself.

A browser can acknowledge the close and then never destroy the page, so neither the close call nor the page's
`close` event ever finishes. When the close does not finish within `_PAGE_CLOSE_TIMEOUT`, the page is released
from its browser anyway, and the browser is retired, so it gets no new pages and is closed once its remaining
pages are closed. The hooks themselves run without a time limit.
"""
original_close = crawlee_page.page.close

async def close_page(*args: Any, **kwargs: Any) -> None:
timeout = self._PAGE_CLOSE_TIMEOUT.total_seconds()
try:
await asyncio.wait_for(original_close(*args, **kwargs), timeout)
except asyncio.TimeoutError:
logger.warning(
f'Closing the page {crawlee_page.id} did not finish within {timeout} seconds, releasing it from '
'the pool anyway.'
)
browser_controller.register_page_closed(crawlee_page.page)
self._retire_browser(browser_controller)

async def close_with_hooks(*args: Any, **kwargs: Any) -> None:
try:
await self._execute_hooks(self._pre_page_close_hooks, crawlee_page, browser_controller)
finally:
await close_page(*args, **kwargs)
await self._execute_hooks(self._post_page_close_hooks, crawlee_page.id, browser_controller)

# `Page.close` is a regular method, so type checkers reject reassigning it directly.
# Install the wrapped version through an untyped reference to the page.
page: Any = crawlee_page.page
page.close = close_with_hooks

def pre_launch_hook(
self, hook: Callable[[str, BrowserPlugin], Awaitable[None]]
Expand Down
9 changes: 5 additions & 4 deletions src/crawlee/browsers/_playwright_browser_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ async def new_page(
page = await self._browser_context.new_page()

# Handle page close event
page.on(event='close', f=self._on_page_close)
page.on(event='close', f=self.register_page_closed)

# Update internal state
self._pages.append(page)
Expand Down Expand Up @@ -225,9 +225,10 @@ async def close(self, *, force: bool = False) -> None:
await self._browser_context.close()
await self._browser.close()

def _on_page_close(self, page: Page) -> None:
"""Handle actions after a page is closed."""
self._pages.remove(page)
@override
def register_page_closed(self, page: Page) -> None:
if page in self._pages:
self._pages.remove(page)

def _filter_context_options(self, options: dict[str, Any]) -> dict[str, Any]:
"""Filter browser context options based on the current mode (incognito vs persistent).
Expand Down
9 changes: 5 additions & 4 deletions src/crawlee/browsers/_stagehand_browser_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
if TYPE_CHECKING:
from collections.abc import Mapping

from playwright.async_api import Browser, BrowserContext, Playwright
from playwright.async_api import Browser, BrowserContext, Page, Playwright
from stagehand import AsyncSession, AsyncStagehand

from crawlee.browsers._stagehand_types import StagehandOptions
Expand Down Expand Up @@ -162,7 +162,7 @@ async def new_page(

raw_page = await self._browser_context.new_page()
page = StagehandPage(raw_page, self._session)
raw_page.on('close', lambda _: self._on_page_close(page))
raw_page.on('close', lambda _: self.register_page_closed(page))

self._pages.append(page)
self._last_page_opened_at = datetime.now(timezone.utc)
Expand Down Expand Up @@ -195,8 +195,9 @@ async def close(self, *, force: bool = False) -> None:
self._session = None
self._browser_context = None

def _on_page_close(self, page: StagehandPage) -> None:
if page in self._pages:
@override
def register_page_closed(self, page: Page) -> None:
if isinstance(page, StagehandPage) and page in self._pages:
self._pages.remove(page)

def _get_extra_http_headers(
Expand Down
54 changes: 52 additions & 2 deletions tests/integration/browsers/test_browser_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
import asyncio
from datetime import timedelta
from typing import TYPE_CHECKING
from unittest.mock import AsyncMock
from unittest.mock import AsyncMock, Mock

import pytest
from playwright.async_api import Page

from crawlee.browsers import BrowserPool, PlaywrightBrowserController, PlaywrightBrowserPlugin
from crawlee.browsers._browser_controller import BrowserController
Expand All @@ -15,7 +16,7 @@
from collections.abc import Mapping
from typing import Any

from playwright.async_api import BrowserContext, Page
from playwright.async_api import BrowserContext
from yarl import URL

from crawlee.browsers._browser_plugin import BrowserPlugin
Expand Down Expand Up @@ -364,6 +365,55 @@ async def hook(page_id: str, controller: BrowserController) -> None:
assert isinstance(controller, BrowserController)


async def test_page_close_that_never_finishes_releases_page(monkeypatch: pytest.MonkeyPatch) -> None:
"""A page whose close never finishes is released from its browser, and the browser is retired."""
monkeypatch.setattr(BrowserPool, '_PAGE_CLOSE_TIMEOUT', timedelta(milliseconds=100))
monkeypatch.setattr(Page, 'close', AsyncMock(side_effect=asyncio.Event().wait))

async with BrowserPool() as browser_pool:
test_page = await browser_pool.new_page()
controller = browser_pool.active_browsers[0]

await asyncio.wait_for(test_page.page.close(), timeout=10)

assert controller.pages_count == 0
assert controller in browser_pool.inactive_browsers

# The page's own `close` event can still arrive later, when the browser is closed. It must be a no-op.
controller.register_page_closed(test_page.page)
assert controller.pages_count == 0


async def test_slow_page_close_hooks_are_not_cancelled(monkeypatch: pytest.MonkeyPatch) -> None:
"""Page close hooks that outlast the close time limit finish, and the browser is not retired."""
monkeypatch.setattr(BrowserPool, '_PAGE_CLOSE_TIMEOUT', timedelta(milliseconds=100))
pre_hook_finished = Mock()
post_hook_finished = Mock()

async with BrowserPool() as browser_pool:

@browser_pool.pre_page_close_hook
async def pre_hook(_crawlee_page: CrawleePage, _controller: BrowserController) -> None:
await asyncio.sleep(0.3)
pre_hook_finished()

@browser_pool.post_page_close_hook
async def post_hook(_page_id: str, _controller: BrowserController) -> None:
await asyncio.sleep(0.3)
post_hook_finished()

test_page = await browser_pool.new_page()
controller = browser_pool.active_browsers[0]

await asyncio.wait_for(test_page.page.close(), timeout=10)

pre_hook_finished.assert_called_once()
post_hook_finished.assert_called_once()
assert test_page.page.is_closed()
assert controller.pages_count == 0
assert controller in browser_pool.active_browsers


async def test_hooks_execution_order() -> None:
call_order: list[str] = []

Expand Down
Loading