Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ request; never edit the version or the changelog by hand.
expiring is terminal for that page's connection; retries require a new page.
- Never block the asyncio loop with file I/O.
- Keep strong references to background tasks until completion.
- Sync callbacks run on the callback dispatcher, never the bridge loop.
- Sync callbacks run on their page's callback dispatcher, never the bridge loop.

## Commands

Expand Down
8 changes: 8 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,14 @@ Ruff has a minimum version rather than an exact pin; use the same version when
comparing generated output across environments, and run the formatting and
codegen drift checks above before submitting changes.

Refresh the offline parity snapshot from the same published package version as
the schema, and commit `tests/fixtures/bap_ts_parity.json` with the schema:

```bash
npm pack @browserless.io/bap-ts@<version>
python scripts/bap_ts_parity.py --package browserless.io-bap-ts-<version>.tgz
```

## API reference

The generated Python API reference is published at
Expand Down
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -287,3 +287,14 @@ resuming handoffs. Handoffs remain unavailable during this cutover. To roll back
pause and drain again and restore both the previous server and previous clients;
rolling back only one side leaves an incompatible pair. No mixed-version
compatibility path is provided by this SDK.

## Sync API events

Callbacks run on a background thread per page, in event order for that page.
A slow callback delays only its own page. Each page buffers up to 1,024 pending
callbacks; beyond that it drops the newest callback with a logged warning.
A `once` listener stays registered until one event is actually queued.
`browser.get_pages()` returns the same page objects, so `off()` works through any
reference to that page.
Register listeners before closing the page or browser; registering on a closed
object does not restart callback delivery.
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ exclude = ["src/browserless_bap/mutations.graphql"]
[tool.pytest.ini_options]
addopts = "--timeout=60 --timeout-method=thread"
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
filterwarnings = ["error::RuntimeWarning"]
markers = ["e2e: tests that require a live Browserless service"]

Expand Down
124 changes: 124 additions & 0 deletions scripts/bap_ts_parity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
#!/usr/bin/env python3
"""Extract an offline parity snapshot from the published bap-ts npm package."""

from __future__ import annotations

import argparse
import difflib
import json
import re
import sys
import tarfile
from collections.abc import Callable
from hashlib import sha256
from pathlib import Path
from typing import Any

TARGET = Path(__file__).resolve().parents[1] / "tests/fixtures/bap_ts_parity.json"
PACKAGE = "@browserless.io/bap-ts"


def extract(read: Callable[[str], bytes]) -> dict[str, Any]:
metadata = json.loads(read("package.json"))
if metadata.get("name") != PACKAGE:
raise ValueError(f"Expected package {PACKAGE}")
js = read("dist/graphql/mutations.js").decode("utf-8")

def table(name: str, *, nested: bool = False) -> dict[str, Any]:
block = re.search(rf"export const {name.upper()} = {{(.*?)\n}};", js, re.S)
if not block:
raise ValueError(f"Missing table: {name}")
if nested:
result = {
key: dict(re.findall(r"(\w+): '([^']*)'", values))
for key, values in re.findall(r"^\s+(\w+): \{(.*)\},?$", block[1], re.M)
}
else:
result = dict(re.findall(r"^\s+(\w+): '([^']*)',", block[1], re.M))
if not result:
raise ValueError(f"Empty table: {name}")
return result

readme = read("README.md").decode("utf-8")
block = re.search(
r"<!-- AUTOGEN:method-table START -->(.*?)<!-- AUTOGEN:method-table END -->",
readme,
re.S,
)
methods = sorted(re.findall(r"^\| `([^`]+)` \|", block[1] if block else "", re.M))
if not methods:
raise ValueError("Empty table: methods")
cushioned = sorted(
{
name
for file in ("dist/page-generated.js", "dist/page.js")
for name, remainder in re.findall(
r"this\.execute\(\s*'(\w+)'(.*?)\);\n",
read(file).decode("utf-8"),
re.S,
)
if "this.transportTimeout(" in remainder
}
)
if not cushioned:
raise ValueError("Empty table: cushioned_mutations")
return {
"package": metadata["name"],
"version": metadata["version"],
"schema_sha256": sha256(read("src/mutations.graphql")).hexdigest(),
"methods": methods,
"response_fields": table("response_fields"),
"optional_fields": table("optional_fields", nested=True),
"arg_types": table("arg_types", nested=True),
"cushioned_mutations": cushioned,
}


def snapshot(package: Path) -> dict[str, Any]:
if package.is_dir():
return extract(lambda name: (package / name).read_bytes())
with tarfile.open(package, "r:*") as archive:

def read(name: str) -> bytes:
member = archive.extractfile(f"package/{name}")
if member is None:
raise ValueError(f"Missing package file: {name}")
with member:
return member.read()

return extract(read)


def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--package", type=Path, required=True)
parser.add_argument("--check", action="store_true")
args = parser.parse_args()
try:
rendered = json.dumps(snapshot(args.package), sort_keys=True, indent=2) + "\n"
if args.check:
current = TARGET.read_text(encoding="utf-8") if TARGET.exists() else ""
if current != rendered:
print(
"".join(
difflib.unified_diff(
current.splitlines(keepends=True),
rendered.splitlines(keepends=True),
fromfile="committed snapshot",
tofile="package snapshot",
)
),
end="",
)
return 1
else:
TARGET.parent.mkdir(parents=True, exist_ok=True)
TARGET.write_text(rendered, encoding="utf-8")
except (OSError, ValueError, KeyError, tarfile.TarError) as error:
print(f"Parity extraction failed: {error}", file=sys.stderr)
return 1
return 0


if __name__ == "__main__":
raise SystemExit(main())
6 changes: 5 additions & 1 deletion src/browserless_bap/page.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@
WaitForResponseResponse,
)

_ASYNC_PREDICATE = re.compile(r"\s*async\b")


@dataclass
class _LiveHandoff:
Expand Down Expand Up @@ -430,7 +432,9 @@ async def wait_for_function(
complete_on_promise_resolution: bool | UnsetType = UNSET,
) -> WaitForFunction:
predicate = expression
if complete_on_promise_resolution is True:
if complete_on_promise_resolution is True and _ASYNC_PREDICATE.match(
expression
):
Comment thread
artiom marked this conversation as resolved.
Resolved
predicate = f"async () => {{ await ({expression})(); return true; }}"
args = _drop_unset(
{
Expand Down
44 changes: 32 additions & 12 deletions src/browserless_bap/sync_api/_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
import threading
import time
import warnings
from collections.abc import Awaitable, Callable, Coroutine
import weakref
from collections.abc import Awaitable, Callable, Coroutine, Iterable
from contextlib import suppress
from typing import Protocol, TypeVar, cast

Expand Down Expand Up @@ -116,7 +117,7 @@ def _run(self) -> None:
finally:
self._queue.task_done()

def close(self) -> None:
def close(self, *, wait: bool = True) -> None:
if self._closed.is_set():
return
self._closed.set()
Expand All @@ -134,9 +135,19 @@ def close(self) -> None:
self._queue.put_nowait(None)
except queue.Full:
return
if threading.current_thread() is not self._thread:
if wait and threading.current_thread() is not self._thread:
self._thread.join(timeout=2)

@staticmethod
def close_all(dispatchers: Iterable[CallbackDispatcher | None]) -> None:
workers = tuple(worker for worker in dispatchers if worker is not None)
deadline = time.monotonic() + 2
for worker in workers:
worker.close(wait=False)
for worker in workers:
if threading.current_thread() is not worker._thread:
worker._thread.join(timeout=max(0, deadline - time.monotonic()))


class LoopThread:
"""Own an asyncio loop on a dedicated daemon thread."""
Expand All @@ -152,21 +163,25 @@ def __init__(self) -> None:
self._pending: set[cf.Future[object]] = set()
self._tracked_lock = threading.Lock()
self._tracked_browsers: dict[int, object] = {}
self._dispatchers: weakref.WeakSet[CallbackDispatcher] = weakref.WeakSet()
self._thread = threading.Thread(
target=self._thread_main,
name="bap-sync-loop",
daemon=True,
)
self._dispatcher = CallbackDispatcher()
self._thread.start()
if not self._ready.wait(timeout=5):
raise RuntimeError("bap sync event loop failed to start")
if self._loop is None:
raise RuntimeError("bap sync event loop failed during startup")

@property
def dispatcher(self) -> CallbackDispatcher:
return self._dispatcher
def new_dispatcher(self, *, max_queue_size: int = 1024) -> CallbackDispatcher:
with self._tracked_lock:
if self._stopping or self._closed.is_set():
raise RuntimeError("the bap sync event loop is shutting down")
dispatcher = CallbackDispatcher(max_queue_size=max_queue_size)
self._dispatchers.add(dispatcher)
return dispatcher

def _thread_main(self) -> None:
loop = asyncio.new_event_loop()
Expand Down Expand Up @@ -300,19 +315,24 @@ def untrack_browser(self, browser: object) -> None:

def close(self) -> None:
if self._closed.is_set():
self._dispatcher.close()
with self._tracked_lock:
dispatchers = tuple(self._dispatchers)
CallbackDispatcher.close_all(dispatchers)
return
if self._stopping:
return
self._stopping = True
with self._tracked_lock:
if self._stopping:
return
self._stopping = True
if (
threading.current_thread() is not self._thread
and self._loop is not None
and self._loop.is_running()
):
with suppress(BaseException):
self.run(self._close_tracked_browsers(), timeout=5)
self._dispatcher.close()
with self._tracked_lock:
dispatchers = tuple(self._dispatchers)
CallbackDispatcher.close_all(dispatchers)
loop = self._loop
if loop is not None and loop.is_running():
loop.call_soon_threadsafe(loop.stop)
Expand Down
43 changes: 40 additions & 3 deletions src/browserless_bap/sync_api/browser.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,44 @@

from collections.abc import Iterator
from contextlib import contextmanager
from threading import Lock
from typing import Any, Self

from ._base import SyncBase
from ._bridge import CallbackDispatcher, LoopThread
from .page import Page


class Browser(SyncBase[Any]):
def __init__(self, impl: Any, bridge: LoopThread | None = None) -> None:
super().__init__(impl, bridge)
self._pages: dict[int, Page] = {}
self._pages_lock = Lock()

def _wrap_page(self, impl: Any) -> Page:
self._prune_pages()
with self._pages_lock:
page = self._pages.get(id(impl))
if page is None or page._impl is not impl:
page = self._pages[id(impl)] = Page(impl, self._bridge)
return page

def new_page(self) -> Page:
return Page(self._invoke("new_page"), self._bridge)
return self._wrap_page(self._invoke("new_page"))
Comment thread
artiom marked this conversation as resolved.
Resolved

def _prune_pages(self) -> None:
with self._pages_lock:
stale = [page for page in self._pages.values() if page.closed]
for page in stale:
del self._pages[id(page._impl)]
CallbackDispatcher.close_all(
page._close_dispatcher(wait=False) for page in stale
)

def get_pages(self) -> list[Page]:
return [Page(page, self._bridge) for page in self._invoke("get_pages")]
self._prune_pages()
impls = self._invoke("get_pages")
return [self._wrap_page(impl) for impl in impls]

def pages(self) -> list[Page]:
return self.get_pages()
Expand All @@ -28,12 +54,23 @@ def page(self) -> Iterator[Page]:
try:
yield page
finally:
page.close()
try:
page.close()
finally:
with self._pages_lock:
if self._pages.get(id(page._impl)) is page:
del self._pages[id(page._impl)]

def close(self) -> None:
try:
self._invoke("close")
finally:
with self._pages_lock:
pages = list(self._pages.values())
self._pages.clear()
CallbackDispatcher.close_all(
page._close_dispatcher(wait=False) for page in pages
)
self._bridge.untrack_browser(self._impl)

def __enter__(self) -> Self:
Expand Down
Loading
Loading