From df55b0b4d1aa48d984fd0b46b8dcc70e3e823a3b Mon Sep 17 00:00:00 2001 From: Amp Date: Fri, 13 Nov 2026 12:48:11 +0000 Subject: [PATCH 1/5] feat!: make element HTML and response bodies opt-in and rename builtin-shadowing errors --- README.md | 26 +++++- llms.txt | 19 +++- scripts/codegen/config/client_only.py | 11 ++- scripts/codegen/config/docs.py | 40 ++++++++ scripts/codegen/config/methods.py | 2 - scripts/codegen/config/schema_map.py | 14 +-- scripts/codegen/emit_operations.py | 20 ++-- scripts/codegen/emit_page.py | 29 +++++- scripts/codegen/emit_responses.py | 15 ++- scripts/codegen/generate.py | 12 +++ scripts/codegen/model.py | 9 ++ scripts/codegen/schema.py | 41 +++++++-- src/browserless_bap/__init__.py | 6 -- src/browserless_bap/_page_generated.py | 74 +++++++++++++++ src/browserless_bap/browser.py | 8 +- src/browserless_bap/element_handle.py | 15 ++- src/browserless_bap/errors.py | 21 +++-- src/browserless_bap/graphql/__init__.py | 6 +- src/browserless_bap/graphql/builder.py | 21 ++++- src/browserless_bap/graphql/mutations.py | 19 +++- src/browserless_bap/page.py | 66 +++++++------ src/browserless_bap/sync_api/_bridge.py | 6 +- .../sync_api/_page_generated.py | 54 +++++++++++ src/browserless_bap/sync_api/browserless.py | 11 ++- src/browserless_bap/sync_api/page.py | 7 ++ src/browserless_bap/transport.py | 40 ++++---- src/browserless_bap/types/options.py | 2 + src/browserless_bap/types/responses.py | 2 +- tests/e2e/test_config.py | 2 +- tests/e2e/test_live.py | 59 ++++++++++-- tests/sync/test_bridge.py | 37 ++++++++ tests/sync/test_sync_events.py | 4 +- tests/sync/test_sync_page.py | 19 ++-- tests/sync/test_wrappers.py | 19 ++-- tests/test_browser.py | 8 +- tests/test_codegen_operations.py | 18 ++-- tests/test_codegen_schema.py | 31 ++++++- tests/test_errors.py | 16 ++-- tests/test_graphql_builder.py | 19 ++++ tests/test_live_complete.py | 36 ++++---- tests/test_page_overrides.py | 92 +++++++++++++++++-- tests/test_public_api.py | 88 ++++++++++++++++++ tests/test_readme_typecheck.py | 34 +++++++ tests/test_transport.py | 7 +- tests/test_transport_protocol.py | 17 ++-- 45 files changed, 890 insertions(+), 212 deletions(-) create mode 100644 tests/test_public_api.py create mode 100644 tests/test_readme_typecheck.py diff --git a/README.md b/README.md index 687dfae..c71b1d5 100644 --- a/README.md +++ b/README.md @@ -108,8 +108,8 @@ contents between the markers by hand. | `content` | `html` | `str` | ✅ | | `cookies` | `cookies` | `list[StandardCookie]` | ✅ | | `emulate_media` | `emulateMediaType` | `EmulateMediaTypeResponse` | ✅ | -| `eval_on_selector` | `text` | `str` | ✅ | -| `eval_on_selector_all` | `mapSelector` | `list[MapSelectorResponse]` | ✅ | +| `eval_on_selector` | `text` | `str` | — | +| `eval_on_selector_all` | `mapSelector` | `list[MapSelectorResponse]` | — | | `evaluate` | `evaluate` | `str \| None` | ✅ | | `fulfill` | `fulfill` | `FulfillResponse` | — | | `go_back` | `back` | `HTTPResponse \| None` | ✅ | @@ -162,6 +162,26 @@ contents between the markers by hand. +## Response keys and optional fields + +Options and keyword arguments use `snake_case`; returned dictionaries keep the +server's field names (`liveURLId`, `httpOnly`, `childElementCount`), as Playwright +for Python does for cookies. + +`query_selector()` and `query_selector_all()` fetch element HTML only with +`html=True`. Reading a handle's `inner_html` or `outer_html` without requesting it +raises `BAPValueError` with a hint. This also applies to element-scoped queries. +`response()` omits bodies by default; pass `body=True` to fetch them, or +`body_encoding=True` to fetch both the body and its `base64Encoded` flag. + +`eval_on_selector()` returns the first matching element's text, and +`eval_on_selector_all()` returns element details. Unlike Playwright's methods, +neither accepts a page function. + +SDK exceptions are `BAPTimeoutError`, `BAPConnectionError` and `BAPValueError`. +They still subclass Python's matching built-in exceptions; the builtin-named SDK +exports have been removed. + ## Schema and generated code `src/browserless_bap/mutations.graphql` is the vendored GraphQL schema. Its full digest, @@ -237,7 +257,7 @@ card, import `LiveURLInstructions` from `browserless_bap.types`. The SDK subscribes before minting and retains completion until the next mint, so a quick completion is not lost between `live_url()` and `wait_for_live_complete()`. -Wait timeouts are in milliseconds and raise the SDK `TimeoutError`. By default +Wait timeouts are in milliseconds and raise `BAPTimeoutError`. By default there is no client wait deadline; pass a positive timeout to bound the wait. `timeout=0` also disables the wait deadline. Timing out or cancelling a wait does not close the live URL. diff --git a/llms.txt b/llms.txt index 07c0cf6..fee5a9f 100644 --- a/llms.txt +++ b/llms.txt @@ -45,6 +45,23 @@ Generated Page methods: `add_cookies`, `add_script_tag`, `add_style_tag`, `authe +## Response keys and optional fields + +Options and keyword arguments use `snake_case`; returned dictionaries keep the +server's field names (`liveURLId`, `httpOnly`, `childElementCount`), as Playwright +for Python does for cookies. + +`query_selector()` and `query_selector_all()` fetch element HTML only with +`html=True`, including element-scoped queries. Reading `inner_html` or +`outer_html` without requesting it raises `BAPValueError` with a fix hint. +`response()` omits bodies unless `body=True` or `body_encoding=True` (which implies +the body and adds `base64Encoded`). + +`eval_on_selector()` returns text; `eval_on_selector_all()` returns element +details. Unlike Playwright's methods, neither accepts a page function. +SDK errors are `BAPTimeoutError`, `BAPConnectionError` and `BAPValueError`, still +subclassing the matching builtins; the builtin-named SDK exports are removed. + ## Live URL handoffs ```python @@ -84,7 +101,7 @@ card, import `LiveURLInstructions` from `browserless_bap.types`. The SDK subscribes before minting and retains completion until the next mint, so a quick completion is not lost between `live_url()` and `wait_for_live_complete()`. -Wait timeouts are in milliseconds and raise the SDK `TimeoutError`. By default +Wait timeouts are in milliseconds and raise `BAPTimeoutError`. By default there is no client wait deadline; pass a positive timeout to bound the wait. `timeout=0` also disables the wait deadline. Timing out or cancelling a wait does not close the live URL. diff --git a/scripts/codegen/config/client_only.py b/scripts/codegen/config/client_only.py index 3f0dab1..39c321d 100644 --- a/scripts/codegen/config/client_only.py +++ b/scripts/codegen/config/client_only.py @@ -8,11 +8,18 @@ class ClientParam: name: str annotation: str transport_timeout: bool = False + opt_in: str | None = None CLIENT_ONLY_PARAMS = { - "GotoOptions": (ClientParam("remote_response_metadata", "bool"),), - "ResponseFilterOptions": (ClientParam("body_encoding", "bool"),), + "GotoOptions": ( + ClientParam("remote_response_metadata", "bool", opt_in="remoteMetadata"), + ), + "ResponseFilterOptions": ( + ClientParam("body", "bool", opt_in="body"), + ClientParam("body_encoding", "bool", opt_in="bodyEncoding"), + ), + "QuerySelectorOptions": (ClientParam("html", "bool", opt_in="html"),), "ScreenshotOptions": (ClientParam("path", "str"),), "PDFOptions": (ClientParam("path", "str"),), "AddScriptTagOptions": (ClientParam("path", "str"),), diff --git a/scripts/codegen/config/docs.py b/scripts/codegen/config/docs.py index 3717827..6fab15b 100644 --- a/scripts/codegen/config/docs.py +++ b/scripts/codegen/config/docs.py @@ -8,4 +8,44 @@ "Start tracking `event` now so a later `wait_for_event` call can return " "immediately if that event has already fired." ), + "eval_on_selector": ( + "Return the text content of the first element matching `selector`. " + "Unlike Playwright's `eval_on_selector`, this takes no page function: " + "it is a shortcut for `text(selector=...)`." + ), + "eval_on_selector_all": ( + "Return details (`innerHTML`, `innerText`, `id`, `class`) for every element " + "matching `selector`. Unlike Playwright's `eval_on_selector_all`, this " + "takes no page function: it is a shortcut for `map_selector(selector)`." + ), + "query_selector": ( + "Return a handle for the first element matching `selector`, or `None`. " + "The element's `innerHTML`/`outerHTML` are fetched only with `html=True`." + ), + "query_selector_all": ( + "Return handles for every element matching `selector`. " + "The elements' `innerHTML`/`outerHTML` are fetched only with `html=True`." + ), } + + +def lead_prose(description: str | None) -> str | None: + if not description: + return None + markers = ( + "```", + "**Usage Example", + "**Field Definition", + "**Simple Example", + "**Advanced", + "Simple Example", + "Advanced Example", + "Example (", + "**Usage", + "**Example", + ) + end = min( + (description.index(marker) for marker in markers if marker in description), + default=len(description), + ) + return description[:end].strip() or None diff --git a/scripts/codegen/config/methods.py b/scripts/codegen/config/methods.py index e955ae8..3abefa8 100644 --- a/scripts/codegen/config/methods.py +++ b/scripts/codegen/config/methods.py @@ -163,8 +163,6 @@ def options_name(mutation: str) -> str: "pdf", "query_selector", "query_selector_all", - "eval_on_selector", - "eval_on_selector_all", "wait_for_selector", "wait_for_function", "evaluate", diff --git a/scripts/codegen/config/schema_map.py b/scripts/codegen/config/schema_map.py index fd78556..8880776 100644 --- a/scripts/codegen/config/schema_map.py +++ b/scripts/codegen/config/schema_map.py @@ -11,6 +11,7 @@ } RESPONSE_TYPE_OVERRIDES = {"Headers": "RequestHeader"} +RESPONSE_FIELD_TYPE_OVERRIDES = {("QuerySelectorResponse", "childElementCount"): "int"} RESPONSE_RENAME_EXCLUDE = {"WaitForFunction", "WaitForEvent"} SCALAR_MAP = { @@ -34,13 +35,8 @@ } SUBSCRIPTION_OMIT_FIELDS = {"response": {"body"}} -UNSUPPORTED_FIELDS = { - "HTTPResponse": { - "fields": {"ip", "port"}, - "variant": "HTTPResponseWithRemoteMetadata", - }, - "ResponseResponse": { - "fields": {"base64Encoded"}, - "variant": "ResponseResponseWithBodyEncoding", - }, +OPTIONAL_FIELDS: dict[str, dict[str, tuple[str, ...]]] = { + "HTTPResponse": {"remoteMetadata": ("ip", "port")}, + "ResponseResponse": {"body": ("body",), "bodyEncoding": ("body", "base64Encoded")}, + "QuerySelectorResponse": {"html": ("innerHTML", "outerHTML")}, } diff --git a/scripts/codegen/emit_operations.py b/scripts/codegen/emit_operations.py index 015659e..f61dffc 100644 --- a/scripts/codegen/emit_operations.py +++ b/scripts/codegen/emit_operations.py @@ -4,21 +4,25 @@ # ruff: noqa: E501 import json -from .config.schema_map import UNSUPPORTED_FIELDS +from .config.schema_map import OPTIONAL_FIELDS from .schema import Schema, render_graphql_type from .writer import HEADER def response_fields(schema: Schema) -> dict[str, str]: - fields = { + return { schema.response_name(name): schema.response_selection(name) for name in schema.response_objects } - for name, config in UNSUPPORTED_FIELDS.items(): - fields[str(config["variant"])] = schema.response_selection( - name, keep_unsupported=True - ) - return fields + + +def optional_fields(schema: Schema) -> dict[str, dict[str, str]]: + return { + schema.response_name(name): { + group: " ".join(fields) for group, fields in groups.items() + } + for name, groups in OPTIONAL_FIELDS.items() + } def argument_types(schema: Schema) -> dict[str, dict[str, str]]: @@ -40,6 +44,8 @@ def emit(schema: Schema) -> str: "# fmt: off", f"RESPONSE_FIELDS: dict[str, str] = {json.dumps(response_fields(schema), sort_keys=True, indent=4)}", "", + f"OPTIONAL_FIELDS: dict[str, dict[str, str]] = {json.dumps(optional_fields(schema), sort_keys=True, indent=4)}", + "", f"ARG_TYPES: dict[str, dict[str, str]] = {json.dumps(argument_types(schema), sort_keys=True, indent=4)}", "# fmt: on", "", diff --git a/scripts/codegen/emit_page.py b/scripts/codegen/emit_page.py index 800fd1b..3531bc2 100644 --- a/scripts/codegen/emit_page.py +++ b/scripts/codegen/emit_page.py @@ -7,7 +7,6 @@ import json -from .config.docs import METHOD_DOCSTRINGS from .model import MethodModel from .writer import HEADER @@ -73,6 +72,7 @@ def _execute_lines(model: MethodModel, prefix: str) -> list[str]: ] if timeout is not None: lines.append(f" timeout=None if {timeout} is UNSET else {timeout},") + lines.extend(_opt_in_lines(model, " ")) lines.append(" )") return lines @@ -91,16 +91,33 @@ def _cast_execute_lines(model: MethodModel) -> list[str]: lines.append( f" timeout=None if {timeout} is UNSET else {timeout}," ) + lines.extend(_opt_in_lines(model, " ")) lines.extend((" ),", " )")) return lines +def _opt_in_lines(model: MethodModel, indent: str) -> list[str]: + params = [param for param in model.params if param.opt_in is not None] + if not params: + return [] + entries = ", ".join( + f"({json.dumps(param.opt_in)}, {param.py_name})" for param in params + ) + return [ + f"{indent}opt_in=tuple(", + f"{indent} group", + f"{indent} for group, enabled in ({entries},)", + f"{indent} if enabled is True", + f"{indent}),", + ] + + def _async_method(model: MethodModel) -> list[str]: lines = [f" async def {model.py_name}("] lines.extend(_signature_lines(model, " ")) lines.append(f" ) -> {model.return_type}:") - if docstring := METHOD_DOCSTRINGS.get(model.py_name): - lines.append(f" {json.dumps(docstring)}") + if docstring := model.doc: + lines.append(f" {json.dumps(docstring, ensure_ascii=False)}") lines.extend(_argument_lines(model)) if model.flavor == "buffer": has_path = any(param.py_name == "path" for param in model.params) @@ -181,6 +198,7 @@ def emit_async(models: list[MethodModel]) -> str: "# ruff: noqa: ASYNC109, F403, F405", "from __future__ import annotations", "", + "from collections.abc import Sequence", "from typing import TYPE_CHECKING, Any, cast", "", "from ._codec import decode_base64, drop_unset, or_default, write_binary_file", @@ -200,6 +218,7 @@ def emit_async(models: list[MethodModel]) -> str: " response_type: str,", " *,", " timeout: float | None = None,", + " opt_in: Sequence[str] = (),", " ) -> Any:", " raise NotImplementedError", "", @@ -258,8 +277,8 @@ def emit_sync(models: list[MethodModel]) -> str: lines.append(f" def {model.py_name}(") lines.extend(_signature_lines(model, " ")) lines.append(f" ) -> {model.return_type}:") - if docstring := METHOD_DOCSTRINGS.get(model.py_name): - lines.append(f" {json.dumps(docstring)}") + if docstring := model.doc: + lines.append(f" {json.dumps(docstring, ensure_ascii=False)}") lines.extend(_sync_result_lines(model)) lines.append("") return "\n".join(lines) diff --git a/scripts/codegen/emit_responses.py b/scripts/codegen/emit_responses.py index 4476b5f..aab94a4 100644 --- a/scripts/codegen/emit_responses.py +++ b/scripts/codegen/emit_responses.py @@ -3,6 +3,7 @@ import json import keyword +from .config.schema_map import RESPONSE_FIELD_TYPE_OVERRIDES from .schema import Schema from .writer import HEADER @@ -32,7 +33,12 @@ def emit(schema: Schema) -> str: lines.append(f' "{name}",') lines.append(" {") for field in fields: - annotation = _nullable(schema.python_type(field.type, response=True)) + annotation = _nullable( + RESPONSE_FIELD_TYPE_OVERRIDES.get( + (gql_name, field.name.value), + schema.python_type(field.type, response=True), + ) + ) lines.append( f" {json.dumps(field.name.value)}: " f"{json.dumps(f'NotRequired[{annotation}]')}," @@ -43,7 +49,12 @@ def emit(schema: Schema) -> str: if not fields: lines.append(" pass") for field in fields: - annotation = _nullable(schema.python_type(field.type, response=True)) + annotation = _nullable( + RESPONSE_FIELD_TYPE_OVERRIDES.get( + (gql_name, field.name.value), + schema.python_type(field.type, response=True), + ) + ) lines.append(f" {field.name.value}: NotRequired[{annotation}]") lines.extend(("", "")) return "\n".join(lines).rstrip() diff --git a/scripts/codegen/generate.py b/scripts/codegen/generate.py index eb3df1a..8a6e1b6 100644 --- a/scripts/codegen/generate.py +++ b/scripts/codegen/generate.py @@ -1,5 +1,7 @@ from __future__ import annotations +import subprocess +import sys from pathlib import Path from .config.docs import LLMS_END, LLMS_START, README_END, README_START @@ -50,6 +52,16 @@ def generate(root: Path, *, docs_output: Path | None = None) -> list[Path]: root / path: content for path, content in zip(GENERATED_OUTPUTS, rendered, strict=True) } + for path in outputs: + if path.name == "_page_generated.py": + outputs[path] = subprocess.run( + [sys.executable, "-m", "ruff", "format", "--stdin-filename", str(path)], + input=outputs[path], + capture_output=True, + text=True, + check=True, + cwd=root, + ).stdout changed = [ path for path, content in outputs.items() if write_if_changed(path, content) ] diff --git a/scripts/codegen/model.py b/scripts/codegen/model.py index 2f928fd..2972d25 100644 --- a/scripts/codegen/model.py +++ b/scripts/codegen/model.py @@ -7,6 +7,7 @@ from graphql.language import FieldDefinitionNode from .config.client_only import CLIENT_ONLY_PARAMS +from .config.docs import METHOD_DOCSTRINGS, lead_prose from .config.methods import ( METHOD_SPECS, NULLABLE_RETURNS, @@ -27,6 +28,7 @@ class Param: positional: bool required: bool transport_timeout: bool = False + opt_in: str | None = None @dataclass(frozen=True) @@ -41,6 +43,7 @@ class MethodModel: unwrap_field: str | None = None unwrap_default: object = None wrap: str | None = None + doc: str | None = None @property def transport_timeout_param(self) -> str | None: @@ -107,6 +110,7 @@ def _params( client.transport_timeout if p.py_name == client.name else p.transport_timeout, + client.opt_in if p.py_name == client.name else p.opt_in, ) for p in params ] @@ -119,6 +123,7 @@ def _params( positional=False, required=False, transport_timeout=client.transport_timeout, + opt_in=client.opt_in, ) ) positional_order = {name: index for index, name in enumerate(spec.positional)} @@ -184,6 +189,10 @@ def build_models(schema: Schema) -> list[MethodModel]: unwrap_field=unwrap_field, unwrap_default=unwrap_default, wrap=spec.wrap, + doc=METHOD_DOCSTRINGS.get(method_name(spec.method)) + or lead_prose( + field.description.value if field.description else None + ), ) ) return models diff --git a/scripts/codegen/schema.py b/scripts/codegen/schema.py index 07956f5..fec03ce 100644 --- a/scripts/codegen/schema.py +++ b/scripts/codegen/schema.py @@ -33,11 +33,12 @@ ENUM_NAME_OVERRIDES, EXCLUDED_MUTATIONS, EXCLUDED_TYPES, + OPTIONAL_FIELDS, + RESPONSE_FIELD_TYPE_OVERRIDES, RESPONSE_RENAME_EXCLUDE, RESPONSE_TYPE_OVERRIDES, SCALAR_MAP, SUBSCRIPTION_OMIT_FIELDS, - UNSUPPORTED_FIELDS, ) @@ -190,15 +191,14 @@ def response_selection( graphql_type: str, *, omit: Iterable[str] = (), - keep_unsupported: bool = False, _seen: frozenset[str] = frozenset(), ) -> str: node = self.objects.get(graphql_type) if node is None: return "" excluded = set(omit) - if not keep_unsupported: - excluded.update(UNSUPPORTED_FIELDS.get(graphql_type, {}).get("fields", ())) + for group in OPTIONAL_FIELDS.get(graphql_type, {}).values(): + excluded.update(group) parts: list[str] = [] for field in node.fields or (): name = field.name.value @@ -263,9 +263,36 @@ def _validate_config(self) -> None: errors.append( f"SUBSCRIPTION_OMIT_FIELDS contains unknown event {event!r}" ) - for type_name in UNSUPPORTED_FIELDS: - if type_name not in self.objects: - errors.append(f"UNSUPPORTED_FIELDS contains unknown type {type_name!r}") + for type_name, groups in OPTIONAL_FIELDS.items(): + node = self.objects.get(type_name) + if node is None: + errors.append(f"OPTIONAL_FIELDS contains unknown type {type_name!r}") + continue + actual_fields = {field.name.value for field in node.fields or ()} + for group in groups.values(): + for name in group: + if name not in actual_fields: + errors.append( + f"OPTIONAL_FIELDS[{type_name!r}] contains unknown " + f"field {name!r}" + ) + for field in fields.values(): + groups = OPTIONAL_FIELDS.get(named_type(field.type), {}) + for client in CLIENT_ONLY_PARAMS.get(options_name(field.name.value), ()): + if client.opt_in is not None and client.opt_in not in groups: + errors.append( + f"CLIENT_ONLY_PARAMS[{options_name(field.name.value)!r}] " + f"contains unknown optional field group {client.opt_in!r}" + ) + for type_name, field_name in RESPONSE_FIELD_TYPE_OVERRIDES: + node = self.objects.get(type_name) + if node is None or field_name not in { + field.name.value for field in node.fields or () + }: + errors.append( + "RESPONSE_FIELD_TYPE_OVERRIDES contains unknown field " + f"{type_name}.{field_name}" + ) builtin_scalars = {"String", "Int", "Float", "Boolean", "ID"} for scalar in self.scalars - builtin_scalars: if scalar not in SCALAR_MAP: diff --git a/src/browserless_bap/__init__.py b/src/browserless_bap/__init__.py index 77273c6..411dbae 100644 --- a/src/browserless_bap/__init__.py +++ b/src/browserless_bap/__init__.py @@ -9,10 +9,7 @@ BAPTimeoutError, BAPValueError, BrowserQLError, - ConnectionError, SchemaMismatchError, - TimeoutError, - ValueError, ) from .page import Page @@ -24,12 +21,9 @@ "Browser", "BrowserQLError", "Browserless", - "ConnectionError", "ElementHandle", "Page", "SchemaMismatchError", - "TimeoutError", - "ValueError", "__version__", "sync_api", ] diff --git a/src/browserless_bap/_page_generated.py b/src/browserless_bap/_page_generated.py index 3887acf..205df38 100644 --- a/src/browserless_bap/_page_generated.py +++ b/src/browserless_bap/_page_generated.py @@ -2,6 +2,7 @@ # ruff: noqa: ASYNC109, F403, F405 from __future__ import annotations +from collections.abc import Sequence from typing import TYPE_CHECKING, Any, cast from ._codec import decode_base64, drop_unset, or_default, write_binary_file @@ -21,6 +22,7 @@ async def _execute( response_type: str, *, timeout: float | None = None, + opt_in: Sequence[str] = (), ) -> Any: raise NotImplementedError @@ -31,6 +33,7 @@ async def authenticate( *, origin: str | UnsetType = UNSET, ) -> HTTPResponse | None: + "Provides credentials for HTTP authentication (RFC 7235). When set, any HTTP\nauthentication challenge (401/407) will be answered with the supplied\ncredentials automatically. If an `origin` is provided, only challenges whose\norigin matches will use these credentials; otherwise every challenge is\nanswered (Puppeteer-style)." args = drop_unset( { "username": username, @@ -53,6 +56,7 @@ async def go_back( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Goes back in browser history, optionally accepting waitUntil and timeout arguments. Returns null if no back is possible" args = drop_unset( { "timeout": timeout, @@ -78,6 +82,7 @@ async def check( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Sets or un-sets the value of a checkbox on the page" args = drop_unset( { "selector": selector, @@ -107,6 +112,7 @@ async def uncheck( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Sets or un-sets the value of a checkbox on the page" args = drop_unset( { "selector": selector, @@ -136,6 +142,7 @@ async def click( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Waits for the element to be visible, scrolls to it, then clicks on it with native events" args = drop_unset( { "selector": selector, @@ -163,6 +170,7 @@ async def set_content( wait_until: WaitUntilHistory | UnsetType = UNSET, wait_until_all: list[WaitUntilHistory] | UnsetType = UNSET, ) -> HTTPResponse | None: + "Sets the given HTML content on the page with an optional waitUntil parameter" args = drop_unset( { "html": html, @@ -184,6 +192,7 @@ async def set_content( async def cookies( self, ) -> list[StandardCookie]: + "Sets and gets cookies on the page" args = None result = await self._execute( "cookies", @@ -204,6 +213,7 @@ async def go_forward( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Goes forward in browser history, optionally accepting waitUntil and timeout arguments. Returns null if no forward is possible" args = drop_unset( { "timeout": timeout, @@ -231,6 +241,7 @@ async def goto( referrer_policy: str | UnsetType = UNSET, remote_response_metadata: bool | UnsetType = UNSET, ) -> HTTPResponse | None: + "Navigates to a URL with an optional waitUntil parameter and timeout parameter" args = drop_unset( { "url": url, @@ -248,6 +259,13 @@ async def goto( args, "HTTPResponse", timeout=None if timeout is UNSET else timeout, + opt_in=tuple( + group + for group, enabled in ( + ("remoteMetadata", remote_response_metadata), + ) + if enabled is True + ), ), ) @@ -260,6 +278,7 @@ async def html( clean: CleanInput | UnsetType = UNSET, rewrite: HTMLRewriteInput | UnsetType = UNSET, ) -> HTMLResponse: + 'Returns the HTML content of the page or selector when specified. This API can also "clean" HTML markup returned by specifying a "clean" argument with numerous options. Features of the "clean" argument include removal of non-text nodes, removal of DOM attributes, as well as removal of excessive whitespace and newlines. Using "clean" can save nearly 1,000 times the payload size. Useful for LLM\'s and other scenarios' args = drop_unset( { "selector": selector, @@ -282,6 +301,7 @@ async def html( async def content( self, ) -> str: + 'Returns the HTML content of the page or selector when specified. This API can also "clean" HTML markup returned by specifying a "clean" argument with numerous options. Features of the "clean" argument include removal of non-text nodes, removal of DOM attributes, as well as removal of excessive whitespace and newlines. Using "clean" can save nearly 1,000 times the payload size. Useful for LLM\'s and other scenarios' args = None result = await self._execute( "html", @@ -300,6 +320,7 @@ async def set_user_agent( self, user_agent: str, ) -> UserAgentResponse: + "Sets the User-Agent string for the browser session" args = drop_unset( { "userAgent": user_agent, @@ -325,6 +346,7 @@ async def hover( x: float | UnsetType = UNSET, y: float | UnsetType = UNSET, ) -> HoverResponse: + "Waits for the element to be visible, scrolls to it, then hover on it with native events" args = drop_unset( { "selector": selector, @@ -350,6 +372,7 @@ async def set_java_script_enabled( self, enabled: bool, ) -> JavaScriptResponse: + "Sets and gets JavaScript execution on the page\n\n**Note: changing this value won't affect scripts that have already been run. It will take full effect on the next navigation.**" args = drop_unset( { "enabled": enabled, @@ -386,6 +409,7 @@ async def live_url( fail_label: str | UnsetType = UNSET, requester_name: str | UnsetType = UNSET, ) -> LiveURLResponse: + "Returns a fully-qualified, user-shareable live-URL for streaming the web-browser to an end-user, optionally interactive.\n\n**Set 30 second timer for interaction**" args = drop_unset( { "captureScale": capture_scale, @@ -425,6 +449,7 @@ async def map_selector( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> list[MapSelectorResponse]: + 'Specify a selector that returns multiple nodes in a document (similar to `document.querySelectorAll`), or JavaScript that returns a NodeList, and this API will respond with details about those DOM nodes. Similar to how "map" works in most functional programming languages and libraries. Useful for mapping over repetitive data in sites and pages like product listings or search results. This will automatically wait for the selector to be present on the page, and is configurable with the "wait" and "timeout" options.\n\nFor getting arbitrary DOM attributes back you can specify them via the `attribute(name: "data-custom-attribute")` property. This will return an object with `name` and `value` properties.\n\nYou may also continuously map further nested items as well, for instance this query might get all books on a page, and then a nested `mapSelector` call might list all sellers of that book, or shipping speeds. Hierarchy of data is preserved to pass through the hierarchical data modeled inside the DOM.\n\nThis API will always return a list of results back regardless if one or more items are found, or `null` if none are found.\n\nUsing aliases can also give the returned JSON more meaning and better model the data returned by this powerful API.' args = drop_unset( { "selector": selector, @@ -450,6 +475,7 @@ async def eval_on_selector_all( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> list[MapSelectorResponse]: + "Return details (`innerHTML`, `innerText`, `id`, `class`) for every element matching `selector`. Unlike Playwright's `eval_on_selector_all`, this takes no page function: it is a shortcut for `map_selector(selector)`." args = drop_unset( { "selector": selector, @@ -496,6 +522,7 @@ async def pdf( transfer_mode: str | UnsetType = UNSET, path: str | UnsetType = UNSET, ) -> bytes: + "Generates a PDF of the page with the print CSS media type" args = drop_unset( { "displayHeaderFooter": display_header_footer, @@ -539,6 +566,7 @@ async def preferences( *, timeout: float | UnsetType = UNSET, ) -> DefaultResponse: + "Sets configuration for the entirety of the session, replacing defaults like the 30 second timeout default" args = drop_unset( { "timeout": timeout, @@ -569,6 +597,7 @@ async def proxy( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, ) -> ProxyResponse: + 'Proxies requests, by a specified set of conditions, through either the Browserless residential proxy or through an external proxy. Only requests that match these conditions are proxied and the rest are sent from the instance\'s own IP address.\n\nUse the "server" argument to specify an external proxy for Browserless to use for requests. For requests with authentication, the username and password should be included in the URL using Basic Authentication. See the examples below for more information on how to format those URLs.\n\n**Using the browserless proxy for all requests and proxy through Brazil**' args = drop_unset( { "network": network, @@ -599,7 +628,9 @@ async def query_selector( *, timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, + html: bool | UnsetType = UNSET, ) -> ElementHandle | None: + "Return a handle for the first element matching `selector`, or `None`. The element's `innerHTML`/`outerHTML` are fetched only with `html=True`." args = drop_unset( { "selector": selector, @@ -612,6 +643,9 @@ async def query_selector( args, "QuerySelectorResponse", timeout=None if timeout is UNSET else timeout, + opt_in=tuple( + group for group, enabled in (("html", html),) if enabled is True + ), ) if result is None: return None @@ -625,7 +659,9 @@ async def query_selector_all( *, timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, + html: bool | UnsetType = UNSET, ) -> list[ElementHandle]: + "Return handles for every element matching `selector`. The elements' `innerHTML`/`outerHTML` are fetched only with `html=True`." args = drop_unset( { "selector": selector, @@ -638,6 +674,9 @@ async def query_selector_all( args, "QuerySelectorResponse", timeout=None if timeout is UNSET else timeout, + opt_in=tuple( + group for group, enabled in (("html", html),) if enabled is True + ), ) if not results: return [] @@ -654,6 +693,7 @@ async def reconnect( *, timeout: float | UnsetType = UNSET, ) -> ReconnectionResponse: + "Returns a payload with reconnection information in order to reconnect back to the same browser session" args = drop_unset( { "timeout": timeout, @@ -672,6 +712,7 @@ async def reconnect( async def stop_session_recording( self, ) -> StopSessionRecordingResponse: + "Stops the current session recording and processes the replay data, similar to closing the context. This mutation will stop RRWeb recording and upload the collected events." args = None return cast( "StopSessionRecordingResponse", @@ -692,6 +733,7 @@ async def reject( url: list[str] | UnsetType = UNSET, regex: list[str] | UnsetType = UNSET, ) -> RejectResponse: + 'Rejects requests by a specified URL pattern, method, or type and operator. You may supply a single pattern, or a list (array) of them. This mutation, by default, will reject any requests that match *any* pattern, which we call an "or" operator. To reject requests where conditions must all match, specify an "and" operator in the mutation. Note that this only has an effect when the query is executing, so scripts that return quickly will likely see assets loading in the editor as these rejections only happen when mutations are executing.\n___\n**Rejecting images or media**' args = drop_unset( { "enabled": enabled, @@ -717,6 +759,7 @@ async def reload( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Reloads the given page with an optional waitUntil parameter and timeout parameter" args = drop_unset( { "timeout": timeout, @@ -743,6 +786,7 @@ async def request( wait: bool | UnsetType = UNSET, operator: OperatorType | UnsetType = UNSET, ) -> list[RequestResponse]: + 'Returns request information made by the Browser with optional filters via arguments. You may filter the returned results by a glob-like URL-pattern, the method of the request or the type of request. Applying an operator to this will then change the behavior by either "and"ing the filters together or "or"ing them. This API will automatically wait for the request to be made if none is immediately found which you can turn off by disabling the "wait" option.\n\n**Getting all "Document" requests**' args = drop_unset( { "type": type, @@ -774,8 +818,10 @@ async def response( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, wait: bool | UnsetType = UNSET, + body: bool | UnsetType = UNSET, body_encoding: bool | UnsetType = UNSET, ) -> list[ResponseResponse]: + 'Returns response information, filtered by the provided arguments, made by the browser. You may optionally filter the returned results by a glob-like URL-pattern, the Method of the response or the Type of response. Applying an operator to this will then change the behavior by either "and"ing the filters together, or "or"ing them. This API will automatically wait for the response to be made if none is immediately found which you can turn off by disabling the "wait" option.\n\n**Getting all "Document" responses**' args = drop_unset( { "status": status, @@ -792,6 +838,14 @@ async def response( args, "ResponseResponse", timeout=None if timeout is UNSET else timeout, + opt_in=tuple( + group + for group, enabled in ( + ("body", body), + ("bodyEncoding", body_encoding), + ) + if enabled is True + ), ) return cast( "list[ResponseResponse]", @@ -814,6 +868,7 @@ async def screenshot( wait_for_images: bool | UnsetType = UNSET, path: str | UnsetType = UNSET, ) -> bytes: + "Screenshots the page or a specific selector" args = drop_unset( { "captureBeyondViewport": capture_beyond_viewport, @@ -851,6 +906,7 @@ async def scroll( x: float | UnsetType = UNSET, y: float | UnsetType = UNSET, ) -> ScrollResponse: + "Waits for a selector, then scrolls to it on the page or an x,y coordinate in pixels" args = drop_unset( { "selector": selector, @@ -879,6 +935,7 @@ async def solve( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> CaptchaResponse: + "🚨 **EXPERIMENTAL** 🚨\nSolves a captcha or other challenge. Can auto-detect the CAPTCHA type or solve a specific type.\nUses the same detection logic as BaaS to automatically identify the CAPTCHA type\nand routes to the appropriate solver. Optionally accepts a specific type to solve." args = drop_unset( { "type": type, @@ -903,6 +960,7 @@ async def solve_image_captcha( input_selector: str, timeout: float | UnsetType = UNSET, ) -> CaptchaResponse: + "Solves an image captcha using caller-provided selectors. Unlike solve, which auto-detects known captcha types, this mutation works on any site by accepting custom captchaSelector and inputSelector arguments. It detects the captcha image, solves it, and fills the result into the specified input field." args = drop_unset( { "captchaSelector": captcha_selector, @@ -928,6 +986,7 @@ async def text( visible: bool | UnsetType = UNSET, clean: CleanInput | UnsetType = UNSET, ) -> TextResponse: + "Returns the text content on the given page or by selector when specified" args = drop_unset( { "timeout": timeout, @@ -950,6 +1009,7 @@ async def eval_on_selector( self, selector: str, ) -> str: + "Return the text content of the first element matching `selector`. Unlike Playwright's `eval_on_selector`, this takes no page function: it is a shortcut for `text(selector=...)`." args = drop_unset( { "selector": selector, @@ -971,6 +1031,7 @@ async def eval_on_selector( async def title( self, ) -> str: + "Returns the title of the page that the browser is currently at" args = None result = await self._execute( "title", @@ -997,6 +1058,7 @@ async def type( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> TypeResponse: + "Types text into an element by scrolling to it, clicking it, then emitting key events for every character. Include a `\\n` (or `\\r`) in `text` to send an Enter/Return key press — useful for submitting search boxes or forms without a separate click. BrowserQL has no standalone key-press mutation, so non-Enter keys cannot be simulated; use `click` for non-keyboard submit interactions (for example, a submit button)." args = drop_unset( { "selector": selector, @@ -1025,6 +1087,7 @@ async def load_secret( *, selector: str | UnsetType = UNSET, ) -> LoadSecretResponse: + "Fills a credential resolved just-in-time from a configured 1Password\nintegration into a target input. The resolved value never appears in your\nquery or in the response. The session must be started with an integration\n(the `integrationId` connection parameter or session field), and the page's\norigin must be in the integration's allowed-domain list. Once any secret has\nbeen filled, capture and page-content readback are disabled for the rest of the\nsession — screenshots, PDFs, screencasts, Live URLs, session recording, and the\ncontent-return mutations (evaluate, html, text, querySelector, querySelectorAll,\ncookies)." args = drop_unset( { "ref": ref, @@ -1043,6 +1106,7 @@ async def load_secret( async def url( self, ) -> str: + "Returns the URL of the page that the browser is currently at" args = None result = await self._execute( "url", @@ -1063,6 +1127,7 @@ async def wait_for_navigation( timeout: float | UnsetType = UNSET, wait_until: WaitUntilGoto | UnsetType = UNSET, ) -> HTTPResponse: + "Waits for a navigation even to fire, useful for clicking an element and waiting for a page load of some" args = drop_unset( { "timeout": timeout, @@ -1086,6 +1151,7 @@ async def wait_for_network_idle( concurrency: int | UnsetType = UNSET, timeout: float | UnsetType = UNSET, ) -> WaitForTimeoutResponse: + "Wait for the page to have no more than the configured number of in-flight network requests for a period of time." args = drop_unset( { "idleTime": idle_time, @@ -1111,6 +1177,7 @@ async def wait_for_selector( visible: bool | UnsetType = UNSET, hidden: bool | UnsetType = UNSET, ) -> WaitForSelectorResponse: + "Waits for a given selector to be present in the DOM, with optional visibility" args = drop_unset( { "selector": selector, @@ -1135,6 +1202,7 @@ async def wait_for_event( *, timeout: float | UnsetType = UNSET, ) -> WaitForEvent: + "Waits for a named event to fire on the page's document or window." args = drop_unset( { "event": event, @@ -1180,6 +1248,7 @@ async def set_viewport_size( has_touch: bool | UnsetType = UNSET, landscape: bool | UnsetType = UNSET, ) -> ViewportResponse: + "Sets the viewport dimensions for the browser session" args = drop_unset( { "width": width, @@ -1208,6 +1277,7 @@ async def switch_to_window( timeout: float | UnsetType = UNSET, wait_until: WaitUntilGoto | UnsetType = UNSET, ) -> SwitchWindowResponse: + "Switches context to a popup window matching specified criteria" args = drop_unset( { "url": url, @@ -1231,6 +1301,7 @@ async def emulate_media( self, type: MediaType, ) -> EmulateMediaTypeResponse: + 'Emulates the given CSS media type for the page, mirroring the behavior of\nprinting ("print") or on-screen rendering ("screen"). Useful before a `pdf`\nor `screenshot` to control how the page renders.' args = drop_unset( { "type": type, @@ -1259,6 +1330,7 @@ async def fulfill( base64_body: str | UnsetType = UNSET, headers: list[HeaderInput] | UnsetType = UNSET, ) -> FulfillResponse: + "Fulfills matching requests with a canned response instead of letting them hit\nthe network, mirroring the request-interceptor mocking used by the REST APIs.\nMatching follows the same glob-style URL/method/type conditions as `reject`." args = drop_unset( { "url": url, @@ -1289,6 +1361,7 @@ async def markdown( timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, ) -> MarkdownResponse: + "Returns the page's content converted to Markdown. The HTML is fetched from\nthe page (optionally scoped to a selector) and converted server-side, with\nscript/style/noscript/iframe nodes stripped." args = drop_unset( { "selector": selector, @@ -1310,6 +1383,7 @@ async def close_live_url( self, live_url_id: str, ) -> CloseLiveURLResponse: + "Closes a Live URL created by this browser session." args = drop_unset( { "liveURLId": live_url_id, diff --git a/src/browserless_bap/browser.py b/src/browserless_bap/browser.py index b8b5209..b109060 100644 --- a/src/browserless_bap/browser.py +++ b/src/browserless_bap/browser.py @@ -8,7 +8,7 @@ from contextlib import asynccontextmanager from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from .errors import ValueError +from .errors import BAPValueError from .page import Page from .transport import Transport, TransportFactory @@ -28,7 +28,7 @@ def __init__( connect_timeout: float | None = None, ) -> None: if browser_ws_endpoint is None and transport is None: - raise ValueError( + raise BAPValueError( "Either `browser_ws_endpoint` or `transport` must be provided" ) self._base_url = browser_ws_endpoint @@ -43,7 +43,7 @@ async def new_page(self) -> Page: """Open a new page with its own BAP WebSocket.""" if self._closed: - raise ValueError("Browser is closed") + raise BAPValueError("Browser is closed") self._pages = [page for page in self._pages if not page.closed] transport = Transport( _with_token(self._base_url, self._token), @@ -58,7 +58,7 @@ async def new_page(self) -> Page: raise if self._closed: await transport.close() - raise ValueError("Browser is closed") + raise BAPValueError("Browser is closed") page = Page(transport) self._pages.append(page) return page diff --git a/src/browserless_bap/element_handle.py b/src/browserless_bap/element_handle.py index 82c7ac5..2466eea 100644 --- a/src/browserless_bap/element_handle.py +++ b/src/browserless_bap/element_handle.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Unpack from urllib.parse import quote +from .errors import BAPValueError from .types.options import ClickOptions, QuerySelectorOptions, TypeOptions from .types.responses import ClickResponse, HoverResponse, SelectResponse, TypeResponse @@ -46,7 +47,16 @@ def __init__( @property def inner_html(self) -> str | None: - return self._properties.get("innerHTML") + """Element HTML. Requires html=True on the query that produced this handle.""" + return self._optional_html("innerHTML", "inner_html") + + def _optional_html(self, key: str, attribute: str) -> str | None: + if key not in self._properties: + raise BAPValueError( + f"{attribute} was not requested; pass html=True to the query " + "that produced this handle" + ) + return self._properties.get(key) @property def inner_text(self) -> str | None: @@ -66,7 +76,8 @@ def local_name(self) -> str | None: @property def outer_html(self) -> str | None: - return self._properties.get("outerHTML") + """Element markup. Requires html=True on the query that produced this handle.""" + return self._optional_html("outerHTML", "outer_html") @property def child_element_count(self) -> int | None: diff --git a/src/browserless_bap/errors.py b/src/browserless_bap/errors.py index f615338..240ad23 100644 --- a/src/browserless_bap/errors.py +++ b/src/browserless_bap/errors.py @@ -8,6 +8,16 @@ ErrorCode = Literal["BQL", "CONNECTION", "TIMEOUT", "CLIENT"] +__all__ = [ + "BAPConnectionError", + "BAPError", + "BAPTimeoutError", + "BAPValueError", + "BrowserQLError", + "ErrorCode", + "SchemaMismatchError", +] + class BAPError(Exception): """Base class for errors produced by the SDK.""" @@ -32,13 +42,13 @@ def __init__(self, errors: Sequence[object]) -> None: super().__init__(message or "Unknown BrowserQL error") -class ConnectionError(BAPError, builtins.ConnectionError): +class BAPConnectionError(BAPError, builtins.ConnectionError): """The BAP transport could not connect or disconnected.""" code = "CONNECTION" -class TimeoutError(BAPError, builtins.TimeoutError): +class BAPTimeoutError(BAPError, builtins.TimeoutError): """A BAP operation exceeded its transport deadline.""" code = "TIMEOUT" @@ -57,12 +67,7 @@ class SchemaMismatchError(BAPError): code = "CLIENT" -class ValueError(BAPError, builtins.ValueError): +class BAPValueError(BAPError, builtins.ValueError): """A public SDK argument failed client-side validation.""" code = "CLIENT" - - -BAPConnectionError = ConnectionError -BAPTimeoutError = TimeoutError -BAPValueError = ValueError diff --git a/src/browserless_bap/graphql/__init__.py b/src/browserless_bap/graphql/__init__.py index 545adff..975cef5 100644 --- a/src/browserless_bap/graphql/__init__.py +++ b/src/browserless_bap/graphql/__init__.py @@ -1,11 +1,13 @@ -from .builder import build_operation -from .mutations import ARG_TYPES, RESPONSE_FIELDS +from .builder import build_operation, selection_for +from .mutations import ARG_TYPES, OPTIONAL_FIELDS, RESPONSE_FIELDS from .subscriptions import SUBSCRIPTIONS, SubscriptionSpec __all__ = [ "ARG_TYPES", + "OPTIONAL_FIELDS", "RESPONSE_FIELDS", "SUBSCRIPTIONS", "SubscriptionSpec", "build_operation", + "selection_for", ] diff --git a/src/browserless_bap/graphql/builder.py b/src/browserless_bap/graphql/builder.py index 390af0f..e96d5cb 100644 --- a/src/browserless_bap/graphql/builder.py +++ b/src/browserless_bap/graphql/builder.py @@ -2,10 +2,27 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import Literal from ..errors import SchemaMismatchError +from .mutations import OPTIONAL_FIELDS, RESPONSE_FIELDS + + +def selection_for(response_type: str, opt_in: Iterable[str] = ()) -> str: + base = RESPONSE_FIELDS.get(response_type) + if base is None: + raise SchemaMismatchError(f"Unknown response type: {response_type}") + selected: dict[str, None] = {} + for group in opt_in: + fields = OPTIONAL_FIELDS.get(response_type, {}).get(group) + if fields is None: + raise SchemaMismatchError( + f'Unknown optional field group "{group}" ' + f"for response type {response_type}" + ) + selected.update(dict.fromkeys(fields.split())) + return " ".join((base, *selected)) def build_operation( @@ -34,4 +51,4 @@ def build_operation( return query, provided -__all__ = ["build_operation"] +__all__ = ["build_operation", "selection_for"] diff --git a/src/browserless_bap/graphql/mutations.py b/src/browserless_bap/graphql/mutations.py index 5800b45..b900ead 100644 --- a/src/browserless_bap/graphql/mutations.py +++ b/src/browserless_bap/graphql/mutations.py @@ -20,7 +20,6 @@ "HTTPHeaders": "name value", "HTTPHeadersResponse": "headers { name value } time", "HTTPResponse": "status time text url", - "HTTPResponseWithRemoteMetadata": "ip port status time text url", "HoverResponse": "selector time x y", "JavaScriptResponse": "enabled time", "LiveURLCompleteEvent": "liveURLId reason", @@ -30,13 +29,12 @@ "MarkdownResponse": "markdown time", "PDFResponse": "base64 size time", "ProxyResponse": "time", - "QuerySelectorResponse": "childElementCount className id innerHTML innerText localName outerHTML", + "QuerySelectorResponse": "childElementCount className id innerText localName", "ReconnectionResponse": "browserQLEndpoint browserWSEndpoint devtoolsFrontendUrl webSocketDebuggerUrl", "RejectResponse": "time enabled", "RequestHeader": "name value", "RequestResponse": "url method type headers { name value }", - "ResponseResponse": "status url method type headers { name value } body", - "ResponseResponseWithBodyEncoding": "status url method type headers { name value } body base64Encoded", + "ResponseResponse": "status url method type headers { name value }", "ScreenshotResponse": "base64 format time", "ScrollResponse": "selector time x y", "SelectResponse": "selector time value", @@ -57,6 +55,19 @@ "WaitForTimeoutResponse": "time" } +OPTIONAL_FIELDS: dict[str, dict[str, str]] = { + "HTTPResponse": { + "remoteMetadata": "ip port" + }, + "QuerySelectorResponse": { + "html": "innerHTML outerHTML" + }, + "ResponseResponse": { + "body": "body", + "bodyEncoding": "body base64Encoded" + } +} + ARG_TYPES: dict[str, dict[str, str]] = { "addScriptTag": { "content": "String", diff --git a/src/browserless_bap/page.py b/src/browserless_bap/page.py index a56c18c..dc868ee 100644 --- a/src/browserless_bap/page.py +++ b/src/browserless_bap/page.py @@ -18,10 +18,9 @@ from ._page_generated import PageGenerated from ._unset import UNSET, UnsetType from .emitter import EventEmitter, Listener -from .errors import ConnectionError, SchemaMismatchError, ValueError -from .errors import TimeoutError as BAPTimeoutError -from .graphql.builder import build_operation -from .graphql.mutations import ARG_TYPES, RESPONSE_FIELDS +from .errors import BAPConnectionError, BAPTimeoutError, BAPValueError +from .graphql.builder import build_operation, selection_for +from .graphql.mutations import ARG_TYPES from .graphql.subscriptions import SUBSCRIPTIONS from .transport import ( OPERATION_TIMEOUT_CUSHION_MS, @@ -138,13 +137,14 @@ async def _execute( response_type: str, *, timeout: float | UnsetType | None = None, + opt_in: Sequence[str] = (), ) -> Any: if mutation_name != "liveURL": return await self._execute_operation( - mutation_name, args, response_type, timeout=timeout + mutation_name, args, response_type, timeout=timeout, opt_in=opt_in ) if self._minting: - raise ValueError( + raise BAPValueError( "Await the previous live_url() call before minting another link" ) self._minting = True @@ -160,13 +160,13 @@ async def _execute( self._on_listener_count_change("liveComplete", 1) await self._wait_subscription_started("liveComplete") if "liveComplete" not in self._unsubscribes: - raise ConnectionError("Live completion subscription unavailable") + raise BAPConnectionError("Live completion subscription unavailable") result = await self._execute_operation( - mutation_name, args, response_type, timeout=timeout + mutation_name, args, response_type, timeout=timeout, opt_in=opt_in ) state.id = result.get("liveURLId") if isinstance(result, Mapping) else None if not state.id: - raise ValueError("live_url returned no liveURLId") + raise BAPValueError("live_url returned no liveURLId") if state.id in state.early and not state.future.done(): state.future.set_result(state.early[state.id]) return result @@ -175,7 +175,7 @@ async def _execute( state.future.set_exception( error if isinstance(error, Exception) - else ConnectionError("live_url() was cancelled or interrupted") + else BAPConnectionError("live_url() was cancelled or interrupted") ) raise finally: @@ -192,10 +192,9 @@ async def _execute_operation( response_type: str, *, timeout: float | UnsetType | None = None, + opt_in: Sequence[str] = (), ) -> Any: - fields = RESPONSE_FIELDS.get(response_type) - if fields is None: - raise SchemaMismatchError(f"Unknown response type: {response_type}") + fields = selection_for(response_type, opt_in) operation = build_operation( mutation_name, args, @@ -227,10 +226,10 @@ async def wait_for_live_complete( Completion or failure is retained until the next live_url() call. """ if self.closed: - raise ConnectionError("Page is closed") + raise BAPConnectionError("Page is closed") deadline = 0 if timeout is None else timeout if not math.isfinite(deadline) or deadline < 0: - raise ValueError("timeout must be a finite non-negative number") + raise BAPValueError("timeout must be a finite non-negative number") if self._live is None or ( self._live.rejected and not self._live.id and not self._minting ): @@ -308,6 +307,7 @@ async def goto( referrer_policy: str | UnsetType = UNSET, remote_response_metadata: bool | UnsetType = UNSET, ) -> HTTPResponse | None: + """Navigate to a URL with optional waitUntil and timeout parameters.""" args = _drop_unset( { "url": url, @@ -318,14 +318,15 @@ async def goto( "referrerPolicy": referrer_policy, } ) - response_type = ( - "HTTPResponseWithRemoteMetadata" - if remote_response_metadata is True - else "HTTPResponse" - ) return cast( HTTPResponse | None, - await self._execute("goto", args, response_type, timeout=timeout), + await self._execute( + "goto", + args, + "HTTPResponse", + timeout=timeout, + opt_in=("remoteMetadata",) if remote_response_metadata is True else (), + ), ) async def response( @@ -338,8 +339,13 @@ async def response( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, wait: bool | UnsetType = UNSET, + body: bool | UnsetType = UNSET, body_encoding: bool | UnsetType = UNSET, ) -> list[ResponseResponse]: + """List responses; fetch bodies only with body=True. + + body_encoding=True implies body=True and adds the base64Encoded flag. + """ args = _drop_unset( { "status": status, @@ -351,12 +357,16 @@ async def response( "wait": wait, } ) - response_type = ( - "ResponseResponseWithBodyEncoding" + opt_in = ( + ("bodyEncoding",) if body_encoding is True - else "ResponseResponse" + else ("body",) + if body is True + else () + ) + result = await self._execute( + "response", args, "ResponseResponse", timeout=timeout, opt_in=opt_in ) - result = await self._execute("response", args, response_type, timeout=timeout) return cast(list[ResponseResponse], list(result or ())) async def select_option( @@ -582,7 +592,7 @@ async def close(self) -> None: if self._closed_flag: return self._closed_flag = True - self._fail_live(ConnectionError("Page is closed")) + self._fail_live(BAPConnectionError("Page is closed")) for ready in self._subscription_ready.values(): ready.set() self._detach_transport() @@ -682,7 +692,7 @@ def on_error(error: Exception) -> None: def on_complete() -> None: clear_if_own() if event == "liveComplete": - self._fail_live(ConnectionError("Live completion stream ended")) + self._fail_live(BAPConnectionError("Live completion stream ended")) try: spec = SUBSCRIPTIONS[event] @@ -772,6 +782,6 @@ def _require_one_source( ) -> None: values = (url, path, content) if sum(value is not UNSET for value in values) != 1: - raise ValueError( + raise BAPValueError( "Exactly one of `url`, `path`, or `content` must be specified." ) diff --git a/src/browserless_bap/sync_api/_bridge.py b/src/browserless_bap/sync_api/_bridge.py index ac23a80..2392f6a 100644 --- a/src/browserless_bap/sync_api/_bridge.py +++ b/src/browserless_bap/sync_api/_bridge.py @@ -19,6 +19,8 @@ from contextlib import suppress from typing import Protocol, TypeVar, cast +from ..errors import BAPTimeoutError + T = TypeVar("T") logger = logging.getLogger("browserless_bap") @@ -247,8 +249,8 @@ def run(self, awaitable: Awaitable[T], *, timeout: float | None = None) -> T: remaining = deadline - time.monotonic() if remaining <= 0: future.cancel() - raise TimeoutError( - f"bap sync bridge timed out after {timeout} seconds" + raise BAPTimeoutError( + "sync bridge", cast(float, timeout) * 1000 ) wait_for = min(wait_for, remaining) try: diff --git a/src/browserless_bap/sync_api/_page_generated.py b/src/browserless_bap/sync_api/_page_generated.py index b012524..aa0909a 100644 --- a/src/browserless_bap/sync_api/_page_generated.py +++ b/src/browserless_bap/sync_api/_page_generated.py @@ -28,6 +28,7 @@ def authenticate( *, origin: str | UnsetType = UNSET, ) -> HTTPResponse | None: + "Provides credentials for HTTP authentication (RFC 7235). When set, any HTTP\nauthentication challenge (401/407) will be answered with the supplied\ncredentials automatically. If an `origin` is provided, only challenges whose\norigin matches will use these credentials; otherwise every challenge is\nanswered (Puppeteer-style)." return cast( "HTTPResponse | None", self._invoke( @@ -44,6 +45,7 @@ def go_back( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Goes back in browser history, optionally accepting waitUntil and timeout arguments. Returns null if no back is possible" return cast( "HTTPResponse | None", self._invoke( @@ -62,6 +64,7 @@ def check( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Sets or un-sets the value of a checkbox on the page" return cast( "ClickResponse", self._invoke( @@ -83,6 +86,7 @@ def uncheck( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Sets or un-sets the value of a checkbox on the page" return cast( "ClickResponse", self._invoke( @@ -104,6 +108,7 @@ def click( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Waits for the element to be visible, scrolls to it, then clicks on it with native events" return cast( "ClickResponse", self._invoke( @@ -124,6 +129,7 @@ def set_content( wait_until: WaitUntilHistory | UnsetType = UNSET, wait_until_all: list[WaitUntilHistory] | UnsetType = UNSET, ) -> HTTPResponse | None: + "Sets the given HTML content on the page with an optional waitUntil parameter" return cast( "HTTPResponse | None", self._invoke( @@ -138,6 +144,7 @@ def set_content( def cookies( self, ) -> list[StandardCookie]: + "Sets and gets cookies on the page" return cast( "list[StandardCookie]", self._invoke( @@ -151,6 +158,7 @@ def go_forward( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Goes forward in browser history, optionally accepting waitUntil and timeout arguments. Returns null if no forward is possible" return cast( "HTTPResponse | None", self._invoke( @@ -171,6 +179,7 @@ def goto( referrer_policy: str | UnsetType = UNSET, remote_response_metadata: bool | UnsetType = UNSET, ) -> HTTPResponse | None: + "Navigates to a URL with an optional waitUntil parameter and timeout parameter" return cast( "HTTPResponse | None", self._invoke( @@ -194,6 +203,7 @@ def html( clean: CleanInput | UnsetType = UNSET, rewrite: HTMLRewriteInput | UnsetType = UNSET, ) -> HTMLResponse: + 'Returns the HTML content of the page or selector when specified. This API can also "clean" HTML markup returned by specifying a "clean" argument with numerous options. Features of the "clean" argument include removal of non-text nodes, removal of DOM attributes, as well as removal of excessive whitespace and newlines. Using "clean" can save nearly 1,000 times the payload size. Useful for LLM\'s and other scenarios' return cast( "HTMLResponse", self._invoke( @@ -209,6 +219,7 @@ def html( def content( self, ) -> str: + 'Returns the HTML content of the page or selector when specified. This API can also "clean" HTML markup returned by specifying a "clean" argument with numerous options. Features of the "clean" argument include removal of non-text nodes, removal of DOM attributes, as well as removal of excessive whitespace and newlines. Using "clean" can save nearly 1,000 times the payload size. Useful for LLM\'s and other scenarios' return cast( "str", self._invoke( @@ -220,6 +231,7 @@ def set_user_agent( self, user_agent: str, ) -> UserAgentResponse: + "Sets the User-Agent string for the browser session" return cast( "UserAgentResponse", self._invoke( @@ -239,6 +251,7 @@ def hover( x: float | UnsetType = UNSET, y: float | UnsetType = UNSET, ) -> HoverResponse: + "Waits for the element to be visible, scrolls to it, then hover on it with native events" return cast( "HoverResponse", self._invoke( @@ -257,6 +270,7 @@ def set_java_script_enabled( self, enabled: bool, ) -> JavaScriptResponse: + "Sets and gets JavaScript execution on the page\n\n**Note: changing this value won't affect scripts that have already been run. It will take full effect on the next navigation.**" return cast( "JavaScriptResponse", self._invoke( @@ -287,6 +301,7 @@ def live_url( fail_label: str | UnsetType = UNSET, requester_name: str | UnsetType = UNSET, ) -> LiveURLResponse: + "Returns a fully-qualified, user-shareable live-URL for streaming the web-browser to an end-user, optionally interactive.\n\n**Set 30 second timer for interaction**" return cast( "LiveURLResponse", self._invoke( @@ -319,6 +334,7 @@ def map_selector( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> list[MapSelectorResponse]: + 'Specify a selector that returns multiple nodes in a document (similar to `document.querySelectorAll`), or JavaScript that returns a NodeList, and this API will respond with details about those DOM nodes. Similar to how "map" works in most functional programming languages and libraries. Useful for mapping over repetitive data in sites and pages like product listings or search results. This will automatically wait for the selector to be present on the page, and is configurable with the "wait" and "timeout" options.\n\nFor getting arbitrary DOM attributes back you can specify them via the `attribute(name: "data-custom-attribute")` property. This will return an object with `name` and `value` properties.\n\nYou may also continuously map further nested items as well, for instance this query might get all books on a page, and then a nested `mapSelector` call might list all sellers of that book, or shipping speeds. Hierarchy of data is preserved to pass through the hierarchical data modeled inside the DOM.\n\nThis API will always return a list of results back regardless if one or more items are found, or `null` if none are found.\n\nUsing aliases can also give the returned JSON more meaning and better model the data returned by this powerful API.' return cast( "list[MapSelectorResponse]", self._invoke( @@ -336,6 +352,7 @@ def eval_on_selector_all( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> list[MapSelectorResponse]: + "Return details (`innerHTML`, `innerText`, `id`, `class`) for every element matching `selector`. Unlike Playwright's `eval_on_selector_all`, this takes no page function: it is a shortcut for `map_selector(selector)`." return cast( "list[MapSelectorResponse]", self._invoke( @@ -374,6 +391,7 @@ def pdf( transfer_mode: str | UnsetType = UNSET, path: str | UnsetType = UNSET, ) -> bytes: + "Generates a PDF of the page with the print CSS media type" return cast( "bytes", self._invoke( @@ -410,6 +428,7 @@ def preferences( *, timeout: float | UnsetType = UNSET, ) -> DefaultResponse: + "Sets configuration for the entirety of the session, replacing defaults like the 30 second timeout default" return cast( "DefaultResponse", self._invoke( @@ -433,6 +452,7 @@ def proxy( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, ) -> ProxyResponse: + 'Proxies requests, by a specified set of conditions, through either the Browserless residential proxy or through an external proxy. Only requests that match these conditions are proxied and the rest are sent from the instance\'s own IP address.\n\nUse the "server" argument to specify an external proxy for Browserless to use for requests. For requests with authentication, the username and password should be included in the URL using Basic Authentication. See the examples below for more information on how to format those URLs.\n\n**Using the browserless proxy for all requests and proxy through Brazil**' return cast( "ProxyResponse", self._invoke( @@ -457,7 +477,9 @@ def query_selector( *, timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, + html: bool | UnsetType = UNSET, ) -> ElementHandle | None: + "Return a handle for the first element matching `selector`, or `None`. The element's `innerHTML`/`outerHTML` are fetched only with `html=True`." return cast( "ElementHandle | None", self._invoke( @@ -465,6 +487,7 @@ def query_selector( selector, timeout=timeout, visible=visible, + html=html, ), ) @@ -474,7 +497,9 @@ def query_selector_all( *, timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, + html: bool | UnsetType = UNSET, ) -> list[ElementHandle]: + "Return handles for every element matching `selector`. The elements' `innerHTML`/`outerHTML` are fetched only with `html=True`." return cast( "list[ElementHandle]", self._invoke( @@ -482,6 +507,7 @@ def query_selector_all( selector, timeout=timeout, visible=visible, + html=html, ), ) @@ -490,6 +516,7 @@ def reconnect( *, timeout: float | UnsetType = UNSET, ) -> ReconnectionResponse: + "Returns a payload with reconnection information in order to reconnect back to the same browser session" return cast( "ReconnectionResponse", self._invoke( @@ -501,6 +528,7 @@ def reconnect( def stop_session_recording( self, ) -> StopSessionRecordingResponse: + "Stops the current session recording and processes the replay data, similar to closing the context. This mutation will stop RRWeb recording and upload the collected events." return cast( "StopSessionRecordingResponse", self._invoke( @@ -518,6 +546,7 @@ def reject( url: list[str] | UnsetType = UNSET, regex: list[str] | UnsetType = UNSET, ) -> RejectResponse: + 'Rejects requests by a specified URL pattern, method, or type and operator. You may supply a single pattern, or a list (array) of them. This mutation, by default, will reject any requests that match *any* pattern, which we call an "or" operator. To reject requests where conditions must all match, specify an "and" operator in the mutation. Note that this only has an effect when the query is executing, so scripts that return quickly will likely see assets loading in the editor as these rejections only happen when mutations are executing.\n___\n**Rejecting images or media**' return cast( "RejectResponse", self._invoke( @@ -537,6 +566,7 @@ def reload( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Reloads the given page with an optional waitUntil parameter and timeout parameter" return cast( "HTTPResponse | None", self._invoke( @@ -556,6 +586,7 @@ def request( wait: bool | UnsetType = UNSET, operator: OperatorType | UnsetType = UNSET, ) -> list[RequestResponse]: + 'Returns request information made by the Browser with optional filters via arguments. You may filter the returned results by a glob-like URL-pattern, the method of the request or the type of request. Applying an operator to this will then change the behavior by either "and"ing the filters together or "or"ing them. This API will automatically wait for the request to be made if none is immediately found which you can turn off by disabling the "wait" option.\n\n**Getting all "Document" requests**' return cast( "list[RequestResponse]", self._invoke( @@ -579,8 +610,10 @@ def response( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, wait: bool | UnsetType = UNSET, + body: bool | UnsetType = UNSET, body_encoding: bool | UnsetType = UNSET, ) -> list[ResponseResponse]: + 'Returns response information, filtered by the provided arguments, made by the browser. You may optionally filter the returned results by a glob-like URL-pattern, the Method of the response or the Type of response. Applying an operator to this will then change the behavior by either "and"ing the filters together, or "or"ing them. This API will automatically wait for the response to be made if none is immediately found which you can turn off by disabling the "wait" option.\n\n**Getting all "Document" responses**' return cast( "list[ResponseResponse]", self._invoke( @@ -592,6 +625,7 @@ def response( type=type, url=url, wait=wait, + body=body, body_encoding=body_encoding, ), ) @@ -612,6 +646,7 @@ def screenshot( wait_for_images: bool | UnsetType = UNSET, path: str | UnsetType = UNSET, ) -> bytes: + "Screenshots the page or a specific selector" return cast( "bytes", self._invoke( @@ -642,6 +677,7 @@ def scroll( x: float | UnsetType = UNSET, y: float | UnsetType = UNSET, ) -> ScrollResponse: + "Waits for a selector, then scrolls to it on the page or an x,y coordinate in pixels" return cast( "ScrollResponse", self._invoke( @@ -663,6 +699,7 @@ def solve( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> CaptchaResponse: + "🚨 **EXPERIMENTAL** 🚨\nSolves a captcha or other challenge. Can auto-detect the CAPTCHA type or solve a specific type.\nUses the same detection logic as BaaS to automatically identify the CAPTCHA type\nand routes to the appropriate solver. Optionally accepts a specific type to solve." return cast( "CaptchaResponse", self._invoke( @@ -680,6 +717,7 @@ def solve_image_captcha( input_selector: str, timeout: float | UnsetType = UNSET, ) -> CaptchaResponse: + "Solves an image captcha using caller-provided selectors. Unlike solve, which auto-detects known captcha types, this mutation works on any site by accepting custom captchaSelector and inputSelector arguments. It detects the captcha image, solves it, and fills the result into the specified input field." return cast( "CaptchaResponse", self._invoke( @@ -698,6 +736,7 @@ def text( visible: bool | UnsetType = UNSET, clean: CleanInput | UnsetType = UNSET, ) -> TextResponse: + "Returns the text content on the given page or by selector when specified" return cast( "TextResponse", self._invoke( @@ -713,6 +752,7 @@ def eval_on_selector( self, selector: str, ) -> str: + "Return the text content of the first element matching `selector`. Unlike Playwright's `eval_on_selector`, this takes no page function: it is a shortcut for `text(selector=...)`." return cast( "str", self._invoke( @@ -724,6 +764,7 @@ def eval_on_selector( def title( self, ) -> str: + "Returns the title of the page that the browser is currently at" return cast( "str", self._invoke( @@ -743,6 +784,7 @@ def type( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> TypeResponse: + "Types text into an element by scrolling to it, clicking it, then emitting key events for every character. Include a `\\n` (or `\\r`) in `text` to send an Enter/Return key press — useful for submitting search boxes or forms without a separate click. BrowserQL has no standalone key-press mutation, so non-Enter keys cannot be simulated; use `click` for non-keyboard submit interactions (for example, a submit button)." return cast( "TypeResponse", self._invoke( @@ -764,6 +806,7 @@ def load_secret( *, selector: str | UnsetType = UNSET, ) -> LoadSecretResponse: + "Fills a credential resolved just-in-time from a configured 1Password\nintegration into a target input. The resolved value never appears in your\nquery or in the response. The session must be started with an integration\n(the `integrationId` connection parameter or session field), and the page's\norigin must be in the integration's allowed-domain list. Once any secret has\nbeen filled, capture and page-content readback are disabled for the rest of the\nsession — screenshots, PDFs, screencasts, Live URLs, session recording, and the\ncontent-return mutations (evaluate, html, text, querySelector, querySelectorAll,\ncookies)." return cast( "LoadSecretResponse", self._invoke( @@ -776,6 +819,7 @@ def load_secret( def url( self, ) -> str: + "Returns the URL of the page that the browser is currently at" return cast( "str", self._invoke( @@ -789,6 +833,7 @@ def wait_for_navigation( timeout: float | UnsetType = UNSET, wait_until: WaitUntilGoto | UnsetType = UNSET, ) -> HTTPResponse: + "Waits for a navigation even to fire, useful for clicking an element and waiting for a page load of some" return cast( "HTTPResponse", self._invoke( @@ -805,6 +850,7 @@ def wait_for_network_idle( concurrency: int | UnsetType = UNSET, timeout: float | UnsetType = UNSET, ) -> WaitForTimeoutResponse: + "Wait for the page to have no more than the configured number of in-flight network requests for a period of time." return cast( "WaitForTimeoutResponse", self._invoke( @@ -823,6 +869,7 @@ def wait_for_selector( visible: bool | UnsetType = UNSET, hidden: bool | UnsetType = UNSET, ) -> WaitForSelectorResponse: + "Waits for a given selector to be present in the DOM, with optional visibility" return cast( "WaitForSelectorResponse", self._invoke( @@ -840,6 +887,7 @@ def wait_for_event( *, timeout: float | UnsetType = UNSET, ) -> WaitForEvent: + "Waits for a named event to fire on the page's document or window." return cast( "WaitForEvent", self._invoke( @@ -872,6 +920,7 @@ def set_viewport_size( has_touch: bool | UnsetType = UNSET, landscape: bool | UnsetType = UNSET, ) -> ViewportResponse: + "Sets the viewport dimensions for the browser session" return cast( "ViewportResponse", self._invoke( @@ -894,6 +943,7 @@ def switch_to_window( timeout: float | UnsetType = UNSET, wait_until: WaitUntilGoto | UnsetType = UNSET, ) -> SwitchWindowResponse: + "Switches context to a popup window matching specified criteria" return cast( "SwitchWindowResponse", self._invoke( @@ -910,6 +960,7 @@ def emulate_media( self, type: MediaType, ) -> EmulateMediaTypeResponse: + 'Emulates the given CSS media type for the page, mirroring the behavior of\nprinting ("print") or on-screen rendering ("screen"). Useful before a `pdf`\nor `screenshot` to control how the page renders.' return cast( "EmulateMediaTypeResponse", self._invoke( @@ -932,6 +983,7 @@ def fulfill( base64_body: str | UnsetType = UNSET, headers: list[HeaderInput] | UnsetType = UNSET, ) -> FulfillResponse: + "Fulfills matching requests with a canned response instead of letting them hit\nthe network, mirroring the request-interceptor mocking used by the REST APIs.\nMatching follows the same glob-style URL/method/type conditions as `reject`." return cast( "FulfillResponse", self._invoke( @@ -956,6 +1008,7 @@ def markdown( timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, ) -> MarkdownResponse: + "Returns the page's content converted to Markdown. The HTML is fetched from\nthe page (optionally scoped to a selector) and converted server-side, with\nscript/style/noscript/iframe nodes stripped." return cast( "MarkdownResponse", self._invoke( @@ -970,6 +1023,7 @@ def close_live_url( self, live_url_id: str, ) -> CloseLiveURLResponse: + "Closes a Live URL created by this browser session." return cast( "CloseLiveURLResponse", self._invoke( diff --git a/src/browserless_bap/sync_api/browserless.py b/src/browserless_bap/sync_api/browserless.py index 69bf68a..1625802 100644 --- a/src/browserless_bap/sync_api/browserless.py +++ b/src/browserless_bap/sync_api/browserless.py @@ -2,16 +2,17 @@ from __future__ import annotations -from typing import Any, Never - from browserless_bap.browserless import Browserless as AsyncBrowserless +from browserless_bap.transport import TransportFactory from ._bridge import get_loop_thread from .browser import Browser class Browserless: - def __new__(cls, *args: object, **kwargs: object) -> Never: + """Factory namespace for blocking browser sessions.""" + + def __new__(cls) -> Browserless: raise TypeError("Browserless is a namespace; call Browserless.connect(...)") @staticmethod @@ -20,9 +21,11 @@ def connect( browser_ws_endpoint: str | None = None, token: str | None = None, timeout: float = 30_000, - transport: Any = None, + transport: TransportFactory | None = None, connect_timeout: float | None = None, ) -> Browser: + """Create a blocking browser for a BAP-compatible WebSocket endpoint.""" + bridge = get_loop_thread() impl = AsyncBrowserless.connect( browser_ws_endpoint=browser_ws_endpoint, diff --git a/src/browserless_bap/sync_api/page.py b/src/browserless_bap/sync_api/page.py index f8472ee..02ec3b0 100644 --- a/src/browserless_bap/sync_api/page.py +++ b/src/browserless_bap/sync_api/page.py @@ -176,6 +176,7 @@ def goto( referrer_policy: str | UnsetType = UNSET, remote_response_metadata: bool | UnsetType = UNSET, ) -> HTTPResponse | None: + """Navigate to a URL with optional waitUntil and timeout parameters.""" return cast( HTTPResponse | None, self._invoke( @@ -200,8 +201,13 @@ def response( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, wait: bool | UnsetType = UNSET, + body: bool | UnsetType = UNSET, body_encoding: bool | UnsetType = UNSET, ) -> list[ResponseResponse]: + """List responses; fetch bodies only with body=True. + + body_encoding=True implies body=True and adds the base64Encoded flag. + """ return cast( list[ResponseResponse], self._invoke( @@ -213,6 +219,7 @@ def response( type=type, url=url, wait=wait, + body=body, body_encoding=body_encoding, ), ) diff --git a/src/browserless_bap/transport.py b/src/browserless_bap/transport.py index b2ac4fc..6a8cbfa 100644 --- a/src/browserless_bap/transport.py +++ b/src/browserless_bap/transport.py @@ -20,7 +20,7 @@ from websockets.exceptions import ConnectionClosed, ConnectionClosedOK from websockets.typing import Subprotocol -from .errors import BrowserQLError, ConnectionError, TimeoutError +from .errors import BAPConnectionError, BAPTimeoutError, BrowserQLError try: from ._version import __version__ @@ -185,7 +185,7 @@ def connect(self) -> Awaitable[None]: async def _connect(self) -> None: if self._closed: - raise ConnectionError("Transport is closed") + raise BAPConnectionError("Transport is closed") deadline = self.operation_timeout(self._connect_timeout) connection: ConnectionTransport | None = None cm = asyncio.timeout(deadline / 1000 if deadline > 0 else None) @@ -213,18 +213,18 @@ async def _connect(self) -> None: return except builtins.TimeoutError as exc: error: Exception = ( - TimeoutError("connect", deadline) + BAPTimeoutError("connect", deadline) if cm.expired() - else ConnectionError( + else BAPConnectionError( f"WebSocket connection failed: {_safe_error_message(exc)}" ) ) - except ConnectionError as exc: - error = ConnectionError(_safe_error_message(exc)) + except BAPConnectionError as exc: + error = BAPConnectionError(_safe_error_message(exc)) except asyncio.CancelledError: raise except BaseException as exc: - error = ConnectionError( + error = BAPConnectionError( f"WebSocket connection failed: {_safe_error_message(exc)}" ) finally: @@ -239,7 +239,7 @@ async def _connect(self) -> None: async def _dial(self) -> ConnectionTransport: if self._url is None: - raise ConnectionError( + raise BAPConnectionError( "A browser_ws_endpoint is required when no transport is provided" ) try: @@ -260,7 +260,7 @@ async def _dial(self) -> ConnectionTransport: message = f"WebSocket connection failed: {_safe_error_message(exc)}" else: return _WebSocketConnection(socket) - raise ConnectionError(message) + raise BAPConnectionError(message) def operation_timeout(self, timeout: float | None, cushion: float = 0) -> float: resolved = self._default_timeout if timeout is None else timeout @@ -303,7 +303,7 @@ async def send( {"id": operation_id, "type": "subscribe", "payload": payload} ) return await future - except TimeoutError: + except BAPTimeoutError: await self._complete_safely(operation_id) raise except asyncio.CancelledError: @@ -361,7 +361,7 @@ async def close(self) -> None: ) with suppress(BaseException): await _maybe_await(self._connection.close()) - self._fail_all(ConnectionError("WebSocket connection closed")) + self._fail_all(BAPConnectionError("WebSocket connection closed")) self._connected = False for task in tuple(self._tasks): task.cancel() @@ -391,17 +391,17 @@ def _stop(self, operation_id: str) -> None: def _ensure_connected(self) -> None: if self._closed or not self._connected or self._connection is None: - raise ConnectionError("WebSocket is not connected") + raise BAPConnectionError("WebSocket is not connected") async def _send_frame(self, frame: Mapping[str, Any]) -> None: connection = self._connection if connection is None: - raise ConnectionError("WebSocket is not connected") + raise BAPConnectionError("WebSocket is not connected") message = json.dumps(frame, separators=(",", ":")) try: async with self._send_lock: await _maybe_await(connection.send(message)) - except ConnectionError: + except BAPConnectionError: raise except asyncio.CancelledError: raise @@ -409,7 +409,7 @@ async def _send_frame(self, frame: Mapping[str, Any]) -> None: message = f"WebSocket send failed: {_safe_error_message(exc)}" else: return - raise ConnectionError(message) + raise BAPConnectionError(message) def _on_frame(self, message: str) -> None: try: @@ -483,13 +483,15 @@ def _on_close(self, _event: object | None = None) -> None: self._connected = False if not self._closing: self._closed = True - self._fail_all(ConnectionError("WebSocket connection closed unexpectedly")) + self._fail_all( + BAPConnectionError("WebSocket connection closed unexpectedly") + ) def _on_error(self, error: object) -> None: self._connected = False self._closed = True self._fail_all( - ConnectionError(f"WebSocket error: {_safe_error_message(error)}") + BAPConnectionError(f"WebSocket error: {_safe_error_message(error)}") ) def _fail_all(self, error: Exception) -> None: @@ -536,12 +538,12 @@ def _expire( timeout_ms: float, ) -> None: if not future.done(): - future.set_exception(TimeoutError(query[:50], timeout_ms)) + future.set_exception(BAPTimeoutError(query[:50], timeout_ms)) async def _send_complete(self, operation_id: str) -> None: if self._closed or self._connection is None: return - with suppress(ConnectionError): + with suppress(BAPConnectionError): await self._send_frame({"id": operation_id, "type": "complete"}) async def _complete_safely(self, operation_id: str) -> None: diff --git a/src/browserless_bap/types/options.py b/src/browserless_bap/types/options.py index 9350342..f9bdfd7 100644 --- a/src/browserless_bap/types/options.py +++ b/src/browserless_bap/types/options.py @@ -154,6 +154,7 @@ class ProxyOptions(TypedDict, total=False): class QuerySelectorOptions(TypedDict, total=False): timeout: NotRequired[float] visible: NotRequired[bool] + html: NotRequired[bool] class ReconnectOptions(TypedDict, total=False): @@ -186,6 +187,7 @@ class ResponseFilterOptions(TypedDict, total=False): type: NotRequired[list[ResourceType]] url: NotRequired[list[str]] wait: NotRequired[bool] + body: NotRequired[bool] body_encoding: NotRequired[bool] diff --git a/src/browserless_bap/types/responses.py b/src/browserless_bap/types/responses.py index 6560fc4..8add206 100644 --- a/src/browserless_bap/types/responses.py +++ b/src/browserless_bap/types/responses.py @@ -164,7 +164,7 @@ class ProxyResponse(TypedDict, total=False): class QuerySelectorResponse(TypedDict, total=False): - childElementCount: NotRequired[float | None] + childElementCount: NotRequired[int | None] className: NotRequired[str | None] id: NotRequired[str | None] innerHTML: NotRequired[str | None] diff --git a/tests/e2e/test_config.py b/tests/e2e/test_config.py index 2690576..fafcf57 100644 --- a/tests/e2e/test_config.py +++ b/tests/e2e/test_config.py @@ -131,5 +131,5 @@ def test_live_suite_needs_explicit_e2e_selection(selection: list[str]) -> None: ) assert result.returncode == 0, result.stdout - assert "6 skipped" in result.stdout + assert "7 skipped" in result.stdout assert "requires -m e2e" in result.stdout diff --git a/tests/e2e/test_live.py b/tests/e2e/test_live.py index fdf514d..b164fb9 100644 --- a/tests/e2e/test_live.py +++ b/tests/e2e/test_live.py @@ -2,7 +2,6 @@ import asyncio from collections.abc import Mapping -from typing import Any, cast import pytest from pytest import CaptureFixture @@ -37,8 +36,7 @@ async def run_async_happy_path(config: LiveConfig) -> None: def run_sync_happy_path(config: LiveConfig) -> None: - sync_browserless = cast(Any, bap_sync.Browserless) - with sync_browserless.connect( + with bap_sync.Browserless.connect( browser_ws_endpoint=config.url, token=config.token, timeout=config.timeout_ms, @@ -74,8 +72,7 @@ def rejected_sync_connection( token: str | None, timeout_ms: float, ) -> None: - sync_browserless = cast(Any, bap_sync.Browserless) - with sync_browserless.connect( + with bap_sync.Browserless.connect( browser_ws_endpoint=url, token=token, timeout=timeout_ms, @@ -185,7 +182,7 @@ def test_missing_and_invalid_auth_are_typed_and_redacted( capsys: CaptureFixture[str], token: str | None, ) -> None: - with pytest.raises(bap.ConnectionError) as async_error: + with pytest.raises(bap.BAPConnectionError) as async_error: asyncio.run( rejected_async_connection( live_config.url, @@ -193,7 +190,7 @@ def test_missing_and_invalid_auth_are_typed_and_redacted( live_config.timeout_ms, ) ) - with pytest.raises(bap.ConnectionError) as sync_error: + with pytest.raises(bap.BAPConnectionError) as sync_error: rejected_sync_connection( live_config.url, token, @@ -220,7 +217,7 @@ def test_malformed_endpoint_is_typed_and_redacted( capsys: CaptureFixture[str], ) -> None: malformed_url = f"http://bql.invalid/chromium/bql?token={MALFORMED_ENDPOINT_TOKEN}" - with pytest.raises(bap.ConnectionError) as async_error: + with pytest.raises(bap.BAPConnectionError) as async_error: asyncio.run( rejected_async_connection( malformed_url, @@ -228,7 +225,7 @@ def test_malformed_endpoint_is_typed_and_redacted( live_config.timeout_ms, ) ) - with pytest.raises(bap.ConnectionError) as sync_error: + with pytest.raises(bap.BAPConnectionError) as sync_error: rejected_sync_connection( malformed_url, live_config.token, @@ -246,3 +243,47 @@ def test_malformed_endpoint_is_typed_and_redacted( ] assert_secret_absent(live_config.token, values) assert_secret_absent(MALFORMED_ENDPOINT_TOKEN, values) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_opt_in_fields_live( + live_config: LiveConfig, + capsys: CaptureFixture[str], +) -> None: + async with bap.Browserless.connect( + browser_ws_endpoint=live_config.url, + token=live_config.token, + timeout=live_config.timeout_ms, + ) as browser: + async with browser.page() as page: + await page.goto(EXAMPLE_URL, timeout=live_config.timeout_ms) + responses = await page.response(type=["document"]) + assert responses + assert all("body" not in item for item in responses) + with_bodies = await page.response(type=["document"], body=True) + assert any( + "Example Domain" in (item.get("body") or "") for item in with_bodies + ) + await page.set_content('

Hello opt-in

') + plain = await page.query_selector("h1") + assert plain is not None + with pytest.raises(bap.BAPValueError, match="html=True"): + _ = plain.inner_html + assert plain.child_element_count == 1 + assert isinstance(plain.child_element_count, int) + html = await page.query_selector("h1", html=True) + assert html is not None + assert html.inner_html == "Hello opt-in" + assert (html.outer_html or "").startswith("opt"] + captured = capsys.readouterr() + assert_secret_absent(live_config.token, [captured.out, captured.err]) diff --git a/tests/sync/test_bridge.py b/tests/sync/test_bridge.py index f5a1328..4206888 100644 --- a/tests/sync/test_bridge.py +++ b/tests/sync/test_bridge.py @@ -9,6 +9,7 @@ import pytest import browserless_bap.sync_api._bridge as bridge_module +from browserless_bap.errors import BAPTimeoutError from browserless_bap.sync_api._bridge import CallbackDispatcher, LoopThread @@ -25,6 +26,42 @@ def test_run_returns_async_result() -> None: bridge.close() +def test_bridge_deadline_raises_typed_timeout_and_cancels_work() -> None: + bridge = LoopThread() + cancelled = threading.Event() + + async def pending() -> None: + try: + await asyncio.Event().wait() + finally: + cancelled.set() + + try: + with pytest.raises(BAPTimeoutError) as caught: + bridge.run(pending(), timeout=0.05) + assert caught.value.operation == "sync bridge" + assert caught.value.timeout_ms == 50 + assert cancelled.wait(timeout=2) + finally: + bridge.close() + + +@pytest.mark.parametrize("sdk_error", [False, True]) +def test_bridge_propagates_completed_timeout_without_retry(sdk_error: bool) -> None: + bridge = LoopThread() + error = BAPTimeoutError("remote", 123) if sdk_error else TimeoutError("user") + + async def fail() -> None: + raise error + + try: + with pytest.raises(type(error)) as caught: + bridge.run(fail(), timeout=1) + assert caught.value is error + finally: + bridge.close() + + def test_run_rejects_active_asyncio_loop_without_leaking_coroutine() -> None: bridge = LoopThread() diff --git a/tests/sync/test_sync_events.py b/tests/sync/test_sync_events.py index a191269..a80b545 100644 --- a/tests/sync/test_sync_events.py +++ b/tests/sync/test_sync_events.py @@ -2,7 +2,6 @@ import threading from collections.abc import Mapping -from typing import Any, cast import browserless_bap.sync_api as sync_bap @@ -10,9 +9,8 @@ def test_sync_event_callback_can_call_page_without_deadlocking() -> None: - browserless = cast(Any, sync_bap.Browserless) with ServerThread() as server: - with browserless.connect( + with sync_bap.Browserless.connect( browser_ws_endpoint=server.url, timeout=2_000, ) as browser: diff --git a/tests/sync/test_sync_page.py b/tests/sync/test_sync_page.py index bf6c572..b72c52a 100644 --- a/tests/sync/test_sync_page.py +++ b/tests/sync/test_sync_page.py @@ -2,21 +2,19 @@ import socket import time -from typing import Any, cast import pytest import browserless_bap.sync_api as sync_bap -from browserless_bap.errors import TimeoutError as BAPTimeoutError +from browserless_bap.errors import BAPTimeoutError from ._server import ServerThread @pytest.mark.timeout(3) def test_sync_operation_timeout_raises() -> None: - browserless = cast(Any, sync_bap.Browserless) with ServerThread(reply=False) as server: - with browserless.connect(browser_ws_endpoint=server.url) as browser: + with sync_bap.Browserless.connect(browser_ws_endpoint=server.url) as browser: with browser.page() as page: started = time.monotonic() with pytest.raises(BAPTimeoutError) as caught: @@ -27,11 +25,10 @@ def test_sync_operation_timeout_raises() -> None: @pytest.mark.timeout(3) def test_sync_connect_timeout_raises() -> None: - browserless = cast(Any, sync_bap.Browserless) with socket.socket() as listener: listener.bind(("127.0.0.1", 0)) listener.listen() - with browserless.connect( + with sync_bap.Browserless.connect( browser_ws_endpoint=f"ws://127.0.0.1:{listener.getsockname()[1]}/bql", connect_timeout=100, ) as browser: @@ -44,9 +41,8 @@ def test_sync_connect_timeout_raises() -> None: def test_sync_page_uses_a_server_on_an_independent_loop_thread() -> None: - browserless = cast(Any, sync_bap.Browserless) with ServerThread() as server: - with browserless.connect( + with sync_bap.Browserless.connect( browser_ws_endpoint=server.url, timeout=2_000, ) as browser: @@ -61,21 +57,20 @@ def test_sync_page_uses_a_server_on_an_independent_loop_thread() -> None: def test_sync_live_url_wait_retains_early_completion() -> None: - browserless = cast(Any, sync_bap.Browserless) with ServerThread() as server: - with browserless.connect( + with sync_bap.Browserless.connect( browser_ws_endpoint=server.url, timeout=2_000 ) as browser: with browser.page() as page: link = page.live_url(instructions="Sign in") assert page.wait_for_live_complete()["reason"] == "userDone" + assert link["liveURLId"] is not None assert page.close_live_url(link["liveURLId"]) == {"liveURLId": "link"} def test_sync_live_url_accepts_rich_instructions() -> None: - browserless = cast(Any, sync_bap.Browserless) with ServerThread() as server: - with browserless.connect( + with sync_bap.Browserless.connect( browser_ws_endpoint=server.url, timeout=2_000 ) as browser: with browser.page() as page: diff --git a/tests/sync/test_wrappers.py b/tests/sync/test_wrappers.py index f3e3896..f7deb6d 100644 --- a/tests/sync/test_wrappers.py +++ b/tests/sync/test_wrappers.py @@ -10,6 +10,7 @@ import browserless_bap.sync_api.page as page_module from browserless_bap._unset import UNSET from browserless_bap.element_handle import ElementHandle as AsyncElementHandle +from browserless_bap.errors import BAPValueError from browserless_bap.sync_api._bridge import LoopThread from browserless_bap.sync_api.browser import Browser from browserless_bap.sync_api.element_handle import ElementHandle @@ -21,7 +22,7 @@ def __init__(self) -> None: self.closed = False self.listeners: dict[str, list[Callable[[object], object]]] = {} self.thread_names: list[str] = [] - self.query_options: dict[str, object] = {} + self.query_options: list[dict[str, object]] = [] async def goto(self, url: str, **options: object) -> dict[str, object]: self.thread_names.append(threading.current_thread().name) @@ -33,7 +34,7 @@ async def title(self) -> str: async def query_selector( self, selector: str, **options: object ) -> AsyncElementHandle: - self.query_options = options + self.query_options.append(options) return AsyncElementHandle( cast(Any, self), selector, @@ -127,11 +128,17 @@ def test_page_calls_run_on_bridge_and_wrap_handles() -> None: assert isinstance(handle, ElementHandle) assert handle.inner_text == "child" assert handle.text_content() == "current child" - assert isinstance(handle.query_selector("span", timeout=5), ElementHandle) - assert impl.query_options == {"timeout": 5} - assert len(handle.query_selector_all("span", visible=True)) == 1 - assert impl.query_options == {"visible": True} + assert isinstance(handle.query_selector("span"), ElementHandle) + assert len(handle.query_selector_all("span")) == 1 + handle.query_selector("span", timeout=5, visible=False, html=True) + assert impl.query_options[-1] == {"timeout": 5, "visible": False, "html": True} + handle.query_selector_all("span", timeout=10, visible=True, html=True) + assert impl.query_options[-1] == {"timeout": 10, "visible": True, "html": True} assert len(page.query_selector_all(".item")) == 1 + plain = ElementHandle(AsyncElementHandle(cast(Any, impl), "p", {}), bridge) + for attribute in ("inner_html", "outer_html"): + with pytest.raises(BAPValueError, match="html=True"): + getattr(plain, attribute) finally: bridge.close() diff --git a/tests/test_browser.py b/tests/test_browser.py index f521532..1298c27 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -5,7 +5,7 @@ import pytest from browserless_bap.browserless import Browserless -from browserless_bap.errors import ConnectionError, ValueError +from browserless_bap.errors import BAPConnectionError, BAPValueError from browserless_bap.page import Page from browserless_bap.transport import TransportFactory, TransportHandlers @@ -27,7 +27,7 @@ async def close(self) -> None: def test_browser_requires_endpoint_or_transport() -> None: - with pytest.raises(ValueError): + with pytest.raises(BAPValueError): Browserless.connect() @@ -76,7 +76,7 @@ async def factory(_url: str | None) -> FakeConnection: await browser.close() release.set() - with pytest.raises(ValueError, match="Browser is closed"): + with pytest.raises(BAPValueError, match="Browser is closed"): await task assert connection.closed assert browser.get_pages() == [] @@ -137,7 +137,7 @@ async def send(self, message: str) -> None: connect_timeout=20 if failure == "timeout" else 0, ) with pytest.raises( - TimeoutError if failure in {"cancel", "timeout"} else ConnectionError + TimeoutError if failure in {"cancel", "timeout"} else BAPConnectionError ): await asyncio.wait_for(browser.new_page(), 0.1) assert connection.closed diff --git a/tests/test_codegen_operations.py b/tests/test_codegen_operations.py index 48ad048..8a82227 100644 --- a/tests/test_codegen_operations.py +++ b/tests/test_codegen_operations.py @@ -7,8 +7,12 @@ import pytest from graphql import build_schema, parse, specified_rules, validate -from browserless_bap.graphql.builder import build_operation -from browserless_bap.graphql.mutations import ARG_TYPES, RESPONSE_FIELDS +from browserless_bap.graphql.builder import build_operation, selection_for +from browserless_bap.graphql.mutations import ( + ARG_TYPES, + OPTIONAL_FIELDS, + RESPONSE_FIELDS, +) from browserless_bap.graphql.subscriptions import SUBSCRIPTIONS ROOT = Path(__file__).resolve().parents[1] @@ -53,15 +57,13 @@ def test_every_default_and_opt_in_response_selection_is_valid() -> None: CODEGEN_SCHEMA.response_name(name): name for name in CODEGEN_SCHEMA.response_objects } - owner_by_response.update( - { - "HTTPResponseWithRemoteMetadata": "HTTPResponse", - "ResponseResponseWithBodyEncoding": "ResponseResponse", - } - ) for response_name, fields in RESPONSE_FIELDS.items(): owner = owner_by_response[response_name] _assert_valid_fragment(f"fragment CodegenSelection on {owner} {{ {fields} }}") + for response_name, groups in OPTIONAL_FIELDS.items(): + owner = owner_by_response[response_name] + fields = selection_for(response_name, groups) + _assert_valid_fragment(f"fragment CodegenSelection on {owner} {{ {fields} }}") def test_every_subscription_document_is_valid() -> None: diff --git a/tests/test_codegen_schema.py b/tests/test_codegen_schema.py index 8e3e0e5..435471d 100644 --- a/tests/test_codegen_schema.py +++ b/tests/test_codegen_schema.py @@ -13,7 +13,7 @@ PAGE_EXCLUDED, POSITIONAL_ARGS, ) -from scripts.codegen.config.schema_map import SCALAR_MAP # noqa: E402 +from scripts.codegen.config.schema_map import OPTIONAL_FIELDS, SCALAR_MAP # noqa: E402 from scripts.codegen.model import build_models # noqa: E402 from scripts.codegen.schema import Schema # noqa: E402 @@ -54,6 +54,11 @@ def test_client_only_parameters_have_no_wire_name() -> None: response = {param.py_name: param for param in models["response"].params} assert goto["remote_response_metadata"].gql_name is None assert response["body_encoding"].gql_name is None + assert response["body"].gql_name is None + for name in ("query_selector", "query_selector_all"): + assert ( + next(p for p in models[name].params if p.py_name == "html").gql_name is None + ) def test_stale_mutation_config_fails_loud() -> None: @@ -82,3 +87,27 @@ def test_unmapped_custom_scalar_fails_loud() -> None: Schema.load(ROOT / "src/browserless_bap/mutations.graphql") finally: SCALAR_MAP["FloatOrString"] = previous + + +@pytest.mark.parametrize( + ("type_name", "field", "message"), + [ + ("MissingType", "body", "unknown type 'MissingType'"), + ("ResponseResponse", "missingField", "unknown field 'missingField'"), + ], +) +def test_invalid_optional_field_configuration_fails_loud( + type_name: str, + field: str, + message: str, +) -> None: + previous = OPTIONAL_FIELDS.get(type_name) + OPTIONAL_FIELDS[type_name] = {"invalid": (field,)} + try: + with pytest.raises(ValueError, match=message): + Schema.load(ROOT / "src/browserless_bap/mutations.graphql") + finally: + if previous is None: + del OPTIONAL_FIELDS[type_name] + else: + OPTIONAL_FIELDS[type_name] = previous diff --git a/tests/test_errors.py b/tests/test_errors.py index 97d7ab3..c56bb4c 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -1,12 +1,12 @@ import builtins from browserless_bap.errors import ( + BAPConnectionError, BAPError, + BAPTimeoutError, + BAPValueError, BrowserQLError, - ConnectionError, SchemaMismatchError, - TimeoutError, - ValueError, ) @@ -16,10 +16,10 @@ def test_error_codes_and_context() -> None: assert bql.code == "BQL" assert bql.errors == [{"message": "first"}, {"message": "second"}] - connection = ConnectionError("closed") + connection = BAPConnectionError("closed") assert connection.code == "CONNECTION" - timeout = TimeoutError("mutation { goto }", 250) + timeout = BAPTimeoutError("mutation { goto }", 250) assert timeout.code == "TIMEOUT" assert timeout.operation == "mutation { goto }" assert timeout.timeout_ms == 250 @@ -29,16 +29,16 @@ def test_error_codes_and_context() -> None: def test_bap_value_error_is_native_value_error_too() -> None: - error = ValueError("invalid") + error = BAPValueError("invalid") assert isinstance(error, BAPError) assert isinstance(error, builtins.ValueError) assert error.code == "CLIENT" def test_transport_errors_are_native_errors_too() -> None: - assert isinstance(ConnectionError("closed"), builtins.ConnectionError) + assert isinstance(BAPConnectionError("closed"), builtins.ConnectionError) assert isinstance( - TimeoutError("mutation { goto }", 250), + BAPTimeoutError("mutation { goto }", 250), builtins.TimeoutError, ) diff --git a/tests/test_graphql_builder.py b/tests/test_graphql_builder.py index e2fdcdb..3c23332 100644 --- a/tests/test_graphql_builder.py +++ b/tests/test_graphql_builder.py @@ -31,3 +31,22 @@ def test_builds_argument_free_subscription() -> None: def test_missing_argument_type_is_a_schema_mismatch() -> None: with pytest.raises(SchemaMismatchError, match="Missing GraphQL type"): build_operation("goto", {"url": "https://example.com"}, "status") + + +def test_optional_selections_preserve_nested_fields_and_deduplicate() -> None: + from browserless_bap.graphql import selection_for + + assert selection_for("QuerySelectorResponse") == ( + "childElementCount className id innerText localName" + ) + assert selection_for("QuerySelectorResponse", ("html",)) == ( + "childElementCount className id innerText localName innerHTML outerHTML" + ) + assert selection_for("ResponseResponse", ("body", "bodyEncoding", "body")) == ( + "status url method type headers { name value } body base64Encoded" + ) + assert selection_for("HTTPResponse", ("remoteMetadata",)).endswith("ip port") + with pytest.raises(SchemaMismatchError, match="Unknown response type"): + selection_for("Unknown") + with pytest.raises(SchemaMismatchError, match="Unknown optional field group"): + selection_for("QuerySelectorResponse", ("body",)) diff --git a/tests/test_live_complete.py b/tests/test_live_complete.py index 0667100..f945de8 100644 --- a/tests/test_live_complete.py +++ b/tests/test_live_complete.py @@ -4,7 +4,7 @@ import pytest -from browserless_bap.errors import ConnectionError, TimeoutError +from browserless_bap.errors import BAPConnectionError, BAPTimeoutError from browserless_bap.page import Page from browserless_bap.transport import Subscriber, Transport @@ -126,7 +126,7 @@ async def test_wait_before_mint_and_independent_timeout() -> None: waiting = asyncio.create_task(page.wait_for_live_complete(timeout=0)) await asyncio.sleep(0) await page.live_url() - with pytest.raises(TimeoutError): + with pytest.raises(BAPTimeoutError): await page.wait_for_live_complete(timeout=5) transport.event("userFailed") assert (await waiting)["reason"] == "userFailed" @@ -143,10 +143,10 @@ async def test_wait_rejects_on_disconnect_or_close(disconnect: bool) -> None: await asyncio.sleep(0) if disconnect: assert transport.subscriber is not None - transport.subscriber.error(ConnectionError("disconnected")) + transport.subscriber.error(BAPConnectionError("disconnected")) else: await page.close() - with pytest.raises(ConnectionError): + with pytest.raises(BAPConnectionError): await waiting await page.close() @@ -204,7 +204,7 @@ async def test_waits_survive_multiple_remints(end: str) -> None: if end == "close": await page.close() results = await asyncio.gather(*waits, return_exceptions=True) - assert all(isinstance(result, ConnectionError) for result in results) + assert all(isinstance(result, BAPConnectionError) for result in results) else: for reason in ["closed", "userDone", "timeout"]: transport.event(reason) @@ -219,7 +219,7 @@ async def test_waits_survive_multiple_remints(end: str) -> None: async def test_mint_timeout_keeps_its_original_error() -> None: transport = LiveTransport() page = Page(cast(Transport, transport)) - error = TimeoutError("liveURL", 25) + error = BAPTimeoutError("liveURL", 25) def fail() -> None: raise error @@ -227,9 +227,9 @@ def fail() -> None: transport.before_reply = fail waiting = asyncio.create_task(page.wait_for_live_complete(timeout=0)) await asyncio.sleep(0) - with pytest.raises(TimeoutError) as mint_error: + with pytest.raises(BAPTimeoutError) as mint_error: await page.live_url() - with pytest.raises(TimeoutError) as wait_error: + with pytest.raises(BAPTimeoutError) as wait_error: await waiting assert mint_error.value is error assert wait_error.value is error @@ -246,14 +246,14 @@ async def test_stream_failure_retained_until_remint(wait_before_failure: bool) - waiting = asyncio.create_task(page.wait_for_live_complete(timeout=0)) await asyncio.sleep(0) assert transport.subscriber is not None - error = ConnectionError("disconnected") + error = BAPConnectionError("disconnected") transport.subscriber.error(error) if waiting is not None: - with pytest.raises(ConnectionError) as wait_error: + with pytest.raises(BAPConnectionError) as wait_error: await waiting assert wait_error.value is error for _ in range(2): - with pytest.raises(ConnectionError) as wait_error: + with pytest.raises(BAPConnectionError) as wait_error: await page.wait_for_live_complete(timeout=5) assert wait_error.value is error await page.live_url() @@ -269,7 +269,7 @@ async def test_failed_handoff_retained_during_mint_then_retry( page = Page(cast(Transport, transport)) entered = asyncio.Event() release = asyncio.Event() - error = ConnectionError("disconnected") + error = BAPConnectionError("disconnected") original_send = transport.send async def send(*_args: Any) -> Any: @@ -283,12 +283,12 @@ async def send(*_args: Any) -> Any: assert transport.subscriber is not None transport.subscriber.error(error) try: - with pytest.raises(ConnectionError) as wait_error: + with pytest.raises(BAPConnectionError) as wait_error: await page.wait_for_live_complete(timeout=5) assert wait_error.value is error finally: release.set() - with pytest.raises(ConnectionError): + with pytest.raises(BAPConnectionError): await mint monkeypatch.setattr(transport, "send", original_send) @@ -342,7 +342,7 @@ def before_reply() -> None: transport.before_reply = before_reply await page.live_url() if not matching: - with pytest.raises(TimeoutError): + with pytest.raises(BAPTimeoutError): await page.wait_for_live_complete(timeout=5) transport.event("closed") assert await page.wait_for_live_complete() == { @@ -355,7 +355,7 @@ def before_reply() -> None: async def test_last_premint_wait_releases_subscription() -> None: transport = LiveTransport() page = Page(cast(Transport, transport)) - with pytest.raises(TimeoutError): + with pytest.raises(BAPTimeoutError): await page.wait_for_live_complete(timeout=5) await asyncio.sleep(0) assert "liveComplete" not in page._unsubscribes @@ -371,7 +371,7 @@ async def test_premint_timeout_preserves_other_waiter() -> None: page = Page(cast(Transport, transport)) waiting = asyncio.create_task(page.wait_for_live_complete()) await asyncio.sleep(0) - with pytest.raises(TimeoutError): + with pytest.raises(BAPTimeoutError): await page.wait_for_live_complete(timeout=5) assert "liveComplete" in page._unsubscribes await page.live_url() @@ -399,7 +399,7 @@ async def send(*_args: Any) -> Any: mint.cancel() with pytest.raises(asyncio.CancelledError): await mint - with pytest.raises(ConnectionError, match="cancelled"): + with pytest.raises(BAPConnectionError, match="cancelled"): await waiting assert not waiting.cancelled() await page.close() diff --git a/tests/test_page_overrides.py b/tests/test_page_overrides.py index c15eba6..8732195 100644 --- a/tests/test_page_overrides.py +++ b/tests/test_page_overrides.py @@ -2,7 +2,7 @@ # ruff: noqa: ASYNC109 import inspect -from collections.abc import Callable, Mapping +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import TYPE_CHECKING, Any, assert_type, cast @@ -96,6 +96,7 @@ def __init__(self) -> None: Mapping[str, Any] | None, str, float | UnsetType | None, + Sequence[str], ] ] = [] @@ -106,8 +107,9 @@ async def _execute( response_type: str, *, timeout: float | UnsetType | None = None, + opt_in: Sequence[str] = (), ) -> Any: - self.calls.append((mutation_name, args, response_type, timeout)) + self.calls.append((mutation_name, args, response_type, timeout, opt_in)) if mutation_name == "evaluate": return {"value": "2"} if mutation_name == "response": @@ -123,8 +125,9 @@ async def _execute( response_type: str, *, timeout: float | UnsetType | None = None, + opt_in: Sequence[str] = (), ) -> None: - self.calls.append((mutation_name, args, response_type, timeout)) + self.calls.append((mutation_name, args, response_type, timeout, opt_in)) return None @@ -136,8 +139,9 @@ async def _execute( response_type: str, *, timeout: float | UnsetType | None = None, + opt_in: Sequence[str] = (), ) -> Any: - self.calls.append((mutation_name, args, response_type, timeout)) + self.calls.append((mutation_name, args, response_type, timeout, opt_in)) if mutation_name == "querySelectorAll": return [{"innerText": "first"}, {"innerText": "second"}] return {"ok": True} @@ -171,12 +175,14 @@ async def test_goto_and_response_opt_ins_never_leak_client_flags() -> None: goto = page.calls[0] assert goto[0] == "goto" assert goto[1] == {"url": "https://example.com", "timeout": 500} - assert goto[2] == "HTTPResponseWithRemoteMetadata" + assert goto[2] == "HTTPResponse" + assert goto[4] == ("remoteMetadata",) assert goto[1] is not None assert "remoteResponseMetadata" not in goto[1] response = page.calls[1] - assert response[2] == "ResponseResponseWithBodyEncoding" + assert response[2] == "ResponseResponse" + assert response[4] == ("bodyEncoding",) assert response[1] is not None assert "bodyEncoding" not in response[1] await page.close() @@ -229,7 +235,7 @@ async def test_wait_for_function_rewrite_preserves_raw_timeout() -> None: complete_on_promise_resolution=True, ) - _, args, _, timeout = page.calls[0] + _, args, _, timeout, _ = page.calls[0] assert args is not None assert args["fn"].startswith("async () => { await (") assert "completeOnPromiseResolution" not in args @@ -289,8 +295,8 @@ async def test_element_handle_descendant_selector_scopes_every_alternative() -> async def test_element_handle_queries_forward_options() -> None: page = SelectorListPage() handle = (await page.query_selector_all(".item"))[0] - await handle.query_selector("li", timeout=5000, visible=True) - await handle.query_selector_all("li", timeout=750, visible=False) + await handle.query_selector("li", timeout=5000, visible=True, html=True) + await handle.query_selector_all("li", timeout=750, visible=False, html=True) assert page.calls[2][1] is not None assert page.calls[2][1]["selector"] == ( @@ -304,6 +310,7 @@ async def test_element_handle_queries_forward_options() -> None: assert call[1]["timeout"] == timeout assert call[1]["visible"] is visible assert call[3] == timeout + assert call[4] == ("html",) await page.close() @@ -347,3 +354,70 @@ async def test_generated_methods_preserve_defaults_for_null_results() -> None: assert await page.screenshot() == b"" await page.close() + + +@pytest.mark.asyncio +async def test_optional_fields_are_selected_only_when_requested() -> None: + class QueryTransport(RecordingTransport): + def __init__(self) -> None: + self.queries: list[str] = [] + + async def send( + self, + query: str, + variables: Mapping[str, Any] | None = None, + timeout: float | None = None, + ) -> dict[str, Any]: + self.queries.append(query) + return { + "querySelector": {"innerText": "hello"}, + "querySelectorAll": [{"innerText": "hello"}], + "response": [], + "goto": {}, + } + + transport = QueryTransport() + page = Page(cast(Transport, transport)) + try: + for query_method in (page.query_selector, page.query_selector_all): + await query_method("p") + assert "innerHTML" not in transport.queries[-1] + assert "outerHTML" not in transport.queries[-1] + await query_method("p", html=True) + assert "innerHTML outerHTML" in transport.queries[-1] + await query_method("p", html=False) + assert "innerHTML" not in transport.queries[-1] + await page.response() + assert "body" not in transport.queries[-1] + await page.response(body=True) + assert "body" in transport.queries[-1].split() + assert "base64Encoded" not in transport.queries[-1] + await page.response(body=True, body_encoding=True) + assert transport.queries[-1].split().count("body") == 1 + assert "base64Encoded" in transport.queries[-1] + await page.goto("https://example.com", remote_response_metadata=True) + assert "ip port" in transport.queries[-1] + finally: + await page.close() + + +@pytest.mark.asyncio +async def test_element_html_requires_opt_in_and_scoped_queries_forward_it() -> None: + from browserless_bap.element_handle import ElementHandle + from browserless_bap.errors import BAPValueError + + page = SelectorListPage() + try: + plain = ElementHandle(page, "ul", {}) + for attribute in ("inner_html", "outer_html"): + with pytest.raises(BAPValueError, match="html=True"): + getattr(plain, attribute) + requested = ElementHandle(page, "ul", {"innerHTML": None, "outerHTML": ""}) + assert requested.inner_html is None + assert requested.outer_html == "" + await plain.query_selector("li", html=True) + assert page.calls[-1][4] == ("html",) + await plain.query_selector_all("li", html=True) + assert page.calls[-1][4] == ("html",) + finally: + await page.close() diff --git a/tests/test_public_api.py b/tests/test_public_api.py new file mode 100644 index 0000000..85fe022 --- /dev/null +++ b/tests/test_public_api.py @@ -0,0 +1,88 @@ +import builtins +import importlib +import inspect +import sys +from pathlib import Path +from typing import NotRequired, get_type_hints + +import pytest + +from browserless_bap import errors +from browserless_bap.page import Page +from browserless_bap.sync_api.page import Page as SyncPage +from browserless_bap.types.responses import QuerySelectorResponse + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from scripts.codegen.model import build_models # noqa: E402 +from scripts.codegen.schema import Schema # noqa: E402 + + +@pytest.mark.parametrize( + "module_name", + ["browserless_bap", "browserless_bap.async_api", "browserless_bap.errors"], +) +def test_public_imports_do_not_shadow_builtins(module_name: str) -> None: + module = importlib.import_module(module_name) + namespace: dict[str, object] = {} + exec(f"from {module_name} import *", namespace) + for name in ("ConnectionError", "TimeoutError", "ValueError"): + assert not hasattr(module, name) + assert name not in namespace + assert eval(name, namespace) is getattr(builtins, name) + + +@pytest.mark.parametrize( + ("name", "base"), + [ + ("BAPConnectionError", builtins.ConnectionError), + ("BAPTimeoutError", builtins.TimeoutError), + ("BAPValueError", builtins.ValueError), + ], +) +def test_sdk_errors_have_distinct_names_and_builtin_bases( + name: str, base: type[Exception] +) -> None: + error = getattr(errors, name) + assert error.__name__ == name + assert issubclass(error, errors.BAPError) + assert issubclass(error, base) + + +def test_errors_export_public_error_types_and_code() -> None: + assert set(errors.__all__) == { + "BAPError", + "BAPConnectionError", + "BAPTimeoutError", + "BAPValueError", + "BrowserQLError", + "ErrorCode", + "SchemaMismatchError", + } + + +def test_generated_methods_have_clean_docstrings() -> None: + models = build_models(Schema.load(ROOT / "src/browserless_bap/mutations.graphql")) + for model in models: + for page in (Page, SyncPage): + doc = inspect.getdoc(getattr(page, model.py_name)) + assert doc, model.py_name + assert "```" not in doc + assert "**Field Definition" not in doc + + +def test_selector_eval_docs_explain_difference_from_playwright() -> None: + readme = (ROOT / "README.md").read_text() + for name in ("eval_on_selector", "eval_on_selector_all"): + for page in (Page, SyncPage): + assert "Unlike Playwright" in (inspect.getdoc(getattr(page, name)) or "") + row = next(line for line in readme.splitlines() if f"`{name}`" in line) + assert row.endswith("| — |") + + +def test_child_element_count_is_typed_as_an_integer() -> None: + assert ( + get_type_hints(QuerySelectorResponse, include_extras=True)["childElementCount"] + == NotRequired[int | None] + ) diff --git a/tests/test_readme_typecheck.py b/tests/test_readme_typecheck.py new file mode 100644 index 0000000..078a24e --- /dev/null +++ b/tests/test_readme_typecheck.py @@ -0,0 +1,34 @@ +import re +from pathlib import Path + +import pytest +from mypy import api + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize("document", ["README.md", "llms.txt"]) +def test_self_contained_examples_typecheck(document: str, tmp_path: Path) -> None: + blocks = [ + block + for block in re.findall( + r"^```python\s*\n(.*?)^```", (ROOT / document).read_text(), re.M | re.S + ) + if "import browserless_bap" in block + ] + if document == "README.md": + assert len(blocks) >= 2 + assert any("import browserless_bap.sync_api" in block for block in blocks) + for index, block in enumerate(blocks): + example = tmp_path / f"example_{index}.py" + example.write_text(block) + stdout, stderr, status = api.run( + [ + "--strict", + "--no-incremental", + "--cache-dir", + str(tmp_path / ".mypy_cache"), + str(example), + ] + ) + assert status == 0, f"{document} block {index}:\n{stdout}{stderr}" diff --git a/tests/test_transport.py b/tests/test_transport.py index 4fb5ef2..a2d4a73 100644 --- a/tests/test_transport.py +++ b/tests/test_transport.py @@ -8,8 +8,7 @@ from websockets.asyncio.server import ServerConnection, serve from browserless_bap import Browserless, __version__ -from browserless_bap.errors import BrowserQLError, ConnectionError -from browserless_bap.errors import TimeoutError as BAPTimeoutError +from browserless_bap.errors import BAPConnectionError, BAPTimeoutError, BrowserQLError from browserless_bap.transport import Transport @@ -38,7 +37,7 @@ async def handler( ) as browser: async with asyncio.timeout(3): with pytest.raises( - BAPTimeoutError if stall else ConnectionError + BAPTimeoutError if stall else BAPConnectionError ) as caught: await browser.new_page() if stall: @@ -59,7 +58,7 @@ async def test_connection_traceback_redacts_token() -> None: async with Browserless.connect( browser_ws_endpoint="https://example.invalid/bql", token="SECRET456" ) as browser: - with pytest.raises(ConnectionError) as caught: + with pytest.raises(BAPConnectionError) as caught: await browser.new_page() assert "SECRET456" not in "".join(traceback.format_exception(caught.value)) diff --git a/tests/test_transport_protocol.py b/tests/test_transport_protocol.py index f511c87..8cf8794 100644 --- a/tests/test_transport_protocol.py +++ b/tests/test_transport_protocol.py @@ -9,8 +9,7 @@ from websockets.exceptions import ConnectionClosedError from websockets.frames import Close -from browserless_bap.errors import BrowserQLError, ConnectionError -from browserless_bap.errors import TimeoutError as BAPTimeoutError +from browserless_bap.errors import BAPConnectionError, BAPTimeoutError, BrowserQLError from browserless_bap.transport import ( Subscriber, Transport, @@ -135,7 +134,7 @@ async def factory(_url: str | None) -> FakeConnection: assert caught.value.timeout_ms == (40 if connect_timeout is None else 20) assert caught.value.__context__ is None assert transport.closed - with pytest.raises((ConnectionError, BAPTimeoutError)): + with pytest.raises((BAPConnectionError, BAPTimeoutError)): await transport.send("query { x }") await transport.close() @@ -166,7 +165,7 @@ async def send(self, message: str) -> None: connection = BrokenConnection() transport = Transport("ws://unused", transport_factory=lambda _url: connection) - with pytest.raises(ConnectionError) as caught: + with pytest.raises(BAPConnectionError) as caught: await transport.connect() assert connection.closed assert caught.value.__cause__ is None @@ -441,7 +440,7 @@ def factory(_url: str | None) -> FakeConnection: "https://example.invalid/bql?token=SECRET", transport_factory=None if site == "dial" else cast(TransportFactory, factory), ) - with pytest.raises(ConnectionError) as caught: + with pytest.raises(BAPConnectionError) as caught: await transport.send("query { x }") assert caught.value.__cause__ is None assert caught.value.__context__ is None @@ -640,7 +639,7 @@ async def test_send_waits_for_complete_and_drop_after_next_rejects() -> None: connection.drop() - with pytest.raises(ConnectionError, match="closed unexpectedly"): + with pytest.raises(BAPConnectionError, match="closed unexpectedly"): await task @@ -711,7 +710,7 @@ async def test_drop_marks_closed_and_fails_subscriber() -> None: assert transport.closed assert len(errors) == 1 - assert isinstance(errors[0], ConnectionError) + assert isinstance(errors[0], BAPConnectionError) await transport.close() @@ -770,7 +769,7 @@ def fail_callback(_error: Exception) -> None: ) ) - with pytest.raises(ConnectionError) as raised: + with pytest.raises(BAPConnectionError) as raised: await task assert transport.closed assert "super-secret" not in str(raised.value) @@ -821,7 +820,7 @@ async def test_abnormal_websocket_close_preserves_reason_and_redacts_token() -> socket.release.set() - with pytest.raises(ConnectionError) as raised: + with pytest.raises(BAPConnectionError) as raised: await task message = str(raised.value) assert transport.closed From 0d0e0d1edc884b0e1d7545d0710a6e00f1158e3e Mon Sep 17 00:00:00 2001 From: Amp Date: Fri, 13 Nov 2026 12:59:39 +0000 Subject: [PATCH 2/5] docs: describe content shortcut without unsupported options --- scripts/codegen/config/docs.py | 1 + src/browserless_bap/_page_generated.py | 2 +- src/browserless_bap/sync_api/_page_generated.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/codegen/config/docs.py b/scripts/codegen/config/docs.py index 6fab15b..7aaf9ef 100644 --- a/scripts/codegen/config/docs.py +++ b/scripts/codegen/config/docs.py @@ -4,6 +4,7 @@ LLMS_END = "" METHOD_DOCSTRINGS = { + "content": "Return the full HTML content of the page as a string.", "watch_event": ( "Start tracking `event` now so a later `wait_for_event` call can return " "immediately if that event has already fired." diff --git a/src/browserless_bap/_page_generated.py b/src/browserless_bap/_page_generated.py index 205df38..a2aff57 100644 --- a/src/browserless_bap/_page_generated.py +++ b/src/browserless_bap/_page_generated.py @@ -301,7 +301,7 @@ async def html( async def content( self, ) -> str: - 'Returns the HTML content of the page or selector when specified. This API can also "clean" HTML markup returned by specifying a "clean" argument with numerous options. Features of the "clean" argument include removal of non-text nodes, removal of DOM attributes, as well as removal of excessive whitespace and newlines. Using "clean" can save nearly 1,000 times the payload size. Useful for LLM\'s and other scenarios' + "Return the full HTML content of the page as a string." args = None result = await self._execute( "html", diff --git a/src/browserless_bap/sync_api/_page_generated.py b/src/browserless_bap/sync_api/_page_generated.py index aa0909a..c8940c9 100644 --- a/src/browserless_bap/sync_api/_page_generated.py +++ b/src/browserless_bap/sync_api/_page_generated.py @@ -219,7 +219,7 @@ def html( def content( self, ) -> str: - 'Returns the HTML content of the page or selector when specified. This API can also "clean" HTML markup returned by specifying a "clean" argument with numerous options. Features of the "clean" argument include removal of non-text nodes, removal of DOM attributes, as well as removal of excessive whitespace and newlines. Using "clean" can save nearly 1,000 times the payload size. Useful for LLM\'s and other scenarios' + "Return the full HTML content of the page as a string." return cast( "str", self._invoke( From 0dd4b8e29d999a4bffed63b1bcaee278861e19f4 Mon Sep 17 00:00:00 2001 From: Amp Date: Fri, 13 Nov 2026 14:27:13 +0000 Subject: [PATCH 3/5] docs: clarify cookie reads and checkbox methods --- CONTRIBUTING.md | 6 ++++++ scripts/codegen/config/docs.py | 3 +++ src/browserless_bap/_page_generated.py | 6 +++--- src/browserless_bap/sync_api/_page_generated.py | 6 +++--- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa10e45..1693a6a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,6 +80,12 @@ the schema. Schema and codegen changes must include `schema.lock` and all regenerated files. Generated files and AUTOGEN regions must not be edited manually. +Code generation formats the generated Page modules with Ruff from the active +Python environment, so install the development dependencies before running it. +Ruff has a minimum version rather than an exact pin; use the same version when +comparing generated output across environments, and run the formatting and +codegen drift checks above before submitting changes. + ## API reference The generated Python API reference is published at diff --git a/scripts/codegen/config/docs.py b/scripts/codegen/config/docs.py index 7aaf9ef..d973c25 100644 --- a/scripts/codegen/config/docs.py +++ b/scripts/codegen/config/docs.py @@ -4,6 +4,9 @@ LLMS_END = "" METHOD_DOCSTRINGS = { + "check": "Check the checkbox matching `selector`.", + "uncheck": "Uncheck the checkbox matching `selector`.", + "cookies": "Return the cookies on the page.", "content": "Return the full HTML content of the page as a string.", "watch_event": ( "Start tracking `event` now so a later `wait_for_event` call can return " diff --git a/src/browserless_bap/_page_generated.py b/src/browserless_bap/_page_generated.py index a2aff57..9a03866 100644 --- a/src/browserless_bap/_page_generated.py +++ b/src/browserless_bap/_page_generated.py @@ -82,7 +82,7 @@ async def check( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: - "Sets or un-sets the value of a checkbox on the page" + "Check the checkbox matching `selector`." args = drop_unset( { "selector": selector, @@ -112,7 +112,7 @@ async def uncheck( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: - "Sets or un-sets the value of a checkbox on the page" + "Uncheck the checkbox matching `selector`." args = drop_unset( { "selector": selector, @@ -192,7 +192,7 @@ async def set_content( async def cookies( self, ) -> list[StandardCookie]: - "Sets and gets cookies on the page" + "Return the cookies on the page." args = None result = await self._execute( "cookies", diff --git a/src/browserless_bap/sync_api/_page_generated.py b/src/browserless_bap/sync_api/_page_generated.py index c8940c9..28c116a 100644 --- a/src/browserless_bap/sync_api/_page_generated.py +++ b/src/browserless_bap/sync_api/_page_generated.py @@ -64,7 +64,7 @@ def check( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: - "Sets or un-sets the value of a checkbox on the page" + "Check the checkbox matching `selector`." return cast( "ClickResponse", self._invoke( @@ -86,7 +86,7 @@ def uncheck( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: - "Sets or un-sets the value of a checkbox on the page" + "Uncheck the checkbox matching `selector`." return cast( "ClickResponse", self._invoke( @@ -144,7 +144,7 @@ def set_content( def cookies( self, ) -> list[StandardCookie]: - "Sets and gets cookies on the page" + "Return the cookies on the page." return cast( "list[StandardCookie]", self._invoke( From 28ed7519354b2f1a709573d869ee31147bb0ef2f Mon Sep 17 00:00:00 2001 From: Amp Date: Fri, 13 Nov 2026 17:04:42 +0000 Subject: [PATCH 4/5] test: check timeout semantics across supported Python versions --- tests/sync/test_bridge.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/sync/test_bridge.py b/tests/sync/test_bridge.py index 4206888..3b6b127 100644 --- a/tests/sync/test_bridge.py +++ b/tests/sync/test_bridge.py @@ -57,7 +57,9 @@ async def fail() -> None: try: with pytest.raises(type(error)) as caught: bridge.run(fail(), timeout=1) - assert caught.value is error + # Python 3.11/3.12 asyncio may recreate builtin TimeoutError instances. + assert type(caught.value) is type(error) + assert caught.value.args == error.args finally: bridge.close() From e61cca23dbbc8db7824ff1fbd8e611e450b403d4 Mon Sep 17 00:00:00 2001 From: Amp Date: Fri, 13 Nov 2026 17:12:22 +0000 Subject: [PATCH 5/5] docs: use renamed SDK error for connection timeouts --- README.md | 2 +- llms.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c71b1d5..5d0cec2 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ once, allowing the server to return its timeout response before the client deadl `connect_timeout` defaults to `timeout` and bounds the entire WebSocket dial, including connection setup and handshake. A value of `0` or less disables that -connection deadline. Connection timeouts raise the SDK's `browserless_bap.TimeoutError` +connection deadline. Connection timeouts raise the SDK's `browserless_bap.BAPTimeoutError` with `operation == "connect"`. A connect deadline expiring is terminal for that page's connection. To retry, diff --git a/llms.txt b/llms.txt index fee5a9f..c58f0c3 100644 --- a/llms.txt +++ b/llms.txt @@ -30,7 +30,7 @@ All timeout values are milliseconds; the default per-step `timeout` is `30000`. Mutations with server-side timeouts get `5000` ms of client headroom once. `connect_timeout` defaults to `timeout`, bounds the entire WebSocket dial (including setup and handshake), and is disabled by values of `0` or less. -Connection timeouts raise the SDK's `browserless_bap.TimeoutError` with +Connection timeouts raise the SDK's `browserless_bap.BAPTimeoutError` with `operation == "connect"`. A connect deadline expiring is terminal for that page's connection; create a