diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa10e45..1693a6a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,6 +80,12 @@ the schema. Schema and codegen changes must include `schema.lock` and all regenerated files. Generated files and AUTOGEN regions must not be edited manually. +Code generation formats the generated Page modules with Ruff from the active +Python environment, so install the development dependencies before running it. +Ruff has a minimum version rather than an exact pin; use the same version when +comparing generated output across environments, and run the formatting and +codegen drift checks above before submitting changes. + ## API reference The generated Python API reference is published at diff --git a/README.md b/README.md index 687dfae..5d0cec2 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ once, allowing the server to return its timeout response before the client deadl `connect_timeout` defaults to `timeout` and bounds the entire WebSocket dial, including connection setup and handshake. A value of `0` or less disables that -connection deadline. Connection timeouts raise the SDK's `browserless_bap.TimeoutError` +connection deadline. Connection timeouts raise the SDK's `browserless_bap.BAPTimeoutError` with `operation == "connect"`. A connect deadline expiring is terminal for that page's connection. To retry, @@ -108,8 +108,8 @@ contents between the markers by hand. | `content` | `html` | `str` | ✅ | | `cookies` | `cookies` | `list[StandardCookie]` | ✅ | | `emulate_media` | `emulateMediaType` | `EmulateMediaTypeResponse` | ✅ | -| `eval_on_selector` | `text` | `str` | ✅ | -| `eval_on_selector_all` | `mapSelector` | `list[MapSelectorResponse]` | ✅ | +| `eval_on_selector` | `text` | `str` | — | +| `eval_on_selector_all` | `mapSelector` | `list[MapSelectorResponse]` | — | | `evaluate` | `evaluate` | `str \| None` | ✅ | | `fulfill` | `fulfill` | `FulfillResponse` | — | | `go_back` | `back` | `HTTPResponse \| None` | ✅ | @@ -162,6 +162,26 @@ contents between the markers by hand. +## Response keys and optional fields + +Options and keyword arguments use `snake_case`; returned dictionaries keep the +server's field names (`liveURLId`, `httpOnly`, `childElementCount`), as Playwright +for Python does for cookies. + +`query_selector()` and `query_selector_all()` fetch element HTML only with +`html=True`. Reading a handle's `inner_html` or `outer_html` without requesting it +raises `BAPValueError` with a hint. This also applies to element-scoped queries. +`response()` omits bodies by default; pass `body=True` to fetch them, or +`body_encoding=True` to fetch both the body and its `base64Encoded` flag. + +`eval_on_selector()` returns the first matching element's text, and +`eval_on_selector_all()` returns element details. Unlike Playwright's methods, +neither accepts a page function. + +SDK exceptions are `BAPTimeoutError`, `BAPConnectionError` and `BAPValueError`. +They still subclass Python's matching built-in exceptions; the builtin-named SDK +exports have been removed. + ## Schema and generated code `src/browserless_bap/mutations.graphql` is the vendored GraphQL schema. Its full digest, @@ -237,7 +257,7 @@ card, import `LiveURLInstructions` from `browserless_bap.types`. The SDK subscribes before minting and retains completion until the next mint, so a quick completion is not lost between `live_url()` and `wait_for_live_complete()`. -Wait timeouts are in milliseconds and raise the SDK `TimeoutError`. By default +Wait timeouts are in milliseconds and raise `BAPTimeoutError`. By default there is no client wait deadline; pass a positive timeout to bound the wait. `timeout=0` also disables the wait deadline. Timing out or cancelling a wait does not close the live URL. diff --git a/llms.txt b/llms.txt index 07c0cf6..c58f0c3 100644 --- a/llms.txt +++ b/llms.txt @@ -30,7 +30,7 @@ All timeout values are milliseconds; the default per-step `timeout` is `30000`. Mutations with server-side timeouts get `5000` ms of client headroom once. `connect_timeout` defaults to `timeout`, bounds the entire WebSocket dial (including setup and handshake), and is disabled by values of `0` or less. -Connection timeouts raise the SDK's `browserless_bap.TimeoutError` with +Connection timeouts raise the SDK's `browserless_bap.BAPTimeoutError` with `operation == "connect"`. A connect deadline expiring is terminal for that page's connection; create a @@ -45,6 +45,23 @@ Generated Page methods: `add_cookies`, `add_script_tag`, `add_style_tag`, `authe +## Response keys and optional fields + +Options and keyword arguments use `snake_case`; returned dictionaries keep the +server's field names (`liveURLId`, `httpOnly`, `childElementCount`), as Playwright +for Python does for cookies. + +`query_selector()` and `query_selector_all()` fetch element HTML only with +`html=True`, including element-scoped queries. Reading `inner_html` or +`outer_html` without requesting it raises `BAPValueError` with a fix hint. +`response()` omits bodies unless `body=True` or `body_encoding=True` (which implies +the body and adds `base64Encoded`). + +`eval_on_selector()` returns text; `eval_on_selector_all()` returns element +details. Unlike Playwright's methods, neither accepts a page function. +SDK errors are `BAPTimeoutError`, `BAPConnectionError` and `BAPValueError`, still +subclassing the matching builtins; the builtin-named SDK exports are removed. + ## Live URL handoffs ```python @@ -84,7 +101,7 @@ card, import `LiveURLInstructions` from `browserless_bap.types`. The SDK subscribes before minting and retains completion until the next mint, so a quick completion is not lost between `live_url()` and `wait_for_live_complete()`. -Wait timeouts are in milliseconds and raise the SDK `TimeoutError`. By default +Wait timeouts are in milliseconds and raise `BAPTimeoutError`. By default there is no client wait deadline; pass a positive timeout to bound the wait. `timeout=0` also disables the wait deadline. Timing out or cancelling a wait does not close the live URL. diff --git a/scripts/codegen/config/client_only.py b/scripts/codegen/config/client_only.py index 3f0dab1..39c321d 100644 --- a/scripts/codegen/config/client_only.py +++ b/scripts/codegen/config/client_only.py @@ -8,11 +8,18 @@ class ClientParam: name: str annotation: str transport_timeout: bool = False + opt_in: str | None = None CLIENT_ONLY_PARAMS = { - "GotoOptions": (ClientParam("remote_response_metadata", "bool"),), - "ResponseFilterOptions": (ClientParam("body_encoding", "bool"),), + "GotoOptions": ( + ClientParam("remote_response_metadata", "bool", opt_in="remoteMetadata"), + ), + "ResponseFilterOptions": ( + ClientParam("body", "bool", opt_in="body"), + ClientParam("body_encoding", "bool", opt_in="bodyEncoding"), + ), + "QuerySelectorOptions": (ClientParam("html", "bool", opt_in="html"),), "ScreenshotOptions": (ClientParam("path", "str"),), "PDFOptions": (ClientParam("path", "str"),), "AddScriptTagOptions": (ClientParam("path", "str"),), diff --git a/scripts/codegen/config/docs.py b/scripts/codegen/config/docs.py index 3717827..d973c25 100644 --- a/scripts/codegen/config/docs.py +++ b/scripts/codegen/config/docs.py @@ -4,8 +4,52 @@ LLMS_END = "" METHOD_DOCSTRINGS = { + "check": "Check the checkbox matching `selector`.", + "uncheck": "Uncheck the checkbox matching `selector`.", + "cookies": "Return the cookies on the page.", + "content": "Return the full HTML content of the page as a string.", "watch_event": ( "Start tracking `event` now so a later `wait_for_event` call can return " "immediately if that event has already fired." ), + "eval_on_selector": ( + "Return the text content of the first element matching `selector`. " + "Unlike Playwright's `eval_on_selector`, this takes no page function: " + "it is a shortcut for `text(selector=...)`." + ), + "eval_on_selector_all": ( + "Return details (`innerHTML`, `innerText`, `id`, `class`) for every element " + "matching `selector`. Unlike Playwright's `eval_on_selector_all`, this " + "takes no page function: it is a shortcut for `map_selector(selector)`." + ), + "query_selector": ( + "Return a handle for the first element matching `selector`, or `None`. " + "The element's `innerHTML`/`outerHTML` are fetched only with `html=True`." + ), + "query_selector_all": ( + "Return handles for every element matching `selector`. " + "The elements' `innerHTML`/`outerHTML` are fetched only with `html=True`." + ), } + + +def lead_prose(description: str | None) -> str | None: + if not description: + return None + markers = ( + "```", + "**Usage Example", + "**Field Definition", + "**Simple Example", + "**Advanced", + "Simple Example", + "Advanced Example", + "Example (", + "**Usage", + "**Example", + ) + end = min( + (description.index(marker) for marker in markers if marker in description), + default=len(description), + ) + return description[:end].strip() or None diff --git a/scripts/codegen/config/methods.py b/scripts/codegen/config/methods.py index e955ae8..3abefa8 100644 --- a/scripts/codegen/config/methods.py +++ b/scripts/codegen/config/methods.py @@ -163,8 +163,6 @@ def options_name(mutation: str) -> str: "pdf", "query_selector", "query_selector_all", - "eval_on_selector", - "eval_on_selector_all", "wait_for_selector", "wait_for_function", "evaluate", diff --git a/scripts/codegen/config/schema_map.py b/scripts/codegen/config/schema_map.py index fd78556..8880776 100644 --- a/scripts/codegen/config/schema_map.py +++ b/scripts/codegen/config/schema_map.py @@ -11,6 +11,7 @@ } RESPONSE_TYPE_OVERRIDES = {"Headers": "RequestHeader"} +RESPONSE_FIELD_TYPE_OVERRIDES = {("QuerySelectorResponse", "childElementCount"): "int"} RESPONSE_RENAME_EXCLUDE = {"WaitForFunction", "WaitForEvent"} SCALAR_MAP = { @@ -34,13 +35,8 @@ } SUBSCRIPTION_OMIT_FIELDS = {"response": {"body"}} -UNSUPPORTED_FIELDS = { - "HTTPResponse": { - "fields": {"ip", "port"}, - "variant": "HTTPResponseWithRemoteMetadata", - }, - "ResponseResponse": { - "fields": {"base64Encoded"}, - "variant": "ResponseResponseWithBodyEncoding", - }, +OPTIONAL_FIELDS: dict[str, dict[str, tuple[str, ...]]] = { + "HTTPResponse": {"remoteMetadata": ("ip", "port")}, + "ResponseResponse": {"body": ("body",), "bodyEncoding": ("body", "base64Encoded")}, + "QuerySelectorResponse": {"html": ("innerHTML", "outerHTML")}, } diff --git a/scripts/codegen/emit_operations.py b/scripts/codegen/emit_operations.py index 015659e..f61dffc 100644 --- a/scripts/codegen/emit_operations.py +++ b/scripts/codegen/emit_operations.py @@ -4,21 +4,25 @@ # ruff: noqa: E501 import json -from .config.schema_map import UNSUPPORTED_FIELDS +from .config.schema_map import OPTIONAL_FIELDS from .schema import Schema, render_graphql_type from .writer import HEADER def response_fields(schema: Schema) -> dict[str, str]: - fields = { + return { schema.response_name(name): schema.response_selection(name) for name in schema.response_objects } - for name, config in UNSUPPORTED_FIELDS.items(): - fields[str(config["variant"])] = schema.response_selection( - name, keep_unsupported=True - ) - return fields + + +def optional_fields(schema: Schema) -> dict[str, dict[str, str]]: + return { + schema.response_name(name): { + group: " ".join(fields) for group, fields in groups.items() + } + for name, groups in OPTIONAL_FIELDS.items() + } def argument_types(schema: Schema) -> dict[str, dict[str, str]]: @@ -40,6 +44,8 @@ def emit(schema: Schema) -> str: "# fmt: off", f"RESPONSE_FIELDS: dict[str, str] = {json.dumps(response_fields(schema), sort_keys=True, indent=4)}", "", + f"OPTIONAL_FIELDS: dict[str, dict[str, str]] = {json.dumps(optional_fields(schema), sort_keys=True, indent=4)}", + "", f"ARG_TYPES: dict[str, dict[str, str]] = {json.dumps(argument_types(schema), sort_keys=True, indent=4)}", "# fmt: on", "", diff --git a/scripts/codegen/emit_page.py b/scripts/codegen/emit_page.py index 800fd1b..3531bc2 100644 --- a/scripts/codegen/emit_page.py +++ b/scripts/codegen/emit_page.py @@ -7,7 +7,6 @@ import json -from .config.docs import METHOD_DOCSTRINGS from .model import MethodModel from .writer import HEADER @@ -73,6 +72,7 @@ def _execute_lines(model: MethodModel, prefix: str) -> list[str]: ] if timeout is not None: lines.append(f" timeout=None if {timeout} is UNSET else {timeout},") + lines.extend(_opt_in_lines(model, " ")) lines.append(" )") return lines @@ -91,16 +91,33 @@ def _cast_execute_lines(model: MethodModel) -> list[str]: lines.append( f" timeout=None if {timeout} is UNSET else {timeout}," ) + lines.extend(_opt_in_lines(model, " ")) lines.extend((" ),", " )")) return lines +def _opt_in_lines(model: MethodModel, indent: str) -> list[str]: + params = [param for param in model.params if param.opt_in is not None] + if not params: + return [] + entries = ", ".join( + f"({json.dumps(param.opt_in)}, {param.py_name})" for param in params + ) + return [ + f"{indent}opt_in=tuple(", + f"{indent} group", + f"{indent} for group, enabled in ({entries},)", + f"{indent} if enabled is True", + f"{indent}),", + ] + + def _async_method(model: MethodModel) -> list[str]: lines = [f" async def {model.py_name}("] lines.extend(_signature_lines(model, " ")) lines.append(f" ) -> {model.return_type}:") - if docstring := METHOD_DOCSTRINGS.get(model.py_name): - lines.append(f" {json.dumps(docstring)}") + if docstring := model.doc: + lines.append(f" {json.dumps(docstring, ensure_ascii=False)}") lines.extend(_argument_lines(model)) if model.flavor == "buffer": has_path = any(param.py_name == "path" for param in model.params) @@ -181,6 +198,7 @@ def emit_async(models: list[MethodModel]) -> str: "# ruff: noqa: ASYNC109, F403, F405", "from __future__ import annotations", "", + "from collections.abc import Sequence", "from typing import TYPE_CHECKING, Any, cast", "", "from ._codec import decode_base64, drop_unset, or_default, write_binary_file", @@ -200,6 +218,7 @@ def emit_async(models: list[MethodModel]) -> str: " response_type: str,", " *,", " timeout: float | None = None,", + " opt_in: Sequence[str] = (),", " ) -> Any:", " raise NotImplementedError", "", @@ -258,8 +277,8 @@ def emit_sync(models: list[MethodModel]) -> str: lines.append(f" def {model.py_name}(") lines.extend(_signature_lines(model, " ")) lines.append(f" ) -> {model.return_type}:") - if docstring := METHOD_DOCSTRINGS.get(model.py_name): - lines.append(f" {json.dumps(docstring)}") + if docstring := model.doc: + lines.append(f" {json.dumps(docstring, ensure_ascii=False)}") lines.extend(_sync_result_lines(model)) lines.append("") return "\n".join(lines) diff --git a/scripts/codegen/emit_responses.py b/scripts/codegen/emit_responses.py index 4476b5f..aab94a4 100644 --- a/scripts/codegen/emit_responses.py +++ b/scripts/codegen/emit_responses.py @@ -3,6 +3,7 @@ import json import keyword +from .config.schema_map import RESPONSE_FIELD_TYPE_OVERRIDES from .schema import Schema from .writer import HEADER @@ -32,7 +33,12 @@ def emit(schema: Schema) -> str: lines.append(f' "{name}",') lines.append(" {") for field in fields: - annotation = _nullable(schema.python_type(field.type, response=True)) + annotation = _nullable( + RESPONSE_FIELD_TYPE_OVERRIDES.get( + (gql_name, field.name.value), + schema.python_type(field.type, response=True), + ) + ) lines.append( f" {json.dumps(field.name.value)}: " f"{json.dumps(f'NotRequired[{annotation}]')}," @@ -43,7 +49,12 @@ def emit(schema: Schema) -> str: if not fields: lines.append(" pass") for field in fields: - annotation = _nullable(schema.python_type(field.type, response=True)) + annotation = _nullable( + RESPONSE_FIELD_TYPE_OVERRIDES.get( + (gql_name, field.name.value), + schema.python_type(field.type, response=True), + ) + ) lines.append(f" {field.name.value}: NotRequired[{annotation}]") lines.extend(("", "")) return "\n".join(lines).rstrip() diff --git a/scripts/codegen/generate.py b/scripts/codegen/generate.py index eb3df1a..8a6e1b6 100644 --- a/scripts/codegen/generate.py +++ b/scripts/codegen/generate.py @@ -1,5 +1,7 @@ from __future__ import annotations +import subprocess +import sys from pathlib import Path from .config.docs import LLMS_END, LLMS_START, README_END, README_START @@ -50,6 +52,16 @@ def generate(root: Path, *, docs_output: Path | None = None) -> list[Path]: root / path: content for path, content in zip(GENERATED_OUTPUTS, rendered, strict=True) } + for path in outputs: + if path.name == "_page_generated.py": + outputs[path] = subprocess.run( + [sys.executable, "-m", "ruff", "format", "--stdin-filename", str(path)], + input=outputs[path], + capture_output=True, + text=True, + check=True, + cwd=root, + ).stdout changed = [ path for path, content in outputs.items() if write_if_changed(path, content) ] diff --git a/scripts/codegen/model.py b/scripts/codegen/model.py index 2f928fd..2972d25 100644 --- a/scripts/codegen/model.py +++ b/scripts/codegen/model.py @@ -7,6 +7,7 @@ from graphql.language import FieldDefinitionNode from .config.client_only import CLIENT_ONLY_PARAMS +from .config.docs import METHOD_DOCSTRINGS, lead_prose from .config.methods import ( METHOD_SPECS, NULLABLE_RETURNS, @@ -27,6 +28,7 @@ class Param: positional: bool required: bool transport_timeout: bool = False + opt_in: str | None = None @dataclass(frozen=True) @@ -41,6 +43,7 @@ class MethodModel: unwrap_field: str | None = None unwrap_default: object = None wrap: str | None = None + doc: str | None = None @property def transport_timeout_param(self) -> str | None: @@ -107,6 +110,7 @@ def _params( client.transport_timeout if p.py_name == client.name else p.transport_timeout, + client.opt_in if p.py_name == client.name else p.opt_in, ) for p in params ] @@ -119,6 +123,7 @@ def _params( positional=False, required=False, transport_timeout=client.transport_timeout, + opt_in=client.opt_in, ) ) positional_order = {name: index for index, name in enumerate(spec.positional)} @@ -184,6 +189,10 @@ def build_models(schema: Schema) -> list[MethodModel]: unwrap_field=unwrap_field, unwrap_default=unwrap_default, wrap=spec.wrap, + doc=METHOD_DOCSTRINGS.get(method_name(spec.method)) + or lead_prose( + field.description.value if field.description else None + ), ) ) return models diff --git a/scripts/codegen/schema.py b/scripts/codegen/schema.py index 07956f5..fec03ce 100644 --- a/scripts/codegen/schema.py +++ b/scripts/codegen/schema.py @@ -33,11 +33,12 @@ ENUM_NAME_OVERRIDES, EXCLUDED_MUTATIONS, EXCLUDED_TYPES, + OPTIONAL_FIELDS, + RESPONSE_FIELD_TYPE_OVERRIDES, RESPONSE_RENAME_EXCLUDE, RESPONSE_TYPE_OVERRIDES, SCALAR_MAP, SUBSCRIPTION_OMIT_FIELDS, - UNSUPPORTED_FIELDS, ) @@ -190,15 +191,14 @@ def response_selection( graphql_type: str, *, omit: Iterable[str] = (), - keep_unsupported: bool = False, _seen: frozenset[str] = frozenset(), ) -> str: node = self.objects.get(graphql_type) if node is None: return "" excluded = set(omit) - if not keep_unsupported: - excluded.update(UNSUPPORTED_FIELDS.get(graphql_type, {}).get("fields", ())) + for group in OPTIONAL_FIELDS.get(graphql_type, {}).values(): + excluded.update(group) parts: list[str] = [] for field in node.fields or (): name = field.name.value @@ -263,9 +263,36 @@ def _validate_config(self) -> None: errors.append( f"SUBSCRIPTION_OMIT_FIELDS contains unknown event {event!r}" ) - for type_name in UNSUPPORTED_FIELDS: - if type_name not in self.objects: - errors.append(f"UNSUPPORTED_FIELDS contains unknown type {type_name!r}") + for type_name, groups in OPTIONAL_FIELDS.items(): + node = self.objects.get(type_name) + if node is None: + errors.append(f"OPTIONAL_FIELDS contains unknown type {type_name!r}") + continue + actual_fields = {field.name.value for field in node.fields or ()} + for group in groups.values(): + for name in group: + if name not in actual_fields: + errors.append( + f"OPTIONAL_FIELDS[{type_name!r}] contains unknown " + f"field {name!r}" + ) + for field in fields.values(): + groups = OPTIONAL_FIELDS.get(named_type(field.type), {}) + for client in CLIENT_ONLY_PARAMS.get(options_name(field.name.value), ()): + if client.opt_in is not None and client.opt_in not in groups: + errors.append( + f"CLIENT_ONLY_PARAMS[{options_name(field.name.value)!r}] " + f"contains unknown optional field group {client.opt_in!r}" + ) + for type_name, field_name in RESPONSE_FIELD_TYPE_OVERRIDES: + node = self.objects.get(type_name) + if node is None or field_name not in { + field.name.value for field in node.fields or () + }: + errors.append( + "RESPONSE_FIELD_TYPE_OVERRIDES contains unknown field " + f"{type_name}.{field_name}" + ) builtin_scalars = {"String", "Int", "Float", "Boolean", "ID"} for scalar in self.scalars - builtin_scalars: if scalar not in SCALAR_MAP: diff --git a/src/browserless_bap/__init__.py b/src/browserless_bap/__init__.py index 77273c6..411dbae 100644 --- a/src/browserless_bap/__init__.py +++ b/src/browserless_bap/__init__.py @@ -9,10 +9,7 @@ BAPTimeoutError, BAPValueError, BrowserQLError, - ConnectionError, SchemaMismatchError, - TimeoutError, - ValueError, ) from .page import Page @@ -24,12 +21,9 @@ "Browser", "BrowserQLError", "Browserless", - "ConnectionError", "ElementHandle", "Page", "SchemaMismatchError", - "TimeoutError", - "ValueError", "__version__", "sync_api", ] diff --git a/src/browserless_bap/_page_generated.py b/src/browserless_bap/_page_generated.py index 3887acf..9a03866 100644 --- a/src/browserless_bap/_page_generated.py +++ b/src/browserless_bap/_page_generated.py @@ -2,6 +2,7 @@ # ruff: noqa: ASYNC109, F403, F405 from __future__ import annotations +from collections.abc import Sequence from typing import TYPE_CHECKING, Any, cast from ._codec import decode_base64, drop_unset, or_default, write_binary_file @@ -21,6 +22,7 @@ async def _execute( response_type: str, *, timeout: float | None = None, + opt_in: Sequence[str] = (), ) -> Any: raise NotImplementedError @@ -31,6 +33,7 @@ async def authenticate( *, origin: str | UnsetType = UNSET, ) -> HTTPResponse | None: + "Provides credentials for HTTP authentication (RFC 7235). When set, any HTTP\nauthentication challenge (401/407) will be answered with the supplied\ncredentials automatically. If an `origin` is provided, only challenges whose\norigin matches will use these credentials; otherwise every challenge is\nanswered (Puppeteer-style)." args = drop_unset( { "username": username, @@ -53,6 +56,7 @@ async def go_back( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Goes back in browser history, optionally accepting waitUntil and timeout arguments. Returns null if no back is possible" args = drop_unset( { "timeout": timeout, @@ -78,6 +82,7 @@ async def check( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Check the checkbox matching `selector`." args = drop_unset( { "selector": selector, @@ -107,6 +112,7 @@ async def uncheck( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Uncheck the checkbox matching `selector`." args = drop_unset( { "selector": selector, @@ -136,6 +142,7 @@ async def click( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Waits for the element to be visible, scrolls to it, then clicks on it with native events" args = drop_unset( { "selector": selector, @@ -163,6 +170,7 @@ async def set_content( wait_until: WaitUntilHistory | UnsetType = UNSET, wait_until_all: list[WaitUntilHistory] | UnsetType = UNSET, ) -> HTTPResponse | None: + "Sets the given HTML content on the page with an optional waitUntil parameter" args = drop_unset( { "html": html, @@ -184,6 +192,7 @@ async def set_content( async def cookies( self, ) -> list[StandardCookie]: + "Return the cookies on the page." args = None result = await self._execute( "cookies", @@ -204,6 +213,7 @@ async def go_forward( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Goes forward in browser history, optionally accepting waitUntil and timeout arguments. Returns null if no forward is possible" args = drop_unset( { "timeout": timeout, @@ -231,6 +241,7 @@ async def goto( referrer_policy: str | UnsetType = UNSET, remote_response_metadata: bool | UnsetType = UNSET, ) -> HTTPResponse | None: + "Navigates to a URL with an optional waitUntil parameter and timeout parameter" args = drop_unset( { "url": url, @@ -248,6 +259,13 @@ async def goto( args, "HTTPResponse", timeout=None if timeout is UNSET else timeout, + opt_in=tuple( + group + for group, enabled in ( + ("remoteMetadata", remote_response_metadata), + ) + if enabled is True + ), ), ) @@ -260,6 +278,7 @@ async def html( clean: CleanInput | UnsetType = UNSET, rewrite: HTMLRewriteInput | UnsetType = UNSET, ) -> HTMLResponse: + 'Returns the HTML content of the page or selector when specified. This API can also "clean" HTML markup returned by specifying a "clean" argument with numerous options. Features of the "clean" argument include removal of non-text nodes, removal of DOM attributes, as well as removal of excessive whitespace and newlines. Using "clean" can save nearly 1,000 times the payload size. Useful for LLM\'s and other scenarios' args = drop_unset( { "selector": selector, @@ -282,6 +301,7 @@ async def html( async def content( self, ) -> str: + "Return the full HTML content of the page as a string." args = None result = await self._execute( "html", @@ -300,6 +320,7 @@ async def set_user_agent( self, user_agent: str, ) -> UserAgentResponse: + "Sets the User-Agent string for the browser session" args = drop_unset( { "userAgent": user_agent, @@ -325,6 +346,7 @@ async def hover( x: float | UnsetType = UNSET, y: float | UnsetType = UNSET, ) -> HoverResponse: + "Waits for the element to be visible, scrolls to it, then hover on it with native events" args = drop_unset( { "selector": selector, @@ -350,6 +372,7 @@ async def set_java_script_enabled( self, enabled: bool, ) -> JavaScriptResponse: + "Sets and gets JavaScript execution on the page\n\n**Note: changing this value won't affect scripts that have already been run. It will take full effect on the next navigation.**" args = drop_unset( { "enabled": enabled, @@ -386,6 +409,7 @@ async def live_url( fail_label: str | UnsetType = UNSET, requester_name: str | UnsetType = UNSET, ) -> LiveURLResponse: + "Returns a fully-qualified, user-shareable live-URL for streaming the web-browser to an end-user, optionally interactive.\n\n**Set 30 second timer for interaction**" args = drop_unset( { "captureScale": capture_scale, @@ -425,6 +449,7 @@ async def map_selector( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> list[MapSelectorResponse]: + 'Specify a selector that returns multiple nodes in a document (similar to `document.querySelectorAll`), or JavaScript that returns a NodeList, and this API will respond with details about those DOM nodes. Similar to how "map" works in most functional programming languages and libraries. Useful for mapping over repetitive data in sites and pages like product listings or search results. This will automatically wait for the selector to be present on the page, and is configurable with the "wait" and "timeout" options.\n\nFor getting arbitrary DOM attributes back you can specify them via the `attribute(name: "data-custom-attribute")` property. This will return an object with `name` and `value` properties.\n\nYou may also continuously map further nested items as well, for instance this query might get all books on a page, and then a nested `mapSelector` call might list all sellers of that book, or shipping speeds. Hierarchy of data is preserved to pass through the hierarchical data modeled inside the DOM.\n\nThis API will always return a list of results back regardless if one or more items are found, or `null` if none are found.\n\nUsing aliases can also give the returned JSON more meaning and better model the data returned by this powerful API.' args = drop_unset( { "selector": selector, @@ -450,6 +475,7 @@ async def eval_on_selector_all( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> list[MapSelectorResponse]: + "Return details (`innerHTML`, `innerText`, `id`, `class`) for every element matching `selector`. Unlike Playwright's `eval_on_selector_all`, this takes no page function: it is a shortcut for `map_selector(selector)`." args = drop_unset( { "selector": selector, @@ -496,6 +522,7 @@ async def pdf( transfer_mode: str | UnsetType = UNSET, path: str | UnsetType = UNSET, ) -> bytes: + "Generates a PDF of the page with the print CSS media type" args = drop_unset( { "displayHeaderFooter": display_header_footer, @@ -539,6 +566,7 @@ async def preferences( *, timeout: float | UnsetType = UNSET, ) -> DefaultResponse: + "Sets configuration for the entirety of the session, replacing defaults like the 30 second timeout default" args = drop_unset( { "timeout": timeout, @@ -569,6 +597,7 @@ async def proxy( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, ) -> ProxyResponse: + 'Proxies requests, by a specified set of conditions, through either the Browserless residential proxy or through an external proxy. Only requests that match these conditions are proxied and the rest are sent from the instance\'s own IP address.\n\nUse the "server" argument to specify an external proxy for Browserless to use for requests. For requests with authentication, the username and password should be included in the URL using Basic Authentication. See the examples below for more information on how to format those URLs.\n\n**Using the browserless proxy for all requests and proxy through Brazil**' args = drop_unset( { "network": network, @@ -599,7 +628,9 @@ async def query_selector( *, timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, + html: bool | UnsetType = UNSET, ) -> ElementHandle | None: + "Return a handle for the first element matching `selector`, or `None`. The element's `innerHTML`/`outerHTML` are fetched only with `html=True`." args = drop_unset( { "selector": selector, @@ -612,6 +643,9 @@ async def query_selector( args, "QuerySelectorResponse", timeout=None if timeout is UNSET else timeout, + opt_in=tuple( + group for group, enabled in (("html", html),) if enabled is True + ), ) if result is None: return None @@ -625,7 +659,9 @@ async def query_selector_all( *, timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, + html: bool | UnsetType = UNSET, ) -> list[ElementHandle]: + "Return handles for every element matching `selector`. The elements' `innerHTML`/`outerHTML` are fetched only with `html=True`." args = drop_unset( { "selector": selector, @@ -638,6 +674,9 @@ async def query_selector_all( args, "QuerySelectorResponse", timeout=None if timeout is UNSET else timeout, + opt_in=tuple( + group for group, enabled in (("html", html),) if enabled is True + ), ) if not results: return [] @@ -654,6 +693,7 @@ async def reconnect( *, timeout: float | UnsetType = UNSET, ) -> ReconnectionResponse: + "Returns a payload with reconnection information in order to reconnect back to the same browser session" args = drop_unset( { "timeout": timeout, @@ -672,6 +712,7 @@ async def reconnect( async def stop_session_recording( self, ) -> StopSessionRecordingResponse: + "Stops the current session recording and processes the replay data, similar to closing the context. This mutation will stop RRWeb recording and upload the collected events." args = None return cast( "StopSessionRecordingResponse", @@ -692,6 +733,7 @@ async def reject( url: list[str] | UnsetType = UNSET, regex: list[str] | UnsetType = UNSET, ) -> RejectResponse: + 'Rejects requests by a specified URL pattern, method, or type and operator. You may supply a single pattern, or a list (array) of them. This mutation, by default, will reject any requests that match *any* pattern, which we call an "or" operator. To reject requests where conditions must all match, specify an "and" operator in the mutation. Note that this only has an effect when the query is executing, so scripts that return quickly will likely see assets loading in the editor as these rejections only happen when mutations are executing.\n___\n**Rejecting images or media**' args = drop_unset( { "enabled": enabled, @@ -717,6 +759,7 @@ async def reload( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Reloads the given page with an optional waitUntil parameter and timeout parameter" args = drop_unset( { "timeout": timeout, @@ -743,6 +786,7 @@ async def request( wait: bool | UnsetType = UNSET, operator: OperatorType | UnsetType = UNSET, ) -> list[RequestResponse]: + 'Returns request information made by the Browser with optional filters via arguments. You may filter the returned results by a glob-like URL-pattern, the method of the request or the type of request. Applying an operator to this will then change the behavior by either "and"ing the filters together or "or"ing them. This API will automatically wait for the request to be made if none is immediately found which you can turn off by disabling the "wait" option.\n\n**Getting all "Document" requests**' args = drop_unset( { "type": type, @@ -774,8 +818,10 @@ async def response( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, wait: bool | UnsetType = UNSET, + body: bool | UnsetType = UNSET, body_encoding: bool | UnsetType = UNSET, ) -> list[ResponseResponse]: + 'Returns response information, filtered by the provided arguments, made by the browser. You may optionally filter the returned results by a glob-like URL-pattern, the Method of the response or the Type of response. Applying an operator to this will then change the behavior by either "and"ing the filters together, or "or"ing them. This API will automatically wait for the response to be made if none is immediately found which you can turn off by disabling the "wait" option.\n\n**Getting all "Document" responses**' args = drop_unset( { "status": status, @@ -792,6 +838,14 @@ async def response( args, "ResponseResponse", timeout=None if timeout is UNSET else timeout, + opt_in=tuple( + group + for group, enabled in ( + ("body", body), + ("bodyEncoding", body_encoding), + ) + if enabled is True + ), ) return cast( "list[ResponseResponse]", @@ -814,6 +868,7 @@ async def screenshot( wait_for_images: bool | UnsetType = UNSET, path: str | UnsetType = UNSET, ) -> bytes: + "Screenshots the page or a specific selector" args = drop_unset( { "captureBeyondViewport": capture_beyond_viewport, @@ -851,6 +906,7 @@ async def scroll( x: float | UnsetType = UNSET, y: float | UnsetType = UNSET, ) -> ScrollResponse: + "Waits for a selector, then scrolls to it on the page or an x,y coordinate in pixels" args = drop_unset( { "selector": selector, @@ -879,6 +935,7 @@ async def solve( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> CaptchaResponse: + "🚨 **EXPERIMENTAL** 🚨\nSolves a captcha or other challenge. Can auto-detect the CAPTCHA type or solve a specific type.\nUses the same detection logic as BaaS to automatically identify the CAPTCHA type\nand routes to the appropriate solver. Optionally accepts a specific type to solve." args = drop_unset( { "type": type, @@ -903,6 +960,7 @@ async def solve_image_captcha( input_selector: str, timeout: float | UnsetType = UNSET, ) -> CaptchaResponse: + "Solves an image captcha using caller-provided selectors. Unlike solve, which auto-detects known captcha types, this mutation works on any site by accepting custom captchaSelector and inputSelector arguments. It detects the captcha image, solves it, and fills the result into the specified input field." args = drop_unset( { "captchaSelector": captcha_selector, @@ -928,6 +986,7 @@ async def text( visible: bool | UnsetType = UNSET, clean: CleanInput | UnsetType = UNSET, ) -> TextResponse: + "Returns the text content on the given page or by selector when specified" args = drop_unset( { "timeout": timeout, @@ -950,6 +1009,7 @@ async def eval_on_selector( self, selector: str, ) -> str: + "Return the text content of the first element matching `selector`. Unlike Playwright's `eval_on_selector`, this takes no page function: it is a shortcut for `text(selector=...)`." args = drop_unset( { "selector": selector, @@ -971,6 +1031,7 @@ async def eval_on_selector( async def title( self, ) -> str: + "Returns the title of the page that the browser is currently at" args = None result = await self._execute( "title", @@ -997,6 +1058,7 @@ async def type( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> TypeResponse: + "Types text into an element by scrolling to it, clicking it, then emitting key events for every character. Include a `\\n` (or `\\r`) in `text` to send an Enter/Return key press — useful for submitting search boxes or forms without a separate click. BrowserQL has no standalone key-press mutation, so non-Enter keys cannot be simulated; use `click` for non-keyboard submit interactions (for example, a submit button)." args = drop_unset( { "selector": selector, @@ -1025,6 +1087,7 @@ async def load_secret( *, selector: str | UnsetType = UNSET, ) -> LoadSecretResponse: + "Fills a credential resolved just-in-time from a configured 1Password\nintegration into a target input. The resolved value never appears in your\nquery or in the response. The session must be started with an integration\n(the `integrationId` connection parameter or session field), and the page's\norigin must be in the integration's allowed-domain list. Once any secret has\nbeen filled, capture and page-content readback are disabled for the rest of the\nsession — screenshots, PDFs, screencasts, Live URLs, session recording, and the\ncontent-return mutations (evaluate, html, text, querySelector, querySelectorAll,\ncookies)." args = drop_unset( { "ref": ref, @@ -1043,6 +1106,7 @@ async def load_secret( async def url( self, ) -> str: + "Returns the URL of the page that the browser is currently at" args = None result = await self._execute( "url", @@ -1063,6 +1127,7 @@ async def wait_for_navigation( timeout: float | UnsetType = UNSET, wait_until: WaitUntilGoto | UnsetType = UNSET, ) -> HTTPResponse: + "Waits for a navigation even to fire, useful for clicking an element and waiting for a page load of some" args = drop_unset( { "timeout": timeout, @@ -1086,6 +1151,7 @@ async def wait_for_network_idle( concurrency: int | UnsetType = UNSET, timeout: float | UnsetType = UNSET, ) -> WaitForTimeoutResponse: + "Wait for the page to have no more than the configured number of in-flight network requests for a period of time." args = drop_unset( { "idleTime": idle_time, @@ -1111,6 +1177,7 @@ async def wait_for_selector( visible: bool | UnsetType = UNSET, hidden: bool | UnsetType = UNSET, ) -> WaitForSelectorResponse: + "Waits for a given selector to be present in the DOM, with optional visibility" args = drop_unset( { "selector": selector, @@ -1135,6 +1202,7 @@ async def wait_for_event( *, timeout: float | UnsetType = UNSET, ) -> WaitForEvent: + "Waits for a named event to fire on the page's document or window." args = drop_unset( { "event": event, @@ -1180,6 +1248,7 @@ async def set_viewport_size( has_touch: bool | UnsetType = UNSET, landscape: bool | UnsetType = UNSET, ) -> ViewportResponse: + "Sets the viewport dimensions for the browser session" args = drop_unset( { "width": width, @@ -1208,6 +1277,7 @@ async def switch_to_window( timeout: float | UnsetType = UNSET, wait_until: WaitUntilGoto | UnsetType = UNSET, ) -> SwitchWindowResponse: + "Switches context to a popup window matching specified criteria" args = drop_unset( { "url": url, @@ -1231,6 +1301,7 @@ async def emulate_media( self, type: MediaType, ) -> EmulateMediaTypeResponse: + 'Emulates the given CSS media type for the page, mirroring the behavior of\nprinting ("print") or on-screen rendering ("screen"). Useful before a `pdf`\nor `screenshot` to control how the page renders.' args = drop_unset( { "type": type, @@ -1259,6 +1330,7 @@ async def fulfill( base64_body: str | UnsetType = UNSET, headers: list[HeaderInput] | UnsetType = UNSET, ) -> FulfillResponse: + "Fulfills matching requests with a canned response instead of letting them hit\nthe network, mirroring the request-interceptor mocking used by the REST APIs.\nMatching follows the same glob-style URL/method/type conditions as `reject`." args = drop_unset( { "url": url, @@ -1289,6 +1361,7 @@ async def markdown( timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, ) -> MarkdownResponse: + "Returns the page's content converted to Markdown. The HTML is fetched from\nthe page (optionally scoped to a selector) and converted server-side, with\nscript/style/noscript/iframe nodes stripped." args = drop_unset( { "selector": selector, @@ -1310,6 +1383,7 @@ async def close_live_url( self, live_url_id: str, ) -> CloseLiveURLResponse: + "Closes a Live URL created by this browser session." args = drop_unset( { "liveURLId": live_url_id, diff --git a/src/browserless_bap/browser.py b/src/browserless_bap/browser.py index b8b5209..b109060 100644 --- a/src/browserless_bap/browser.py +++ b/src/browserless_bap/browser.py @@ -8,7 +8,7 @@ from contextlib import asynccontextmanager from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from .errors import ValueError +from .errors import BAPValueError from .page import Page from .transport import Transport, TransportFactory @@ -28,7 +28,7 @@ def __init__( connect_timeout: float | None = None, ) -> None: if browser_ws_endpoint is None and transport is None: - raise ValueError( + raise BAPValueError( "Either `browser_ws_endpoint` or `transport` must be provided" ) self._base_url = browser_ws_endpoint @@ -43,7 +43,7 @@ async def new_page(self) -> Page: """Open a new page with its own BAP WebSocket.""" if self._closed: - raise ValueError("Browser is closed") + raise BAPValueError("Browser is closed") self._pages = [page for page in self._pages if not page.closed] transport = Transport( _with_token(self._base_url, self._token), @@ -58,7 +58,7 @@ async def new_page(self) -> Page: raise if self._closed: await transport.close() - raise ValueError("Browser is closed") + raise BAPValueError("Browser is closed") page = Page(transport) self._pages.append(page) return page diff --git a/src/browserless_bap/element_handle.py b/src/browserless_bap/element_handle.py index 82c7ac5..2466eea 100644 --- a/src/browserless_bap/element_handle.py +++ b/src/browserless_bap/element_handle.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Unpack from urllib.parse import quote +from .errors import BAPValueError from .types.options import ClickOptions, QuerySelectorOptions, TypeOptions from .types.responses import ClickResponse, HoverResponse, SelectResponse, TypeResponse @@ -46,7 +47,16 @@ def __init__( @property def inner_html(self) -> str | None: - return self._properties.get("innerHTML") + """Element HTML. Requires html=True on the query that produced this handle.""" + return self._optional_html("innerHTML", "inner_html") + + def _optional_html(self, key: str, attribute: str) -> str | None: + if key not in self._properties: + raise BAPValueError( + f"{attribute} was not requested; pass html=True to the query " + "that produced this handle" + ) + return self._properties.get(key) @property def inner_text(self) -> str | None: @@ -66,7 +76,8 @@ def local_name(self) -> str | None: @property def outer_html(self) -> str | None: - return self._properties.get("outerHTML") + """Element markup. Requires html=True on the query that produced this handle.""" + return self._optional_html("outerHTML", "outer_html") @property def child_element_count(self) -> int | None: diff --git a/src/browserless_bap/errors.py b/src/browserless_bap/errors.py index f615338..240ad23 100644 --- a/src/browserless_bap/errors.py +++ b/src/browserless_bap/errors.py @@ -8,6 +8,16 @@ ErrorCode = Literal["BQL", "CONNECTION", "TIMEOUT", "CLIENT"] +__all__ = [ + "BAPConnectionError", + "BAPError", + "BAPTimeoutError", + "BAPValueError", + "BrowserQLError", + "ErrorCode", + "SchemaMismatchError", +] + class BAPError(Exception): """Base class for errors produced by the SDK.""" @@ -32,13 +42,13 @@ def __init__(self, errors: Sequence[object]) -> None: super().__init__(message or "Unknown BrowserQL error") -class ConnectionError(BAPError, builtins.ConnectionError): +class BAPConnectionError(BAPError, builtins.ConnectionError): """The BAP transport could not connect or disconnected.""" code = "CONNECTION" -class TimeoutError(BAPError, builtins.TimeoutError): +class BAPTimeoutError(BAPError, builtins.TimeoutError): """A BAP operation exceeded its transport deadline.""" code = "TIMEOUT" @@ -57,12 +67,7 @@ class SchemaMismatchError(BAPError): code = "CLIENT" -class ValueError(BAPError, builtins.ValueError): +class BAPValueError(BAPError, builtins.ValueError): """A public SDK argument failed client-side validation.""" code = "CLIENT" - - -BAPConnectionError = ConnectionError -BAPTimeoutError = TimeoutError -BAPValueError = ValueError diff --git a/src/browserless_bap/graphql/__init__.py b/src/browserless_bap/graphql/__init__.py index 545adff..975cef5 100644 --- a/src/browserless_bap/graphql/__init__.py +++ b/src/browserless_bap/graphql/__init__.py @@ -1,11 +1,13 @@ -from .builder import build_operation -from .mutations import ARG_TYPES, RESPONSE_FIELDS +from .builder import build_operation, selection_for +from .mutations import ARG_TYPES, OPTIONAL_FIELDS, RESPONSE_FIELDS from .subscriptions import SUBSCRIPTIONS, SubscriptionSpec __all__ = [ "ARG_TYPES", + "OPTIONAL_FIELDS", "RESPONSE_FIELDS", "SUBSCRIPTIONS", "SubscriptionSpec", "build_operation", + "selection_for", ] diff --git a/src/browserless_bap/graphql/builder.py b/src/browserless_bap/graphql/builder.py index 390af0f..e96d5cb 100644 --- a/src/browserless_bap/graphql/builder.py +++ b/src/browserless_bap/graphql/builder.py @@ -2,10 +2,27 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from typing import Literal from ..errors import SchemaMismatchError +from .mutations import OPTIONAL_FIELDS, RESPONSE_FIELDS + + +def selection_for(response_type: str, opt_in: Iterable[str] = ()) -> str: + base = RESPONSE_FIELDS.get(response_type) + if base is None: + raise SchemaMismatchError(f"Unknown response type: {response_type}") + selected: dict[str, None] = {} + for group in opt_in: + fields = OPTIONAL_FIELDS.get(response_type, {}).get(group) + if fields is None: + raise SchemaMismatchError( + f'Unknown optional field group "{group}" ' + f"for response type {response_type}" + ) + selected.update(dict.fromkeys(fields.split())) + return " ".join((base, *selected)) def build_operation( @@ -34,4 +51,4 @@ def build_operation( return query, provided -__all__ = ["build_operation"] +__all__ = ["build_operation", "selection_for"] diff --git a/src/browserless_bap/graphql/mutations.py b/src/browserless_bap/graphql/mutations.py index 5800b45..b900ead 100644 --- a/src/browserless_bap/graphql/mutations.py +++ b/src/browserless_bap/graphql/mutations.py @@ -20,7 +20,6 @@ "HTTPHeaders": "name value", "HTTPHeadersResponse": "headers { name value } time", "HTTPResponse": "status time text url", - "HTTPResponseWithRemoteMetadata": "ip port status time text url", "HoverResponse": "selector time x y", "JavaScriptResponse": "enabled time", "LiveURLCompleteEvent": "liveURLId reason", @@ -30,13 +29,12 @@ "MarkdownResponse": "markdown time", "PDFResponse": "base64 size time", "ProxyResponse": "time", - "QuerySelectorResponse": "childElementCount className id innerHTML innerText localName outerHTML", + "QuerySelectorResponse": "childElementCount className id innerText localName", "ReconnectionResponse": "browserQLEndpoint browserWSEndpoint devtoolsFrontendUrl webSocketDebuggerUrl", "RejectResponse": "time enabled", "RequestHeader": "name value", "RequestResponse": "url method type headers { name value }", - "ResponseResponse": "status url method type headers { name value } body", - "ResponseResponseWithBodyEncoding": "status url method type headers { name value } body base64Encoded", + "ResponseResponse": "status url method type headers { name value }", "ScreenshotResponse": "base64 format time", "ScrollResponse": "selector time x y", "SelectResponse": "selector time value", @@ -57,6 +55,19 @@ "WaitForTimeoutResponse": "time" } +OPTIONAL_FIELDS: dict[str, dict[str, str]] = { + "HTTPResponse": { + "remoteMetadata": "ip port" + }, + "QuerySelectorResponse": { + "html": "innerHTML outerHTML" + }, + "ResponseResponse": { + "body": "body", + "bodyEncoding": "body base64Encoded" + } +} + ARG_TYPES: dict[str, dict[str, str]] = { "addScriptTag": { "content": "String", diff --git a/src/browserless_bap/page.py b/src/browserless_bap/page.py index a56c18c..dc868ee 100644 --- a/src/browserless_bap/page.py +++ b/src/browserless_bap/page.py @@ -18,10 +18,9 @@ from ._page_generated import PageGenerated from ._unset import UNSET, UnsetType from .emitter import EventEmitter, Listener -from .errors import ConnectionError, SchemaMismatchError, ValueError -from .errors import TimeoutError as BAPTimeoutError -from .graphql.builder import build_operation -from .graphql.mutations import ARG_TYPES, RESPONSE_FIELDS +from .errors import BAPConnectionError, BAPTimeoutError, BAPValueError +from .graphql.builder import build_operation, selection_for +from .graphql.mutations import ARG_TYPES from .graphql.subscriptions import SUBSCRIPTIONS from .transport import ( OPERATION_TIMEOUT_CUSHION_MS, @@ -138,13 +137,14 @@ async def _execute( response_type: str, *, timeout: float | UnsetType | None = None, + opt_in: Sequence[str] = (), ) -> Any: if mutation_name != "liveURL": return await self._execute_operation( - mutation_name, args, response_type, timeout=timeout + mutation_name, args, response_type, timeout=timeout, opt_in=opt_in ) if self._minting: - raise ValueError( + raise BAPValueError( "Await the previous live_url() call before minting another link" ) self._minting = True @@ -160,13 +160,13 @@ async def _execute( self._on_listener_count_change("liveComplete", 1) await self._wait_subscription_started("liveComplete") if "liveComplete" not in self._unsubscribes: - raise ConnectionError("Live completion subscription unavailable") + raise BAPConnectionError("Live completion subscription unavailable") result = await self._execute_operation( - mutation_name, args, response_type, timeout=timeout + mutation_name, args, response_type, timeout=timeout, opt_in=opt_in ) state.id = result.get("liveURLId") if isinstance(result, Mapping) else None if not state.id: - raise ValueError("live_url returned no liveURLId") + raise BAPValueError("live_url returned no liveURLId") if state.id in state.early and not state.future.done(): state.future.set_result(state.early[state.id]) return result @@ -175,7 +175,7 @@ async def _execute( state.future.set_exception( error if isinstance(error, Exception) - else ConnectionError("live_url() was cancelled or interrupted") + else BAPConnectionError("live_url() was cancelled or interrupted") ) raise finally: @@ -192,10 +192,9 @@ async def _execute_operation( response_type: str, *, timeout: float | UnsetType | None = None, + opt_in: Sequence[str] = (), ) -> Any: - fields = RESPONSE_FIELDS.get(response_type) - if fields is None: - raise SchemaMismatchError(f"Unknown response type: {response_type}") + fields = selection_for(response_type, opt_in) operation = build_operation( mutation_name, args, @@ -227,10 +226,10 @@ async def wait_for_live_complete( Completion or failure is retained until the next live_url() call. """ if self.closed: - raise ConnectionError("Page is closed") + raise BAPConnectionError("Page is closed") deadline = 0 if timeout is None else timeout if not math.isfinite(deadline) or deadline < 0: - raise ValueError("timeout must be a finite non-negative number") + raise BAPValueError("timeout must be a finite non-negative number") if self._live is None or ( self._live.rejected and not self._live.id and not self._minting ): @@ -308,6 +307,7 @@ async def goto( referrer_policy: str | UnsetType = UNSET, remote_response_metadata: bool | UnsetType = UNSET, ) -> HTTPResponse | None: + """Navigate to a URL with optional waitUntil and timeout parameters.""" args = _drop_unset( { "url": url, @@ -318,14 +318,15 @@ async def goto( "referrerPolicy": referrer_policy, } ) - response_type = ( - "HTTPResponseWithRemoteMetadata" - if remote_response_metadata is True - else "HTTPResponse" - ) return cast( HTTPResponse | None, - await self._execute("goto", args, response_type, timeout=timeout), + await self._execute( + "goto", + args, + "HTTPResponse", + timeout=timeout, + opt_in=("remoteMetadata",) if remote_response_metadata is True else (), + ), ) async def response( @@ -338,8 +339,13 @@ async def response( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, wait: bool | UnsetType = UNSET, + body: bool | UnsetType = UNSET, body_encoding: bool | UnsetType = UNSET, ) -> list[ResponseResponse]: + """List responses; fetch bodies only with body=True. + + body_encoding=True implies body=True and adds the base64Encoded flag. + """ args = _drop_unset( { "status": status, @@ -351,12 +357,16 @@ async def response( "wait": wait, } ) - response_type = ( - "ResponseResponseWithBodyEncoding" + opt_in = ( + ("bodyEncoding",) if body_encoding is True - else "ResponseResponse" + else ("body",) + if body is True + else () + ) + result = await self._execute( + "response", args, "ResponseResponse", timeout=timeout, opt_in=opt_in ) - result = await self._execute("response", args, response_type, timeout=timeout) return cast(list[ResponseResponse], list(result or ())) async def select_option( @@ -582,7 +592,7 @@ async def close(self) -> None: if self._closed_flag: return self._closed_flag = True - self._fail_live(ConnectionError("Page is closed")) + self._fail_live(BAPConnectionError("Page is closed")) for ready in self._subscription_ready.values(): ready.set() self._detach_transport() @@ -682,7 +692,7 @@ def on_error(error: Exception) -> None: def on_complete() -> None: clear_if_own() if event == "liveComplete": - self._fail_live(ConnectionError("Live completion stream ended")) + self._fail_live(BAPConnectionError("Live completion stream ended")) try: spec = SUBSCRIPTIONS[event] @@ -772,6 +782,6 @@ def _require_one_source( ) -> None: values = (url, path, content) if sum(value is not UNSET for value in values) != 1: - raise ValueError( + raise BAPValueError( "Exactly one of `url`, `path`, or `content` must be specified." ) diff --git a/src/browserless_bap/sync_api/_bridge.py b/src/browserless_bap/sync_api/_bridge.py index ac23a80..2392f6a 100644 --- a/src/browserless_bap/sync_api/_bridge.py +++ b/src/browserless_bap/sync_api/_bridge.py @@ -19,6 +19,8 @@ from contextlib import suppress from typing import Protocol, TypeVar, cast +from ..errors import BAPTimeoutError + T = TypeVar("T") logger = logging.getLogger("browserless_bap") @@ -247,8 +249,8 @@ def run(self, awaitable: Awaitable[T], *, timeout: float | None = None) -> T: remaining = deadline - time.monotonic() if remaining <= 0: future.cancel() - raise TimeoutError( - f"bap sync bridge timed out after {timeout} seconds" + raise BAPTimeoutError( + "sync bridge", cast(float, timeout) * 1000 ) wait_for = min(wait_for, remaining) try: diff --git a/src/browserless_bap/sync_api/_page_generated.py b/src/browserless_bap/sync_api/_page_generated.py index b012524..28c116a 100644 --- a/src/browserless_bap/sync_api/_page_generated.py +++ b/src/browserless_bap/sync_api/_page_generated.py @@ -28,6 +28,7 @@ def authenticate( *, origin: str | UnsetType = UNSET, ) -> HTTPResponse | None: + "Provides credentials for HTTP authentication (RFC 7235). When set, any HTTP\nauthentication challenge (401/407) will be answered with the supplied\ncredentials automatically. If an `origin` is provided, only challenges whose\norigin matches will use these credentials; otherwise every challenge is\nanswered (Puppeteer-style)." return cast( "HTTPResponse | None", self._invoke( @@ -44,6 +45,7 @@ def go_back( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Goes back in browser history, optionally accepting waitUntil and timeout arguments. Returns null if no back is possible" return cast( "HTTPResponse | None", self._invoke( @@ -62,6 +64,7 @@ def check( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Check the checkbox matching `selector`." return cast( "ClickResponse", self._invoke( @@ -83,6 +86,7 @@ def uncheck( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Uncheck the checkbox matching `selector`." return cast( "ClickResponse", self._invoke( @@ -104,6 +108,7 @@ def click( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> ClickResponse: + "Waits for the element to be visible, scrolls to it, then clicks on it with native events" return cast( "ClickResponse", self._invoke( @@ -124,6 +129,7 @@ def set_content( wait_until: WaitUntilHistory | UnsetType = UNSET, wait_until_all: list[WaitUntilHistory] | UnsetType = UNSET, ) -> HTTPResponse | None: + "Sets the given HTML content on the page with an optional waitUntil parameter" return cast( "HTTPResponse | None", self._invoke( @@ -138,6 +144,7 @@ def set_content( def cookies( self, ) -> list[StandardCookie]: + "Return the cookies on the page." return cast( "list[StandardCookie]", self._invoke( @@ -151,6 +158,7 @@ def go_forward( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Goes forward in browser history, optionally accepting waitUntil and timeout arguments. Returns null if no forward is possible" return cast( "HTTPResponse | None", self._invoke( @@ -171,6 +179,7 @@ def goto( referrer_policy: str | UnsetType = UNSET, remote_response_metadata: bool | UnsetType = UNSET, ) -> HTTPResponse | None: + "Navigates to a URL with an optional waitUntil parameter and timeout parameter" return cast( "HTTPResponse | None", self._invoke( @@ -194,6 +203,7 @@ def html( clean: CleanInput | UnsetType = UNSET, rewrite: HTMLRewriteInput | UnsetType = UNSET, ) -> HTMLResponse: + 'Returns the HTML content of the page or selector when specified. This API can also "clean" HTML markup returned by specifying a "clean" argument with numerous options. Features of the "clean" argument include removal of non-text nodes, removal of DOM attributes, as well as removal of excessive whitespace and newlines. Using "clean" can save nearly 1,000 times the payload size. Useful for LLM\'s and other scenarios' return cast( "HTMLResponse", self._invoke( @@ -209,6 +219,7 @@ def html( def content( self, ) -> str: + "Return the full HTML content of the page as a string." return cast( "str", self._invoke( @@ -220,6 +231,7 @@ def set_user_agent( self, user_agent: str, ) -> UserAgentResponse: + "Sets the User-Agent string for the browser session" return cast( "UserAgentResponse", self._invoke( @@ -239,6 +251,7 @@ def hover( x: float | UnsetType = UNSET, y: float | UnsetType = UNSET, ) -> HoverResponse: + "Waits for the element to be visible, scrolls to it, then hover on it with native events" return cast( "HoverResponse", self._invoke( @@ -257,6 +270,7 @@ def set_java_script_enabled( self, enabled: bool, ) -> JavaScriptResponse: + "Sets and gets JavaScript execution on the page\n\n**Note: changing this value won't affect scripts that have already been run. It will take full effect on the next navigation.**" return cast( "JavaScriptResponse", self._invoke( @@ -287,6 +301,7 @@ def live_url( fail_label: str | UnsetType = UNSET, requester_name: str | UnsetType = UNSET, ) -> LiveURLResponse: + "Returns a fully-qualified, user-shareable live-URL for streaming the web-browser to an end-user, optionally interactive.\n\n**Set 30 second timer for interaction**" return cast( "LiveURLResponse", self._invoke( @@ -319,6 +334,7 @@ def map_selector( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> list[MapSelectorResponse]: + 'Specify a selector that returns multiple nodes in a document (similar to `document.querySelectorAll`), or JavaScript that returns a NodeList, and this API will respond with details about those DOM nodes. Similar to how "map" works in most functional programming languages and libraries. Useful for mapping over repetitive data in sites and pages like product listings or search results. This will automatically wait for the selector to be present on the page, and is configurable with the "wait" and "timeout" options.\n\nFor getting arbitrary DOM attributes back you can specify them via the `attribute(name: "data-custom-attribute")` property. This will return an object with `name` and `value` properties.\n\nYou may also continuously map further nested items as well, for instance this query might get all books on a page, and then a nested `mapSelector` call might list all sellers of that book, or shipping speeds. Hierarchy of data is preserved to pass through the hierarchical data modeled inside the DOM.\n\nThis API will always return a list of results back regardless if one or more items are found, or `null` if none are found.\n\nUsing aliases can also give the returned JSON more meaning and better model the data returned by this powerful API.' return cast( "list[MapSelectorResponse]", self._invoke( @@ -336,6 +352,7 @@ def eval_on_selector_all( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> list[MapSelectorResponse]: + "Return details (`innerHTML`, `innerText`, `id`, `class`) for every element matching `selector`. Unlike Playwright's `eval_on_selector_all`, this takes no page function: it is a shortcut for `map_selector(selector)`." return cast( "list[MapSelectorResponse]", self._invoke( @@ -374,6 +391,7 @@ def pdf( transfer_mode: str | UnsetType = UNSET, path: str | UnsetType = UNSET, ) -> bytes: + "Generates a PDF of the page with the print CSS media type" return cast( "bytes", self._invoke( @@ -410,6 +428,7 @@ def preferences( *, timeout: float | UnsetType = UNSET, ) -> DefaultResponse: + "Sets configuration for the entirety of the session, replacing defaults like the 30 second timeout default" return cast( "DefaultResponse", self._invoke( @@ -433,6 +452,7 @@ def proxy( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, ) -> ProxyResponse: + 'Proxies requests, by a specified set of conditions, through either the Browserless residential proxy or through an external proxy. Only requests that match these conditions are proxied and the rest are sent from the instance\'s own IP address.\n\nUse the "server" argument to specify an external proxy for Browserless to use for requests. For requests with authentication, the username and password should be included in the URL using Basic Authentication. See the examples below for more information on how to format those URLs.\n\n**Using the browserless proxy for all requests and proxy through Brazil**' return cast( "ProxyResponse", self._invoke( @@ -457,7 +477,9 @@ def query_selector( *, timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, + html: bool | UnsetType = UNSET, ) -> ElementHandle | None: + "Return a handle for the first element matching `selector`, or `None`. The element's `innerHTML`/`outerHTML` are fetched only with `html=True`." return cast( "ElementHandle | None", self._invoke( @@ -465,6 +487,7 @@ def query_selector( selector, timeout=timeout, visible=visible, + html=html, ), ) @@ -474,7 +497,9 @@ def query_selector_all( *, timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, + html: bool | UnsetType = UNSET, ) -> list[ElementHandle]: + "Return handles for every element matching `selector`. The elements' `innerHTML`/`outerHTML` are fetched only with `html=True`." return cast( "list[ElementHandle]", self._invoke( @@ -482,6 +507,7 @@ def query_selector_all( selector, timeout=timeout, visible=visible, + html=html, ), ) @@ -490,6 +516,7 @@ def reconnect( *, timeout: float | UnsetType = UNSET, ) -> ReconnectionResponse: + "Returns a payload with reconnection information in order to reconnect back to the same browser session" return cast( "ReconnectionResponse", self._invoke( @@ -501,6 +528,7 @@ def reconnect( def stop_session_recording( self, ) -> StopSessionRecordingResponse: + "Stops the current session recording and processes the replay data, similar to closing the context. This mutation will stop RRWeb recording and upload the collected events." return cast( "StopSessionRecordingResponse", self._invoke( @@ -518,6 +546,7 @@ def reject( url: list[str] | UnsetType = UNSET, regex: list[str] | UnsetType = UNSET, ) -> RejectResponse: + 'Rejects requests by a specified URL pattern, method, or type and operator. You may supply a single pattern, or a list (array) of them. This mutation, by default, will reject any requests that match *any* pattern, which we call an "or" operator. To reject requests where conditions must all match, specify an "and" operator in the mutation. Note that this only has an effect when the query is executing, so scripts that return quickly will likely see assets loading in the editor as these rejections only happen when mutations are executing.\n___\n**Rejecting images or media**' return cast( "RejectResponse", self._invoke( @@ -537,6 +566,7 @@ def reload( timeout: float | UnsetType = UNSET, wait_until: WaitUntilHistory | UnsetType = UNSET, ) -> HTTPResponse | None: + "Reloads the given page with an optional waitUntil parameter and timeout parameter" return cast( "HTTPResponse | None", self._invoke( @@ -556,6 +586,7 @@ def request( wait: bool | UnsetType = UNSET, operator: OperatorType | UnsetType = UNSET, ) -> list[RequestResponse]: + 'Returns request information made by the Browser with optional filters via arguments. You may filter the returned results by a glob-like URL-pattern, the method of the request or the type of request. Applying an operator to this will then change the behavior by either "and"ing the filters together or "or"ing them. This API will automatically wait for the request to be made if none is immediately found which you can turn off by disabling the "wait" option.\n\n**Getting all "Document" requests**' return cast( "list[RequestResponse]", self._invoke( @@ -579,8 +610,10 @@ def response( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, wait: bool | UnsetType = UNSET, + body: bool | UnsetType = UNSET, body_encoding: bool | UnsetType = UNSET, ) -> list[ResponseResponse]: + 'Returns response information, filtered by the provided arguments, made by the browser. You may optionally filter the returned results by a glob-like URL-pattern, the Method of the response or the Type of response. Applying an operator to this will then change the behavior by either "and"ing the filters together, or "or"ing them. This API will automatically wait for the response to be made if none is immediately found which you can turn off by disabling the "wait" option.\n\n**Getting all "Document" responses**' return cast( "list[ResponseResponse]", self._invoke( @@ -592,6 +625,7 @@ def response( type=type, url=url, wait=wait, + body=body, body_encoding=body_encoding, ), ) @@ -612,6 +646,7 @@ def screenshot( wait_for_images: bool | UnsetType = UNSET, path: str | UnsetType = UNSET, ) -> bytes: + "Screenshots the page or a specific selector" return cast( "bytes", self._invoke( @@ -642,6 +677,7 @@ def scroll( x: float | UnsetType = UNSET, y: float | UnsetType = UNSET, ) -> ScrollResponse: + "Waits for a selector, then scrolls to it on the page or an x,y coordinate in pixels" return cast( "ScrollResponse", self._invoke( @@ -663,6 +699,7 @@ def solve( timeout: float | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> CaptchaResponse: + "🚨 **EXPERIMENTAL** 🚨\nSolves a captcha or other challenge. Can auto-detect the CAPTCHA type or solve a specific type.\nUses the same detection logic as BaaS to automatically identify the CAPTCHA type\nand routes to the appropriate solver. Optionally accepts a specific type to solve." return cast( "CaptchaResponse", self._invoke( @@ -680,6 +717,7 @@ def solve_image_captcha( input_selector: str, timeout: float | UnsetType = UNSET, ) -> CaptchaResponse: + "Solves an image captcha using caller-provided selectors. Unlike solve, which auto-detects known captcha types, this mutation works on any site by accepting custom captchaSelector and inputSelector arguments. It detects the captcha image, solves it, and fills the result into the specified input field." return cast( "CaptchaResponse", self._invoke( @@ -698,6 +736,7 @@ def text( visible: bool | UnsetType = UNSET, clean: CleanInput | UnsetType = UNSET, ) -> TextResponse: + "Returns the text content on the given page or by selector when specified" return cast( "TextResponse", self._invoke( @@ -713,6 +752,7 @@ def eval_on_selector( self, selector: str, ) -> str: + "Return the text content of the first element matching `selector`. Unlike Playwright's `eval_on_selector`, this takes no page function: it is a shortcut for `text(selector=...)`." return cast( "str", self._invoke( @@ -724,6 +764,7 @@ def eval_on_selector( def title( self, ) -> str: + "Returns the title of the page that the browser is currently at" return cast( "str", self._invoke( @@ -743,6 +784,7 @@ def type( visible: bool | UnsetType = UNSET, wait: bool | UnsetType = UNSET, ) -> TypeResponse: + "Types text into an element by scrolling to it, clicking it, then emitting key events for every character. Include a `\\n` (or `\\r`) in `text` to send an Enter/Return key press — useful for submitting search boxes or forms without a separate click. BrowserQL has no standalone key-press mutation, so non-Enter keys cannot be simulated; use `click` for non-keyboard submit interactions (for example, a submit button)." return cast( "TypeResponse", self._invoke( @@ -764,6 +806,7 @@ def load_secret( *, selector: str | UnsetType = UNSET, ) -> LoadSecretResponse: + "Fills a credential resolved just-in-time from a configured 1Password\nintegration into a target input. The resolved value never appears in your\nquery or in the response. The session must be started with an integration\n(the `integrationId` connection parameter or session field), and the page's\norigin must be in the integration's allowed-domain list. Once any secret has\nbeen filled, capture and page-content readback are disabled for the rest of the\nsession — screenshots, PDFs, screencasts, Live URLs, session recording, and the\ncontent-return mutations (evaluate, html, text, querySelector, querySelectorAll,\ncookies)." return cast( "LoadSecretResponse", self._invoke( @@ -776,6 +819,7 @@ def load_secret( def url( self, ) -> str: + "Returns the URL of the page that the browser is currently at" return cast( "str", self._invoke( @@ -789,6 +833,7 @@ def wait_for_navigation( timeout: float | UnsetType = UNSET, wait_until: WaitUntilGoto | UnsetType = UNSET, ) -> HTTPResponse: + "Waits for a navigation even to fire, useful for clicking an element and waiting for a page load of some" return cast( "HTTPResponse", self._invoke( @@ -805,6 +850,7 @@ def wait_for_network_idle( concurrency: int | UnsetType = UNSET, timeout: float | UnsetType = UNSET, ) -> WaitForTimeoutResponse: + "Wait for the page to have no more than the configured number of in-flight network requests for a period of time." return cast( "WaitForTimeoutResponse", self._invoke( @@ -823,6 +869,7 @@ def wait_for_selector( visible: bool | UnsetType = UNSET, hidden: bool | UnsetType = UNSET, ) -> WaitForSelectorResponse: + "Waits for a given selector to be present in the DOM, with optional visibility" return cast( "WaitForSelectorResponse", self._invoke( @@ -840,6 +887,7 @@ def wait_for_event( *, timeout: float | UnsetType = UNSET, ) -> WaitForEvent: + "Waits for a named event to fire on the page's document or window." return cast( "WaitForEvent", self._invoke( @@ -872,6 +920,7 @@ def set_viewport_size( has_touch: bool | UnsetType = UNSET, landscape: bool | UnsetType = UNSET, ) -> ViewportResponse: + "Sets the viewport dimensions for the browser session" return cast( "ViewportResponse", self._invoke( @@ -894,6 +943,7 @@ def switch_to_window( timeout: float | UnsetType = UNSET, wait_until: WaitUntilGoto | UnsetType = UNSET, ) -> SwitchWindowResponse: + "Switches context to a popup window matching specified criteria" return cast( "SwitchWindowResponse", self._invoke( @@ -910,6 +960,7 @@ def emulate_media( self, type: MediaType, ) -> EmulateMediaTypeResponse: + 'Emulates the given CSS media type for the page, mirroring the behavior of\nprinting ("print") or on-screen rendering ("screen"). Useful before a `pdf`\nor `screenshot` to control how the page renders.' return cast( "EmulateMediaTypeResponse", self._invoke( @@ -932,6 +983,7 @@ def fulfill( base64_body: str | UnsetType = UNSET, headers: list[HeaderInput] | UnsetType = UNSET, ) -> FulfillResponse: + "Fulfills matching requests with a canned response instead of letting them hit\nthe network, mirroring the request-interceptor mocking used by the REST APIs.\nMatching follows the same glob-style URL/method/type conditions as `reject`." return cast( "FulfillResponse", self._invoke( @@ -956,6 +1008,7 @@ def markdown( timeout: float | UnsetType = UNSET, visible: bool | UnsetType = UNSET, ) -> MarkdownResponse: + "Returns the page's content converted to Markdown. The HTML is fetched from\nthe page (optionally scoped to a selector) and converted server-side, with\nscript/style/noscript/iframe nodes stripped." return cast( "MarkdownResponse", self._invoke( @@ -970,6 +1023,7 @@ def close_live_url( self, live_url_id: str, ) -> CloseLiveURLResponse: + "Closes a Live URL created by this browser session." return cast( "CloseLiveURLResponse", self._invoke( diff --git a/src/browserless_bap/sync_api/browserless.py b/src/browserless_bap/sync_api/browserless.py index 69bf68a..1625802 100644 --- a/src/browserless_bap/sync_api/browserless.py +++ b/src/browserless_bap/sync_api/browserless.py @@ -2,16 +2,17 @@ from __future__ import annotations -from typing import Any, Never - from browserless_bap.browserless import Browserless as AsyncBrowserless +from browserless_bap.transport import TransportFactory from ._bridge import get_loop_thread from .browser import Browser class Browserless: - def __new__(cls, *args: object, **kwargs: object) -> Never: + """Factory namespace for blocking browser sessions.""" + + def __new__(cls) -> Browserless: raise TypeError("Browserless is a namespace; call Browserless.connect(...)") @staticmethod @@ -20,9 +21,11 @@ def connect( browser_ws_endpoint: str | None = None, token: str | None = None, timeout: float = 30_000, - transport: Any = None, + transport: TransportFactory | None = None, connect_timeout: float | None = None, ) -> Browser: + """Create a blocking browser for a BAP-compatible WebSocket endpoint.""" + bridge = get_loop_thread() impl = AsyncBrowserless.connect( browser_ws_endpoint=browser_ws_endpoint, diff --git a/src/browserless_bap/sync_api/page.py b/src/browserless_bap/sync_api/page.py index f8472ee..02ec3b0 100644 --- a/src/browserless_bap/sync_api/page.py +++ b/src/browserless_bap/sync_api/page.py @@ -176,6 +176,7 @@ def goto( referrer_policy: str | UnsetType = UNSET, remote_response_metadata: bool | UnsetType = UNSET, ) -> HTTPResponse | None: + """Navigate to a URL with optional waitUntil and timeout parameters.""" return cast( HTTPResponse | None, self._invoke( @@ -200,8 +201,13 @@ def response( type: list[ResourceType] | UnsetType = UNSET, url: list[str] | UnsetType = UNSET, wait: bool | UnsetType = UNSET, + body: bool | UnsetType = UNSET, body_encoding: bool | UnsetType = UNSET, ) -> list[ResponseResponse]: + """List responses; fetch bodies only with body=True. + + body_encoding=True implies body=True and adds the base64Encoded flag. + """ return cast( list[ResponseResponse], self._invoke( @@ -213,6 +219,7 @@ def response( type=type, url=url, wait=wait, + body=body, body_encoding=body_encoding, ), ) diff --git a/src/browserless_bap/transport.py b/src/browserless_bap/transport.py index b2ac4fc..6a8cbfa 100644 --- a/src/browserless_bap/transport.py +++ b/src/browserless_bap/transport.py @@ -20,7 +20,7 @@ from websockets.exceptions import ConnectionClosed, ConnectionClosedOK from websockets.typing import Subprotocol -from .errors import BrowserQLError, ConnectionError, TimeoutError +from .errors import BAPConnectionError, BAPTimeoutError, BrowserQLError try: from ._version import __version__ @@ -185,7 +185,7 @@ def connect(self) -> Awaitable[None]: async def _connect(self) -> None: if self._closed: - raise ConnectionError("Transport is closed") + raise BAPConnectionError("Transport is closed") deadline = self.operation_timeout(self._connect_timeout) connection: ConnectionTransport | None = None cm = asyncio.timeout(deadline / 1000 if deadline > 0 else None) @@ -213,18 +213,18 @@ async def _connect(self) -> None: return except builtins.TimeoutError as exc: error: Exception = ( - TimeoutError("connect", deadline) + BAPTimeoutError("connect", deadline) if cm.expired() - else ConnectionError( + else BAPConnectionError( f"WebSocket connection failed: {_safe_error_message(exc)}" ) ) - except ConnectionError as exc: - error = ConnectionError(_safe_error_message(exc)) + except BAPConnectionError as exc: + error = BAPConnectionError(_safe_error_message(exc)) except asyncio.CancelledError: raise except BaseException as exc: - error = ConnectionError( + error = BAPConnectionError( f"WebSocket connection failed: {_safe_error_message(exc)}" ) finally: @@ -239,7 +239,7 @@ async def _connect(self) -> None: async def _dial(self) -> ConnectionTransport: if self._url is None: - raise ConnectionError( + raise BAPConnectionError( "A browser_ws_endpoint is required when no transport is provided" ) try: @@ -260,7 +260,7 @@ async def _dial(self) -> ConnectionTransport: message = f"WebSocket connection failed: {_safe_error_message(exc)}" else: return _WebSocketConnection(socket) - raise ConnectionError(message) + raise BAPConnectionError(message) def operation_timeout(self, timeout: float | None, cushion: float = 0) -> float: resolved = self._default_timeout if timeout is None else timeout @@ -303,7 +303,7 @@ async def send( {"id": operation_id, "type": "subscribe", "payload": payload} ) return await future - except TimeoutError: + except BAPTimeoutError: await self._complete_safely(operation_id) raise except asyncio.CancelledError: @@ -361,7 +361,7 @@ async def close(self) -> None: ) with suppress(BaseException): await _maybe_await(self._connection.close()) - self._fail_all(ConnectionError("WebSocket connection closed")) + self._fail_all(BAPConnectionError("WebSocket connection closed")) self._connected = False for task in tuple(self._tasks): task.cancel() @@ -391,17 +391,17 @@ def _stop(self, operation_id: str) -> None: def _ensure_connected(self) -> None: if self._closed or not self._connected or self._connection is None: - raise ConnectionError("WebSocket is not connected") + raise BAPConnectionError("WebSocket is not connected") async def _send_frame(self, frame: Mapping[str, Any]) -> None: connection = self._connection if connection is None: - raise ConnectionError("WebSocket is not connected") + raise BAPConnectionError("WebSocket is not connected") message = json.dumps(frame, separators=(",", ":")) try: async with self._send_lock: await _maybe_await(connection.send(message)) - except ConnectionError: + except BAPConnectionError: raise except asyncio.CancelledError: raise @@ -409,7 +409,7 @@ async def _send_frame(self, frame: Mapping[str, Any]) -> None: message = f"WebSocket send failed: {_safe_error_message(exc)}" else: return - raise ConnectionError(message) + raise BAPConnectionError(message) def _on_frame(self, message: str) -> None: try: @@ -483,13 +483,15 @@ def _on_close(self, _event: object | None = None) -> None: self._connected = False if not self._closing: self._closed = True - self._fail_all(ConnectionError("WebSocket connection closed unexpectedly")) + self._fail_all( + BAPConnectionError("WebSocket connection closed unexpectedly") + ) def _on_error(self, error: object) -> None: self._connected = False self._closed = True self._fail_all( - ConnectionError(f"WebSocket error: {_safe_error_message(error)}") + BAPConnectionError(f"WebSocket error: {_safe_error_message(error)}") ) def _fail_all(self, error: Exception) -> None: @@ -536,12 +538,12 @@ def _expire( timeout_ms: float, ) -> None: if not future.done(): - future.set_exception(TimeoutError(query[:50], timeout_ms)) + future.set_exception(BAPTimeoutError(query[:50], timeout_ms)) async def _send_complete(self, operation_id: str) -> None: if self._closed or self._connection is None: return - with suppress(ConnectionError): + with suppress(BAPConnectionError): await self._send_frame({"id": operation_id, "type": "complete"}) async def _complete_safely(self, operation_id: str) -> None: diff --git a/src/browserless_bap/types/options.py b/src/browserless_bap/types/options.py index 9350342..f9bdfd7 100644 --- a/src/browserless_bap/types/options.py +++ b/src/browserless_bap/types/options.py @@ -154,6 +154,7 @@ class ProxyOptions(TypedDict, total=False): class QuerySelectorOptions(TypedDict, total=False): timeout: NotRequired[float] visible: NotRequired[bool] + html: NotRequired[bool] class ReconnectOptions(TypedDict, total=False): @@ -186,6 +187,7 @@ class ResponseFilterOptions(TypedDict, total=False): type: NotRequired[list[ResourceType]] url: NotRequired[list[str]] wait: NotRequired[bool] + body: NotRequired[bool] body_encoding: NotRequired[bool] diff --git a/src/browserless_bap/types/responses.py b/src/browserless_bap/types/responses.py index 6560fc4..8add206 100644 --- a/src/browserless_bap/types/responses.py +++ b/src/browserless_bap/types/responses.py @@ -164,7 +164,7 @@ class ProxyResponse(TypedDict, total=False): class QuerySelectorResponse(TypedDict, total=False): - childElementCount: NotRequired[float | None] + childElementCount: NotRequired[int | None] className: NotRequired[str | None] id: NotRequired[str | None] innerHTML: NotRequired[str | None] diff --git a/tests/e2e/test_config.py b/tests/e2e/test_config.py index 2690576..fafcf57 100644 --- a/tests/e2e/test_config.py +++ b/tests/e2e/test_config.py @@ -131,5 +131,5 @@ def test_live_suite_needs_explicit_e2e_selection(selection: list[str]) -> None: ) assert result.returncode == 0, result.stdout - assert "6 skipped" in result.stdout + assert "7 skipped" in result.stdout assert "requires -m e2e" in result.stdout diff --git a/tests/e2e/test_live.py b/tests/e2e/test_live.py index fdf514d..b164fb9 100644 --- a/tests/e2e/test_live.py +++ b/tests/e2e/test_live.py @@ -2,7 +2,6 @@ import asyncio from collections.abc import Mapping -from typing import Any, cast import pytest from pytest import CaptureFixture @@ -37,8 +36,7 @@ async def run_async_happy_path(config: LiveConfig) -> None: def run_sync_happy_path(config: LiveConfig) -> None: - sync_browserless = cast(Any, bap_sync.Browserless) - with sync_browserless.connect( + with bap_sync.Browserless.connect( browser_ws_endpoint=config.url, token=config.token, timeout=config.timeout_ms, @@ -74,8 +72,7 @@ def rejected_sync_connection( token: str | None, timeout_ms: float, ) -> None: - sync_browserless = cast(Any, bap_sync.Browserless) - with sync_browserless.connect( + with bap_sync.Browserless.connect( browser_ws_endpoint=url, token=token, timeout=timeout_ms, @@ -185,7 +182,7 @@ def test_missing_and_invalid_auth_are_typed_and_redacted( capsys: CaptureFixture[str], token: str | None, ) -> None: - with pytest.raises(bap.ConnectionError) as async_error: + with pytest.raises(bap.BAPConnectionError) as async_error: asyncio.run( rejected_async_connection( live_config.url, @@ -193,7 +190,7 @@ def test_missing_and_invalid_auth_are_typed_and_redacted( live_config.timeout_ms, ) ) - with pytest.raises(bap.ConnectionError) as sync_error: + with pytest.raises(bap.BAPConnectionError) as sync_error: rejected_sync_connection( live_config.url, token, @@ -220,7 +217,7 @@ def test_malformed_endpoint_is_typed_and_redacted( capsys: CaptureFixture[str], ) -> None: malformed_url = f"http://bql.invalid/chromium/bql?token={MALFORMED_ENDPOINT_TOKEN}" - with pytest.raises(bap.ConnectionError) as async_error: + with pytest.raises(bap.BAPConnectionError) as async_error: asyncio.run( rejected_async_connection( malformed_url, @@ -228,7 +225,7 @@ def test_malformed_endpoint_is_typed_and_redacted( live_config.timeout_ms, ) ) - with pytest.raises(bap.ConnectionError) as sync_error: + with pytest.raises(bap.BAPConnectionError) as sync_error: rejected_sync_connection( malformed_url, live_config.token, @@ -246,3 +243,47 @@ def test_malformed_endpoint_is_typed_and_redacted( ] assert_secret_absent(live_config.token, values) assert_secret_absent(MALFORMED_ENDPOINT_TOKEN, values) + + +@pytest.mark.e2e +@pytest.mark.asyncio +async def test_opt_in_fields_live( + live_config: LiveConfig, + capsys: CaptureFixture[str], +) -> None: + async with bap.Browserless.connect( + browser_ws_endpoint=live_config.url, + token=live_config.token, + timeout=live_config.timeout_ms, + ) as browser: + async with browser.page() as page: + await page.goto(EXAMPLE_URL, timeout=live_config.timeout_ms) + responses = await page.response(type=["document"]) + assert responses + assert all("body" not in item for item in responses) + with_bodies = await page.response(type=["document"], body=True) + assert any( + "Example Domain" in (item.get("body") or "") for item in with_bodies + ) + await page.set_content('