diff --git a/AGENTS.md b/AGENTS.md index 41ea58d..4329c46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,8 @@ # Working on the polyfill -This package implements document-local WebMCP registration and discovery. Use -the official `webmcp-types` dependency without duplicating or augmenting its -declarations. Execution is deferred to a separate change alongside types PR #3. +This package implements the document-local imperative WebMCP draft. Use the +official `webmcp-types` dependency; do not duplicate its declarations. Keep the +temporary `executeTool` augmentation only until the upstream package ships it. No MCP server, transport, extension product, navigator aliases, or legacy API compatibility belongs here. diff --git a/README.md b/README.md index b13432b..1644019 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ await context.registerTool( { signal: registration.signal }, ); -console.log(await context.getTools()); +const [tool] = await context.getTools(); +console.log(await context.executeTool(tool, {})); // {"title":"WebMCP demo"} // Remove the tool when it is no longer needed. registration.abort(); @@ -49,9 +50,9 @@ To install explicitly, import and call `installWebMCP` from `webmcp-polyfill`. I Tools stay in the current document. Cross-document tools, declarative forms, lifecycle window events, and browser agent integration aren't implemented. Nonempty `exposedTo` and `fromOrigins` options reject. -This version supports registration, discovery, and `toolchange`. It does not invoke registered callbacks; `executeTool()` follows alongside [the types update](https://github.com/webmachinelearning/webmcp-types/pull/3). +`executeTool()` accepts an object and returns a JSON-serialized result. Callbacks must validate their inputs; schema inference provides TypeScript checks only. -The implementation follows [draft source `cc45efc`](https://github.com/webmachinelearning/webmcp/blob/cc45efcaf0/index.bs) and passes all 27 selected assertions of the [upstream WPT](https://github.com/web-platform-tests/wpt/tree/1a21db90adf8a264370ad806ed761f39e1d435a0/webmcp) at the pin, with no expected failures. [TESTING.md](https://github.com/webmachinelearning/webmcp-polyfill/blob/main/TESTING.md) records the selection and what it excludes. +The implementation follows [draft source `cc45efc`](https://github.com/webmachinelearning/webmcp/blob/cc45efcaf0/index.bs). Of the 56 selected assertions of the [upstream WPT](https://github.com/web-platform-tests/wpt/tree/1a21db90adf8a264370ad806ed761f39e1d435a0/webmcp) at the pin, 51 pass and five fail where the pinned tests disagree with the draft's `executeTool()` input and result rules. [TESTING.md](https://github.com/webmachinelearning/webmcp-polyfill/blob/main/TESTING.md) records each one and what the selection excludes. The API tracks the draft. Breaking changes ship with notes: in minor releases while the version is 0.x, in majors after 1.0. diff --git a/TESTING.md b/TESTING.md index e4a2a4e..19e2b40 100644 --- a/TESTING.md +++ b/TESTING.md @@ -11,14 +11,15 @@ pnpm test pnpm test:package ``` -| Check | What it runs | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| `index.test-d.ts` | TypeScript against the built package declarations, including upstream schema inference | -| `index.test.ts` | Built bundle served over HTTP in Chromium (native WebMCP disabled), Firefox, and WebKit; coercion, metadata, events, registration abort, errors, detached documents | -| `app.test.ts` | A served application, real button interactions, discovery, unregistration, and reload | -| `native.test.ts` | Real native Chromium registration, then polyfill loading; context and getter identities must survive | -| `pnpm test:package` | Packed tarball installed into a fresh consumer, public type imports, SSR-safe entry points, and package contents | -| `pnpm test:wpt` | Unmodified upstream registration/discovery WPT in real Chrome Canary, with native WebMCP disabled | +| Check | What it runs | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `index.test-d.ts` | TypeScript against the built package declarations, including upstream schema inference | +| `index.test.ts` | Built bundle served over HTTP in Chromium (native WebMCP disabled), Firefox, and WebKit; coercion, metadata, events, errors, detached documents | +| `execute.test.ts` | The same three engines: object input, JSON results, cancellation, concurrent calls, and dispatch-time failures | +| `app.test.ts` | A served application, real button interactions, callback side effects, invalid input, unregistration, and reload | +| `native.test.ts` | Real native Chromium registration, then polyfill loading; context and getter identities must survive | +| `pnpm test:package` | Packed tarball installed into a fresh consumer, public type imports, SSR-safe entry points, and package contents | +| `pnpm test:wpt` | Unmodified upstream WPT and IDL in real Chrome Canary, with native WebMCP disabled | The fixture server binds 127.0.0.1:8793 and sets the required `Origin-Agent-Cluster` header; Playwright never reuses an existing server, so free that port first. @@ -46,39 +47,74 @@ WPT_ROOT=../wpt CHROME_BIN=/path/to/chrome-canary pnpm test:wpt ``` `WPT_PYTHON` and `WPT_VENV` optionally select the interpreter and environment. -The runner checks the revision, rejects tracked source changes, requires all 13 selected files +The runner checks the revision, rejects tracked source changes, requires all 18 selected files to run exactly once, checks assertion counts, and retains `wpt-results/report.json` with browser and upstream revisions. Runner errors, unexpected failures, and unexpected passes fail the command, and nothing is retried. -At this pin, **all 27 selected assertions pass**, with no expected failures. +At this pin, 56 assertions run: **51 pass and five have explicit expected FAIL +metadata**. All 22 IDL assertions pass. API shape coverage does not prove runtime +defaults or complete conformance. + +### Known draft disagreements + +The [published draft](https://webmachinelearning.github.io/webmcp/#dom-modelcontext-executetool) +accepts optional `any inputObject`, rejects non-objects, and has no `{}` default. +It also JSON-serializes callback results, including strings. The pinned WPT +expects a default object and raw string results in the following cases: + +| Upstream file | Expected failures | Local coverage | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------- | +| `executeTool-invalid-dictionary.https.html` | Missing tool invoked without input expects `UnknownError`, rather than the draft's earlier input `TypeError` | Descriptor validation and input errors | +| `executeTool-error-window-onerror.https.html` | Both cases omit input and expect execution to start | Explicit object input, callback/serialization failures, and absence of page errors | +| `executeTool-unregister-resolution-race.https.html` | Invocation omits input | Unregistration during an invocation and its successful result | +| `object-arguments.https.html` | Expects an unquoted string result; also expects omitted input to become `{}` | Array/object inputs, rejected primitive inputs, and JSON result serialization | + +These are five assertions in four files, recorded individually in `wpt-metadata`. +The pin, [1a21db9](https://github.com/web-platform-tests/wpt/commit/1a21db90adf8a264370ad806ed761f39e1d435a0), +is the WPT export of the Chromium change for +[spec PR #246](https://github.com/webmachinelearning/webmcp/pull/246) and +[#251](https://github.com/webmachinelearning/webmcp/pull/251), and it rewrote all four of these +files. The draft's input and result rules have not changed since #251 merged, so these are not +stale tests awaiting an update: four of the assertions invoke `executeTool()` with no input, +which the draft rejects with a `TypeError` before it looks the tool up, and `object-arguments` +asserts an unquoted `"Success"` from a callback whose result the draft JSON-serializes. Re-read the live +draft before changing any expectation. An expected failure can stop at its first assertion, so +the local tests cover the behavior after that point. + +`webmcp-types` PR #3 and the augmentation in `index.ts` both declare `inputObject?: object`, +which is narrower than the IDL's `any`: omitting the argument type-checks and then rejects at +runtime, as the draft requires. ### Excluded coverage -Execution tests and the full IDL harness are deferred with `executeTool()`. -The IDL includes that method, so running the full shape suite against this -registration/discovery subset would intentionally fail. Keep that exclusion -explicit rather than adding expected failures for an API not yet included. - -Cross-document discovery, frame-tree routing, declarative forms, browser -permissions integration, CSS states, navigation/BFCache, and lifecycle window -events are also outside this initial implementation. +Cross-document discovery/execution, frame-tree routing, navigation cancellation, +declarative forms, browser permissions integration, CSS states, and lifecycle +window events are outside this initial implementation. The pinned +`executeTool-abort.https.html` is excluded rather than carried as expected +failures: its second subtest waits forever on a `toolactivated` event that the +draft still leaves [unspecified](https://github.com/webmachinelearning/webmcp/issues/146), +so the file times out and its last three subtests never run at all. Expected-failure +metadata cannot express that, and cancellation is exercised directly in all three +engines instead. +`exposedTo-invalid-origins.https.html` is excluded for the `exposedTo` divergence +recorded below, not for cross-document routing. WPT's `--inject-script` modifies testharness pages; it does not install the bundle in `/common/blank.html` helper frames. Detached-frame WPT therefore cannot run unchanged in this lane. The local test serves an instrumented iframe, removes -it, and checks registration, discovery, and exception realms. +it, and checks all three operations and exception realms. ## Where to look when upstream changes -| Source | Use | -| --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | -| [Draft source and history](https://github.com/webmachinelearning/webmcp/commits/main/index.bs) | Normative algorithms, Web IDL, open issues; compare with the revision below | -| [WPT webmcp](https://github.com/web-platform-tests/wpt/tree/master/webmcp) and [results](https://wpt.fyi/results/webmcp) | Executable assertions and native cross-browser results; these results are not polyfill results | -| [WPT IDL](https://github.com/web-platform-tests/wpt/blob/master/interfaces/webmcp.idl) | Generated interface snapshot; it can lag the published draft | -| [Official types](https://github.com/webmachinelearning/webmcp-types) | Public declarations, schema inference, and pending API updates | -| [Blink script_tools](https://chromium.googlesource.com/chromium/src/+/main/third_party/blink/renderer/core/script_tools/) | Chromium IDL, implementation, tests, and commit-linked bugs | -| [Gecko source search](https://searchfox.org/mozilla-central/search?q=ModelContext) and [Mozilla position](https://github.com/mozilla/standards-positions/issues/1412) | Locate Firefox implementation work and discussion; a position is not evidence of shipped support | +| Source | Use | +| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| [Draft source and history](https://github.com/webmachinelearning/webmcp/commits/main/index.bs) | Normative algorithms, Web IDL, open issues; compare with the revision below | +| [WPT webmcp](https://github.com/web-platform-tests/wpt/tree/master/webmcp) and [results](https://wpt.fyi/results/webmcp) | Executable assertions and native cross-browser results; these results are not polyfill results | +| [WPT IDL](https://github.com/web-platform-tests/wpt/blob/master/interfaces/webmcp.idl) | Generated interface snapshot; it can lag the published draft | +| [Official types](https://github.com/webmachinelearning/webmcp-types) | Public declarations, schema inference, and pending API updates | +| [Blink script_tools](https://chromium.googlesource.com/chromium/src/+/main/third_party/blink/renderer/core/script_tools/) | Chromium IDL, implementation, tests, and commit-linked bugs | +| [Gecko source search](https://searchfox.org/mozilla-central/search?q=ModelContext) and [Mozilla position](https://github.com/mozilla/standards-positions/issues/1412) | Locate Firefox implementation work and discussion; a position is not evidence of shipped support | Reproduce a disagreement before changing code or expectations, and record what changed in the draft, types, WPT, and browser implementation separately. @@ -93,7 +129,10 @@ explicit `any`. Warnings fail the command, and `pnpm test` runs it before compil The implementation was compared with [draft source `cc45efc`](https://github.com/webmachinelearning/webmcp/blob/cc45efcaf0/index.bs). It uses timers to queue tasks; JavaScript cannot reproduce the browser's WebMCP task source, nor the draft's abort *algorithms*, which run before an abort event -rather than as a listener. +rather than as a listener. One consequence is observable: a signal aborted before +the dispatch timer fires rejects the caller and never runs the callback, where the +draft dispatches to the target document and then cancels through the callback's own +signal. The runtime checks the `tools` Permissions Policy when the browser exposes it. No engine lists `tools` in `permissionsPolicy.features()` today, so that branch @@ -110,6 +149,10 @@ implementation has nowhere to expose a tool to; rejecting keeps the polyfill fro implying cross-document support it does not have. Origins are validated first, so an untrustworthy one still fails with the `SecurityError` the draft requires. +`webmcp-types@0.1.7` does not declare `executeTool()`. The method augmentation +in `index.ts` is temporary; remove it when [types PR #3](https://github.com/webmachinelearning/webmcp-types/pull/3) +is included in a release. + To test types changes, keep `webmcp-types` beside this checkout and run `pnpm link ../webmcp-types`, then `pnpm typecheck`. Restore the published dependency with `pnpm install --force` before package checks. Do not commit diff --git a/app.test.ts b/app.test.ts index 7e61787..e7d6620 100644 --- a/app.test.ts +++ b/app.test.ts @@ -1,6 +1,6 @@ import { test, expect } from "@playwright/test"; -test("a served application registers, discovers, unregisters, and resets on reload", async ({ +test("a served application executes, rejects invalid input, unregisters, and resets on reload", async ({ page, browser, }, testInfo) => { @@ -12,11 +12,13 @@ test("a served application registers, discovers, unregisters, and resets on relo page.on("pageerror", (error) => errors.push(error)); await page.goto("/app"); await expect(page.locator("#status")).toHaveText("registered"); - expect( - await page.evaluate(async () => - (await document.modelContext!.getTools()).map((tool) => tool.name), - ), - ).toEqual(["increment"]); + await page.getByRole("button", { name: "Execute on page" }).click(); + await expect(page.locator("#result")).toHaveText('{"count":2}'); + await expect(page.locator("#count")).toHaveText("2"); + await page.getByLabel("Amount").fill("-1"); + await page.getByRole("button", { name: "Execute on page" }).click(); + await expect(page.locator("#result")).toHaveText("UnknownError"); + await expect(page.locator("#count")).toHaveText("2"); await page.getByRole("button", { name: "Unregister", exact: true }).click(); await expect(page.locator("#status")).toHaveText("unregistered"); expect(await page.evaluate(() => document.modelContext!.getTools())).toEqual([]); diff --git a/execute.test.ts b/execute.test.ts new file mode 100644 index 0000000..7fe98e4 --- /dev/null +++ b/execute.test.ts @@ -0,0 +1,537 @@ +import { test, expect } from "@playwright/test"; +import "./index.js"; + +test.beforeEach(async ({ page }) => { + await page.goto("/"); + expect(await page.evaluate(() => "modelContext" in document)).toBe(false); +}); + +test("ignores late results after cancellation, including serialization side effects", async ({ + page, +}) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + let finish!: (value: object) => void; + let serialized = false; + await context.registerTool({ + name: "late", + description: "Late", + execute() { + entered(); + return new Promise((resolve) => { + finish = resolve; + }); + }, + }); + const [tool] = await context.getTools(); + const caller = new AbortController(); + const result = context + .executeTool(tool, {}, { signal: caller.signal }) + .catch((error) => error); + await started; + caller.abort("cancelled"); + await result; + finish({ + toJSON() { + serialized = true; + return {}; + }, + }); + await context.getTools(); + return serialized; + }), + ).toBe(false); +}); + +test("concurrent calls to the same tool have independent cancellation", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + const signals: AbortSignal[] = []; + let start!: () => void; + const bothStarted = new Promise((resolve) => { + start = resolve; + }); + let finish!: (value: object) => void; + await context.registerTool({ + name: "concurrent", + description: "Concurrent", + execute(_, { signal }) { + signals.push(signal); + if (signals.length === 2) start(); + return new Promise((resolve) => { + finish = resolve; + }); + }, + }); + const [tool] = await context.getTools(); + const caller = new AbortController(); + const first = context + .executeTool(tool, {}, { signal: caller.signal }) + .catch((error) => error); + const second = context.executeTool(tool, {}); + await bothStarted; + const aborted = new Promise((resolve) => + signals[0].addEventListener("abort", () => resolve(), { once: true }), + ); + caller.abort("first only"); + await aborted; + finish({ second: true }); + return { + first: await first, + second: await second, + aborted: signals.map((signal) => signal.aborted), + }; + }), + ).toEqual({ first: "first only", second: '{"second":true}', aborted: [true, false] }); +}); + +test("executes copied object and array inputs with a fresh callback signal", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + "use strict"; + const context = document.modelContext!; + const input = { nested: { value: 1 } }; + const caller = new AbortController(); + await context.registerTool({ + name: "echo", + description: "Echo", + execute(args, { signal }) { + return { + args, + fresh: signal instanceof AbortSignal && signal !== caller.signal, + unbound: this === undefined, + }; + }, + }); + const [tool] = await context.getTools(); + const pending = context.executeTool(tool, input, { signal: caller.signal }); + input.nested.value = 9; + return [JSON.parse(await pending), JSON.parse(await context.executeTool(tool, [1, 2]))]; + }), + ).toEqual([ + { args: { nested: { value: 1 } }, fresh: true, unbound: true }, + { args: [1, 2], fresh: true, unbound: true }, + ]); +}); + +test("rejects legacy JSON strings and preserves input serialization errors", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + await context.registerTool({ name: "x", description: "X", execute: () => null }); + const [tool] = await context.getTools(); + const errors = []; + for (const input of [ + "{}", + null, + undefined, + 1, + { + toJSON() { + throw new RangeError("input"); + }, + }, + ]) { + try { + // @ts-expect-error Exercise primitive inputs from JavaScript callers. + await context.executeTool(tool, input); + errors.push("resolved"); + } catch (error) { + if (!(error instanceof Error)) throw error; + errors.push(error.name); + } + } + return errors; + }), + ).toEqual(["TypeError", "TypeError", "TypeError", "TypeError", "RangeError"]); +}); + +test("serializes results as JSON and rejects callback or serialization failures", async ({ + page, +}) => { + const pageErrors: Error[] = []; + page.on("pageerror", (error) => pageErrors.push(error)); + await page.addScriptTag({ url: "/auto.js" }); + const results = await page.evaluate(async () => { + const context = document.modelContext!; + const circular = {}; + Object.assign(circular, { self: circular }); + await context.registerTool({ + name: "x", + description: "X", + execute({ value }) { + if (value === "throw") throw new Error("callback"); + if (value === "circular") return circular; + if (value === "undefined") return undefined; + return value; + }, + }); + const [tool] = await context.getTools(); + const output = []; + for (const value of ["hello", null, 42, { success: true }, "throw", "circular", "undefined"]) { + try { + output.push(await context.executeTool(tool, { value })); + } catch (error) { + if (!(error instanceof Error)) throw error; + output.push(error.name); + } + } + return output; + }); + expect(results).toEqual([ + '"hello"', + "null", + "42", + '{"success":true}', + "UnknownError", + "UnknownError", + "UnknownError", + ]); + expect(pageErrors).toEqual([]); +}); + +test("cancels the caller immediately and sends a default AbortError to the callback", async ({ + page, +}) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + let observed!: (reason: string) => void; + const callbackAborted = new Promise((resolve) => { + observed = resolve; + }); + await context.registerTool({ + name: "pending", + description: "Pending", + execute(_input, { signal }) { + entered(); + return new Promise((resolve) => + signal.addEventListener( + "abort", + () => { + observed(signal.reason.name); + resolve("late result"); + }, + { once: true }, + ), + ); + }, + }); + const [tool] = await context.getTools(); + const controller = new AbortController(); + controller.signal.addEventListener("abort", (event) => event.stopImmediatePropagation()); + const result = context + .executeTool(tool, {}, { signal: controller.signal }) + .catch((error) => error); + await started; + controller.abort("caller reason"); + return [await result, await callbackAborted]; + }), + ).toEqual(["caller reason", "AbortError"]); +}); + +test("unregistration leaves an already-running invocation alive", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + const registration = new AbortController(); + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + let complete!: (value: string) => void; + let callbackSignal!: AbortSignal; + await context.registerTool( + { + name: "pending", + description: "Pending", + execute(_input, { signal }) { + callbackSignal = signal; + entered(); + return new Promise((resolve) => { + complete = resolve; + }); + }, + }, + { signal: registration.signal }, + ); + const [tool] = await context.getTools(); + const pending = context.executeTool(tool, {}); + await started; + registration.abort(); + const count = (await context.getTools()).length; + complete("finished"); + return { count, aborted: callbackSignal.aborted, result: await pending }; + }), + ).toEqual({ count: 0, aborted: false, result: '"finished"' }); +}); + +// A polyfill divergence, not a draft requirement: the draft dispatches to the target +// document and cancels through the callback's own signal. See TESTING.md. +test("aborting before the dispatch task rejects without starting the callback", async ({ + page, +}) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + let executions = 0; + await context.registerTool({ + name: "x", + description: "X", + execute() { + executions++; + }, + }); + const [tool] = await context.getTools(); + const controller = new AbortController(); + const pending = context + .executeTool(tool, {}, { signal: controller.signal }) + .catch((error) => error); + controller.abort("before dispatch"); + const reason = await pending; + await context.getTools(); + return [executions, reason]; + }), + ).toEqual([0, "before dispatch"]); +}); + +test("execution converts every descriptor member before invoking the tool", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + const result = await page.evaluate(async () => { + const context = document.modelContext!; + let calls = 0; + await context.registerTool({ + name: "descriptor", + description: "Descriptor conversion", + execute() { + return ++calls; + }, + }); + const [tool] = await context.getTools(); + const errors = []; + const fakeWindow = {}; + Object.assign(fakeWindow, { window: fakeWindow }); + const invalid: unknown[] = [ + { ...tool, title: Symbol() }, + { ...tool, inputSchema: 1 }, + { ...tool, annotations: 1 }, + { ...tool, window: fakeWindow }, + { ...tool, window: null }, + ]; + for (const descriptor of invalid) { + try { + // @ts-expect-error Exercise invalid JavaScript descriptor members. + await context.executeTool(descriptor, {}); + errors.push("resolved"); + } catch (error) { + if (!(error instanceof Error)) throw error; + errors.push(error.name); + } + } + const invalidCalls = calls; + const reads: string[] = []; + const descriptor = { + get annotations() { + reads.push("annotations"); + return { + get consequentialHint() { + reads.push("consequentialHint"); + return false; + }, + get readOnlyHint() { + reads.push("readOnlyHint"); + return false; + }, + get untrustedContentHint() { + reads.push("untrustedContentHint"); + return false; + }, + }; + }, + get description() { + reads.push("description"); + return tool.description; + }, + get inputSchema() { + reads.push("inputSchema"); + return {}; + }, + get name() { + reads.push("name"); + return tool.name; + }, + get origin() { + reads.push("origin"); + return tool.origin; + }, + get title() { + reads.push("title"); + return "Descriptor"; + }, + get window() { + reads.push("window"); + return tool.window; + }, + }; + await context.executeTool(descriptor, {}); + return { errors, invalidCalls, reads, calls }; + }); + expect(result).toEqual({ + errors: ["TypeError", "TypeError", "TypeError", "TypeError", "TypeError"], + invalidCalls: 0, + reads: [ + "annotations", + "consequentialHint", + "readOnlyHint", + "untrustedContentHint", + "description", + "inputSchema", + "name", + "origin", + "title", + "window", + ], + calls: 1, + }); +}); + + +test("a tool belonging to another window cannot be executed", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + await context.registerTool({ name: "local", description: "Local", execute: () => null }); + const [tool] = await context.getTools(); + const iframe = document.createElement("iframe"); + iframe.src = "/app"; + await new Promise((resolve) => { + iframe.onload = () => resolve(); + document.body.append(iframe); + }); + try { + await context.executeTool({ ...tool, window: iframe.contentWindow! }, {}); + return "resolved"; + } catch (error) { + if (!(error instanceof Error)) throw error; + return error.name; + } + }), + ).toBe("UnknownError"); +}); + +test("descriptor origins are parsed, and a mismatch fails like a missing tool", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + await context.registerTool({ name: "x", description: "X", execute: () => null }); + const [tool] = await context.getTools(); + const results: [string, string][] = []; + for (const [label, origin] of [ + ["unparseable", "not a url"], + ["opaque", "data:,x"], + ["another origin", "https://other.test"], + ]) { + try { + await context.executeTool({ ...tool, origin }, {}); + results.push([label, "resolved"]); + } catch (error) { + if (!(error instanceof Error)) throw error; + results.push([label, error.name]); + } + } + return results; + }), + ).toEqual([ + ["unparseable", "NotSupportedError"], + ["opaque", "NotSupportedError"], + ["another origin", "UnknownError"], + ]); +}); + +test("a signal option that is not an AbortSignal is rejected", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + await context.registerTool({ name: "x", description: "X", execute: () => null }); + const [tool] = await context.getTools(); + const results: [string, string][] = []; + for (const signal of [1, {}, null, "abort"] as unknown[]) { + const label = JSON.stringify(signal)!; + try { + // @ts-expect-error Exercise invalid JavaScript callers at the Web IDL boundary. + await context.executeTool(tool, {}, { signal }); + results.push([label, "resolved"]); + } catch (error) { + if (!(error instanceof Error)) throw error; + results.push([label, error.name]); + } + } + return results; + }), + ).toEqual([ + ["1", "TypeError"], + ["{}", "TypeError"], + ["null", "TypeError"], + ['"abort"', "TypeError"], + ]); +}); + +test("unregistering before dispatch rejects without running the callback", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + const registration = new AbortController(); + let executions = 0; + await context.registerTool( + { name: "x", description: "X", execute: () => ++executions }, + { signal: registration.signal }, + ); + const [tool] = await context.getTools(); + const pending = context.executeTool(tool, {}).catch((error: Error) => error.name); + registration.abort(); + return [await pending, executions]; + }), + ).toEqual(["UnknownError", 0]); +}); + +test("input that serializes to a non-object rejects before the callback runs", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + let executions = 0; + await context.registerTool({ + name: "x", + description: "X", + execute: () => ++executions, + }); + const [tool] = await context.getTools(); + const name = await context.executeTool(tool, { toJSON: () => 5 }).then( + () => "resolved", + (error: Error) => error.name, + ); + return [name, executions]; + }), + ).toEqual(["UnknownError", 0]); +}); diff --git a/fixtures/app.js b/fixtures/app.js index c078bb1..c8b01a9 100644 --- a/fixtures/app.js +++ b/fixtures/app.js @@ -31,4 +31,14 @@ element("unregister").onclick = async () => { await context.getTools(); element("status").textContent = "unregistered"; }; +element("execute").onclick = async () => { + try { + const [tool] = await context.getTools(); + element("result").textContent = await context.executeTool(tool, { + amount: Number(element("amount").value), + }); + } catch (error) { + element("result").textContent = error.name; + } +}; await register(); diff --git a/fixtures/server.mjs b/fixtures/server.mjs index 18e5252..495fbf3 100644 --- a/fixtures/server.mjs +++ b/fixtures/server.mjs @@ -3,8 +3,10 @@ import { readFile } from "node:fs/promises"; const app = `WebMCP counter

Counter

0 + -loading`; + +loading`; createServer(async (request, response) => { const path = new URL(request.url, "http://localhost").pathname; diff --git a/index.test-d.ts b/index.test-d.ts index af01976..da7bcf2 100644 --- a/index.test-d.ts +++ b/index.test-d.ts @@ -14,8 +14,13 @@ if (document.modelContext) { return { name, aborted, invalid }; }, }); - const tools: WebMCP.RegisteredTool[] = await context.getTools(); - // @ts-expect-error Execution is deferred to a separate change. - void context.executeTool; - void tools; + const [tool] = await context.getTools(); + const result: string = await context.executeTool( + tool, + {}, + { signal: new AbortController().signal }, + ); + // @ts-expect-error The current draft accepts objects, not serialized JSON. + context.executeTool(tool, "{}"); + void result; } diff --git a/index.test.ts b/index.test.ts index 875573f..368cb78 100644 --- a/index.test.ts +++ b/index.test.ts @@ -18,6 +18,11 @@ test("every operation rejects when the server opts out of origin-keyed agent clu for (const operation of [ () => context.getTools(), () => context.registerTool({ name: "x", description: "X", execute: () => null }), + () => + context.executeTool( + { name: "x", title: "", description: "X", window, origin: location.origin }, + {}, + ), ]) { try { await operation(); @@ -29,7 +34,7 @@ test("every operation rejects when the server opts out of origin-keyed agent clu } return errors; }), - ).toEqual(["SecurityError", "SecurityError"]); + ).toEqual(["SecurityError", "SecurityError", "SecurityError"]); }); test("only potentially trustworthy origins reach the cross-document refusal", async ({ page }) => { @@ -250,6 +255,7 @@ test("operations on a real detached frame reject in the frame's realm", async ({ document.body.append(iframe); await loaded; const context = iframe.contentDocument!.modelContext!; + const tool = (await context.getTools())[0]; const FrameException = iframe.contentDocument!.defaultView!.DOMException; iframe.remove(); const errors = []; @@ -257,6 +263,7 @@ test("operations on a real detached frame reject in the frame's realm", async ({ () => context.getTools(), () => context.registerTool({ name: "detached", description: "Detached", execute: () => null }), + () => context.executeTool(tool, {}), ]) { try { await operation(); @@ -267,7 +274,7 @@ test("operations on a real detached frame reject in the frame's realm", async ({ } return errors; }); - expect(frame).toEqual(["InvalidStateError", "InvalidStateError"]); + expect(frame).toEqual(["InvalidStateError", "InvalidStateError", "InvalidStateError"]); }); test("installs once, exposes only standard members, and keeps document identity", async ({ @@ -308,14 +315,14 @@ test("installs once, exposes only standard members, and keeps document identity" writable: descriptor.set !== undefined, alias: "modelContext" in navigator, testing: "modelContextTesting" in navigator, - lengths: [context.registerTool.length, context.getTools.length], + lengths: [context.registerTool.length, context.getTools.length, context.executeTool.length], constructionError, getterErrors, }; }); expect(result).toEqual({ same: true, - members: ["getTools", "ontoolchange", "registerTool"], + members: ["executeTool", "getTools", "ontoolchange", "registerTool"], own: [], brand: "[object ModelContext]", instance: true, @@ -324,7 +331,7 @@ test("installs once, exposes only standard members, and keeps document identity" writable: false, alias: false, testing: false, - lengths: [1, 0], + lengths: [1, 0, 1], constructionError: "TypeError", getterErrors: ["TypeError", "TypeError", "TypeError", "TypeError", "TypeError"], }); @@ -450,7 +457,7 @@ test("rejects invalid descriptors, duplicates and unserializable schemas", async expect(registered).toEqual(["valid"]); }); -test("coerces registration dictionary members", async ({ page }) => { +test("uses dictionary coercion without retaining or binding the tool object", async ({ page }) => { await page.addScriptTag({ url: "/auto.js" }); const result = await page.evaluate(async () => { "use strict"; @@ -460,7 +467,9 @@ test("coerces registration dictionary members", async ({ page }) => { title: "\ud800", description: true, annotations: { readOnlyHint: 1 }, - execute: () => null, + execute() { + return this === undefined; + }, }; // @ts-expect-error Web IDL coerces the deliberately non-string fields. await context.registerTool(descriptor); @@ -470,6 +479,7 @@ test("coerces registration dictionary members", async ({ page }) => { title: tool.title, description: tool.description, annotations: tool.annotations, + result: await context.executeTool(tool, {}), }; }); expect(result).toEqual({ @@ -477,6 +487,7 @@ test("coerces registration dictionary members", async ({ page }) => { title: "\ufffd", description: "true", annotations: { consequentialHint: false, readOnlyHint: true, untrustedContentHint: false }, + result: "true", }); }); @@ -574,7 +585,12 @@ test("validates origins and refuses cross-document exposure", async ({ page }) = } return errors; }), - ).toEqual(["SecurityError", "SecurityError", "NotSupportedError", "NotSupportedError"]); + ).toEqual([ + "SecurityError", + "SecurityError", + "NotSupportedError", + "NotSupportedError", + ]); }); test("inactive documents get their own context but cannot register tools", async ({ page }) => { diff --git a/index.ts b/index.ts index 5d179ff..8ed3932 100644 --- a/index.ts +++ b/index.ts @@ -6,8 +6,27 @@ export type { WebMCP } from "webmcp-types"; * SPDX-License-Identifier: MIT */ -// Capture the constructor before a detached WindowProxy stops exposing it. +// Mirrors webmachinelearning/webmcp-types#3; delete once that ships in a release. +declare global { + namespace WebMCP { + interface ModelContextExecuteToolOptions { + signal?: AbortSignal; + } + interface ModelContext { + executeTool( + tool: RegisteredTool, + inputObject?: object, + options?: ModelContextExecuteToolOptions, + ): Promise; + } + } +} + +// Capture native bindings before a detached WindowProxy stops exposing them. HTML declares +// `window` as [LegacyUnforgeable], so its getter is an own property of every Window; a realm +// without one has no Document either, and `installWebMCP` exits early there. const NativeDOMException = globalThis.DOMException; +const windowGetter = Object.getOwnPropertyDescriptor(globalThis, "window")?.get; interface Tool { metadata: Omit; @@ -156,7 +175,7 @@ function activeView(owner: Document): Window { } // Timers approximate the unavailable WebMCP task source; exact -// inter-source ordering requires native support. +// inter-source ordering and document-navigation semantics require native support. function queueTask(callback: () => void): void { setTimeout(callback, 0); } @@ -199,7 +218,7 @@ class ModelContextPolyfill extends EventTarget implements WebMCP.ModelContext { const description = domString(required(descriptor.description, "description")); const callback = required(descriptor.execute, "execute"); if (typeof callback !== "function") throw new TypeError("execute must be a function"); - // SAFETY: Web IDL requires callability only. + // SAFETY: Web IDL requires callability only; arguments and results are converted at invocation. const execute = callback as WebMCP.ToolExecuteCallback; const inputSchema = descriptor.inputSchema; if (inputSchema !== undefined && !isObject(inputSchema)) @@ -275,6 +294,112 @@ class ModelContextPolyfill extends EventTarget implements WebMCP.ModelContext { .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); return new Promise((resolve) => queueTask(() => resolve(tools))); } + + async executeTool( + tool: WebMCP.RegisteredTool, + inputObject: object | undefined = undefined, + options: WebMCP.ModelContextExecuteToolOptions = {}, + ): Promise { + // Web IDL converts every member of RegisteredTool, in lexicographical order, before the + // algorithm runs, so the conversions whose results are unused below are kept for their + // observable effects: reading the caller's getters, and throwing TypeError. + const descriptor = dictionary(tool); + toolAnnotations(descriptor.annotations); + domString(required(descriptor.description, "description")); + const inputSchema = descriptor.inputSchema; + if (inputSchema !== undefined && !isObject(inputSchema)) + throw new TypeError("inputSchema must be an object"); + const name = domString(required(descriptor.name, "name")); + const origin = domString(required(descriptor.origin, "origin")).toWellFormed(); + const title = descriptor.title; + if (title !== undefined) domString(title); + const target = required(descriptor.window, "window"); + if (!isObject(target)) throw new TypeError("window must be a Window"); + // SAFETY: the native getter performs the Window brand check, including for cross-origin + // WindowProxy objects, and throws for anything else. Its value is unused. + windowGetter!.call(target); + const signal = signalOption(dictionary(options).signal); + activeView(this.#owner); + let expectedOrigin = "null"; + try { + expectedOrigin = new URL(origin).origin; + } catch { + // An unparseable origin falls through to the opaque check below. + } + // `URL.origin` serializes an opaque origin as "null", which no document's origin matches. + if (expectedOrigin === "null") { + throw new NativeDOMException("Invalid or opaque origin", "NotSupportedError"); + } + if (!isObject(inputObject)) throw new TypeError("inputObject must be an object"); + const input = serialize(inputObject); + signal?.throwIfAborted(); + if (target !== this.#owner.defaultView) { + // The draft rejects a target in another traversable with UnknownError, and reports a + // target document that has no such tool the same way. Routing to another document in this + // traversable needs native WebMCP, so both arrive here indistinguishably. + throw new NativeDOMException("Tool execution failed", "UnknownError"); + } + + return new Promise((resolve, reject) => { + const controller = new AbortController(); + let settled = false; + let callbackSettled = false; + // Returns true to the first caller only; whoever wins owns settling the promise. + const claimSettlement = (): boolean => { + if (settled) return false; + settled = true; + signal?.removeEventListener("abort", abort); + return true; + }; + const abort = (): void => { + if (!claimSettlement()) return; + reject(signal!.reason); + // The callback receives a fresh signal and a default AbortError, while the caller + // receives its own abort reason. Aborting a task later lets the caller's rejection + // arrive first, and leaves a callback that has already settled alone. + queueTask(() => { + if (!callbackSettled) controller.abort(); + }); + }; + const fail = (): void => { + callbackSettled = true; + queueTask(() => { + if (claimSettlement()) + reject(new NativeDOMException("Tool execution failed", "UnknownError")); + }); + }; + const complete = (value: unknown): void => { + callbackSettled = true; + // Checked before serializing: a cancelled call must not run the author's toJSON. + if (settled) return; + try { + const result = serialize(value); + queueTask(() => { + if (claimSettlement()) resolve(result); + }); + } catch { + fail(); + } + }; + signal?.addEventListener("abort", abort, { once: true }); + queueTask(() => { + if (settled) return; + try { + // Every dispatch failure is indistinguishable to the caller by design, so each bare + // throw funnels into fail() and the draft's UnknownError. + const view = activeView(this.#owner); + const entry = this.#tools.get(name); + if (!entry || expectedOrigin !== view.origin) throw new Error(); + const args: unknown = JSON.parse(input); + if (!isObject(args)) throw new Error(); + const { execute } = entry; + Promise.resolve(execute(args, { signal: controller.signal })).then(complete, fail); + } catch { + fail(); + } + }); + }); + } } const contexts = new WeakMap(); @@ -299,6 +424,7 @@ Object.defineProperties(ModelContextPolyfill.prototype, { // Class members are non-enumerable; Web IDL interface members are enumerable. registerTool: { enumerable: true }, getTools: { enumerable: true }, + executeTool: { enumerable: true }, ontoolchange: { enumerable: true }, }); diff --git a/package.test.mjs b/package.test.mjs index 7af0308..0b4dba2 100644 --- a/package.test.mjs +++ b/package.test.mjs @@ -64,9 +64,12 @@ try { return { count }; }, }); - async function discover(context: WebMCP.ModelContext) { - const tools: WebMCP.RegisteredTool[] = await context.getTools(); - return tools; + async function invoke(context: WebMCP.ModelContext) { + const [tool] = await context.getTools(); + const result: string = await context.executeTool(tool, { count: 1 }); + // @ts-expect-error legacy JSON-string input is not supported + await context.executeTool(tool, '{}'); + return result; } `, ); diff --git a/playwright.config.ts b/playwright.config.ts index 5e731e4..3c6d3ec 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "@playwright/test"; export default defineConfig({ - testMatch: /(?:index|app)\.test\.ts$/, + testMatch: /(?:index|execute|app)\.test\.ts$/, forbidOnly: !!process.env.CI, fullyParallel: true, workers: 3, diff --git a/tsconfig.test.json b/tsconfig.test.json index 6fa915b..058e7d9 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -4,6 +4,7 @@ "files": [ "index.test-d.ts", "index.test.ts", + "execute.test.ts", "app.test.ts", "native.test.ts", "playwright.config.ts" diff --git a/wpt-metadata/webmcp/imperative/executeTool-error-window-onerror.https.html.ini b/wpt-metadata/webmcp/imperative/executeTool-error-window-onerror.https.html.ini new file mode 100644 index 0000000..a0fff5f --- /dev/null +++ b/wpt-metadata/webmcp/imperative/executeTool-error-window-onerror.https.html.ini @@ -0,0 +1,6 @@ +# Draft differences and matching local coverage: TESTING.md. +[executeTool-error-window-onerror.https.html] + [Failed tool execution does not trigger window.onerror] + expected: FAIL + [Tool execution returning circular object rejects and does not trigger window.onerror] + expected: FAIL diff --git a/wpt-metadata/webmcp/imperative/executeTool-invalid-dictionary.https.html.ini b/wpt-metadata/webmcp/imperative/executeTool-invalid-dictionary.https.html.ini new file mode 100644 index 0000000..7e3f29e --- /dev/null +++ b/wpt-metadata/webmcp/imperative/executeTool-invalid-dictionary.https.html.ini @@ -0,0 +1,4 @@ +# Draft differences and matching local coverage: TESTING.md. +[executeTool-invalid-dictionary.https.html] + [executeTool() rejects with UnknownError when a non-existent tool object is supplied] + expected: FAIL diff --git a/wpt-metadata/webmcp/imperative/executeTool-unregister-resolution-race.https.html.ini b/wpt-metadata/webmcp/imperative/executeTool-unregister-resolution-race.https.html.ini new file mode 100644 index 0000000..054efdf --- /dev/null +++ b/wpt-metadata/webmcp/imperative/executeTool-unregister-resolution-race.https.html.ini @@ -0,0 +1,4 @@ +# Draft differences and matching local coverage: TESTING.md. +[executeTool-unregister-resolution-race.https.html] + [executeTool resolves successfully even when tool is unregistered synchronously after its promise resolves] + expected: FAIL diff --git a/wpt-metadata/webmcp/imperative/object-arguments.https.html.ini b/wpt-metadata/webmcp/imperative/object-arguments.https.html.ini new file mode 100644 index 0000000..59b25bc --- /dev/null +++ b/wpt-metadata/webmcp/imperative/object-arguments.https.html.ini @@ -0,0 +1,4 @@ +# Draft differences and matching local coverage: TESTING.md. +[object-arguments.https.html] + [tool execution requires a JSON Object input argument; and Arrays are Objects] + expected: FAIL diff --git a/wpt.mjs b/wpt.mjs index 3af4b0a..9ac8910 100644 --- a/wpt.mjs +++ b/wpt.mjs @@ -11,7 +11,7 @@ if (!root || !chrome) const revision = "1a21db90adf8a264370ad806ed761f39e1d435a0"; // Pinned together with `revision` and `tests`; keep TESTING.md in sync. -const EXPECTED_ASSERTIONS = 27; +const EXPECTED_ASSERTIONS = 56; const head = spawnSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }); if (head.error) throw head.error; if (head.status !== 0 || head.stdout.trim() !== revision) { @@ -23,8 +23,8 @@ if (clean.status !== 0) { throw new Error("WPT has tracked changes; restore the pinned sources before running conformance"); } -// Unmodified upstream registration/discovery tests. Execution and full IDL -// coverage belong to the executeTool follow-up. +// Unmodified upstream files. Draft disagreements are tracked by exact subtest +// in wpt-metadata, never by patching tests or silently skipping failures. const tests = [ "imperative/register_tool_name_validation.https.html", "imperative/register_tool_signal.https.html", @@ -36,9 +36,14 @@ const tests = [ "imperative/getTools-imperative-schema.https.html", "imperative/model_context.https.html", "imperative/non-secure.html", + "idlharness.https.window.html", "imperative/register-tool-title.https.html", "imperative/register_tool_with_empty_annotation.https.html", "imperative/getTools-imperative-annotations.https.html", + "imperative/executeTool-invalid-dictionary.https.html", + "imperative/executeTool-error-window-onerror.https.html", + "imperative/executeTool-unregister-resolution-race.https.html", + "imperative/object-arguments.https.html", ]; const report = fileURLToPath(new URL("./wpt-results/report.json", import.meta.url)); mkdirSync(dirname(report), { recursive: true }); @@ -70,6 +75,8 @@ const result = spawnSync( fileURLToPath(new URL("./dist/polyfill.js", import.meta.url)), "--manifest", resolve(root, "MANIFEST.json"), + "--metadata", + fileURLToPath(new URL("./wpt-metadata", import.meta.url)), "--log-mach=-", "--log-wptreport", report,