From 0cb2b89e65b6e487fcc8da22dd19dad32148e147 Mon Sep 17 00:00:00 2001 From: Alex Nahas Date: Mon, 14 Sep 2026 08:10:30 -0700 Subject: [PATCH] feat(polyfill): add executeTool with execution conformance tests Implement executeTool() against the current draft: object input, JSON-serialized results, and abort handling that gives the callback its own signal and a default AbortError while the caller receives its own reason. Extend the WPT selection from 13 files to 18, and from 27 assertions to 56, now that the interface is complete enough for the IDL harness. Record as per-subtest expected failures in wpt-metadata the four upstream files that disagree with the draft on omitted input and on result serialization, rather than patching tests or relaxing the implementation. Carry a temporary ModelContext augmentation mirroring webmcp-types#3 until that ships. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 6 +- README.md | 7 +- TESTING.md | 99 +++- app.test.ts | 14 +- execute.test.ts | 537 ++++++++++++++++++ fixtures/app.js | 10 + fixtures/server.mjs | 4 +- index.test-d.ts | 13 +- index.test.ts | 32 +- index.ts | 132 ++++- package.test.mjs | 9 +- playwright.config.ts | 2 +- tsconfig.test.json | 1 + ...teTool-error-window-onerror.https.html.ini | 6 + ...cuteTool-invalid-dictionary.https.html.ini | 4 + ...-unregister-resolution-race.https.html.ini | 4 + .../object-arguments.https.html.ini | 4 + wpt.mjs | 13 +- 18 files changed, 834 insertions(+), 63 deletions(-) create mode 100644 execute.test.ts create mode 100644 wpt-metadata/webmcp/imperative/executeTool-error-window-onerror.https.html.ini create mode 100644 wpt-metadata/webmcp/imperative/executeTool-invalid-dictionary.https.html.ini create mode 100644 wpt-metadata/webmcp/imperative/executeTool-unregister-resolution-race.https.html.ini create mode 100644 wpt-metadata/webmcp/imperative/object-arguments.https.html.ini diff --git a/AGENTS.md b/AGENTS.md index 41ea58d..4329c46 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,8 @@ # Working on the polyfill -This package implements document-local WebMCP registration and discovery. Use -the official `webmcp-types` dependency without duplicating or augmenting its -declarations. Execution is deferred to a separate change alongside types PR #3. +This package implements the document-local imperative WebMCP draft. Use the +official `webmcp-types` dependency; do not duplicate its declarations. Keep the +temporary `executeTool` augmentation only until the upstream package ships it. No MCP server, transport, extension product, navigator aliases, or legacy API compatibility belongs here. diff --git a/README.md b/README.md index b13432b..1644019 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ await context.registerTool( { signal: registration.signal }, ); -console.log(await context.getTools()); +const [tool] = await context.getTools(); +console.log(await context.executeTool(tool, {})); // {"title":"WebMCP demo"} // Remove the tool when it is no longer needed. registration.abort(); @@ -49,9 +50,9 @@ To install explicitly, import and call `installWebMCP` from `webmcp-polyfill`. I Tools stay in the current document. Cross-document tools, declarative forms, lifecycle window events, and browser agent integration aren't implemented. Nonempty `exposedTo` and `fromOrigins` options reject. -This version supports registration, discovery, and `toolchange`. It does not invoke registered callbacks; `executeTool()` follows alongside [the types update](https://github.com/webmachinelearning/webmcp-types/pull/3). +`executeTool()` accepts an object and returns a JSON-serialized result. Callbacks must validate their inputs; schema inference provides TypeScript checks only. -The implementation follows [draft source `cc45efc`](https://github.com/webmachinelearning/webmcp/blob/cc45efcaf0/index.bs) and passes all 27 selected assertions of the [upstream WPT](https://github.com/web-platform-tests/wpt/tree/1a21db90adf8a264370ad806ed761f39e1d435a0/webmcp) at the pin, with no expected failures. [TESTING.md](https://github.com/webmachinelearning/webmcp-polyfill/blob/main/TESTING.md) records the selection and what it excludes. +The implementation follows [draft source `cc45efc`](https://github.com/webmachinelearning/webmcp/blob/cc45efcaf0/index.bs). Of the 56 selected assertions of the [upstream WPT](https://github.com/web-platform-tests/wpt/tree/1a21db90adf8a264370ad806ed761f39e1d435a0/webmcp) at the pin, 51 pass and five fail where the pinned tests disagree with the draft's `executeTool()` input and result rules. [TESTING.md](https://github.com/webmachinelearning/webmcp-polyfill/blob/main/TESTING.md) records each one and what the selection excludes. The API tracks the draft. Breaking changes ship with notes: in minor releases while the version is 0.x, in majors after 1.0. diff --git a/TESTING.md b/TESTING.md index e4a2a4e..19e2b40 100644 --- a/TESTING.md +++ b/TESTING.md @@ -11,14 +11,15 @@ pnpm test pnpm test:package ``` -| Check | What it runs | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| `index.test-d.ts` | TypeScript against the built package declarations, including upstream schema inference | -| `index.test.ts` | Built bundle served over HTTP in Chromium (native WebMCP disabled), Firefox, and WebKit; coercion, metadata, events, registration abort, errors, detached documents | -| `app.test.ts` | A served application, real button interactions, discovery, unregistration, and reload | -| `native.test.ts` | Real native Chromium registration, then polyfill loading; context and getter identities must survive | -| `pnpm test:package` | Packed tarball installed into a fresh consumer, public type imports, SSR-safe entry points, and package contents | -| `pnpm test:wpt` | Unmodified upstream registration/discovery WPT in real Chrome Canary, with native WebMCP disabled | +| Check | What it runs | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `index.test-d.ts` | TypeScript against the built package declarations, including upstream schema inference | +| `index.test.ts` | Built bundle served over HTTP in Chromium (native WebMCP disabled), Firefox, and WebKit; coercion, metadata, events, errors, detached documents | +| `execute.test.ts` | The same three engines: object input, JSON results, cancellation, concurrent calls, and dispatch-time failures | +| `app.test.ts` | A served application, real button interactions, callback side effects, invalid input, unregistration, and reload | +| `native.test.ts` | Real native Chromium registration, then polyfill loading; context and getter identities must survive | +| `pnpm test:package` | Packed tarball installed into a fresh consumer, public type imports, SSR-safe entry points, and package contents | +| `pnpm test:wpt` | Unmodified upstream WPT and IDL in real Chrome Canary, with native WebMCP disabled | The fixture server binds 127.0.0.1:8793 and sets the required `Origin-Agent-Cluster` header; Playwright never reuses an existing server, so free that port first. @@ -46,39 +47,74 @@ WPT_ROOT=../wpt CHROME_BIN=/path/to/chrome-canary pnpm test:wpt ``` `WPT_PYTHON` and `WPT_VENV` optionally select the interpreter and environment. -The runner checks the revision, rejects tracked source changes, requires all 13 selected files +The runner checks the revision, rejects tracked source changes, requires all 18 selected files to run exactly once, checks assertion counts, and retains `wpt-results/report.json` with browser and upstream revisions. Runner errors, unexpected failures, and unexpected passes fail the command, and nothing is retried. -At this pin, **all 27 selected assertions pass**, with no expected failures. +At this pin, 56 assertions run: **51 pass and five have explicit expected FAIL +metadata**. All 22 IDL assertions pass. API shape coverage does not prove runtime +defaults or complete conformance. + +### Known draft disagreements + +The [published draft](https://webmachinelearning.github.io/webmcp/#dom-modelcontext-executetool) +accepts optional `any inputObject`, rejects non-objects, and has no `{}` default. +It also JSON-serializes callback results, including strings. The pinned WPT +expects a default object and raw string results in the following cases: + +| Upstream file | Expected failures | Local coverage | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------- | +| `executeTool-invalid-dictionary.https.html` | Missing tool invoked without input expects `UnknownError`, rather than the draft's earlier input `TypeError` | Descriptor validation and input errors | +| `executeTool-error-window-onerror.https.html` | Both cases omit input and expect execution to start | Explicit object input, callback/serialization failures, and absence of page errors | +| `executeTool-unregister-resolution-race.https.html` | Invocation omits input | Unregistration during an invocation and its successful result | +| `object-arguments.https.html` | Expects an unquoted string result; also expects omitted input to become `{}` | Array/object inputs, rejected primitive inputs, and JSON result serialization | + +These are five assertions in four files, recorded individually in `wpt-metadata`. +The pin, [1a21db9](https://github.com/web-platform-tests/wpt/commit/1a21db90adf8a264370ad806ed761f39e1d435a0), +is the WPT export of the Chromium change for +[spec PR #246](https://github.com/webmachinelearning/webmcp/pull/246) and +[#251](https://github.com/webmachinelearning/webmcp/pull/251), and it rewrote all four of these +files. The draft's input and result rules have not changed since #251 merged, so these are not +stale tests awaiting an update: four of the assertions invoke `executeTool()` with no input, +which the draft rejects with a `TypeError` before it looks the tool up, and `object-arguments` +asserts an unquoted `"Success"` from a callback whose result the draft JSON-serializes. Re-read the live +draft before changing any expectation. An expected failure can stop at its first assertion, so +the local tests cover the behavior after that point. + +`webmcp-types` PR #3 and the augmentation in `index.ts` both declare `inputObject?: object`, +which is narrower than the IDL's `any`: omitting the argument type-checks and then rejects at +runtime, as the draft requires. ### Excluded coverage -Execution tests and the full IDL harness are deferred with `executeTool()`. -The IDL includes that method, so running the full shape suite against this -registration/discovery subset would intentionally fail. Keep that exclusion -explicit rather than adding expected failures for an API not yet included. - -Cross-document discovery, frame-tree routing, declarative forms, browser -permissions integration, CSS states, navigation/BFCache, and lifecycle window -events are also outside this initial implementation. +Cross-document discovery/execution, frame-tree routing, navigation cancellation, +declarative forms, browser permissions integration, CSS states, and lifecycle +window events are outside this initial implementation. The pinned +`executeTool-abort.https.html` is excluded rather than carried as expected +failures: its second subtest waits forever on a `toolactivated` event that the +draft still leaves [unspecified](https://github.com/webmachinelearning/webmcp/issues/146), +so the file times out and its last three subtests never run at all. Expected-failure +metadata cannot express that, and cancellation is exercised directly in all three +engines instead. +`exposedTo-invalid-origins.https.html` is excluded for the `exposedTo` divergence +recorded below, not for cross-document routing. WPT's `--inject-script` modifies testharness pages; it does not install the bundle in `/common/blank.html` helper frames. Detached-frame WPT therefore cannot run unchanged in this lane. The local test serves an instrumented iframe, removes -it, and checks registration, discovery, and exception realms. +it, and checks all three operations and exception realms. ## Where to look when upstream changes -| Source | Use | -| --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | -| [Draft source and history](https://github.com/webmachinelearning/webmcp/commits/main/index.bs) | Normative algorithms, Web IDL, open issues; compare with the revision below | -| [WPT webmcp](https://github.com/web-platform-tests/wpt/tree/master/webmcp) and [results](https://wpt.fyi/results/webmcp) | Executable assertions and native cross-browser results; these results are not polyfill results | -| [WPT IDL](https://github.com/web-platform-tests/wpt/blob/master/interfaces/webmcp.idl) | Generated interface snapshot; it can lag the published draft | -| [Official types](https://github.com/webmachinelearning/webmcp-types) | Public declarations, schema inference, and pending API updates | -| [Blink script_tools](https://chromium.googlesource.com/chromium/src/+/main/third_party/blink/renderer/core/script_tools/) | Chromium IDL, implementation, tests, and commit-linked bugs | -| [Gecko source search](https://searchfox.org/mozilla-central/search?q=ModelContext) and [Mozilla position](https://github.com/mozilla/standards-positions/issues/1412) | Locate Firefox implementation work and discussion; a position is not evidence of shipped support | +| Source | Use | +| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| [Draft source and history](https://github.com/webmachinelearning/webmcp/commits/main/index.bs) | Normative algorithms, Web IDL, open issues; compare with the revision below | +| [WPT webmcp](https://github.com/web-platform-tests/wpt/tree/master/webmcp) and [results](https://wpt.fyi/results/webmcp) | Executable assertions and native cross-browser results; these results are not polyfill results | +| [WPT IDL](https://github.com/web-platform-tests/wpt/blob/master/interfaces/webmcp.idl) | Generated interface snapshot; it can lag the published draft | +| [Official types](https://github.com/webmachinelearning/webmcp-types) | Public declarations, schema inference, and pending API updates | +| [Blink script_tools](https://chromium.googlesource.com/chromium/src/+/main/third_party/blink/renderer/core/script_tools/) | Chromium IDL, implementation, tests, and commit-linked bugs | +| [Gecko source search](https://searchfox.org/mozilla-central/search?q=ModelContext) and [Mozilla position](https://github.com/mozilla/standards-positions/issues/1412) | Locate Firefox implementation work and discussion; a position is not evidence of shipped support | Reproduce a disagreement before changing code or expectations, and record what changed in the draft, types, WPT, and browser implementation separately. @@ -93,7 +129,10 @@ explicit `any`. Warnings fail the command, and `pnpm test` runs it before compil The implementation was compared with [draft source `cc45efc`](https://github.com/webmachinelearning/webmcp/blob/cc45efcaf0/index.bs). It uses timers to queue tasks; JavaScript cannot reproduce the browser's WebMCP task source, nor the draft's abort *algorithms*, which run before an abort event -rather than as a listener. +rather than as a listener. One consequence is observable: a signal aborted before +the dispatch timer fires rejects the caller and never runs the callback, where the +draft dispatches to the target document and then cancels through the callback's own +signal. The runtime checks the `tools` Permissions Policy when the browser exposes it. No engine lists `tools` in `permissionsPolicy.features()` today, so that branch @@ -110,6 +149,10 @@ implementation has nowhere to expose a tool to; rejecting keeps the polyfill fro implying cross-document support it does not have. Origins are validated first, so an untrustworthy one still fails with the `SecurityError` the draft requires. +`webmcp-types@0.1.7` does not declare `executeTool()`. The method augmentation +in `index.ts` is temporary; remove it when [types PR #3](https://github.com/webmachinelearning/webmcp-types/pull/3) +is included in a release. + To test types changes, keep `webmcp-types` beside this checkout and run `pnpm link ../webmcp-types`, then `pnpm typecheck`. Restore the published dependency with `pnpm install --force` before package checks. Do not commit diff --git a/app.test.ts b/app.test.ts index 7e61787..e7d6620 100644 --- a/app.test.ts +++ b/app.test.ts @@ -1,6 +1,6 @@ import { test, expect } from "@playwright/test"; -test("a served application registers, discovers, unregisters, and resets on reload", async ({ +test("a served application executes, rejects invalid input, unregisters, and resets on reload", async ({ page, browser, }, testInfo) => { @@ -12,11 +12,13 @@ test("a served application registers, discovers, unregisters, and resets on relo page.on("pageerror", (error) => errors.push(error)); await page.goto("/app"); await expect(page.locator("#status")).toHaveText("registered"); - expect( - await page.evaluate(async () => - (await document.modelContext!.getTools()).map((tool) => tool.name), - ), - ).toEqual(["increment"]); + await page.getByRole("button", { name: "Execute on page" }).click(); + await expect(page.locator("#result")).toHaveText('{"count":2}'); + await expect(page.locator("#count")).toHaveText("2"); + await page.getByLabel("Amount").fill("-1"); + await page.getByRole("button", { name: "Execute on page" }).click(); + await expect(page.locator("#result")).toHaveText("UnknownError"); + await expect(page.locator("#count")).toHaveText("2"); await page.getByRole("button", { name: "Unregister", exact: true }).click(); await expect(page.locator("#status")).toHaveText("unregistered"); expect(await page.evaluate(() => document.modelContext!.getTools())).toEqual([]); diff --git a/execute.test.ts b/execute.test.ts new file mode 100644 index 0000000..7fe98e4 --- /dev/null +++ b/execute.test.ts @@ -0,0 +1,537 @@ +import { test, expect } from "@playwright/test"; +import "./index.js"; + +test.beforeEach(async ({ page }) => { + await page.goto("/"); + expect(await page.evaluate(() => "modelContext" in document)).toBe(false); +}); + +test("ignores late results after cancellation, including serialization side effects", async ({ + page, +}) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + let finish!: (value: object) => void; + let serialized = false; + await context.registerTool({ + name: "late", + description: "Late", + execute() { + entered(); + return new Promise((resolve) => { + finish = resolve; + }); + }, + }); + const [tool] = await context.getTools(); + const caller = new AbortController(); + const result = context + .executeTool(tool, {}, { signal: caller.signal }) + .catch((error) => error); + await started; + caller.abort("cancelled"); + await result; + finish({ + toJSON() { + serialized = true; + return {}; + }, + }); + await context.getTools(); + return serialized; + }), + ).toBe(false); +}); + +test("concurrent calls to the same tool have independent cancellation", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + const signals: AbortSignal[] = []; + let start!: () => void; + const bothStarted = new Promise((resolve) => { + start = resolve; + }); + let finish!: (value: object) => void; + await context.registerTool({ + name: "concurrent", + description: "Concurrent", + execute(_, { signal }) { + signals.push(signal); + if (signals.length === 2) start(); + return new Promise((resolve) => { + finish = resolve; + }); + }, + }); + const [tool] = await context.getTools(); + const caller = new AbortController(); + const first = context + .executeTool(tool, {}, { signal: caller.signal }) + .catch((error) => error); + const second = context.executeTool(tool, {}); + await bothStarted; + const aborted = new Promise((resolve) => + signals[0].addEventListener("abort", () => resolve(), { once: true }), + ); + caller.abort("first only"); + await aborted; + finish({ second: true }); + return { + first: await first, + second: await second, + aborted: signals.map((signal) => signal.aborted), + }; + }), + ).toEqual({ first: "first only", second: '{"second":true}', aborted: [true, false] }); +}); + +test("executes copied object and array inputs with a fresh callback signal", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + "use strict"; + const context = document.modelContext!; + const input = { nested: { value: 1 } }; + const caller = new AbortController(); + await context.registerTool({ + name: "echo", + description: "Echo", + execute(args, { signal }) { + return { + args, + fresh: signal instanceof AbortSignal && signal !== caller.signal, + unbound: this === undefined, + }; + }, + }); + const [tool] = await context.getTools(); + const pending = context.executeTool(tool, input, { signal: caller.signal }); + input.nested.value = 9; + return [JSON.parse(await pending), JSON.parse(await context.executeTool(tool, [1, 2]))]; + }), + ).toEqual([ + { args: { nested: { value: 1 } }, fresh: true, unbound: true }, + { args: [1, 2], fresh: true, unbound: true }, + ]); +}); + +test("rejects legacy JSON strings and preserves input serialization errors", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + await context.registerTool({ name: "x", description: "X", execute: () => null }); + const [tool] = await context.getTools(); + const errors = []; + for (const input of [ + "{}", + null, + undefined, + 1, + { + toJSON() { + throw new RangeError("input"); + }, + }, + ]) { + try { + // @ts-expect-error Exercise primitive inputs from JavaScript callers. + await context.executeTool(tool, input); + errors.push("resolved"); + } catch (error) { + if (!(error instanceof Error)) throw error; + errors.push(error.name); + } + } + return errors; + }), + ).toEqual(["TypeError", "TypeError", "TypeError", "TypeError", "RangeError"]); +}); + +test("serializes results as JSON and rejects callback or serialization failures", async ({ + page, +}) => { + const pageErrors: Error[] = []; + page.on("pageerror", (error) => pageErrors.push(error)); + await page.addScriptTag({ url: "/auto.js" }); + const results = await page.evaluate(async () => { + const context = document.modelContext!; + const circular = {}; + Object.assign(circular, { self: circular }); + await context.registerTool({ + name: "x", + description: "X", + execute({ value }) { + if (value === "throw") throw new Error("callback"); + if (value === "circular") return circular; + if (value === "undefined") return undefined; + return value; + }, + }); + const [tool] = await context.getTools(); + const output = []; + for (const value of ["hello", null, 42, { success: true }, "throw", "circular", "undefined"]) { + try { + output.push(await context.executeTool(tool, { value })); + } catch (error) { + if (!(error instanceof Error)) throw error; + output.push(error.name); + } + } + return output; + }); + expect(results).toEqual([ + '"hello"', + "null", + "42", + '{"success":true}', + "UnknownError", + "UnknownError", + "UnknownError", + ]); + expect(pageErrors).toEqual([]); +}); + +test("cancels the caller immediately and sends a default AbortError to the callback", async ({ + page, +}) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + let observed!: (reason: string) => void; + const callbackAborted = new Promise((resolve) => { + observed = resolve; + }); + await context.registerTool({ + name: "pending", + description: "Pending", + execute(_input, { signal }) { + entered(); + return new Promise((resolve) => + signal.addEventListener( + "abort", + () => { + observed(signal.reason.name); + resolve("late result"); + }, + { once: true }, + ), + ); + }, + }); + const [tool] = await context.getTools(); + const controller = new AbortController(); + controller.signal.addEventListener("abort", (event) => event.stopImmediatePropagation()); + const result = context + .executeTool(tool, {}, { signal: controller.signal }) + .catch((error) => error); + await started; + controller.abort("caller reason"); + return [await result, await callbackAborted]; + }), + ).toEqual(["caller reason", "AbortError"]); +}); + +test("unregistration leaves an already-running invocation alive", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + const registration = new AbortController(); + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + let complete!: (value: string) => void; + let callbackSignal!: AbortSignal; + await context.registerTool( + { + name: "pending", + description: "Pending", + execute(_input, { signal }) { + callbackSignal = signal; + entered(); + return new Promise((resolve) => { + complete = resolve; + }); + }, + }, + { signal: registration.signal }, + ); + const [tool] = await context.getTools(); + const pending = context.executeTool(tool, {}); + await started; + registration.abort(); + const count = (await context.getTools()).length; + complete("finished"); + return { count, aborted: callbackSignal.aborted, result: await pending }; + }), + ).toEqual({ count: 0, aborted: false, result: '"finished"' }); +}); + +// A polyfill divergence, not a draft requirement: the draft dispatches to the target +// document and cancels through the callback's own signal. See TESTING.md. +test("aborting before the dispatch task rejects without starting the callback", async ({ + page, +}) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + let executions = 0; + await context.registerTool({ + name: "x", + description: "X", + execute() { + executions++; + }, + }); + const [tool] = await context.getTools(); + const controller = new AbortController(); + const pending = context + .executeTool(tool, {}, { signal: controller.signal }) + .catch((error) => error); + controller.abort("before dispatch"); + const reason = await pending; + await context.getTools(); + return [executions, reason]; + }), + ).toEqual([0, "before dispatch"]); +}); + +test("execution converts every descriptor member before invoking the tool", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + const result = await page.evaluate(async () => { + const context = document.modelContext!; + let calls = 0; + await context.registerTool({ + name: "descriptor", + description: "Descriptor conversion", + execute() { + return ++calls; + }, + }); + const [tool] = await context.getTools(); + const errors = []; + const fakeWindow = {}; + Object.assign(fakeWindow, { window: fakeWindow }); + const invalid: unknown[] = [ + { ...tool, title: Symbol() }, + { ...tool, inputSchema: 1 }, + { ...tool, annotations: 1 }, + { ...tool, window: fakeWindow }, + { ...tool, window: null }, + ]; + for (const descriptor of invalid) { + try { + // @ts-expect-error Exercise invalid JavaScript descriptor members. + await context.executeTool(descriptor, {}); + errors.push("resolved"); + } catch (error) { + if (!(error instanceof Error)) throw error; + errors.push(error.name); + } + } + const invalidCalls = calls; + const reads: string[] = []; + const descriptor = { + get annotations() { + reads.push("annotations"); + return { + get consequentialHint() { + reads.push("consequentialHint"); + return false; + }, + get readOnlyHint() { + reads.push("readOnlyHint"); + return false; + }, + get untrustedContentHint() { + reads.push("untrustedContentHint"); + return false; + }, + }; + }, + get description() { + reads.push("description"); + return tool.description; + }, + get inputSchema() { + reads.push("inputSchema"); + return {}; + }, + get name() { + reads.push("name"); + return tool.name; + }, + get origin() { + reads.push("origin"); + return tool.origin; + }, + get title() { + reads.push("title"); + return "Descriptor"; + }, + get window() { + reads.push("window"); + return tool.window; + }, + }; + await context.executeTool(descriptor, {}); + return { errors, invalidCalls, reads, calls }; + }); + expect(result).toEqual({ + errors: ["TypeError", "TypeError", "TypeError", "TypeError", "TypeError"], + invalidCalls: 0, + reads: [ + "annotations", + "consequentialHint", + "readOnlyHint", + "untrustedContentHint", + "description", + "inputSchema", + "name", + "origin", + "title", + "window", + ], + calls: 1, + }); +}); + + +test("a tool belonging to another window cannot be executed", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + await context.registerTool({ name: "local", description: "Local", execute: () => null }); + const [tool] = await context.getTools(); + const iframe = document.createElement("iframe"); + iframe.src = "/app"; + await new Promise((resolve) => { + iframe.onload = () => resolve(); + document.body.append(iframe); + }); + try { + await context.executeTool({ ...tool, window: iframe.contentWindow! }, {}); + return "resolved"; + } catch (error) { + if (!(error instanceof Error)) throw error; + return error.name; + } + }), + ).toBe("UnknownError"); +}); + +test("descriptor origins are parsed, and a mismatch fails like a missing tool", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + await context.registerTool({ name: "x", description: "X", execute: () => null }); + const [tool] = await context.getTools(); + const results: [string, string][] = []; + for (const [label, origin] of [ + ["unparseable", "not a url"], + ["opaque", "data:,x"], + ["another origin", "https://other.test"], + ]) { + try { + await context.executeTool({ ...tool, origin }, {}); + results.push([label, "resolved"]); + } catch (error) { + if (!(error instanceof Error)) throw error; + results.push([label, error.name]); + } + } + return results; + }), + ).toEqual([ + ["unparseable", "NotSupportedError"], + ["opaque", "NotSupportedError"], + ["another origin", "UnknownError"], + ]); +}); + +test("a signal option that is not an AbortSignal is rejected", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + await context.registerTool({ name: "x", description: "X", execute: () => null }); + const [tool] = await context.getTools(); + const results: [string, string][] = []; + for (const signal of [1, {}, null, "abort"] as unknown[]) { + const label = JSON.stringify(signal)!; + try { + // @ts-expect-error Exercise invalid JavaScript callers at the Web IDL boundary. + await context.executeTool(tool, {}, { signal }); + results.push([label, "resolved"]); + } catch (error) { + if (!(error instanceof Error)) throw error; + results.push([label, error.name]); + } + } + return results; + }), + ).toEqual([ + ["1", "TypeError"], + ["{}", "TypeError"], + ["null", "TypeError"], + ['"abort"', "TypeError"], + ]); +}); + +test("unregistering before dispatch rejects without running the callback", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + const registration = new AbortController(); + let executions = 0; + await context.registerTool( + { name: "x", description: "X", execute: () => ++executions }, + { signal: registration.signal }, + ); + const [tool] = await context.getTools(); + const pending = context.executeTool(tool, {}).catch((error: Error) => error.name); + registration.abort(); + return [await pending, executions]; + }), + ).toEqual(["UnknownError", 0]); +}); + +test("input that serializes to a non-object rejects before the callback runs", async ({ page }) => { + await page.addScriptTag({ url: "/auto.js" }); + expect( + await page.evaluate(async () => { + const context = document.modelContext!; + let executions = 0; + await context.registerTool({ + name: "x", + description: "X", + execute: () => ++executions, + }); + const [tool] = await context.getTools(); + const name = await context.executeTool(tool, { toJSON: () => 5 }).then( + () => "resolved", + (error: Error) => error.name, + ); + return [name, executions]; + }), + ).toEqual(["UnknownError", 0]); +}); diff --git a/fixtures/app.js b/fixtures/app.js index c078bb1..c8b01a9 100644 --- a/fixtures/app.js +++ b/fixtures/app.js @@ -31,4 +31,14 @@ element("unregister").onclick = async () => { await context.getTools(); element("status").textContent = "unregistered"; }; +element("execute").onclick = async () => { + try { + const [tool] = await context.getTools(); + element("result").textContent = await context.executeTool(tool, { + amount: Number(element("amount").value), + }); + } catch (error) { + element("result").textContent = error.name; + } +}; await register(); diff --git a/fixtures/server.mjs b/fixtures/server.mjs index 18e5252..495fbf3 100644 --- a/fixtures/server.mjs +++ b/fixtures/server.mjs @@ -3,8 +3,10 @@ import { readFile } from "node:fs/promises"; const app = `WebMCP counter

Counter

0 + -loading`; + +loading`; createServer(async (request, response) => { const path = new URL(request.url, "http://localhost").pathname; diff --git a/index.test-d.ts b/index.test-d.ts index af01976..da7bcf2 100644 --- a/index.test-d.ts +++ b/index.test-d.ts @@ -14,8 +14,13 @@ if (document.modelContext) { return { name, aborted, invalid }; }, }); - const tools: WebMCP.RegisteredTool[] = await context.getTools(); - // @ts-expect-error Execution is deferred to a separate change. - void context.executeTool; - void tools; + const [tool] = await context.getTools(); + const result: string = await context.executeTool( + tool, + {}, + { signal: new AbortController().signal }, + ); + // @ts-expect-error The current draft accepts objects, not serialized JSON. + context.executeTool(tool, "{}"); + void result; } diff --git a/index.test.ts b/index.test.ts index 875573f..368cb78 100644 --- a/index.test.ts +++ b/index.test.ts @@ -18,6 +18,11 @@ test("every operation rejects when the server opts out of origin-keyed agent clu for (const operation of [ () => context.getTools(), () => context.registerTool({ name: "x", description: "X", execute: () => null }), + () => + context.executeTool( + { name: "x", title: "", description: "X", window, origin: location.origin }, + {}, + ), ]) { try { await operation(); @@ -29,7 +34,7 @@ test("every operation rejects when the server opts out of origin-keyed agent clu } return errors; }), - ).toEqual(["SecurityError", "SecurityError"]); + ).toEqual(["SecurityError", "SecurityError", "SecurityError"]); }); test("only potentially trustworthy origins reach the cross-document refusal", async ({ page }) => { @@ -250,6 +255,7 @@ test("operations on a real detached frame reject in the frame's realm", async ({ document.body.append(iframe); await loaded; const context = iframe.contentDocument!.modelContext!; + const tool = (await context.getTools())[0]; const FrameException = iframe.contentDocument!.defaultView!.DOMException; iframe.remove(); const errors = []; @@ -257,6 +263,7 @@ test("operations on a real detached frame reject in the frame's realm", async ({ () => context.getTools(), () => context.registerTool({ name: "detached", description: "Detached", execute: () => null }), + () => context.executeTool(tool, {}), ]) { try { await operation(); @@ -267,7 +274,7 @@ test("operations on a real detached frame reject in the frame's realm", async ({ } return errors; }); - expect(frame).toEqual(["InvalidStateError", "InvalidStateError"]); + expect(frame).toEqual(["InvalidStateError", "InvalidStateError", "InvalidStateError"]); }); test("installs once, exposes only standard members, and keeps document identity", async ({ @@ -308,14 +315,14 @@ test("installs once, exposes only standard members, and keeps document identity" writable: descriptor.set !== undefined, alias: "modelContext" in navigator, testing: "modelContextTesting" in navigator, - lengths: [context.registerTool.length, context.getTools.length], + lengths: [context.registerTool.length, context.getTools.length, context.executeTool.length], constructionError, getterErrors, }; }); expect(result).toEqual({ same: true, - members: ["getTools", "ontoolchange", "registerTool"], + members: ["executeTool", "getTools", "ontoolchange", "registerTool"], own: [], brand: "[object ModelContext]", instance: true, @@ -324,7 +331,7 @@ test("installs once, exposes only standard members, and keeps document identity" writable: false, alias: false, testing: false, - lengths: [1, 0], + lengths: [1, 0, 1], constructionError: "TypeError", getterErrors: ["TypeError", "TypeError", "TypeError", "TypeError", "TypeError"], }); @@ -450,7 +457,7 @@ test("rejects invalid descriptors, duplicates and unserializable schemas", async expect(registered).toEqual(["valid"]); }); -test("coerces registration dictionary members", async ({ page }) => { +test("uses dictionary coercion without retaining or binding the tool object", async ({ page }) => { await page.addScriptTag({ url: "/auto.js" }); const result = await page.evaluate(async () => { "use strict"; @@ -460,7 +467,9 @@ test("coerces registration dictionary members", async ({ page }) => { title: "\ud800", description: true, annotations: { readOnlyHint: 1 }, - execute: () => null, + execute() { + return this === undefined; + }, }; // @ts-expect-error Web IDL coerces the deliberately non-string fields. await context.registerTool(descriptor); @@ -470,6 +479,7 @@ test("coerces registration dictionary members", async ({ page }) => { title: tool.title, description: tool.description, annotations: tool.annotations, + result: await context.executeTool(tool, {}), }; }); expect(result).toEqual({ @@ -477,6 +487,7 @@ test("coerces registration dictionary members", async ({ page }) => { title: "\ufffd", description: "true", annotations: { consequentialHint: false, readOnlyHint: true, untrustedContentHint: false }, + result: "true", }); }); @@ -574,7 +585,12 @@ test("validates origins and refuses cross-document exposure", async ({ page }) = } return errors; }), - ).toEqual(["SecurityError", "SecurityError", "NotSupportedError", "NotSupportedError"]); + ).toEqual([ + "SecurityError", + "SecurityError", + "NotSupportedError", + "NotSupportedError", + ]); }); test("inactive documents get their own context but cannot register tools", async ({ page }) => { diff --git a/index.ts b/index.ts index 5d179ff..8ed3932 100644 --- a/index.ts +++ b/index.ts @@ -6,8 +6,27 @@ export type { WebMCP } from "webmcp-types"; * SPDX-License-Identifier: MIT */ -// Capture the constructor before a detached WindowProxy stops exposing it. +// Mirrors webmachinelearning/webmcp-types#3; delete once that ships in a release. +declare global { + namespace WebMCP { + interface ModelContextExecuteToolOptions { + signal?: AbortSignal; + } + interface ModelContext { + executeTool( + tool: RegisteredTool, + inputObject?: object, + options?: ModelContextExecuteToolOptions, + ): Promise; + } + } +} + +// Capture native bindings before a detached WindowProxy stops exposing them. HTML declares +// `window` as [LegacyUnforgeable], so its getter is an own property of every Window; a realm +// without one has no Document either, and `installWebMCP` exits early there. const NativeDOMException = globalThis.DOMException; +const windowGetter = Object.getOwnPropertyDescriptor(globalThis, "window")?.get; interface Tool { metadata: Omit; @@ -156,7 +175,7 @@ function activeView(owner: Document): Window { } // Timers approximate the unavailable WebMCP task source; exact -// inter-source ordering requires native support. +// inter-source ordering and document-navigation semantics require native support. function queueTask(callback: () => void): void { setTimeout(callback, 0); } @@ -199,7 +218,7 @@ class ModelContextPolyfill extends EventTarget implements WebMCP.ModelContext { const description = domString(required(descriptor.description, "description")); const callback = required(descriptor.execute, "execute"); if (typeof callback !== "function") throw new TypeError("execute must be a function"); - // SAFETY: Web IDL requires callability only. + // SAFETY: Web IDL requires callability only; arguments and results are converted at invocation. const execute = callback as WebMCP.ToolExecuteCallback; const inputSchema = descriptor.inputSchema; if (inputSchema !== undefined && !isObject(inputSchema)) @@ -275,6 +294,112 @@ class ModelContextPolyfill extends EventTarget implements WebMCP.ModelContext { .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); return new Promise((resolve) => queueTask(() => resolve(tools))); } + + async executeTool( + tool: WebMCP.RegisteredTool, + inputObject: object | undefined = undefined, + options: WebMCP.ModelContextExecuteToolOptions = {}, + ): Promise { + // Web IDL converts every member of RegisteredTool, in lexicographical order, before the + // algorithm runs, so the conversions whose results are unused below are kept for their + // observable effects: reading the caller's getters, and throwing TypeError. + const descriptor = dictionary(tool); + toolAnnotations(descriptor.annotations); + domString(required(descriptor.description, "description")); + const inputSchema = descriptor.inputSchema; + if (inputSchema !== undefined && !isObject(inputSchema)) + throw new TypeError("inputSchema must be an object"); + const name = domString(required(descriptor.name, "name")); + const origin = domString(required(descriptor.origin, "origin")).toWellFormed(); + const title = descriptor.title; + if (title !== undefined) domString(title); + const target = required(descriptor.window, "window"); + if (!isObject(target)) throw new TypeError("window must be a Window"); + // SAFETY: the native getter performs the Window brand check, including for cross-origin + // WindowProxy objects, and throws for anything else. Its value is unused. + windowGetter!.call(target); + const signal = signalOption(dictionary(options).signal); + activeView(this.#owner); + let expectedOrigin = "null"; + try { + expectedOrigin = new URL(origin).origin; + } catch { + // An unparseable origin falls through to the opaque check below. + } + // `URL.origin` serializes an opaque origin as "null", which no document's origin matches. + if (expectedOrigin === "null") { + throw new NativeDOMException("Invalid or opaque origin", "NotSupportedError"); + } + if (!isObject(inputObject)) throw new TypeError("inputObject must be an object"); + const input = serialize(inputObject); + signal?.throwIfAborted(); + if (target !== this.#owner.defaultView) { + // The draft rejects a target in another traversable with UnknownError, and reports a + // target document that has no such tool the same way. Routing to another document in this + // traversable needs native WebMCP, so both arrive here indistinguishably. + throw new NativeDOMException("Tool execution failed", "UnknownError"); + } + + return new Promise((resolve, reject) => { + const controller = new AbortController(); + let settled = false; + let callbackSettled = false; + // Returns true to the first caller only; whoever wins owns settling the promise. + const claimSettlement = (): boolean => { + if (settled) return false; + settled = true; + signal?.removeEventListener("abort", abort); + return true; + }; + const abort = (): void => { + if (!claimSettlement()) return; + reject(signal!.reason); + // The callback receives a fresh signal and a default AbortError, while the caller + // receives its own abort reason. Aborting a task later lets the caller's rejection + // arrive first, and leaves a callback that has already settled alone. + queueTask(() => { + if (!callbackSettled) controller.abort(); + }); + }; + const fail = (): void => { + callbackSettled = true; + queueTask(() => { + if (claimSettlement()) + reject(new NativeDOMException("Tool execution failed", "UnknownError")); + }); + }; + const complete = (value: unknown): void => { + callbackSettled = true; + // Checked before serializing: a cancelled call must not run the author's toJSON. + if (settled) return; + try { + const result = serialize(value); + queueTask(() => { + if (claimSettlement()) resolve(result); + }); + } catch { + fail(); + } + }; + signal?.addEventListener("abort", abort, { once: true }); + queueTask(() => { + if (settled) return; + try { + // Every dispatch failure is indistinguishable to the caller by design, so each bare + // throw funnels into fail() and the draft's UnknownError. + const view = activeView(this.#owner); + const entry = this.#tools.get(name); + if (!entry || expectedOrigin !== view.origin) throw new Error(); + const args: unknown = JSON.parse(input); + if (!isObject(args)) throw new Error(); + const { execute } = entry; + Promise.resolve(execute(args, { signal: controller.signal })).then(complete, fail); + } catch { + fail(); + } + }); + }); + } } const contexts = new WeakMap(); @@ -299,6 +424,7 @@ Object.defineProperties(ModelContextPolyfill.prototype, { // Class members are non-enumerable; Web IDL interface members are enumerable. registerTool: { enumerable: true }, getTools: { enumerable: true }, + executeTool: { enumerable: true }, ontoolchange: { enumerable: true }, }); diff --git a/package.test.mjs b/package.test.mjs index 7af0308..0b4dba2 100644 --- a/package.test.mjs +++ b/package.test.mjs @@ -64,9 +64,12 @@ try { return { count }; }, }); - async function discover(context: WebMCP.ModelContext) { - const tools: WebMCP.RegisteredTool[] = await context.getTools(); - return tools; + async function invoke(context: WebMCP.ModelContext) { + const [tool] = await context.getTools(); + const result: string = await context.executeTool(tool, { count: 1 }); + // @ts-expect-error legacy JSON-string input is not supported + await context.executeTool(tool, '{}'); + return result; } `, ); diff --git a/playwright.config.ts b/playwright.config.ts index 5e731e4..3c6d3ec 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "@playwright/test"; export default defineConfig({ - testMatch: /(?:index|app)\.test\.ts$/, + testMatch: /(?:index|execute|app)\.test\.ts$/, forbidOnly: !!process.env.CI, fullyParallel: true, workers: 3, diff --git a/tsconfig.test.json b/tsconfig.test.json index 6fa915b..058e7d9 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -4,6 +4,7 @@ "files": [ "index.test-d.ts", "index.test.ts", + "execute.test.ts", "app.test.ts", "native.test.ts", "playwright.config.ts" diff --git a/wpt-metadata/webmcp/imperative/executeTool-error-window-onerror.https.html.ini b/wpt-metadata/webmcp/imperative/executeTool-error-window-onerror.https.html.ini new file mode 100644 index 0000000..a0fff5f --- /dev/null +++ b/wpt-metadata/webmcp/imperative/executeTool-error-window-onerror.https.html.ini @@ -0,0 +1,6 @@ +# Draft differences and matching local coverage: TESTING.md. +[executeTool-error-window-onerror.https.html] + [Failed tool execution does not trigger window.onerror] + expected: FAIL + [Tool execution returning circular object rejects and does not trigger window.onerror] + expected: FAIL diff --git a/wpt-metadata/webmcp/imperative/executeTool-invalid-dictionary.https.html.ini b/wpt-metadata/webmcp/imperative/executeTool-invalid-dictionary.https.html.ini new file mode 100644 index 0000000..7e3f29e --- /dev/null +++ b/wpt-metadata/webmcp/imperative/executeTool-invalid-dictionary.https.html.ini @@ -0,0 +1,4 @@ +# Draft differences and matching local coverage: TESTING.md. +[executeTool-invalid-dictionary.https.html] + [executeTool() rejects with UnknownError when a non-existent tool object is supplied] + expected: FAIL diff --git a/wpt-metadata/webmcp/imperative/executeTool-unregister-resolution-race.https.html.ini b/wpt-metadata/webmcp/imperative/executeTool-unregister-resolution-race.https.html.ini new file mode 100644 index 0000000..054efdf --- /dev/null +++ b/wpt-metadata/webmcp/imperative/executeTool-unregister-resolution-race.https.html.ini @@ -0,0 +1,4 @@ +# Draft differences and matching local coverage: TESTING.md. +[executeTool-unregister-resolution-race.https.html] + [executeTool resolves successfully even when tool is unregistered synchronously after its promise resolves] + expected: FAIL diff --git a/wpt-metadata/webmcp/imperative/object-arguments.https.html.ini b/wpt-metadata/webmcp/imperative/object-arguments.https.html.ini new file mode 100644 index 0000000..59b25bc --- /dev/null +++ b/wpt-metadata/webmcp/imperative/object-arguments.https.html.ini @@ -0,0 +1,4 @@ +# Draft differences and matching local coverage: TESTING.md. +[object-arguments.https.html] + [tool execution requires a JSON Object input argument; and Arrays are Objects] + expected: FAIL diff --git a/wpt.mjs b/wpt.mjs index 3af4b0a..9ac8910 100644 --- a/wpt.mjs +++ b/wpt.mjs @@ -11,7 +11,7 @@ if (!root || !chrome) const revision = "1a21db90adf8a264370ad806ed761f39e1d435a0"; // Pinned together with `revision` and `tests`; keep TESTING.md in sync. -const EXPECTED_ASSERTIONS = 27; +const EXPECTED_ASSERTIONS = 56; const head = spawnSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }); if (head.error) throw head.error; if (head.status !== 0 || head.stdout.trim() !== revision) { @@ -23,8 +23,8 @@ if (clean.status !== 0) { throw new Error("WPT has tracked changes; restore the pinned sources before running conformance"); } -// Unmodified upstream registration/discovery tests. Execution and full IDL -// coverage belong to the executeTool follow-up. +// Unmodified upstream files. Draft disagreements are tracked by exact subtest +// in wpt-metadata, never by patching tests or silently skipping failures. const tests = [ "imperative/register_tool_name_validation.https.html", "imperative/register_tool_signal.https.html", @@ -36,9 +36,14 @@ const tests = [ "imperative/getTools-imperative-schema.https.html", "imperative/model_context.https.html", "imperative/non-secure.html", + "idlharness.https.window.html", "imperative/register-tool-title.https.html", "imperative/register_tool_with_empty_annotation.https.html", "imperative/getTools-imperative-annotations.https.html", + "imperative/executeTool-invalid-dictionary.https.html", + "imperative/executeTool-error-window-onerror.https.html", + "imperative/executeTool-unregister-resolution-race.https.html", + "imperative/object-arguments.https.html", ]; const report = fileURLToPath(new URL("./wpt-results/report.json", import.meta.url)); mkdirSync(dirname(report), { recursive: true }); @@ -70,6 +75,8 @@ const result = spawnSync( fileURLToPath(new URL("./dist/polyfill.js", import.meta.url)), "--manifest", resolve(root, "MANIFEST.json"), + "--metadata", + fileURLToPath(new URL("./wpt-metadata", import.meta.url)), "--log-mach=-", "--log-wptreport", report,