From b411af4b047f8cdd64a08f58ce277d13abfef148 Mon Sep 17 00:00:00 2001 From: Austin Hamilton Date: Mon, 5 Oct 2026 01:30:47 +0000 Subject: [PATCH] feat(examples): demonstrate semantic input bindings --- docs/ai-agent-integration.md | 29 ++++-- .../basic-dry-run/ai-agent-integration.ts | 92 +++++++++++++++--- examples/basic-dry-run/check.ts | 95 ++++++++++++++++--- 3 files changed, 183 insertions(+), 33 deletions(-) diff --git a/docs/ai-agent-integration.md b/docs/ai-agent-integration.md index 4078f7b..4423518 100644 --- a/docs/ai-agent-integration.md +++ b/docs/ai-agent-integration.md @@ -37,14 +37,31 @@ The patch is partial: controls not listed keep their current state. The runtime still performs profile normalization, safety checks, replay logging, and one adapter state-sync message. -For safer model-facing control, use named action maps. Run the canonical -hold-and-release example from the repository root: +For safer model-facing control, use named action maps. The runnable +[semantic binding example](../examples/basic-dry-run/ai-agent-integration.ts) +defines stable action IDs (`moveLeft`, `moveRight`, and `confirm`) separately +from caller-owned source identifiers. It supplies two binding sets: one for +WASD-style identifiers and one for arrow/Enter identifiers. Both route to the +same `createActionMap`, which defines the OpenController commands for those +semantic actions. + +The consuming application owns input events and passes a source identifier to +its selected binding set. The example validates the full set first, rejecting +unknown semantic actions and duplicate source identifiers before creating or +commanding a controller. It then resolves the source identifier and dispatches +the mapped semantic action. OpenController does not capture keyboard or +physical gamepad input, change OS or Steam remapping, or provide a settings UI +or persistent user profile. The consuming application still owns its input +capture, conflict policy, persistence, and recovery experience. + +Run both binding sets from the repository root: ```bash bun run examples/basic-dry-run/ai-agent-integration.ts ``` -The [runnable source](../examples/basic-dry-run/ai-agent-integration.ts) holds -and releases a button through named actions. CI executes that same source and -checks both states and replay intents with `bun run check:dry-run-examples`. -Action maps keep model output constrained to named, reviewed behaviors. +The dry-run smoke check covers both valid sets and invalid unknown/conflicting +bindings with `bun run check:dry-run-examples`. It verifies OpenController +commands and dry-run state only; it does not establish host, browser, Steam, +physical device, or game input behavior. Action maps keep model output +constrained to named, reviewed behaviors. diff --git a/examples/basic-dry-run/ai-agent-integration.ts b/examples/basic-dry-run/ai-agent-integration.ts index 46c9fa1..63be74c 100644 --- a/examples/basic-dry-run/ai-agent-integration.ts +++ b/examples/basic-dry-run/ai-agent-integration.ts @@ -2,28 +2,90 @@ import { randomUUID } from "node:crypto"; import type { ControllerState } from "@opencontroller/core"; import { createActionMap, createController } from "@opencontroller/core"; -export async function runAgentActionMapExample( - replayDir = `replays/ai-agent-action-map-${randomUUID()}`, -): Promise<{ heldState: ControllerState; releasedState: ControllerState }> { +const semanticActions = { + moveLeft: [{ type: "setStick", stick: "LEFT", x: -1, y: 0 }], + moveRight: [{ type: "setStick", stick: "LEFT", x: 1, y: 0 }], + confirm: [{ type: "setButton", button: "A", pressed: true }], +} as const; + +export type SemanticActionId = keyof typeof semanticActions; + +export type InputBinding = { + /** An input identifier supplied by the consuming application. */ + sourceId: string; + /** A stable semantic action understood by this application. */ + actionId: string; +}; + +export const wasdBindings: readonly InputBinding[] = [ + { sourceId: "KeyA", actionId: "moveLeft" }, + { sourceId: "KeyD", actionId: "moveRight" }, + { sourceId: "Space", actionId: "confirm" }, +]; + +export const arrowBindings: readonly InputBinding[] = [ + { sourceId: "ArrowLeft", actionId: "moveLeft" }, + { sourceId: "ArrowRight", actionId: "moveRight" }, + { sourceId: "Enter", actionId: "confirm" }, +]; + +export function validateInputBindings(bindings: readonly InputBinding[]): void { + const validActions = new Set(Object.keys(semanticActions)); + const seenSources = new Set(); + + for (const binding of bindings) { + if (seenSources.has(binding.sourceId)) { + throw new Error(`Conflicting source identifier: ${binding.sourceId}`); + } + seenSources.add(binding.sourceId); + + if (!validActions.has(binding.actionId)) { + throw new Error(`Unknown semantic action: ${binding.actionId}`); + } + } +} + +export async function runSemanticBindingExample( + bindings: readonly InputBinding[], + sourceId: string, + replayDir = `replays/ai-agent-bindings-${randomUUID()}`, +): Promise { + // Validate the complete caller configuration before creating or commanding + // a controller. Invalid configurations cannot dispatch a partial action. + validateInputBindings(bindings); + const selectedBinding = bindings.find( + (binding) => binding.sourceId === sourceId, + ); + if (!selectedBinding) { + throw new Error(`No binding for source identifier: ${sourceId}`); + } + const controller = await createController({ profile: "xbox", adapter: "dry-run", - replay: { dir: replayDir, source: "ai-agent-action-map" }, + replay: { dir: replayDir, source: "semantic-input-bindings" }, }); - const actions = createActionMap(controller, { - holdBlock: [{ type: "setButton", button: "LB", pressed: true }], - releaseBlock: [{ type: "setButton", button: "LB", pressed: false }], - }); - - await actions.run("holdBlock"); - const heldState = controller.getState(); - await actions.run("releaseBlock"); - const releasedState = controller.getState(); + const actions = createActionMap(controller, semanticActions); + // The consuming application owns input capture and invokes this function + // with its own source identifier. OpenController only executes the action. + if (!actions.has(selectedBinding.actionId)) { + throw new Error(`Unknown semantic action: ${selectedBinding.actionId}`); + } + await actions.run(selectedBinding.actionId, { + intent: selectedBinding.actionId, + source: selectedBinding.sourceId, + }); + const state = controller.getState(); await controller.disconnect(); - return { heldState, releasedState }; + return state; } if (import.meta.main) { - console.log(await runAgentActionMapExample()); + const wasdState = await runSemanticBindingExample(wasdBindings, "KeyA"); + const arrowsState = await runSemanticBindingExample( + arrowBindings, + "ArrowLeft", + ); + console.log({ wasdState, arrowsState }); } diff --git a/examples/basic-dry-run/check.ts b/examples/basic-dry-run/check.ts index 2f98968..910c066 100644 --- a/examples/basic-dry-run/check.ts +++ b/examples/basic-dry-run/check.ts @@ -1,8 +1,13 @@ import assert from "node:assert/strict"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { access, mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { runAgentActionMapExample } from "./ai-agent-integration"; +import { + arrowBindings, + runSemanticBindingExample, + validateInputBindings, + wasdBindings, +} from "./ai-agent-integration"; import { runGettingStartedDryRun } from "./getting-started"; import { runBasicDryRunExample } from "./index"; @@ -61,19 +66,83 @@ try { ); assert.equal(gettingStartedStates.at(-1)?.state?.connected, false); - const agentDir = join(replayRoot, "agent-action-map"); - const { heldState, releasedState } = await runAgentActionMapExample(agentDir); - assert.equal(heldState.buttons.LB, true); - assert.equal(releasedState.buttons.LB, false); - const agentCommands = await readEvents( - agentDir, + const wasdDir = join(replayRoot, "wasd"); + const wasdState = await runSemanticBindingExample( + wasdBindings, + "KeyA", + wasdDir, + ); + assert.deepEqual(wasdState.sticks.left, { x: -1, y: 0 }); + const wasdCommands = await readEvents( + wasdDir, + "commands.jsonl", + ); + assert.deepEqual(wasdCommands.map((event) => event.intent).filter(Boolean), [ + "moveLeft", + ]); + assert.equal(wasdCommands[0]?.command.type, "setStick"); + + const arrowsDir = join(replayRoot, "arrows"); + const arrowsState = await runSemanticBindingExample( + arrowBindings, + "ArrowLeft", + arrowsDir, + ); + assert.deepEqual(arrowsState.sticks.left, { x: -1, y: 0 }); + const arrowCommands = await readEvents( + arrowsDir, + "commands.jsonl", + ); + assert.deepEqual(arrowCommands.map((event) => event.intent).filter(Boolean), [ + "moveLeft", + ]); + assert.equal(arrowCommands[0]?.command.type, "setStick"); + + const confirmDir = join(replayRoot, "wasd-confirm"); + const confirmState = await runSemanticBindingExample( + wasdBindings, + "Space", + confirmDir, + ); + assert.equal(confirmState.buttons.A, true); + const confirmCommands = await readEvents( + confirmDir, "commands.jsonl", ); assert.deepEqual( - agentCommands.slice(0, 2).map((event) => event.intent), - ["holdBlock", "releaseBlock"], + confirmCommands.map((event) => event.intent).filter(Boolean), + ["confirm"], + ); + assert.equal(confirmCommands[0]?.command.button, "A"); + + const unknownBindingsDir = join(replayRoot, "unknown-action"); + await assert.rejects( + runSemanticBindingExample( + [{ sourceId: "KeyA", actionId: "teleport" }], + "KeyA", + unknownBindingsDir, + ), + /Unknown semantic action: teleport/, + ); + await assert.rejects(access(unknownBindingsDir)); + + const conflictBindingsDir = join(replayRoot, "conflicting-source"); + await assert.rejects( + runSemanticBindingExample( + [ + { sourceId: "KeyA", actionId: "moveLeft" }, + { sourceId: "KeyA", actionId: "confirm" }, + ], + "KeyA", + conflictBindingsDir, + ), + /Conflicting source identifier: KeyA/, + ); + await assert.rejects(access(conflictBindingsDir)); + assert.throws( + () => validateInputBindings([{ sourceId: "KeyA", actionId: "nope" }]), + /Unknown semantic action/, ); - assert.equal(agentCommands[0]?.command.button, "LB"); const basicDir = join(replayRoot, "basic"); const basicState = await runBasicDryRunExample(basicDir); @@ -91,7 +160,9 @@ try { ), ); - console.log("Dry-run documentation examples updated state and replay logs."); + console.log( + "Dry-run documentation examples updated state, replay logs, and rejected invalid bindings before dispatch.", + ); } finally { await rm(replayRoot, { recursive: true, force: true }); }