From 38c40392aa79974e8cbf1e597f93166d5b4f77f1 Mon Sep 17 00:00:00 2001 From: Bob Lee Date: Mon, 12 Oct 2026 18:56:33 +0800 Subject: [PATCH 1/3] feat(cua): add explicit app sharing and scoped cross-platform control --- docs/architecture/computer-use-control.md | 242 +++- docs/interactive-capabilities/README.md | 16 +- .../capabilities.json | 196 ++- .../capabilities/feature.computer-use.md | 6 + .../technical/product-control-open-audit.json | 62 +- .../technical/tauri-command-map.json | 152 +- .../rules/source/required-rules.mjs | 10 +- scripts/fixtures/computer-use-capture.m | 2 +- .../fixtures/computer-use-input-controls.m | 6 +- scripts/fixtures/computer-use-pointer.m | 2 +- scripts/test-computer-use-context.mjs | 10 + scripts/test-linux-computer-use-native.mjs | 25 +- scripts/test-macos-control-roundtrip.mjs | 38 +- scripts/test-windows-computer-use-native.mjs | 91 ++ src/apps/desktop/Cargo.toml | 2 +- src/apps/desktop/src/api/computer_use_api.rs | 159 ++- src/apps/desktop/src/computer_use/AGENTS.md | 41 + .../src/computer_use/control_session.rs | 440 +++--- .../computer_use/control_session/admission.rs | 673 +++++++++ .../computer_use/control_session/sharing.rs | 204 +++ .../desktop_host/ax_orchestration.rs | 59 +- .../desktop_host/background_guard.rs | 234 ++++ .../src/computer_use/desktop_host/macos.rs | 25 +- .../src/computer_use/desktop_host/mod.rs | 575 ++------ .../native_control_roundtrip_tests.rs | 177 ++- .../computer_use/desktop_host/observation.rs | 174 +++ .../desktop_host/pointer_input.rs | 175 ++- .../computer_use/desktop_host/screenshot.rs | 394 ++---- .../desktop_host/shared_preview.rs | 314 +++++ .../src/computer_use/desktop_host/sharing.rs | 219 +++ .../src/computer_use/desktop_host/state.rs | 125 ++ src/apps/desktop/src/computer_use/grants.rs | 673 +++++++++ .../src/computer_use/integration_e2e.rs | 1 + .../desktop/src/computer_use/linux_control.rs | 14 + .../src/computer_use/linux_control_ax.rs | 39 +- .../desktop/src/computer_use/macos_ax_dump.rs | 56 +- .../desktop/src/computer_use/macos_ax_ui.rs | 4 +- .../src/computer_use/macos_bg_input.rs | 2 +- .../desktop/src/computer_use/macos_capture.m | 17 +- .../desktop/src/computer_use/macos_capture.rs | 42 +- .../src/computer_use/macos_list_apps.rs | 37 +- src/apps/desktop/src/computer_use/mod.rs | 9 +- .../src/computer_use/observation_cache.rs | 159 +++ .../desktop/src/computer_use/permissions.rs | 203 +++ .../src/computer_use/process_identity.rs | 225 +++ .../desktop/src/computer_use/share_grants.rs | 207 +++ .../desktop/src/computer_use/windows_ax_ui.rs | 27 +- .../src/computer_use/windows_capture.rs | 558 +------- .../src/computer_use/windows_list_apps.rs | 30 +- .../src/computer_use/windows_wgc_capture.rs | 44 +- src/apps/desktop/src/lib.rs | 9 + .../prompts/agents/computer_use_mode.md | 5 +- .../src/agentic/tools/computer_use_host.rs | 8 + .../implementations/computer_use_actions.rs | 3 + .../implementations/computer_use_program.rs | 94 +- .../implementations/computer_use_tool.rs | 238 +++- .../generated/product-control-catalog.json | 196 ++- .../generated/remote-surface-registry.json | 112 +- .../src/remote_surface/capabilities.rs | 4 + .../src/remote_surface/table.rs | 9 + src/crates/execution/tool-contracts/AGENTS.md | 2 +- .../tool-contracts/src/computer_use.rs | 171 ++- .../src/computer_use_control.rs | 184 ++- .../src/computer_use_geometry.rs | 352 +++++ .../src/computer_use_permissions.rs | 109 ++ .../execution/tool-contracts/src/lib.rs | 4 + src/crates/execution/tool-execution/AGENTS.md | 12 +- .../tool-execution/src/computer_use.rs | 445 ++---- .../interactive-capabilities/catalog.json | 62 +- .../ComputerUseControlCard.appearance.ts | 2 +- .../components/ComputerUseControlCard.scss | 35 + .../ComputerUseControlCard.test.tsx | 12 + .../app/components/ComputerUseControlCard.tsx | 52 +- .../app/components/ComputerUsePermissions.tsx | 90 ++ .../components/ComputerUseSharingControls.tsx | 84 ++ .../components/computer-use/PreviewCursor.tsx | 49 + .../computer-use/SharedPreview.test.tsx | 114 ++ .../components/computer-use/SharedPreview.tsx | 159 +++ .../computer-use/VirtualCursor.test.ts | 47 + .../components/computer-use/VirtualCursor.ts | 240 ++++ .../computer-use/cursor-reference.json | 1239 +++++++++++++++++ .../computer-use/previewGeometry.test.ts | 20 + .../computer-use/previewGeometry.ts | 26 + .../generated/interactive-capabilities.json | 196 ++- .../pages/tools/DeviceControlSettingsPage.tsx | 29 +- .../api/adapters/peer-device-adapter.test.ts | 20 + .../api/adapters/peer-device-adapter.ts | 11 + .../api/generated/productControl.ts | 2 +- .../api/generated/remoteSurface.ts | 4 +- .../api/service-api/ComputerUseControlAPI.ts | 119 ++ .../peer-device/PeerConnectionManager.ts | 6 + src/web-ui/src/locales/en-US/common.json | 63 + src/web-ui/src/locales/zh-CN/common.json | 63 + src/web-ui/src/locales/zh-TW/common.json | 63 + 94 files changed, 9581 insertions(+), 2378 deletions(-) create mode 100644 scripts/test-windows-computer-use-native.mjs create mode 100644 src/apps/desktop/src/computer_use/control_session/admission.rs create mode 100644 src/apps/desktop/src/computer_use/control_session/sharing.rs create mode 100644 src/apps/desktop/src/computer_use/desktop_host/background_guard.rs create mode 100644 src/apps/desktop/src/computer_use/desktop_host/observation.rs create mode 100644 src/apps/desktop/src/computer_use/desktop_host/shared_preview.rs create mode 100644 src/apps/desktop/src/computer_use/desktop_host/sharing.rs create mode 100644 src/apps/desktop/src/computer_use/desktop_host/state.rs create mode 100644 src/apps/desktop/src/computer_use/grants.rs create mode 100644 src/apps/desktop/src/computer_use/observation_cache.rs create mode 100644 src/apps/desktop/src/computer_use/permissions.rs create mode 100644 src/apps/desktop/src/computer_use/process_identity.rs create mode 100644 src/apps/desktop/src/computer_use/share_grants.rs create mode 100644 src/crates/execution/tool-contracts/src/computer_use_geometry.rs create mode 100644 src/crates/execution/tool-contracts/src/computer_use_permissions.rs create mode 100644 src/web-ui/src/app/components/ComputerUsePermissions.tsx create mode 100644 src/web-ui/src/app/components/ComputerUseSharingControls.tsx create mode 100644 src/web-ui/src/app/components/computer-use/PreviewCursor.tsx create mode 100644 src/web-ui/src/app/components/computer-use/SharedPreview.test.tsx create mode 100644 src/web-ui/src/app/components/computer-use/SharedPreview.tsx create mode 100644 src/web-ui/src/app/components/computer-use/VirtualCursor.test.ts create mode 100644 src/web-ui/src/app/components/computer-use/VirtualCursor.ts create mode 100644 src/web-ui/src/app/components/computer-use/cursor-reference.json create mode 100644 src/web-ui/src/app/components/computer-use/previewGeometry.test.ts create mode 100644 src/web-ui/src/app/components/computer-use/previewGeometry.ts diff --git a/docs/architecture/computer-use-control.md b/docs/architecture/computer-use-control.md index b456cbad3a..6da8fab774 100644 --- a/docs/architecture/computer-use-control.md +++ b/docs/architecture/computer-use-control.md @@ -1,10 +1,12 @@ # Computer Use control-session SPEC -Status: implemented control-session and input-program architecture, with platform -validation at different levels. The covered macOS semantic-input pipeline and -Linux GTK/AT-SPI text selection have native evidence. The directed macOS background pointer route has passed a complete five-action -production Tool/native program on dedicated controls. Windows native execution, GNOME Wayland -portal execution and remote end-to-end control remain acceptance gaps. Compilation is not native acceptance. +Status: the October 2026 migration implements permissions, user-selected App +sharing and control grants, coordinate mapping, virtual cursor feedback and +platform-scoped background input. Validation includes native macOS Tool/input/ +capture checks, Linux GTK/AT-SPI execution and Windows native-module +cross-compilation. Windows native execution, GNOME Wayland portal execution and +remote end-to-end control remain acceptance gaps. Compilation is not native +acceptance, and background support is limited to each provider's declared routes. ## Product contract @@ -49,6 +51,65 @@ Portable DTOs and `AppInputAction` live in `tool-contracts`; the host port lives Core. Desktop owns native resources and platform adapters. Assembly must not own OS APIs, native focus workarounds or concrete capture implementations. +The shared action catalog and structural target validation live in +`tool-contracts`. Desktop's `control_session/admission.rs` is a deterministic +owner/mode/lease state machine; its facade alone performs native cleanup and +carries tokens across workers. `desktop_host/observation.rs` coordinates native +reads, while `observation_cache.rs` stores bounded references without owning input +permission. `computer_use_geometry` adapts the independent `computer-use-rust` +project's image/content/surface mapping for all three platforms. The reference +project's macOS-only capture/input and standalone MCP/session shell are not a +replacement for the existing Windows, Linux or product runtime providers. + +### Reference implementation migration + +The five requested areas are adapted from `computer-use-rust` into the existing +product owners. The reference repository is development evidence, never a +runtime, build, test or filesystem dependency of OpenBitFun. + +| Area | Reference source | OpenBitFun owner and adaptation | +|---|---|---| +| Permission query/request | `src/native/permissions.rs`, native permission probes | `computer_use_permissions` contracts, Desktop `permissions.rs`, `ComputerUsePermissions.tsx`; four separate read-only probes and explicit request receipts; Windows capability/UIPI facts and Linux AT-SPI/session consent replace macOS-specific assumptions | +| App sharing and control grants | `src/sharing.rs`, `src/grants.rs`, sharing controls in `src/gui.rs` | Admission state machine plus Desktop `sharing.rs`, `shared_preview.rs`, `process_identity.rs`, `grants.rs`, `share_grants.rs`; user-selected native window, observe first, separate revocable grant, executable-bound remembered preference, manual preview gestures | +| Coordinate conversion | `CoordinateMap` in `src/sharing.rs` | Portable `CaptureGeometry` and Web `previewGeometry.ts`; image/content/global/preview round trips, aspect fit, negative origins, independent X/Y scales, strict padding rejection and immutable screenshot identity | +| Virtual mouse | `src/cursor.rs`, GUI cursor rendering | Web `VirtualCursor.ts` ports the deterministic physical model, rendered by `PreviewCursor.tsx`; 63 Rust-generated reference samples cover movement, retargeting, immediate placement, resize and visibility. Existing passive native overlays remain platform adapters | +| Background control | Native directed/semantic input and desktop-state comparison | Existing AX/UIA/AT-SPI and directed-input providers plus `background_guard.rs`; focus/pointer/window-order evidence, explicit unsupported routes, revocation on observed interference, no restore or replay. The reference's implicit Command modifier is not copied because it changes the requested click | + +### Permissions and user sharing + +Opening settings, checking status or reading AX never requests OS permission. +The user selects each request explicitly. Receipts distinguish a request API +invocation from the current grant and from observing a visible OS dialog. Input +Monitoring is optional. Windows does not invent a macOS-style grant; protected +desktops, capture policy and target integrity still apply. Linux's portal grant +belongs to one selected session; AT-SPI availability does not grant seat input. + +`start_control` and implicit tool admission return `awaiting_share` / a clear +`SHARE_REQUIRED` result until the user selects a target in the executing host's +control card. Discovery returns opaque choices bound to the process lifetime, +not authority derived from a title or a client-supplied PID. Starting the selected +share validates the live process and exact window, obtains real pixels or the +supported AT-SPI tree, then exposes observation. Input requires a separate user +grant for the selected background or foreground mode. A tool cannot select a +different window, grant itself input or widen the mode. + +Remembering a grant is optional and never starts a share or selects a window. +The preference binds the canonical executable path, full executable SHA-256, +bundle identity when available and input mode; process start identity and file +metadata are checked around hashing. An updated executable requires a new +grant. The versioned bounded file preserves unknown fields and unreadable user +data; writes are atomic, privately permissioned and protected against concurrent +changes. Forgetting an active matching preference revokes input before disk IO. +Portal streams have no verified executable identity and remain session-only. + +Manual preview input joins the same exclusive action lease as the Agent. Pointer +gestures retain their screenshot until release; resizing or revocation cancels +them. Click/right click/double click, drag, scroll, text and key input use the +native provider. The preview cannot pass Agent-owned AX indices. Input errors +revoke the grant and are never automatically replayed. Linux background sharing +shows the AT-SPI tree and exposes semantic Agent actions; it does not fabricate +pixels or coordinate input for an unrelated portal surface. + The app-input pipeline has one execution owner: ```text @@ -98,7 +159,11 @@ Receipts distinguish attempted steps, submitted steps, the failing step and observation failure. A native error may follow partial submission. Neither that error, a missing screenshot nor an unchanged AX digest authorizes replay. Submission means accepted by the delivery path, not verified application success; -completion must be established by the resulting application state. +completion must be established by the resulting application state. The evidence +helper fingerprints every encoded byte, but hashes cannot measure changed image +area or establish completion. Its legacy loop API makes no loop claim from +repeated action names, mouse usage or identical frames. Automatic retry policy +is restricted to transient read failures; mutation errors never authorize replay. Main agents can use ComputerUse directly when the host and tool configuration allow it. The ComputerUse subagent is optional. Delegated execution keeps @@ -109,24 +174,33 @@ authorization boundary for native chrome, dialogs or unattached windows. ## Session lifecycle and revocation -The resource lifecycle is `idle → starting → active → stopped|failed`, with an -active observing/background/foreground mode. One Runtime session owns the GUI +The user-facing resource lifecycle is +`idle → awaiting_share → active(observe) → active(authorized mode) → stopped|failed`, +with internal starting/cleanup transitions. One Runtime session owns the GUI resource. Competing ownership returns `control_busy`; it does not create a second permission owner or Agent loop. -An explicit start selects mode. Target-scoped observation can establish a -background session for compatible callers, but never foreground authority. -Discovery alone does not create capture. The first app observation binds an -exact window; Linux portal capture binds the consented surface separately from -semantic application identity. Stop/status are idempotent. A stopped generation -requires an explicit restart, not a screenshot-driven resurrection. +An explicit tool start requests a mode; the user's share selection and grant +establish authority. Discovery alone does not create capture. Legacy wire fields +remain readable but do not bypass the new host's user-selection gate. Linux +portal capture binds the consented surface separately from semantic application +identity. Stop/status are idempotent. A stopped generation requires an explicit +restart, not a screenshot-driven resurrection. Revoking control preserves the +shared observation and increments the authorization revision immediately; stale +workers lose input authority and held native inputs are released. Stop revokes admission and advances generation before cancelling native work, releasing owned pressed keys/buttons, stopping capture and hiding feedback. Native Closed/Stop Sharing, cancellation, permission revocation and target loss use the same owner path. Generation/sequence tokens cross blocking/native worker -boundaries; delayed work cannot borrow a later action's admission. Startup and +boundaries; observation leases never carry input authority. Unknown actions fail +before admission. An unrelated executor cannot observe or rebind an in-flight +action without carrying its original token. Target revisions distinguish every bind, including A → B → A +in one generation, and native stop callbacks carry the stream's original revision. +Delayed work cannot borrow a later action's admission. Startup and cleanup epochs prevent late callbacks from resurrecting or clearing newer state. +Native cleanup is serialized and rechecks its epoch after acquiring the cleanup +lock; a delayed cleanup job cannot tear down a restarted session. Already submitted external effects cannot be rolled back. ## Observation, coordinates and model transport @@ -141,13 +215,29 @@ remain readable, without advertising their obsolete navigation workflow. Image coordinates are meaningful only with the corresponding `screenshot_id`, `image_content_rect` and `image_global_bounds`. Apply scaling once. Reject unknown -IDs, target changes and invalid geometry. macOS can translate a retained image's +IDs, padding, out-of-range points, target changes and invalid geometry. No +clamping can redirect a bad coordinate onto an edge control. macOS and Windows +query live native bounds and can translate a retained image's coordinates for pure window movement when the image projection and size remain valid; resizing or changed projection requires fresh observation. Retina, mixed DPI, negative origins and cross-monitor movement are acceptance cases. +Screenshot geometry is scoped by control generation and target revision and +retained in a bounded history of 16 images; pixels are not retained in that cache. +Stop, restart and rebind invalidate it. Preview frames carry the same revision, +and controllers reject stale responses even when the target string matches. + Observed node indices refer to retained native identities from that observation. -Do not rebuild a tree and reinterpret an old index. Bound-window input validates +`observation_id` identifies the node address space independently of the content +`digest`. Every application snapshot creates a new identity, invalidates derived +interactive/visual views, and stamps all returned views with that identity. +New callers echo it on node/index input; hosts reject unknown/stale explicit IDs +before dispatch. Legacy callers can omit it, and older payloads default it to +absent. Native AX/UIA/AT-SPI caches independently validate generation and target +revision and release retained observations on Stop. macOS node lookups retain +their own guard before unlocking the cache, so concurrent cleanup cannot free an +in-flight reference. Do not rebuild a tree and +reinterpret an old index. Bound-window input validates the element's actual owning window, including same-PID foreign-window rejection. Application-level menu exceptions must be explicit, not a general unknown-window escape hatch. @@ -176,12 +266,26 @@ application work. Windows drains WGC frames to the newest available frame; it does not claim an equivalent input-clock barrier. Windows input preparation validates and reuses the live WGC session without -consuming or encoding a frame. Only an actual observation requests pixels. +consuming or encoding a frame. Only an actual observation requests pixels. WGC BGRA converts directly to RGBA +and the host encodes JPEG; there is no intermediate PNG, GDI/PrintWindow path or +desktop capture fallback. Preview encoding supplies its known dimensions rather +than decoding its own JPEG. Unchanged legitimate dark frames are preserved. Explicit action waits subscribe to session changes; Stop, revocation or target replacement interrupts them immediately, including waits between batch steps. ## Native input rules +Background actions capture desktop witnesses before and after submission. +macOS checks foreground identity plus its activation epoch, the physical pointer +and relative order of existing windows; Windows checks foreground HWND, pointer +and window order. Linux can inspect active AT-SPI top-level windows but cannot +observe compositor cursor/stacking, which remain unknown in the receipt. A +missing preflight witness sends no input. Changed or unavailable post-action +evidence revokes the session without focus restoration or retry. Concurrent +human/system input can also change these witnesses, so an interference error +does not blame the application or claim that already submitted effects were +undone. Boundary samples cannot prove all transient or later application behavior. + ### macOS Retain the selected content window across observations. A sharing-indicator @@ -255,8 +359,7 @@ control lifecycle, not to a model-generated sequence of clicks. The production Tool/native fixture passed a five-action background program: click an ordinary non-AX canvas, type text, invoke the real menu with Cmd+Shift+K, -scroll by 8, then type an emoji. It completed in 466 ms including its one final -observation, excluding model reasoning. The covered target received one content +scroll by 8, then type an emoji, followed by one final observation. The covered target received one content click with unchanged modifiers, text reached the canvas while the separate text field remained unchanged, and the menu/scroll actions took effect. Exactly one outside-frame focus pair prepared the entire program. The observer retained @@ -264,11 +367,11 @@ active/key/frontmost and stacking state, received no input, and the human cursor matched its baseline. Stop deactivated the internally prepared target. Semantic Unicode field input also passed in the fixture before the program. -The standalone raw native runner also passed its one native test (4.62 seconds -including capture and waits), with strict cursor, lifecycle and event-scope -checks. Both runners use ordinary inactive controls and preserve the observer -baseline; neither substitutes a permissive first-click implementation. Unit -coverage includes 143 passing Desktop tests. +The standalone raw native runner has prior passing evidence with strict cursor, +lifecycle and event-scope checks; it was not rerun during this refactor. Both +runners use ordinary inactive controls and preserve the observer baseline; +neither substitutes a permissive first-click implementation. Current validation +counts and platform gaps are recorded in the verification section below. Complete acceptance must verify no off-frame content hit, exactly one requested content click, correct text destination, unchanged human foreground/key/stacking @@ -283,7 +386,7 @@ capture errors. No unlock, wake or automatic activation is a recovery action. ### Windows Observation and semantic actions share a retained MTA UIA worker/cache. Validate -PID, HWND, generation, enabled state and geometry before supported Invoke, +PID, HWND, generation, target revision, enabled state and geometry before supported Invoke, Toggle, SelectionItem or Scroll operations. Release retained COM references with the cache; do not rebuild indices during input. @@ -312,7 +415,7 @@ unconsented X11/global path. Portal authorization has a bounded deadline and explicit local-authorization failure; late responses cannot revive old sessions. Native-thread Stop schedules release/closure through the retained runtime. -AT-SPI caches retain bus/object identity and control generation. Default actions +AT-SPI caches retain bus/object identity, control generation and target revision. Default actions execute once without coordinate fallback. Text insertion uses character caret and selection offsets with UTF-8 byte length. One selected range is replaced; unselected text remains intact. Invalid offsets or multiple selections fail @@ -332,6 +435,14 @@ preview feedback conservatively when native visibility cannot be established. Ubuntu preview does not invent a Wayland global overlay. Reduced-motion retains static position feedback without movement transitions. +The physical preview cursor is a deterministic 240 Hz simulation sampled by +`requestAnimationFrame`, with retargeting, directional rotation, speed stretch, +arrival deformation, opacity/blur and resize behavior from the reference. Motion +never acknowledges, schedules or delays native input. Reduced-motion uses +immediate feedback. Rust-generated golden traces test the mathematical port, +not native OS delivery. The click ring expires independently of the persistent +pointer. + The product surface shows authoritative target, mode, status and Stop while the chat card is collapsed. Preview sampling is bounded and stops when unobserved; React component lifetime does not own native capture lifetime. Generation/target @@ -342,8 +453,8 @@ checks discard stale preview responses and device-switch results. | Scenario | Required behavior | |---|---| | Remote workspace | Existing SSH/Docker workspace refusal; never operate the controller's local GUI as fallback | -| Remote control | Use the executing host's registered provider; local-only consent must report `local_authorization_required`; task cancellation remains reachable | -| Peer Device | Follow operation-registry routing and negotiated provider capability; controller permission is not peer permission; CLI without GUI rejects explicitly | +| Remote control | Mobile/IM have no App picker or grant UI and receive an explicit unsupported/`SHARE_REQUIRED` state; no hidden blocking prompt; task cancellation remains reachable | +| Peer Device | Share/permission/preview commands proxy to the peer only after `computer_use_sharing_v1` negotiation; controller permission is not peer permission; old peers and CLI without GUI reject explicitly | | Detached Dispatch | Headless execution has no implied GUI capability or submitter UI dependency; require an explicitly negotiated target provider | Missing new capabilities on old peers mean unknown/unsupported. Portable fields @@ -354,19 +465,54 @@ Local fixtures and unsupported-host unit tests are not remote end-to-end evidenc ## Verification and acceptance -The final integration baseline is upstream `178e9b555`. The macOS Desktop debug -build, 65 Core tests, 143 Desktop tests, ten control-card/API frontend tests, -frontend TypeScript check and Windows production-source cross-compilation passed -on this baseline. The broader `check:web` gate stops at four existing typography -violations in unchanged mobile `chat.scss` and `host-queue.scss`; it is not a pass. - -Focused validation includes 65 Core Computer Use tests (including the 45 tool -schema/execution tests), 143 Desktop tests, eight prompt catalog tests and 19 -Computer Use contract tests. Opt-in native fixtures ignored by unit-test commands -do not count as native passes. The full macOS Tool program above was executed -separately and passed. Its fixture records hardware idle time and activation -history, retains strict failure assertions, and never restores observer focus to -manufacture a pass. Native Windows, Wayland and remote execution remain untested. +The 2026-10-12 refactor started from `f5b6d3ab2`. Focused checks passed: 72 Core +Computer Use tests, 160 Desktop tests, 33 contract tests, five runtime-evidence +tests, eight prompt-catalog tests, 63 isolated production-source context tests, +19 operation-registry tests, nine Desktop remote-workspace policy tests and +82 frontend tests. The frontend command covers sharing/permissions, preview +gestures, stale responses, peer capability gates, Appearance registration and +cursor/geometry parity. The cursor oracle contains 63 samples generated by the +unmodified reference Rust implementation. Counts from separate suites overlap +and must not be added into a unique-test total. Nine Desktop native tests and one +browser context test remain opt-in/ignored in those unit commands; they are not +counted as native passes. The macOS Desktop debug build, `check:web`, i18n audit, +all-surface color audit, repository hygiene, generated capability check and +core-boundary checks also passed. + +Dedicated macOS fixtures separately verified retained AX references after cache +cleanup, stale observation-ID rejection before input, the complete Tool/native +roundtrip, strict background input, capture and passive feedback. The final +strict background run used the production App selection, observe-only capture +and explicit grant flow, then exercised native manual-preview input and rejected +its stale request after grant revocation. Observer foreground/key/stacking, its +input-event history and the system cursor assertions all passed, including Stop. +Earlier attempts include a hardware-idle precondition timeout and two failures +of the final unchanged-cursor assertion; they remain failure/inconclusive evidence. +An unchanged rerun during a user-provided quiet interval passed. Neither input +replay, focus restoration nor relaxed assertions were used to obtain that pass. +The separate Unicode/OCR roundtrip initially stopped at its background-start +precondition because macOS selected the disposable target as the foreground app. +Its runner now starts an independent observer after the host initializes; the +rerun passed exact input counts, Unicode text, stale-reference rejection, OCR, +generation/binding continuity and foreground preservation. + +The Windows production native modules pass cross-compilation with +`test-windows-computer-use-native.mjs --check --target x86_64-pc-windows-msvc`. +A full Desktop cross-build on macOS is blocked by the unavailable Windows C SDK +headers required by `ring`; module type-checking does not cover that build or +native execution. Linux production modules were compiled and executed in an +ARM64 Debian container with real D-Bus, AT-SPI, GTK and Xvfb: 65 focused tests +passed, with two opt-in native tests ignored; the separate GTK fixture then +passed button activation and UTF-8 EditableText insertion while preserving +existing text. This validates the production AT-SPI provider under Xvfb, not a +full Linux Desktop build or compositor integration. No GNOME Wayland compositor, +portal chooser or live PipeWire stream was available for the portal fixture. + +The sharing card was rendered in Chromium with the real design-system styles at +1000×800 and 390×700; the narrow layout keeps Stop visible while permissions are +expanded. That visual harness mocks transport. It is not an end-to-end Tauri +permission-dialog test. Full OS denied-to-granted/restart flows and system Stop +Sharing UI interaction remain manual acceptance work. Run focused checks from the [Desktop Computer Use guide](../../src/apps/desktop/src/computer_use/AGENTS.md) and [Core guide](../../src/crates/assembly/core/AGENTS.md). A build-only result, @@ -375,11 +521,13 @@ background-input run. | Area | Recorded evidence | Remaining scope | |---|---|---| -| macOS strict production Tool pipeline | Five-action non-AX canvas/text/menu/scroll/emoji program in 466 ms including final observation; one content click and one outside-frame focus pair; observer foreground/key/stacking and human cursor unchanged; no observer input; Stop deactivated target | Wider applications, cross-Space behavior and visual system Sharing UI acceptance | -| macOS capture/feedback | Dedicated native fixtures exercised obscured target pixels, binding retention, movement/resize, hidden/minimized errors, closure, pointer lifetime and locked-session rejection | System Stop Sharing interaction and broader platform/application matrix | -| Linux text semantics and scope | 45 focused tests; a separate real GTK/AT-SPI fixture verified selected text replacement while unselected text survived | GNOME Wayland portal consent, real PipeWire/seat input and sharing UI | -| Windows text semantics | Production-source cross-compilation, including Edit/RichEdit selection replacement | Interactive Windows native delivery, WGC border/stop and foreground preservation | -| Remote scenarios | Scope/unsupported-provider contracts | No remote-workspace, Remote Connect, Peer Device or Detached Dispatch end-to-end GUI validation | +| macOS strict production Tool pipeline | User-selected share and separate grant; five-action non-AX canvas/text/menu/scroll/emoji program with one final observation; exactly one content click and one outside-frame focus pair; native preview input and revocation; observer foreground/key/stacking and human cursor unchanged; no observer input; Stop deactivated target | Wider applications, cross-Space behavior and visual system Sharing UI acceptance | +| macOS capture/feedback | Dedicated native fixtures exercised obscured target pixels, binding retention, movement/resize, hidden/minimized errors, closure and pointer lifetime; earlier locked-session rejection evidence was not rerun | System Stop Sharing interaction and broader platform/application matrix | +| Linux text semantics and scope | 65 production-module tests plus a real GTK/AT-SPI activation and UTF-8 insertion fixture under D-Bus/Xvfb; portable admission, permission, grant and geometry checks | Full Linux Desktop build, GNOME Wayland portal consent, real PipeWire/seat input and sharing UI; arbitrary background coordinates remain explicitly unsupported | +| Windows native modules | Production-source cross-compilation of control admission, UIA/MSAA, WGC, pointer feedback, permissions, process identity, grants and background witnesses | Full Windows Desktop build, native delivery, WGC border/stop and foreground preservation | +| Permissions and remembered grants | Read-only probe/explicit request tests, process identity checks, private atomic persistence, legacy/future-field retention, grant revocation and no automatic sharing | Real OS permission denial/approval/restart matrix and broader executable upgrade scenarios | +| Preview and virtual cursor | Rust golden parity, padded/negative-origin/resize geometry, one-input dispatch, pointer ownership, cancellation and two responsive visual renders; native preview Escape delivered once | Full desktop UI-to-native pointer/drag matrix across all three OSes | +| Remote scenarios | Explicit remote-workspace/headless-provider refusal tests; frontend peer-scope, old-peer capability denial and stale-preview tests | No remote-workspace, Remote Connect, Peer Device or Detached Dispatch end-to-end GUI validation | The macOS strict fixture builds before opening windows, initializes the host, uses separate application bundle identities, and waits for a bounded hardware diff --git a/docs/interactive-capabilities/README.md b/docs/interactive-capabilities/README.md index dbf962b8b6..181a75590c 100644 --- a/docs/interactive-capabilities/README.md +++ b/docs/interactive-capabilities/README.md @@ -1,9 +1,9 @@ # OpenBitFun 功能与设置目录 / OpenBitFun Features & Settings -OpenBitFun Playbook 当前包含 **22 个功能**和 **21 个设置页**，共 **43 个**用户可理解的条目、**325 项**有源码证据的子能力。每个条目有独立 Markdown，并直接服务于说明书网站、OpenBitFun 全局搜索和 `OpenBitFunControl` 智能体工具。 +OpenBitFun Playbook 当前包含 **22 个功能**和 **21 个设置页**，共 **43 个**用户可理解的条目、**328 项**有源码证据的子能力。每个条目有独立 Markdown，并直接服务于说明书网站、OpenBitFun 全局搜索和 `OpenBitFunControl` 智能体工具。 -OpenBitFun Playbook currently contains **22 features**, **21 settings pages**, and **325** source-backed sub-capabilities across **43** user-facing entries. Every entry has its own Markdown page and directly powers the website, in-app global search, and the `OpenBitFunControl` agent tool. +OpenBitFun Playbook currently contains **22 features**, **21 settings pages**, and **328** source-backed sub-capabilities across **43** user-facing entries. Every entry has its own Markdown page and directly powers the website, in-app global search, and the `OpenBitFunControl` agent tool. ## 唯一事实源 / Single source of truth @@ -27,20 +27,20 @@ OpenBitFun Playbook currently contains **22 features**, **21 settings pages**, a - Generated per-item interaction audit: `docs/interactive-capabilities/technical/product-control-open-audit.json` - Generated low-level audit map: `docs/interactive-capabilities/technical/tauri-command-map.json` -说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **670** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 +说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **679** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 -Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **670** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. +Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **679** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. ## 控制边界 / Control boundary -- 每个子能力都明确标记为直接控制、委托给专用智能体工具、需交互打开或不支持；“打开页面”不会再被统计成“智能体已控制”。当前覆盖：直接 **49**、委托 **62**、需交互 **214**、不支持 **0**。 +- 每个子能力都明确标记为直接控制、委托给专用智能体工具、需交互打开或不支持；“打开页面”不会再被统计成“智能体已控制”。当前覆盖：直接 **49**、委托 **62**、需交互 **217**、不支持 **0**。 - 稳定行为声明为带 JSON 输入契约的 `operations` 或 `options`，并绑定原生产品控制 Provider；智能体不接触原始 Tauri Command。 -- `OpenBitFunControl list` 和 `search` 都返回带 `nextCursor` 的精简分页结果；目录可持续增长，不靠固定总量上限。完整目录和 325 项子能力都不会写入 system prompt。 +- `OpenBitFunControl list` 和 `search` 都返回带 `nextCursor` 的精简分页结果；目录可持续增长，不靠固定总量上限。完整目录和 328 项子能力都不会写入 system prompt。 - 目录发现与契约读取不依赖 React 或可见窗口。普通配置型 option 统一由 Product Assembly 的共享 ConfigService 执行器读、写并回读，因此 Desktop、CLI 与 Headless 表面走同一份实现；只有宿主原生 operation/provider option 和界面导航按表面注册适配器，缺失时必须明确返回不可用，禁止静默回退本机。只读智能体只能发现和读取目录。 -- Every documented item is classified as direct control, delegated Agent control, interactive opening, or unsupported; opening a page is never counted as direct control. Current coverage is **49 direct**, **62 delegated**, **214 interactive**, and **0 unsupported**. +- Every documented item is classified as direct control, delegated Agent control, interactive opening, or unsupported; opening a page is never counted as direct control. Current coverage is **49 direct**, **62 delegated**, **217 interactive**, and **0 unsupported**. - Stable behavior becomes a typed `operation` or `option` with a JSON input contract and a native product-control provider. Agents never receive raw Tauri commands. -- `OpenBitFunControl list` and `search` return compact pages with a `nextCursor`; the catalog can grow without a fixed total-size ceiling. Neither the full catalog nor its 325 documented items enters the system prompt. +- `OpenBitFunControl list` and `search` return compact pages with a `nextCursor`; the catalog can grow without a fixed total-size ceiling. Neither the full catalog nor its 328 documented items enters the system prompt. - Discovery and contract lookup do not depend on React or a visible window. Ordinary config-backed options are read, written, and read back by one Product Assembly ConfigService executor shared by Desktop, CLI, and headless surfaces. Only host-native operations/provider options and presentation routes install surface adapters; missing adapters return explicit unavailability without local fallback. Read-only agents may only discover and inspect entries. ## 防腐化门禁 / Anti-drift gates diff --git a/docs/interactive-capabilities/capabilities.json b/docs/interactive-capabilities/capabilities.json index 6eacd2da54..c9273cdeae 100644 --- a/docs/interactive-capabilities/capabilities.json +++ b/docs/interactive-capabilities/capabilities.json @@ -4,7 +4,7 @@ "title": "OpenBitFun Playbook", "origin": "https://playbook.openbitfun.com", "source": "src/shared/interactive-capabilities/catalog.json", - "digest": "e05dbc222e1267523832f82d6392701ab011c9d8d5ad1582dbe407904dc80d3d", + "digest": "156caca2db3ed5820e9bf6683d4df89db5bbbfac8c6c8910d31262c05450ec6e", "ownerDigest": "2269c2432e5db44706b2bce44df3c60672141a47c36141aeb1257ea7fd2507f8", "searchAcceptance": [ { @@ -138,11 +138,11 @@ "features": 22, "settings": 21, "userFacing": 43, - "documentedItems": 325, + "documentedItems": 328, "controlCoverage": { "direct": 49, "delegated": 62, - "interactive": 214, + "interactive": 217, "unsupported": 0 } }, @@ -4869,7 +4869,10 @@ "default-cdp", "agent-actions", "control-session-preview", - "stop-control-session" + "stop-control-session", + "app-sharing", + "preview-input", + "remembered-control" ], "kind": "query", "risk": "read", @@ -5322,6 +5325,144 @@ "pageId": "tools.execution" } }, + { + "id": "feature.computer-use:open:app-sharing", + "capabilityId": "feature.computer-use", + "itemIds": [ + "app-sharing" + ], + "kind": "open", + "risk": "ui", + "executionHost": "presentationSurface", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1", + "product_control_presentation_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": false + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "openReason": "visualSelection", + "presentationTarget": { + "kind": "settings", + "pageId": "tools.execution" + } + }, + { + "id": "feature.computer-use:open:preview-input", + "capabilityId": "feature.computer-use", + "itemIds": [ + "preview-input" + ], + "kind": "open", + "risk": "ui", + "executionHost": "presentationSurface", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1", + "product_control_presentation_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": false + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "openReason": "visualSelection", + "presentationTarget": { + "kind": "settings", + "pageId": "tools.execution" + } + }, + { + "id": "feature.computer-use:open:remembered-control", + "capabilityId": "feature.computer-use", + "itemIds": [ + "remembered-control" + ], + "kind": "open", + "risk": "ui", + "executionHost": "presentationSurface", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1", + "product_control_presentation_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": false + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "openReason": "visualSelection", + "presentationTarget": { + "kind": "settings", + "pageId": "tools.execution" + } + }, { "id": "feature.skills:query", "capabilityId": "feature.skills", @@ -21970,13 +22111,15 @@ "启用或停用 Computer Use", "连接或断开受支持的浏览器", "检查并打开系统权限设置", - "查看当前控制目标与预览，并随时停止控制" + "查看当前控制目标与预览，并随时停止控制", + "先选择共享 App，再授权输入；可记住授权或随时撤销" ], "highlightsEn": [ "Enable or disable Computer Use", "Connect or disconnect a supported browser", "Inspect and open OS permission settings", - "Inspect the current control target and preview, and stop control at any time" + "Inspect the current control target and preview, and stop control at any time", + "Select an App share before granting input; remember or revoke the grant" ], "items": [ { @@ -22093,6 +22236,39 @@ "Use ComputerUse stop_control to release the tool-owned session. Users can also stop from the executing Desktop control card; its button addresses the displayed generation rather than a subsequently created session." ] } + }, + { + "id": "app-sharing", + "titleZh": "选择 App 窗口并单独授权或撤销控制", + "titleEn": "Select an App window and separately grant or revoke control", + "control": { + "kind": "open", + "reasonCode": "visualSelection", + "reasonZh": "共享窗口及控制模式由用户明确选择；智能体只能请求共享，不能代替用户授权。", + "reasonEn": "Window selection and input mode require an explicit user choice; agents can request sharing but cannot grant themselves control." + } + }, + { + "id": "preview-input", + "titleZh": "在共享预览中点击、拖动、滚动及输入", + "titleEn": "Click, drag, scroll and type through the shared preview", + "control": { + "kind": "open", + "reasonCode": "visualSelection", + "reasonZh": "预览输入需要用户授权，并绑定共享窗口、图像坐标及授权版本。", + "reasonEn": "Preview input requires user authorization and is bound to the shared window, image coordinates and authorization revision." + } + }, + { + "id": "remembered-control", + "titleZh": "按应用执行文件和控制模式记住或忘记授权", + "titleEn": "Remember or forget control by application executable and input mode", + "control": { + "kind": "open", + "reasonCode": "visualSelection", + "reasonZh": "记住授权只在用户选定并共享同一应用后生效，应用文件或输入模式改变时需要重新授权。", + "reasonEn": "Remembered control applies only after the user shares the same application; changed executable contents or input modes require a new grant." + } } ], "stepsZh": [ @@ -22140,10 +22316,12 @@ "连接或断开受支持的浏览器", "检查并打开系统权限设置", "查看当前控制目标与预览，并随时停止控制", + "先选择共享 App，再授权输入；可记住授权或随时撤销", "Enable or disable Computer Use", "Connect or disconnect a supported browser", "Inspect and open OS permission settings", "Inspect the current control target and preview, and stop control at any time", + "Select an App share before granting input; remember or revoke the grant", "检查 Computer Use 可用性、辅助功能和屏幕录制授权状态", "Check Computer Use availability plus accessibility and screen-recording permission status", "请求桌面控制所需的系统权限", @@ -22162,6 +22340,12 @@ "Inspect the current target, mode, and authorized capture preview in the control card", "停止当前桌面控制会话并释放捕获与输入资源", "Stop the current desktop-control session and release capture and input resources", + "选择 App 窗口并单独授权或撤销控制", + "Select an App window and separately grant or revoke control", + "在共享预览中点击、拖动、滚动及输入", + "Click, drag, scroll and type through the shared preview", + "按应用执行文件和控制模式记住或忘记授权", + "Remember or forget control by application executable and input mode", "打开桌面控制设置", "带我配置浏览器控制", "Open desktop-control settings", diff --git a/docs/interactive-capabilities/capabilities/feature.computer-use.md b/docs/interactive-capabilities/capabilities/feature.computer-use.md index 3a3bd82d5b..2c83d4fd9c 100644 --- a/docs/interactive-capabilities/capabilities/feature.computer-use.md +++ b/docs/interactive-capabilities/capabilities/feature.computer-use.md @@ -35,6 +35,12 @@ Allow agents to operate a browser or local UI while explicitly managing accessib - Inspect the current target, mode, and authorized capture preview in the control card - **由专用智能体工具控制 / Delegated Agent tool** · `ComputerUse` · 停止当前桌面控制会话并释放捕获与输入资源 - Stop the current desktop-control session and release capture and input resources +- **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 选择 App 窗口并单独授权或撤销控制 + - Select an App window and separately grant or revoke control +- **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 在共享预览中点击、拖动、滚动及输入 + - Click, drag, scroll and type through the shared preview +- **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 按应用执行文件和控制模式记住或忘记授权 + - Remember or forget control by application executable and input mode ## 怎么用 / How to use it diff --git a/docs/interactive-capabilities/technical/product-control-open-audit.json b/docs/interactive-capabilities/technical/product-control-open-audit.json index fd51f91eae..40ec72cffd 100644 --- a/docs/interactive-capabilities/technical/product-control-open-audit.json +++ b/docs/interactive-capabilities/technical/product-control-open-audit.json @@ -1,13 +1,13 @@ { "schemaVersion": 1, "generatedFrom": "src/shared/interactive-capabilities/catalog.json", - "catalogDigest": "e05dbc222e1267523832f82d6392701ab011c9d8d5ad1582dbe407904dc80d3d", - "count": 214, + "catalogDigest": "156caca2db3ed5820e9bf6683d4df89db5bbbfac8c6c8910d31262c05450ec6e", + "count": 217, "reasonCounts": { "externalAuth": 4, "secretEntry": 5, "unstructuredInteraction": 185, - "visualSelection": 20 + "visualSelection": 23 }, "entries": [ { @@ -957,7 +957,8 @@ "pageId": "tools.execution" }, "evidence": [ - "command:computer_use_request_permissions" + "command:computer_use_request_permissions", + "command:computer_use_request_permission" ] }, { @@ -1045,6 +1046,59 @@ "source:src/web-ui/src/app/components/ComputerUseControlCard.tsx#ComputerUseControlCard" ] }, + { + "capabilityId": "feature.computer-use", + "itemId": "app-sharing", + "titleZh": "选择 App 窗口并单独授权或撤销控制", + "titleEn": "Select an App window and separately grant or revoke control", + "reasonCode": "visualSelection", + "reasonZh": "共享窗口及控制模式由用户明确选择；智能体只能请求共享，不能代替用户授权。", + "reasonEn": "Window selection and input mode require an explicit user choice; agents can request sharing but cannot grant themselves control.", + "presentationTarget": { + "kind": "settings", + "pageId": "tools.execution" + }, + "evidence": [ + "command:computer_use_share_request", + "command:computer_use_share_targets", + "command:computer_use_share_start", + "command:computer_use_share_control" + ] + }, + { + "capabilityId": "feature.computer-use", + "itemId": "preview-input", + "titleZh": "在共享预览中点击、拖动、滚动及输入", + "titleEn": "Click, drag, scroll and type through the shared preview", + "reasonCode": "visualSelection", + "reasonZh": "预览输入需要用户授权，并绑定共享窗口、图像坐标及授权版本。", + "reasonEn": "Preview input requires user authorization and is bound to the shared window, image coordinates and authorization revision.", + "presentationTarget": { + "kind": "settings", + "pageId": "tools.execution" + }, + "evidence": [ + "command:computer_use_share_observe", + "command:computer_use_share_input" + ] + }, + { + "capabilityId": "feature.computer-use", + "itemId": "remembered-control", + "titleZh": "按应用执行文件和控制模式记住或忘记授权", + "titleEn": "Remember or forget control by application executable and input mode", + "reasonCode": "visualSelection", + "reasonZh": "记住授权只在用户选定并共享同一应用后生效，应用文件或输入模式改变时需要重新授权。", + "reasonEn": "Remembered control applies only after the user shares the same application; changed executable contents or input modes require a new grant.", + "presentationTarget": { + "kind": "settings", + "pageId": "tools.execution" + }, + "evidence": [ + "command:computer_use_share_grants", + "command:computer_use_share_forget_grant" + ] + }, { "capabilityId": "feature.skills", "itemId": "installed-catalog", diff --git a/docs/interactive-capabilities/technical/tauri-command-map.json b/docs/interactive-capabilities/technical/tauri-command-map.json index f51bbba056..dccbfc27c5 100644 --- a/docs/interactive-capabilities/technical/tauri-command-map.json +++ b/docs/interactive-capabilities/technical/tauri-command-map.json @@ -1,11 +1,11 @@ { "schemaVersion": 2, "generatedFrom": "src/shared/interactive-capabilities/catalog.json", - "catalogDigest": "e05dbc222e1267523832f82d6392701ab011c9d8d5ad1582dbe407904dc80d3d", - "commandCount": 670, + "catalogDigest": "156caca2db3ed5820e9bf6683d4df89db5bbbfac8c6c8910d31262c05450ec6e", + "commandCount": 679, "coverage": { - "commandCount": 670, - "documentedCommandCount": 622, + "commandCount": 679, + "documentedCommandCount": 631, "implementationCommandCount": 48, "implementationDigest": "f6d38a24a70708988cb47ada81d07eccf0668684e8734c9ee5155b9ffa7e3db8" }, @@ -1432,6 +1432,22 @@ "signature": "fn computer_use_open_system_settings( request: ComputerUseOpenSettingsRequest, ) -> Result<(), String>", "remoteWorkspacePolicy": "LocalOnly" }, + { + "id": "computer_use_request_permission", + "moduleId": "computer_use", + "capabilityId": "feature.computer-use", + "capabilityIds": [ + "feature.computer-use" + ], + "documentedItemIds": [ + "feature.computer-use:request-permissions" + ], + "visibility": "documented", + "rustPath": "computer_use_request_permission", + "sourceFile": "src/apps/desktop/src/api/computer_use_api.rs", + "signature": "fn computer_use_request_permission( request: ComputerPermissionRequest, ) -> Result", + "remoteWorkspacePolicy": "LocalOnly" + }, { "id": "computer_use_request_permissions", "moduleId": "computer_use", @@ -1452,6 +1468,134 @@ "signature": "fn computer_use_request_permissions() -> Result<(), String>", "remoteWorkspacePolicy": "LocalOnly" }, + { + "id": "computer_use_share_control", + "moduleId": "computer_use", + "capabilityId": "feature.computer-use", + "capabilityIds": [ + "feature.computer-use" + ], + "documentedItemIds": [ + "feature.computer-use:app-sharing" + ], + "visibility": "documented", + "rustPath": "computer_use_share_control", + "sourceFile": "src/apps/desktop/src/api/computer_use_api.rs", + "signature": "fn computer_use_share_control( state: State<'_, AppState>, request: ComputerUseShareControlRequest, ) -> Result", + "remoteWorkspacePolicy": "LocalOnly" + }, + { + "id": "computer_use_share_forget_grant", + "moduleId": "computer_use", + "capabilityId": "feature.computer-use", + "capabilityIds": [ + "feature.computer-use" + ], + "documentedItemIds": [ + "feature.computer-use:remembered-control" + ], + "visibility": "documented", + "rustPath": "computer_use_share_forget_grant", + "sourceFile": "src/apps/desktop/src/api/computer_use_api.rs", + "signature": "fn computer_use_share_forget_grant( state: State<'_, AppState>, request: ComputerUseForgetGrantRequest, ) -> Result, String>", + "remoteWorkspacePolicy": "LocalOnly" + }, + { + "id": "computer_use_share_grants", + "moduleId": "computer_use", + "capabilityId": "feature.computer-use", + "capabilityIds": [ + "feature.computer-use" + ], + "documentedItemIds": [ + "feature.computer-use:remembered-control" + ], + "visibility": "documented", + "rustPath": "computer_use_share_grants", + "sourceFile": "src/apps/desktop/src/api/computer_use_api.rs", + "signature": "fn computer_use_share_grants( state: State<'_, AppState>, ) -> Result, String>", + "remoteWorkspacePolicy": "LocalOnly" + }, + { + "id": "computer_use_share_input", + "moduleId": "computer_use", + "capabilityId": "feature.computer-use", + "capabilityIds": [ + "feature.computer-use" + ], + "documentedItemIds": [ + "feature.computer-use:preview-input" + ], + "visibility": "documented", + "rustPath": "computer_use_share_input", + "sourceFile": "src/apps/desktop/src/api/computer_use_api.rs", + "signature": "fn computer_use_share_input( request: crate::computer_use::shared_preview::SharedInputRequest, ) -> Result", + "remoteWorkspacePolicy": "LocalOnly" + }, + { + "id": "computer_use_share_observe", + "moduleId": "computer_use", + "capabilityId": "feature.computer-use", + "capabilityIds": [ + "feature.computer-use" + ], + "documentedItemIds": [ + "feature.computer-use:preview-input" + ], + "visibility": "documented", + "rustPath": "computer_use_share_observe", + "sourceFile": "src/apps/desktop/src/api/computer_use_api.rs", + "signature": "fn computer_use_share_observe( request: crate::computer_use::shared_preview::SharedPreviewRequest, ) -> Result", + "remoteWorkspacePolicy": "LocalOnly" + }, + { + "id": "computer_use_share_request", + "moduleId": "computer_use", + "capabilityId": "feature.computer-use", + "capabilityIds": [ + "feature.computer-use" + ], + "documentedItemIds": [ + "feature.computer-use:app-sharing" + ], + "visibility": "documented", + "rustPath": "computer_use_share_request", + "sourceFile": "src/apps/desktop/src/api/computer_use_api.rs", + "signature": "fn computer_use_share_request( request: ComputerUseShareRequest, ) -> Result", + "remoteWorkspacePolicy": "LocalOnly" + }, + { + "id": "computer_use_share_start", + "moduleId": "computer_use", + "capabilityId": "feature.computer-use", + "capabilityIds": [ + "feature.computer-use" + ], + "documentedItemIds": [ + "feature.computer-use:app-sharing" + ], + "visibility": "documented", + "rustPath": "computer_use_share_start", + "sourceFile": "src/apps/desktop/src/api/computer_use_api.rs", + "signature": "fn computer_use_share_start( state: State<'_, AppState>, request: ComputerUseShareStartRequest, ) -> Result", + "remoteWorkspacePolicy": "LocalOnly" + }, + { + "id": "computer_use_share_targets", + "moduleId": "computer_use", + "capabilityId": "feature.computer-use", + "capabilityIds": [ + "feature.computer-use" + ], + "documentedItemIds": [ + "feature.computer-use:app-sharing" + ], + "visibility": "documented", + "rustPath": "computer_use_share_targets", + "sourceFile": "src/apps/desktop/src/api/computer_use_api.rs", + "signature": "fn computer_use_share_targets( ) -> Result, String>", + "remoteWorkspacePolicy": "LocalOnly" + }, { "id": "confirm_frontend_update", "moduleId": "frontend_workbench", diff --git a/scripts/core-boundaries/rules/source/required-rules.mjs b/scripts/core-boundaries/rules/source/required-rules.mjs index 183fd6ff41..b307440219 100644 --- a/scripts/core-boundaries/rules/source/required-rules.mjs +++ b/scripts/core-boundaries/rules/source/required-rules.mjs @@ -4401,7 +4401,7 @@ export const requiredContentRules = [ { path: 'src/crates/execution/tool-execution/src/computer_use.rs', reason: - 'tool-runtime must own provider-neutral Computer Use loop detection, screenshot hashing, verification, and retry policy while core keeps host adapters', + 'tool-runtime must own provider-neutral Computer Use observation evidence, screenshot hashing, and read retry policy while core keeps host adapters', patterns: [ { regex: /\bpub struct ComputerUseOptimizer\b/, @@ -4420,12 +4420,12 @@ export const requiredContentRules = [ message: 'missing provider-neutral Computer Use retry decision owner', }, { - regex: /\bdetects_repeated_action_loop\b/, - message: 'missing Computer Use loop detection regression', + regex: /\blegitimate_repeated_work_is_not_classified_as_a_loop\b/, + message: 'missing Computer Use repeated-work evidence regression', }, { - regex: /\bretry_decision_uses_error_text_without_core_error_type\b/, - message: 'missing Computer Use retry decision regression', + regex: /\btimeout_or_stale_errors_never_authorize_input_replay\b/, + message: 'missing Computer Use mutation replay rejection regression', }, ], }, diff --git a/scripts/fixtures/computer-use-capture.m b/scripts/fixtures/computer-use-capture.m index 979c1323b8..dd8f353d28 100644 --- a/scripts/fixtures/computer-use-capture.m +++ b/scripts/fixtures/computer-use-capture.m @@ -19,7 +19,7 @@ @interface NSObject (OBFCaptureTimingFixture) extern void obf_capture_stop(void *); extern void obf_pointer_show(uint32_t, double, double, bool); extern void obf_pointer_hide(void); -void obf_control_native_stopped(uint64_t generation, int32_t pid, uint32_t windowID, const char *reason) { +void obf_control_native_stopped(uint64_t generation, uint64_t revision, int32_t pid, uint32_t windowID, const char *reason) { atomic_fetch_add(&stopCount, 1); if(generation == expectedGeneration && pid == getpid() && windowID == expectedWindow) atomic_store(&stoppedWithIdentity, true); fprintf(stderr, "Stopped: %s\n", reason); diff --git a/scripts/fixtures/computer-use-input-controls.m b/scripts/fixtures/computer-use-input-controls.m index c41d8d66af..5b561390d4 100644 --- a/scripts/fixtures/computer-use-input-controls.m +++ b/scripts/fixtures/computer-use-input-controls.m @@ -10,7 +10,7 @@ static NSWindow *targetWindow, *foreignWindow; static NSMutableArray *events; static NSMutableString *canvasText; -static NSUInteger buttonActions, canvasDowns, canvasCommandDowns, foreignActions, canvasScrolls, shortcutActions; +static NSUInteger buttonActions, canvasDowns, canvasCommandDowns, foreignActions, canvasScrolls, shortcutActions, canvasEscapes; static double canvasScrollX, canvasScrollY; static NSDictionary *targets; static void installLifecycleHistory(void) { @@ -23,7 +23,7 @@ static void installLifecycleHistory(void) { } } static void publish(void) { - NSDictionary *data = @{@"bundle_id":NSBundle.mainBundle.bundleIdentifier ?: @"", @"frontmost_pid":@(NSWorkspace.sharedWorkspace.frontmostApplication.processIdentifier), @"activation_history":activationHistory ?: @[], @"lifecycle_history":lifecycleHistory ?: @[], @"foreign_actions":@(foreignActions), @"selected_count":@(table.selectedRowIndexes.count), @"selected_row":@(table.selectedRow), @"canvas_scrolls":@(canvasScrolls), @"canvas_scroll_x":@(canvasScrollX), @"canvas_scroll_y":@(canvasScrollY), @"button_actions":@(buttonActions), @"shortcut_actions":@(shortcutActions), @"field_text":field.stringValue ?: @"", @"canvas_command_downs":@(canvasCommandDowns), @"canvas_downs":@(canvasDowns), @"canvas_text":canvasText, @"events":events, @"targets":targets ?: @{}, @"window_id":@(targetWindow.windowNumber), @"key_window":@(targetWindow.keyWindow), @"active":@(NSApp.active)}; + NSDictionary *data = @{@"bundle_id":NSBundle.mainBundle.bundleIdentifier ?: @"", @"frontmost_pid":@(NSWorkspace.sharedWorkspace.frontmostApplication.processIdentifier), @"activation_history":activationHistory ?: @[], @"lifecycle_history":lifecycleHistory ?: @[], @"foreign_actions":@(foreignActions), @"selected_count":@(table.selectedRowIndexes.count), @"selected_row":@(table.selectedRow), @"canvas_scrolls":@(canvasScrolls), @"canvas_scroll_x":@(canvasScrollX), @"canvas_scroll_y":@(canvasScrollY), @"button_actions":@(buttonActions), @"shortcut_actions":@(shortcutActions), @"field_text":field.stringValue ?: @"", @"canvas_command_downs":@(canvasCommandDowns), @"canvas_downs":@(canvasDowns), @"canvas_text":canvasText, @"canvas_escape_presses":@(canvasEscapes), @"events":events, @"targets":targets ?: @{}, @"window_id":@(targetWindow.windowNumber), @"key_window":@(targetWindow.keyWindow), @"active":@(NSApp.active)}; [[NSJSONSerialization dataWithJSONObject:data options:0 error:nil] writeToFile:resultPath atomically:YES]; } @interface OBFControlsApp : NSApplication @@ -50,7 +50,7 @@ - (BOOL)isAccessibilityElement { return NO; } - (void)drawRect:(NSRect)dirty { [NSColor.systemBlueColor setFill]; NSRectFill(self.bounds); } - (void)mouseDown:(NSEvent *)event { canvasDowns++; if (event.modifierFlags & NSEventModifierFlagCommand) canvasCommandDowns++; [self.window makeFirstResponder:self]; publish(); } - (void)scrollWheel:(NSEvent *)event { canvasScrolls++; canvasScrollX += event.scrollingDeltaX; canvasScrollY += event.scrollingDeltaY; publish(); } -- (void)keyDown:(NSEvent *)event { if (event.characters) [canvasText appendString:event.characters]; publish(); } +- (void)keyDown:(NSEvent *)event { if (event.keyCode == 53) canvasEscapes++; else if (event.characters) [canvasText appendString:event.characters]; publish(); } @end @interface OBFControlsActions : NSObject @end diff --git a/scripts/fixtures/computer-use-pointer.m b/scripts/fixtures/computer-use-pointer.m index 0466717c09..5429c3c47c 100644 --- a/scripts/fixtures/computer-use-pointer.m +++ b/scripts/fixtures/computer-use-pointer.m @@ -2,7 +2,7 @@ #import "../../src/apps/desktop/src/computer_use/macos_capture.m" #include -void obf_control_native_stopped(uint64_t generation, int32_t pid, uint32_t windowID, const char *reason) { +void obf_control_native_stopped(uint64_t generation, uint64_t revision, int32_t pid, uint32_t windowID, const char *reason) { (void)generation; (void)pid; (void)windowID; (void)reason; } diff --git a/scripts/test-computer-use-context.mjs b/scripts/test-computer-use-context.mjs index 255216b3e8..1bcf030e66 100644 --- a/scripts/test-computer-use-context.mjs +++ b/scripts/test-computer-use-context.mjs @@ -26,8 +26,18 @@ sha1 = "0.10" // no replacement algorithms, fake screenshots, or native-host stubs here. await writeFile(join(dir, 'lib.rs'), `#![allow(dead_code)] extern crate self as openbitfun_core; +extern crate self as openbitfun_agent_tools; #[path = ${source('src/crates/execution/tool-contracts/src/computer_use.rs')}] pub mod computer_use_contract; +pub use computer_use_contract as computer_use; +#[path = ${source('src/crates/execution/tool-contracts/src/computer_use_control.rs')}] +pub mod computer_use_control; +#[path = ${source('src/crates/execution/tool-contracts/src/computer_use_geometry.rs')}] +pub mod computer_use_geometry; +#[path = ${source('src/apps/desktop/src/computer_use/control_session/admission.rs')}] +mod control_admission; +#[path = ${source('src/apps/desktop/src/computer_use/observation_cache.rs')}] +mod observation_cache; pub mod agentic { pub mod tools { pub mod computer_use_host { pub use crate::computer_use_contract::*; }}} diff --git a/scripts/test-linux-computer-use-native.mjs b/scripts/test-linux-computer-use-native.mjs index 6c0c9fc12d..cbddcec14e 100644 --- a/scripts/test-linux-computer-use-native.mjs +++ b/scripts/test-linux-computer-use-native.mjs @@ -33,6 +33,9 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time" serde = { version = "1", features = ["derive"] } serde_json = "1" sha1 = "0.10" +sha2 = "0.10" +tempfile = "3" +libc = "0.2" screenshots = "0.8" base64 = "0.22" image = { version = "0.25", default-features = false, features = ["jpeg", "png"] } @@ -45,6 +48,8 @@ extern crate self as openbitfun_agent_tools; pub mod computer_use_contract; #[path = ${source('src/crates/execution/tool-contracts/src/computer_use_control.rs')}] pub mod computer_use_control; +#[path = ${source('src/crates/execution/tool-contracts/src/computer_use_permissions.rs')}] +pub mod computer_use_permissions; pub mod agentic { pub mod tools { pub mod computer_use_host { pub use crate::computer_use_contract::*; ${lease} @@ -52,18 +57,14 @@ pub mod agentic { pub mod tools { pub mod computer_use_host { pub mod util { pub mod errors { pub use crate::computer_use_contract::{ComputerUseContractError as OpenBitFunError, ComputerUseContractResult as OpenBitFunResult}; }} -#[path = ${source('src/apps/desktop/src/computer_use/control_session.rs')}] -mod control_session; -#[path = ${source('src/apps/desktop/src/computer_use/linux_control.rs')}] -mod linux_control; -#[path = ${source('src/apps/desktop/src/computer_use/linux_control_ax.rs')}] -mod linux_control_ax; -#[path = ${source('src/apps/desktop/src/computer_use/linux_ax_ui.rs')}] -mod linux_ax_ui; -#[path = ${source('src/apps/desktop/src/computer_use/ui_locate_common.rs')}] -mod ui_locate_common; -#[path = ${source('src/apps/desktop/src/computer_use/ax_snapshot_digest.rs')}] -mod ax_snapshot_digest; +mod computer_use { + ${['control_session', 'process_identity', 'grants', 'permissions', 'linux_control', 'linux_control_ax', 'linux_ax_ui', 'ui_locate_common', 'ax_snapshot_digest'].map(name => `#[path = ${source(`src/apps/desktop/src/computer_use/${name}.rs`)}] mod ${name};`).join('\n ')} + mod desktop_host { + use crate::util::errors::{OpenBitFunError, OpenBitFunResult}; + #[path = ${source('src/apps/desktop/src/computer_use/desktop_host/background_guard.rs')}] mod background_guard; + } +} + `); const code = await new Promise((resolveExit, reject) => { const child = spawn('cargo', ['test', '--offline', '--manifest-path', join(dir, 'Cargo.toml'), '--lib', ...process.argv.slice(2)], { diff --git a/scripts/test-macos-control-roundtrip.mjs b/scripts/test-macos-control-roundtrip.mjs index 5c339d6df8..bd763fef1c 100644 --- a/scripts/test-macos-control-roundtrip.mjs +++ b/scripts/test-macos-control-roundtrip.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { mkdtemp, rm, readFile } from 'node:fs/promises'; +import { mkdtemp, rm, readFile, mkdir, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { spawn } from 'node:child_process'; @@ -7,16 +7,27 @@ import { fileURLToPath } from 'node:url'; assert.equal(process.platform, 'darwin', 'ComputerUse tool roundtrip fixture requires macOS'); const root = fileURLToPath(new URL('../', import.meta.url)); const directory = await mkdtemp(join(tmpdir(), 'openbitfun-control-roundtrip-')); -const run = (command, args, env = process.env) => new Promise((resolve, reject) => { +const run = (command, args, env = process.env, onSpawn) => new Promise((resolve, reject) => { const child = spawn(command, args, { cwd: root, stdio: 'inherit', windowsHide: true, env }); + onSpawn?.(child); child.on('error', reject); child.on('close', code => code === 0 ? resolve() : reject(new Error(`${command} exited ${code}`))); }); let fixture; +let observer; +let harness; try { const binary = join(directory, 'input-fixture'); const result = join(directory, 'counts.json'); + const contents = join(directory, 'OpenBitFun Roundtrip Observer.app', 'Contents'); + await mkdir(join(contents, 'MacOS'), { recursive: true }); + await writeFile(join(contents, 'Info.plist'), `CFBundleIdentifierdev.openbitfun.roundtrip-fixture.observerCFBundleNameOpenBitFun Roundtrip ObserverCFBundleExecutablefixtureCFBundlePackageTypeAPPL`); + const observerBinary = join(contents, 'MacOS', 'fixture'); await run('xcrun', ['clang', '-fobjc-arc', '-Wall', '-Werror', 'scripts/fixtures/computer-use-roundtrip.m', '-framework', 'AppKit', '-o', binary]); + await run('xcrun', ['clang', '-fobjc-arc', '-Wall', '-Werror', 'scripts/fixtures/computer-use-input-controls.m', '-framework', 'AppKit', '-o', observerBinary]); + // Build before either disposable window is opened. The host's AppKit startup + // must also finish before the observer establishes the foreground baseline. + if (!process.env.OPENBITFUN_TEST_BINARY) await run('cargo', ['test', '-p', 'openbitfun-desktop', '--features', 'devtools', '--test', 'computer_use_native_roundtrip', '--no-run']); fixture = spawn(binary, [result], { cwd: root, stdio: ['ignore', 'pipe', 'inherit'], windowsHide: true }); const [pid, x, y] = await new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error('Input fixture failed to become ready')), 10_000); @@ -31,10 +42,27 @@ try { }); const command = process.env.OPENBITFUN_TEST_BINARY || 'cargo'; const args = process.env.OPENBITFUN_TEST_BINARY ? [] : ['test', '-p', 'openbitfun-desktop', '--features', 'devtools', '--test', 'computer_use_native_roundtrip']; - await run(command, args, { + const hostReady = join(directory, 'host-ready'); + const observerResult = join(directory, 'observer.json'); + const roundtrip = run(command, args, { ...process.env, OPENBITFUN_ROUNDTRIP_FIXTURE_PID: pid, OPENBITFUN_ROUNDTRIP_FIXTURE_X: x, OPENBITFUN_ROUNDTRIP_FIXTURE_Y: y, OPENBITFUN_ROUNDTRIP_FIXTURE_RESULT: result, - }); + OPENBITFUN_INPUT_HOST_READY: hostReady, OPENBITFUN_INPUT_OBSERVER_RESULT: observerResult, + }, child => { harness = child; }).then(() => null, error => error); + const deadline = Date.now() + 10_000; + while (true) { + try { await readFile(hostReady); break; } catch (error) { + if (error.code !== 'ENOENT' || Date.now() >= deadline) throw error; + await new Promise(resolve => setTimeout(resolve, 20)); + } + } + observer = spawn(observerBinary, ['--observer', observerResult], { cwd: root, stdio: ['ignore', 'pipe', 'inherit'], windowsHide: true }); + const failure = await roundtrip; + if (failure) throw failure; + const foreground = JSON.parse(await readFile(observerResult, 'utf8')); + assert.equal(foreground.active, true); + assert.equal(foreground.key_window, true); + assert.equal(foreground.frontmost_pid, observer.pid); const received = JSON.parse(await readFile(result, 'utf8')); assert.equal(received.downs, 1); assert.equal(received.ups, 1); @@ -43,6 +71,8 @@ try { assert.equal(received.text, '背景输入 Test'); console.log('PASS: tool observation/action/verification loop delivered exactly one click, semantic activation and Enter with exact Unicode text'); } finally { + if (harness && harness.exitCode === null) await new Promise(resolve => { harness.once('exit', resolve); harness.kill('SIGTERM'); }); + if (observer && observer.exitCode === null) await new Promise(resolve => { observer.once('exit', resolve); observer.kill('SIGTERM'); }); if (fixture && fixture.exitCode === null) await new Promise(resolve => { fixture.once('exit', resolve); fixture.kill('SIGTERM'); }); await rm(directory, { recursive: true, force: true }); } diff --git a/scripts/test-windows-computer-use-native.mjs b/scripts/test-windows-computer-use-native.mjs new file mode 100644 index 0000000000..071d29bf83 --- /dev/null +++ b/scripts/test-windows-computer-use-native.mjs @@ -0,0 +1,91 @@ +// Compile production WGC/UIA/MSAA/control modules without linking the desktop shell. +// Cross-host mode checks types only; it never claims native Windows execution. +import { mkdtemp, readFile, writeFile, copyFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawn } from 'node:child_process'; + +const root = fileURLToPath(new URL('../', import.meta.url)); +const args = process.argv.slice(2); +const check = args[0] === '--check'; +if (check) args.shift(); +if (process.platform !== 'win32' && !check) { + throw new Error('Native tests require Windows. Use --check --target x86_64-pc-windows-msvc for a compile-only check.'); +} +const dir = await mkdtemp(join(tmpdir(), 'openbitfun-windows-native-')); +const source = (path) => JSON.stringify(resolve(root, path)); +try { + const host = await readFile(join(root, 'src/crates/assembly/core/src/agentic/tools/computer_use_host.rs'), 'utf8'); + const lease = host.match(/pub trait ComputerUseActionLease:[\s\S]*?\n\}/)?.[0]; + if (!lease) throw new Error('The production lease trait could not be extracted.'); + await writeFile(join(dir, 'Cargo.toml'), `[package] +name = "windows-computer-use-native-tests" +version = "0.0.0" +edition = "2021" +[workspace] +[lib] +path = "lib.rs" +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha1 = "0.10" +sha2 = "0.10" +tempfile = "3" +log = "0.4" +tokio = { version = "1", default-features = false, features = ["rt-multi-thread", "macros", "sync", "time"] } +screenshots = "0.8" +base64 = "0.22" +image = { version = "0.25", default-features = false, features = ["jpeg", "png"] } +windows = { version = "0.61", features = [ + "Foundation", "Graphics_Capture", "Graphics_DirectX", "Graphics_DirectX_Direct3D11", + "Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", + "Win32_Graphics_Dwm", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Gdi", + "Win32_System_Com", "Win32_System_Ole", "Win32_System_Variant", "Win32_System_LibraryLoader", + "Win32_System_Threading", "Win32_System_WinRT", "Win32_System_WinRT_Direct3D11", + "Win32_System_WinRT_Graphics_Capture", "Win32_UI_Accessibility", "Win32_UI_Input_KeyboardAndMouse", + "Win32_UI_WindowsAndMessaging" +] } +`); + await copyFile(join(root, 'Cargo.lock'), join(dir, 'Cargo.lock')); + // Only the error-category constructors are adapted to the portable error + // envelope. Every native API, DTO, cache, lease and algorithm is production code. + await writeFile(join(dir, 'lib.rs'), `#![allow(dead_code, unused_imports)] +extern crate self as openbitfun_core; +extern crate self as openbitfun_agent_tools; +#[path = ${source('src/crates/execution/tool-contracts/src/computer_use.rs')}] +pub mod computer_use_contract; +#[path = ${source('src/crates/execution/tool-contracts/src/computer_use_control.rs')}] +pub mod computer_use_control; +#[path = ${source('src/crates/execution/tool-contracts/src/computer_use_permissions.rs')}] +pub mod computer_use_permissions; +impl computer_use_contract::ComputerUseContractError { + pub fn service(message: impl Into) -> Self { Self::tool(message) } + pub fn io(message: impl Into) -> Self { Self::tool(message) } +} +pub mod agentic { pub mod tools { pub mod computer_use_host { + pub use crate::computer_use_contract::*; + ${lease} +}}} +pub mod util { pub mod errors { + pub use crate::computer_use_contract::{ComputerUseContractError as OpenBitFunError, ComputerUseContractResult as OpenBitFunResult}; +}} +mod computer_use { + ${['control_session', 'process_identity', 'grants', 'permissions', 'windows_list_apps', 'windows_ax_ui', 'windows_msaa', 'windows_capture', 'windows_wgc_capture', 'windows_pointer_feedback', 'ui_locate_common', 'ax_snapshot_digest'].map(name => `#[path = ${source(`src/apps/desktop/src/computer_use/${name}.rs`)}] mod ${name};`).join('\n ')} + mod desktop_host { + use crate::util::errors::{OpenBitFunError, OpenBitFunResult}; + #[path = ${source('src/apps/desktop/src/computer_use/desktop_host/background_guard.rs')}] mod background_guard; + } +} +`); + process.exitCode = await new Promise((resolveExit, reject) => { + const child = spawn('cargo', [check ? 'check' : 'test', '--offline', '--manifest-path', join(dir, 'Cargo.toml'), check ? '--tests' : '--lib', ...args], { + cwd: root, stdio: 'inherit', windowsHide: true, + env: { ...process.env, CARGO_TARGET_DIR: process.env.CARGO_TARGET_DIR ?? join(root, 'target/windows-computer-use-native') }, + }); + child.on('error', reject); + child.on('close', code => resolveExit(code ?? 1)); + }); +} finally { + await rm(dir, { recursive: true, force: true }); +} diff --git a/src/apps/desktop/Cargo.toml b/src/apps/desktop/Cargo.toml index 7e2319ba26..c051934b26 100644 --- a/src/apps/desktop/Cargo.toml +++ b/src/apps/desktop/Cargo.toml @@ -130,7 +130,6 @@ windows = { workspace = true, features = [ "Win32_System_LibraryLoader", "Win32_System_SystemServices", "Win32_System_Threading", - "Win32_Storage_Xps", "Win32_System_Com", "Win32_System_Ole", "Win32_System_Variant", @@ -158,6 +157,7 @@ default = [] devtools = ["tauri/devtools", "openbitfun-webdriver/embedded"] [target.'cfg(target_os = "linux")'.dependencies] +libc = { workspace = true } atspi = { workspace = true, features = ["zbus"] } gstreamer = { workspace = true } gstreamer-app = { workspace = true } diff --git a/src/apps/desktop/src/api/computer_use_api.rs b/src/apps/desktop/src/api/computer_use_api.rs index d3dec095e1..f2cd48566e 100644 --- a/src/apps/desktop/src/api/computer_use_api.rs +++ b/src/apps/desktop/src/api/computer_use_api.rs @@ -1,10 +1,11 @@ //! Tauri commands for Computer use (permissions + settings deep links). use crate::api::app_state::AppState; -use crate::computer_use::DesktopComputerUseHost; -use openbitfun_core::agentic::tools::computer_use_host::ComputerUseHost; +use openbitfun_agent_tools::computer_use_permissions::{ + ComputerPermissionKind, ComputerPermissionReceipt, ComputerPermissionRequest, + ComputerPermissions, +}; use openbitfun_core::service::config::types::AIConfig; -#[cfg(target_os = "windows")] use openbitfun_core::util::process_manager; use serde::{Deserialize, Serialize}; use tauri::State; @@ -16,6 +17,7 @@ pub struct ComputerUseStatusResponse { pub accessibility_granted: bool, pub screen_capture_granted: bool, pub platform_note: Option, + pub permissions: ComputerPermissions, } #[derive(Debug, Deserialize)] @@ -35,27 +37,39 @@ pub async fn computer_use_get_status( .await .map_err(|e| e.to_string())?; - let host = DesktopComputerUseHost::new(); - let snap = host - .permission_snapshot() - .await - .map_err(|e| e.to_string())?; + let permissions = crate::computer_use::permissions::status().await; Ok(ComputerUseStatusResponse { computer_use_enabled: ai.computer_use_enabled, - accessibility_granted: snap.accessibility_granted, - screen_capture_granted: snap.screen_capture_granted, - platform_note: snap.platform_note, + accessibility_granted: permissions.usable(ComputerPermissionKind::Accessibility), + screen_capture_granted: permissions.usable(ComputerPermissionKind::ScreenRecording), + platform_note: Some( + permissions + .permissions + .iter() + .map(|permission| permission.detail.as_str()) + .collect::>() + .join(" "), + ), + permissions, }) } #[tauri::command] pub async fn computer_use_request_permissions() -> Result<(), String> { - let host = DesktopComputerUseHost::new(); - host.prompt_for_missing_permissions(); + // Compatibility command: older clients still invoke this from a user gesture. + crate::computer_use::permissions::request(ComputerPermissionKind::Accessibility).await?; + crate::computer_use::permissions::request(ComputerPermissionKind::ScreenRecording).await?; Ok(()) } +#[tauri::command] +pub async fn computer_use_request_permission( + request: ComputerPermissionRequest, +) -> Result { + crate::computer_use::permissions::request(request.kind).await +} + #[tauri::command] pub async fn computer_use_open_system_settings( request: ComputerUseOpenSettingsRequest, @@ -63,15 +77,18 @@ pub async fn computer_use_open_system_settings( #[cfg(target_os = "macos")] { let url = match request.pane.as_str() { - "accessibility" => { + "accessibility" | "input_posting" => { "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility" } - "screen_capture" => { + "screen_capture" | "screen_recording" => { "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture" } + "input_monitoring" => { + "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent" + } _ => return Err(format!("Unknown settings pane: {}", request.pane)), }; - std::process::Command::new("open") + process_manager::create_command("open") .arg(url) .status() .map_err(|e| e.to_string())?; @@ -134,3 +151,113 @@ pub async fn computer_use_control_preview( ) -> Result, String> { crate::computer_use::control_session::preview(request.generation) } + +#[derive(Debug, Deserialize)] +pub struct ComputerUseShareStartRequest { + pub generation: u64, + pub target: String, + #[serde(default)] + pub mode: openbitfun_agent_tools::computer_use_control::ControlMode, +} +#[derive(Debug, Deserialize)] +pub struct ComputerUseShareControlRequest { + pub generation: u64, + pub target_revision: u64, + pub authorization_revision: u64, + pub enabled: bool, + #[serde(default)] + pub remember: bool, +} +#[tauri::command] +pub async fn computer_use_share_targets( +) -> Result, String> { + crate::computer_use::share_targets().await +} +#[tauri::command] +pub async fn computer_use_share_start( + state: State<'_, AppState>, + request: ComputerUseShareStartRequest, +) -> Result { + let snapshot = crate::computer_use::shared_preview::host() + .share_selected_target(request.generation, &request.target, request.mode) + .await?; + crate::computer_use::share_grants::apply_remembered(computer_use_grants_path(&state), snapshot) + .await +} +#[tauri::command] +pub async fn computer_use_share_control( + state: State<'_, AppState>, + request: ComputerUseShareControlRequest, +) -> Result { + let snapshot = crate::computer_use::control_session::set_share_control( + request.generation, + request.target_revision, + request.authorization_revision, + request.enabled, + )?; + if request.enabled && request.remember { + crate::computer_use::share_grants::remember(computer_use_grants_path(&state), snapshot) + .await + } else { + Ok(snapshot) + } +} + +fn computer_use_grants_path(state: &AppState) -> std::path::PathBuf { + state + .workspace_service + .path_manager() + .user_data_dir() + .join("computer-use") + .join("control-grants.json") +} + +#[derive(Debug, Deserialize)] +pub struct ComputerUseShareRequest { + #[serde(default)] + pub owner: Option, + #[serde(default)] + pub mode: openbitfun_agent_tools::computer_use_control::ControlMode, +} +#[tauri::command] +pub async fn computer_use_share_request( + request: ComputerUseShareRequest, +) -> Result { + // Manual shares have an explicit owner. An Agent cannot silently adopt it. + crate::computer_use::control_session::request_share( + request.owner.as_deref().unwrap_or("user:manual-sharing"), + request.mode, + ) +} +#[tauri::command] +pub async fn computer_use_share_grants( + state: State<'_, AppState>, +) -> Result, String> { + crate::computer_use::share_grants::list(computer_use_grants_path(&state)).await +} +#[derive(Debug, Deserialize)] +pub struct ComputerUseForgetGrantRequest { + pub identity: crate::computer_use::grants::GrantIdentity, +} +#[tauri::command] +pub async fn computer_use_share_forget_grant( + state: State<'_, AppState>, + request: ComputerUseForgetGrantRequest, +) -> Result, String> { + crate::computer_use::share_grants::forget(computer_use_grants_path(&state), request.identity) + .await +} + +#[tauri::command] +pub async fn computer_use_share_observe( + request: crate::computer_use::shared_preview::SharedPreviewRequest, +) -> Result { + crate::computer_use::shared_preview::observe(request).await +} + +#[tauri::command] +pub async fn computer_use_share_input( + request: crate::computer_use::shared_preview::SharedInputRequest, +) -> Result { + crate::computer_use::shared_preview::input(request).await +} diff --git a/src/apps/desktop/src/computer_use/AGENTS.md b/src/apps/desktop/src/computer_use/AGENTS.md index 927a9b6e23..e76bc8b004 100644 --- a/src/apps/desktop/src/computer_use/AGENTS.md +++ b/src/apps/desktop/src/computer_use/AGENTS.md @@ -51,6 +51,13 @@ remaining native acceptance scenarios separately. ## Module map - `desktop_host/` — `ComputerUseHost` trait impl; entry for all actions +- `control_session/admission.rs` — pure mode/owner/lease/start/stop state machine +- `control_session/sharing.rs`, `desktop_host/sharing.rs` — user-selected window, observe-first sharing and revocable input grant +- `permissions.rs`, `process_identity.rs`, `grants.rs`, `share_grants.rs` — explicit OS requests and executable-bound remembered preferences +- `desktop_host/shared_preview.rs` — scoped manual preview input through the same native host/lease as the Agent +- `desktop_host/background_guard.rs` — before/after desktop witnesses; no focus restoration or replay +- `desktop_host/observation.rs`, `observation_cache.rs` — observation identity and bounded screenshot geometry; never input authority +- `desktop_host/state.rs` — derived host feedback and view caches - `macos_*` / `windows_*` — platform AX, capture, list-apps, shortcuts - `windows_capture.rs` — WGC frame conversion and authoritative window bounds - `windows_wgc_capture.rs` — Windows.Graphics.Capture (Direct3D11) @@ -156,3 +163,37 @@ cargo build -p openbitfun-desktop ``` Windows-only paths (`windows_wgc_capture`, UIA) compile on CI (`windows-latest`). +The isolated native-module harness compiles the production control owner, UIA, +MSAA, WGC and pointer modules. Its portable error-envelope adapter replaces no +native APIs or algorithms. Cross-compilation is type-checking, not Windows +execution; full Desktop compilation and native fixtures still belong on Windows. + +```bash +# Native Windows: unit tests; native delivery fixtures remain opt-in/ignored. +node scripts/test-windows-computer-use-native.mjs +# Other hosts with the target installed: compile production native modules only. +node scripts/test-windows-computer-use-native.mjs --check --target x86_64-pc-windows-msvc +``` + +Observation IDs and target revisions are additive wire fields. Keep accepting +legacy payloads, advertise `observation-id-v1`, and reject explicit stale IDs +before input. New observations invalidate prior AX/view indices even when the +content digest is unchanged. Do not replace namespace identity with a digest. + +New native fixture setup must use the production target inventory, selected-share +capture and explicit grant. Tool `start_control` only requests sharing. Do not +bypass that gate to make a native fixture pass. Permission probes must never +prompt, and remembered grants must never select or share a window automatically. + +The Web preview's deterministic cursor is verified against Rust-generated +`computer-use-rust` samples retained in the repository. The reference source is +not required to build or run tests. After preview, sharing or permission UI edits: + +```bash +pnpm --dir src/web-ui run test:run src/app/components/ComputerUseControlCard.test.tsx src/app/components/computer-use src/infrastructure/api/adapters/peer-device-adapter.test.ts +``` + +These UI tests cover permission requests, stale device/grant responses, geometry, +gesture cancellation and cursor math. They do not establish native delivery or +remote end-to-end behavior. The strict macOS fixture also exercises native shared +preview capture/input and stale preview rejection after revocation. diff --git a/src/apps/desktop/src/computer_use/control_session.rs b/src/apps/desktop/src/computer_use/control_session.rs index 3349f6f42f..5b15700834 100644 --- a/src/apps/desktop/src/computer_use/control_session.rs +++ b/src/apps/desktop/src/computer_use/control_session.rs @@ -1,12 +1,22 @@ //! Desktop GUI resource admission. Runtime remains the task/permission owner. use openbitfun_agent_tools::computer_use_control::{ - action_is_observation, action_requires_foreground, ControlClick, ControlMode, ControlPointer, - ControlSnapshot, + ControlClick, ControlMode, ControlPointer, ControlSnapshot, ObservationScope, }; use openbitfun_core::agentic::tools::computer_use_host::ComputerUseActionLease; use std::cell::Cell; use std::sync::{Mutex, OnceLock}; +#[path = "control_session/admission.rs"] +mod admission; +#[path = "control_session/sharing.rs"] +mod sharing; +use admission::Admission; +pub(crate) use admission::ControlToken; +pub(crate) use sharing::{ + acquire_shared, begin_share, finish_share, grant_preference_result, request_share, + set_share_control, shared_process, shared_target_allowed, +}; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Executor { Task(tokio::task::Id), @@ -18,30 +28,16 @@ fn executor() -> Executor { .unwrap_or_else(|| Executor::Thread(std::thread::current().id())) } -/// An in-process capability, never deserialized from the controller. Blocking -/// workers retain the action identity even after the async tool future is gone. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct ControlToken { - generation: u64, - sequence: u64, - input: bool, -} -impl ControlToken { - pub(crate) fn generation(self) -> u64 { - self.generation - } -} thread_local! { static TOKEN: Cell> = const { Cell::new(None) }; } #[derive(Default)] struct Resource { - snapshot: ControlSnapshot, - executor: Option, - busy: bool, - cleaning: bool, + admission: Admission, preview: Option, + process: Option, } static RESOURCE: OnceLock> = OnceLock::new(); +static NATIVE_CLEANUP: Mutex<()> = Mutex::new(()); static SCOPE_CHANGED: tokio::sync::Notify = tokio::sync::Notify::const_new(); fn resource() -> &'static Mutex { RESOURCE.get_or_init(|| Mutex::new(Resource::default())) @@ -52,24 +48,34 @@ fn locked() -> Result, String> { .map_err(|_| "[CONTROL_FAILED] Control resource lock poisoned".into()) } fn active(r: &Resource) -> bool { - r.snapshot.state == "active" && !r.cleaning + r.admission.active() } fn token_for(r: &Resource) -> ControlToken { - ControlToken { - generation: r.snapshot.generation, - sequence: r.snapshot.sequence, - input: r.busy && r.executor == Some(executor()), - } + r.admission.token(executor()) } fn matches_token(r: &Resource, token: ControlToken) -> bool { - token.generation == r.snapshot.generation && token.sequence == r.snapshot.sequence + r.admission.matches_token(token) } fn check_context(r: &Resource) -> Result<(), String> { - if !active(r) { - return Err("[CONTROL_STOPPED] Start a new control session before capturing".into()); + r.admission.check_observer(executor(), TOKEN.get())?; + if let Some(process) = &r.process { + process.validate()?; } - if TOKEN.get().is_some_and(|token| !matches_token(r, token)) { - return Err("[CONTROL_STALE] This worker belongs to an earlier action".into()); + Ok(()) +} + +/// Captured before crossing an async/native boundary; never mint a replacement +/// scope after the work completes. +pub(crate) fn observation_scope() -> Result { + let r = locked()?; + check_context(&r)?; + Ok(ObservationScope::from_control(&r.admission.snapshot)) +} +pub(crate) fn validate_observation_scope(scope: &ObservationScope) -> Result<(), String> { + let r = locked()?; + check_context(&r)?; + if *scope != ObservationScope::from_control(&r.admission.snapshot) { + return Err("[CONTROL_STALE] Observation belongs to a stopped or rebound target".into()); } Ok(()) } @@ -91,11 +97,7 @@ pub(crate) fn with_token(token: ControlToken, f: impl FnOnce() -> T) -> T { /// Drop-in replacement for host blocking operations. Stale workers never gain /// the authority of a later task just because its global busy flag is true. pub(crate) fn dispatch_token() -> ControlToken { - capture_token().unwrap_or(ControlToken { - generation: u64::MAX, - sequence: u64::MAX, - input: false, - }) + capture_token().unwrap_or(ControlToken::invalid()) } pub(crate) fn spawn_blocking(f: F) -> tokio::task::JoinHandle where @@ -130,11 +132,13 @@ async fn wait_with_scope_check( pub(crate) async fn wait(ms: u64) -> Result<(), String> { let token = capture_token()?; - let target = snapshot().target; + let scope = observation_scope()?; wait_with_scope_check(&SCOPE_CHANGED, std::time::Duration::from_millis(ms), || { let r = locked()?; check_context(&r)?; - if !matches_token(&r, token) || r.snapshot.target != target { + if !matches_token(&r, token) + || ObservationScope::from_control(&r.admission.snapshot) != scope + { return Err("[CONTROL_STALE] Control changed during wait".into()); } Ok(()) @@ -143,12 +147,16 @@ pub(crate) async fn wait(ms: u64) -> Result<(), String> { } fn projected(r: &Resource) -> ControlSnapshot { - let mut s = r.snapshot.clone(); + let mut s = r.admission.snapshot.clone(); s.supported = true; s.capabilities = vec![ "control-session-v1".into(), "stop-control".into(), "action-feedback".into(), + "observation-scope-v1".into(), + "observation-id-v1".into(), + "app-sharing-v1".into(), + "permission-requests-v1".into(), ]; // Backend routes, not a claim that every control implements them. These // facts let the agent plan before sending unsupported canvas operations. @@ -197,58 +205,28 @@ pub(crate) fn snapshot() -> ControlSnapshot { ..Default::default() }) } -fn begin_start(r: &mut Resource, owner: &str, mode: ControlMode) -> Result { - if owner.trim().is_empty() { - return Err("[CONTROL_OWNER_REQUIRED] A runtime session is required".into()); - } - if r.busy || r.cleaning || (active(r) && r.snapshot.owner.as_deref() != Some(owner)) { - return Err("[CONTROL_BUSY] Another action or session owns desktop control".into()); - } - r.cleaning = true; - r.snapshot.generation += 1; - r.snapshot.owner = Some(owner.into()); - r.snapshot.mode = mode; - r.snapshot.target = None; - r.snapshot.pointer = None; - r.snapshot.reason = None; - r.snapshot.action = None; - r.snapshot.state = "starting".into(); - r.preview = None; - Ok(r.snapshot.generation) -} -fn finish_start(r: &mut Resource, generation: u64) -> Result<(), String> { - if r.snapshot.generation != generation || r.snapshot.state != "starting" { - return Err("[CONTROL_STOPPED] Start was cancelled during native cleanup".into()); - } - r.cleaning = false; - r.snapshot.state = "active".into(); - Ok(()) -} pub(crate) fn start(owner: &str, mode: ControlMode) -> Result { - let generation = begin_start(&mut *locked()?, owner, mode)?; - stop_native(); + let generation = { + let mut r = locked()?; + let generation = r.admission.begin_start(owner, mode)?; + r.preview = None; + r.process = None; + SCOPE_CHANGED.notify_waiters(); + generation + }; + stop_native(generation)?; let mut r = locked()?; - finish_start(&mut r, generation)?; + r.admission.finish_start(generation)?; Ok(projected(&r)) } fn revoke(r: &mut Resource, reason: &str) -> u64 { SCOPE_CHANGED.notify_waiters(); - r.snapshot.generation += 1; - r.snapshot.state = "stopped".into(); - r.snapshot.reason = Some(reason.into()); - r.snapshot.pointer = None; - r.snapshot.action = None; - r.busy = false; - r.executor = None; - r.cleaning = true; r.preview = None; - r.snapshot.generation + r.process = None; + r.admission.revoke(reason) } fn finish_cleanup(r: &mut Resource, ticket: u64) { - // An older callback cannot open admission while a newer cleanup is pending. - if r.snapshot.generation == ticket && r.snapshot.state == "stopped" { - r.cleaning = false; - } + r.admission.finish_cleanup(ticket); } pub(crate) fn stop(owner: Option<&str>, reason: &str) -> Result { stop_checked(owner, None, reason) @@ -260,27 +238,24 @@ pub(crate) fn stop_checked( ) -> Result { let ticket = { let mut r = locked()?; - if generation.is_some_and(|g| r.snapshot.generation != g) { - return Err("[CONTROL_STALE] The control session has changed".into()); - } - if owner.is_some_and(|owner| { - r.snapshot - .owner - .as_deref() - .is_some_and(|current| current != owner) - }) { - return Err("[CONTROL_OWNER_MISMATCH] This task does not own desktop control".into()); - } - revoke(&mut r, reason) + let Some(ticket) = r.admission.request_stop(owner, generation, reason)? else { + return Ok(projected(&r)); + }; + r.preview = None; + r.process = None; + SCOPE_CHANGED.notify_waiters(); + ticket }; - stop_native(); + stop_native(ticket)?; let mut r = locked()?; finish_cleanup(&mut r, ticket); Ok(projected(&r)) } fn cleanup_async(ticket: u64) { std::thread::spawn(move || { - stop_native(); + if stop_native(ticket).is_err() { + return; + } if let Ok(mut r) = locked() { finish_cleanup(&mut r, ticket); } @@ -290,29 +265,36 @@ fn cleanup_async(ticket: u64) { /// Never call an unscoped "stop current" from a delayed native callback. #[cfg(target_os = "linux")] pub(crate) fn native_stopped_generation(generation: u64, reason: &str) { - native_stopped_inner(generation, None, reason); + native_stopped_inner(generation, None, None, reason); } -pub(crate) fn native_stopped_target(generation: u64, target: &str, reason: &str) { - native_stopped_inner(generation, Some(target), reason); +pub(crate) fn native_stopped_target(generation: u64, revision: u64, target: &str, reason: &str) { + native_stopped_inner(generation, Some(revision), Some(target), reason); } fn revoke_native( r: &mut Resource, generation: u64, + revision: Option, target: Option<&str>, reason: &str, ) -> Option { - if r.snapshot.generation != generation + if r.admission.snapshot.generation != generation || !active(r) - || target.is_some_and(|target| r.snapshot.target.as_deref() != Some(target)) + || revision.is_some_and(|revision| r.admission.snapshot.target_revision != revision) + || target.is_some_and(|target| r.admission.snapshot.target.as_deref() != Some(target)) { return None; } Some(revoke(r, reason)) } -fn native_stopped_inner(generation: u64, target: Option<&str>, reason: &str) { +fn native_stopped_inner( + generation: u64, + revision: Option, + target: Option<&str>, + reason: &str, +) { let ticket = locked() .ok() - .and_then(|mut r| revoke_native(&mut r, generation, target, reason)); + .and_then(|mut r| revoke_native(&mut r, generation, revision, target, reason)); if let Some(ticket) = ticket { cleanup_async(ticket); } @@ -320,21 +302,25 @@ fn native_stopped_inner(generation: u64, target: Option<&str>, reason: &str) { pub(crate) fn capture_allowed() -> Result<(), String> { check_context(&*locked()?) } +pub(crate) fn record_background_observation( + scope: &ObservationScope, + evidence: openbitfun_agent_tools::computer_use_control::BackgroundObservation, +) { + if let Ok(mut r) = locked() { + if active(&r) && ObservationScope::from_control(&r.admission.snapshot) == *scope { + r.admission.snapshot.background_observation = Some(evidence); + } + } +} pub(crate) fn input_allowed() -> Result<(), String> { let r = locked()?; check_context(&r)?; - let token = TOKEN.get().unwrap_or_else(|| token_for(&r)); - if !r.busy || !token.input || !matches_token(&r, token) { - return Err("[CONTROL_STOPPED] Input lease has been revoked".into()); - } - if r.snapshot.mode == ControlMode::Observe { - return Err("[CONTROL_OBSERVE_ONLY] Observation does not authorize input".into()); - } - Ok(()) + r.admission + .check_input(TOKEN.get().unwrap_or_else(|| token_for(&r))) } pub(crate) fn foreground_allowed() -> Result<(), String> { input_allowed()?; - if locked()?.snapshot.mode != ControlMode::Foreground { + if locked()?.admission.snapshot.mode != ControlMode::Foreground { return Err( "[FOREGROUND_REQUIRED] Seat input requires start_control with mode=foreground".into(), ); @@ -344,16 +330,11 @@ pub(crate) fn foreground_allowed() -> Result<(), String> { pub(crate) fn bind_target(target: String) -> Result<(), String> { let mut r = locked()?; check_context(&r)?; - if target.trim().is_empty() { - return Err("[CONTROL_TARGET_REQUIRED] A native target is required".into()); - } - let changed = r.snapshot.target.as_deref() != Some(&target); + let changed = r.admission.bind(target)?; if changed { SCOPE_CHANGED.notify_waiters(); r.preview = None; - r.snapshot.pointer = None; } - r.snapshot.target = Some(target); drop(r); if changed { #[cfg(target_os = "macos")] @@ -369,7 +350,7 @@ pub(crate) fn bind_target(target: String) -> Result<(), String> { pub(crate) fn target_allowed(target: &str) -> Result<(), String> { let r = locked()?; check_context(&r)?; - if r.snapshot.target.as_deref() != Some(target) { + if r.admission.snapshot.target.as_deref() != Some(target) { return Err( "[CONTROL_TARGET_CHANGED] The input target differs from the captured window".into(), ); @@ -381,7 +362,7 @@ pub(crate) fn record_pointer(x: f64, y: f64, click: bool) { return; } let target = if let Ok(mut r) = locked() { - if check_context(&r).is_err() || !r.busy { + if check_context(&r).is_err() || !r.admission.busy { return; } let occurred_at_ms = std::time::SystemTime::now() @@ -389,6 +370,7 @@ pub(crate) fn record_pointer(x: f64, y: f64, click: bool) { .unwrap_or_default() .as_millis() as u64; let sequence = r + .admission .snapshot .pointer .as_ref() @@ -401,12 +383,13 @@ pub(crate) fn record_pointer(x: f64, y: f64, click: bool) { occurred_at_ms, }) } else { - r.snapshot + r.admission + .snapshot .pointer .as_ref() .and_then(|p| p.last_click.clone()) }; - r.snapshot.pointer = Some(ControlPointer { + r.admission.snapshot.pointer = Some(ControlPointer { x, y, click, @@ -414,7 +397,7 @@ pub(crate) fn record_pointer(x: f64, y: f64, click: bool) { occurred_at_ms, last_click, }); - r.snapshot.target.clone() + r.admission.snapshot.target.clone() } else { return; }; @@ -439,48 +422,11 @@ pub(crate) fn record_pointer(x: f64, y: f64, click: bool) { #[cfg(target_os = "linux")] let _ = target; } -fn admit( - r: &mut Resource, - owner: &str, - action: &str, - execution: Executor, -) -> Result { - if owner.trim().is_empty() { - return Err("[CONTROL_OWNER_REQUIRED] A runtime session is required".into()); - } - if r.snapshot.state.is_empty() || r.snapshot.state == "idle" { - r.snapshot.owner = Some(owner.into()); - r.snapshot.mode = ControlMode::Background; - r.snapshot.state = "active".into(); - r.snapshot.generation += 1; - } - if !active(r) { - return Err("[CONTROL_STOPPED] Use start_control to begin a new session".into()); - } - if r.snapshot.owner.as_deref() != Some(owner) || r.busy { - return Err("[CONTROL_BUSY] Desktop control belongs to another task or action".into()); - } - if r.snapshot.mode == ControlMode::Observe && !action_is_observation(action) { - return Err("[CONTROL_OBSERVE_ONLY] Start a control session to send input".into()); - } - if action_requires_foreground(action) && r.snapshot.mode != ControlMode::Foreground { - return Err("[FOREGROUND_REQUIRED] This action uses desktop focus or seat input; explicitly start_control with mode=foreground".into()); - } - r.busy = true; - r.executor = Some(execution); - r.snapshot.sequence += 1; - r.snapshot.action = Some(action.into()); - Ok(ControlToken { - generation: r.snapshot.generation, - sequence: r.snapshot.sequence, - input: true, - }) -} pub(crate) fn acquire( owner: &str, action: &str, ) -> Result, String> { - let token = admit(&mut *locked()?, owner, action, executor())?; + let token = locked()?.admission.admit(owner, action, executor())?; Ok(Box::new(Lease { token, complete: false, @@ -495,45 +441,51 @@ impl ComputerUseActionLease for Lease { self.complete = true; } } -fn release(r: &mut Resource, token: ControlToken, complete: bool) -> Option { - if !matches_token(r, token) { - return None; - } - r.busy = false; - r.executor = None; - if complete { - r.snapshot.action = None; - None - } else { - Some(revoke(r, "action_cancelled")) - } -} impl Drop for Lease { fn drop(&mut self) { - let ticket = locked() - .ok() - .and_then(|mut r| release(&mut r, self.token, self.complete)); + let ticket = locked().ok().and_then(|mut r| { + let ticket = r.admission.release(self.token, self.complete); + if ticket.is_some() { + r.preview = None; + SCOPE_CHANGED.notify_waiters(); + } + ticket + }); if let Some(ticket) = ticket { cleanup_async(ticket); } } } -fn stop_native() { +fn stop_native(ticket: u64) -> Result<(), String> { + // A cancelled Start and Stop can both enqueue cleanup. Serialize native + // teardown and recheck the ticket after waiting, so delayed old cleanup + // cannot stop resources created by a later successful Start. + let _cleanup = NATIVE_CLEANUP + .lock() + .map_err(|_| "[CONTROL_FAILED] Native cleanup lock poisoned".to_string())?; + if !locked()?.admission.owns_cleanup(ticket) { + return Ok(()); + } #[cfg(target_os = "macos")] { + super::macos_bg_input::release_held_inputs(); super::macos_capture::stop_capture(); + super::macos_ax_dump::clear_observations(); super::macos_input_focus::stop(); super::macos_capture::hide_pointer(); } #[cfg(target_os = "windows")] { super::windows_wgc_capture::stop_capture(); + super::windows_ax_ui::clear_observations(); super::windows_pointer_feedback::hide_pointer(); } #[cfg(target_os = "linux")] { super::linux_control::stop_session(); + super::linux_control_ax::clear_observations(); } + Ok(()) } #[derive(Clone, serde::Serialize)] @@ -548,13 +500,13 @@ pub struct ControlPreview { pub span_width: f64, pub span_height: f64, pub generation: u64, + pub target_revision: u64, } /// Capture callers pass the identity saved before native work, never retag a /// late frame with snapshot() after a new task has acquired the device. -pub(crate) fn publish_rgba_generation( - generation: u64, - target: &str, +pub(crate) fn publish_rgba( + scope: &ObservationScope, bytes: &[u8], width: u32, height: u32, @@ -571,38 +523,41 @@ pub(crate) fn publish_rgba_generation( .encode_image(&image) .is_ok() { - publish_jpeg_generation(generation, target, &jpeg, bounds); + publish_jpeg(scope, &jpeg, [image.width(), image.height()], bounds); } } -pub(crate) fn publish_jpeg_generation( - generation: u64, - target: &str, +pub(crate) fn publish_jpeg( + scope: &ObservationScope, bytes: &[u8], + dimensions: [u32; 2], bounds: [f64; 4], ) { use base64::Engine; if bounds.iter().any(|v| !v.is_finite()) || bounds[2] <= 0.0 || bounds[3] <= 0.0 { return; } - let Ok(image) = image::load_from_memory(bytes) else { + let Some(target) = scope.target.as_ref() else { return; }; + if dimensions.contains(&0) { + return; + } let frame = ControlPreview { - target: target.to_owned(), + target: target.clone(), image_base64: base64::engine::general_purpose::STANDARD.encode(bytes), mime_type: "image/jpeg", - width: image.width(), - height: image.height(), + width: dimensions[0], + height: dimensions[1], origin_x: bounds[0], origin_y: bounds[1], span_width: bounds[2], span_height: bounds[3], - generation, + generation: scope.generation, + target_revision: scope.target_revision, }; if let Ok(mut r) = locked() { if check_context(&r).is_ok() - && r.snapshot.generation == generation - && r.snapshot.target.as_deref() == Some(target) + && ObservationScope::from_control(&r.admission.snapshot) == *scope { r.preview = Some(frame); } @@ -610,93 +565,60 @@ pub(crate) fn publish_jpeg_generation( } pub(crate) fn preview(generation: u64) -> Result, String> { let r = locked()?; - if r.snapshot.generation != generation || !active(&r) { + if r.admission.snapshot.generation != generation || !active(&r) { return Err("[CONTROL_STALE] Control session changed".into()); } Ok(r.preview .as_ref() - .filter(|p| p.generation == generation) + .filter(|p| { + p.generation == generation && p.target_revision == r.admission.snapshot.target_revision + }) .cloned()) } #[cfg(test)] mod tests { use super::*; - fn running() -> Resource { - let mut r = Resource::default(); - let g = begin_start(&mut r, "a", ControlMode::Background).unwrap(); - finish_start(&mut r, g).unwrap(); - r - } #[test] - fn stop_during_start_cannot_resurrect_admission() { + fn old_native_stream_cannot_stop_a_return_to_the_same_window() { let mut r = Resource::default(); - let started = begin_start(&mut r, "a", ControlMode::Background).unwrap(); - let stopped = revoke(&mut r, "user_stopped"); - assert!(finish_start(&mut r, started).is_err()); - assert!(!active(&r)); - finish_cleanup(&mut r, stopped); - assert!(!active(&r)); - } - #[test] - fn old_cleanup_cannot_clear_new_cleanup_barrier() { - let mut r = running(); - let first = revoke(&mut r, "first"); - let second = revoke(&mut r, "second"); - finish_cleanup(&mut r, first); - assert!(r.cleaning); - assert!(begin_start(&mut r, "a", ControlMode::Background).is_err()); - finish_cleanup(&mut r, second); - assert!(!r.cleaning); - } - #[test] - fn old_worker_and_old_lease_cannot_borrow_new_action() { - let mut r = running(); - let first = admit(&mut r, "a", "app_click", executor()).unwrap(); - let stopped = revoke(&mut r, "cancelled"); - finish_cleanup(&mut r, stopped); - let g = begin_start(&mut r, "a", ControlMode::Background).unwrap(); - finish_start(&mut r, g).unwrap(); - let second = admit(&mut r, "a", "app_click", executor()).unwrap(); - assert!(!matches_token(&r, first)); - assert!(matches_token(&r, second)); - with_token(first, || assert!(check_context(&r).is_err())); - assert!(release(&mut r, first, false).is_none()); - assert!(r.busy); - } - #[test] - fn successive_actions_in_same_session_have_distinct_authority() { - let mut r = running(); - let first = admit(&mut r, "a", "app_click", executor()).unwrap(); - release(&mut r, first, true); - let second = admit(&mut r, "a", "app_click", executor()).unwrap(); - assert_eq!(first.generation, second.generation); - assert_ne!(first.sequence, second.sequence); - with_token(first, || assert!(check_context(&r).is_err())); + let generation = r + .admission + .begin_start("a", ControlMode::Background) + .unwrap(); + r.admission.finish_start(generation).unwrap(); + r.admission.bind("window:a".into()).unwrap(); + r.admission.bind("window:b".into()).unwrap(); + r.admission.bind("window:a".into()).unwrap(); + assert!( + revoke_native(&mut r, generation, Some(1), Some("window:a"), "late close").is_none() + ); + assert!(active(&r)); + assert!(revoke_native( + &mut r, + generation, + Some(3), + Some("window:a"), + "current close" + ) + .is_some()); } #[test] fn native_stop_from_old_stream_cannot_revoke_retargeted_window() { - let mut r = running(); - let generation = r.snapshot.generation; - r.snapshot.target = Some("window:new".into()); - assert!(revoke_native(&mut r, generation, Some("window:old"), "closed").is_none()); + let mut r = Resource::default(); + let generation = r + .admission + .begin_start("a", ControlMode::Background) + .unwrap(); + r.admission.finish_start(generation).unwrap(); + r.admission.bind("window:new".into()).unwrap(); + assert!(revoke_native(&mut r, generation, Some(1), Some("window:old"), "closed").is_none()); assert!(active(&r)); - let ticket = revoke_native(&mut r, generation, Some("window:new"), "closed").unwrap(); - assert!(r.cleaning); + let ticket = + revoke_native(&mut r, generation, Some(1), Some("window:new"), "closed").unwrap(); assert!(!active(&r)); finish_cleanup(&mut r, ticket); } - #[test] - fn observer_foreground_and_owner_boundaries_are_enforced() { - let mut r = running(); - r.snapshot.mode = ControlMode::Observe; - assert!(admit(&mut r, "a", "app_click", executor()).is_err()); - assert!(admit(&mut r, "b", "get_app_state", executor()).is_err()); - r.snapshot.mode = ControlMode::Background; - assert!(admit(&mut r, "a", "click", executor()).is_err()); - assert!(admit(&mut r, "", "get_app_state", executor()).is_err()); - assert!(!r.busy); - } } #[cfg(test)] diff --git a/src/apps/desktop/src/computer_use/control_session/admission.rs b/src/apps/desktop/src/computer_use/control_session/admission.rs new file mode 100644 index 0000000000..a7a890ba9c --- /dev/null +++ b/src/apps/desktop/src/computer_use/control_session/admission.rs @@ -0,0 +1,673 @@ +//! Deterministic resource admission. No OS handles, threads, timers or IO. +//! Native cleanup is performed by the desktop facade using the returned epoch. + +use openbitfun_agent_tools::computer_use_control::{ + ControlActionPolicy, ControlMode, ControlSharing, ControlSnapshot, SharingPhase, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ControlToken { + pub(super) generation: u64, + pub(super) sequence: u64, + pub(super) input: bool, + authorization_revision: u64, + leased: bool, +} + +impl ControlToken { + pub(crate) fn generation(self) -> u64 { + self.generation + } + + pub(super) const fn invalid() -> Self { + Self { + generation: u64::MAX, + sequence: u64::MAX, + input: false, + authorization_revision: u64::MAX, + leased: true, + } + } +} + +pub(super) struct Admission { + pub(super) snapshot: ControlSnapshot, + pub(super) executor: Option, + pub(super) busy: bool, + cleaning: bool, + last_revoked_generation: Option, +} + +impl Default for Admission { + fn default() -> Self { + Self { + snapshot: ControlSnapshot::default(), + executor: None, + busy: false, + cleaning: false, + last_revoked_generation: None, + } + } +} + +impl Admission { + pub(super) fn active(&self) -> bool { + self.snapshot.state == "active" && !self.cleaning + } + + pub(super) fn token(&self, executor: E) -> ControlToken { + ControlToken { + generation: self.snapshot.generation, + sequence: self.snapshot.sequence, + input: self.busy + && self.executor == Some(executor) + && self + .snapshot + .action + .as_deref() + .and_then(ControlActionPolicy::for_action) + .is_some_and(|policy| !policy.observation), + leased: self.busy, + authorization_revision: self.authorization_revision(), + } + } + + pub(super) fn matches_token(&self, token: ControlToken) -> bool { + token.generation == self.snapshot.generation + && token.sequence == self.snapshot.sequence + && (!token.leased || self.busy) + && token.authorization_revision == self.authorization_revision() + } + + fn authorization_revision(&self) -> u64 { + self.snapshot + .sharing + .as_ref() + .map_or(0, |sharing| sharing.authorization_revision) + } + + /// A tool can request sharing, but cannot select a window or grant input. + pub(super) fn request_share(&mut self, owner: &str, mode: ControlMode) -> Result<(), String> { + if self.snapshot.owner.as_deref() != Some(owner) || self.busy || self.cleaning { + return Err("[CONTROL_BUSY] Another task or action owns desktop control".into()); + } + self.snapshot.mode = ControlMode::Observe; + self.snapshot.state = "awaiting_share".into(); + self.snapshot.sharing = Some(ControlSharing { + phase: SharingPhase::Selecting, + requested_mode: mode, + target_label: None, + control_enabled: false, + authorization_revision: 0, + remembered: false, + grant_note: None, + }); + Ok(()) + } + + pub(super) fn begin_share( + &mut self, + generation: u64, + target: String, + label: String, + mode: ControlMode, + ) -> Result<(), String> { + if self.snapshot.generation != generation + || self.busy + || self.cleaning + || self + .snapshot + .sharing + .as_ref() + .is_none_or(|sharing| sharing.phase != SharingPhase::Selecting) + { + return Err( + "[CONTROL_STALE] Start a new share selection before selecting a target".into(), + ); + } + self.bind(target)?; + let sharing = self + .snapshot + .sharing + .as_mut() + .ok_or("[SHARE_REQUIRED] No share request")?; + sharing.phase = SharingPhase::Starting; + sharing.requested_mode = mode; + sharing.target_label = Some(label); + self.snapshot.mode = ControlMode::Observe; + self.snapshot.state = "active".into(); + Ok(()) + } + + pub(super) fn finish_share(&mut self, generation: u64, revision: u64) -> Result<(), String> { + if self.snapshot.generation != generation + || self.snapshot.target_revision != revision + || !self.active() + { + return Err("[CONTROL_STALE] Share was stopped during native capture".into()); + } + let sharing = self + .snapshot + .sharing + .as_mut() + .ok_or("[SHARE_REQUIRED] No share request")?; + if sharing.phase != SharingPhase::Starting { + return Err("[CONTROL_STALE] Native share completion is no longer current".into()); + } + sharing.phase = SharingPhase::Shared; + Ok(()) + } + + pub(super) fn set_control( + &mut self, + generation: u64, + target_revision: u64, + expected_revision: u64, + enabled: bool, + ) -> Result<(), String> { + if self.snapshot.generation != generation + || self.snapshot.target_revision != target_revision + || !self.active() + { + return Err("[CONTROL_STALE] Share changed before authorization".into()); + } + if enabled && self.busy { + return Err( + "[CONTROL_BUSY] Wait for the current action before granting control".into(), + ); + } + let sharing = self + .snapshot + .sharing + .as_mut() + .ok_or("[SHARE_REQUIRED] This host has no user-selected share")?; + if sharing.phase != SharingPhase::Shared + || sharing.authorization_revision != expected_revision + { + return Err("[CONTROL_STALE] Authorization request is stale".into()); + } + if enabled && sharing.requested_mode == ControlMode::Observe { + return Err("[CONTROL_OBSERVE_ONLY] The user selected observation only".into()); + } + sharing.authorization_revision = sharing + .authorization_revision + .checked_add(1) + .ok_or("[CONTROL_FAILED] Authorization revision exhausted")?; + sharing.control_enabled = enabled; + self.snapshot.mode = if enabled { + sharing.requested_mode + } else { + ControlMode::Observe + }; + self.snapshot.pointer = None; + Ok(()) + } + + pub(super) fn shared_target_allowed(&self, target: &str) -> Result<(), String> { + if self.snapshot.sharing.is_some() && self.snapshot.target.as_deref() != Some(target) { + return Err( + "[SHARED_TARGET_ONLY] This operation does not address the user's shared window" + .into(), + ); + } + Ok(()) + } + + pub(super) fn check(&self, token: Option) -> Result<(), String> { + if !self.active() { + return Err("[CONTROL_STOPPED] Start a new control session before capturing".into()); + } + if token.is_some_and(|token| !self.matches_token(token)) { + return Err("[CONTROL_STALE] This worker belongs to an earlier action".into()); + } + Ok(()) + } + + pub(super) fn check_observer( + &self, + execution: E, + token: Option, + ) -> Result<(), String> { + self.check(token)?; + if token.is_none() && self.busy && self.executor != Some(execution) { + return Err("[CONTROL_BUSY] Observation belongs to another action; native workers must carry the original lease".into()); + } + Ok(()) + } + + pub(super) fn owns_cleanup(&self, ticket: u64) -> bool { + self.cleaning && self.snapshot.generation == ticket + } + + pub(super) fn check_input(&self, token: ControlToken) -> Result<(), String> { + self.check(Some(token))?; + if !self.busy || !token.input { + return Err("[CONTROL_OBSERVE_ONLY] This action has no input lease".into()); + } + if self.snapshot.mode == ControlMode::Observe { + return Err("[CONTROL_OBSERVE_ONLY] Observation does not authorize input".into()); + } + if self.snapshot.sharing.as_ref().is_some_and(|sharing| { + !sharing.control_enabled || sharing.phase != SharingPhase::Shared + }) { + return Err( + "[CONTROL_NOT_GRANTED] The user has not authorized input for this share".into(), + ); + } + Ok(()) + } + + pub(super) fn begin_start(&mut self, owner: &str, mode: ControlMode) -> Result { + if owner.trim().is_empty() { + return Err("[CONTROL_OWNER_REQUIRED] A runtime session is required".into()); + } + if self.busy + || self.cleaning + || ((self.active() || self.snapshot.state == "awaiting_share") + && self.snapshot.owner.as_deref() != Some(owner)) + { + return Err("[CONTROL_BUSY] Another action or session owns desktop control".into()); + } + let generation = self + .snapshot + .generation + .checked_add(1) + .ok_or("[CONTROL_FAILED] Control generation exhausted")?; + self.cleaning = true; + self.last_revoked_generation = None; + self.snapshot.generation = generation; + self.snapshot.target_revision = 0; + self.snapshot.owner = Some(owner.into()); + self.snapshot.mode = mode; + self.snapshot.target = None; + self.snapshot.pointer = None; + self.snapshot.reason = None; + self.snapshot.action = None; + self.snapshot.sharing = None; + self.snapshot.background_observation = None; + self.snapshot.state = "starting".into(); + Ok(generation) + } + + pub(super) fn finish_start(&mut self, generation: u64) -> Result<(), String> { + if self.snapshot.generation != generation || self.snapshot.state != "starting" { + return Err("[CONTROL_STOPPED] Start was cancelled during native cleanup".into()); + } + self.cleaning = false; + self.snapshot.state = "active".into(); + Ok(()) + } + + /// Retried Stop requests acknowledge the original revocation, but can never + /// stop a subsequently started session, even if its owner is unchanged. + pub(super) fn request_stop( + &mut self, + owner: Option<&str>, + generation: Option, + reason: &str, + ) -> Result, String> { + if generation.is_some_and(|generation| { + generation != self.snapshot.generation + && !(self.snapshot.state == "stopped" + && self.last_revoked_generation == Some(generation)) + }) { + return Err("[CONTROL_STALE] The control session has changed".into()); + } + if owner.is_some_and(|owner| { + self.snapshot + .owner + .as_deref() + .is_some_and(|current| current != owner) + }) { + return Err("[CONTROL_OWNER_MISMATCH] This task does not own desktop control".into()); + } + if self.snapshot.state == "stopped" + || self.snapshot.state == "idle" + || self.snapshot.state.is_empty() + { + return Ok(None); + } + Ok(Some(self.revoke(reason))) + } + + pub(super) fn revoke(&mut self, reason: &str) -> u64 { + // Saturation is safe only for revocation: admission uses checked_add + // and can never restart an exhausted generation. + self.last_revoked_generation = Some(self.snapshot.generation); + self.snapshot.generation = self.snapshot.generation.saturating_add(1); + self.snapshot.state = "stopped".into(); + self.snapshot.reason = Some(reason.into()); + self.snapshot.pointer = None; + self.snapshot.action = None; + if let Some(sharing) = self.snapshot.sharing.as_mut() { + sharing.control_enabled = false; + } + self.busy = false; + self.executor = None; + self.cleaning = true; + self.snapshot.generation + } + + pub(super) fn finish_cleanup(&mut self, ticket: u64) { + if self.snapshot.generation == ticket && self.snapshot.state == "stopped" { + self.cleaning = false; + } + } + + pub(super) fn bind(&mut self, target: String) -> Result { + if target.trim().is_empty() { + return Err("[CONTROL_TARGET_REQUIRED] A native target is required".into()); + } + if self.snapshot.target.as_ref() == Some(&target) { + return Ok(false); + } + let portal_selection = self.snapshot.target.as_deref() == Some("portal:pending") + && target.starts_with("portal:/") + && self + .snapshot + .sharing + .as_ref() + .is_some_and(|sharing| sharing.phase == SharingPhase::Starting); + if !portal_selection + && self + .snapshot + .sharing + .as_ref() + .is_some_and(|sharing| sharing.phase != SharingPhase::Selecting) + { + self.shared_target_allowed(&target)?; + } + self.snapshot.target_revision = self + .snapshot + .target_revision + .checked_add(1) + .ok_or("[CONTROL_FAILED] Target revision exhausted")?; + self.snapshot.target = Some(target); + self.snapshot.pointer = None; + Ok(true) + } + + pub(super) fn admit( + &mut self, + owner: &str, + action: &str, + execution: E, + ) -> Result { + let policy = ControlActionPolicy::for_action(action) + .filter(|policy| policy.lease) + .ok_or_else(|| format!("[CONTROL_ACTION_INVALID] {action} is not a resource action"))?; + if owner.trim().is_empty() { + return Err("[CONTROL_OWNER_REQUIRED] A runtime session is required".into()); + } + let initial = self.snapshot.state.is_empty() || self.snapshot.state == "idle"; + if self.snapshot.state == "awaiting_share" { + return Err("[SHARE_REQUIRED] Select an App in the Computer Use sharing panel on the executing Desktop or its connected peer controller. Mobile/IM/headless surfaces cannot select a native window; this call is not waiting for a hidden dialog.".into()); + } + if !initial && !self.active() { + return Err("[CONTROL_STOPPED] Use start_control to begin a new session".into()); + } + if self.busy || (!initial && self.snapshot.owner.as_deref() != Some(owner)) { + return Err("[CONTROL_BUSY] Desktop control belongs to another task or action".into()); + } + let mode = if initial { + ControlMode::Background + } else { + self.snapshot.mode + }; + if self.snapshot.sharing.is_some() + && (policy.foreground && !policy.target || matches!(action, "clipboard_set")) + { + return Err("[SHARED_TARGET_ONLY] Window sharing does not authorize global clipboard, launching or scripts".into()); + } + if mode == ControlMode::Observe && !policy.observation { + return Err("[CONTROL_OBSERVE_ONLY] Start a control session to send input".into()); + } + if policy.foreground && mode != ControlMode::Foreground { + return Err("[FOREGROUND_REQUIRED] This action uses desktop focus or seat input; explicitly start_control with mode=foreground".into()); + } + let sequence = self + .snapshot + .sequence + .checked_add(1) + .ok_or("[CONTROL_FAILED] Action sequence exhausted")?; + if initial { + self.snapshot.generation = self + .snapshot + .generation + .checked_add(1) + .ok_or("[CONTROL_FAILED] Control generation exhausted")?; + self.snapshot.owner = Some(owner.into()); + self.snapshot.mode = mode; + self.snapshot.state = "active".into(); + } + self.busy = true; + self.executor = Some(execution); + self.snapshot.sequence = sequence; + self.snapshot.action = Some(action.into()); + Ok(self.token(execution)) + } + + pub(super) fn release(&mut self, token: ControlToken, complete: bool) -> Option { + // A control revocation invalidates input immediately, but the old lease + // still releases its own busy slot. Never release a later action. + if token.generation != self.snapshot.generation + || token.sequence != self.snapshot.sequence + || !self.busy + { + return None; + } + self.busy = false; + self.executor = None; + if complete { + self.snapshot.action = None; + None + } else { + Some(self.revoke("action_cancelled")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn sharing_requires_selection_capture_and_a_separate_revocable_grant() { + let mut state = running(); + let generation = state.snapshot.generation; + state + .request_share("owner", ControlMode::Background) + .unwrap(); + assert_eq!(state.snapshot.mode, ControlMode::Observe); + assert!(state + .admit("owner", "app_click", 1) + .unwrap_err() + .contains("SHARE_REQUIRED")); + assert!(state.begin_start("other", ControlMode::Background).is_err()); + state + .begin_share( + generation, + "pid:1/window:2".into(), + "App".into(), + ControlMode::Background, + ) + .unwrap(); + let revision = state.snapshot.target_revision; + assert!(state.set_control(generation, revision, 0, true).is_err()); + state.finish_share(generation, revision).unwrap(); + assert!(state.admit("owner", "app_click", 1).is_err()); + let read = state.admit("owner", "get_app_state", 1).unwrap(); + assert!(state.set_control(generation, revision, 0, true).is_err()); + state.release(read, true); + state.set_control(generation, revision, 0, true).unwrap(); + let input = state.admit("owner", "app_drag", 1).unwrap(); + assert!(state.check_input(input).is_ok()); + state.set_control(generation, revision, 1, false).unwrap(); + assert!(state.check_input(input).is_err()); + assert!(state.release(input, true).is_none()); + assert!(!state.busy); + assert!(state.set_control(generation, revision, 1, true).is_err()); + assert!(state.bind("pid:1/window:3".into()).is_err()); + assert_eq!(state.snapshot.target.as_deref(), Some("pid:1/window:2")); + } + + #[test] + fn stopped_sharing_and_observe_only_selection_cannot_be_granted_by_late_completion() { + let mut state = running(); + let generation = state.snapshot.generation; + state.request_share("owner", ControlMode::Observe).unwrap(); + state + .begin_share( + generation, + "window:a".into(), + "App".into(), + ControlMode::Observe, + ) + .unwrap(); + state.finish_share(generation, 1).unwrap(); + assert!(state.set_control(generation, 1, 0, true).is_err()); + state.revoke("user_stopped"); + assert!(state.finish_share(generation, 1).is_err()); + assert!(state.set_control(generation, 1, 0, true).is_err()); + } + + #[test] + fn shared_foreground_mode_does_not_authorize_global_scripts_or_clipboard() { + let mut state = running(); + let generation = state.snapshot.generation; + state + .request_share("owner", ControlMode::Foreground) + .unwrap(); + state + .begin_share( + generation, + "window:a".into(), + "App".into(), + ControlMode::Foreground, + ) + .unwrap(); + state.finish_share(generation, 1).unwrap(); + state.set_control(generation, 1, 0, true).unwrap(); + for action in ["clipboard_set", "open_app", "run_jxa"] { + assert!(state.admit("owner", action, 1).is_err(), "{action}"); + assert!(!state.busy); + } + } + fn running() -> Admission { + let mut state = Admission::default(); + let generation = state.begin_start("owner", ControlMode::Background).unwrap(); + state.finish_start(generation).unwrap(); + state + } + + #[test] + fn retried_stop_is_idempotent_but_cannot_revoke_a_restart() { + let mut resource = Admission::::default(); + let generation = resource + .begin_start("task", ControlMode::Background) + .unwrap(); + resource.finish_start(generation).unwrap(); + let ticket = resource + .request_stop(Some("task"), Some(generation), "stop") + .unwrap() + .unwrap(); + assert_eq!( + resource + .request_stop(Some("task"), Some(generation), "retry") + .unwrap(), + None + ); + resource.finish_cleanup(ticket); + assert_eq!( + resource.request_stop(Some("task"), None, "retry").unwrap(), + None + ); + let restarted = resource + .begin_start("task", ControlMode::Background) + .unwrap(); + resource.finish_start(restarted).unwrap(); + assert!(resource + .request_stop(Some("task"), Some(generation), "late retry") + .is_err()); + assert!(resource.active()); + } + + #[test] + fn stop_during_start_cannot_resurrect_admission() { + let mut state = Admission::::default(); + let start = state.begin_start("owner", ControlMode::Background).unwrap(); + let stop = state.revoke("stopped"); + assert!(state.finish_start(start).is_err()); + state.finish_cleanup(stop); + assert!(!state.active()); + } + + #[test] + fn stale_cleanup_and_lease_cannot_change_a_new_owner() { + let mut state = running(); + let old = state.admit("owner", "app_click", 1).unwrap(); + let first = state.revoke("first"); + let second = state.revoke("second"); + assert!(!state.owns_cleanup(first)); + assert!(state.owns_cleanup(second)); + state.finish_cleanup(first); + assert!(state.begin_start("new", ControlMode::Background).is_err()); + state.finish_cleanup(second); + let generation = state.begin_start("new", ControlMode::Background).unwrap(); + state.finish_start(generation).unwrap(); + let new = state.admit("new", "app_click", 2).unwrap(); + assert!(state.check(Some(old)).is_err()); + assert!(state.release(old, false).is_none()); + assert!(state.check_input(new).is_ok()); + } + + #[test] + fn read_actions_and_foreign_executors_never_receive_input_authority() { + let mut state = running(); + let read = state.admit("owner", "get_app_state", 1).unwrap(); + assert!(state.check(Some(read)).is_ok()); + assert!(state.check_observer(1, None).is_ok()); + assert!(state.check_observer(2, None).is_err()); + assert!(state.check_observer(2, Some(read)).is_ok()); + assert!(state.check_input(read).is_err()); + state.release(read, true); + let input = state.admit("owner", "app_click", 1).unwrap(); + assert!(state.check_input(input).is_ok()); + assert!(state.check_input(state.token(2)).is_err()); + assert!(state.check(Some(read)).is_err()); + assert!(state.check_observer(2, Some(read)).is_err()); + state.release(input, true); + assert!(state.check(Some(input)).is_err()); + } + + #[test] + fn rejected_actions_do_not_create_or_take_over_sessions() { + let mut state = Admission::::default(); + for action in ["unrecognized", "click", "start_control"] { + assert!(state.admit("owner", action, 1).is_err()); + assert!(state.snapshot.owner.is_none()); + } + let mut state = running(); + state.snapshot.mode = ControlMode::Observe; + assert!(state.admit("owner", "app_click", 1).is_err()); + assert!(state.admit("other", "get_app_state", 1).is_err()); + assert!(state.admit("", "get_app_state", 1).is_err()); + assert!(!state.busy); + } + + #[test] + fn target_rebinding_and_exhausted_ids_cannot_alias_old_observations() { + let mut state = running(); + assert!(state.bind("a".into()).unwrap()); + assert!(!state.bind("a".into()).unwrap()); + assert!(state.bind("b".into()).unwrap()); + assert!(state.bind("a".into()).unwrap()); + assert_eq!(state.snapshot.target_revision, 3); + state.snapshot.sequence = u64::MAX; + assert!(state.admit("owner", "app_click", 1).is_err()); + state.snapshot.generation = u64::MAX; + let stop = state.revoke("stopped"); + state.finish_cleanup(stop); + assert!(state.begin_start("owner", ControlMode::Background).is_err()); + assert!(!state.active()); + } +} diff --git a/src/apps/desktop/src/computer_use/control_session/sharing.rs b/src/apps/desktop/src/computer_use/control_session/sharing.rs new file mode 100644 index 0000000000..ea0ede1602 --- /dev/null +++ b/src/apps/desktop/src/computer_use/control_session/sharing.rs @@ -0,0 +1,204 @@ +//! User-owned sharing transitions around the existing native resource owner. +use super::*; + +pub(crate) fn request_share(owner: &str, mode: ControlMode) -> Result { + { + let r = locked()?; + if r.admission.snapshot.owner.as_deref() != Some(owner) + && (active(&r) || r.admission.snapshot.state == "awaiting_share") + { + return Err("[CONTROL_BUSY] Another task owns the share request".into()); + } + if let Some(sharing) = &r.admission.snapshot.sharing { + if r.admission.snapshot.owner.as_deref() == Some(owner) + && (active(&r) || r.admission.snapshot.state == "awaiting_share") + { + if sharing.requested_mode != mode { + return Err("[CONTROL_USER_MODE_REQUIRED] The user must stop sharing and select the new input mode".into()); + } + return Ok(projected(&r)); + } + } + } + let generation = { + let mut r = locked()?; + let generation = r.admission.begin_start(owner, ControlMode::Observe)?; + r.preview = None; + r.process = None; + SCOPE_CHANGED.notify_waiters(); + generation + }; + stop_native(generation)?; + let mut r = locked()?; + r.admission.finish_start(generation)?; + r.admission.request_share(owner, mode)?; + Ok(projected(&r)) +} + +pub(crate) fn begin_share( + generation: u64, + target: String, + label: String, + mode: ControlMode, + process: Option, +) -> Result<(ControlSnapshot, Box), String> { + let mut r = locked()?; + if let Some(process) = &process { + process.validate()?; + } + r.admission.begin_share(generation, target, label, mode)?; + let owner = r + .admission + .snapshot + .owner + .clone() + .ok_or("[CONTROL_OWNER_REQUIRED] No sharing task")?; + let token = r.admission.admit(&owner, "get_app_state", executor())?; + r.preview = None; + r.process = process; + SCOPE_CHANGED.notify_waiters(); + Ok(( + projected(&r), + Box::new(Lease { + token, + complete: false, + }), + )) +} + +pub(crate) fn finish_share(generation: u64, revision: u64) -> Result { + let mut r = locked()?; + r.admission.finish_share(generation, revision)?; + Ok(projected(&r)) +} + +pub(crate) fn set_share_control( + generation: u64, + target_revision: u64, + expected_revision: u64, + enabled: bool, +) -> Result { + let _cleanup = NATIVE_CLEANUP + .lock() + .map_err(|_| "[CONTROL_FAILED] Native cleanup lock poisoned")?; + let result = { + let mut r = locked()?; + if let Some(process) = &r.process { + process.validate()?; + } + r.admission + .set_control(generation, target_revision, expected_revision, enabled)?; + SCOPE_CHANGED.notify_waiters(); + projected(&r) + }; + if !enabled { + // Serialize with native teardown and keep the revocation's identity. + // A delayed cleanup must never hide a newer share's feedback. + let current = snapshot(); + if current.generation != generation + || current.target_revision != target_revision + || current + .sharing + .as_ref() + .is_none_or(|sharing| sharing.authorization_revision != expected_revision + 1) + { + return Ok(result); + } + #[cfg(target_os = "macos")] + { + super::super::macos_bg_input::release_held_inputs(); + super::super::macos_input_focus::stop(); + super::super::macos_capture::hide_pointer(); + } + #[cfg(target_os = "windows")] + super::super::windows_pointer_feedback::hide_pointer(); + } + Ok(result) +} + +/// UI preview actions join the existing exclusive action lease. A controller +/// cannot choose the task owner or race a grant/target change into a new lease. +pub(crate) fn acquire_shared( + generation: u64, + target_revision: u64, + authorization_revision: u64, + action: &str, +) -> Result<(ControlSnapshot, Box), String> { + let mut r = locked()?; + if r.admission.snapshot.generation != generation + || r.admission.snapshot.target_revision != target_revision + || r.admission.snapshot.sharing.as_ref().is_none_or(|sharing| { + sharing.phase != openbitfun_agent_tools::computer_use_control::SharingPhase::Shared + || sharing.authorization_revision != authorization_revision + }) + { + return Err("[CONTROL_STALE] Shared preview or authorization changed".into()); + } + if let Some(process) = &r.process { + process.validate()?; + } + let owner = r + .admission + .snapshot + .owner + .clone() + .ok_or("[SHARE_REQUIRED] No shared application")?; + let token = r.admission.admit(&owner, action, executor())?; + Ok(( + projected(&r), + Box::new(Lease { + token, + complete: false, + }), + )) +} + +/// Must run before native capture, AX observation or input, not after pixels +/// have already been fetched from an unshared application. +pub(crate) fn shared_target_allowed(target: &str) -> Result<(), String> { + locked()?.admission.shared_target_allowed(target) +} + +pub(crate) fn shared_process( + generation: u64, + revision: u64, +) -> Result { + let r = locked()?; + if !active(&r) + || r.admission.snapshot.generation != generation + || r.admission.snapshot.target_revision != revision + { + return Err("[CONTROL_STALE] Shared application changed".into()); + } + let process = r.process.clone().ok_or("[GRANT_IDENTITY_UNAVAILABLE] Portal streams have no verifiable application executable; consent is session-only")?; + process.validate()?; + Ok(process) +} + +pub(crate) fn grant_preference_result( + generation: u64, + revision: u64, + authorization_revision: u64, + remembered: bool, + note: Option, +) -> Result { + let mut r = locked()?; + if !active(&r) + || r.admission.snapshot.generation != generation + || r.admission.snapshot.target_revision != revision + || r.admission + .snapshot + .sharing + .as_ref() + .is_none_or(|sharing| sharing.authorization_revision != authorization_revision) + { + return Err( + "[CONTROL_STALE] Shared application changed while reading its preference".into(), + ); + } + if let Some(sharing) = r.admission.snapshot.sharing.as_mut() { + sharing.remembered = remembered; + sharing.grant_note = note; + } + Ok(projected(&r)) +} diff --git a/src/apps/desktop/src/computer_use/desktop_host/ax_orchestration.rs b/src/apps/desktop/src/computer_use/desktop_host/ax_orchestration.rs index 80ea3e7771..b1ee8d31a2 100644 --- a/src/apps/desktop/src/computer_use/desktop_host/ax_orchestration.rs +++ b/src/apps/desktop/src/computer_use/desktop_host/ax_orchestration.rs @@ -197,11 +197,58 @@ fn observed_node_center( } impl DesktopComputerUseHost { + async fn dispatch_app_click_impl(&self, params: AppClickParams) -> OpenBitFunResult<()> { + super::background_guard::run(self.dispatch_app_click_native(params)).await + } + async fn dispatch_app_type_text_impl( + &self, + app: AppSelector, + text: &str, + focus: Option, + ) -> OpenBitFunResult<()> { + super::background_guard::run(self.dispatch_app_type_text_native(app, text, focus)).await + } + async fn dispatch_app_scroll_impl( + &self, + app: AppSelector, + focus: Option, + dx: i32, + dy: i32, + ) -> OpenBitFunResult<()> { + super::background_guard::run(self.dispatch_app_scroll_native(app, focus, dx, dy)).await + } + async fn dispatch_app_key_chord_impl( + &self, + app: AppSelector, + keys: Vec, + focus_idx: Option, + ) -> OpenBitFunResult<()> { + super::background_guard::run(self.dispatch_app_key_chord_native(app, keys, focus_idx)).await + } + async fn dispatch_app_drag_impl( + &self, + app: AppSelector, + from: ClickTarget, + to: ClickTarget, + button: String, + duration_ms: u64, + ) -> OpenBitFunResult<()> { + super::background_guard::run(self.dispatch_app_drag_native( + app, + from, + to, + button, + duration_ms, + )) + .await + } + pub(super) async fn dispatch_app_input_impl( &self, app: AppSelector, action: AppInputAction, ) -> OpenBitFunResult<()> { + action.validate().map_err(OpenBitFunError::tool)?; crate::computer_use::control_session::input_allowed().map_err(OpenBitFunError::tool)?; match action { AppInputAction::Click { @@ -243,7 +290,7 @@ impl DesktopComputerUseHost { } } - async fn dispatch_app_drag_impl( + async fn dispatch_app_drag_native( &self, app: AppSelector, from: ClickTarget, @@ -344,7 +391,7 @@ impl DesktopComputerUseHost { }) } - async fn dispatch_app_click_impl(&self, params: AppClickParams) -> OpenBitFunResult<()> { + async fn dispatch_app_click_native(&self, params: AppClickParams) -> OpenBitFunResult<()> { #[cfg(target_os = "macos")] { let pid = resolve_pid_macos(self, ¶ms.app).await?; @@ -640,7 +687,7 @@ impl DesktopComputerUseHost { })).await.map_err(|error| OpenBitFunError::tool(error.to_string()))? } - async fn dispatch_app_type_text_impl( + async fn dispatch_app_type_text_native( &self, app: AppSelector, text: &str, @@ -853,7 +900,7 @@ impl DesktopComputerUseHost { self.observe_after_app_input(app).await } - async fn dispatch_app_scroll_impl( + async fn dispatch_app_scroll_native( &self, app: AppSelector, focus: Option, @@ -926,7 +973,7 @@ impl DesktopComputerUseHost { self.observe_after_app_input(app).await } - async fn dispatch_app_key_chord_impl( + async fn dispatch_app_key_chord_native( &self, app: AppSelector, keys: Vec, @@ -1111,6 +1158,7 @@ impl DesktopComputerUseHost { .unwrap_or_default(); let view = InteractiveView { + observation_id: snap.observation_id.clone(), app: snap.app.clone(), window_title: snap.window_title.clone(), elements: elements.clone(), @@ -1303,6 +1351,7 @@ impl DesktopComputerUseHost { .map(|d| d.as_millis() as u64) .unwrap_or_default(); let view = VisualMarkView { + observation_id: snap.observation_id.clone(), app: snap.app.clone(), window_title: snap.window_title.clone(), marks: marks.clone(), diff --git a/src/apps/desktop/src/computer_use/desktop_host/background_guard.rs b/src/apps/desktop/src/computer_use/desktop_host/background_guard.rs new file mode 100644 index 0000000000..5da92f69a3 --- /dev/null +++ b/src/apps/desktop/src/computer_use/desktop_host/background_guard.rs @@ -0,0 +1,234 @@ +//! Operation-boundary desktop witnesses adapted from computer-use-rust's +//! background-interference check. They never restore focus or replay input. +use super::*; +use openbitfun_agent_tools::computer_use_control::{ + BackgroundObservation, ControlMode, ObservationScope, +}; + +#[derive(Debug, Clone)] +struct Witness { + foreground: Vec, + pointer: Option<(f64, f64)>, + window_order: Option>, + activation_epoch: Option, +} + +// New/closed windows do not establish a raise. Compare the relative order of +// windows that existed at both boundaries, including the human foreground. +fn common_order_unchanged(before: &[String], after: &[String]) -> bool { + let common_before: Vec<_> = before + .iter() + .filter(|window| after.contains(window)) + .collect(); + let common_after: Vec<_> = after + .iter() + .filter(|window| before.contains(window)) + .collect(); + common_before == common_after +} + +fn evidence(before: &Witness, after: &Witness, sequence: u64) -> BackgroundObservation { + BackgroundObservation { + sequence, + foreground_unchanged: before.foreground == after.foreground + && match (before.activation_epoch, after.activation_epoch) { (Some(a),Some(b)) => a == b, _ => true }, + cursor_unchanged: before.pointer.zip(after.pointer).map(|(a,b)| a == b), + window_order_unchanged: before.window_order.as_ref().zip(after.window_order.as_ref()) + .map(|(a,b)| common_order_unchanged(a,b)), + scope: if cfg!(target_os = "linux") { + "AT-SPI active top-level windows at call boundaries; compositor cursor and stacking are not observable. Semantic actions only; no seat input." + } else { + "Foreground identity, physical cursor and relative order of existing top-level windows at call boundaries. This does not prove delivery or later application behavior." + }.into(), + } +} + +pub(super) async fn run( + input: impl std::future::Future>, +) -> OpenBitFunResult { + let control = crate::computer_use::control_session::snapshot(); + if control.mode != ControlMode::Background { + return input.await; + } + let scope = ObservationScope::from_control(&control); + let before = capture().await.map_err(|error| { + OpenBitFunError::tool(format!( + "[BACKGROUND_OBSERVATION_UNAVAILABLE] No input sent: {error}" + )) + })?; + let result = input.await; + // Once an operation may have been submitted, any failure must preserve + // ambiguity. A fallback click or a foreground repair can duplicate effects. + let after = capture().await; + match after { + Ok(after) => { + let observation = evidence(&before, &after, control.sequence); + let changed = !observation.foreground_unchanged + || observation.cursor_unchanged == Some(false) + || observation.window_order_unchanged == Some(false); + crate::computer_use::control_session::record_background_observation( + &scope, + observation, + ); + if !changed { + return result; + } + revoke(&scope, "background_desktop_changed"); + Err(OpenBitFunError::tool(format!("[BACKGROUND_INTERFERENCE] Desktop focus, physical pointer or window ordering changed during the operation; control was revoked. Cause may be application behavior or concurrent user/system input. Effects may already have occurred. No restoration or retry was sent. Operation error: {}", result.err().map(|error| error.to_string()).unwrap_or_else(|| "none".into())))) + } + Err(error) => { + revoke(&scope, "background_observation_failed"); + Err(OpenBitFunError::tool(format!("[BACKGROUND_OBSERVATION_FAILED] Could not verify the desktop after input; control was revoked. Effects may already have occurred; observe before deciding another action. {error}"))) + } + } +} + +fn revoke(scope: &ObservationScope, reason: &str) { + if let Some(target) = &scope.target { + crate::computer_use::control_session::native_stopped_target( + scope.generation, + scope.target_revision, + target, + reason, + ); + } +} + +#[cfg(any(target_os = "macos", target_os = "windows"))] +async fn capture() -> Result { + crate::computer_use::control_session::spawn_blocking(native_witness) + .await + .map_err(|error| error.to_string())? +} + +#[cfg(target_os = "macos")] +fn native_witness() -> Result { + let foreground = objc2_app_kit::NSWorkspace::sharedWorkspace() + .frontmostApplication() + .ok_or("No foreground application")? + .processIdentifier(); + let pointer = macos::quartz_mouse_location().map_err(|error| error.to_string())?; + let windows = crate::computer_use::macos_list_apps::list_share_targets() + .map_err(|error| error.to_string())?; + let epoch = crate::computer_use::macos_capture::activation_epoch(); + Ok(Witness { + foreground: vec![foreground.to_string()], + pointer: Some(pointer), + window_order: Some(windows.into_iter().map(|window| window.id).collect()), + activation_epoch: (epoch != 0).then_some(epoch), + }) +} + +#[cfg(target_os = "windows")] +fn native_witness() -> Result { + use windows::Win32::{ + Foundation::POINT, + UI::WindowsAndMessaging::{GetCursorPos, GetForegroundWindow}, + }; + let foreground = unsafe { GetForegroundWindow() }; + if foreground.0.is_null() { + return Err("No foreground window in this desktop session".into()); + } + let mut pointer = POINT::default(); + unsafe { GetCursorPos(&mut pointer) }.map_err(|error| error.to_string())?; + Ok(Witness { + foreground: vec![(foreground.0 as usize).to_string()], + pointer: Some((pointer.x as f64, pointer.y as f64)), + window_order: Some( + crate::computer_use::windows_list_apps::list_share_targets() + .into_iter() + .map(|window| window.id) + .collect(), + ), + activation_epoch: None, + }) +} + +#[cfg(target_os = "linux")] +async fn capture() -> Result { + use atspi::{connection::P2P, AccessibilityConnection, State}; + fn error(error: impl std::fmt::Display) -> String { + error.to_string() + } + let connection = AccessibilityConnection::new() + .await + .map_err(|error| error.to_string())?; + let root = connection + .root_accessible_on_registry() + .await + .map_err(error)?; + let mut active = Vec::new(); + for application in root.get_children().await.map_err(error)? { + let app = connection + .object_as_accessible(&application) + .await + .map_err(error)?; + for window in app.get_children().await.map_err(error)? { + let accessible = connection + .object_as_accessible(&window) + .await + .map_err(error)?; + if accessible + .get_state() + .await + .map_err(error)? + .contains(State::Active) + { + active.push(format!("{:?}", window)); + } + } + } + active.sort(); + Ok(Witness { + foreground: active, + pointer: None, + window_order: None, + activation_epoch: None, + }) +} + +#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] +async fn capture() -> Result { + Err("No desktop observation provider".into()) +} + +#[cfg(test)] +mod tests { + use super::*; + fn witness(order: &[&str]) -> Witness { + Witness { + foreground: vec!["human".into()], + pointer: Some((12.0, 34.0)), + window_order: Some(order.iter().map(|value| (*value).into()).collect()), + activation_epoch: Some(4), + } + } + #[test] + fn detects_raise_pointer_and_transient_activation_without_blame_or_repair() { + let before = witness(&["human", "target", "other"]); + assert!( + !evidence(&before, &witness(&["target", "human", "other"]), 1) + .window_order_unchanged + .unwrap() + ); + let mut after = before.clone(); + after.pointer = Some((13.0, 34.0)); + assert_eq!(evidence(&before, &after, 1).cursor_unchanged, Some(false)); + after = before.clone(); + after.activation_epoch = Some(6); + assert!(!evidence(&before, &after, 1).foreground_unchanged); + } + #[test] + fn new_background_window_is_not_a_raise_and_missing_coverage_is_not_success() { + let before = witness(&["human", "target"]); + assert_eq!( + evidence(&before, &witness(&["human", "dialog", "target"]), 1).window_order_unchanged, + Some(true) + ); + let mut partial = before.clone(); + partial.pointer = None; + partial.window_order = None; + assert_eq!(evidence(&partial, &partial, 1).cursor_unchanged, None); + assert_eq!(evidence(&partial, &partial, 1).window_order_unchanged, None); + } +} diff --git a/src/apps/desktop/src/computer_use/desktop_host/macos.rs b/src/apps/desktop/src/computer_use/desktop_host/macos.rs index 7f3637692f..88af914da6 100644 --- a/src/apps/desktop/src/computer_use/desktop_host/macos.rs +++ b/src/apps/desktop/src/computer_use/desktop_host/macos.rs @@ -125,7 +125,6 @@ unsafe extern "C" { #[link(name = "CoreGraphics", kind = "framework")] unsafe extern "C" { - fn CGPreflightScreenCaptureAccess() -> bool; fn CGRequestScreenCaptureAccess() -> bool; fn CGEventCreate(source: *const c_void) -> *const c_void; fn CGEventGetLocation(event: *const c_void) -> CGPoint; @@ -150,10 +149,6 @@ pub(super) fn ax_trusted() -> bool { unsafe { AXIsProcessTrusted() } } -pub(super) fn screen_capture_preflight() -> bool { - unsafe { CGPreflightScreenCaptureAccess() } -} - pub(super) fn request_ax_prompt() { let key = CFString::new("AXTrustedCheckOptionPrompt"); let val = CFBoolean::true_value(); @@ -167,28 +162,16 @@ pub(super) fn request_screen_capture() -> bool { unsafe { CGRequestScreenCaptureAccess() } } -/// Shared AX-first desktop guard: `get_app_state_inner` and -/// `get_app_shortcuts_inner` both silently degrade (truncated tree / empty -/// menu bar) instead of erroring when Accessibility trust is missing, which -/// looks like "nothing here" rather than "OpenBitFun lacks permission". Fail -/// fast with a structured `[PERMISSION_DENIED]` error and re-trigger the -/// system prompt so the user has a way back to the dialog without digging -/// through System Settings manually. -/// -/// `retry_hint` is the caller-specific closing sentence (e.g. `"After -/// granting, retry \`desktop.get_app_state\` and the AX tree will include -/// all WebView subtree nodes."`), so each call site keeps its own precise -/// guidance while the shared boilerplate (check + prompt + explanation) -/// cannot drift between call sites. +/// Observation never invokes consent prompts. The user's permission action +/// owns the explicit request and the OS owns its outcome. pub(super) fn require_ax_trust_for(retry_hint: &str) -> OpenBitFunResult<()> { if ax_trusted() { return Ok(()); } - request_ax_prompt(); Err(OpenBitFunError::tool(format!( "[PERMISSION_DENIED] macOS Accessibility permission not granted to OpenBitFun. \ - The system has been asked to surface the permission dialog (System Settings → \ - Privacy & Security → Accessibility → enable OpenBitFun). {}", + Use Computer Use permissions to request access, or open System Settings → \ + Privacy & Security → Accessibility → enable OpenBitFun. {}", retry_hint ))) } diff --git a/src/apps/desktop/src/computer_use/desktop_host/mod.rs b/src/apps/desktop/src/computer_use/desktop_host/mod.rs index 0cb9b51718..0785c89516 100644 --- a/src/apps/desktop/src/computer_use/desktop_host/mod.rs +++ b/src/apps/desktop/src/computer_use/desktop_host/mod.rs @@ -1,13 +1,19 @@ -//! Cross-platform `ComputerUseHost` via `screenshots` + `enigo`. +//! Desktop ComputerUseHost: scoped observations and platform-native capture/input providers. mod screenshot; use screenshot::PointerMap; +mod background_guard; +mod observation; +pub(crate) mod shared_preview; +pub(crate) mod sharing; +mod state; +use state::ComputerUseSessionMutableState; +#[cfg(any(target_os = "macos", target_os = "windows"))] +use state::{CachedInteractiveView, CachedVisualMarkView}; use async_trait::async_trait; #[cfg(any(target_os = "macos", target_os = "windows"))] use log::debug; -#[cfg(any(target_os = "macos", target_os = "windows"))] -use openbitfun_core::agentic::tools::computer_use_host::VisualMark; use openbitfun_core::agentic::tools::computer_use_host::{ ActionRecord, AppInfo, AppSelector, AppShortcutsSnapshot, AppStateSnapshot, AppWaitPredicate, ComputerScreenshot, ComputerUseDisplayInfo, ComputerUseHost, ComputerUseInteractionState, @@ -21,7 +27,6 @@ use openbitfun_core::agentic::tools::computer_use_host::{ use openbitfun_core::agentic::tools::computer_use_host::{ ComputerUseForegroundApplication, ComputerUsePointerGlobal, }; -use openbitfun_core::agentic::tools::computer_use_optimizer::ComputerUseOptimizer; use openbitfun_core::util::errors::{OpenBitFunError, OpenBitFunResult}; #[cfg(any(target_os = "windows", target_os = "linux"))] use screenshots::display_info::DisplayInfo; @@ -348,121 +353,6 @@ mod windows_foreground_tests { } } -/// Unified mutable session state for computer use — one mutex instead of five. -/// State transitions are applied centrally after each action (screenshot, pointer move, click, etc.). -#[derive(Debug)] -struct ComputerUseSessionMutableState { - pointer_map: Option, - /// When true, a fresh `screenshot_display` is required before `click` and before `key_chord` that sends Return/Enter - /// (set after pointer moves / click; cleared after screenshot). - click_needs_fresh_screenshot: bool, - /// After click / key / type / scroll / drag: recommend a **`screenshot`** to confirm UI state (Cowork verify). - /// Cleared on the next successful `screenshot_display`. - pending_verify_screenshot: bool, - /// Action optimizer for loop detection, history, and visual verification. - optimizer: ComputerUseOptimizer, - /// Most-recent action **kind** that mutated UI / pointer state. Surfaced - /// to the model via `interaction_state.last_mutation` so it can pair the - /// right verification step (e.g. after `Click` + `pending_verify` ⇒ take - /// a confirming `screenshot`; after `TypeText` ⇒ may chain Enter without - /// re-screenshotting because typing does not move the pointer). - last_mutation_kind: Option, - /// Caller-pinned target display (set via `desktop.focus_display`). - /// When set, all subsequent screenshots / peeks / locates use this - /// display instead of "screen under the mouse pointer". The model - /// uses this to disambiguate multi-monitor targets explicitly. - preferred_display_id: Option, - /// Most-recent Set-of-Mark interactive view per pid. Used to resolve - /// `interactive_*` numeric `i` indices back to AX node indices and to - /// detect stale-view usage via `before_view_digest`. - #[cfg(any(target_os = "macos", target_os = "windows"))] - interactive_view_cache: std::collections::HashMap, - #[cfg(any(target_os = "macos", target_os = "windows"))] - visual_mark_cache: std::collections::HashMap, - /// Most-recent focused-window screenshot coordinate map per application - /// pid. `app_click(target: image_xy | image_grid)` must use the same - /// image basis the model saw from `get_app_state`, not whichever global - /// computer-use screenshot happened to run last. - #[cfg(any(target_os = "macos", target_os = "windows"))] - app_pointer_maps: std::collections::HashMap, - /// Exact screenshot-id keyed coordinate maps. This is the strongest - /// addressing basis for arbitrary visual targets because it survives - /// interleaved app_state / screenshot / interactive_view calls. - screenshot_pointer_maps: std::collections::HashMap, - screenshot_targets: std::collections::HashMap, - app_pointer_targets: std::collections::HashMap, -} - -#[derive(Debug, Clone)] -#[cfg(any(target_os = "macos", target_os = "windows"))] -struct CachedInteractiveView { - digest: String, - /// `i` → `node_idx` map (dense, indexed by `i`). - elements: Vec, -} - -#[derive(Debug, Clone)] -#[cfg(any(target_os = "macos", target_os = "windows"))] -struct CachedVisualMarkView { - digest: String, - marks: Vec, - screenshot_id: Option, -} - -impl ComputerUseSessionMutableState { - fn new() -> Self { - Self { - pointer_map: None, - click_needs_fresh_screenshot: true, - pending_verify_screenshot: false, - optimizer: ComputerUseOptimizer::new(), - last_mutation_kind: None, - preferred_display_id: None, - #[cfg(any(target_os = "macos", target_os = "windows"))] - interactive_view_cache: std::collections::HashMap::new(), - #[cfg(any(target_os = "macos", target_os = "windows"))] - visual_mark_cache: std::collections::HashMap::new(), - #[cfg(any(target_os = "macos", target_os = "windows"))] - app_pointer_maps: std::collections::HashMap::new(), - screenshot_pointer_maps: std::collections::HashMap::new(), - screenshot_targets: std::collections::HashMap::new(), - app_pointer_targets: std::collections::HashMap::new(), - } - } - - /// Called after a successful screenshot capture. - fn transition_after_screenshot(&mut self, map: PointerMap) { - self.pointer_map = Some(map); - self.click_needs_fresh_screenshot = false; - self.pending_verify_screenshot = false; - self.last_mutation_kind = Some(ComputerUseLastMutationKind::Screenshot); - } - - /// Called after pointer mutation (move, step, relative), click, scroll, key_chord, or type_text. - fn transition_after_pointer_mutation(&mut self) { - self.click_needs_fresh_screenshot = true; - // Note: `last_mutation_kind` is set explicitly by the calling - // action (PointerMove / Click / Scroll / KeyChord / TypeText / Drag) - // so we do not overwrite it here with a generic value. - } - - /// Called after click (same effect as pointer mutation for freshness). - fn transition_after_click(&mut self) { - self.click_needs_fresh_screenshot = true; - self.pending_verify_screenshot = true; - self.last_mutation_kind = Some(ComputerUseLastMutationKind::Click); - } - - /// Called after key, typing, scroll, or drag — UI likely changed; next `screenshot` should confirm. - fn transition_after_committed_ui_action(&mut self) { - self.pending_verify_screenshot = true; - } - - fn record_mutation(&mut self, kind: ComputerUseLastMutationKind) { - self.last_mutation_kind = Some(kind); - } -} - pub struct DesktopComputerUseHost { state: Mutex, } @@ -481,16 +371,25 @@ impl Default for DesktopComputerUseHost { } impl DesktopComputerUseHost { + fn state_for_scope( + &self, + scope: &openbitfun_agent_tools::computer_use_control::ObservationScope, + ) -> OpenBitFunResult> { + crate::computer_use::control_session::validate_observation_scope(scope) + .map_err(OpenBitFunError::tool)?; + let mut state = self.state.lock().map_err(|_| { + OpenBitFunError::tool("[CONTROL_FAILED] Observation cache lock poisoned") + })?; + state.synchronize(scope); + Ok(state) + } + pub fn new() -> Self { Self { state: Mutex::new(ComputerUseSessionMutableState::new()), } } - pub fn prompt_for_missing_permissions(&self) { - self.run_background_input_self_check(); - } - fn next_screenshot_id() -> String { let seq = SCREENSHOT_ID_COUNTER.fetch_add(1, Ordering::Relaxed); let ms = std::time::SystemTime::now() @@ -500,55 +399,6 @@ impl DesktopComputerUseHost { format!("shot_{}_{}", ms, seq) } - /// Codex-style startup probe: log whether AX/background-input capabilities - /// are available so operators can diagnose missing permissions early. - /// - /// Behaviour parity with Codex: if the process is NOT yet - /// Accessibility-trusted, immediately call - /// `AXIsProcessTrustedWithOptions({kAXTrustedCheckOptionPrompt: true})` - /// once. macOS responds by surfacing the system-modal "允许 X 通过辅助功能 - /// 控制您的电脑" dialog (deep-linked to System Settings → Privacy & Security - /// → Accessibility). Without this call, the OS NEVER prompts and AX tree - /// reads against other apps return only the top-level window structure - /// (root window + a few descendants) — which is exactly the "shallow tree - /// / agent goes blind" symptom we observed against the OpenBitFun WebView. - fn run_background_input_self_check(&self) { - #[cfg(target_os = "macos")] - { - let bg_ok = crate::computer_use::macos_bg_input::supports_background_input(); - if bg_ok { - log::info!( - "AX-first computer use ready: AXIsProcessTrustedWithOptions=true; CGEventPostToPid background input enabled" - ); - } else { - log::warn!( - "AX-first computer use disabled: process is NOT marked Accessibility-trusted. Triggering one-shot system prompt via AXIsProcessTrustedWithOptions(prompt:true) so macOS surfaces the Accessibility permission dialog (deep-link: System Settings → Privacy & Security → Accessibility)." - ); - // Fire-and-forget. The dialog is async and modal at the macOS - // level; we do not block startup waiting for the user to - // approve. The next CU invocation will simply succeed once - // permission lands. Subsequent OpenBitFun launches skip the - // prompt because `ax_trusted()` will already be true. - macos::request_ax_prompt(); - } - // Same idea for Screen Recording. Without it, focused-window - // screenshots fall back to a desktop-wallpaper placeholder, which - // is the second half of the "blind agent" failure mode. - if !macos::screen_capture_preflight() { - log::warn!( - "Screen Recording permission missing; window screenshots will be incomplete. Triggering CGRequestScreenCaptureAccess() to surface the system prompt." - ); - let _ = macos::request_screen_capture(); - } - } - #[cfg(not(target_os = "macos"))] - { - log::info!( - "AX-first background input is macOS-only in this build; legacy screen-coordinate desktop actions remain available" - ); - } - } - /// Best-effort foreground app + pointer; safe to call from `spawn_blocking`. fn collect_session_snapshot_sync() -> ComputerUseSessionSnapshot { #[cfg(target_os = "macos")] @@ -835,138 +685,6 @@ Check the app directly, or ask the user to bring it up.", ComputerUseSessionSnapshot::default() } - fn permission_sync() -> ComputerUsePermissionSnapshot { - #[cfg(target_os = "windows")] - fn is_process_elevated() -> bool { - use windows::Win32::Foundation::HANDLE; - use windows::Win32::Security::{ - GetTokenInformation, TokenElevation, TOKEN_ELEVATION, TOKEN_QUERY, - }; - use windows::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken}; - unsafe { - let mut token = HANDLE::default(); - if OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut token).is_err() { - return false; - } - let mut elevation = TOKEN_ELEVATION::default(); - let mut ret_len: u32 = 0; - let ok = GetTokenInformation( - token, - TokenElevation, - Some(&mut elevation as *mut _ as *mut _), - std::mem::size_of::() as u32, - &mut ret_len, - ) - .is_ok(); - let _ = windows::Win32::Foundation::CloseHandle(token); - ok && elevation.TokenIsElevated != 0 - } - } - - #[cfg(target_os = "macos")] - { - let platform_note = if cfg!(debug_assertions) && !macos::ax_trusted() { - Some( - "Development build: grant Accessibility to target/debug/openbitfun-desktop (path appears in errors if mouse fails)." - .to_string(), - ) - } else { - None - }; - ComputerUsePermissionSnapshot { - accessibility_granted: macos::ax_trusted(), - screen_capture_granted: macos::screen_capture_preflight(), - platform_note, - } - } - #[cfg(target_os = "windows")] - { - // Phase 4: real probe instead of always returning `true`. - // Screen capture: enumerating displays via the `screenshots` crate - // exercises the same DXGI/GDI path used for actual capture, so a - // failure here is a strong signal that capture won't work either - // (e.g. running under Session 0 / blocked by group policy). - let screen_capture_granted = DisplayInfo::all().map(|d| !d.is_empty()).unwrap_or(false); - - // Accessibility / input injection: there is no opt-in permission - // on Windows, but UIPI silently blocks input into elevated windows - // when we are not elevated. Detect elevation so the model can warn - // the user instead of silently mis-clicking. - let elevated = is_process_elevated(); - let mut notes: Vec<&'static str> = Vec::new(); - if !screen_capture_granted { - notes.push( - "Screen capture probe failed: no displays enumerated (Session 0 / RDP / policy?).", - ); - } - if !elevated { - notes - .push("Not running elevated: UIPI may block input into Administrator windows."); - } - ComputerUsePermissionSnapshot { - accessibility_granted: true, - screen_capture_granted, - platform_note: if notes.is_empty() { - None - } else { - Some(notes.join(" ")) - }, - } - } - #[cfg(target_os = "linux")] - { - // Phase 4: probe display server type *and* the actual capture path. - let session_type = std::env::var("XDG_SESSION_TYPE").unwrap_or_default(); - let wayland = std::env::var("WAYLAND_DISPLAY").is_ok() - || session_type.eq_ignore_ascii_case("wayland"); - let x11_display = std::env::var("DISPLAY").is_ok(); - - let screen_capture_granted = DisplayInfo::all().map(|d| !d.is_empty()).unwrap_or(false); - - // Global keyboard / mouse injection on Linux requires either an - // X11 session with XTEST (`enigo` / `rdev` work) *or* uinput on - // Wayland (root). Without DISPLAY we can't inject synthetic input - // even on a Wayland session running XWayland. - let accessibility_granted = if wayland { false } else { x11_display }; - - let mut notes: Vec = Vec::new(); - if wayland { - notes.push( - "Wayland session: synthetic input is unsupported; screen capture relies on xdg-desktop-portal." - .to_string(), - ); - } - if !x11_display && !wayland { - notes.push( - "DISPLAY not set: no X server reachable for input injection.".to_string(), - ); - } - if !screen_capture_granted { - notes.push( - "Screen capture probe failed: no displays enumerated by the screenshots crate." - .to_string(), - ); - } - ComputerUsePermissionSnapshot { - accessibility_granted, - screen_capture_granted, - platform_note: if notes.is_empty() { - None - } else { - Some(notes.join(" ")) - }, - } - } - #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] - { - ComputerUsePermissionSnapshot { - accessibility_granted: false, - screen_capture_granted: false, - platform_note: Some("Computer use is not supported on this OS.".to_string()), - } - } - } - /// Best-effort current mouse position in global screen coordinates. pub(super) fn current_mouse_position() -> (f64, f64) { #[cfg(target_os = "macos")] @@ -1012,146 +730,6 @@ mod macos; mod pointer_input; impl DesktopComputerUseHost { - /// Internal `get_app_state` that lets callers opt out of the focused-window - /// screenshot. The public trait method always passes `capture_screenshot=true` - /// (Codex parity). Internal re-snapshots from `app_click` / `app_type_text` / - /// `app_scroll` / `app_key_chord` pass `false` to avoid a redundant capture - /// — the **outer** call (e.g. the one returned to the model) gets the image. - pub(crate) async fn get_app_state_inner( - &self, - app: AppSelector, - max_depth: u32, - focus_window_only: bool, - capture_screenshot: bool, - ) -> OpenBitFunResult { - #[cfg(target_os = "macos")] - { - // Pre-flight: without Accessibility trust macOS silently truncates - // the AX subtree to the top-level window/container (~7 nodes for - // a Tauri WebView app), with no exception. The agent then has no - // actionable widgets to act on. Fail fast with a structured - // `[PERMISSION_DENIED]` error so the model can surface the issue - // (and the host's startup prompt is what produces the dialog). - macos::require_ax_trust_for( - "After granting, retry `desktop.get_app_state` and the AX tree will include all WebView subtree nodes.", - )?; - let pid = resolve_pid_macos(self, &app).await?; - let mut snap = crate::computer_use::control_session::spawn_blocking(move || { - // Wrap in @try/@catch — AX APIs can throw NSException for - // sandboxed / partially-loaded / dying processes, and an - // unwound foreign exception aborts the whole openbitfun process - // (`Rust cannot catch foreign exceptions, aborting`). - macos::catch_objc(|| { - crate::computer_use::macos_ax_dump::dump_app_ax( - pid, - crate::computer_use::macos_ax_dump::DumpOpts { - max_depth, - focus_window_only, - ..Default::default() - }, - ) - }) - }) - .await - .map_err(|e| OpenBitFunError::tool(e.to_string()))??; - - // Auto-attach focused-window screenshot. Failures are non-fatal — - // worst case the model still has the AX tree. - if capture_screenshot { - let started = std::time::Instant::now(); - match self.screenshot_for_app_pid(pid).await { - Ok(shot) => { - debug!( - "computer_use.app_state: attached screenshot ({}x{} jpeg, {} bytes, {}ms)", - shot.image_width, - shot.image_height, - shot.bytes.len(), - started.elapsed().as_millis() - ); - snap.screenshot = Some(shot); - } - Err(e) => { - snap.tree_text.push_str(&format!( - "\n[note] CAPTURE_UNAVAILABLE: {e}. Accessibility facts remain available; no substitute window was captured.\n" - )); - } - } - } - // Register the snapshot in the element-token registry so - // subsequent `app_click` calls can resolve `s{hex}:{idx}` - // tokens back to this snapshot's element indices. - let reg_pid = snap.app.pid.unwrap_or(0); - let _ = openbitfun_agent_tools::element_token::global().register_snapshot( - reg_pid, - 0, - snap.nodes.len(), - ); - Ok(snap) - } - #[cfg(target_os = "windows")] - { - let (_, hwnd_raw) = self.windows_target(&app).await?; - - let mut snap = crate::computer_use::control_session::spawn_blocking(move || { - let hwnd = windows::Win32::Foundation::HWND(hwnd_raw as *mut std::ffi::c_void); - crate::computer_use::windows_ax_ui::get_app_state_snapshot_for_window( - hwnd, - max_depth, - focus_window_only, - ) - }) - .await - .map_err(|e| OpenBitFunError::tool(e.to_string()))??; - - let reg_pid = snap.app.pid.unwrap_or(0); - - // Auto-attach window screenshot (Codex parity). Failures are non-fatal. - if capture_screenshot { - let started = std::time::Instant::now(); - match self - .screenshot_for_foreground_window(reg_pid, hwnd_raw) - .await - { - Ok(shot) => { - debug!( - "computer_use.app_state: attached window screenshot ({}x{}, {} bytes, {}ms)", - shot.image_width, - shot.image_height, - shot.bytes.len(), - started.elapsed().as_millis() - ); - snap.screenshot = Some(shot); - } - Err(e) => { - snap.tree_text.push_str(&format!( - "\n[note] CAPTURE_UNAVAILABLE: {e}. Accessibility facts remain available; no substitute window was captured.\n" - )); - } - } - } - - // Register snapshot in element-token registry. - let _ = openbitfun_agent_tools::element_token::global().register_snapshot( - reg_pid, - 0, - snap.nodes.len(), - ); - Ok(snap) - } - #[cfg(target_os = "linux")] - { - let _ = capture_screenshot; - crate::computer_use::linux_control_ax::snapshot(app, max_depth, focus_window_only).await - } - #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] - { - let _ = (app, max_depth, focus_window_only, capture_screenshot); - Err(OpenBitFunError::tool( - "[CONTROL_UNSUPPORTED] No accessibility provider", - )) - } - } - /// Enumerate the registered keyboard shortcuts in `app`'s menu bar. /// Read-only counterpart to `key_chord` / `app_key_chord` (which only /// **send** keys). Unlike `get_app_state_inner`, macOS does not need @@ -1263,6 +841,30 @@ impl ComputerUseHost for DesktopComputerUseHost { } } + fn validate_observation(&self, id: &str) -> OpenBitFunResult<()> { + let scope = crate::computer_use::control_session::observation_scope() + .map_err(OpenBitFunError::tool)?; + let pid = scope + .target + .as_deref() + .and_then(|target| { + target + .strip_prefix("pid:") + .or_else(|| target.strip_prefix("atspi:")) + }) + .and_then(|target| target.split('/').next()) + .and_then(|pid| pid.parse::().ok()) + .ok_or_else(|| { + OpenBitFunError::tool( + "[OBSERVATION_TARGET_REQUIRED] Prepare the observed application before input", + ) + })?; + self.state_for_scope(&scope)? + .observations + .validate_application(pid, id) + .map_err(OpenBitFunError::tool) + } + async fn prepare_control_target(&self, mut app: AppSelector) -> OpenBitFunResult<()> { if app.is_empty() { if let Some(pid) = crate::computer_use::control_session::snapshot() @@ -1306,6 +908,9 @@ impl ComputerUseHost for DesktopComputerUseHost { crate::computer_use::linux_control_ax::bind_app_selector(&app).await?; } } + let scope = crate::computer_use::control_session::observation_scope() + .map_err(OpenBitFunError::tool)?; + drop(self.state_for_scope(&scope)?); Ok(()) } @@ -1314,31 +919,10 @@ impl ComputerUseHost for DesktopComputerUseHost { owner: &str, request: openbitfun_core::agentic::tools::computer_use_host::ControlStartRequest, ) -> OpenBitFunResult { - let snapshot = crate::computer_use::control_session::start(owner, request.mode) - .map_err(OpenBitFunError::tool)?; - if let Ok(mut state) = self.state.lock() { - state.screenshot_pointer_maps.clear(); - state.screenshot_targets.clear(); - state.app_pointer_targets.clear(); - #[cfg(any(target_os = "macos", target_os = "windows"))] - state.app_pointer_maps.clear(); - state.pointer_map = None; - } - #[cfg(target_os = "linux")] - if let Err(error) = crate::computer_use::linux_control::start_session( - request.mode == openbitfun_agent_tools::computer_use_control::ControlMode::Foreground, - ) - .await - { - let _ = crate::computer_use::control_session::stop_checked( - Some(owner), - Some(snapshot.generation), - &error, - ); - return Err(OpenBitFunError::tool(error)); - } - Ok(snapshot) + crate::computer_use::control_session::request_share(owner, request.mode) + .map_err(OpenBitFunError::tool) } + fn control_snapshot( &self, ) -> openbitfun_core::agentic::tools::computer_use_host::ControlSnapshot { @@ -1370,30 +954,51 @@ impl ComputerUseHost for DesktopComputerUseHost { ) -> OpenBitFunResult< Option>, > { - if matches!( - action, - "list_apps" | "list_displays" | "get_os_info" | "clipboard_get" | "control_status" - ) { + let policy = + openbitfun_agent_tools::computer_use_control::ControlActionPolicy::for_action(action) + .ok_or_else(|| { + OpenBitFunError::tool(format!( + "[CONTROL_ACTION_INVALID] Unknown ComputerUse action: {action}" + )) + })?; + if !policy.lease { return Ok(None); } + let current = crate::computer_use::control_session::snapshot(); + if current.state == "idle" { + crate::computer_use::control_session::request_share( + owner, + openbitfun_agent_tools::computer_use_control::ControlMode::Background, + ) + .map_err(OpenBitFunError::tool)?; + } crate::computer_use::control_session::acquire(owner, action) .map(Some) .map_err(OpenBitFunError::tool) } async fn permission_snapshot(&self) -> OpenBitFunResult { - Ok( - crate::computer_use::control_session::spawn_blocking(Self::permission_sync) - .await - .map_err(|e| OpenBitFunError::tool(e.to_string()))?, - ) + use openbitfun_agent_tools::computer_use_permissions::ComputerPermissionKind as Kind; + let status = crate::computer_use::permissions::status().await; + Ok(ComputerUsePermissionSnapshot { + accessibility_granted: status.usable(Kind::Accessibility), + screen_capture_granted: status.usable(Kind::ScreenRecording), + platform_note: Some( + status + .permissions + .iter() + .map(|permission| permission.detail.as_str()) + .collect::>() + .join(" "), + ), + }) } fn computer_use_interaction_state(&self) -> ComputerUseInteractionState { let (has_capture, click_needs_fresh, pending_verify, last_mutation, preferred_display_id) = { let s = self.state.lock().unwrap(); ( - s.pointer_map.is_some(), + s.observations.latest_image().is_some(), s.click_needs_fresh_screenshot, s.pending_verify_screenshot, s.last_mutation_kind.clone(), @@ -1958,10 +1563,14 @@ impl ComputerUseHost for DesktopComputerUseHost { } #[cfg(target_os = "windows")] { - // Windows uses PostMessageW / SendInput for background input. + // Semantic UIA and targeted window messages; SendInput remains foreground-only. true } - #[cfg(not(any(target_os = "macos", target_os = "windows")))] + #[cfg(target_os = "linux")] + { + true // AT-SPI semantic input; arbitrary background coordinates remain unsupported. + } + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] { false } @@ -2111,10 +1720,10 @@ impl ComputerUseHost for DesktopComputerUseHost { } /// Linux Computer Use is a **legacy compatibility layer** only: basic -/// screenshot + enigo input + AT-SPI locate/OCR. AX-first APIs (`get_app_state`, -/// `app_*`, interactive/visual views, shortcuts) are intentionally unavailable. +/// Unsupported legacy operations must identify the available Linux providers; +/// semantic AT-SPI input and portal foreground input have separate capabilities. #[cfg(not(any(target_os = "macos", target_os = "windows")))] -pub(crate) const LINUX_LEGACY_AX_UNAVAILABLE: &str = "This Computer Use action requires macOS or Windows AX-first support. Linux only provides legacy screenshot, OCR locate, and pointer/keyboard input (X11 session required)."; +pub(crate) const LINUX_LEGACY_AX_UNAVAILABLE: &str = "[CONTROL_CAPABILITY_UNSUPPORTED] This operation has no Linux provider. Linux supports portal capture/foreground input and AT-SPI node activation/EditableText; select an advertised capability."; #[cfg(target_os = "windows")] fn app_selector_is_unspecified(app: &AppSelector) -> bool { diff --git a/src/apps/desktop/src/computer_use/desktop_host/native_control_roundtrip_tests.rs b/src/apps/desktop/src/computer_use/desktop_host/native_control_roundtrip_tests.rs index 80a55cfc0a..6de4dfc527 100644 --- a/src/apps/desktop/src/computer_use/desktop_host/native_control_roundtrip_tests.rs +++ b/src/apps/desktop/src/computer_use/desktop_host/native_control_roundtrip_tests.rs @@ -7,6 +7,62 @@ use serde_json::{json, Value}; use std::sync::Arc; const OWNER: &str = "native-control-roundtrip-fixture"; +async fn select_and_grant_fixture(host: &DesktopComputerUseHost, pid: i32, window: Option) { + let pending = host.control_snapshot(); + assert_eq!(pending.state, "awaiting_share"); + assert!(crate::computer_use::control_session::acquire(OWNER, "app_click").is_err()); + let target = crate::computer_use::share_targets() + .await + .unwrap() + .into_iter() + .find(|target| { + target.pid == Some(pid) + && window.map_or(target.title == "Computer Use Roundtrip Fixture", |id| { + target.window_id == Some(id) + }) + }) + .expect("fixture sharing choice"); + let shared = host + .share_selected_target( + pending.generation, + &target.selection_id, + openbitfun_agent_tools::computer_use_control::ControlMode::Background, + ) + .await + .unwrap(); + assert_eq!( + shared.mode, + openbitfun_agent_tools::computer_use_control::ControlMode::Observe + ); + assert!(crate::computer_use::control_session::acquire(OWNER, "app_click").is_err()); + use crate::computer_use::shared_preview::{self, SharedInputRequest, SharedPreviewRequest}; + let preview_scope = || SharedPreviewRequest { + generation: shared.generation, + target_revision: shared.target_revision, + authorization_revision: shared.sharing.as_ref().unwrap().authorization_revision, + }; + let preview = shared_preview::observe(preview_scope()).await.unwrap(); + assert!(preview.frame.as_ref().is_some_and(|frame| frame.width > 0 + && frame.height > 0 + && !frame.screenshot_id.is_empty())); + let denied = shared_preview::input(SharedInputRequest { + scope: preview_scope(), + input: openbitfun_core::agentic::tools::computer_use_host::AppInputAction::KeyChord { + keys: vec!["escape".into()], + focus_idx: None, + }, + }) + .await + .unwrap_err(); + assert!(denied.contains("CONTROL_OBSERVE_ONLY"), "{denied}"); + crate::computer_use::control_session::set_share_control( + shared.generation, + shared.target_revision, + shared.sharing.as_ref().unwrap().authorization_revision, + true, + ) + .unwrap(); +} struct StopOnExit; impl Drop for StopOnExit { fn drop(&mut self) { @@ -60,6 +116,29 @@ fn assert_session( "tool pipeline changed the foreground application; human switching also invalidates this assertion"); } +async fn wait_for_observer(path: &str) -> Value { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let ready = std::fs::read_to_string(path) + .ok() + .and_then(|data| serde_json::from_str::(&data).ok()); + if let Some(state) = ready.filter(|state| { + state["ready"] == true + && state["active"] == true + && state["key_window"] == true + && state["pid"].as_i64().map(|pid| pid as i32) + == crate::computer_use::macos_bg_input::frontmost_pid_macos() + }) { + return state; + } + assert!( + std::time::Instant::now() < deadline, + "foreground observer failed to establish the startup baseline" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } +} + pub(crate) async fn run() { if std::env::var_os("OPENBITFUN_INPUT_FIXTURE_PID").is_some() { run_covered_semantic().await; @@ -78,6 +157,9 @@ pub(crate) async fn run() { .parse() .unwrap(); let result_path = std::env::var("OPENBITFUN_ROUNDTRIP_FIXTURE_RESULT").unwrap(); + if let Ok(observer_path) = std::env::var("OPENBITFUN_INPUT_OBSERVER_RESULT") { + wait_for_observer(&observer_path).await; + } let foreground = crate::computer_use::macos_bg_input::frontmost_pid_macos(); assert_ne!( foreground, @@ -99,6 +181,7 @@ pub(crate) async fn run() { ) .await; let generation = start["generation"].as_u64().expect("control generation"); + select_and_grant_fixture(&host, pid, None).await; let apps = call(&tool, &context, json!({"action":"list_apps"})).await; assert!( apps.to_string().contains(&pid.to_string()), @@ -110,6 +193,29 @@ pub(crate) async fn run() { json!({"action":"get_app_state", "app":{"pid":pid}}), ) .await; + let old_observation_id = observation["app_state"]["observation_id"] + .as_str() + .expect("scoped observation identity") + .to_owned(); + let observation = call( + &tool, + &context, + json!({"action":"get_app_state", "app":{"pid":pid}}), + ) + .await; + let observation_id = observation["app_state"]["observation_id"].as_str().unwrap(); + assert_ne!( + old_observation_id, observation_id, + "identical trees still have different reference lifetimes" + ); + let stale = tool + .call_impl( + &json!({"action":"app_click", "app":{"pid":pid}, "observation_id":old_observation_id, + "target":{"kind":"node_idx","idx":0}}), + &context, + ) + .await; + assert!(stale.unwrap_err().to_string().contains("STALE_OBSERVATION")); let target = host .control_snapshot() .target @@ -134,7 +240,7 @@ pub(crate) async fn run() { call( &tool, &context, - json!({"action":"app_click", "app":{"pid":pid}, + json!({"action":"app_click", "app":{"pid":pid}, "observation_id":observation_id, "target":{"kind":"node_idx", "idx":button_idx}}), ) .await; @@ -222,26 +328,7 @@ async fn run_covered_semantic() { let read = |path: &str| -> Value { serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() }; - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); - loop { - let ready = std::fs::read_to_string(&observer_path) - .ok() - .and_then(|data| serde_json::from_str::(&data).ok()); - if ready.as_ref().is_some_and(|state| { - state["ready"] == true - && state["active"] == true - && state["key_window"] == true - && state["pid"].as_i64().map(|pid| pid as i32) - == crate::computer_use::macos_bg_input::frontmost_pid_macos() - }) { - break; - } - assert!( - std::time::Instant::now() < deadline, - "foreground observer failed to establish the startup baseline" - ); - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - } + wait_for_observer(&observer_path).await; let initial = read(&path); let initial_observer = read(&observer_path); assert_eq!(initial["bundle_id"], "dev.openbitfun.input-fixture.target"); @@ -271,6 +358,7 @@ async fn run_covered_semantic() { json!({"action":"start_control","mode":"background"}), ) .await; + select_and_grant_fixture(&host, pid, initial["window_id"].as_u64()).await; let observation = call( &tool, &context, @@ -452,9 +540,56 @@ async fn run_covered_semantic() { eprintln!( "PASS non-AX canvas receives one unmodified click and text without foreground change" ); + // Human preview input uses a separate host instance but the same native + // sharing resource, lease and background guard as the Agent. + use crate::computer_use::shared_preview::{self, SharedInputRequest, SharedPreviewRequest}; + let shared = host.control_snapshot(); + let scope = || SharedPreviewRequest { + generation: shared.generation, + target_revision: shared.target_revision, + authorization_revision: shared.sharing.as_ref().unwrap().authorization_revision, + }; + shared_preview::observe(scope()).await.unwrap(); + shared_preview::input(SharedInputRequest { + scope: scope(), + input: openbitfun_core::agentic::tools::computer_use_host::AppInputAction::KeyChord { + keys: vec!["escape".into()], + focus_idx: None, + }, + }) + .await + .unwrap(); + shared_preview::observe(scope()).await.unwrap(); + assert_eq!(read(&path)["canvas_escape_presses"], 1); + crate::computer_use::control_session::set_share_control( + shared.generation, + shared.target_revision, + shared.sharing.as_ref().unwrap().authorization_revision, + false, + ) + .unwrap(); + assert!(shared_preview::input(SharedInputRequest { + scope: scope(), + input: openbitfun_core::agentic::tools::computer_use_host::AppInputAction::KeyChord { + keys: vec!["escape".into()], + focus_idx: None, + }, + }) + .await + .unwrap_err() + .contains("CONTROL_STALE")); + assert!(!host.control_snapshot().sharing.unwrap().control_enabled); + eprintln!( + "PASS native preview input shares the Agent lease; revocation rejects stale preview work" + ); call(&tool, &context, json!({"action":"stop_control"})).await; tokio::time::sleep(std::time::Duration::from_millis(80)).await; let final_observer = read(&observer_path); + assert_eq!( + read(&path)["canvas_escape_presses"], + 1, + "revoked preview input must not be delivered" + ); assert_eq!( read(&path)["active"], false, diff --git a/src/apps/desktop/src/computer_use/desktop_host/observation.rs b/src/apps/desktop/src/computer_use/desktop_host/observation.rs new file mode 100644 index 0000000000..963fe52e77 --- /dev/null +++ b/src/apps/desktop/src/computer_use/desktop_host/observation.rs @@ -0,0 +1,174 @@ +//! Observation lifecycle shared by AX, UIA and AT-SPI providers. +use super::*; + +impl DesktopComputerUseHost { + pub(crate) async fn get_app_state_inner( + &self, + app: AppSelector, + max_depth: u32, + focus_window_only: bool, + capture_screenshot: bool, + ) -> OpenBitFunResult { + #[cfg(target_os = "linux")] + if !app.is_empty() { + crate::computer_use::linux_control_ax::bind_app_selector(&app).await?; + } + let scope = crate::computer_use::control_session::observation_scope() + .map_err(OpenBitFunError::tool)?; + let mut snapshot = self + .get_native_app_state_inner(app, max_depth, focus_window_only, capture_screenshot) + .await?; + let pid = snapshot.app.pid.ok_or_else(|| { + OpenBitFunError::tool("[APP_IDENTITY_UNAVAILABLE] Observation has no process identity") + })?; + let mut state = self.state_for_scope(&scope)?; + snapshot.observation_id = Some( + state + .observations + .record_application(pid) + .map_err(OpenBitFunError::tool)?, + ); + // Native node indices were rebuilt, including same-content observations. + #[cfg(any(target_os = "macos", target_os = "windows"))] + { + state.interactive_view_cache.clear(); + state.visual_mark_cache.clear(); + } + Ok(snapshot) + } + + /// Internal `get_app_state` that lets callers opt out of the focused-window + /// screenshot. The public trait method always passes `capture_screenshot=true` + /// (Codex parity). Internal re-snapshots from `app_click` / `app_type_text` / + /// `app_scroll` / `app_key_chord` pass `false` to avoid a redundant capture + /// — the **outer** call (e.g. the one returned to the model) gets the image. + async fn get_native_app_state_inner( + &self, + app: AppSelector, + max_depth: u32, + focus_window_only: bool, + capture_screenshot: bool, + ) -> OpenBitFunResult { + #[cfg(target_os = "macos")] + { + // Pre-flight: without Accessibility trust macOS silently truncates + // the AX subtree to the top-level window/container (~7 nodes for + // a Tauri WebView app), with no exception. The agent then has no + // actionable widgets to act on. Fail fast with a structured + // `[PERMISSION_DENIED]` error so the model can surface the issue + // Permission requests remain an explicit user action in settings. + macos::require_ax_trust_for( + "After granting, retry `desktop.get_app_state` and the AX tree will include all WebView subtree nodes.", + )?; + let pid = resolve_pid_macos(self, &app).await?; + let shared = crate::computer_use::control_session::snapshot(); + if shared.sharing.is_some() + && !shared + .target + .as_deref() + .is_some_and(|target| target.starts_with(&format!("pid:{pid}/window:"))) + { + return Err(OpenBitFunError::tool("[SHARED_TARGET_ONLY] Accessibility observation requires the shared application")); + } + let focus_window_only = focus_window_only || shared.sharing.is_some(); + let mut snap = crate::computer_use::control_session::spawn_blocking(move || { + // Wrap in @try/@catch — AX APIs can throw NSException for + // sandboxed / partially-loaded / dying processes, and an + // unwound foreign exception aborts the whole openbitfun process + // (`Rust cannot catch foreign exceptions, aborting`). + macos::catch_objc(|| { + crate::computer_use::macos_ax_dump::dump_app_ax( + pid, + crate::computer_use::macos_ax_dump::DumpOpts { + max_depth, + focus_window_only, + ..Default::default() + }, + ) + }) + }) + .await + .map_err(|e| OpenBitFunError::tool(e.to_string()))??; + + // Auto-attach focused-window screenshot. Failures are non-fatal — + // worst case the model still has the AX tree. + if capture_screenshot { + let started = std::time::Instant::now(); + match self.screenshot_for_app_pid(pid).await { + Ok(shot) => { + debug!( + "computer_use.app_state: attached screenshot ({}x{} jpeg, {} bytes, {}ms)", + shot.image_width, + shot.image_height, + shot.bytes.len(), + started.elapsed().as_millis() + ); + snap.screenshot = Some(shot); + } + Err(e) => { + snap.tree_text.push_str(&format!( + "\n[note] CAPTURE_UNAVAILABLE: {e}. Accessibility facts remain available; no substitute window was captured.\n" + )); + } + } + } + Ok(snap) + } + #[cfg(target_os = "windows")] + { + let (_, hwnd_raw) = self.windows_target(&app).await?; + + let mut snap = crate::computer_use::control_session::spawn_blocking(move || { + let hwnd = windows::Win32::Foundation::HWND(hwnd_raw as *mut std::ffi::c_void); + crate::computer_use::windows_ax_ui::get_app_state_snapshot_for_window( + hwnd, + max_depth, + focus_window_only, + ) + }) + .await + .map_err(|e| OpenBitFunError::tool(e.to_string()))??; + + let reg_pid = snap.app.pid.unwrap_or(0); + + // Auto-attach window screenshot (Codex parity). Failures are non-fatal. + if capture_screenshot { + let started = std::time::Instant::now(); + match self + .screenshot_for_foreground_window(reg_pid, hwnd_raw) + .await + { + Ok(shot) => { + debug!( + "computer_use.app_state: attached window screenshot ({}x{}, {} bytes, {}ms)", + shot.image_width, + shot.image_height, + shot.bytes.len(), + started.elapsed().as_millis() + ); + snap.screenshot = Some(shot); + } + Err(e) => { + snap.tree_text.push_str(&format!( + "\n[note] CAPTURE_UNAVAILABLE: {e}. Accessibility facts remain available; no substitute window was captured.\n" + )); + } + } + } + + Ok(snap) + } + #[cfg(target_os = "linux")] + { + let _ = capture_screenshot; + crate::computer_use::linux_control_ax::snapshot(app, max_depth, focus_window_only).await + } + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] + { + let _ = (app, max_depth, focus_window_only, capture_screenshot); + Err(OpenBitFunError::tool( + "[CONTROL_UNSUPPORTED] No accessibility provider", + )) + } + } +} diff --git a/src/apps/desktop/src/computer_use/desktop_host/pointer_input.rs b/src/apps/desktop/src/computer_use/desktop_host/pointer_input.rs index 3399d18dd8..74298eb385 100644 --- a/src/apps/desktop/src/computer_use/desktop_host/pointer_input.rs +++ b/src/apps/desktop/src/computer_use/desktop_host/pointer_input.rs @@ -10,7 +10,7 @@ #[cfg(target_os = "macos")] use super::macos; -use super::DesktopComputerUseHost; +use super::{DesktopComputerUseHost, PointerMap}; use enigo::{Axis, Button, Coordinate, Direction, Enigo, Key, Keyboard, Mouse, Settings}; use log::debug; #[cfg(target_os = "windows")] @@ -293,81 +293,85 @@ impl DesktopComputerUseHost { Ok(()) } - #[cfg(any(target_os = "macos", target_os = "windows"))] - pub(super) fn map_app_image_coords_to_pointer_f64( + /// Resolve image/normalized input against the observation scope and live + /// native bounds, including legacy foreground actions. + pub(super) fn observed_pointer_map( &self, - pid: i32, - x: i32, - y: i32, + expected_pid: Option, screenshot_id: Option<&str>, - ) -> OpenBitFunResult<(f64, f64)> { + ) -> OpenBitFunResult { + let scope = crate::computer_use::control_session::observation_scope() + .map_err(OpenBitFunError::tool)?; + if expected_pid.is_some_and(|pid| { + !scope + .target + .as_deref() + .is_some_and(|target| target.starts_with(&format!("pid:{pid}/window:"))) + }) { + return Err(OpenBitFunError::tool( + "[STALE_CAPTURE] Target window changed; observe it again", + )); + } let map = { - let s = self - .state - .lock() - .map_err(|e| OpenBitFunError::tool(format!("lock: {}", e)))?; - let control = crate::computer_use::control_session::snapshot(); - if s.app_pointer_targets.get(&pid) != control.target.as_ref() - || control.target.is_none() + let state = self.state_for_scope(&scope)?; + let selected = state + .observations + .image(screenshot_id) + .map_err(OpenBitFunError::tool)?; + if !state + .observations + .latest_image() + .is_some_and(|current| current.same_window_projection(&selected)) { return Err(OpenBitFunError::tool( - "[STALE_CAPTURE] Target window changed; observe it again", + "[STALE_CAPTURE] Screenshot does not match the target's current geometry", )); } - let current = s.app_pointer_maps.get(&pid).copied(); - match screenshot_id { - Some(id) => { - if s.screenshot_targets.get(id) != control.target.as_ref() { - return Err(OpenBitFunError::tool( - "[STALE_CAPTURE] Screenshot belongs to another target or expired", - )); - } - let selected = s.screenshot_pointer_maps.get(id).copied().ok_or_else(|| { - OpenBitFunError::tool( - "[STALE_CAPTURE] Unknown screenshot_id; observe the target again", - ) - })?; - if !current.is_some_and(|current| current.same_window_projection(&selected)) { - return Err(OpenBitFunError::tool("[STALE_CAPTURE] Screenshot does not match the target's current geometry")); - } - Some(selected) - } - None => current, - } + selected }; - let Some(mut map) = map else { - return Err(OpenBitFunError::tool( - "No screenshot coordinate map is available for this app. Call desktop.get_app_state for the target app first, then use app_click image_xy/image_grid against that returned screenshot_id.".to_string(), - )); - }; - if x < 0 || y < 0 || x as u32 >= map.image_w || y as u32 >= map.image_h { - return Err(OpenBitFunError::tool( - "[INVALID_COORDINATES] Point is outside the target image", - )); - } #[cfg(target_os = "macos")] - { + let map = { + let pid = scope + .target + .as_deref() + .and_then(|target| target.strip_prefix("pid:")) + .and_then(|target| target.split('/').next()) + .and_then(|id| id.parse::().ok()) + .ok_or_else(|| OpenBitFunError::tool("[STALE_CAPTURE] Missing process identity"))?; let window = crate::computer_use::macos_capture::bound_window_id(pid) .map_err(OpenBitFunError::tool)?; - let bounds = crate::computer_use::macos_capture::window_bounds(pid, window) - .map_err(OpenBitFunError::tool)?; - map = map.at_window_bounds(bounds)?; - } + map.at_window_bounds( + crate::computer_use::macos_capture::window_bounds(pid, window) + .map_err(OpenBitFunError::tool)?, + )? + }; #[cfg(target_os = "windows")] - { - let control = crate::computer_use::control_session::snapshot(); - if let Ok(Some(preview)) = - crate::computer_use::control_session::preview(control.generation) - { - map = map.at_window_bounds([ - preview.origin_x, - preview.origin_y, - preview.span_width, - preview.span_height, - ])?; - } - } - map.map_image_to_global_f64(x, y) + let map = { + let hwnd = scope + .target + .as_deref() + .and_then(|target| target.split("/window:").nth(1)) + .and_then(|id| id.parse::().ok()) + .ok_or_else(|| OpenBitFunError::tool("[STALE_CAPTURE] Missing window identity"))?; + map.at_window_bounds(crate::computer_use::windows_capture::window_bounds( + windows::Win32::Foundation::HWND(hwnd as *mut _), + )?)? + }; + crate::computer_use::control_session::validate_observation_scope(&scope) + .map_err(OpenBitFunError::tool)?; + Ok(map) + } + + #[cfg(any(target_os = "macos", target_os = "windows"))] + pub(super) fn map_app_image_coords_to_pointer_f64( + &self, + pid: i32, + x: i32, + y: i32, + screenshot_id: Option<&str>, + ) -> OpenBitFunResult<(f64, f64)> { + self.observed_pointer_map(Some(pid), screenshot_id)? + .map_image_to_global_f64(x, y) } #[cfg(any(target_os = "macos", target_os = "windows"))] @@ -453,6 +457,14 @@ impl DesktopComputerUseHost { } return Ok((pid, window)); } + if crate::computer_use::control_session::snapshot() + .sharing + .is_some() + { + return Err(OpenBitFunError::tool( + "[SHARED_TARGET_ONLY] Application was not shared by the user", + )); + } let hwnd = crate::computer_use::windows_list_apps::find_top_window_for_pid(pid as u32) .ok_or_else(|| OpenBitFunError::tool("[TARGET_UNAVAILABLE] Target has no window"))?; Ok((pid, hwnd.0 as isize)) @@ -584,16 +596,8 @@ impl DesktopComputerUseHost { x: i32, y: i32, ) -> OpenBitFunResult<(f64, f64)> { - let s = self - .state - .lock() - .map_err(|e| OpenBitFunError::tool(format!("lock: {}", e)))?; - let Some(map) = s.pointer_map else { - return Err(OpenBitFunError::tool( - "No screenshot yet in this session: run action screenshot first, then use x,y in the screenshot image pixel grid (image_width x image_height), or set use_screen_coordinates true with global screen pixels.".to_string(), - )); - }; - map.map_image_to_global_f64(x, y) + self.observed_pointer_map(None, None)? + .map_image_to_global_f64(x, y) } pub(super) fn map_normalized_coords_to_pointer_f64_impl( @@ -601,16 +605,8 @@ impl DesktopComputerUseHost { x: i32, y: i32, ) -> OpenBitFunResult<(f64, f64)> { - let s = self - .state - .lock() - .map_err(|e| OpenBitFunError::tool(format!("lock: {}", e)))?; - let Some(map) = s.pointer_map else { - return Err(OpenBitFunError::tool( - "No screenshot yet: run screenshot first. For coordinate_mode \"normalized\", use x and y each in 0..=1000.".to_string(), - )); - }; - map.map_normalized_to_global_f64(x, y) + self.observed_pointer_map(None, None)? + .map_normalized_to_global_f64(x, y) } pub(super) async fn mouse_move_global_f64_impl( @@ -675,14 +671,9 @@ impl DesktopComputerUseHost { // enigo `Coordinate::Rel` uses `location()` on macOS, which mixes NSEvent + main-display // pixel height — not the same space as `CGEvent` / our screenshot mapping. Use Quartz // position + scale from the last capture (display points per screenshot pixel). - let (dpt_x, dpt_y) = { - let state = self - .state - .lock() - .map_err(|e| OpenBitFunError::tool(format!("lock: {e}")))?; - let map = state.pointer_map.ok_or_else(|| OpenBitFunError::tool("CAPTURE_REQUIRED: Observe the authorized target before converting image-pixel deltas"))?; - map.image_delta_to_global(dx, dy)? - }; + let (dpt_x, dpt_y) = self + .observed_pointer_map(None, None)? + .image_delta_to_global(dx, dy)?; crate::computer_use::control_session::spawn_blocking(move || { Self::run_enigo_job(|e| { diff --git a/src/apps/desktop/src/computer_use/desktop_host/screenshot.rs b/src/apps/desktop/src/computer_use/desktop_host/screenshot.rs index d69511f069..aec4d6b971 100644 --- a/src/apps/desktop/src/computer_use/desktop_host/screenshot.rs +++ b/src/apps/desktop/src/computer_use/desktop_host/screenshot.rs @@ -6,7 +6,7 @@ use super::macos; use super::DesktopComputerUseHost; use image::codecs::jpeg::JpegEncoder; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "windows"))] use image::DynamicImage; use image::RgbImage; use openbitfun_core::agentic::tools::computer_use_host::{ @@ -28,177 +28,53 @@ fn bound_window_identity(target: &str) -> OpenBitFunResult<(i32, isize)> { }) } -/// macOS: map JPEG/bitmap pixels to/from **CoreGraphics global display coordinates** (same as -/// `CGDisplayBounds` / `CGEventGetLocation`): origin at the **top-left of the main display**, Y -/// increases **downward**. Not AppKit bottom-left / Y-up. -#[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug, PartialEq)] -pub(super) struct MacPointerGeo { - pub(super) disp_ox: f64, - pub(super) disp_oy: f64, - pub(super) disp_w: f64, - pub(super) disp_h: f64, - pub(super) full_px_w: u32, - pub(super) full_px_h: u32, - crop_x0: u32, - crop_y0: u32, -} - -#[cfg(target_os = "macos")] -impl MacPointerGeo { - /// Map **continuous** framebuffer pixel center `(cx, cy)` (0.5 = middle of left/top pixel) to CG global. - fn full_pixel_center_to_global_f64(&self, cx: f64, cy: f64) -> OpenBitFunResult<(f64, f64)> { - if self.disp_w <= 0.0 || self.disp_h <= 0.0 || self.full_px_w == 0 || self.full_px_h == 0 { - return Err(OpenBitFunError::tool( - "Invalid macOS pointer geometry.".to_string(), - )); - } - let px_w = self.full_px_w as f64; - let px_h = self.full_px_h as f64; - let max_cx = (self.full_px_w.saturating_sub(1) as f64) + 0.5; - let max_cy = (self.full_px_h.saturating_sub(1) as f64) + 0.5; - let cx = cx.clamp(0.5, max_cx); - let cy = cy.clamp(0.5, max_cy); - let gx = self.disp_ox + (cx / px_w) * self.disp_w; - let gy = self.disp_oy + (cy / px_h) * self.disp_h; - Ok((gx, gy)) - } -} - +/// One validated image-to-surface transform shared by all native backends. #[derive(Clone, Copy, Debug, PartialEq)] -pub(super) struct PointerMap { - /// Screenshot JPEG width/height (same as capture when there is no frame padding). - pub(super) image_w: u32, - pub(super) image_h: u32, - /// Top-left of capture inside the JPEG (0 when there is no padding). - content_origin_x: u32, - content_origin_y: u32, - /// Native capture pixel size (the cropped/visible bitmap). - content_w: u32, - content_h: u32, - pub(super) native_w: u32, - pub(super) native_h: u32, - pub(super) origin_x: i32, - pub(super) origin_y: i32, - #[cfg(target_os = "macos")] - pub(super) macos_geo: Option, -} +pub(super) struct PointerMap(openbitfun_agent_tools::computer_use_geometry::CaptureGeometry); impl PointerMap { - /// Relative image-pixel displacement uses the same scale as the captured - /// image, without clamping a delta to an absolute image location. + fn from_screenshot(shot: &ComputerScreenshot) -> OpenBitFunResult { + let content = shot.image_content_rect.as_ref().ok_or_else(|| { + OpenBitFunError::tool("[INVALID_GEOMETRY] Capture content rectangle unavailable") + })?; + let bounds = shot.image_global_bounds.as_ref().ok_or_else(|| { + OpenBitFunError::tool( + "[COORDINATES_UNAVAILABLE] Capture has no authoritative surface bounds", + ) + })?; + openbitfun_agent_tools::computer_use_geometry::CaptureGeometry::new( + shot.image_width, + shot.image_height, + content, + bounds, + ) + .map(Self) + .map_err(OpenBitFunError::tool) + } #[cfg(target_os = "macos")] pub(super) fn image_delta_to_global(&self, dx: i32, dy: i32) -> OpenBitFunResult<(f64, f64)> { - if self.content_w == 0 || self.content_h == 0 || self.native_w == 0 || self.native_h == 0 { - return Err(OpenBitFunError::tool( - "Invalid screenshot coordinate map (zero dimension)", - )); - } - let mut sx = self.native_w as f64 / self.content_w as f64; - let mut sy = self.native_h as f64 / self.content_h as f64; - if let Some(geo) = self.macos_geo { - if geo.full_px_w == 0 - || geo.full_px_h == 0 - || !geo.disp_w.is_finite() - || !geo.disp_h.is_finite() - || geo.disp_w <= 0.0 - || geo.disp_h <= 0.0 - { - return Err(OpenBitFunError::tool("Invalid macOS pointer geometry")); - } - sx *= geo.disp_w / geo.full_px_w as f64; - sy *= geo.disp_h / geo.full_px_h as f64; - } - Ok((dx as f64 * sx, dy as f64 * sy)) + Ok(self.0.image_delta_to_global(dx, dy)) } - - /// Window-local pixels survive a translation, but not resizing or a change - /// in padding/scaling. Compare every mapping field except the global origin. pub(super) fn same_window_projection(&self, other: &Self) -> bool { - let mut translated = *self; - translated.origin_x = other.origin_x; - translated.origin_y = other.origin_y; - translated == *other + self.0.same_projection(&other.0) } - - pub(super) fn at_window_bounds(mut self, bounds: [f64; 4]) -> OpenBitFunResult { - if bounds.iter().any(|v| !v.is_finite()) - || (bounds[2] - self.native_w as f64).abs() > 1.0 - || (bounds[3] - self.native_h as f64).abs() > 1.0 - { - return Err(OpenBitFunError::tool( - "[STALE_CAPTURE] Target resized; observe again before coordinate input", - )); - } - self.origin_x = bounds[0].round() as i32; - self.origin_y = bounds[1].round() as i32; - Ok(self) + pub(super) fn at_window_bounds(self, bounds: [f64; 4]) -> OpenBitFunResult { + self.0 + .at_bounds(bounds) + .map(Self) + .map_err(OpenBitFunError::tool) } - - /// Continuous mapping: **composed JPEG** pixel `(x,y)` -> global (macOS CG). pub(super) fn map_image_to_global_f64(&self, x: i32, y: i32) -> OpenBitFunResult<(f64, f64)> { - if self.image_w == 0 - || self.image_h == 0 - || self.content_w == 0 - || self.content_h == 0 - || self.native_w == 0 - || self.native_h == 0 - { - return Err(OpenBitFunError::tool( - "Invalid screenshot coordinate map (zero dimension).".to_string(), - )); - } - let ox = self.content_origin_x as i32; - let oy = self.content_origin_y as i32; - let cx_img = x - ox; - let cy_img = y - oy; - let max_cx = self.content_w.saturating_sub(1) as i32; - let max_cy = self.content_h.saturating_sub(1) as i32; - let cx_img = cx_img.clamp(0, max_cx) as f64; - let cy_img = cy_img.clamp(0, max_cy) as f64; - let cw = self.content_w as f64; - let ch = self.content_h as f64; - let nw = self.native_w as f64; - let nh = self.native_h as f64; - - #[cfg(target_os = "macos")] - if let Some(g) = self.macos_geo { - let cx = g.crop_x0 as f64 + (cx_img + 0.5) * nw / cw; - let cy = g.crop_y0 as f64 + (cy_img + 0.5) * nh / ch; - return g.full_pixel_center_to_global_f64(cx, cy); - } - - let center_full_x = self.origin_x as f64 + (cx_img + 0.5) * nw / cw; - let center_full_y = self.origin_y as f64 + (cy_img + 0.5) * nh / ch; - Ok((center_full_x, center_full_y)) + self.0.image_to_global(x, y).map_err(OpenBitFunError::tool) } - - /// Normalized 0..=1000 maps to the **capture** bitmap. pub(super) fn map_normalized_to_global_f64( &self, x: i32, y: i32, ) -> OpenBitFunResult<(f64, f64)> { - if self.native_w == 0 || self.native_h == 0 { - return Err(OpenBitFunError::tool( - "Invalid screenshot coordinate map (zero native dimension).".to_string(), - )); - } - let nw = self.native_w as f64; - let nh = self.native_h as f64; - let tx = (x.clamp(0, 1000) as f64) / 1000.0; - let ty = (y.clamp(0, 1000) as f64) / 1000.0; - - #[cfg(target_os = "macos")] - if let Some(g) = self.macos_geo { - let cx = g.crop_x0 as f64 + tx * (nw - 1.0).max(0.0) + 0.5; - let cy = g.crop_y0 as f64 + ty * (nh - 1.0).max(0.0) + 0.5; - return g.full_pixel_center_to_global_f64(cx, cy); - } - - let gx = self.origin_x as f64 + tx * (nw - 1.0).max(0.0) + 0.5; - let gy = self.origin_y as f64 + ty * (nh - 1.0).max(0.0) + 0.5; - Ok((gx, gy)) + self.0 + .normalized_to_global(x, y) + .map_err(OpenBitFunError::tool) } } @@ -335,9 +211,16 @@ impl DesktopComputerUseHost { pid: i32, update_navigation: bool, ) -> OpenBitFunResult { - let cap = crate::computer_use::control_session::spawn_blocking(move || { - crate::computer_use::macos_capture::capture_frame(pid, None) - .map_err(OpenBitFunError::tool) + let (cap, scope) = crate::computer_use::control_session::spawn_blocking(move || { + crate::computer_use::macos_capture::ensure_capture(pid, None) + .map_err(OpenBitFunError::tool)?; + let scope = crate::computer_use::control_session::observation_scope() + .map_err(OpenBitFunError::tool)?; + let cap = crate::computer_use::macos_capture::capture_frame(pid, None) + .map_err(OpenBitFunError::tool)?; + crate::computer_use::control_session::validate_observation_scope(&scope) + .map_err(OpenBitFunError::tool)?; + Ok::<_, OpenBitFunError>((cap, scope)) }) .await .map_err(|e| OpenBitFunError::tool(e.to_string()))??; @@ -349,35 +232,24 @@ impl DesktopComputerUseHost { let logical_w = cap.bounds[2].round() as u32; let logical_h = cap.bounds[3].round() as u32; let mut shot = Self::raw_shot_from_rgb_crop(rgb, origin_x, origin_y, logical_w, logical_h)?; + shot.image_global_bounds = Some(ComputerUseImageGlobalBounds { + left: cap.bounds[0], + top: cap.bounds[1], + width: cap.bounds[2], + height: cap.bounds[3], + }); shot.quadrant_navigation_click_ready = true; - let map = PointerMap { - image_w: shot.image_width, - image_h: shot.image_height, - content_origin_x: 0, - content_origin_y: 0, - content_w: shot.image_width, - content_h: shot.image_height, - native_w: logical_w, - native_h: logical_h, - origin_x, - origin_y, - macos_geo: None, - }; + crate::computer_use::control_session::validate_observation_scope(&scope) + .map_err(OpenBitFunError::tool)?; + let map = PointerMap::from_screenshot(&shot)?; if !update_navigation { return Ok(shot); } - let mut state = self - .state - .lock() - .map_err(|e| OpenBitFunError::tool(format!("lock: {e}")))?; - state.transition_after_screenshot(map); - state.app_pointer_maps.insert(pid, map); - let target = format!("pid:{pid}/window:{}", cap.window_id); - state.app_pointer_targets.insert(pid, target.clone()); - if let Some(id) = shot.screenshot_id.clone() { - state.screenshot_targets.insert(id.clone(), target); - state.screenshot_pointer_maps.insert(id, map); - } + let mut state = self.state_for_scope(&scope)?; + state.transition_after_screenshot( + shot.screenshot_id.clone().expect("capture identity"), + map, + ); Ok(shot) } @@ -408,6 +280,8 @@ impl DesktopComputerUseHost { ) -> OpenBitFunResult { use windows::Win32::Foundation::HWND; + let scope = crate::computer_use::control_session::observation_scope() + .map_err(OpenBitFunError::tool)?; let cap = crate::computer_use::control_session::spawn_blocking(move || { let hwnd = HWND(hwnd_raw as *mut std::ffi::c_void); crate::computer_use::windows_capture::screenshot_window_capture(hwnd) @@ -415,9 +289,9 @@ impl DesktopComputerUseHost { .await .map_err(|e| OpenBitFunError::tool(e.to_string()))??; - let img = image::load_from_memory(&cap.png) - .map_err(|e| OpenBitFunError::tool(format!("decode window capture PNG: {}", e)))?; - let rgb = img.to_rgb8(); + let rgba = image::RgbaImage::from_raw(cap.width, cap.height, cap.rgba) + .ok_or_else(|| OpenBitFunError::tool("[CAPTURE_INVALID_FRAME] Invalid RGBA buffer"))?; + let rgb = DynamicImage::ImageRgba8(rgba).to_rgb8(); let native_w = rgb.width(); let native_h = rgb.height(); @@ -427,31 +301,14 @@ impl DesktopComputerUseHost { // Image pixels map 1:1 to the captured window rectangle (no downscale), // so content == image == native and the screen origin is the window's // (DWM-frame-adjusted) top-left. - let map = PointerMap { - image_w: shot.image_width, - image_h: shot.image_height, - content_origin_x: 0, - content_origin_y: 0, - content_w: shot.image_width, - content_h: shot.image_height, - native_w, - native_h, - origin_x: cap.origin_x, - origin_y: cap.origin_y, - }; + let map = PointerMap::from_screenshot(&shot)?; + crate::computer_use::control_session::validate_observation_scope(&scope) + .map_err(OpenBitFunError::tool)?; if update_navigation { - let mut s = self - .state - .lock() - .map_err(|e| OpenBitFunError::tool(format!("lock: {}", e)))?; - s.transition_after_screenshot(map); - s.app_pointer_maps.insert(pid, map); - let target = format!("pid:{pid}/window:{hwnd_raw}"); - s.app_pointer_targets.insert(pid, target.clone()); - if let Some(id) = shot.screenshot_id.clone() { - s.screenshot_targets.insert(id.clone(), target); - s.screenshot_pointer_maps.insert(id, map); - } + self.state_for_scope(&scope)?.transition_after_screenshot( + shot.screenshot_id.clone().expect("capture identity"), + map, + ); } Ok(shot) } @@ -466,7 +323,11 @@ impl DesktopComputerUseHost { session: &crate::computer_use::linux_control::LinuxControlSession, update_navigation: bool, ) -> OpenBitFunResult { + let scope = crate::computer_use::control_session::observation_scope() + .map_err(OpenBitFunError::tool)?; let jpeg = session.capture().await.map_err(OpenBitFunError::tool)?; + crate::computer_use::control_session::validate_observation_scope(&scope) + .map_err(OpenBitFunError::tool)?; let rgb = image::load_from_memory(&jpeg) .map_err(|e| OpenBitFunError::tool(format!("[CAPTURE_INVALID_FRAME] {e}")))? .to_rgb8(); @@ -481,33 +342,20 @@ impl DesktopComputerUseHost { if size.is_none() { shot.image_global_bounds = None; } - let map = PointerMap { - image_w: shot.image_width, - image_h: shot.image_height, - content_origin_x: 0, - content_origin_y: 0, - content_w: shot.image_width, - content_h: shot.image_height, - native_w: logical_w, - native_h: logical_h, - origin_x: 0, - origin_y: 0, - }; - if update_navigation { - let mut state = self - .state - .lock() - .map_err(|e| OpenBitFunError::tool(format!("lock: {e}")))?; - state.transition_after_screenshot(map); - if let Some(id) = shot.screenshot_id.clone() { - state.screenshot_pointer_maps.insert(id, map); - } + if update_navigation && shot.image_global_bounds.is_none() { + self.state_for_scope(&scope)?.observations.clear_images(); } - let target = session.target_identity(); - crate::computer_use::control_session::publish_jpeg_generation( - session.generation(), - &target, + if update_navigation && shot.image_global_bounds.is_some() { + let map = PointerMap::from_screenshot(&shot)?; + self.state_for_scope(&scope)?.transition_after_screenshot( + shot.screenshot_id.clone().expect("capture identity"), + map, + ); + } + crate::computer_use::control_session::publish_jpeg( + &scope, &shot.bytes, + [shot.image_width, shot.image_height], [0.0, 0.0, logical_w as f64, logical_h as f64], ); Ok(shot) @@ -737,20 +585,25 @@ mod window_projection_tests { } fn capture_map() -> PointerMap { - PointerMap { - image_w: 900, - image_h: 600, - content_origin_x: 0, - content_origin_y: 0, - content_w: 900, - content_h: 600, - native_w: 1800, - native_h: 1200, - origin_x: 139, - origin_y: 40, - #[cfg(target_os = "macos")] - macos_geo: None, - } + PointerMap( + openbitfun_agent_tools::computer_use_geometry::CaptureGeometry::new( + 900, + 600, + &super::ComputerUseImageContentRect { + left: 0, + top: 0, + width: 900, + height: 600, + }, + &super::ComputerUseImageGlobalBounds { + left: 139.0, + top: 40.0, + width: 1800.0, + height: 1200.0, + }, + ) + .unwrap(), + ) } #[cfg(target_os = "macos")] @@ -758,13 +611,19 @@ mod window_projection_tests { fn relative_movement_after_capture_uses_geometry_without_ocr_navigation_gate() { let host = DesktopComputerUseHost::new(); let map = capture_map(); - host.state.lock().unwrap().transition_after_screenshot(map); - let retained = host.state.lock().unwrap().pointer_map.unwrap(); + host.state + .lock() + .unwrap() + .transition_after_screenshot("fixture".into(), map); + let retained = host + .state + .lock() + .unwrap() + .observations + .latest_image() + .unwrap(); assert_eq!(retained.image_delta_to_global(4, -3).unwrap(), (8.0, -6.0)); assert!(host.last_screenshot_refinement().is_none()); - let mut invalid = retained; - invalid.content_w = 0; - assert!(invalid.image_delta_to_global(4, -3).is_err()); // Absolute coordinates still use the existing validated map; this // delta helper does not waive target binding, leases or OS authority. assert_eq!(retained, map); @@ -791,12 +650,25 @@ mod window_projection_tests { assert!(observed .at_window_bounds([f64::NAN, 40.0, 1800.0, 1200.0]) .is_err()); - let mut resampled = observed; - resampled.image_w /= 2; - resampled.content_w /= 2; + let resampled = PointerMap( + openbitfun_agent_tools::computer_use_geometry::CaptureGeometry::new( + 450, + 300, + &super::ComputerUseImageContentRect { + left: 0, + top: 0, + width: 450, + height: 300, + }, + &super::ComputerUseImageGlobalBounds { + left: 139.0, + top: 40.0, + width: 1800.0, + height: 1200.0, + }, + ) + .unwrap(), + ); assert!(!observed.same_window_projection(&resampled)); - let mut padded = observed; - padded.content_origin_x = 10; - assert!(!observed.same_window_projection(&padded)); } } diff --git a/src/apps/desktop/src/computer_use/desktop_host/shared_preview.rs b/src/apps/desktop/src/computer_use/desktop_host/shared_preview.rs new file mode 100644 index 0000000000..e0423f8037 --- /dev/null +++ b/src/apps/desktop/src/computer_use/desktop_host/shared_preview.rs @@ -0,0 +1,314 @@ +//! Human preview control uses the existing host and lease; animation never +//! submits input. Image coordinates carry the immutable capture identity. +use super::*; +use crate::computer_use::control_session; +use openbitfun_agent_tools::computer_use_control::{ControlSnapshot, ObservationScope}; +use openbitfun_core::agentic::tools::computer_use_host::{ + AppInputAction, ClickTarget, ComputerUseActionLease, ComputerUseImageContentRect, + ComputerUseImageGlobalBounds, +}; +use serde::{Deserialize, Serialize}; +use std::sync::OnceLock; + +pub(crate) fn host() -> &'static DesktopComputerUseHost { + static HOST: OnceLock = OnceLock::new(); + HOST.get_or_init(DesktopComputerUseHost::new) +} + +#[derive(Debug, Deserialize)] +pub struct SharedPreviewRequest { + pub generation: u64, + pub target_revision: u64, + pub authorization_revision: u64, +} +impl SharedPreviewRequest { + fn acquire( + &self, + action: &str, + ) -> Result<(ControlSnapshot, Box), String> { + control_session::acquire_shared( + self.generation, + self.target_revision, + self.authorization_revision, + action, + ) + } +} + +#[derive(Serialize)] +pub struct SharedFrame { + pub screenshot_id: String, + pub image_base64: String, + pub mime_type: String, + pub width: u32, + pub height: u32, + pub content: ComputerUseImageContentRect, + pub bounds: Option, +} +#[derive(Serialize)] +pub struct SharedObservation { + pub control: ControlSnapshot, + pub frame: Option, + pub tree: Option, +} +#[derive(Debug, Deserialize)] +pub struct SharedInputRequest { + #[serde(flatten)] + pub scope: SharedPreviewRequest, + pub input: AppInputAction, +} + +fn pid(control: &ControlSnapshot) -> Result { + let target = control + .target + .as_deref() + .ok_or("[SHARE_REQUIRED] No shared target")?; + target + .strip_prefix("pid:") + .and_then(|value| value.split('/').next()) + .or_else(|| target.strip_prefix("atspi:")) + .and_then(|value| value.parse().ok()) + .ok_or_else(|| "[TARGET_REQUIRED] Shared target has no application process".into()) +} + +pub(crate) async fn observe(request: SharedPreviewRequest) -> Result { + let (control, mut lease) = request.acquire("get_app_state")?; + let result = capture(&control).await; + lease.complete(); + result +} + +async fn capture(control: &ControlSnapshot) -> Result { + let scope = ObservationScope::from_control(control); + let shot: ComputerScreenshot; + #[cfg(target_os = "macos")] + { + shot = host() + .screenshot_for_app_pid(pid(control)?) + .await + .map_err(|e| e.to_string())?; + } + #[cfg(target_os = "windows")] + { + let hwnd = control + .target + .as_deref() + .and_then(|target| target.split("/window:").nth(1)) + .and_then(|window| window.parse().ok()) + .ok_or("[TARGET_REQUIRED] Missing shared window")?; + shot = host() + .screenshot_for_foreground_window(pid(control)?, hwnd) + .await + .map_err(|e| e.to_string())?; + } + #[cfg(target_os = "linux")] + { + if control + .target + .as_deref() + .is_some_and(|target| target.starts_with("atspi:")) + { + let state = crate::computer_use::linux_control_ax::preview_snapshot( + AppSelector::by_pid(pid(control)?), + 8, + ) + .await + .map_err(|e| e.to_string())?; + control_session::validate_observation_scope(&scope)?; + return Ok(SharedObservation { + control: control_session::snapshot(), + frame: None, + tree: Some(state.tree_text), + }); + } + shot = host() + .screenshot_display(Default::default()) + .await + .map_err(|e| e.to_string())?; + } + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] + { + let _ = scope; + return Err("[CONTROL_UNSUPPORTED] No shared preview provider".into()); + } + #[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))] + { + use base64::Engine; + control_session::validate_observation_scope(&scope)?; + Ok(SharedObservation { + control: control_session::snapshot(), + tree: None, + frame: Some(SharedFrame { + screenshot_id: shot + .screenshot_id + .ok_or("[CAPTURE_ID_REQUIRED] Missing frame identity")?, + image_base64: base64::engine::general_purpose::STANDARD.encode(shot.bytes), + mime_type: shot.mime_type, + width: shot.image_width, + height: shot.image_height, + content: shot + .image_content_rect + .unwrap_or(ComputerUseImageContentRect { + left: 0, + top: 0, + width: shot.image_width, + height: shot.image_height, + }), + bounds: shot.image_global_bounds, + }), + }) + } +} + +fn validate_preview_input(input: &AppInputAction) -> Result<(), String> { + input.validate()?; + let image = |target: &ClickTarget| { + matches!( + target, + ClickTarget::ImageXy { + screenshot_id: Some(_), + .. + } + ) + }; + let valid = match input { + AppInputAction::Click { target, .. } => image(target), + AppInputAction::TypeText { focus, .. } | AppInputAction::Scroll { focus, .. } => { + focus.as_ref().is_none_or(image) + } + AppInputAction::KeyChord { focus_idx, .. } => focus_idx.is_none(), + AppInputAction::Drag { from, to, .. } => image(from) && image(to), + AppInputAction::Wait { .. } => false, + }; + if valid { + Ok(()) + } else { + Err("[PREVIEW_IMAGE_REQUIRED] Preview input requires coordinates from its observed image; AX indices belong to the Agent observation".into()) + } +} + +pub(crate) async fn input(request: SharedInputRequest) -> Result { + validate_preview_input(&request.input)?; + let (control, mut lease) = request.scope.acquire(request.input.name())?; + #[cfg(target_os = "linux")] + let result = if control + .target + .as_deref() + .is_some_and(|target| target.starts_with("portal:")) + { + portal_input(request.input).await + } else { + host() + .dispatch_app_input(AppSelector::by_pid(pid(&control)?), request.input) + .await + }; + #[cfg(not(target_os = "linux"))] + let result = host() + .dispatch_app_input(AppSelector::by_pid(pid(&control)?), request.input) + .await; + lease.complete(); + if let Err(error) = result { + // A partial gesture must not be retried by the preview, nor keep its grant. + if let Some(sharing) = &control.sharing { + let _ = control_session::set_share_control( + control.generation, + control.target_revision, + sharing.authorization_revision, + false, + ); + } + return Err(error.to_string()); + } + control_session::validate_observation_scope(&ObservationScope::from_control(&control))?; + Ok(control_session::snapshot()) +} + +#[cfg(target_os = "linux")] +async fn portal_input(input: AppInputAction) -> OpenBitFunResult<()> { + control_session::foreground_allowed().map_err(OpenBitFunError::tool)?; + // Portal coordinates are stream-local. They never address an AT-SPI PID. + let point = |target: ClickTarget| -> OpenBitFunResult<(f64, f64)> { + if let ClickTarget::ImageXy { + x, + y, + screenshot_id: Some(id), + } = target + { + host() + .observed_pointer_map(None, Some(&id))? + .map_image_to_global_f64(x, y) + } else { + Err(OpenBitFunError::tool( + "[PREVIEW_IMAGE_REQUIRED] Missing image coordinates", + )) + } + }; + match input { + AppInputAction::Click { + target, + click_count, + mouse_button, + modifier_keys, + .. + } => { + if !modifier_keys.is_empty() { + return Err(OpenBitFunError::tool( + "[PREVIEW_UNSUPPORTED] Portal modified clicks are unavailable", + )); + } + let (x, y) = point(target)?; + host().mouse_move_global_f64(x, y).await?; + for _ in 0..click_count { + host().mouse_click_authoritative(&mouse_button).await?; + } + Ok(()) + } + AppInputAction::TypeText { text, focus: None } => host().type_text(&text).await, + AppInputAction::KeyChord { keys, .. } => host().key_chord(keys).await, + AppInputAction::Scroll { dx, dy, focus } => { + if let Some(target) = focus { + let (x, y) = point(target)?; + host().mouse_move_global_f64(x, y).await?; + } + host().scroll(dx, dy).await + } + AppInputAction::Drag { + from, + to, + mouse_button, + duration_ms, + } => { + host() + .drag(point(from)?, point(to)?, &mouse_button, duration_ms) + .await + } + _ => Err(OpenBitFunError::tool( + "[PREVIEW_UNSUPPORTED] Unsupported portal preview input", + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn preview_cannot_smuggle_agent_nodes_or_unscoped_coordinates() { + for target in [ + ClickTarget::NodeIdx { idx: 1 }, + ClickTarget::ImageXy { + x: 1, + y: 1, + screenshot_id: None, + }, + ] { + assert!(validate_preview_input(&AppInputAction::Click { + target, + click_count: 1, + mouse_button: "left".into(), + modifier_keys: vec![], + wait_ms_after: None + }) + .is_err()); + } + } +} diff --git a/src/apps/desktop/src/computer_use/desktop_host/sharing.rs b/src/apps/desktop/src/computer_use/desktop_host/sharing.rs new file mode 100644 index 0000000000..f5cb9ef44e --- /dev/null +++ b/src/apps/desktop/src/computer_use/desktop_host/sharing.rs @@ -0,0 +1,219 @@ +//! User-selected native sharing, built on the same capture and input provider +//! as ComputerUse. No separate browser, input loop or permission owner. +use super::*; +use openbitfun_agent_tools::computer_use_control::{ControlMode, ControlSnapshot, ShareTarget}; + +async fn native_targets() -> Result, String> { + #[cfg(target_os = "macos")] + { + return tokio::task::spawn_blocking( + crate::computer_use::macos_list_apps::list_share_targets, + ) + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string()); + } + #[cfg(target_os = "windows")] + { + return Ok(crate::computer_use::windows_list_apps::list_share_targets()); + } + #[cfg(target_os = "linux")] + { + let mut result = vec![ShareTarget { + selection_id: String::new(), + unavailable_reason: None, + id: "portal:pending".into(), + pid: None, + window_id: None, + application: "Desktop portal".into(), + title: String::new(), + bundle_id: None, + background_pointer: false, + background_semantic: false, + selection_required: true, + }]; + result.extend( + crate::computer_use::linux_control_ax::list_apps() + .await + .map_err(|e| e.to_string())? + .into_iter() + .filter_map(|app| { + app.pid.map(|pid| ShareTarget { + selection_id: String::new(), + unavailable_reason: None, + id: format!("atspi:{pid}"), + pid: Some(pid), + window_id: None, + application: app.name, + title: String::new(), + bundle_id: None, + background_pointer: false, + background_semantic: true, + selection_required: false, + }) + }), + ); + return Ok(result); + } + #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] + Err("[CONTROL_UNSUPPORTED] No native sharing provider".into()) +} + +use crate::computer_use::process_identity::ProcessIdentity; +use std::{ + collections::{hash_map::RandomState, VecDeque}, + hash::{BuildHasher, Hasher}, + sync::{Mutex, OnceLock}, +}; +#[derive(Clone)] +struct Choice { + target: ShareTarget, + process: Option, +} +static CHOICES: OnceLock>> = OnceLock::new(); + +pub(crate) async fn targets() -> Result, String> { + let mut targets = native_targets().await?; + let result = tokio::task::spawn_blocking(move || { + let mut choices = CHOICES + .get_or_init(Default::default) + .lock() + .map_err(|_| "[CONTROL_FAILED] Target inventory lock poisoned")?; + for target in &mut targets { + target.selection_id = + format!("window_{:016x}", RandomState::new().build_hasher().finish()); + let process = target.pid.map(ProcessIdentity::read).transpose(); + match process { + Ok(process) => choices.push_back(Choice { + target: target.clone(), + process, + }), + Err(error) => target.unavailable_reason = Some(error), + } + } + // Bounded discovery references are distinct from input leases. Older + // references are rejected, never resolved to a new process with its PID. + while choices.len() > 512 { + choices.pop_front(); + } + Ok::<_, String>(targets) + }) + .await + .map_err(|e| e.to_string())?; + result +} + +impl DesktopComputerUseHost { + pub(crate) async fn share_selected_target( + &self, + generation: u64, + target_id: &str, + mode: ControlMode, + ) -> Result { + // Re-resolve the inventory at the authorization boundary. A title or + // client-provided PID is not evidence that the original window exists. + let choice = CHOICES + .get_or_init(Default::default) + .lock() + .map_err(|_| "[CONTROL_FAILED] Target inventory lock poisoned")? + .iter() + .find(|choice| choice.target.selection_id == target_id) + .cloned() + .ok_or( + "[TARGET_WINDOW_UNAVAILABLE] Target selection expired; refresh the window list", + )?; + if let Some(process) = &choice.process { + process.validate()?; + } + let target = choice.target; + if let Some(reason) = target.unavailable_reason { + return Err(reason); + } + if target.selection_required && mode == ControlMode::Background { + return Err("[BACKGROUND_UNSUPPORTED] Portal input controls the foreground seat. Select an AT-SPI application for background semantic control, or explicitly choose foreground mode.".into()); + } + let label = if target.title.is_empty() { + target.application.clone() + } else { + format!("{} — {}", target.application, target.title) + }; + let (started, mut lease) = crate::computer_use::control_session::begin_share( + generation, + target.id.clone(), + label, + mode, + choice.process, + )?; + let result = self.prepare_shared_surface(&target, mode).await; + if let Err(error) = result { + let _ = crate::computer_use::control_session::stop_checked( + None, + Some(started.generation), + &error, + ); + lease.complete(); + return Err(error); + } + let current = crate::computer_use::control_session::snapshot(); + let snapshot = crate::computer_use::control_session::finish_share( + started.generation, + current.target_revision, + )?; + lease.complete(); + Ok(snapshot) + } + + async fn prepare_shared_surface( + &self, + target: &ShareTarget, + mode: ControlMode, + ) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + let _ = mode; + let pid = target.pid.ok_or("[TARGET_REQUIRED] Missing process")?; + let window = target + .window_id + .and_then(|id| u32::try_from(id).ok()) + .ok_or("[TARGET_REQUIRED] Missing window")?; + crate::computer_use::control_session::spawn_blocking(move || { + crate::computer_use::macos_capture::ensure_capture(pid, Some(window)) + }) + .await + .map_err(|e| e.to_string())??; + self.screenshot_for_app_pid(pid) + .await + .map_err(|e| e.to_string())?; + } + #[cfg(target_os = "windows")] + { + let _ = mode; + let pid = target.pid.ok_or("[TARGET_REQUIRED] Missing process")?; + let window = target + .window_id + .and_then(|id| isize::try_from(id).ok()) + .ok_or("[TARGET_REQUIRED] Missing window")?; + self.screenshot_for_foreground_window(pid, window) + .await + .map_err(|e| e.to_string())?; + } + #[cfg(target_os = "linux")] + if target.selection_required { + crate::computer_use::linux_control::start_session(mode == ControlMode::Foreground) + .await?; + self.screenshot_display(Default::default()) + .await + .map_err(|e| e.to_string())?; + } else { + self.get_app_state_inner( + AppSelector::by_pid(target.pid.ok_or("[TARGET_REQUIRED] Missing process")?), + 8, + false, + false, + ) + .await + .map_err(|e| e.to_string())?; + } + Ok(()) + } +} diff --git a/src/apps/desktop/src/computer_use/desktop_host/state.rs b/src/apps/desktop/src/computer_use/desktop_host/state.rs new file mode 100644 index 0000000000..06790b029b --- /dev/null +++ b/src/apps/desktop/src/computer_use/desktop_host/state.rs @@ -0,0 +1,125 @@ +//! Host-local observations and interaction feedback; control authority lives in control_session. +use super::PointerMap; +use openbitfun_core::agentic::tools::computer_use_host::ComputerUseLastMutationKind; +#[cfg(any(target_os = "macos", target_os = "windows"))] +use openbitfun_core::agentic::tools::computer_use_host::VisualMark; +use openbitfun_core::agentic::tools::computer_use_optimizer::ComputerUseOptimizer; + +/// Unified mutable session state for computer use — one mutex instead of five. +/// State transitions are applied centrally after each action (screenshot, pointer move, click, etc.). +#[derive(Debug)] +pub(super) struct ComputerUseSessionMutableState { + pub(super) observations: crate::computer_use::observation_cache::ObservationCache, + /// When true, a fresh `screenshot_display` is required before `click` and before `key_chord` that sends Return/Enter + /// (set after pointer moves / click; cleared after screenshot). + pub(super) click_needs_fresh_screenshot: bool, + /// After click / key / type / scroll / drag: recommend a **`screenshot`** to confirm UI state (Cowork verify). + /// Cleared on the next successful `screenshot_display`. + pub(super) pending_verify_screenshot: bool, + /// Bounded action evidence; hashes alone never verify task completion. + pub(super) optimizer: ComputerUseOptimizer, + /// Most-recent action **kind** that mutated UI / pointer state. Surfaced + /// to the model via `interaction_state.last_mutation` so it can pair the + /// right verification step (e.g. after `Click` + `pending_verify` ⇒ take + /// a confirming `screenshot`; after `TypeText` ⇒ may chain Enter without + /// re-screenshotting because typing does not move the pointer). + pub(super) last_mutation_kind: Option, + /// Caller-pinned target display (set via `desktop.focus_display`). + /// When set, all subsequent screenshots / peeks / locates use this + /// display instead of "screen under the mouse pointer". The model + /// uses this to disambiguate multi-monitor targets explicitly. + pub(super) preferred_display_id: Option, + /// Most-recent Set-of-Mark interactive view per pid. Used to resolve + /// `interactive_*` numeric `i` indices back to AX node indices and to + /// detect stale-view usage via `before_view_digest`. + #[cfg(any(target_os = "macos", target_os = "windows"))] + pub(super) interactive_view_cache: std::collections::HashMap, + #[cfg(any(target_os = "macos", target_os = "windows"))] + pub(super) visual_mark_cache: std::collections::HashMap, +} + +#[derive(Debug, Clone)] +#[cfg(any(target_os = "macos", target_os = "windows"))] +pub(super) struct CachedInteractiveView { + pub(super) digest: String, + /// `i` → `node_idx` map (dense, indexed by `i`). + pub(super) elements: + Vec, +} + +#[derive(Debug, Clone)] +#[cfg(any(target_os = "macos", target_os = "windows"))] +pub(super) struct CachedVisualMarkView { + pub(super) digest: String, + pub(super) marks: Vec, + pub(super) screenshot_id: Option, +} + +impl ComputerUseSessionMutableState { + pub(super) fn synchronize( + &mut self, + scope: &openbitfun_agent_tools::computer_use_control::ObservationScope, + ) { + if self.observations.synchronize(scope) { + self.clear_derived_state(); + } + } + fn clear_derived_state(&mut self) { + self.click_needs_fresh_screenshot = true; + self.pending_verify_screenshot = false; + self.last_mutation_kind = None; + self.optimizer.clear_history(); + #[cfg(any(target_os = "macos", target_os = "windows"))] + { + self.interactive_view_cache.clear(); + self.visual_mark_cache.clear(); + } + } + + pub(super) fn new() -> Self { + Self { + observations: Default::default(), + click_needs_fresh_screenshot: true, + pending_verify_screenshot: false, + optimizer: ComputerUseOptimizer::new(), + last_mutation_kind: None, + preferred_display_id: None, + #[cfg(any(target_os = "macos", target_os = "windows"))] + interactive_view_cache: std::collections::HashMap::new(), + #[cfg(any(target_os = "macos", target_os = "windows"))] + visual_mark_cache: std::collections::HashMap::new(), + } + } + + /// Called after a successful screenshot capture. + pub(super) fn transition_after_screenshot(&mut self, id: String, map: PointerMap) { + self.observations.record_image(id, map); + self.click_needs_fresh_screenshot = false; + self.pending_verify_screenshot = false; + self.last_mutation_kind = Some(ComputerUseLastMutationKind::Screenshot); + } + + /// Called after pointer mutation (move, step, relative), click, scroll, key_chord, or type_text. + pub(super) fn transition_after_pointer_mutation(&mut self) { + self.click_needs_fresh_screenshot = true; + // Note: `last_mutation_kind` is set explicitly by the calling + // action (PointerMove / Click / Scroll / KeyChord / TypeText / Drag) + // so we do not overwrite it here with a generic value. + } + + /// Called after click (same effect as pointer mutation for freshness). + pub(super) fn transition_after_click(&mut self) { + self.click_needs_fresh_screenshot = true; + self.pending_verify_screenshot = true; + self.last_mutation_kind = Some(ComputerUseLastMutationKind::Click); + } + + /// Called after key, typing, scroll, or drag — UI likely changed; next `screenshot` should confirm. + pub(super) fn transition_after_committed_ui_action(&mut self) { + self.pending_verify_screenshot = true; + } + + pub(super) fn record_mutation(&mut self, kind: ComputerUseLastMutationKind) { + self.last_mutation_kind = Some(kind); + } +} diff --git a/src/apps/desktop/src/computer_use/grants.rs b/src/apps/desktop/src/computer_use/grants.rs new file mode 100644 index 0000000000..e135eb4612 --- /dev/null +++ b/src/apps/desktop/src/computer_use/grants.rs @@ -0,0 +1,673 @@ +//! Application control preferences. A remembered grant never selects or shares a window. +// Migrated from computer-use-rust/grants.rs. OpenBitFun supplies its own +// product-isolated path and preserves additive fields written by newer versions. +#[derive(Debug)] +pub(crate) struct GrantError { + pub code: String, + message: String, + details: Option, +} +impl GrantError { + fn new(code: impl Into, message: impl Into) -> Self { + Self { + code: code.into(), + message: message.into(), + details: None, + } + } + fn details(mut self, details: serde_json::Value) -> Self { + self.details = Some(details); + self + } +} +impl std::fmt::Display for GrantError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[{}] {}", self.code, self.message) + } +} +impl std::error::Error for GrantError {} +impl From for GrantError { + fn from(error: std::io::Error) -> Self { + Self::new("grant_io", error.to_string()) + } +} +impl From for GrantError { + fn from(error: serde_json::Error) -> Self { + Self::new("invalid_grant_store", error.to_string()) + } +} +type Result = std::result::Result; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use sha2::{Digest, Sha256}; +use std::{ + fs::{self, File, Metadata, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, + time::{SystemTime, UNIX_EPOCH}, +}; + +const STORE_VERSION: u32 = 1; +// A local preference file has a bounded parsing/serialization allocation budget. +const MAX_STORE_BYTES: u64 = 1024 * 1024; +static FILE_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct GrantIdentity { + pub bundle_id: Option, + pub executable: PathBuf, + pub executable_sha256: String, + pub input_mode: String, + #[serde(default, flatten)] + pub extensions: std::collections::BTreeMap, +} + +impl GrantIdentity { + /// Call only after the sharing boundary verifies the selected process lifetime. + /// Hashing runs on the native worker, never on the UI thread. + /// This identifies the on-disk executable, not the process's mapped image or + /// its loaded libraries. Unix metadata change time also has to predate the + /// process, because an updater can preserve mtime while replacing the file. + pub(crate) fn capture( + bundle_id: Option, + executable: &Path, + input_mode: &str, + process_started: SystemTime, + ) -> Result { + validate_mode(input_mode)?; + if !executable.is_absolute() { + return Err(GrantError::new( + "grant_identity_unavailable", + "A remembered grant requires an absolute executable path", + )); + } + let canonical = executable.canonicalize()?; + let mut file = File::open(&canonical)?; + let before = file.metadata()?; + if !before.is_file() || changed_after_process_start(&before, process_started)? { + return Err(GrantError::new( + "grant_identity_unavailable", + "The executable is not a regular file or its content or metadata changed after this process started; restart the app before remembering control", + )); + } + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + digest.update(&buffer[..count]); + } + let after = file.metadata()?; + let at_path = fs::metadata(&canonical)?; + if !same_file_version(&before, &after) + || !same_file_version(&after, &at_path) + || executable.canonicalize()? != canonical + { + return Err(GrantError::new( + "grant_identity_changed", + "The executable changed while its authorization identity was being read", + )); + } + let identity = Self { + bundle_id, + executable: canonical, + executable_sha256: format!("{:x}", digest.finalize()), + input_mode: input_mode.into(), + extensions: Default::default(), + }; + identity.validate()?; + Ok(identity) + } + + fn validate(&self) -> Result<()> { + validate_mode(&self.input_mode)?; + if !self.executable.is_absolute() + || self.executable_sha256.len() != 64 + || !self + .executable_sha256 + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + || self.bundle_id.as_ref().is_some_and(|id| id.is_empty()) + { + return Err(GrantError::new( + "invalid_grant_store", + "A remembered grant has an invalid application identity", + )); + } + Ok(()) + } +} + +fn changed_after_process_start(metadata: &Metadata, process_started: SystemTime) -> Result { + if metadata.modified()? > process_started { + return Ok(true); + } + #[cfg(unix)] + { + use std::{os::unix::fs::MetadataExt, time::Duration}; + let invalid_time = || { + GrantError::new( + "grant_identity_unavailable", + "The executable metadata change time cannot be verified", + ) + }; + let nanoseconds = u32::try_from(metadata.ctime_nsec()) + .ok() + .filter(|value| *value < 1_000_000_000) + .ok_or_else(invalid_time)?; + let seconds = metadata.ctime(); + let whole_seconds = Duration::from_secs(seconds.unsigned_abs()); + let base = if seconds >= 0 { + UNIX_EPOCH.checked_add(whole_seconds) + } else { + UNIX_EPOCH.checked_sub(whole_seconds) + }; + let changed = base + .and_then(|base| base.checked_add(Duration::from_nanos(u64::from(nanoseconds)))) + .ok_or_else(invalid_time)?; + Ok(changed > process_started) + } + #[cfg(not(unix))] + { + Ok(false) + } +} + +fn validate_mode(mode: &str) -> Result<()> { + if matches!(mode, "foreground" | "background") { + Ok(()) + } else { + Err(GrantError::new("invalid_grant_store", "Unknown input mode")) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GrantRecord { + pub identity: GrantIdentity, + /// A presentation label only; it is never used to match an application. + pub application_name: String, + pub granted_at_unix_ms: u64, + #[serde(default, flatten)] + pub extensions: std::collections::BTreeMap, +} + +#[derive(Serialize, Deserialize)] +struct Document { + #[serde(default = "store_version")] + version: u32, + #[serde(default)] + grants: Vec, + #[serde(default, flatten)] + extensions: std::collections::BTreeMap, +} + +fn store_version() -> u32 { + STORE_VERSION +} + +pub(crate) struct GrantStore { + extensions: std::collections::BTreeMap, + path: PathBuf, + records: Vec, + loaded_bytes: Option>, +} + +#[derive(Debug)] +pub(crate) struct Persisted { + pub backup: Option, + /// Rename completed. A durability/unlock error must not pretend it rolled back. + pub warning: Option, +} + +impl GrantStore { + pub(crate) fn load(path: PathBuf) -> Result { + if !path.is_absolute() || path.file_name().is_none() { + return Err(GrantError::new( + "invalid_grant_path", + "The grant file must have an absolute filesystem path", + )); + } + let loaded_bytes = read_store(&path)?; + let (records, extensions) = match &loaded_bytes { + None => (Vec::new(), Default::default()), + Some(bytes) => { + let document: Document = serde_json::from_slice(bytes) + .map_err(|error| GrantError::new("invalid_grant_store", error.to_string()))?; + if document.version != STORE_VERSION { + return Err(GrantError::new( + "unsupported_grant_store", + format!("Unsupported grant file version {}", document.version), + )); + } + for (index, grant) in document.grants.iter().enumerate() { + grant.identity.validate()?; + if document.grants[..index] + .iter() + .any(|prior| prior.identity == grant.identity) + { + return Err(GrantError::new( + "invalid_grant_store", + "Duplicate application authorization identity", + )); + } + } + (document.grants, document.extensions) + } + }; + Ok(Self { + path, + records, + loaded_bytes, + extensions, + }) + } + + pub(crate) fn records(&self) -> &[GrantRecord] { + &self.records + } + + pub(crate) fn contains(&self, identity: &GrantIdentity) -> bool { + self.records.iter().any(|grant| &grant.identity == identity) + } + + pub(crate) fn remember( + &mut self, + identity: GrantIdentity, + application_name: String, + ) -> Result { + identity.validate()?; + if self.contains(&identity) { + return Ok(Persisted { + backup: None, + warning: None, + }); + } + let mut records = self.records.clone(); + records.push(GrantRecord { + identity, + application_name, + granted_at_unix_ms: unix_millis()?, + extensions: Default::default(), + }); + self.persist(records) + } + + pub(crate) fn forget(&mut self, identity: &GrantIdentity) -> Result { + let mut records = self.records.clone(); + records.retain(|grant| &grant.identity != identity); + if records.len() == self.records.len() { + return Ok(Persisted { + backup: None, + warning: None, + }); + } + self.persist(records) + } + + fn persist(&mut self, records: Vec) -> Result { + let mut bytes = serde_json::to_vec_pretty(&Document { + version: STORE_VERSION, + extensions: self.extensions.clone(), + grants: records.clone(), + })?; + bytes.push(b'\n'); + if bytes.len() as u64 > MAX_STORE_BYTES { + return Err(GrantError::new( + "capacity_exceeded", + "Remembered grants exceed the 1 MiB preference-file budget", + )); + } + let parent = self.path.parent().expect("absolute file has parent"); + create_private_directories(parent)?; + // An OS lock survives an atomic data-file replacement and is released on + // crash/exit. Keeping the lock inode avoids stale or split lock ownership. + let lock = open_lock(&lock_path(&self.path))?; + lock.try_lock().map_err(|error| { + GrantError::new( + "grant_store_busy", + format!("Another writer holds the grant file: {error}"), + ) + })?; + if read_store(&self.path)? != self.loaded_bytes { + return Err(GrantError::new( + "grant_store_changed", + "The grant file changed in another process; reload it before editing", + )); + } + let backup = self + .loaded_bytes + .as_ref() + .map(|original| write_unique(&self.path, "backup", original)) + .transpose()?; + let temporary = write_unique(&self.path, "tmp", &bytes)?; + if let Err(error) = fs::rename(&temporary, &self.path) { + let cleanup = fs::remove_file(&temporary).err(); + return Err(GrantError::new("grant_store_write", error.to_string()).details(json!({ + "temporary_path":temporary,"temporary_cleanup_error":cleanup.map(|error|error.to_string()),"backup":backup + }))); + } + self.records = records; + self.loaded_bytes = Some(bytes); + let mut warnings = Vec::new(); + if let Err(error) = File::open(parent).and_then(|directory| directory.sync_all()) { + warnings.push(format!( + "Saved grant file, but directory sync failed: {error}" + )); + } + if let Err(error) = lock.unlock() { + warnings.push(format!( + "Saved grant file, but explicit lock release failed: {error}" + )); + } + Ok(Persisted { + backup, + warning: (!warnings.is_empty()).then(|| warnings.join("; ")), + }) + } +} + +fn unix_millis() -> Result { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .and_then(|time| u64::try_from(time.as_millis()).ok()) + .ok_or_else(|| GrantError::new("invalid_clock", "Cannot record the authorization time")) +} + +fn same_file_version(left: &Metadata, right: &Metadata) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + left.dev() == right.dev() + && left.ino() == right.ino() + && left.len() == right.len() + && left.mtime() == right.mtime() + && left.mtime_nsec() == right.mtime_nsec() + && left.ctime() == right.ctime() + && left.ctime_nsec() == right.ctime_nsec() + } + #[cfg(not(unix))] + { + left.len() == right.len() && left.modified().ok() == right.modified().ok() + } +} + +fn require_private_regular(metadata: &Metadata) -> Result<()> { + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(GrantError::new( + "invalid_grant_store", + "The grant path must be a regular file, not a symlink", + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if metadata.permissions().mode() & 0o077 != 0 { + return Err(GrantError::new( + "invalid_grant_store", + "Grant and lock files must have private permissions (0600)", + )); + } + } + Ok(()) +} + +fn read_store(path: &Path) -> Result>> { + let before = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + require_private_regular(&before)?; + if before.len() > MAX_STORE_BYTES { + return Err(GrantError::new( + "capacity_exceeded", + "Grant file exceeds 1 MiB", + )); + } + let file = File::open(path)?; + let opened = file.metadata()?; + if !same_file_version(&before, &opened) { + return Err(GrantError::new( + "grant_store_changed", + "Grant file changed while opening", + )); + } + let mut bytes = Vec::new(); + (&file).take(MAX_STORE_BYTES + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_STORE_BYTES + || !same_file_version(&opened, &file.metadata()?) + || !same_file_version(&opened, &fs::symlink_metadata(path)?) + { + return Err(GrantError::new( + "grant_store_changed", + "Grant file changed while reading", + )); + } + Ok(Some(bytes)) +} + +fn private_options() -> OpenOptions { + let mut options = OpenOptions::new(); + options.read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options +} + +fn lock_path(path: &Path) -> PathBuf { + let mut name = path + .file_name() + .expect("absolute grant path has filename") + .to_os_string(); + name.push(".lock"); + path.with_file_name(name) +} + +fn open_lock(path: &Path) -> Result { + match private_options().create_new(true).open(path) { + Ok(file) => Ok(file), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let before = fs::symlink_metadata(path)?; + require_private_regular(&before)?; + let file = private_options().open(path)?; + if !same_file_version(&before, &file.metadata()?) { + return Err(GrantError::new( + "grant_store_changed", + "Grant lock changed while opening", + )); + } + Ok(file) + } + Err(error) => Err(error.into()), + } +} + +fn create_private_directories(path: &Path) -> Result<()> { + let mut builder = fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(path)?; + Ok(()) +} + +fn write_unique(path: &Path, kind: &str, bytes: &[u8]) -> Result { + let sequence = FILE_SEQUENCE.fetch_add(1, Ordering::SeqCst); + let mut name = path + .file_name() + .expect("absolute grant path has filename") + .to_os_string(); + name.push(format!( + ".{kind}-{}-{}-{sequence}", + std::process::id(), + unix_millis()? + )); + let destination = path.with_file_name(name); + let mut file = private_options().create_new(true).open(&destination)?; + if let Err(error) = file.write_all(bytes).and_then(|_| file.sync_all()) { + let cleanup = fs::remove_file(&destination).err(); + return Err( + GrantError::new("grant_store_write", error.to_string()).details(json!({ + "path":destination,"cleanup_error":cleanup.map(|error|error.to_string()) + })), + ); + } + Ok(destination) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn legacy_missing_version_and_future_additive_fields_survive_writes() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("grants.json"); + let id = identity(directory.path(), "background"); + GrantStore::load(path.clone()) + .unwrap() + .remember(id.clone(), "Old app".into()) + .unwrap(); + let document = json!({"future_document": {"keep": true}, "grants": [{ + "identity": id, "application_name": "Old app", "granted_at_unix_ms": 1, "future_record": 7 + }]}); + fs::write(&path, serde_json::to_vec(&document).unwrap()).unwrap(); + let mut store = GrantStore::load(path.clone()).unwrap(); + assert!(store.contains(&id)); + let second = identity(directory.path(), "foreground"); + store.remember(second, "Second mode".into()).unwrap(); + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(written["future_document"], document["future_document"]); + assert_eq!(written["grants"][0]["future_record"], 7); + let mut future = written; + future["grants"][0]["identity"]["future_identity_requirement"] = json!("unknown"); + fs::write(&path, serde_json::to_vec(&future).unwrap()).unwrap(); + let reloaded = GrantStore::load(path).unwrap(); + assert!( + !reloaded.contains(&id), + "unknown identity constraints must never broaden an old grant" + ); + assert_eq!( + reloaded.records()[0].identity.extensions["future_identity_requirement"], + "unknown" + ); + } + use std::time::Duration; + + fn identity(directory: &Path, mode: &str) -> GrantIdentity { + let executable = directory.join("application"); + if !executable.exists() { + fs::write(&executable, b"executable version one").unwrap(); + } + GrantIdentity::capture( + Some("test.application".into()), + &executable, + mode, + SystemTime::now() + Duration::from_secs(1), + ) + .unwrap() + } + + #[test] + fn persists_exact_identity_and_mode_then_revokes_with_original_backup() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("preferences/grants.json"); + let id = identity(directory.path(), "foreground"); + let mut store = GrantStore::load(path.clone()).unwrap(); + assert!(!path.exists()); + store.remember(id.clone(), "My Application".into()).unwrap(); + let original = fs::read(&path).unwrap(); + let mut reopened = GrantStore::load(path.clone()).unwrap(); + assert!(reopened.contains(&id)); + let mut changed = id.clone(); + changed.input_mode = "background".into(); + assert!(!reopened.contains(&changed)); + changed = id.clone(); + changed.bundle_id = Some("another.application".into()); + assert!(!reopened.contains(&changed)); + changed = id.clone(); + changed.executable = directory.path().join("another"); + assert!(!reopened.contains(&changed)); + changed = id.clone(); + changed.executable_sha256 = "f".repeat(64); + assert!(!reopened.contains(&changed)); + let saved = reopened.forget(&id).unwrap(); + assert_eq!(fs::read(saved.backup.unwrap()).unwrap(), original); + assert!(!GrantStore::load(path.clone()).unwrap().contains(&id)); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + assert_eq!( + fs::metadata(path.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); + } + } + + #[test] + fn changed_binary_and_binary_newer_than_process_need_a_new_grant() { + let directory = tempfile::tempdir().unwrap(); + let first = identity(directory.path(), "foreground"); + fs::write(&first.executable, b"executable version two").unwrap(); + let second = identity(directory.path(), "foreground"); + assert_ne!(first, second); + assert!(GrantIdentity::capture(None, &first.executable, "foreground", UNIX_EPOCH).is_err()); + } + + #[test] + fn corrupt_and_concurrently_changed_files_are_never_replaced() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("grants.json"); + let id = identity(directory.path(), "foreground"); + let mut first = GrantStore::load(path.clone()).unwrap(); + let mut stale = GrantStore::load(path.clone()).unwrap(); + first.remember(id.clone(), "App".into()).unwrap(); + let original = fs::read(&path).unwrap(); + assert_eq!( + stale.remember(id.clone(), "App".into()).unwrap_err().code, + "grant_store_changed" + ); + assert!(stale.records().is_empty()); + assert_eq!(fs::read(&path).unwrap(), original); + fs::write(&path, b"{broken").unwrap(); + assert!(GrantStore::load(path.clone()).is_err()); + assert!(first.forget(&id).is_err()); + assert!(first.contains(&id)); + assert_eq!(fs::read(&path).unwrap(), b"{broken"); + } + + #[test] + fn lock_contention_keeps_old_memory_and_disk_and_recovers_after_owner_drops() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("grants.json"); + let id = identity(directory.path(), "foreground"); + let mut store = GrantStore::load(path.clone()).unwrap(); + store.remember(id.clone(), "App".into()).unwrap(); + let original = fs::read(&path).unwrap(); + let lock = open_lock(&lock_path(&path)).unwrap(); + lock.try_lock().unwrap(); + assert_eq!(store.forget(&id).unwrap_err().code, "grant_store_busy"); + assert!(store.contains(&id)); + assert_eq!(fs::read(&path).unwrap(), original); + drop(lock); + store.forget(&id).unwrap(); + assert!(!store.contains(&id)); + } +} diff --git a/src/apps/desktop/src/computer_use/integration_e2e.rs b/src/apps/desktop/src/computer_use/integration_e2e.rs index de82a0e3a2..b80956fe22 100644 --- a/src/apps/desktop/src/computer_use/integration_e2e.rs +++ b/src/apps/desktop/src/computer_use/integration_e2e.rs @@ -358,6 +358,7 @@ mod tests { }; let snap = AppStateSnapshot { + observation_id: None, app: AppInfo { name: "TestApp".to_string(), bundle_id: None, diff --git a/src/apps/desktop/src/computer_use/linux_control.rs b/src/apps/desktop/src/computer_use/linux_control.rs index 7f8448e9ea..9b36adbe2a 100644 --- a/src/apps/desktop/src/computer_use/linux_control.rs +++ b/src/apps/desktop/src/computer_use/linux_control.rs @@ -760,6 +760,20 @@ pub(super) fn stop_session() { } /// None means an untouched legacy X11 host. A revoked portal never falls back. +pub(super) fn session_is_active() -> bool { + SESSION + .lock() + .ok() + .is_some_and(|slot| slot.as_ref().is_some_and(|session| session.is_active())) +} + +pub(super) fn session_has_input_consent() -> bool { + SESSION.lock().ok().is_some_and(|slot| { + slot.as_ref() + .is_some_and(|session| session.is_active() && session.input_devices & 3 == 3) + }) +} + pub(super) fn session() -> Result>, String> { let current = SESSION .lock() diff --git a/src/apps/desktop/src/computer_use/linux_control_ax.rs b/src/apps/desktop/src/computer_use/linux_control_ax.rs index 865fd855ef..6666da8929 100644 --- a/src/apps/desktop/src/computer_use/linux_control_ax.rs +++ b/src/apps/desktop/src/computer_use/linux_control_ax.rs @@ -15,7 +15,7 @@ use std::{ #[derive(Clone)] struct CachedNode { - generation: u64, + scope: openbitfun_agent_tools::computer_use_control::ObservationScope, pid: i32, object: ObjectRefOwned, role: String, @@ -25,6 +25,11 @@ static CACHE: OnceLock>>> = OnceLock::new(); fn cache() -> &'static Mutex>> { CACHE.get_or_init(|| Mutex::new(HashMap::new())) } +pub(super) fn clear_observations() { + if let Ok(mut entries) = cache().lock() { + entries.clear(); + } +} fn error(message: impl std::fmt::Display) -> OpenBitFunError { OpenBitFunError::tool(format!("[AT_SPI] {message}")) } @@ -147,19 +152,37 @@ pub(super) async fn snapshot( selector: AppSelector, max_depth: u32, focus_only: bool, +) -> OpenBitFunResult { + snapshot_impl(selector, max_depth, focus_only, true).await +} + +/// The human's semantic preview must not replace indices held by the Agent's +/// last observation. Only the Agent observation path publishes actionable nodes. +pub(super) async fn preview_snapshot( + selector: AppSelector, + max_depth: u32, +) -> OpenBitFunResult { + snapshot_impl(selector, max_depth, false, false).await +} + +async fn snapshot_impl( + selector: AppSelector, + max_depth: u32, + focus_only: bool, + publish_nodes: bool, ) -> OpenBitFunResult { let selector = if selector.is_empty() { bound_app_selector()? } else { selector }; - let generation = super::control_session::snapshot().generation; let connection = AccessibilityConnection::new().await.map_err(error)?; let (app, root) = resolve(&connection, &selector).await?; let pid = app .pid .ok_or_else(|| error("[APP_IDENTITY_UNAVAILABLE] Missing PID"))?; super::control_session::bind_target(format!("atspi:{pid}")).map_err(error)?; + let scope = super::control_session::observation_scope().map_err(error)?; let selected_window = if focus_only { let accessible = connection .object_as_accessible(&root) @@ -265,7 +288,7 @@ pub(super) async fn snapshot( actions )); cached.push(CachedNode { - generation, + scope: scope.clone(), pid, object: object.clone(), role: role.clone(), @@ -310,8 +333,14 @@ pub(super) async fn snapshot( let pid = app .pid .ok_or_else(|| error("[APP_IDENTITY_UNAVAILABLE] AT-SPI application PID unavailable"))?; - cache().lock().map_err(error)?.insert(pid, cached); + super::control_session::validate_observation_scope(&scope).map_err(error)?; + if publish_nodes { + let mut entries = cache().lock().map_err(error)?; + entries.clear(); + entries.insert(pid, cached); + } Ok(AppStateSnapshot { + observation_id: None, app, window_title, tree_text: tree, @@ -342,7 +371,7 @@ async fn cached_target( .and_then(|nodes| nodes.get(index as usize)) .cloned() .ok_or_else(|| error("[STALE_NODE] Read get_app_state before acting on a node index."))?; - if target.generation != super::control_session::snapshot().generation { + if super::control_session::validate_observation_scope(&target.scope).is_err() { return Err(error( "[STALE_NODE] Control session changed; read a fresh application snapshot.", )); diff --git a/src/apps/desktop/src/computer_use/macos_ax_dump.rs b/src/apps/desktop/src/computer_use/macos_ax_dump.rs index 34f523bf2a..10c141fff4 100644 --- a/src/apps/desktop/src/computer_use/macos_ax_dump.rs +++ b/src/apps/desktop/src/computer_use/macos_ax_dump.rs @@ -104,6 +104,7 @@ impl AxRef { // `AX_NODE_STALE`. struct CachedSnapshot { + scope: openbitfun_agent_tools::computer_use_control::ObservationScope, digest: String, refs: Vec, } @@ -122,24 +123,46 @@ fn snapshot_cache() -> &'static Mutex> { SNAPSHOT_CACHE.get_or_init(|| Mutex::new(HashMap::new())) } -/// Resolve `(pid, idx)` to a live AX ref. Caller must NOT release it; the -/// cache owns the retain. Returns `None` if the snapshot has been replaced -/// (i.e. the digest no longer matches) or the idx is out of range. -pub(crate) fn cached_ref(pid: i32, expected_digest: Option<&str>, idx: u32) -> Option { +pub(super) fn clear_observations() { + if let Ok(mut cache) = snapshot_cache().lock() { + cache.clear(); + } +} + +/// Resolve and retain `(pid, idx)` while the cache is locked. Stop or a new +/// observation can clear the cache immediately after this lookup; the caller's +/// reference stays alive until its guard is dropped. +pub(crate) fn cached_ref( + pid: i32, + expected_digest: Option<&str>, + idx: u32, +) -> Option { let cache = snapshot_cache().lock().ok()?; let snap = cache.get(&pid)?; + if snap.scope + != openbitfun_agent_tools::computer_use_control::ObservationScope::from_control( + &super::control_session::snapshot(), + ) + { + return None; + } if let Some(want) = expected_digest { if snap.digest != want { return None; } } - snap.refs.get(idx as usize).copied() + let reference = *snap.refs.get(idx as usize)?; + if reference.0.is_null() { + return None; + } + unsafe { CFRetain(reference.0 as CFTypeRef) }; + Some(RetainedCachedTarget(reference)) } /// Like `cached_ref` but does not require a digest match. Used for /// best-effort follow-up actions where the caller did not have a chance to /// re-snapshot (e.g. `app_wait_for` polling). -pub(crate) fn cached_ref_loose(pid: i32, idx: u32) -> Option { +pub(crate) fn cached_ref_loose(pid: i32, idx: u32) -> Option { cached_ref(pid, None, idx) } @@ -192,15 +215,7 @@ impl Drop for RetainedCachedTarget { } pub(crate) fn retained_cached_target(pid: i32, idx: u32) -> Option { - let cache = snapshot_cache().lock().ok()?; - let reference = *cache.get(&pid)?.refs.get(idx as usize)?; - if reference.0.is_null() { - return None; - } - unsafe { - CFRetain(reference.0 as CFTypeRef); - } - Some(RetainedCachedTarget(reference)) + cached_ref(pid, None, idx) } pub(crate) fn validate_bound_target(pid: i32, target: AxRef) -> OpenBitFunResult<()> { @@ -688,6 +703,9 @@ fn is_closed_menu_container(role: &str, frame: Option<(f64, f64, f64, f64)>) -> } pub(super) fn dump_app_ax(pid: i32, opts: DumpOpts) -> OpenBitFunResult { + let scope = openbitfun_agent_tools::computer_use_control::ObservationScope::from_control( + &super::control_session::snapshot(), + ); let app = unsafe { AXUIElementCreateApplication(pid) }; if app.is_null() { return Err(OpenBitFunError::tool(format!( @@ -903,9 +921,11 @@ equivalents, or AXPress the menu first.\n", let mut cache = snapshot_cache() .lock() .map_err(|_| OpenBitFunError::tool("AX snapshot cache poisoned".to_string()))?; + cache.clear(); cache.insert( pid, CachedSnapshot { + scope, digest: digest.clone(), refs, }, @@ -913,6 +933,7 @@ equivalents, or AXPress the menu first.\n", } Ok(AppStateSnapshot { + observation_id: None, app: openbitfun_core::agentic::tools::computer_use_host::AppInfo { name: window_title.clone().unwrap_or_default(), bundle_id: None, @@ -1156,14 +1177,17 @@ mod tests { .find(|node| node.title.as_deref() == Some("Save report")) .unwrap(); let retained = retained_cached_target(pid, save.idx).expect("observed target retained"); + let located = + cached_ref(pid, Some(&snapshot.digest), save.idx).expect("located target retained"); let frame = retained.frame_global().expect("observed target frame"); - snapshot_cache().lock().unwrap().remove(&pid); + clear_observations(); assert!(cached_ref_loose(pid, save.idx).is_none()); assert_eq!( retained.frame_global(), Some(frame), "cache replacement must not release an in-flight target reference" ); + assert_eq!(located.frame_global(), Some(frame)); dump_app_ax(pid, DumpOpts::default()).expect("restore fixture observation cache"); let elements = crate::computer_use::interactive_filter::build_interactive_elements( diff --git a/src/apps/desktop/src/computer_use/macos_ax_ui.rs b/src/apps/desktop/src/computer_use/macos_ax_ui.rs index 8f02c62b61..4425858141 100644 --- a/src/apps/desktop/src/computer_use/macos_ax_ui.rs +++ b/src/apps/desktop/src/computer_use/macos_ax_ui.rs @@ -554,8 +554,8 @@ pub(super) fn locate_ui_element_center_for_pid( ))); } }; - let nt = unsafe { read_node_text(ax.0) }; - let frame = unsafe { element_frame_global(ax.0) }.ok_or_else(|| { + let nt = unsafe { read_node_text(ax.reference().0) }; + let frame = unsafe { element_frame_global(ax.reference().0) }.ok_or_else(|| { OpenBitFunError::tool(format!( "[AX_IDX_STALE] node_idx={} resolved but has no AXFrame (off-screen / minimised). \ Re-call `desktop.get_app_state`.", diff --git a/src/apps/desktop/src/computer_use/macos_bg_input.rs b/src/apps/desktop/src/computer_use/macos_bg_input.rs index e821868cbb..ae2f74e450 100644 --- a/src/apps/desktop/src/computer_use/macos_bg_input.rs +++ b/src/apps/desktop/src/computer_use/macos_bg_input.rs @@ -1796,7 +1796,7 @@ mod tests { super::super::macos_ax_dump::cached_ref_loose(pid, foreign.idx).unwrap(); assert!( matches!( - super::super::macos_ax_write::try_ax_press(foreign_ref), + super::super::macos_ax_write::try_ax_press(foreign_ref.reference()), super::super::macos_ax_write::AxWriteOutcome::Unavailable(_) ), "same-pid foreign-window write must be rejected" diff --git a/src/apps/desktop/src/computer_use/macos_capture.m b/src/apps/desktop/src/computer_use/macos_capture.m index 5e841a6bad..59f5a72831 100644 --- a/src/apps/desktop/src/computer_use/macos_capture.m +++ b/src/apps/desktop/src/computer_use/macos_capture.m @@ -8,7 +8,7 @@ #include #include #include -extern void obf_control_native_stopped(uint64_t generation, int32_t pid, uint32_t windowID, const char *reason); +extern void obf_control_native_stopped(uint64_t generation, uint64_t revision, int32_t pid, uint32_t windowID, const char *reason); // Zero means the main-thread observer is not installed yet. Consumers must // not cache focus preparation against that uninitialized value. Installation @@ -50,6 +50,7 @@ @interface OBFWindowCapture : NSObject @property(nonatomic) CGRect bounds; @property(nonatomic) uint64_t sequence; @property(nonatomic) uint64_t generation; +@property(atomic) uint64_t targetRevision; @property(nonatomic) CMTime lastObservedSample; @property(nonatomic) CMTime inputBarrier; @property(nonatomic) uint64_t receivedFrames; @@ -77,13 +78,13 @@ - (void)fail:(NSString *)reason { } - (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { [self fail:[NSString stringWithFormat:@"CAPTURE_STOPPED: %@", error.localizedDescription]]; - obf_control_native_stopped(self.generation, self.pid, self.windowID, self.failure.UTF8String); + obf_control_native_stopped(self.generation, self.targetRevision, self.pid, self.windowID, self.failure.UTF8String); } // macOS 15.2+: a closed target must immediately revoke input, even if the // framework keeps the stream available for a later reopened window. - (void)streamDidBecomeInactive:(SCStream *)stream { [self fail:@"TARGET_WINDOW_UNAVAILABLE: Shared window became inactive"]; - obf_control_native_stopped(self.generation, self.pid, self.windowID, self.failure.UTF8String); + obf_control_native_stopped(self.generation, self.targetRevision, self.pid, self.windowID, self.failure.UTF8String); [stream stopCaptureWithCompletionHandler:^(NSError *error) { (void)error; }]; } - (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sample ofType:(SCStreamOutputType)type { @@ -234,6 +235,7 @@ uint32_t obf_capture_validate_target(int32_t pid, uint32_t requestedWindow, char OBFWindowCapture *session = [[OBFWindowCapture alloc] init]; session.pid = pid; session.generation = generation; + session.targetRevision = UINT64_MAX; [SCShareableContent getShareableContentExcludingDesktopWindows:YES onScreenWindowsOnly:NO completionHandler:^(SCShareableContent *content, NSError *contentError) { if (contentError) { [session fail:contentError.localizedDescription]; return; } @@ -303,7 +305,7 @@ int obf_capture_status(void *handle, char *error, size_t capacity) { OBFWindowCapture *session = (__bridge OBFWindowCapture *)handle; if (obf_session_locked()) { [session fail:@"SESSION_LOCKED: Execution host screen is locked; start control again after unlocking"]; - obf_control_native_stopped(session.generation, session.pid, session.windowID, session.failure.UTF8String); + obf_control_native_stopped(session.generation, session.targetRevision, session.pid, session.windowID, session.failure.UTF8String); [session.stream stopCaptureWithCompletionHandler:^(NSError *e) { (void)e; }]; obf_error(error, capacity, session.failure); return 0; @@ -320,7 +322,7 @@ int obf_capture_status(void *handle, char *error, size_t capacity) { [session.condition unlock]; if (!wasStopped && !valid) { [session fail:@"TARGET_WINDOW_UNAVAILABLE: Original target window is no longer capturable (closed or hidden)"]; - obf_control_native_stopped(session.generation, session.pid, session.windowID, session.failure.UTF8String); + obf_control_native_stopped(session.generation, session.targetRevision, session.pid, session.windowID, session.failure.UTF8String); [session.stream stopCaptureWithCompletionHandler:^(NSError *e) { (void)e; }]; } [session.condition lock]; @@ -354,6 +356,11 @@ int obf_capture_bounds(void *handle, double *bounds, char *error, size_t capacit // Establish a presentation-time barrier after input. A queued pre-input frame // cannot satisfy the next observation merely because it arrives late. +void obf_capture_bind_revision(void *handle, uint64_t revision) { + OBFWindowCapture *session = (__bridge OBFWindowCapture *)handle; + session.targetRevision = revision; +} + void obf_capture_mark_input(void *handle) { OBFWindowCapture *session = (__bridge OBFWindowCapture *)handle; [session.condition lock]; diff --git a/src/apps/desktop/src/computer_use/macos_capture.rs b/src/apps/desktop/src/computer_use/macos_capture.rs index 8167ced873..dd9113b5ce 100644 --- a/src/apps/desktop/src/computer_use/macos_capture.rs +++ b/src/apps/desktop/src/computer_use/macos_capture.rs @@ -37,6 +37,7 @@ unsafe extern "C" { error: *mut c_char, capacity: usize, ) -> i32; + fn obf_capture_bind_revision(handle: *mut c_void, revision: u64); fn obf_capture_mark_input(handle: *mut c_void); fn obf_capture_free(bytes: *mut c_void); fn obf_capture_stop(handle: *mut c_void); @@ -56,7 +57,7 @@ pub(super) struct CaptureFrame { pub height: u32, pub window_id: u32, pub bounds: [f64; 4], - pub generation: u64, + pub scope: openbitfun_agent_tools::computer_use_control::ObservationScope, } struct CaptureSession { @@ -64,6 +65,7 @@ struct CaptureSession { pid: i32, window_id: u32, generation: u64, + target_revision: u64, } // Native state is protected by NSCondition. All Rust accesses additionally use // CAPTURE; AppKit presentation is explicitly dispatched to the main queue. @@ -96,6 +98,7 @@ impl CaptureSession { pid, window_id: window_id.unwrap_or(0), generation, + target_revision: u64::MAX, }; session.window_id = session.frame()?.window_id; Ok(session) @@ -143,7 +146,11 @@ impl CaptureSession { height, window_id, bounds, - generation: self.generation, + scope: openbitfun_agent_tools::computer_use_control::ObservationScope { + generation: self.generation, + target_revision: self.target_revision, + target: Some(format!("pid:{}/window:{}", self.pid, window_id)), + }, }) } } @@ -158,6 +165,23 @@ fn capture() -> &'static Mutex> { } pub(super) fn ensure_capture(pid: i32, mut window_id: Option) -> Result<(), String> { + let selected = super::control_session::snapshot(); + if selected.sharing.is_some() { + let prefix = format!("pid:{pid}/window:"); + let shared_window = selected + .target + .as_deref() + .and_then(|target| target.strip_prefix(&prefix)) + .and_then(|id| id.parse::().ok()) + .ok_or("[SHARED_TARGET_ONLY] Application was not shared by the user")?; + if window_id.is_some_and(|window| window != shared_window) { + return Err("[SHARED_TARGET_ONLY] Window was not shared by the user".into()); + } + window_id = Some(shared_window); + super::control_session::shared_target_allowed(&format!( + "pid:{pid}/window:{shared_window}" + ))?; + } super::control_session::capture_allowed()?; let mut active = capture().lock().map_err(|_| "Capture lock poisoned")?; let generation = super::control_session::capture_token()?.generation(); @@ -189,10 +213,13 @@ pub(super) fn ensure_capture(pid: i32, mut window_id: Option) -> Result<(), } active.take(); super::control_session::capture_allowed()?; - let session = CaptureSession::start(pid, window_id)?; + let mut session = CaptureSession::start(pid, window_id)?; // Stop can race a slow permission/capture callback. super::control_session::capture_allowed()?; super::control_session::bind_target(format!("pid:{pid}/window:{}", session.window_id))?; + session.target_revision = super::control_session::observation_scope()?.target_revision; + unsafe { obf_capture_bind_revision(session.handle.as_ptr(), session.target_revision) }; + session.check()?; *active = Some(session); Ok(()) } @@ -226,10 +253,9 @@ pub(super) fn capture_frame(pid: i32, window_id: Option) -> Result Option { unsafe { CFNumber::wrap_under_get_rule(value.cast()) }.to_f64() } -fn window_owners() -> OpenBitFunResult> { +pub(super) fn list_share_targets( +) -> OpenBitFunResult> { #[link(name = "CoreGraphics", kind = "framework")] unsafe extern "C" { fn CGWindowListCopyWindowInfo( @@ -46,7 +47,7 @@ fn window_owners() -> OpenBitFunResult> { )); } let windows: CFArray = unsafe { CFArray::wrap_under_create_rule(raw) }; - let mut owners = HashSet::new(); + let mut targets = Vec::new(); for window in windows.iter() { if unsafe { CFGetTypeID(*window) } != Dictionary::type_id() { continue; @@ -68,10 +69,38 @@ fn window_owners() -> OpenBitFunResult> { continue; } if let Some(pid) = number(&window, "kCGWindowOwnerPID").filter(|pid| *pid > 0.0) { - owners.insert(pid as i32); + let Some(id) = number(&window, "kCGWindowNumber").filter(|id| *id > 0.0) else { + continue; + }; + let string = |key: &str| { + value(&window, key) + .filter(|value| unsafe { CFGetTypeID(*value) } == CFString::type_id()) + .map(|value| unsafe { CFString::wrap_under_get_rule(value.cast()) }.to_string()) + }; + targets.push(openbitfun_agent_tools::computer_use_control::ShareTarget { + selection_id: String::new(), + unavailable_reason: None, + id: format!("pid:{}/window:{}", pid as i32, id as u64), + pid: Some(pid as i32), + window_id: Some(id as u64), + application: string("kCGWindowOwnerName") + .unwrap_or_else(|| format!("Application {}", pid as i32)), + title: string("kCGWindowName").unwrap_or_default(), + bundle_id: None, + background_pointer: true, + background_semantic: true, + selection_required: false, + }); } } - Ok(owners) + Ok(targets) +} + +fn window_owners() -> OpenBitFunResult> { + Ok(list_share_targets()? + .into_iter() + .filter_map(|window| window.pid) + .collect()) } fn include_application( diff --git a/src/apps/desktop/src/computer_use/mod.rs b/src/apps/desktop/src/computer_use/mod.rs index 28c372093d..d8bd306973 100644 --- a/src/apps/desktop/src/computer_use/mod.rs +++ b/src/apps/desktop/src/computer_use/mod.rs @@ -1,9 +1,10 @@ -//! Desktop Computer use host (screenshots + enigo). +//! Desktop Computer Use: portable admission/observation lifecycle, native OS providers. mod ax_snapshot_digest; pub(crate) mod control_session; mod debug_overlay; mod desktop_host; +pub(crate) mod grants; mod interactive_filter; #[cfg(target_os = "linux")] mod linux_ax_ui; @@ -29,8 +30,12 @@ mod macos_input_focus; mod macos_list_apps; #[cfg(target_os = "macos")] mod macos_skylight; +mod observation_cache; mod ocr_context; +pub(crate) mod permissions; +pub(crate) mod process_identity; mod screen_ocr; +pub(crate) mod share_grants; mod som_overlay; mod terminal_detect; mod ui_locate_common; @@ -51,6 +56,8 @@ mod windows_pointer_feedback; #[cfg(target_os = "windows")] mod windows_wgc_capture; +pub(crate) use desktop_host::shared_preview; +pub(crate) use desktop_host::sharing::targets as share_targets; pub use desktop_host::DesktopComputerUseHost; #[cfg(test)] diff --git a/src/apps/desktop/src/computer_use/observation_cache.rs b/src/apps/desktop/src/computer_use/observation_cache.rs new file mode 100644 index 0000000000..c3c4306571 --- /dev/null +++ b/src/apps/desktop/src/computer_use/observation_cache.rs @@ -0,0 +1,159 @@ +//! Bounded observation references. This cache never owns input permission. +//! Like computer-use-rust's native State, each instance has a fresh namespace, +//! and every new AX observation invalidates the preceding node address space. + +use openbitfun_agent_tools::computer_use_control::ObservationScope; +use std::collections::{hash_map::RandomState, VecDeque}; +use std::hash::{BuildHasher, Hasher}; + +// Geometry only (no pixels). A bounded history permits interleaved screenshot +// and AX/view observations without retaining an entire task's capture history. +const RETAINED_IMAGES: usize = 16; + +#[derive(Debug)] +pub(super) struct ObservationCache { + scope: Option, + namespace: u64, + revision: u64, + application: Option<(i32, String)>, + images: VecDeque<(String, G)>, + latest: Option, +} + +impl Default for ObservationCache { + fn default() -> Self { + Self { + scope: None, + namespace: RandomState::new().build_hasher().finish(), + revision: 0, + application: None, + images: VecDeque::new(), + latest: None, + } + } +} + +impl ObservationCache { + pub(super) fn synchronize(&mut self, scope: &ObservationScope) -> bool { + if self.scope.as_ref() == Some(scope) { + return false; + } + self.clear(); + self.scope = Some(scope.clone()); + true + } + + pub(super) fn clear(&mut self) { + self.scope = None; + self.application = None; + self.clear_images(); + } + + pub(super) fn clear_images(&mut self) { + self.images.clear(); + self.latest = None; + } + + pub(super) fn record_image(&mut self, id: String, geometry: G) { + self.images.retain(|(existing, _)| *existing != id); + self.images.push_back((id, geometry)); + while self.images.len() > RETAINED_IMAGES { + self.images.pop_front(); + } + self.latest = Some(geometry); + } + + pub(super) fn latest_image(&self) -> Option { + self.latest + } + + pub(super) fn image(&self, id: Option<&str>) -> Result { + let geometry = match id { + Some(id) => self + .images + .iter() + .find(|(key, _)| key == id) + .map(|(_, geometry)| *geometry), + None => self.latest, + }; + geometry.ok_or_else(|| "[STALE_CAPTURE] Unknown or expired screenshot_id; observe the authorized target again".into()) + } + + pub(super) fn record_application(&mut self, pid: i32) -> Result { + self.application = None; + self.revision = self + .revision + .checked_add(1) + .ok_or("[OBSERVATION_EXHAUSTED] Create a new host observation cache")?; + let id = format!("obs_{:016x}_{:x}", self.namespace, self.revision); + self.application = Some((pid, id.clone())); + Ok(id) + } + + pub(super) fn validate_application(&self, pid: i32, id: &str) -> Result<(), String> { + if id.is_empty() + || self + .application + .as_ref() + .is_none_or(|(current_pid, current_id)| *current_pid != pid || current_id != id) + { + return Err("[STALE_OBSERVATION] Node references belong to a different observation; read the target again before choosing an action".into()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn scope(revision: u64) -> ObservationScope { + ObservationScope { + generation: 1, + target_revision: revision, + target: Some("pid:12/window:34".into()), + } + } + + #[test] + fn interleaved_images_are_bounded_without_changing_their_geometry() { + let mut cache = ObservationCache::default(); + cache.synchronize(&scope(1)); + for index in 0..=RETAINED_IMAGES { + cache.record_image(format!("shot_{index}"), index); + } + assert!(cache.image(Some("shot_0")).is_err()); + assert_eq!(cache.image(Some("shot_1")).unwrap(), 1); + assert_eq!(cache.image(None).unwrap(), RETAINED_IMAGES); + assert_eq!(cache.images.len(), RETAINED_IMAGES); + } + + #[test] + fn new_observation_invalidates_nodes_even_if_native_indices_and_content_are_identical() { + let mut cache = ObservationCache::::default(); + cache.synchronize(&scope(1)); + let first = cache.record_application(12).unwrap(); + assert!(cache.validate_application(12, &first).is_ok()); + let second = cache.record_application(12).unwrap(); + assert_ne!(first, second); + assert!(cache.validate_application(12, &first).is_err()); + assert!(cache.validate_application(13, &second).is_err()); + assert!(cache.validate_application(12, &second).is_ok()); + } + + #[test] + fn stop_restart_and_return_to_same_target_cannot_restore_references() { + let mut cache = ObservationCache::default(); + cache.synchronize(&scope(1)); + let old = cache.record_application(12).unwrap(); + cache.record_image("shot".into(), 7); + assert!(!cache.synchronize(&scope(1))); + assert_eq!(cache.image(Some("shot")).unwrap(), 7); + assert!(cache.synchronize(&scope(3))); + assert!(cache.validate_application(12, &old).is_err()); + assert!(cache.image(Some("shot")).is_err()); + let replacement = cache.record_application(12).unwrap(); + cache.clear(); + cache.synchronize(&scope(3)); + assert_ne!(replacement, cache.record_application(12).unwrap()); + } +} diff --git a/src/apps/desktop/src/computer_use/permissions.rs b/src/apps/desktop/src/computer_use/permissions.rs new file mode 100644 index 0000000000..3867932ba5 --- /dev/null +++ b/src/apps/desktop/src/computer_use/permissions.rs @@ -0,0 +1,203 @@ +//! OS consent adapters. All probes are read-only; only request() may prompt. +use openbitfun_agent_tools::computer_use_permissions::{ + ComputerPermission, ComputerPermissionKind as Kind, ComputerPermissionReceipt, + ComputerPermissionState as State, ComputerPermissions, +}; + +const KINDS: [Kind; 4] = [ + Kind::Accessibility, + Kind::ScreenRecording, + Kind::InputPosting, + Kind::InputMonitoring, +]; + +pub(crate) async fn status() -> ComputerPermissions { + let permissions = platform_status().await; + ComputerPermissions { + platform: std::env::consts::OS.into(), + process_id: std::process::id(), + executable: std::env::current_exe() + .ok() + .map(|path| path.display().to_string()), + permissions, + prompts_requested: false, + } +} + +pub(crate) async fn request(kind: Kind) -> Result { + let before = status().await; + let already_granted = before.usable(kind); + #[cfg(target_os = "macos")] + let api_returned_granted = if already_granted { + None + } else { + Some( + tokio::task::spawn_blocking(move || macos::request(kind)) + .await + .map_err(|e| e.to_string())?, + ) + }; + #[cfg(not(target_os = "macos"))] + let api_returned_granted: Option = None; + let after = status().await; + let current_granted = after.usable(kind); + let permission = after + .permissions + .into_iter() + .find(|permission| permission.kind == kind) + .ok_or("[PERMISSION_UNAVAILABLE] No permission provider")?; + Ok(ComputerPermissionReceipt { + kind, already_granted, current_granted, + request_api_invoked: api_returned_granted.is_some(), + prompt_requested: api_returned_granted.is_some(), + prompt_visibility_observed: false, + requires_user_action: !current_granted, + restart_may_be_required: cfg!(target_os = "macos") && !already_granted, + api_returned_granted, permission, + permission_owner: if cfg!(target_os = "macos") { + "macOS chooses the responsible executable. Requesting does not grant access or prove a visible prompt." + } else if cfg!(target_os = "linux") { + "The desktop portal grants a selected session when sharing starts. AT-SPI access belongs to the desktop user." + } else { + "Windows desktop, capture policy and target integrity level govern access. There is no application-wide consent prompt." + }.into(), + }) +} + +#[cfg(target_os = "macos")] +async fn platform_status() -> Vec { + KINDS.into_iter().map(|kind| ComputerPermission { + kind, + state: if macos::granted(kind) { State::Granted } else { State::Required }, + requestable: true, + settings_url: Some(macos::settings_url(kind).into()), + detail: match kind { + Kind::Accessibility => "Required for application accessibility and semantic controls.", + Kind::ScreenRecording => "Required for the selected application's window capture.", + Kind::InputPosting => "Required for directed input; does not authorize foreground input.", + Kind::InputMonitoring => "Optional diagnostic capability; normal sharing does not request global input monitoring.", + }.into(), + }).collect() +} + +#[cfg(target_os = "windows")] +async fn platform_status() -> Vec { + let capture = + windows::Graphics::Capture::GraphicsCaptureSession::IsSupported().unwrap_or(false); + KINDS.into_iter().map(|kind| ComputerPermission { + kind, requestable: false, settings_url: None, + state: if kind == Kind::ScreenRecording && !capture { State::Unavailable } else { State::NotRequired }, + detail: if kind == Kind::ScreenRecording { + "Windows Graphics Capture availability is a capability probe. A protected window or desktop can still refuse capture." + } else if kind == Kind::InputMonitoring { + "Global input monitoring is not required." + } else { + "Windows has no consent prompt for this route. UIPI, the target's integrity level and session isolation are checked per operation." + }.into(), + }).collect() +} + +#[cfg(target_os = "linux")] +async fn platform_status() -> Vec { + // Opening the accessibility bus and reading the registry does not request + // portal selection or seat input. Do not infer consent from DISPLAY. + let accessible = match atspi::AccessibilityConnection::new().await { + Ok(connection) => connection.root_accessible_on_registry().await.is_ok(), + Err(_) => false, + }; + let portal_active = super::linux_control::session_is_active(); + let portal_input = super::linux_control::session_has_input_consent(); + KINDS.into_iter().map(|kind| ComputerPermission { + kind, requestable: false, settings_url: None, + state: match kind { + Kind::Accessibility if accessible => State::Granted, + Kind::Accessibility => State::Unavailable, + Kind::ScreenRecording if portal_active => State::Granted, + Kind::ScreenRecording => State::SessionRequired, + Kind::InputPosting if portal_input => State::Granted, + Kind::InputPosting => State::SessionRequired, + Kind::InputMonitoring => State::NotRequired, + }, + detail: match kind { + Kind::Accessibility => "AT-SPI provides background semantic actions for accessible applications; it does not provide arbitrary background coordinates.", + Kind::ScreenRecording => "Start sharing to open the desktop portal's window selector. Consent is limited to that native session.", + Kind::InputPosting => "Portal seat input requires an explicitly authorized foreground session. Background input is limited to AT-SPI semantic actions.", + Kind::InputMonitoring => "Global input monitoring is not required.", + }.into(), + }).collect() +} + +#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] +async fn platform_status() -> Vec { + KINDS + .into_iter() + .map(|kind| ComputerPermission { + kind, + state: State::Unavailable, + requestable: false, + settings_url: None, + detail: "No native Computer Use provider for this platform.".into(), + }) + .collect() +} + +#[cfg(target_os = "macos")] +pub(crate) mod macos { + use super::Kind; + use core_foundation::{ + base::TCFType, boolean::CFBoolean, dictionary::CFDictionary, string::CFString, + }; + #[link(name = "ApplicationServices", kind = "framework")] + unsafe extern "C" { + fn AXIsProcessTrusted() -> bool; + fn AXIsProcessTrustedWithOptions(options: *const std::ffi::c_void) -> bool; + fn CGPreflightScreenCaptureAccess() -> bool; + fn CGRequestScreenCaptureAccess() -> bool; + fn CGPreflightPostEventAccess() -> bool; + fn CGRequestPostEventAccess() -> bool; + fn CGPreflightListenEventAccess() -> bool; + fn CGRequestListenEventAccess() -> bool; + } + pub(crate) fn granted(kind: Kind) -> bool { + unsafe { + match kind { + Kind::Accessibility => AXIsProcessTrusted(), + Kind::ScreenRecording => CGPreflightScreenCaptureAccess(), + Kind::InputPosting => CGPreflightPostEventAccess(), + Kind::InputMonitoring => CGPreflightListenEventAccess(), + } + } + } + pub(crate) fn request(kind: Kind) -> bool { + if granted(kind) { + return true; + } + unsafe { + match kind { + Kind::Accessibility => { + let key = CFString::new("AXTrustedCheckOptionPrompt"); + let value = CFBoolean::true_value(); + let options = + CFDictionary::from_CFType_pairs(&[(key.as_CFType(), value.as_CFType())]); + AXIsProcessTrustedWithOptions(options.as_concrete_TypeRef() as *const _) + } + Kind::ScreenRecording => CGRequestScreenCaptureAccess(), + Kind::InputPosting => CGRequestPostEventAccess(), + Kind::InputMonitoring => CGRequestListenEventAccess(), + } + } + } + pub(crate) const fn settings_url(kind: Kind) -> &'static str { + match kind { + Kind::Accessibility | Kind::InputPosting => { + "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility" + } + Kind::ScreenRecording => { + "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture" + } + Kind::InputMonitoring => { + "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent" + } + } + } +} diff --git a/src/apps/desktop/src/computer_use/process_identity.rs b/src/apps/desktop/src/computer_use/process_identity.rs new file mode 100644 index 0000000000..176b51b087 --- /dev/null +++ b/src/apps/desktop/src/computer_use/process_identity.rs @@ -0,0 +1,225 @@ +//! Native process lifetime and executable identity for revocable sharing. +//! Adapted from computer-use-rust/native/macos.rs; equivalent native identities +//! are used on Windows and Linux instead of treating a recycled PID as an app. +use std::{ + path::PathBuf, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProcessIdentity { + pub(crate) pid: i32, + pub(crate) executable: PathBuf, + pub(crate) started: SystemTime, + pub(crate) bundle_id: Option, + lifetime: u64, +} + +impl ProcessIdentity { + pub(crate) fn read(pid: i32) -> Result { + if pid <= 0 { + return Err("[PROCESS_IDENTITY_UNAVAILABLE] Invalid process ID".into()); + } + native_read(pid) + } + pub(crate) fn validate(&self) -> Result<(), String> { + if Self::read(self.pid).as_ref() != Ok(self) { + return Err("[STALE_PROCESS] Shared application exited or its process/executable identity changed; select it again".into()); + } + Ok(()) + } +} + +#[cfg(target_os = "macos")] +fn native_read(pid: i32) -> Result { + use std::os::unix::ffi::OsStringExt; + #[repr(C)] + #[derive(Default)] + struct BsdInfo { + flags: u32, + status: u32, + xstatus: u32, + pid: u32, + ppid: u32, + uid: u32, + gid: u32, + ruid: u32, + rgid: u32, + svuid: u32, + svgid: u32, + rfu: u32, + comm: [u8; 16], + name: [u8; 32], + nfiles: u32, + pgid: u32, + pjobc: u32, + tdev: u32, + tpgid: u32, + nice: i32, + start_sec: u64, + start_usec: u64, + } + unsafe extern "C" { + fn proc_pidinfo( + pid: i32, + flavor: i32, + arg: u64, + buffer: *mut std::ffi::c_void, + size: i32, + ) -> i32; + fn proc_pidpath(pid: i32, buffer: *mut std::ffi::c_void, size: u32) -> i32; + } + let mut info = BsdInfo::default(); + let size = std::mem::size_of::() as i32; + if unsafe { proc_pidinfo(pid, 3, 0, (&mut info as *mut BsdInfo).cast(), size) } != size + || info.pid != pid as u32 + || info.status == 5 + || info.start_usec >= 1_000_000 + { + return Err( + "[PROCESS_IDENTITY_UNAVAILABLE] Cannot read the live process start time".into(), + ); + } + let mut path = [0u8; 4096]; + if unsafe { proc_pidpath(pid, path.as_mut_ptr().cast(), path.len() as u32) } <= 0 { + return Err("[PROCESS_IDENTITY_UNAVAILABLE] Cannot read the live executable path".into()); + } + let length = path + .iter() + .position(|byte| *byte == 0) + .ok_or("[PROCESS_IDENTITY_UNAVAILABLE] Truncated executable path")?; + let started = UNIX_EPOCH + .checked_add(Duration::from_secs(info.start_sec)) + .and_then(|time| time.checked_add(Duration::from_micros(info.start_usec))) + .ok_or("[PROCESS_IDENTITY_UNAVAILABLE] Invalid process start time")?; + Ok(ProcessIdentity { + pid, + executable: std::ffi::OsString::from_vec(path[..length].to_vec()).into(), + started, + bundle_id: objc2_app_kit::NSWorkspace::sharedWorkspace() + .runningApplications() + .iter() + .find(|app| app.processIdentifier() == pid) + .and_then(|app| app.bundleIdentifier()) + .map(|id| id.to_string()), + lifetime: info + .start_sec + .wrapping_mul(1_000_000) + .wrapping_add(info.start_usec), + }) +} + +#[cfg(target_os = "windows")] +fn native_read(pid: i32) -> Result { + use windows::Win32::{ + Foundation::{CloseHandle, FILETIME}, + System::Threading::{ + GetProcessTimes, OpenProcess, QueryFullProcessImageNameW, PROCESS_NAME_WIN32, + PROCESS_QUERY_LIMITED_INFORMATION, + }, + }; + let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid as u32) } + .map_err(|error| format!("[PROCESS_IDENTITY_UNAVAILABLE] {error}"))?; + struct Handle(windows::Win32::Foundation::HANDLE); + impl Drop for Handle { + fn drop(&mut self) { + let _ = unsafe { CloseHandle(self.0) }; + } + } + let handle = Handle(handle); + let mut creation = FILETIME::default(); + let mut exit = FILETIME::default(); + let mut kernel = FILETIME::default(); + let mut user = FILETIME::default(); + unsafe { GetProcessTimes(handle.0, &mut creation, &mut exit, &mut kernel, &mut user) } + .map_err(|error| format!("[PROCESS_IDENTITY_UNAVAILABLE] {error}"))?; + if exit.dwHighDateTime != 0 || exit.dwLowDateTime != 0 { + return Err("[STALE_PROCESS] Application has exited".into()); + } + let ticks = (u64::from(creation.dwHighDateTime) << 32) | u64::from(creation.dwLowDateTime); + let unix_ticks = ticks + .checked_sub(116_444_736_000_000_000) + .ok_or("[PROCESS_IDENTITY_UNAVAILABLE] Invalid creation time")?; + let mut path = vec![0u16; 32_768]; + let mut length = path.len() as u32; + unsafe { + QueryFullProcessImageNameW( + handle.0, + PROCESS_NAME_WIN32, + windows::core::PWSTR(path.as_mut_ptr()), + &mut length, + ) + } + .map_err(|error| format!("[PROCESS_IDENTITY_UNAVAILABLE] {error}"))?; + use std::os::windows::ffi::OsStringExt; + Ok(ProcessIdentity { + pid, + executable: std::ffi::OsString::from_wide(&path[..length as usize]).into(), + bundle_id: None, + started: UNIX_EPOCH + + Duration::from_secs(unix_ticks / 10_000_000) + + Duration::from_nanos((unix_ticks % 10_000_000) * 100), + lifetime: ticks, + }) +} + +#[cfg(target_os = "linux")] +fn native_read(pid: i32) -> Result { + let failed = |error: std::io::Error| format!("[PROCESS_IDENTITY_UNAVAILABLE] {error}"); + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(failed)?; + let rest = stat + .rsplit_once(") ") + .ok_or("[PROCESS_IDENTITY_UNAVAILABLE] Invalid process record")? + .1; + let fields: Vec<_> = rest.split_ascii_whitespace().collect(); + if fields + .first() + .is_none_or(|state| matches!(*state, "Z" | "X")) + { + return Err("[STALE_PROCESS] Application is not live".into()); + } + let ticks = fields + .get(19) + .and_then(|value| value.parse::().ok()) + .ok_or("[PROCESS_IDENTITY_UNAVAILABLE] Missing process start time")?; + let hz = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; + if hz <= 0 { + return Err("[PROCESS_IDENTITY_UNAVAILABLE] Cannot read the system clock rate".into()); + } + let system = std::fs::read_to_string("/proc/stat").map_err(failed)?; + let boot = system + .lines() + .find_map(|line| line.strip_prefix("btime ")) + .and_then(|value| value.trim().parse::().ok()) + .ok_or("[PROCESS_IDENTITY_UNAVAILABLE] Missing system boot time")?; + let executable = std::fs::read_link(format!("/proc/{pid}/exe")).map_err(failed)?; + Ok(ProcessIdentity { + pid, + executable, + started: UNIX_EPOCH + + Duration::from_secs(boot + ticks / hz as u64) + + Duration::from_nanos((ticks % hz as u64) * 1_000_000_000 / hz as u64), + lifetime: ticks, + bundle_id: None, + }) +} + +#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] +fn native_read(_pid: i32) -> Result { + Err("[PROCESS_IDENTITY_UNAVAILABLE] No native provider".into()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn live_identity_is_stable_and_modified_lifetime_is_rejected() { + let identity = ProcessIdentity::read(std::process::id() as i32).unwrap(); + assert!(identity.executable.is_absolute()); + identity.validate().unwrap(); + let mut changed = identity; + changed.lifetime = changed.lifetime.wrapping_add(1); + assert!(changed.validate().is_err()); + assert!(ProcessIdentity::read(-1).is_err()); + } +} diff --git a/src/apps/desktop/src/computer_use/share_grants.rs b/src/apps/desktop/src/computer_use/share_grants.rs new file mode 100644 index 0000000000..c20c0bbd82 --- /dev/null +++ b/src/apps/desktop/src/computer_use/share_grants.rs @@ -0,0 +1,207 @@ +//! A remembered preference can authorize input only after explicit selection +//! and successful native sharing. No preference starts a capture or picks an app. +use super::{ + control_session, + grants::{GrantIdentity, GrantRecord, GrantStore}, +}; +use openbitfun_agent_tools::computer_use_control::{ControlMode, ControlSnapshot}; +use std::path::PathBuf; +// Serializes preference intentions across windows/controllers. Forget revokes +// input before waiting; a pending remember cannot reinsert it afterwards. +static PREFERENCES: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn authorization(snapshot: &ControlSnapshot) -> u64 { + snapshot + .sharing + .as_ref() + .map_or(0, |sharing| sharing.authorization_revision) +} +fn current_grant(snapshot: &ControlSnapshot) -> Result<(), String> { + let current = control_session::snapshot(); + if current.generation != snapshot.generation + || current.target_revision != snapshot.target_revision + || authorization(¤t) != authorization(snapshot) + || current + .sharing + .as_ref() + .is_none_or(|sharing| !sharing.control_enabled) + { + return Err("[CONTROL_STALE] Grant changed before its preference was saved".into()); + } + Ok(()) +} + +async fn identity(snapshot: &ControlSnapshot) -> Result { + let process = control_session::shared_process(snapshot.generation, snapshot.target_revision)?; + let mode = match snapshot + .sharing + .as_ref() + .ok_or("[SHARE_REQUIRED] No share")? + .requested_mode + { + ControlMode::Background => "background", + ControlMode::Foreground => "foreground", + ControlMode::Observe => { + return Err("[CONTROL_OBSERVE_ONLY] Observation has no remembered input grant".into()) + } + }; + let result = tokio::task::spawn_blocking(move || { + process.validate()?; + let identity = GrantIdentity::capture( + process.bundle_id.clone(), + &process.executable, + mode, + process.started, + ) + .map_err(|error| error.to_string())?; + process.validate()?; + Ok::<_, String>(identity) + }) + .await + .map_err(|error| error.to_string())??; + control_session::shared_process(snapshot.generation, snapshot.target_revision)?; + Ok(result) +} + +pub(crate) async fn apply_remembered( + path: PathBuf, + snapshot: ControlSnapshot, +) -> Result { + let _operation = PREFERENCES.lock().await; + if snapshot + .sharing + .as_ref() + .is_none_or(|sharing| sharing.requested_mode == ControlMode::Observe) + || snapshot + .target + .as_deref() + .is_some_and(|target| target.starts_with("portal:")) + { + return Ok(snapshot); + } + let result = async { + let identity = identity(&snapshot).await?; + tokio::task::spawn_blocking(move || { + GrantStore::load(path) + .map(|store| store.contains(&identity)) + .map_err(|e| e.to_string()) + }) + .await + .map_err(|error| error.to_string())? + } + .await; + let (remembered, note) = match result { + Ok(found) => (found, None), + Err(error) => (false, Some(error)), + }; + let snapshot = if remembered { + control_session::set_share_control( + snapshot.generation, + snapshot.target_revision, + authorization(&snapshot), + true, + )? + } else { + snapshot + }; + control_session::grant_preference_result( + snapshot.generation, + snapshot.target_revision, + authorization(&snapshot), + remembered, + note, + ) +} + +pub(crate) async fn remember( + path: PathBuf, + snapshot: ControlSnapshot, +) -> Result { + let _operation = PREFERENCES.lock().await; + let result = async { + let identity = identity(&snapshot).await?; + current_grant(&snapshot)?; + let label = snapshot + .sharing + .as_ref() + .and_then(|sharing| sharing.target_label.clone()) + .unwrap_or_default(); + tokio::task::spawn_blocking(move || { + GrantStore::load(path) + .and_then(|mut store| store.remember(identity, label)) + .map(|result| result.warning) + .map_err(|error| error.to_string()) + }) + .await + .map_err(|error| error.to_string())? + } + .await; + let (remembered, note) = match result { + Ok(warning) => (true, warning), + Err(error) => (false, Some(error)), + }; + control_session::grant_preference_result( + snapshot.generation, + snapshot.target_revision, + authorization(&snapshot), + remembered, + note, + ) +} + +pub(crate) async fn list(path: PathBuf) -> Result, String> { + tokio::task::spawn_blocking(move || { + GrantStore::load(path) + .map(|store| store.records().to_vec()) + .map_err(|error| error.to_string()) + }) + .await + .map_err(|error| error.to_string())? +} + +pub(crate) async fn forget( + path: PathBuf, + identity: GrantIdentity, +) -> Result, String> { + // Forgetting matching active consent revokes it immediately; disk IO cannot + // keep input alive. Unreadable preferences are retained and reported. + let snapshot = control_session::snapshot(); + if let Some(sharing) = &snapshot.sharing { + if let Ok(process) = + control_session::shared_process(snapshot.generation, snapshot.target_revision) + { + let mode = if sharing.requested_mode == ControlMode::Foreground { + "foreground" + } else { + "background" + }; + if process.executable.canonicalize().ok().as_ref() == Some(&identity.executable) + && identity.input_mode == mode + { + if let Ok(revoked) = control_session::set_share_control( + snapshot.generation, + snapshot.target_revision, + sharing.authorization_revision, + false, + ) { + let _ = control_session::grant_preference_result( + snapshot.generation, + snapshot.target_revision, + authorization(&revoked), + false, + None, + ); + } + } + } + } + let _operation = PREFERENCES.lock().await; + tokio::task::spawn_blocking(move || { + GrantStore::load(path) + .and_then(|mut store| store.forget(&identity)) + .map(|result| result.warning) + .map_err(|error| error.to_string()) + }) + .await + .map_err(|error| error.to_string())? +} diff --git a/src/apps/desktop/src/computer_use/windows_ax_ui.rs b/src/apps/desktop/src/computer_use/windows_ax_ui.rs index 1c4d3234d1..14453e17a7 100644 --- a/src/apps/desktop/src/computer_use/windows_ax_ui.rs +++ b/src/apps/desktop/src/computer_use/windows_ax_ui.rs @@ -1027,6 +1027,7 @@ fn snapshot_and_nodes( Ok(( AppStateSnapshot { + observation_id: None, app, window_title, tree_text, @@ -1102,6 +1103,7 @@ enum SemanticAction { Center(u32), } enum UiaRequest { + Clear, InsertNative( isize, Option<(f64, f64)>, @@ -1125,14 +1127,21 @@ enum UiaRequest { } struct ObservedWindow { hwnd: isize, - generation: u64, + scope: openbitfun_agent_tools::computer_use_control::ObservationScope, pid: u32, nodes: Vec, } +static UIA_WORKER: std::sync::OnceLock> = + std::sync::OnceLock::new(); + +pub(super) fn clear_observations() { + if let Some(worker) = UIA_WORKER.get() { + let _ = worker.send(UiaRequest::Clear); + } +} + fn uia_worker() -> &'static std::sync::mpsc::Sender { - static WORKER: std::sync::OnceLock> = - std::sync::OnceLock::new(); - WORKER.get_or_init(|| { + UIA_WORKER.get_or_init(|| { let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { // UIA client work belongs on an MTA thread separate from the UI. @@ -1140,6 +1149,7 @@ fn uia_worker() -> &'static std::sync::mpsc::Sender { let mut observed: Option = None; while let Ok(request) = rx.recv() { match request { + UiaRequest::Clear => { observed = None; }, UiaRequest::InsertNative(hwnd, point, text, token, reply) => { let result = super::control_session::with_token(token, || insert_native_target(hwnd, point, &text)); let _ = reply.send(result); @@ -1147,10 +1157,12 @@ fn uia_worker() -> &'static std::sync::mpsc::Sender { UiaRequest::Observe(hwnd, depth, focus, token, reply) => { let result = super::control_session::with_token(token, || { super::control_session::capture_allowed().map_err(OpenBitFunError::tool)?; + let scope = super::control_session::observation_scope().map_err(OpenBitFunError::tool)?; let handle = windows::Win32::Foundation::HWND(hwnd as *mut _); let (snapshot, nodes) = snapshot_and_nodes(handle, depth, focus)?; super::control_session::capture_allowed().map_err(OpenBitFunError::tool)?; - observed = Some(ObservedWindow { hwnd, generation: token.generation(), + super::control_session::validate_observation_scope(&scope).map_err(OpenBitFunError::tool)?; + observed = Some(ObservedWindow { hwnd, scope, pid: window_pid_for(handle).unwrap_or(0), nodes }); Ok(snapshot) }); @@ -1161,7 +1173,7 @@ fn uia_worker() -> &'static std::sync::mpsc::Sender { let result = super::control_session::with_token(token, || { let cache = observed.as_ref().ok_or_else(|| OpenBitFunError::tool( "[AX_OBSERVATION_REQUIRED] Read the target window before using a node index"))?; - apply_semantic(cache, hwnd, token.generation(), action) + apply_semantic(cache, hwnd, action) }); let _ = reply.send(result); } @@ -1449,7 +1461,6 @@ pub(super) fn cached_node_center(hwnd: isize, idx: u32) -> OpenBitFunResult<(f64 fn apply_semantic( cache: &ObservedWindow, hwnd: isize, - generation: u64, action: SemanticAction, ) -> OpenBitFunResult<(f64, f64)> { use windows::Win32::UI::Accessibility::{ @@ -1459,7 +1470,7 @@ fn apply_semantic( }; super::control_session::capture_allowed().map_err(OpenBitFunError::tool)?; if cache.hwnd != hwnd - || cache.generation != generation + || super::control_session::validate_observation_scope(&cache.scope).is_err() || window_pid_for(windows::Win32::Foundation::HWND(hwnd as *mut _)) != Some(cache.pid) { return Err(OpenBitFunError::tool( diff --git a/src/apps/desktop/src/computer_use/windows_capture.rs b/src/apps/desktop/src/computer_use/windows_capture.rs index 7cd10aeccd..d9adecb0db 100644 --- a/src/apps/desktop/src/computer_use/windows_capture.rs +++ b/src/apps/desktop/src/computer_use/windows_capture.rs @@ -1,289 +1,20 @@ -//! Window-only Windows.Graphics.Capture observations. Persistent streams retain -//! the OS sharing border. Failure never falls back to another window's pixels. -//! Legacy GDI helpers remain private for existing geometry unit tests only. - -#![allow(dead_code)] - -use image::{DynamicImage, ImageBuffer, ImageFormat, Rgba}; -use log::warn; +//! Window-only WGC frames and authoritative DWM geometry. +//! There is no desktop/GDI fallback and no encode/decode round trip. use openbitfun_core::util::errors::{OpenBitFunError, OpenBitFunResult}; -use windows::Win32::Foundation::{HWND, POINT, RECT}; +use windows::Win32::Foundation::{HWND, RECT}; use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS}; -use windows::Win32::Graphics::Gdi::{ - BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDC, GetDIBits, - GetWindowDC, ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, DIB_RGB_COLORS, - RGBQUAD, SRCCOPY, -}; -use windows::Win32::Storage::Xps::{PrintWindow, PRINT_WINDOW_FLAGS}; -use windows::Win32::UI::WindowsAndMessaging::{ - GetAncestor, GetWindowRect, IsIconic, WindowFromPoint, GA_ROOT, -}; - -/// `PW_RENDERFULLCONTENT` (0x2): render window contents even when occluded or -/// off-screen (GDI-backed surfaces only). -const PW_RENDERFULLCONTENT: PRINT_WINDOW_FLAGS = PRINT_WINDOW_FLAGS(2u32); -/// 1-px inset applied to the DWM extended-frame crop to strip the dark hairline -/// Win11 dialogs paint at the rounded-corner edge. -const DWM_CROP_INSET_PX: i32 = 1; - -/// Encode raw BGRA bytes (top-down, row-major, as `GetDIBits` returns) as PNG. -/// -/// Swaps B <-> R in place then defers to the `image` crate's encoder (BGRA is -/// not a PNG-encodable channel order). Alpha is preserved as-is, matching the -/// cua source. Caller guarantees `bgra.len() == width * height * 4`. -fn encode_bgra_to_png(bgra: &[u8], width: u32, height: u32) -> OpenBitFunResult> { - if bgra.len() as u64 != (width as u64) * (height as u64) * 4 { - return Err(OpenBitFunError::service(format!( - "encode_bgra_to_png: buffer size {} != width({width}) * height({height}) * 4", - bgra.len() - ))); - } - let mut rgba = bgra.to_vec(); - for px in rgba.chunks_exact_mut(4) { - px.swap(0, 2); // B <-> R, keep G + A - } - let buf: ImageBuffer, Vec> = ImageBuffer::from_raw(width, height, rgba) - .ok_or_else(|| { - OpenBitFunError::io(format!( - "invalid RGBA buffer for width={width} height={height}" - )) - })?; - let mut out = Vec::new(); - DynamicImage::ImageRgba8(buf) - .write_to(&mut std::io::Cursor::new(&mut out), ImageFormat::Png) - .map_err(|e| OpenBitFunError::io(format!("PNG encode failed: {e}")))?; - Ok(out) -} - -/// Detect the all-black bitmap `PrintWindow` returns for DirectComposition-backed -/// UWP / WinUI3 surfaces. -/// -/// Sparse-samples (every Nth pixel so the heuristic is cheap even on 4K windows) -/// and reports `true` when > 99.5% of sampled pixels are black (`B+G+R == 0`, -/// alpha ignored — that's the all-zero pattern DirectComposition leaves behind). -/// The threshold is intentionally aggressive so legitimate dark UI does not trip -/// the fallback. -fn is_mostly_black_bgra(data: &[u8], width: u32, height: u32) -> bool { - if data.len() < 16 { - return true; - } - let pixel_count = (width as usize).saturating_mul(height as usize); - if pixel_count == 0 { - return true; - } - let available = data.len() / 4; - if available == 0 { - return true; - } - let sample_count = available.min(pixel_count); - let stride = (sample_count / 1024).max(1); - let mut sampled = 0usize; - let mut black = 0usize; - for i in (0..sample_count).step_by(stride) { - let off = i * 4; - if off + 2 < data.len() { - if data[off] == 0 && data[off + 1] == 0 && data[off + 2] == 0 { - black += 1; - } - sampled += 1; - } - } - // > 99.5% of sampled pixels are black -> treat as failed render. - sampled > 0 && (black * 200) >= (sampled * 199) -} - -/// Probe whether `hwnd` is currently obscured by another window. -/// -/// Samples `WindowFromPoint` at 5 points (4 corners inset 2 px + center) and -/// considers the target occluded when 2+ samples return a window whose root -/// ancestor isn't `hwnd`'s root ancestor. The 2-of-5 threshold avoids false -/// positives from a single corner covered by a non-opaque layered overlay (e.g. -/// an agent cursor). Callers that surface a screen-region `BitBlt` result should -/// use this to warn that the bitmap may show the *covering* window's pixels. -fn target_is_obscured(hwnd: HWND) -> bool { - if hwnd.is_invalid() { - return false; - } - let mut rect = RECT::default(); - if unsafe { GetWindowRect(hwnd, &mut rect) }.is_err() { - return false; - } - let w = rect.right - rect.left; - let h = rect.bottom - rect.top; - if w <= 4 || h <= 4 { - return false; - } - // 5 sample points: 4 corners (inset 2 px) + center. - let pts: [(i32, i32); 5] = [ - (rect.left + 2, rect.top + 2), - (rect.right - 3, rect.top + 2), - (rect.left + 2, rect.bottom - 3), - (rect.right - 3, rect.bottom - 3), - ((rect.left + rect.right) / 2, (rect.top + rect.bottom) / 2), - ]; - let target_root = unsafe { GetAncestor(hwnd, GA_ROOT) }; - let mut covered = 0usize; - for (x, y) in &pts { - let owner = unsafe { WindowFromPoint(POINT { x: *x, y: *y }) }; - if owner.is_invalid() { - continue; - } - let owner_root = unsafe { GetAncestor(owner, GA_ROOT) }; - if owner_root != target_root { - covered += 1; - } - } - // 2-of-5 threshold: a single corner covered can be a non-opaque layered - // overlay; two or more sample points missing means real content is covered. - covered >= 2 -} - -/// Return `true` when `hwnd` is minimized (iconic). -/// -/// `GetWindowRect` on an iconic HWND returns the off-screen "iconic position" -/// and `PrintWindow` paints nothing — the result is a degenerate all-black -/// ~28x160 PNG that an agent can't tell apart from a real blank screen. -/// Guarding here lets callers restore the window before retrying. -fn is_iconic(hwnd: HWND) -> bool { - if hwnd.is_invalid() { - return false; - } - unsafe { IsIconic(hwnd).as_bool() } -} - -/// Capture a window via Windows.Graphics.Capture (WGC), returning BGRA pixels + -/// `(width, height)`. -/// -/// WGC is the only API that returns a UWP target's own composited pixels even -/// when occluded by another window. -fn screenshot_window_via_wgc(hwnd: HWND) -> OpenBitFunResult<(Vec, u32, u32)> { - crate::computer_use::windows_wgc_capture::capture_window_bgra(hwnd) -} - -/// Fallback capture path: `BitBlt` the desktop DC over the rectangle covered by -/// `hwnd`'s on-screen bounds. -/// -/// Works for UWP / DirectComposition surfaces that `PrintWindow` can't reach, -/// as long as the window is on-screen and not occluded. Returns -/// `(bgra_pixels, width, height)`. -unsafe fn screenshot_via_screen_region(hwnd: HWND) -> OpenBitFunResult<(Vec, i32, i32)> { - let mut rect = RECT::default(); - // SAFETY: `rect` is a valid out-parameter and a stale/invalid HWND is - // reported by the Win32 API. - unsafe { GetWindowRect(hwnd, &mut rect) }.map_err(|e| { - OpenBitFunError::service(format!("screen-region fallback: GetWindowRect failed: {e}")) - })?; - // Under Per-Monitor V2 DPI awareness, GetWindowRect returns PHYSICAL pixels - // and BitBlt operates in physical pixels too — use the rect as-is. - let physical_left = rect.left; - let physical_top = rect.top; - let w = rect.right - rect.left; - let h = rect.bottom - rect.top; - if w <= 0 || h <= 0 { - return Err(OpenBitFunError::service(format!( - "screen-region fallback: window has zero/negative bounds: {w}x{h}" - ))); - } - // SAFETY: the DC and bitmap handles are created in this block, remain live - // through the copy, and are restored/released before the function returns. - let screen_dc = unsafe { GetDC(None) }; // NULL HWND -> desktop DC - let mem_dc = unsafe { CreateCompatibleDC(Some(screen_dc)) }; - let bitmap = unsafe { CreateCompatibleBitmap(screen_dc, w, h) }; - let old_bitmap = unsafe { SelectObject(mem_dc, bitmap.into()) }; - let blt_ok = unsafe { - BitBlt( - mem_dc, - 0, - 0, - w, - h, - Some(screen_dc), - physical_left, - physical_top, - SRCCOPY, - ) - }; - let mut bmi = BITMAPINFO { - bmiHeader: BITMAPINFOHEADER { - biSize: std::mem::size_of::() as u32, - biWidth: w, - biHeight: -h, // top-down - biPlanes: 1, - biBitCount: 32, - biCompression: BI_RGB.0, - biSizeImage: (w * h * 4) as u32, - ..Default::default() - }, - bmiColors: [RGBQUAD::default(); 1], - }; - let pixel_count = (w * h) as usize; - let mut pixels = vec![0u8; pixel_count * 4]; - let ok = unsafe { - GetDIBits( - mem_dc, - bitmap, - 0, - h as u32, - Some(pixels.as_mut_ptr() as *mut _), - &mut bmi, - DIB_RGB_COLORS, - ) - }; - unsafe { - SelectObject(mem_dc, old_bitmap); - let _ = DeleteObject(bitmap.into()); - let _ = DeleteDC(mem_dc); - ReleaseDC(None, screen_dc); - } - if blt_ok.is_err() { - return Err(OpenBitFunError::service(format!( - "screen-region fallback: BitBlt failed: {blt_ok:?}" - ))); - } - if ok == 0 { - return Err(OpenBitFunError::service( - "screen-region fallback: GetDIBits returned 0", - )); - } - Ok((pixels, w, h)) -} - -/// A captured window bitmap plus the screen-space geometry it maps to. -/// -/// `origin_x`/`origin_y` are the **physical** screen coordinates of the -/// returned bitmap's top-left pixel, and `width`/`height` are its pixel -/// dimensions. Together they let the desktop host build a `PointerMap` that -/// converts an image pixel the vision model picked back into the screen -/// coordinate a background click should target. pub(super) struct WindowCapture { - pub png: Vec, - pub occluded: bool, + pub rgba: Vec, pub origin_x: i32, pub origin_y: i32, pub width: u32, pub height: u32, } -/// Capture the target's own surface; never substitute screen-region pixels. -pub(super) fn screenshot_window_capture(hwnd: HWND) -> OpenBitFunResult { - if is_iconic(hwnd) { - return Err(OpenBitFunError::service( - "capture_unavailable: minimized window has no rendered content", - )); - } - let (pixels, width, height) = screenshot_window_via_wgc(hwnd)?; - window_capture_from_bgra(hwnd, &pixels, width, height) -} - -/// Encode a frame obtained from the control session's persistent WGC stream. -/// WGC includes the visible DWM frame, not GetWindowRect's invisible margins. -pub(super) fn window_capture_from_bgra( - hwnd: HWND, - pixels: &[u8], - width: u32, - height: u32, -) -> OpenBitFunResult { +/// Live physical frame geometry is also used immediately before input. Preview +/// metadata is an observation, never evidence of the window's current position. +pub(super) fn window_bounds(hwnd: HWND) -> OpenBitFunResult<[f64; 4]> { let mut bounds = RECT::default(); unsafe { DwmGetWindowAttribute( @@ -293,265 +24,46 @@ pub(super) fn window_capture_from_bgra( std::mem::size_of::() as u32, ) } - .map_err(|e| OpenBitFunError::service(format!("capture_geometry_unavailable: {e}")))?; - if bounds.right - bounds.left != width as i32 || bounds.bottom - bounds.top != height as i32 { + .map_err(|e| OpenBitFunError::service(format!("[CAPTURE_GEOMETRY_UNAVAILABLE] {e}")))?; + let width = i64::from(bounds.right) - i64::from(bounds.left); + let height = i64::from(bounds.bottom) - i64::from(bounds.top); + if width <= 0 || height <= 0 { return Err(OpenBitFunError::service( - "capture_geometry_changed: refresh the window observation before input", + "[CAPTURE_GEOMETRY_UNAVAILABLE] Empty window bounds", )); } - let mut rgba = pixels.to_vec(); - for pixel in rgba.chunks_exact_mut(4) { - pixel.swap(0, 2); - } - let generation = crate::computer_use::control_session::capture_token() - .map_err(OpenBitFunError::service)? - .generation(); - let mut pid = 0; - unsafe { - windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId(hwnd, Some(&mut pid)); - } - let target = format!("pid:{pid}/window:{}", hwnd.0 as isize); - crate::computer_use::control_session::publish_rgba_generation( - generation, - &target, - &rgba, - width, - height, - [ - bounds.left as f64, - bounds.top as f64, - width as f64, - height as f64, - ], - ); - Ok(WindowCapture { - png: encode_bgra_to_png(pixels, width, height)?, - occluded: false, - origin_x: bounds.left, - origin_y: bounds.top, - width, - height, - }) + Ok([ + bounds.left as f64, + bounds.top as f64, + width as f64, + height as f64, + ]) } -unsafe fn screenshot_window_bytes_unsafe(hwnd: HWND) -> OpenBitFunResult { - if hwnd.is_invalid() { - return Err(OpenBitFunError::service( - "screenshot_window_bytes: invalid HWND", - )); - } - // Bail on minimized (iconic) windows before any capture path: GetWindowRect - // on an iconic HWND returns the off-screen iconic position and PrintWindow - // paints nothing. The degenerate all-black PNG wastes model turns retrying - // against a window minimized to the taskbar. - if is_iconic(hwnd) { +pub(super) fn screenshot_window_capture(hwnd: HWND) -> OpenBitFunResult { + let scope = super::control_session::observation_scope().map_err(OpenBitFunError::service)?; + let (mut rgba, width, height) = super::windows_wgc_capture::capture_window_bgra(hwnd)?; + let bounds = window_bounds(hwnd)?; + if bounds[2] != f64::from(width) || bounds[3] != f64::from(height) { return Err(OpenBitFunError::service( - "cannot capture minimized window: it has no rendered content. \ - Restore the window first.", + "[CAPTURE_GEOMETRY_CHANGED] Refresh the resized window before input", )); } - // Tracks the screen-space top-left of the returned bitmap; set from the - // window rect below and updated when the DWM crop trims the drop-shadow - // margin so coordinate mapping stays exact. - let mut origin_x: i32; - let mut origin_y: i32; - - // Size the buffer to the WHOLE window (GetWindowRect), not just the client - // area — PrintWindow draws the entire window at 1:1 from (0, 0). A - // client-sized buffer loses non-client chrome (e.g. VCL/SAL dialogs put the - // bottom button strip outside the standard Win32 client area). - let mut win_rect = RECT::default(); - // SAFETY: `win_rect` is a valid out-parameter and `hwnd` was checked above. - unsafe { GetWindowRect(hwnd, &mut win_rect) }.map_err(|e| { - OpenBitFunError::service(format!( - "screenshot_window_bytes: GetWindowRect failed: {e}" - )) - })?; - let w = win_rect.right - win_rect.left; - let h = win_rect.bottom - win_rect.top; - if w <= 0 || h <= 0 { - return Err(OpenBitFunError::service(format!( - "screenshot_window_bytes: window has zero/negative size: {w}x{h}" - ))); - } - origin_x = win_rect.left; - origin_y = win_rect.top; - - // SAFETY: the DC and bitmap handles are created here, remain live through - // the capture, and are restored/released immediately after `GetDIBits`. - let screen_dc = unsafe { GetWindowDC(Some(hwnd)) }; - let mem_dc = unsafe { CreateCompatibleDC(Some(screen_dc)) }; - let bitmap = unsafe { CreateCompatibleBitmap(screen_dc, w, h) }; - let old_bitmap = unsafe { SelectObject(mem_dc, bitmap.into()) }; - - // Primary: PrintWindow with PW_RENDERFULLCONTENT. If it refuses, BitBlt - // straight from the window DC as a last resort (best-effort — a failure - // here surfaces downstream via the mostly-black detection + fallback). - let pw_ok = unsafe { PrintWindow(hwnd, mem_dc, PW_RENDERFULLCONTENT) }; - if !pw_ok.as_bool() { - let _ = unsafe { BitBlt(mem_dc, 0, 0, w, h, Some(screen_dc), 0, 0, SRCCOPY) }; - } - - // DWM extended-frame bounds: strip the invisible drop-shadow margin that - // GetWindowRect counts but PrintWindow doesn't paint (leaves a black trim). - // Best-effort — if the DWM call fails, keep the full-window bitmap as-is. - let dwm_rect: Option = { - let mut r = RECT::default(); - // SAFETY: `r` is a live `RECT` out-buffer with the exact byte size - // required by `DWMWA_EXTENDED_FRAME_BOUNDS`. - let hr = unsafe { - DwmGetWindowAttribute( - hwnd, - DWMWA_EXTENDED_FRAME_BOUNDS, - &mut r as *mut _ as *mut _, - std::mem::size_of::() as u32, - ) - }; - hr.ok().map(|_| r) - }; - - let mut bmi = BITMAPINFO { - bmiHeader: BITMAPINFOHEADER { - biSize: std::mem::size_of::() as u32, - biWidth: w, - biHeight: -h, // top-down - biPlanes: 1, - biBitCount: 32, - biCompression: BI_RGB.0, - biSizeImage: (w * h * 4) as u32, - ..Default::default() - }, - bmiColors: [RGBQUAD::default(); 1], - }; - - let pixel_count = (w * h) as usize; - let mut pixels = vec![0u8; pixel_count * 4]; - let ok = unsafe { - GetDIBits( - mem_dc, - bitmap, - 0, - h as u32, - Some(pixels.as_mut_ptr() as *mut _), - &mut bmi, - DIB_RGB_COLORS, - ) - }; - - unsafe { - SelectObject(mem_dc, old_bitmap); - let _ = DeleteObject(bitmap.into()); - let _ = DeleteDC(mem_dc); - ReleaseDC(Some(hwnd), screen_dc); - } - - if ok == 0 { + if width == 0 || height == 0 || u64::from(width) * u64::from(height) * 4 != rgba.len() as u64 { return Err(OpenBitFunError::service( - "screenshot_window_bytes: GetDIBits returned 0", + "[CAPTURE_INVALID_FRAME] Invalid BGRA buffer", )); } - - // Crop to the DWM extended-frame bounds (with a 1-px inset) to remove the - // invisible-shadow margin and the Win11 rounded-corner hairline. The crop - // offset shifts the bitmap origin in screen space, so fold it into - // origin_x/origin_y for coordinate mapping. - let (pixels, w, h, crop_off_x, crop_off_y) = - crop_to_dwm_frame(pixels, w, h, win_rect, dwm_rect); - origin_x += crop_off_x; - origin_y += crop_off_y; - - // Detect the all-black bitmap PrintWindow returns for DirectComposition- - // backed surfaces. Recovery order: - // 1. WGC (occlusion-immune; works for UWP / DirectComposition). - // 2. Screen-region BitBlt (works when target is on-screen & visible), - // flagged occluded via target_is_obscured when another window covers it. - if is_mostly_black_bgra(&pixels, w as u32, h as u32) { - if let Ok((alt_pixels, alt_w, alt_h)) = screenshot_window_via_wgc(hwnd) { - return Ok(WindowCapture { - png: encode_bgra_to_png(&alt_pixels, alt_w, alt_h)?, - occluded: false, - origin_x: win_rect.left, - origin_y: win_rect.top, - width: alt_w, - height: alt_h, - }); - } - let occluded = target_is_obscured(hwnd); - match unsafe { screenshot_via_screen_region(hwnd) } { - Ok((alt_pixels, alt_w, alt_h)) => { - // Screen-region BitBlt captures the full GetWindowRect region - // (no DWM crop), so its origin is the raw window top-left. - return Ok(WindowCapture { - png: encode_bgra_to_png(&alt_pixels, alt_w as u32, alt_h as u32)?, - occluded, - origin_x: win_rect.left, - origin_y: win_rect.top, - width: alt_w as u32, - height: alt_h as u32, - }); - } - Err(e) => { - warn!( - "screenshot_window_bytes: PrintWindow returned a mostly-black bitmap \ - (UWP / DirectComposition target?); screen-region fallback failed: {e}" - ); - // Fall through — return the (black) PrintWindow result so the - // caller still gets an image rather than an outright error. - } - } + for pixel in rgba.chunks_exact_mut(4) { + pixel.swap(0, 2); } - - // PrintWindow reads from the target's own DC, so the bitmap is the target's - // pixels even when occluded — no occluded warning on this path. + super::control_session::validate_observation_scope(&scope).map_err(OpenBitFunError::service)?; + super::control_session::publish_rgba(&scope, &rgba, width, height, bounds); Ok(WindowCapture { - png: encode_bgra_to_png(&pixels, w as u32, h as u32)?, - occluded: false, - origin_x, - origin_y, - width: w as u32, - height: h as u32, + rgba, + origin_x: bounds[0] as i32, + origin_y: bounds[1] as i32, + width, + height, }) } - -/// Crop `pixels` (BGRA, top-down) to the DWM extended-frame bounds, removing the -/// invisible drop-shadow margin PrintWindow doesn't paint. No-op when the DWM -/// rect is unavailable or the computed crop is out of bounds. -/// -/// Returns `(pixels, width, height, off_x, off_y)` where `off_x`/`off_y` are the -/// offset (in window-local pixels) from the original window top-left to the -/// cropped content's top-left — `0` when the crop is a no-op. Callers fold these -/// into the screen-space origin so coordinate mapping stays exact. -fn crop_to_dwm_frame( - pixels: Vec, - w: i32, - h: i32, - win_rect: RECT, - dwm_rect: Option, -) -> (Vec, i32, i32, i32, i32) { - let Some(dwm) = dwm_rect else { - return (pixels, w, h, 0, 0); - }; - let off_x = (dwm.left - win_rect.left) + DWM_CROP_INSET_PX; - let off_y = (dwm.top - win_rect.top) + DWM_CROP_INSET_PX; - let crop_w = (dwm.right - dwm.left) - 2 * DWM_CROP_INSET_PX; - let crop_h = (dwm.bottom - dwm.top) - 2 * DWM_CROP_INSET_PX; - if off_x < 0 - || off_y < 0 - || crop_w <= 0 - || crop_h <= 0 - || off_x + crop_w > w - || off_y + crop_h > h - { - return (pixels, w, h, 0, 0); - } - let stride_full = (w * 4) as usize; - let stride_crop = (crop_w * 4) as usize; - let mut cropped = vec![0u8; (crop_w * crop_h * 4) as usize]; - for row in 0..crop_h as usize { - let src_row = (off_y as usize + row) * stride_full + (off_x as usize) * 4; - let dst_row = row * stride_crop; - cropped[dst_row..dst_row + stride_crop] - .copy_from_slice(&pixels[src_row..src_row + stride_crop]); - } - (cropped, crop_w, crop_h, off_x, off_y) -} diff --git a/src/apps/desktop/src/computer_use/windows_list_apps.rs b/src/apps/desktop/src/computer_use/windows_list_apps.rs index e98b9da184..8e044a4a74 100644 --- a/src/apps/desktop/src/computer_use/windows_list_apps.rs +++ b/src/apps/desktop/src/computer_use/windows_list_apps.rs @@ -41,6 +41,7 @@ extern "system" { /// One visible top-level window discovered during enumeration. struct WindowEntry { + hwnd: usize, pid: u32, title: String, } @@ -82,6 +83,29 @@ pub(super) fn list_running_apps(_include_hidden: bool) -> OpenBitFunResult Vec +{ + enumerate_windows() + .into_iter() + .map( + |window| openbitfun_agent_tools::computer_use_control::ShareTarget { + selection_id: String::new(), + unavailable_reason: None, + id: format!("pid:{}/window:{}", window.pid, window.hwnd), + pid: Some(window.pid as i32), + window_id: Some(window.hwnd as u64), + application: exe_basename_for_pid(window.pid) + .unwrap_or_else(|| window.title.clone()), + title: window.title, + bundle_id: None, + background_pointer: true, + background_semantic: true, + selection_required: false, + }, + ) + .collect() +} + fn sort_apps_by_name(apps: &mut [AppInfo]) { apps.sort_by_cached_key(|app| app.name.to_lowercase()); } @@ -163,7 +187,11 @@ unsafe extern "system" fn enum_windows_cb(hwnd: HWND, lparam: LPARAM) -> BOOL { } if let Ok(mut s) = state.lock() { - s.windows.push(WindowEntry { pid, title }); + s.windows.push(WindowEntry { + hwnd: hwnd.0 as usize, + pid, + title, + }); } TRUE } diff --git a/src/apps/desktop/src/computer_use/windows_wgc_capture.rs b/src/apps/desktop/src/computer_use/windows_wgc_capture.rs index 08138fc5c1..18d02b6d90 100644 --- a/src/apps/desktop/src/computer_use/windows_wgc_capture.rs +++ b/src/apps/desktop/src/computer_use/windows_wgc_capture.rs @@ -44,6 +44,8 @@ impl WgcCaptureSession { if hwnd.is_invalid() { return Err(OpenBitFunError::service("WGC capture: invalid HWND")); } + let scope = crate::computer_use::control_session::observation_scope() + .map_err(OpenBitFunError::service)?; unsafe { let (device, context) = create_d3d11_device()?; let direct_device = create_winrt_d3d_device(&device)?; @@ -85,9 +87,7 @@ impl WgcCaptureSession { } let closed = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); let signal = closed.clone(); - let generation = crate::computer_use::control_session::capture_token() - .map_err(OpenBitFunError::service)? - .generation(); + let mut pid = 0; windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId(hwnd, Some(&mut pid)); let target = format!("pid:{pid}/window:{}", hwnd.0 as isize); @@ -95,7 +95,8 @@ impl WgcCaptureSession { .Closed(&windows::Foundation::TypedEventHandler::new(move |_, _| { signal.store(true, std::sync::atomic::Ordering::Release); crate::computer_use::control_session::native_stopped_target( - generation, + scope.generation, + scope.target_revision, &target, "target_closed", ); @@ -220,9 +221,12 @@ enum CaptureCommand { // A generation change must not retain a prior owner's native capture session. fn ensure_stream( - active: &mut Option<(usize, u64, WgcCaptureSession)>, + active: &mut Option<( + usize, + openbitfun_agent_tools::computer_use_control::ObservationScope, + WgcCaptureSession, + )>, handle: usize, - generation: u64, ) -> OpenBitFunResult<&mut WgcCaptureSession> { use windows::Win32::UI::WindowsAndMessaging::{GetWindowThreadProcessId, IsIconic, IsWindow}; crate::computer_use::control_session::capture_allowed().map_err(OpenBitFunError::service)?; @@ -241,9 +245,11 @@ fn ensure_stream( handle as isize )) .map_err(OpenBitFunError::service)?; - if active.as_ref().map(|(id, owner, _)| (*id, *owner)) != Some((handle, generation)) { + let scope = crate::computer_use::control_session::observation_scope() + .map_err(OpenBitFunError::service)?; + if active.as_ref().map(|(id, owner, _)| (*id, owner)) != Some((handle, &scope)) { *active = None; - *active = Some((handle, generation, WgcCaptureSession::start(hwnd)?)); + *active = Some((handle, scope, WgcCaptureSession::start(hwnd)?)); } let stream = &mut active.as_mut().expect("capture initialized").2; stream.ensure_live()?; @@ -252,14 +258,19 @@ fn ensure_stream( // All COM initialization, interfaces and D3D immediate-context calls stay on // this dedicated worker. The registry only contains a thread-safe sender. +static CAPTURE_WORKER: std::sync::OnceLock> = + std::sync::OnceLock::new(); + fn capture_worker() -> &'static std::sync::mpsc::Sender { - static WORKER: std::sync::OnceLock> = - std::sync::OnceLock::new(); - WORKER.get_or_init(|| { + CAPTURE_WORKER.get_or_init(|| { let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let initialized = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }.is_ok(); - let mut active: Option<(usize, u64, WgcCaptureSession)> = None; + let mut active: Option<( + usize, + openbitfun_agent_tools::computer_use_control::ObservationScope, + WgcCaptureSession, + )> = None; while let Ok(command) = rx.recv() { match command { CaptureCommand::Stop(reply) => { @@ -269,7 +280,7 @@ fn capture_worker() -> &'static std::sync::mpsc::Sender { CaptureCommand::Ensure(handle, token, reply) => { let result = crate::computer_use::control_session::with_token(token, || { - ensure_stream(&mut active, handle, token.generation()).map(|_| ()) + ensure_stream(&mut active, handle).map(|_| ()) }); if result.is_err() { active = None; @@ -279,7 +290,7 @@ fn capture_worker() -> &'static std::sync::mpsc::Sender { CaptureCommand::Capture(handle, token, reply) => { let result = crate::computer_use::control_session::with_token(token, || { - ensure_stream(&mut active, handle, token.generation())?.capture() + ensure_stream(&mut active, handle)?.capture() }); if result.is_err() { active = None; @@ -299,7 +310,10 @@ fn capture_worker() -> &'static std::sync::mpsc::Sender { pub(super) fn stop_capture() { let (tx, rx) = std::sync::mpsc::sync_channel(1); - if capture_worker().send(CaptureCommand::Stop(tx)).is_ok() { + if CAPTURE_WORKER + .get() + .is_some_and(|worker| worker.send(CaptureCommand::Stop(tx)).is_ok()) + { let _ = rx.recv_timeout(Duration::from_secs(3)); } } diff --git a/src/apps/desktop/src/lib.rs b/src/apps/desktop/src/lib.rs index 52b38e0f99..3664b088f7 100644 --- a/src/apps/desktop/src/lib.rs +++ b/src/apps/desktop/src/lib.rs @@ -1450,6 +1450,15 @@ pub async fn run() { computer_use_control_stop, computer_use_control_preview, computer_use_request_permissions, + computer_use_request_permission, + computer_use_share_targets, + computer_use_share_observe, + computer_use_share_input, + computer_use_share_start, + computer_use_share_control, + computer_use_share_request, + computer_use_share_grants, + computer_use_share_forget_grant, computer_use_open_system_settings, set_config, save_cloud_speech_config, diff --git a/src/crates/assembly/agent-content/prompts/agents/computer_use_mode.md b/src/crates/assembly/agent-content/prompts/agents/computer_use_mode.md index a4a48b914f..a972e0f4b3 100644 --- a/src/crates/assembly/agent-content/prompts/agents/computer_use_mode.md +++ b/src/crates/assembly/agent-content/prompts/agents/computer_use_mode.md @@ -25,7 +25,9 @@ Choose the control surface that owns the task: # Control Session And Target -For an app task, call `start_control` with `mode: "background"`. Identify the running application through `list_apps`, then use the same `app` selector for `get_app_state` and `app_*` actions, for example `{"pid":421}` or `{"bundle_id":"com.example.app"}`. Keep the session active across observations, actions and verification; do not stop and restart between calls. Use `control_status` to inspect the owner, mode and target, and `stop_control` when the task is finished. A system stop requires an explicit new start. +For an app task, call `start_control` with `mode: "background"`. On hosts advertising `app-sharing-v1`, this requests a user selection; it does not grant control. When the result is `awaiting_share` or `SHARE_REQUIRED`, explain that the user must select the App window in the executing Desktop's sharing panel (or its connected peer controller). Stop this attempt until the user completes that choice. Sharing starts in observation mode, and input needs a separate user grant or an exact remembered application grant. Never invoke permission, selection or grant UI commands on the user's behalf. Mobile/IM and headless callers that cannot select a native window must receive an explicit unsupported/selection-required result, not wait for a hidden dialog. + +After sharing is established, use the bound application from `control_status` and the same `app` selector for `get_app_state` and `app_*` actions, for example `{"pid":421}`. Other windows, including windows in the same application, are outside this selected capture and input scope. Keep the session active across observations, actions and verification; do not stop and restart between calls. Use `control_status` to inspect the owner, mode and target, and `stop_control` when the task is finished. A system stop requires a new user-selected share. Older hosts without `app-sharing-v1` retain their advertised control contract; never assume the new grant flow exists remotely. The controlled target can be behind another application. The human's foreground application, display and cursor are metadata, not a reason to switch the task's target. An observation with no explicit selector follows the bound target when one exists. App-scoped actions require an explicit application identity from the observation. @@ -41,6 +43,7 @@ Use fresh, observed targets from the same application: - `app_click` with `target: {"kind":"node_idx","idx":3}` addresses a node in that application's current observation. - `app_click` with `target: {"kind":"ocr_text","needle":"Search"}` targets observed text in the authorized application window. Resolve ambiguous labels from current context before acting. +- Echo the latest `observation_id` on node/index input when the host advertises `observation-id-v1`. It identifies the node address space, while `digest` describes content. A new application observation invalidates the previous node references even if the digest is unchanged; a target switch or Stop invalidates every reference. Older hosts may omit this field. - When the model can see the attached image, `app_click` with `target: {"kind":"image_xy","x":120,"y":80,"screenshot_id":"capture-1"}` uses pixels from that exact screenshot. `image_grid` also requires the matching screenshot identity and observed grid bounds. Never reinterpret image pixels as global screen coordinates or reuse references after a target/geometry change. - `app_type_text` takes the exact Unicode `text`, including CJK, Arabic, emoji and multiline content. Its optional `focus` uses the same tagged target shape, for example `{"kind":"node_idx","idx":3}`. Do not switch to global clipboard input solely because the text is non-Latin or long. - `app_scroll` takes `dx`/`dy` and an optional tagged `focus`. `app_key_chord` sends a supported app-scoped shortcut; use `get_app_shortcuts` to discover an unknown shortcut instead of guessing. diff --git a/src/crates/assembly/core/src/agentic/tools/computer_use_host.rs b/src/crates/assembly/core/src/agentic/tools/computer_use_host.rs index 07402b83b3..475d6ce1a1 100644 --- a/src/crates/assembly/core/src/agentic/tools/computer_use_host.rs +++ b/src/crates/assembly/core/src/agentic/tools/computer_use_host.rs @@ -40,6 +40,14 @@ pub trait ComputerUseHost: Send + Sync + std::fmt::Debug { None } + /// Validate an opaque observation reference against the prepared target. + /// Older hosts must reject explicit references they cannot validate. + fn validate_observation(&self, _observation_id: &str) -> OpenBitFunResult<()> { + Err(OpenBitFunError::tool( + "[OBSERVATION_REFERENCE_UNSUPPORTED] This host cannot validate observation_id", + )) + } + async fn prepare_control_target(&self, _app: AppSelector) -> OpenBitFunResult<()> { Ok(()) } diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_actions.rs b/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_actions.rs index 37ac7a8c7f..d7a668b2d1 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_actions.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_actions.rs @@ -507,6 +507,7 @@ impl ComputerUseActions { "app": snap.app, "window_title": snap.window_title, "digest": snap.digest, + "observation_id": snap.observation_id, "captured_at_ms": snap.captured_at_ms, "tree_text": tree_text, "node_count": snap.nodes.len(), @@ -578,6 +579,7 @@ impl ComputerUseActions { "app": view.app, "window_title": view.window_title, "digest": view.digest, + "observation_id": view.observation_id, "captured_at_ms": view.captured_at_ms, "elements": view.elements, "omitted_element_count": view.omitted_element_count, @@ -614,6 +616,7 @@ impl ComputerUseActions { "app": view.app, "window_title": view.window_title, "digest": view.digest, + "observation_id": view.observation_id, "captured_at_ms": view.captured_at_ms, "marks": view.marks, "has_screenshot": view.screenshot.is_some(), diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_program.rs b/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_program.rs index 5f0852511d..1a51a69f25 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_program.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_program.rs @@ -1,7 +1,7 @@ //! The single app-input executor: a validated program, ordered receipts and one observation. //! Never retries a submitted step; a new model decision starts a new program. use crate::agentic::tools::computer_use_host::{ - AppInputAction, AppSelector, AppStateSnapshot, ClickTarget, ComputerUseHost, ControlSnapshot, + AppInputAction, AppSelector, AppStateSnapshot, ComputerUseHost, ControlSnapshot, }; use crate::agentic::tools::framework::ToolUseContext; use serde_json::{json, Value}; @@ -14,94 +14,7 @@ pub(super) struct ProgramObservation { /// Check structural target errors before the first mutation. Native providers /// still validate current window bounds, retained identities and screenshot IDs. pub(super) fn validate_step(step: &AppInputAction) -> Result<(), String> { - step.validate()?; - fn target(target: &ClickTarget) -> Result<(), String> { - match target { - ClickTarget::ImageXy { - x, - y, - screenshot_id, - } => { - if *x < 0 - || *y < 0 - || screenshot_id - .as_deref() - .is_none_or(|id| id.trim().is_empty()) - { - return Err("image_xy requires nonnegative image coordinates and the observed screenshot_id".into()); - } - } - ClickTarget::ImageGrid { - x0, - y0, - width, - height, - rows, - cols, - row, - col, - intersections, - screenshot_id, - } => { - if *x0 < 0 - || *y0 < 0 - || *width == 0 - || *height == 0 - || *rows == 0 - || *cols == 0 - || row >= rows - || col >= cols - || (*intersections && (*rows < 2 || *cols < 2)) - || screenshot_id - .as_deref() - .is_none_or(|id| id.trim().is_empty()) - { - return Err("image_grid requires a nonempty observed screenshot_id, positive rectangle/grid dimensions and an in-range row/col".into()); - } - } - ClickTarget::VisualGrid { - rows, - cols, - row, - col, - intersections, - .. - } => { - if *rows == 0 - || *cols == 0 - || row >= rows - || col >= cols - || (*intersections && (*rows < 2 || *cols < 2)) - { - return Err( - "visual_grid requires positive dimensions and an in-range row/col".into(), - ); - } - } - ClickTarget::OcrText { needle } if needle.trim().is_empty() => { - return Err("ocr_text requires non-empty observed text".into()) - } - ClickTarget::ScreenXy { x, y } if !x.is_finite() || !y.is_finite() => { - return Err("screen_xy coordinates must be finite".into()) - } - _ => {} - } - Ok(()) - } - match step { - AppInputAction::Click { target: point, .. } => target(point), - AppInputAction::TypeText { - focus: Some(point), .. - } - | AppInputAction::Scroll { - focus: Some(point), .. - } => target(point), - AppInputAction::Drag { from, to, .. } => { - target(from)?; - target(to) - } - _ => Ok(()), - } + step.validate() } pub(super) async fn execute( @@ -220,7 +133,7 @@ pub(super) async fn observe_after_input( } } -fn same_active_control(before: &ControlSnapshot, after: &ControlSnapshot) -> bool { +pub(super) fn same_active_control(before: &ControlSnapshot, after: &ControlSnapshot) -> bool { if !before.supported { return !after.supported; } @@ -230,4 +143,5 @@ fn same_active_control(before: &ControlSnapshot, after: &ControlSnapshot) -> boo && after.owner == before.owner && after.mode == before.mode && after.target == before.target + && after.target_revision == before.target_revision } diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_tool.rs index 86774e79e9..daaf407739 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/computer_use_tool.rs @@ -83,7 +83,7 @@ pub(crate) async fn computer_use_augment_result_json( let success = body .get("success") .and_then(|v| v.as_bool()) - .unwrap_or(true); + .unwrap_or(false); host.record_action(&action_type, &action_params, success); // Check for action loops @@ -455,6 +455,7 @@ If neither accessibility nor built-in OCR exposes the required content, report t "text_query": { "type": "string", "description": "For `move_to_text`, `move_to_target`, `click_target`: visible text to OCR-match on screen (case-insensitive substring)." }, "identifier_contains": { "type": "string", "description": "For `locate`, `click_element`: case-insensitive substring on AXIdentifier." }, "node_idx": { "type": "integer", "minimum": 0, "description": "For `locate`, `click_element`: jump straight to a node returned by the most recent `get_app_state` (field `idx`). Bypasses BFS. macOS only; other platforms return AX_IDX_NOT_SUPPORTED." }, + "observation_id": { "type": "string", "minLength": 1, "description": "Echo the opaque observation_id from get_app_state or the latest app action when sending node/index input. It is invalidated by every new app observation, target change, or stop. Omit only for older hosts without observation-id-v1." }, "app_state_digest": { "type": "string", "description": "For `locate`, `click_element`: optional `state_digest` from the same `get_app_state` call that produced `node_idx`. Stale digest yields AX_IDX_STALE so you re-snapshot." }, "max_depth": { "type": "integer", "minimum": 1, "maximum": 200, "description": "For get_app_state: AX depth (default 32). For locate/click_element: max BFS depth (default 48), ignored with node_idx." }, "filter_combine": { "type": "string", "enum": ["all", "any"], "description": "For `locate`, `click_element`: `all` (default, AND) or `any` (OR) for filter combination. Priority: `node_idx` > `text_contains` > `title_contains`+`role_substring`." }, @@ -560,6 +561,7 @@ If neither accessibility nor built-in OCR exposes the required content, report t | "text" | "keys" | "focus_idx" + | "observation_id" | "dx" | "dy" | "app_name" @@ -1509,9 +1511,9 @@ Use foreground only when the user explicitly requests taking over the visible de Use list_apps once to identify the target, then get_app_state with its app selector (pid, bundle_id, or name). \ An empty app selector follows the bound target, or the current app when no target is bound yet. \ For GUI tasks, use this tool's capture, accessibility and input interfaces. Do not replace failed observations with ad hoc AppleScript, screen-capture, or OCR programs in ExecCommand. \ -Read the returned structured data: application identities, tree_text, node indices, screenshot_id, coordinate bounds, and errors. \ +Read the returned structured data: application identities, tree_text, observation_id, node indices, screenshot_id, coordinate bounds, and errors. \ Prefer app_batch for already-decided app_click, app_type_text, app_key_chord, app_scroll or app_drag steps; it returns one final observation. Stop before choosing a target that depends on new pixels. Never replay completed receipts after a partial failure. \ -Use observed image_xy coordinates with the matching screenshot_id; node_idx and ocr_text are optional precision aids, not prerequisites for graphical controls. \ +Use observed image_xy coordinates with the matching screenshot_id; echo observation_id for node/index input when available. node_idx and ocr_text are optional precision aids, not prerequisites for graphical controls. \ Screenshot image coordinates are valid only with that screenshot identity and its image_content_rect/image_global_bounds; never reinterpret them as screen coordinates. \ Execute an action or batch once, then verify the intended change. A submitted event is not evidence of successful delivery; do not repeat a mutation merely because the AX digest is unchanged. \ If the app exposes only window chrome, use the authorized window screenshot and built-in OCR targeting instead of guessing controls from a sparse tree. \ @@ -1689,6 +1691,13 @@ impl ComputerUseTool { .get("action") .and_then(Value::as_str) .ok_or_else(|| OpenBitFunError::tool("action is required"))?; + let policy = + openbitfun_agent_tools::computer_use_control::ControlActionPolicy::for_action(action) + .ok_or_else(|| { + OpenBitFunError::tool(format!( + "[CONTROL_ACTION_INVALID] Unknown ComputerUse action: {action}" + )) + })?; let host = context .computer_use_host .as_ref() @@ -1727,27 +1736,15 @@ impl ComputerUseTool { _ => {} } let previous_generation = host.control_snapshot().generation; - let mut lease = host.acquire_control_action(owner, action).await?; + let lease_result = host.acquire_control_action(owner, action).await; let current_generation = host.control_snapshot().generation; if current_generation != previous_generation { Self::watch_control_cancellation(host.clone(), context, current_generation); } + let mut lease = lease_result?; let result = async { let mut capture_preparation_error = None; - if !matches!( - action, - "list_apps" - | "list_displays" - | "wait" - | "get_os_info" - | "clipboard_get" - | "clipboard_set" - | "open_app" - | "open_url" - | "open_file" - | "run_script" - | "run_apple_script" - ) { + if policy.target { let app: crate::agentic::tools::computer_use_host::AppSelector = match input.get("app").filter(|v| v.is_object()) { Some(app) => serde_json::from_value(app.clone()) .map_err(|e| OpenBitFunError::tool(format!("Invalid app selector: {e}")))?, @@ -1769,7 +1766,19 @@ impl ComputerUseTool { } } } + if !policy.observation { + if let Some(value) = input.get("observation_id") { + let id = value.as_str().filter(|id| !id.is_empty()).ok_or_else(|| OpenBitFunError::tool("[INVALID_PARAMS] observation_id must be a nonempty string"))?; + host.validate_observation(id)?; + } + } + let observation_scope = host.control_snapshot(); let mut results = self.call_controlled(input, context).await?; + if policy.observation && policy.lease + && !super::computer_use_program::same_active_control(&observation_scope, &host.control_snapshot()) + { + return Err(OpenBitFunError::tool("[CONTROL_STALE] Observation discarded because control stopped or changed target")); + } if let Some(error) = capture_preparation_error { for result in &mut results { if let ToolResult::Result { data, .. } = result { @@ -3347,6 +3356,12 @@ mod tests { fn control_snapshot(&self) -> crate::agentic::tools::computer_use_host::ControlSnapshot { crate::agentic::tools::computer_use_host::ControlSnapshot { supported: self.control_supported, + state: if self.control_supported { + "active" + } else { + "idle" + } + .into(), target: self.control_supported.then(|| "pid:421/window:22".into()), ..Default::default() } @@ -3702,6 +3717,7 @@ mod tests { state: Mutex, events: Arc>>, wait_started: tokio::sync::Notify, + require_share: bool, fail_capture: bool, fail_observation: bool, replace_control_during_observation: Option, @@ -3718,6 +3734,7 @@ mod tests { }), events: Arc::default(), wait_started: tokio::sync::Notify::new(), + require_share: false, fail_capture: false, fail_observation: false, replace_control_during_observation: None, @@ -3750,6 +3767,15 @@ mod tests { } #[async_trait::async_trait] impl ComputerUseHost for ControlRecordingHost { + fn validate_observation(&self, id: &str) -> OpenBitFunResult<()> { + self.events.lock().unwrap().push(format!("validate:{id}")); + if id == "fixture-current" { + Ok(()) + } else { + Err(OpenBitFunError::tool("[STALE_OBSERVATION] Read again")) + } + } + async fn dispatch_app_input( &self, app: crate::agentic::tools::computer_use_host::AppSelector, @@ -3848,6 +3874,23 @@ mod tests { state.state = "stopped".into(); Ok(state.clone()) } + async fn stop_control_generation( + &self, + owner: &str, + generation: u64, + ) -> OpenBitFunResult { + let snapshot = { + let mut state = self.state.lock().unwrap(); + if state.generation != generation { + return Ok(state.clone()); + } + state.generation += 1; + state.state = "stopped".into(); + state.clone() + }; + self.events.lock().unwrap().push(format!("stop:{owner}")); + Ok(snapshot) + } async fn acquire_control_action( &self, owner: &str, @@ -3857,6 +3900,21 @@ mod tests { .lock() .unwrap() .push(format!("acquire:{owner}:{action}")); + { + let mut state = self.state.lock().unwrap(); + if self.require_share { + state.generation += 1; + state.owner = Some(owner.into()); + state.state = "awaiting_share".into(); + return Err(OpenBitFunError::tool( + "[SHARE_REQUIRED] Select an application", + )); + } + if state.state == "idle" { + state.state = "active".into(); + state.owner = Some(owner.into()); + } + } Ok(Some(Box::new(RecordingLease { events: self.events.clone(), complete: false, @@ -3925,6 +3983,108 @@ mod tests { context } + #[test] + fn every_advertised_action_has_explicit_admission_policy() { + for schema in [ + ComputerUseTool::new().input_schema(), + ComputerUseTool::input_schema_text_only(), + ] { + for action in schema["properties"]["action"]["enum"].as_array().unwrap() { + let name = action.as_str().unwrap(); + assert!( + openbitfun_agent_tools::computer_use_control::ControlActionPolicy::for_action( + name + ) + .is_some(), + "{name}" + ); + } + } + } + + #[tokio::test] + async fn remote_workspace_and_headless_hosts_never_fall_back_to_controller_gui() { + let host = Arc::new(ControlRecordingHost::default()); + let mut remote = control_context(host.clone()); + let identity = openbitfun_services_core::workspace_identity::workspace_session_identity( + "/srv/project", + Some("fixture-ssh"), + Some("ssh.fixture"), + ) + .unwrap(); + remote.workspace = Some(crate::agentic::WorkspaceBinding::new_remote( + None, + std::path::PathBuf::from("/srv/project"), + "fixture-ssh".into(), + "Fixture".into(), + identity, + )); + let result = ComputerUseTool::new() + .call_impl(&json!({"action":"start_control"}), &remote) + .await; + assert!(result + .unwrap_err() + .to_string() + .contains("workspace is remote")); + assert!(host.events.lock().unwrap().is_empty()); + let headless = ToolUseContext::for_tool_listing(None, None); + let result = ComputerUseTool::new() + .call_impl( + &json!({"action":"get_app_state","app":{"pid":421}}), + &headless, + ) + .await; + assert!(result + .unwrap_err() + .to_string() + .contains("provider unavailable")); + } + + #[tokio::test] + async fn stale_observation_references_fail_before_any_input() { + let host = Arc::new(ControlRecordingHost::default()); + let context = control_context(host.clone()); + let result = ComputerUseTool::new().call_impl(&json!({ + "action":"app_click", "app":{"pid":421}, "observation_id":"fixture-old", "target":{"node_idx":0} + }), &context).await; + assert!(result + .unwrap_err() + .to_string() + .contains("STALE_OBSERVATION")); + let events = host.events.lock().unwrap(); + assert!(events.iter().any(|event| event == "validate:fixture-old")); + assert!(!events.iter().any(|event| event.starts_with("input:"))); + } + + #[tokio::test] + async fn read_discarded_after_target_revision_changes_even_when_target_is_identical() { + let initial = ControlSnapshot { + supported: true, + state: "active".into(), + owner: Some("control-task".into()), + generation: 1, + target: Some("window:a".into()), + target_revision: 1, + ..Default::default() + }; + let host = Arc::new(ControlRecordingHost { + state: Mutex::new(initial.clone()), + replace_control_during_observation: Some(ControlSnapshot { + target_revision: 3, + ..initial + }), + ..Default::default() + }); + let context = control_context(host); + let result = ComputerUseTool::new() + .call_impl( + &json!({"action":"get_app_state","app":{"pid":421}}), + &context, + ) + .await; + assert!(result.unwrap_err().to_string().contains("CONTROL_STALE")); + } + #[tokio::test] async fn control_entrypoints_route_owner_mode_and_do_not_take_action_leases() { let host = Arc::new(ControlRecordingHost::default()); @@ -3990,6 +4150,48 @@ mod tests { } } + #[tokio::test] + async fn cancelling_a_failed_share_request_closes_only_its_own_generation() { + for superseded in [false, true] { + let host = Arc::new(ControlRecordingHost { + require_share: true, + ..Default::default() + }); + let mut context = control_context(host.clone()); + let token = tokio_util::sync::CancellationToken::new(); + context.runtime_handles = + openbitfun_runtime_ports::ToolRuntimeHandles::new(None, Some(token.clone())); + let error = ComputerUseTool::new() + .call_impl( + &json!({"action":"get_app_state","app":{"pid":421}}), + &context, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("SHARE_REQUIRED")); + assert_eq!(host.control_snapshot().state, "awaiting_share"); + if superseded { + host.state.lock().unwrap().generation += 1; + } + token.cancel(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + assert_eq!( + host.control_snapshot().state, + if superseded { + "awaiting_share" + } else { + "stopped" + } + ); + assert!(!host + .events + .lock() + .unwrap() + .iter() + .any(|event| event.starts_with("observe:") || event.starts_with("input:"))); + } + } + #[tokio::test] async fn completed_action_releases_lease_without_cancelling_control() { let host = Arc::new(ControlRecordingHost::default()); diff --git a/src/crates/contracts/product-domains/src/generated/product-control-catalog.json b/src/crates/contracts/product-domains/src/generated/product-control-catalog.json index 954e70afd1..7c70862c5b 100644 --- a/src/crates/contracts/product-domains/src/generated/product-control-catalog.json +++ b/src/crates/contracts/product-domains/src/generated/product-control-catalog.json @@ -4,7 +4,7 @@ "title": "OpenBitFun Playbook", "origin": "https://playbook.openbitfun.com", "source": "src/shared/interactive-capabilities/catalog.json", - "digest": "e05dbc222e1267523832f82d6392701ab011c9d8d5ad1582dbe407904dc80d3d", + "digest": "156caca2db3ed5820e9bf6683d4df89db5bbbfac8c6c8910d31262c05450ec6e", "ownerDigest": "2269c2432e5db44706b2bce44df3c60672141a47c36141aeb1257ea7fd2507f8", "searchAcceptance": [ { @@ -138,11 +138,11 @@ "features": 22, "settings": 21, "userFacing": 43, - "documentedItems": 325, + "documentedItems": 328, "controlCoverage": { "direct": 49, "delegated": 62, - "interactive": 214, + "interactive": 217, "unsupported": 0 } }, @@ -4869,7 +4869,10 @@ "default-cdp", "agent-actions", "control-session-preview", - "stop-control-session" + "stop-control-session", + "app-sharing", + "preview-input", + "remembered-control" ], "kind": "query", "risk": "read", @@ -5322,6 +5325,144 @@ "pageId": "tools.execution" } }, + { + "id": "feature.computer-use:open:app-sharing", + "capabilityId": "feature.computer-use", + "itemIds": [ + "app-sharing" + ], + "kind": "open", + "risk": "ui", + "executionHost": "presentationSurface", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1", + "product_control_presentation_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": false + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "openReason": "visualSelection", + "presentationTarget": { + "kind": "settings", + "pageId": "tools.execution" + } + }, + { + "id": "feature.computer-use:open:preview-input", + "capabilityId": "feature.computer-use", + "itemIds": [ + "preview-input" + ], + "kind": "open", + "risk": "ui", + "executionHost": "presentationSurface", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1", + "product_control_presentation_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": false + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "openReason": "visualSelection", + "presentationTarget": { + "kind": "settings", + "pageId": "tools.execution" + } + }, + { + "id": "feature.computer-use:open:remembered-control", + "capabilityId": "feature.computer-use", + "itemIds": [ + "remembered-control" + ], + "kind": "open", + "risk": "ui", + "executionHost": "presentationSurface", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1", + "product_control_presentation_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": false, + "reason": "This delivery profile has no live presentation surface" + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": false + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "openReason": "visualSelection", + "presentationTarget": { + "kind": "settings", + "pageId": "tools.execution" + } + }, { "id": "feature.skills:query", "capabilityId": "feature.skills", @@ -21986,13 +22127,15 @@ "启用或停用 Computer Use", "连接或断开受支持的浏览器", "检查并打开系统权限设置", - "查看当前控制目标与预览，并随时停止控制" + "查看当前控制目标与预览，并随时停止控制", + "先选择共享 App，再授权输入；可记住授权或随时撤销" ], "highlightsEn": [ "Enable or disable Computer Use", "Connect or disconnect a supported browser", "Inspect and open OS permission settings", - "Inspect the current control target and preview, and stop control at any time" + "Inspect the current control target and preview, and stop control at any time", + "Select an App share before granting input; remember or revoke the grant" ], "items": [ { @@ -22109,6 +22252,39 @@ "Use ComputerUse stop_control to release the tool-owned session. Users can also stop from the executing Desktop control card; its button addresses the displayed generation rather than a subsequently created session." ] } + }, + { + "id": "app-sharing", + "titleZh": "选择 App 窗口并单独授权或撤销控制", + "titleEn": "Select an App window and separately grant or revoke control", + "control": { + "kind": "open", + "reasonCode": "visualSelection", + "reasonZh": "共享窗口及控制模式由用户明确选择；智能体只能请求共享，不能代替用户授权。", + "reasonEn": "Window selection and input mode require an explicit user choice; agents can request sharing but cannot grant themselves control." + } + }, + { + "id": "preview-input", + "titleZh": "在共享预览中点击、拖动、滚动及输入", + "titleEn": "Click, drag, scroll and type through the shared preview", + "control": { + "kind": "open", + "reasonCode": "visualSelection", + "reasonZh": "预览输入需要用户授权，并绑定共享窗口、图像坐标及授权版本。", + "reasonEn": "Preview input requires user authorization and is bound to the shared window, image coordinates and authorization revision." + } + }, + { + "id": "remembered-control", + "titleZh": "按应用执行文件和控制模式记住或忘记授权", + "titleEn": "Remember or forget control by application executable and input mode", + "control": { + "kind": "open", + "reasonCode": "visualSelection", + "reasonZh": "记住授权只在用户选定并共享同一应用后生效，应用文件或输入模式改变时需要重新授权。", + "reasonEn": "Remembered control applies only after the user shares the same application; changed executable contents or input modes require a new grant." + } } ], "stepsZh": [ @@ -22156,10 +22332,12 @@ "连接或断开受支持的浏览器", "检查并打开系统权限设置", "查看当前控制目标与预览，并随时停止控制", + "先选择共享 App，再授权输入；可记住授权或随时撤销", "Enable or disable Computer Use", "Connect or disconnect a supported browser", "Inspect and open OS permission settings", "Inspect the current control target and preview, and stop control at any time", + "Select an App share before granting input; remember or revoke the grant", "检查 Computer Use 可用性、辅助功能和屏幕录制授权状态", "Check Computer Use availability plus accessibility and screen-recording permission status", "请求桌面控制所需的系统权限", @@ -22178,6 +22356,12 @@ "Inspect the current target, mode, and authorized capture preview in the control card", "停止当前桌面控制会话并释放捕获与输入资源", "Stop the current desktop-control session and release capture and input resources", + "选择 App 窗口并单独授权或撤销控制", + "Select an App window and separately grant or revoke control", + "在共享预览中点击、拖动、滚动及输入", + "Click, drag, scroll and type through the shared preview", + "按应用执行文件和控制模式记住或忘记授权", + "Remember or forget control by application executable and input mode", "打开桌面控制设置", "带我配置浏览器控制", "Open desktop-control settings", diff --git a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json index e0211eedd0..f162024e8f 100644 --- a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json +++ b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "digest": "fnv1a64:7a6e4c114b970186", + "digest": "fnv1a64:c48277b5f23f04e7", "retiredCommandPrefixes": [ { "prefix": "lsp_", @@ -19,6 +19,7 @@ "token_usage_statistics", "miniapp_agent_context_files_v1", "wsl_workspaces_v1", + "computer_use_sharing_v1", "product_control_v1", "product_control_native_v1", "product_control_presentation_v1", @@ -40,6 +41,7 @@ "token_usage_statistics", "miniapp_agent_context_files_v1", "wsl_workspaces_v1", + "computer_use_sharing_v1", "product_control_v1", "product_control_native_v1", "product_control_presentation_v1", @@ -1114,6 +1116,18 @@ "reason": "the CLI peer host has no browser or OS automation runtime" } }, + { + "id": "computer_use_request_permission", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no browser or OS automation runtime" + } + }, { "id": "computer_use_request_permissions", "surface": "tauri_command", @@ -1126,6 +1140,102 @@ "reason": "the CLI peer host has no browser or OS automation runtime" } }, + { + "id": "computer_use_share_control", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no browser or OS automation runtime" + } + }, + { + "id": "computer_use_share_forget_grant", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no browser or OS automation runtime" + } + }, + { + "id": "computer_use_share_grants", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no browser or OS automation runtime" + } + }, + { + "id": "computer_use_share_input", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no browser or OS automation runtime" + } + }, + { + "id": "computer_use_share_observe", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no browser or OS automation runtime" + } + }, + { + "id": "computer_use_share_request", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no browser or OS automation runtime" + } + }, + { + "id": "computer_use_share_start", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no browser or OS automation runtime" + } + }, + { + "id": "computer_use_share_targets", + "surface": "tauri_command", + "remoteWorkspace": "LocalOnly", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no browser or OS automation runtime" + } + }, { "id": "confirm_frontend_update", "surface": "tauri_command", diff --git a/src/crates/contracts/product-domains/src/remote_surface/capabilities.rs b/src/crates/contracts/product-domains/src/remote_surface/capabilities.rs index 3f20b1ef5d..6a4c0c5a03 100644 --- a/src/crates/contracts/product-domains/src/remote_surface/capabilities.rs +++ b/src/crates/contracts/product-domains/src/remote_surface/capabilities.rs @@ -42,6 +42,8 @@ pub enum PeerHostCapability { MiniappAgentContextFilesV1, /// WSL profiles and discovery are understood; availability is probed on the host. WslWorkspacesV1, + /// User-selected native window sharing and explicit permission receipts. + ComputerUseSharingV1, /// `product_control_invoke` is implemented (shared config contract). ProductControlV1, /// ProductControl definitions that need a host-native provider run here. @@ -75,6 +77,7 @@ impl PeerHostCapability { Self::TokenUsageStatistics, Self::MiniappAgentContextFilesV1, Self::WslWorkspacesV1, + Self::ComputerUseSharingV1, Self::ProductControlV1, Self::ProductControlNativeV1, Self::ProductControlPresentationV1, @@ -99,6 +102,7 @@ impl PeerHostCapability { Self::TokenUsageStatistics => "token_usage_statistics", Self::MiniappAgentContextFilesV1 => "miniapp_agent_context_files_v1", Self::WslWorkspacesV1 => "wsl_workspaces_v1", + Self::ComputerUseSharingV1 => "computer_use_sharing_v1", Self::ProductControlV1 => "product_control_v1", Self::ProductControlNativeV1 => "product_control_native_v1", Self::ProductControlPresentationV1 => "product_control_presentation_v1", diff --git a/src/crates/contracts/product-domains/src/remote_surface/table.rs b/src/crates/contracts/product-domains/src/remote_surface/table.rs index 026a4582dd..4c60dac8a9 100644 --- a/src/crates/contracts/product-domains/src/remote_surface/table.rs +++ b/src/crates/contracts/product-domains/src/remote_surface/table.rs @@ -172,7 +172,16 @@ pub(super) const OPERATIONS: &[OperationDefinition] = &[ op("computer_use_control_stop", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), op("computer_use_get_status", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), op("computer_use_open_system_settings", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), + op("computer_use_request_permission", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), op("computer_use_request_permissions", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), + op("computer_use_share_control", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), + op("computer_use_share_forget_grant", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), + op("computer_use_share_grants", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), + op("computer_use_share_input", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), + op("computer_use_share_observe", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), + op("computer_use_share_request", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), + op("computer_use_share_start", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), + op("computer_use_share_targets", LocalOnly, Proxied, CLI_NO_BROWSER_OR_OS_AUTOMATION), op("confirm_frontend_update", LocalOnly, ControllerLocal, REFUSED), op("control_background_command", Unaudited, Proxied, CLI_NOT_IMPLEMENTED), op("control_deep_review_queue", Unaudited, Proxied, CLI_NOT_IMPLEMENTED), diff --git a/src/crates/execution/tool-contracts/AGENTS.md b/src/crates/execution/tool-contracts/AGENTS.md index 65800deefa..51be8b1e7c 100644 --- a/src/crates/execution/tool-contracts/AGENTS.md +++ b/src/crates/execution/tool-contracts/AGENTS.md @@ -41,7 +41,7 @@ the product tool runtime. cargo test --locked -p openbitfun-agent-tools --no-default-features cargo test --locked -p openbitfun-agent-tools --no-default-features --features acp-bridge --test tool_contracts acp_external_agent_bridge_preserves_tool_contract cargo test --locked -p openbitfun-agent-tools --no-default-features --features mcp-bridge --test tool_contracts mcp_tool_bridge -cargo test --locked -p openbitfun-agent-tools --no-default-features --features computer-use-contract --lib computer_use:: +cargo test --locked -p openbitfun-agent-tools --no-default-features --features computer-use-contract --lib computer_use cargo test --locked -p openbitfun-agent-tools --no-default-features --features element-token --lib element_token:: node scripts/check-core-boundaries.mjs ``` diff --git a/src/crates/execution/tool-contracts/src/computer_use.rs b/src/crates/execution/tool-contracts/src/computer_use.rs index 9a45b48119..8fc144f456 100644 --- a/src/crates/execution/tool-contracts/src/computer_use.rs +++ b/src/crates/execution/tool-contracts/src/computer_use.rs @@ -620,6 +620,10 @@ fn normalize_windows_key(token: &str) -> String { /// `app_*` mutation so the model can verify changes in one round-trip. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct AppStateSnapshot { + /// Opaque node address-space identity. Echo on subsequent input; unlike the + /// content digest this changes even when two observations look identical. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observation_id: Option, /// Identity of the captured application. pub app: AppInfo, /// Title of the focused window when `focus_window_only=true`, else @@ -758,6 +762,9 @@ fn default_true() -> bool { /// `ComputerUseHost::build_interactive_view`. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct InteractiveView { + /// Node/view address space from the underlying application observation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observation_id: Option, /// Identity of the captured application. pub app: AppInfo, /// Title of the focused window (or `None` when the host could not @@ -946,6 +953,9 @@ pub struct VisualMark { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct VisualMarkView { + /// Node/view address space from the underlying application observation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observation_id: Option, pub app: AppInfo, #[serde(default, skip_serializing_if = "Option::is_none")] pub window_title: Option, @@ -1141,18 +1151,143 @@ impl AppInputAction { } pub fn validate(&self) -> Result<(), String> { match self { - Self::Click {click_count, mouse_button, ..} if !(1..=3).contains(click_count) || !matches!(mouse_button.as_str(), "left"|"right"|"middle") => Err("click_count must be 1..3 and mouse_button must be left/right/middle".into()), - Self::KeyChord {keys, ..} if keys.is_empty() || keys.iter().any(|key| key.trim().is_empty()) => Err("keys must contain non-empty key names".into()), - Self::Drag {from, to, ..} if !matches!((from, to), ( - ClickTarget::ImageXy {x:x0,y:y0,screenshot_id:Some(a)}, - ClickTarget::ImageXy {x:x1,y:y1,screenshot_id:Some(b)} - ) if *x0 >= 0 && *y0 >= 0 && *x1 >= 0 && *y1 >= 0 && !a.is_empty() && a == b) => Err("drag endpoints must use nonnegative image_xy coordinates from the same screenshot_id".into()), - Self::Drag {mouse_button, duration_ms, ..} if !matches!(mouse_button.as_str(), "left"|"right"|"middle") || *duration_ms == 0 => Err("drag requires a valid button and positive duration_ms".into()), + Self::Click { + click_count, + mouse_button, + .. + } if !(1..=3).contains(click_count) + || !matches!(mouse_button.as_str(), "left" | "right" | "middle") => + { + return Err( + "click_count must be 1..3 and mouse_button must be left/right/middle".into(), + ); + } + Self::KeyChord { keys, .. } + if keys.is_empty() || keys.iter().any(|key| key.trim().is_empty()) => + { + return Err("keys must contain non-empty key names".into()); + } + Self::Drag { + from, + to, + mouse_button, + duration_ms, + } => { + if !matches!((from, to), ( + ClickTarget::ImageXy { screenshot_id: Some(a), .. }, + ClickTarget::ImageXy { screenshot_id: Some(b), .. } + ) if a == b) + { + return Err( + "drag endpoints must use image_xy coordinates from the same screenshot_id" + .into(), + ); + } + if !matches!(mouse_button.as_str(), "left" | "right" | "middle") + || *duration_ms == 0 + { + return Err("drag requires a valid button and positive duration_ms".into()); + } + } + _ => {} + } + match self { + Self::Click { target, .. } => target.validate(), + Self::TypeText { + focus: Some(target), + .. + } + | Self::Scroll { + focus: Some(target), + .. + } => target.validate(), + Self::Drag { from, to, .. } => { + from.validate()?; + to.validate() + } _ => Ok(()), } } } +impl ClickTarget { + pub fn validate(&self) -> Result<(), String> { + match self { + ClickTarget::ImageXy { + x, + y, + screenshot_id, + } => { + if *x < 0 + || *y < 0 + || screenshot_id + .as_deref() + .is_none_or(|id| id.trim().is_empty()) + { + return Err("image_xy requires nonnegative image coordinates and the observed screenshot_id".into()); + } + } + ClickTarget::ImageGrid { + x0, + y0, + width, + height, + rows, + cols, + row, + col, + intersections, + screenshot_id, + } => { + if *x0 < 0 + || *y0 < 0 + || *width == 0 + || *height == 0 + || i64::from(*x0) + i64::from(*width) > i64::from(i32::MAX) + 1 + || i64::from(*y0) + i64::from(*height) > i64::from(i32::MAX) + 1 + || *rows == 0 + || *cols == 0 + || row >= rows + || col >= cols + || (*intersections && (*rows < 2 || *cols < 2)) + || screenshot_id + .as_deref() + .is_none_or(|id| id.trim().is_empty()) + { + return Err("image_grid requires a nonempty observed screenshot_id, positive rectangle/grid dimensions and an in-range row/col".into()); + } + } + ClickTarget::VisualGrid { + rows, + cols, + row, + col, + intersections, + .. + } => { + if *rows == 0 + || *cols == 0 + || row >= rows + || col >= cols + || (*intersections && (*rows < 2 || *cols < 2)) + { + return Err( + "visual_grid requires positive dimensions and an in-range row/col".into(), + ); + } + } + ClickTarget::OcrText { needle } if needle.trim().is_empty() => { + return Err("ocr_text requires non-empty observed text".into()) + } + ClickTarget::ScreenXy { x, y } if !x.is_finite() || !y.is_finite() => { + return Err("screen_xy coordinates must be finite".into()) + } + _ => {} + } + Ok(()) + } +} + /// Predicate for `ComputerUseHost::app_wait_for`. /// /// Hosts that don't yet implement AX waiting can simply return the @@ -1832,6 +1967,28 @@ mod tool_body_tests { #[cfg(test)] mod app_input_program_tests { + #[test] + fn legacy_app_observation_omits_identity_without_losing_content() { + let old = serde_json::json!({"app":{"name":"Editor","pid":7,"running":true},"tree_text":"[0] Button","digest":"same-content","captured_at_ms":1}); + let snapshot: super::AppStateSnapshot = serde_json::from_value(old).unwrap(); + assert!(snapshot.observation_id.is_none()); + let round: super::AppStateSnapshot = + serde_json::from_value(serde_json::to_value(&snapshot).unwrap()).unwrap(); + assert_eq!(snapshot, round); + } + + #[test] + fn all_program_targets_are_validated_before_native_dispatch() { + for value in [ + serde_json::json!({"action":"app_click","target":{"kind":"image_xy","x":1,"y":1}}), + serde_json::json!({"action":"app_type_text","text":"x","focus":{"kind":"ocr_text","needle":" "}}), + serde_json::json!({"action":"app_scroll","dx":0,"dy":1,"focus":{"kind":"image_grid","x0":2147483647,"y0":0,"width":2,"height":1,"rows":1,"cols":1,"row":0,"col":0,"screenshot_id":"shot"}}), + ] { + let input: super::AppInputAction = serde_json::from_value(value).unwrap(); + assert!(input.validate().is_err()); + } + } + use super::*; use serde_json::json; diff --git a/src/crates/execution/tool-contracts/src/computer_use_control.rs b/src/crates/execution/tool-contracts/src/computer_use_control.rs index 7d8817d826..d9fdcce927 100644 --- a/src/crates/execution/tool-contracts/src/computer_use_control.rs +++ b/src/crates/execution/tool-contracts/src/computer_use_control.rs @@ -19,11 +19,69 @@ pub struct ControlSnapshot { pub mode: ControlMode, pub state: String, pub target: Option, + /// Changes on every rebind, including A -> B -> A in one generation. + /// Older peers omit this and retain their existing target comparison. + pub target_revision: u64, pub action: Option, pub sequence: u64, pub reason: Option, pub pointer: Option, pub capabilities: Vec, + /// Missing on older hosts. A native share grants observation first; input + /// is authorized separately by the driving user, never by a tool call. + #[serde(skip_serializing_if = "Option::is_none")] + pub sharing: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub background_observation: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BackgroundObservation { + pub sequence: u64, + pub foreground_unchanged: bool, + pub cursor_unchanged: Option, + pub window_order_unchanged: Option, + pub scope: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ControlSharing { + pub phase: SharingPhase, + pub requested_mode: ControlMode, + pub target_label: Option, + pub control_enabled: bool, + pub authorization_revision: u64, + #[serde(default)] + pub remembered: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub grant_note: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SharingPhase { + Selecting, + Starting, + Shared, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ShareTarget { + /// Native identity. Titles are labels, never authorization identities. + pub id: String, + /// Opaque selection reference bound to the process lifetime at discovery. + #[serde(default)] + pub selection_id: String, + #[serde(default)] + pub unavailable_reason: Option, + pub pid: Option, + pub window_id: Option, + pub application: String, + pub title: String, + pub bundle_id: Option, + pub background_pointer: bool, + pub background_semantic: bool, + pub selection_required: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -53,48 +111,85 @@ pub struct ControlStartRequest { pub mode: ControlMode, } -/// Explicit action classification, not a loop/count heuristic. -pub fn action_requires_foreground(action: &str) -> bool { - matches!( - action, - "click" - | "mouse_move" - | "pointer_move_rel" - | "scroll" - | "drag" - | "key_chord" - | "type_text" - | "paste" - | "click_target" - | "move_to_target" - | "click_element" - | "move_to_text" - | "open_app" - | "focus_display" - | "run_apple_script" - | "run_script" - | "open_url" - | "open_file" - ) +/// Admission facts shared by the tool, native lease and read-only retry policy. +/// Unknown names have no policy and must fail before acquiring any resource. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ControlActionPolicy { + pub observation: bool, + pub foreground: bool, + pub target: bool, + pub lease: bool, } -pub fn action_is_observation(action: &str) -> bool { - matches!( - action, - "screenshot" +impl ControlActionPolicy { + pub fn for_action(action: &str) -> Option { + let (observation, foreground, target, lease) = match action { + "start_control" | "stop_control" => (false, false, false, false), + "control_status" | "list_apps" | "list_displays" | "get_os_info" | "clipboard_get" => { + (true, false, false, false) + } + "wait" => (true, false, false, true), + "screenshot" | "describe_screen" | "locate" - | "wait" - | "list_displays" - | "list_apps" | "get_app_state" | "get_app_shortcuts" | "app_wait_for" | "build_interactive_view" - | "build_visual_mark_view" - | "get_os_info" - | "clipboard_get" - ) + | "build_visual_mark_view" => (true, false, true, true), + "app_click" + | "app_type_text" + | "app_scroll" + | "app_key_chord" + | "app_drag" + | "app_batch" + | "interactive_click" + | "interactive_type_text" + | "interactive_scroll" + | "visual_click" => (false, false, true, true), + "clipboard_set" => (false, false, false, true), + "open_app" | "run_apple_script" | "run_script" | "open_url" | "open_file" => { + (false, true, false, true) + } + "click" | "mouse_move" | "pointer_move_rel" | "scroll" | "drag" | "key_chord" + | "type_text" | "paste" | "click_target" | "move_to_target" | "click_element" + | "move_to_text" | "focus_display" => (false, true, true, true), + _ => return None, + }; + Some(Self { + observation, + foreground, + target, + lease, + }) + } +} + +pub fn action_requires_foreground(action: &str) -> bool { + ControlActionPolicy::for_action(action).is_some_and(|policy| policy.foreground) +} + +pub fn action_is_observation(action: &str) -> bool { + ControlActionPolicy::for_action(action).is_some_and(|policy| policy.observation) +} + +/// Identity of an observation binding, separate from the input action lease. +/// It is an in-process fact, never authority supplied by a model/controller. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ObservationScope { + pub generation: u64, + pub target_revision: u64, + pub target: Option, +} + +impl ObservationScope { + pub fn from_control(control: &ControlSnapshot) -> Self { + Self { + generation: control.generation, + target_revision: control.target_revision, + target: control.target.clone(), + } + } } #[cfg(test)] @@ -104,6 +199,7 @@ mod tests { fn legacy_payloads_do_not_grant_control() { let old: ControlSnapshot = serde_json::from_str("{}").unwrap(); assert!(!old.supported); + assert_eq!(old.target_revision, 0); assert!(old.owner.is_none()); let round: ControlSnapshot = serde_json::from_value(serde_json::to_value(old).unwrap()).unwrap(); @@ -131,4 +227,24 @@ mod tests { assert!(!action_requires_foreground("app_click")); assert!(action_is_observation("get_app_state")); } + + #[test] + fn unknown_actions_cannot_acquire_a_default_input_policy() { + assert!(ControlActionPolicy::for_action("native_surprise").is_none()); + let observation = ControlActionPolicy::for_action("get_app_state").unwrap(); + assert!(observation.observation && observation.target && observation.lease); + let discovery = ControlActionPolicy::for_action("list_apps").unwrap(); + assert!(discovery.observation && !discovery.target && !discovery.lease); + } + + #[test] + fn returning_to_the_same_target_does_not_restore_old_observation_scope() { + let mut control = ControlSnapshot { + target: Some("pid:1/window:2".into()), + ..Default::default() + }; + let old = ObservationScope::from_control(&control); + control.target_revision += 2; + assert_ne!(old, ObservationScope::from_control(&control)); + } } diff --git a/src/crates/execution/tool-contracts/src/computer_use_geometry.rs b/src/crates/execution/tool-contracts/src/computer_use_geometry.rs new file mode 100644 index 0000000000..9939a9a3b5 --- /dev/null +++ b/src/crates/execution/tool-contracts/src/computer_use_geometry.rs @@ -0,0 +1,352 @@ +//! Coordinate transforms for an observed surface, independent of the host OS. +//! +//! Adapted from computer-use-rust's sharing::CoordinateMap. The image's actual +//! content rectangle and the captured surface bounds are the only scale source; +//! display DPI, the human pointer and the current foreground window are not. + +use crate::computer_use::{ComputerUseImageContentRect, ComputerUseImageGlobalBounds}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CaptureGeometry { + image: [u32; 2], + content: [u32; 4], + bounds: [f64; 4], +} + +impl CaptureGeometry { + pub fn new( + width: u32, + height: u32, + content: &ComputerUseImageContentRect, + bounds: &ComputerUseImageGlobalBounds, + ) -> Result { + let geometry = Self { + image: [width, height], + content: [content.left, content.top, content.width, content.height], + bounds: [bounds.left, bounds.top, bounds.width, bounds.height], + }; + geometry.validate()?; + Ok(geometry) + } + + fn validate(&self) -> Result<(), String> { + let [left, top, width, height] = self.content; + let [x, y, span_x, span_y] = self.bounds; + if self.image.contains(&0) + || width == 0 + || height == 0 + || left + .checked_add(width) + .is_none_or(|right| right > self.image[0]) + || top + .checked_add(height) + .is_none_or(|bottom| bottom > self.image[1]) + || !self.bounds.iter().all(|value| value.is_finite()) + || span_x <= 0.0 + || span_y <= 0.0 + || !(x + span_x).is_finite() + || !(y + span_y).is_finite() + { + return Err("[INVALID_GEOMETRY] Capture requires finite bounds and a nonempty content rectangle inside the image".into()); + } + Ok(()) + } + + /// Image pixel indices address their centers. Padding and out-of-range + /// points fail; clamping them could silently click a different control. + pub fn image_to_global(&self, x: i32, y: i32) -> Result<(f64, f64), String> { + let [left, top, width, height] = self.content; + let x = i64::from(x) - i64::from(left); + let y = i64::from(y) - i64::from(top); + if x < 0 || y < 0 || x >= i64::from(width) || y >= i64::from(height) { + return Err("[INVALID_COORDINATES] Point is outside the captured image content (including padding)".into()); + } + Ok(( + self.bounds[0] + (x as f64 + 0.5) * self.bounds[2] / f64::from(width), + self.bounds[1] + (y as f64 + 0.5) * self.bounds[3] / f64::from(height), + )) + } + + /// Legacy normalized coordinates select the surface's native point grid. + /// Preserve endpoint semantics, while rejecting out-of-range values. + pub fn normalized_to_global(&self, x: i32, y: i32) -> Result<(f64, f64), String> { + if !(0..=1000).contains(&x) || !(0..=1000).contains(&y) { + return Err( + "[INVALID_COORDINATES] Normalized coordinates must be within 0..=1000".into(), + ); + } + let axis = |origin: f64, span: f64, value: i32| { + origin + f64::from(value) / 1000.0 * (span - 1.0).max(0.0) + span.min(1.0) / 2.0 + }; + Ok(( + axis(self.bounds[0], self.bounds[2], x), + axis(self.bounds[1], self.bounds[3], y), + )) + } + + pub fn image_delta_to_global(&self, dx: i32, dy: i32) -> (f64, f64) { + ( + f64::from(dx) * self.bounds[2] / f64::from(self.content[2]), + f64::from(dy) * self.bounds[3] / f64::from(self.content[3]), + ) + } + + /// Continuous image coordinates are used by a preview gesture. Unlike + /// integer pixel indices they already describe their sub-pixel location. + pub fn image_point_to_global(&self, x: f64, y: f64) -> Result<(f64, f64), String> { + let [left, top, width, height] = self.content.map(f64::from); + if !x.is_finite() + || !y.is_finite() + || x < left + || y < top + || x >= left + width + || y >= top + height + { + return Err("[INVALID_COORDINATES] Point is outside captured image content".into()); + } + Ok(( + self.bounds[0] + (x - left) * self.bounds[2] / width, + self.bounds[1] + (y - top) * self.bounds[3] / height, + )) + } + + pub fn global_to_image(&self, x: f64, y: f64) -> Result<(f64, f64), String> { + if !contains(self.bounds, [x, y]) { + return Err("[INVALID_COORDINATES] Point is outside the captured window".into()); + } + Ok(( + f64::from(self.content[0]) + + (x - self.bounds[0]) / self.bounds[2] * f64::from(self.content[2]), + f64::from(self.content[1]) + + (y - self.bounds[1]) / self.bounds[3] * f64::from(self.content[3]), + )) + } + + pub fn fit_preview(&self, available: [f64; 4]) -> Result { + if !valid_bounds(available) { + return Err("[INVALID_GEOMETRY] Preview bounds must be finite and positive".into()); + } + let scale = + (available[2] / f64::from(self.image[0])).min(available[3] / f64::from(self.image[1])); + let width = f64::from(self.image[0]) * scale; + let height = f64::from(self.image[1]) * scale; + let bounds = [ + available[0] + (available[2] - width) / 2.0, + available[1] + (available[3] - height) / 2.0, + width, + height, + ]; + if !valid_bounds(bounds) { + return Err("[INVALID_GEOMETRY] Preview projection overflowed".into()); + } + Ok(PreviewGeometry { + capture: *self, + bounds, + }) + } + + pub fn same_projection(&self, other: &Self) -> bool { + self.image == other.image + && self.content == other.content + && self.bounds[2..] == other.bounds[2..] + } + + /// A moved window preserves local coordinates; resizing requires another + /// observation. Bounds stay fractional (macOS points) or physical pixels + /// (Windows), according to the native provider's coordinate contract. + pub fn at_bounds(mut self, bounds: [f64; 4]) -> Result { + if bounds[2..] != self.bounds[2..] { + return Err( + "[STALE_CAPTURE] Target resized; observe again before coordinate input".into(), + ); + } + self.bounds = bounds; + self.validate()?; + Ok(self) + } + + pub fn bounds(&self) -> [f64; 4] { + self.bounds + } +} + +fn valid_bounds(bounds: [f64; 4]) -> bool { + bounds.iter().all(|value| value.is_finite()) + && bounds[2] > 0.0 + && bounds[3] > 0.0 + && (bounds[0] + bounds[2]).is_finite() + && (bounds[1] + bounds[3]).is_finite() +} +fn contains(bounds: [f64; 4], point: [f64; 2]) -> bool { + point.iter().all(|value| value.is_finite()) + && point[0] >= bounds[0] + && point[1] >= bounds[1] + && point[0] < bounds[0] + bounds[2] + && point[1] < bounds[1] + bounds[3] +} + +/// Aspect-fit preview geometry, migrated from sharing::CoordinateMap. Preview +/// letterboxing and padding inside the capture are different excluded regions. +#[derive(Debug, Clone, Copy)] +pub struct PreviewGeometry { + capture: CaptureGeometry, + bounds: [f64; 4], +} +impl PreviewGeometry { + pub fn bounds(&self) -> [f64; 4] { + self.bounds + } + pub fn preview_to_image(&self, x: f64, y: f64) -> Result<(f64, f64), String> { + if !contains(self.bounds, [x, y]) { + return Err("[INVALID_COORDINATES] Point is outside the preview image, including letterbox margins".into()); + } + let image = ( + (x - self.bounds[0]) / self.bounds[2] * f64::from(self.capture.image[0]), + (y - self.bounds[1]) / self.bounds[3] * f64::from(self.capture.image[1]), + ); + self.capture.image_point_to_global(image.0, image.1)?; + Ok(image) + } + pub fn preview_to_global(&self, x: f64, y: f64) -> Result<(f64, f64), String> { + let image = self.preview_to_image(x, y)?; + self.capture.image_point_to_global(image.0, image.1) + } + pub fn image_to_preview(&self, x: f64, y: f64) -> Result<(f64, f64), String> { + self.capture.image_point_to_global(x, y)?; + Ok(( + self.bounds[0] + x / f64::from(self.capture.image[0]) * self.bounds[2], + self.bounds[1] + y / f64::from(self.capture.image[1]) * self.bounds[3], + )) + } + pub fn global_to_preview(&self, x: f64, y: f64) -> Result<(f64, f64), String> { + let image = self.capture.global_to_image(x, y)?; + self.image_to_preview(image.0, image.1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn geometry() -> CaptureGeometry { + CaptureGeometry::new( + 220, + 140, + &ComputerUseImageContentRect { + left: 10, + top: 20, + width: 200, + height: 100, + }, + &ComputerUseImageGlobalBounds { + left: -1200.25, + top: 40.5, + width: 400.0, + height: 150.0, + }, + ) + .unwrap() + } + + #[test] + fn mixed_scale_negative_origin_and_padding_use_the_observed_surface() { + let map = geometry(); + assert_eq!(map.image_to_global(10, 20).unwrap(), (-1199.25, 41.25)); + assert_eq!(map.image_to_global(209, 119).unwrap(), (-801.25, 189.75)); + assert_eq!(map.image_delta_to_global(4, -3), (8.0, -4.5)); + for point in [ + (9, 20), + (10, 19), + (210, 20), + (10, 120), + (-1, 20), + (i32::MIN, i32::MAX), + ] { + assert!(map.image_to_global(point.0, point.1).is_err(), "{point:?}"); + } + } + + #[test] + fn movement_preserves_fractional_coordinates_but_even_small_resizes_invalidate() { + let before = geometry(); + let after = before.at_bounds([10.25, -50.0, 400.0, 150.0]).unwrap(); + assert!(before.same_projection(&after)); + assert_eq!(after.image_to_global(10, 20).unwrap(), (11.25, -49.25)); + assert!(before.at_bounds([0.0, 0.0, 400.5, 150.0]).is_err()); + assert!(before.at_bounds([f64::NAN, 0.0, 400.0, 150.0]).is_err()); + } + + #[test] + fn malformed_or_overflowing_geometry_is_never_actionable() { + let mut content = ComputerUseImageContentRect { + left: 0, + top: 0, + width: 10, + height: 10, + }; + let mut bounds = ComputerUseImageGlobalBounds { + left: 0.0, + top: 0.0, + width: 10.0, + height: 10.0, + }; + assert!(CaptureGeometry::new(0, 10, &content, &bounds).is_err()); + content.left = u32::MAX; + assert!(CaptureGeometry::new(u32::MAX, 10, &content, &bounds).is_err()); + content.left = 0; + bounds.width = f64::INFINITY; + assert!(CaptureGeometry::new(10, 10, &content, &bounds).is_err()); + bounds.width = f64::MAX; + bounds.left = f64::MAX; + assert!(CaptureGeometry::new(10, 10, &content, &bounds).is_err()); + } + + #[test] + fn normalized_endpoints_preserve_legacy_mapping_without_clamping() { + let map = geometry(); + assert_eq!( + map.normalized_to_global(0, 1000).unwrap(), + (-1199.75, 190.0) + ); + assert!(map.normalized_to_global(-1, 0).is_err()); + assert!(map.normalized_to_global(0, 1001).is_err()); + } + + #[test] + fn preview_letterbox_and_capture_padding_are_rejected_independently() { + let capture = geometry(); + let map = capture.fit_preview([10.0, 20.0, 440.0, 440.0]).unwrap(); + assert_eq!(map.bounds(), [10.0, 100.0, 440.0, 280.0]); + assert!(map.preview_to_image(20.0, 30.0).is_err()); // preview margin + assert!(map.preview_to_image(20.0, 110.0).is_err()); // capture padding + assert!(map.preview_to_image(450.0, 200.0).is_err()); // right edge + let point = map.image_to_preview(42.25, 62.5).unwrap(); + assert_eq!(point, (94.5, 225.0)); + assert_eq!( + map.preview_to_image(point.0, point.1).unwrap(), + (42.25, 62.5) + ); + let global = map.preview_to_global(point.0, point.1).unwrap(); + assert_eq!(global, (-1135.75, 104.25)); + assert_eq!(map.global_to_preview(global.0, global.1).unwrap(), point); + } + + #[test] + fn preview_resize_preserves_the_same_image_point() { + let capture = geometry(); + for size in [ + [-50.0, 20.0, 600.0, 130.0], + [0.0, 0.0, 100.0, 800.0], + [0.0, 0.0, 0.25, 0.5], + ] { + let preview = capture.fit_preview(size).unwrap(); + let point = preview.image_to_preview(100.5, 50.5).unwrap(); + let recovered = preview.preview_to_image(point.0, point.1).unwrap(); + assert!((recovered.0 - 100.5).abs() < 1e-9 && (recovered.1 - 50.5).abs() < 1e-9); + } + assert!(capture.fit_preview([0.0, 0.0, 0.0, 10.0]).is_err()); + assert!(capture + .fit_preview([f64::MAX, 0.0, f64::MAX, 10.0]) + .is_err()); + } +} diff --git a/src/crates/execution/tool-contracts/src/computer_use_permissions.rs b/src/crates/execution/tool-contracts/src/computer_use_permissions.rs new file mode 100644 index 0000000000..3413507eca --- /dev/null +++ b/src/crates/execution/tool-contracts/src/computer_use_permissions.rs @@ -0,0 +1,109 @@ +//! Permission facts and explicit request receipts. Probing never requests consent. +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ComputerPermissionKind { + Accessibility, + #[serde(alias = "screen_capture")] + ScreenRecording, + InputPosting, + InputMonitoring, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ComputerPermissionState { + Granted, + Required, + /// Authorization belongs to the selected native session, not the whole OS. + SessionRequired, + NotRequired, + Unavailable, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComputerPermission { + pub kind: ComputerPermissionKind, + pub state: ComputerPermissionState, + pub requestable: bool, + pub settings_url: Option, + pub detail: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComputerPermissions { + pub platform: String, + pub process_id: u32, + pub executable: Option, + pub permissions: Vec, + /// A status read is never an authorization gesture. + pub prompts_requested: bool, +} + +impl ComputerPermissions { + pub fn usable(&self, kind: ComputerPermissionKind) -> bool { + self.permissions.iter().any(|item| { + item.kind == kind + && matches!( + item.state, + ComputerPermissionState::Granted | ComputerPermissionState::NotRequired + ) + }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComputerPermissionRequest { + pub kind: ComputerPermissionKind, +} + +/// Adapted from computer-use-rust/native/permissions.rs. An invoked OS API is +/// not evidence that a prompt was visible or that permission was granted. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComputerPermissionReceipt { + pub kind: ComputerPermissionKind, + pub already_granted: bool, + pub current_granted: bool, + pub request_api_invoked: bool, + pub prompt_requested: bool, + pub prompt_visibility_observed: bool, + pub api_returned_granted: Option, + pub requires_user_action: bool, + pub restart_may_be_required: bool, + pub permission: ComputerPermission, + pub permission_owner: String, +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn legacy_capture_kind_and_future_fields_remain_compatible() { + let request: ComputerPermissionRequest = serde_json::from_value(serde_json::json!({ + "kind": "screen_capture", "future": true, + })) + .unwrap(); + assert_eq!(request.kind, ComputerPermissionKind::ScreenRecording); + let round: ComputerPermissionRequest = + serde_json::from_value(serde_json::to_value(request).unwrap()).unwrap(); + assert_eq!(round.kind, ComputerPermissionKind::ScreenRecording); + } + #[test] + fn session_consent_is_not_global_permission() { + let status = ComputerPermissions { + platform: "linux".into(), + process_id: 1, + executable: None, + prompts_requested: false, + permissions: vec![ComputerPermission { + kind: ComputerPermissionKind::ScreenRecording, + state: ComputerPermissionState::SessionRequired, + requestable: false, + settings_url: None, + detail: String::new(), + }], + }; + assert!(!status.usable(ComputerPermissionKind::ScreenRecording)); + } +} diff --git a/src/crates/execution/tool-contracts/src/lib.rs b/src/crates/execution/tool-contracts/src/lib.rs index f6724bbb18..6d894a9091 100644 --- a/src/crates/execution/tool-contracts/src/lib.rs +++ b/src/crates/execution/tool-contracts/src/lib.rs @@ -9,6 +9,10 @@ pub mod acp_tool_bridge; pub mod computer_use; #[cfg(feature = "computer-use-contract")] pub mod computer_use_control; +#[cfg(feature = "computer-use-contract")] +pub mod computer_use_geometry; +#[cfg(feature = "computer-use-contract")] +pub mod computer_use_permissions; pub mod deferred_tool; #[cfg(feature = "element-token")] pub mod element_token; diff --git a/src/crates/execution/tool-execution/AGENTS.md b/src/crates/execution/tool-execution/AGENTS.md index 1785ebed34..7d8301af48 100644 --- a/src/crates/execution/tool-execution/AGENTS.md +++ b/src/crates/execution/tool-execution/AGENTS.md @@ -4,7 +4,7 @@ Scope: this guide applies to `src/crates/execution/tool-execution`. `tool-runtime` owns low-level reusable tool execution helpers such as filesystem and search utilities, provider-neutral pipeline planning/retry/token policy, -ExecCommand presentation/control facts, Computer Use loop/retry policies, +ExecCommand presentation/control facts, Computer Use evidence/read-retry policies, prompt-safe tool context facts/custom-data materialization and extension merge, background exec-output capture state, and provider-neutral Web tool result processing. It is not the product tool registry, permission model, or @@ -48,9 +48,12 @@ agent-facing tool surface. output buffers, cursors, lifecycle metadata, assistant response text, and provider-neutral completion shapes; concrete local/remote process managers stay in services or core adapters. -- Computer Use helpers here may own provider-neutral loop detection, screenshot - hash, verification, and retry policy. Host APIs, permissions, captures, OCR, - accessibility, and OS input remain in host adapters. +- Computer Use helpers own bounded action evidence, full-buffer screenshot + fingerprints and read-only retry policy. Repeated action names or unchanged + pixels do not prove a loop; hash changes do not verify task completion. Errors + cannot prove that mutation input was never submitted, so they must not authorize + automatic input replay. Host APIs, permissions, captures, OCR, accessibility, + and OS input remain in host adapters. - Provider-neutral contracts belong in `tool-contracts` (`openbitfun-agent-tools`); product provider grouping belongs in `tool-provider-groups` (`openbitfun-tool-packs`). @@ -63,6 +66,7 @@ agent-facing tool surface. ```bash cargo test -p tool-runtime +cargo test -p tool-runtime --no-default-features --lib computer_use cargo test -p tool-runtime --features shell-analysis --lib shell_analysis cargo test -p tool-runtime --lib fs::read_file::tests:: cargo test -p tool-runtime --no-default-features --lib search:: diff --git a/src/crates/execution/tool-execution/src/computer_use.rs b/src/crates/execution/tool-execution/src/computer_use.rs index 5b0fcbf2bf..bda7693758 100644 --- a/src/crates/execution/tool-execution/src/computer_use.rs +++ b/src/crates/execution/tool-execution/src/computer_use.rs @@ -1,4 +1,5 @@ -//! Computer Use optimization: action verification, loop detection, and retry logic. +//! Bounded Computer Use evidence and conservative read-retry policy. +//! Input submission, observed pixel changes and task completion are different facts. use serde::{Deserialize, Serialize}; use std::collections::VecDeque; @@ -7,12 +8,6 @@ use std::time::{SystemTime, UNIX_EPOCH}; /// Maximum actions to track in history const MAX_HISTORY_SIZE: usize = 50; -/// Loop detection window (check last N actions) -const LOOP_DETECTION_WINDOW: usize = 10; - -/// Maximum identical action sequences before triggering loop detection -const MAX_LOOP_REPETITIONS: usize = 3; - /// Action record for history tracking #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ActionRecord { @@ -73,76 +68,11 @@ impl ComputerUseOptimizer { self.last_screenshot_hash = Some(hash); } - /// Detect if agent is stuck in a loop + /// Compatibility projection for existing host APIs. Repeated action names, + /// byte-identical frames and mouse usage alone cannot prove a task loop. + /// Keep factual history for the caller's next decision without inventing + /// retries, focus changes or keyboard input. pub fn detect_loop(&self) -> LoopDetectionResult { - if self.action_history.len() < LOOP_DETECTION_WINDOW { - return LoopDetectionResult { - is_loop: false, - pattern_length: 0, - repetitions: 0, - suggestion: String::new(), - }; - } - - // Check for repeating action patterns - for pattern_len in 2..=5 { - if let Some(result) = self.check_pattern_repetition(pattern_len) { - if result.repetitions >= MAX_LOOP_REPETITIONS { - return result; - } - } - } - - // Check for screenshot stagnation (same view, different actions) - if self.check_screenshot_stagnation() { - return LoopDetectionResult { - is_loop: true, - pattern_length: 0, - repetitions: 0, - suggestion: "Screen state unchanged after multiple actions. Try: 1) Use `key_chord` (Enter, Escape, Tab) instead of mouse, 2) Use `click_element` or `move_to_text` for precise targeting instead of screenshot drill, 3) Verify app is focused.".to_string(), - }; - } - - // Check for excessive mouse usage without keyboard - if self.check_excessive_mouse_usage() { - return LoopDetectionResult { - is_loop: true, - pattern_length: 0, - repetitions: 0, - suggestion: "Detected heavy mouse usage without keyboard. Consider: 1) Use `key_chord` with Enter/Escape/Tab/Space instead of clicking buttons, 2) Use `move_to_text` (OCR) instead of screenshot-based targeting, 3) Use `click_element` (accessibility tree) when possible.".to_string(), - }; - } - - // Check for screenshot → mouse_move → click pattern without using precise coordinates - if self.check_screenshot_mouse_pattern() { - return LoopDetectionResult { - is_loop: true, - pattern_length: 0, - repetitions: 0, - suggestion: "Detected screenshot + mouse move pattern. Use `move_to_text` for visible text or `click_element` for accessibility elements instead of estimating from JPEG. Use `global_center_x/y` from prior tool results with `use_screen_coordinates: true`.".to_string(), - }; - } - - // Check for repeated move_to_text failures without trying keyboard navigation - if self.check_repeated_move_to_text_failures() { - return LoopDetectionResult { - is_loop: true, - pattern_length: 0, - repetitions: 0, - suggestion: "Detected repeated move_to_text failures. Try: 1) Use `key_chord` with Tab/Shift+Tab to navigate focus instead of OCR, 2) Try a shorter substring in `move_to_text`, 3) Verify you're targeting the correct window/app.".to_string(), - }; - } - - // Check for screenshot → mouse_move loop without any clicks or progress - if self.check_screenshot_mouse_loop() { - return LoopDetectionResult { - is_loop: true, - pattern_length: 0, - repetitions: 0, - suggestion: "Detected screenshot → mouse_move loop without progress. Stop guessing coordinates! Try: 1) Use `key_chord` with Tab to navigate focus, 2) Use `move_to_text` with a visible text target, 3) Verify the correct app is focused.".to_string(), - }; - } - LoopDetectionResult { is_loop: false, pattern_length: 0, @@ -151,170 +81,6 @@ impl ComputerUseOptimizer { } } - fn check_pattern_repetition(&self, pattern_len: usize) -> Option { - let recent: Vec<_> = self - .action_history - .iter() - .rev() - .take(LOOP_DETECTION_WINDOW) - .collect(); - if recent.len() < pattern_len * MAX_LOOP_REPETITIONS { - return None; - } - - let pattern: Vec<_> = recent - .iter() - .take(pattern_len) - .map(|r| &r.action_type) - .collect(); - let mut reps = 1; - - for chunk in recent.chunks(pattern_len).skip(1) { - if chunk.len() != pattern_len { - break; - } - let chunk_types: Vec<_> = chunk.iter().map(|r| &r.action_type).collect(); - if chunk_types == pattern { - reps += 1; - } else { - break; - } - } - - if reps >= MAX_LOOP_REPETITIONS { - Some(LoopDetectionResult { - is_loop: true, - pattern_length: pattern_len, - repetitions: reps, - suggestion: format!( - "Detected repeating pattern of {} actions (repeated {} times). Try: 1) Use `key_chord` (Enter/Escape/Tab/Space) instead of mouse clicks, 2) Use `click_element` (accessibility tree) or `move_to_text` (OCR) instead of vision-based targeting, 3) Take a fresh screenshot to verify current state.", - pattern_len, reps - ), - }) - } else { - None - } - } - - fn check_screenshot_stagnation(&self) -> bool { - let recent: Vec<_> = self.action_history.iter().rev().take(6).collect(); - if recent.len() < 6 { - return false; - } - - // Check if last 6 actions had same screenshot hash (no visual change) - if let Some(first_hash) = recent[0].screenshot_hash { - recent - .iter() - .skip(1) - .all(|r| r.screenshot_hash == Some(first_hash)) - } else { - false - } - } - - /// Detect excessive mouse usage without any keyboard actions - fn check_excessive_mouse_usage(&self) -> bool { - let recent: Vec<_> = self.action_history.iter().rev().take(10).collect(); - if recent.len() < 10 { - return false; - } - - let mouse_actions = ["click", "mouse_move", "scroll", "drag", "pointer_move_rel"]; - let has_keyboard = recent - .iter() - .any(|r| r.action_type == "key_chord" || r.action_type == "type_text"); - - let mouse_count = recent - .iter() - .filter(|r| mouse_actions.contains(&r.action_type.as_str())) - .count(); - - // If 8+ of last 10 actions are mouse and no keyboard usage - !has_keyboard && mouse_count >= 8 - } - - /// Detect screenshot → mouse_move → click pattern without precise coordinates - fn check_screenshot_mouse_pattern(&self) -> bool { - let recent: Vec<_> = self.action_history.iter().rev().take(12).collect(); - if recent.len() < 9 { - return false; - } - - let mut screenshot_count = 0; - let mut mouse_move_count = 0; - let mut has_move_to_text = false; - let mut has_click_element = false; - - for action in &recent { - match action.action_type.as_str() { - "screenshot" => screenshot_count += 1, - "mouse_move" => mouse_move_count += 1, - "move_to_text" => has_move_to_text = true, - "click_element" => has_click_element = true, - _ => {} - } - } - - // If we have many screenshots + mouse moves but no move_to_text/click_element - screenshot_count >= 3 && mouse_move_count >= 2 && !has_move_to_text && !has_click_element - } - - /// Detect repeated move_to_text failures without trying keyboard navigation - fn check_repeated_move_to_text_failures(&self) -> bool { - let recent: Vec<_> = self.action_history.iter().rev().take(8).collect(); - if recent.len() < 5 { - return false; - } - - let mut move_to_text_failures = 0; - let mut has_keyboard = false; - - for action in &recent { - if action.action_type == "move_to_text" && !action.success { - move_to_text_failures += 1; - } - if action.action_type == "key_chord" { - has_keyboard = true; - } - } - - // 3+ move_to_text failures and no keyboard attempts - move_to_text_failures >= 3 && !has_keyboard - } - - /// Detect screenshot → mouse_move loop without any clicks or progress - fn check_screenshot_mouse_loop(&self) -> bool { - let recent: Vec<_> = self.action_history.iter().rev().take(10).collect(); - if recent.len() < 6 { - return false; - } - - let mut screenshot_count = 0; - let mut mouse_move_count = 0; - let mut has_click = false; - let mut has_keyboard = false; - let mut has_move_to_text = false; - - for action in &recent { - match action.action_type.as_str() { - "screenshot" => screenshot_count += 1, - "mouse_move" => mouse_move_count += 1, - "click" => has_click = true, - "key_chord" | "type_text" => has_keyboard = true, - "move_to_text" => has_move_to_text = true, - _ => {} - } - } - - // Many screenshots + mouse moves, but no clicks/keyboard/move_to_text - screenshot_count >= 3 - && mouse_move_count >= 2 - && !has_click - && !has_keyboard - && !has_move_to_text - } - /// Get action history for backtracking pub fn get_history(&self) -> Vec { self.action_history.iter().cloned().collect() @@ -333,10 +99,10 @@ impl Default for ComputerUseOptimizer { } } -/// Simple hash function for screenshot comparison +/// Full-buffer fingerprint of encoded bytes, not a perceptual or task-success metric. pub fn hash_screenshot_bytes(bytes: &[u8]) -> u64 { let mut hash: u64 = 0xcbf29ce484222325; - for &byte in bytes.iter().step_by(1000) { + for &byte in bytes.iter() { hash ^= byte as u64; hash = hash.wrapping_mul(0x100000001b3); } @@ -346,8 +112,10 @@ pub fn hash_screenshot_bytes(bytes: &[u8]) -> u64 { /// Verification result after an action #[derive(Debug, Clone, Serialize, Deserialize)] pub struct VerificationResult { + /// Hash comparison alone never verifies the requested outcome. pub verified: bool, pub visual_change_detected: bool, + /// Legacy compatibility field. Always zero: hashes cannot measure an area. pub change_percentage: f32, pub suggestion: Option, } @@ -366,13 +134,13 @@ impl RetryStrategy { Self { max_attempts, current_attempt: 0, - should_retry: true, + should_retry: max_attempts > 0, retry_delay_ms: 500, } } pub fn next_attempt(&mut self) -> bool { - self.current_attempt += 1; + self.current_attempt = self.current_attempt.saturating_add(1); self.should_retry = self.current_attempt < self.max_attempts; self.should_retry } @@ -382,133 +150,134 @@ impl RetryStrategy { } } -/// Compare two screenshot hashes to detect visual changes +/// Compare encoded frame fingerprints. A clock, caret or animation can change +/// pixels after failed input; successful input can also leave pixels unchanged. pub fn detect_visual_change(hash_before: u64, hash_after: u64) -> VerificationResult { - let changed = hash_before != hash_after; - VerificationResult { - verified: changed, - visual_change_detected: changed, - change_percentage: if changed { 100.0 } else { 0.0 }, - suggestion: if !changed { - Some("No visual change detected. Action may have failed or UI did not update. Consider: 1) Retry the action, 2) Verify element is clickable, 3) Try keyboard shortcut instead.".to_string()) - } else { - None - }, + verified: false, + visual_change_detected: hash_before != hash_after, + change_percentage: 0.0, + suggestion: Some("Inspect the observation against the requested outcome. Pixel changes do not prove completion; an unchanged image does not justify replaying input.".into()), } } -/// Determine if an action should be retried based on provider-neutral error text. +/// Only side-effect-free observations can be retried after a transient failure. +/// Text errors cannot establish that a mutation was never submitted. pub fn should_retry_action_message(error_message: &str, action_type: &str) -> bool { - let error_msg = error_message.to_lowercase(); - - if error_msg.contains("timeout") - || error_msg.contains("not found") - || error_msg.contains("element moved") - || error_msg.contains("stale") - { - return true; + if !matches!( + action_type, + "screenshot" | "get_app_state" | "get_app_shortcuts" | "locate" | "list_apps" | "read_text" + ) { + return false; } - - if error_msg.contains("permission") - || error_msg.contains("not enabled") - || error_msg.contains("not available") + let error = error_message.to_ascii_lowercase(); + if [ + "permission", + "denied", + "unsupported", + "not available", + "control_stale", + "control_stopped", + "cancelled", + ] + .iter() + .any(|term| error.contains(term)) { return false; } - - matches!(action_type, "click" | "click_element" | "locate") + error.contains("timeout") || error.contains("temporarily unavailable") } -/// Generate retry suggestion based on failure context -pub fn generate_retry_suggestion(action_type: &str, attempt: u32) -> String { - match action_type { - "click" | "click_element" => { - if attempt == 1 { - "First retry: Taking fresh screenshot to verify element position.".to_string() - } else { - "Retry failed. Try: 1) Use accessibility tree (click_element), 2) Use keyboard shortcut, 3) Verify element is visible and clickable.".to_string() - } - } - "locate" => { - "Element not found. Try: 1) Broaden search criteria (use filter_combine: 'any'), 2) Use only role_substring or title_contains, 3) Verify app is focused.".to_string() - } - _ => format!("Retry attempt {} for action: {}", attempt, action_type), - } +pub fn generate_retry_suggestion(action_type: &str, _attempt: u32) -> String { + format!("{action_type} did not establish the requested outcome. Read the authorized target again and inspect the result before choosing new input. Submitted input must not be automatically replayed.") } #[cfg(test)] mod tests { - use super::{ - detect_visual_change, generate_retry_suggestion, should_retry_action_message, - ComputerUseOptimizer, RetryStrategy, - }; + use super::*; #[test] - fn detects_repeated_action_loop() { - let mut optimizer = ComputerUseOptimizer::new(); - for _ in 0..3 { - optimizer.record_action("screenshot".to_string(), "{}".to_string(), true); - optimizer.record_action("mouse_move".to_string(), "{}".to_string(), true); - } - for _ in 0..4 { - optimizer.record_action("screenshot".to_string(), "{}".to_string(), true); + fn legitimate_repeated_work_is_not_classified_as_a_loop() { + for action in ["scroll", "click", "mouse_move", "screenshot", "type_text"] { + let mut evidence = ComputerUseOptimizer::new(); + evidence.update_screenshot_hash(7); + for index in 0..100 { + evidence.record_action(action.into(), format!("target:{index}"), true); + } + assert!(!evidence.detect_loop().is_loop); + assert_eq!(evidence.get_history().len(), MAX_HISTORY_SIZE); + evidence.clear_history(); + assert!(evidence.get_history().is_empty()); } - - let result = optimizer.detect_loop(); - - assert!(result.is_loop); - assert!(!result.suggestion.is_empty()); } #[test] - fn hashes_screenshots_deterministically() { - let bytes = b"abcdef0123456789"; + fn same_and_changed_frames_never_claim_completion_or_area() { + for (before, after) in [(1, 1), (1, 2)] { + let evidence = detect_visual_change(before, after); + assert!(!evidence.verified); + assert_eq!(evidence.visual_change_detected, before != after); + assert_eq!(evidence.change_percentage, 0.0); + } + } - assert_eq!( - super::hash_screenshot_bytes(bytes), - super::hash_screenshot_bytes(bytes) - ); + #[test] + fn fingerprint_covers_pixels_between_previous_sparse_samples() { + let before = vec![0; 4001]; + let mut after = before.clone(); + after[3999] = 1; assert_ne!( - super::hash_screenshot_bytes(bytes), - super::hash_screenshot_bytes(b"z") + hash_screenshot_bytes(&before), + hash_screenshot_bytes(&after) + ); + assert_eq!( + hash_screenshot_bytes(&before), + hash_screenshot_bytes(&before) ); } #[test] - fn verification_reports_visual_change_without_host_dependencies() { - let changed = detect_visual_change(1, 2); - let unchanged = detect_visual_change(1, 1); - - assert!(changed.verified); - assert_eq!(changed.change_percentage, 100.0); - assert!(!unchanged.verified); - assert!(unchanged.suggestion.is_some()); + fn timeout_or_stale_errors_never_authorize_input_replay() { + for action in [ + "click", + "app_click", + "click_element", + "drag", + "type_text", + "app_batch", + "key_chord", + "open_app", + "unknown", + ] { + for error in [ + "timeout", + "element not found", + "stale", + "temporarily unavailable", + ] { + assert!(!should_retry_action_message(error, action)); + } + } + assert!(should_retry_action_message("timeout", "get_app_state")); + assert!(!should_retry_action_message( + "permission denied timeout", + "get_app_state" + )); + assert!(!should_retry_action_message( + "CONTROL_STALE timeout", + "screenshot" + )); } #[test] - fn retry_strategy_preserves_attempt_semantics() { + fn read_retry_budget_handles_zero_and_overflow() { + let mut zero = RetryStrategy::new(0); + assert!(!zero.should_retry); + assert!(!zero.next_attempt()); let mut retry = RetryStrategy::new(2); - assert!(retry.next_attempt()); assert!(!retry.next_attempt()); - assert!(retry.is_exhausted()); - } - - #[test] - fn retry_decision_uses_error_text_without_core_error_type() { - assert!(should_retry_action_message("operation timeout", "drag")); - assert!(!should_retry_action_message("permission denied", "click")); - assert!(should_retry_action_message( - "other failure", - "click_element" - )); - assert!(!should_retry_action_message("other failure", "type_text")); - } - - #[test] - fn retry_suggestion_matches_existing_action_copy() { - assert!(generate_retry_suggestion("click", 1).contains("fresh screenshot")); - assert!(generate_retry_suggestion("locate", 2).contains("Broaden search criteria")); + retry.current_attempt = u32::MAX; + assert!(!retry.next_attempt()); } } diff --git a/src/shared/interactive-capabilities/catalog.json b/src/shared/interactive-capabilities/catalog.json index 0006c7e673..6221678955 100644 --- a/src/shared/interactive-capabilities/catalog.json +++ b/src/shared/interactive-capabilities/catalog.json @@ -2623,13 +2623,15 @@ "启用或停用 Computer Use", "连接或断开受支持的浏览器", "检查并打开系统权限设置", - "查看当前控制目标与预览，并随时停止控制" + "查看当前控制目标与预览，并随时停止控制", + "先选择共享 App，再授权输入；可记住授权或随时撤销" ], "highlightsEn": [ "Enable or disable Computer Use", "Connect or disconnect a supported browser", "Inspect and open OS permission settings", - "Inspect the current control target and preview, and stop control at any time" + "Inspect the current control target and preview, and stop control at any time", + "Select an App share before granting input; remember or revoke the grant" ], "items": [ { @@ -2657,7 +2659,8 @@ "reasonEn": "“Request the operating-system permissions required for desktop control” spans multiple live-state-dependent steps and currently has no single structured Command that can deterministically complete the whole workflow; the Agent opens the exact entry and keeps the remaining interaction visible to the user." }, "evidence": [ - "command:computer_use_request_permissions" + "command:computer_use_request_permissions", + "command:computer_use_request_permission" ] }, { @@ -2779,6 +2782,53 @@ "command:computer_use_control_stop", "source:src/web-ui/src/app/components/ComputerUseControlCard.tsx#ComputerUseControlCard" ] + }, + { + "id": "app-sharing", + "titleZh": "选择 App 窗口并单独授权或撤销控制", + "titleEn": "Select an App window and separately grant or revoke control", + "control": { + "kind": "open", + "reasonCode": "visualSelection", + "reasonZh": "共享窗口及控制模式由用户明确选择；智能体只能请求共享，不能代替用户授权。", + "reasonEn": "Window selection and input mode require an explicit user choice; agents can request sharing but cannot grant themselves control." + }, + "evidence": [ + "command:computer_use_share_request", + "command:computer_use_share_targets", + "command:computer_use_share_start", + "command:computer_use_share_control" + ] + }, + { + "id": "preview-input", + "titleZh": "在共享预览中点击、拖动、滚动及输入", + "titleEn": "Click, drag, scroll and type through the shared preview", + "control": { + "kind": "open", + "reasonCode": "visualSelection", + "reasonZh": "预览输入需要用户授权，并绑定共享窗口、图像坐标及授权版本。", + "reasonEn": "Preview input requires user authorization and is bound to the shared window, image coordinates and authorization revision." + }, + "evidence": [ + "command:computer_use_share_observe", + "command:computer_use_share_input" + ] + }, + { + "id": "remembered-control", + "titleZh": "按应用执行文件和控制模式记住或忘记授权", + "titleEn": "Remember or forget control by application executable and input mode", + "control": { + "kind": "open", + "reasonCode": "visualSelection", + "reasonZh": "记住授权只在用户选定并共享同一应用后生效，应用文件或输入模式改变时需要重新授权。", + "reasonEn": "Remembered control applies only after the user shares the same application; changed executable contents or input modes require a new grant." + }, + "evidence": [ + "command:computer_use_share_grants", + "command:computer_use_share_forget_grant" + ] } ], "stepsZh": [ @@ -5899,7 +5949,11 @@ "control": { "kind": "direct", "operations": [], - "options": [{ "id": "auto-show-selection-toolbar" }] + "options": [ + { + "id": "auto-show-selection-toolbar" + } + ] }, "evidence": [ "source:src/web-ui/src/app/scenes/settings/pages/application/TextSelectionSettingsSection.tsx#AUTO_SHOW_SELECTION_TOOLBAR_CONFIG_PATH", diff --git a/src/web-ui/src/app/components/ComputerUseControlCard.appearance.ts b/src/web-ui/src/app/components/ComputerUseControlCard.appearance.ts index 954fa7f471..0963739a0a 100644 --- a/src/web-ui/src/app/components/ComputerUseControlCard.appearance.ts +++ b/src/web-ui/src/app/components/ComputerUseControlCard.appearance.ts @@ -2,5 +2,5 @@ import type { AppearanceSurfaceDescriptor } from '@/infrastructure/appearance'; export const computerUseControlAppearanceDescriptor: AppearanceSurfaceDescriptor = { id: 'computer-control', - parts: [{ id: 'root' }, { id: 'status' }, { id: 'preview' }, { id: 'pointer' }, { id: 'click' }], + parts: [{ id: 'root' }, { id: 'status' }, { id: 'preview' }, { id: 'pointer' }, { id: 'click' }, { id: 'sharing' }, { id: 'permissions' }], }; diff --git a/src/web-ui/src/app/components/ComputerUseControlCard.scss b/src/web-ui/src/app/components/ComputerUseControlCard.scss index 2e3a34cb5b..1b463a4d4e 100644 --- a/src/web-ui/src/app/components/ComputerUseControlCard.scss +++ b/src/web-ui/src/app/components/ComputerUseControlCard.scss @@ -1,13 +1,31 @@ .computer-control-card { inline-size: 368px; max-inline-size: calc(100vw - var(--openbitfun-space-6)); + max-block-size: calc(100dvh - var(--openbitfun-space-6)); + overflow: hidden; + + &__panel { max-block-size: inherit; } + &__body { min-block-size: 0; overflow-y: auto; } &__status { margin: 0; } + &__sharing, &__permissions, &__permission { + display: flex; + flex-direction: column; + gap: var(--openbitfun-space-2); + } + &__permissions { gap: var(--openbitfun-space-4); } + &__row { display: flex; align-items: center; flex-wrap: wrap; gap: var(--openbitfun-space-2); } + &__sharing p, &__permissions p { margin: 0; } + &__permission { padding-block: var(--openbitfun-space-2); } + &__tree { max-block-size: 240px; overflow: auto; white-space: pre-wrap; } + + &__preview { position: relative; overflow: hidden; border-radius: var(--openbitfun-layout-card-radius-sm); + touch-action: none; img { display: block; inline-size: 100%; block-size: auto; } } @@ -40,8 +58,25 @@ border-radius: 50%; transform: translate(-50%, -50%); } + &__pointer--physical { + inline-size: 24px; + block-size: 24px; + transform-origin: 4.8px 4.8px; + transition: none; + } + &__click--shared { + inline-size: 24px; + block-size: 24px; + animation: computer-control-click 550ms linear forwards; + } +} + +@keyframes computer-control-click { + from { opacity: 1; transform: translate(-50%, -50%) scale(1); } + to { opacity: 0; transform: translate(-50%, -50%) scale(3.5); } } @media (prefers-reduced-motion: reduce) { .computer-control-card__pointer { transition: none; } + .computer-control-card__click--shared { animation: none; } } diff --git a/src/web-ui/src/app/components/ComputerUseControlCard.test.tsx b/src/web-ui/src/app/components/ComputerUseControlCard.test.tsx index 7ac24df4e4..2ccd776179 100644 --- a/src/web-ui/src/app/components/ComputerUseControlCard.test.tsx +++ b/src/web-ui/src/app/components/ComputerUseControlCard.test.tsx @@ -148,3 +148,15 @@ it('removes the old target pointer while waiting for the new target preview', as expect(container.querySelector('[data-openbitfun-part="pointer"]')).toBeNull(); expect(container.querySelector('img')).toBeNull(); }); + +it('discards a preview from an earlier visit to the same target', async () => { + mocks.status.mockResolvedValue({ ...active('Window A'), target_revision: 3 }); + mocks.preview.mockResolvedValue({ generation: 4, target: 'Window A', target_revision: 1, + image_base64: 'AA==', mime_type: 'image/png', width: 100, height: 100, + origin_x: 0, origin_y: 0, span_width: 100, span_height: 100 }); + await act(async () => root.render()); + await act(async () => [...container.querySelectorAll('button')].find(b => b.textContent === 'computerControl.showPreview')!.click()); + await act(async () => vi.advanceTimersByTime(1000)); + expect(container.querySelector('img')).toBeNull(); + expect(container.textContent).toContain('computerControl.previewUnavailable'); +}); diff --git a/src/web-ui/src/app/components/ComputerUseControlCard.tsx b/src/web-ui/src/app/components/ComputerUseControlCard.tsx index 4d5a693b2a..3b5dd665c5 100644 --- a/src/web-ui/src/app/components/ComputerUseControlCard.tsx +++ b/src/web-ui/src/app/components/ComputerUseControlCard.tsx @@ -7,6 +7,8 @@ import { type ComputerUseControlSnapshot, type ComputerUseControlPreview, } from '@/infrastructure/api/service-api/ComputerUseControlAPI'; import './ComputerUseControlCard.scss'; +import { ComputerUseSharingControls } from './ComputerUseSharingControls'; +import { SharedPreview } from './computer-use/SharedPreview'; interface View { epoch: number; @@ -16,7 +18,13 @@ interface View { error?: boolean; } -/** One passive card in the shared notification region; never intercepts input. */ +function acceptsUpdate(current: ComputerUseControlSnapshot | undefined, next: ComputerUseControlSnapshot) { + if (!current || current.generation !== next.generation) return !current || next.generation > current.generation; + if ((current.target_revision ?? 0) !== (next.target_revision ?? 0)) return (next.target_revision ?? 0) > (current.target_revision ?? 0); + return (next.sharing?.authorization_revision ?? 0) >= (current.sharing?.authorization_revision ?? 0); +} + +/** One card for the executing host in the shared notification region. */ export function ComputerUseControlCard() { const { t } = useI18n('common'); const scope = useSyncExternalStore(onSurfaceActivated, getActiveSurfaceScope, getActiveSurfaceScope); @@ -30,7 +38,7 @@ export function ComputerUseControlCard() { const [dismissed, setDismissed] = useState(); const current = view?.epoch === scope.epoch ? view : undefined; const snapshot = current?.snapshot; - const active = snapshot?.state === 'active' || snapshot?.state === 'starting'; + const active = snapshot?.state === 'active' || snapshot?.state === 'starting' || snapshot?.state === 'awaiting_share'; useEffect(() => { let disposed = false; @@ -45,13 +53,15 @@ export function ComputerUseControlCard() { if (disposed || !scope.isCurrent() || revision !== mutation.current) return; setView(previous => ({ epoch: scope.epoch, snapshot: next, ...(next.state === 'active' && previous?.epoch === scope.epoch && previous.snapshot.generation === next.generation - && previous.snapshot.target === next.target ? { frame: previous.frame, framePointer: previous.framePointer } : {}), + && previous.snapshot.target === next.target + && (previous.snapshot.target_revision ?? 0) === (next.target_revision ?? 0) ? { frame: previous.frame, framePointer: previous.framePointer } : {}), })); - if (expandedRef.current && next.supported && next.state === 'active') { + if (expandedRef.current && next.supported && next.state === 'active' && !next.sharing) { try { const frame = await computerUseControlAPI.preview(next.generation); if (disposed || !scope.isCurrent() || revision !== mutation.current) return; - if (frame.generation !== next.generation || frame.target !== next.target) { + if (!frame || frame.generation !== next.generation || frame.target !== next.target + || (frame.target_revision ?? 0) !== (next.target_revision ?? 0)) { throw new Error('Control preview target changed'); } if (!/^image\/(png|jpeg|webp)$/.test(frame.mime_type) || frame.width <= 0 || frame.height <= 0) { @@ -59,6 +69,7 @@ export function ComputerUseControlCard() { } setView(previous => previous?.snapshot.generation === next.generation && previous.snapshot.target === next.target + && (previous.snapshot.target_revision ?? 0) === (next.target_revision ?? 0) ? { ...previous, frame, framePointer: next.pointer } : previous); } catch { if (!disposed && scope.isCurrent() && revision === mutation.current) setView(previous => previous && { @@ -98,7 +109,7 @@ export function ComputerUseControlCard() { }; const lastClick = current?.framePointer?.last_click; - const clickIdentity = `${scope.epoch}:${snapshot?.generation}:${snapshot?.target}:${lastClick?.sequence}`; + const clickIdentity = `${scope.epoch}:${snapshot?.generation}:${snapshot?.target}:${snapshot?.target_revision ?? 0}:${lastClick?.sequence}`; useEffect(() => { setClickFeedback(null); if (!expanded || !current?.frame || !lastClick) return; @@ -120,15 +131,30 @@ export function ComputerUseControlCard() { return
- {t('computerControl.title')} - +

- {current?.error ? t('computerControl.unavailable') : active ? mode : t('computerControl.stopped')} + {current?.error ? t('computerControl.unavailable') : snapshot.state === 'awaiting_share' ? t('computerSharing.selectApp') : active ? mode : t('computerControl.stopped')}

- {snapshot.target && {snapshot.target}} - {expanded && current?.frame &&
+ {snapshot.target && {snapshot.sharing?.target_label || snapshot.target}} + {active && snapshot.sharing && { mutation.current += 1; }} + onSnapshot={next => { + if (!scope.isCurrent()) return; + mutation.current += 1; + setView(previous => !acceptsUpdate(previous?.snapshot, next) ? previous : { epoch: scope.epoch, snapshot: next }); + if (next.sharing?.phase === 'shared') setExpanded(true); + }} />} + {snapshot.reason && !active &&

{snapshot.reason}

} + {expanded && snapshot.sharing?.phase === 'shared' && { + if (!scope.isCurrent()) return; + mutation.current += 1; + setView(previous => !acceptsUpdate(previous?.snapshot, next) ? previous : { epoch: scope.epoch, snapshot: next }); + }} />} + {expanded && !snapshot.sharing && current?.frame &&
{t('computerControl.previewAlt')} {pointer &&
} - {expanded && !current?.frame &&

{t('computerControl.previewUnavailable')}

} + {expanded && !snapshot.sharing && !current?.frame &&

{t('computerControl.previewUnavailable')}

} - {active && } diff --git a/src/web-ui/src/app/components/ComputerUsePermissions.tsx b/src/web-ui/src/app/components/ComputerUsePermissions.tsx new file mode 100644 index 0000000000..f70e5de244 --- /dev/null +++ b/src/web-ui/src/app/components/ComputerUsePermissions.tsx @@ -0,0 +1,90 @@ +import { useEffect, useState, useSyncExternalStore } from 'react'; +import { Button, OverflowText, StatusPill } from '@openbitfun/ui'; +import { useI18n } from '@/infrastructure/i18n'; +import { getActiveSurfaceScope, onSurfaceActivated } from '@/infrastructure/peer-device/deviceSurface'; +import { computerUseControlAPI, type ComputerUseGrant, type ComputerUsePermissionKind, type ComputerUsePermissionReceipt, + type ComputerUsePermissions as PermissionStatus } from '@/infrastructure/api/service-api/ComputerUseControlAPI'; + +/** The query path only probes. Each request button is a distinct user gesture. */ +export function ComputerUsePermissions({ showGrants = false }: { showGrants?: boolean }) { + const { t } = useI18n('common'); + const scope = useSyncExternalStore(onSurfaceActivated, getActiveSurfaceScope, getActiveSurfaceScope); + const [status, setStatus] = useState<{ epoch: number; value: PermissionStatus }>(); + const [receipt, setReceipt] = useState(); + const [error, setError] = useState(); + const [busy, setBusy] = useState(false); + const [grants, setGrants] = useState(); + const current = status?.epoch === scope.epoch ? status.value : undefined; + const refresh = async () => { + const response = await computerUseControlAPI.permissions(); + if (!scope.isCurrent()) return; + if (!response.permissions) throw new Error(t('computerSharing.updateHost')); + setStatus({ epoch: scope.epoch, value: response.permissions }); + }; + useEffect(() => { + let disposed = false; + setStatus(undefined); setReceipt(undefined); setGrants(undefined); setError(undefined); setBusy(false); + void computerUseControlAPI.permissions().then(response => { + if (disposed || !scope.isCurrent()) return; + if (response.permissions) setStatus({ epoch: scope.epoch, value: response.permissions }); + else setError(t('computerSharing.updateHost')); + }).catch(error => { if (!disposed && scope.isCurrent()) setError(String(error)); }); + return () => { disposed = true; }; + }, [scope, t]); + const perform = async (work: () => Promise) => { + if (!scope.isCurrent()) return; + setBusy(true); setError(undefined); + try { await work(); } catch (error) { if (scope.isCurrent()) setError(String(error)); } + finally { if (scope.isCurrent()) setBusy(false); } + }; + const request = (kind: ComputerUsePermissionKind) => perform(async () => { + const next = await computerUseControlAPI.requestPermission(kind); + if (!scope.isCurrent()) return; + setReceipt(next); + await refresh(); + }); + return
+
+ {t('computerSharing.permissions')} + +
+ {current?.permissions.map(permission =>
+
+ {t(`computerSharing.permissionNames.${permission.kind}`)} + + {t(`computerSharing.permissionStates.${permission.state}`)} + +
+

{t(`computerSharing.permissionHelp.${permission.kind}`)}

+
+ {permission.requestable && permission.state !== 'granted' && } + {permission.settings_url && } +
+
)} + {current?.executable &&
{t('computerSharing.permissionOwner')}{current.executable}
} + {receipt &&

{t(receipt.current_granted ? 'computerSharing.permissionGranted' + : receipt.request_api_invoked ? 'computerSharing.permissionRequested' : 'computerSharing.permissionSession')} + {receipt.restart_may_be_required && <> {t('computerSharing.restartPermission')}}

} + {showGrants &&
{ + if (event.currentTarget.open && !grants) void perform(async () => { + const next = await computerUseControlAPI.grants(); if (scope.isCurrent()) setGrants(next); + }); + }}>{t('computerSharing.rememberedGrants')} + {grants?.length === 0 &&

{t('computerSharing.noGrants')}

} + {grants?.map(grant =>
+ {grant.application_name} + {grant.identity.executable} + {t(grant.identity.input_mode === 'foreground' ? 'computerSharing.modeForeground' : 'computerSharing.modeBackground')} + +
)} +
} + {error &&

{error}

} +
; +} diff --git a/src/web-ui/src/app/components/ComputerUseSharingControls.tsx b/src/web-ui/src/app/components/ComputerUseSharingControls.tsx new file mode 100644 index 0000000000..c4fd77d226 --- /dev/null +++ b/src/web-ui/src/app/components/ComputerUseSharingControls.tsx @@ -0,0 +1,84 @@ +import { useEffect, useState, useSyncExternalStore } from 'react'; +import { Button, Checkbox, Select } from '@openbitfun/ui'; +import { useI18n } from '@/infrastructure/i18n'; +import { getActiveSurfaceScope, onSurfaceActivated } from '@/infrastructure/peer-device/deviceSurface'; +import { computerUseControlAPI, type ComputerUseControlSnapshot, type ComputerUseShareTarget } from '@/infrastructure/api/service-api/ComputerUseControlAPI'; +import { ComputerUsePermissions } from './ComputerUsePermissions'; + +export function ComputerUseSharingControls({ snapshot, onSnapshot, onMutation }: { + snapshot: ComputerUseControlSnapshot; + onSnapshot: (snapshot: ComputerUseControlSnapshot) => void; + onMutation: () => void; +}) { + const { t } = useI18n('common'); + const scope = useSyncExternalStore(onSurfaceActivated, getActiveSurfaceScope, getActiveSurfaceScope); + const [targets, setTargets] = useState([]); + const [selected, setSelected] = useState(''); + const [mode, setMode] = useState(snapshot.sharing?.requested_mode ?? 'background'); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(); + const [remember, setRemember] = useState(false); + const selecting = snapshot.sharing?.phase === 'selecting'; + const refreshTargets = async () => { + const next = await computerUseControlAPI.targets(); + if (!scope.isCurrent()) return; + setTargets(next); setSelected(''); + }; + useEffect(() => { + let disposed = false; + setTargets([]); setSelected(''); setError(undefined); setBusy(false); setRemember(false); + setMode(snapshot.sharing?.requested_mode ?? 'background'); + if (selecting) void computerUseControlAPI.targets().then(next => { + if (!disposed && scope.isCurrent()) setTargets(next); + }).catch(error => { if (!disposed && scope.isCurrent()) setError(String(error)); }); + return () => { disposed = true; }; + // The target selection belongs to one resource generation on one device. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [scope, snapshot.generation, selecting]); + const perform = async (work: () => Promise) => { + if (!scope.isCurrent()) return; + setBusy(true); setError(undefined); onMutation(); + try { await work(); } catch (error) { if (scope.isCurrent()) setError(String(error)); } + finally { if (scope.isCurrent()) setBusy(false); } + }; + const target = targets.find(target => target.selection_id === selected); + return
+ {selecting && <> +

{t('computerSharing.selectDescription')}

+ setMode(value as ComputerUseControlSnapshot['mode'])} + options={[{ value: 'background', label: t('computerSharing.modeBackground') }, + { value: 'observe', label: t('computerSharing.modeObserve') }, + { value: 'foreground', label: t('computerSharing.modeForeground') }]} /> + {mode === 'foreground' &&

{t('computerSharing.foregroundNotice')}

} + {target && !target.background_pointer &&

{t(target.background_semantic ? 'computerSharing.semanticOnly' : 'computerSharing.portalOnly')}

} +
+ + +
+ } + {snapshot.sharing?.phase === 'shared' && snapshot.sharing.requested_mode !== 'observe' && <> +

{t(snapshot.sharing.control_enabled ? 'computerSharing.controlGranted' : 'computerSharing.observeFirst')}

+ {!snapshot.sharing.control_enabled && !snapshot.target?.startsWith('portal:') && setRemember(event.target.checked)} disabled={busy} + label={t('computerSharing.remember')} />} + + {snapshot.sharing.remembered &&

{t('computerSharing.remembered')}

} + {snapshot.sharing.grant_note &&

{snapshot.sharing.grant_note}

} + } +
{t('computerSharing.permissions')}
+ {error &&

{error}

} +
; +} diff --git a/src/web-ui/src/app/components/computer-use/PreviewCursor.tsx b/src/web-ui/src/app/components/computer-use/PreviewCursor.tsx new file mode 100644 index 0000000000..45307365b1 --- /dev/null +++ b/src/web-ui/src/app/components/computer-use/PreviewCursor.tsx @@ -0,0 +1,49 @@ +import { useEffect, useRef } from 'react'; +import { VirtualCursor } from './VirtualCursor'; + +/** Feedback has no transport or input callback. Its hot spot is independent of native coordinates. */ +export function PreviewCursor({ x, y, immediate = false }: { x: number; y: number; immediate?: boolean }) { + const element = useRef(null); + const state = useRef<{ cursor: VirtualCursor; sequence: number; frame: number; last: number; reduced: boolean; width: number; height: number }>(); + const point = useRef({ x, y, immediate }); point.current = { x, y, immediate }; + const update = useRef<() => void>(); + useEffect(() => { + const span = element.current, parent = span?.parentElement; + if (!span || !parent) return; + const query = window.matchMedia('(prefers-reduced-motion: reduce)'); + const s = { cursor: new VirtualCursor(Math.max(parent.clientWidth, 1), Math.max(parent.clientHeight, 1)), sequence: 0, + frame: 0, last: 0, reduced: query.matches, width: parent.clientWidth, height: parent.clientHeight }; + state.current = s; + const render = () => { + const position = s.cursor.position(), visual = s.cursor.visual(); + span.style.left = `${position.x}px`; span.style.top = `${position.y}px`; + span.style.opacity = String(visual.opacity); + span.style.filter = `blur(${visual.blur}px)`; + span.style.transform = `translate(-4.8px,-4.8px) rotate(${visual.axisRotation}deg) scale(1,${visual.axisScale}) rotate(${-visual.axisRotation}deg) rotate(${visual.rotation + 44}deg) scale(${visual.scaleX},${visual.scaleY})`; + }; + const tick = (now: number) => { + s.frame = 0; + s.cursor.advance(s.last ? (now - s.last) / 1000 : 0); s.last = now; + s.cursor.takeArrival(); render(); + if (!s.reduced && s.cursor.needsAnimation()) s.frame = requestAnimationFrame(tick); + }; + const move = () => { + s.cursor.moveTo({ x: point.current.x * s.width, y: point.current.y * s.height }, ++s.sequence, !s.reduced && !point.current.immediate); + s.cursor.takeArrival(); render(); + if (!s.frame && !s.reduced && s.cursor.needsAnimation()) { s.last = performance.now(); s.frame = requestAnimationFrame(tick); } + }; + update.current = move; + const resize = new ResizeObserver(() => { + s.width = parent.clientWidth; s.height = parent.clientHeight; + if (s.width > 0 && s.height > 0) { s.cursor.resize(s.width, s.height); move(); } + }); + const motionChanged = () => { s.reduced = query.matches; if (s.reduced) { cancelAnimationFrame(s.frame); s.frame = 0; } move(); }; + query.addEventListener('change', motionChanged); resize.observe(parent); move(); + return () => { resize.disconnect(); query.removeEventListener('change', motionChanged); cancelAnimationFrame(s.frame); state.current = undefined; update.current = undefined; }; + }, []); + useEffect(() => { update.current?.(); }, [x, y, immediate]); + return ; +} diff --git a/src/web-ui/src/app/components/computer-use/SharedPreview.test.tsx b/src/web-ui/src/app/components/computer-use/SharedPreview.test.tsx new file mode 100644 index 0000000000..9138eab020 --- /dev/null +++ b/src/web-ui/src/app/components/computer-use/SharedPreview.test.tsx @@ -0,0 +1,114 @@ +// @vitest-environment jsdom +import React, { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; +import { SharedPreview } from './SharedPreview'; +import { ComputerUsePermissions } from '../ComputerUsePermissions'; +import type { ComputerUseControlSnapshot } from '@/infrastructure/api/service-api/ComputerUseControlAPI'; +const mocks = vi.hoisted(() => ({ observeShare: vi.fn(), inputShare: vi.fn(), permissions: vi.fn(), requestPermission: vi.fn(), + scope: { epoch: 1, isCurrent: () => true }, listeners: new Set<() => void>(), t: (key: string) => key })); +vi.mock('@/infrastructure/api/service-api/ComputerUseControlAPI', () => ({ computerUseControlAPI: mocks })); +vi.mock('@/infrastructure/peer-device/deviceSurface', () => ({ + getActiveSurfaceScope: () => mocks.scope, + onSurfaceActivated: (fn: () => void) => { mocks.listeners.add(fn); return () => mocks.listeners.delete(fn); }, +})); +vi.mock('@/infrastructure/i18n', () => ({ useI18n: () => ({ t: mocks.t }) })); +vi.mock('./PreviewCursor', () => ({ PreviewCursor: () => })); +vi.mock('@openbitfun/ui', () => ({ + Button: ({ children, variant: _variant, size: _size, ...props }: React.ButtonHTMLAttributes & { variant?: string; size?: string }) => , + Input: ({ size: _size, ...props }: Omit, 'size'> & { size?: string }) => , + Select: () => setClickCount(Number(value))} options={[ + { value: 1, label: t('computerSharing.singleClick') }, { value: 2, label: t('computerSharing.doubleClick') }, + ]} /> +
+ setText(event.target.value)} disabled={busy} /> + +
+
+