From ffcc06685fd03453e7b6d5584acc4a340afe09b4 Mon Sep 17 00:00:00 2001 From: Ryan Ewen Date: Sat, 3 Oct 2026 15:49:07 -0400 Subject: [PATCH 1/5] Improve shared browser interaction and display quality Add direct keyboard, pointer and touch controls, display presets and favorites, toolbar menus, sharp captures, native desktop hover, and stable frame buffering. Preserve page state during mode changes and reduce redundant frame transfers. Include regression tests and fork reconciliation notes. --- paseo-shared-browser/README.md | 99 +- .../client/browser-canvas-input.test.ts | 102 ++ .../client/browser-canvas-input.ts | 185 +++ .../client/browser-canvas-keyboard-web.ts | 226 +++ .../client/browser-canvas-keyboard.test.ts | 108 ++ .../client/browser-canvas-keyboard.ts | 122 ++ .../client/browser-canvas-layout.test.ts | 109 ++ .../client/browser-canvas-layout.ts | 51 + .../browser-canvas-viewport-web.test.ts | 35 + .../client/browser-canvas-viewport-web.ts | 58 + .../client/browser-canvas-viewport.tsx | 152 ++ .../client/browser-capture-cadence.test.ts | 8 + .../client/browser-capture-cadence.ts | 9 + .../client/browser-emulation-mode.test.ts | 75 + .../client/browser-emulation-mode.ts | 98 ++ .../client/browser-frame-buffer.test.ts | 153 ++ .../client/browser-frame-buffer.ts | 146 ++ .../client/browser-frame-image.tsx | 62 + .../client/browser-image-load.test.ts | 21 + .../client/browser-image-load.ts | 24 + .../client/browser-input-queue.test.ts | 577 +++++++ .../client/browser-input-queue.ts | 435 +++++ .../client/browser-menu-placement.test.ts | 73 + .../client/browser-menu-placement.ts | 78 + .../client/browser-native-keyboard.test.ts | 116 ++ .../client/browser-native-keyboard.ts | 145 ++ .../client/browser-resolution-picker.tsx | 174 ++ .../client/browser-toolbar-menu-web.test.ts | 179 +++ .../client/browser-toolbar-menu-web.ts | 148 ++ .../client/browser-toolbar-menu.tsx | 233 +++ .../client/browser-toolbar-submenu.tsx | 93 ++ .../client/browser-viewer-recovery.test.ts | 92 ++ .../client/browser-viewer-recovery.ts | 60 + paseo-shared-browser/client/browser.tsx | 1418 +++++++++-------- paseo-shared-browser/client/favorite-star.tsx | 23 + .../client/frame-lifecycle.ts | 119 +- .../client/use-browser-canvas-input.ts | 382 +++++ .../client/use-browser-emulation-mode.ts | 56 + .../client/use-browser-frame-buffer.ts | 66 + .../client/use-browser-native-keyboard.ts | 129 ++ .../client/use-browser-viewer-recovery.ts | 26 + .../client/use-resolution-favorites.ts | 91 ++ paseo-shared-browser/client/web.ts | 271 ++++ .../docs/fork-reconciliation.md | 172 ++ .../docs/input-forwarding-notes.md | 76 + paseo-shared-browser/index.server.ts | 12 + paseo-shared-browser/package.json | 4 +- paseo-shared-browser/paseo-plugin.json | 16 +- .../server/agent-browser-runtime.ts | 793 ++++++--- .../server/browser-gesture.ts | 111 ++ paseo-shared-browser/server/browser-policy.ts | 354 +++- paseo-shared-browser/server/browser.ts | 34 + .../server/capture-transport-policy.test.ts | 105 ++ .../server/capture-transport-policy.ts | 153 ++ .../server/capture-transport-runtime.test.ts | 205 +++ .../server/emulation-restoration.test.ts | 258 +++ .../server/input-generation.test.ts | 17 + .../server/input-generation.ts | 28 + paseo-shared-browser/server/jpeg-frame.ts | 27 + .../server/keyboard-input.test.ts | 33 + paseo-shared-browser/server/keyboard-input.ts | 68 + .../server/live-input-policy.test.ts | 466 ++++++ .../server/live-input-runtime.test.ts | 365 +++++ .../server/live-input-supervisor.test.ts | 138 ++ paseo-shared-browser/server/mcp-entry.ts | 18 +- .../server/runtime-frame-reuse.test.ts | 265 +++ .../server/runtime-launch-environment.test.ts | 98 ++ .../server/runtime-owner-display.test.ts | 91 ++ paseo-shared-browser/server/runtime-owner.ts | 133 +- .../server/runtime-page-state.test.ts | 97 ++ .../server/runtime-protocol.ts | 6 +- .../server/stale-wheel-admission.test.ts | 135 ++ paseo-shared-browser/server/supervisor.ts | 26 +- .../server/virtual-display.test.ts | 171 ++ .../server/virtual-display.ts | 222 +++ .../shared/browser-display-preferences.ts | 31 + paseo-shared-browser/shared/browser.ts | 172 +- .../shared/capture-settings.ts | 16 + paseo-shared-browser/shared/device-presets.ts | 99 ++ .../shared/resolution-menu.test.ts | 114 ++ .../shared/resolution-menu.ts | 78 + .../shared/viewport-limits.ts | 3 + paseo-shared-browser/tests/contracts.test.ts | 10 +- .../tests/frame-lifecycle.test.ts | 6 +- paseo-shared-browser/tests/lease.test.ts | 105 ++ .../tests/viewport-resync.test.ts | 44 +- 86 files changed, 11135 insertions(+), 1067 deletions(-) create mode 100644 paseo-shared-browser/client/browser-canvas-input.test.ts create mode 100644 paseo-shared-browser/client/browser-canvas-input.ts create mode 100644 paseo-shared-browser/client/browser-canvas-keyboard-web.ts create mode 100644 paseo-shared-browser/client/browser-canvas-keyboard.test.ts create mode 100644 paseo-shared-browser/client/browser-canvas-keyboard.ts create mode 100644 paseo-shared-browser/client/browser-canvas-layout.test.ts create mode 100644 paseo-shared-browser/client/browser-canvas-layout.ts create mode 100644 paseo-shared-browser/client/browser-canvas-viewport-web.test.ts create mode 100644 paseo-shared-browser/client/browser-canvas-viewport-web.ts create mode 100644 paseo-shared-browser/client/browser-canvas-viewport.tsx create mode 100644 paseo-shared-browser/client/browser-capture-cadence.test.ts create mode 100644 paseo-shared-browser/client/browser-capture-cadence.ts create mode 100644 paseo-shared-browser/client/browser-emulation-mode.test.ts create mode 100644 paseo-shared-browser/client/browser-emulation-mode.ts create mode 100644 paseo-shared-browser/client/browser-frame-buffer.test.ts create mode 100644 paseo-shared-browser/client/browser-frame-buffer.ts create mode 100644 paseo-shared-browser/client/browser-frame-image.tsx create mode 100644 paseo-shared-browser/client/browser-image-load.test.ts create mode 100644 paseo-shared-browser/client/browser-image-load.ts create mode 100644 paseo-shared-browser/client/browser-input-queue.test.ts create mode 100644 paseo-shared-browser/client/browser-input-queue.ts create mode 100644 paseo-shared-browser/client/browser-menu-placement.test.ts create mode 100644 paseo-shared-browser/client/browser-menu-placement.ts create mode 100644 paseo-shared-browser/client/browser-native-keyboard.test.ts create mode 100644 paseo-shared-browser/client/browser-native-keyboard.ts create mode 100644 paseo-shared-browser/client/browser-resolution-picker.tsx create mode 100644 paseo-shared-browser/client/browser-toolbar-menu-web.test.ts create mode 100644 paseo-shared-browser/client/browser-toolbar-menu-web.ts create mode 100644 paseo-shared-browser/client/browser-toolbar-menu.tsx create mode 100644 paseo-shared-browser/client/browser-toolbar-submenu.tsx create mode 100644 paseo-shared-browser/client/browser-viewer-recovery.test.ts create mode 100644 paseo-shared-browser/client/browser-viewer-recovery.ts create mode 100644 paseo-shared-browser/client/favorite-star.tsx create mode 100644 paseo-shared-browser/client/use-browser-canvas-input.ts create mode 100644 paseo-shared-browser/client/use-browser-emulation-mode.ts create mode 100644 paseo-shared-browser/client/use-browser-frame-buffer.ts create mode 100644 paseo-shared-browser/client/use-browser-native-keyboard.ts create mode 100644 paseo-shared-browser/client/use-browser-viewer-recovery.ts create mode 100644 paseo-shared-browser/client/use-resolution-favorites.ts create mode 100644 paseo-shared-browser/client/web.ts create mode 100644 paseo-shared-browser/docs/fork-reconciliation.md create mode 100644 paseo-shared-browser/docs/input-forwarding-notes.md create mode 100644 paseo-shared-browser/server/browser-gesture.ts create mode 100644 paseo-shared-browser/server/capture-transport-policy.test.ts create mode 100644 paseo-shared-browser/server/capture-transport-policy.ts create mode 100644 paseo-shared-browser/server/capture-transport-runtime.test.ts create mode 100644 paseo-shared-browser/server/emulation-restoration.test.ts create mode 100644 paseo-shared-browser/server/input-generation.test.ts create mode 100644 paseo-shared-browser/server/input-generation.ts create mode 100644 paseo-shared-browser/server/jpeg-frame.ts create mode 100644 paseo-shared-browser/server/keyboard-input.test.ts create mode 100644 paseo-shared-browser/server/keyboard-input.ts create mode 100644 paseo-shared-browser/server/live-input-policy.test.ts create mode 100644 paseo-shared-browser/server/live-input-runtime.test.ts create mode 100644 paseo-shared-browser/server/live-input-supervisor.test.ts create mode 100644 paseo-shared-browser/server/runtime-frame-reuse.test.ts create mode 100644 paseo-shared-browser/server/runtime-launch-environment.test.ts create mode 100644 paseo-shared-browser/server/runtime-owner-display.test.ts create mode 100644 paseo-shared-browser/server/runtime-page-state.test.ts create mode 100644 paseo-shared-browser/server/stale-wheel-admission.test.ts create mode 100644 paseo-shared-browser/server/virtual-display.test.ts create mode 100644 paseo-shared-browser/server/virtual-display.ts create mode 100644 paseo-shared-browser/shared/browser-display-preferences.ts create mode 100644 paseo-shared-browser/shared/capture-settings.ts create mode 100644 paseo-shared-browser/shared/device-presets.ts create mode 100644 paseo-shared-browser/shared/resolution-menu.test.ts create mode 100644 paseo-shared-browser/shared/resolution-menu.ts create mode 100644 paseo-shared-browser/shared/viewport-limits.ts diff --git a/paseo-shared-browser/README.md b/paseo-shared-browser/README.md index b8a6ad8f..fbe4d09a 100644 --- a/paseo-shared-browser/README.md +++ b/paseo-shared-browser/README.md @@ -38,7 +38,8 @@ organization names. `AGENT_BROWSER_IDLE_TIMEOUT_MS=0`, `AGENT_BROWSER_STREAM_PORT=0`, and `AGENT_BROWSER_NO_AUTO_DIALOG=1` itself. - Frames stream from Chromium through CDP `Page.startScreencast`, with a bounded screenshot fallback. - Remote input supports tap, double-tap, right-click, drag, swipe scrolling, text, and special keys. + Remote input supports mouse hover, wheel scrolling, continuous dragging, native touch pan/pinch, + tap, double-tap, right-click, text, and special keys. - Device presets for Desktop Chrome, iPhone 15 Pro, Pixel 7, and iPad Pro 11 change Chromium's viewport, device pixel ratio, touch behavior, platform, and user agent. A phone can view and control the shared browser, but the rendered browser remains Chromium. An iPhone preset is mobile @@ -108,7 +109,7 @@ unchanged. Pi agents continue to receive the adapter, but they require Pi's opti launch it. The injected MCP server exposes exactly these tools: `shared_browser_status`, -`shared_browser_capture`, `shared_browser_acquire_control`, `shared_browser_release_control`, +`shared_browser_capture`, `shared_browser_device`, `shared_browser_acquire_control`, `shared_browser_release_control`, `shared_browser_navigate`, `shared_browser_input`, and `shared_browser_viewport`. It does not expose arbitrary CDP commands, JavaScript or page evaluation, browser profile access, or filesystem access. @@ -137,11 +138,92 @@ User-supplied `AGENT_BROWSER_*` variables are deliberately ignored. ## Controls -- Toolbar: back, forward, reload, address bar, and device emulation. +- Toolbar: back, forward, reload, address bar, a combined monitor menu, mobile emulation toggle + and a vertical-dots browser actions menu. +- Custom viewports support 320 to 2560 pixels wide and 480 to 2560 pixels high. Invalid sizes + show their error inside the device dialog; a successful Apply closes it. +- Desktop presets also include 1280 × 800 (16:10) and 1280 × 1280 (1:1). +- Pixel 7 (high resolution) keeps the same 412 × 839 phone layout and input coordinates, + with a sharper 824 × 1678 capture. +- Desktop presets include 1920 × 1080 and 2560 × 1440 (16:9), 1920 × 1200 and + 2560 × 1600 (16:10), and 1920 × 1920 and 2560 × 2560 (1:1). +- The Resolution and quality menu groups desktop choices by 16:9, 16:10 and 1:1, + then mobile. Each group sorts by width. Filled stars mark favorites; the monitor + menu provides favorites and access to the full resolution and quality list. +- Monitor and browser actions open compact anchored menus rather than dialogs. Menus stay + within the pane, scroll long lists, and dismiss with an outside press, Escape or native Back. +- The monitor menu offers **Fit to panel** (default) and **Actual size (100%)**. Actual size + uses browser layout pixels, not the larger JPEG dimensions of sharper phone captures. + This choice affects only the current viewer, without resizing the shared browser. + Desktop local scrollbars reveal overflow while wheel events on a controlled frame go to + the remote page. Phone viewers can pan while observing; controlling sends swipes to the + page instead. Fitting images stay centered without empty scroll ranges; a scrollbar on + one axis does not force one on the other. Returning to Fit resets local offsets without + reloading the page or image. +- Quality choices Low, Medium and High request JPEG quality 70, 90 and 95 respectively. + Detailed large views use more bandwidth; frames reduce quality further only if they + exceed the 4 MiB frame limit. Preferences persist on the connected Paseo host. +- Captures refresh after input. A stalled screencast falls back to a fresh screenshot instead + of indefinitely showing an old image. +- The transport sends individual JPEG frames, not encoded video. Repeated reads of the exact + same cached runtime frame send metadata only; new captures and changed input authority + retain their normal frame validation. Image quality and cached-pixel age limits are unchanged. +- Quiet pages stay on screenshot fallback until sustained fresh streaming resumes. The transport + label shows how the displayed frame was captured; switching to fallback does not mean the + browser disconnected. Cached captures expire within one second and refresh after input. +- Reconnecting or switching browser tabs restores the selected device viewport, capture density, + touch and user agent before accepting new frames. +- Actual JPEG dimensions must match the selected capture resolution before a frame is accepted. + The fallback captures the complete visible viewport, preserves scroll position, and accounts + for device pixel ratio and capture scale so inputs still use the original layout coordinates. +- The last decoded frame stays visible while its replacement loads, with native image fading + disabled. Input targets the displayed frame, and obsolete image callbacks cannot replace it. - Status row: session state, viewer count, controller, and lease expiry. +- Returning after viewer expiry reattaches viewing once automatically while preserving the + page. Expired control is cleared; take control again to send input. Expected expiry is not + shown as an action failure; a failed reattachment still offers manual retry. +- Clicking a link or submitting a form may navigate before its input reply arrives. A completed + input returns the new viewing state without a false failure or repeating the action. + Replaced controls, targets and uncertain sends remain rejected. - Human control: **Take control**, **Release**, and **Take over** for explicit handoff. Agent MCP calls have no forced-takeover operation. -- Mobile: swipe scrolls by default; pointer and keyboard options open as bottom sheets. +- With control, mouse movement forwards real hover effects and standard browser cursor changes. + Wheel scrolling stays inside the browser canvas; held drags update before release. Leaving the + canvas clears remote hover, while dragging beyond its edges still releases the held button. +- Hidden Linux browsers use an authenticated private virtual display when `/usr/bin/Xvfb` + is already installed. This supplies native desktop pointer/hover capabilities after switching + back from phone mode. It opens no visible window or TCP listener, changes no host display + settings, and adds about 25 MiB of framebuffer memory per workspace plus server overhead. + The owned display is stopped with its browser. Other platforms, missing Xvfb, or a fully + cleaned display-startup failure retain the existing launch behavior; headless Linux may + still lack media-query-gated hover. The plugin does not install Xvfb automatically. +- Native touch forwards active fingers continuously, including pan and pinch on desktop and mobile + presets. Physical mouse buttons and double clicks retain their natural actions; touch never + changes into a synthetic mouse gesture. + Losing control, changing page or resizing cancels held input; old contacts must lift before a + fresh touch can target the replacement page. No uncertain input is replayed after a failure. +- Desktop viewports show native scrollbars. Dragging a scrollbar thumb scrolls the page + continuously before release. Wheel scrolling and touch panning remain available; + mobile emulation retains its normal mobile scrollbar behavior. +- Click the controlled canvas to type directly on desktop and web. Shortcuts, repeat, + modifier clicks and committed composition text are forwarded; other Paseo inputs stay local. +- The address-bar Browser menu contains a Send keys submenu and reconnect actions. On desktop, + keys open beside the menu; on compact screens, Back returns to the parent menu. Extra keys + are no longer shown below the canvas or in a separate dialog. +- On native phones, Keyboard in that menu opens the software keyboard for basic live typing + and Backspace + without a visible text box. Autocorrection is disabled. Compose opens an explicit local + draft for IME text, inserted once with Done. Native hardware shortcuts are not supported + by this software-keyboard relay. Extra keys remain available in the Send keys submenu. +- Local plain-text paste is forwarded; remote copy/cut are not synchronized to the local clipboard. +- The address-bar mobile toggle changes shared emulation while preserving the current display + dimensions and capture density. Choose a resolution explicitly to change the display. + Your device default applies the first time you take control, never while observing, and + also preserves the current display. Narrow desktop layouts do not count as phones. +- The mobile address field keeps a full text line with compact padding. Read-only addresses + stay legible while observing; taking control is still required to edit or navigate. +- Mode changes preserve the current page and unsaved fields. Sites that select layouts only + at page load may need an explicit Reload. There are no click, swipe or scroll mode controls. ## Security boundary @@ -184,5 +266,12 @@ persistence, and archive teardown. Release Please maintains the version, changelog, component tag, and GitHub release from Conventional Commits in the monorepo. -Both the Paseo daemon and app must satisfy `^0.9.0 || ^0.10.0 || ^0.11.0`. The client surface uses React Native primitives +Both the Paseo daemon and app must satisfy the version range in `paseo-plugin.json`. +Stable 0.11 and the tested `0.11.0-beta.3` allowance are both retained. The client surface uses React Native primitives and works in desktop, web, iOS, and Android Paseo clients. + +A wheel capture can finish decoding after input revoked its frame token. The server +returns a known non-admission receipt before creating a new channel; the canvas waits +for another decoded frame before admitting that still-unsent gesture. Recovery allows +up to three admission attempts within one four-second decoded-frame wait budget. +Published input, unknown outcomes, expired leases, and replaced contexts are never retried. diff --git a/paseo-shared-browser/client/browser-canvas-input.test.ts b/paseo-shared-browser/client/browser-canvas-input.test.ts new file mode 100644 index 00000000..3fafc9f9 --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-input.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import type { BrowserGestureEvent } from "../shared/browser"; +import { createBrowserCanvasInput } from "./browser-canvas-input"; + +const point = { x: 40, y: 20, width: 400, height: 300 }; +function fixture() { + const events: BrowserGestureEvent[] = []; + let ended = 0; + const input = createBrowserCanvasInput({ + enabled: () => true, + viewport: () => ({ width: 800, height: 600 }), + enqueue: (event) => { + events.push(event); + return true; + }, + finish: () => { + ended += 1; + }, + cancel: () => {}, + onPoint: () => {}, + onActivity: () => {}, + }); + return { input, events, ended: () => ended }; +} + +describe("natural canvas input", () => { + it("keeps actual right, middle and double-click edges without a mode override", () => { + const f = fixture(); + for (const [button, clickCount] of [ + ["left", 1], + ["left", 2], + ["right", 1], + ["middle", 1], + ] as const) { + f.input.mouseDown(point, button, clickCount); + f.input.mouseUp(point, button, clickCount); + } + expect( + f.events.map((event) => + event.kind === "down" || event.kind === "up" + ? [event.kind, event.button, event.clickCount] + : null, + ), + ).toEqual([ + ["down", "left", 1], + ["up", "left", 1], + ["down", "left", 2], + ["up", "left", 2], + ["down", "right", 1], + ["up", "right", 1], + ["down", "middle", 1], + ["up", "middle", 1], + ]); + }); + + it("forwards held mouse motion before release rather than synthesizing a final drag", () => { + const f = fixture(); + f.input.mouseDown(point, "left", 1); + f.input.mouseMove({ ...point, x: 60 }); + f.input.mouseMove({ ...point, x: 100 }); + expect(f.ended()).toBe(0); + expect(f.events.map((event) => event.kind)).toEqual([ + "down", + "move", + "move", + ]); + f.input.mouseUp({ ...point, x: 100 }, "left", 1); + expect(f.ended()).toBe(1); + }); + + it("always forwards genuine touch contacts, including held press and pinch partial release", () => { + const f = fixture(); + const first = { ...point, id: 7 }; + const second = { ...point, x: 200, id: 9 }; + f.input.touch("start", [first]); + expect(f.events).toEqual([ + { kind: "touch", type: "start", points: [first] }, + ]); + expect(f.ended()).toBe(0); + f.input.touch("start", [first, second]); + f.input.touch("move", [ + { ...first, x: 20 }, + { ...second, x: 240 }, + ]); + f.input.touch("end", [second]); + f.input.touch("end", []); + expect( + f.events.map((event) => + event.kind === "touch" + ? [event.type, event.points.map((contact) => contact.id)] + : event.kind, + ), + ).toEqual([ + ["start", [7]], + ["start", [7, 9]], + ["move", [7, 9]], + ["move", [9]], + ["end", []], + ]); + expect(f.ended()).toBe(1); + }); +}); diff --git a/paseo-shared-browser/client/browser-canvas-input.ts b/paseo-shared-browser/client/browser-canvas-input.ts new file mode 100644 index 00000000..11101ec9 --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-input.ts @@ -0,0 +1,185 @@ +/** + * Platform-neutral canvas gestures. Coordinate mapping belongs to the actual + * displayed image rectangle, not capture pixels (including sharp mobile images). + * Touch is always genuine multi-touch on every remote preset. Mouse buttons and + * click counts come from physical events; surrounding Paseo chrome is untouched. + */ +import type { BrowserGestureEvent } from "../shared/browser"; +import type { + BrowserGesturePoint, + BrowserTouchPoint, +} from "./browser-input-queue"; + +export interface CanvasInputOptions { + enabled(): boolean; + viewport(): { width: number; height: number } | null; + enqueue(event: BrowserGestureEvent): boolean; + finish(): void; + cancel(): void; + onPoint(point: BrowserGesturePoint): void; + onActivity(active: boolean): void; +} +const MAX_SCROLL_DELTA = 4_000; + +/** Preserve wheel distance by splitting oversized deltas instead of clipping it. */ +function scroll( + options: CanvasInputOptions, + point: BrowserGesturePoint, + deltaX: number, + deltaY: number, +) { + let remainingX = deltaX; + let remainingY = deltaY; + while (Math.abs(remainingX) > 0.001 || Math.abs(remainingY) > 0.001) { + const x = Math.max( + -MAX_SCROLL_DELTA, + Math.min(MAX_SCROLL_DELTA, remainingX), + ); + const y = Math.max( + -MAX_SCROLL_DELTA, + Math.min(MAX_SCROLL_DELTA, remainingY), + ); + if (!options.enqueue({ kind: "scroll", point, deltaX: x, deltaY: y })) + return; + remainingX -= x; + remainingY -= y; + } +} + +/** Consume normalized real pointer/touch events with stable touch identities. */ +export function createBrowserCanvasInput(options: CanvasInputOptions) { + const mouseButtons = new Set<"left" | "right" | "middle">(); + let activeTouchIds = new Set(); + let touchBlockedUntilRelease = false; + + // Quarantine cancelled held contacts until actual liftoff. A move or added + // finger cannot invent a new press on a replacement page or controller. + const reset = () => { + if (activeTouchIds.size) touchBlockedUntilRelease = true; + mouseButtons.clear(); + activeTouchIds.clear(); + options.onActivity(false); + }; + const cancel = () => { + reset(); + options.cancel(); + }; + const mouseMove = (point: BrowserGesturePoint) => { + if (!options.enabled()) return false; + options.onPoint(point); + if (!mouseButtons.size) options.onActivity(false); + return options.enqueue({ kind: "move", point }); + }; + const mouseDown = ( + point: BrowserGesturePoint, + button: "left" | "right" | "middle", + clickCount: 1 | 2, + ) => { + if (!options.enabled()) return false; + options.onPoint(point); + const accepted = options.enqueue({ + kind: "down", + point, + button, + clickCount, + }); + if (accepted) { + mouseButtons.add(button); + options.onActivity(true); + } + return accepted; + }; + const mouseUp = ( + point: BrowserGesturePoint, + button: "left" | "right" | "middle", + clickCount: 1 | 2, + ) => { + if (!mouseButtons.has(button)) return false; + options.onPoint(point); + const accepted = options.enqueue({ kind: "up", point, button, clickCount }); + mouseButtons.delete(button); + if (!mouseButtons.size) { + options.onActivity(false); + options.finish(); + } + return accepted; + }; + const wheel = ( + point: BrowserGesturePoint, + deltaX: number, + deltaY: number, + ) => { + if (!options.enabled()) return false; + const viewport = options.viewport(); + if (!viewport) return false; + options.onPoint(point); + options.onActivity(true); + scroll( + options, + point, + (deltaX * viewport.width) / point.width, + (deltaY * viewport.height) / point.height, + ); + return true; + }; + + const touch = ( + phase: "start" | "move" | "end" | "cancel", + points: BrowserTouchPoint[], + ) => { + if (phase === "cancel") { + cancel(); + // Native touchCancel terminates the physical contact session itself. + touchBlockedUntilRelease = false; + return true; + } + if (touchBlockedUntilRelease) { + if (points.length === 0) touchBlockedUntilRelease = false; + return true; + } + if (!activeTouchIds.size && phase !== "start") return true; + if (!options.enabled()) { + if (points.length > 0) touchBlockedUntilRelease = true; + cancel(); + return false; + } + if ( + points.length > 5 || + new Set(points.map((point) => point.id)).size !== points.length + ) { + cancel(); + return false; + } + const hasNewPoint = points.some((point) => !activeTouchIds.has(point.id)); + if (hasNewPoint && phase !== "start") { + cancel(); + return true; + } + const type = points.length === 0 ? "end" : hasNewPoint ? "start" : "move"; + if (points.length === 0 && activeTouchIds.size === 0) return false; + const accepted = options.enqueue({ kind: "touch", type, points }); + activeTouchIds = new Set(points.map((point) => point.id)); + options.onActivity(points.length > 0); + if (points.length === 0) options.finish(); + if (points[0]) options.onPoint(points[0]); + return accepted; + }; + + return { + mouseMove, + mouseDown, + mouseUp, + wheel, + touch, + cancel, + reset, + isHoldingMouse: () => mouseButtons.size > 0, + isTouchQuarantined: () => touchBlockedUntilRelease, + leave: () => { + if (!mouseButtons.size && !activeTouchIds.size) { + options.enqueue({ kind: "leave" }); + options.finish(); + } + }, + }; +} diff --git a/paseo-shared-browser/client/browser-canvas-keyboard-web.ts b/paseo-shared-browser/client/browser-canvas-keyboard-web.ts new file mode 100644 index 00000000..ef2d6340 --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-keyboard-web.ts @@ -0,0 +1,226 @@ +/** + * Focused web keyboard/IME bridge. Its invisible editable sink gives the browser + * a real composition target without modifying image children. Only controlled + * canvas focus consumes host keys; blur releases the original remote channel. + * Browser-reserved shortcuts are outside JavaScript's interception guarantees. + */ +import { + createBrowserCanvasKeyboard, + type CanvasKeyEnvelope, + type CanvasKeyboardEvent, +} from "./browser-canvas-keyboard"; + +interface KeyboardDomEvent extends CanvasKeyEnvelope { + data?: string | null; + inputType?: string; + clipboardData?: { getData(type: string): string } | null; + preventDefault(): void; + stopPropagation(): void; +} +interface KeyboardElement { + tabIndex: number; + value: string; + style: Record; + setAttribute(name: string, value: string): void; + focus(options?: { preventScroll: boolean }): void; + remove(): void; + addEventListener( + type: string, + listener: (event: KeyboardDomEvent) => void, + options?: { capture?: boolean }, + ): void; + removeEventListener( + type: string, + listener: (event: KeyboardDomEvent) => void, + options?: { capture?: boolean }, + ): void; +} +export interface KeyboardCanvasNode extends KeyboardElement { + appendChild(child: KeyboardElement): void; + ownerDocument: { + activeElement: unknown; + hidden: boolean; + createElement(tag: "textarea"): KeyboardElement; + addEventListener(type: string, listener: (event: KeyboardDomEvent) => void): void; + removeEventListener(type: string, listener: (event: KeyboardDomEvent) => void): void; + defaultView: { + addEventListener(type: string, listener: () => void): void; + removeEventListener(type: string, listener: () => void): void; + } | null; + }; +} +interface BindingOptions { + enabled(): boolean; + enqueue(event: CanvasKeyboardEvent): boolean; + finish(): void; + cancel(): void; + onError(error: Error): void; +} + +/** DOM-only binding; callers must dispose it and reset on route/control changes. */ +export function bindBrowserCanvasKeyboard(node: KeyboardCanvasNode, options: BindingOptions) { + const document = node.ownerDocument; + const previousTabIndex = node.tabIndex; + const sink = document.createElement("textarea"); + sink.tabIndex = -1; + sink.setAttribute("aria-label", "Type in the shared browser"); + sink.setAttribute("autocomplete", "off"); + sink.setAttribute("autocapitalize", "off"); + sink.setAttribute("spellcheck", "false"); + Object.assign(sink.style, { + position: "absolute", + width: "1px", + height: "1px", + opacity: "0", + left: "0", + top: "0", + padding: "0", + border: "0", + pointerEvents: "none", + }); + node.tabIndex = 0; + node.appendChild(sink); + let composing = false; + let compositionAdmitted = false; + let altTextPending: string | null = null; + const focused = () => document.activeElement === sink || document.activeElement === node; + const keyboard = createBrowserCanvasKeyboard({ + ...options, + enabled: () => options.enabled() && focused() && !document.hidden, + }); + const consume = (event: KeyboardDomEvent) => { + event.preventDefault(); + event.stopPropagation(); + }; + const reset = () => { + keyboard.reset(); + compositionAdmitted = false; + composing = false; + altTextPending = null; + sink.value = ""; + }; + const cancel = () => { + reset(); + options.cancel(); + }; + const focus = () => { + if (options.enabled() && !document.hidden) { + sink.focus({ preventScroll: true }); + } + }; + const down = (event: KeyboardDomEvent) => { + if (!focused()) { + return; + } + // Local paste supplies clipboard bytes separately. Sending Ctrl/Meta+V would + // additionally paste an unrelated remote clipboard and duplicate the action. + const pasteShortcut = !event.altKey && ( + ((event.ctrlKey || event.metaKey) && event.code === "KeyV") || + (event.shiftKey && !event.ctrlKey && !event.metaKey && event.code === "Insert") + ); + if (pasteShortcut) { + if (options.enabled()) { + event.stopPropagation(); + } + return; + } + if (keyboard.key("down", event)) { + const altPrintable = event.altKey && !event.ctrlKey && !event.metaKey && + !event.getModifierState?.("AltGraph") && Array.from(event.key).length === 1; + altTextPending = altPrintable ? event.key : null; + if (altTextPending) { + // Option layouts can produce text with Alt held. Only the browser's + // ensuing insertion event qualifies it; an Alt shortcut may emit none. + event.stopPropagation(); + } else { + consume(event); + } + } else if (options.enabled() && (composing || event.isComposing)) { + event.stopPropagation(); + } + }; + const up = (event: KeyboardDomEvent) => { + // Document observes releases after focus loss without consuming Paseo input. + if (keyboard.key("up", event) && focused()) { + consume(event); + } + altTextPending = null; + }; + const compositionStart = () => { + altTextPending = null; + composing = true; + compositionAdmitted = options.enabled() && focused() && !document.hidden; + }; + const commitText = (value: string) => { + if (value.length > 16_000) { + options.onError(new Error("Text is too long. Send up to 16,000 characters at once.")); + return; + } + keyboard.text(value); + }; + const compositionEnd = (event: KeyboardDomEvent) => { + const accepted = compositionAdmitted && options.enabled() && focused(); + composing = false; + compositionAdmitted = false; + if (accepted) { + commitText(event.data ?? ""); + } + sink.value = ""; + }; + const beforeInput = (event: KeyboardDomEvent) => { + if (!composing && options.enabled() && focused()) { + consume(event); + if (altTextPending && event.inputType === "insertText" && event.data === altTextPending) { + commitText(event.data); + } + altTextPending = null; + } + }; + const paste = (event: KeyboardDomEvent) => { + if (!options.enabled() || !focused()) { + return; + } + consume(event); + altTextPending = null; + commitText(event.clipboardData?.getData("text/plain") ?? ""); + sink.value = ""; + }; + const visibility = () => { + if (document.hidden) { + cancel(); + } + }; + const listeners: Array<[KeyboardElement, string, (event: KeyboardDomEvent) => void]> = [ + [node, "pointerdown", focus], + [node, "mousedown", focus], + [node, "touchstart", focus], + [node, "focus", focus], + [sink, "keydown", down], + [node, "keydown", down], + [sink, "compositionstart", compositionStart], + [sink, "compositionend", compositionEnd], + [sink, "beforeinput", beforeInput], + [sink, "paste", paste], + [sink, "blur", cancel], + ]; + for (const [element, type, listener] of listeners) { + element.addEventListener(type, listener); + } + document.addEventListener("keyup", up); + document.addEventListener("visibilitychange", visibility); + document.defaultView?.addEventListener("blur", cancel); + return { + reset, + dispose() { + cancel(); + for (const [element, type, listener] of listeners) { + element.removeEventListener(type, listener); + } + document.removeEventListener("keyup", up); + document.removeEventListener("visibilitychange", visibility); + document.defaultView?.removeEventListener("blur", cancel); + node.tabIndex = previousTabIndex; + sink.remove(); + }, + }; +} diff --git a/paseo-shared-browser/client/browser-canvas-keyboard.test.ts b/paseo-shared-browser/client/browser-canvas-keyboard.test.ts new file mode 100644 index 00000000..3f71fd76 --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-keyboard.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; +import { + canvasKeyModifiers, + createBrowserCanvasKeyboard, + type CanvasKeyboardEvent, + type CanvasKeyEnvelope, +} from "./browser-canvas-keyboard"; + +const envelope = (overrides: Partial = {}): CanvasKeyEnvelope => ({ + key: "a", + code: "KeyA", + altKey: false, + ctrlKey: false, + metaKey: false, + shiftKey: false, + repeat: false, + ...overrides, +}); +function fixture() { + const events: CanvasKeyboardEvent[] = []; + let enabled = true; + let finishes = 0; + const input = createBrowserCanvasKeyboard({ + enabled: () => enabled, + enqueue: (event) => { + events.push(event); + return true; + }, + finish: () => { + finishes += 1; + }, + }); + return { + input, + events, + disable: () => { + enabled = false; + }, + enable: () => { + enabled = true; + }, + finishes: () => finishes, + }; +} +describe("physical canvas keyboard", () => { + it("commits qualified AltGraph text once but never converts Ctrl+Alt shortcuts", () => { + const f = fixture(); + const euro = envelope({ key: "€", code: "KeyE", ctrlKey: true, altKey: true, getModifierState: value => value === "AltGraph" }); + f.input.key("down", euro); + f.input.key("up", euro); + f.input.key("down", envelope({ key: "e", code: "KeyE", ctrlKey: true, altKey: true })); + expect(f.events.filter(event => event.kind === "text")).toEqual([{ kind: "text", text: "€" }]); + expect(f.events.filter(event => event.kind === "key")).toHaveLength(3); + expect(f.events.filter(event => event.kind === "key").every(event => event.text === undefined)).toBe(true); + const separate = fixture(); + separate.input.key("down", envelope({ key: "€", getModifierState: value => value === "AltGraph" })); + expect(separate.events).toEqual([{ kind: "key", type: "down", key: "€", code: "KeyA", modifiers: 0, repeat: false, text: "€" }]); + }); + it("preserves Unicode text, physical repeat/code and all modifier bits", () => { + const f = fixture(); + f.input.key("down", envelope({ key: "é" })); + f.input.key("down", envelope({ key: "é", repeat: true })); + f.input.key("up", envelope({ key: "é", repeat: true })); + expect(f.events).toEqual([ + { kind: "key", type: "down", key: "é", code: "KeyA", modifiers: 0, repeat: false, text: "é" }, + { kind: "key", type: "down", key: "é", code: "KeyA", modifiers: 0, repeat: true, text: "é" }, + { kind: "key", type: "up", key: "é", code: "KeyA", modifiers: 0, repeat: false }, + ]); + expect( + canvasKeyModifiers(envelope({ altKey: true, ctrlKey: true, metaKey: true, shiftKey: true })), + ).toBe(15); + f.input.key("down", envelope({ ctrlKey: true })); + expect(f.events.at(-1)).not.toHaveProperty("text"); + }); + it("never starts from a cancelled repeat or a repeat that began observe-only", () => { + const f = fixture(); + f.input.key("down", envelope()); + f.input.reset(); + expect(f.input.key("down", envelope({ repeat: true }))).toBe(false); + expect(f.input.key("up", envelope())).toBe(false); + f.disable(); + f.input.key("down", envelope()); + f.enable(); + expect(f.input.key("down", envelope({ repeat: true }))).toBe(false); + expect(f.input.key("up", envelope())).toBe(false); + expect(f.input.key("down", envelope())).toBe(true); + expect(f.events).toHaveLength(2); + }); + it("does not type composition intermediates; commits bounded text separately", () => { + const f = fixture(); + expect(f.input.key("down", envelope({ isComposing: true }))).toBe(false); + expect(f.input.key("down", envelope({ key: "Dead" }))).toBe(false); + expect(f.input.key("down", envelope({ keyCode: 229 }))).toBe(false); + expect(f.input.text("日本語🙂")).toBe(true); + expect(f.input.text("x".repeat(16_001))).toBe(false); + expect(f.events).toEqual([{ kind: "text", text: "日本語🙂" }]); + }); + it("keeps modifier holds through other keys and finishes only after last keyup", () => { + const f = fixture(); + const ctrl = envelope({ key: "Control", code: "ControlLeft", ctrlKey: true }); + f.input.key("down", ctrl); + f.input.key("down", envelope({ ctrlKey: true })); + f.input.key("up", envelope({ ctrlKey: true })); + expect(f.finishes()).toBe(0); + f.input.key("up", { ...ctrl, ctrlKey: false }); + expect(f.finishes()).toBe(1); + }); +}); diff --git a/paseo-shared-browser/client/browser-canvas-keyboard.ts b/paseo-shared-browser/client/browser-canvas-keyboard.ts new file mode 100644 index 00000000..8b1f3148 --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-keyboard.ts @@ -0,0 +1,122 @@ +/** + * Normalize physical canvas keys without inventing presses after control loss. + * IME and clipboard commits use text insertion; ordinary typing retains real + * key/code/repeat/modifier events. The DOM adapter owns focus and composition. + */ +import type { BrowserGestureEvent } from "../shared/browser"; + +export interface CanvasKeyEnvelope { + key: string; + code: string; + altKey: boolean; + ctrlKey: boolean; + metaKey: boolean; + shiftKey: boolean; + repeat: boolean; + isComposing?: boolean; + keyCode?: number; + getModifierState?(key: string): boolean; +} +export type CanvasKeyboardEvent = Extract; +interface KeyboardOptions { + enabled(): boolean; + enqueue(event: CanvasKeyboardEvent): boolean; + finish(): void; +} + +/** CDP modifier bits preserve the actual host combination, including Shift. */ +export function canvasKeyModifiers(event: CanvasKeyEnvelope): number { + return ( + (event.altKey ? 1 : 0) | + (event.ctrlKey ? 2 : 0) | + (event.metaKey ? 4 : 0) | + (event.shiftKey ? 8 : 0) + ); +} + +/** A cancelled held key's repeat/up cannot begin input on a replacement page. */ +export function createBrowserCanvasKeyboard(options: KeyboardOptions) { + const admitted = new Set(); + const blocked = new Set(); + const identity = (event: CanvasKeyEnvelope) => event.code || event.key; + const reset = () => { + for (const key of admitted) blocked.add(key); + admitted.clear(); + }; + const key = (type: "down" | "up", event: CanvasKeyEnvelope): boolean => { + const id = identity(event); + if (type === "up") { + blocked.delete(id); + if (!admitted.delete(id)) { + return false; + } + } else { + if (!options.enabled()) { + blocked.add(id); + return false; + } + if ( + event.isComposing || + event.key === "Dead" || + event.key === "Process" || + event.keyCode === 229 + ) { + return false; + } + // A non-repeat is an authentic new physical edge after a missed blur keyup. + if (!event.repeat) { + blocked.delete(id); + } + if (blocked.has(id) || (event.repeat && !admitted.has(id))) { + return false; + } + } + if ( + !options.enabled() || + !event.key || + event.key.length > 64 || + !/^[A-Za-z][A-Za-z0-9]{0,63}$/.test(event.code) + ) { + return false; + } + const modifiers = canvasKeyModifiers(event); + const printable = + type === "down" && + !event.altKey && + !event.ctrlKey && + !event.metaKey && + Array.from(event.key).length === 1; + const accepted = options.enqueue({ + kind: "key", + type, + key: event.key, + code: event.code, + modifiers, + repeat: type === "down" && event.repeat, + ...(printable ? { text: event.key } : {}), + }); + if (accepted && type === "down") { + admitted.add(id); + // AltGraph can report Ctrl+Alt on international layouts. The real host + // qualifier permits one text commit without rewriting a genuine shortcut. + if (event.getModifierState?.("AltGraph") && (modifiers & 7) !== 0 && Array.from(event.key).length === 1) { + options.enqueue({ kind: "text", text: event.key }); + } + } + if (accepted && type === "up" && !admitted.size) { + options.finish(); + } + return accepted; + }; + const text = (value: string): boolean => { + if (!options.enabled() || !value || value.length > 16_000) { + return false; + } + const accepted = options.enqueue({ kind: "text", text: value }); + if (accepted && !admitted.size) { + options.finish(); + } + return accepted; + }; + return { key, text, reset }; +} diff --git a/paseo-shared-browser/client/browser-canvas-layout.test.ts b/paseo-shared-browser/client/browser-canvas-layout.test.ts new file mode 100644 index 00000000..f9ac35a8 --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-layout.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; +import { containedRect } from "../shared/browser"; +import { getBrowserCanvasLayout } from "./browser-canvas-layout"; + +describe("local browser canvas geometry", () => { + it("defaults to Fit and exactly preserves containedRect's letterboxing", () => { + const container = { width: 1000, height: 700 }; + const frame = { width: 824, height: 1678 }; + expect(getBrowserCanvasLayout(container, frame, { width: 412, height: 839 })).toEqual({ + contentSize: container, + frameRect: containedRect(container, frame), + }); + }); + + it("displays sharp Pixel capture at canonical CSS size rather than twice its size", () => { + expect( + getBrowserCanvasLayout( + { width: 300, height: 600 }, + { width: 824, height: 1678 }, + { width: 412, height: 839 }, + "actual", + ), + ).toEqual({ + contentSize: { width: 412, height: 839 }, + frameRect: { x: 0, y: 0, width: 412, height: 839 }, + }); + }); + + it("keeps Actual size unchanged at DPR1,2 or3", () => { + const viewport = { width: 1280, height: 800 }; + for (const scale of [1, 2, 3]) { + expect( + getBrowserCanvasLayout( + { width: 600, height: 400 }, + { width: viewport.width * scale, height: viewport.height * scale }, + viewport, + "actual", + ), + ).toEqual({ + contentSize: viewport, + frameRect: { x: 0, y: 0, ...viewport }, + }); + } + }); + + it("lets the scroll host center a smaller Actual frame without an outer-sized spacer", () => { + expect( + getBrowserCanvasLayout( + { width: 1000, height: 1000 }, + { width: 824, height: 1678 }, + { width: 412, height: 839 }, + "actual", + ), + ).toEqual({ + contentSize: { width: 412, height: 839 }, + frameRect: { x: 0, y: 0, width: 412, height: 839 }, + }); + }); + + it("uses exact frame extents on both axes so scrollbar allocation cannot create unused overflow", () => { + expect( + getBrowserCanvasLayout( + { width: 1000, height: 500 }, + { width: 824, height: 1678 }, + { width: 412, height: 839 }, + "actual", + ), + ).toEqual({ + contentSize: { width: 412, height: 839 }, + frameRect: { x: 0, y: 0, width: 412, height: 839 }, + }); + expect( + getBrowserCanvasLayout( + { width: 300, height: 1000 }, + { width: 824, height: 1678 }, + { width: 412, height: 839 }, + "actual", + ), + ).toEqual({ + contentSize: { width: 412, height: 839 }, + frameRect: { x: 0, y: 0, width: 412, height: 839 }, + }); + }); + + it("rejects invalid/unmeasured dimensions without publishing NaN overlay coordinates", () => { + const valid = { width: 600, height: 400 }; + for (const invalid of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + for (const axis of ["width", "height"] as const) { + const size = { ...valid, [axis]: invalid }; + expect(getBrowserCanvasLayout(size, valid, valid)).toBeNull(); + expect(getBrowserCanvasLayout(valid, size, valid)).toBeNull(); + expect(getBrowserCanvasLayout(valid, valid, size, "actual")).toBeNull(); + } + } + }); + + it("does not require resolved remote CSS size before the existing Fit geometry can render", () => { + expect( + getBrowserCanvasLayout( + { width: 800, height: 600 }, + { width: 1280, height: 800 }, + { width: 0, height: 0 }, + ), + ).toEqual({ + contentSize: { width: 800, height: 600 }, + frameRect: containedRect({ width: 800, height: 600 }, { width: 1280, height: 800 }), + }); + }); +}); diff --git a/paseo-shared-browser/client/browser-canvas-layout.ts b/paseo-shared-browser/client/browser-canvas-layout.ts new file mode 100644 index 00000000..09555bf7 --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-layout.ts @@ -0,0 +1,51 @@ +/** + * Local viewer geometry only. Frame pixels determine Fit's aspect ratio; Actual + * size uses the remote viewport's CSS dimensions, independently of capture DPR. + * Neither mode changes the remote viewport or its frame/input authority. + */ +import { containedRect, type FrameRect, type FrameSize } from "../shared/browser"; + +export type BrowserCanvasDisplayMode = "fit" | "actual"; + +export interface BrowserCanvasLayout { + /** Frame wrapper size; Actual's scroll host centers it within the available inner viewport. */ + contentSize: FrameSize; + /** Position and displayed size inside that extent, also used by the input overlay. */ + frameRect: FrameRect; +} + +/** + * Calculate displayed geometry without treating sharp capture pixels as CSS + * dimensions. Invalid/unmeasured sizes return null so no frame or input overlay + * receives non-finite coordinates. Fit preserves containedRect's exact geometry. + */ +export function getBrowserCanvasLayout( + containerSize: FrameSize, + framePixelSize: FrameSize, + viewportCssSize: FrameSize, + mode: BrowserCanvasDisplayMode = "fit", +): BrowserCanvasLayout | null { + if (!validSize(containerSize) || !validSize(framePixelSize)) return null; + + if (mode === "fit") { + const frameRect = containedRect(containerSize, framePixelSize); + return frameRect ? { contentSize: { ...containerSize }, frameRect } : null; + } + + if (!validSize(viewportCssSize)) return null; + return { + contentSize: { ...viewportCssSize }, + frameRect: { + x: 0, + y: 0, + width: viewportCssSize.width, + height: viewportCssSize.height, + }, + }; +} + +function validSize(size: FrameSize): boolean { + return ( + Number.isFinite(size.width) && size.width > 0 && Number.isFinite(size.height) && size.height > 0 + ); +} diff --git a/paseo-shared-browser/client/browser-canvas-viewport-web.test.ts b/paseo-shared-browser/client/browser-canvas-viewport-web.test.ts new file mode 100644 index 00000000..9f286490 --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-viewport-web.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { configureBrowserCanvasWebScrolling } from "./browser-canvas-viewport-web"; + +describe("local web browser viewport overflow", () => { + it("enables both overflow axes on the same public scroll node, preserving Actual offsets", () => { + const node = {style:{overflowX:"hidden",overflowY:"auto"},scrollLeft:300,scrollTop:150}; + configureBrowserCanvasWebScrolling({getScrollableNode:()=>node},"actual",true); + expect(node).toEqual({style:{overflowX:"auto",overflowY:"auto"},scrollLeft:300,scrollTop:150}); + }); + + it("hides overflow and resets both local offsets when returning to Fit", () => { + const node = {style:{overflowX:"auto",overflowY:"auto"},scrollLeft:300,scrollTop:150}; + configureBrowserCanvasWebScrolling({getScrollableNode:()=>node},"fit",true); + expect(node).toEqual({style:{overflowX:"hidden",overflowY:"hidden"},scrollLeft:0,scrollTop:0}); + }); + + it("retains an Actual observation position when local panning is disabled", () => { + const node = {style:{overflowX:"auto",overflowY:"auto"},scrollLeft:300,scrollTop:150}; + configureBrowserCanvasWebScrolling({getScrollableNode:()=>node},"actual",false); + expect(node).toEqual({style:{overflowX:"hidden",overflowY:"hidden"},scrollLeft:300,scrollTop:150}); + }); + + it("uses the documented native host fallback while preserving the handle receiver", () => { + const node = {style:{overflowX:"",overflowY:""},scrollLeft:0,scrollTop:0}; + const handle = {node,getScrollableNode:()=>null,getNativeScrollRef(){return this.node;}}; + configureBrowserCanvasWebScrolling(handle,"actual",true); + expect(node.style).toEqual({overflowX:"auto",overflowY:"auto"}); + }); + + it("ignores missing, disappearing and non-DOM platform refs without changing unrelated nodes", () => { + expect(()=>configureBrowserCanvasWebScrolling(null,"actual",true)).not.toThrow(); + expect(()=>configureBrowserCanvasWebScrolling({getScrollableNode:()=>42},"actual",true)).not.toThrow(); + expect(()=>configureBrowserCanvasWebScrolling({getScrollableNode:()=>{throw new Error("Detached");}},"actual",true)).not.toThrow(); + }); +}); diff --git a/paseo-shared-browser/client/browser-canvas-viewport-web.ts b/paseo-shared-browser/client/browser-canvas-viewport-web.ts new file mode 100644 index 00000000..8c74d98e --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-viewport-web.ts @@ -0,0 +1,58 @@ +/** + * Web-only overflow adapter for the public React Native ScrollView host. A single + * DOM scroll node keeps both native desktop scrollbars at the viewer's edges. + * Only local overflow/offsets are changed; child remote-input listeners are untouched. + */ +import type { BrowserCanvasDisplayMode } from "./browser-canvas-layout"; + +interface WebScrollNode { + style: { overflowX: string; overflowY: string }; + scrollLeft: number; + scrollTop: number; +} + +export interface BrowserCanvasScrollHandle { + getScrollableNode?(): unknown; + getNativeScrollRef?(): unknown; +} + +/** + * Apply public ScrollView's second overflow axis before paint. Fit resets both + * offsets without replacing children. An unavailable/non-web host is a no-op. + * Actual disabled panning retains its previous observation offset. + */ +export function configureBrowserCanvasWebScrolling( + handle: BrowserCanvasScrollHandle | null, + mode: BrowserCanvasDisplayMode, + localPanEnabled: boolean, +): void { + if (!handle) return; + const node = readScrollNode(handle); + if (!node) return; + + const overflow = mode === "actual" && localPanEnabled ? "auto" : "hidden"; + node.style.overflowX = overflow; + node.style.overflowY = overflow; + if (mode === "fit") { + node.scrollLeft = 0; + node.scrollTop = 0; + } +} + +/** Prefer the documented scrollable host; tolerate platform refs that expose only the native scroll ref. */ +function readScrollNode(handle: BrowserCanvasScrollHandle): WebScrollNode | null { + for (const read of [handle.getScrollableNode, handle.getNativeScrollRef]) { + if (!read) continue; + try { + const candidate: unknown = read.call(handle); + if (!candidate || typeof candidate !== "object") continue; + const node = candidate as Partial; + if (!node.style || typeof node.style.overflowX !== "string" || typeof node.style.overflowY !== "string") continue; + if (typeof node.scrollLeft !== "number" || typeof node.scrollTop !== "number") continue; + return node as WebScrollNode; + } catch { + // A disappearing host must not break frame rendering or input cleanup. + } + } + return null; +} diff --git a/paseo-shared-browser/client/browser-canvas-viewport.tsx b/paseo-shared-browser/client/browser-canvas-viewport.tsx new file mode 100644 index 00000000..1452d790 --- /dev/null +++ b/paseo-shared-browser/client/browser-canvas-viewport.tsx @@ -0,0 +1,152 @@ +/** + * Stable local scroll container for browser frames and their input overlay. + * Wrappers and children never change identity when Fit/Actual changes, preserving + * decoded frame slots and refs. Only Actual enables local scrolling. The parent + * cancels active remote input before changing geometry and owns overlay authority. + */ +import { type ReactNode, type Ref, useLayoutEffect, useRef } from "react"; +import { + type LayoutChangeEvent, + Platform, + ScrollView, + type StyleProp, + StyleSheet, + View, + type ViewStyle, +} from "react-native"; +import type { FrameSize } from "../shared/browser"; +import type { BrowserCanvasDisplayMode } from "./browser-canvas-layout"; +import { configureBrowserCanvasWebScrolling } from "./browser-canvas-viewport-web"; + +export interface BrowserCanvasViewportProps { + children: ReactNode; + /** Frame wrapper from getBrowserCanvasLayout, measured in local logical pixels. */ + contentSize: FrameSize; + mode?: BrowserCanvasDisplayMode; + /** Desktop permits local scrollbars; native observation permits pan, remote control does not. */ + localPanEnabled: boolean; + onLayout(event: LayoutChangeEvent): void; + style?: StyleProp; + viewportRef?: Ref; + contentRef?: Ref; +} + +const styles = StyleSheet.create({ + viewport: { flex: 1, minWidth: 0, minHeight: 0, overflow: "hidden" }, + scroll: { flex: 1 }, + horizontalContent: { height: "100%" }, + vertical: { height: "100%" }, + content: { position: "relative" }, +}); + +/** Keep the frame coordinate origin inside the scrolling content, rather than the clipped viewport. */ +export function BrowserCanvasViewport({ + children, + contentSize, + mode = "fit", + localPanEnabled, + onLayout, + style, + viewportRef, + contentRef, +}: BrowserCanvasViewportProps) { + const horizontalRef = useRef(null); + const verticalRef = useRef(null); + const actual = mode === "actual"; + const scrollEnabled = actual && localPanEnabled; + const web = Platform.OS === "web"; + // Center a fitting Actual frame in the scrollport, not a spacer sized from + // the outer pane. A scrollbar on one axis must not create overflow on the + // other. Minimums keep overflowing frames reachable from origin zero. + const actualContent = { + flexGrow: 1, + alignItems: "center" as const, + justifyContent: "center" as const, + }; + + useLayoutEffect(() => { + if (web) { + configureBrowserCanvasWebScrolling(horizontalRef.current, mode, localPanEnabled); + return; + } + if (actual) return; + // Reset local offsets without remounting either frame-buffer layer. A hidden + // Actual offset must not shift Fit's overlay away from its displayed pixels. + horizontalRef.current?.scrollTo({ x: 0, y: 0, animated: false }); + verticalRef.current?.scrollTo({ x: 0, y: 0, animated: false }); + }, [actual, mode, localPanEnabled, web, contentSize.width, contentSize.height]); + + const content = ( + + {children} + + ); + + if (web) { + return ( + + + {content} + + + ); + } + + return ( + + + + {content} + + + + ); +} diff --git a/paseo-shared-browser/client/browser-capture-cadence.test.ts b/paseo-shared-browser/client/browser-capture-cadence.test.ts new file mode 100644 index 00000000..628723ce --- /dev/null +++ b/paseo-shared-browser/client/browser-capture-cadence.test.ts @@ -0,0 +1,8 @@ +import { expect, it } from "vitest"; +import { browserCaptureInterval } from "./browser-capture-cadence"; +it("keeps idle/hover polling bounded and only accelerates actual ready input", () => { + expect(browserCaptureInterval("ready", false)).toBe(250); + expect(browserCaptureInterval("ready", true)).toBe(100); + expect(browserCaptureInterval("starting", true)).toBe(1_500); + expect(browserCaptureInterval(undefined, false)).toBe(1_500); +}); diff --git a/paseo-shared-browser/client/browser-capture-cadence.ts b/paseo-shared-browser/client/browser-capture-cadence.ts new file mode 100644 index 00000000..8403c697 --- /dev/null +++ b/paseo-shared-browser/client/browser-capture-cadence.ts @@ -0,0 +1,9 @@ +/** Single-flight capture cadence; active input improves playback without changing image quality. */ +export function browserCaptureInterval( + status: string | undefined, + activeInput: boolean, +): number { + if (status !== "ready") return 1_500; + if (activeInput) return 100; + return 250; +} diff --git a/paseo-shared-browser/client/browser-emulation-mode.test.ts b/paseo-shared-browser/client/browser-emulation-mode.test.ts new file mode 100644 index 00000000..c48b4653 --- /dev/null +++ b/paseo-shared-browser/client/browser-emulation-mode.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { + createEmulationChoices, + localEmulationMode, + matchingResolutionPresetId, + selectionMode, +} from "./browser-emulation-mode"; + +describe("device defaults for shared emulation", () => { + it("uses actual platform/browser device hints, not narrow layout", () => { + expect(localEmulationMode({ platform: "web", userAgent: "Desktop Chrome" })).toBe("desktop"); + expect(localEmulationMode({ platform: "web", userAgent: "Android Mobile Chrome" })).toBe( + "mobile", + ); + expect(localEmulationMode({ platform: "web", mobile: true })).toBe("mobile"); + expect( + localEmulationMode({ platform: "web", userAgent: "Macintosh Safari", maxTouchPoints: 5 }), + ).toBe("mobile"); + expect( + localEmulationMode({ platform: "web", userAgent: "Windows Chrome", maxTouchPoints: 10 }), + ).toBe("desktop"); + expect(localEmulationMode({ platform: "ios" })).toBe("mobile"); + expect(localEmulationMode({ platform: "android" })).toBe("mobile"); + }); + it("applies a different local mode once, then honors explicit choices across reacquisition", () => { + const choices = createEmulationChoices({ platform: "ios" }); + expect(choices.claimDefault("desktop")).toEqual({ + presetId: "iphone-15-pro", + preserveDisplay: true, + }); + expect(choices.claimDefault("desktop")).toBeNull(); + }); + it("keeps an already matching mode and never replays a remembered viewport on toggle", () => { + const choices = createEmulationChoices({ platform: "web" }); + choices.remember(null); + expect(choices.claimDefault("desktop")).toBeNull(); + expect(choices.toggle("desktop")).toEqual({ presetId: "pixel-7-sharp", preserveDisplay: true }); + expect(choices.toggle("mobile")).toEqual({ + presetId: "desktop-chrome", + preserveDisplay: true, + }); + expect(selectionMode(null)).toBe("desktop"); + expect(selectionMode("pixel-7")).toBe("mobile"); + }); + it("remembers behavior profiles without claiming their native dimensions", () => { + const choices = createEmulationChoices({ platform: "web" }); + choices.remember("desktop-1920x1080"); + choices.remember("iphone-15-pro"); + expect(choices.toggle("desktop")).toEqual({ presetId: "iphone-15-pro", preserveDisplay: true }); + expect(choices.toggle("mobile")).toEqual({ + presetId: "desktop-1920x1080", + preserveDisplay: true, + }); + expect( + matchingResolutionPresetId({ + devicePresetId: "iphone-15-pro", + viewport: { width: 1920, height: 1080 }, + captureScale: 1, + }), + ).toBeNull(); + }); + it("checks exact viewport and capture density in resolution selection", () => { + const base = { + devicePresetId: "pixel-7-sharp" as const, + viewport: { width: 412, height: 839 }, + captureScale: 2, + }; + expect(matchingResolutionPresetId(base)).toBe("pixel-7-sharp"); + expect(matchingResolutionPresetId({ ...base, captureScale: 1 })).toBeNull(); + expect( + matchingResolutionPresetId({ ...base, viewport: { width: 1920, height: 1200 } }), + ).toBeNull(); + expect(matchingResolutionPresetId(null)).toBeNull(); + }); +}); diff --git a/paseo-shared-browser/client/browser-emulation-mode.ts b/paseo-shared-browser/client/browser-emulation-mode.ts new file mode 100644 index 00000000..5f3fc82a --- /dev/null +++ b/paseo-shared-browser/client/browser-emulation-mode.ts @@ -0,0 +1,98 @@ +/** Local mode defaults and remembered behavior profiles, independent of shared display size. */ + +import type { BrowserState } from "../shared/browser"; +import { DEVICE_PRESETS, type DevicePresetId } from "../shared/device-presets"; + +export type BrowserEmulationMode = "desktop" | "mobile"; +export type EmulationSelection = { presetId: DevicePresetId; preserveDisplay: true }; +interface LocalDevice { + platform: "ios" | "android" | "web"; + userAgent?: string; + mobile?: boolean; + maxTouchPoints?: number; +} + +/** Compact layout can mean a narrow desktop window, so it never determines device identity. */ +export function localEmulationMode(device: LocalDevice): BrowserEmulationMode { + if (device.platform !== "web") return "mobile"; + if ( + device.mobile === true || + (/Macintosh/i.test(device.userAgent ?? "") && (device.maxTouchPoints ?? 0) > 1) || + /Android|iPhone|iPad|iPod|Mobile/i.test(device.userAgent ?? "") + ) + return "mobile"; + return "desktop"; +} + +/** Browser hints are local presentation facts, not remote browser or authentication data. */ +export function readLocalDevice(platform: LocalDevice["platform"]): LocalDevice { + const browser = globalThis as unknown as { + navigator?: { + userAgent?: string; + maxTouchPoints?: number; + userAgentData?: { mobile?: boolean }; + }; + }; + const userAgent = browser.navigator?.userAgent; + const mobile = browser.navigator?.userAgentData?.mobile; + const maxTouchPoints = browser.navigator?.maxTouchPoints; + return { + platform, + ...(userAgent === undefined ? {} : { userAgent }), + ...(mobile === undefined ? {} : { mobile }), + ...(maxTouchPoints === undefined ? {} : { maxTouchPoints }), + }; +} + +/** The configured profile defines behavior even when its native dimensions are overridden. */ +export function selectionMode(presetId: DevicePresetId | null): BrowserEmulationMode { + return DEVICE_PRESETS.find((preset) => preset.id === presetId)?.isMobile ? "mobile" : "desktop"; +} + +/** A profile ID alone must not check a resolution that differs from the actual display. */ +export function matchingResolutionPresetId( + state: Pick | null, +): DevicePresetId | null { + if (!state) return null; + const preset = DEVICE_PRESETS.find((item) => item.id === state.devicePresetId); + return preset && + preset.viewport.width === state.viewport.width && + preset.viewport.height === state.viewport.height && + preset.captureScale === state.captureScale + ? preset.id + : null; +} + +/** One device default per mounted host/workspace; later explicit choices survive releasing control. */ +export function createEmulationChoices(device: LocalDevice) { + const defaultMode = localEmulationMode(device); + const mobilePreset: DevicePresetId = + device.platform === "ios" || + /iPhone|iPad|iPod/i.test(device.userAgent ?? "") || + (/Macintosh/i.test(device.userAgent ?? "") && (device.maxTouchPoints ?? 0) > 1) + ? "iphone-15-pro" + : "pixel-7-sharp"; + let initialized = false; + const profiles: Record = { + desktop: "desktop-chrome", + mobile: mobilePreset, + }; + return { + remember(presetId: DevicePresetId | null) { + if (presetId) profiles[selectionMode(presetId)] = presetId; + }, + claimDefault(currentMode: BrowserEmulationMode): EmulationSelection | null { + if (initialized) return null; + initialized = true; + return currentMode === defaultMode + ? null + : { presetId: profiles[defaultMode], preserveDisplay: true }; + }, + toggle(currentMode: BrowserEmulationMode): EmulationSelection { + return { + presetId: profiles[currentMode === "mobile" ? "desktop" : "mobile"], + preserveDisplay: true, + }; + }, + }; +} diff --git a/paseo-shared-browser/client/browser-frame-buffer.test.ts b/paseo-shared-browser/client/browser-frame-buffer.test.ts new file mode 100644 index 00000000..d6987c6f --- /dev/null +++ b/paseo-shared-browser/client/browser-frame-buffer.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from "vitest"; +import { createBrowserFrameBuffer, type BrowserFrameBuffer, type FrameCandidate } from "./browser-frame-buffer"; + +function candidate(id: string, pixels = id, generation = 0): FrameCandidate { + return { + viewerToken: "viewer-a", + mutationEpoch: 0, + frame: { + sessionId: "session-a", + frameId: id, + mimeType: "image/jpeg", + transport: "screenshot", + dataBase64: pixels, + byteLength: pixels.length, + width: 1280, + height: 800, + navigationGeneration: generation, + viewportGeneration: 0, + runtimeId: "runtime-a", + capturedAt: "2026-10-02T12:00:00.000Z", + }, + }; +} +function pendingTicket(buffer: BrowserFrameBuffer): number { + const state = buffer.snapshot(); + if (state.pending === null) throw new Error("No pending decoder"); + return state.layers[state.pending]!.ticket; +} +function visible(buffer: BrowserFrameBuffer) { + const state = buffer.snapshot(); + return state.front === null ? null : state.layers[state.front]; +} +function decodeFirst(buffer: BrowserFrameBuffer) { + buffer.offer(candidate("front")); + buffer.settle(pendingTicket(buffer), true, () => true); +} + +describe("decoded Shared Browser frame handoff", () => { + it("keeps initial capture hidden until successful decode, then retains it during replacement", () => { + const buffer = createBrowserFrameBuffer(); + buffer.offer(candidate("front")); + expect(visible(buffer)).toBeNull(); + buffer.settle(pendingTicket(buffer), true, () => true); + const original = visible(buffer); + buffer.offer(candidate("next")); + expect(visible(buffer)).toBe(original); + expect(visible(buffer)?.candidate.frame.frameId).toBe("front"); + buffer.settle(pendingTicket(buffer), true, () => true); + expect(visible(buffer)?.candidate.frame.frameId).toBe("next"); + }); + + it("retains the decoded front and input frame after a failed replacement", () => { + const buffer = createBrowserFrameBuffer(); + decodeFirst(buffer); + const original = visible(buffer); + buffer.offer(candidate("failed")); + buffer.settle(pendingTicket(buffer), false, () => true); + expect(visible(buffer)).toBe(original); + expect(buffer.snapshot().imageError).toBe(false); + buffer.offer(candidate("retry")); + buffer.settle(pendingTicket(buffer), true, () => true); + expect(visible(buffer)?.candidate.frame.frameId).toBe("retry"); + }); + + it("shows the established initial error only when no decoded front exists", () => { + const buffer = createBrowserFrameBuffer(); + buffer.offer(candidate("bad-first")); + buffer.settle(pendingTicket(buffer), false, () => true); + expect(buffer.snapshot().imageError).toBe(true); + expect(visible(buffer)).toBeNull(); + }); + + it("advances identical-pixel frame authority without replacing its decoded source", () => { + const buffer = createBrowserFrameBuffer(); + decodeFirst(buffer); + const source = visible(buffer)?.source; + const ticket = visible(buffer)?.ticket; + buffer.offer(candidate("new-authority", "front", 1)); + expect(visible(buffer)?.candidate.frame.frameId).toBe("new-authority"); + expect(visible(buffer)?.candidate.frame.navigationGeneration).toBe(1); + expect(visible(buffer)?.source).toBe(source); + expect(visible(buffer)?.ticket).toBe(ticket); + expect(buffer.snapshot().pending).toBeNull(); + }); + + it("coalesces fast captures without cancelling the slow decoder or accumulating images", () => { + const buffer = createBrowserFrameBuffer(); + decodeFirst(buffer); + buffer.offer(candidate("slow")); + const ticket = pendingTicket(buffer); + for (let i = 0; i < 100; i++) buffer.offer(candidate(`queued-${i}`)); + expect(pendingTicket(buffer)).toBe(ticket); + expect(buffer.snapshot().layers.filter(Boolean)).toHaveLength(2); + expect(buffer.snapshot().queued?.frame.frameId).toBe("queued-99"); + buffer.settle(ticket, true, () => true); + expect(visible(buffer)?.candidate.frame.frameId).toBe("slow"); + buffer.settle(pendingTicket(buffer), true, () => true); + expect(visible(buffer)?.candidate.frame.frameId).toBe("queued-99"); + }); + + it("ignores late callbacks when the same native slot was reused for a newer generation", () => { + const buffer = createBrowserFrameBuffer(); + decodeFirst(buffer); + buffer.offer(candidate("obsolete")); + const oldTicket = pendingTicket(buffer); + buffer.invalidatePending(); + buffer.offer(candidate("current", "current", 1)); + const newTicket = pendingTicket(buffer); + buffer.settle(oldTicket, true, () => true); + buffer.settle(oldTicket, false, () => true); + expect(pendingTicket(buffer)).toBe(newTicket); + expect(visible(buffer)?.candidate.frame.frameId).toBe("front"); + buffer.settle(newTicket, true, () => true); + expect(visible(buffer)?.candidate.frame.frameId).toBe("current"); + }); + + it("never promotes a decoded frame whose viewer/runtime/generation authority expired", () => { + const buffer = createBrowserFrameBuffer(); + decodeFirst(buffer); + buffer.offer(candidate("obsolete")); + buffer.offer({ ...candidate("fresh"), viewerToken: "viewer-b" }); + const isCurrent = (item: FrameCandidate) => item.viewerToken === "viewer-b"; + buffer.settle(pendingTicket(buffer), true, isCurrent); + expect(visible(buffer)?.candidate.frame.frameId).toBe("front"); + buffer.settle(pendingTicket(buffer), true, isCurrent); + expect(visible(buffer)?.candidate.frame.frameId).toBe("fresh"); + }); + + it("clears workspace pixels and prevents ticket replay after reset", () => { + const buffer = createBrowserFrameBuffer(); + decodeFirst(buffer); + buffer.offer(candidate("old-workspace")); + const oldTicket = pendingTicket(buffer); + buffer.reset(); + expect(visible(buffer)).toBeNull(); + buffer.offer(candidate("new-workspace")); + const fresh = buffer.snapshot(); + buffer.settle(oldTicket, true, () => true); + expect(buffer.snapshot()).toBe(fresh); + buffer.settle(pendingTicket(buffer), true, () => true); + expect(visible(buffer)?.candidate.frame.frameId).toBe("new-workspace"); + }); + + it("a freshest capture identical to the front cancels obsolete pending promotion", () => { + const buffer = createBrowserFrameBuffer(); + decodeFirst(buffer); + buffer.offer(candidate("transient")); + const oldTicket = pendingTicket(buffer); + buffer.offer(candidate("back-to-front", "front")); + buffer.settle(oldTicket, true, () => true); + expect(visible(buffer)?.candidate.frame.frameId).toBe("back-to-front"); + }); +}); diff --git a/paseo-shared-browser/client/browser-frame-buffer.ts b/paseo-shared-browser/client/browser-frame-buffer.ts new file mode 100644 index 00000000..6eea840a --- /dev/null +++ b/paseo-shared-browser/client/browser-frame-buffer.ts @@ -0,0 +1,146 @@ +/** + * Owns decoded browser-frame handoff independently of transport capture cadence. + * Two stable image slots retain the decoded front while one replacement loads; + * one newest queued capture bounds memory without starving a slow decoder. + * Frame IDs are opaque. A decode ticket, not capture timestamps, rejects late + * native callbacks. Callers qualify session/generation ownership separately. + */ +import type { BrowserFrame } from "../shared/browser"; + +export type FrameCandidate = { + frame: BrowserFrame; + /** Canonical CSS layout captured with these pixels, independent of capture density. */ + viewport?: { width: number; height: number }; + viewerToken: string; + mutationEpoch: number; +}; +export type FrameLayer = { + ticket: number; + candidate: FrameCandidate; + source: { uri: string }; +}; +export type FrameBufferSnapshot = { + layers: readonly [FrameLayer | null, FrameLayer | null]; + front: 0 | 1 | null; + pending: 0 | 1 | null; + queued: FrameCandidate | null; + imageError: boolean; +}; + +/** Exact JPEG reuse avoids reloading an already decoded image for a fresh frame ID. */ +function samePixels(a: BrowserFrame, b: BrowserFrame): boolean { + return ( + a.mimeType === b.mimeType && + a.width === b.width && + a.height === b.height && + a.dataBase64 === b.dataBase64 + ); +} + +/** + * Create a bounded handoff model. Use plain closures rather than a class because + * the client is evaluated dynamically across Paseo hosts, outside Metro's normal + * class lowering. Visible input authority is published only after React commits. + */ +export function createBrowserFrameBuffer() { + let nextTicket = 0; + let value: FrameBufferSnapshot = { + layers: [null, null], + front: null, + pending: null, + queued: null, + imageError: false, + }; + + function snapshot(): FrameBufferSnapshot { + return value; + } + + /** Replace an obsolete pending decode without discarding the last decoded front. */ + function invalidatePending(): FrameBufferSnapshot { + const layers: [FrameLayer | null, FrameLayer | null] = [...value.layers]; + if (value.pending !== null) { + layers[value.pending] = null; + } + value = { ...value, layers, pending: null, queued: null }; + return value; + } + + /** Clear workspace-owned pixels. Ticket numbers never reset, including across workspaces. */ + function reset(): FrameBufferSnapshot { + value = { + layers: [null, null], + front: null, + pending: null, + queued: null, + imageError: false, + }; + return value; + } + + /** Offer a qualified capture; newest queued metadata replaces earlier queued captures. */ + function offer(candidate: FrameCandidate): FrameBufferSnapshot { + const front = value.front; + const frontLayer = front === null ? null : value.layers[front]; + if (front !== null && frontLayer && samePixels(frontLayer.candidate.frame, candidate.frame)) { + invalidatePending(); + const layers: [FrameLayer | null, FrameLayer | null] = [...value.layers]; + layers[front] = { ...frontLayer, candidate }; + value = { ...value, layers, imageError: false }; + return value; + } + + if (value.pending !== null) { + value = { ...value, queued: candidate }; + return value; + } + + const slot = front === 0 ? 1 : 0; + const layers: [FrameLayer | null, FrameLayer | null] = [...value.layers]; + layers[slot] = { + ticket: ++nextTicket, + candidate, + source: { uri: `data:${candidate.frame.mimeType};base64,${candidate.frame.dataBase64}` }, + }; + value = { ...value, layers, pending: slot, queued: null, imageError: false }; + return value; + } + + /** + * Accept only this pending image's successful load. Failure or expired authority + * never removes the front. Only a still-qualified newest queued capture is loaded. + */ + function settle( + ticket: number, + succeeded: boolean, + isCurrent: (candidate: FrameCandidate) => boolean, + ): FrameBufferSnapshot { + const pending = value.pending; + const layer = pending === null ? null : value.layers[pending]; + if (pending === null || !layer || layer.ticket !== ticket) { + return value; + } + + const queued = value.queued; + const accepted = succeeded && isCurrent(layer.candidate); + const layers: [FrameLayer | null, FrameLayer | null] = [...value.layers]; + if (!accepted) { + layers[pending] = null; + } + value = { + ...value, + layers, + front: accepted ? pending : value.front, + pending: null, + queued: null, + imageError: !succeeded && value.front === null && isCurrent(layer.candidate), + }; + if (queued && isCurrent(queued)) { + offer(queued); + } + return value; + } + return { snapshot, invalidatePending, reset, offer, settle }; +} + +export type BrowserFrameBuffer = ReturnType; diff --git a/paseo-shared-browser/client/browser-frame-image.tsx b/paseo-shared-browser/client/browser-frame-image.tsx new file mode 100644 index 00000000..a7588cd8 --- /dev/null +++ b/paseo-shared-browser/client/browser-frame-image.tsx @@ -0,0 +1,62 @@ +/** + * Owns one visual JPEG/decode slot. Input authority belongs to the frame-buffer + * hook, not this image. Metadata-only captures and mouse state must not rerender + * the native image or recreate its load callbacks when its pixels are unchanged. + */ +import { memo, useCallback, useMemo } from "react"; +import { Image, StyleSheet } from "react-native"; +import { createBrowserImageLoadHandler } from "./browser-image-load"; + +interface BrowserFrameImageProps { + ticket: number; + source: { uri: string }; + visible: boolean; + label: string; + x: number; + y: number; + width: number; + height: number; + cornerRadius: number; + settled(ticket: number, succeeded: boolean): boolean; + retry(): void; +} + +const styles = StyleSheet.create({ frame: { position: "absolute" } }); + +/** Memoize visual values only; newer input frame IDs still commit through the parent hook. */ +export const BrowserFrameImage = memo(function BrowserFrameImage(props: BrowserFrameImageProps) { + const { ticket, source, settled, retry } = props; + const onLoad = useMemo( + () => createBrowserImageLoadHandler(ticket, source.uri, settled), + [ticket, source.uri, settled], + ); + const onError = useCallback(() => { + if (settled(ticket, false)) { + retry(); + } + }, [ticket, settled, retry]); + + return ( + + ); +}); diff --git a/paseo-shared-browser/client/browser-image-load.test.ts b/paseo-shared-browser/client/browser-image-load.test.ts new file mode 100644 index 00000000..c1138cb2 --- /dev/null +++ b/paseo-shared-browser/client/browser-image-load.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { createBrowserImageLoadHandler } from "./browser-image-load"; + +describe("cross-platform browser image decode", () => { + it("accepts the desktop DOM load envelope without inventing a reported URI", () => { + const delivered: unknown[] = []; + const handler = createBrowserImageLoadHandler(1, "data:image/jpeg;base64,frame", (...args) => delivered.push(args)); + expect(() => handler({ nativeEvent: new Event("load") })).not.toThrow(); + expect(delivered).toEqual([[1, true]]); + }); + + it("accepts the native exact source and refuses a contradictory source", () => { + const delivered: unknown[] = []; + const handler = createBrowserImageLoadHandler(7, "expected", (...args) => delivered.push(args)); + handler({ nativeEvent: { source: { uri: "other" } } }); + handler({ nativeEvent: { source: { uri: 42 } } }); + expect(delivered).toEqual([]); + handler({ nativeEvent: { source: { uri: "expected" } } }); + expect(delivered).toEqual([[7, true]]); + }); +}); diff --git a/paseo-shared-browser/client/browser-image-load.ts b/paseo-shared-browser/client/browser-image-load.ts new file mode 100644 index 00000000..3df64277 --- /dev/null +++ b/paseo-shared-browser/client/browser-image-load.ts @@ -0,0 +1,24 @@ +/** Qualify native and web image-load events before the current decode-ticket handoff. */ + +/** + * Native Image may report its loaded source URI; React Native Web instead wraps + * a DOM load Event without that field. The unique decoder ticket remains the + * authority when no URI is reported. A contradictory reported URI is rejected. + * The settlement callback must still check current viewer, mutation and browser + * generations before committing pixels or input authority. + */ +export function createBrowserImageLoadHandler( + ticket: number, + expectedUri: string, + settle: (ticket: number, succeeded: boolean) => unknown, +): (event: unknown) => void { + return (event) => { + const nativeEvent = event && typeof event === "object" && "nativeEvent" in event + ? event.nativeEvent : undefined; + const source = nativeEvent && typeof nativeEvent === "object" && "source" in nativeEvent + ? nativeEvent.source : undefined; + const uri = source && typeof source === "object" && "uri" in source ? source.uri : undefined; + if (uri && uri !== expectedUri) return; + settle(ticket, true); + }; +} diff --git a/paseo-shared-browser/client/browser-input-queue.test.ts b/paseo-shared-browser/client/browser-input-queue.test.ts new file mode 100644 index 00000000..fd0487f0 --- /dev/null +++ b/paseo-shared-browser/client/browser-input-queue.test.ts @@ -0,0 +1,577 @@ +import { describe, expect, it, vi } from "vitest"; +import { + createBrowserInputQueue, + type BrowserGestureAuthority, + type BrowserGestureTransport, +} from "./browser-input-queue"; +import type { BrowserGestureEvent, BrowserState } from "../shared/browser"; + +const authority: BrowserGestureAuthority = { + viewerToken: "viewer", + controlToken: "control", + expected: { + sessionId: "session", + runtimeId: "runtime", + bridgeEpoch: 1, + navigationGeneration: 1, + viewportGeneration: 1, + }, + target: { + frameId: "decoded-front", + navigationGeneration: 1, + viewportGeneration: 1, + }, +}; +const state = { captureScale: 1 } as BrowserState; +const point = (x: number) => ({ x, y: 1, width: 800, height: 600 }); +const move = (x: number): BrowserGestureEvent => ({ + kind: "move", + point: point(x), +}); +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: Error) => void; + return { + promise: new Promise((yes, no) => { + resolve = yes; + reject = no; + }), + resolve, + reject, + }; +} +async function flush() { + for (let i = 0; i < 12; i++) await Promise.resolve(); +} +function fixture(waitForFrame?: (afterFrameId: string, maxWaitMs?: number) => Promise) { + let current: BrowserGestureAuthority | null = authority; + const sent: Parameters[0][] = []; + const ended: Parameters[0][] = []; + const errors: unknown[] = []; + const cursors: unknown[] = []; + const navigationCompleted: boolean[] = []; + const states: BrowserState[] = []; + let projectState: ((state: BrowserState) => void) | null = null; + let finished = 0; + const beginGate = + deferred>>(); + const transport: BrowserGestureTransport = { + begin: async () => await beginGate.promise, + update: async (input) => { + sent.push(input); + return { + state, + gestureId: input.gestureId, + nextSequence: input.sequence + 1, + cursor: "pointer", + }; + }, + end: async (input) => { + ended.push(input); + return { state, cursor: null }; + }, + }; + const queue = createBrowserInputQueue({ + transport, + authority: () => current, + onState: next => { states.push(next); projectState?.(next); }, + onCursor: (cursor) => cursors.push(cursor), + onError: (error) => errors.push(error), + onFinish: () => { finished += 1; }, + onNavigationComplete: () => navigationCompleted.push(true), + ...(waitForFrame ? { waitForFrame } : {}), + }); + return { + queue, + sent, + ended, + errors, + cursors, + beginGate, + transport, + navigationCompleted, + states, + finishes: () => finished, + projectState(callback: (state: BrowserState) => void) { projectState = callback; }, + changeAuthority(next: BrowserGestureAuthority | null) { + current = next; + }, + }; +} + +describe("bounded browser input queue", () => { + it("completes acknowledged Enter navigation before state projection revokes the decoded frame", async () => { + const f = fixture(); + const next: BrowserState = { + ...state, status: "ready", controller: "self", sessionId: "session", runtimeId: "runtime", + bridgeEpoch: 1, viewportGeneration: 1, navigationGeneration: 2, + }; + f.projectState(reply => { if (reply.navigationGeneration === 2) f.changeAuthority(null); }); + f.transport.update = async input => { + f.sent.push(input); + return { state: next, gestureId: input.gestureId, nextSequence: input.sequence + 1, + cursor: null, completion: "navigation" }; + }; + f.queue.enqueue({ kind: "key", type: "down", key: "Enter", code: "Enter", modifiers: 0, repeat: false }); + f.queue.enqueue({ kind: "key", type: "up", key: "Enter", code: "Enter", modifiers: 0, repeat: false }); + f.queue.enqueue({ kind: "text", text: "never type on the destination" }); + f.queue.finish(); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(f.sent).toHaveLength(1); + expect(f.errors).toEqual([]); + expect(f.ended).toEqual([]); + expect(f.navigationCompleted).toEqual([true]); + expect(f.finishes()).toBe(1); + expect(f.cursors.at(-1)).toBeNull(); + }); + + it("keeps the acknowledged click release but drops later old-page wheel and end", async () => { + const f = fixture(); + f.transport.update = async input => { + f.sent.push(input); + if (input.event.kind !== "up") { + return { state, gestureId: input.gestureId, nextSequence: input.sequence + 1, cursor: null }; + } + return { state: { ...state, status: "ready", controller: "self", sessionId: "session", runtimeId: "runtime", + bridgeEpoch: 1, viewportGeneration: 1, navigationGeneration: 2 }, + gestureId: input.gestureId, nextSequence: input.sequence + 1, cursor: null, completion: "navigation" }; + }; + f.projectState(reply => { if (reply.navigationGeneration === 2) f.changeAuthority(null); }); + f.queue.enqueue({ kind: "down", point: point(1), button: "left", clickCount: 1 }); + f.queue.enqueue({ kind: "up", point: point(1), button: "left", clickCount: 1 }); + f.queue.enqueue({ kind: "scroll", point: point(1), deltaX: 0, deltaY: 10 }); + f.queue.finish(); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(f.sent.map(input => input.event.kind)).toEqual(["down", "up"]); + expect(f.errors).toEqual([]); + expect(f.navigationCompleted).toEqual([true]); + expect(f.ended).toEqual([]); + }); + + it("does not grant navigation completion for changed runtime, bridge, viewport, controller or sequence", async () => { + const next: BrowserState = { + ...state, status: "ready", controller: "self", sessionId: "session", runtimeId: "runtime", + bridgeEpoch: 1, viewportGeneration: 1, navigationGeneration: 2, + }; + for (const altered of [ + { sessionId: "another-session" }, { runtimeId: "another-runtime" }, { bridgeEpoch: 2 }, + { viewportGeneration: 2 }, { controller: "other" as const }, { navigationGeneration: 1 }, + { status: "error" as const }, + ]) { + const f = fixture(); + f.transport.update = async input => { + f.sent.push(input); + return { state: { ...next, ...altered }, gestureId: input.gestureId, + nextSequence: input.sequence + 1, cursor: null, completion: "navigation" }; + }; + f.queue.enqueue(move(1)); f.queue.enqueue(move(2)); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(f.navigationCompleted).toEqual([]); + expect(f.sent).toHaveLength(1); + expect(f.errors).toHaveLength(1); + expect(f.ended).toHaveLength(1); + } + const sequence = fixture(); + sequence.transport.update = async input => ({ state: next, gestureId: input.gestureId, + nextSequence: input.sequence + 2, cursor: null, completion: "navigation" }); + sequence.queue.enqueue(move(1)); + sequence.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(sequence.errors).toHaveLength(1); + expect(sequence.navigationCompleted).toEqual([]); + }); + + it("unmarked navigation and replacement control retain the existing context rejection", async () => { + const f = fixture(); + const next: BrowserState = { ...state, navigationGeneration: 2 }; + f.projectState(reply => { if (reply.navigationGeneration === 2) f.changeAuthority(null); }); + f.transport.update = async input => ({ state: next, gestureId: input.gestureId, + nextSequence: input.sequence + 1, cursor: null }); + f.queue.enqueue(move(1)); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(f.errors).toHaveLength(1); + expect(f.navigationCompleted).toEqual([]); + + const changed = fixture(); + changed.transport.update = async input => { + changed.changeAuthority({ ...authority, controlToken: "replacement" }); + return { state: { ...next, status: "ready", controller: "self", sessionId: "session", + runtimeId: "runtime", bridgeEpoch: 1, viewportGeneration: 1 }, + gestureId: input.gestureId, nextSequence: input.sequence + 1, cursor: null, completion: "navigation" }; + }; + changed.queue.enqueue(move(1)); + changed.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(changed.navigationCompleted).toEqual([]); + expect(changed.states).toEqual([state]); + expect(changed.ended).toHaveLength(1); + }); + it("coalesces only motion and wheel distance while keeping press/release sequence", async () => { + const f = fixture(); + f.queue.enqueue(move(1)); + for (let i = 2; i < 100; i++) f.queue.enqueue(move(i)); + f.queue.enqueue({ + kind: "down", + point: point(99), + button: "left", + clickCount: 1, + }); + f.queue.enqueue({ kind: "scroll", point: point(99), deltaX: 2, deltaY: 5 }); + f.queue.enqueue({ + kind: "scroll", + point: point(100), + deltaX: 3, + deltaY: 7, + }); + f.queue.enqueue({ + kind: "up", + point: point(100), + button: "left", + clickCount: 1, + }); + f.queue.finish(); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(f.sent.map((input) => input.event.kind)).toEqual([ + "move", + "move", + "down", + "scroll", + "up", + ]); + expect(f.sent.map((input) => input.sequence)).toEqual([1, 2, 3, 4, 5]); + expect(f.sent[2]?.target).toEqual(authority.target); + expect(f.sent[0]?.target).toBeUndefined(); + expect(f.sent[3]?.event).toMatchObject({ deltaX: 5, deltaY: 12 }); + expect(f.ended[0]).toMatchObject({ sequence: 6, cancel: false }); + expect(f.errors).toEqual([]); + }); + + it("keeps touch-ID edges through 60Hz motion and partial release", async () => { + const f = fixture(); + const touch = ( + type: "start" | "move" | "end", + ids: number[], + x: number, + ): BrowserGestureEvent => ({ + kind: "touch", + type, + points: ids.map((id) => ({ ...point(x + id), id })), + }); + f.queue.enqueue(touch("start", [0], 1)); + f.queue.enqueue(touch("start", [0, 1], 2)); + for (let i = 0; i < 180; i++) f.queue.enqueue(touch("move", [0, 1], i)); + f.queue.enqueue(touch("move", [1], 181)); + f.queue.enqueue(touch("end", [], 182)); + f.queue.finish(); + f.beginGate.resolve({ state, gestureId: "touch-channel", nextSequence: 1 }); + await flush(); + expect( + f.sent.map((input) => + input.event.kind === "touch" + ? [input.event.type, input.event.points.map((point) => point.id)] + : null, + ), + ).toEqual([ + ["start", [0]], + ["start", [0, 1]], + ["move", [0, 1]], + ["move", [1]], + ["end", []], + ]); + expect(f.sent[0]?.target).toEqual(authority.target); + expect(f.sent[1]?.target).toBeUndefined(); + expect(f.errors).toEqual([]); + }); + + it("revokes late begin replies on lease replacement and never sends queued presses", async () => { + const f = fixture(); + f.queue.enqueue({ + kind: "down", + point: point(1), + button: "left", + clickCount: 1, + }); + f.changeAuthority({ ...authority, controlToken: "replacement" }); + f.queue.cancel(); + f.beginGate.resolve({ state, gestureId: "obsolete", nextSequence: 1 }); + await flush(); + expect(f.sent).toEqual([]); + expect(f.ended[0]).toMatchObject({ gestureId: "obsolete", cancel: true }); + expect(f.cursors.filter(Boolean)).toEqual([]); + }); + + it("does not replay an unknown update and cancels with original identity", async () => { + const f = fixture(); + let attempts = 0; + f.transport.update = async () => { + attempts++; + throw new Error("Unknown outcome"); + }; + f.queue.enqueue({ + kind: "down", + point: point(1), + button: "left", + clickCount: 1, + }); + f.queue.enqueue(move(2)); + f.queue.enqueue({ + kind: "up", + point: point(2), + button: "left", + clickCount: 1, + }); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(attempts).toBe(1); + expect(f.ended[0]).toMatchObject({ gestureId: "owned", cancel: true }); + expect(f.errors).toHaveLength(1); + }); + + it("holds at one in-flight update and never publishes a late cursor into a new generation", async () => { + const f = fixture(); + const updateGate = + deferred>>(); + let attempts = 0; + f.transport.update = async () => { + attempts++; + return await updateGate.promise; + }; + f.queue.enqueue(move(1)); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + f.queue.enqueue(move(2)); + f.changeAuthority({ + ...authority, + expected: { ...authority.expected, viewportGeneration: 2 }, + }); + f.queue.cancel(); + updateGate.resolve({ + state, + gestureId: "owned", + nextSequence: 2, + cursor: "text", + }); + await flush(); + expect(attempts).toBe(1); + expect(f.cursors.filter(Boolean)).toEqual([]); + expect(f.ended[0]).toMatchObject({ sequence: 2, cancel: true }); + }); + + it("changes hover mouse to touch without discarding the first unsent touch", async () => { + const f = fixture(); + const began: string[] = []; + f.transport.begin = async (input) => { + began.push(input.pointerKind); + return { state, gestureId: `channel-${began.length}`, nextSequence: 1 }; + }; + f.queue.enqueue(move(1)); + await flush(); + f.queue.enqueue({ + kind: "touch", + type: "start", + points: [{ ...point(2), id: 7 }], + }); + await flush(); + expect(began).toEqual(["mouse", "touch"]); + expect(f.ended[0]).toMatchObject({ gestureId: "channel-1", cancel: true }); + expect(f.sent[1]).toMatchObject({ + gestureId: "channel-2", + event: { type: "start" }, + target: authority.target, + }); + expect(f.errors).toEqual([]); + }); + + it("waits for actual decoded pixels between independent presses, never for a held second finger", async () => { + const frame = deferred(); + const waited: string[] = []; + const f = fixture(async (id) => { + waited.push(id); + await frame.promise; + }); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + f.queue.enqueue({ kind: "scroll", point: point(1), deltaX: 0, deltaY: 20 }); + await flush(); + f.queue.enqueue({ + kind: "down", + point: point(1), + button: "left", + clickCount: 1, + }); + f.queue.enqueue({ + kind: "up", + point: point(1), + button: "left", + clickCount: 1, + }); + await flush(); + expect(f.sent.map((input) => input.event.kind)).toEqual(["scroll"]); + expect(waited).toEqual(["decoded-front"]); + f.changeAuthority({ + ...authority, + target: { ...authority.target, frameId: "new-front" }, + }); + frame.resolve(); + await flush(); + expect(f.sent[1]?.target?.frameId).toBe("new-front"); + expect(f.sent.map((input) => input.event.kind)).toEqual([ + "scroll", + "down", + "up", + ]); + expect(f.errors).toEqual([]); + }); + + it("drains queued mouseup then leave before normal cleanup without fresh-frame waits", async () => { + const waits: string[] = []; + const f = fixture(async (id) => { + waits.push(id); + }); + f.queue.enqueue({ + kind: "down", + point: point(1), + button: "left", + clickCount: 1, + }); + f.queue.enqueue({ + kind: "up", + point: point(1), + button: "left", + clickCount: 1, + }); + f.queue.finish(); + f.queue.enqueue({ kind: "leave" }); + f.queue.finish(); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(f.sent.map((input) => input.event.kind)).toEqual([ + "down", + "up", + "leave", + ]); + expect(f.ended).toHaveLength(1); + expect(f.ended[0]?.cancel).toBe(false); + expect(waits).toEqual([]); + expect(f.errors).toEqual([]); + }); + it("refreshes only a known unadmitted begin, then sends the pending wheel once", async () => { + const f = fixture(async () => { + f.changeAuthority({ ...authority, target: { ...authority.target, frameId: "post-scroll-decoded" } }); + }); + let begins = 0; + f.transport.begin = async () => { + begins += 1; + return begins === 1 ? { state, admission: "stale-frame" } : { state, gestureId: "owned", nextSequence: 1 }; + }; + f.queue.enqueue({ kind: "scroll", point: point(1), deltaX: 0, deltaY: 20 }); + await flush(); + expect(begins).toBe(2); + expect(f.sent).toHaveLength(1); + expect(f.errors).toEqual([]); + }); + + it("does not retry begin or update after transport error or uncertain physical publication", async () => { + for (const failure of ["begin", "update"] as const) { + const f = fixture(async () => { throw new Error("Must not wait/retry"); }); + let begins = 0; + let updates = 0; + f.transport.begin = async () => { + begins += 1; + if (failure === "begin") throw new Error("Outcome unknown"); + return { state, gestureId: "owned", nextSequence: 1 }; + }; + f.transport.update = async () => { updates += 1; throw new Error("Outcome unknown"); }; + f.queue.enqueue({ kind: "scroll", point: point(1), deltaX: 0, deltaY: 20 }); + await flush(); + expect(begins).toBe(1); + expect(updates).toBe(failure === "begin" ? 0 : 1); + expect(f.errors).toHaveLength(1); + } + }); + + it("bounds stale admission and refuses replacement control while waiting for decode", async () => { + const f = fixture(async () => { + f.changeAuthority({ ...authority, target: { ...authority.target, frameId: String(++front) } }); + }); + let front = 0; + let begins = 0; + f.transport.begin = async () => { begins += 1; return { state, admission: "stale-frame" }; }; + f.queue.enqueue(move(1)); + await flush(); + expect(begins).toBe(3); + expect(f.sent).toHaveLength(0); + expect(f.errors).toHaveLength(1); + const changed = fixture(async () => { changed.changeAuthority({ ...authority, controlToken: "replacement" }); }); + let changedBegins = 0; + changed.transport.begin = async () => { changedBegins += 1; return { state, admission: "stale-frame" }; }; + changed.queue.enqueue(move(1)); + await flush(); + expect(changedBegins).toBe(1); + expect(changed.sent).toHaveLength(0); + }); + + it("shares one four-second admission wait budget across stale decoder handoffs", async () => { + let clock = 1_000; + const now = vi.spyOn(Date, "now").mockImplementation(() => clock); + try { + const budgets: number[] = []; + const f = fixture(async (_id, budget) => { + budgets.push(budget!); + clock += budgets.length === 1 ? 3_000 : 1_001; + f.changeAuthority({ ...authority, target: { ...authority.target, frameId: String(clock) } }); + }); + let begins = 0; + f.transport.begin = async () => { begins += 1; return { state, admission: "stale-frame" }; }; + f.queue.enqueue(move(1)); + await flush(); + expect(budgets).toEqual([4_000, 1_000]); + expect(begins).toBe(2); + expect(f.sent).toHaveLength(0); + expect(f.errors).toHaveLength(1); + } finally { now.mockRestore(); } + }); + + it("does not begin under replacement control after the initial invalidated-frame wait", async () => { + const f = fixture(async () => { f.changeAuthority({ ...authority, controlToken: "replacement" }); }); + let begins = 0; + f.transport.begin = async () => { begins += 1; return { state, gestureId: "owned", nextSequence: 1 }; }; + f.queue.enqueue({ kind: "scroll", point: point(1), deltaX: 0, deltaY: 20 }); + await flush(); + f.queue.finish(); + await flush(); + f.queue.enqueue(move(1)); + await flush(); + expect(begins).toBe(1); + expect(f.sent).toHaveLength(1); + expect(f.errors).toEqual([]); + }); + +}); + +it("keeps keyboard repeat/Unicode edges ordered and a held modifier alive across mouse release", async () => { + const f = fixture(); + const key = { kind: "key", type: "down", key: "Control", code: "ControlLeft", modifiers: 2, repeat: false } as const; + f.queue.enqueue(key); + f.queue.enqueue({ kind: "down", point: point(1), button: "left", clickCount: 1 }); + f.queue.enqueue({ kind: "up", point: point(1), button: "left", clickCount: 1 }); + f.queue.finish(); + f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); + await flush(); + expect(f.ended).toEqual([]); + f.queue.enqueue({ ...key, type: "up", modifiers: 0 }); + f.queue.enqueue({ kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: false }); + f.queue.enqueue({ kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: true }); + f.queue.enqueue({ kind: "key", type: "up", key: "é", code: "KeyE", modifiers: 0, repeat: false }); + f.queue.enqueue({ kind: "text", text: "漢字\n😀" }); + f.queue.finish(); + await flush(); + expect(f.sent.map((input) => input.event.kind)).toEqual(["key", "down", "up", "key", "key", "key", "key", "text"]); + expect(f.sent.map((input) => input.sequence)).toEqual([1,2,3,4,5,6,7,8]); + expect(f.ended).toHaveLength(1); + expect(f.errors).toEqual([]); +}); diff --git a/paseo-shared-browser/client/browser-input-queue.ts b/paseo-shared-browser/client/browser-input-queue.ts new file mode 100644 index 00000000..8bf0cefb --- /dev/null +++ b/paseo-shared-browser/client/browser-input-queue.ts @@ -0,0 +1,435 @@ +/** + * Ordered, bounded live browser input. A channel starts from decoded front-frame + * authority; its own motion continues with opaque server identity/sequence. Only + * replaceable motion with unchanged pressed identities and wheel deltas coalesce. + * Press/release and touch identity changes remain ordered. No uncertain mutation is replayed. + */ +import type { RpcInput, RpcOutput } from "@getpaseo/plugin"; +import type { + beginBrowserGestureRpc, + updateBrowserGestureRpc, + endBrowserGestureRpc, + BrowserGestureEvent, + BrowserCursor, + BrowserState, +} from "../shared/browser"; + +export type BrowserGestureAuthority = Omit< + RpcInput, + "pointerKind" +>; +export type BrowserGesturePoint = Extract< + BrowserGestureEvent, + { kind: "move" } +>["point"]; +export type BrowserTouchPoint = Extract< + BrowserGestureEvent, + { kind: "touch" } +>["points"][number]; +/** Preserve the admission union while using schema-normalized state defaults. */ +type NormalizedAdmission = T extends unknown ? Omit & { state: BrowserState } : never; +export interface BrowserGestureTransport { + begin(input: RpcInput): Promise< + NormalizedAdmission> + >; + update(input: RpcInput): Promise< + Omit, "state"> & { + state: BrowserState; + } + >; + end(input: RpcInput): Promise< + Omit, "state"> & { + state: BrowserState; + } + >; +} +interface QueueOptions { + transport: BrowserGestureTransport; + authority(): BrowserGestureAuthority | null; + onState(state: BrowserState): void; + onCursor(cursor: BrowserCursor | null): void; + onError(error: unknown): void; + onFinish(): void; + /** Reset local held contacts after an acknowledged navigation, without cancelling this queue again. */ + onNavigationComplete?(): void; + /** Resolves only after a different decoded front frame commits, not after a timer. */ + waitForFrame?(afterFrameId: string, maxWaitMs?: number): Promise; +} +type Command = { event: BrowserGestureEvent } | { end: true }; +interface Channel { + authority: BrowserGestureAuthority; + pointerKind: "mouse" | "touch"; + gestureId: string; + sequence: number; + heldButtons: Set; + heldKeys: Set; + touchCount: number; +} +const MAX_QUEUED_COMMANDS = 64; +const MAX_WHEEL_DELTA = 4_000; +const MAX_FRAME_ADMISSION_ATTEMPTS = 3; +const FRAME_ADMISSION_WAIT_MS = 4_000; + +/** Compare control/route incarnation, never the changing decoded frame identity. */ +function sameAuthority( + a: BrowserGestureAuthority, + b: BrowserGestureAuthority | null, +): boolean { + return Boolean( + b && + a.viewerToken === b.viewerToken && + a.controlToken === b.controlToken && + a.expected.sessionId === b.expected.sessionId && + a.expected.runtimeId === b.expected.runtimeId && + a.expected.bridgeEpoch === b.expected.bridgeEpoch && + a.expected.navigationGeneration === b.expected.navigationGeneration && + a.expected.viewportGeneration === b.expected.viewportGeneration, + ); +} +function transportContext(authority: BrowserGestureAuthority) { + return { + viewerToken: authority.viewerToken, + controlToken: authority.controlToken, + expected: authority.expected, + }; +} +function needsFreshPress( + event: BrowserGestureEvent, + channel: Channel, +): boolean { + return ( + event.kind === "down" || + (event.kind === "touch" && + event.type === "start" && + channel.touchCount === 0) + ); +} + +/** A qualified reply completes old input; it never authorizes input on the new page. */ +function isAcknowledgedNavigation( + original: BrowserGestureAuthority, + state: BrowserState, +): boolean { + return state.status === "ready" && state.controller === "self" && + state.sessionId === original.expected.sessionId && + state.runtimeId === original.expected.runtimeId && + state.bridgeEpoch === original.expected.bridgeEpoch && + state.viewportGeneration === original.expected.viewportGeneration && + state.navigationGeneration > original.expected.navigationGeneration; +} + +/** One in-flight RPC, with fail-closed cancellation on pressure or authority loss. */ +export function createBrowserInputQueue(options: QueueOptions) { + let epoch = 0; + let running = false; + let channel: Channel | null = null; + let commands: Command[] = []; + let invalidatedFrameId: string | null = null; + + const cancelChannel = async (old: Channel) => { + try { + await options.transport.end({ + ...transportContext(old.authority), + gestureId: old.gestureId, + sequence: old.sequence, + cancel: true, + }); + } catch { + // Best-effort cleanup, never retry physical input. Server's bounded idle + // expiry releases an uncertain channel if the cleanup reply is lost. + } + }; + const cancel = () => { + epoch += 1; + commands = []; + invalidatedFrameId = null; + const old = channel; + channel = null; + options.onCursor(null); + if (old && !running) void cancelChannel(old); + }; + + const drain = async () => { + if (running) return; + running = true; + const currentEpoch = epoch; + let owned = channel; + try { + while (commands.length && currentEpoch === epoch) { + const command = commands.shift(); + if (!command) break; + if ("end" in command) { + if (!owned) continue; + // A pointer release must not release a still-held keyboard chord. + if (owned.heldKeys.size || owned.heldButtons.size || owned.touchCount) continue; + const ending = owned; + const result = await options.transport.end({ + ...transportContext(ending.authority), + gestureId: ending.gestureId, + sequence: ending.sequence, + cancel: false, + }); + owned = null; + channel = null; + if ( + currentEpoch !== epoch || + !sameAuthority(ending.authority, options.authority()) + ) + break; + options.onState(result.state); + if (!sameAuthority(ending.authority, options.authority())) { + throw new Error( + "Browser input context changed. Release the gesture and try again.", + ); + } + // Normal channel completion keeps the last qualified hover cursor. + // A null end receipt denotes cleanup, not a new pointer location. + if (result.cursor !== null) options.onCursor(result.cursor); + options.onFinish(); + continue; + } + + let current = options.authority(); + if (!current) + throw new Error( + "A decoded frame and active browser control are required.", + ); + const pointerKind = command.event.kind === "touch" ? "touch" + : command.event.kind === "key" || command.event.kind === "text" + ? owned?.pointerKind ?? "mouse" : "mouse"; + if (owned && !sameAuthority(owned.authority, current)) { + throw new Error( + "Browser input context changed. Release the gesture and try again.", + ); + } + if (owned && owned.pointerKind !== pointerKind) { + // Changing input devices is ordinary on touch laptops. Close only this + // exact old channel before sending the still-unsent event on a new one. + await cancelChannel(owned); + owned = null; + channel = null; + if (currentEpoch !== epoch) break; + if (!sameAuthority(current, options.authority())) { + throw new Error( + "Browser input context changed. Release the gesture and try again.", + ); + } + } + if (!owned) { + const admissionDeadline = Date.now() + FRAME_ADMISSION_WAIT_MS; + if ( + invalidatedFrameId === current.target.frameId && + options.waitForFrame + ) { + const waitingAuthority = current; + await options.waitForFrame(current.target.frameId, FRAME_ADMISSION_WAIT_MS); + if (currentEpoch !== epoch) break; + current = options.authority(); + if (!sameAuthority(waitingAuthority, current)) break; + if (!current) throw new Error("Browser input context changed."); + } + for (let attempt = 0; attempt < MAX_FRAME_ADMISSION_ATTEMPTS; attempt += 1) { + if (Date.now() >= admissionDeadline) break; + const sentAuthority = current; + const result = await options.transport.begin({ ...sentAuthority, pointerKind }); + if (!("admission" in result)) { + owned = { + authority: sentAuthority, + pointerKind, + gestureId: result.gestureId, + sequence: result.nextSequence, + heldButtons: new Set(), + heldKeys: new Set(), + touchCount: 0, + }; + } + if (currentEpoch !== epoch || !sameAuthority(sentAuthority, options.authority())) break; + options.onState(result.state); + if (!sameAuthority(sentAuthority, options.authority())) break; + if (owned) { + channel = owned; + break; + } + // Only this validated non-publication receipt permits another begin. + // A runtime/transport exception never retries an action. Captures + // already decoding may also be revoked, so admission remains bounded. + if (attempt === MAX_FRAME_ADMISSION_ATTEMPTS - 1) break; + options.onFinish(); + const latest = options.authority(); + if (latest?.target.frameId === sentAuthority.target.frameId) { + if (!options.waitForFrame) throw new Error("Waiting for a current decoded frame."); + const remaining = admissionDeadline - Date.now(); + if (remaining <= 0) break; + await options.waitForFrame(sentAuthority.target.frameId, remaining); + } + if (currentEpoch !== epoch) break; + current = options.authority(); + if (!current || !sameAuthority(sentAuthority, current)) { + throw new Error("Browser input context changed. Release the gesture and try again."); + } + } + if (currentEpoch !== epoch || !sameAuthority(current, options.authority())) break; + if (!owned) throw new Error("Waiting for a current decoded frame. Release the gesture and try again."); + } + + let fresh = options.authority(); + if ( + needsFreshPress(command.event, owned) && + fresh && + invalidatedFrameId === fresh.target.frameId && + options.waitForFrame + ) { + await options.waitForFrame(fresh.target.frameId); + if (currentEpoch !== epoch) break; + fresh = options.authority(); + } + if (!sameAuthority(owned.authority, fresh)) { + throw new Error( + "Browser input context changed. Release the gesture and try again.", + ); + } + if ( + (command.event.kind !== "move" && command.event.kind !== "leave") || + owned.heldButtons.size > 0 + ) { + invalidatedFrameId = fresh?.target.frameId ?? null; + } + const requiresTarget = needsFreshPress(command.event, owned); + if (command.event.kind === "key" && command.event.type === "down") owned.heldKeys.add(command.event.code); + if (command.event.kind === "key" && command.event.type === "up") owned.heldKeys.delete(command.event.code); + if (command.event.kind === "touch") + owned.touchCount = command.event.points.length; + if (command.event.kind === "down") + owned.heldButtons.add(command.event.button); + if (command.event.kind === "up") + owned.heldButtons.delete(command.event.button); + const expectedNextSequence = owned.sequence + 1; + const result = await options.transport.update({ + ...transportContext(owned.authority), + gestureId: owned.gestureId, + sequence: owned.sequence, + event: command.event, + ...(requiresTarget && fresh ? { target: fresh.target } : {}), + }); + owned.sequence = result.nextSequence; + if ( + currentEpoch !== epoch || + !sameAuthority(owned.authority, options.authority()) + ) + break; + if (result.gestureId !== owned.gestureId) + throw new Error("Browser gesture identity changed."); + if (result.completion === "navigation") { + if (!isAcknowledgedNavigation(owned.authority, result.state) || + result.nextSequence !== expectedNextSequence) { + throw new Error("Browser navigation completion identity changed."); + } + // Native publication and cleanup are already acknowledged. Discard + // queued releases/typing from the old document before projecting the + // new state, which intentionally invalidates the old decoded frame. + epoch += 1; + commands = []; + invalidatedFrameId = null; + channel = null; + owned = null; + options.onCursor(null); + options.onNavigationComplete?.(); + options.onState(result.state); + options.onFinish(); + break; + } + options.onState(result.state); + if (!sameAuthority(owned.authority, options.authority())) { + throw new Error( + "Browser input context changed. Release the gesture and try again.", + ); + } + options.onCursor(result.cursor); + } + } catch (error) { + if (currentEpoch === epoch) { + epoch += 1; + commands = []; + channel = null; + options.onCursor(null); + options.onError(error); + } + } finally { + if ( + owned && + (currentEpoch !== epoch || + !sameAuthority(owned.authority, options.authority())) + ) { + if (currentEpoch === epoch) { + epoch += 1; + commands = []; + options.onCursor(null); + } + await cancelChannel(owned); + if (channel === owned) channel = null; + } + running = false; + if (commands.length) void drain(); + } + }; + + const enqueue = (event: BrowserGestureEvent) => { + if (!options.authority()) return false; + // Keep leave behind the queued release, ahead of normal channel cleanup. + const trailing = commands.at(-1); + if (event.kind === "leave" && trailing && "end" in trailing) commands.pop(); + const previous = commands.at(-1); + if (previous && "event" in previous) { + if (event.kind === "move" && previous.event.kind === "move") { + previous.event = event; + return true; + } + if ( + event.kind === "touch" && + previous.event.kind === "touch" && + event.type === "move" && + previous.event.type === "move" && + event.points.length === previous.event.points.length && + event.points.every( + (point) => + previous.event.kind === "touch" && + previous.event.points.some((other) => other.id === point.id), + ) + ) { + previous.event = event; + return true; + } + if (event.kind === "scroll" && previous.event.kind === "scroll") { + const deltaX = previous.event.deltaX + event.deltaX; + const deltaY = previous.event.deltaY + event.deltaY; + if ( + Math.abs(deltaX) <= MAX_WHEEL_DELTA && + Math.abs(deltaY) <= MAX_WHEEL_DELTA + ) { + previous.event = { ...event, deltaX, deltaY }; + return true; + } + } + } + if (commands.length >= MAX_QUEUED_COMMANDS) { + cancel(); + options.onError( + new Error( + "Browser input cannot keep up. Release the gesture and try again.", + ), + ); + return false; + } + commands.push({ event }); + void drain(); + return true; + }; + const finish = () => { + const last = commands.at(-1); + if (last && "end" in last) return; + commands.push({ end: true }); + void drain(); + }; + + return { enqueue, finish, cancel }; +} diff --git a/paseo-shared-browser/client/browser-menu-placement.test.ts b/paseo-shared-browser/client/browser-menu-placement.test.ts new file mode 100644 index 00000000..d595b036 --- /dev/null +++ b/paseo-shared-browser/client/browser-menu-placement.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { browserMenuPlacement, browserSubmenuPlacement } from "./browser-menu-placement"; + +describe("bounded toolbar menus", () => { + it("aligns under a desktop trigger without exceeding the pane", () => { + expect( + browserMenuPlacement( + { width: 800, height: 600 }, + { x: 740, y: 8, width: 28, height: 28 }, + 240, + ), + ).toEqual({ x: 468, y: 40, width: 300, height: 240 }); + }); + it("fits a narrow phone pane and bounds a long favorites list for scrolling", () => { + const rect = browserMenuPlacement( + { width: 280, height: 240 }, + { x: 20, y: 8, width: 28, height: 28 }, + 900, + )!; + expect(rect).toEqual({ x: 8, y: 40, width: 264, height: 192 }); + }); + it("opens above a lower anchor when there is more space there", () => { + expect( + browserMenuPlacement( + { width: 400, height: 300 }, + { x: 340, y: 260, width: 28, height: 28 }, + 160, + ), + ).toEqual({ x: 68, y: 96, width: 300, height: 160 }); + }); + it("refuses unknown or invalid measured geometry", () => { + for (const bad of [0, -1, Infinity, NaN]) { + expect( + browserMenuPlacement( + { width: bad, height: 600 }, + { x: 20, y: 8, width: 28, height: 28 }, + 240, + ), + ).toBeNull(); + expect( + browserMenuPlacement( + { width: 800, height: 600 }, + { x: 20, y: 8, width: bad, height: 28 }, + 240, + ), + ).toBeNull(); + } + }); +}); + + +describe("bounded child menu placement", () => { + it("opens beside a parent with enough room", () => { + expect(browserSubmenuPlacement({ width: 900, height: 600 }, + { x: 8, y: 40, width: 300, height: 100 }, + { x: 8, y: 80, width: 300, height: 36 }, 200)) + .toEqual({ x: 312, y: 80, width: 240, height: 200 }); + }); + it("flips left and clamps a long submenu for internal scrolling", () => { + expect(browserSubmenuPlacement({ width: 800, height: 300 }, + { x: 468, y: 40, width: 300, height: 100 }, + { x: 468, y: 260, width: 300, height: 36 }, 900)) + .toEqual({ x: 224, y: 8, width: 240, height: 284 }); + }); + it("refuses unknown pane or row geometry", () => { + expect(browserSubmenuPlacement({ width: 0, height: 300 }, + { x: 0, y: 0, width: 100, height: 80 }, + { x: 0, y: 20, width: 100, height: 30 }, 160)).toBeNull(); + expect(browserSubmenuPlacement({ width: 800, height: 300 }, + { x: 0, y: 0, width: 100, height: 80 }, + { x: 0, y: NaN, width: 100, height: 30 }, 160)).toBeNull(); + }); +}); diff --git a/paseo-shared-browser/client/browser-menu-placement.ts b/paseo-shared-browser/client/browser-menu-placement.ts new file mode 100644 index 00000000..b3146688 --- /dev/null +++ b/paseo-shared-browser/client/browser-menu-placement.ts @@ -0,0 +1,78 @@ +/** Panel-relative menu placement; the pane already excludes the host's safe areas. */ +export interface MenuRect { + x: number; + y: number; + width: number; + height: number; +} + +/** Anchor to the trigger's right edge and prefer below, with bounded internal scrolling. */ +export function browserMenuPlacement( + pane: { width: number; height: number }, + anchor: MenuRect, + preferredHeight: number, +): MenuRect | null { + const values = [ + pane.width, + pane.height, + anchor.x, + anchor.y, + anchor.width, + anchor.height, + preferredHeight, + ]; + if ( + values.some((value) => !Number.isFinite(value)) || + pane.width <= 16 || + pane.height <= 16 || + anchor.width <= 0 || + anchor.height <= 0 || + preferredHeight <= 0 + ) { + return null; + } + + const margin = 8; + const gap = 4; + const width = Math.min(300, pane.width - margin * 2); + const x = Math.max( + margin, + Math.min(pane.width - margin - width, anchor.x + anchor.width - width), + ); + const below = Math.max(0, pane.height - margin - anchor.y - anchor.height - gap); + const above = Math.max(0, anchor.y - margin - gap); + const useBelow = below >= preferredHeight || below >= above; + const height = Math.min(preferredHeight, useBelow ? below : above, pane.height - margin * 2); + if (height < 1) return null; + + const y = useBelow ? anchor.y + anchor.height + gap : anchor.y - gap - height; + return { x, y: Math.max(margin, Math.min(pane.height - margin - height, y)), width, height }; +} + +/** Place a flyout beside its parent, flip left when needed, and clamp to the pane. */ +export function browserSubmenuPlacement( + pane: { width: number; height: number }, + parent: MenuRect, + row: MenuRect, + preferredHeight: number, +): MenuRect | null { + const values = [pane.width, pane.height, parent.x, parent.width, row.y, preferredHeight]; + if ( + values.some(value => !Number.isFinite(value)) || + pane.width <= 16 || + pane.height <= 16 || + preferredHeight <= 0 + ) { + return null; + } + const width = Math.min(240, pane.width - 16); + const height = Math.min(preferredHeight, pane.height - 16); + const right = parent.x + parent.width + 4; + const x = right + width <= pane.width - 8 ? right : parent.x - width - 4; + return { + x: Math.max(8, Math.min(pane.width - 8 - width, x)), + y: Math.max(8, Math.min(pane.height - 8 - height, row.y)), + width, + height, + }; +} diff --git a/paseo-shared-browser/client/browser-native-keyboard.test.ts b/paseo-shared-browser/client/browser-native-keyboard.test.ts new file mode 100644 index 00000000..a324e522 --- /dev/null +++ b/paseo-shared-browser/client/browser-native-keyboard.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; +import { createBrowserNativeKeyboard } from "./browser-native-keyboard"; +import type { CanvasKeyboardEvent } from "./browser-canvas-keyboard"; + +function fixture() { + const events: CanvasKeyboardEvent[] = []; + const errors: Error[] = []; + let value = ""; + let enabled = true; + const relay = createBrowserNativeKeyboard({ + enabled: () => enabled, + enqueue(event) { + events.push(event); + return true; + }, + finish() {}, + onValue(next) { + value = next; + }, + onError(error) { + errors.push(error); + }, + }); + return { + relay, + events, + errors, + value: () => value, + disable: () => { + enabled = false; + }, + }; +} +describe("native Basic and Compose relay", () => { + it("inserts only appended basic text and never duplicates physical Backspace on change", () => { + const f = fixture(); + f.relay.changeText("a"); + f.relay.changeText("ab"); + f.relay.changeText("ab🙂"); + f.relay.keyPress("Backspace"); + f.relay.changeText("ab"); + expect(f.events.filter((event) => event.kind === "text")).toEqual([ + { kind: "text", text: "a" }, + { kind: "text", text: "b" }, + { kind: "text", text: "🙂" }, + ]); + expect(f.events.filter((event) => event.kind === "key").map((event) => event.type)).toEqual([ + "down", + "up", + ]); + expect(f.value()).toBe(""); + f.relay.changeText("c"); + expect(f.events.at(-1)).toEqual({ kind: "text", text: "c" }); + }); + it("forwards empty Backspace and Enter as deliberate physical keys, not inferred field edits", () => { + const f = fixture(); + f.relay.keyPress("Backspace"); + f.relay.keyPress("Enter"); + expect( + f.events.map((event) => (event.kind === "key" ? [event.key, event.type] : null)), + ).toEqual([ + ["Backspace", "down"], + ["Backspace", "up"], + ["Enter", "down"], + ["Enter", "up"], + ]); + }); + it("refuses local autocorrect, middle edits, and bulk deletion without rewriting remote text", () => { + for (const edit of ["abd", "aXbc", "a"]) { + const f = fixture(); + f.relay.changeText("abc"); + f.relay.changeText(edit); + expect(f.events).toEqual([{ kind: "text", text: "abc" }]); + expect(f.errors).toHaveLength(1); + expect(f.value()).toBe(""); + } + }); + it("one complete Unicode grapheme deletion is one Backspace when its key event is absent", () => { + const f = fixture(); + f.relay.changeText("👩‍👩‍👧‍👦"); + f.relay.changeText(""); + expect(f.events.filter((event) => event.kind === "key")).toHaveLength(2); + expect(f.errors).toHaveLength(0); + }); + it("never infers Backspace from removing only part of a combining or joined cluster", () => { + for (const [before, after] of [ + ["e\u0301", "e"], + ["👩‍👩‍👧‍👦", "👩"], + ] as const) { + const f = fixture(); + f.relay.changeText(before); + f.relay.changeText(after); + expect(f.events.filter((event) => event.kind === "key")).toHaveLength(0); + expect(f.errors).toHaveLength(1); + } + }); + it("Compose inserts the explicit Done draft once and rejects oversized or unowned commits", () => { + const f = fixture(); + expect(f.events).toHaveLength(0); + expect(f.relay.compose("日本語🙂")).toBe(true); + expect(f.events).toEqual([{ kind: "text", text: "日本語🙂" }]); + expect(f.relay.compose("x".repeat(16_001))).toBe(false); + f.disable(); + expect(f.relay.compose("different owner")).toBe(false); + expect(f.events).toHaveLength(1); + }); + it("bounds local context independently of already inserted remote text", () => { + const f = fixture(); + f.relay.changeText("a".repeat(1_025)); + expect(f.value()).toBe(""); + f.relay.changeText("b"); + expect(f.events.at(-1)).toEqual({ kind: "text", text: "b" }); + f.relay.selection(0, 1); + expect(f.value()).toBe(""); + }); +}); diff --git a/paseo-shared-browser/client/browser-native-keyboard.ts b/paseo-shared-browser/client/browser-native-keyboard.ts new file mode 100644 index 00000000..cfca9b87 --- /dev/null +++ b/paseo-shared-browser/client/browser-native-keyboard.ts @@ -0,0 +1,145 @@ +/** + * Basic native software typing, separate from Compose's explicit Done commit. + * This relay knows only text it just inserted at the remote caret. It never + * replaces a remote field or infers a correction outside that local suffix. + * RN does not expose composition commits; complex IME belongs in Compose mode. + */ +import type { CanvasKeyboardEvent } from "./browser-canvas-keyboard"; + +interface NativeKeyboardOptions { + enabled(): boolean; + enqueue(event: CanvasKeyboardEvent): boolean; + finish(): void; + onValue(value: string): void; + onError(error: Error): void; +} +const MAX_LOCAL_SUFFIX = 1_024; + +/** Infer only a whole final grapheme from the original known suffix boundaries. */ +function isOneTrailingGrapheme(before: string, after: string): boolean { + if (typeof Intl.Segmenter === "function") { + const segments = [ + ...new Intl.Segmenter(undefined, { granularity: "grapheme" }).segment(before), + ]; + return segments.at(-1)?.index === after.length; + } + // Hermes versions without Segmenter can safely count plain ASCII only. A code + // point count would corrupt emoji/combining clusters under native Backspace. + return /^[\x20-\x7e]*$/.test(before) && before.length - after.length === 1; +} + +/** Event envelopes come from TextInput, not inferred native hardware shortcuts. */ +export function createBrowserNativeKeyboard(options: NativeKeyboardOptions) { + let suffix = ""; + let pendingBackspace: string | null = null; + const setSuffix = (value: string) => { + suffix = value; + options.onValue(value); + }; + const reset = () => { + pendingBackspace = null; + setSuffix(""); + }; + const rejectEdit = () => { + reset(); + options.onError( + new Error( + "Use Compose and Done for IME or text corrections. Live typing supports basic text and Backspace.", + ), + ); + }; + const press = (key: "Backspace" | "Enter", code: "Backspace" | "Enter") => { + if (!options.enabled()) return false; + const down = options.enqueue({ + kind: "key", + type: "down", + key, + code, + modifiers: 0, + repeat: false, + }); + if (!down) return false; + const up = options.enqueue({ kind: "key", type: "up", key, code, modifiers: 0, repeat: false }); + if (up) options.finish(); + return up; + }; + const changeText = (value: string) => { + if (!options.enabled()) { + reset(); + return; + } + if (pendingBackspace !== null) { + const previous = pendingBackspace; + pendingBackspace = null; + if (previous.startsWith(value) && value.length < previous.length) { + // onKeyPress already forwarded the physical Backspace. Drop local + // context so an OS-specific deletion width cannot authorize later edits. + setSuffix(""); + return; + } + rejectEdit(); + return; + } + if (value === suffix) return; + if (value.startsWith(suffix)) { + const appended = value.slice(suffix.length); + if (appended.length > 16_000) { + reset(); + options.onError(new Error("Text is too long. Send up to 16,000 characters at once.")); + return; + } + if (options.enqueue({ kind: "text", text: appended })) { + setSuffix(value.length > MAX_LOCAL_SUFFIX ? "" : value); + options.finish(); + } + return; + } + if (suffix.startsWith(value)) { + // A missing onKeyPress can be repaired only for one complete known + // grapheme. Bulk replacement/deletion is a Compose edit, not live input. + if (isOneTrailingGrapheme(suffix, value) && press("Backspace", "Backspace")) { + setSuffix(""); + return; + } + } + rejectEdit(); + }; + const keyPress = (key: string) => { + if (!options.enabled()) return; + if (key === "Backspace") { + if (press("Backspace", "Backspace")) pendingBackspace = suffix || null; + } else if (key === "Enter") { + press("Enter", "Enter"); + reset(); + } + }; + const selection = (start: number, end: number) => { + // Native selection notifications may precede the matching text change. + // A collapsed caret at/after the suffix, or the pending physical deletion, + // grants no edit authority and can wait for that qualified change callback. + if (start === end && (end >= suffix.length || pendingBackspace !== null)) { + return true; + } + if (start !== end || end !== suffix.length) { + // Hidden Live input has no selection editor. A moved local caret cannot + // establish an equivalent remote caret and therefore revokes the suffix. + reset(); + return false; + } + return true; + }; + const compose = (value: string): boolean => { + if (!options.enabled() || !value) return false; + if (value.length > 16_000) { + options.onError(new Error("Text is too long. Send up to 16,000 characters at once.")); + return false; + } + const accepted = options.enqueue({ kind: "text", text: value }); + if (accepted) { + reset(); + options.finish(); + } + return accepted; + }; + return { reset, changeText, keyPress, selection, compose }; +} diff --git a/paseo-shared-browser/client/browser-resolution-picker.tsx b/paseo-shared-browser/client/browser-resolution-picker.tsx new file mode 100644 index 00000000..8365ceff --- /dev/null +++ b/paseo-shared-browser/client/browser-resolution-picker.tsx @@ -0,0 +1,174 @@ +/** Grouped resolution choices with independent host favorites and capture quality. */ +import type { PluginHostProps } from "@getpaseo/plugin/client"; +import { Icon } from "@getpaseo/plugin/client/react-native"; +import { Pressable, StyleSheet, Text, View } from "react-native"; +import type { BrowserDisplayPreferences } from "../shared/browser-display-preferences"; +import type { DevicePresetId } from "../shared/device-presets"; +import type { ResolutionGroup } from "../shared/resolution-menu"; +import { FavoriteStar } from "./favorite-star"; + +export interface BrowserResolutionPickerProps { + theme: PluginHostProps["theme"]; + groups: readonly ResolutionGroup[]; + selectedPresetId: DevicePresetId | null; + favoritePresetIds: readonly DevicePresetId[]; + selectDisabled: boolean; + favoriteDisabled: boolean; + captureQuality: BrowserDisplayPreferences["captureQuality"]; + onSelect(id: DevicePresetId): void; + onToggleFavorite(id: DevicePresetId): void; + onQualityChange(quality: BrowserDisplayPreferences["captureQuality"]): void; +} + +const STYLES = StyleSheet.create({ + group: { gap: 8 }, + header: { fontSize: 11, fontWeight: "600" }, + row: { flexDirection: "row", gap: 8, alignItems: "center" }, + choice: { + minHeight: 46, + paddingHorizontal: 12, + paddingVertical: 8, + borderWidth: 1, + borderRadius: 8, + flexDirection: "row", + alignItems: "center", + gap: 8, + flex: 1, + }, + title: { flex: 1, fontSize: 13, fontWeight: "600" }, + detail: { fontSize: 11 }, + star: { + width: 44, + minHeight: 44, + alignItems: "center", + justifyContent: "center", + borderRadius: 8, + }, + quality: { + minHeight: 44, + paddingHorizontal: 12, + paddingVertical: 8, + borderWidth: 1, + borderRadius: 8, + flexDirection: "row", + alignItems: "center", + gap: 8, + }, +}); +const QUALITY_OPTIONS = [ + { id: "low", label: "Low", detail: "Smaller images, less detail" }, + { id: "medium", label: "Medium", detail: "Balanced detail and image size" }, + { id: "high", label: "High", detail: "Sharper images, more data" }, +] as const; + +/** Keeps star presses outside the resolution button; favoriting never resizes the browser. */ +export function BrowserResolutionPicker(props: BrowserResolutionPickerProps) { + const { theme } = props; + const colors = theme.colors; + const selectedStyle = { + borderColor: colors.accent, + backgroundColor: colors.surface2, + }; + const baseStyle = { + borderColor: colors.border, + backgroundColor: colors.surface1, + }; + return ( + + {props.groups.map((group) => ( + + {group.label} + {group.presets.map((preset) => { + const selected = preset.id === props.selectedPresetId; + const favorite = props.favoritePresetIds.includes(preset.id); + return ( + + props.onSelect(preset.id)} + style={({ pressed }) => [ + STYLES.choice, + baseStyle, + selected ? selectedStyle : null, + { + opacity: props.selectDisabled ? 0.45 : pressed ? 0.72 : 1, + }, + ]} + > + + {preset.label} + + {preset.viewport.width} × {preset.viewport.height} + + + props.onToggleFavorite(preset.id)} + style={({ pressed }) => [ + STYLES.star, + { + backgroundColor: "transparent", + opacity: props.favoriteDisabled ? 0.45 : pressed ? 0.72 : 1, + }, + ]} + > + + + + ); + })} + + ))} + + Image quality + {QUALITY_OPTIONS.map((option) => { + const selected = option.id === props.captureQuality; + return ( + props.onQualityChange(option.id)} + style={({ pressed }) => [ + STYLES.quality, + baseStyle, + selected ? selectedStyle : null, + { opacity: props.favoriteDisabled ? 0.45 : pressed ? 0.72 : 1 }, + ]} + > + {option.label} + + {option.detail} + + + ); + })} + + + ); +} diff --git a/paseo-shared-browser/client/browser-toolbar-menu-web.test.ts b/paseo-shared-browser/client/browser-toolbar-menu-web.test.ts new file mode 100644 index 00000000..77b060b4 --- /dev/null +++ b/paseo-shared-browser/client/browser-toolbar-menu-web.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("react-native", () => ({ Platform: { OS: "web" } })); + +import { bindBrowserToolbarMenuWeb } from "./browser-toolbar-menu-web"; + +/** Deterministic host nodes exercise focus/ownership, without a browser or shared page. */ +interface MenuEvent { + key?: string; + shiftKey?: boolean; + target: unknown; + preventDefault(): void; + stopPropagation(): void; +} +function fixture(hierarchy?: Parameters[5]) { + const listeners = new Map void>(); + const doc = { + activeElement: null as unknown, + addEventListener: (name: string, fn: (event: MenuEvent) => void) => listeners.set(name, fn), + removeEventListener: (name: string) => listeners.delete(name), + }; + const item = (disabled = false) => { + const result = { + focus: vi.fn(() => { + doc.activeElement = result; + listeners.get("focusin")?.({ target: result, preventDefault() {}, stopPropagation() {} }); + }), + contains: (node: unknown) => node === result, + getAttribute: (name: string) => (name === "aria-disabled" ? String(disabled) : null), + }; + return result; + }; + const items = [item(), item(true), item()]; + const triggerButton = item(); + const menu = { + ownerDocument: doc, + contains: (node: unknown) => items.some((item) => item === node), + querySelectorAll: () => items, + }; + const trigger = { + contains: (node: unknown) => node === triggerButton, + querySelectorAll: () => [triggerButton], + }; + const backdrop = {}; + const overlay = { contains: (node: unknown) => node === backdrop || menu.contains(node) }; + const close = vi.fn(); + let restore = true; + const dispose = bindBrowserToolbarMenuWeb(menu, trigger, close, () => restore, overlay, hierarchy); + const event = (key: string, target = doc.activeElement) => ({ + key, + target, + preventDefault: vi.fn(), + stopPropagation: vi.fn(), + }); + return { + doc, + menu, + items, + triggerButton, + backdrop, + listeners, + close, + dispose, + event, + handoff: () => { + restore = false; + }, + }; +} + +describe("local toolbar menu keyboard and dismissal", () => { + it("Tab and Shift+Tab exit through the surrounding host controls", () => { + for (const backwards of [false, true]) { + const f = fixture(); + const before = { focus: vi.fn(), getAttribute: () => null }; + const after = { focus: vi.fn(), getAttribute: () => null }; + Object.assign(f.doc, { + querySelectorAll: () => [before, f.triggerButton, after, ...f.items], + }); + f.listeners.get("keydown")!({ ...f.event("Tab"), shiftKey: backwards }); + expect(f.close).toHaveBeenCalledOnce(); + f.dispose(); + expect((backwards ? before : after).focus).toHaveBeenCalledOnce(); + expect(f.triggerButton.focus).not.toHaveBeenCalled(); + } + }); + it("focuses the first item and uses arrows/Home/End without forwarding keys", () => { + const f = fixture(); + expect(f.doc.activeElement).toBe(f.items[0]); + for (const [key, index] of [ + ["ArrowDown", 2], + ["ArrowDown", 0], + ["ArrowUp", 2], + ["Home", 0], + ["End", 2], + ] as const) { + const event = f.event(key); + f.listeners.get("keydown")!(event); + expect(f.doc.activeElement).toBe(f.items[index]); + expect(event.preventDefault).toHaveBeenCalledOnce(); + expect(event.stopPropagation).toHaveBeenCalledOnce(); + } + f.dispose(); + }); + it("Escape dismisses and returns focus even if the row is removed before cleanup", () => { + const f = fixture(); + const event = f.event("Escape"); + f.listeners.get("keydown")!(event); + expect(f.close).toHaveBeenCalledOnce(); + f.doc.activeElement = {}; + f.dispose(); + expect(f.triggerButton.focus).toHaveBeenCalledOnce(); + expect(f.listeners.size).toBe(0); + }); + it("does not restore focus when an action hands it to another dialog or keyboard", () => { + const f = fixture(); + f.handoff(); + f.dispose(); + expect(f.triggerButton.focus).not.toHaveBeenCalled(); + }); + it("consumes local backdrop dismissal so it cannot click through to the remote page", () => { + const f = fixture(); + const event = f.event("", f.backdrop); + f.listeners.get("pointerdown")!(event); + expect(f.close).toHaveBeenCalledOnce(); + expect(event.preventDefault).toHaveBeenCalledOnce(); + expect(event.stopPropagation).toHaveBeenCalledOnce(); + f.dispose(); + }); + it("leaves deliberate outside-host focus alone", () => { + const f = fixture(); + const event = f.event("", {}); + f.listeners.get("pointerdown")!(event); + expect(f.close).toHaveBeenCalledOnce(); + expect(event.preventDefault).not.toHaveBeenCalled(); + f.dispose(); + expect(f.triggerButton.focus).not.toHaveBeenCalled(); + }); +}); + + +describe("root-owned submenu Tab exit", () => { + it("Tab and Shift+Tab from a descendant skip the whole popup", () => { + for (const backwards of [false, true]) { + const f = fixture(); + const childMenu = {}; + const child = { + focus: vi.fn(), + getAttribute: () => null, + closest: () => childMenu, + }; + const originalContains = f.menu.contains; + f.menu.contains = node => node === child || originalContains(node); + const before = { focus: vi.fn(), getAttribute: () => null }; + const after = { focus: vi.fn(), getAttribute: () => null }; + Object.assign(f.doc, { + querySelectorAll: () => [before, f.triggerButton, after, ...f.items, child], + }); + f.doc.activeElement = child; + const event = { ...f.event("Tab", child), shiftKey: backwards }; + f.listeners.get("keydown")!(event); + expect(f.close).toHaveBeenCalledOnce(); + f.dispose(); + expect((backwards ? before : after).focus).toHaveBeenCalledOnce(); + expect(child.focus).not.toHaveBeenCalled(); + expect(f.triggerButton.focus).not.toHaveBeenCalled(); + } + }); + it("nested binders leave Tab to the root without publishing a second close", () => { + const closeRoot = vi.fn(); + const f = fixture({ nested: true, closeRoot }); + const event = f.event("Tab"); + f.listeners.get("keydown")!(event); + expect(f.close).not.toHaveBeenCalled(); + expect(closeRoot).not.toHaveBeenCalled(); + expect(event.preventDefault).not.toHaveBeenCalled(); + f.dispose(); + }); +}); diff --git a/paseo-shared-browser/client/browser-toolbar-menu-web.ts b/paseo-shared-browser/client/browser-toolbar-menu-web.ts new file mode 100644 index 00000000..06944354 --- /dev/null +++ b/paseo-shared-browser/client/browser-toolbar-menu-web.ts @@ -0,0 +1,148 @@ +/** Web focus and dismissal for local menus; never forwards menu keys to Chromium. */ +import { Platform } from "react-native"; + +interface FocusNode { + focus(): void; + contains(node: unknown): boolean; + getAttribute(name: string): string | null; + getClientRects?(): ArrayLike; + closest?(selector: string): unknown; +} +interface MenuNode extends FocusNode { + querySelectorAll(selector: string): ArrayLike; + ownerDocument: { + activeElement: unknown; + querySelectorAll?(selector: string): ArrayLike; + addEventListener(type: string, listener: (event: MenuEvent) => void, capture: boolean): void; + removeEventListener(type: string, listener: (event: MenuEvent) => void, capture: boolean): void; + }; +} +interface MenuEvent { + key?: string; + shiftKey?: boolean; + target: unknown; + preventDefault(): void; + stopPropagation(): void; +} + +/** Restore trigger focus only if the menu still owns it and the action did not hand it away. */ +export function bindBrowserToolbarMenuWeb( + node: unknown, + trigger: unknown, + close: () => void, + restoreFocus: () => boolean, + overlay?: unknown, + hierarchy?: { openSubmenu?(): void; back?(): void; closeRoot?(): void; nested?: boolean }, +): () => void { + if (Platform.OS !== "web" || !node || typeof node !== "object" || !("ownerDocument" in node)) { + return () => {}; + } + const menu = node as MenuNode; + const anchor = trigger as MenuNode | null; + const enabledItems = () => + Array.from(menu.querySelectorAll('[role="menuitem"]')).filter( + (item) => + item.getAttribute("aria-disabled") !== "true" && item.getAttribute("disabled") === null && + (!item.closest || item.closest('[role="menu"]') === menu), + ); + enabledItems()[0]?.focus(); + let focusOwned = menu.contains(menu.ownerDocument.activeElement); + let returnTarget: FocusNode | null = null; + const focusin = (event: MenuEvent) => { + focusOwned = menu.contains(event.target); + }; + + const keydown = (event: MenuEvent) => { + if (!menu.contains(event.target)) return; + const target = event.target as FocusNode; + // Only the root owns Tab exit. Its host-order calculation excludes the + // entire popup, and its cleanup performs the one final focus restoration. + if (event.key === "Tab" && hierarchy?.nested) return; + if (event.key !== "Tab" && target.closest && target.closest('[role="menu"]') !== menu) return; + if (event.key === "ArrowRight" && target.getAttribute("aria-expanded") !== null && hierarchy?.openSubmenu) { + event.preventDefault(); + event.stopPropagation(); + hierarchy.openSubmenu(); + return; + } + if ((event.key === "ArrowLeft" || event.key === "Escape") && hierarchy?.back) { + event.preventDefault(); + event.stopPropagation(); + hierarchy.back(); + return; + } + if (event.key === "Escape" || event.key === "Tab") { + if (event.key === "Tab") { + // Menu Tab leaves the popup rather than walking its command rows or + // trapping focus. Follow the trigger's surrounding host tab order. + const triggerButton = Array.from(anchor?.querySelectorAll('[role="button"], [role="menuitem"]') ?? [])[0]; + const focusable = Array.from( + menu.ownerDocument.querySelectorAll?.( + 'button, [role="button"], input:not([type="hidden"]), select, textarea, a[href], [tabindex]', + ) ?? [], + ).filter( + (item) => + !menu.contains(item) && + item.getAttribute("disabled") === null && + item.getAttribute("aria-disabled") !== "true" && + Number(item.getAttribute("tabindex") ?? 0) >= 0 && + !item.closest?.('[aria-hidden="true"], [inert]') && + (!item.getClientRects || item.getClientRects().length > 0), + ); + const index = triggerButton ? focusable.indexOf(triggerButton) : -1; + if (index >= 0) + returnTarget = focusable[index + (event.shiftKey ? -1 : 1)] ?? triggerButton ?? null; + } + event.preventDefault(); + event.stopPropagation(); + if (event.key === "Tab" && hierarchy?.closeRoot) hierarchy.closeRoot(); + else close(); + return; + } + if (!["ArrowDown", "ArrowUp", "Home", "End"].includes(event.key ?? "")) return; + event.preventDefault(); + event.stopPropagation(); + const items = enabledItems(); + if (!items.length) return; + const index = items.indexOf(menu.ownerDocument.activeElement as FocusNode); + let next: number; + if (event.key === "Home") next = 0; + else if (event.key === "End") next = items.length - 1; + else if (event.key === "ArrowDown") next = (index + 1) % items.length; + else next = index <= 0 ? items.length - 1 : index - 1; + items[next]?.focus(); + }; + const outside = (event: MenuEvent) => { + if (hierarchy?.nested || menu.contains(event.target) || anchor?.contains(event.target)) return; + // Consume backdrop presses before re-enabling the image below it. A local + // dismiss must never become a newly controlled remote mouse-down. + if (overlay && (overlay as FocusNode).contains(event.target)) { + event.preventDefault(); + event.stopPropagation(); + } else { + // The user deliberately focused another host surface; leave it alone. + focusOwned = false; + } + close(); + }; + menu.ownerDocument.addEventListener("keydown", keydown, true); + menu.ownerDocument.addEventListener("pointerdown", outside, true); + menu.ownerDocument.addEventListener("focusin", focusin, true); + return () => { + menu.ownerDocument.removeEventListener("keydown", keydown, true); + menu.ownerDocument.removeEventListener("pointerdown", outside, true); + menu.ownerDocument.removeEventListener("focusin", focusin, true); + // Native React unmount can remove the focused row before passive cleanup. + // Remember ownership rather than consulting only the now-detached node. + if (restoreFocus() && focusOwned) { + (returnTarget ?? Array.from(anchor?.querySelectorAll('[role="button"], [role="menuitem"]') ?? [])[0])?.focus(); + } + }; +} + +/** Restore an in-menu trigger after compact Back, without touching native hosts. */ +export function focusBrowserMenuTrigger(node: unknown): void { + if (Platform.OS !== "web" || !node || typeof node !== "object" || !("querySelectorAll" in node)) return; + const trigger = node as MenuNode; + Array.from(trigger.querySelectorAll('[role="menuitem"], [role="button"]'))[0]?.focus(); +} diff --git a/paseo-shared-browser/client/browser-toolbar-menu.tsx b/paseo-shared-browser/client/browser-toolbar-menu.tsx new file mode 100644 index 00000000..52d3e4a2 --- /dev/null +++ b/paseo-shared-browser/client/browser-toolbar-menu.tsx @@ -0,0 +1,233 @@ +/** + * One anchored menu surface for workspace-panel toolbar controls. The public SDK + * has no inline menu, so use its theme and public RN primitives without importing + * host internals. Placement stays inside the pane, including compact clients. + */ +import type { PluginHostProps } from "@getpaseo/plugin/client"; +import { Icon } from "@getpaseo/plugin/client/react-native"; +import { + type ReactNode, + type RefObject, + useEffect, + useLayoutEffect, + useRef, + useState, +} from "react"; +import { BackHandler, Platform, Pressable, ScrollView, StyleSheet, Text, View } from "react-native"; +import { browserMenuPlacement, type MenuRect } from "./browser-menu-placement"; +import { BrowserToolbarSubmenu, type BrowserSubmenu } from "./browser-toolbar-submenu"; +import { bindBrowserToolbarMenuWeb, focusBrowserMenuTrigger } from "./browser-toolbar-menu-web"; + +type Theme = PluginHostProps["theme"]; +const STYLES = StyleSheet.create({ + overlay: { ...StyleSheet.absoluteFillObject, zIndex: 100 }, + menu: { position: "absolute", borderWidth: 1, borderRadius: 8, overflow: "hidden", elevation: 8 }, + content: { paddingVertical: 4 }, + item: { + minHeight: 36, + paddingHorizontal: 12, + paddingVertical: 8, + flexDirection: "row", + alignItems: "center", + gap: 8, + }, + label: { fontSize: 13, flexShrink: 1, flex: 1 }, + heading: { + fontSize: 11, + fontWeight: "600", + paddingHorizontal: 12, + paddingTop: 10, + paddingBottom: 4, + }, + separator: { borderBottomWidth: 1, marginVertical: 4 }, +}); + +export interface BrowserToolbarMenuProps { + theme: Theme; + title: string; + compact: boolean; + paneRef: RefObject; + anchorRef: RefObject; + paneSize: { width: number; height: number }; + /** Estimated body height; overflow remains scrollable instead of growing outside the pane. */ + preferredHeight: number; + onClose(): void; + shouldRestoreFocus(): boolean; + children: ReactNode; + submenu?: BrowserSubmenu | undefined; + onSubmenuOpen?(): void; +} + +/** Measure after layout, reject obsolete callbacks, and re-anchor on pane resizing. */ +export function BrowserToolbarMenu(props: BrowserToolbarMenuProps) { + const compact = props.compact || Platform.OS !== "web"; + const previousSubmenu = useRef(undefined); + const [placement, setPlacement] = useState(null); + const menuRef = useRef(null); + const overlayRef = useRef(null); + const current = useRef(props); + current.current = props; + useLayoutEffect(() => { + let active = true; + props.paneRef.current?.measureInWindow((paneX, paneY) => { + if (!active) return; + props.anchorRef.current?.measureInWindow((x, y, width, height) => { + if (!active) return; + setPlacement( + browserMenuPlacement( + props.paneSize, + { x: x - paneX, y: y - paneY, width, height }, + compact && props.submenu ? props.submenu.preferredHeight + 52 : props.preferredHeight, + ), + ); + }); + }); + return () => { + active = false; + }; + }, [ + props.paneRef, + props.anchorRef, + props.paneSize.width, + props.paneSize.height, + props.preferredHeight, + compact, + Boolean(props.submenu), + ]); + + useEffect(() => { + if (!placement) return; + const dispose = bindBrowserToolbarMenuWeb( + menuRef.current, + props.anchorRef.current, + () => current.current.onClose(), + () => current.current.shouldRestoreFocus(), + overlayRef.current, + { openSubmenu: () => current.current.onSubmenuOpen?.(), + ...(compact && props.submenu ? { back: () => current.current.submenu?.onBack() } : {}) }, + ); + if (compact && previousSubmenu.current && !props.submenu) { + focusBrowserMenuTrigger(previousSubmenu.current.anchorRef.current); + } + previousSubmenu.current = props.submenu; + return dispose; + }, [Boolean(placement), props.anchorRef, compact, compact && Boolean(props.submenu)]); + useEffect(() => { + if (Platform.OS === "web") return; + const subscription = BackHandler.addEventListener("hardwareBackPress", () => { + if (current.current.submenu) current.current.submenu.onBack(); + else current.current.onClose(); + return true; + }); + return () => subscription.remove(); + }, []); + + return ( + + + {placement ? ( + + + {compact && props.submenu ? ( + <> + + {props.submenu.title} + {props.submenu.children} + + ) : props.children} + + {!compact && props.submenu ? ( + + ) : null} + + ) : null} + + ); +} + +/** Standard menu row; checked choices use an indicator instead of a raised action button. */ +export function BrowserMenuItem({ + theme, + compact, + label, + icon, + selected, + disabled = false, + expanded, + onPress, +}: { + theme: Theme; + compact: boolean; + label: string; + icon?: string; + selected?: boolean; + disabled?: boolean; + expanded?: boolean; + onPress(): void; +}) { + const [hovered, setHovered] = useState(false); + const [focused, setFocused] = useState(false); + return ( + setHovered(true)} + onHoverOut={() => setHovered(false)} + onFocus={() => setFocused(true)} + onBlur={() => setFocused(false)} + onPress={onPress} + style={({ pressed }) => [ + STYLES.item, + { + minHeight: compact ? 44 : 36, + backgroundColor: hovered || focused ? theme.colors.surface2 : "transparent", + opacity: disabled ? 0.45 : pressed ? 0.72 : 1, + }, + ]} + > + {icon ? : null} + {label} + {expanded !== undefined ? : null} + {selected ? : null} + + ); +} + +/** Group labels and separators share the same themed menu chrome. */ +export function BrowserMenuHeading({ theme, children }: { theme: Theme; children: ReactNode }) { + return {children}; +} + +export function BrowserMenuSeparator({ theme }: { theme: Theme }) { + return ; +} diff --git a/paseo-shared-browser/client/browser-toolbar-submenu.tsx b/paseo-shared-browser/client/browser-toolbar-submenu.tsx new file mode 100644 index 00000000..3552ea9d --- /dev/null +++ b/paseo-shared-browser/client/browser-toolbar-submenu.tsx @@ -0,0 +1,93 @@ +/** Desktop flyouts remain descendants of the root menu and share its one backdrop. */ +import type { PluginHostProps } from "@getpaseo/plugin/client"; +import { useLayoutEffect, useRef, useState, type ReactNode, type RefObject } from "react"; +import { ScrollView, View } from "react-native"; +import { browserSubmenuPlacement, type MenuRect } from "./browser-menu-placement"; +import { bindBrowserToolbarMenuWeb } from "./browser-toolbar-menu-web"; + +export interface BrowserSubmenu { + title: string; + anchorRef: RefObject; + preferredHeight: number; + children: ReactNode; + onBack(): void; +} + +/** Measure the actual trigger row; refuse obsolete callbacks after close/resize. */ +export function BrowserToolbarSubmenu(props: { + theme: PluginHostProps["theme"]; + submenu: BrowserSubmenu; + paneRef: RefObject; + paneSize: { width: number; height: number }; + parent: MenuRect; + onClose(): void; +}) { + const [placement, setPlacement] = useState(null); + const menu = useRef(null); + const current = useRef(props); + current.current = props; + useLayoutEffect(() => { + let active = true; + props.paneRef.current?.measureInWindow((paneX, paneY) => { + props.submenu.anchorRef.current?.measureInWindow((x, y, width, height) => { + if (!active) return; + setPlacement( + browserSubmenuPlacement( + props.paneSize, + props.parent, + { x: x - paneX, y: y - paneY, width, height }, + props.submenu.preferredHeight, + ), + ); + }); + }); + return () => { + active = false; + }; + }, [props.paneSize.width, props.paneSize.height, props.parent, props.submenu.anchorRef]); + useLayoutEffect(() => { + if (!placement) return; + return bindBrowserToolbarMenuWeb( + menu.current, + props.submenu.anchorRef.current, + () => current.current.submenu.onBack(), + () => true, + undefined, + { + nested: true, + back: () => current.current.submenu.onBack(), + closeRoot: () => current.current.onClose(), + }, + ); + }, [Boolean(placement), props.submenu.anchorRef]); + if (!placement) return null; + return ( + + + {props.submenu.children} + + + ); +} diff --git a/paseo-shared-browser/client/browser-viewer-recovery.test.ts b/paseo-shared-browser/client/browser-viewer-recovery.test.ts new file mode 100644 index 00000000..1ed66e03 --- /dev/null +++ b/paseo-shared-browser/client/browser-viewer-recovery.test.ts @@ -0,0 +1,92 @@ +import { QueryClient, QueryObserver } from "@tanstack/react-query"; +import { describe, expect, it } from "vitest"; +import { + createBrowserViewerRecovery, + isExpiredBrowserViewerError, +} from "./browser-viewer-recovery"; + +const expired = Object.assign( + new Error( + "Request failed: Viewer token is invalid or expired requestType=plugin.rpc.invoke.request code=handler_error", + ), + { code: "handler_error" }, +); +const observation = { + identity: "host-a/workspace-a", + viewerToken: "old-viewer", + failedViewerToken: "old-viewer", + captureError: expired, + pending: false, +}; + +describe("expired browser viewing recovery", () => { + it("matches the actual SDK handler-error suffix but not transport, control or arbitrary failures", () => { + expect(isExpiredBrowserViewerError(expired)).toBe(true); + expect(isExpiredBrowserViewerError(Object.assign(new Error("Request failed: Viewer token is invalid or expired"), { code: "handler_error" }))).toBe(true); + expect(isExpiredBrowserViewerError(new Error("Viewer token is invalid or expired"))).toBe(true); + for (const error of [ + new Error("Control lease is invalid or expired"), + new Error("Browser frame is stale"), + new Error("Bridge disconnected"), + Object.assign(new Error("Viewer token is invalid or expired"), { code: "unknown_outcome" }), + Object.assign(new Error("Viewer token is invalid or expired"), { code: "rpc_timeout" }), + { code: "handler_error", message: "Other error: Viewer token is invalid or expired" }, + ]) + expect(isExpiredBrowserViewerError(error)).toBe(false); + }); + + it("reattaches once for each expired token and leaves failures for explicit reconnect", () => { + const claim = createBrowserViewerRecovery(); + expect(claim({ ...observation, pending: true })).toBe(false); + expect(claim(observation)).toBe(true); + expect(claim(observation)).toBe(false); + expect(claim({ ...observation, pending: true })).toBe(false); + expect(claim({ ...observation, viewerToken: "new", failedViewerToken: "new" })).toBe(true); + }); + + it("ignores obsolete token errors, missing attachments and unrelated current capture errors", () => { + const claim = createBrowserViewerRecovery(); + expect(claim({ ...observation, viewerToken: "replacement" })).toBe(false); + expect(claim({ ...observation, viewerToken: null })).toBe(false); + expect(claim({ ...observation, captureError: new Error("Timeout") })).toBe(false); + expect(claim(observation)).toBe(true); + }); + + it("permits a distinct scope's attachment while retaining per-token loop protection", () => { + const claim = createBrowserViewerRecovery(); + expect(claim(observation)).toBe(true); + expect(claim(observation)).toBe(false); + expect(claim({ ...observation, identity: "host-b/workspace-a" })).toBe(true); + expect(claim({ ...observation, identity: "host-b/workspace-a" })).toBe(false); + }); +}); + + +it("the actual capture query observer discards old token errors when its token key changes", async () => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + const replacement = Promise.withResolvers(); + const observer = new QueryObserver(client, { + queryKey: ["shared-browser", "capture", "old-viewer"], + queryFn: async (): Promise => { throw expired; }, + }); + const unsubscribe = observer.subscribe(() => {}); + try { + await observer.refetch(); + expect(observer.getCurrentResult().error).toBe(expired); + observer.setOptions({ + queryKey: ["shared-browser", "capture", "new-viewer"], + queryFn: () => replacement.promise, + retry: false, + }); + expect(observer.getCurrentResult().error).toBeNull(); + expect(observer.getCurrentResult().status).toBe("pending"); + replacement.resolve("fresh capture"); + await observer.refetch({ cancelRefetch: false }); + expect(observer.getCurrentResult().error).toBeNull(); + expect(observer.getCurrentResult().data).toBe("fresh capture"); + } finally { + replacement.resolve("cleanup"); + unsubscribe(); + client.clear(); + } +}); diff --git a/paseo-shared-browser/client/browser-viewer-recovery.ts b/paseo-shared-browser/client/browser-viewer-recovery.ts new file mode 100644 index 00000000..c566ae48 --- /dev/null +++ b/paseo-shared-browser/client/browser-viewer-recovery.ts @@ -0,0 +1,60 @@ +/** + * Recover viewing after its short-lived viewer token expires during background + * suspension. Only a capture error can use this policy; mutation failure must + * never trigger action replay, control acquisition or lease takeover. + */ +const VIEWER_EXPIRED_MESSAGE = "Viewer token is invalid or expired"; + +/** Match the owned handler error, including Paseo's documented RPC error suffix. */ +export function isExpiredBrowserViewerError(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const value = error as { message?: unknown; code?: unknown }; + if (typeof value.message !== "string") return false; + if (value.code !== undefined && value.code !== "handler_error") return false; + // Daemon session.js applies this exact prefix before the client appends + // requestType/code. Arbitrary error text never grants automatic recovery. + const message = value.code === "handler_error" && value.message.startsWith("Request failed: ") + ? value.message.slice("Request failed: ".length) + : value.message; + if (message === VIEWER_EXPIRED_MESSAGE) return true; + return value.code === "handler_error" && + message.startsWith(`${VIEWER_EXPIRED_MESSAGE} requestType=`); +} + +export interface BrowserViewerRecoveryObservation { + /** Host/workspace scope of the mounted panel, never a printer or page identity. */ + identity: string; + viewerToken: string | null; + /** Exact token of the failed capture query; an old viewer cannot recover its replacement. */ + failedViewerToken: string | null; + captureError: unknown; + pending: boolean; +} + +/** + * Claim at most one automatic viewing reattachment per expired token. Failure + * keeps manual Reconnect available. A new token or panel scope permits its own + * later recovery; this helper neither issues input nor holds browser authority. + */ +export function createBrowserViewerRecovery() { + let scope: string | null = null; + let attemptedToken: string | null = null; + return (observation: BrowserViewerRecoveryObservation): boolean => { + if (scope !== observation.identity) { + scope = observation.identity; + attemptedToken = null; + } + const token = observation.viewerToken; + if ( + observation.pending || + !token || + observation.failedViewerToken !== token || + attemptedToken === token || + !isExpiredBrowserViewerError(observation.captureError) + ) { + return false; + } + attemptedToken = token; + return true; + }; +} diff --git a/paseo-shared-browser/client/browser.tsx b/paseo-shared-browser/client/browser.tsx index 024018e2..06c31d5c 100644 --- a/paseo-shared-browser/client/browser.tsx +++ b/paseo-shared-browser/client/browser.tsx @@ -3,15 +3,13 @@ import { type PluginWorkspacePanelProps, useRpc, } from "@getpaseo/plugin/client"; -import { Icon, Modal, ScrollView, TextInput } from "@getpaseo/plugin/client/react-native"; +import { Icon, Modal, TextInput } from "@getpaseo/plugin/client/react-native"; import { useMutation, useQuery } from "@tanstack/react-query"; import { type ReactNode, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ActivityIndicator, - type GestureResponderEvent, - Image, type LayoutChangeEvent, - PanResponder, + TextInput as NativeTextInput, Pressable, type StyleProp, StyleSheet, @@ -28,13 +26,13 @@ import { type BrowserFrame, type BrowserInputEvent, type BrowserState, + beginBrowserGestureRpc, captureBrowserRpc, - containedRect, - createImageSizeLoader, DEVICE_PRESETS, type DevicePresetId, detachBrowserRpc, didBrowserRuntimeRestart, + endBrowserGestureRpc, isBrowserStateCurrent, listOpenBrowserWorkspacesRpc, MAX_VIEWPORT, @@ -43,9 +41,29 @@ import { releaseControlRpc, resizeBrowserRpc, sendBrowserInputRpc, - toViewportPoint, + updateBrowserGestureRpc, } from "../shared/browser"; -import { FrameLifecycle } from "./frame-lifecycle"; +import { groupResolutionPresets } from "../shared/resolution-menu"; +import { type BrowserCanvasDisplayMode, getBrowserCanvasLayout } from "./browser-canvas-layout"; +import { BrowserCanvasViewport } from "./browser-canvas-viewport"; +import { BrowserFrameImage } from "./browser-frame-image"; +import { browserCaptureInterval } from "./browser-capture-cadence"; +import { matchingResolutionPresetId, type EmulationSelection } from "./browser-emulation-mode"; +import type { FrameCandidate } from "./browser-frame-buffer"; +import { BrowserResolutionPicker } from "./browser-resolution-picker"; +import { + BrowserMenuHeading, + BrowserMenuItem, + BrowserMenuSeparator, + BrowserToolbarMenu, +} from "./browser-toolbar-menu"; +import { isExpiredBrowserViewerError } from "./browser-viewer-recovery"; +import { createFrameLifecycle } from "./frame-lifecycle"; +import { useBrowserCanvasInput } from "./use-browser-canvas-input"; +import { useBrowserEmulationMode } from "./use-browser-emulation-mode"; +import { useBrowserFrameBuffer } from "./use-browser-frame-buffer"; +import { useBrowserViewerRecovery } from "./use-browser-viewer-recovery"; +import { useResolutionFavorites } from "./use-resolution-favorites"; const SPACE = { xxs: 2, @@ -59,32 +77,21 @@ const TYPE = { caption: 11, body: 13, title: 14 } as const; const DIMENSION = { control: 34, touch: 44, - pad: 60, icon: 15, - addressCompact: 120, addressRegular: 220, canvasCompact: 220, canvasRegular: 320, helperMax: 420, viewportField: 72, - typeField: 180, screenRadius: 20, } as const; -const DRAG_THRESHOLD = 6; -const SCROLL_STEP = 520; -const CAPTURE_INTERVAL_READY = 250; -const CAPTURE_INTERVAL_WAITING = 1_500; const MAX_VIEWER_LABEL_LENGTH = 64; const PILL_PRESENCE_POLL_MS = 2_000; const AGENT_DIRECTORY_PAGE_LIMIT = 200; const MAX_URL_LENGTH = 8_192; -const MAX_TEXT_LENGTH = 4_000; const BYTES_PER_KIBIBYTE = 1_024; -const MAX_SCROLL_DELTA = 4_000; type Theme = PluginWorkspacePanelProps["theme"]; -type InteractionMode = "click" | "double" | "right"; -type SwipeMode = "scroll" | "drag"; type SpecialKey = Extract["key"]; interface Size { @@ -92,11 +99,6 @@ interface Size { height: number; } -interface DisplayPoint { - x: number; - y: number; -} - const SPECIAL_KEYS: readonly { key: SpecialKey; label: string }[] = [ { key: "Enter", label: "Enter" }, { key: "Tab", label: "Tab" }, @@ -146,10 +148,6 @@ function isFrameCurrent(frame: BrowserFrame, state: BrowserState): boolean { ); } -function clampScrollDelta(delta: number): number { - return Math.max(-MAX_SCROLL_DELTA, Math.min(MAX_SCROLL_DELTA, Math.round(delta))); -} - function createStyles(theme: Theme, compact: boolean) { return StyleSheet.create({ screen: { @@ -216,12 +214,20 @@ function createStyles(theme: Theme, compact: boolean) { }, addressInput: { flex: 1, - minWidth: compact ? DIMENSION.addressCompact : DIMENSION.addressRegular, + minWidth: compact ? 40 : DIMENSION.addressRegular, }, chromeAddressInput: { - height: 28, + height: compact ? 34 : 28, borderRadius: RADIUS.md, backgroundColor: theme.colors.surface1, + // Compact chrome must override the themed input's form-sized padding. + // Keep a full text line even on native hosts with their own font padding. + paddingVertical: 0, + paddingHorizontal: SPACE.xs, + fontSize: TYPE.body, + lineHeight: 18, + includeFontPadding: false, + textAlignVertical: "center", }, chromeIconButton: { width: 28, @@ -239,11 +245,7 @@ function createStyles(theme: Theme, compact: boolean) { chromeIconButtonDisabled: { opacity: 0.45, }, - toolbarContent: { - flexDirection: "row", - alignItems: "center", - gap: SPACE.xs, - }, + button: { minHeight: DIMENSION.control, minWidth: DIMENSION.control, @@ -335,10 +337,6 @@ function createStyles(theme: Theme, compact: boolean) { minHeight: 0, overflow: "hidden", }, - frame: { - position: "absolute", - borderRadius: compact ? DIMENSION.screenRadius - SPACE.xs : 0, - }, interactionLayer: { position: "absolute", }, @@ -377,15 +375,7 @@ function createStyles(theme: Theme, compact: boolean) { fontSize: TYPE.caption, flexShrink: 1, }, - controls: { - paddingHorizontal: compact ? SPACE.md : SPACE.sm, - paddingTop: compact ? SPACE.md : SPACE.sm, - paddingBottom: compact ? SPACE.lg : SPACE.sm, - gap: compact ? SPACE.sm : SPACE.xs, - borderTopWidth: 1, - borderTopColor: theme.colors.border, - backgroundColor: theme.colors.surface0, - }, + mobileRow: { flexDirection: "row", alignItems: "center", @@ -400,19 +390,6 @@ function createStyles(theme: Theme, compact: boolean) { flexWrap: "wrap", gap: SPACE.sm, }, - sheetPad: { - alignItems: "center", - gap: SPACE.sm, - }, - sheetPadRow: { - flexDirection: "row", - alignItems: "center", - justifyContent: "center", - gap: SPACE.sm, - }, - padSpacer: { - width: DIMENSION.pad, - }, controlStrip: { minHeight: DIMENSION.control, flexDirection: "row", @@ -425,13 +402,6 @@ function createStyles(theme: Theme, compact: boolean) { fontWeight: "600", marginRight: SPACE.xs, }, - separator: { - width: 1, - alignSelf: "stretch", - marginVertical: SPACE.xs, - marginHorizontal: SPACE.xs, - backgroundColor: theme.colors.border, - }, viewportField: { width: DIMENSION.viewportField, textAlign: "center", @@ -440,20 +410,7 @@ function createStyles(theme: Theme, compact: boolean) { color: theme.colors.foregroundMuted, fontSize: TYPE.body, }, - typeRow: { - flexDirection: compact ? "column" : "row", - gap: SPACE.xs, - }, - typeInput: { - flex: 1, - minWidth: compact ? undefined : DIMENSION.typeField, - }, - keysContent: { - flexDirection: "row", - alignItems: "center", - gap: SPACE.xs, - paddingTop: SPACE.xs, - }, + deviceModalContent: { gap: SPACE.sm, padding: SPACE.sm, @@ -497,10 +454,6 @@ function createStyles(theme: Theme, compact: boolean) { buttonFill: { flex: 1, }, - buttonPad: { - width: DIMENSION.pad, - minHeight: DIMENSION.touch, - }, }); } @@ -515,7 +468,6 @@ interface ControlButtonStyles { buttonDisabled: ViewStyle; buttonLarge: ViewStyle; buttonFill: ViewStyle; - buttonPad: ViewStyle; buttonText: TextStyle; buttonTextSelected: TextStyle; } @@ -548,7 +500,6 @@ interface ControlButtonProps { danger?: boolean; large?: boolean; fill?: boolean; - pad?: boolean; disabled?: boolean; onPress(): void; } @@ -564,7 +515,6 @@ function ControlButton({ danger = false, large = false, fill = false, - pad = false, disabled = false, onPress, }: ControlButtonProps) { @@ -591,9 +541,8 @@ function ControlButton({ pressed ? styles.buttonPressed : null, focused ? styles.buttonFocused : null, disabled ? styles.buttonDisabled : null, - large || pad ? styles.buttonLarge : null, + large ? styles.buttonLarge : null, fill ? styles.buttonFill : null, - pad ? styles.buttonPad : null, ]} > {icon ? ( @@ -622,7 +571,9 @@ function ChromeIconButton({ theme, label, icon, + iconNode, selected = false, + expanded, disabled = false, onPress, }: { @@ -630,6 +581,8 @@ function ChromeIconButton({ theme: Theme; label: string; icon: string; + iconNode?: ReactNode; + expanded?: boolean; selected?: boolean; disabled?: boolean; onPress(): void; @@ -639,7 +592,7 @@ function ChromeIconButton({ setHovered(true)} onHoverOut={() => setHovered(false)} @@ -651,11 +604,13 @@ function ChromeIconButton({ disabled ? styles.chromeIconButtonDisabled : null, ]} > - + {iconNode ?? ( + + )} ); } @@ -667,6 +622,8 @@ interface FieldProps { accessibilityLabel: string; placeholder?: string; editable?: boolean; + /** Read-only information can stay legible without implying it is editable. */ + dimWhenReadOnly?: boolean; maxLength?: number; keyboardType?: TextInputProps["keyboardType"]; inputMode?: TextInputProps["inputMode"]; @@ -686,6 +643,7 @@ function Field({ accessibilityLabel, placeholder, editable = true, + dimWhenReadOnly = true, maxLength, keyboardType, inputMode, @@ -727,7 +685,7 @@ function Field({ styles.field, style, focused ? styles.fieldFocused : null, - !editable ? styles.fieldDisabled : null, + !editable && dimWhenReadOnly ? styles.fieldDisabled : null, ]} value={value} /> @@ -964,6 +922,127 @@ export function contributeSharedBrowserClient(client: PluginClientContext) { agents.clear(); }; } +/** Native software keyboard access; complex drafts are inserted only by explicit Done. */ +function NativeKeyboardControls({ + styles, + theme, + relay, + enabled, + ownershipKey, + request, + onRequestHandled, +}: { + styles: ReturnType; + theme: Theme; + relay: ReturnType["nativeKeyboard"]; + enabled: boolean; + ownershipKey: string; + request: { id: number; kind: "keyboard" | "compose"; ownershipKey: string } | null; + onRequestHandled(id: number): void; +}) { + const handledRequest = useRef(null); + const [composeOpen, setComposeOpen] = useState(false); + const [draft, setDraft] = useState(""); + const composeOwner = useRef(null); + const liveAuthority = useRef({ enabled, ownershipKey }); + liveAuthority.current = { enabled, ownershipKey }; + const canCommit = enabled && composeOwner.current === ownershipKey; + const closeCompose = () => { + setComposeOpen(false); + setDraft(""); + composeOwner.current = null; + }; + const openCompose = () => { + relay.inputRef.current?.blur(); + relay.reset(); + composeOwner.current = ownershipKey; + setComposeOpen(true); + }; + const commit = () => { + // Consume before publication: a rapid second Done or retained old handler + // must not repeat insertion or borrow a replacement page's control. + if ( + !liveAuthority.current.enabled || + liveAuthority.current.ownershipKey !== ownershipKey || + composeOwner.current !== ownershipKey || + !draft + ) + return; + composeOwner.current = null; + if (relay.composeText(draft)) { + closeCompose(); + } else { + // false means nothing was admitted, so the same draft remains reviewable. + composeOwner.current = ownershipKey; + } + }; + // Menu dismissal commits before this focus effect. The hidden input remains + // mounted outside the menu so closing it cannot remove the typing sink. + useEffect(() => { + if (!request || handledRequest.current === request.id) return; + handledRequest.current = request.id; + if (enabled && request.ownershipKey === ownershipKey) { + if (request.kind === "keyboard") relay.focus(); + else openCompose(); + } + onRequestHandled(request.id); + }, [request, enabled, ownershipKey, relay, onRequestHandled]); + return ( + <> + + { + if (!open) closeCompose(); + }} + > + + + + Compose with your keyboard, then choose Done to insert the text into the focused page + field. + + + {!canCommit ? ( + + Browser control or page changed. Close this draft and focus the field again. + + ) : null} + + + + + + + + + ); +} + export function SharedBrowserPanel({ theme, host, @@ -986,34 +1065,56 @@ export function SharedBrowserPanel({ const resizeBrowser = useRpc(resizeBrowserRpc); const applyDevicePreset = useRpc(applyDevicePresetRpc); const sendBrowserInput = useRpc(sendBrowserInputRpc); + const beginBrowserGesture = useRpc(beginBrowserGestureRpc); + const updateBrowserGesture = useRpc(updateBrowserGestureRpc); + const endBrowserGesture = useRpc(endBrowserGestureRpc); const mountedRef = useRef(false); const activeViewerTokenRef = useRef(null); const stateRef = useRef(null); - const lifecycleRef = useRef(new FrameLifecycle()); - const lifecycle = lifecycleRef.current; + const frameRef = useRef(null); const lastPointRef = useRef<{ x: number; y: number } | null>(null); - const dragStartRef = useRef<{ x: number; y: number } | null>(null); + const [inputLifecycle] = useState(createFrameLifecycle); + const [legacyInputBusy, setLegacyInputBusy] = useState(false); const captureInFlightRef = useRef(false); const [state, setState] = useState(null); - const [frame, setFrame] = useState(null); - const [inputBusy, setInputBusy] = useState(false); const [controlToken, setControlToken] = useState(null); const [operationError, setOperationError] = useState(null); - const [imageError, setImageError] = useState(false); const [reconnecting, setReconnecting] = useState(false); const [runtimeNotice, setRuntimeNotice] = useState(null); const [addressDraft, setAddressDraft] = useState(""); const [addressFocused, setAddressFocused] = useState(false); const [viewportWidth, setViewportWidth] = useState(""); const [viewportHeight, setViewportHeight] = useState(""); - const [typeDraft, setTypeDraft] = useState(""); - const [interactionMode, setInteractionMode] = useState("click"); const [devicePickerOpen, setDevicePickerOpen] = useState(false); - const [pointerSheetOpen, setPointerSheetOpen] = useState(false); - const [keysSheetOpen, setKeysSheetOpen] = useState(false); - const [swipeMode, setSwipeMode] = useState(layout.compact ? "scroll" : "drag"); + const preferences = useResolutionFavorites(host.id); + const [keysSubmenuOpen, setKeysSubmenuOpen] = useState(false); + const keysAnchorRef = useRef(null); + const [toolbarMenu, setToolbarMenu] = useState<"display" | "actions" | null>(null); + const [scaleMode, setScaleMode] = useState("fit"); + const paneRef = useRef(null); + const displayAnchorRef = useRef(null); + const actionsAnchorRef = useRef(null); + const [paneSize, setPaneSize] = useState({ width: 0, height: 0 }); + const menuRestoreFocus = useRef(true); + const closeToolbarMenu = useCallback((restoreFocus = true) => { + menuRestoreFocus.current = restoreFocus; + setKeysSubmenuOpen(false); + setToolbarMenu(null); + }, []); + const toggleToolbarMenu = (menu: "display" | "actions") => { + menuRestoreFocus.current = true; + setKeysSubmenuOpen(false); + setToolbarMenu((current) => (current === menu ? null : menu)); + }; + const [nativeRequest, setNativeRequest] = useState<{ + id: number; + kind: "keyboard" | "compose"; + ownershipKey: string; + } | null>(null); + const nextNativeRequest = useRef(0); + const [activeInput, setActiveInput] = useState(false); const [containerSize, setContainerSize] = useState({ width: 0, height: 0 }); useEffect(() => { @@ -1023,28 +1124,47 @@ export function SharedBrowserPanel({ }; }, []); - const acceptState = useCallback((next: BrowserState) => { - const previous = stateRef.current; - if (previous && !isBrowserStateCurrent(previous, next)) return false; - const currentFrame = lifecycle.frame; - if (previous && didBrowserRuntimeRestart(previous, next)) { - setRuntimeNotice( - "Browser restarted. The preserved viewer connection now targets the new runtime.", - ); - setControlToken(null); - } - if (currentFrame && !isFrameCurrent(currentFrame, next)) { - lifecycle.drop(); - setFrame(null); - lastPointRef.current = null; - } - stateRef.current = next; - setState(next); - return true; + const isCandidateCurrent = useCallback((candidate: FrameCandidate) => { + const current = stateRef.current; + return ( + mountedRef.current && + candidate.viewerToken === activeViewerTokenRef.current && + candidate.mutationEpoch === inputLifecycle.epoch && + current !== null && + isFrameCurrent(candidate.frame, current) + ); }, []); + const { buffer, frame, receive, settled, discardObsoletePending, reset } = useBrowserFrameBuffer( + frameRef, + isCandidateCurrent, + ); + const imageError = buffer.imageError; + + const acceptState = useCallback( + (next: BrowserState) => { + const previous = stateRef.current; + if (previous && !isBrowserStateCurrent(previous, next)) return false; + const currentFrame = frameRef.current; + if (previous && didBrowserRuntimeRestart(previous, next)) { + setRuntimeNotice( + "Browser restarted. The preserved viewer connection now targets the new runtime.", + ); + setControlToken(null); + } + if (currentFrame && !isFrameCurrent(currentFrame, next)) { + lastPointRef.current = null; + } + stateRef.current = next; + // Repeated captures in the new generation must not cancel its slow decoder. + discardObsoletePending(); + setState(next); + return true; + }, + [discardObsoletePending], + ); const attachQuery = useQuery({ - queryKey: ["shared-browser", "attach", workspaceId, viewerLabel], + queryKey: ["shared-browser", "attach", host.id, workspaceId, viewerLabel], queryFn: async () => { const result = await attachBrowser({ workspaceId, viewerLabel }); if (!mountedRef.current) { @@ -1076,39 +1196,43 @@ export function SharedBrowserPanel({ useEffect(() => { stateRef.current = null; - lifecycle.reset(); + frameRef.current = null; setState(null); - setFrame(null); - setInputBusy(false); + reset(); + inputLifecycle.bump(); + inputLifecycle.reset(); + setLegacyInputBusy(false); setControlToken(null); setOperationError(null); setRuntimeNotice(null); - }, [workspaceId]); + }, [reset, workspaceId, host.id, inputLifecycle]); const captureQuery = useQuery({ - queryKey: ["shared-browser", "capture", viewerToken], + queryKey: ["shared-browser", "capture", viewerToken, preferences.captureQuality], queryFn: async () => { if (!viewerToken) throw new Error("The browser viewer is not attached."); - const mutationEpoch = lifecycle.epoch; - const knownFrame = lifecycle.frame; + const mutationEpoch = inputLifecycle.epoch; + const knownFrame = frameRef.current; captureInFlightRef.current = true; try { const result = await captureBrowser({ viewerToken, - quality: "medium", + quality: preferences.captureQuality, knownFrameId: knownFrame?.frameId ?? null, }); - return { ...result, mutationEpoch }; + return { + ...result, + mutationEpoch, + viewerToken, + captureQuality: preferences.captureQuality, + }; } finally { captureInFlightRef.current = false; } }, enabled: Boolean(viewerToken), retry: false, - refetchInterval: (query) => - query.state.data?.state.status === "ready" - ? CAPTURE_INTERVAL_READY - : CAPTURE_INTERVAL_WAITING, + refetchInterval: (query) => browserCaptureInterval(query.state.data?.state.status, activeInput), refetchIntervalInBackground: false, refetchOnWindowFocus: true, staleTime: 0, @@ -1116,16 +1240,24 @@ export function SharedBrowserPanel({ useEffect(() => { const result = captureQuery.data; - if (!result || !lifecycle.isCurrent(result.mutationEpoch) || !acceptState(result.state)) { + if ( + !result || + result.viewerToken !== activeViewerTokenRef.current || + result.captureQuality !== preferences.captureQuality || + result.mutationEpoch !== inputLifecycle.epoch || + !acceptState(result.state) + ) { return; } if (result.frame && isFrameCurrent(result.frame, result.state)) { - lifecycle.accept(result.mutationEpoch, result.frame); - setImageError(false); - setFrame(lifecycle.visible); - setInputBusy(lifecycle.busy); + receive({ + frame: result.frame, + viewport: result.state.viewport, + viewerToken: result.viewerToken, + mutationEpoch: result.mutationEpoch, + }); } - }, [acceptState, captureQuery.data]); + }, [acceptState, captureQuery.data, preferences.captureQuality, receive]); useEffect(() => { if (state?.controller !== "self" && controlToken) setControlToken(null); @@ -1142,10 +1274,6 @@ export function SharedBrowserPanel({ setViewportHeight(String(state.viewport.height)); }, [state?.sessionId, state?.viewport.height, state?.viewport.width]); - useEffect(() => { - setImageError(false); - }, [frame?.frameId]); - const refreshCapture = useCallback(() => { const requestWasInFlight = captureInFlightRef.current; const request = captureQuery.refetch({ cancelRefetch: false }); @@ -1154,6 +1282,10 @@ export function SharedBrowserPanel({ } }, [captureQuery.refetch]); + const retryFrameCapture = useCallback(() => { + void captureQuery.refetch({ cancelRefetch: false }); + }, [captureQuery.refetch]); + const mutationFailed = useCallback( (error: unknown) => { const message = errorMessage(error); @@ -1203,7 +1335,10 @@ export function SharedBrowserPanel({ const resizeMutation = useMutation({ mutationFn: resizeBrowser, retry: false, - onSuccess: (result) => mutationSucceeded(result.state), + onSuccess: (result) => { + setDevicePickerOpen(false); + mutationSucceeded(result.state); + }, onError: mutationFailed, }); const deviceMutation = useMutation({ @@ -1212,22 +1347,21 @@ export function SharedBrowserPanel({ onSuccess: (result) => mutationSucceeded(result.state), onError: mutationFailed, }); - const settleInput = () => { - lifecycle.settle(); + // Upstream's settlement gate protects discrete key controls and native text fallback. + // Continuous gestures retain their own ordered channel without waiting per event. + const settleLegacyInput = () => { + inputLifecycle.settle(); + setLegacyInputBusy(true); }; const inputMutation = useMutation({ mutationFn: sendBrowserInput, retry: false, - onSuccess: (result, variables) => { - if (variables.event.kind === "type") { - const sentText = variables.event.text; - setTypeDraft((current) => (current === sentText ? "" : current)); - } - settleInput(); + onSuccess: (result) => { + settleLegacyInput(); mutationSucceeded(result.state); }, onError: (error) => { - settleInput(); + settleLegacyInput(); mutationFailed(error); }, }); @@ -1239,45 +1373,48 @@ export function SharedBrowserPanel({ resizeMutation.isPending || deviceMutation.isPending || inputMutation.isPending; - const canControl = Boolean(viewerToken && controlToken && state?.controller === "self"); - const currentFrame = frame && state && isFrameCurrent(frame, state) ? frame : null; - const canSendInput = - canControl && Boolean(currentFrame) && !inputMutation.isPending && !inputBusy; - - const frameUri = useMemo( - () => (currentFrame ? `data:${currentFrame.mimeType};base64,${currentFrame.dataBase64}` : null), - [currentFrame], - ); - const [imageSize, setImageSize] = useState<{ uri: string; size: Size } | null>(null); - const loadImageSize = useMemo( - () => - createImageSizeLoader( - (uri, success, failure) => Image.getSize(uri, success, failure), - (uri, size) => setImageSize(size ? { uri, size } : null), - ), - [], + const viewingExpired = isExpiredBrowserViewerError(captureQuery.error); + const canControl = Boolean( + viewerToken && controlToken && state?.controller === "self" && !viewingExpired, ); + const frontLayer = buffer.front === null ? null : buffer.layers[buffer.front]; + const currentFrame = + frame && + state && + frontLayer?.candidate.viewerToken === viewerToken && + isFrameCurrent(frame, state) + ? frame + : null; useEffect(() => { - if (frameUri) loadImageSize(frameUri); - }, [frameUri, loadImageSize]); - // The bitmap can disagree with the frame metadata; its own aspect ratio decides the rectangle. - const displayRect = useMemo( + const candidate = frontLayer?.candidate; + if (!frame || !candidate || candidate.mutationEpoch !== inputLifecycle.epoch) return; + inputLifecycle.accept(candidate.mutationEpoch, frame); + setLegacyInputBusy(inputLifecycle.busy); + }, [frame, frontLayer, inputLifecycle]); + const canSendInput = + canControl && Boolean(currentFrame) && !inputMutation.isPending && !legacyInputBusy; + + // Layout belongs to the decoded front frame. A denser capture adds detail, + // not a larger 100% layout; scale never writes to the shared browser state. + const canvasLayout = useMemo( () => - currentFrame - ? containedRect( + frame + ? getBrowserCanvasLayout( containerSize, - imageSize?.uri === frameUri - ? imageSize.size - : { width: currentFrame.width, height: currentFrame.height }, + frame, + frontLayer?.candidate.viewport ?? state?.viewport ?? frame, + scaleMode, ) : null, - [containerSize, currentFrame, frameUri, imageSize], + [containerSize, frame, frontLayer, state?.viewport, scaleMode], ); + const displayRect = canvasLayout?.frameRect ?? null; const controlContext = useCallback(() => { const viewer = activeViewerTokenRef.current; const current = stateRef.current; - if (!viewer || !controlToken || !current || current.controller !== "self") return null; + if (viewingExpired || !viewer || !controlToken || !current || current.controller !== "self") + return null; return { viewerToken: viewer, controlToken, @@ -1287,13 +1424,21 @@ export function SharedBrowserPanel({ viewportGeneration: current.viewportGeneration, }, }; - }, [controlToken]); + }, [controlToken, viewingExpired]); const inputContext = useCallback(() => { const context = controlContext(); const current = stateRef.current; - const targetFrame = lifecycle.frame; - if (!context || !current || !targetFrame || !isFrameCurrent(targetFrame, current)) return null; + const targetFrame = frameRef.current; + if ( + !context || + !current || + !targetFrame || + !isFrameCurrent(targetFrame, current) || + frontLayer?.candidate.viewerToken !== context.viewerToken + ) { + return null; + } return { ...context, target: { @@ -1302,7 +1447,7 @@ export function SharedBrowserPanel({ viewportGeneration: targetFrame.viewportGeneration, }, }; - }, [controlContext]); + }, [controlContext, frontLayer]); const requireControlContext = useCallback(() => { const context = controlContext(); @@ -1325,81 +1470,87 @@ export function SharedBrowserPanel({ const sendEvent = useCallback( (event: BrowserInputEvent) => { - if (lifecycle.busy) return; const context = requireInputContext(); - if (!context || !lifecycle.begin()) return; - setInputBusy(true); + if (!context || inputMutation.isPending || !inputLifecycle.begin()) return; + setLegacyInputBusy(true); inputMutation.mutate({ ...context, event }); }, - [inputMutation, lifecycle, requireInputContext], + [inputMutation, requireInputContext], ); - const pointFromEvent = useCallback( - (event: GestureResponderEvent): DisplayPoint | null => { - if (!displayRect) return null; - const x = Math.min(displayRect.width, Math.max(0, event.nativeEvent.locationX)); - const y = Math.min(displayRect.height, Math.max(0, event.nativeEvent.locationY)); - return { x, y }; + const liveInputEnabled = + canSendInput && + !navigateMutation.isPending && + !resizeMutation.isPending && + !deviceMutation.isPending && + !releaseMutation.isPending && + !acquireMutation.isPending && + toolbarMenu === null; + // Forward physical mouse events and genuine touch on every remote preset. + const canvasInput = useBrowserCanvasInput({ + authority: () => { + const context = inputContext(); + const current = stateRef.current; + if (!context || !current?.runtimeId || current.bridgeEpoch === undefined) return null; + return { + ...context, + expected: { + ...context.expected, + runtimeId: current.runtimeId, + bridgeEpoch: current.bridgeEpoch, + }, + }; }, - [displayRect], - ); - - const finishPointerGesture = useCallback( - (event: GestureResponderEvent) => { - const start = dragStartRef.current; - const end = pointFromEvent(event); - const viewport = stateRef.current?.viewport; - dragStartRef.current = null; - if (!start || !end || !displayRect || !viewport) return; - const toViewport = (point: DisplayPoint) => toViewportPoint(point, displayRect, viewport); - lastPointRef.current = { x: end.x, y: end.y }; - const distance = Math.hypot(end.x - start.x, end.y - start.y); - if (distance >= DRAG_THRESHOLD) { - if (swipeMode === "scroll") { - const scale = viewport.width / displayRect.width; - sendEvent({ - kind: "scroll", - point: toViewport(start), - deltaX: clampScrollDelta((start.x - end.x) * scale), - deltaY: clampScrollDelta((start.y - end.y) * scale), - }); - return; - } - sendEvent({ - kind: "drag", - start: toViewport(start), - end: toViewport(end), - button: interactionMode === "right" ? "right" : "left", - }); - return; - } - sendEvent({ - kind: "click", - point: toViewport(end), - button: interactionMode === "right" ? "right" : "left", - clickCount: interactionMode === "double" ? 2 : 1, - }); + transport: { begin: beginBrowserGesture, update: updateBrowserGesture, end: endBrowserGesture }, + ownershipKey: JSON.stringify([ + viewerToken, + controlToken, + state?.sessionId, + state?.runtimeId, + state?.bridgeEpoch, + state?.navigationGeneration, + state?.viewportGeneration, + displayRect?.width, + displayRect?.height, + containerSize.width, + containerSize.height, + scaleMode, + liveInputEnabled, + ]), + decodedFrameId: currentFrame?.frameId ?? null, + enabled: liveInputEnabled, + displaySize: displayRect, + viewport: state?.viewport ?? null, + onState: acceptState, + onError: mutationFailed, + onFinish: refreshCapture, + onPoint: (point) => { + lastPointRef.current = { x: point.x, y: point.y }; }, - [displayRect, interactionMode, pointFromEvent, sendEvent, swipeMode], - ); + onActivity: setActiveInput, + onInputBoundary: () => { + inputLifecycle.bump(); + }, + }); - const panResponder = useMemo( - () => - PanResponder.create({ - onStartShouldSetPanResponder: () => canSendInput, - onMoveShouldSetPanResponder: () => canSendInput, - onPanResponderGrant: (event) => { - const point = pointFromEvent(event); - dragStartRef.current = point ? { x: point.x, y: point.y } : null; - }, - onPanResponderRelease: finishPointerGesture, - onPanResponderTerminate: () => { - dragStartRef.current = null; - }, - onPanResponderTerminationRequest: () => false, - }), - [canSendInput, finishPointerGesture, pointFromEvent], - ); + const nativeOwnershipKey = JSON.stringify([ + viewerToken, + controlToken, + state?.sessionId, + state?.runtimeId, + state?.bridgeEpoch, + state?.navigationGeneration, + state?.viewportGeneration, + ]); + const requestNativeKeyboard = (kind: "keyboard" | "compose") => { + if (!canSendInput) return; + nextNativeRequest.current += 1; + closeToolbarMenu(false); + setNativeRequest({ id: nextNativeRequest.current, kind, ownershipKey: nativeOwnershipKey }); + }; + const nativeRequestHandled = useCallback((id: number) => { + setNativeRequest((current) => (current?.id === id ? null : current)); + }, []); const handleCanvasLayout = useCallback((event: LayoutChangeEvent) => { const { width, height } = event.nativeEvent.layout; @@ -1411,7 +1562,7 @@ export function SharedBrowserPanel({ const takeControl = useCallback( (takeover: boolean) => { if (!viewerToken || acquireMutation.isPending) return; - lifecycle.bump(); + inputLifecycle.bump(); acquireMutation.mutate({ viewerToken, takeover }); }, [acquireMutation, viewerToken], @@ -1426,7 +1577,7 @@ export function SharedBrowserPanel({ ) { return; } - lifecycle.bump(); + inputLifecycle.bump(); releaseMutation.mutate({ viewerToken: viewer, controlToken }); }, [controlToken, releaseMutation]); @@ -1440,11 +1591,11 @@ export function SharedBrowserPanel({ setOperationError("Enter an address to navigate."); return; } - lifecycle.bump(); + inputLifecycle.bump(); navigateMutation.mutate({ ...context, action: { kind: "goto", url: nextUrl } }); return; } - lifecycle.bump(); + inputLifecycle.bump(); navigateMutation.mutate({ ...context, action: { kind: action } }); }, [navigateMutation, requireControlContext], @@ -1468,7 +1619,7 @@ export function SharedBrowserPanel({ ); return; } - lifecycle.bump(); + inputLifecycle.bump(); resizeMutation.mutate({ ...context, viewport: { width, height } }); }, [requireControlContext, resizeMutation, viewportHeight, viewportWidth]); @@ -1476,38 +1627,36 @@ export function SharedBrowserPanel({ (presetId: DevicePresetId) => { const context = requireControlContext(); if (!context || deviceMutation.isPending) return; - lifecycle.bump(); + inputLifecycle.bump(); setDevicePickerOpen(false); deviceMutation.mutate({ ...context, presetId }); }, [deviceMutation, requireControlContext], ); - const scroll = useCallback( - (deltaX: number, deltaY: number) => { - const viewport = stateRef.current?.viewport; - if (!displayRect || !viewport) return; - const previous = lastPointRef.current; - const x = previous && previous.x <= displayRect.width ? previous.x : displayRect.width / 2; - const y = previous && previous.y <= displayRect.height ? previous.y : displayRect.height / 2; - sendEvent({ - kind: "scroll", - point: toViewportPoint({ x, y }, displayRect, viewport), - deltaX, - deltaY, - }); + const applyEmulationSelection = useCallback( + (selection: EmulationSelection) => { + const context = requireControlContext(); + if (!context || anyMutationPending) return; + inputLifecycle.bump(); + deviceMutation.mutate({ ...context, ...selection }); }, - [displayRect, sendEvent], + [requireControlContext, anyMutationPending, inputLifecycle, deviceMutation], ); - - const sendText = useCallback(() => { - if (!typeDraft || inputMutation.isPending) return; - sendEvent({ kind: "type", text: typeDraft }); - }, [inputMutation.isPending, sendEvent, typeDraft]); + const emulation = useBrowserEmulationMode({ + identity: JSON.stringify([host.id, workspaceId]), + platform: layout.platform, + state, + canControl, + pending: anyMutationPending, + apply: applyEmulationSelection, + }); const reconnect = useCallback(() => { if (attachQuery.isFetching) return; - lifecycle.bump(); + inputLifecycle.bump(); + // Immediately revoke local input while the viewing-only reattachment awaits. + activeViewerTokenRef.current = null; setReconnecting(true); setControlToken(null); setOperationError(null); @@ -1516,7 +1665,18 @@ export function SharedBrowserPanel({ }); }, [attachQuery.isFetching, attachQuery.refetch]); - const connectionError = attachQuery.error ?? captureQuery.error; + useBrowserViewerRecovery({ + identity: JSON.stringify([host.id, workspaceId]), + viewerToken, + failedViewerToken: viewerToken, + captureError: captureQuery.error, + pending: reconnecting || attachQuery.isFetching, + reconnect, + }); + + // Expiry belongs to the viewing-only recovery path, not an action failure. + // A failed reattachment still surfaces its attachQuery error and manual retry. + const connectionError = attachQuery.error ?? (viewingExpired ? null : captureQuery.error); const recoveryError = state?.recoveryState === "runtime-unavailable" ? (state.error ?? "Browser runtime unavailable.") @@ -1549,7 +1709,10 @@ export function SharedBrowserPanel({ const activeDevicePreset = state?.devicePresetId ? DEVICE_PRESETS.find(({ id }) => id === state.devicePresetId) : null; - const deviceLabel = activeDevicePreset?.label ?? "Custom display"; + const selectedResolutionPresetId = matchingResolutionPresetId(state); + const deviceLabel = selectedResolutionPresetId + ? activeDevicePreset?.label ?? "Custom display" + : `${emulation.mode === "mobile" ? "Mobile" : "Desktop"} · custom display`; const transportLabel = currentFrame?.transport === "cdp-screencast" ? "CDP" : "fallback"; const frameSummary = currentFrame ? layout.compact @@ -1597,17 +1760,6 @@ export function SharedBrowserPanel({ ); } - const imageStyle = displayRect - ? [ - styles.frame, - { - left: displayRect.x, - top: displayRect.y, - width: displayRect.width, - height: displayRect.height, - }, - ] - : styles.frame; const interactionStyle = displayRect ? [ styles.interactionLayer, @@ -1621,7 +1773,17 @@ export function SharedBrowserPanel({ : styles.interactionLayer; return ( - + { + const { width, height } = event.nativeEvent.layout; + setPaneSize((previous) => + previous.width === width && previous.height === height ? previous : { width, height }, + ); + }} + > setAddressFocused(false)} onSubmit={() => navigate("goto", addressDraft)} /> + + toggleToolbarMenu("display")} + /> + setDevicePickerOpen(true)} + label={`${emulation.mode === "mobile" ? "Disable" : "Enable"} mobile emulation`} + icon="Smartphone" + selected={emulation.mode === "mobile"} + disabled={!canControl || anyMutationPending} + onPress={emulation.toggle} /> + + toggleToolbarMenu("actions")} + /> + @@ -1703,30 +1889,50 @@ export function SharedBrowserPanel({ ) : null} - - {frameUri && displayRect && !imageError ? ( - <> - + {buffer.layers.map((layer, slot) => { + if (!layer) return null; + const rect = getBrowserCanvasLayout( + containerSize, + layer.candidate.frame, + layer.candidate.viewport ?? state?.viewport ?? layer.candidate.frame, + scaleMode, + )?.frameRect; + if (!rect) return null; + const visible = slot === buffer.front; + return ( + { - lifecycle.frame = null; - setImageError(true); - void captureQuery.refetch({ cancelRefetch: false }); - }} - resizeMode="contain" - source={{ uri: frameUri }} - style={imageStyle} + settled={settled} + retry={retryFrameCapture} + source={layer.source} + x={rect.x} + y={rect.y} + width={rect.width} + height={rect.height} + cornerRadius={layout.compact ? DIMENSION.screenRadius - SPACE.xs : 0} /> - - + ); + })} + {frame && displayRect ? ( + ) : imageError ? ( )} - + {state?.title || state?.url || "Shared browser"} @@ -1772,377 +1978,59 @@ export function SharedBrowserPanel({ - - {layout.compact ? ( - <> - - - - - - setPointerSheetOpen(true)} - /> - setKeysSheetOpen(true)} - /> - - - ) : ( - <> - - - Tap - setInteractionMode("click")} - /> - setInteractionMode("double")} - /> - setInteractionMode("right")} - /> - - Drag - setSwipeMode("scroll")} - /> - setSwipeMode("drag")} - /> - - Scroll - scroll(-SCROLL_STEP, 0)} - /> - scroll(0, -SCROLL_STEP)} - /> - scroll(0, SCROLL_STEP)} - /> - scroll(SCROLL_STEP, 0)} - /> - - - - - - - - - Keys - {SPECIAL_KEYS.map(({ key, label }) => ( - sendEvent({ kind: "key", key })} - /> - ))} - - - )} - + {layout.platform !== "web" ? ( + + ) : null} } - open={pointerSheetOpen} - onOpenChange={setPointerSheetOpen} + title="Resolution and quality" + icon={} + open={devicePickerOpen} + onOpenChange={setDevicePickerOpen} > - - Swipe gesture - - setSwipeMode("scroll")} - /> - setSwipeMode("drag")} - /> - - Tap action - - setInteractionMode("click")} - /> - setInteractionMode("double")} - /> - setInteractionMode("right")} - /> - - Nudge scroll - - scroll(0, -SCROLL_STEP)} - /> - - scroll(-SCROLL_STEP, 0)} - /> - + + { + void preferences.toggleFavorite(id); + }} + onQualityChange={(quality) => { + void preferences.changeQuality(quality); + }} + /> + {preferences.error ? ( + + + {preferences.error} + scroll(SCROLL_STEP, 0)} + label="Reload preferences" + onPress={preferences.reload} /> - scroll(0, SCROLL_STEP)} - /> - - - Tap the page to click. Swipe to scroll or drag, depending on the gesture above. - - - - - - } - open={keysSheetOpen} - onOpenChange={setKeysSheetOpen} - > - - - - {SPECIAL_KEYS.map(({ key, label }) => ( - sendEvent({ kind: "key", key })} - /> - ))} - - - Keys go to the page element that currently has focus in the shared browser. - - - - - } - open={devicePickerOpen} - onOpenChange={setDevicePickerOpen} - > - - - {DEVICE_PRESETS.map((preset) => { - const selected = state?.devicePresetId === preset.id; - return ( - selectDevicePreset(preset.id)} - style={({ pressed }) => [ - styles.devicePresetRow, - selected ? styles.devicePresetRowSelected : null, - pressed ? styles.buttonPressed : null, - !canControl ? styles.buttonDisabled : null, - ]} - > - - {preset.label} - - {preset.viewport.width} × {preset.viewport.height} - - - ); - })} + ) : null} Custom viewport + {visibleError ? ( + + ) : null} + {toolbarMenu ? ( + closeToolbarMenu()} + shouldRestoreFocus={() => menuRestoreFocus.current} + onSubmenuOpen={() => { if (canSendInput) setKeysSubmenuOpen(true); }} + submenu={toolbarMenu === "actions" && keysSubmenuOpen ? { + title: "Send keys", + anchorRef: keysAnchorRef, + preferredHeight: 40 + SPECIAL_KEYS.length * (layout.compact ? 44 : 36), + onBack: () => setKeysSubmenuOpen(false), + children: SPECIAL_KEYS.map(({ key, label }) => ( + { + canvasInput.nativeKeyboard.reset(); + sendEvent({ kind: "key", key }); + }} /> + )), + } : undefined} + > + {toolbarMenu === "display" ? ( + <> + View size + { + closeToolbarMenu(); + setScaleMode("fit"); + }} + /> + { + closeToolbarMenu(); + setScaleMode("actual"); + }} + /> + {layout.platform !== "web" && scaleMode === "actual" ? ( + + Release control to pan this view. While controlling, swipes go to the page. + + ) : null} + + Favorite resolutions + {preferences.favoritePresetIds.length === 0 ? ( + + Star resolutions in the full list to add them here. + + ) : ( + preferences.favoritePresetIds.map((id) => { + const preset = DEVICE_PRESETS.find((item) => item.id === id); + return preset ? ( + { + closeToolbarMenu(); + selectDevicePreset(id); + }} + /> + ) : null; + }) + )} + + { + closeToolbarMenu(false); + setDevicePickerOpen(true); + }} + /> + + ) : ( + <> + {layout.platform !== "web" ? ( + <> + requestNativeKeyboard("keyboard")} + /> + requestNativeKeyboard("compose")} + /> + + ) : null} + + setKeysSubmenuOpen(true)} + /> + + { + closeToolbarMenu(); + reconnect(); + }} + /> + + )} + + ) : null} ); } diff --git a/paseo-shared-browser/client/favorite-star.tsx b/paseo-shared-browser/client/favorite-star.tsx new file mode 100644 index 00000000..84d3fa1b --- /dev/null +++ b/paseo-shared-browser/client/favorite-star.tsx @@ -0,0 +1,23 @@ +/** Filled and outlined favorite indicators with identical cross-platform font geometry. */ +import { Text } from "react-native"; + +/** The host Icon API exposes stroke only; paired system glyphs provide real fill on phones too. */ +export function FavoriteStar({ + filled, + color, + size = 18, +}: { + filled: boolean; + color: string; + size?: number; +}) { + return ( + + {filled ? "\u2605" : "\u2606"} + + ); +} diff --git a/paseo-shared-browser/client/frame-lifecycle.ts b/paseo-shared-browser/client/frame-lifecycle.ts index a7a19090..94ce29f1 100644 --- a/paseo-shared-browser/client/frame-lifecycle.ts +++ b/paseo-shared-browser/client/frame-lifecycle.ts @@ -1,65 +1,68 @@ import type { BrowserFrame } from "../shared/browser"; /** - * Tracks which captured frame input may target. After an input is submitted - * and again when it settles, the actionable frame is dropped and the epoch - * advances, so captures that began earlier are ignored. Input stays blocked - * until a capture taken after settlement is accepted. The last accepted image - * stays visible meanwhile. + * Tracks actionable captures across discrete input settlement. The decoded + * image remains visible while admission waits for a capture begun after settle. + * A factory keeps this upstream policy compatible with Paseo's mobile Hermes + * compiler, which rejects the emitted class declaration in client bundles. */ -export class FrameLifecycle { - epoch = 0; - /** Frame input may target; null while a fresh capture is awaited. */ - frame: BrowserFrame | null = null; - /** Last accepted frame, kept on screen while input is blocked. */ - visible: BrowserFrame | null = null; - private gate: "idle" | "pending" | "refreshing" = "idle"; +export function createFrameLifecycle() { + let epoch = 0; + let frame: BrowserFrame | null = null; + let visible: BrowserFrame | null = null; + let gate: "idle" | "pending" | "refreshing" = "idle"; - get busy(): boolean { - return this.gate !== "idle"; - } + const bump = () => { + epoch += 1; + }; + const drop = () => { + frame = null; + visible = null; + }; - bump(): void { - this.epoch += 1; - } - - isCurrent(captureEpoch: number): boolean { - return captureEpoch === this.epoch; - } - - /** Claims the input slot; false if an input is already in flight or awaiting a fresh frame. */ - begin(): boolean { - if (this.busy) return false; - this.gate = "pending"; - this.frame = null; - this.bump(); - return true; - } - - /** Called when an input succeeds or fails. Never replays the action. */ - settle(): void { - this.frame = null; - this.bump(); - this.gate = "refreshing"; - } - - /** Accepts a capture started at `captureEpoch`; false if it predates the latest boundary. */ - accept(captureEpoch: number, frame: BrowserFrame): boolean { - if (!this.isCurrent(captureEpoch)) return false; - this.frame = frame; - this.visible = frame; - if (this.gate === "refreshing") this.gate = "idle"; - return true; - } - - /** Forgets all frames, e.g. when the session no longer matches them. */ - drop(): void { - this.frame = null; - this.visible = null; - } - - reset(): void { - this.drop(); - this.gate = "idle"; - } + return { + get epoch() { + return epoch; + }, + get frame() { + return frame; + }, + get visible() { + return visible; + }, + get busy() { + return gate !== "idle"; + }, + bump, + isCurrent(captureEpoch: number) { + return captureEpoch === epoch; + }, + /** Blocks further discrete input until settlement and a fresh decoded frame. */ + begin() { + if (gate !== "idle") return false; + gate = "pending"; + frame = null; + bump(); + return true; + }, + /** Advances even on failure; an uncertain action is never replayed. */ + settle() { + frame = null; + bump(); + gate = "refreshing"; + }, + /** A capture begun before settlement cannot restore admission. */ + accept(captureEpoch: number, captured: BrowserFrame) { + if (captureEpoch !== epoch) return false; + frame = captured; + visible = captured; + if (gate === "refreshing") gate = "idle"; + return true; + }, + drop, + reset() { + drop(); + gate = "idle"; + }, + }; } diff --git a/paseo-shared-browser/client/use-browser-canvas-input.ts b/paseo-shared-browser/client/use-browser-canvas-input.ts new file mode 100644 index 00000000..0db45a29 --- /dev/null +++ b/paseo-shared-browser/client/use-browser-canvas-input.ts @@ -0,0 +1,382 @@ +/** + * One live input queue per mounted canvas, shared by native responders and real + * web listeners. Ownership changes cancel old channels; decode-only frame updates + * do not. Cursor replies are consumed only by the queue incarnation that sent + * them. The existing capture polling keeps playback running during a drag. + */ +import { + useCallback, + useEffect, + useLayoutEffect, + useRef, + useState, +} from "react"; +import { + AppState, + PanResponder, + Platform, + type GestureResponderEvent, + type View, +} from "react-native"; +import type { BrowserCursor, BrowserState } from "../shared/browser"; +import { + createBrowserInputQueue, + type BrowserGestureAuthority, + type BrowserGesturePoint, + type BrowserGestureTransport, + type BrowserTouchPoint, +} from "./browser-input-queue"; +import { createBrowserCanvasInput } from "./browser-canvas-input"; +import { bindBrowserCanvasKeyboard, type KeyboardCanvasNode } from "./browser-canvas-keyboard-web"; +import type { CanvasKeyboardEvent } from "./browser-canvas-keyboard"; +import { useBrowserNativeKeyboard } from "./use-browser-native-keyboard"; +import { + bindBrowserCanvasWeb, + setBrowserCanvasCursor, + type BrowserCanvasNode, +} from "./web"; + +interface CanvasOptions { + authority(): BrowserGestureAuthority | null; + transport: BrowserGestureTransport; + /** Control/session/generations/viewport/display rectangle, excluding frameId. */ + ownershipKey: string; + decodedFrameId: string | null; + enabled: boolean; + displaySize: { width: number; height: number } | null; + viewport: { width: number; height: number } | null; + onState(state: BrowserState): void; + onError(error: unknown): void; + onFinish(): void; + onPoint(point: BrowserGesturePoint): void; + onActivity(active: boolean): void; + onInputBoundary(): void; +} + +/** Native touch coordinates are relative to the controlled overlay, with stable ID mapping. */ +function nativePoints( + event: GestureResponderEvent, + size: CanvasOptions["displaySize"], + ids: Map, +): BrowserTouchPoint[] { + if (!size || size.width <= 0 || size.height <= 0) return []; + return event.nativeEvent.touches.map((touch) => { + let id = ids.get(touch.identifier); + if (id === undefined) { + id = ids.size; + ids.set(touch.identifier, id); + } + return { + id, + x: Math.max(0, Math.min(size.width, touch.locationX)), + y: Math.max(0, Math.min(size.height, touch.locationY)), + width: size.width, + height: size.height, + }; + }); +} + +/** Attach the same authority-preserving input model to native View or web host events. */ +export function useBrowserCanvasInput(options: CanvasOptions) { + const current = useRef(options); + current.current = options; + const nodeRef = useRef(null); + const [node, setNode] = useState(null); + const alive = useRef(true); + const inputModel = useRef | null>( + null, + ); + const keyboard = useRef | null>(null); + const nativeRelay = useRef<{ reset(): void } | null>(null); + const cursorVisible = useRef(false); + const frameWaiters = useRef( + new Set<{ + afterFrameId: string; + resolve(): void; + reject(error: Error): void; + }>(), + ); + const rejectFrameWaiters = () => { + for (const waiter of frameWaiters.current) + waiter.reject(new Error("Browser input context changed.")); + frameWaiters.current.clear(); + }; + const [queue] = useState(() => + createBrowserInputQueue({ + transport: { + begin: (input) => current.current.transport.begin(input), + update: (input) => current.current.transport.update(input), + end: (input) => current.current.transport.end(input), + }, + authority: () => (alive.current ? current.current.authority() : null), + onState: (state) => { + if (alive.current) current.current.onState(state); + }, + onCursor: (cursor: BrowserCursor | null) => { + if (cursorVisible.current) + setBrowserCanvasCursor(nodeRef.current, cursor); + }, + onError: (error) => { + // Queue owns cleanup; reset local contacts without recursively cancelling. + inputModel.current?.reset(); + keyboard.current?.reset(); + nativeRelay.current?.reset(); + cursorVisible.current = false; + setBrowserCanvasCursor(nodeRef.current, null); + clearIdle(); + if (alive.current) current.current.onError(error); + }, + onNavigationComplete: () => { + // Server already closed the acknowledged channel. Quarantine local + // holds before state projection; never replay their old-page release. + inputModel.current?.reset(); + keyboard.current?.reset(); + nativeRelay.current?.reset(); + cursorVisible.current = false; + setBrowserCanvasCursor(nodeRef.current, null); + clearIdle(); + rejectFrameWaiters(); + }, + onFinish: () => { + if (alive.current) current.current.onFinish(); + }, + waitForFrame(afterFrameId, maxWaitMs = 4_000) { + return new Promise((resolve, reject) => { + const waiter = { + afterFrameId, + resolve: () => { + clearTimeout(timeout); + frameWaiters.current.delete(waiter); + resolve(); + }, + reject: (error: Error) => { + clearTimeout(timeout); + frameWaiters.current.delete(waiter); + reject(error); + }, + }; + // The existing server gesture idle bound is five seconds. Stop an unsent + // press before it expires; time alone never authorizes new image pixels. + const timeout = setTimeout( + () => + waiter.reject( + new Error( + "Waiting for a current decoded frame. Release the gesture and try again.", + ), + ), + Math.min(4_000, maxWaitMs), + ); + frameWaiters.current.add(waiter); + }); + }, + }), + ); + const idleTimer = useRef | null>(null); + const clearIdle = () => { + if (idleTimer.current) clearTimeout(idleTimer.current); + idleTimer.current = null; + }; + const [input] = useState(() => + createBrowserCanvasInput({ + enabled: () => alive.current && current.current.enabled, + viewport: () => current.current.viewport, + enqueue(event) { + clearIdle(); + if ( + event.kind === "down" || + (event.kind === "touch" && event.type === "start") + ) { + nativeRelay.current?.reset(); + current.current.onInputBoundary(); + } + const accepted = queue.enqueue(event); + if (event.kind === "move" || event.kind === "scroll") { + idleTimer.current = setTimeout(() => { + // Never end a slow held drag just because the user stopped moving. + if (!input.isHoldingMouse()) { + current.current.onActivity(false); + queue.finish(); + } + }, 1_000); + } + return accepted; + }, + finish: () => { + clearIdle(); + queue.finish(); + }, + cancel: () => { + clearIdle(); + keyboard.current?.reset(); + nativeRelay.current?.reset(); + cursorVisible.current = false; + setBrowserCanvasCursor(nodeRef.current, null); + queue.cancel(); + rejectFrameWaiters(); + }, + onPoint: (point) => current.current.onPoint(point), + onActivity: (active) => { + if (alive.current) current.current.onActivity(active); + }, + }), + ); + + inputModel.current = input; + + const enqueueKeyboard = useCallback((event: CanvasKeyboardEvent) => { + clearIdle(); + const accepted = queue.enqueue(event); + if (accepted) { + if (event.kind === "text" || (event.type === "down" && !event.repeat)) { + current.current.onInputBoundary(); + } + current.current.onActivity(true); + } + return accepted; + }, [queue]); + const finishKeyboard = useCallback(() => { + current.current.onActivity(false); + // Keep one channel through ordinary inter-key gaps. New typing clears this + // timer, while blur/control loss cancels immediately through the input model. + clearIdle(); + idleTimer.current = setTimeout(() => queue.finish(), 1_000); + }, [queue]); + const nativeKeyboard = useBrowserNativeKeyboard({ + enabled: () => alive.current && current.current.enabled, + ownershipKey: options.ownershipKey, + enqueue: enqueueKeyboard, + finish: finishKeyboard, + cancel: input.cancel, + onError: error => { if (alive.current) current.current.onError(error); }, + }); + nativeRelay.current = nativeKeyboard; + + const previousOwnership = useRef(options.ownershipKey); + useLayoutEffect(() => { + if (previousOwnership.current === options.ownershipKey) return; + previousOwnership.current = options.ownershipKey; + input.cancel(); + }, [input, options.ownershipKey]); + useLayoutEffect(() => { + for (const waiter of frameWaiters.current) { + if ( + options.decodedFrameId && + options.decodedFrameId !== waiter.afterFrameId + ) + waiter.resolve(); + } + }, [options.decodedFrameId]); + useEffect(() => { + alive.current = true; + const subscription = AppState.addEventListener("change", (state) => { + if (state !== "active") input.cancel(); + }); + return () => { + alive.current = false; + input.cancel(); + subscription.remove(); + }; + }, [input]); + useEffect(() => { + if ( + Platform.OS !== "web" || + !node || + typeof node !== "object" || + !("addEventListener" in node) + ) + return; + return bindBrowserCanvasWeb( + node as BrowserCanvasNode, + input, + () => alive.current && current.current.enabled, + (visible) => { + cursorVisible.current = visible; + if (!visible) setBrowserCanvasCursor(nodeRef.current, null); + }, + ); + }, [input, node]); + + useEffect(() => { + if (Platform.OS !== "web" || !node || typeof node !== "object" || !("ownerDocument" in node)) return; + const binding = bindBrowserCanvasKeyboard(node as KeyboardCanvasNode, { + enabled: () => alive.current && current.current.enabled, + enqueue: enqueueKeyboard, + finish: finishKeyboard, + cancel: input.cancel, + onError: (error) => { + if (alive.current) current.current.onError(error); + }, + }); + keyboard.current = binding; + return () => { + binding.dispose(); + if (keyboard.current === binding) keyboard.current = null; + }; + }, [input, node, enqueueKeyboard, finishKeyboard]); + + const ids = useRef(new Map()); + const [panResponder] = useState(() => + PanResponder.create({ + onStartShouldSetPanResponder: (event) => { + if (Platform.OS === "web" || !current.current.enabled) return false; + // A newly granted responder must contain only genuinely new contacts. + // A second finger cannot import a first contact that began observe-only. + const changed = new Set( + (event.nativeEvent.changedTouches ?? []).map( + (touch) => touch.identifier, + ), + ); + return ( + event.nativeEvent.touches.length > 0 && + event.nativeEvent.touches.every((touch) => + changed.has(touch.identifier), + ) + ); + }, + // Moving a contact after lease acquisition is not a fresh start edge. + onMoveShouldSetPanResponder: () => false, + onPanResponderGrant: (event) => { + ids.current.clear(); + input.touch( + "start", + nativePoints(event, current.current.displaySize, ids.current), + ); + }, + onPanResponderStart: (event) => + input.touch( + "start", + nativePoints(event, current.current.displaySize, ids.current), + ), + onPanResponderMove: (event) => + input.touch( + "move", + nativePoints(event, current.current.displaySize, ids.current), + ), + onPanResponderEnd: (event) => + input.touch( + "end", + nativePoints(event, current.current.displaySize, ids.current), + ), + onPanResponderRelease: () => { + input.touch("end", []); + ids.current.clear(); + }, + onPanResponderTerminate: () => { + input.touch("cancel", []); + ids.current.clear(); + }, + onPanResponderTerminationRequest: () => false, + }), + ); + const canvasRef = useCallback((next: View | null) => { + nodeRef.current = next; + setNode(next); + }, []); + + return { + canvasRef, + panHandlers: Platform.OS === "web" ? {} : panResponder.panHandlers, + cancel: input.cancel, + nativeKeyboard, + }; +} diff --git a/paseo-shared-browser/client/use-browser-emulation-mode.ts b/paseo-shared-browser/client/use-browser-emulation-mode.ts new file mode 100644 index 00000000..391470c0 --- /dev/null +++ b/paseo-shared-browser/client/use-browser-emulation-mode.ts @@ -0,0 +1,56 @@ +/** Apply the local device default only after this viewer first owns the shared browser. */ +import { useEffect, useRef } from "react"; +import type { BrowserState } from "../shared/browser"; +import { + createEmulationChoices, + type EmulationSelection, + readLocalDevice, + selectionMode, +} from "./browser-emulation-mode"; + +interface ModeOptions { + identity: string; + platform: "ios" | "android" | "web"; + state: BrowserState | null; + canControl: boolean; + pending: boolean; + apply(selection: EmulationSelection): void; +} + +/** Observers never mutate emulation; failed changes remain explicit errors and are not retried. */ +export function useBrowserEmulationMode(options: ModeOptions) { + const owned = useRef({ + identity: options.identity, + choices: createEmulationChoices(readLocalDevice(options.platform)), + }); + if (owned.current.identity !== options.identity) { + owned.current = { + identity: options.identity, + choices: createEmulationChoices(readLocalDevice(options.platform)), + }; + } + const choices = owned.current.choices; + const presetId = options.state?.devicePresetId ?? null; + const mode = selectionMode(presetId); + useEffect(() => { + if (options.state) choices.remember(presetId); + }, [choices, presetId, Boolean(options.state)]); + useEffect(() => { + if (!options.canControl || options.pending || options.state?.status !== "ready") return; + const selection = choices.claimDefault(mode); + if (selection) options.apply(selection); + }, [choices, mode, options.canControl, options.pending, options.state?.status, options.apply]); + return { + mode, + toggle: () => { + if ( + owned.current.choices !== choices || + !options.canControl || + options.pending || + options.state?.status !== "ready" + ) + return; + options.apply(choices.toggle(mode)); + }, + }; +} diff --git a/paseo-shared-browser/client/use-browser-frame-buffer.ts b/paseo-shared-browser/client/use-browser-frame-buffer.ts new file mode 100644 index 00000000..896238f6 --- /dev/null +++ b/paseo-shared-browser/client/use-browser-frame-buffer.ts @@ -0,0 +1,66 @@ +/** + * Connects native Image decode callbacks to the bounded handoff model. The + * visible-frame ref is changed in a layout effect, after the image-layer commit, + * so transport metadata cannot authorize input against pixels not yet displayed. + */ +import { useCallback, useLayoutEffect, useState, type RefObject } from "react"; +import type { BrowserFrame } from "../shared/browser"; +import { createBrowserFrameBuffer, type FrameCandidate } from "./browser-frame-buffer"; + +/** Bind qualified captures and native callbacks to committed visible input authority. */ +export function useBrowserFrameBuffer( + visibleFrameRef: RefObject, + isCurrent: (candidate: FrameCandidate) => boolean, +) { + const [model] = useState(createBrowserFrameBuffer); + const [buffer, setBuffer] = useState(model.snapshot); + const frame = + buffer.front === null ? null : (buffer.layers[buffer.front]?.candidate.frame ?? null); + + useLayoutEffect(() => { + visibleFrameRef.current = frame; + return () => { + visibleFrameRef.current = null; + }; + }, [frame, visibleFrameRef]); + + const discardObsoletePending = useCallback(() => { + const current = model.snapshot(); + const pending = current.pending === null ? null : current.layers[current.pending]; + if (pending && !isCurrent(pending.candidate)) { + setBuffer(model.invalidatePending()); + } + }, [isCurrent, model]); + const reset = useCallback(() => { + visibleFrameRef.current = null; + setBuffer(model.reset()); + }, [model, visibleFrameRef]); + const receive = useCallback( + (candidate: FrameCandidate) => { + if (!isCurrent(candidate)) { + return; + } + const current = model.snapshot(); + const pending = current.pending === null ? null : current.layers[current.pending]; + if (pending && !isCurrent(pending.candidate)) { + model.invalidatePending(); + } + setBuffer(model.offer(candidate)); + }, + [isCurrent, model], + ); + const settled = useCallback( + (ticket: number, succeeded: boolean) => { + const current = model.snapshot(); + const pending = current.pending === null ? null : current.layers[current.pending]; + if (!pending || pending.ticket !== ticket || !isCurrent(pending.candidate)) { + return false; + } + setBuffer(model.settle(ticket, succeeded, isCurrent)); + return true; + }, + [isCurrent, model], + ); + + return { buffer, frame, receive, settled, discardObsoletePending, reset }; +} diff --git a/paseo-shared-browser/client/use-browser-native-keyboard.ts b/paseo-shared-browser/client/use-browser-native-keyboard.ts new file mode 100644 index 00000000..0b058327 --- /dev/null +++ b/paseo-shared-browser/client/use-browser-native-keyboard.ts @@ -0,0 +1,129 @@ +/** + * Stock RN TextInput binding for the native Keyboard button. The caller places + * the hidden Live input and owns the visible Compose draft/Done sheet. Android + * hardware-key shortcuts are not exposed by this RN software-keyboard API. + */ +import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"; +import { Platform, type TextInput, type TextInputProps } from "react-native"; +import type { CanvasKeyboardEvent } from "./browser-canvas-keyboard"; +import { createBrowserNativeKeyboard } from "./browser-native-keyboard"; + +interface NativeRelayOptions { + enabled(): boolean; + ownershipKey: string; + enqueue(event: CanvasKeyboardEvent): boolean; + finish(): void; + cancel(): void; + onError(error: Error): void; +} + +/** No native input is focused automatically; focus runs only from Keyboard. */ +export function useBrowserNativeKeyboard(options: NativeRelayOptions) { + const current = useRef(options); + current.current = options; + const alive = useRef(true); + const inputRef = useRef(null); + const [value, setValue] = useState(""); + const [generation, setGeneration] = useState(0); + const currentGeneration = useRef(0); + const focusedTarget = useRef(null); + const lastEventCount = useRef(-1); + const rotateInput = useCallback(() => { + focusedTarget.current = null; + lastEventCount.current = -1; + currentGeneration.current += 1; + if (alive.current && Platform.OS !== "web") { + setGeneration(currentGeneration.current); + } + }, []); + const [relay] = useState(() => + createBrowserNativeKeyboard({ + enabled: () => alive.current && Platform.OS !== "web" && current.current.enabled(), + enqueue: (event) => current.current.enqueue(event), + finish: () => current.current.finish(), + onError: (error) => { + rotateInput(); + current.current.onError(error); + }, + onValue: (next) => { + if (alive.current) setValue(next); + }, + }), + ); + const reset = useCallback(() => { + relay.reset(); + rotateInput(); + }, [relay, rotateInput]); + const previousOwner = useRef(options.ownershipKey); + useLayoutEffect(() => { + if (previousOwner.current === options.ownershipKey) return; + previousOwner.current = options.ownershipKey; + reset(); + }, [reset, options.ownershipKey]); + useEffect(() => { + alive.current = true; + return () => { + alive.current = false; + relay.reset(); + }; + }, [relay]); + const focus = useCallback(() => { + if (Platform.OS !== "web" && current.current.enabled()) inputRef.current?.focus(); + }, []); + const callbackCurrent = () => + alive.current && + currentGeneration.current === generation && + current.current.ownershipKey === options.ownershipKey && + current.current.enabled(); + const inputProps: TextInputProps = { + value, + autoCorrect: false, + autoCapitalize: "none", + autoComplete: "off", + spellCheck: false, + contextMenuHidden: true, + multiline: false, + submitBehavior: "submit", + onFocus: (event) => { + const target = event.nativeEvent.target; + if (callbackCurrent() && Number.isSafeInteger(target) && target > 0) { + focusedTarget.current = target; + } + }, + onChange: (event) => { + const { target, text, eventCount } = event.nativeEvent; + if ( + !callbackCurrent() || + target !== focusedTarget.current || + !Number.isSafeInteger(eventCount) || + eventCount < 0 || + eventCount <= lastEventCount.current + ) + return; + lastEventCount.current = eventCount; + relay.changeText(text); + }, + onKeyPress: (event) => { + if (callbackCurrent() && focusedTarget.current !== null) + relay.keyPress(event.nativeEvent.key); + }, + onSelectionChange: (event) => { + if (!callbackCurrent() || event.nativeEvent.target !== focusedTarget.current) return; + const { start, end } = event.nativeEvent.selection; + if (!relay.selection(start, end)) reset(); + }, + onBlur: () => { + if (!callbackCurrent() || focusedTarget.current === null) return; + reset(); + current.current.cancel(); + }, + }; + return { + inputRef, + inputProps, + inputKey: `${options.ownershipKey}:${generation}`, + focus, + reset, + composeText: relay.compose, + }; +} diff --git a/paseo-shared-browser/client/use-browser-viewer-recovery.ts b/paseo-shared-browser/client/use-browser-viewer-recovery.ts new file mode 100644 index 00000000..4ba808b9 --- /dev/null +++ b/paseo-shared-browser/client/use-browser-viewer-recovery.ts @@ -0,0 +1,26 @@ +/** Attach viewing only after the current capture token expires; never replay control or input. */ +import { useEffect, useState } from "react"; +import { + type BrowserViewerRecoveryObservation, + createBrowserViewerRecovery, +} from "./browser-viewer-recovery"; + +interface ViewerRecoveryOptions extends BrowserViewerRecoveryObservation { + reconnect(): void; +} + +/** Bounded recovery follows the existing manual Reconnect path and its lifecycle fences. */ +export function useBrowserViewerRecovery(options: ViewerRecoveryOptions): void { + const [claim] = useState(createBrowserViewerRecovery); + useEffect(() => { + if (claim(options)) options.reconnect(); + }, [ + claim, + options.identity, + options.viewerToken, + options.failedViewerToken, + options.captureError, + options.pending, + options.reconnect, + ]); +} diff --git a/paseo-shared-browser/client/use-resolution-favorites.ts b/paseo-shared-browser/client/use-resolution-favorites.ts new file mode 100644 index 00000000..71efedf7 --- /dev/null +++ b/paseo-shared-browser/client/use-resolution-favorites.ts @@ -0,0 +1,91 @@ +/** + * Revisioned host favorites. Stars use the same settings/CAS service as neighboring + * plugins; conflicts stay visible until reload, and no write is replayed. An old + * host's completion cannot clear a newer host's pending action or expose its error. + */ +import { useSettings } from "@getpaseo/plugin/client"; +import { useRef, useState } from "react"; +import { + browserDisplayPreferences, + type BrowserDisplayPreferences, +} from "../shared/browser-display-preferences"; +import { DEFAULT_CAPTURE_QUALITY } from "../shared/capture-settings"; +import { + normalizeResolutionFavorites, + toggleResolutionFavorite, +} from "../shared/resolution-menu"; +import type { DevicePresetId } from "../shared/device-presets"; + +/** Call with the surface's host.id; persistence itself is scoped by Paseo. */ +export function useResolutionFavorites(hostId: string) { + const settings = useSettings(browserDisplayPreferences); + const currentHost = useRef(hostId); + currentHost.current = hostId; + const pending = useRef<{ hostId: string } | null>(null); + const [pendingHost, setPendingHost] = useState(null); + const saving = settings.saving || pendingHost === hostId; + const disabled = settings.status !== "ready" || saving; + const favoritePresetIds = + settings.status === "ready" + ? normalizeResolutionFavorites(settings.values.favoritePresetIds) + : []; + const error = + settings.status === "error" || settings.status === "invalid" + ? settings.error + : settings.saveError; + + const savePreferences = async ( + update: (values: BrowserDisplayPreferences) => BrowserDisplayPreferences, + ): Promise => { + if ( + currentHost.current !== hostId || + settings.status !== "ready" || + settings.saving || + pending.current?.hostId === hostId + ) + return false; + const operation = { hostId }; + pending.current = operation; + setPendingHost(hostId); + try { + const saved = await settings.save( + update(settings.values), + settings.revision, + ); + return currentHost.current === hostId && saved; + } finally { + if (pending.current === operation) { + pending.current = null; + setPendingHost(null); + } + } + }; + + return { + favoritePresetIds, + captureQuality: + settings.status === "ready" + ? settings.values.captureQuality + : DEFAULT_CAPTURE_QUALITY, + loading: settings.status === "loading", + saving, + disabled, + error, + toggleFavorite: (id: DevicePresetId) => { + if (normalizeResolutionFavorites([id]).length === 0) + return Promise.resolve(false); + return savePreferences((values) => ({ + ...values, + favoritePresetIds: toggleResolutionFavorite( + values.favoritePresetIds, + id, + ), + })); + }, + changeQuality: ( + captureQuality: BrowserDisplayPreferences["captureQuality"], + ) => savePreferences((values) => ({ ...values, captureQuality })), + reload: settings.reload, + resetPreferences: settings.reset, + }; +} diff --git a/paseo-shared-browser/client/web.ts b/paseo-shared-browser/client/web.ts new file mode 100644 index 00000000..98cb8afb --- /dev/null +++ b/paseo-shared-browser/client/web.ts @@ -0,0 +1,271 @@ +/** + * Web-only input adapter for the native View's DOM host node. No DOM globals or + * handlers leak into the cross-platform panel. Non-passive wheel/touch listeners + * capture only a controlled image canvas; document mouse listeners preserve drags outside + * its rectangle and visibility/blur immediately cancels held remote input. + */ +import { Platform } from "react-native"; +import type { BrowserCursor } from "../shared/browser"; +import type { createBrowserCanvasInput } from "./browser-canvas-input"; + +type CanvasInput = ReturnType; +interface Rect { + left: number; + top: number; + width: number; + height: number; +} +interface DomTouch { + identifier: number; + clientX: number; + clientY: number; +} +interface DomEvent { + clientX?: number; + clientY?: number; + button?: number; + detail?: number; + deltaX?: number; + deltaY?: number; + deltaMode?: number; + touches?: ArrayLike; + sourceCapabilities?: { firesTouchEvents?: boolean }; + preventDefault(): void; + stopPropagation(): void; +} +export interface BrowserCanvasNode { + getBoundingClientRect(): Rect; + addEventListener( + type: string, + listener: (event: DomEvent) => void, + options?: { passive: boolean }, + ): void; + removeEventListener(type: string, listener: (event: DomEvent) => void): void; + style: { cursor: string; touchAction: string }; + ownerDocument: { + hidden: boolean; + addEventListener( + type: string, + listener: (event: DomEvent) => void, + options?: { passive?: boolean; capture?: boolean }, + ): void; + removeEventListener( + type: string, + listener: (event: DomEvent) => void, + options?: { capture?: boolean }, + ): void; + defaultView: { + addEventListener(type: string, listener: () => void): void; + removeEventListener(type: string, listener: () => void): void; + } | null; + }; +} + +/** Bind real web envelopes only on web; native touch uses the responder adapter. */ +export function bindBrowserCanvasWeb( + node: BrowserCanvasNode, + input: CanvasInput, + enabled: () => boolean, + onCursorVisibility: (visible: boolean) => void, +) { + if (Platform.OS !== "web") return () => {}; + const previousTouchAction = node.style.touchAction; + node.style.touchAction = "none"; + const listeners = new Map void>(); + const point = (x = 0, y = 0) => { + const rect = node.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return null; + return { + x: Math.max(0, Math.min(rect.width, x - rect.left)), + y: Math.max(0, Math.min(rect.height, y - rect.top)), + width: rect.width, + height: rect.height, + }; + }; + const listen = (type: string, listener: (event: DomEvent) => void) => { + listeners.set(type, listener); + node.addEventListener(type, listener, { passive: false }); + }; + const consume = (event: DomEvent) => { + event.preventDefault(); + event.stopPropagation(); + }; + let touchActive = false; + const mouse = (type: "move" | "down" | "up", event: DomEvent) => { + // Consumed touch events suppress compatibility mouse events in browsers. + // Chromium also explicitly identifies them; never forward a second tap. + if ( + touchActive || + event.sourceCapabilities?.firesTouchEvents || + !enabled() + ) { + return; + } + const location = point(event.clientX, event.clientY); + if (!location) return; + onCursorVisibility(true); + const button = + event.button === 2 ? "right" : event.button === 1 ? "middle" : "left"; + const count = event.detail === 2 ? 2 : 1; + if (type === "down") { + if (input.mouseDown(location, button, count)) consume(event); + } else if (type === "up") { + if (input.mouseUp(location, button, count)) consume(event); + } else { + const held = input.isHoldingMouse(); + input.mouseMove(location); + if (held) consume(event); + } + }; + // MouseEvent.detail carries physical double-click counts. PointerEvent.detail + // commonly stays zero. Only mouse events publish input, avoiding duplicate edges. + listen("mousedown", (event) => mouse("down", event)); + listen("mousemove", (event) => mouse("move", event)); + listen("mouseup", (event) => mouse("up", event)); + listen("mouseleave", () => { + onCursorVisibility(false); + input.leave(); + node.style.cursor = "default"; + }); + listen("pointercancel", () => input.cancel()); + const isOutside = (event: DomEvent) => { + const rect = node.getBoundingClientRect(); + return ( + event.clientX === undefined || + event.clientY === undefined || + event.clientX < rect.left || + event.clientX > rect.left + rect.width || + event.clientY < rect.top || + event.clientY > rect.top + rect.height + ); + }; + const outsideMove = (event: DomEvent) => { + if (input.isHoldingMouse()) { + mouse("move", event); + onCursorVisibility(!isOutside(event)); + } + }; + const outsideUp = (event: DomEvent) => { + if (input.isHoldingMouse()) { + mouse("up", event); + const outside = isOutside(event); + onCursorVisibility(!outside); + if (outside) input.leave(); + } + }; + node.ownerDocument.addEventListener("mousemove", outsideMove); + node.ownerDocument.addEventListener("mouseup", outsideUp); + listen("contextmenu", (event) => { + if (enabled()) consume(event); + }); + listen("wheel", (event) => { + if (!enabled()) return; + const location = point(event.clientX, event.clientY); + if (!location) return; + // Wheel deltaMode is local CSS pixels, lines, or pages. Line-mode uses the + // browser's conventional 16 CSS-pixel step; page mode uses this image's size. + const multiplierX = + event.deltaMode === 2 ? location.width : event.deltaMode === 1 ? 16 : 1; + const multiplierY = + event.deltaMode === 2 ? location.height : event.deltaMode === 1 ? 16 : 1; + if ( + input.wheel( + location, + (event.deltaX ?? 0) * multiplierX, + (event.deltaY ?? 0) * multiplierY, + ) + ) { + consume(event); + } + }); + const documentTouchListeners = new Map void>(); + for (const phase of ["start", "move", "end", "cancel"] as const) { + // An observe-only overlay has pointerEvents:none. Track only contacts that + // start over its image (without consuming them), so acquiring control mid- + // contact cannot turn a later move/second finger into a remote fresh press. + const trackUncontrolledContact = (event: DomEvent) => { + const rect = node.getBoundingClientRect(); + const touches = Array.from(event.touches ?? []); + const overImage = touches.some( + (touch) => + touch.clientX >= rect.left && + touch.clientX <= rect.left + rect.width && + touch.clientY >= rect.top && + touch.clientY <= rect.top + rect.height, + ); + if (input.isTouchQuarantined() || (!enabled() && overImage)) { + const points = touches + .map((touch) => { + const location = point(touch.clientX, touch.clientY); + return location ? { ...location, id: touch.identifier } : null; + }) + .filter((item) => item !== null); + input.touch(phase, points); + } + }; + documentTouchListeners.set(`touch${phase}`, trackUncontrolledContact); + node.ownerDocument.addEventListener( + `touch${phase}`, + trackUncontrolledContact, + { capture: true, passive: true }, + ); + listen(`touch${phase}`, (event) => { + touchActive = phase !== "cancel" && (event.touches?.length ?? 0) > 0; + const points = Array.from(event.touches ?? []).map((touch) => { + const location = point(touch.clientX, touch.clientY); + return location ? { ...location, id: touch.identifier } : null; + }); + if (points.some((item) => !item)) { + input.cancel(); + return; + } + const accepted = input.touch( + phase, + points.filter((item) => item !== null), + ); + if (accepted && enabled()) { + consume(event); + } + }); + } + const cancel = () => { + onCursorVisibility(false); + touchActive = false; + input.cancel(); + node.style.cursor = "default"; + }; + const visibility = () => { + if (node.ownerDocument.hidden) cancel(); + }; + node.ownerDocument.addEventListener("visibilitychange", visibility); + node.ownerDocument.defaultView?.addEventListener("blur", cancel); + return () => { + cancel(); + for (const [type, listener] of listeners) { + node.removeEventListener(type, listener); + } + for (const [type, listener] of documentTouchListeners) { + node.ownerDocument.removeEventListener(type, listener, { capture: true }); + } + node.ownerDocument.removeEventListener("visibilitychange", visibility); + node.ownerDocument.removeEventListener("mousemove", outsideMove); + node.ownerDocument.removeEventListener("mouseup", outsideUp); + node.ownerDocument.defaultView?.removeEventListener("blur", cancel); + node.style.touchAction = previousTouchAction; + }; +} + +/** Cursor enums originate from the guarded remote response, never custom URLs. */ +export function setBrowserCanvasCursor( + node: unknown, + cursor: BrowserCursor | null, +): void { + if ( + Platform.OS === "web" && + node && + typeof node === "object" && + "style" in node + ) { + (node as BrowserCanvasNode).style.cursor = cursor ?? "default"; + } +} diff --git a/paseo-shared-browser/docs/fork-reconciliation.md b/paseo-shared-browser/docs/fork-reconciliation.md new file mode 100644 index 00000000..93ed094c --- /dev/null +++ b/paseo-shared-browser/docs/fork-reconciliation.md @@ -0,0 +1,172 @@ +# Shared Browser interaction and display fork + +This branch is rebased onto upstream main `d7b3e654` (Shared Browser 1.2.0). It is uncommitted and has not been submitted as a pull request. Dependencies +retain the repository catalog declarations. Runtime assets, browser profiles, +credentials, local backups and application data are excluded. + +## Changes preserved + +- Display sizes through 2560 by 2560, desktop aspect ratio presets, 1280 by 800 + and 1280 square presets, and a sharper Pixel capture mode. +- Higher capture quality, exact capture dimensions, reconnect restoration, + stable capture transport and decoded frame buffering. +- Physical hover, cursor feedback, wheel, right and double clicks, continuous + mouse dragging, native scrollbar dragging and genuine touch gestures. +- Natural input without click or drag mode selectors, with preserved keyboard, + text, navigation, display and control lease surfaces. +- Ordered gesture channels, bounded pressure, cleanup on lost ownership and + safe recovery of stale gesture admission. Published actions are not replayed. + +## Validation + +The combined registered suite passed 226 tests, including native relay, viewer +recovery, acknowledged-navigation, menu and local-view sizing additions. Actual Chromium checks verified trusted native keyboard +input, Unicode, repeat, shortcuts, focus traversal, Ctrl+click and key release. +Quality 70, 90 and 95 produced distinct 9,868, 11,131 and 11,951-byte frames on the +same isolated page. Mounted tests separately cover DOM keyboard/composition, +preference revision conflicts, observer safety and first-control device defaults. +The actual Paseo client/server compiler and mobile Hermes compiler passed. +The loaded runtime also rendered the signed-in nine-printer fleet. Low, Medium +and High captures returned distinct 93,155, 149,326 and 198,233-byte JPEGs, each +1280 by 800 pixels, before restoring the earlier square display. + +These checks use disposable profiles and fixtures. They do not establish human +pane forwarding or physical-phone software-keyboard behavior. The user confirmed +native scrollbar dragging in the real pane. Discrete MCP retains strict stale +frame validation and does not exercise the human pane gesture queue. + +## Native desktop hover capability + +Chromium headless desktop currently reports no pointer or hover capability. +Media-query-gated hover styles therefore do not activate on every control. +Startup Blink settings cannot restore those capabilities after touch emulation +is disabled. Preserve page state: do not reset the page on device changes or +compensate with application CSS. Hidden Linux runtimes now use an authenticated, +owned Xvfb display when the existing executable is available. It supplies genuine +embedder mouse preferences, so desktop -> phone -> desktop and CDP reconnect +retain the correct pointer/hover media. Abstract-only Unix transport avoids WSLg's +protected pathname sockets; TCP and host display access remain disabled. Startup +fallback is allowed only after full display cleanup and before Chromium exists. +Missing Xvfb and other platforms retain their existing behavior; they are not +covered by the Linux capability correction. + +## Upstream reconciliation + +| Upstream change | Reconciliation | +| --- | --- | +| #263 pointer mapping | Reuse upstream contained geometry helpers, retain capture-scale-aware coordinates and decoded-frame authority. | +| #264 frame settlement | Preserve the upstream discrete-input epoch gate and regressions; gate on decoded frames. Use a factory so the actual mobile Hermes compiler accepts the client bundle. Continuous input keeps its ordered channel. | +| #266 viewport resync | Integrate one bounded reapplication of saved emulation for valid dimension mismatches, using capture scale rather than device pixel ratio. Malformed JPEG and superseded capture remain rejected. | +| #258 sidebar and OMP MCP | Retain upstream sidebar, capability guards, entrypoint and tests. | +| #248 Paseo compatibility | Retain stable 0.11 support and the specifically tested 0.11.0-beta.3 allowance. | +| Release metadata | Keep upstream version 1.2.0 and catalogue dependencies; do not publish an independent release. | + +Input limitations and primary references are in [input-forwarding-notes.md](input-forwarding-notes.md). +Keep runtime assets, browser profiles, machine-specific paths and backups outside +the source diff. Changes remain uncommitted for review. The original pre-rebase +stash remains a recovery backup; it is not a pending patch to apply again. + +## Toolbar and completed-input follow-up + +The toolbar now owns mobile emulation, favorite resolution selection and the +browser action menu. Bottom favorite, emulation and key rows are removed. +Favorite indicators use paired filled/outline star glyphs because the host Icon +API exposes stroke only. No selected raised surface is used for the star button. + +Post-update daemon logs contained viewer expiry, stale navigation and live input +attachment changes. Isolated real Chromium reproduced a completed click that +navigated but returned a stale-navigation failure. Completion now accepts only +acknowledged publication plus a document navigation on the same attachment, +closes the old gesture and returns fresh viewing state. Unpublished and uncertain +commands, replaced controls, viewports, runtime targets and bridges still fail. +A separate actual Enter form-submit test confirms one submission and released +held keys. The shared browser is never used for these write-capable fixtures. + +The completed-input follow-up also passes mounted desktop/compact toolbar, +filled-star preference and native keyboard request checks. Mounted keyboard and +touch fixtures confirm held inputs are quarantined until physical release after +acknowledged navigation. Old-document continuations are discarded; no action is +replayed and no obsolete channel-end request is sent. Independent regression +checks retain refusal of a target replacement after publication. + +## Combined toolbar menus and local view sizing + +The monitor control combines favorites, the full resolution/quality list and +local Fit/Actual size selection. The vertical-dots browser menu contains actions, +without duplicated resolution access. Both use one shared anchored-menu surface +with bounded placement, keyboard focus and dismissal. Only public RN primitives +and themed SDK icons are used; the SDK has no public inline menu API. + +Actual size preserves canonical CSS layout dimensions and decoded frame slots. +Image and input overlay share one geometry, including high-density captures. +Desktop uses one scroll host so both local scrollbars stay at the pane edges. +Native observation permits local panning; active control reserves gestures for +the remote page. Scale selection is local to the mounted viewer, and changing +scale cancels held remote input. The browser page and shared resolution persist. + +The combined follow-up passes 226 registered tests, four mounted toolbar/canvas +cases, TypeScript, the actual Paseo compiler and the mobile Hermes compiler. +These validate rendering structure, focus, local scroll styles and input mapping; +they do not certify physical-phone native panning or software-keyboard behavior. + +## Mode-only display preservation and key submenu + +The mobile toggle and the first-control device default now retain the current +CSS viewport dimensions and JPEG capture density. The policy resolves these +values under the session lock, cancels held input, and invalidates old frames +even when dimensions do not change. Explicit resolution presets retain their +full display behavior. Preset checkmarks and the frame label require an exact +display and capture-density match, preventing a remembered behavior profile +from incorrectly claiming a selected resolution. + +Send keys is a nested toolbar menu, not a separate dialog. Desktop uses a +bounded flyout; compact/native clients use a Back row. One backdrop owns outside +dismissal. Keyboard navigation scopes each menu level, returns to the parent +trigger, and retains the existing command guards and dispatch callback. + +The combined registered suite passes 236 tests. Disposable Chromium confirms +unchanged 1280 x 800 mode switches and preserved 824 x 1678 sharp captures, +with page drafts intact and no reload. Seven mounted mode-hook cases and five +mounted submenu cases cover ownership, pending/stale callbacks, keyboard +hierarchy and single command dispatch. TypeScript, the actual Paseo compiler +and the mobile Hermes compiler pass. Human host-pane interactions still need +direct observation; browser MCP images cannot establish their behavior. + +## Actual-size bounds, native hover and frame delivery + +Actual size now sizes the canvas to the remote CSS dimensions and lets the scroll +host center it. An outer border or a single scrollbar no longer creates overflow +on the other axis. Container size changes cancel held input because recentering +can move the frame origin without changing its CSS dimensions. Disposable +Chromium measured 16 before/after cases covering fitting and exact-edge frames, +fractional borders, each overflowing axis, classic and zero-width scrollbar +geometry, edge reachability, sharp captures and pointer mapping after panning. +These use the public React Native Web CSS shape, not a physical phone. + +An owned private Linux display supplies genuine desktop hover, including after +phone emulation and CDP reconnect. The actual authenticated runtime preserves a +page draft through those switches and produces 824 x 1678 sharp phone and +2560 x 2560 desktop captures. Activating this launch change for an existing +headless browser requires one browser restart; later mode switches do not reload +the page. Missing Xvfb and other platforms keep their existing behavior. + +Frame polling reuses only the exact runtime receipt under matching session, +runtime, bridge, navigation, viewport and quality authority. Repeated cached +frames return metadata instead of sending the JPEG again. Runtime age and +new-capture invalidation remain enforced. A disposable idle-page fixture reduced +three repeated poll payloads by 72.94%, without changing image quality. Fresh +page metadata reads run concurrently under attachment/document guards. The +measured median was 3.434 ms before and 2.121 ms after; guarded input median was +16.971 ms before and 15.504 ms after, with no established tail-latency improvement. +This remains JPEG-frame transport, not an encoded video stream or a frame-rate +claim. A memoized image leaf also avoids decoding/rendering unchanged visual +props on unrelated parent updates. + +The compact address field removes excess vertical padding and stays at full +contrast while observing. Its control and navigation permissions are unchanged. + +The registered suite passes 264 tests. Four mounted toolbar/canvas checks and +two mounted image authority/memoization checks also pass, as do TypeScript, the +actual Paseo compiler and mobile Hermes compiler. Linux fixture checks prove +native hover and display lifecycle; human host-pane performance and physical +native-phone input/layout still need direct observation. diff --git a/paseo-shared-browser/docs/input-forwarding-notes.md b/paseo-shared-browser/docs/input-forwarding-notes.md new file mode 100644 index 00000000..99585989 --- /dev/null +++ b/paseo-shared-browser/docs/input-forwarding-notes.md @@ -0,0 +1,76 @@ +# Input forwarding constraints and checks + +The shared canvas forwards input only while its viewer owns control. Page state +survives display-mode changes. Control loss, focus loss, navigation and display +changes release held input against its original browser attachment. Never replay +an action whose publication or outcome is uncertain. + +## Keyboard and text + +Physical keys carry key/code, modifiers, repeat and press/release edges. Committed +text uses Chromium's separate insertion operation. Dead keys, AltGraph, Option +text and composition cannot be inferred reliably from a printable key alone. +The web relay uses DOM composition and insertion evidence to avoid duplicate or +lost text. Tab changes remote focus while the local relay keeps receiving keys. +Paseo inputs outside the focused canvas remain local. + +Native phone typing uses an invisible TextInput with autocorrection disabled. +Only basic append/backspace operations are live. Complex IME text is composed +locally and inserted once with Done. React Native provides no portable explicit +composition-commit boundary; replacement diffs must not delete unknown remote +text. Android TextInput key callbacks do not expose the full hardware-key stream. +Phone hardware shortcuts therefore remain unverified and unsupported by this relay. + +Local plain-text paste is forwarded. Copy/cut target the remote browser clipboard; +there is no remote-to-local clipboard synchronization. Browser and operating-system +reserved shortcuts may remain local. Global keyboard lock is not used. + +## Mouse, touch and scrolling + +Coordinates use the displayed image rectangle and CSS viewport, not image pixels +or device pixel ratio. Mouse movement carries both the held button and button +mask: native scrollbar dragging fails if movement reports button:none despite a +held left button. Hover, wheel and held drags use ordered continuous input. + +Touch routes retain contact identities and cancel on ownership changes. Avoid +forwarding synthesized compatibility mouse events a second time. Canvas-scoped +non-passive listeners and touch-action control avoid scrolling the host while +allowing host controls outside the canvas to work normally. + +Right-click forwards native page input and suppresses the local viewer context +menu. This does not guarantee that Chromium browser chrome context menus appear +in page screenshots. Native mouse dragging is distinct from HTML5 data-transfer +drag/drop; file-drop and cross-application drag payloads are not implemented. + +A stale decoded frame can fail initial admission. Only confirmed unpublished +admission is refreshed and retried within one bounded budget. Published updates +are never replayed. Discrete MCP input retains its strict frame validation and is +a different path from the human pane's ordered gesture channel. + +## Evidence and remaining acceptance + +Actual isolated Chromium checks cover trusted typing, Unicode, repeat, shortcuts, +focus traversal, Ctrl+click and held-key cleanup. Mounted DOM checks cover IME, +AltGraph, Option insertion, paste and control replacement. Native phone relay +checks validate event routing and lifecycle, but do not replace physical-device +software-keyboard acceptance. Human pane scrollbar dragging was confirmed. + +Headless Linux Chromium can lack pointer/hover media even when mouse events work. +When existing Xvfb is available, a private authenticated display supplies native +desktop mouse capabilities and survives touch-disable and CDP reconnect. Desktop +-> phone -> desktop preserves the current document and draft. Missing Xvfb retains +headless behavior; other operating systems are outside this verified correction. +Do not compensate by changing PrintStream styling or resetting the page per toggle. + +## Primary references + +- [Chromium Input protocol](https://chromedevtools.github.io/devtools-protocol/tot/Input/): key/text and mouse/touch contracts. +- [Playwright input](https://playwright.dev/docs/input): drag/drop event sequencing and native input operations. +- [Playwright keyboard](https://playwright.dev/docs/api/class-keyboard): text insertion differs from physical keys. +- [React Native TextInput](https://reactnative.dev/docs/textinput): native text/key events and platform limitations. +- [W3C Pointer Events](https://www.w3.org/TR/pointerevents/): touch-action, cancellation and compatibility mouse events. +- [W3C UI Events](https://www.w3.org/TR/uievents/): key, code, modifiers and composition. +- [Keyboard API](https://developer.mozilla.org/en-US/docs/Web/API/Keyboard_API): platform availability and reserved-key constraints. + +These are engineering constraints, not evidence of the upstream author's reasons +for choosing the previous controls. diff --git a/paseo-shared-browser/index.server.ts b/paseo-shared-browser/index.server.ts index 0d4e6b70..dfa67e70 100644 --- a/paseo-shared-browser/index.server.ts +++ b/paseo-shared-browser/index.server.ts @@ -15,6 +15,9 @@ import { handleReleaseControl, handleResizeBrowser, handleSendBrowserInput, + handleBeginBrowserGesture, + handleUpdateBrowserGesture, + handleEndBrowserGesture, handleWorkspaceArchived, issueAgentTicket, revokeAgentBrowserAccess, @@ -31,8 +34,13 @@ import { releaseControlRpc, resizeBrowserRpc, sendBrowserInputRpc, + beginBrowserGestureRpc, + updateBrowserGestureRpc, + endBrowserGestureRpc, } from "./shared/browser"; +import { browserDisplayPreferences } from "./shared/browser-display-preferences"; + const TICKET_ENV = "PASEO_SHARED_BROWSER_TICKET"; const MCP_SERVER_ID = "shared-browser"; const TICKET_ISSUE_TIMEOUT_MS = 2_000; @@ -67,12 +75,16 @@ function mcpBundlePath(): string { } export default function contribute(server: PluginServerContext) { + server.registerSettings(browserDisplayPreferences); server.handle(attachBrowserRpc, handleAttachBrowser); server.handle(detachBrowserRpc, handleDetachBrowser); server.handle(captureBrowserRpc, handleCaptureBrowser); server.handle(listOpenBrowserWorkspacesRpc, handleListOpenBrowserWorkspaces); server.handle(acquireControlRpc, handleAcquireControl); server.handle(releaseControlRpc, handleReleaseControl); + server.handle(beginBrowserGestureRpc, handleBeginBrowserGesture); + server.handle(updateBrowserGestureRpc, handleUpdateBrowserGesture); + server.handle(endBrowserGestureRpc, handleEndBrowserGesture); server.handle(navigateBrowserRpc, handleNavigateBrowser); server.handle(resizeBrowserRpc, handleResizeBrowser); server.handle(applyDevicePresetRpc, handleApplyDevicePreset); diff --git a/paseo-shared-browser/package.json b/paseo-shared-browser/package.json index 8547b9fa..ca15651d 100644 --- a/paseo-shared-browser/package.json +++ b/paseo-shared-browser/package.json @@ -43,8 +43,8 @@ "check:write": "biome check --write .", "typecheck": "tsc --noEmit", "prepare:runtime": "node scripts/prepare-runtime.mjs", - "test": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts", - "test:unit": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts", + "test": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts client server", + "test:unit": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts client server", "test:smoke": "vitest run tests/browser.smoke.test.ts --testTimeout=30000" }, "dependencies": { diff --git a/paseo-shared-browser/paseo-plugin.json b/paseo-shared-browser/paseo-plugin.json index 28b65397..6b244031 100644 --- a/paseo-shared-browser/paseo-plugin.json +++ b/paseo-shared-browser/paseo-plugin.json @@ -1,6 +1,18 @@ { "id": "shared-browser", "description": "One real Chromium session per Paseo workspace, shared live across every connected client.", - "requirements": { "paseo": "^0.9.0 || ^0.10.0 || ^0.11.0" }, - "build": [["node", "scripts/prepare-dependencies.mjs"], ["npm", "run", "prepare:runtime"]] + "requirements": { + "paseo": "^0.9.0 || ^0.10.0 || ^0.11.0 || 0.11.0-beta.3" + }, + "build": [ + [ + "node", + "scripts/prepare-dependencies.mjs" + ], + [ + "npm", + "run", + "prepare:runtime" + ] + ] } diff --git a/paseo-shared-browser/server/agent-browser-runtime.ts b/paseo-shared-browser/server/agent-browser-runtime.ts index eefc616e..ff444c31 100644 --- a/paseo-shared-browser/server/agent-browser-runtime.ts +++ b/paseo-shared-browser/server/agent-browser-runtime.ts @@ -13,6 +13,17 @@ import { listPageTargets, } from "./cdp"; +import { DEFAULT_JPEG_QUALITY, captureDimensions } from "../shared/capture-settings"; +import { MAX_VIEWPORT } from "../shared/viewport-limits"; +import { readJpegFrameDimensions } from "./jpeg-frame"; +export { readJpegFrameDimensions as jpegDimensions } from "./jpeg-frame"; +import { CAPTURE_MAX_AGE_MS, createCaptureTransportPolicy } from "./capture-transport-policy"; +import { browserCursorSchema, type BrowserCursor } from "../shared/browser"; +import { nativeKeyEvent, heldKeyModifiers } from "./keyboard-input"; +import { browserGestureKeySchema, type BrowserGestureKeyEvent } from "../shared/browser"; +import { GESTURE_IDLE_MS, GESTURE_LIFETIME_MS } from "./browser-gesture"; +import { formatRuntimeInputGeneration } from "./input-generation"; + const execFileAsync = promisify(execFile); export const AGENT_BROWSER_VERSION = "0.37.1"; const PRIVATE_DIRECTORY_MODE = 0o700; @@ -34,6 +45,8 @@ export interface BrowserViewport { width: number; height: number; deviceScaleFactor: number; + /** Capture density does not change the emulated layout or pointer coordinate system. */ + captureScale?: number; mobile: boolean; touch: boolean; userAgent?: string; @@ -68,6 +81,8 @@ export interface RuntimePageState { title: string; canGoBack: boolean; canGoForward: boolean; + /** Process-local attachment/document revision, including same-URL reloads. */ + inputGeneration: string; } export interface RuntimeFrame { @@ -94,6 +109,8 @@ export interface AgentBrowserRuntimeOptions { session: string; initialUrl?: string; headed?: boolean; + /** Trusted per-instance private X11 context, only DISPLAY/XAUTHORITY; never mutates host environment. */ + launchEnvironment?: Readonly>; timeoutMs?: number; } @@ -115,7 +132,7 @@ interface PageLifecycleEvent { interface ScreencastFrame { data: string; - metadata: { deviceWidth: number; deviceHeight: number }; + metadata: { deviceWidth: number; deviceHeight: number; timestamp?: number }; sessionId: number; } @@ -124,7 +141,10 @@ function requireAbsolute(path: string, label: string): string { return resolve(path); } -function runtimeEnvironment(ipcDirectory: string): NodeJS.ProcessEnv { +function runtimeEnvironment( + ipcDirectory: string, + launchEnvironment?: Readonly>, +): NodeJS.ProcessEnv { const environment: NodeJS.ProcessEnv = {}; for (const key of [ "PATH", @@ -153,6 +173,26 @@ function runtimeEnvironment(ipcDirectory: string): NodeJS.ProcessEnv { const value = process.env[key]; if (value !== undefined) environment[key] = value; } + if (launchEnvironment !== undefined) { + const keys = Object.keys(launchEnvironment); + const display = launchEnvironment.DISPLAY; + const authority = launchEnvironment.XAUTHORITY; + if ( + keys.length !== 2 || + keys.some(key => key !== "DISPLAY" && key !== "XAUTHORITY") || + typeof display !== "string" || + !/^:[0-9]{1,5}(?:\.[0-9]{1,2})?$/.test(display) || + typeof authority !== "string" || + !isAbsolute(authority) + ) { + // Do not include display/authentication values in errors or child arguments. + throw new AgentBrowserIncompatibleError("Private browser display environment is invalid"); + } + environment.DISPLAY = display; + environment.XAUTHORITY = authority; + // A private X11 display must not inherit an unrelated host Wayland session. + delete environment.WAYLAND_DISPLAY; + } environment.AGENT_BROWSER_SOCKET_DIR = ipcDirectory; environment.AGENT_BROWSER_IDLE_TIMEOUT_MS = "0"; environment.AGENT_BROWSER_STREAM_PORT = "0"; @@ -183,26 +223,6 @@ function findString(value: unknown, keys: readonly string[]): string | null { return null; } -export function jpegDimensions(base64: string): { width: number; height: number } | null { - const bytes = Buffer.from(base64, "base64"); - let offset = 2; - while (bytes[0] === 0xff && bytes[1] === 0xd8 && offset + 9 <= bytes.length) { - if (bytes[offset] !== 0xff) return null; - const marker = bytes[offset + 1]; - // SOFn carries the frame size; DHT (c4), JPG (c8) and DAC (cc) share the range but do not. - if ( - marker !== undefined && - marker >= 0xc0 && - marker <= 0xcf && - ![0xc4, 0xc8, 0xcc].includes(marker) - ) - return { width: bytes.readUInt16BE(offset + 7), height: bytes.readUInt16BE(offset + 5) }; - if (marker === 0xda) return null; - offset += 2 + bytes.readUInt16BE(offset + 2); - } - return null; -} - export class AgentBrowserRuntime { readonly binaryPath: string; readonly executablePath: string; @@ -216,13 +236,22 @@ export class AgentBrowserRuntime { private connection: CdpConnection | null = null; private page: CdpSession | null = null; private targetId: string | null = null; - private viewport: BrowserViewport | null = null; + private viewport: DeviceEmulation | null = null; + private emulationAppliedPage: CdpSession | null = null; + private attachmentGeneration = 0; + private readonly capturePolicy = createCaptureTransportPolicy(); + private streamEventSequence = 0; private screencastFrame: RuntimeFrame | null = null; + private screencastFrameReceivedAt: number | null = null; private screencastWaiters = new Set<(frame: RuntimeFrame | null) => void>(); private screencastActive = false; - private screencastQuality = 65; + private screencastQuality: number = DEFAULT_JPEG_QUALITY; private heldButtons = new Set(); - private heldKeys = new Set(); + private heldKeys = new Map(); + private heldTouches = false; + private activeTouches = new Map(); + private documentGeneration = 0; + private liveInput: { id: string; page: CdpSession; generation: number; documentGeneration: number; idleUntil: number; expiresAt: number; timer: ReturnType | null } | null = null; private stopping = false; constructor(options: AgentBrowserRuntimeOptions) { @@ -236,7 +265,7 @@ export class AgentBrowserRuntime { this.session = options.session; this.headed = options.headed ?? false; this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; - this.environment = runtimeEnvironment(this.ipcDirectory); + this.environment = runtimeEnvironment(this.ipcDirectory, options.launchEnvironment); this.initialUrl = options.initialUrl ?? "about:blank"; } @@ -259,6 +288,8 @@ export class AgentBrowserRuntime { this.profilePath, "--executable-path", this.executablePath, + "--hide-scrollbars", + "false", ...(this.headed ? ["--headed"] : []), ...(chromiumArguments ? ["--args", chromiumArguments] : []), "--json", @@ -272,6 +303,7 @@ export class AgentBrowserRuntime { async reconnect(): Promise { const targetId = this.targetId; + this.attachmentGeneration += 1; this.invalidateScreencastFrame(); this.connection?.close(); this.connection = null; @@ -325,21 +357,46 @@ export class AgentBrowserRuntime { } async selectTarget(targetId: string): Promise { + const attachmentGeneration = ++this.attachmentGeneration; + const connection = this.requireConnection(); const target = (await this.targets()).find((candidate) => candidate.targetId === targetId); + this.assertAttachmentCurrent(connection, attachmentGeneration); if (!target) throw new CdpUnavailableError(`Unknown page target: ${targetId}`); this.invalidateScreencastFrame(); - if (this.page) { + const previous = this.page; + if (previous) { if (this.screencastActive) { - await this.page.send("Page.stopScreencast", {}, { mutation: true }); + await previous.send("Page.stopScreencast", {}, { mutation: true }); + this.assertAttachmentCurrent(connection, attachmentGeneration); } - await this.page.detach(); + await previous.detach(); + this.assertAttachmentCurrent(connection, attachmentGeneration); } this.invalidateScreencastFrame(); - await this.requireConnection().send("Target.activateTarget", { targetId }, { mutation: true }); - this.page = await attachToTarget(this.requireConnection(), targetId); - this.targetId = targetId; - await this.preparePage(this.page); - if (this.screencastActive) await this.startScreencastSession(this.page); + await connection.send("Target.activateTarget", { targetId }, { mutation: true }); + this.assertAttachmentCurrent(connection, attachmentGeneration); + // CDP emulation is session-owned: detaching the former session restores native + // window metrics. Publish a replacement only after its retained settings succeed. + this.page = null; + this.emulationAppliedPage = null; + const replacement = await attachToTarget(connection, targetId); + try { + this.assertAttachmentCurrent(connection, attachmentGeneration); + await this.bindPageEvents(replacement); + this.assertAttachmentCurrent(connection, attachmentGeneration); + await this.restoreConfiguredEmulation(replacement); + this.assertAttachmentCurrent(connection, attachmentGeneration); + this.page = replacement; + this.targetId = targetId; + } catch (error) { + await replacement.detach().catch(() => undefined); + // Detaching a partial emulation can clear target metrics even if a newer + // attachment won. Its next access must restore intent before admitting pixels. + this.emulationAppliedPage = null; + this.invalidateScreencastFrame(); + throw error; + } + if (this.screencastActive) await this.startScreencastSession(replacement); } private async navigationHistory(page: CdpSession): Promise { @@ -357,20 +414,35 @@ export class AgentBrowserRuntime { } } } + /** Read fresh metadata concurrently; never combine observations across native document changes. */ async state(): Promise { const page = await this.requirePage(); - const history = await this.navigationHistory(page); - const evaluated = await page.send<{ - result: { value?: { url?: string; title?: string } }; - }>("Runtime.evaluate", { - expression: "({url: location.href, title: document.title})", - returnByValue: true, - }); + const attachmentGeneration = this.attachmentGeneration; + const documentGeneration = this.documentGeneration; + // Native history and fixed DOM metadata are independent reads. Overlap their + // CDP round trips without caching metadata or weakening input attachment checks. + const [history, evaluated] = await Promise.all([ + this.navigationHistory(page), + page.send<{ + result: { value?: { url?: string; title?: string } }; + }>("Runtime.evaluate", { + expression: "({url: location.href, title: document.title})", + returnByValue: true, + }), + ]); + if ( + page !== this.page || + attachmentGeneration !== this.attachmentGeneration || + documentGeneration !== this.documentGeneration + ) { + throw new CdpUnavailableError("Browser metadata changed while being read"); + } return { url: evaluated.result.value?.url ?? history.entries[history.currentIndex]?.url ?? "", title: evaluated.result.value?.title ?? history.entries[history.currentIndex]?.title ?? "", canGoBack: history.currentIndex > 0, canGoForward: history.currentIndex < history.entries.length - 1, + inputGeneration: formatRuntimeInputGeneration(this.attachmentGeneration, this.documentGeneration), }; } @@ -445,12 +517,72 @@ export class AgentBrowserRuntime { async emulate(device: DeviceEmulation): Promise { this.assertViewport(device); await this.withInvalidatedScreencast(async (page) => { + const attachmentGeneration = this.attachmentGeneration; + this.emulationAppliedPage = null; await this.applyEmulation(page, device); + this.assertAttachmentCurrent(page.connection, attachmentGeneration); + if (this.page !== page) throw new CdpUnavailableError("Browser page changed during emulation"); this.viewport = { ...device }; + this.emulationAppliedPage = page; }); } - async startScreencast(quality = 65): Promise { + /** Apply the entire device contract to one CDP session without changing saved intent. */ + private async applyEmulation(page: CdpSession, device: DeviceEmulation): Promise { + await page.send( + "Emulation.setDeviceMetricsOverride", + { + width: device.width, + height: device.height, + deviceScaleFactor: device.deviceScaleFactor, + mobile: device.mobile, + screenWidth: device.screenWidth ?? device.width, + screenHeight: device.screenHeight ?? device.height, + screenOrientation: { + type: device.width > device.height ? "landscapePrimary" : "portraitPrimary", + angle: device.width > device.height ? 90 : 0, + }, + }, + { mutation: true }, + ); + await page.send( + "Emulation.setTouchEmulationEnabled", + { enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 }, + { mutation: true }, + ); + if (device.userAgent) { + await page.send( + "Emulation.setUserAgentOverride", + { userAgent: device.userAgent, platform: device.platform ?? "" }, + { mutation: true }, + ); + } + } + + /** No frame/input may use a session whose emulated viewport has not been restored. */ + private async restoreConfiguredEmulation(page: CdpSession): Promise { + const device = this.viewport; + if (!device || this.emulationAppliedPage === page) return; + const attachmentGeneration = this.attachmentGeneration; + this.invalidateScreencastFrame(); + this.emulationAppliedPage = null; + await this.applyEmulation(page, device); + this.assertAttachmentCurrent(page.connection, attachmentGeneration); + if (device !== this.viewport) { + throw new CdpUnavailableError("Browser device settings changed during restoration"); + } + this.emulationAppliedPage = page; + this.invalidateScreencastFrame(); + } + + /** Reject late configuration completions after a disconnect or newer target selection. */ + private assertAttachmentCurrent(connection: CdpConnection, generation: number): void { + if (connection !== this.connection || !connection.isOpen || generation !== this.attachmentGeneration) { + throw new CdpUnavailableError("Browser attachment changed during emulation restoration"); + } + } + + async startScreencast(quality: number = DEFAULT_JPEG_QUALITY): Promise { if (!Number.isInteger(quality) || quality < 1 || quality > 100) { throw new RangeError("Screencast quality must be an integer from 1 to 100"); } @@ -471,82 +603,214 @@ export class AgentBrowserRuntime { } } - async frame(maxBytes: number, quality = 65, waitMs = 500): Promise { - if (!Number.isInteger(maxBytes) || maxBytes < 1) + /** Return truthful fresh stream/fallback pixels; mutation/session changes revoke pending captures. */ + async frame( + maxBytes: number, + quality: number = DEFAULT_JPEG_QUALITY, + waitMs = 500, + ): Promise { + if (!Number.isInteger(maxBytes) || maxBytes < 1) { throw new RangeError("maxBytes must be positive"); - const streamed = this.screencastFrame ?? (await this.waitForFrame(waitMs)); - if (streamed && streamed.byteLength <= maxBytes) return streamed; - let frame = await this.screenshot(await this.requirePage(), maxBytes, quality); - if (!this.sizeMismatch(frame)) return frame; - // The browser stopped honouring the cached viewport. Re-apply it once; never loop. - await this.emulate(this.requireViewport()); - frame = await this.screenshot(await this.requirePage(), maxBytes, quality); - const mismatch = this.sizeMismatch(frame); - if (mismatch) throw new Error(`Browser returned a ${mismatch} after viewport resync`); - return frame; - } - - private async screenshot(page: CdpSession, maxBytes: number, quality: number) { - for (const candidate of [quality, 50, 35, 20, 10, 1]) { - const boundedQuality = Math.max(1, Math.min(100, Math.round(candidate))); - const result = await page.send<{ data: string }>("Page.captureScreenshot", { - format: "jpeg", - quality: boundedQuality, - fromSurface: true, - captureBeyondViewport: false, - }); - const byteLength = Buffer.byteLength(result.data, "base64"); - if (byteLength <= maxBytes) { - const viewport = this.requireViewport(); + } + if (!Number.isFinite(quality)) throw new RangeError("JPEG quality must be finite"); + await this.requirePage(); + const requestedQuality = Math.max(1, Math.min(100, Math.round(quality))); + if (requestedQuality !== this.screencastQuality) { + // Viewers choose their own quality; never restart the shared stream or + // classify its health from another viewer's screenshot preference. + const cached = this.capturePolicy.readScreenshot(performance.now(), requestedQuality, maxBytes); + return cached ?? await this.captureScreenshot(maxBytes, requestedQuality, true); + } + // Expiry discards pixels without resetting fallback dwell/recovery evidence. + // Mutations and connection changes use the stronger invalidation below. + const generation = this.capturePolicy.currentGeneration(); + const receivedAt = this.screencastFrameReceivedAt; + if (receivedAt === null || performance.now() - receivedAt > CAPTURE_MAX_AGE_MS) { + this.screencastFrame = null; + this.screencastFrameReceivedAt = null; + } + if (this.capturePolicy.canUseStream(performance.now())) { + const streamed = this.screencastFrame ?? (await this.waitForFrame(waitMs)); + if (!this.capturePolicy.isCurrent(generation)) { + throw new CdpUnavailableError("Browser capture was invalidated while waiting for a frame"); + } + if (streamed && streamed.byteLength <= maxBytes) { + return streamed; + } + } + + this.capturePolicy.enterFallback(performance.now()); + const cached = this.capturePolicy.readScreenshot(performance.now(), requestedQuality, maxBytes); + if (cached) { + return cached; + } + return this.captureScreenshot(maxBytes, requestedQuality); + } + + /** Capture the configured viewport and cache only an unchanged-session/mutation completion. */ + private async captureScreenshot(maxBytes: number, requestedQuality: number, preserveStreamRecovery = false): Promise { + const page = await this.requirePage(); + const screenshotGeneration = this.capturePolicy.beginScreenshot({ preserveRecovery: preserveStreamRecovery }); + try { + const viewport = this.requireViewport(); + const expectedPixels = captureDimensions(viewport, viewport.captureScale); + const readClip = async () => { + const metrics = await page.send<{ cssVisualViewport: { pageX: number; pageY: number } }>( + "Page.getLayoutMetrics", + ); + this.assertCaptureCurrent(page, screenshotGeneration); return { - dataBase64: result.data, - byteLength, + x: metrics.cssVisualViewport.pageX, + y: metrics.cssVisualViewport.pageY, width: viewport.width, height: viewport.height, + scale: (viewport.captureScale ?? 1) / viewport.deviceScaleFactor, + }; + }; + // Explicitly capture the visible CSS viewport, independent of the native window size. + // Preserve scroll position and normalize device pixel ratio to the frame/input contract. + let clip = await readClip(); + let resynced = false; + // Reduce quality gradually only when a detailed frame exceeds the byte budget. + const initialQuality = requestedQuality; + const candidates = [...new Set([initialQuality, 90, 85, 75, 65, 50, 35, 20, 10, 1])].filter( + (candidate) => candidate <= initialQuality, + ); + const capture = async (quality: number) => { + this.assertCaptureCurrent(page, screenshotGeneration); + const result = await page.send<{ data: string }>("Page.captureScreenshot", { + format: "jpeg", + quality, + fromSurface: true, + // Beyond-viewport capture replaces emulation and resets enabled touch. + captureBeyondViewport: false, + clip, + }); + this.assertCaptureCurrent(page, screenshotGeneration); + const dimensions = readJpegFrameDimensions(result.data); + if (!dimensions) throw new Error("Chromium returned a malformed JPEG screenshot"); + return { data: result.data, dimensions, byteLength: Buffer.byteLength(result.data, "base64") }; + }; + const matchesViewport = (dimensions: { width: number; height: number }) => + dimensions.width === expectedPixels.width && dimensions.height === expectedPixels.height; + + for (const candidate of candidates) { + const boundedQuality = Math.max(1, Math.min(100, Math.round(candidate))); + let pixels = await capture(boundedQuality); + if (!matchesViewport(pixels.dimensions) && !resynced) { + // Retain upstream viewport recovery, with captureScale rather than DPR: + // mobile CSS emulation and image density are independently configured. + // Recapture pixels only; uncertain input is never replayed. + resynced = true; + const attachmentGeneration = this.attachmentGeneration; + await this.applyEmulation(page, viewport); + this.assertAttachmentCurrent(page.connection, attachmentGeneration); + this.assertCaptureCurrent(page, screenshotGeneration); + if (viewport !== this.viewport) throw new CdpUnavailableError("Browser viewport changed during resync"); + this.emulationAppliedPage = page; + clip = await readClip(); + pixels = await capture(boundedQuality); + } + if (!matchesViewport(pixels.dimensions)) { + throw new Error(`Chromium returned a ${pixels.dimensions.width}x${pixels.dimensions.height} JPEG instead of ${expectedPixels.width}x${expectedPixels.height}`); + } + if (pixels.byteLength > maxBytes) continue; + + const frame: RuntimeFrame = { + dataBase64: pixels.data, + byteLength: pixels.byteLength, + width: pixels.dimensions.width, + height: pixels.dimensions.height, transport: "screenshot", capturedAt: new Date().toISOString(), - } satisfies RuntimeFrame; + }; + this.capturePolicy.rememberScreenshot( + screenshotGeneration, + frame, + performance.now(), + requestedQuality, + maxBytes, + ); + return frame; } + throw new Error(`JPEG screenshot exceeds ${maxBytes} bytes at minimum quality`); + } finally { + this.capturePolicy.endScreenshot(screenshotGeneration, performance.now()); } - throw new Error(`JPEG screenshot exceeds ${maxBytes} bytes at minimum quality`); } - private sizeMismatch(frame: RuntimeFrame): string | null { - const { width, height, deviceScaleFactor: scale } = this.requireViewport(); - const size = jpegDimensions(frame.dataBase64); - // Chromium rounds fractional device pixels, so allow one pixel of slack. - if ( - size && - Math.abs(size.width - width * scale) <= 1 && - Math.abs(size.height - height * scale) <= 1 - ) - return null; - return `${size ? `${size.width}x${size.height}` : "unreadable"} JPEG for a ${width}x${height} viewport at ${scale}x`; + /** Refuse async pixels from a superseded session or mutation, without silently replaying capture. */ + private assertCaptureCurrent(page: CdpSession, generation: number): void { + if (page !== this.page || !this.capturePolicy.isCurrent(generation)) { + throw new CdpUnavailableError("Browser capture was invalidated while taking a screenshot"); + } } - async mouseMove(x: number, y: number): Promise { + /** Invalidate pre-input pixels even if Chromium cannot confirm the mutation. */ + private async dispatchInput(method: string, params: Record, gestureId?: string): Promise { + try { + const page = await this.requirePage(); + if (gestureId) await this.assertLiveInput(gestureId); + const originalInput = gestureId ? this.liveInput : null; + const modifiers = heldKeyModifiers(this.heldKeys.values()); + const nativeParams = method === "Input.dispatchMouseEvent" && modifiers + ? { ...params, modifiers } + : params; + await page.send(method, nativeParams, { mutation: true }); + if (gestureId) { + // An input can synchronously activate a link or submit a form. Its CDP + // acknowledgment succeeds even when navigation ends the old channel. + // Attachment replacement and uncertain sends retain their failure path. + if (originalInput && page === this.page + && originalInput.generation === this.attachmentGeneration + && originalInput.documentGeneration !== this.documentGeneration) { + await this.endLiveInput(gestureId); + return; + } + await this.assertLiveInput(gestureId); + this.renewLiveInput(gestureId); + } + } finally { + this.invalidateScreencastFrame(); + } + } + + async mouseMove(x: number, y: number, gestureId?: string): Promise { this.assertPoint(x, y); - await (await this.requirePage()).send( + await this.dispatchInput( "Input.dispatchMouseEvent", { type: "mouseMoved", x, y, - button: "none", + // Native scrollbar dragging needs the held button as well as its mask. + button: this.heldMoveButton(), buttons: this.buttonMask(), - }, - { mutation: true }, + }, gestureId, ); } + /** Fixed native mouse departure; never maps an outside point to a page control. */ + async mouseLeave(gestureId: string): Promise { + if (this.heldButtons.size > 0) throw new Error("Release held mouse buttons before leaving"); + await this.dispatchInput("Input.dispatchMouseEvent", { + type: "mouseMoved", x: -1, y: -1, button: "none", buttons: 0, + }, gestureId); + } + async mouseDown( x: number, y: number, button: MouseButton = "left", clickCount = 1, + gestureId?: string, ): Promise { this.assertPoint(x, y); - await (await this.requirePage()).send( + // Publication may succeed before an acknowledgement is lost. Track intent + // first so cleanup still releases a possibly held button. + if (gestureId) await this.assertLiveInput(gestureId); + this.heldButtons.add(button); + await this.dispatchInput( "Input.dispatchMouseEvent", { type: "mousePressed", @@ -555,15 +819,13 @@ export class AgentBrowserRuntime { button, buttons: this.buttonMask(button), clickCount, - }, - { mutation: true }, + }, gestureId, ); - this.heldButtons.add(button); } - async mouseUp(x: number, y: number, button: MouseButton = "left", clickCount = 1): Promise { + async mouseUp(x: number, y: number, button: MouseButton = "left", clickCount = 1, gestureId?: string): Promise { this.assertPoint(x, y); - await (await this.requirePage()).send( + await this.dispatchInput( "Input.dispatchMouseEvent", { type: "mouseReleased", @@ -572,15 +834,14 @@ export class AgentBrowserRuntime { button, buttons: this.buttonMask(undefined, button), clickCount, - }, - { mutation: true }, + }, gestureId, ); this.heldButtons.delete(button); } - async wheel(x: number, y: number, deltaX: number, deltaY: number): Promise { + async wheel(x: number, y: number, deltaX: number, deltaY: number, gestureId?: string): Promise { this.assertPoint(x, y); - await (await this.requirePage()).send( + await this.dispatchInput( "Input.dispatchMouseEvent", { type: "mouseWheel", @@ -590,69 +851,157 @@ export class AgentBrowserRuntime { deltaY, button: "none", buttons: this.buttonMask(), - }, - { mutation: true }, + }, gestureId, ); } - async insertText(text: string): Promise { - await (await this.requirePage()).send("Input.insertText", { text }, { mutation: true }); + /** Insert a committed paste/IME value on the original guarded attachment. */ + async insertText(text: string, gestureId?: string): Promise { + await this.dispatchInput("Input.insertText", { text }, gestureId); + } + + /** Track cleanup intent before publication, including an uncertain key-down reply. */ + async dispatchKey(event: BrowserGestureKeyEvent, gestureId: string): Promise { + const parsed = browserGestureKeySchema.parse(event); + await this.assertLiveInput(gestureId); + const held = this.heldKeys.has(parsed.code); + if (parsed.type === "down" && held !== parsed.repeat) { + throw new Error(parsed.repeat ? "Key repeat has no matching press" : "Key is already pressed"); + } + if (parsed.type === "up" && !held) throw new Error("Key release has no matching press"); + if (parsed.type === "down") this.heldKeys.set(parsed.code, { key: parsed.key, code: parsed.code }); + await this.dispatchInput("Input.dispatchKeyEvent", nativeKeyEvent(parsed), gestureId); + if (parsed.type === "up") this.heldKeys.delete(parsed.code); } async keyDown(key: string, code = key): Promise { - await (await this.requirePage()).send( - "Input.dispatchKeyEvent", - { - type: "keyDown", - key, - code, - text: key.length === 1 ? key : undefined, - }, - { mutation: true }, - ); - this.heldKeys.add(key); + this.heldKeys.set(code, { key, code }); + await this.dispatchInput("Input.dispatchKeyEvent", { + type: "keyDown", key, code, text: key.length === 1 ? key : undefined, + }); } async keyUp(key: string, code = key): Promise { - await (await this.requirePage()).send( - "Input.dispatchKeyEvent", - { type: "keyUp", key, code }, - { mutation: true }, - ); - this.heldKeys.delete(key); + await this.dispatchInput("Input.dispatchKeyEvent", { type: "keyUp", key, code }); + this.heldKeys.delete(code); } + /** + * The caller supplies the complete active contact set. CDP touchMove does not + * release omitted contacts, so publish explicit contact ends before moving the + * survivors. Any uncertain publication retains cleanup intent until cancel. + */ async touch( type: "touchStart" | "touchMove" | "touchEnd" | "touchCancel", points: Array<{ x: number; y: number; id?: number }>, + gestureId?: string, ): Promise { for (const point of points) this.assertPoint(point.x, point.y); - await (await this.requirePage()).send( - "Input.dispatchTouchEvent", - { - type, - touchPoints: points.map((point, index) => ({ - x: point.x, - y: point.y, - id: point.id ?? index, - })), - }, - { mutation: true }, - ); + if (gestureId) await this.assertLiveInput(gestureId); + const contacts = points.map((point, index) => ({ ...point, id: point.id ?? index })); + const activeIds = new Set(contacts.map(point => point.id)); + const removed = type === "touchMove" + ? [...this.activeTouches.values()].filter(point => !activeIds.has(point.id)) + : []; + this.heldTouches = true; + if (removed.length > 0) { + await this.dispatchInput("Input.dispatchTouchEvent", { type: "touchEnd", touchPoints: removed }, gestureId); + } + await this.dispatchInput("Input.dispatchTouchEvent", { type, touchPoints: contacts }, gestureId); + // Navigation cleanup already discarded this channel's contacts. Do not + // recreate held-touch state after its acknowledged start caused navigation. + if (gestureId && this.liveInput?.id !== gestureId) return; + this.activeTouches.clear(); + if (type === "touchStart" || type === "touchMove") { + for (const point of contacts) this.activeTouches.set(point.id, point); + } + this.heldTouches = this.activeTouches.size > 0; } - async releaseHeldInput(): Promise { - const page = this.page; + /** Pin this channel to one CDP attachment. Reconnects never replay held input on a replacement. */ + async beginLiveInput(id: string): Promise { + if (this.liveInput) await this.endLiveInput(this.liveInput.id); + const page = await this.requirePage(); + const now = Date.now(); + this.liveInput = { id, page, generation: this.attachmentGeneration, documentGeneration: this.documentGeneration, + idleUntil: now + GESTURE_IDLE_MS, expiresAt: now + GESTURE_LIFETIME_MS, timer: null }; + this.scheduleLiveInputExpiry(); + } + + /** Compare after every asynchronous transport boundary, including automatic reconnect/reattach. */ + async assertLiveInput(id: string): Promise { + const page = await this.requirePage(); + if (this.liveInput?.id === id && (Date.now() >= this.liveInput.idleUntil || Date.now() >= this.liveInput.expiresAt)) { + await this.endLiveInput(id); + } + if (!this.liveInput || this.liveInput.id !== id || this.liveInput.page !== page + || this.liveInput.generation !== this.attachmentGeneration || this.liveInput.documentGeneration !== this.documentGeneration) { + throw new CdpUnavailableError("Live browser input attachment changed"); + } + } + + /** Cleanup is idempotent and targets the original attachment, never whatever is currently active. */ + async endLiveInput(id: string): Promise { + const current = this.liveInput; + if (!current || current.id !== id) return; + this.liveInput = null; + if (current.timer) clearTimeout(current.timer); + await this.releaseHeldInput(current.page); + } + + /** Only acknowledged input extends idle time; metadata/checks cannot keep a stranded press alive. */ + private renewLiveInput(id: string): void { + if (this.liveInput?.id !== id) return; + this.liveInput.idleUntil = Date.now() + GESTURE_IDLE_MS; + this.scheduleLiveInputExpiry(); + } + + /** Runtime-owned expiry survives plugin disconnect or a fenced supervisor IPC client. */ + private scheduleLiveInputExpiry(): void { + const current = this.liveInput; + if (!current) return; + if (current.timer) clearTimeout(current.timer); + const delay = Math.max(0, Math.min(current.idleUntil, current.expiresAt) - Date.now()); + current.timer = setTimeout(() => { + if (this.liveInput !== current) return; + void this.endLiveInput(current.id).catch(() => undefined); + }, delay); + current.timer.unref?.(); + } + + /** Fixed hit-test, safe cursor names only. Cursor failures cannot fail an acknowledged input. */ + async cursorAt(x: number, y: number, gestureId: string): Promise { + try { + this.assertPoint(x, y); + await this.assertLiveInput(gestureId); + const page = this.liveInput!.page; + const result = await page.send<{ result: { value?: unknown } }>("Runtime.evaluate", { + expression: `(() => { const e = document.elementFromPoint(${x}, ${y}); return e ? getComputedStyle(e).cursor : null; })()`, + returnByValue: true, + }); + await this.assertLiveInput(gestureId); + const parsed = browserCursorSchema.safeParse(result.result.value); + return parsed.success ? parsed.data : null; + } catch { return null; } + } + + async releaseHeldInput(page = this.page): Promise { if (!page) { this.heldButtons.clear(); this.heldKeys.clear(); + this.heldTouches = false; + this.activeTouches.clear(); return; } const buttons = [...this.heldButtons]; - const keys = [...this.heldKeys]; + const keys = [...this.heldKeys.values()]; this.heldButtons.clear(); this.heldKeys.clear(); + const touchHeld = this.heldTouches; + this.heldTouches = false; + this.activeTouches.clear(); await Promise.allSettled([ + ...(touchHeld ? [page.send("Input.dispatchTouchEvent", { type: "touchCancel", touchPoints: [] }, { mutation: true })] : []), ...buttons.map((button) => page.send( "Input.dispatchMouseEvent", @@ -667,13 +1016,13 @@ export class AgentBrowserRuntime { { mutation: true }, ), ), - ...keys.map((key) => + ...keys.map(({ key, code }) => page.send( "Input.dispatchKeyEvent", { type: "keyUp", key, - code: key, + code, }, { mutation: true }, ), @@ -684,6 +1033,7 @@ export class AgentBrowserRuntime { async shutdown(force = false): Promise { if (this.stopping) return; this.stopping = true; + if (this.liveInput) await this.endLiveInput(this.liveInput.id); await this.releaseHeldInput(); this.invalidateScreencastFrame(); if (!force) { @@ -776,6 +1126,8 @@ export class AgentBrowserRuntime { if (this.connection !== connection) return; this.page = null; this.targetId = null; + this.emulationAppliedPage = null; + this.attachmentGeneration += 1; this.invalidateScreencastFrame(); }); const targets = await listPageTargets(this.connection); @@ -786,7 +1138,14 @@ export class AgentBrowserRuntime { } private async requirePage(): Promise { - if (this.page && this.connection?.isOpen) return this.page; + if (this.page && this.connection?.isOpen) { + const page = this.page; + await this.restoreConfiguredEmulation(page); + if (this.page !== page || !this.connection?.isOpen) { + throw new CdpUnavailableError("Browser page changed during emulation restoration"); + } + return page; + } await this.reconnect(); if (!this.page) throw new CdpUnavailableError("No page target is attached"); return this.page; @@ -802,46 +1161,36 @@ export class AgentBrowserRuntime { return this.viewport; } - private async preparePage(page: CdpSession): Promise { - page.on("Page.screencastFrame", this.onScreencastFrame); + private async bindPageEvents(page: CdpSession): Promise { + // URL equality is insufficient for reload. Main-document and same-document + // navigation revoke the channel without treating child-frame loads as a reset. + let rootFrameId: string | undefined; + page.on("Page.frameNavigated", (event: { frame: { id: string; parentId?: string } }) => { + if (page !== this.page || event.frame.parentId) return; + rootFrameId = event.frame.id; + this.documentGeneration += 1; + void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined); + }); + page.on("Page.navigatedWithinDocument", (event: { frameId: string }) => { + if (page !== this.page || event.frameId !== rootFrameId) return; + this.documentGeneration += 1; + void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined); + }); + page.on("Page.screencastFrame", (event: ScreencastFrame) => + this.onScreencastFrame(page, event), + ); page.on("event", (event: CdpEvent) => { - if (event.method === "Inspector.targetCrashed") this.invalidateScreencastFrame(); + if (event.method === "Inspector.targetCrashed" && this.page === page) { + this.documentGeneration += 1; + void this.endLiveInput(this.liveInput?.id ?? "").catch(() => undefined); + this.emulationAppliedPage = null; + this.invalidateScreencastFrame(); + } }); await Promise.all([page.send("Page.enable"), page.send("Runtime.enable")]); + const tree = await page.send<{ frameTree?: { frame: { id: string } } }>("Page.getFrameTree"); + rootFrameId = tree.frameTree?.frame.id ?? rootFrameId; await this.navigationHistory(page); - // Emulation is scoped to the CDP session, so every fresh attach starts without it. - if (this.viewport) await this.applyEmulation(page, this.viewport); - } - - private async applyEmulation(page: CdpSession, device: DeviceEmulation): Promise { - await page.send( - "Emulation.setDeviceMetricsOverride", - { - width: device.width, - height: device.height, - deviceScaleFactor: device.deviceScaleFactor, - mobile: device.mobile, - screenWidth: device.screenWidth ?? device.width, - screenHeight: device.screenHeight ?? device.height, - screenOrientation: { - type: device.width > device.height ? "landscapePrimary" : "portraitPrimary", - angle: device.width > device.height ? 90 : 0, - }, - }, - { mutation: true }, - ); - await page.send( - "Emulation.setTouchEmulationEnabled", - { enabled: device.touch, maxTouchPoints: device.touch ? 5 : 1 }, - { mutation: true }, - ); - if (device.userAgent) { - await page.send( - "Emulation.setUserAgentOverride", - { userAgent: device.userAgent, platform: device.platform ?? "" }, - { mutation: true }, - ); - } } private async startScreencastSession(page: CdpSession): Promise { @@ -855,6 +1204,10 @@ export class AgentBrowserRuntime { candidate = await this.requirePage(); continue; } + await this.restoreConfiguredEmulation(candidate); + if (this.page !== candidate) { + throw new CdpUnavailableError("Browser page changed before screencast start"); + } await candidate.send( "Page.startScreencast", { format: "jpeg", quality: this.screencastQuality, everyNthFrame: 1 }, @@ -876,8 +1229,14 @@ export class AgentBrowserRuntime { } private async reattachPageForScreencast(previous: CdpSession): Promise { + if (this.page !== previous) { + if (!this.page) throw new CdpUnavailableError("Browser page changed during reattachment"); + return this.page; + } + const attachmentGeneration = ++this.attachmentGeneration; const connection = this.requireConnection(); const targets = await listPageTargets(connection); + this.assertAttachmentCurrent(connection, attachmentGeneration); const target = targets.find((candidate) => candidate.targetId === this.targetId) ?? targets[0]; if (!target) throw new CdpUnavailableError("Chromium has no page target"); await connection.send( @@ -885,16 +1244,41 @@ export class AgentBrowserRuntime { { targetId: target.targetId }, { mutation: true }, ); + this.assertAttachmentCurrent(connection, attachmentGeneration); const replacement = await attachToTarget(connection, target.targetId); - await this.preparePage(replacement); - if (this.page === previous) { + if (this.page !== previous || attachmentGeneration !== this.attachmentGeneration) { + await replacement.detach().catch(() => undefined); + // The losing controller's detach may clear target overrides. Repair the + // current controller on its next access rather than trust earlier pixels. + this.emulationAppliedPage = null; + this.invalidateScreencastFrame(); + if (!this.page) throw new CdpUnavailableError("Browser page changed during reattachment"); + return this.page; + } + this.assertAttachmentCurrent(connection, attachmentGeneration); + + this.invalidateScreencastFrame(); + this.page = null; + this.emulationAppliedPage = null; + // Detach first: the former session can otherwise clear the newly applied override. + try { + await previous.detach(); + this.assertAttachmentCurrent(connection, attachmentGeneration); + await this.bindPageEvents(replacement); + this.assertAttachmentCurrent(connection, attachmentGeneration); + await this.restoreConfiguredEmulation(replacement); + this.assertAttachmentCurrent(connection, attachmentGeneration); this.page = replacement; this.targetId = target.targetId; - } else { + return replacement; + } catch (error) { await replacement.detach().catch(() => undefined); + // Detaching a partial emulation can clear target metrics even if a newer + // attachment won. Its next access must restore intent before admitting pixels. + this.emulationAppliedPage = null; + this.invalidateScreencastFrame(); + throw error; } - await previous.detach().catch(() => undefined); - return this.page ?? replacement; } private async withInvalidatedScreencast( @@ -908,23 +1292,33 @@ export class AgentBrowserRuntime { await mutation(page); } finally { this.invalidateScreencastFrame(); - if (restart && this.page === page) await this.startScreencastSession(page); + if (restart && this.page === page) { + await this.restoreConfiguredEmulation(page); + await this.startScreencastSession(page); + } } } + /** Clear both sources and revoke every in-flight capture after input or transport changes. */ private invalidateScreencastFrame(): void { + this.capturePolicy.invalidate(); this.screencastFrame = null; + this.screencastFrameReceivedAt = null; this.resolveFrameWaiters(null); } - private readonly onScreencastFrame = (event: ScreencastFrame): void => { - void this.page - ?.send("Page.screencastFrameAck", { sessionId: event.sessionId }) - .catch(() => undefined); - const width = Math.round(event.metadata.deviceWidth); - const height = Math.round(event.metadata.deviceHeight); + private onScreencastFrame(page: CdpSession, event: ScreencastFrame): void { + // A late frame belongs to its emitting session, even after page reattachment. + void page.send("Page.screencastFrameAck", { sessionId: event.sessionId }).catch(() => undefined); + if (page !== this.page || !this.screencastActive || this.emulationAppliedPage !== page) return; + const dimensions = readJpegFrameDimensions(event.data); const viewport = this.viewport; - if (!viewport || width !== viewport.width || height !== viewport.height) return; + // Chromium can report the new device dimensions while emitting a clipped transition image. + // Reject those pixels and use the bounded screenshot fallback rather than mislabeling them. + if (!viewport || !dimensions) return; + const expectedPixels = captureDimensions(viewport, viewport.captureScale); + if (dimensions.width !== expectedPixels.width || dimensions.height !== expectedPixels.height) return; + const { width, height } = dimensions; const frame: RuntimeFrame = { dataBase64: event.data, byteLength: Buffer.byteLength(event.data, "base64"), @@ -933,9 +1327,16 @@ export class AgentBrowserRuntime { transport: "cdp-screencast", capturedAt: new Date().toISOString(), }; + const now = performance.now(); + const timestamp = event.metadata.timestamp; + const identity = typeof timestamp === "number" && Number.isFinite(timestamp) + ? `provider:${timestamp}` + : `event:${++this.streamEventSequence}`; + this.capturePolicy.observeStream(identity, now); this.screencastFrame = frame; + this.screencastFrameReceivedAt = now; this.resolveFrameWaiters(frame); - }; + } private waitForFrame(waitMs: number): Promise { const { promise, resolve } = Promise.withResolvers(); @@ -958,6 +1359,12 @@ export class AgentBrowserRuntime { } private assertViewport(viewport: BrowserViewport): void { + const captureScale = viewport.captureScale ?? 1; + const pixels = captureDimensions(viewport, captureScale); + if (!Number.isFinite(captureScale) || captureScale < 1 || captureScale > 2 + || pixels.width > MAX_VIEWPORT.width || pixels.height > MAX_VIEWPORT.height) { + throw new RangeError("Capture density exceeds the supported image bounds"); + } if ( !Number.isInteger(viewport.width) || viewport.width < 1 || @@ -986,6 +1393,18 @@ export class AgentBrowserRuntime { } } + /** CDP/Puppeteer priority for supported held buttons, independent of press order. + * https://github.com/puppeteer/puppeteer/blob/main/packages/puppeteer-core/src/cdp/Input.ts + */ + private heldMoveButton(): MouseButton | "none" { + for (const button of ["left", "right", "middle"] as const) { + if (this.heldButtons.has(button)) { + return button; + } + } + return "none"; + } + private buttonMask(add?: MouseButton, remove?: MouseButton): number { const active = new Set(this.heldButtons); if (add) active.add(add); diff --git a/paseo-shared-browser/server/browser-gesture.ts b/paseo-shared-browser/server/browser-gesture.ts new file mode 100644 index 00000000..6e2697f7 --- /dev/null +++ b/paseo-shared-browser/server/browser-gesture.ts @@ -0,0 +1,111 @@ +/** + * One ordered live input channel, anchored to a decoded frame by browser-policy. + * Owns only bounded gesture state/coordinates. The host supplies serialized + * authority checks and transport cleanup; this module never grants a lease. + */ +import type { BrowserGestureEvent, Viewport } from "../shared/browser"; +import { mapDisplayedPoint } from "../shared/browser"; + +export const GESTURE_IDLE_MS = 5_000; +export const GESTURE_LIFETIME_MS = 5 * 60_000; + +export class BrowserGesture { + nextSequence = 1; + idleUntil: number; + readonly expiresAt: number; + readonly buttons = new Set(); + readonly touches = new Set(); + readonly keys = new Set(); + private display: { width: number; height: number } | null = null; + timer: ReturnType | null = null; + lastPoint: { x: number; y: number } | null = null; + + constructor( + readonly id: string, + readonly viewerToken: string, + readonly controlToken: string, + readonly pointerKind: "mouse" | "touch", + readonly expected: { + sessionId: string; + runtimeId: string; + bridgeEpoch: number; + navigationGeneration: number; + viewportGeneration: number; + }, + readonly viewport: Viewport, + now: number, + readonly runtimeInputGeneration: string | null = null, + ) { + this.idleUntil = now + GESTURE_IDLE_MS; + this.expiresAt = now + GESTURE_LIFETIME_MS; + } + + /** Reject dropped/replayed packets and expire even if a delayed queue resumes later. */ + assertSequence(sequence: number, now: number): void { + if (sequence !== this.nextSequence) throw new Error("Browser gesture sequence is stale"); + if (now >= this.idleUntil || now >= this.expiresAt) throw new Error("Browser gesture expired"); + } + + /** Pin displayed geometry on first point; a mid-gesture local layout change must cancel. */ + mapPoint(point: { x: number; y: number; width: number; height: number }) { + if (!this.display) this.display = { width: point.width, height: point.height }; + if (this.display.width !== point.width || this.display.height !== point.height) { + throw new Error("Browser gesture display geometry changed"); + } + const mapped = mapDisplayedPoint(point, this.viewport); + this.lastPoint = mapped; + return mapped; + } + + /** Validate edges before publication. Continuation never invents a missing button/touch start. */ + validate(event: BrowserGestureEvent): void { + if (event.kind === "leave" && this.buttons.size > 0) + throw new Error("Release held mouse buttons before leaving"); + if (event.kind === "key") { + if (event.type === "down" && this.keys.has(event.code) !== event.repeat) { + throw new Error(event.repeat ? "Key repeat has no matching press" : "Key is already pressed"); + } + if (event.type === "up" && !this.keys.has(event.code)) { + throw new Error("Key release has no matching press"); + } + return; + } + if (event.kind === "text") return; + if ((event.kind === "touch") !== (this.pointerKind === "touch")) + throw new Error("Browser gesture pointer kind changed"); + if (event.kind === "down" && this.buttons.has(event.button)) + throw new Error("Mouse button is already pressed"); + if (event.kind === "up" && !this.buttons.has(event.button)) + throw new Error("Mouse button is not pressed"); + if (event.kind === "touch") { + if ( + event.type === "move" && + (this.touches.size === 0 || event.points.some((point) => !this.touches.has(point.id))) + ) { + throw new Error("Touch move has no matching start"); + } + if (event.type === "start" && event.points.every((point) => this.touches.has(point.id))) { + throw new Error("Touch start must add a touch identifier"); + } + } + } + + /** Commit the acknowledged edge, preserving explicitly released touch IDs. */ + acknowledge(event: BrowserGestureEvent, now: number): void { + if (event.kind === "key" && event.type === "down") this.keys.add(event.code); + if (event.kind === "key" && event.type === "up") this.keys.delete(event.code); + if (event.kind === "down") this.buttons.add(event.button); + if (event.kind === "up") this.buttons.delete(event.button); + if (event.kind === "touch") { + this.touches.clear(); + for (const point of event.points) this.touches.add(point.id); + } + this.nextSequence += 1; + this.idleUntil = now + GESTURE_IDLE_MS; + } + + /** Only an unpressed mouse move leaves existing bounded frame authority usable for a new press. */ + isHover(event: BrowserGestureEvent): boolean { + return (event.kind === "move" || event.kind === "leave") && this.buttons.size === 0; + } +} diff --git a/paseo-shared-browser/server/browser-policy.ts b/paseo-shared-browser/server/browser-policy.ts index 5e680267..7010c60d 100644 --- a/paseo-shared-browser/server/browser-policy.ts +++ b/paseo-shared-browser/server/browser-policy.ts @@ -1,9 +1,10 @@ import { randomBytes } from "node:crypto"; -import type { RpcInput } from "@getpaseo/plugin"; +import type { RpcInput, RpcOutput } from "@getpaseo/plugin"; import { type applyDevicePresetRpc, type BrowserFrame, type BrowserInputEvent, + type BrowserGestureEvent, type BrowserState, DEFAULT_VIEWPORT, DEVICE_PRESETS, @@ -12,17 +13,29 @@ import { MAX_VIEWPORT, MIN_VIEWPORT, mapDisplayedPoint, + type beginBrowserGestureRpc, + type updateBrowserGestureRpc, + type endBrowserGestureRpc, + browserGestureEventSchema, + browserCursorSchema, type navigateBrowserRpc, type resizeBrowserRpc, type sendBrowserInputRpc, type Viewport, } from "../shared/browser"; import type { JsonValue } from "./runtime-protocol"; +import { DEFAULT_CAPTURE_QUALITY, DEFAULT_JPEG_QUALITY, JPEG_QUALITY, captureDimensions } from "../shared/capture-settings"; +import { BrowserGesture } from "./browser-gesture"; +import { sameRuntimeInputAttachment } from "./input-generation"; export type WorkspaceValidator = (workspaceId: string) => Promise; type NavigateInput = RpcInput; type ResizeInput = RpcInput; type ApplyDevicePresetInput = RpcInput; type SendInput = RpcInput; +type BeginGestureInput = RpcInput; +type UpdateGestureInput = RpcInput; +type UpdateGestureReply = Omit, "state"> & { state: BrowserState }; +type EndGestureInput = RpcInput; type CaptureQuality = "low" | "medium" | "high"; type InputTarget = SendInput["target"]; @@ -33,10 +46,7 @@ const FRAME_TOKEN_TTL_MS = 5_000; const MAX_RECENT_FRAMES = 32; const MAX_VIEWERS_PER_SESSION = 16; const MAX_SESSIONS = 8; -const SCREENCAST_QUALITY = 65; -const RUNTIME_FRAME_MAX_BYTES = 750_000; const SCREENCAST_WAIT_MS = 500; -const JPEG_QUALITY = { low: 40, medium: 65, high: 85 } as const; export interface BrowserRuntimeClient { connect(): Promise<{ epoch: number }>; @@ -77,6 +87,7 @@ interface BrowserSession { runtimeCreatedAt: number; bridgeEpoch: number; viewport: Viewport; + captureScale: number; navigationGeneration: number; viewportGeneration: number; devicePresetId: DevicePresetId | null; @@ -85,7 +96,12 @@ interface BrowserSession { viewers: Map; controller: Controller | null; mutationTail: Promise; - frameCache: Map; + frameCache: Map; + }>; recentFrames: Map< string, { navigationGeneration: number; viewportGeneration: number; expiresAt: number } @@ -96,6 +112,8 @@ interface BrowserSession { canGoForward: boolean; error: string | null; archived: boolean; + gesture: BrowserGesture | null; + inputGeneration: string | null; } function defaultToken(): string { @@ -179,7 +197,10 @@ export class SessionManager { setBridgeEpoch(epoch: number): void { this.bridgeEpoch = epoch; - for (const session of this.sessions.values()) session.bridgeEpoch = epoch; + for (const session of this.sessions.values()) { + if (session.bridgeEpoch !== epoch) void this.cancelGesture(session); + session.bridgeEpoch = epoch; + } } async attach( @@ -200,7 +221,7 @@ export class SessionManager { const viewerToken = this.issueUniqueToken(); session.viewers.set(viewerToken, { label, expiresAt: this.now() + this.viewerTtlMs }); this.viewerSessions.set(viewerToken, session); - await this.request(session, "screencast.start", { quality: SCREENCAST_QUALITY }).catch( + await this.request(session, "screencast.start", { quality: DEFAULT_JPEG_QUALITY }).catch( () => undefined, ); try { @@ -220,7 +241,10 @@ export class SessionManager { this.pruneExpired(session); const detached = session.viewers.delete(viewerToken); this.viewerSessions.delete(viewerToken); - if (session.controller?.viewerToken === viewerToken) session.controller = null; + if (session.controller?.viewerToken === viewerToken) { + await this.cancelGesture(session); + session.controller = null; + } if (session.viewers.size === 0) await this.request(session, "screencast.stop", null).catch(() => undefined); return { detached }; @@ -236,6 +260,7 @@ export class SessionManager { const session = this.sessions.get(workspaceId); if (!session) return; session.archived = true; + await this.cancelGesture(session); for (const token of session.viewers.keys()) this.viewerSessions.delete(token); session.viewers.clear(); session.controller = null; @@ -264,7 +289,7 @@ export class SessionManager { async capture( viewerToken: string, - quality: CaptureQuality = "medium", + quality: CaptureQuality = DEFAULT_CAPTURE_QUALITY, knownFrameId: string | null = null, ): Promise<{ state: BrowserState; frame: BrowserFrame | null }> { const session = this.requireViewer(viewerToken); @@ -290,6 +315,7 @@ export class SessionManager { const current = session.controller; if (current && current.viewerToken !== viewerToken && !takeover) throw new Error("Browser control is held by another viewer"); + if (current?.viewerToken !== viewerToken) await this.cancelGesture(session); const controller = current?.viewerToken === viewerToken ? current @@ -310,6 +336,7 @@ export class SessionManager { const session = this.requireViewer(viewerToken); return this.serialize(session, async () => { this.requireController(session, viewerToken, controlToken); + await this.cancelGesture(session); session.controller = null; this.heartbeatViewer(session, viewerToken); return { state: await this.snapshotState(session, viewerToken) }; @@ -320,6 +347,7 @@ export class SessionManager { const session = this.requireViewer(input.viewerToken); return this.serialize(session, async () => { this.requireMutationAccess(session, input); + await this.cancelGesture(session); const navigationGeneration = session.navigationGeneration; if (input.action.kind === "goto") await this.request(session, "navigate", { url: normalizeBrowserUrl(input.action.url) }); @@ -344,6 +372,7 @@ export class SessionManager { const session = this.requireViewer(input.viewerToken); return this.serialize(session, async () => { this.requireMutationAccess(session, input); + await this.cancelGesture(session); assertViewport(input.viewport); if ( session.viewport.width !== input.viewport.width || @@ -360,6 +389,7 @@ export class SessionManager { }); session.viewport = { ...input.viewport }; session.devicePresetId = null; + session.captureScale = 1; session.userAgent = session.defaultUserAgent; session.viewportGeneration += 1; this.invalidateFrames(session); @@ -373,24 +403,31 @@ export class SessionManager { const session = this.requireViewer(input.viewerToken); return this.serialize(session, async () => { this.requireMutationAccess(session, input); + await this.cancelGesture(session); const preset = DEVICE_PRESETS.find(({ id }) => id === input.presetId); if (!preset) throw new Error("Unknown device preset"); + // Resolve the display under the session lock. A mode toggle must neither + // replay remembered dimensions nor change the capture's pixel resolution. + const viewport = input.preserveDisplay ? session.viewport : preset.viewport; + const captureScale = input.preserveDisplay ? session.captureScale : preset.captureScale; await this.request(session, "emulate", { - width: preset.viewport.width, - height: preset.viewport.height, + width: viewport.width, + height: viewport.height, deviceScaleFactor: preset.deviceScaleFactor, + captureScale, mobile: preset.isMobile, touch: preset.hasTouch, userAgent: preset.userAgent, platform: preset.platform, }); - session.viewport = { ...preset.viewport }; + session.viewport = { ...viewport }; session.devicePresetId = preset.id; + session.captureScale = captureScale; session.userAgent = preset.userAgent; session.viewportGeneration += 1; - session.navigationGeneration += 1; + // Emulation invalidates input geometry even when display dimensions stay + // fixed. UA-sniffing sites can use Reload; never discard a draft automatically. this.invalidateFrames(session); - await this.request(session, "reload", null); this.renewController(session, input.viewerToken); return { state: await this.snapshotState(session, input.viewerToken) }; }); @@ -401,6 +438,7 @@ export class SessionManager { return this.serialize(session, async () => { this.requireMutationAccess(session, input); this.requireRecentFrame(session, input.target); + await this.cancelGesture(session); await this.dispatchInput(session, input.event, input.target); this.invalidateFrames(session); this.renewController(session, input.viewerToken); @@ -408,8 +446,218 @@ export class SessionManager { }); } + /** Anchor a live channel to exact frame/control/runtime identity, without pressing anything. */ + async beginGesture(input: BeginGestureInput) { + const session = this.requireViewer(input.viewerToken); + return this.serialize(session, async () => { + this.requireMutationAccess(session, input); + await this.refreshPageMetadata(session); + this.requireMutationAccess(session, input); + if (!this.isRecentFrame(session, input.target)) { + // A pre-input capture can decode after scroll revoked its token. Report + // known non-publication, not an uncertain runtime failure. The client + // may obtain another decoded frame before admitting its still-unsent input. + const state = await this.snapshotState(session, input.viewerToken); + this.requireMutationAccess(session, input); + if (state.status !== "ready") throw new Error(state.error ?? "Browser runtime is unavailable"); + return { state, admission: "stale-frame" as const }; + } + await this.cancelGesture(session); + const gesture = new BrowserGesture(this.issueUniqueToken(), input.viewerToken, input.controlToken, + input.pointerKind, { ...input.expected }, { ...session.viewport }, this.now(), session.inputGeneration); + session.gesture = gesture; + try { + await this.request(session, "input.begin", { gestureId: gesture.id }); + await this.assertGestureCurrent(session, gesture); + this.renewController(session, input.viewerToken); + this.scheduleGestureTimeout(session, gesture); + const state = await this.snapshotState(session, input.viewerToken); + await this.assertGestureCurrent(session, gesture); + return { state, gestureId: gesture.id, nextSequence: gesture.nextSequence }; + } catch (error) { + await this.cancelGesture(session, gesture); + throw error; + } + }); + } + + /** Deliver one strictly ordered packet; an uncertain failure cancels without retrying its action. */ + async updateGesture(input: UpdateGestureInput): Promise { + const session = this.requireViewer(input.viewerToken); + return this.serialize(session, async () => { + const gesture = this.requireGesture(session, input); + let publishedEvent: BrowserGestureEvent | null = null; + let eventAcknowledged = false; + try { + this.assertGesturePacketContext(gesture, input.expected); + const event = browserGestureEventSchema.parse(input.event); + gesture.assertSequence(input.sequence, this.now()); + await this.assertGestureCurrent(session, gesture); + gesture.validate(event); + if (event.kind === "down" || (event.kind === "touch" && event.type === "start" && gesture.touches.size === 0)) { + if (!input.target) throw new Error("A current frame is required for a new press"); + this.requireRecentFrame(session, input.target); + } + const hover = gesture.isHover(event); + if (event.kind === "key") { + await this.request(session, "input.key", { gestureId: gesture.id, event: { + kind: event.kind, type: event.type, key: event.key, code: event.code, + modifiers: event.modifiers, repeat: event.repeat, + ...(event.text !== undefined ? { text: event.text } : {}), + } }); + } else if (event.kind === "text") { + await this.request(session, "input.text", { gestureId: gesture.id, text: event.text }); + } else if (event.kind === "leave") { + await this.request(session, "mouse.leave", { gestureId: gesture.id }); + gesture.lastPoint = null; + } else if (event.kind === "touch") { + const points = event.points.map(point => ({ ...gesture.mapPoint(point), id: point.id })); + await this.request(session, "touch", { gestureId: gesture.id, type: event.type, points }); + } else { + const point = gesture.mapPoint(event.point); + const operation = event.kind === "move" ? "mouse.move" : event.kind === "down" ? "mouse.down" + : event.kind === "up" ? "mouse.up" : "mouse.wheel"; + await this.request(session, operation, { ...point, gestureId: gesture.id, + ...(event.kind === "down" || event.kind === "up" ? { button: event.button, clickCount: event.clickCount } : {}), + ...(event.kind === "scroll" ? { deltaX: event.deltaX, deltaY: event.deltaY } : {}) }); + } + publishedEvent = event; + await this.assertGestureCurrent(session, gesture); + gesture.acknowledge(event, this.now()); + eventAcknowledged = true; + // Hover may alter pixels but still leaves the bounded decoded-frame token + // usable for a subsequent press. Scroll/held input revokes that authority. + session.frameCache.clear(); + if (!hover) session.recentFrames.clear(); + this.renewController(session, input.viewerToken); + this.scheduleGestureTimeout(session, gesture); + const cursor = await this.gestureCursor(session, gesture); + const state = await this.snapshotState(session, input.viewerToken); + await this.assertGestureCurrent(session, gesture); + return { state, gestureId: gesture.id, nextSequence: gesture.nextSequence, cursor }; + } catch (error) { + if (publishedEvent) { + const completion = await this.completeNavigatingInput(session, gesture, publishedEvent, eventAcknowledged).catch(() => null); + if (completion) return completion; + } + await this.cancelGesture(session, gesture); + throw error; + } + }); + } + + /** Return a fresh viewing state after acknowledged input navigated, without admitting another input. */ + private async completeNavigatingInput(session: BrowserSession, gesture: BrowserGesture, event: BrowserGestureEvent, eventAcknowledged: boolean) { + const state = await this.snapshotState(session, gesture.viewerToken); + if (state.navigationGeneration === gesture.expected.navigationGeneration) return null; + // Runtime inputGeneration is the private native attachment:document tuple. + // A target replacement can also bump policy navigationGeneration, but must + // never be accepted as same-attachment navigation after an acknowledged input. + if (!sameRuntimeInputAttachment(gesture.runtimeInputGeneration, session.inputGeneration)) return null; + + // Only the old document constraint may change after successful publication. + // Takeover, viewport changes and runtime/bridge fences are still refused. + this.requireMutationAccess(session, { + viewerToken: gesture.viewerToken, + controlToken: gesture.controlToken, + expected: { ...gesture.expected, navigationGeneration: state.navigationGeneration }, + }); + if (!eventAcknowledged) gesture.acknowledge(event, this.now()); + await this.cancelGesture(session, gesture); + const finalState = await this.snapshotState(session, gesture.viewerToken); + if (!sameRuntimeInputAttachment(gesture.runtimeInputGeneration, session.inputGeneration)) return null; + this.requireMutationAccess(session, { + viewerToken: gesture.viewerToken, + controlToken: gesture.controlToken, + expected: { ...gesture.expected, navigationGeneration: finalState.navigationGeneration }, + }); + this.invalidateFrames(session); + this.renewController(session, gesture.viewerToken); + return { state: finalState, gestureId: gesture.id, nextSequence: gesture.nextSequence, cursor: null, completion: "navigation" as const }; + } + + /** Cancellation releases only this channel, even after a lost update reply or navigation drift. */ + async endGesture(input: EndGestureInput) { + const session = this.requireViewer(input.viewerToken); + return this.serialize(session, async () => { + const gesture = session.gesture; + if (!gesture && input.cancel) { + this.requireController(session, input.viewerToken, input.controlToken); + return { state: await this.snapshotState(session, input.viewerToken), cursor: null }; + } + const owned = this.requireGesture(session, input); + try { + if (!input.cancel) { + this.assertGesturePacketContext(owned, input.expected); + owned.assertSequence(input.sequence, this.now()); + await this.assertGestureCurrent(session, owned); + } + } finally { + await this.cancelGesture(session, owned); + } + return { state: await this.snapshotState(session, input.viewerToken), cursor: null }; + }); + } + + private requireGesture(session: BrowserSession, input: UpdateGestureInput | EndGestureInput): BrowserGesture { + const gesture = session.gesture; + if (!gesture || gesture.id !== input.gestureId || gesture.viewerToken !== input.viewerToken + || gesture.controlToken !== input.controlToken) throw new Error("Browser gesture is unavailable"); + return gesture; + } + + private assertGesturePacketContext(gesture: BrowserGesture, expected: BeginGestureInput["expected"]): void { + // JSON member order is irrelevant; identity fields must match individually. + for (const key of ["sessionId", "runtimeId", "bridgeEpoch", "navigationGeneration", "viewportGeneration"] as const) { + if (gesture.expected[key] !== expected[key]) throw new Error("Browser gesture context changed"); + } + } + + private async assertGestureCurrent(session: BrowserSession, gesture: BrowserGesture): Promise { + this.requireMutationAccess(session, { viewerToken: gesture.viewerToken, controlToken: gesture.controlToken, expected: gesture.expected }); + if (session.gesture !== gesture || this.now() >= gesture.idleUntil || this.now() >= gesture.expiresAt) throw new Error("Browser gesture expired"); + // Runtime check pins the CDP document/attachment without repeating a full + // metadata read for every drag point. snapshotState supplies final metadata. + await this.request(session, "input.check", { gestureId: gesture.id }); + this.requireMutationAccess(session, { viewerToken: gesture.viewerToken, controlToken: gesture.controlToken, expected: gesture.expected }); + if (session.gesture !== gesture) throw new Error("Browser gesture was cancelled"); + } + + private async gestureCursor(session: BrowserSession, gesture: BrowserGesture) { + if (!gesture.lastPoint || gesture.pointerKind !== "mouse") return null; + try { + const raw = await this.request(session, "cursor", { ...gesture.lastPoint, gestureId: gesture.id }); + const parsed = browserCursorSchema.safeParse(raw); + return parsed.success ? parsed.data : null; + } catch { return null; } + } + + private scheduleGestureTimeout(session: BrowserSession, gesture: BrowserGesture): void { + if (gesture.timer) clearTimeout(gesture.timer); + gesture.timer = setTimeout(() => { + void this.serialize(session, async () => { + if (session.gesture === gesture && this.now() >= Math.min(gesture.idleUntil, gesture.expiresAt)) { + await this.cancelGesture(session, gesture); + } + }).catch(() => undefined); + }, Math.max(1, Math.min(gesture.idleUntil, gesture.expiresAt) - this.now())); + gesture.timer.unref?.(); + } + + private async cancelGesture(session: BrowserSession, expected?: BrowserGesture): Promise { + const gesture = session.gesture; + if (!gesture || (expected && gesture !== expected)) return; + session.gesture = null; + if (gesture.timer) clearTimeout(gesture.timer); + // Cleanup targets the runtime's original attachment, never a newly navigated + // or replaced page. Failure is best-effort and cannot replay the input. + try { await this.request(session, "input.end", { gestureId: gesture.id }); } + catch { /* Best-effort old-attachment cleanup must not escape teardown. */ } + } + reset(): void { this.lifecycleGeneration += 1; + for (const session of this.sessions.values()) void this.cancelGesture(session); this.sessions.clear(); this.sessionCreations.clear(); this.viewerSessions.clear(); @@ -481,6 +729,7 @@ export class SessionManager { runtimeCreatedAt: descriptor.createdAt, bridgeEpoch: this.bridgeEpoch, viewport: { ...DEFAULT_VIEWPORT }, + captureScale: 1, navigationGeneration: 0, viewportGeneration: 0, devicePresetId: null, @@ -497,6 +746,8 @@ export class SessionManager { canGoForward: Boolean(state.canGoForward), error: null, archived: false, + gesture: null, + inputGeneration: typeof state.inputGeneration === "string" ? state.inputGeneration : null, }; } catch (error) { if (ensured) await this.client.archiveWorkspace(workspaceId).catch(() => undefined); @@ -540,8 +791,10 @@ export class SessionManager { if ( session.controller && (session.controller.expiresAt <= now || !session.viewers.has(session.controller.viewerToken)) - ) + ) { session.controller = null; + void this.serialize(session, () => this.cancelGesture(session)).catch(() => undefined); + } } private requireController( session: BrowserSession, @@ -586,16 +839,21 @@ export class SessionManager { session: BrowserSession, target: { frameId: string; navigationGeneration: number; viewportGeneration: number }, ): void { + if (!this.isRecentFrame(session, target)) throw new Error("Browser frame is stale"); + } + + /** Check issued frame authority without admitting or cancelling a live channel. */ + private isRecentFrame( + session: BrowserSession, + target: { frameId: string; navigationGeneration: number; viewportGeneration: number }, + ): boolean { this.pruneRecentFrames(session); const frame = session.recentFrames.get(target.frameId); - if ( - !frame || - target.navigationGeneration !== session.navigationGeneration || - target.viewportGeneration !== session.viewportGeneration || - frame.navigationGeneration !== session.navigationGeneration || - frame.viewportGeneration !== session.viewportGeneration - ) - throw new Error("Browser frame is stale"); + return Boolean(frame && + target.navigationGeneration === session.navigationGeneration && + target.viewportGeneration === session.viewportGeneration && + frame.navigationGeneration === session.navigationGeneration && + frame.viewportGeneration === session.viewportGeneration); } private renewController(session: BrowserSession, token: string): void { this.heartbeatViewer(session, token); @@ -650,23 +908,32 @@ export class SessionManager { const cached = session.frameCache.get(quality); if ( cached && + cached.frame.sessionId === session.sessionId && + cached.frame.runtimeId === session.runtimeId && + cached.frame.captureEpoch === session.bridgeEpoch && cached.frame.navigationGeneration === session.navigationGeneration && cached.frame.viewportGeneration === session.viewportGeneration && this.now() - cached.cachedAt <= this.frameCacheMs ) return cached.frame; const generation = { + sessionId: session.sessionId, + runtimeId: session.runtimeId, + captureEpoch: session.bridgeEpoch, navigationGeneration: session.navigationGeneration, viewportGeneration: session.viewportGeneration, }; const raw = asRecord( await this.request(session, "frame", { - maxBytes: RUNTIME_FRAME_MAX_BYTES, + maxBytes: FRAME_MAX_BYTES, quality: JPEG_QUALITY[quality], waitMs: SCREENCAST_WAIT_MS, }), ); if ( + generation.sessionId !== session.sessionId || + generation.runtimeId !== session.runtimeId || + generation.captureEpoch !== session.bridgeEpoch || generation.navigationGeneration !== session.navigationGeneration || generation.viewportGeneration !== session.viewportGeneration ) @@ -680,8 +947,32 @@ export class SessionManager { Buffer.byteLength(dataBase64, "base64") !== byteLength ) throw new Error("Browser returned an invalid frame"); + const expectedPixels = captureDimensions(session.viewport, session.captureScale); + if (Number(raw.width) !== expectedPixels.width || Number(raw.height) !== expectedPixels.height) + throw new Error("Browser returned a frame for a stale viewport"); + + // Runtime fallback receipts remain fresh for at most one second. A 100ms + // policy-cache expiry must not create a new token and resend the same large + // JPEG at every poll. Reuse only its exact receipt, not merely equal pixels. + if ( + cached && + typeof raw.capturedAt === "string" && + Number.isFinite(Date.parse(raw.capturedAt)) && + (raw.transport === "screenshot" || raw.transport === "cdp-screencast") && + session.frameCache.get(quality) === cached && + cached.frame.sessionId === generation.sessionId && + cached.frame.runtimeId === generation.runtimeId && + cached.frame.captureEpoch === generation.captureEpoch && + cached.frame.navigationGeneration === generation.navigationGeneration && + cached.frame.viewportGeneration === generation.viewportGeneration && + (["dataBase64", "byteLength", "width", "height", "capturedAt", "transport"] as const) + .every(key => cached.rawFrame[key] === raw[key]) + ) { + cached.cachedAt = this.now(); + return cached.frame; + } + const frame: BrowserFrame = { - sessionId: session.sessionId, frameId: this.issueUniqueToken(), mimeType: "image/jpeg", transport: raw.transport === "cdp-screencast" ? "cdp-screencast" : "screenshot", @@ -690,13 +981,9 @@ export class SessionManager { width: Number(raw.width), height: Number(raw.height), ...generation, - runtimeId: session.runtimeId, - captureEpoch: session.bridgeEpoch, capturedAt: String(raw.capturedAt), }; - if (frame.width !== session.viewport.width || frame.height !== session.viewport.height) - throw new Error("Browser returned a frame for a stale viewport"); - session.frameCache.set(quality, { frame, cachedAt: this.now() }); + session.frameCache.set(quality, { frame, cachedAt: this.now(), rawFrame: { ...raw } }); return frame; } @@ -789,10 +1076,14 @@ export class SessionManager { private async refreshPageMetadata(session: BrowserSession): Promise { const raw = asRecord(await this.request(session, "state", null)); const url = boundedText(String(raw.url ?? ""), 8192); - if (session.lastUrl && session.lastUrl !== url) { + const inputGeneration = typeof raw.inputGeneration === "string" ? raw.inputGeneration : null; + const documentChanged = session.inputGeneration !== null && inputGeneration !== session.inputGeneration; + if ((session.lastUrl && session.lastUrl !== url) || documentChanged) { + await this.cancelGesture(session); session.navigationGeneration += 1; this.invalidateFrames(session); } + session.inputGeneration = inputGeneration; session.lastUrl = url; session.lastTitle = boundedText(String(raw.title ?? ""), 1024); session.canGoBack = Boolean(raw.canGoBack); @@ -822,6 +1113,7 @@ export class SessionManager { canGoBack: session.canGoBack, canGoForward: session.canGoForward, viewport: { ...session.viewport }, + captureScale: session.captureScale, navigationGeneration: session.navigationGeneration, viewportGeneration: session.viewportGeneration, devicePresetId: session.devicePresetId, diff --git a/paseo-shared-browser/server/browser.ts b/paseo-shared-browser/server/browser.ts index 050268cc..f2200218 100644 --- a/paseo-shared-browser/server/browser.ts +++ b/paseo-shared-browser/server/browser.ts @@ -16,6 +16,9 @@ import type { releaseControlRpc, resizeBrowserRpc, sendBrowserInputRpc, + beginBrowserGestureRpc, + updateBrowserGestureRpc, + endBrowserGestureRpc, } from "../shared/browser"; import { resolveBrowserRuntimeRoot } from "./runtime-path"; import type { JsonValue } from "./runtime-protocol"; @@ -48,6 +51,12 @@ type ApplyDevicePresetInput = RpcInput; type ApplyDevicePresetOutput = RpcOutput; type SendInput = RpcInput; type SendOutput = RpcOutput; +type BeginGestureInput = RpcInput; +type BeginGestureOutput = RpcOutput; +type UpdateGestureInput = RpcInput; +type UpdateGestureOutput = RpcOutput; +type EndGestureInput = RpcInput; +type EndGestureOutput = RpcOutput; function paseoHome(): string { return process.env.PASEO_HOME ?? join(homedir(), ".paseo"); @@ -113,6 +122,18 @@ class RemoteBrowserManager { return this.client.requestBrowser("input", input as unknown as JsonValue); } + beginGesture(input: BeginGestureInput): Promise { + return this.client.requestBrowser("gesture.begin", input as unknown as JsonValue); + } + + updateGesture(input: UpdateGestureInput): Promise { + return this.client.requestBrowser("gesture.update", input as unknown as JsonValue); + } + + endGesture(input: EndGestureInput): Promise { + return this.client.requestBrowser("gesture.end", input as unknown as JsonValue); + } + async listOpenWorkspaceIds(): Promise { const result = await this.client.requestBrowser("list", {}); if ( @@ -293,6 +314,19 @@ export async function handleSendBrowserInput(input: SendInput): Promise { + return (await getProductionManager()).beginGesture(input); +} + +export async function handleUpdateBrowserGesture(input: UpdateGestureInput): Promise { + return (await getProductionManager()).updateGesture(input); +} + +export async function handleEndBrowserGesture(input: EndGestureInput): Promise { + return (await getProductionManager()).endGesture(input); +} + export async function cleanupBrowserServer(): Promise { productionStopped = true; if (productionStart) await productionStart.catch(() => undefined); diff --git a/paseo-shared-browser/server/capture-transport-policy.test.ts b/paseo-shared-browser/server/capture-transport-policy.test.ts new file mode 100644 index 00000000..9c58ec2c --- /dev/null +++ b/paseo-shared-browser/server/capture-transport-policy.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "vitest"; +import { createCaptureTransportPolicy } from "./capture-transport-policy"; +import type { RuntimeFrame } from "./agent-browser-runtime"; + +const frame: RuntimeFrame = { + dataBase64: "Zml4dHVyZQ==", + byteLength: 7, + width: 1280, + height: 800, + transport: "screenshot", + capturedAt: "2026-10-02T12:00:00.000Z", +}; + +describe("capture transport policy", () => { + it("keeps the true screenshot and original capture time only through1000ms at exact quality/budget", () => { + const policy = createCaptureTransportPolicy(); + const token = policy.beginScreenshot(); + expect(policy.rememberScreenshot(token, frame, 100, 100, 1000)).toBe(true); + policy.endScreenshot(token, 100); + expect(policy.readScreenshot(1100, 100, 1000)).toBe(frame); + expect(policy.readScreenshot(1101, 100, 1000)).toBeNull(); + expect(policy.readScreenshot(500, 90, 1000)).toBeNull(); + expect(policy.readScreenshot(500, 100, 1001)).toBeNull(); + }); + + it("revokes old asynchronous completions on input/session invalidation", () => { + const policy = createCaptureTransportPolicy(); + const old = policy.beginScreenshot(); + policy.invalidate(); + const fresh = policy.beginScreenshot(); + expect(policy.rememberScreenshot(old, frame, 100, 100, 1000)).toBe(false); + policy.endScreenshot(old, 100); + expect(policy.readScreenshot(100, 100, 1000)).toBeNull(); + expect(policy.rememberScreenshot(fresh, frame, 200, 100, 1000)).toBe(true); + policy.invalidate(); + expect(policy.readScreenshot(200, 100, 1000)).toBeNull(); + }); + + it("cannot recover from screenshot-induced frames or repeated sparse capture cycles", () => { + const policy = createCaptureTransportPolicy(); + policy.enterFallback(0); + for (let time = 0; time < 6000; time += 1000) { + const token = policy.beginScreenshot(); + policy.observeStream(`inside-${time}`, time + 10); + policy.endScreenshot(token, time + 20); + policy.observeStream(`queued-${time}`, time + 100); + expect(policy.canUseStream(time + 500)).toBe(false); + } + }); + + it("requires3 distinct events over500ms with bounded gaps and retains the original dwell", () => { + const policy = createCaptureTransportPolicy(); + policy.enterFallback(0); + policy.observeStream("one", 1100); + policy.enterFallback(1200); + policy.observeStream("two", 1350); + expect(policy.canUseStream(1500)).toBe(false); + policy.observeStream("three", 1600); + expect(policy.canUseStream(1600)).toBe(true); + }); + + it("rejects duplicate events, burst-only evidence and stale recovery", () => { + const duplicate = createCaptureTransportPolicy(); + duplicate.enterFallback(0); + duplicate.observeStream("same", 1000); + duplicate.observeStream("same", 1250); + duplicate.observeStream("same", 1500); + expect(duplicate.canUseStream(1500)).toBe(false); + + const burst = createCaptureTransportPolicy(); + burst.enterFallback(0); + burst.observeStream("one", 1400); + burst.observeStream("two", 1450); + burst.observeStream("three", 1500); + expect(burst.canUseStream(1500)).toBe(false); + expect(burst.canUseStream(2100)).toBe(false); + }); + + it("a long inter-event gap resets evidence and invalidation clears all recovery", () => { + const policy = createCaptureTransportPolicy(); + policy.enterFallback(0); + policy.observeStream("one", 1000); + policy.observeStream("two", 1250); + policy.observeStream("three", 1800); + expect(policy.canUseStream(1800)).toBe(false); + policy.observeStream("four", 2050); + policy.invalidate(); + policy.observeStream("five", 2300); + expect(policy.canUseStream(2300)).toBe(false); + }); +}); + +it("quality-only captures preserve independent recovery and still exclude capture-induced events", () => { + const policy = createCaptureTransportPolicy(); + policy.enterFallback(0); + policy.observeStream("one", 1100); + const token = policy.beginScreenshot({ preserveRecovery: true }); + policy.observeStream("induced", 1120); + policy.endScreenshot(token, 1130); + policy.observeStream("queued", 1250); + expect(policy.canUseStream(1500)).toBe(false); + policy.observeStream("two", 1500); + policy.observeStream("three", 1750); + expect(policy.canUseStream(1750)).toBe(true); +}); diff --git a/paseo-shared-browser/server/capture-transport-policy.ts b/paseo-shared-browser/server/capture-transport-policy.ts new file mode 100644 index 00000000..8e1b4e9d --- /dev/null +++ b/paseo-shared-browser/server/capture-transport-policy.ts @@ -0,0 +1,153 @@ +/** + * Owns transport hysteresis and bounded screenshot reuse, independently of JPEG + * decoding. A quiet CDP stream can emit one frame because captureScreenshot ran; + * those frames are still acknowledged/cached, but cannot establish recovery. + * The 250ms post-capture exclusion is conservative because CDP provides no + * causal request ID on screencast events. Cached receipt age is monotonic. + */ +import type { RuntimeFrame } from "./agent-browser-runtime"; + +export const CAPTURE_MAX_AGE_MS = 1_000; +const FALLBACK_MIN_DWELL_MS = 1_500; +const RECOVERY_MIN_EVENTS = 3; +const RECOVERY_MIN_SPAN_MS = 500; +const RECOVERY_MAX_GAP_MS = 500; +const SCREENSHOT_EVENT_GUARD_MS = 250; +const RECOVERY_ID_MEMORY = 32; + +type ScreenshotCache = { + frame: RuntimeFrame; + receivedAt: number; + quality: number; + maxBytes: number; +}; + +/** Create one policy per runtime. Invalidation revokes outstanding screenshot completion tokens. */ +export function createCaptureTransportPolicy() { + let generation = 0; + let fallbackSince: number | null = null; + let screenshot: ScreenshotCache | null = null; + let screenshotsInFlight = 0; + let ignoreStreamUntil = 0; + let recoveryFirstAt: number | null = null; + let recoveryLastAt: number | null = null; + let recoveryCount = 0; + const recoveryIds = new Set(); + + function clearRecovery(): void { + recoveryFirstAt = null; + recoveryLastAt = null; + recoveryCount = 0; + recoveryIds.clear(); + } + + /** + * Clear cached screenshots/evidence and revoke old async completions. Keep an + * existing fallback dwell: input/session changes require fresh recovery evidence. + */ + function invalidate(): void { + generation += 1; + screenshot = null; + screenshotsInFlight = 0; + ignoreStreamUntil = 0; + clearRecovery(); + } + + /** Enter fallback once; recapture/cache expiry must not extend the minimum dwell. */ + function enterFallback(now: number): void { + if (fallbackSince !== null) return; + fallbackSince = now; + clearRecovery(); + } + + /** Feed only valid current-session CDP events, with distinct provider timestamps when available. */ + function observeStream(identity: string, now: number): void { + if (fallbackSince === null) return; + if (screenshotsInFlight > 0 || now <= ignoreStreamUntil) return; + if (recoveryLastAt !== null && now - recoveryLastAt > RECOVERY_MAX_GAP_MS) { + clearRecovery(); + } + if (recoveryIds.has(identity)) return; + if (recoveryFirstAt === null) recoveryFirstAt = now; + recoveryLastAt = now; + recoveryCount = Math.min(RECOVERY_MIN_EVENTS, recoveryCount + 1); + recoveryIds.add(identity); + if (recoveryIds.size > RECOVERY_ID_MEMORY) { + const oldest = recoveryIds.values().next().value; + if (oldest !== undefined) recoveryIds.delete(oldest); + } + } + + /** Require sustained independent activity before allowing a fresh stream to replace fallback. */ + function canUseStream(now: number): boolean { + if (fallbackSince === null) return true; + if ( + recoveryFirstAt === null || + recoveryLastAt === null || + now - fallbackSince < FALLBACK_MIN_DWELL_MS || + recoveryCount < RECOVERY_MIN_EVENTS || + recoveryLastAt - recoveryFirstAt < RECOVERY_MIN_SPAN_MS || + now - recoveryLastAt > RECOVERY_MAX_GAP_MS + ) + return false; + fallbackSince = null; + clearRecovery(); + return true; + } + + /** Reuse only identical quality/budget requests; larger budgets deserve a new quality decision. */ + function readScreenshot(now: number, quality: number, maxBytes: number): RuntimeFrame | null { + if ( + !screenshot || + now - screenshot.receivedAt > CAPTURE_MAX_AGE_MS || + screenshot.quality !== quality || + screenshot.maxBytes !== maxBytes + ) + return null; + return screenshot.frame; + } + + /** + * Begin capture and return its mutation/session epoch. A quality-only viewer + * capture keeps independent stream recovery evidence, while still excluding + * capture-induced events until the same causal guard expires. + */ + function beginScreenshot(options: { preserveRecovery?: boolean } = {}): number { + screenshotsInFlight += 1; + if (!options.preserveRecovery) clearRecovery(); + return generation; + } + + /** End causal exclusion only for this still-current capture generation. */ + function endScreenshot(token: number, now: number): void { + if (token !== generation) return; + screenshotsInFlight = Math.max(0, screenshotsInFlight - 1); + ignoreStreamUntil = Math.max(ignoreStreamUntil, now + SCREENSHOT_EVENT_GUARD_MS); + } + + /** Never install a pre-input/session screenshot after its asynchronous RPC resolves. */ + function rememberScreenshot( + token: number, + frame: RuntimeFrame, + now: number, + quality: number, + maxBytes: number, + ): boolean { + if (token !== generation) return false; + screenshot = { frame, receivedAt: now, quality, maxBytes }; + return true; + } + + return { + invalidate, + enterFallback, + observeStream, + canUseStream, + readScreenshot, + beginScreenshot, + endScreenshot, + rememberScreenshot, + currentGeneration: () => generation, + isCurrent: (token: number) => token === generation, + }; +} diff --git a/paseo-shared-browser/server/capture-transport-runtime.test.ts b/paseo-shared-browser/server/capture-transport-runtime.test.ts new file mode 100644 index 00000000..1a747f07 --- /dev/null +++ b/paseo-shared-browser/server/capture-transport-runtime.test.ts @@ -0,0 +1,205 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { AgentBrowserRuntime } from "./agent-browser-runtime"; + +/** Header fixture exercises actual bounded JPEG validation without launching Chromium. */ +function jpeg(tag = 1): string { + return Buffer.from([ + 255, + 216, + 255, + 224, + 0, + 3, + tag, + 255, + 192, + 0, + 11, + 8, + 3, + 32, + 5, + 0, + 1, + 1, + 17, + 0, + 255, + 217, + ]).toString("base64"); +} +let clock = 10_000; +beforeEach(() => { + clock = 10_000; + vi.spyOn(performance, "now").mockImplementation(() => clock); +}); +afterEach(() => vi.restoreAllMocks()); + +function fixture() { + const runtime = new AgentBrowserRuntime({ + binaryPath: "/tmp/unlaunched-browser", + executablePath: "/tmp/unlaunched-chromium", + profilePath: "/tmp/uncreated-profile", + ipcDirectory: "/tmp/uncreated-ipc", + session: "test", + }); + const calls: string[] = []; + let capture: (params: Record) => Promise<{ data: string }> = async () => ({ data: jpeg() }); + const page = { + send: async (method: string, params: Record = {}) => { + calls.push(method); + if (method === "Page.captureScreenshot") return capture(params); + if (method === "Page.getLayoutMetrics") return { cssVisualViewport: { pageX: 0, pageY: 0 } }; + return {}; + }, + }; + const control = runtime as unknown as { + requirePage: () => Promise; + page: typeof page; + invalidateScreencastFrame: () => void; + onScreencastFrame: (source: typeof page, event: object) => void; + }; + Object.assign(runtime, { + page, + emulationAppliedPage: page, + screencastActive: true, + viewport: { width: 1280, height: 800, deviceScaleFactor: 1, mobile: false, touch: false }, + }); + control.requirePage = async () => control.page; + const emit = (timestamp: number, source = page) => + control.onScreencastFrame(source, { + data: jpeg(2), + metadata: { deviceWidth: 1280, deviceHeight: 800, timestamp }, + sessionId: 17, + }); + return { + runtime, + control, + page, + calls, + emit, + setCapture: (next: typeof capture) => { + capture = next; + }, + }; +} + +describe("runtime capture hysteresis", () => { + it("captures only the visible viewport without Chromium's touch-resetting beyond-viewport mode", async () => { + const { runtime, setCapture } = fixture(); + setCapture(async params => { + expect(params.captureBeyondViewport).toBe(false); + expect(params.clip).toEqual({ x: 0, y: 0, width: 1280, height: 800, scale: 1 }); + return { data: jpeg() }; + }); + expect((await runtime.frame(100, 95, 0)).transport).toBe("screenshot"); + }); + + it("does not bounce to CDP after a screenshot-induced event; reuses only fresh exact-request pixels", async () => { + const { runtime, calls, emit, setCapture } = fixture(); + setCapture(async () => { + emit(clock); + return { data: jpeg() }; + }); + const first = await runtime.frame(100, 95, 0); + expect(first.transport).toBe("screenshot"); + clock += 250; + expect(await runtime.frame(100, 95, 0)).toBe(first); + expect(calls.filter((method) => method === "Page.captureScreenshot")).toHaveLength(1); + clock += 751; + expect(await runtime.frame(100, 95, 0)).not.toBe(first); + expect(calls.filter((method) => method === "Page.captureScreenshot")).toHaveLength(2); + await runtime.frame(100, 90, 0); + await runtime.frame(101, 90, 0); + expect(calls.filter((method) => method === "Page.captureScreenshot")).toHaveLength(4); + }); + + it("only sustained valid current-session events recover CDP after dwell", async () => { + const { runtime, page, emit } = fixture(); + await runtime.frame(100, 95, 0); + clock += 1100; + emit(1); + clock += 250; + emit(2); + clock += 250; + emit(3, { ...page }); + expect((await runtime.frame(100, 95, 0)).transport).toBe("screenshot"); + clock += 300; + emit(3); + clock += 250; + emit(4); + clock += 250; + emit(5); + expect((await runtime.frame(100, 95, 0)).transport).toBe("cdp-screencast"); + }); + + it("late screenshot completion after input cannot return or restore old cached pixels", async () => { + const { runtime, control, setCapture } = fixture(); + const shot = Promise.withResolvers<{ data: string }>(); + const started = Promise.withResolvers(); + setCapture(async () => { + started.resolve(); + return shot.promise; + }); + const pending = runtime.frame(100, 95, 0); + await started.promise; + await runtime.insertText("owned-fixture"); + shot.resolve({ data: jpeg() }); + await expect(pending).rejects.toThrow("capture was invalidated"); + setCapture(async () => ({ data: jpeg(3) })); + const fresh = await runtime.frame(100, 95, 0); + expect(fresh.dataBase64).toBe(jpeg(3)); + control.invalidateScreencastFrame(); + setCapture(async () => ({ data: jpeg(4) })); + expect((await runtime.frame(100, 95, 0)).dataBase64).toBe(jpeg(4)); + }); + + it("late screenshot completion from a replaced page cannot authorize a frame", async () => { + const { runtime, control, page, setCapture } = fixture(); + const shot = Promise.withResolvers<{ data: string }>(); + const started = Promise.withResolvers(); + setCapture(async () => { + started.resolve(); + return shot.promise; + }); + const pending = runtime.frame(100, 95, 0); + await started.promise; + control.page = { ...page }; + control.invalidateScreencastFrame(); + shot.resolve({ data: jpeg() }); + await expect(pending).rejects.toThrow("capture was invalidated"); + }); +}); + + +it("honors low/medium quality independently while high viewers keep the shared stream", async () => { + const { runtime, calls, emit, setCapture } = fixture(); + const qualities: unknown[] = []; + setCapture(async params => { + qualities.push(params.quality); + return { data: jpeg() }; + }); + emit(1); + expect((await runtime.frame(100, 95, 0)).transport).toBe("cdp-screencast"); + const low = await runtime.frame(100, 70, 0); + expect(low.transport).toBe("screenshot"); + expect(await runtime.frame(100, 70, 0)).toBe(low); + expect((await runtime.frame(100, 95, 0)).transport).toBe("cdp-screencast"); + expect((await runtime.frame(100, 90, 0)).transport).toBe("screenshot"); + expect((await runtime.frame(100, 95, 0)).transport).toBe("cdp-screencast"); + expect(qualities).toEqual([70, 90]); + expect(calls).not.toContain("Page.startScreencast"); + expect(calls).not.toContain("Page.stopScreencast"); +}); + +it("reduces quality only to satisfy the byte cap and never substitutes a cached high-quality frame", async () => { + const { runtime, setCapture } = fixture(); + const qualities: unknown[] = []; + setCapture(async params => { + qualities.push(params.quality); + return { data: params.quality === 70 ? jpeg() + "AAAA".repeat(50) : jpeg() }; + }); + await runtime.frame(100, 95, 0); + expect((await runtime.frame(100, 70, 0)).transport).toBe("screenshot"); + expect(qualities).toEqual([95, 70, 65]); +}); diff --git a/paseo-shared-browser/server/emulation-restoration.test.ts b/paseo-shared-browser/server/emulation-restoration.test.ts new file mode 100644 index 00000000..3fa2551c --- /dev/null +++ b/paseo-shared-browser/server/emulation-restoration.test.ts @@ -0,0 +1,258 @@ +import { EventEmitter } from "node:events"; +import { describe, expect, it } from "vitest"; +import { AgentBrowserRuntime, type DeviceEmulation } from "./agent-browser-runtime"; +import { CdpSession, type CdpConnection } from "./cdp"; + +/** Protocol fixture exercises real attachment/configuration methods without a browser process. */ +class ConnectionFixture extends EventEmitter { + isOpen = true; + nextSession = 0; + calls: { method: string; params: Record; sessionId: string | undefined }[] = []; + intercept?: (method: string, sessionId?: string) => Promise; + + async send( + method: string, + params: Record = {}, + options: { sessionId?: string } = {}, + ): Promise { + this.calls.push({ method, params, sessionId: options.sessionId }); + await this.intercept?.(method, options.sessionId); + if (method === "Target.getTargets") { + return { + targetInfos: ["a", "b"].map((targetId) => ({ + targetId, + type: "page", + url: "about:blank", + title: targetId, + })), + } as T; + } + if (method === "Target.attachToTarget") return { sessionId: `new-${++this.nextSession}` } as T; + if (method === "Page.getNavigationHistory") return { currentIndex: 0, entries: [] } as T; + return {} as T; + } + + close(): void { + this.isOpen = false; + } +} + +interface RuntimeControl { + connection: CdpConnection; + page: CdpSession | null; + targetId: string | null; + viewport: DeviceEmulation; + emulationAppliedPage: CdpSession | null; + screencastActive: boolean; + screencastFrame: unknown; + attachmentGeneration: number; + assertVersion(): Promise; + connectCdp(targetId?: string): Promise; + requirePage(): Promise; + reattachPageForScreencast(previous: CdpSession): Promise; + onScreencastFrame(page: CdpSession, event: unknown): void; +} + +const pixel: DeviceEmulation = { + width: 412, + height: 839, + deviceScaleFactor: 2.625, + captureScale: 2, + mobile: true, + touch: true, + screenWidth: 430, + screenHeight: 900, + userAgent: "fixture-mobile-agent", + platform: "Android", +}; + +function fixture() { + const runtime = new AgentBrowserRuntime({ + binaryPath: "/tmp/unlaunched-browser", + executablePath: "/tmp/unlaunched-chromium", + profilePath: "/tmp/uncreated-profile", + ipcDirectory: "/tmp/uncreated-ipc", + session: "emulation-test", + }); + const connection = new ConnectionFixture(); + const typedConnection = connection as unknown as CdpConnection; + const previous = new CdpSession(typedConnection, "previous", "a"); + const control = runtime as unknown as RuntimeControl; + Object.assign(control, { + connection: typedConnection, + page: previous, + targetId: "a", + viewport: { ...pixel }, + emulationAppliedPage: previous, + }); + return { runtime, connection, previous, control }; +} + +function assertDeviceRestored(connection: ConnectionFixture, sessionId: string) { + const calls = connection.calls.filter((call) => call.sessionId === sessionId); + expect( + calls.find((call) => call.method === "Emulation.setDeviceMetricsOverride")?.params, + ).toEqual({ + width: 412, + height: 839, + deviceScaleFactor: 2.625, + mobile: true, + screenWidth: 430, + screenHeight: 900, + screenOrientation: { type: "portraitPrimary", angle: 0 }, + }); + expect( + calls.find((call) => call.method === "Emulation.setTouchEmulationEnabled")?.params, + ).toEqual({ enabled: true, maxTouchPoints: 5 }); + expect(calls.find((call) => call.method === "Emulation.setUserAgentOverride")?.params).toEqual({ + userAgent: "fixture-mobile-agent", + platform: "Android", + }); +} + +describe("session-owned device emulation restoration", () => { + it("restores full retained settings before publishing a newly selected target", async () => { + const { runtime, connection, control } = fixture(); + const publications: (CdpSession | null)[] = []; + connection.intercept = async (method) => { + if (method.startsWith("Emulation.")) publications.push(control.page); + }; + await runtime.selectTarget("b"); + expect(publications).toEqual([null, null, null]); + expect(control.page?.targetId).toBe("b"); + expect(control.emulationAppliedPage).toBe(control.page); + expect(control.viewport.captureScale).toBe(2); + assertDeviceRestored(connection, control.page!.sessionId); + }); + + it("reconnect restores the device on the replacement transport rather than the old connection", async () => { + const { runtime, connection, control } = fixture(); + const replacement = new ConnectionFixture(); + control.assertVersion = async () => {}; + control.connectCdp = async () => { + control.connection = replacement as unknown as CdpConnection; + await runtime.selectTarget("a"); + }; + await runtime.reconnect(); + expect(connection.isOpen).toBe(false); + expect(control.page?.connection).toBe(control.connection); + assertDeviceRestored(replacement, control.page!.sessionId); + }); + + it("screencast retry detaches the old controller before applying replacement metrics", async () => { + const { connection, control, previous } = fixture(); + const restored = await control.reattachPageForScreencast(previous); + const detach = connection.calls.findIndex((call) => call.method === "Target.detachFromTarget"); + const metrics = connection.calls.findIndex( + (call) => call.method === "Emulation.setDeviceMetricsOverride", + ); + expect(detach).toBeGreaterThanOrEqual(0); + expect(metrics).toBeGreaterThan(detach); + expect(control.page).toBe(restored); + assertDeviceRestored(connection, restored.sessionId); + }); + + it("refuses frame admission after partial restoration fails", async () => { + const { runtime, connection, control } = fixture(); + connection.intercept = async (method) => { + if (method === "Emulation.setTouchEmulationEnabled") + throw new Error("touch restore rejected"); + }; + await expect(runtime.selectTarget("b")).rejects.toThrow("touch restore rejected"); + expect(control.page).toBeNull(); + expect(control.emulationAppliedPage).toBeNull(); + expect( + connection.calls.some( + (call) => call.method === "Target.detachFromTarget" && call.params.sessionId === "new-1", + ), + ).toBe(true); + control.assertVersion = async () => { + throw new Error("transport unavailable"); + }; + await expect(runtime.frame(1024, 95, 0)).rejects.toThrow("transport unavailable"); + expect( + connection.calls.some( + (call) => + call.method === "Page.captureScreenshot" || call.method === "Page.startScreencast", + ), + ).toBe(false); + }); + + it("rejects a disconnect while restoring rather than publishing the detached session", async () => { + const { runtime, connection, control } = fixture(); + connection.intercept = async (method) => { + if (method === "Emulation.setUserAgentOverride") connection.close(); + }; + await expect(runtime.selectTarget("b")).rejects.toThrow("attachment changed"); + expect(control.page).toBeNull(); + expect(control.emulationAppliedPage).toBeNull(); + }); + + it("repairs the winning attachment after a stale restoring controller is detached", async () => { + const { runtime, connection, control } = fixture(); + const gate = Promise.withResolvers(); + const reached = Promise.withResolvers(); + connection.intercept = async (method, sessionId) => { + if (method === "Emulation.setUserAgentOverride" && sessionId === "new-1") { + reached.resolve(); + await gate.promise; + } + }; + const older = runtime.selectTarget("a"); + await reached.promise; + await runtime.selectTarget("b"); + const winner = control.page; + expect(control.emulationAppliedPage).toBe(winner); + gate.resolve(); + await expect(older).rejects.toThrow("attachment changed"); + expect(control.page).toBe(winner); + expect(control.emulationAppliedPage).toBeNull(); + control.screencastActive = true; + control.onScreencastFrame(winner!, { data: "ignored", metadata: {}, sessionId: 4 }); + expect(control.screencastFrame).toBeNull(); + expect(await control.requirePage()).toBe(winner); + expect(control.emulationAppliedPage).toBe(winner); + assertDeviceRestored(connection, winner!.sessionId); + }); + + it("refuses retry publication if detaching the prior emulation controller fails", async () => { + const { connection, control, previous } = fixture(); + connection.intercept = async (method) => { + if (method === "Target.detachFromTarget") throw new Error("detach outcome unknown"); + }; + await expect(control.reattachPageForScreencast(previous)).rejects.toThrow( + "detach outcome unknown", + ); + expect(control.page).toBeNull(); + expect(control.emulationAppliedPage).toBeNull(); + expect( + connection.calls.some( + (call) => call.method.startsWith("Emulation.") || call.method === "Page.startScreencast", + ), + ).toBe(false); + }); + + it("invalidates winner readiness after detaching a superseded retry controller", async () => { + const { runtime, connection, control, previous } = fixture(); + const reached = Promise.withResolvers(); + const gate = Promise.withResolvers(); + let first = true; + connection.intercept = async (method) => { + if (method === "Target.attachToTarget" && first) { + first = false; + reached.resolve(); + await gate.promise; + } + }; + const retry = control.reattachPageForScreencast(previous); + await reached.promise; + await runtime.selectTarget("b"); + const winner = control.page; + gate.resolve(); + expect(await retry).toBe(winner); + expect(control.page).toBe(winner); + expect(control.emulationAppliedPage).toBeNull(); + expect(await control.requirePage()).toBe(winner); + expect(control.emulationAppliedPage).toBe(winner); + }); +}); diff --git a/paseo-shared-browser/server/input-generation.test.ts b/paseo-shared-browser/server/input-generation.test.ts new file mode 100644 index 00000000..de699ab4 --- /dev/null +++ b/paseo-shared-browser/server/input-generation.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { formatRuntimeInputGeneration, sameRuntimeInputAttachment } from "./input-generation"; + +describe("private native input generation codec", () => { + it("admits document changes only within the same exact native attachment", () => { + expect(sameRuntimeInputAttachment(formatRuntimeInputGeneration(3,4), formatRuntimeInputGeneration(3,5))).toBe(true); + expect(sameRuntimeInputAttachment("3:4", "4:4")).toBe(false); + }); + + it("refuses missing, opaque, truncated and unbounded generation formats", () => { + for (const invalid of [null,"opaque","3","3:","03:4","3:-1","3:9007199254740992","3:4:5"]) { + expect(sameRuntimeInputAttachment("3:4",invalid)).toBe(false); + expect(sameRuntimeInputAttachment(invalid,"3:4")).toBe(false); + } + expect(()=>formatRuntimeInputGeneration(-1,0)).toThrow("Invalid"); + }); +}); diff --git a/paseo-shared-browser/server/input-generation.ts b/paseo-shared-browser/server/input-generation.ts new file mode 100644 index 00000000..d640adac --- /dev/null +++ b/paseo-shared-browser/server/input-generation.ts @@ -0,0 +1,28 @@ +/** + * Private runtime metadata codec for attachment and document counters. This is + * produced by agent-browser-runtime and consumed only by browser-policy. Unknown + * older formats cannot prove same-attachment completion after input publication. + */ + +/** Serialize native counters without treating document navigation as target replacement. */ +export function formatRuntimeInputGeneration(attachment: number, document: number): string { + if (!Number.isSafeInteger(attachment) || attachment < 0 || !Number.isSafeInteger(document) || document < 0) { + throw new Error("Invalid native input generation"); + } + return `${attachment}:${document}`; +} + +/** Compare only independently serialized attachment identity; malformed or absent metadata refuses completion. */ +export function sameRuntimeInputAttachment(original: string | null, current: string | null): boolean { + const before = readNativeCounters(original); + const after = readNativeCounters(current); + return before !== null && after !== null && before.attachment === after.attachment; +} + +/** Require the complete native codec, including a bounded valid document counter. */ +function readNativeCounters(value: string | null): { attachment: number; document: number } | null { + if (!value || value.length > 33 || !/^(0|[1-9]\d*):(0|[1-9]\d*)$/.test(value)) return null; + const [attachment, document] = value.split(":").map(Number); + if (!Number.isSafeInteger(attachment) || !Number.isSafeInteger(document)) return null; + return { attachment: attachment!, document: document! }; +} diff --git a/paseo-shared-browser/server/jpeg-frame.ts b/paseo-shared-browser/server/jpeg-frame.ts new file mode 100644 index 00000000..cf5cc856 --- /dev/null +++ b/paseo-shared-browser/server/jpeg-frame.ts @@ -0,0 +1,27 @@ +/** Read actual JPEG dimensions so CDP viewport metadata cannot authorize input on cropped pixels. */ +export function readJpegFrameDimensions(dataBase64: string): { width: number; height: number } | null { + const bytes = Buffer.from(dataBase64, "base64"); + if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8) return null; + + let offset = 2; + while (offset < bytes.length) { + if (bytes[offset++] !== 0xff) return null; + while (bytes[offset] === 0xff) offset += 1; + const marker = bytes[offset++]; + if (marker === undefined || marker === 0xda || marker === 0xd9) return null; + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd8)) continue; + if (offset + 2 > bytes.length) return null; + + const length = bytes.readUInt16BE(offset); + if (length < 2 || offset + length > bytes.length) return null; + // All JPEG start-of-frame markers except the Huffman/arithmetic table markers. + if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) { + if (length < 8) return null; + const height = bytes.readUInt16BE(offset + 3); + const width = bytes.readUInt16BE(offset + 5); + return width > 0 && height > 0 ? { width, height } : null; + } + offset += length; + } + return null; +} diff --git a/paseo-shared-browser/server/keyboard-input.test.ts b/paseo-shared-browser/server/keyboard-input.test.ts new file mode 100644 index 00000000..a103c88c --- /dev/null +++ b/paseo-shared-browser/server/keyboard-input.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { browserGestureEventSchema, browserGestureKeySchema } from "../shared/browser"; +import { heldKeyModifiers, nativeKeyEvent } from "./keyboard-input"; + +const key = (key: string, code: string, modifiers = 0) => + browserGestureKeySchema.parse({ kind: "key", type: "down", key, code, modifiers }); + +describe("native keyboard envelopes", () => { + it("preserves Unicode and space while Enter and Tab retain native meanings", () => { + expect(nativeKeyEvent(key("😀", "KeyA"))).toMatchObject({ type: "keyDown", text: "😀" }); + expect(nativeKeyEvent(key(" ", "Space"))).toMatchObject({ text: " ", windowsVirtualKeyCode: 32 }); + expect(nativeKeyEvent(key("Enter", "Enter"))).toMatchObject({ text: "\r", windowsVirtualKeyCode: 13 }); + expect(nativeKeyEvent(key("Tab", "Tab"))).toMatchObject({ type: "rawKeyDown", windowsVirtualKeyCode: 9 }); + expect(nativeKeyEvent(key("Tab", "Tab"))).not.toHaveProperty("text"); + expect(nativeKeyEvent(key("ArrowLeft", "ArrowLeft"))).toMatchObject({ windowsVirtualKeyCode: 37 }); + expect(nativeKeyEvent(key("A", "KeyA", 8))).toMatchObject({ text: "A", modifiers: 8 }); + }); + + it("forwards shortcuts and repeat without synthesizing printable characters", () => { + expect(nativeKeyEvent(key("a", "KeyA", 2))).toMatchObject({ type: "rawKeyDown", windowsVirtualKeyCode: 65, modifiers: 2 }); + expect(nativeKeyEvent(key("a", "KeyA", 2))).not.toHaveProperty("text"); + expect(nativeKeyEvent({ ...key("a", "KeyA"), repeat: true })).toHaveProperty("autoRepeat", true); + expect(heldKeyModifiers([{ key: "Control" }, { key: "Shift" }])).toBe(10); + }); + + it("rejects shortcut text, keyup text/repeat and oversized committed text", () => { + expect(browserGestureKeySchema.safeParse({ ...key("a", "KeyA", 2), text: "a" }).success).toBe(false); + expect(browserGestureKeySchema.safeParse({ ...key("a", "KeyA"), type: "up", text: "a" }).success).toBe(false); + expect(browserGestureKeySchema.safeParse({ ...key("a", "KeyA"), type: "up", repeat: true }).success).toBe(false); + expect(browserGestureEventSchema.safeParse({ kind: "text", text: "漢字\n😀" }).success).toBe(true); + expect(browserGestureEventSchema.safeParse({ kind: "text", text: "x".repeat(16001) }).success).toBe(false); + }); +}); diff --git a/paseo-shared-browser/server/keyboard-input.ts b/paseo-shared-browser/server/keyboard-input.ts new file mode 100644 index 00000000..1455ebc7 --- /dev/null +++ b/paseo-shared-browser/server/keyboard-input.ts @@ -0,0 +1,68 @@ +/** + * Converts bounded physical keyboard envelopes to native CDP events. No page + * script or browser editing command is exposed. CDP uses Windows virtual key + * values on all platforms; unsupported physical codes retain DOM key/code only. + */ +import type { BrowserGestureKeyEvent } from "../shared/browser"; + +const SPECIAL_CODES: Record = { + Backspace: 8, Tab: 9, Enter: 13, NumpadEnter: 13, ShiftLeft: 16, ShiftRight: 16, + ControlLeft: 17, ControlRight: 17, AltLeft: 18, AltRight: 18, Pause: 19, + CapsLock: 20, Escape: 27, Space: 32, PageUp: 33, PageDown: 34, End: 35, + Home: 36, ArrowLeft: 37, ArrowUp: 38, ArrowRight: 39, ArrowDown: 40, + Insert: 45, Delete: 46, MetaLeft: 91, MetaRight: 92, ContextMenu: 93, + NumpadMultiply: 106, NumpadAdd: 107, NumpadSubtract: 109, NumpadDecimal: 110, + NumpadDivide: 111, NumLock: 144, ScrollLock: 145, Semicolon: 186, Equal: 187, + Comma: 188, Minus: 189, Period: 190, Slash: 191, Backquote: 192, + BracketLeft: 219, Backslash: 220, BracketRight: 221, Quote: 222, +}; + +/** Physical code chooses the accelerator code independently of keyboard layout text. */ +export function virtualKeyCode(code: string): number | undefined { + if (/^Key[A-Z]$/.test(code)) return code.charCodeAt(3); + if (/^Digit[0-9]$/.test(code)) return code.charCodeAt(5); + if (/^Numpad[0-9]$/.test(code)) return 96 + Number(code.slice(6)); + const functionKey = /^F([1-9]|1[0-9]|2[0-4])$/.exec(code); + if (functionKey) return 111 + Number(functionKey[1]); + return SPECIAL_CODES[code]; +} + +/** Native text is separate from shortcut keys; Tab stays a native focus command. */ +function nativeKeyText(event: BrowserGestureKeyEvent): string | undefined { + if (event.type !== "down" || (event.modifiers & 7) !== 0) return undefined; + if (event.text !== undefined) return event.text; + if (event.key === "Enter") return "\r"; + if (Array.from(event.key).length === 1) return event.key; + return undefined; +} + +/** Preserve real edge/repeat/text; never synthesize text for modifier shortcuts. */ +export function nativeKeyEvent(event: BrowserGestureKeyEvent): Record { + const windowsVirtualKeyCode = virtualKeyCode(event.code); + const text = nativeKeyText(event); + const modifier = /^(Shift|Control|Alt|Meta)(Left|Right)$/.test(event.code); + const location = modifier ? (event.code.endsWith("Left") ? 1 : 2) : undefined; + return { + type: event.type === "up" ? "keyUp" : text ? "keyDown" : "rawKeyDown", + key: event.key, + code: event.code, + modifiers: event.modifiers, + autoRepeat: event.repeat, + ...(text !== undefined ? { text } : {}), + ...(windowsVirtualKeyCode !== undefined ? { windowsVirtualKeyCode } : {}), + ...(event.code.startsWith("Numpad") ? { isKeypad: true } : {}), + ...(location !== undefined ? { location } : {}), + }; +} + +/** Mouse chords use keys already published on this exact input attachment. */ +export function heldKeyModifiers(keys: Iterable<{ key: string }>): number { + let mask = 0; + for (const { key } of keys) { + if (key === "Alt" || key === "AltGraph") mask |= 1; + if (key === "Control" || key === "AltGraph") mask |= 2; + if (key === "Meta") mask |= 4; + if (key === "Shift") mask |= 8; + } + return mask; +} diff --git a/paseo-shared-browser/server/live-input-policy.test.ts b/paseo-shared-browser/server/live-input-policy.test.ts new file mode 100644 index 00000000..520109fc --- /dev/null +++ b/paseo-shared-browser/server/live-input-policy.test.ts @@ -0,0 +1,466 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { SessionManager, type BrowserRuntimeClient } from "./browser-policy"; +import type { BrowserGestureEvent } from "../shared/browser"; + +/** Exercise the real policy with a deterministic owned runtime; no browser or services are started. */ +async function fixture() { + let counter = 0; + let url = "https://fixture.invalid/"; + let inputGeneration = "0:0"; + let pinnedGeneration: string | null = null; + let pinnedUrl: string | null = null; + let intercept: (operation: string) => Promise = async () => {}; + const calls: { operation: string; input: unknown }[] = []; + const client: BrowserRuntimeClient = { + connect: async () => ({ epoch: 1 }), + ensureWorkspace: async (workspaceId) => ({ + workspaceId, + runtimeId: "r".repeat(32), + createdAt: 1, + }), + requestWorkspace: async (_workspace, operation, input) => { + calls.push({ operation, input }); + await intercept(operation); + if (operation === "input.begin") { + pinnedGeneration = inputGeneration; + pinnedUrl = url; + } + if ( + operation === "input.check" && + (pinnedGeneration !== inputGeneration || pinnedUrl !== url) + ) + throw new Error("Live browser input attachment changed"); + if (operation === "identity") return { userAgent: "Fixture Chromium" }; + if (operation === "state") + return { url, title: "Fixture", canGoBack: false, canGoForward: false, inputGeneration }; + if (operation === "frame") + return { + dataBase64: "eA==", + byteLength: 1, + width: 1280, + height: 800, + capturedAt: new Date().toISOString(), + }; + if (operation === "cursor") return "pointer"; + return null; + }, + archiveWorkspace: async () => {}, + disconnect: () => {}, + }; + const manager = new SessionManager({ + validateWorkspace: async () => true, + client, + issueToken: () => (++counter).toString().padStart(32, "0"), + }); + await manager.connect(); + const { viewerToken } = await manager.attach("owned-workspace", "Viewer"); + const { controlToken } = await manager.acquireControl(viewerToken); + const capture = await manager.capture(viewerToken); + const frame = capture.frame!; + const expected = { + sessionId: capture.state.sessionId, + runtimeId: capture.state.runtimeId!, + bridgeEpoch: capture.state.bridgeEpoch!, + navigationGeneration: capture.state.navigationGeneration, + viewportGeneration: capture.state.viewportGeneration, + }; + const context = { viewerToken, controlToken, expected }; + const target = { + frameId: frame.frameId, + navigationGeneration: frame.navigationGeneration, + viewportGeneration: frame.viewportGeneration, + }; + let gestureId = ""; + let sequence = 1; + return { + manager, + calls, + context, + target, + begin: async (pointerKind: "mouse" | "touch" = "mouse") => { + const reply = await manager.beginGesture({ ...context, target, pointerKind }); + if ("admission" in reply) throw new Error("Fixture expected accepted frame"); + gestureId = reply.gestureId; + sequence = reply.nextSequence; + return reply; + }, + update: async (event: BrowserGestureEvent, withTarget = false) => { + const result = await manager.updateGesture({ + ...context, + gestureId, + sequence, + event, + ...(withTarget ? { target } : {}), + }); + sequence = result.nextSequence; + return result; + }, + end: async (cancel = false, overrideSequence?: number) => + manager.endGesture({ ...context, gestureId, sequence: overrideSequence ?? sequence, cancel }), + changeUrl: (value: string) => { + url = value; + }, + reloadSameUrl: () => { + inputGeneration = "0:1"; + }, + replaceTarget: () => { inputGeneration = "1:0"; }, + intercept: (fn: typeof intercept) => { + intercept = fn; + }, + }; +} + +const point = (x = 50, y = 40) => ({ x, y, width: 640, height: 400 }); +afterEach(() => vi.useRealTimers()); + +describe("owned ordered live input", () => { + it("releases held input before a mode-only change with unchanged display dimensions", async () => { + const state = await fixture(); + try { + await state.begin(); + await state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true); + const beforeChange = state.calls.length; + const changed = await state.manager.applyDevicePreset({ + ...state.context, + presetId: "pixel-7-sharp", + preserveDisplay: true, + }); + + expect(changed.state.viewport).toEqual({ width: 1280, height: 800 }); + expect(changed.state.captureScale).toBe(1); + // The runtime's input.end owns release of every held mouse/touch/key input. + const changeCalls = state.calls.slice(beforeChange); + const released = changeCalls.findIndex(({ operation }) => operation === "input.end"); + const emulated = changeCalls.findIndex(({ operation }) => operation === "emulate"); + expect(released).toBeGreaterThan(-1); + expect(emulated).toBeGreaterThan(released); + await expect(state.update({ kind: "move", point: point(90, 60) })).rejects.toThrow(); + } finally { + await state.manager.disconnect(); + } + }); + + it("returns fresh state and closes the channel after an acknowledged mouse release navigates", async () => { + const state = await fixture(); + try { + await state.begin(); + await state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true); + state.intercept(async operation => { + if (operation === "mouse.up") state.changeUrl("https://fixture.invalid/after"); + }); + const reply = await state.update({ kind: "up", button: "left", clickCount: 1, point: point() }); + expect(reply.state.url).toBe("https://fixture.invalid/after"); + expect(reply.state.navigationGeneration).toBe(state.context.expected.navigationGeneration + 1); + expect(reply.cursor).toBeNull(); + expect(reply.completion).toBe("navigation"); + expect(state.calls.filter(call => call.operation === "mouse.up")).toHaveLength(1); + await expect(state.update({ kind: "move", point: point() })).rejects.toThrow("unavailable"); + } finally { state.manager.disconnect(); } + }); + + it("acknowledges a form-submit key before same-URL document replacement without replay", async () => { + const state = await fixture(); + try { + await state.begin(); + state.intercept(async operation => { + if (operation === "input.key") state.reloadSameUrl(); + }); + const reply = await state.update({kind:"key",type:"down",key:"Enter",code:"Enter",modifiers:0,repeat:false}); + expect(reply.completion).toBe("navigation"); + expect(reply.state.navigationGeneration).toBe(state.context.expected.navigationGeneration + 1); + expect(state.calls.filter(call => call.operation === "input.key")).toHaveLength(1); + expect(state.calls.filter(call => call.operation === "input.end")).toHaveLength(1); + await expect(state.update({kind:"key",type:"up",key:"Enter",code:"Enter",modifiers:0,repeat:false})).rejects.toThrow("unavailable"); + } finally { state.manager.disconnect(); } + }); + + it("keeps pre-publication document drift and uncertain publication failures rejected", async () => { + for (const scenario of ["before", "uncertain"] as const) { + const state = await fixture(); + try { + await state.begin(); + if (scenario === "before") state.reloadSameUrl(); + else state.intercept(async operation => { + if (operation !== "mouse.down") return; + state.changeUrl("https://fixture.invalid/after"); + throw new Error("Mutation outcome is unknown"); + }); + await expect(state.update({kind:"down",button:"left",clickCount:1,point:point()},true)).rejects.toThrow(scenario === "before" ? "attachment" : "unknown"); + expect(state.calls.filter(call => call.operation === "mouse.down")).toHaveLength(scenario === "before" ? 0 : 1); + } finally { state.manager.disconnect(); } + } + }); + + it("does not treat a bridge fence after publication as document-only completion", async () => { + const state = await fixture(); + try { + await state.begin(); + state.intercept(async operation => { + if (operation !== "mouse.down") return; + state.changeUrl("https://fixture.invalid/after"); + state.manager.setBridgeEpoch(2); + }); + await expect(state.update({kind:"down",button:"left",clickCount:1,point:point()},true)).rejects.toThrow("bridge"); + expect(state.calls.filter(call => call.operation === "mouse.down")).toHaveLength(1); + } finally { state.manager.disconnect(); } + }); + + it("refuses target replacement after a successful native input acknowledgment", async () => { + const state = await fixture(); + try { + await state.begin(); + let published = false; + state.intercept(async operation => { + if (operation === "mouse.down") published = true; + if (operation === "input.check" && published) state.replaceTarget(); + }); + await expect(state.update({kind:"down",button:"left",clickCount:1,point:point()},true)).rejects.toThrow("attachment"); + expect(state.calls.filter(call => call.operation === "mouse.down")).toHaveLength(1); + expect(state.calls.filter(call => call.operation === "input.end")).toHaveLength(1); + } finally { state.manager.disconnect(); } + }); + + it("retains a bounded decoded frame after hover, then streams every pressed drag move without requiring new frames", async () => { + const state = await fixture(); + await state.begin(); + expect((await state.update({ kind: "move", point: point() })).cursor).toBe("pointer"); + await state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true); + await state.update({ kind: "move", point: point(70) }); + await state.update({ kind: "move", point: point(90) }); + await state.update({ kind: "up", button: "left", clickCount: 1, point: point(90) }); + await state.end(); + expect( + state.calls + .filter((call) => call.operation === "mouse.move") + .map((call) => (call.input as { x: number }).x), + ).toEqual([100, 140, 180]); + expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); + state.manager.disconnect(); + }); + + it("returns known non-admission for revoked frame before channel publication or old-channel cancellation", async () => { + const state = await fixture(); + await state.begin(); + await state.update({ kind: "scroll", point: point(), deltaX: 0, deltaY: 40 }); + const before = state.calls.length; + const reply = await state.manager.beginGesture({ ...state.context, target: state.target, pointerKind: "mouse" }); + expect(reply).toHaveProperty("admission", "stale-frame"); + expect(reply).not.toHaveProperty("gestureId"); + expect(state.calls.slice(before).map(call => call.operation)).not.toContain("input.begin"); + expect(state.calls.slice(before).map(call => call.operation)).not.toContain("input.end"); + await state.update({ kind: "scroll", point: point(), deltaX: 0, deltaY: 40 }); + await expect(state.manager.beginGesture({ ...state.context, controlToken: "wrong", target: state.target, pointerKind: "mouse" })).rejects.toThrow(); + state.manager.disconnect(); + }); + + + it("does not return stale admission if its final metadata read fails or discovers replacement navigation", async () => { + for (const failure of ["runtime", "navigation"] as const) { + const state = await fixture(); + try { + await state.begin(); + await state.update({ kind: "scroll", point: point(), deltaX: 0, deltaY: 40 }); + let reads = 0; + state.intercept(async operation => { + if (operation !== "state" || ++reads !== 2) return; + if (failure === "runtime") throw new Error("Runtime read unavailable"); + state.changeUrl("https://replacement.invalid/"); + }); + const before = state.calls.length; + await expect(state.manager.beginGesture({ ...state.context, target: state.target, pointerKind: "mouse" })).rejects.toThrow(failure === "runtime" ? "Runtime read unavailable" : "navigation"); + expect(state.calls.slice(before).map(call => call.operation)).not.toContain("input.begin"); + } finally { state.manager.disconnect(); } + } + }); + + it("leaves native hover through a coordinate-free owned event while retaining frame admission", async () => { + const state = await fixture(); + await state.begin(); + await state.update({ kind: "move", point: point() }); + expect((await state.update({ kind: "leave" })).cursor).toBeNull(); + expect(state.calls.find((call) => call.operation === "mouse.leave")?.input).not.toHaveProperty( + "x", + ); + await state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true); + await expect(state.update({ kind: "leave" })).rejects.toThrow("Release held"); + expect(state.calls.filter((call) => call.operation === "mouse.leave")).toHaveLength(1); + state.manager.disconnect(); + }); + + it("new presses require a current decoded target; cancellation after a lost reply accepts the old sequence", async () => { + const state = await fixture(); + await state.begin(); + await expect( + state.update({ kind: "down", button: "left", clickCount: 1, point: point() }), + ).rejects.toThrow("current frame"); + expect(state.calls.some((call) => call.operation === "mouse.down")).toBe(false); + await state.begin(); + await state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true); + await state.end(true, 1); + expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(2); + state.manager.disconnect(); + }); + + it("maps native touch coordinates, preserves stable identifiers and accepts partial release via the active set", async () => { + const state = await fixture(); + await state.begin("touch"); + await state.update( + { + kind: "touch", + type: "start", + points: [ + { ...point(), id: 31 }, + { ...point(80), id: 99 }, + ], + }, + true, + ); + await state.update({ kind: "touch", type: "move", points: [{ ...point(60), id: 31 }] }); + await state.update({ kind: "touch", type: "end", points: [] }); + await state.end(); + const touches = state.calls + .filter((call) => call.operation === "touch") + .map((call) => call.input as { points: unknown[] }); + expect(touches[0]!.points).toEqual([ + { x: 100, y: 80, id: 31 }, + { x: 160, y: 80, id: 99 }, + ]); + expect(touches[1]!.points).toEqual([{ x: 120, y: 80, id: 31 }]); + expect(touches[2]!.points).toEqual([]); + state.manager.disconnect(); + }); + + it("accepts a staggered second finger within one held gesture, but refuses a new independent start from an old frame", async () => { + const state = await fixture(); + await state.begin("touch"); + await state.update({ kind: "touch", type: "start", points: [{ ...point(), id: 1 }] }, true); + await state.update({ + kind: "touch", + type: "start", + points: [ + { ...point(), id: 1 }, + { ...point(80), id: 2 }, + ], + }); + await state.update({ kind: "touch", type: "end", points: [] }); + await expect( + state.update({ kind: "touch", type: "start", points: [{ ...point(), id: 3 }] }, true), + ).rejects.toThrow("frame"); + expect(state.calls.filter((call) => call.operation === "touch")).toHaveLength(3); + state.manager.disconnect(); + }); + + it("rejects dropped sequence/layout and never replays an uncertain published packet", async () => { + const state = await fixture(); + const begin = await state.begin(); + await expect( + state.manager.updateGesture({ + ...state.context, + gestureId: begin.gestureId, + sequence: 2, + event: { kind: "move", point: point() }, + }), + ).rejects.toThrow("sequence"); + expect(state.calls.some((call) => call.operation === "mouse.move")).toBe(false); + await state.begin(); + await state.update({ kind: "move", point: point() }); + await expect(state.update({ kind: "move", point: { ...point(), width: 600 } })).rejects.toThrow( + "geometry", + ); + await state.begin(); + state.intercept(async (op) => { + if (op === "mouse.down") throw new Error("Unknown publication outcome"); + }); + await expect( + state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true), + ).rejects.toThrow("Unknown publication"); + expect(state.calls.filter((call) => call.operation === "mouse.down")).toHaveLength(1); + expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(3); + state.manager.disconnect(); + }); + + it("same-URL reload, navigation and bridge epoch changes refuse and release the old channel", async () => { + for (const change of ["reload", "url", "bridge"] as const) { + const state = await fixture(); + await state.begin(); + if (change === "reload") state.reloadSameUrl(); + if (change === "url") state.changeUrl("https://fixture.invalid/replacement"); + if (change === "bridge") state.manager.setBridgeEpoch(2); + await expect(state.update({ kind: "move", point: point() })).rejects.toThrow(); + expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); + expect(state.calls.some((call) => call.operation === "mouse.move")).toBe(false); + state.manager.disconnect(); + } + }); + + it("closes the old channel without denying an acknowledged navigation-causing press", async () => { + const state = await fixture(); + await state.begin(); + state.intercept(async (op) => { + if (op === "mouse.down") state.reloadSameUrl(); + }); + const reply = await state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true); + expect(reply.state.navigationGeneration).toBe(state.context.expected.navigationGeneration + 1); + await expect(state.update({ kind: "move", point: point() })).rejects.toThrow("unavailable"); + expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); + state.manager.disconnect(); + }); + + it("idle timeout fires without another request and takeover/detach release immediately", async () => { + vi.useFakeTimers(); + const timed = await fixture(); + await timed.begin(); + await timed.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true); + await vi.advanceTimersByTimeAsync(5_001); + expect(timed.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); + timed.manager.disconnect(); + for (const operation of ["takeover", "detach", "release"] as const) { + const state = await fixture(); + await state.begin(); + if (operation === "takeover") { + const other = await state.manager.attach("owned-workspace", "Other"); + await state.manager.acquireControl(other.viewerToken, true); + } else if (operation === "detach") await state.manager.detach(state.context.viewerToken); + else + await state.manager.releaseControl(state.context.viewerToken, state.context.controlToken); + expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); + state.manager.disconnect(); + } + }); + + it("cursor metadata failure is optional while input/context failures remain visible", async () => { + const state = await fixture(); + await state.begin(); + state.intercept(async (op) => { + if (op === "cursor") throw new Error("Unsupported hit test"); + }); + expect((await state.update({ kind: "move", point: point() })).cursor).toBeNull(); + await state.end(); + state.manager.disconnect(); + }); +}); + +it("streams keyboard edges and committed text without a fresh frame per keystroke", async () => { + const state = await fixture(); + await state.begin(); + const key = { kind: "key", type: "down", key: "a", code: "KeyA", modifiers: 0, repeat: false } as const; + await state.update(key); + await state.update({ ...key, repeat: true }); + await state.update({ ...key, type: "up" }); + await state.update({ kind: "text", text: "漢字\n😀" }); + await state.end(); + expect(state.calls.filter((call) => call.operation === "input.key")).toHaveLength(3); + expect(state.calls.filter((call) => call.operation === "input.text")).toEqual([{ operation: "input.text", input: expect.objectContaining({ text: "漢字\n😀" }) }]); +}); + +it("cancels a held gesture on device mode changes without reloading or inventing navigation", async () => { + const state = await fixture(); + await state.begin(); + await state.update({ kind: "key", type: "down", key: "Control", code: "ControlLeft", modifiers: 2, repeat: false }); + const reply = await state.manager.applyDevicePreset({ ...state.context, presetId: "pixel-7" }); + expect(reply.state.navigationGeneration).toBe(state.context.expected.navigationGeneration); + expect(reply.state.viewportGeneration).toBe(state.context.expected.viewportGeneration + 1); + expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); + expect(state.calls.filter((call) => call.operation === "reload" || call.operation === "navigate")).toEqual([]); + expect(state.calls.filter((call) => call.operation === "emulate").at(-1)?.input).toMatchObject({ mobile: true, touch: true }); +}); diff --git a/paseo-shared-browser/server/live-input-runtime.test.ts b/paseo-shared-browser/server/live-input-runtime.test.ts new file mode 100644 index 00000000..8fdb2503 --- /dev/null +++ b/paseo-shared-browser/server/live-input-runtime.test.ts @@ -0,0 +1,365 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { AgentBrowserRuntime } from "./agent-browser-runtime"; +import { createRuntimeOwner } from "./runtime-owner"; + +/** Exercise native publication/cleanup without starting Chromium or touching any profile. */ +function fixture() { + const runtime = new AgentBrowserRuntime({ + binaryPath: "/tmp/unlaunched-browser", + executablePath: "/tmp/unlaunched-chromium", + profilePath: "/tmp/uncreated-profile", + ipcDirectory: "/tmp/uncreated-ipc", + session: "test", + }); + const calls: { method: string; params: Record }[] = []; + let intercept: (method: string) => Promise = async () => {}; + const page = { + send: async (method: string, params: Record = {}) => { + calls.push({ method, params }); + await intercept(method); + return method === "Runtime.evaluate" ? { result: { value: "pointer" } } : {}; + }, + }; + const control = runtime as unknown as { + page: typeof page; + requirePage: () => Promise; + documentGeneration: number; + }; + Object.assign(runtime, { + page, + emulationAppliedPage: page, + viewport: { width: 1280, height: 800, deviceScaleFactor: 1, mobile: false, touch: false }, + }); + control.requirePage = async () => control.page; + return { + runtime, + control, + page, + calls, + intercept: (next: typeof intercept) => { + intercept = next; + }, + }; +} + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); +describe("runtime-owned live input", () => { + it("completes acknowledged input that navigates while closing original held-input cleanup", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + state.intercept(async method => { + if (method !== "Input.dispatchKeyEvent" || state.calls.at(-1)?.params.type !== "keyDown") return; + state.control.documentGeneration++; + await state.runtime.endLiveInput("owned"); + }); + await expect(state.runtime.dispatchKey({kind:"key",type:"down",key:"Enter",code:"Enter",modifiers:0,repeat:false},"owned")).resolves.toBeUndefined(); + expect(state.calls.filter(call => call.params.type === "keyDown")).toHaveLength(1); + expect(state.calls.filter(call => call.params.type === "keyUp")).toHaveLength(1); + await expect(state.runtime.assertLiveInput("owned")).rejects.toThrow("attachment"); + }); + + it("does not complete target replacement or an unacknowledged send as successful navigation", async () => { + for (const scenario of ["replacement", "uncertain"] as const) { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + state.intercept(async method => { + if (method !== "Input.dispatchMouseEvent") return; + state.control.documentGeneration++; + if (scenario === "uncertain") throw new Error("Mutation outcome is unknown"); + state.control.page = {send:async()=>({})}; + }); + await expect(state.runtime.mouseDown(20,30,"left",1,"owned")).rejects.toThrow(scenario === "replacement" ? "attachment" : "unknown"); + state.intercept(async () => {}); + await state.runtime.endLiveInput("owned"); + } + }); + + it("does not restore held touches after an acknowledged touch start navigates", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + state.intercept(async method => { + if (method !== "Input.dispatchTouchEvent" || state.calls.at(-1)?.params.type !== "touchStart") return; + state.control.documentGeneration++; + await state.runtime.endLiveInput("owned"); + }); + await state.runtime.touch("touchStart",[{x:20,y:30,id:7}],"owned"); + await state.runtime.releaseHeldInput(); + expect(state.calls.filter(call => call.params.type === "touchCancel")).toHaveLength(1); + await expect(state.runtime.assertLiveInput("owned")).rejects.toThrow("attachment"); + }); + + it("releases original held mouse after plugin/IPC disappearance without another policy request", async () => { + vi.useFakeTimers(); + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + await state.runtime.mouseDown(10, 20, "left", 1, "owned"); + await vi.advanceTimersByTimeAsync(4_000); + await state.runtime.assertLiveInput("owned"); + await state.runtime.cursorAt(10, 20, "owned"); + await vi.advanceTimersByTimeAsync(1_001); + expect(state.calls.filter((call) => call.params.type === "mouseReleased")).toHaveLength(1); + await expect(state.runtime.mouseMove(20, 30, "owned")).rejects.toThrow("attachment"); + }); + + it("acknowledged moves renew idle expiry but never the absolute five-minute limit", async () => { + vi.useFakeTimers(); + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + await state.runtime.mouseDown(10, 20, "left", 1, "owned"); + for (let index = 0; index < 74; index++) { + await vi.advanceTimersByTimeAsync(4_000); + await state.runtime.mouseMove(20, 30, "owned"); + } + await vi.advanceTimersByTimeAsync(4_001); + expect(state.calls.filter((call) => call.params.type === "mouseReleased")).toHaveLength(1); + }); + + it("cancels possibly published touch after lost acknowledgment and keeps original attachment on replacement", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + state.intercept(async (method) => { + if (method === "Input.dispatchTouchEvent") throw new Error("Unknown publication outcome"); + }); + await expect( + state.runtime.touch("touchStart", [{ x: 20, y: 30, id: 7 }], "owned"), + ).rejects.toThrow("Unknown publication"); + const replacementCalls: string[] = []; + state.control.page = { + send: async (method) => { + replacementCalls.push(method); + return {}; + }, + }; + await state.runtime.endLiveInput("owned"); + expect(state.calls.filter((call) => call.params.type === "touchCancel")).toHaveLength(1); + expect(replacementCalls).toEqual([]); + }); + + it("ends removed touch contacts before moving survivors and cancels remaining held contacts", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + await state.runtime.touch( + "touchStart", + [ + { x: 20, y: 30, id: 7 }, + { x: 60, y: 30, id: 9 }, + ], + "owned", + ); + await state.runtime.touch("touchMove", [{ x: 25, y: 35, id: 7 }], "owned"); + expect( + state.calls + .filter((call) => call.method === "Input.dispatchTouchEvent") + .map((call) => call.params), + ).toEqual([ + { + type: "touchStart", + touchPoints: [ + { x: 20, y: 30, id: 7 }, + { x: 60, y: 30, id: 9 }, + ], + }, + { type: "touchEnd", touchPoints: [{ x: 60, y: 30, id: 9 }] }, + { type: "touchMove", touchPoints: [{ x: 25, y: 35, id: 7 }] }, + ]); + await state.runtime.endLiveInput("owned"); + expect(state.calls.filter((call) => call.params.type === "touchCancel")).toHaveLength(1); + }); + + it("same-URL document replacement refuses publication and optional cursor rejects CSS URL values", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + state.control.documentGeneration++; + await expect(state.runtime.mouseMove(20, 30, "owned")).rejects.toThrow("attachment"); + expect(state.calls).toEqual([]); + await state.runtime.endLiveInput("owned"); + await state.runtime.beginLiveInput("next"); + state.control.page.send = async () => ({ + result: { value: "url(https://secret.invalid/cursor), pointer" }, + }); + expect(await state.runtime.cursorAt(20, 30, "next")).toBeNull(); + await state.runtime.endLiveInput("next"); + }); + + it("native leave uses only the fixed outside point and refuses departure while a button is held", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + await state.runtime.mouseLeave("owned"); + expect(state.calls[0]?.params).toEqual({ + type: "mouseMoved", + x: -1, + y: -1, + button: "none", + buttons: 0, + }); + await state.runtime.mouseDown(20, 30, "left", 1, "owned"); + await expect(state.runtime.mouseLeave("owned")).rejects.toThrow("Release held"); + await state.runtime.endLiveInput("owned"); + await expect(state.runtime.mouseLeave("owned")).rejects.toThrow("attachment"); + }); + + it("private runtime routing preserves gesture identity and refuses malformed native contact sets", async () => { + const owner = await createRuntimeOwner(); + const runtime = { + beginLiveInput: vi.fn(), + assertLiveInput: vi.fn(), + endLiveInput: vi.fn(), + touch: vi.fn(), + mouseMove: vi.fn(), + dispatchKey: vi.fn(), + insertText: vi.fn(), + }; + const owned = { runtimeId: "owned", runtime } as unknown as Parameters[0]; + await owner.request(owned, "input.begin", { gestureId: "gesture" }); + await owner.request(owned, "mouse.move", { gestureId: "gesture", x: 20, y: 30 }); + await owner.request(owned, "touch", { + gestureId: "gesture", + type: "start", + points: [{ id: 7, x: 20, y: 30 }], + }); + expect(runtime.beginLiveInput).toHaveBeenCalledWith("gesture"); + expect(runtime.mouseMove).toHaveBeenCalledWith(20, 30, "gesture"); + expect(runtime.touch).toHaveBeenCalledWith("touchStart", [{ id: 7, x: 20, y: 30 }], "gesture"); + for (const points of [ + [ + { id: 7, x: 20, y: 30 }, + { id: 7, x: 30, y: 40 }, + ], + [], + [{ id: true, x: 20, y: 30 }], + ]) { + await expect( + owner.request(owned, "touch", { gestureId: "gesture", type: "start", points }), + ).rejects.toThrow(); + } + expect(runtime.touch).toHaveBeenCalledTimes(1); + const event = { kind: "key", type: "down", key: "Enter", code: "Enter", modifiers: 0, repeat: false }; + await owner.request(owned, "input.key", { gestureId: "gesture", event }); + await owner.request(owned, "input.text", { gestureId: "gesture", text: "漢字😀" }); + expect(runtime.dispatchKey).toHaveBeenCalledWith(event, "gesture"); + expect(runtime.insertText).toHaveBeenCalledWith("漢字😀", "gesture"); + await expect(owner.request(owned, "input.key", { event })).rejects.toThrow(); + await expect(owner.request(owned, "input.key", { gestureId: "gesture", event: { ...event, modifiers: 99 } })).rejects.toThrow(); + await expect(owner.request(owned, "input.text", { gestureId: "gesture", text: "x".repeat(16001) })).rejects.toThrow(); + expect(runtime.dispatchKey).toHaveBeenCalledTimes(1); + expect(runtime.insertText).toHaveBeenCalledTimes(1); + }); + + it("unpressed movement retains the ordinary no-button packet", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + await state.runtime.mouseMove(20, 30, "owned"); + expect(state.calls[0]?.params).toEqual({ + type: "mouseMoved", + x: 20, + y: 30, + button: "none", + buttons: 0, + }); + await state.runtime.endLiveInput("owned"); + }); + + for (const [button, mask] of [ + ["left", 1], + ["right", 2], + ["middle", 4], + ] as const) { + it(`held ${button} movement carries its native button and mask until release`, async () => { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + await state.runtime.mouseDown(10, 20, button, 1, "owned"); + await state.runtime.mouseMove(20, 30, "owned"); + await state.runtime.mouseUp(20, 30, button, 1, "owned"); + await state.runtime.mouseMove(30, 40, "owned"); + expect( + state.calls + .filter((call) => call.params.type === "mouseMoved") + .map((call) => call.params), + ).toEqual([ + { type: "mouseMoved", x: 20, y: 30, button, buttons: mask }, + { type: "mouseMoved", x: 30, y: 40, button: "none", buttons: 0 }, + ]); + await state.runtime.endLiveInput("owned"); + }); + } + + it("multiple held buttons use canonical priority, and cleanup leaves the next gesture unpressed", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("owned"); + await state.runtime.mouseDown(10, 20, "middle", 1, "owned"); + await state.runtime.mouseDown(10, 20, "right", 1, "owned"); + await state.runtime.mouseMove(20, 30, "owned"); + await state.runtime.mouseDown(10, 20, "left", 1, "owned"); + await state.runtime.mouseMove(20, 30, "owned"); + await state.runtime.mouseUp(10, 20, "left", 1, "owned"); + await state.runtime.mouseMove(20, 30, "owned"); + await state.runtime.mouseUp(10, 20, "right", 1, "owned"); + await state.runtime.mouseMove(20, 30, "owned"); + await state.runtime.endLiveInput("owned"); + await state.runtime.beginLiveInput("next"); + await state.runtime.mouseMove(20, 30, "next"); + expect( + state.calls + .filter((call) => call.params.type === "mouseMoved") + .map((call) => [call.params.button, call.params.buttons]), + ).toEqual([ + ["right", 6], + ["left", 7], + ["right", 6], + ["middle", 4], + ["none", 0], + ]); + expect( + state.calls.filter((call) => call.params.type === "mouseReleased").at(-1) + ?.params.button, + ).toBe("middle"); + await state.runtime.endLiveInput("next"); + }); +}); + +it("publishes ordered native Unicode/repeat/shortcut keys and committed text on the pinned attachment", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("keys"); + await state.runtime.dispatchKey({ kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: false }, "keys"); + await state.runtime.dispatchKey({ kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: true }, "keys"); + await state.runtime.dispatchKey({ kind: "key", type: "up", key: "é", code: "KeyE", modifiers: 0, repeat: false }, "keys"); + await state.runtime.dispatchKey({ kind: "key", type: "down", key: "Control", code: "ControlLeft", modifiers: 2, repeat: false }, "keys"); + await state.runtime.mouseDown(10, 20, "left", 1, "keys"); + await state.runtime.insertText("漢字\n😀", "keys"); + expect(state.calls.filter((call) => call.method === "Input.dispatchKeyEvent").map((call) => call.params)).toMatchObject([ + { type: "keyDown", text: "é", code: "KeyE", autoRepeat: false }, + { type: "keyDown", text: "é", autoRepeat: true }, + { type: "keyUp", code: "KeyE" }, + { type: "rawKeyDown", code: "ControlLeft", modifiers: 2 }, + ]); + expect(state.calls.find((call) => call.params.type === "mousePressed")?.params.modifiers).toBe(2); + expect(state.calls.find((call) => call.method === "Input.insertText")?.params).toEqual({ text: "漢字\n😀" }); + await state.runtime.endLiveInput("keys"); + expect(state.calls.at(-1)?.params).toMatchObject({ type: "keyUp", key: "Control", code: "ControlLeft" }); +}); + +it("cleans possibly published keys on the original page and rejects unmatched repeat/up", async () => { + const state = fixture(); + await state.runtime.beginLiveInput("keys"); + const down = { kind: "key", type: "down", key: "Shift", code: "ShiftRight", modifiers: 8, repeat: false } as const; + await expect(state.runtime.dispatchKey({ ...down, repeat: true }, "keys")).rejects.toThrow(); + await expect(state.runtime.dispatchKey({ ...down, type: "up" }, "keys")).rejects.toThrow(); + state.intercept(async (method) => { if (method === "Input.dispatchKeyEvent") throw new Error("Lost key acknowledgment"); }); + await expect(state.runtime.dispatchKey(down, "keys")).rejects.toThrow("Lost key"); + state.control.page = { send: async () => ({}) }; + await state.runtime.endLiveInput("keys"); + expect(state.calls.filter((call) => call.params.type === "keyUp")).toHaveLength(1); +}); + +it("expires held keyboard modifiers without another client request", async () => { + vi.useFakeTimers(); + const state = fixture(); + await state.runtime.beginLiveInput("keys"); + await state.runtime.dispatchKey({ kind: "key", type: "down", key: "Alt", code: "AltLeft", modifiers: 1, repeat: false }, "keys"); + await vi.advanceTimersByTimeAsync(5001); + expect(state.calls.filter((call) => call.params.type === "keyUp")).toHaveLength(1); + await expect(state.runtime.insertText("late", "keys")).rejects.toThrow("attachment"); +}); diff --git a/paseo-shared-browser/server/live-input-supervisor.test.ts b/paseo-shared-browser/server/live-input-supervisor.test.ts new file mode 100644 index 00000000..606b68cb --- /dev/null +++ b/paseo-shared-browser/server/live-input-supervisor.test.ts @@ -0,0 +1,138 @@ +import { expect, it } from "vitest"; +import { RuntimeSupervisor } from "./supervisor"; +import type { JsonValue, RuntimeRequest } from "./runtime-protocol"; +import { browserStateSchema } from "../shared/browser"; + +/** Exercise the immutable supervisor's human RPC path without a plugin or browser process. */ +it("routes validated human gestures, fences replaced plugin bridges and never exposes live gesture operations to agent tickets", async () => { + const calls: string[] = []; + const supervisor = new RuntimeSupervisor({ + owner: { + create: async () => ({ runtimeId: "r".repeat(32) }), + request: async (_runtime, operation) => { + calls.push(operation); + if (operation === "identity") return { userAgent: "Fixture Chromium" }; + if (operation === "state") + return { + url: "https://fixture.invalid", + title: "Fixture", + canGoBack: false, + canGoForward: false, + inputGeneration: "0:0", + }; + if (operation === "frame") + return { + dataBase64: "eA==", + byteLength: 1, + width: 1280, + height: 800, + capturedAt: new Date().toISOString(), + }; + if (operation === "cursor") return "pointer"; + return null; + }, + stop: async () => {}, + }, + }); + const lease = supervisor.claimBridge("bridge"); + const request = async ( + operation: string, + input: unknown, + epoch = lease.epoch, + bridgeId = "bridge", + ) => + supervisor.dispatch({ + version: 2, + id: "request", + token: "unused-by-direct-fixture", + bridgeId, + epoch, + method: "browser.request", + operation, + input: input as JsonValue, + }); + try { + const attached = (await request("attach", { + workspaceId: "workspace", + viewerLabel: "Viewer", + })) as { viewerToken: string }; + const acquired = (await request("acquire-control", { viewerToken: attached.viewerToken })) as { + controlToken: string; + }; + const capture = (await request("capture", { viewerToken: attached.viewerToken })) as { + state: unknown; + frame: { frameId: string }; + }; + const state = browserStateSchema.parse(capture.state); + const expected = { + sessionId: state.sessionId, + runtimeId: state.runtimeId!, + bridgeEpoch: state.bridgeEpoch!, + navigationGeneration: state.navigationGeneration, + viewportGeneration: state.viewportGeneration, + }; + const context = { + viewerToken: attached.viewerToken, + controlToken: acquired.controlToken, + expected, + }; + const target = { + frameId: capture.frame.frameId, + navigationGeneration: state.navigationGeneration, + viewportGeneration: state.viewportGeneration, + }; + const begun = (await request("gesture.begin", { + ...context, + target, + pointerKind: "mouse", + })) as { gestureId: string; nextSequence: number }; + await request("gesture.update", { + ...context, + gestureId: begun.gestureId, + sequence: 1, + event: { kind: "key", type: "down", key: "Control", code: "ControlLeft", modifiers: 2, repeat: false }, + }); + expect(calls.filter((value) => value === "input.key")).toHaveLength(1); + const afterKey = (await request("capture", { viewerToken: attached.viewerToken })) as { frame: { frameId: string } }; + await request("gesture.update", { + ...context, + gestureId: begun.gestureId, + sequence: 2, + target: { ...target, frameId: afterKey.frame.frameId }, + event: { kind: "down", point: { x: 10, y: 20, width: 1280, height: 800 } }, + }); + expect(calls.filter((value) => value === "mouse.down")).toHaveLength(1); + await expect( + request("gesture.update", { + ...context, + gestureId: begun.gestureId, + sequence: 3, + event: { + kind: "scroll", + point: { x: 10, y: 20, width: 1280, height: 800 }, + deltaX: 0, + deltaY: Infinity, + }, + }), + ).rejects.toThrow(); + supervisor.claimBridge("replacement", true); + await expect( + request("gesture.end", { ...context, gestureId: begun.gestureId, sequence: 1, cancel: true }), + ).rejects.toThrow("lease"); + // The old channel's supervisor-side cleanup still reaches its owned runtime. + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(calls.filter((value) => value === "input.end")).toHaveLength(1); + await expect( + supervisor.dispatch({ + version: 2, + id: "request", + method: "agent.request", + ticket: "invalid", + operation: "gesture.begin", + input: {}, + } as unknown as RuntimeRequest), + ).rejects.toThrow(); + } finally { + await supervisor.stopAll(); + } +}); diff --git a/paseo-shared-browser/server/mcp-entry.ts b/paseo-shared-browser/server/mcp-entry.ts index a3848dd8..9a6a7706 100644 --- a/paseo-shared-browser/server/mcp-entry.ts +++ b/paseo-shared-browser/server/mcp-entry.ts @@ -3,13 +3,14 @@ import { join } from "node:path"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { z } from "zod"; +import { MIN_VIEWPORT, MAX_VIEWPORT } from "../shared/viewport-limits"; +import { DEFAULT_CAPTURE_QUALITY } from "../shared/capture-settings"; +import { DEVICE_PRESET_IDS } from "../shared/device-presets"; import type { JsonValue } from "./runtime-protocol"; import { resolveSupervisorPaths } from "./supervisor"; import { AgentSupervisorClient } from "./supervisor-client"; const TICKET_ENV = "PASEO_SHARED_BROWSER_TICKET"; -const MIN_VIEWPORT = { width: 320, height: 480 } as const; -const MAX_VIEWPORT = { width: 1600, height: 1200 } as const; const pointSchema = z.object({ x: z.number().finite().nonnegative(), @@ -106,7 +107,7 @@ async function main(): Promise { description: "Capture the current shared browser frame and state. Capture before sending input.", inputSchema: z.object({ - quality: z.enum(["low", "medium", "high"]).default("medium"), + quality: z.enum(["low", "medium", "high"]).default(DEFAULT_CAPTURE_QUALITY), }), annotations: { readOnlyHint: true, idempotentHint: true }, }, @@ -225,6 +226,17 @@ async function main(): Promise { ), ); + server.registerTool( + "shared_browser_device", + { + description: + "Apply the same device or display preset as the human picker, including sharper Pixel capture. Requires control and reloads the current page.", + inputSchema: z.object({ presetId: z.enum(DEVICE_PRESET_IDS) }), + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false }, + }, + async ({ presetId }) => textResult(await client.request("device", { presetId })), + ); + const transport = new StdioServerTransport(); process.once("exit", () => client.disconnect()); await server.connect(transport); diff --git a/paseo-shared-browser/server/runtime-frame-reuse.test.ts b/paseo-shared-browser/server/runtime-frame-reuse.test.ts new file mode 100644 index 00000000..e948ac69 --- /dev/null +++ b/paseo-shared-browser/server/runtime-frame-reuse.test.ts @@ -0,0 +1,265 @@ +/** Exact fresh runtime receipts avoid resending cached JPEGs without reviving invalidated input. */ +import { describe, expect, it } from "vitest"; +import { SessionManager, type BrowserRuntimeClient } from "./browser-policy"; +import type { BrowserFrame, BrowserState } from "../shared/browser"; +import type { JsonValue } from "./runtime-protocol"; + +async function fixture() { + let now = 1_000; + let token = 0; + let frames = 0; + let inputGeneration = "0:0"; + const raw: Record = { + dataBase64: Buffer.alloc(200_000, 120).toString("base64"), + byteLength: 200_000, + width: 1280, + height: 800, + transport: "screenshot", + capturedAt: new Date(now).toISOString(), + }; + const client: BrowserRuntimeClient = { + connect: async () => ({ epoch: 1 }), + ensureWorkspace: async (workspaceId) => ({ + workspaceId, + runtimeId: "runtime-one", + createdAt: 1, + }), + requestWorkspace: async (_workspace, operation, input) => { + if (operation === "identity") return { userAgent: "Fixture" }; + if (operation === "state") + return { + url: "https://fixture.invalid/", + title: "Fixture", + canGoBack: false, + canGoForward: false, + inputGeneration, + }; + if (operation === "frame") { + frames++; + return { ...raw }; + } + if (operation === "emulate" && input && typeof input === "object" && !Array.isArray(input)) { + raw.width = input.width!; + raw.height = input.height!; + } + if (operation === "cursor") return null; + return null; + }, + archiveWorkspace: async () => {}, + disconnect: () => {}, + }; + const manager = new SessionManager({ + client, + now: () => now, + validateWorkspace: async () => true, + issueToken: () => String(++token).padStart(32, "0"), + }); + await manager.connect(); + const viewer = await manager.attach("owned-perf-fixture", "Viewer"); + const control = await manager.acquireControl(viewer.viewerToken); + const expected = (state: BrowserState) => ({ + sessionId: state.sessionId, + runtimeId: state.runtimeId!, + bridgeEpoch: state.bridgeEpoch!, + navigationGeneration: state.navigationGeneration, + viewportGeneration: state.viewportGeneration, + }); + return { + manager, + raw, + viewer, + control, + expected, + get frames() { + return frames; + }, + advance(ms = 250) { + now += ms; + }, + documentChange() { + inputGeneration = "0:1"; + }, + session: () => + ( + manager as unknown as { sessions: Map } + ).sessions.get("owned-perf-fixture")!, + }; +} +const target = (frame: BrowserFrame) => ({ + frameId: frame.frameId, + navigationGeneration: frame.navigationGeneration, + viewportGeneration: frame.viewportGeneration, +}); + +describe("exact runtime receipt reuse across policy cache expiry", () => { + it("returns metadata-only repeated polls while preserving current-frame input authority", async () => { + const f = await fixture(); + try { + const first = await f.manager.capture(f.viewer.viewerToken); + for (let index = 0; index < 3; index++) { + f.advance(); + const repeated = await f.manager.capture( + f.viewer.viewerToken, + "high", + first.frame!.frameId, + ); + expect(repeated.frame).toBeNull(); + } + expect(f.frames).toBe(4); + const result = await f.manager.beginGesture({ + viewerToken: f.viewer.viewerToken, + controlToken: f.control.controlToken, + expected: f.expected(first.state), + target: target(first.frame!), + pointerKind: "mouse", + }); + expect(result).toHaveProperty("gestureId"); + } finally { + f.manager.disconnect(); + } + }); + + it("assigns a new token for any changed runtime receipt field, including equal pixels at a new capture time", async () => { + for (const [field, value] of [ + ["capturedAt", "2026-10-03T12:00:00.000Z"], + ["transport", "cdp-screencast"], + ["dataBase64", Buffer.alloc(200_000, 121).toString("base64")], + ["byteLength", 1], + ] as const) { + const f = await fixture(); + try { + const first = await f.manager.capture(f.viewer.viewerToken); + f.advance(); + f.raw[field] = value; + if (field === "byteLength") f.raw.dataBase64 = "eA=="; + const next = await f.manager.capture(f.viewer.viewerToken, "high", first.frame!.frameId); + expect(next.frame?.frameId).not.toBe(first.frame!.frameId); + } finally { + f.manager.disconnect(); + } + } + }); + + it("keeps per-quality tokens separate even for identical runtime receipts", async () => { + const f = await fixture(); + try { + const first = await f.manager.capture(f.viewer.viewerToken, "high"); + f.advance(); + const next = await f.manager.capture(f.viewer.viewerToken, "medium", first.frame!.frameId); + expect(next.frame?.frameId).not.toBe(first.frame!.frameId); + } finally { + f.manager.disconnect(); + } + }); + + it("cannot revive an identical runtime receipt after input invalidation", async () => { + const f = await fixture(); + try { + const first = await f.manager.capture(f.viewer.viewerToken); + await f.manager.sendInput({ + viewerToken: f.viewer.viewerToken, + controlToken: f.control.controlToken, + expected: f.expected(first.state), + target: target(first.frame!), + event: { kind: "type", text: "x" }, + }); + f.advance(); + const next = await f.manager.capture(f.viewer.viewerToken, "high", first.frame!.frameId); + expect(next.frame?.frameId).not.toBe(first.frame!.frameId); + await expect( + f.manager.beginGesture({ + viewerToken: f.viewer.viewerToken, + controlToken: f.control.controlToken, + expected: f.expected(next.state), + target: target(first.frame!), + pointerKind: "mouse", + }), + ).resolves.toMatchObject({ admission: "stale-frame" }); + } finally { + f.manager.disconnect(); + } + }); + + it("changes token for exact resized geometry and refuses malformed old dimensions", async () => { + const f = await fixture(); + try { + const first = await f.manager.capture(f.viewer.viewerToken); + await f.manager.resize({ + viewerToken: f.viewer.viewerToken, + controlToken: f.control.controlToken, + expected: f.expected(first.state), + viewport: { width: 1920, height: 1200 }, + }); + const next = await f.manager.capture(f.viewer.viewerToken, "high", first.frame!.frameId); + expect(next.frame).toMatchObject({ width: 1920, height: 1200 }); + expect(next.frame!.frameId).not.toBe(first.frame!.frameId); + f.advance(); + f.raw.width = 1280; + await expect(f.manager.capture(f.viewer.viewerToken)).rejects.toThrow("stale viewport"); + } finally { + f.manager.disconnect(); + } + }); + + it("does not reuse receipts under changed runtime identity or bridge epoch", async () => { + for (const field of ["runtimeId", "bridgeEpoch"] as const) { + const f = await fixture(); + try { + const first = await f.manager.capture(f.viewer.viewerToken); + f.advance(); + if (field === "runtimeId") f.session().runtimeId = "runtime-two"; + else f.session().bridgeEpoch = 2; + const next = await f.manager.capture(f.viewer.viewerToken, "high", first.frame!.frameId); + expect(next.frame!.frameId).not.toBe(first.frame!.frameId); + } finally { + f.manager.disconnect(); + } + } + }); + + it("cannot deduplicate legacy unmarked receipts or metadata after a document change", async () => { + for (const field of ["capturedAt", "transport"] as const) { + const f = await fixture(); + try { + delete f.raw[field]; + const first = await f.manager.capture(f.viewer.viewerToken); + f.advance(); + const next = await f.manager.capture(f.viewer.viewerToken, "high", first.frame!.frameId); + expect(next.frame!.frameId).not.toBe(first.frame!.frameId); + } finally { + f.manager.disconnect(); + } + } + const f = await fixture(); + try { + const first = await f.manager.capture(f.viewer.viewerToken); + f.documentChange(); + await f.manager.status(f.viewer.viewerToken); + f.advance(); + const next = await f.manager.capture(f.viewer.viewerToken, "high", first.frame!.frameId); + expect(next.frame!.frameId).not.toBe(first.frame!.frameId); + expect(next.state.navigationGeneration).toBeGreaterThan(first.state.navigationGeneration); + } finally { + f.manager.disconnect(); + } + }); + + it("avoids repeated large payloads while every expired policy read still asks the runtime for fresh pixels", async () => { + const f = await fixture(); + try { + const first = await f.manager.capture(f.viewer.viewerToken); + let before = JSON.stringify(first).length; + let after = before; + for (let index = 0; index < 3; index++) { + f.advance(); + const reply = await f.manager.capture(f.viewer.viewerToken, "high", first.frame!.frameId); + after += JSON.stringify(reply).length; + before += JSON.stringify({ state: reply.state, frame: first.frame }).length; + } + expect(after).toBeLessThan(before * 0.26); + expect(f.frames).toBe(4); + } finally { + f.manager.disconnect(); + } + }); +}); diff --git a/paseo-shared-browser/server/runtime-launch-environment.test.ts b/paseo-shared-browser/server/runtime-launch-environment.test.ts new file mode 100644 index 00000000..1258e1ec --- /dev/null +++ b/paseo-shared-browser/server/runtime-launch-environment.test.ts @@ -0,0 +1,98 @@ +/** Per-instance display context reaches native CLI children without changing host or session authority. */ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { AgentBrowserRuntime } from "./agent-browser-runtime"; + +const temporary: string[] = []; +afterEach(async () => { + await Promise.all(temporary.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +async function fixture(launchEnvironment?: Readonly>) { + const directory = await mkdtemp("/tmp/printstream-runtime-env-"); + temporary.push(directory); + const binary = join(directory, "owned-cli"); + await writeFile( + binary, + "#!" + + process.execPath + + '\nif(process.argv.includes("--version")){console.log("0.37.1");}else{console.log(JSON.stringify({display:process.env.DISPLAY??null,authority:process.env.XAUTHORITY??null,wayland:process.env.WAYLAND_DISPLAY??null,socket:process.env.AGENT_BROWSER_SOCKET_DIR,profile:process.env.AGENT_BROWSER_PROFILE??null,args:process.argv.slice(2)}));}\n', + { mode: 0o700 }, + ); + const runtime = new AgentBrowserRuntime({ + binaryPath: binary, + executablePath: join(directory, "unlaunched-chromium"), + profilePath: join(directory, "owned-profile"), + ipcDirectory: join(directory, "owned-ipc"), + session: "owned-environment", + ...(launchEnvironment ? { launchEnvironment } : {}), + }); + const native = runtime as unknown as { + assertVersion(): Promise; + invoke( + args: string[], + ): Promise<{ + display: string | null; + authority: string | null; + wayland: string | null; + socket: string; + profile: null; + args: string[]; + }>; + }; + return { directory, runtime, native }; +} + +describe("trusted private display launch environment", () => { + it("passes display/auth to every CLI child, copies settings, and keeps host environment unchanged", async () => { + const before = { + display: process.env.DISPLAY, + authority: process.env.XAUTHORITY, + wayland: process.env.WAYLAND_DISPLAY, + }; + const environment = { DISPLAY: ":24701", XAUTHORITY: "/tmp/owned-private-auth" }; + const f = await fixture(environment); + environment.DISPLAY = ":24702"; + await f.native.assertVersion(); + for (const args of [["owned-request"], ["owned-reconnect"]]) { + const result = await f.native.invoke(args); + expect(result).toEqual({ + display: ":24701", + authority: "/tmp/owned-private-auth", + wayland: null, + socket: join(f.directory, "owned-ipc"), + profile: null, + args, + }); + } + expect({ + display: process.env.DISPLAY, + authority: process.env.XAUTHORITY, + wayland: process.env.WAYLAND_DISPLAY, + }).toEqual(before); + await f.runtime.shutdown(); + }); + it("preserves the ordinary host display path when no private environment is provided", async () => { + const f = await fixture(); + const result = await f.native.invoke(["ordinary"]); + expect(result).toMatchObject({ + display: process.env.DISPLAY ?? null, + authority: process.env.XAUTHORITY ?? null, + wayland: process.env.WAYLAND_DISPLAY ?? null, + socket: join(f.directory, "owned-ipc"), + }); + await f.runtime.shutdown(); + }); + it("refuses missing, remote or extra values that could override managed IPC/profile/session identity", async () => { + for (const value of [ + { DISPLAY: ":24701" }, + { DISPLAY: "remote.example:0", XAUTHORITY: "/tmp/auth" }, + { DISPLAY: ":24701", XAUTHORITY: "relative-auth" }, + { DISPLAY: ":24701", XAUTHORITY: "/tmp/auth", AGENT_BROWSER_SOCKET_DIR: "/tmp/other-ipc" }, + { DISPLAY: ":24701", XAUTHORITY: "/tmp/auth", HOME: "/tmp/other-home" }, + ]) { + await expect(fixture(value)).rejects.toThrow("display environment is invalid"); + } + }); +}); diff --git a/paseo-shared-browser/server/runtime-owner-display.test.ts b/paseo-shared-browser/server/runtime-owner-display.test.ts new file mode 100644 index 00000000..d11fabbc --- /dev/null +++ b/paseo-shared-browser/server/runtime-owner-display.test.ts @@ -0,0 +1,91 @@ +import { beforeEach, expect, it, vi } from "vitest"; +import { createRuntimeOwner } from "./runtime-owner"; + +const fake = vi.hoisted(() => ({ + display: vi.fn(), + runtime: vi.fn(), + launch: vi.fn(), + shutdown: vi.fn(), + assertAvailable: vi.fn(), + stop: vi.fn(), +})); +vi.mock("./virtual-display", () => ({ createPrivateVirtualDisplay: fake.display })); +vi.mock("./runtime-path", () => ({ + resolveBrowserRuntimeRoot: () => "/tmp/owned-runtime-fixture", +})); +vi.mock("./agent-browser-runtime", () => ({ + AgentBrowserRuntime: class { + launch = fake.launch; + shutdown = fake.shutdown; + constructor(options: unknown) { + fake.runtime(options); + } + }, +})); +beforeEach(() => { + vi.clearAllMocks(); + fake.launch.mockResolvedValue(undefined); + fake.shutdown.mockResolvedValue(undefined); + fake.stop.mockResolvedValue(undefined); + fake.display.mockResolvedValue({ + launchEnvironment: { DISPLAY: ":23067", XAUTHORITY: "/tmp/private/authority" }, + assertAvailable: fake.assertAvailable, + stop: fake.stop, + }); +}); + +it("supplies private display environment only to hidden runtime creation", async () => { + const owner = await createRuntimeOwner(); + const runtime = await owner.create("workspace-fixture"); + expect(fake.runtime.mock.calls[0]![0]).toMatchObject({ + headed: true, + launchEnvironment: { DISPLAY: ":23067", XAUTHORITY: "/tmp/private/authority" }, + }); + await owner.stop(runtime); + expect(fake.shutdown).toHaveBeenCalledTimes(1); + expect(fake.stop).toHaveBeenCalledTimes(1); +}); + +it("keeps explicit headed mode and unavailable-display headless behavior unchanged", async () => { + await (await createRuntimeOwner({ headed: true })).create("headed-fixture"); + expect(fake.display).not.toHaveBeenCalled(); + expect(fake.runtime.mock.calls[0]![0]).toMatchObject({ headed: true }); + expect(fake.runtime.mock.calls[0]![0]).not.toHaveProperty("launchEnvironment"); + fake.display.mockResolvedValue(null); + await (await createRuntimeOwner()).create("absent-fixture"); + expect(fake.runtime.mock.calls[1]![0]).toMatchObject({ headed: false }); +}); + +it("falls back only before Chromium construction when optional display startup fails", async () => { + fake.display.mockRejectedValue(new Error("fixture private startup failure")); + const warning = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await (await createRuntimeOwner()).create("startup-fallback-fixture"); + expect(fake.runtime).toHaveBeenCalledTimes(1); + expect(fake.runtime.mock.calls[0]![0]).toMatchObject({ headed: false }); + expect(fake.runtime.mock.calls[0]![0]).not.toHaveProperty("launchEnvironment"); + expect(warning).toHaveBeenCalledWith( + "[shared-browser] Private display unavailable; using existing headless mode.", + ); + } finally { + warning.mockRestore(); + } +}); + +it("cleans the private display after launch failure and even failing browser shutdown", async () => { + fake.launch.mockRejectedValue(new Error("fixture launch failure")); + fake.shutdown.mockRejectedValue(new Error("fixture shutdown failure")); + await expect((await createRuntimeOwner()).create("failure-fixture")).rejects.toThrow(); + expect(fake.stop).toHaveBeenCalledTimes(1); +}); + +it("refuses runtime requests after private display loss", async () => { + const owner = await createRuntimeOwner(); + const runtime = await owner.create("loss-fixture"); + fake.assertAvailable.mockImplementation(() => { + throw new Error("Private display ended"); + }); + await expect(owner.request(runtime, "state", null)).rejects.toThrow("ended"); + await owner.stop(runtime); + expect(fake.stop).toHaveBeenCalledTimes(1); +}); diff --git a/paseo-shared-browser/server/runtime-owner.ts b/paseo-shared-browser/server/runtime-owner.ts index e7deb001..4bb40e3a 100644 --- a/paseo-shared-browser/server/runtime-owner.ts +++ b/paseo-shared-browser/server/runtime-owner.ts @@ -1,16 +1,19 @@ import { createHash, randomUUID } from "node:crypto"; import { homedir } from "node:os"; import { join } from "node:path"; +import { browserGestureKeySchema } from "../shared/browser"; import { AgentBrowserRuntime, type BrowserViewport } from "./agent-browser-runtime"; import { resolveBrowserRuntimeRoot } from "./runtime-path"; import type { JsonValue } from "./runtime-protocol"; import type { RuntimeOwner } from "./supervisor"; +import { createPrivateVirtualDisplay, type PrivateVirtualDisplay } from "./virtual-display"; const DEFAULT_BROWSER_URL = "https://example.com/"; interface OwnedRuntime { runtimeId: string; runtime: AgentBrowserRuntime; + display: PrivateVirtualDisplay | null; } interface RuntimeOwnerOptions { initialUrl?: string; @@ -48,23 +51,51 @@ export async function createRuntimeOwner( return { async create(workspaceId) { const hash = createHash("sha256").update(workspaceId).digest("hex"); - const runtime = new AgentBrowserRuntime({ - binaryPath, - executablePath, - profilePath: join(root, "profiles", hash), - ipcDirectory, - session: `ws-${hash.slice(0, 16)}`, - initialUrl: options.initialUrl ?? DEFAULT_BROWSER_URL, - headed: options.headed ?? false, - }); - await runtime.launch(); - return { runtimeId: randomUUID(), runtime }; + // Explicit headed mode uses the caller's real display. Hidden Linux mode + // gets a private virtual mouse so native pointer media survive emulation. + let display: PrivateVirtualDisplay | null = null; + if (!options.headed) { + try { + display = await createPrivateVirtualDisplay(); + } catch { + // The helper has finished cleanup and no Chromium was constructed yet. + // Optional display support must not break existing headless installs. + console.warn( + "[shared-browser] Private display unavailable; using existing headless mode.", + ); + } + } + let runtime: AgentBrowserRuntime | null = null; + try { + runtime = new AgentBrowserRuntime({ + binaryPath, + executablePath, + profilePath: join(root, "profiles", hash), + ipcDirectory, + session: `ws-${hash.slice(0, 16)}`, + initialUrl: options.initialUrl ?? DEFAULT_BROWSER_URL, + headed: display ? true : (options.headed ?? false), + ...(display ? { launchEnvironment: display.launchEnvironment } : {}), + }); + await runtime.launch(); + display?.assertAvailable(); + return { runtimeId: randomUUID(), runtime, display }; + } catch (error) { + try { + await runtime?.shutdown(); + } finally { + await display?.stop(); + } + throw error; + } }, async request(owned, operation, input) { + owned.display?.assertAvailable(); const data = input && typeof input === "object" && !Array.isArray(input) ? (input as Record) : {}; + const gestureId = typeof data.gestureId === "string" ? data.gestureId : undefined; switch (operation) { case "identity": return owned.runtime.identity() as unknown as JsonValue; @@ -98,7 +129,7 @@ export async function createRuntimeOwner( Number(data.waitMs), )) as unknown as JsonValue; case "mouse.move": - await owned.runtime.mouseMove(Number(data.x), Number(data.y)); + await owned.runtime.mouseMove(Number(data.x), Number(data.y), gestureId); return null; case "mouse.down": await owned.runtime.mouseDown( @@ -106,6 +137,7 @@ export async function createRuntimeOwner( Number(data.y), String(data.button) as "left" | "middle" | "right", Number(data.clickCount), + gestureId, ); return null; case "mouse.up": @@ -114,6 +146,7 @@ export async function createRuntimeOwner( Number(data.y), String(data.button) as "left" | "middle" | "right", Number(data.clickCount), + gestureId, ); return null; case "mouse.wheel": @@ -122,8 +155,78 @@ export async function createRuntimeOwner( Number(data.y), Number(data.deltaX), Number(data.deltaY), + gestureId, ); return null; + case "mouse.leave": + if (!gestureId) throw new Error("Gesture identity is required"); + await owned.runtime.mouseLeave(gestureId); + return null; + case "input.begin": + if (!gestureId) throw new Error("Gesture identity is required"); + await owned.runtime.beginLiveInput(gestureId); + return null; + case "input.check": + if (!gestureId) throw new Error("Gesture identity is required"); + await owned.runtime.assertLiveInput(gestureId); + return null; + case "input.end": + if (!gestureId) throw new Error("Gesture identity is required"); + await owned.runtime.endLiveInput(gestureId); + return null; + case "cursor": + if (!gestureId) throw new Error("Gesture identity is required"); + return owned.runtime.cursorAt(Number(data.x), Number(data.y), gestureId); + case "touch": { + if (!gestureId || !Array.isArray(data.points) || data.points.length > 5) + throw new Error("Invalid touch input"); + const points = data.points.map((value) => { + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error("Invalid touch point"); + const id = value.id; + if ( + typeof id !== "number" || + typeof value.x !== "number" || + typeof value.y !== "number" + ) + throw new Error("Invalid touch point numbers"); + if (!Number.isSafeInteger(id) || id < 0 || id > 2_147_483_647) + throw new Error("Invalid touch identifier"); + return { x: Number(value.x), y: Number(value.y), id }; + }); + if (new Set(points.map((point) => point.id)).size !== points.length) + throw new Error("Duplicate touch identifiers"); + const type = data.type; + if (type !== "start" && type !== "move" && type !== "end" && type !== "cancel") + throw new Error("Invalid touch phase"); + if ((type === "end" || type === "cancel") !== (points.length === 0)) + throw new Error("Invalid touch phase points"); + const nativeType = + type === "start" + ? "touchStart" + : type === "move" + ? "touchMove" + : type === "end" + ? "touchEnd" + : "touchCancel"; + await owned.runtime.touch(nativeType, points, gestureId); + return null; + } + case "input.key": + if (!gestureId) throw new Error("Gesture identity is required"); + await owned.runtime.dispatchKey(browserGestureKeySchema.parse(data.event), gestureId); + return null; + case "input.text": + if ( + !gestureId || + typeof data.text !== "string" || + data.text.length < 1 || + data.text.length > 16_000 + ) { + throw new Error("Invalid committed text input"); + } + await owned.runtime.insertText(data.text, gestureId); + return null; case "text.insert": await owned.runtime.insertText(String(data.text)); return null; @@ -138,7 +241,11 @@ export async function createRuntimeOwner( } }, async stop(owned) { - await owned.runtime.shutdown(); + try { + await owned.runtime.shutdown(); + } finally { + await owned.display?.stop(); + } }, }; } diff --git a/paseo-shared-browser/server/runtime-page-state.test.ts b/paseo-shared-browser/server/runtime-page-state.test.ts new file mode 100644 index 00000000..e7906dd4 --- /dev/null +++ b/paseo-shared-browser/server/runtime-page-state.test.ts @@ -0,0 +1,97 @@ +/** Fresh metadata reads overlap CDP latency but never accept mixed native identities. */ +import { describe, expect, it } from "vitest"; +import { AgentBrowserRuntime } from "./agent-browser-runtime"; + +function fixture() { + const runtime = new AgentBrowserRuntime({ + binaryPath: "/tmp/unlaunched-browser", + executablePath: "/tmp/unlaunched-chromium", + profilePath: "/tmp/uncreated-state-profile", + ipcDirectory: "/tmp/uncreated-state-ipc", + session: "state-test", + }); + const calls: string[] = []; + const history = Promise.withResolvers<{ + currentIndex: number; + entries: { url: string; title: string }[]; + }>(); + const metadata = Promise.withResolvers<{ + result: { value?: { url?: string; title?: string } }; + }>(); + const page = { + send: (method: string) => { + calls.push(method); + return method === "Page.getNavigationHistory" ? history.promise : metadata.promise; + }, + }; + const control = runtime as unknown as { + page: typeof page; + attachmentGeneration: number; + documentGeneration: number; + requirePage(): Promise; + }; + control.page = page; + control.requirePage = async () => page; + return { runtime, control, page, calls, history, metadata }; +} +const entries = [ + { url: "https://example.test/previous", title: "Previous" }, + { url: "https://example.test/current", title: "History title" }, +]; + +describe("fresh parallel native page metadata", () => { + it("starts both independent reads before either resolves and retains native history facts", async () => { + const f = fixture(); + const pending = f.runtime.state(); + await Promise.resolve(); + expect(f.calls).toEqual(["Page.getNavigationHistory", "Runtime.evaluate"]); + f.metadata.resolve({ + result: { value: { url: "https://example.test/current#hash", title: "Fresh title" } }, + }); + f.history.resolve({ currentIndex: 1, entries }); + expect(await pending).toMatchObject({ + url: "https://example.test/current#hash", + title: "Fresh title", + canGoBack: true, + canGoForward: false, + }); + }); + it("preserves existing native-history fallback when DOM values are absent", async () => { + const f = fixture(); + const pending = f.runtime.state(); + f.metadata.resolve({ result: {} }); + f.history.resolve({ currentIndex: 0, entries }); + expect(await pending).toMatchObject({ + url: entries[0]!.url, + title: "Previous", + canGoBack: false, + canGoForward: true, + }); + }); + it("refuses document, attachment or page replacement during either pending observation", async () => { + for (const kind of ["document", "attachment", "page"] as const) { + const f = fixture(); + const pending = f.runtime.state(); + await Promise.resolve(); + if (kind === "document") f.control.documentGeneration++; + if (kind === "attachment") f.control.attachmentGeneration++; + if (kind === "page") + f.control.page = { send: () => Promise.resolve({}) } as unknown as typeof f.page; + f.history.resolve({ currentIndex: 1, entries }); + f.metadata.resolve({ result: { value: { title: "Obsolete document" } } }); + await expect(pending).rejects.toThrow("metadata changed"); + } + }); + it("propagates failed native reads rather than reusing earlier metadata", async () => { + for (const failed of ["history", "metadata"]) { + const f = fixture(); + const pending = f.runtime.state(); + const refusal = expect(pending).rejects.toThrow("Read unavailable"); + if (failed === "history") f.history.reject(new Error("Read unavailable")); + else f.history.resolve({ currentIndex: 1, entries }); + if (failed === "metadata") f.metadata.reject(new Error("Read unavailable")); + else f.metadata.resolve({ result: {} }); + await refusal; + } + }); +}); diff --git a/paseo-shared-browser/server/runtime-protocol.ts b/paseo-shared-browser/server/runtime-protocol.ts index 1153b3ee..02d777a2 100644 --- a/paseo-shared-browser/server/runtime-protocol.ts +++ b/paseo-shared-browser/server/runtime-protocol.ts @@ -36,7 +36,8 @@ export type AgentBrowserOperation = | "release-control" | "navigate" | "input" - | "viewport"; + | "viewport" + | "device"; export type RuntimeRequest = | (AdminRequestBase & { method: "bridge.claim"; takeover?: boolean }) @@ -267,7 +268,8 @@ function requireAgentOperation(value: unknown): AgentBrowserOperation { value === "release-control" || value === "navigate" || value === "input" || - value === "viewport" + value === "viewport" || + value === "device" ) return value; throw new RuntimeProtocolError("INVALID_REQUEST", "Unknown agent browser operation"); diff --git a/paseo-shared-browser/server/stale-wheel-admission.test.ts b/paseo-shared-browser/server/stale-wheel-admission.test.ts new file mode 100644 index 00000000..f9e319e3 --- /dev/null +++ b/paseo-shared-browser/server/stale-wheel-admission.test.ts @@ -0,0 +1,135 @@ +import assert from "node:assert/strict"; +import { it } from "vitest"; +import { SessionManager, type BrowserRuntimeClient } from "./browser-policy"; +import { + createBrowserInputQueue, + type BrowserGestureAuthority, +} from "../client/browser-input-queue"; + +it("recovers a captured-before-wheel frame that decodes after invalidation without repeating the first wheel", async () => { + // No native process or profile: actual queue + policy, with delayed decoder and deterministic clock. + let token = 0; + let now = 1_000; + const calls: string[] = []; + const client: BrowserRuntimeClient = { + connect: async () => ({ epoch: 1 }), + ensureWorkspace: async (workspaceId) => ({ + workspaceId, + runtimeId: "r".repeat(32), + createdAt: 1, + }), + requestWorkspace: async (_workspace, operation) => { + calls.push(operation); + if (operation === "identity") return { userAgent: "Fixture Chromium" }; + if (operation === "state") + return { + url: "https://fixture.invalid/", + title: "Fixture", + canGoBack: false, + canGoForward: false, + inputGeneration: "0:0", + }; + if (operation === "frame") + return { + dataBase64: "eA==", + byteLength: 1, + width: 1280, + height: 800, + capturedAt: new Date(now).toISOString(), + }; + if (operation === "cursor") return "default"; + return null; + }, + archiveWorkspace: async () => {}, + disconnect: () => {}, + }; + const manager = new SessionManager({ + client, + validateWorkspace: async () => true, + now: () => now, + frameCacheMs: 0, + issueToken: () => String(++token).padStart(32, "0"), + }); + await manager.connect(); + const { viewerToken } = await manager.attach("owned-fixture", "Viewer"); + const { controlToken } = await manager.acquireControl(viewerToken); + const a = await manager.capture(viewerToken); + const context = { + viewerToken, + controlToken, + expected: { + sessionId: a.state.sessionId, + runtimeId: a.state.runtimeId!, + bridgeEpoch: a.state.bridgeEpoch!, + navigationGeneration: a.state.navigationGeneration, + viewportGeneration: a.state.viewportGeneration, + }, + }; + const target = (frame: NonNullable) => ({ + frameId: frame.frameId, + navigationGeneration: frame.navigationGeneration, + viewportGeneration: frame.viewportGeneration, + }); + let current: BrowserGestureAuthority = { + ...context, + target: target(a.frame!), + }; + const errors: unknown[] = []; + let decoderPending: NonNullable | null = null; + let waited = 0; + const queue = createBrowserInputQueue({ + authority: () => current, + transport: { + begin: (input) => manager.beginGesture(input), + end: (input) => manager.endGesture(input), + update: async (input) => { + // A different capture is already issued before input; its slow decoder commits after the wheel reply. + now++; + decoderPending = (await manager.capture(viewerToken)).frame!; + const result = await manager.updateGesture(input); + current = { ...context, target: target(decoderPending) }; + return result; + }, + }, + onState: () => {}, + onCursor: () => {}, + onError: (error) => errors.push(error), + onFinish: () => {}, + waitForFrame: async () => { + waited++; + now++; + const fresh = await manager.capture(viewerToken); + current = { ...context, target: target(fresh.frame!) }; + }, + }); + const flush = async () => { + for (let i = 0; i < 100; i++) await Promise.resolve(); + }; + try { + queue.enqueue({ + kind: "scroll", + point: { x: 100, y: 100, width: 1280, height: 800 }, + deltaX: 0, + deltaY: 40, + }); + await flush(); + queue.finish(); + await flush(); + assert.equal(errors.length, 0); + assert.notEqual(current.target.frameId, a.frame!.frameId); + queue.enqueue({ + kind: "scroll", + point: { x: 100, y: 100, width: 1280, height: 800 }, + deltaX: 0, + deltaY: 40, + }); + await flush(); + assert.equal(waited, 1); + assert.equal(errors.length, 0); + assert.equal(calls.filter((call) => call === "mouse.wheel").length, 2); + assert.equal(calls.filter((call) => call === "input.begin").length, 2); + } finally { + queue.cancel(); + manager.disconnect(); + } +}); diff --git a/paseo-shared-browser/server/supervisor.ts b/paseo-shared-browser/server/supervisor.ts index 218a0348..a4e74259 100644 --- a/paseo-shared-browser/server/supervisor.ts +++ b/paseo-shared-browser/server/supervisor.ts @@ -3,8 +3,10 @@ import { chmod, mkdir, open, readFile, rm, writeFile } from "node:fs/promises"; import { createServer, type Socket } from "node:net"; import { homedir, tmpdir } from "node:os"; import { dirname, join } from "node:path"; -import type { BrowserFrame, BrowserInputEvent, BrowserState, Viewport } from "../shared/browser"; +import type { BrowserFrame, BrowserInputEvent, BrowserState, Viewport, DevicePresetId } from "../shared/browser"; +import { beginBrowserGestureRpc, updateBrowserGestureRpc, endBrowserGestureRpc } from "../shared/browser"; import { SessionManager } from "./browser-policy"; +import { DEFAULT_CAPTURE_QUALITY } from "../shared/capture-settings"; import { CdpUnknownOutcomeError } from "./cdp"; import { type BridgeLease, @@ -297,7 +299,7 @@ export class RuntimeSupervisor ({ + access: vi.fn(), + chmod: vi.fn(), + mkdtemp: vi.fn(), + rm: vi.fn(), + writeFile: vi.fn(), + spawn: vi.fn(), + connect: vi.fn(), +})); +vi.mock("node:fs/promises", () => fake); +vi.mock("node:child_process", () => ({ spawn: fake.spawn })); +vi.mock("node:net", () => ({ createConnection: fake.connect })); + +let child: EventEmitter & { + pid: number; + exitCode: number | null; + signalCode: string | null; + stdio: unknown[]; + kill: ReturnType; +}; +let ready: Readable; +beforeEach(() => { + vi.clearAllMocks(); + fake.access.mockResolvedValue(undefined); + fake.chmod.mockResolvedValue(undefined); + fake.mkdtemp.mockResolvedValue("/tmp/private-display-fixture"); + fake.writeFile.mockResolvedValue(undefined); + fake.rm.mockResolvedValue(undefined); + ready = new Readable({ read() {} }); + child = Object.assign(new EventEmitter(), { + pid: 123, + exitCode: null, + signalCode: null, + stdio: [null, null, null, ready], + kill: vi.fn(() => { + child.signalCode = "SIGTERM"; + child.emit("exit", null, "SIGTERM"); + return true; + }), + }); + fake.spawn.mockImplementation((_path, args) => { + queueMicrotask(() => ready.push(`${args[0].slice(1)}\n`)); + return child; + }); + fake.connect.mockImplementation(() => { + const socket = Object.assign(new EventEmitter(), { + destroy: vi.fn(), + write: vi.fn((packet: Buffer) => { + const response = Buffer.alloc(8); + response[0] = packet.readUInt16LE(8) === 16 ? 1 : 0; + response.writeUInt16LE(11, 2); + queueMicrotask(() => socket.emit("data", response)); + }), + }); + queueMicrotask(() => socket.emit("connect")); + return socket; + }); +}); + +it("keeps non-Linux and missing Xvfb on the original path without side effects", async () => { + expect(await createPrivateVirtualDisplay({ platform: "win32" })).toBeNull(); + expect(fake.access).not.toHaveBeenCalled(); + fake.access.mockRejectedValue(Object.assign(new Error(), { code: "ENOENT" })); + expect(await createPrivateVirtualDisplay({ platform: "linux" })).toBeNull(); + expect(fake.spawn).not.toHaveBeenCalled(); + expect(fake.mkdtemp).not.toHaveBeenCalled(); +}); + +it("uses an authenticated private high abstract-only display and idempotent cleanup", async () => { + const originalDisplay = process.env.DISPLAY; + const display = await createPrivateVirtualDisplay({ platform: "linux" }); + expect(display).not.toBeNull(); + const args = fake.spawn.mock.calls[0]![1] as string[]; + expect(Number(args[0]!.slice(1))).toBeGreaterThanOrEqual(20_000); + expect(args).toEqual([ + args[0], + "-screen", + "0", + "2560x2560x24", + "-nolisten", + "unix", + "-nolisten", + "tcp", + "-nolock", + "-auth", + "/tmp/private-display-fixture/authority", + "-displayfd", + "3", + ]); + expect(fake.chmod).toHaveBeenCalledWith("/tmp/private-display-fixture", 0o700); + expect(fake.writeFile.mock.calls[0]![2]).toEqual({ mode: 0o600, flag: "wx" }); + expect(fake.connect).toHaveBeenCalledTimes(2); // Authorized and anonymous denial. + expect(fake.connect.mock.calls[0]![0].path).toBe(`\0/tmp/.X11-unix/X${args[0]!.slice(1)}`); + expect(process.env.DISPLAY).toBe(originalDisplay); + display!.assertAvailable(); + await display!.stop(); + await display!.stop(); + expect(child.kill).toHaveBeenCalledTimes(1); + expect(fake.rm).toHaveBeenCalledTimes(1); + expect(() => display!.assertAvailable()).toThrow("ended"); +}); + +it("refuses a changed readiness identity before transmitting the secret", async () => { + fake.spawn.mockImplementation(() => { + queueMicrotask(() => ready.push("0\n")); + return child; + }); + await expect(createPrivateVirtualDisplay()).rejects.toThrow("identity changed"); + expect(fake.connect).not.toHaveBeenCalled(); + expect(child.kill).toHaveBeenCalledTimes(1); + expect(fake.rm).toHaveBeenCalledTimes(1); +}); + +it("bounds a never-ready child and removes its authority file", async () => { + fake.spawn.mockReturnValue(child); + await expect(createPrivateVirtualDisplay({ timeoutMs: 5 })).rejects.toThrow("not ready"); + expect(child.kill).toHaveBeenCalledTimes(1); + expect(fake.rm).toHaveBeenCalledTimes(1); +}); + +it("handles spawn failure and prevents silent headless fallback", async () => { + fake.spawn.mockImplementation(() => { + queueMicrotask(() => child.emit("error", new Error("fixture spawn failure"))); + return child; + }); + await expect(createPrivateVirtualDisplay()).rejects.toThrow("failed to start"); + expect(fake.rm).toHaveBeenCalledTimes(1); +}); + +it("refuses a display which accepts clients without the private cookie", async () => { + fake.connect.mockImplementation(() => { + const socket = Object.assign(new EventEmitter(), { + destroy: vi.fn(), + write: () => { + const response = Buffer.alloc(8); + response[0] = 1; + response.writeUInt16LE(11, 2); + queueMicrotask(() => socket.emit("data", response)); + }, + }); + queueMicrotask(() => socket.emit("connect")); + return socket; + }); + await expect(createPrivateVirtualDisplay()).rejects.toThrow("unauthenticated"); + expect(child.kill).toHaveBeenCalledTimes(1); +}); + +it("invalidates on unexpected child exit without starting a replacement", async () => { + const display = await createPrivateVirtualDisplay(); + child.exitCode = 1; + child.emit("exit", 1); + expect(() => display!.assertAvailable()).toThrow("ended"); + await display!.stop(); + expect(fake.spawn).toHaveBeenCalledTimes(1); + expect(child.kill).not.toHaveBeenCalled(); +}); + +it("encodes the display-scoped authority format and rejects malformed inputs", () => { + const cookie = Buffer.alloc(16, 7); + const authority = encodeDisplayAuthority("23067", cookie); + expect(authority.readUInt16BE(0)).toBe(65_535); + expect(authority.includes(Buffer.from("MIT-MAGIC-COOKIE-1"))).toBe(true); + expect(authority.subarray(-16)).toEqual(cookie); + expect(() => encodeDisplayAuthority("0", cookie)).toThrow(); + expect(() => encodeDisplayAuthority("23067", Buffer.alloc(2))).toThrow(); +}); diff --git a/paseo-shared-browser/server/virtual-display.ts b/paseo-shared-browser/server/virtual-display.ts new file mode 100644 index 00000000..38c36c2f --- /dev/null +++ b/paseo-shared-browser/server/virtual-display.ts @@ -0,0 +1,222 @@ +/** + * Owns one authenticated, invisible Linux X display per browser runtime. Abstract + * Unix transport avoids WSLg's protected pathname directory; TCP is never enabled. + * The display stays alive across CDP reconnects. Startup failure is definitive: + * callers must not silently relaunch a partly initialized browser headlessly. + */ +import { type ChildProcess, spawn } from "node:child_process"; +import { randomBytes, randomInt } from "node:crypto"; +import { constants } from "node:fs"; +import { access, chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createConnection } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Readable } from "node:stream"; +import { MAX_VIEWPORT } from "../shared/viewport-limits"; + +const AUTH_NAME = "MIT-MAGIC-COOKIE-1"; +const STARTUP_MS = 5_000; + +export interface PrivateVirtualDisplay { + readonly launchEnvironment: Readonly<{ DISPLAY: string; XAUTHORITY: string }>; + /** Refuses work after the display exits; it never creates a replacement display. */ + assertAvailable(): void; + /** Idempotently terminates only the owned child and removes its private auth file. */ + stop(): Promise; +} + +/** Serializes a FamilyWild Xauthority record, scoped to this display and cookie. */ +export function encodeDisplayAuthority(display: string, cookie: Buffer): Buffer { + if (!/^\d{4,5}$/.test(display) || Number(display) > 65_535 || cookie.length !== 16) { + throw new Error("Invalid private display authority"); + } + const fields = [Buffer.alloc(0), Buffer.from(display), Buffer.from(AUTH_NAME), cookie]; + const family = Buffer.alloc(2); + family.writeUInt16BE(65_535); + return Buffer.concat([ + family, + ...fields.flatMap((field) => { + const length = Buffer.alloc(2); + length.writeUInt16BE(field.length); + return [length, field]; + }), + ]); +} + +/** Performs a bounded X11 setup exchange to prove the owned secret is accepted. */ +export async function authenticateDisplay(display: string, cookie: Buffer): Promise { + return await new Promise((resolve) => { + const socket = createConnection({ path: `\0/tmp/.X11-unix/X${display}` }); + let settled = false; + let received = Buffer.alloc(0); + const finish = (ready: boolean) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + resolve(ready); + }; + const timer = setTimeout(() => finish(false), 200); + socket.once("error", () => finish(false)); + socket.once("close", () => finish(false)); + socket.once("connect", () => { + const name = Buffer.from(AUTH_NAME); + const header = Buffer.alloc(12); + header[0] = 0x6c; // X11 little-endian client, protocol 11. + header.writeUInt16LE(11, 2); + header.writeUInt16LE(name.length, 6); + header.writeUInt16LE(cookie.length, 8); + socket.write(Buffer.concat([header, name, Buffer.alloc(2), cookie])); + }); + socket.on("data", (chunk) => { + if (received.length + chunk.length > 32_768) return finish(false); + received = Buffer.concat([received, chunk]); + if (received.length >= 8) { + finish(received[0] === 1 && received.readUInt16LE(2) === 11); + } + }); + }); +} + +/** Reads only the exact child readiness FD before sending its cookie to a socket. */ +async function waitDisplayReady( + child: ChildProcess, + display: string, + timeoutMs: number, +): Promise { + await new Promise((resolve, reject) => { + const ready = child.stdio[3] as Readable; + let response = ""; + const finish = (error?: Error) => { + clearTimeout(timer); + ready.removeListener("data", onData); + ready.removeListener("error", onError); + child.removeListener("error", onError); + child.removeListener("exit", onExit); + ready.destroy(); + if (error) reject(error); + else resolve(); + }; + const onError = () => finish(new Error("Private browser display failed to start")); + const onExit = () => finish(new Error("Private browser display ended before readiness")); + const onData = (chunk: Buffer) => { + response += chunk.toString("ascii"); + if (response.length > 16) return finish(new Error("Invalid private display readiness")); + if (response.includes("\n")) { + finish( + response === `${display}\n` ? undefined : new Error("Private display identity changed"), + ); + } + }; + const timer = setTimeout( + () => finish(new Error("Private browser display was not ready")), + timeoutMs, + ); + ready.on("data", onData); + ready.once("error", onError); + child.once("error", onError); + child.once("exit", onExit); + }); +} + +/** Stops a spawned child before deleting the secret it may still be reading. */ +async function stopDisplayChild(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null || !child.pid) return; + await new Promise((resolve) => { + const timer = setTimeout(() => child.kill("SIGKILL"), 1_000); + child.once("exit", () => { + clearTimeout(timer); + resolve(); + }); + child.kill("SIGTERM"); + }); +} + +/** + * Starts Xvfb only for Linux hidden browsers when the existing executable exists. + * Absence returns null. Present-but-broken Xvfb fails closed with bounded cleanup. + * No process.env mutation, network listener, host display, or dependency install. + * One 2560x2560x24 framebuffer is bounded at about 25 MiB before server overhead; + * Chromium CPU/memory costs remain those of the existing browser runtime. + */ +export async function createPrivateVirtualDisplay( + options: { platform?: NodeJS.Platform; executablePath?: string; timeoutMs?: number } = {}, +): Promise { + if ((options.platform ?? process.platform) !== "linux") return null; + const executable = options.executablePath ?? "/usr/bin/Xvfb"; + try { + await access(executable, constants.X_OK); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw new Error("Private browser display executable is not usable"); + } + const display = String(randomInt(20_000, 60_000)); + const directory = await mkdtemp(join(tmpdir(), "paseo-browser-display-")); + const authPath = join(directory, "authority"); + const cookie = randomBytes(16); + let child: ChildProcess | null = null; + let stopPromise: Promise | null = null; + let exited = false; + const stop = () => + (stopPromise ??= (async () => { + exited = true; + try { + if (child) await stopDisplayChild(child); + } finally { + await rm(directory, { recursive: true, force: true }); + } + })()); + try { + await chmod(directory, 0o700); + await writeFile(authPath, encodeDisplayAuthority(display, cookie), { mode: 0o600, flag: "wx" }); + child = spawn( + executable, + [ + `:${display}`, + "-screen", + "0", + `${MAX_VIEWPORT.width}x${MAX_VIEWPORT.height}x24`, + "-nolisten", + "unix", + "-nolisten", + "tcp", + "-nolock", + "-auth", + authPath, + "-displayfd", + "3", + ], + { stdio: ["ignore", "ignore", "ignore", "pipe"] }, + ); + child.once("error", () => { + exited = true; + }); + child.once("exit", () => { + exited = true; + }); + // An explicit high display plus readiness FD does not invoke Xvfb's scan + // starting at :0. The child's pipe proves it bound that exact listener first. + await waitDisplayReady( + child, + display, + Math.min(STARTUP_MS, Math.max(1, options.timeoutMs ?? STARTUP_MS)), + ); + if (exited || !(await authenticateDisplay(display, cookie))) { + throw new Error("Private browser display authentication failed"); + } + if (await authenticateDisplay(display, Buffer.alloc(0))) { + throw new Error("Private browser display accepted an unauthenticated client"); + } + if (exited) throw new Error("Private browser display ended"); + return { + launchEnvironment: Object.freeze({ DISPLAY: `:${display}`, XAUTHORITY: authPath }), + assertAvailable() { + if (exited) throw new Error("Private browser display ended"); + }, + stop, + }; + } catch (error) { + await stop(); + throw error; + } +} diff --git a/paseo-shared-browser/shared/browser-display-preferences.ts b/paseo-shared-browser/shared/browser-display-preferences.ts new file mode 100644 index 00000000..ccdfa9fc --- /dev/null +++ b/paseo-shared-browser/shared/browser-display-preferences.ts @@ -0,0 +1,31 @@ +/** + * Host-persisted display favorites through Paseo's settings service. These values + * change picker convenience only, never browser input ownership or emulation. + * Retired IDs remain parseable; the current catalogue prunes them at the view. + */ +import { defineSettings } from "@getpaseo/plugin"; +import { z } from "zod"; +import { DEVICE_PRESETS } from "./device-presets"; +import { DEFAULT_CAPTURE_QUALITY } from "./capture-settings"; + +export const browserDisplayPreferencesSchema = z.object({ + captureQuality: z + .enum(["low", "medium", "high"]) + .default(DEFAULT_CAPTURE_QUALITY), + favoritePresetIds: z + .array(z.string().min(1).max(128)) + .max(DEVICE_PRESETS.length) + .default([]), +}); + +export type BrowserDisplayPreferences = z.output< + typeof browserDisplayPreferencesSchema +>; + +/** One revisioned document per plugin host, shared across that host's panels. */ +export const browserDisplayPreferences = defineSettings({ + id: "display-preferences", + scope: "host", + version: 1, + schema: browserDisplayPreferencesSchema, +}); diff --git a/paseo-shared-browser/shared/browser.ts b/paseo-shared-browser/shared/browser.ts index 74d669a8..20e7894b 100644 --- a/paseo-shared-browser/shared/browser.ts +++ b/paseo-shared-browser/shared/browser.ts @@ -1,71 +1,13 @@ import { defineRpc } from "@getpaseo/plugin"; import { z } from "zod"; +import { MIN_VIEWPORT, MAX_VIEWPORT } from "./viewport-limits"; +import { DEVICE_PRESET_IDS } from "./device-presets"; +import { DEFAULT_CAPTURE_QUALITY, FRAME_MAX_BYTES, FRAME_MAX_BASE64_CHARS } from "./capture-settings"; +export { DEFAULT_CAPTURE_QUALITY, FRAME_MAX_BYTES, FRAME_MAX_BASE64_CHARS } from "./capture-settings"; +export { DEVICE_PRESETS, DEVICE_PRESET_IDS, type DevicePresetId } from "./device-presets"; export const DEFAULT_VIEWPORT = { width: 1280, height: 800 } as const; -export const MIN_VIEWPORT = { width: 320, height: 480 } as const; -export const MAX_VIEWPORT = { width: 1600, height: 1200 } as const; -export const FRAME_MAX_BYTES = 800_000; -export const FRAME_MAX_BASE64_CHARS = Math.ceil(FRAME_MAX_BYTES / 3) * 4; - -export const DEVICE_PRESETS = [ - { - id: "desktop-chrome", - label: "Desktop Chrome", - shortLabel: "Desktop", - viewport: { width: 1280, height: 720 }, - deviceScaleFactor: 1, - isMobile: false, - hasTouch: false, - platform: "Win32", - userAgent: - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Safari/537.36", - }, - { - id: "iphone-15-pro", - label: "iPhone 15 Pro", - shortLabel: "iPhone 15", - viewport: { width: 393, height: 659 }, - deviceScaleFactor: 3, - isMobile: true, - hasTouch: true, - platform: "iPhone", - userAgent: - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1", - }, - { - id: "pixel-7", - label: "Pixel 7", - shortLabel: "Pixel 7", - viewport: { width: 412, height: 839 }, - deviceScaleFactor: 2.625, - isMobile: true, - hasTouch: true, - platform: "Linux armv81", - userAgent: - "Mozilla/5.0 (Linux; Android 14; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Mobile Safari/537.36", - }, - { - id: "ipad-pro-11", - label: "iPad Pro 11", - shortLabel: "iPad 11", - viewport: { width: 834, height: 1194 }, - deviceScaleFactor: 2, - isMobile: true, - hasTouch: true, - platform: "iPad", - userAgent: - "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1", - }, -] as const; - -export const DEVICE_PRESET_IDS = [ - "desktop-chrome", - "iphone-15-pro", - "pixel-7", - "ipad-pro-11", -] as const; - -export type DevicePresetId = (typeof DEVICE_PRESET_IDS)[number]; +export { MIN_VIEWPORT, MAX_VIEWPORT } from "./viewport-limits"; const devicePresetIdSchema = z.enum(DEVICE_PRESET_IDS); const workspaceIdSchema = z @@ -102,6 +44,7 @@ export const browserStateSchema = z.object({ canGoBack: z.boolean(), canGoForward: z.boolean(), viewport: viewportSchema, + captureScale: z.number().min(1).max(2).default(1), navigationGeneration: generationSchema, viewportGeneration: generationSchema, devicePresetId: devicePresetIdSchema.nullable(), @@ -161,7 +104,7 @@ export const captureBrowserRpc = defineRpc({ name: "shared-browser.capture", input: z.object({ viewerToken: opaqueTokenSchema, - quality: z.enum(["low", "medium", "high"]).default("medium"), + quality: z.enum(["low", "medium", "high"]).default(DEFAULT_CAPTURE_QUALITY), knownFrameId: opaqueTokenSchema.nullable().default(null), }), output: z.object({ @@ -236,6 +179,8 @@ export const applyDevicePresetRpc = defineRpc({ controlToken: opaqueTokenSchema, expected: expectedStateSchema, presetId: devicePresetIdSchema, + /** Mode-only changes retain current CSS dimensions and JPEG capture density. */ + preserveDisplay: z.boolean().optional(), }), output: z.object({ state: browserStateSchema }), }); @@ -312,6 +257,103 @@ export const sendBrowserInputRpc = defineRpc({ output: z.object({ state: browserStateSchema }), }); +/** Safe CSS cursor names only. Custom cursor URLs never reach the local client. */ +export const browserCursorSchema = z.enum([ + "default", "none", "pointer", "text", "vertical-text", "crosshair", "move", + "grab", "grabbing", "wait", "progress", "help", "not-allowed", "no-drop", + "copy", "alias", "context-menu", "cell", "all-scroll", "col-resize", "row-resize", + "n-resize", "s-resize", "e-resize", "w-resize", "ne-resize", "nw-resize", + "se-resize", "sw-resize", "ew-resize", "ns-resize", "nesw-resize", "nwse-resize", + "zoom-in", "zoom-out", +]); +export type BrowserCursor = z.output; + +const touchPointSchema = displayedPointSchema.extend({ id: z.number().int().min(0).max(2_147_483_647) }); +/** Native keyboard modifiers use CDP's bits: Alt 1, Control 2, Meta 4, Shift 8. */ +export const browserGestureKeySchema = z.object({ + kind: z.literal("key"), + type: z.enum(["down", "up"]), + key: z.string().min(1).max(64), + code: z.string().min(1).max(64).regex(/^[A-Za-z][A-Za-z0-9]*$/), + modifiers: z.number().int().min(0).max(15), + repeat: z.boolean().default(false), + text: z.string().min(1).max(32).optional(), +}).superRefine((event, context) => { + if (event.type === "up" && (event.repeat || event.text !== undefined)) { + context.addIssue({ code: "custom", message: "Key release cannot repeat or insert text" }); + } + if (event.text !== undefined && (event.modifiers & 7) !== 0) { + context.addIssue({ code: "custom", message: "Shortcut keys cannot insert printable text" }); + } +}); +export type BrowserGestureKeyEvent = z.output; + +/** + * Ordered live input, in displayed-image coordinates. Touch start/move contain + * the complete active ID set; removing contacts in move releases those contacts. + * End/cancel contain no points. Wheel deltas are bounded browser CSS pixels. + */ +export const browserGestureEventSchema = z.discriminatedUnion("kind", [ + browserGestureKeySchema, + z.object({ kind: z.literal("text"), text: z.string().min(1).max(16_000) }), + z.object({ kind: z.literal("leave") }), + z.object({ kind: z.literal("move"), point: displayedPointSchema }), + z.object({ kind: z.literal("down"), point: displayedPointSchema, + button: z.enum(["left", "right", "middle"]).default("left"), clickCount: z.union([z.literal(1), z.literal(2)]).default(1) }), + z.object({ kind: z.literal("up"), point: displayedPointSchema, + button: z.enum(["left", "right", "middle"]).default("left"), clickCount: z.union([z.literal(1), z.literal(2)]).default(1) }), + z.object({ kind: z.literal("scroll"), point: displayedPointSchema, + deltaX: z.number().finite().min(-4_000).max(4_000), deltaY: z.number().finite().min(-4_000).max(4_000) }), + z.object({ kind: z.literal("touch"), type: z.enum(["start", "move", "end", "cancel"]), + points: z.array(touchPointSchema).max(5) }).superRefine((event, context) => { + if ((event.type === "end" || event.type === "cancel") !== (event.points.length === 0)) { + context.addIssue({ code: "custom", message: "Touch start/move require points; end/cancel require none" }); + } + if (new Set(event.points.map(point => point.id)).size !== event.points.length) { + context.addIssue({ code: "custom", message: "Touch identifiers must be unique" }); + } + }), +]); +export type BrowserGestureEvent = z.output; + +const gestureContextSchema = z.object({ viewerToken: opaqueTokenSchema, controlToken: opaqueTokenSchema, + expected: expectedStateSchema.extend({ runtimeId: runtimeIdSchema, bridgeEpoch: epochSchema }) }); +const gestureContinuationSchema = gestureContextSchema.extend({ + gestureId: opaqueTokenSchema, sequence: z.number().int().positive().max(100_000), +}); +/** Begin pins a decoded recent frame but sends no physical input. One live channel per controller. Keyboard events may share either pointer channel. */ +export const beginBrowserGestureRpc = defineRpc({ + name: "shared-browser.gesture.begin", + input: gestureContextSchema.extend({ target: targetFrameSchema, pointerKind: z.enum(["mouse", "touch"]) }), + output: z.union([ + z.object({ state: browserStateSchema, gestureId: opaqueTokenSchema, nextSequence: z.number().int().positive() }), + // This receipt exists only before input.begin, so no physical action or channel is replayed. + z.object({ state: browserStateSchema, admission: z.literal("stale-frame") }), + ]), +}); +/** + * Strict sequence continues the original frame context despite this channel's + * input invalidations. Independent mouse down/initial touch start require target; + * additional contacts in a held touch gesture continue its pinned geometry. + */ +export const updateBrowserGestureRpc = defineRpc({ + name: "shared-browser.gesture.update", + input: gestureContinuationSchema.extend({ event: browserGestureEventSchema, target: targetFrameSchema.optional() }), + // The native input was acknowledged and navigated on its original attachment. + // This closes the old channel; it never authorizes a continuation or a replay. + output: z.object({ state: browserStateSchema, gestureId: opaqueTokenSchema, nextSequence: z.number().int().positive(), cursor: browserCursorSchema.nullable(), completion: z.literal("navigation").optional() }), +}); +/** + * End/cancel releases the original held input without retrying an uncertain action. + * cancel permits a stale sequence after a lost reply, but still requires the exact + * owned gesture/controller. Normal end requires the acknowledged next sequence. + */ +export const endBrowserGestureRpc = defineRpc({ + name: "shared-browser.gesture.end", + input: gestureContinuationSchema.extend({ cancel: z.boolean().default(false) }), + output: z.object({ state: browserStateSchema, cursor: browserCursorSchema.nullable() }), +}); + export type BrowserState = z.output; export type BrowserFrame = z.output; export type BrowserInputEvent = z.output; diff --git a/paseo-shared-browser/shared/capture-settings.ts b/paseo-shared-browser/shared/capture-settings.ts new file mode 100644 index 00000000..048dac4f --- /dev/null +++ b/paseo-shared-browser/shared/capture-settings.ts @@ -0,0 +1,16 @@ +/** JPEG quality and byte bounds shared by the human viewer, agent adapter and browser runtime. */ +export const DEFAULT_CAPTURE_QUALITY = "high" as const; +export const JPEG_QUALITY = { low: 70, medium: 90, high: 95 } as const; +export const DEFAULT_JPEG_QUALITY = JPEG_QUALITY[DEFAULT_CAPTURE_QUALITY]; + +// Detailed large views need enough room to retain text without exceeding bounded RPC payloads. +export const FRAME_MAX_BYTES = 4 * 1024 * 1024; +export const FRAME_MAX_BASE64_CHARS = Math.ceil(FRAME_MAX_BYTES / 3) * 4; + +/** Actual image pixels are independent of CSS layout and input coordinates for sharper phone capture. */ +export function captureDimensions(viewport: { width: number; height: number }, captureScale = 1) { + return { + width: Math.round(viewport.width * captureScale), + height: Math.round(viewport.height * captureScale), + }; +} diff --git a/paseo-shared-browser/shared/device-presets.ts b/paseo-shared-browser/shared/device-presets.ts new file mode 100644 index 00000000..49980dab --- /dev/null +++ b/paseo-shared-browser/shared/device-presets.ts @@ -0,0 +1,99 @@ +/** Device emulation facts and display sizes shared by the picker and server policy. */ +const DESKTOP_CHROME_EMULATION = { + deviceScaleFactor: 1, + captureScale: 1, + isMobile: false, + hasTouch: false, + platform: "Win32", + userAgent: + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Safari/537.36", +} as const; + +/** Larger desktop presets change resolution while retaining the same input and browser behavior. */ +function createDesktopPreset(id: Id, width: number, height: number) { + const dimensions = `${width} × ${height}`; + let ratio = "16:9"; + if (width === height) { + ratio = "1:1"; + } else if (width * 10 === height * 16) { + ratio = "16:10"; + } + + return { + ...DESKTOP_CHROME_EMULATION, + id, + label: `Desktop ${dimensions} (${ratio})`, + shortLabel: dimensions, + viewport: { width, height }, + } as const; +} + +const PIXEL_7_EMULATION = { + id: "pixel-7", + label: "Pixel 7", + shortLabel: "Pixel 7", + viewport: { width: 412, height: 839 }, + deviceScaleFactor: 2.625, + captureScale: 1, + isMobile: true, + hasTouch: true, + platform: "Linux armv81", + userAgent: + "Mozilla/5.0 (Linux; Android 14; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.8010.12 Mobile Safari/537.36", +} as const; + +export const DEVICE_PRESETS = [ + { + ...DESKTOP_CHROME_EMULATION, + id: "desktop-chrome", + label: "Desktop Chrome", + shortLabel: "Desktop", + viewport: { width: 1280, height: 720 }, + }, + createDesktopPreset("desktop-1280x800", 1280, 800), + createDesktopPreset("desktop-1280x1280", 1280, 1280), + createDesktopPreset("desktop-1920x1080", 1920, 1080), + createDesktopPreset("desktop-2560x1440", 2560, 1440), + createDesktopPreset("desktop-1920x1200", 1920, 1200), + createDesktopPreset("desktop-2560x1600", 2560, 1600), + createDesktopPreset("desktop-1920x1920", 1920, 1920), + createDesktopPreset("desktop-2560x2560", 2560, 2560), + { + id: "iphone-15-pro", + label: "iPhone 15 Pro", + shortLabel: "iPhone 15", + viewport: { width: 393, height: 659 }, + deviceScaleFactor: 3, + captureScale: 1, + isMobile: true, + hasTouch: true, + platform: "iPhone", + userAgent: + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1", + }, + PIXEL_7_EMULATION, + { + ...PIXEL_7_EMULATION, + id: "pixel-7-sharp", + label: "Pixel 7 (high resolution)", + shortLabel: "Pixel 7 HD", + captureScale: 2, + }, + { + id: "ipad-pro-11", + label: "iPad Pro 11", + shortLabel: "iPad 11", + viewport: { width: 834, height: 1194 }, + deviceScaleFactor: 2, + captureScale: 1, + isMobile: true, + hasTouch: true, + platform: "iPad", + userAgent: + "Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.6 Mobile/15E148 Safari/604.1", + }, +] as const; + +// The schema and picker derive from one catalogue, so adding a preset cannot leave it unselectable. +export const DEVICE_PRESET_IDS = DEVICE_PRESETS.map((preset) => preset.id); +export type DevicePresetId = (typeof DEVICE_PRESET_IDS)[number]; diff --git a/paseo-shared-browser/shared/resolution-menu.test.ts b/paseo-shared-browser/shared/resolution-menu.test.ts new file mode 100644 index 00000000..831d0b8d --- /dev/null +++ b/paseo-shared-browser/shared/resolution-menu.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "vitest"; +import { DEVICE_PRESETS } from "./device-presets"; +import { + groupResolutionPresets, + normalizeResolutionFavorites, + toggleResolutionFavorite, +} from "./resolution-menu"; +import { + browserDisplayPreferences, + browserDisplayPreferencesSchema, +} from "./browser-display-preferences"; + +describe("resolution choices and host preferences", () => { + it("groups the whole catalogue by viewport ratio, sorts width and retains mobile ties", () => { + const groups = groupResolutionPresets(); + expect(groups.map((group) => group.label)).toEqual([ + "16:9", + "16:10", + "1:1", + "Mobile", + ]); + expect( + groups + .flatMap((group) => group.presets) + .map((preset) => preset.id) + .sort(), + ).toEqual(DEVICE_PRESETS.map((preset) => preset.id).sort()); + for (const group of groups) { + expect(group.presets.map((preset) => preset.viewport.width)).toEqual( + group.presets + .map((preset) => preset.viewport.width) + .sort((a, b) => a - b), + ); + for (const preset of group.presets) { + if (group.id === "mobile") expect(preset.isMobile).toBe(true); + else { + expect(preset.isMobile).toBe(false); + const [width, height] = group.id.split(":").map(Number); + if (width === undefined || height === undefined) throw new Error("Invalid ratio fixture"); + expect(preset.viewport.width * height).toBe(preset.viewport.height * width); + } + } + } + expect(groups[3]?.presets.map((preset) => preset.id)).toEqual([ + "iphone-15-pro", + "pixel-7", + "pixel-7-sharp", + "ipad-pro-11", + ]); + }); + + it("prunes removed favorites and duplicates without changing stable favorite order", () => { + expect( + normalizeResolutionFavorites([ + "retired", + "pixel-7-sharp", + "desktop-chrome", + "pixel-7-sharp", + "unknown", + ]), + ).toEqual(["pixel-7-sharp", "desktop-chrome"]); + expect(normalizeResolutionFavorites(Array(100).fill("pixel-7"))).toEqual([ + "pixel-7", + ]); + expect( + normalizeResolutionFavorites( + DEVICE_PRESETS.flatMap((preset) => [preset.id, preset.id]), + ), + ).toHaveLength(DEVICE_PRESETS.length); + }); + + it("toggles only catalogue entries and does not mutate stored input", () => { + const input = Object.freeze([ + "retired", + "desktop-chrome", + "desktop-chrome", + ]); + expect(toggleResolutionFavorite(input, "pixel-7")).toEqual([ + "desktop-chrome", + "pixel-7", + ]); + expect(toggleResolutionFavorite(input, "desktop-chrome")).toEqual([]); + expect(toggleResolutionFavorite(input, "unknown")).toEqual([ + "desktop-chrome", + ]); + expect(input).toEqual(["retired", "desktop-chrome", "desktop-chrome"]); + }); + + it("defaults host favorites and quality but allows retired IDs to remain readable", () => { + expect(browserDisplayPreferences.scope).toBe("host"); + expect(browserDisplayPreferencesSchema.parse({})).toEqual({ + favoritePresetIds: [], + captureQuality: "high", + }); + expect( + browserDisplayPreferencesSchema.parse({ favoritePresetIds: ["retired"] }) + .favoritePresetIds, + ).toEqual(["retired"]); + expect( + browserDisplayPreferencesSchema.safeParse({ captureQuality: "ultra" }) + .success, + ).toBe(false); + expect( + browserDisplayPreferencesSchema.safeParse({ + favoritePresetIds: Array(DEVICE_PRESETS.length + 1).fill("pixel-7"), + }).success, + ).toBe(false); + expect( + browserDisplayPreferencesSchema.safeParse({ + favoritePresetIds: ["x".repeat(129)], + }).success, + ).toBe(false); + }); +}); diff --git a/paseo-shared-browser/shared/resolution-menu.ts b/paseo-shared-browser/shared/resolution-menu.ts new file mode 100644 index 00000000..d2a8c55c --- /dev/null +++ b/paseo-shared-browser/shared/resolution-menu.ts @@ -0,0 +1,78 @@ +/** Catalogue grouping and stable-ID favorites for every resolution picker surface. */ +import { + DEVICE_PRESETS, + DEVICE_PRESET_IDS, + type DevicePresetId, +} from "./device-presets"; + +export type ResolutionPreset = (typeof DEVICE_PRESETS)[number]; +export type ResolutionGroupId = "16:9" | "16:10" | "1:1" | "mobile"; +export interface ResolutionGroup { + id: ResolutionGroupId; + label: string; + presets: readonly ResolutionPreset[]; +} + +const GROUPS = [ + { id: "16:9", label: "16:9" }, + { id: "16:10", label: "16:10" }, + { id: "1:1", label: "1:1" }, + { id: "mobile", label: "Mobile" }, +] as const; +const KNOWN_PRESET_IDS: ReadonlySet = new Set(DEVICE_PRESET_IDS); + +/** Group by actual viewport facts, keeping equal-width presets in catalogue order. */ +export function groupResolutionPresets(): ResolutionGroup[] { + const groups: ResolutionGroup[] = GROUPS.map((group) => ({ + ...group, + presets: [], + })); + for (const preset of DEVICE_PRESETS) { + const { width, height } = preset.viewport; + let id: ResolutionGroupId; + if (preset.isMobile) id = "mobile"; + else if (width === height) id = "1:1"; + else if (width * 9 === height * 16) id = "16:9"; + else if (width * 10 === height * 16) id = "16:10"; + else + throw new Error( + `Resolution preset ${preset.id} requires an aspect-ratio group`, + ); + const group = groups.find((candidate) => candidate.id === id); + if (!group) throw new Error(`Resolution group ${id} is missing`); + group.presets = [...group.presets, preset]; + } + return groups.map((group) => ({ + ...group, + presets: [...group.presets].sort( + (a, b) => a.viewport.width - b.viewport.width, + ), + })); +} + +/** Drop removed/unknown IDs and duplicates, retaining the user's favorite order. */ +export function normalizeResolutionFavorites( + ids: readonly string[], +): DevicePresetId[] { + const favorites: DevicePresetId[] = []; + const seen = new Set(); + for (const id of ids) { + if (!KNOWN_PRESET_IDS.has(id) || seen.has(id)) continue; + seen.add(id); + favorites.push(id as DevicePresetId); + if (favorites.length === DEVICE_PRESETS.length) break; + } + return favorites; +} + +/** Toggle only a current preset. Unknown IDs cannot become persisted shortcuts. */ +export function toggleResolutionFavorite( + ids: readonly string[], + id: string, +): DevicePresetId[] { + const current = normalizeResolutionFavorites(ids); + if (!KNOWN_PRESET_IDS.has(id)) return current; + if (current.includes(id as DevicePresetId)) + return current.filter((existing) => existing !== id); + return [...current, id as DevicePresetId]; +} diff --git a/paseo-shared-browser/shared/viewport-limits.ts b/paseo-shared-browser/shared/viewport-limits.ts new file mode 100644 index 00000000..8168ae93 --- /dev/null +++ b/paseo-shared-browser/shared/viewport-limits.ts @@ -0,0 +1,3 @@ +/** Shared bounds for custom viewports, frame contracts, and the agent adapter. */ +export const MIN_VIEWPORT = { width: 320, height: 480 } as const; +export const MAX_VIEWPORT = { width: 2560, height: 2560 } as const; diff --git a/paseo-shared-browser/tests/contracts.test.ts b/paseo-shared-browser/tests/contracts.test.ts index 46e5b436..c6622cb4 100644 --- a/paseo-shared-browser/tests/contracts.test.ts +++ b/paseo-shared-browser/tests/contracts.test.ts @@ -29,6 +29,9 @@ const browserMocks = vi.hoisted(() => ({ handleReleaseControl: vi.fn(), handleResizeBrowser: vi.fn(), handleSendBrowserInput: vi.fn(), + handleBeginBrowserGesture: vi.fn(), + handleUpdateBrowserGesture: vi.fn(), + handleEndBrowserGesture: vi.fn(), handleWorkspaceArchived: vi.fn(), issueAgentTicket: vi.fn().mockResolvedValue(undefined), revokeAgentBrowserAccess: vi.fn(), @@ -52,6 +55,7 @@ type AgentCreateHook = (input: { request: AgentCreateRequest }) => Promise { if (name === "agent.create") hook = handler as AgentCreateHook; @@ -76,6 +80,7 @@ const BASE_STATE: BrowserState = { navigationGeneration: 4, viewportGeneration: 2, devicePresetId: null, + captureScale: 1, userAgent: "Chromium", controller: "none", controllerLabel: null, @@ -109,7 +114,10 @@ describe("shared RPC validation", () => { expect(viewportSchema.safeParse(MIN_VIEWPORT).success).toBe(true); expect(viewportSchema.safeParse(MAX_VIEWPORT).success).toBe(true); expect(viewportSchema.safeParse({ width: 320, height: 240 }).success).toBe(false); - expect(viewportSchema.safeParse({ width: 2560, height: 1440 }).success).toBe(false); + expect(viewportSchema.safeParse({ width: 2560, height: 1440 }).success).toBe(true); + expect(viewportSchema.safeParse({ width: 2560, height: 2560 }).success).toBe(true); + expect(viewportSchema.safeParse({ width: 2561, height: 1440 }).success).toBe(false); + expect(viewportSchema.safeParse({ width: 1280, height: 2561 }).success).toBe(false); }); it("rejects unbounded input text and scroll deltas", () => { diff --git a/paseo-shared-browser/tests/frame-lifecycle.test.ts b/paseo-shared-browser/tests/frame-lifecycle.test.ts index 0296daac..f2902948 100644 --- a/paseo-shared-browser/tests/frame-lifecycle.test.ts +++ b/paseo-shared-browser/tests/frame-lifecycle.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { FrameLifecycle } from "../client/frame-lifecycle"; +import { createFrameLifecycle } from "../client/frame-lifecycle"; import type { BrowserFrame } from "../shared/browser"; function frame(frameId: string): BrowserFrame { @@ -7,8 +7,8 @@ function frame(frameId: string): BrowserFrame { } /** A lifecycle holding an accepted, actionable frame. */ -function ready(): FrameLifecycle { - const lifecycle = new FrameLifecycle(); +function ready(): ReturnType { + const lifecycle = createFrameLifecycle(); expect(lifecycle.accept(lifecycle.epoch, frame("base"))).toBe(true); return lifecycle; } diff --git a/paseo-shared-browser/tests/lease.test.ts b/paseo-shared-browser/tests/lease.test.ts index e6a98fc9..61e472c7 100644 --- a/paseo-shared-browser/tests/lease.test.ts +++ b/paseo-shared-browser/tests/lease.test.ts @@ -221,6 +221,36 @@ describe("SessionManager control leases", () => { ); }); + it("expired viewing reattaches to the preserved page without taking another viewer's control or replaying input", async () => { + let now = 1_000; + const { manager, client } = createManager({ now: () => now }); + const returning = await manager.attach("workspace-one", "Returning client"); + const observer = await manager.attach("workspace-one", "Other client"); + now += 10_001; + await expect(manager.capture(returning.viewerToken, "medium", null)).rejects.toThrow( + "Viewer token is invalid or expired", + ); + const other = await manager.attach("workspace-one", "Other current client"); + await manager.acquireControl(other.viewerToken, false); + const previousCalls = client.operations.length; + const recovered = await manager.attach("workspace-one", "Returning client"); + expect(recovered.viewerToken).not.toBe(returning.viewerToken); + expect(recovered.state.sessionId).toBe(returning.state.sessionId); + expect(recovered.state.runtimeId).toBe(returning.state.runtimeId); + expect(recovered.state.url).toBe(returning.state.url); + expect(recovered.state.controller).toBe("other"); + expect((await manager.status(other.viewerToken)).state.controller).toBe("self"); + expect(client.workspaces.size).toBe(1); + expect(client.archiveCalls).toEqual([]); + expect(client.operations.slice(previousCalls).map((call) => call.operation)).not.toContain( + "navigate", + ); + expect(client.operations.slice(previousCalls).map((call) => call.operation)).not.toContain( + "mouse.down", + ); + expect(await manager.detach(observer.viewerToken)).toEqual({ detached: false }); + }); + it("accepts a shared recent frame and rejects it after viewport invalidation", async () => { const { manager } = createManager({ frameCacheMs: 0 }); const first = await manager.attach("workspace-one", "First client"); @@ -352,6 +382,81 @@ describe("SessionManager control leases", () => { }); }); + it("switches mobile and desktop behavior while preserving the current custom display", async () => { + const { manager, client } = createManager(); + const viewer = await manager.attach("workspace-mode", "Client"); + const control = await manager.acquireControl(viewer.viewerToken, false); + const resized = await manager.resize({ + viewerToken: viewer.viewerToken, + controlToken: control.controlToken, + expected: expected(control.state), + viewport: { width: 1920, height: 1200 }, + }); + const capture = await manager.capture(viewer.viewerToken, "high", null); + const mobile = await manager.applyDevicePreset({ + viewerToken: viewer.viewerToken, + controlToken: control.controlToken, + expected: expected(resized.state), + presetId: "pixel-7-sharp", + preserveDisplay: true, + }); + expect(mobile.state.viewport).toEqual({ width: 1920, height: 1200 }); + expect(mobile.state.captureScale).toBe(1); + expect(mobile.state.viewportGeneration).toBeGreaterThan(resized.state.viewportGeneration); + expect(mobile.state.navigationGeneration).toBe(resized.state.navigationGeneration); + expect( + client.operations.filter((op) => op.operation === "emulate").at(-1)?.input, + ).toMatchObject({ width: 1920, height: 1200, captureScale: 1, mobile: true, touch: true }); + await expect( + manager.sendInput({ + viewerToken: viewer.viewerToken, + controlToken: control.controlToken, + expected: expected(capture.state), + target: capture.frame!, + event: { kind: "key", key: "Enter" }, + }), + ).rejects.toThrow("viewport"); + const desktop = await manager.applyDevicePreset({ + viewerToken: viewer.viewerToken, + controlToken: control.controlToken, + expected: expected(mobile.state), + presetId: "desktop-chrome", + preserveDisplay: true, + }); + expect(desktop.state.viewport).toEqual({ width: 1920, height: 1200 }); + expect(desktop.state.captureScale).toBe(1); + expect( + client.operations.filter((op) => op.operation === "emulate").at(-1)?.input, + ).toMatchObject({ width: 1920, height: 1200, captureScale: 1, mobile: false, touch: false }); + expect(client.operations.some((op) => ["reload", "navigate"].includes(op.operation))).toBe( + false, + ); + }); + + it("preserves sharp capture density during a mode-only desktop switch", async () => { + const { manager, client } = createManager(); + const viewer = await manager.attach("workspace-sharp-mode", "Client"); + const control = await manager.acquireControl(viewer.viewerToken, false); + const mobile = await manager.applyDevicePreset({ + viewerToken: viewer.viewerToken, + controlToken: control.controlToken, + expected: expected(control.state), + presetId: "pixel-7-sharp", + }); + const desktop = await manager.applyDevicePreset({ + viewerToken: viewer.viewerToken, + controlToken: control.controlToken, + expected: expected(mobile.state), + presetId: "desktop-chrome", + preserveDisplay: true, + }); + expect(desktop.state.viewport).toEqual(mobile.state.viewport); + expect(desktop.state.captureScale).toBe(2); + expect( + client.operations.filter((op) => op.operation === "emulate").at(-1)?.input, + ).toMatchObject({ width: 412, height: 839, captureScale: 2, mobile: false, touch: false }); + }); + it("archives and tears down the workspace runtime", async () => { const { manager, client } = createManager(); const viewer = await manager.attach("workspace-archive", "Client"); diff --git a/paseo-shared-browser/tests/viewport-resync.test.ts b/paseo-shared-browser/tests/viewport-resync.test.ts index 6a654191..20ea2b45 100644 --- a/paseo-shared-browser/tests/viewport-resync.test.ts +++ b/paseo-shared-browser/tests/viewport-resync.test.ts @@ -26,15 +26,16 @@ function jpeg(width: number, height: number): string { ]).toString("base64"); } -function fakeRuntime(heights: number[], viewport = { width: 1280, height: 800 }) { +function fakeRuntime(heights: number[], viewport = { width: 1280, height: 800 }, captureScale = 1, dpr = 1) { const calls: string[] = []; const page = { async send(method: string) { calls.push(method); if (method === "Page.captureScreenshot") { const height = heights.shift() ?? 800; - return { data: jpeg(viewport.width, height) }; + return { data: jpeg(viewport.width * captureScale, height) }; } + if (method === "Page.getLayoutMetrics") return { cssVisualViewport: { pageX: 3, pageY: 20 } }; return {}; }, }; @@ -47,10 +48,14 @@ function fakeRuntime(heights: number[], viewport = { width: 1280, height: 800 }) }); Object.assign(runtime, { page, + emulationAppliedPage: page, connection: { isOpen: true }, - viewport: { ...viewport, deviceScaleFactor: 1, mobile: false, touch: false }, + viewport: { ...viewport, deviceScaleFactor: dpr, captureScale, mobile: false, touch: false }, }); - return { runtime, calls }; + const control = runtime as unknown as { requirePage: () => Promise }; + Object.assign(page, { connection: (runtime as any).connection }); + control.requirePage = async () => page; + return { runtime, calls, page }; } describe("jpegDimensions", () => { @@ -83,3 +88,34 @@ describe("fallback frame viewport resync", () => { expect(calls.filter((call) => call === "Page.captureScreenshot")).toHaveLength(2); }); }); + + +it("recovers to captureScale pixels independently of mobile DPR", async () => { + const { runtime, calls } = fakeRuntime([600, 1600], { width: 1280, height: 800 }, 2, 3); + expect(await runtime.frame(1000000, 65, 1)).toMatchObject({ width: 2560, height: 1600 }); + expect(calls.filter((method) => method === "Page.captureScreenshot")).toHaveLength(2); +}); + +it("rejects malformed JPEG without using it to trigger viewport recovery", async () => { + const { runtime, calls, page } = fakeRuntime([800]); + const original = page.send; + page.send = async (method: string) => method === "Page.captureScreenshot" + ? { data: Buffer.from("not a jpeg").toString("base64") } as any + : original(method); + await expect(runtime.frame(1000000, 65, 1)).rejects.toThrow("malformed JPEG"); + expect(calls).not.toContain("Emulation.setDeviceMetricsOverride"); +}); + +it("drops viewport recovery after a mutation supersedes capture", async () => { + const { runtime, calls, page } = fakeRuntime([633, 800]); + const original = page.send; + page.send = async (method: string) => { + const result = await original(method); + if (method === "Emulation.setDeviceMetricsOverride") { + (runtime as any).invalidateScreencastFrame(); + } + return result; + }; + await expect(runtime.frame(1000000, 65, 1)).rejects.toThrow("invalidated"); + expect(calls.filter((method) => method === "Page.captureScreenshot")).toHaveLength(1); +}); From 99155310aa0eb8bee2ef29969bf079104e7fab52 Mon Sep 17 00:00:00 2001 From: Ryan Ewen Date: Sun, 4 Oct 2026 11:59:18 -0400 Subject: [PATCH 2/5] chore(paseo-shared-browser): prepare interaction changes for upstream review Apply pinned formatting, type the viewport fixture, align the smoke with default capture quality, and document isolated validation and the supported SDK. --- paseo-shared-browser/README.md | 15 +- .../client/browser-canvas-input.test.ts | 10 +- .../client/browser-canvas-input.ts | 34 +-- .../client/browser-canvas-keyboard-web.ts | 20 +- .../client/browser-canvas-keyboard.test.ts | 31 +- .../client/browser-canvas-keyboard.ts | 6 +- .../browser-canvas-viewport-web.test.ts | 74 ++++- .../client/browser-canvas-viewport-web.ts | 7 +- .../client/browser-capture-cadence.test.ts | 1 + .../client/browser-capture-cadence.ts | 5 +- .../client/browser-frame-buffer.test.ts | 6 +- .../client/browser-image-load.test.ts | 4 +- .../client/browser-image-load.ts | 10 +- .../client/browser-input-queue.test.ts | 286 +++++++++++++----- .../client/browser-input-queue.ts | 147 ++++----- .../client/browser-menu-placement.test.ts | 47 ++- .../client/browser-menu-placement.ts | 2 +- .../client/browser-native-keyboard.test.ts | 2 +- .../client/browser-toolbar-menu-web.test.ts | 12 +- .../client/browser-toolbar-menu-web.ts | 21 +- .../client/browser-toolbar-menu.tsx | 40 ++- .../client/browser-toolbar-submenu.tsx | 2 +- .../client/browser-viewer-recovery.test.ts | 13 +- .../client/browser-viewer-recovery.ts | 12 +- paseo-shared-browser/client/browser.tsx | 51 ++-- .../client/use-browser-canvas-input.ts | 106 +++---- .../client/use-browser-frame-buffer.ts | 2 +- .../client/use-resolution-favorites.ts | 29 +- paseo-shared-browser/client/web.ts | 42 +-- .../docs/fork-reconciliation.md | 231 ++++---------- paseo-shared-browser/index.server.ts | 8 +- paseo-shared-browser/paseo-plugin.json | 14 +- .../server/agent-browser-runtime.ts | 233 ++++++++++---- .../server/browser-gesture.ts | 4 +- paseo-shared-browser/server/browser-policy.ts | 241 +++++++++++---- paseo-shared-browser/server/browser.ts | 12 +- .../server/capture-transport-policy.test.ts | 2 +- .../server/capture-transport-runtime.test.ts | 11 +- .../server/emulation-restoration.test.ts | 2 +- .../server/input-generation.test.ts | 24 +- .../server/input-generation.ts | 12 +- paseo-shared-browser/server/jpeg-frame.ts | 4 +- .../server/keyboard-input.test.ts | 50 ++- paseo-shared-browser/server/keyboard-input.ts | 54 +++- .../server/live-input-policy.test.ts | 182 ++++++++--- .../server/live-input-runtime.test.ts | 114 +++++-- .../server/live-input-supervisor.test.ts | 17 +- paseo-shared-browser/server/mcp-entry.ts | 2 +- .../server/runtime-frame-reuse.test.ts | 2 +- .../server/runtime-launch-environment.test.ts | 7 +- .../server/stale-wheel-admission.test.ts | 4 +- paseo-shared-browser/server/supervisor.ts | 24 +- .../shared/browser-display-preferences.ts | 15 +- paseo-shared-browser/shared/browser.ts | 177 ++++++++--- .../shared/resolution-menu.test.ts | 52 +--- .../shared/resolution-menu.ts | 27 +- .../tests/browser.smoke.test.ts | 19 +- .../tests/viewport-resync.test.ts | 26 +- 58 files changed, 1605 insertions(+), 1002 deletions(-) diff --git a/paseo-shared-browser/README.md b/paseo-shared-browser/README.md index fbe4d09a..82802395 100644 --- a/paseo-shared-browser/README.md +++ b/paseo-shared-browser/README.md @@ -104,8 +104,7 @@ The plugin automatically injects its stdio MCP adapter only when a new, non-inte created with a provider that accepts external MCP servers. Agents that already exist, resumed sessions, imported sessions, and Paseo's internal agents are not modified. Paseo's built-in OMP provider accepts session MCP servers from Paseo 0.11, so new OMP agents receive the adapter there. -On Paseo 0.9 and 0.10 the built-in OMP adapter rejects external MCP servers, so OMP agents are left -unchanged. Pi agents continue to receive the adapter, but they require Pi's optional MCP support to +Pi agents also receive the adapter, but require Pi's optional MCP support to launch it. The injected MCP server exposes exactly these tools: `shared_browser_status`, @@ -253,13 +252,17 @@ User-supplied `AGENT_BROWSER_*` variables are deliberately ignored. ```bash bun install bun run typecheck -bun run lint -bun run format:check +bun run check bun run test:unit -bun run prepare:runtime +PASEO_HOME="$(mktemp -d)" bun run prepare:runtime ``` -`bun run test:smoke` launches the configured real Chromium runtime and exercises two viewers, +Treat the default Paseo daemon and profile as live user state. Use an isolated +`PASEO_HOME` for runtime preparation and tests; do not run development lifecycle +commands against the default daemon. Keep the prepared home available for smoke +tests so they can resolve its runtime assets. + +`PASEO_HOME= bun run test:smoke` launches the configured real Chromium runtime and exercises two viewers, control handoff, reconnect, stale-frame rejection, viewport changes, device emulation, profile persistence, and archive teardown. diff --git a/paseo-shared-browser/client/browser-canvas-input.test.ts b/paseo-shared-browser/client/browser-canvas-input.test.ts index 3fafc9f9..b6476055 100644 --- a/paseo-shared-browser/client/browser-canvas-input.test.ts +++ b/paseo-shared-browser/client/browser-canvas-input.test.ts @@ -59,11 +59,7 @@ describe("natural canvas input", () => { f.input.mouseMove({ ...point, x: 60 }); f.input.mouseMove({ ...point, x: 100 }); expect(f.ended()).toBe(0); - expect(f.events.map((event) => event.kind)).toEqual([ - "down", - "move", - "move", - ]); + expect(f.events.map((event) => event.kind)).toEqual(["down", "move", "move"]); f.input.mouseUp({ ...point, x: 100 }, "left", 1); expect(f.ended()).toBe(1); }); @@ -73,9 +69,7 @@ describe("natural canvas input", () => { const first = { ...point, id: 7 }; const second = { ...point, x: 200, id: 9 }; f.input.touch("start", [first]); - expect(f.events).toEqual([ - { kind: "touch", type: "start", points: [first] }, - ]); + expect(f.events).toEqual([{ kind: "touch", type: "start", points: [first] }]); expect(f.ended()).toBe(0); f.input.touch("start", [first, second]); f.input.touch("move", [ diff --git a/paseo-shared-browser/client/browser-canvas-input.ts b/paseo-shared-browser/client/browser-canvas-input.ts index 11101ec9..e396dd85 100644 --- a/paseo-shared-browser/client/browser-canvas-input.ts +++ b/paseo-shared-browser/client/browser-canvas-input.ts @@ -5,10 +5,7 @@ * click counts come from physical events; surrounding Paseo chrome is untouched. */ import type { BrowserGestureEvent } from "../shared/browser"; -import type { - BrowserGesturePoint, - BrowserTouchPoint, -} from "./browser-input-queue"; +import type { BrowserGesturePoint, BrowserTouchPoint } from "./browser-input-queue"; export interface CanvasInputOptions { enabled(): boolean; @@ -31,16 +28,9 @@ function scroll( let remainingX = deltaX; let remainingY = deltaY; while (Math.abs(remainingX) > 0.001 || Math.abs(remainingY) > 0.001) { - const x = Math.max( - -MAX_SCROLL_DELTA, - Math.min(MAX_SCROLL_DELTA, remainingX), - ); - const y = Math.max( - -MAX_SCROLL_DELTA, - Math.min(MAX_SCROLL_DELTA, remainingY), - ); - if (!options.enqueue({ kind: "scroll", point, deltaX: x, deltaY: y })) - return; + const x = Math.max(-MAX_SCROLL_DELTA, Math.min(MAX_SCROLL_DELTA, remainingX)); + const y = Math.max(-MAX_SCROLL_DELTA, Math.min(MAX_SCROLL_DELTA, remainingY)); + if (!options.enqueue({ kind: "scroll", point, deltaX: x, deltaY: y })) return; remainingX -= x; remainingY -= y; } @@ -104,11 +94,7 @@ export function createBrowserCanvasInput(options: CanvasInputOptions) { } return accepted; }; - const wheel = ( - point: BrowserGesturePoint, - deltaX: number, - deltaY: number, - ) => { + const wheel = (point: BrowserGesturePoint, deltaX: number, deltaY: number) => { if (!options.enabled()) return false; const viewport = options.viewport(); if (!viewport) return false; @@ -123,10 +109,7 @@ export function createBrowserCanvasInput(options: CanvasInputOptions) { return true; }; - const touch = ( - phase: "start" | "move" | "end" | "cancel", - points: BrowserTouchPoint[], - ) => { + const touch = (phase: "start" | "move" | "end" | "cancel", points: BrowserTouchPoint[]) => { if (phase === "cancel") { cancel(); // Native touchCancel terminates the physical contact session itself. @@ -143,10 +126,7 @@ export function createBrowserCanvasInput(options: CanvasInputOptions) { cancel(); return false; } - if ( - points.length > 5 || - new Set(points.map((point) => point.id)).size !== points.length - ) { + if (points.length > 5 || new Set(points.map((point) => point.id)).size !== points.length) { cancel(); return false; } diff --git a/paseo-shared-browser/client/browser-canvas-keyboard-web.ts b/paseo-shared-browser/client/browser-canvas-keyboard-web.ts index ef2d6340..8ad9f8ab 100644 --- a/paseo-shared-browser/client/browser-canvas-keyboard-web.ts +++ b/paseo-shared-browser/client/browser-canvas-keyboard-web.ts @@ -5,9 +5,9 @@ * Browser-reserved shortcuts are outside JavaScript's interception guarantees. */ import { - createBrowserCanvasKeyboard, - type CanvasKeyEnvelope, type CanvasKeyboardEvent, + type CanvasKeyEnvelope, + createBrowserCanvasKeyboard, } from "./browser-canvas-keyboard"; interface KeyboardDomEvent extends CanvasKeyEnvelope { @@ -114,10 +114,10 @@ export function bindBrowserCanvasKeyboard(node: KeyboardCanvasNode, options: Bin } // Local paste supplies clipboard bytes separately. Sending Ctrl/Meta+V would // additionally paste an unrelated remote clipboard and duplicate the action. - const pasteShortcut = !event.altKey && ( - ((event.ctrlKey || event.metaKey) && event.code === "KeyV") || - (event.shiftKey && !event.ctrlKey && !event.metaKey && event.code === "Insert") - ); + const pasteShortcut = + !event.altKey && + (((event.ctrlKey || event.metaKey) && event.code === "KeyV") || + (event.shiftKey && !event.ctrlKey && !event.metaKey && event.code === "Insert")); if (pasteShortcut) { if (options.enabled()) { event.stopPropagation(); @@ -125,8 +125,12 @@ export function bindBrowserCanvasKeyboard(node: KeyboardCanvasNode, options: Bin return; } if (keyboard.key("down", event)) { - const altPrintable = event.altKey && !event.ctrlKey && !event.metaKey && - !event.getModifierState?.("AltGraph") && Array.from(event.key).length === 1; + const altPrintable = + event.altKey && + !event.ctrlKey && + !event.metaKey && + !event.getModifierState?.("AltGraph") && + Array.from(event.key).length === 1; altTextPending = altPrintable ? event.key : null; if (altTextPending) { // Option layouts can produce text with Alt held. Only the browser's diff --git a/paseo-shared-browser/client/browser-canvas-keyboard.test.ts b/paseo-shared-browser/client/browser-canvas-keyboard.test.ts index 3f71fd76..e0950765 100644 --- a/paseo-shared-browser/client/browser-canvas-keyboard.test.ts +++ b/paseo-shared-browser/client/browser-canvas-keyboard.test.ts @@ -1,9 +1,9 @@ import { describe, expect, it } from "vitest"; import { - canvasKeyModifiers, - createBrowserCanvasKeyboard, type CanvasKeyboardEvent, type CanvasKeyEnvelope, + canvasKeyModifiers, + createBrowserCanvasKeyboard, } from "./browser-canvas-keyboard"; const envelope = (overrides: Partial = {}): CanvasKeyEnvelope => ({ @@ -45,16 +45,31 @@ function fixture() { describe("physical canvas keyboard", () => { it("commits qualified AltGraph text once but never converts Ctrl+Alt shortcuts", () => { const f = fixture(); - const euro = envelope({ key: "€", code: "KeyE", ctrlKey: true, altKey: true, getModifierState: value => value === "AltGraph" }); + const euro = envelope({ + key: "€", + code: "KeyE", + ctrlKey: true, + altKey: true, + getModifierState: (value) => value === "AltGraph", + }); f.input.key("down", euro); f.input.key("up", euro); f.input.key("down", envelope({ key: "e", code: "KeyE", ctrlKey: true, altKey: true })); - expect(f.events.filter(event => event.kind === "text")).toEqual([{ kind: "text", text: "€" }]); - expect(f.events.filter(event => event.kind === "key")).toHaveLength(3); - expect(f.events.filter(event => event.kind === "key").every(event => event.text === undefined)).toBe(true); + expect(f.events.filter((event) => event.kind === "text")).toEqual([ + { kind: "text", text: "€" }, + ]); + expect(f.events.filter((event) => event.kind === "key")).toHaveLength(3); + expect( + f.events.filter((event) => event.kind === "key").every((event) => event.text === undefined), + ).toBe(true); const separate = fixture(); - separate.input.key("down", envelope({ key: "€", getModifierState: value => value === "AltGraph" })); - expect(separate.events).toEqual([{ kind: "key", type: "down", key: "€", code: "KeyA", modifiers: 0, repeat: false, text: "€" }]); + separate.input.key( + "down", + envelope({ key: "€", getModifierState: (value) => value === "AltGraph" }), + ); + expect(separate.events).toEqual([ + { kind: "key", type: "down", key: "€", code: "KeyA", modifiers: 0, repeat: false, text: "€" }, + ]); }); it("preserves Unicode text, physical repeat/code and all modifier bits", () => { const f = fixture(); diff --git a/paseo-shared-browser/client/browser-canvas-keyboard.ts b/paseo-shared-browser/client/browser-canvas-keyboard.ts index 8b1f3148..d4d05d4a 100644 --- a/paseo-shared-browser/client/browser-canvas-keyboard.ts +++ b/paseo-shared-browser/client/browser-canvas-keyboard.ts @@ -99,7 +99,11 @@ export function createBrowserCanvasKeyboard(options: KeyboardOptions) { admitted.add(id); // AltGraph can report Ctrl+Alt on international layouts. The real host // qualifier permits one text commit without rewriting a genuine shortcut. - if (event.getModifierState?.("AltGraph") && (modifiers & 7) !== 0 && Array.from(event.key).length === 1) { + if ( + event.getModifierState?.("AltGraph") && + (modifiers & 7) !== 0 && + Array.from(event.key).length === 1 + ) { options.enqueue({ kind: "text", text: event.key }); } } diff --git a/paseo-shared-browser/client/browser-canvas-viewport-web.test.ts b/paseo-shared-browser/client/browser-canvas-viewport-web.test.ts index 9f286490..d51d1f69 100644 --- a/paseo-shared-browser/client/browser-canvas-viewport-web.test.ts +++ b/paseo-shared-browser/client/browser-canvas-viewport-web.test.ts @@ -3,33 +3,75 @@ import { configureBrowserCanvasWebScrolling } from "./browser-canvas-viewport-we describe("local web browser viewport overflow", () => { it("enables both overflow axes on the same public scroll node, preserving Actual offsets", () => { - const node = {style:{overflowX:"hidden",overflowY:"auto"},scrollLeft:300,scrollTop:150}; - configureBrowserCanvasWebScrolling({getScrollableNode:()=>node},"actual",true); - expect(node).toEqual({style:{overflowX:"auto",overflowY:"auto"},scrollLeft:300,scrollTop:150}); + const node = { + style: { overflowX: "hidden", overflowY: "auto" }, + scrollLeft: 300, + scrollTop: 150, + }; + configureBrowserCanvasWebScrolling({ getScrollableNode: () => node }, "actual", true); + expect(node).toEqual({ + style: { overflowX: "auto", overflowY: "auto" }, + scrollLeft: 300, + scrollTop: 150, + }); }); it("hides overflow and resets both local offsets when returning to Fit", () => { - const node = {style:{overflowX:"auto",overflowY:"auto"},scrollLeft:300,scrollTop:150}; - configureBrowserCanvasWebScrolling({getScrollableNode:()=>node},"fit",true); - expect(node).toEqual({style:{overflowX:"hidden",overflowY:"hidden"},scrollLeft:0,scrollTop:0}); + const node = { + style: { overflowX: "auto", overflowY: "auto" }, + scrollLeft: 300, + scrollTop: 150, + }; + configureBrowserCanvasWebScrolling({ getScrollableNode: () => node }, "fit", true); + expect(node).toEqual({ + style: { overflowX: "hidden", overflowY: "hidden" }, + scrollLeft: 0, + scrollTop: 0, + }); }); it("retains an Actual observation position when local panning is disabled", () => { - const node = {style:{overflowX:"auto",overflowY:"auto"},scrollLeft:300,scrollTop:150}; - configureBrowserCanvasWebScrolling({getScrollableNode:()=>node},"actual",false); - expect(node).toEqual({style:{overflowX:"hidden",overflowY:"hidden"},scrollLeft:300,scrollTop:150}); + const node = { + style: { overflowX: "auto", overflowY: "auto" }, + scrollLeft: 300, + scrollTop: 150, + }; + configureBrowserCanvasWebScrolling({ getScrollableNode: () => node }, "actual", false); + expect(node).toEqual({ + style: { overflowX: "hidden", overflowY: "hidden" }, + scrollLeft: 300, + scrollTop: 150, + }); }); it("uses the documented native host fallback while preserving the handle receiver", () => { - const node = {style:{overflowX:"",overflowY:""},scrollLeft:0,scrollTop:0}; - const handle = {node,getScrollableNode:()=>null,getNativeScrollRef(){return this.node;}}; - configureBrowserCanvasWebScrolling(handle,"actual",true); - expect(node.style).toEqual({overflowX:"auto",overflowY:"auto"}); + const node = { style: { overflowX: "", overflowY: "" }, scrollLeft: 0, scrollTop: 0 }; + const handle = { + node, + getScrollableNode: () => null, + getNativeScrollRef() { + return this.node; + }, + }; + configureBrowserCanvasWebScrolling(handle, "actual", true); + expect(node.style).toEqual({ overflowX: "auto", overflowY: "auto" }); }); it("ignores missing, disappearing and non-DOM platform refs without changing unrelated nodes", () => { - expect(()=>configureBrowserCanvasWebScrolling(null,"actual",true)).not.toThrow(); - expect(()=>configureBrowserCanvasWebScrolling({getScrollableNode:()=>42},"actual",true)).not.toThrow(); - expect(()=>configureBrowserCanvasWebScrolling({getScrollableNode:()=>{throw new Error("Detached");}},"actual",true)).not.toThrow(); + expect(() => configureBrowserCanvasWebScrolling(null, "actual", true)).not.toThrow(); + expect(() => + configureBrowserCanvasWebScrolling({ getScrollableNode: () => 42 }, "actual", true), + ).not.toThrow(); + expect(() => + configureBrowserCanvasWebScrolling( + { + getScrollableNode: () => { + throw new Error("Detached"); + }, + }, + "actual", + true, + ), + ).not.toThrow(); }); }); diff --git a/paseo-shared-browser/client/browser-canvas-viewport-web.ts b/paseo-shared-browser/client/browser-canvas-viewport-web.ts index 8c74d98e..c3547458 100644 --- a/paseo-shared-browser/client/browser-canvas-viewport-web.ts +++ b/paseo-shared-browser/client/browser-canvas-viewport-web.ts @@ -47,7 +47,12 @@ function readScrollNode(handle: BrowserCanvasScrollHandle): WebScrollNode | null const candidate: unknown = read.call(handle); if (!candidate || typeof candidate !== "object") continue; const node = candidate as Partial; - if (!node.style || typeof node.style.overflowX !== "string" || typeof node.style.overflowY !== "string") continue; + if ( + !node.style || + typeof node.style.overflowX !== "string" || + typeof node.style.overflowY !== "string" + ) + continue; if (typeof node.scrollLeft !== "number" || typeof node.scrollTop !== "number") continue; return node as WebScrollNode; } catch { diff --git a/paseo-shared-browser/client/browser-capture-cadence.test.ts b/paseo-shared-browser/client/browser-capture-cadence.test.ts index 628723ce..9a964ad1 100644 --- a/paseo-shared-browser/client/browser-capture-cadence.test.ts +++ b/paseo-shared-browser/client/browser-capture-cadence.test.ts @@ -1,5 +1,6 @@ import { expect, it } from "vitest"; import { browserCaptureInterval } from "./browser-capture-cadence"; + it("keeps idle/hover polling bounded and only accelerates actual ready input", () => { expect(browserCaptureInterval("ready", false)).toBe(250); expect(browserCaptureInterval("ready", true)).toBe(100); diff --git a/paseo-shared-browser/client/browser-capture-cadence.ts b/paseo-shared-browser/client/browser-capture-cadence.ts index 8403c697..7722fbaa 100644 --- a/paseo-shared-browser/client/browser-capture-cadence.ts +++ b/paseo-shared-browser/client/browser-capture-cadence.ts @@ -1,8 +1,5 @@ /** Single-flight capture cadence; active input improves playback without changing image quality. */ -export function browserCaptureInterval( - status: string | undefined, - activeInput: boolean, -): number { +export function browserCaptureInterval(status: string | undefined, activeInput: boolean): number { if (status !== "ready") return 1_500; if (activeInput) return 100; return 250; diff --git a/paseo-shared-browser/client/browser-frame-buffer.test.ts b/paseo-shared-browser/client/browser-frame-buffer.test.ts index d6987c6f..cc084fe5 100644 --- a/paseo-shared-browser/client/browser-frame-buffer.test.ts +++ b/paseo-shared-browser/client/browser-frame-buffer.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { createBrowserFrameBuffer, type BrowserFrameBuffer, type FrameCandidate } from "./browser-frame-buffer"; +import { + type BrowserFrameBuffer, + createBrowserFrameBuffer, + type FrameCandidate, +} from "./browser-frame-buffer"; function candidate(id: string, pixels = id, generation = 0): FrameCandidate { return { diff --git a/paseo-shared-browser/client/browser-image-load.test.ts b/paseo-shared-browser/client/browser-image-load.test.ts index c1138cb2..d4f6ebd6 100644 --- a/paseo-shared-browser/client/browser-image-load.test.ts +++ b/paseo-shared-browser/client/browser-image-load.test.ts @@ -4,7 +4,9 @@ import { createBrowserImageLoadHandler } from "./browser-image-load"; describe("cross-platform browser image decode", () => { it("accepts the desktop DOM load envelope without inventing a reported URI", () => { const delivered: unknown[] = []; - const handler = createBrowserImageLoadHandler(1, "data:image/jpeg;base64,frame", (...args) => delivered.push(args)); + const handler = createBrowserImageLoadHandler(1, "data:image/jpeg;base64,frame", (...args) => + delivered.push(args), + ); expect(() => handler({ nativeEvent: new Event("load") })).not.toThrow(); expect(delivered).toEqual([[1, true]]); }); diff --git a/paseo-shared-browser/client/browser-image-load.ts b/paseo-shared-browser/client/browser-image-load.ts index 3df64277..a7b39575 100644 --- a/paseo-shared-browser/client/browser-image-load.ts +++ b/paseo-shared-browser/client/browser-image-load.ts @@ -13,10 +13,12 @@ export function createBrowserImageLoadHandler( settle: (ticket: number, succeeded: boolean) => unknown, ): (event: unknown) => void { return (event) => { - const nativeEvent = event && typeof event === "object" && "nativeEvent" in event - ? event.nativeEvent : undefined; - const source = nativeEvent && typeof nativeEvent === "object" && "source" in nativeEvent - ? nativeEvent.source : undefined; + const nativeEvent = + event && typeof event === "object" && "nativeEvent" in event ? event.nativeEvent : undefined; + const source = + nativeEvent && typeof nativeEvent === "object" && "source" in nativeEvent + ? nativeEvent.source + : undefined; const uri = source && typeof source === "object" && "uri" in source ? source.uri : undefined; if (uri && uri !== expectedUri) return; settle(ticket, true); diff --git a/paseo-shared-browser/client/browser-input-queue.test.ts b/paseo-shared-browser/client/browser-input-queue.test.ts index fd0487f0..604ff190 100644 --- a/paseo-shared-browser/client/browser-input-queue.test.ts +++ b/paseo-shared-browser/client/browser-input-queue.test.ts @@ -1,10 +1,10 @@ import { describe, expect, it, vi } from "vitest"; +import type { BrowserGestureEvent, BrowserState } from "../shared/browser"; import { - createBrowserInputQueue, type BrowserGestureAuthority, type BrowserGestureTransport, + createBrowserInputQueue, } from "./browser-input-queue"; -import type { BrowserGestureEvent, BrowserState } from "../shared/browser"; const authority: BrowserGestureAuthority = { viewerToken: "viewer", @@ -53,8 +53,7 @@ function fixture(waitForFrame?: (afterFrameId: string, maxWaitMs?: number) => Pr const states: BrowserState[] = []; let projectState: ((state: BrowserState) => void) | null = null; let finished = 0; - const beginGate = - deferred>>(); + const beginGate = deferred>>(); const transport: BrowserGestureTransport = { begin: async () => await beginGate.promise, update: async (input) => { @@ -74,10 +73,15 @@ function fixture(waitForFrame?: (afterFrameId: string, maxWaitMs?: number) => Pr const queue = createBrowserInputQueue({ transport, authority: () => current, - onState: next => { states.push(next); projectState?.(next); }, + onState: (next) => { + states.push(next); + projectState?.(next); + }, onCursor: (cursor) => cursors.push(cursor), onError: (error) => errors.push(error), - onFinish: () => { finished += 1; }, + onFinish: () => { + finished += 1; + }, onNavigationComplete: () => navigationCompleted.push(true), ...(waitForFrame ? { waitForFrame } : {}), }); @@ -92,7 +96,9 @@ function fixture(waitForFrame?: (afterFrameId: string, maxWaitMs?: number) => Pr navigationCompleted, states, finishes: () => finished, - projectState(callback: (state: BrowserState) => void) { projectState = callback; }, + projectState(callback: (state: BrowserState) => void) { + projectState = callback; + }, changeAuthority(next: BrowserGestureAuthority | null) { current = next; }, @@ -103,17 +109,44 @@ describe("bounded browser input queue", () => { it("completes acknowledged Enter navigation before state projection revokes the decoded frame", async () => { const f = fixture(); const next: BrowserState = { - ...state, status: "ready", controller: "self", sessionId: "session", runtimeId: "runtime", - bridgeEpoch: 1, viewportGeneration: 1, navigationGeneration: 2, + ...state, + status: "ready", + controller: "self", + sessionId: "session", + runtimeId: "runtime", + bridgeEpoch: 1, + viewportGeneration: 1, + navigationGeneration: 2, }; - f.projectState(reply => { if (reply.navigationGeneration === 2) f.changeAuthority(null); }); - f.transport.update = async input => { + f.projectState((reply) => { + if (reply.navigationGeneration === 2) f.changeAuthority(null); + }); + f.transport.update = async (input) => { f.sent.push(input); - return { state: next, gestureId: input.gestureId, nextSequence: input.sequence + 1, - cursor: null, completion: "navigation" }; + return { + state: next, + gestureId: input.gestureId, + nextSequence: input.sequence + 1, + cursor: null, + completion: "navigation", + }; }; - f.queue.enqueue({ kind: "key", type: "down", key: "Enter", code: "Enter", modifiers: 0, repeat: false }); - f.queue.enqueue({ kind: "key", type: "up", key: "Enter", code: "Enter", modifiers: 0, repeat: false }); + f.queue.enqueue({ + kind: "key", + type: "down", + key: "Enter", + code: "Enter", + modifiers: 0, + repeat: false, + }); + f.queue.enqueue({ + kind: "key", + type: "up", + key: "Enter", + code: "Enter", + modifiers: 0, + repeat: false, + }); f.queue.enqueue({ kind: "text", text: "never type on the destination" }); f.queue.finish(); f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); @@ -128,23 +161,43 @@ describe("bounded browser input queue", () => { it("keeps the acknowledged click release but drops later old-page wheel and end", async () => { const f = fixture(); - f.transport.update = async input => { + f.transport.update = async (input) => { f.sent.push(input); if (input.event.kind !== "up") { - return { state, gestureId: input.gestureId, nextSequence: input.sequence + 1, cursor: null }; + return { + state, + gestureId: input.gestureId, + nextSequence: input.sequence + 1, + cursor: null, + }; } - return { state: { ...state, status: "ready", controller: "self", sessionId: "session", runtimeId: "runtime", - bridgeEpoch: 1, viewportGeneration: 1, navigationGeneration: 2 }, - gestureId: input.gestureId, nextSequence: input.sequence + 1, cursor: null, completion: "navigation" }; + return { + state: { + ...state, + status: "ready", + controller: "self", + sessionId: "session", + runtimeId: "runtime", + bridgeEpoch: 1, + viewportGeneration: 1, + navigationGeneration: 2, + }, + gestureId: input.gestureId, + nextSequence: input.sequence + 1, + cursor: null, + completion: "navigation", + }; }; - f.projectState(reply => { if (reply.navigationGeneration === 2) f.changeAuthority(null); }); + f.projectState((reply) => { + if (reply.navigationGeneration === 2) f.changeAuthority(null); + }); f.queue.enqueue({ kind: "down", point: point(1), button: "left", clickCount: 1 }); f.queue.enqueue({ kind: "up", point: point(1), button: "left", clickCount: 1 }); f.queue.enqueue({ kind: "scroll", point: point(1), deltaX: 0, deltaY: 10 }); f.queue.finish(); f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); await flush(); - expect(f.sent.map(input => input.event.kind)).toEqual(["down", "up"]); + expect(f.sent.map((input) => input.event.kind)).toEqual(["down", "up"]); expect(f.errors).toEqual([]); expect(f.navigationCompleted).toEqual([true]); expect(f.ended).toEqual([]); @@ -152,21 +205,37 @@ describe("bounded browser input queue", () => { it("does not grant navigation completion for changed runtime, bridge, viewport, controller or sequence", async () => { const next: BrowserState = { - ...state, status: "ready", controller: "self", sessionId: "session", runtimeId: "runtime", - bridgeEpoch: 1, viewportGeneration: 1, navigationGeneration: 2, + ...state, + status: "ready", + controller: "self", + sessionId: "session", + runtimeId: "runtime", + bridgeEpoch: 1, + viewportGeneration: 1, + navigationGeneration: 2, }; for (const altered of [ - { sessionId: "another-session" }, { runtimeId: "another-runtime" }, { bridgeEpoch: 2 }, - { viewportGeneration: 2 }, { controller: "other" as const }, { navigationGeneration: 1 }, + { sessionId: "another-session" }, + { runtimeId: "another-runtime" }, + { bridgeEpoch: 2 }, + { viewportGeneration: 2 }, + { controller: "other" as const }, + { navigationGeneration: 1 }, { status: "error" as const }, ]) { const f = fixture(); - f.transport.update = async input => { + f.transport.update = async (input) => { f.sent.push(input); - return { state: { ...next, ...altered }, gestureId: input.gestureId, - nextSequence: input.sequence + 1, cursor: null, completion: "navigation" }; + return { + state: { ...next, ...altered }, + gestureId: input.gestureId, + nextSequence: input.sequence + 1, + cursor: null, + completion: "navigation", + }; }; - f.queue.enqueue(move(1)); f.queue.enqueue(move(2)); + f.queue.enqueue(move(1)); + f.queue.enqueue(move(2)); f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); await flush(); expect(f.navigationCompleted).toEqual([]); @@ -175,8 +244,13 @@ describe("bounded browser input queue", () => { expect(f.ended).toHaveLength(1); } const sequence = fixture(); - sequence.transport.update = async input => ({ state: next, gestureId: input.gestureId, - nextSequence: input.sequence + 2, cursor: null, completion: "navigation" }); + sequence.transport.update = async (input) => ({ + state: next, + gestureId: input.gestureId, + nextSequence: input.sequence + 2, + cursor: null, + completion: "navigation", + }); sequence.queue.enqueue(move(1)); sequence.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); await flush(); @@ -187,9 +261,15 @@ describe("bounded browser input queue", () => { it("unmarked navigation and replacement control retain the existing context rejection", async () => { const f = fixture(); const next: BrowserState = { ...state, navigationGeneration: 2 }; - f.projectState(reply => { if (reply.navigationGeneration === 2) f.changeAuthority(null); }); - f.transport.update = async input => ({ state: next, gestureId: input.gestureId, - nextSequence: input.sequence + 1, cursor: null }); + f.projectState((reply) => { + if (reply.navigationGeneration === 2) f.changeAuthority(null); + }); + f.transport.update = async (input) => ({ + state: next, + gestureId: input.gestureId, + nextSequence: input.sequence + 1, + cursor: null, + }); f.queue.enqueue(move(1)); f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); await flush(); @@ -197,11 +277,23 @@ describe("bounded browser input queue", () => { expect(f.navigationCompleted).toEqual([]); const changed = fixture(); - changed.transport.update = async input => { + changed.transport.update = async (input) => { changed.changeAuthority({ ...authority, controlToken: "replacement" }); - return { state: { ...next, status: "ready", controller: "self", sessionId: "session", - runtimeId: "runtime", bridgeEpoch: 1, viewportGeneration: 1 }, - gestureId: input.gestureId, nextSequence: input.sequence + 1, cursor: null, completion: "navigation" }; + return { + state: { + ...next, + status: "ready", + controller: "self", + sessionId: "session", + runtimeId: "runtime", + bridgeEpoch: 1, + viewportGeneration: 1, + }, + gestureId: input.gestureId, + nextSequence: input.sequence + 1, + cursor: null, + completion: "navigation", + }; }; changed.queue.enqueue(move(1)); changed.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); @@ -334,8 +426,7 @@ describe("bounded browser input queue", () => { it("holds at one in-flight update and never publishes a late cursor into a new generation", async () => { const f = fixture(); - const updateGate = - deferred>>(); + const updateGate = deferred>>(); let attempts = 0; f.transport.update = async () => { attempts++; @@ -419,11 +510,7 @@ describe("bounded browser input queue", () => { frame.resolve(); await flush(); expect(f.sent[1]?.target?.frameId).toBe("new-front"); - expect(f.sent.map((input) => input.event.kind)).toEqual([ - "scroll", - "down", - "up", - ]); + expect(f.sent.map((input) => input.event.kind)).toEqual(["scroll", "down", "up"]); expect(f.errors).toEqual([]); }); @@ -449,11 +536,7 @@ describe("bounded browser input queue", () => { f.queue.finish(); f.beginGate.resolve({ state, gestureId: "owned", nextSequence: 1 }); await flush(); - expect(f.sent.map((input) => input.event.kind)).toEqual([ - "down", - "up", - "leave", - ]); + expect(f.sent.map((input) => input.event.kind)).toEqual(["down", "up", "leave"]); expect(f.ended).toHaveLength(1); expect(f.ended[0]?.cancel).toBe(false); expect(waits).toEqual([]); @@ -461,12 +544,17 @@ describe("bounded browser input queue", () => { }); it("refreshes only a known unadmitted begin, then sends the pending wheel once", async () => { const f = fixture(async () => { - f.changeAuthority({ ...authority, target: { ...authority.target, frameId: "post-scroll-decoded" } }); + f.changeAuthority({ + ...authority, + target: { ...authority.target, frameId: "post-scroll-decoded" }, + }); }); let begins = 0; f.transport.begin = async () => { begins += 1; - return begins === 1 ? { state, admission: "stale-frame" } : { state, gestureId: "owned", nextSequence: 1 }; + return begins === 1 + ? { state, admission: "stale-frame" } + : { state, gestureId: "owned", nextSequence: 1 }; }; f.queue.enqueue({ kind: "scroll", point: point(1), deltaX: 0, deltaY: 20 }); await flush(); @@ -477,7 +565,9 @@ describe("bounded browser input queue", () => { it("does not retry begin or update after transport error or uncertain physical publication", async () => { for (const failure of ["begin", "update"] as const) { - const f = fixture(async () => { throw new Error("Must not wait/retry"); }); + const f = fixture(async () => { + throw new Error("Must not wait/retry"); + }); let begins = 0; let updates = 0; f.transport.begin = async () => { @@ -485,7 +575,10 @@ describe("bounded browser input queue", () => { if (failure === "begin") throw new Error("Outcome unknown"); return { state, gestureId: "owned", nextSequence: 1 }; }; - f.transport.update = async () => { updates += 1; throw new Error("Outcome unknown"); }; + f.transport.update = async () => { + updates += 1; + throw new Error("Outcome unknown"); + }; f.queue.enqueue({ kind: "scroll", point: point(1), deltaX: 0, deltaY: 20 }); await flush(); expect(begins).toBe(1); @@ -496,19 +589,30 @@ describe("bounded browser input queue", () => { it("bounds stale admission and refuses replacement control while waiting for decode", async () => { const f = fixture(async () => { - f.changeAuthority({ ...authority, target: { ...authority.target, frameId: String(++front) } }); + f.changeAuthority({ + ...authority, + target: { ...authority.target, frameId: String(++front) }, + }); }); let front = 0; let begins = 0; - f.transport.begin = async () => { begins += 1; return { state, admission: "stale-frame" }; }; + f.transport.begin = async () => { + begins += 1; + return { state, admission: "stale-frame" }; + }; f.queue.enqueue(move(1)); await flush(); expect(begins).toBe(3); expect(f.sent).toHaveLength(0); expect(f.errors).toHaveLength(1); - const changed = fixture(async () => { changed.changeAuthority({ ...authority, controlToken: "replacement" }); }); + const changed = fixture(async () => { + changed.changeAuthority({ ...authority, controlToken: "replacement" }); + }); let changedBegins = 0; - changed.transport.begin = async () => { changedBegins += 1; return { state, admission: "stale-frame" }; }; + changed.transport.begin = async () => { + changedBegins += 1; + return { state, admission: "stale-frame" }; + }; changed.queue.enqueue(move(1)); await flush(); expect(changedBegins).toBe(1); @@ -523,23 +627,36 @@ describe("bounded browser input queue", () => { const f = fixture(async (_id, budget) => { budgets.push(budget!); clock += budgets.length === 1 ? 3_000 : 1_001; - f.changeAuthority({ ...authority, target: { ...authority.target, frameId: String(clock) } }); + f.changeAuthority({ + ...authority, + target: { ...authority.target, frameId: String(clock) }, + }); }); let begins = 0; - f.transport.begin = async () => { begins += 1; return { state, admission: "stale-frame" }; }; + f.transport.begin = async () => { + begins += 1; + return { state, admission: "stale-frame" }; + }; f.queue.enqueue(move(1)); await flush(); expect(budgets).toEqual([4_000, 1_000]); expect(begins).toBe(2); expect(f.sent).toHaveLength(0); expect(f.errors).toHaveLength(1); - } finally { now.mockRestore(); } + } finally { + now.mockRestore(); + } }); it("does not begin under replacement control after the initial invalidated-frame wait", async () => { - const f = fixture(async () => { f.changeAuthority({ ...authority, controlToken: "replacement" }); }); + const f = fixture(async () => { + f.changeAuthority({ ...authority, controlToken: "replacement" }); + }); let begins = 0; - f.transport.begin = async () => { begins += 1; return { state, gestureId: "owned", nextSequence: 1 }; }; + f.transport.begin = async () => { + begins += 1; + return { state, gestureId: "owned", nextSequence: 1 }; + }; f.queue.enqueue({ kind: "scroll", point: point(1), deltaX: 0, deltaY: 20 }); await flush(); f.queue.finish(); @@ -550,12 +667,18 @@ describe("bounded browser input queue", () => { expect(f.sent).toHaveLength(1); expect(f.errors).toEqual([]); }); - }); it("keeps keyboard repeat/Unicode edges ordered and a held modifier alive across mouse release", async () => { const f = fixture(); - const key = { kind: "key", type: "down", key: "Control", code: "ControlLeft", modifiers: 2, repeat: false } as const; + const key = { + kind: "key", + type: "down", + key: "Control", + code: "ControlLeft", + modifiers: 2, + repeat: false, + } as const; f.queue.enqueue(key); f.queue.enqueue({ kind: "down", point: point(1), button: "left", clickCount: 1 }); f.queue.enqueue({ kind: "up", point: point(1), button: "left", clickCount: 1 }); @@ -564,14 +687,37 @@ it("keeps keyboard repeat/Unicode edges ordered and a held modifier alive across await flush(); expect(f.ended).toEqual([]); f.queue.enqueue({ ...key, type: "up", modifiers: 0 }); - f.queue.enqueue({ kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: false }); - f.queue.enqueue({ kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: true }); + f.queue.enqueue({ + kind: "key", + type: "down", + key: "é", + code: "KeyE", + modifiers: 0, + repeat: false, + }); + f.queue.enqueue({ + kind: "key", + type: "down", + key: "é", + code: "KeyE", + modifiers: 0, + repeat: true, + }); f.queue.enqueue({ kind: "key", type: "up", key: "é", code: "KeyE", modifiers: 0, repeat: false }); f.queue.enqueue({ kind: "text", text: "漢字\n😀" }); f.queue.finish(); await flush(); - expect(f.sent.map((input) => input.event.kind)).toEqual(["key", "down", "up", "key", "key", "key", "key", "text"]); - expect(f.sent.map((input) => input.sequence)).toEqual([1,2,3,4,5,6,7,8]); + expect(f.sent.map((input) => input.event.kind)).toEqual([ + "key", + "down", + "up", + "key", + "key", + "key", + "key", + "text", + ]); + expect(f.sent.map((input) => input.sequence)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); expect(f.ended).toHaveLength(1); expect(f.errors).toEqual([]); }); diff --git a/paseo-shared-browser/client/browser-input-queue.ts b/paseo-shared-browser/client/browser-input-queue.ts index 8bf0cefb..c77f47c7 100644 --- a/paseo-shared-browser/client/browser-input-queue.ts +++ b/paseo-shared-browser/client/browser-input-queue.ts @@ -6,32 +6,25 @@ */ import type { RpcInput, RpcOutput } from "@getpaseo/plugin"; import type { - beginBrowserGestureRpc, - updateBrowserGestureRpc, - endBrowserGestureRpc, - BrowserGestureEvent, BrowserCursor, + BrowserGestureEvent, BrowserState, + beginBrowserGestureRpc, + endBrowserGestureRpc, + updateBrowserGestureRpc, } from "../shared/browser"; -export type BrowserGestureAuthority = Omit< - RpcInput, - "pointerKind" ->; -export type BrowserGesturePoint = Extract< - BrowserGestureEvent, - { kind: "move" } ->["point"]; -export type BrowserTouchPoint = Extract< - BrowserGestureEvent, - { kind: "touch" } ->["points"][number]; +export type BrowserGestureAuthority = Omit, "pointerKind">; +export type BrowserGesturePoint = Extract["point"]; +export type BrowserTouchPoint = Extract["points"][number]; /** Preserve the admission union while using schema-normalized state defaults. */ -type NormalizedAdmission = T extends unknown ? Omit & { state: BrowserState } : never; +type NormalizedAdmission = T extends unknown + ? Omit & { state: BrowserState } + : never; export interface BrowserGestureTransport { - begin(input: RpcInput): Promise< - NormalizedAdmission> - >; + begin( + input: RpcInput, + ): Promise>>; update(input: RpcInput): Promise< Omit, "state"> & { state: BrowserState; @@ -71,10 +64,7 @@ const MAX_FRAME_ADMISSION_ATTEMPTS = 3; const FRAME_ADMISSION_WAIT_MS = 4_000; /** Compare control/route incarnation, never the changing decoded frame identity. */ -function sameAuthority( - a: BrowserGestureAuthority, - b: BrowserGestureAuthority | null, -): boolean { +function sameAuthority(a: BrowserGestureAuthority, b: BrowserGestureAuthority | null): boolean { return Boolean( b && a.viewerToken === b.viewerToken && @@ -93,29 +83,24 @@ function transportContext(authority: BrowserGestureAuthority) { expected: authority.expected, }; } -function needsFreshPress( - event: BrowserGestureEvent, - channel: Channel, -): boolean { +function needsFreshPress(event: BrowserGestureEvent, channel: Channel): boolean { return ( event.kind === "down" || - (event.kind === "touch" && - event.type === "start" && - channel.touchCount === 0) + (event.kind === "touch" && event.type === "start" && channel.touchCount === 0) ); } /** A qualified reply completes old input; it never authorizes input on the new page. */ -function isAcknowledgedNavigation( - original: BrowserGestureAuthority, - state: BrowserState, -): boolean { - return state.status === "ready" && state.controller === "self" && +function isAcknowledgedNavigation(original: BrowserGestureAuthority, state: BrowserState): boolean { + return ( + state.status === "ready" && + state.controller === "self" && state.sessionId === original.expected.sessionId && state.runtimeId === original.expected.runtimeId && state.bridgeEpoch === original.expected.bridgeEpoch && state.viewportGeneration === original.expected.viewportGeneration && - state.navigationGeneration > original.expected.navigationGeneration; + state.navigationGeneration > original.expected.navigationGeneration + ); } /** One in-flight RPC, with fail-closed cancellation on pressure or authority loss. */ @@ -171,16 +156,11 @@ export function createBrowserInputQueue(options: QueueOptions) { }); owned = null; channel = null; - if ( - currentEpoch !== epoch || - !sameAuthority(ending.authority, options.authority()) - ) + if (currentEpoch !== epoch || !sameAuthority(ending.authority, options.authority())) break; options.onState(result.state); if (!sameAuthority(ending.authority, options.authority())) { - throw new Error( - "Browser input context changed. Release the gesture and try again.", - ); + throw new Error("Browser input context changed. Release the gesture and try again."); } // Normal channel completion keeps the last qualified hover cursor. // A null end receipt denotes cleanup, not a new pointer location. @@ -190,17 +170,15 @@ export function createBrowserInputQueue(options: QueueOptions) { } let current = options.authority(); - if (!current) - throw new Error( - "A decoded frame and active browser control are required.", - ); - const pointerKind = command.event.kind === "touch" ? "touch" - : command.event.kind === "key" || command.event.kind === "text" - ? owned?.pointerKind ?? "mouse" : "mouse"; + if (!current) throw new Error("A decoded frame and active browser control are required."); + const pointerKind = + command.event.kind === "touch" + ? "touch" + : command.event.kind === "key" || command.event.kind === "text" + ? (owned?.pointerKind ?? "mouse") + : "mouse"; if (owned && !sameAuthority(owned.authority, current)) { - throw new Error( - "Browser input context changed. Release the gesture and try again.", - ); + throw new Error("Browser input context changed. Release the gesture and try again."); } if (owned && owned.pointerKind !== pointerKind) { // Changing input devices is ordinary on touch laptops. Close only this @@ -210,17 +188,12 @@ export function createBrowserInputQueue(options: QueueOptions) { channel = null; if (currentEpoch !== epoch) break; if (!sameAuthority(current, options.authority())) { - throw new Error( - "Browser input context changed. Release the gesture and try again.", - ); + throw new Error("Browser input context changed. Release the gesture and try again."); } } if (!owned) { const admissionDeadline = Date.now() + FRAME_ADMISSION_WAIT_MS; - if ( - invalidatedFrameId === current.target.frameId && - options.waitForFrame - ) { + if (invalidatedFrameId === current.target.frameId && options.waitForFrame) { const waitingAuthority = current; await options.waitForFrame(current.target.frameId, FRAME_ADMISSION_WAIT_MS); if (currentEpoch !== epoch) break; @@ -269,7 +242,10 @@ export function createBrowserInputQueue(options: QueueOptions) { } } if (currentEpoch !== epoch || !sameAuthority(current, options.authority())) break; - if (!owned) throw new Error("Waiting for a current decoded frame. Release the gesture and try again."); + if (!owned) + throw new Error( + "Waiting for a current decoded frame. Release the gesture and try again.", + ); } let fresh = options.authority(); @@ -284,9 +260,7 @@ export function createBrowserInputQueue(options: QueueOptions) { fresh = options.authority(); } if (!sameAuthority(owned.authority, fresh)) { - throw new Error( - "Browser input context changed. Release the gesture and try again.", - ); + throw new Error("Browser input context changed. Release the gesture and try again."); } if ( (command.event.kind !== "move" && command.event.kind !== "leave") || @@ -295,14 +269,13 @@ export function createBrowserInputQueue(options: QueueOptions) { invalidatedFrameId = fresh?.target.frameId ?? null; } const requiresTarget = needsFreshPress(command.event, owned); - if (command.event.kind === "key" && command.event.type === "down") owned.heldKeys.add(command.event.code); - if (command.event.kind === "key" && command.event.type === "up") owned.heldKeys.delete(command.event.code); - if (command.event.kind === "touch") - owned.touchCount = command.event.points.length; - if (command.event.kind === "down") - owned.heldButtons.add(command.event.button); - if (command.event.kind === "up") - owned.heldButtons.delete(command.event.button); + if (command.event.kind === "key" && command.event.type === "down") + owned.heldKeys.add(command.event.code); + if (command.event.kind === "key" && command.event.type === "up") + owned.heldKeys.delete(command.event.code); + if (command.event.kind === "touch") owned.touchCount = command.event.points.length; + if (command.event.kind === "down") owned.heldButtons.add(command.event.button); + if (command.event.kind === "up") owned.heldButtons.delete(command.event.button); const expectedNextSequence = owned.sequence + 1; const result = await options.transport.update({ ...transportContext(owned.authority), @@ -312,16 +285,14 @@ export function createBrowserInputQueue(options: QueueOptions) { ...(requiresTarget && fresh ? { target: fresh.target } : {}), }); owned.sequence = result.nextSequence; - if ( - currentEpoch !== epoch || - !sameAuthority(owned.authority, options.authority()) - ) - break; + if (currentEpoch !== epoch || !sameAuthority(owned.authority, options.authority())) break; if (result.gestureId !== owned.gestureId) throw new Error("Browser gesture identity changed."); if (result.completion === "navigation") { - if (!isAcknowledgedNavigation(owned.authority, result.state) || - result.nextSequence !== expectedNextSequence) { + if ( + !isAcknowledgedNavigation(owned.authority, result.state) || + result.nextSequence !== expectedNextSequence + ) { throw new Error("Browser navigation completion identity changed."); } // Native publication and cleanup are already acknowledged. Discard @@ -340,9 +311,7 @@ export function createBrowserInputQueue(options: QueueOptions) { } options.onState(result.state); if (!sameAuthority(owned.authority, options.authority())) { - throw new Error( - "Browser input context changed. Release the gesture and try again.", - ); + throw new Error("Browser input context changed. Release the gesture and try again."); } options.onCursor(result.cursor); } @@ -357,8 +326,7 @@ export function createBrowserInputQueue(options: QueueOptions) { } finally { if ( owned && - (currentEpoch !== epoch || - !sameAuthority(owned.authority, options.authority())) + (currentEpoch !== epoch || !sameAuthority(owned.authority, options.authority())) ) { if (currentEpoch === epoch) { epoch += 1; @@ -402,10 +370,7 @@ export function createBrowserInputQueue(options: QueueOptions) { if (event.kind === "scroll" && previous.event.kind === "scroll") { const deltaX = previous.event.deltaX + event.deltaX; const deltaY = previous.event.deltaY + event.deltaY; - if ( - Math.abs(deltaX) <= MAX_WHEEL_DELTA && - Math.abs(deltaY) <= MAX_WHEEL_DELTA - ) { + if (Math.abs(deltaX) <= MAX_WHEEL_DELTA && Math.abs(deltaY) <= MAX_WHEEL_DELTA) { previous.event = { ...event, deltaX, deltaY }; return true; } @@ -414,9 +379,7 @@ export function createBrowserInputQueue(options: QueueOptions) { if (commands.length >= MAX_QUEUED_COMMANDS) { cancel(); options.onError( - new Error( - "Browser input cannot keep up. Release the gesture and try again.", - ), + new Error("Browser input cannot keep up. Release the gesture and try again."), ); return false; } diff --git a/paseo-shared-browser/client/browser-menu-placement.test.ts b/paseo-shared-browser/client/browser-menu-placement.test.ts index d595b036..1885cc99 100644 --- a/paseo-shared-browser/client/browser-menu-placement.test.ts +++ b/paseo-shared-browser/client/browser-menu-placement.test.ts @@ -48,26 +48,43 @@ describe("bounded toolbar menus", () => { }); }); - describe("bounded child menu placement", () => { it("opens beside a parent with enough room", () => { - expect(browserSubmenuPlacement({ width: 900, height: 600 }, - { x: 8, y: 40, width: 300, height: 100 }, - { x: 8, y: 80, width: 300, height: 36 }, 200)) - .toEqual({ x: 312, y: 80, width: 240, height: 200 }); + expect( + browserSubmenuPlacement( + { width: 900, height: 600 }, + { x: 8, y: 40, width: 300, height: 100 }, + { x: 8, y: 80, width: 300, height: 36 }, + 200, + ), + ).toEqual({ x: 312, y: 80, width: 240, height: 200 }); }); it("flips left and clamps a long submenu for internal scrolling", () => { - expect(browserSubmenuPlacement({ width: 800, height: 300 }, - { x: 468, y: 40, width: 300, height: 100 }, - { x: 468, y: 260, width: 300, height: 36 }, 900)) - .toEqual({ x: 224, y: 8, width: 240, height: 284 }); + expect( + browserSubmenuPlacement( + { width: 800, height: 300 }, + { x: 468, y: 40, width: 300, height: 100 }, + { x: 468, y: 260, width: 300, height: 36 }, + 900, + ), + ).toEqual({ x: 224, y: 8, width: 240, height: 284 }); }); it("refuses unknown pane or row geometry", () => { - expect(browserSubmenuPlacement({ width: 0, height: 300 }, - { x: 0, y: 0, width: 100, height: 80 }, - { x: 0, y: 20, width: 100, height: 30 }, 160)).toBeNull(); - expect(browserSubmenuPlacement({ width: 800, height: 300 }, - { x: 0, y: 0, width: 100, height: 80 }, - { x: 0, y: NaN, width: 100, height: 30 }, 160)).toBeNull(); + expect( + browserSubmenuPlacement( + { width: 0, height: 300 }, + { x: 0, y: 0, width: 100, height: 80 }, + { x: 0, y: 20, width: 100, height: 30 }, + 160, + ), + ).toBeNull(); + expect( + browserSubmenuPlacement( + { width: 800, height: 300 }, + { x: 0, y: 0, width: 100, height: 80 }, + { x: 0, y: NaN, width: 100, height: 30 }, + 160, + ), + ).toBeNull(); }); }); diff --git a/paseo-shared-browser/client/browser-menu-placement.ts b/paseo-shared-browser/client/browser-menu-placement.ts index b3146688..35b965fc 100644 --- a/paseo-shared-browser/client/browser-menu-placement.ts +++ b/paseo-shared-browser/client/browser-menu-placement.ts @@ -58,7 +58,7 @@ export function browserSubmenuPlacement( ): MenuRect | null { const values = [pane.width, pane.height, parent.x, parent.width, row.y, preferredHeight]; if ( - values.some(value => !Number.isFinite(value)) || + values.some((value) => !Number.isFinite(value)) || pane.width <= 16 || pane.height <= 16 || preferredHeight <= 0 diff --git a/paseo-shared-browser/client/browser-native-keyboard.test.ts b/paseo-shared-browser/client/browser-native-keyboard.test.ts index a324e522..b149cea9 100644 --- a/paseo-shared-browser/client/browser-native-keyboard.test.ts +++ b/paseo-shared-browser/client/browser-native-keyboard.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { createBrowserNativeKeyboard } from "./browser-native-keyboard"; import type { CanvasKeyboardEvent } from "./browser-canvas-keyboard"; +import { createBrowserNativeKeyboard } from "./browser-native-keyboard"; function fixture() { const events: CanvasKeyboardEvent[] = []; diff --git a/paseo-shared-browser/client/browser-toolbar-menu-web.test.ts b/paseo-shared-browser/client/browser-toolbar-menu-web.test.ts index 77b060b4..a1118249 100644 --- a/paseo-shared-browser/client/browser-toolbar-menu-web.test.ts +++ b/paseo-shared-browser/client/browser-toolbar-menu-web.test.ts @@ -45,7 +45,14 @@ function fixture(hierarchy?: Parameters[5]) { const overlay = { contains: (node: unknown) => node === backdrop || menu.contains(node) }; const close = vi.fn(); let restore = true; - const dispose = bindBrowserToolbarMenuWeb(menu, trigger, close, () => restore, overlay, hierarchy); + const dispose = bindBrowserToolbarMenuWeb( + menu, + trigger, + close, + () => restore, + overlay, + hierarchy, + ); const event = (key: string, target = doc.activeElement) => ({ key, target, @@ -138,7 +145,6 @@ describe("local toolbar menu keyboard and dismissal", () => { }); }); - describe("root-owned submenu Tab exit", () => { it("Tab and Shift+Tab from a descendant skip the whole popup", () => { for (const backwards of [false, true]) { @@ -150,7 +156,7 @@ describe("root-owned submenu Tab exit", () => { closest: () => childMenu, }; const originalContains = f.menu.contains; - f.menu.contains = node => node === child || originalContains(node); + f.menu.contains = (node) => node === child || originalContains(node); const before = { focus: vi.fn(), getAttribute: () => null }; const after = { focus: vi.fn(), getAttribute: () => null }; Object.assign(f.doc, { diff --git a/paseo-shared-browser/client/browser-toolbar-menu-web.ts b/paseo-shared-browser/client/browser-toolbar-menu-web.ts index 06944354..d51f8777 100644 --- a/paseo-shared-browser/client/browser-toolbar-menu-web.ts +++ b/paseo-shared-browser/client/browser-toolbar-menu-web.ts @@ -42,7 +42,8 @@ export function bindBrowserToolbarMenuWeb( const enabledItems = () => Array.from(menu.querySelectorAll('[role="menuitem"]')).filter( (item) => - item.getAttribute("aria-disabled") !== "true" && item.getAttribute("disabled") === null && + item.getAttribute("aria-disabled") !== "true" && + item.getAttribute("disabled") === null && (!item.closest || item.closest('[role="menu"]') === menu), ); enabledItems()[0]?.focus(); @@ -59,7 +60,11 @@ export function bindBrowserToolbarMenuWeb( // entire popup, and its cleanup performs the one final focus restoration. if (event.key === "Tab" && hierarchy?.nested) return; if (event.key !== "Tab" && target.closest && target.closest('[role="menu"]') !== menu) return; - if (event.key === "ArrowRight" && target.getAttribute("aria-expanded") !== null && hierarchy?.openSubmenu) { + if ( + event.key === "ArrowRight" && + target.getAttribute("aria-expanded") !== null && + hierarchy?.openSubmenu + ) { event.preventDefault(); event.stopPropagation(); hierarchy.openSubmenu(); @@ -75,7 +80,9 @@ export function bindBrowserToolbarMenuWeb( if (event.key === "Tab") { // Menu Tab leaves the popup rather than walking its command rows or // trapping focus. Follow the trigger's surrounding host tab order. - const triggerButton = Array.from(anchor?.querySelectorAll('[role="button"], [role="menuitem"]') ?? [])[0]; + const triggerButton = Array.from( + anchor?.querySelectorAll('[role="button"], [role="menuitem"]') ?? [], + )[0]; const focusable = Array.from( menu.ownerDocument.querySelectorAll?.( 'button, [role="button"], input:not([type="hidden"]), select, textarea, a[href], [tabindex]', @@ -135,14 +142,18 @@ export function bindBrowserToolbarMenuWeb( // Native React unmount can remove the focused row before passive cleanup. // Remember ownership rather than consulting only the now-detached node. if (restoreFocus() && focusOwned) { - (returnTarget ?? Array.from(anchor?.querySelectorAll('[role="button"], [role="menuitem"]') ?? [])[0])?.focus(); + ( + returnTarget ?? + Array.from(anchor?.querySelectorAll('[role="button"], [role="menuitem"]') ?? [])[0] + )?.focus(); } }; } /** Restore an in-menu trigger after compact Back, without touching native hosts. */ export function focusBrowserMenuTrigger(node: unknown): void { - if (Platform.OS !== "web" || !node || typeof node !== "object" || !("querySelectorAll" in node)) return; + if (Platform.OS !== "web" || !node || typeof node !== "object" || !("querySelectorAll" in node)) + return; const trigger = node as MenuNode; Array.from(trigger.querySelectorAll('[role="menuitem"], [role="button"]'))[0]?.focus(); } diff --git a/paseo-shared-browser/client/browser-toolbar-menu.tsx b/paseo-shared-browser/client/browser-toolbar-menu.tsx index 52d3e4a2..a043254e 100644 --- a/paseo-shared-browser/client/browser-toolbar-menu.tsx +++ b/paseo-shared-browser/client/browser-toolbar-menu.tsx @@ -15,8 +15,8 @@ import { } from "react"; import { BackHandler, Platform, Pressable, ScrollView, StyleSheet, Text, View } from "react-native"; import { browserMenuPlacement, type MenuRect } from "./browser-menu-placement"; -import { BrowserToolbarSubmenu, type BrowserSubmenu } from "./browser-toolbar-submenu"; import { bindBrowserToolbarMenuWeb, focusBrowserMenuTrigger } from "./browser-toolbar-menu-web"; +import { type BrowserSubmenu, BrowserToolbarSubmenu } from "./browser-toolbar-submenu"; type Theme = PluginHostProps["theme"]; const STYLES = StyleSheet.create({ @@ -103,8 +103,10 @@ export function BrowserToolbarMenu(props: BrowserToolbarMenuProps) { () => current.current.onClose(), () => current.current.shouldRestoreFocus(), overlayRef.current, - { openSubmenu: () => current.current.onSubmenuOpen?.(), - ...(compact && props.submenu ? { back: () => current.current.submenu?.onBack() } : {}) }, + { + openSubmenu: () => current.current.onSubmenuOpen?.(), + ...(compact && props.submenu ? { back: () => current.current.submenu?.onBack() } : {}), + }, ); if (compact && previousSubmenu.current && !props.submenu) { focusBrowserMenuTrigger(previousSubmenu.current.anchorRef.current); @@ -157,15 +159,29 @@ export function BrowserToolbarMenu(props: BrowserToolbarMenuProps) { > {compact && props.submenu ? ( <> - + {props.submenu.title} {props.submenu.children} - ) : props.children} + ) : ( + props.children + )} {!compact && props.submenu ? ( - + ) : null} ) : null} @@ -199,7 +215,11 @@ export function BrowserMenuItem({ setHovered(true)} onHoverOut={() => setHovered(false)} @@ -217,7 +237,9 @@ export function BrowserMenuItem({ > {icon ? : null} {label} - {expanded !== undefined ? : null} + {expanded !== undefined ? ( + + ) : null} {selected ? : null} ); diff --git a/paseo-shared-browser/client/browser-toolbar-submenu.tsx b/paseo-shared-browser/client/browser-toolbar-submenu.tsx index 3552ea9d..13f373f4 100644 --- a/paseo-shared-browser/client/browser-toolbar-submenu.tsx +++ b/paseo-shared-browser/client/browser-toolbar-submenu.tsx @@ -1,6 +1,6 @@ /** Desktop flyouts remain descendants of the root menu and share its one backdrop. */ import type { PluginHostProps } from "@getpaseo/plugin/client"; -import { useLayoutEffect, useRef, useState, type ReactNode, type RefObject } from "react"; +import { type ReactNode, type RefObject, useLayoutEffect, useRef, useState } from "react"; import { ScrollView, View } from "react-native"; import { browserSubmenuPlacement, type MenuRect } from "./browser-menu-placement"; import { bindBrowserToolbarMenuWeb } from "./browser-toolbar-menu-web"; diff --git a/paseo-shared-browser/client/browser-viewer-recovery.test.ts b/paseo-shared-browser/client/browser-viewer-recovery.test.ts index 1ed66e03..e6dd278b 100644 --- a/paseo-shared-browser/client/browser-viewer-recovery.test.ts +++ b/paseo-shared-browser/client/browser-viewer-recovery.test.ts @@ -22,7 +22,13 @@ const observation = { describe("expired browser viewing recovery", () => { it("matches the actual SDK handler-error suffix but not transport, control or arbitrary failures", () => { expect(isExpiredBrowserViewerError(expired)).toBe(true); - expect(isExpiredBrowserViewerError(Object.assign(new Error("Request failed: Viewer token is invalid or expired"), { code: "handler_error" }))).toBe(true); + expect( + isExpiredBrowserViewerError( + Object.assign(new Error("Request failed: Viewer token is invalid or expired"), { + code: "handler_error", + }), + ), + ).toBe(true); expect(isExpiredBrowserViewerError(new Error("Viewer token is invalid or expired"))).toBe(true); for (const error of [ new Error("Control lease is invalid or expired"), @@ -61,13 +67,14 @@ describe("expired browser viewing recovery", () => { }); }); - it("the actual capture query observer discards old token errors when its token key changes", async () => { const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); const replacement = Promise.withResolvers(); const observer = new QueryObserver(client, { queryKey: ["shared-browser", "capture", "old-viewer"], - queryFn: async (): Promise => { throw expired; }, + queryFn: async (): Promise => { + throw expired; + }, }); const unsubscribe = observer.subscribe(() => {}); try { diff --git a/paseo-shared-browser/client/browser-viewer-recovery.ts b/paseo-shared-browser/client/browser-viewer-recovery.ts index c566ae48..beaad17e 100644 --- a/paseo-shared-browser/client/browser-viewer-recovery.ts +++ b/paseo-shared-browser/client/browser-viewer-recovery.ts @@ -13,12 +13,14 @@ export function isExpiredBrowserViewerError(error: unknown): boolean { if (value.code !== undefined && value.code !== "handler_error") return false; // Daemon session.js applies this exact prefix before the client appends // requestType/code. Arbitrary error text never grants automatic recovery. - const message = value.code === "handler_error" && value.message.startsWith("Request failed: ") - ? value.message.slice("Request failed: ".length) - : value.message; + const message = + value.code === "handler_error" && value.message.startsWith("Request failed: ") + ? value.message.slice("Request failed: ".length) + : value.message; if (message === VIEWER_EXPIRED_MESSAGE) return true; - return value.code === "handler_error" && - message.startsWith(`${VIEWER_EXPIRED_MESSAGE} requestType=`); + return ( + value.code === "handler_error" && message.startsWith(`${VIEWER_EXPIRED_MESSAGE} requestType=`) + ); } export interface BrowserViewerRecoveryObservation { diff --git a/paseo-shared-browser/client/browser.tsx b/paseo-shared-browser/client/browser.tsx index 06c31d5c..619292db 100644 --- a/paseo-shared-browser/client/browser.tsx +++ b/paseo-shared-browser/client/browser.tsx @@ -46,10 +46,10 @@ import { import { groupResolutionPresets } from "../shared/resolution-menu"; import { type BrowserCanvasDisplayMode, getBrowserCanvasLayout } from "./browser-canvas-layout"; import { BrowserCanvasViewport } from "./browser-canvas-viewport"; -import { BrowserFrameImage } from "./browser-frame-image"; import { browserCaptureInterval } from "./browser-capture-cadence"; -import { matchingResolutionPresetId, type EmulationSelection } from "./browser-emulation-mode"; +import { type EmulationSelection, matchingResolutionPresetId } from "./browser-emulation-mode"; import type { FrameCandidate } from "./browser-frame-buffer"; +import { BrowserFrameImage } from "./browser-frame-image"; import { BrowserResolutionPicker } from "./browser-resolution-picker"; import { BrowserMenuHeading, @@ -1711,7 +1711,7 @@ export function SharedBrowserPanel({ : null; const selectedResolutionPresetId = matchingResolutionPresetId(state); const deviceLabel = selectedResolutionPresetId - ? activeDevicePreset?.label ?? "Custom display" + ? (activeDevicePreset?.label ?? "Custom display") : `${emulation.mode === "mobile" ? "Mobile" : "Desktop"} · custom display`; const transportLabel = currentFrame?.transport === "cdp-screencast" ? "CDP" : "fallback"; const frameSummary = currentFrame @@ -1911,9 +1911,7 @@ export function SharedBrowserPanel({ key={layer.ticket} ticket={layer.ticket} visible={visible} - label={ - state?.title ? `Shared browser: ${state.title}` : "Shared browser frame" - } + label={state?.title ? `Shared browser: ${state.title}` : "Shared browser frame"} settled={settled} retry={retryFrameCapture} source={layer.source} @@ -2093,21 +2091,32 @@ export function SharedBrowserPanel({ } onClose={() => closeToolbarMenu()} shouldRestoreFocus={() => menuRestoreFocus.current} - onSubmenuOpen={() => { if (canSendInput) setKeysSubmenuOpen(true); }} - submenu={toolbarMenu === "actions" && keysSubmenuOpen ? { - title: "Send keys", - anchorRef: keysAnchorRef, - preferredHeight: 40 + SPECIAL_KEYS.length * (layout.compact ? 44 : 36), - onBack: () => setKeysSubmenuOpen(false), - children: SPECIAL_KEYS.map(({ key, label }) => ( - { - canvasInput.nativeKeyboard.reset(); - sendEvent({ kind: "key", key }); - }} /> - )), - } : undefined} + onSubmenuOpen={() => { + if (canSendInput) setKeysSubmenuOpen(true); + }} + submenu={ + toolbarMenu === "actions" && keysSubmenuOpen + ? { + title: "Send keys", + anchorRef: keysAnchorRef, + preferredHeight: 40 + SPECIAL_KEYS.length * (layout.compact ? 44 : 36), + onBack: () => setKeysSubmenuOpen(false), + children: SPECIAL_KEYS.map(({ key, label }) => ( + { + canvasInput.nativeKeyboard.reset(); + sendEvent({ kind: "key", key }); + }} + /> + )), + } + : undefined + } > {toolbarMenu === "display" ? ( <> diff --git a/paseo-shared-browser/client/use-browser-canvas-input.ts b/paseo-shared-browser/client/use-browser-canvas-input.ts index 0db45a29..a027f215 100644 --- a/paseo-shared-browser/client/use-browser-canvas-input.ts +++ b/paseo-shared-browser/client/use-browser-canvas-input.ts @@ -4,37 +4,27 @@ * do not. Cursor replies are consumed only by the queue incarnation that sent * them. The existing capture polling keeps playback running during a drag. */ -import { - useCallback, - useEffect, - useLayoutEffect, - useRef, - useState, -} from "react"; +import { useCallback, useEffect, useLayoutEffect, useRef, useState } from "react"; import { AppState, + type GestureResponderEvent, PanResponder, Platform, - type GestureResponderEvent, type View, } from "react-native"; import type { BrowserCursor, BrowserState } from "../shared/browser"; +import { createBrowserCanvasInput } from "./browser-canvas-input"; +import type { CanvasKeyboardEvent } from "./browser-canvas-keyboard"; +import { bindBrowserCanvasKeyboard, type KeyboardCanvasNode } from "./browser-canvas-keyboard-web"; import { - createBrowserInputQueue, type BrowserGestureAuthority, type BrowserGesturePoint, type BrowserGestureTransport, type BrowserTouchPoint, + createBrowserInputQueue, } from "./browser-input-queue"; -import { createBrowserCanvasInput } from "./browser-canvas-input"; -import { bindBrowserCanvasKeyboard, type KeyboardCanvasNode } from "./browser-canvas-keyboard-web"; -import type { CanvasKeyboardEvent } from "./browser-canvas-keyboard"; import { useBrowserNativeKeyboard } from "./use-browser-native-keyboard"; -import { - bindBrowserCanvasWeb, - setBrowserCanvasCursor, - type BrowserCanvasNode, -} from "./web"; +import { type BrowserCanvasNode, bindBrowserCanvasWeb, setBrowserCanvasCursor } from "./web"; interface CanvasOptions { authority(): BrowserGestureAuthority | null; @@ -83,9 +73,7 @@ export function useBrowserCanvasInput(options: CanvasOptions) { const nodeRef = useRef(null); const [node, setNode] = useState(null); const alive = useRef(true); - const inputModel = useRef | null>( - null, - ); + const inputModel = useRef | null>(null); const keyboard = useRef | null>(null); const nativeRelay = useRef<{ reset(): void } | null>(null); const cursorVisible = useRef(false); @@ -113,8 +101,7 @@ export function useBrowserCanvasInput(options: CanvasOptions) { if (alive.current) current.current.onState(state); }, onCursor: (cursor: BrowserCursor | null) => { - if (cursorVisible.current) - setBrowserCanvasCursor(nodeRef.current, cursor); + if (cursorVisible.current) setBrowserCanvasCursor(nodeRef.current, cursor); }, onError: (error) => { // Queue owns cleanup; reset local contacts without recursively cancelling. @@ -182,10 +169,7 @@ export function useBrowserCanvasInput(options: CanvasOptions) { viewport: () => current.current.viewport, enqueue(event) { clearIdle(); - if ( - event.kind === "down" || - (event.kind === "touch" && event.type === "start") - ) { + if (event.kind === "down" || (event.kind === "touch" && event.type === "start")) { nativeRelay.current?.reset(); current.current.onInputBoundary(); } @@ -223,17 +207,20 @@ export function useBrowserCanvasInput(options: CanvasOptions) { inputModel.current = input; - const enqueueKeyboard = useCallback((event: CanvasKeyboardEvent) => { - clearIdle(); - const accepted = queue.enqueue(event); - if (accepted) { - if (event.kind === "text" || (event.type === "down" && !event.repeat)) { - current.current.onInputBoundary(); + const enqueueKeyboard = useCallback( + (event: CanvasKeyboardEvent) => { + clearIdle(); + const accepted = queue.enqueue(event); + if (accepted) { + if (event.kind === "text" || (event.type === "down" && !event.repeat)) { + current.current.onInputBoundary(); + } + current.current.onActivity(true); } - current.current.onActivity(true); - } - return accepted; - }, [queue]); + return accepted; + }, + [queue], + ); const finishKeyboard = useCallback(() => { current.current.onActivity(false); // Keep one channel through ordinary inter-key gaps. New typing clears this @@ -247,7 +234,9 @@ export function useBrowserCanvasInput(options: CanvasOptions) { enqueue: enqueueKeyboard, finish: finishKeyboard, cancel: input.cancel, - onError: error => { if (alive.current) current.current.onError(error); }, + onError: (error) => { + if (alive.current) current.current.onError(error); + }, }); nativeRelay.current = nativeKeyboard; @@ -259,10 +248,7 @@ export function useBrowserCanvasInput(options: CanvasOptions) { }, [input, options.ownershipKey]); useLayoutEffect(() => { for (const waiter of frameWaiters.current) { - if ( - options.decodedFrameId && - options.decodedFrameId !== waiter.afterFrameId - ) + if (options.decodedFrameId && options.decodedFrameId !== waiter.afterFrameId) waiter.resolve(); } }, [options.decodedFrameId]); @@ -278,12 +264,7 @@ export function useBrowserCanvasInput(options: CanvasOptions) { }; }, [input]); useEffect(() => { - if ( - Platform.OS !== "web" || - !node || - typeof node !== "object" || - !("addEventListener" in node) - ) + if (Platform.OS !== "web" || !node || typeof node !== "object" || !("addEventListener" in node)) return; return bindBrowserCanvasWeb( node as BrowserCanvasNode, @@ -297,7 +278,8 @@ export function useBrowserCanvasInput(options: CanvasOptions) { }, [input, node]); useEffect(() => { - if (Platform.OS !== "web" || !node || typeof node !== "object" || !("ownerDocument" in node)) return; + if (Platform.OS !== "web" || !node || typeof node !== "object" || !("ownerDocument" in node)) + return; const binding = bindBrowserCanvasKeyboard(node as KeyboardCanvasNode, { enabled: () => alive.current && current.current.enabled, enqueue: enqueueKeyboard, @@ -322,41 +304,25 @@ export function useBrowserCanvasInput(options: CanvasOptions) { // A newly granted responder must contain only genuinely new contacts. // A second finger cannot import a first contact that began observe-only. const changed = new Set( - (event.nativeEvent.changedTouches ?? []).map( - (touch) => touch.identifier, - ), + (event.nativeEvent.changedTouches ?? []).map((touch) => touch.identifier), ); return ( event.nativeEvent.touches.length > 0 && - event.nativeEvent.touches.every((touch) => - changed.has(touch.identifier), - ) + event.nativeEvent.touches.every((touch) => changed.has(touch.identifier)) ); }, // Moving a contact after lease acquisition is not a fresh start edge. onMoveShouldSetPanResponder: () => false, onPanResponderGrant: (event) => { ids.current.clear(); - input.touch( - "start", - nativePoints(event, current.current.displaySize, ids.current), - ); + input.touch("start", nativePoints(event, current.current.displaySize, ids.current)); }, onPanResponderStart: (event) => - input.touch( - "start", - nativePoints(event, current.current.displaySize, ids.current), - ), + input.touch("start", nativePoints(event, current.current.displaySize, ids.current)), onPanResponderMove: (event) => - input.touch( - "move", - nativePoints(event, current.current.displaySize, ids.current), - ), + input.touch("move", nativePoints(event, current.current.displaySize, ids.current)), onPanResponderEnd: (event) => - input.touch( - "end", - nativePoints(event, current.current.displaySize, ids.current), - ), + input.touch("end", nativePoints(event, current.current.displaySize, ids.current)), onPanResponderRelease: () => { input.touch("end", []); ids.current.clear(); diff --git a/paseo-shared-browser/client/use-browser-frame-buffer.ts b/paseo-shared-browser/client/use-browser-frame-buffer.ts index 896238f6..bab5cdf8 100644 --- a/paseo-shared-browser/client/use-browser-frame-buffer.ts +++ b/paseo-shared-browser/client/use-browser-frame-buffer.ts @@ -3,7 +3,7 @@ * visible-frame ref is changed in a layout effect, after the image-layer commit, * so transport metadata cannot authorize input against pixels not yet displayed. */ -import { useCallback, useLayoutEffect, useState, type RefObject } from "react"; +import { type RefObject, useCallback, useLayoutEffect, useState } from "react"; import type { BrowserFrame } from "../shared/browser"; import { createBrowserFrameBuffer, type FrameCandidate } from "./browser-frame-buffer"; diff --git a/paseo-shared-browser/client/use-resolution-favorites.ts b/paseo-shared-browser/client/use-resolution-favorites.ts index 71efedf7..2602bd44 100644 --- a/paseo-shared-browser/client/use-resolution-favorites.ts +++ b/paseo-shared-browser/client/use-resolution-favorites.ts @@ -6,15 +6,12 @@ import { useSettings } from "@getpaseo/plugin/client"; import { useRef, useState } from "react"; import { - browserDisplayPreferences, type BrowserDisplayPreferences, + browserDisplayPreferences, } from "../shared/browser-display-preferences"; import { DEFAULT_CAPTURE_QUALITY } from "../shared/capture-settings"; -import { - normalizeResolutionFavorites, - toggleResolutionFavorite, -} from "../shared/resolution-menu"; import type { DevicePresetId } from "../shared/device-presets"; +import { normalizeResolutionFavorites, toggleResolutionFavorite } from "../shared/resolution-menu"; /** Call with the surface's host.id; persistence itself is scoped by Paseo. */ export function useResolutionFavorites(hostId: string) { @@ -48,10 +45,7 @@ export function useResolutionFavorites(hostId: string) { pending.current = operation; setPendingHost(hostId); try { - const saved = await settings.save( - update(settings.values), - settings.revision, - ); + const saved = await settings.save(update(settings.values), settings.revision); return currentHost.current === hostId && saved; } finally { if (pending.current === operation) { @@ -64,27 +58,20 @@ export function useResolutionFavorites(hostId: string) { return { favoritePresetIds, captureQuality: - settings.status === "ready" - ? settings.values.captureQuality - : DEFAULT_CAPTURE_QUALITY, + settings.status === "ready" ? settings.values.captureQuality : DEFAULT_CAPTURE_QUALITY, loading: settings.status === "loading", saving, disabled, error, toggleFavorite: (id: DevicePresetId) => { - if (normalizeResolutionFavorites([id]).length === 0) - return Promise.resolve(false); + if (normalizeResolutionFavorites([id]).length === 0) return Promise.resolve(false); return savePreferences((values) => ({ ...values, - favoritePresetIds: toggleResolutionFavorite( - values.favoritePresetIds, - id, - ), + favoritePresetIds: toggleResolutionFavorite(values.favoritePresetIds, id), })); }, - changeQuality: ( - captureQuality: BrowserDisplayPreferences["captureQuality"], - ) => savePreferences((values) => ({ ...values, captureQuality })), + changeQuality: (captureQuality: BrowserDisplayPreferences["captureQuality"]) => + savePreferences((values) => ({ ...values, captureQuality })), reload: settings.reload, resetPreferences: settings.reset, }; diff --git a/paseo-shared-browser/client/web.ts b/paseo-shared-browser/client/web.ts index 98cb8afb..cba54a06 100644 --- a/paseo-shared-browser/client/web.ts +++ b/paseo-shared-browser/client/web.ts @@ -94,18 +94,13 @@ export function bindBrowserCanvasWeb( const mouse = (type: "move" | "down" | "up", event: DomEvent) => { // Consumed touch events suppress compatibility mouse events in browsers. // Chromium also explicitly identifies them; never forward a second tap. - if ( - touchActive || - event.sourceCapabilities?.firesTouchEvents || - !enabled() - ) { + if (touchActive || event.sourceCapabilities?.firesTouchEvents || !enabled()) { return; } const location = point(event.clientX, event.clientY); if (!location) return; onCursorVisibility(true); - const button = - event.button === 2 ? "right" : event.button === 1 ? "middle" : "left"; + const button = event.button === 2 ? "right" : event.button === 1 ? "middle" : "left"; const count = event.detail === 2 ? 2 : 1; if (type === "down") { if (input.mouseDown(location, button, count)) consume(event); @@ -164,16 +159,10 @@ export function bindBrowserCanvasWeb( if (!location) return; // Wheel deltaMode is local CSS pixels, lines, or pages. Line-mode uses the // browser's conventional 16 CSS-pixel step; page mode uses this image's size. - const multiplierX = - event.deltaMode === 2 ? location.width : event.deltaMode === 1 ? 16 : 1; - const multiplierY = - event.deltaMode === 2 ? location.height : event.deltaMode === 1 ? 16 : 1; + const multiplierX = event.deltaMode === 2 ? location.width : event.deltaMode === 1 ? 16 : 1; + const multiplierY = event.deltaMode === 2 ? location.height : event.deltaMode === 1 ? 16 : 1; if ( - input.wheel( - location, - (event.deltaX ?? 0) * multiplierX, - (event.deltaY ?? 0) * multiplierY, - ) + input.wheel(location, (event.deltaX ?? 0) * multiplierX, (event.deltaY ?? 0) * multiplierY) ) { consume(event); } @@ -204,11 +193,10 @@ export function bindBrowserCanvasWeb( } }; documentTouchListeners.set(`touch${phase}`, trackUncontrolledContact); - node.ownerDocument.addEventListener( - `touch${phase}`, - trackUncontrolledContact, - { capture: true, passive: true }, - ); + node.ownerDocument.addEventListener(`touch${phase}`, trackUncontrolledContact, { + capture: true, + passive: true, + }); listen(`touch${phase}`, (event) => { touchActive = phase !== "cancel" && (event.touches?.length ?? 0) > 0; const points = Array.from(event.touches ?? []).map((touch) => { @@ -256,16 +244,8 @@ export function bindBrowserCanvasWeb( } /** Cursor enums originate from the guarded remote response, never custom URLs. */ -export function setBrowserCanvasCursor( - node: unknown, - cursor: BrowserCursor | null, -): void { - if ( - Platform.OS === "web" && - node && - typeof node === "object" && - "style" in node - ) { +export function setBrowserCanvasCursor(node: unknown, cursor: BrowserCursor | null): void { + if (Platform.OS === "web" && node && typeof node === "object" && "style" in node) { (node as BrowserCanvasNode).style.cursor = cursor ?? "default"; } } diff --git a/paseo-shared-browser/docs/fork-reconciliation.md b/paseo-shared-browser/docs/fork-reconciliation.md index 93ed094c..a9af5b29 100644 --- a/paseo-shared-browser/docs/fork-reconciliation.md +++ b/paseo-shared-browser/docs/fork-reconciliation.md @@ -1,172 +1,59 @@ -# Shared Browser interaction and display fork - -This branch is rebased onto upstream main `d7b3e654` (Shared Browser 1.2.0). It is uncommitted and has not been submitted as a pull request. Dependencies -retain the repository catalog declarations. Runtime assets, browser profiles, -credentials, local backups and application data are excluded. - -## Changes preserved - -- Display sizes through 2560 by 2560, desktop aspect ratio presets, 1280 by 800 - and 1280 square presets, and a sharper Pixel capture mode. -- Higher capture quality, exact capture dimensions, reconnect restoration, - stable capture transport and decoded frame buffering. -- Physical hover, cursor feedback, wheel, right and double clicks, continuous - mouse dragging, native scrollbar dragging and genuine touch gestures. -- Natural input without click or drag mode selectors, with preserved keyboard, - text, navigation, display and control lease surfaces. -- Ordered gesture channels, bounded pressure, cleanup on lost ownership and - safe recovery of stale gesture admission. Published actions are not replayed. - -## Validation - -The combined registered suite passed 226 tests, including native relay, viewer -recovery, acknowledged-navigation, menu and local-view sizing additions. Actual Chromium checks verified trusted native keyboard -input, Unicode, repeat, shortcuts, focus traversal, Ctrl+click and key release. -Quality 70, 90 and 95 produced distinct 9,868, 11,131 and 11,951-byte frames on the -same isolated page. Mounted tests separately cover DOM keyboard/composition, -preference revision conflicts, observer safety and first-control device defaults. -The actual Paseo client/server compiler and mobile Hermes compiler passed. -The loaded runtime also rendered the signed-in nine-printer fleet. Low, Medium -and High captures returned distinct 93,155, 149,326 and 198,233-byte JPEGs, each -1280 by 800 pixels, before restoring the earlier square display. - -These checks use disposable profiles and fixtures. They do not establish human -pane forwarding or physical-phone software-keyboard behavior. The user confirmed -native scrollbar dragging in the real pane. Discrete MCP retains strict stale -frame validation and does not exercise the human pane gesture queue. - -## Native desktop hover capability - -Chromium headless desktop currently reports no pointer or hover capability. -Media-query-gated hover styles therefore do not activate on every control. -Startup Blink settings cannot restore those capabilities after touch emulation -is disabled. Preserve page state: do not reset the page on device changes or -compensate with application CSS. Hidden Linux runtimes now use an authenticated, -owned Xvfb display when the existing executable is available. It supplies genuine -embedder mouse preferences, so desktop -> phone -> desktop and CDP reconnect -retain the correct pointer/hover media. Abstract-only Unix transport avoids WSLg's -protected pathname sockets; TCP and host display access remain disabled. Startup -fallback is allowed only after full display cleanup and before Chromium exists. -Missing Xvfb and other platforms retain their existing behavior; they are not -covered by the Linux capability correction. - -## Upstream reconciliation - -| Upstream change | Reconciliation | -| --- | --- | -| #263 pointer mapping | Reuse upstream contained geometry helpers, retain capture-scale-aware coordinates and decoded-frame authority. | -| #264 frame settlement | Preserve the upstream discrete-input epoch gate and regressions; gate on decoded frames. Use a factory so the actual mobile Hermes compiler accepts the client bundle. Continuous input keeps its ordered channel. | -| #266 viewport resync | Integrate one bounded reapplication of saved emulation for valid dimension mismatches, using capture scale rather than device pixel ratio. Malformed JPEG and superseded capture remain rejected. | -| #258 sidebar and OMP MCP | Retain upstream sidebar, capability guards, entrypoint and tests. | -| #248 Paseo compatibility | Retain stable 0.11 support and the specifically tested 0.11.0-beta.3 allowance. | -| Release metadata | Keep upstream version 1.2.0 and catalogue dependencies; do not publish an independent release. | - -Input limitations and primary references are in [input-forwarding-notes.md](input-forwarding-notes.md). -Keep runtime assets, browser profiles, machine-specific paths and backups outside -the source diff. Changes remain uncommitted for review. The original pre-rebase -stash remains a recovery backup; it is not a pending patch to apply again. - -## Toolbar and completed-input follow-up - -The toolbar now owns mobile emulation, favorite resolution selection and the -browser action menu. Bottom favorite, emulation and key rows are removed. -Favorite indicators use paired filled/outline star glyphs because the host Icon -API exposes stroke only. No selected raised surface is used for the star button. - -Post-update daemon logs contained viewer expiry, stale navigation and live input -attachment changes. Isolated real Chromium reproduced a completed click that -navigated but returned a stale-navigation failure. Completion now accepts only -acknowledged publication plus a document navigation on the same attachment, -closes the old gesture and returns fresh viewing state. Unpublished and uncertain -commands, replaced controls, viewports, runtime targets and bridges still fail. -A separate actual Enter form-submit test confirms one submission and released -held keys. The shared browser is never used for these write-capable fixtures. - -The completed-input follow-up also passes mounted desktop/compact toolbar, -filled-star preference and native keyboard request checks. Mounted keyboard and -touch fixtures confirm held inputs are quarantined until physical release after -acknowledged navigation. Old-document continuations are discarded; no action is -replayed and no obsolete channel-end request is sent. Independent regression -checks retain refusal of a target replacement after publication. - -## Combined toolbar menus and local view sizing - -The monitor control combines favorites, the full resolution/quality list and -local Fit/Actual size selection. The vertical-dots browser menu contains actions, -without duplicated resolution access. Both use one shared anchored-menu surface -with bounded placement, keyboard focus and dismissal. Only public RN primitives -and themed SDK icons are used; the SDK has no public inline menu API. - -Actual size preserves canonical CSS layout dimensions and decoded frame slots. -Image and input overlay share one geometry, including high-density captures. -Desktop uses one scroll host so both local scrollbars stay at the pane edges. -Native observation permits local panning; active control reserves gestures for -the remote page. Scale selection is local to the mounted viewer, and changing -scale cancels held remote input. The browser page and shared resolution persist. - -The combined follow-up passes 226 registered tests, four mounted toolbar/canvas -cases, TypeScript, the actual Paseo compiler and the mobile Hermes compiler. -These validate rendering structure, focus, local scroll styles and input mapping; -they do not certify physical-phone native panning or software-keyboard behavior. - -## Mode-only display preservation and key submenu - -The mobile toggle and the first-control device default now retain the current -CSS viewport dimensions and JPEG capture density. The policy resolves these -values under the session lock, cancels held input, and invalidates old frames -even when dimensions do not change. Explicit resolution presets retain their -full display behavior. Preset checkmarks and the frame label require an exact -display and capture-density match, preventing a remembered behavior profile -from incorrectly claiming a selected resolution. - -Send keys is a nested toolbar menu, not a separate dialog. Desktop uses a -bounded flyout; compact/native clients use a Back row. One backdrop owns outside -dismissal. Keyboard navigation scopes each menu level, returns to the parent -trigger, and retains the existing command guards and dispatch callback. - -The combined registered suite passes 236 tests. Disposable Chromium confirms -unchanged 1280 x 800 mode switches and preserved 824 x 1678 sharp captures, -with page drafts intact and no reload. Seven mounted mode-hook cases and five -mounted submenu cases cover ownership, pending/stale callbacks, keyboard -hierarchy and single command dispatch. TypeScript, the actual Paseo compiler -and the mobile Hermes compiler pass. Human host-pane interactions still need -direct observation; browser MCP images cannot establish their behavior. - -## Actual-size bounds, native hover and frame delivery - -Actual size now sizes the canvas to the remote CSS dimensions and lets the scroll -host center it. An outer border or a single scrollbar no longer creates overflow -on the other axis. Container size changes cancel held input because recentering -can move the frame origin without changing its CSS dimensions. Disposable -Chromium measured 16 before/after cases covering fitting and exact-edge frames, -fractional borders, each overflowing axis, classic and zero-width scrollbar -geometry, edge reachability, sharp captures and pointer mapping after panning. -These use the public React Native Web CSS shape, not a physical phone. - -An owned private Linux display supplies genuine desktop hover, including after -phone emulation and CDP reconnect. The actual authenticated runtime preserves a -page draft through those switches and produces 824 x 1678 sharp phone and -2560 x 2560 desktop captures. Activating this launch change for an existing -headless browser requires one browser restart; later mode switches do not reload -the page. Missing Xvfb and other platforms keep their existing behavior. - -Frame polling reuses only the exact runtime receipt under matching session, -runtime, bridge, navigation, viewport and quality authority. Repeated cached -frames return metadata instead of sending the JPEG again. Runtime age and -new-capture invalidation remain enforced. A disposable idle-page fixture reduced -three repeated poll payloads by 72.94%, without changing image quality. Fresh -page metadata reads run concurrently under attachment/document guards. The -measured median was 3.434 ms before and 2.121 ms after; guarded input median was -16.971 ms before and 15.504 ms after, with no established tail-latency improvement. -This remains JPEG-frame transport, not an encoded video stream or a frame-rate -claim. A memoized image leaf also avoids decoding/rendering unchanged visual -props on unrelated parent updates. - -The compact address field removes excess vertical padding and stays at full -contrast while observing. Its control and navigation permissions are unchanged. - -The registered suite passes 264 tests. Four mounted toolbar/canvas checks and -two mounted image authority/memoization checks also pass, as do TypeScript, the -actual Paseo compiler and mobile Hermes compiler. Linux fixture checks prove -native hover and display lifecycle; human host-pane performance and physical -native-phone input/layout still need direct observation. +# Shared Browser interaction and display integration + +This document records the contracts behind the interaction and display changes +on the Shared Browser 1.2 runtime. Installation and controls are documented in +[README](../README.md). Input caveats and primary sources are documented in +[input-forwarding-notes](input-forwarding-notes.md). + +## Upstream contracts retained + +- Reuse upstream contained-image pointer mapping. CSS coordinates remain + independent of device pixel ratio and capture density. +- Retain the discrete-input frame epoch gate and viewport-resync checks. A + malformed or superseded capture cannot trigger emulation recovery. +- Preserve sidebar and MCP integration, plugin identity, catalog dependencies + and package version. Release Please continues to own release metadata. +- Require the SDK generation used by upstream's sidebar and server APIs: + stable Paseo 0.11 and the explicitly tested 0.11.0-beta.3 allowance. + +## Input and presentation ownership + +Human input uses ordered gesture channels. Motion can coalesce only without +crossing a press, release, key or text transition. Ownership loss, page changes +and viewport replacement cancel held state. No published action is replayed +when its acknowledgement is uncertain. Agent commands remain fresh-frame bound. + +A decoded image and its targeting receipt settle together. Late image loads +cannot replace a newer paint. Cached pixels may avoid a redundant payload only +under matching runtime, bridge, navigation, viewport and quality authority; +source age and input invalidation still apply. + +Fit and Actual size are local viewer choices. Actual size uses CSS dimensions, +not sharper image dimensions. Overflow is measured independently per axis, and +container geometry changes cancel active gestures. Observing phone users can +pan locally; controlling phone users send their gestures to the remote page. + +Shared emulation changes preserve viewport dimensions. Device defaults apply +only on first human control, so an observer cannot resize another user's page. +Native-phone composition is explicitly committed rather than applied as remote +caret edits when the operating system changes a local draft. + +## Linux hover capability + +Headless Chromium may report no desktop pointer/hover capability even after +touch emulation is disabled. When Xvfb is available, an owned private display +supplies genuine desktop capability without reloading the document on mode +changes. It uses authenticated abstract Unix transport and disables TCP and +pathname listeners. The display is stopped with its browser. Other platforms +or missing Xvfb retain existing launch behavior. + +## Verification boundaries + +Run the suite, formatting, TypeScript and real browser smoke from the active +checkout. Use the isolated runtime workflow in README. Test profiles, runtime +assets and credentials must never enter the source tree. + +Unit fixtures cover input ordering, frame authority and local geometry. Owned +Chromium fixtures cover native input and display lifecycle. These checks do not +establish physical-phone software keyboard behavior or latency through every +network topology; those require direct device observation. diff --git a/paseo-shared-browser/index.server.ts b/paseo-shared-browser/index.server.ts index dfa67e70..120ff7ea 100644 --- a/paseo-shared-browser/index.server.ts +++ b/paseo-shared-browser/index.server.ts @@ -8,16 +8,16 @@ import { handleAcquireControl, handleApplyDevicePreset, handleAttachBrowser, + handleBeginBrowserGesture, handleCaptureBrowser, handleDetachBrowser, + handleEndBrowserGesture, handleListOpenBrowserWorkspaces, handleNavigateBrowser, handleReleaseControl, handleResizeBrowser, handleSendBrowserInput, - handleBeginBrowserGesture, handleUpdateBrowserGesture, - handleEndBrowserGesture, handleWorkspaceArchived, issueAgentTicket, revokeAgentBrowserAccess, @@ -27,16 +27,16 @@ import { acquireControlRpc, applyDevicePresetRpc, attachBrowserRpc, + beginBrowserGestureRpc, captureBrowserRpc, detachBrowserRpc, + endBrowserGestureRpc, listOpenBrowserWorkspacesRpc, navigateBrowserRpc, releaseControlRpc, resizeBrowserRpc, sendBrowserInputRpc, - beginBrowserGestureRpc, updateBrowserGestureRpc, - endBrowserGestureRpc, } from "./shared/browser"; import { browserDisplayPreferences } from "./shared/browser-display-preferences"; diff --git a/paseo-shared-browser/paseo-plugin.json b/paseo-shared-browser/paseo-plugin.json index 6b244031..e14a566f 100644 --- a/paseo-shared-browser/paseo-plugin.json +++ b/paseo-shared-browser/paseo-plugin.json @@ -2,17 +2,7 @@ "id": "shared-browser", "description": "One real Chromium session per Paseo workspace, shared live across every connected client.", "requirements": { - "paseo": "^0.9.0 || ^0.10.0 || ^0.11.0 || 0.11.0-beta.3" + "paseo": "^0.11.0 || 0.11.0-beta.3" }, - "build": [ - [ - "node", - "scripts/prepare-dependencies.mjs" - ], - [ - "npm", - "run", - "prepare:runtime" - ] - ] + "build": [["node", "scripts/prepare-dependencies.mjs"], ["npm", "run", "prepare:runtime"]] } diff --git a/paseo-shared-browser/server/agent-browser-runtime.ts b/paseo-shared-browser/server/agent-browser-runtime.ts index ff444c31..03914aec 100644 --- a/paseo-shared-browser/server/agent-browser-runtime.ts +++ b/paseo-shared-browser/server/agent-browser-runtime.ts @@ -2,6 +2,8 @@ import { execFile } from "node:child_process"; import { chmod, mkdir, readFile, rm } from "node:fs/promises"; import { isAbsolute, resolve } from "node:path"; import { promisify } from "node:util"; +import { captureDimensions, DEFAULT_JPEG_QUALITY } from "../shared/capture-settings"; +import { MAX_VIEWPORT } from "../shared/viewport-limits"; import { attachToTarget, CdpConnection, @@ -12,17 +14,20 @@ import { CdpUnknownOutcomeError, listPageTargets, } from "./cdp"; - -import { DEFAULT_JPEG_QUALITY, captureDimensions } from "../shared/capture-settings"; -import { MAX_VIEWPORT } from "../shared/viewport-limits"; import { readJpegFrameDimensions } from "./jpeg-frame"; + export { readJpegFrameDimensions as jpegDimensions } from "./jpeg-frame"; -import { CAPTURE_MAX_AGE_MS, createCaptureTransportPolicy } from "./capture-transport-policy"; -import { browserCursorSchema, type BrowserCursor } from "../shared/browser"; -import { nativeKeyEvent, heldKeyModifiers } from "./keyboard-input"; -import { browserGestureKeySchema, type BrowserGestureKeyEvent } from "../shared/browser"; + +import { + type BrowserCursor, + type BrowserGestureKeyEvent, + browserCursorSchema, + browserGestureKeySchema, +} from "../shared/browser"; import { GESTURE_IDLE_MS, GESTURE_LIFETIME_MS } from "./browser-gesture"; +import { CAPTURE_MAX_AGE_MS, createCaptureTransportPolicy } from "./capture-transport-policy"; import { formatRuntimeInputGeneration } from "./input-generation"; +import { heldKeyModifiers, nativeKeyEvent } from "./keyboard-input"; const execFileAsync = promisify(execFile); export const AGENT_BROWSER_VERSION = "0.37.1"; @@ -179,7 +184,7 @@ function runtimeEnvironment( const authority = launchEnvironment.XAUTHORITY; if ( keys.length !== 2 || - keys.some(key => key !== "DISPLAY" && key !== "XAUTHORITY") || + keys.some((key) => key !== "DISPLAY" && key !== "XAUTHORITY") || typeof display !== "string" || !/^:[0-9]{1,5}(?:\.[0-9]{1,2})?$/.test(display) || typeof authority !== "string" || @@ -251,7 +256,15 @@ export class AgentBrowserRuntime { private heldTouches = false; private activeTouches = new Map(); private documentGeneration = 0; - private liveInput: { id: string; page: CdpSession; generation: number; documentGeneration: number; idleUntil: number; expiresAt: number; timer: ReturnType | null } | null = null; + private liveInput: { + id: string; + page: CdpSession; + generation: number; + documentGeneration: number; + idleUntil: number; + expiresAt: number; + timer: ReturnType | null; + } | null = null; private stopping = false; constructor(options: AgentBrowserRuntimeOptions) { @@ -442,7 +455,10 @@ export class AgentBrowserRuntime { title: evaluated.result.value?.title ?? history.entries[history.currentIndex]?.title ?? "", canGoBack: history.currentIndex > 0, canGoForward: history.currentIndex < history.entries.length - 1, - inputGeneration: formatRuntimeInputGeneration(this.attachmentGeneration, this.documentGeneration), + inputGeneration: formatRuntimeInputGeneration( + this.attachmentGeneration, + this.documentGeneration, + ), }; } @@ -521,7 +537,8 @@ export class AgentBrowserRuntime { this.emulationAppliedPage = null; await this.applyEmulation(page, device); this.assertAttachmentCurrent(page.connection, attachmentGeneration); - if (this.page !== page) throw new CdpUnavailableError("Browser page changed during emulation"); + if (this.page !== page) + throw new CdpUnavailableError("Browser page changed during emulation"); this.viewport = { ...device }; this.emulationAppliedPage = page; }); @@ -577,7 +594,11 @@ export class AgentBrowserRuntime { /** Reject late configuration completions after a disconnect or newer target selection. */ private assertAttachmentCurrent(connection: CdpConnection, generation: number): void { - if (connection !== this.connection || !connection.isOpen || generation !== this.attachmentGeneration) { + if ( + connection !== this.connection || + !connection.isOpen || + generation !== this.attachmentGeneration + ) { throw new CdpUnavailableError("Browser attachment changed during emulation restoration"); } } @@ -618,8 +639,12 @@ export class AgentBrowserRuntime { if (requestedQuality !== this.screencastQuality) { // Viewers choose their own quality; never restart the shared stream or // classify its health from another viewer's screenshot preference. - const cached = this.capturePolicy.readScreenshot(performance.now(), requestedQuality, maxBytes); - return cached ?? await this.captureScreenshot(maxBytes, requestedQuality, true); + const cached = this.capturePolicy.readScreenshot( + performance.now(), + requestedQuality, + maxBytes, + ); + return cached ?? (await this.captureScreenshot(maxBytes, requestedQuality, true)); } // Expiry discards pixels without resetting fallback dwell/recovery evidence. // Mutations and connection changes use the stronger invalidation below. @@ -648,9 +673,15 @@ export class AgentBrowserRuntime { } /** Capture the configured viewport and cache only an unchanged-session/mutation completion. */ - private async captureScreenshot(maxBytes: number, requestedQuality: number, preserveStreamRecovery = false): Promise { + private async captureScreenshot( + maxBytes: number, + requestedQuality: number, + preserveStreamRecovery = false, + ): Promise { const page = await this.requirePage(); - const screenshotGeneration = this.capturePolicy.beginScreenshot({ preserveRecovery: preserveStreamRecovery }); + const screenshotGeneration = this.capturePolicy.beginScreenshot({ + preserveRecovery: preserveStreamRecovery, + }); try { const viewport = this.requireViewport(); const expectedPixels = captureDimensions(viewport, viewport.captureScale); @@ -689,7 +720,11 @@ export class AgentBrowserRuntime { this.assertCaptureCurrent(page, screenshotGeneration); const dimensions = readJpegFrameDimensions(result.data); if (!dimensions) throw new Error("Chromium returned a malformed JPEG screenshot"); - return { data: result.data, dimensions, byteLength: Buffer.byteLength(result.data, "base64") }; + return { + data: result.data, + dimensions, + byteLength: Buffer.byteLength(result.data, "base64"), + }; }; const matchesViewport = (dimensions: { width: number; height: number }) => dimensions.width === expectedPixels.width && dimensions.height === expectedPixels.height; @@ -706,13 +741,16 @@ export class AgentBrowserRuntime { await this.applyEmulation(page, viewport); this.assertAttachmentCurrent(page.connection, attachmentGeneration); this.assertCaptureCurrent(page, screenshotGeneration); - if (viewport !== this.viewport) throw new CdpUnavailableError("Browser viewport changed during resync"); + if (viewport !== this.viewport) + throw new CdpUnavailableError("Browser viewport changed during resync"); this.emulationAppliedPage = page; clip = await readClip(); pixels = await capture(boundedQuality); } if (!matchesViewport(pixels.dimensions)) { - throw new Error(`Chromium returned a ${pixels.dimensions.width}x${pixels.dimensions.height} JPEG instead of ${expectedPixels.width}x${expectedPixels.height}`); + throw new Error( + `Chromium returned a ${pixels.dimensions.width}x${pixels.dimensions.height} JPEG instead of ${expectedPixels.width}x${expectedPixels.height}`, + ); } if (pixels.byteLength > maxBytes) continue; @@ -747,23 +785,29 @@ export class AgentBrowserRuntime { } /** Invalidate pre-input pixels even if Chromium cannot confirm the mutation. */ - private async dispatchInput(method: string, params: Record, gestureId?: string): Promise { + private async dispatchInput( + method: string, + params: Record, + gestureId?: string, + ): Promise { try { const page = await this.requirePage(); if (gestureId) await this.assertLiveInput(gestureId); const originalInput = gestureId ? this.liveInput : null; const modifiers = heldKeyModifiers(this.heldKeys.values()); - const nativeParams = method === "Input.dispatchMouseEvent" && modifiers - ? { ...params, modifiers } - : params; + const nativeParams = + method === "Input.dispatchMouseEvent" && modifiers ? { ...params, modifiers } : params; await page.send(method, nativeParams, { mutation: true }); if (gestureId) { // An input can synchronously activate a link or submit a form. Its CDP // acknowledgment succeeds even when navigation ends the old channel. // Attachment replacement and uncertain sends retain their failure path. - if (originalInput && page === this.page - && originalInput.generation === this.attachmentGeneration - && originalInput.documentGeneration !== this.documentGeneration) { + if ( + originalInput && + page === this.page && + originalInput.generation === this.attachmentGeneration && + originalInput.documentGeneration !== this.documentGeneration + ) { await this.endLiveInput(gestureId); return; } @@ -786,16 +830,25 @@ export class AgentBrowserRuntime { // Native scrollbar dragging needs the held button as well as its mask. button: this.heldMoveButton(), buttons: this.buttonMask(), - }, gestureId, + }, + gestureId, ); } /** Fixed native mouse departure; never maps an outside point to a page control. */ async mouseLeave(gestureId: string): Promise { if (this.heldButtons.size > 0) throw new Error("Release held mouse buttons before leaving"); - await this.dispatchInput("Input.dispatchMouseEvent", { - type: "mouseMoved", x: -1, y: -1, button: "none", buttons: 0, - }, gestureId); + await this.dispatchInput( + "Input.dispatchMouseEvent", + { + type: "mouseMoved", + x: -1, + y: -1, + button: "none", + buttons: 0, + }, + gestureId, + ); } async mouseDown( @@ -819,11 +872,18 @@ export class AgentBrowserRuntime { button, buttons: this.buttonMask(button), clickCount, - }, gestureId, + }, + gestureId, ); } - async mouseUp(x: number, y: number, button: MouseButton = "left", clickCount = 1, gestureId?: string): Promise { + async mouseUp( + x: number, + y: number, + button: MouseButton = "left", + clickCount = 1, + gestureId?: string, + ): Promise { this.assertPoint(x, y); await this.dispatchInput( "Input.dispatchMouseEvent", @@ -834,12 +894,19 @@ export class AgentBrowserRuntime { button, buttons: this.buttonMask(undefined, button), clickCount, - }, gestureId, + }, + gestureId, ); this.heldButtons.delete(button); } - async wheel(x: number, y: number, deltaX: number, deltaY: number, gestureId?: string): Promise { + async wheel( + x: number, + y: number, + deltaX: number, + deltaY: number, + gestureId?: string, + ): Promise { this.assertPoint(x, y); await this.dispatchInput( "Input.dispatchMouseEvent", @@ -851,7 +918,8 @@ export class AgentBrowserRuntime { deltaY, button: "none", buttons: this.buttonMask(), - }, gestureId, + }, + gestureId, ); } @@ -866,10 +934,13 @@ export class AgentBrowserRuntime { await this.assertLiveInput(gestureId); const held = this.heldKeys.has(parsed.code); if (parsed.type === "down" && held !== parsed.repeat) { - throw new Error(parsed.repeat ? "Key repeat has no matching press" : "Key is already pressed"); + throw new Error( + parsed.repeat ? "Key repeat has no matching press" : "Key is already pressed", + ); } if (parsed.type === "up" && !held) throw new Error("Key release has no matching press"); - if (parsed.type === "down") this.heldKeys.set(parsed.code, { key: parsed.key, code: parsed.code }); + if (parsed.type === "down") + this.heldKeys.set(parsed.code, { key: parsed.key, code: parsed.code }); await this.dispatchInput("Input.dispatchKeyEvent", nativeKeyEvent(parsed), gestureId); if (parsed.type === "up") this.heldKeys.delete(parsed.code); } @@ -877,7 +948,10 @@ export class AgentBrowserRuntime { async keyDown(key: string, code = key): Promise { this.heldKeys.set(code, { key, code }); await this.dispatchInput("Input.dispatchKeyEvent", { - type: "keyDown", key, code, text: key.length === 1 ? key : undefined, + type: "keyDown", + key, + code, + text: key.length === 1 ? key : undefined, }); } @@ -899,15 +973,24 @@ export class AgentBrowserRuntime { for (const point of points) this.assertPoint(point.x, point.y); if (gestureId) await this.assertLiveInput(gestureId); const contacts = points.map((point, index) => ({ ...point, id: point.id ?? index })); - const activeIds = new Set(contacts.map(point => point.id)); - const removed = type === "touchMove" - ? [...this.activeTouches.values()].filter(point => !activeIds.has(point.id)) - : []; + const activeIds = new Set(contacts.map((point) => point.id)); + const removed = + type === "touchMove" + ? [...this.activeTouches.values()].filter((point) => !activeIds.has(point.id)) + : []; this.heldTouches = true; if (removed.length > 0) { - await this.dispatchInput("Input.dispatchTouchEvent", { type: "touchEnd", touchPoints: removed }, gestureId); + await this.dispatchInput( + "Input.dispatchTouchEvent", + { type: "touchEnd", touchPoints: removed }, + gestureId, + ); } - await this.dispatchInput("Input.dispatchTouchEvent", { type, touchPoints: contacts }, gestureId); + await this.dispatchInput( + "Input.dispatchTouchEvent", + { type, touchPoints: contacts }, + gestureId, + ); // Navigation cleanup already discarded this channel's contacts. Do not // recreate held-touch state after its acknowledged start caused navigation. if (gestureId && this.liveInput?.id !== gestureId) return; @@ -923,19 +1006,34 @@ export class AgentBrowserRuntime { if (this.liveInput) await this.endLiveInput(this.liveInput.id); const page = await this.requirePage(); const now = Date.now(); - this.liveInput = { id, page, generation: this.attachmentGeneration, documentGeneration: this.documentGeneration, - idleUntil: now + GESTURE_IDLE_MS, expiresAt: now + GESTURE_LIFETIME_MS, timer: null }; + this.liveInput = { + id, + page, + generation: this.attachmentGeneration, + documentGeneration: this.documentGeneration, + idleUntil: now + GESTURE_IDLE_MS, + expiresAt: now + GESTURE_LIFETIME_MS, + timer: null, + }; this.scheduleLiveInputExpiry(); } /** Compare after every asynchronous transport boundary, including automatic reconnect/reattach. */ async assertLiveInput(id: string): Promise { const page = await this.requirePage(); - if (this.liveInput?.id === id && (Date.now() >= this.liveInput.idleUntil || Date.now() >= this.liveInput.expiresAt)) { + if ( + this.liveInput?.id === id && + (Date.now() >= this.liveInput.idleUntil || Date.now() >= this.liveInput.expiresAt) + ) { await this.endLiveInput(id); } - if (!this.liveInput || this.liveInput.id !== id || this.liveInput.page !== page - || this.liveInput.generation !== this.attachmentGeneration || this.liveInput.documentGeneration !== this.documentGeneration) { + if ( + !this.liveInput || + this.liveInput.id !== id || + this.liveInput.page !== page || + this.liveInput.generation !== this.attachmentGeneration || + this.liveInput.documentGeneration !== this.documentGeneration + ) { throw new CdpUnavailableError("Live browser input attachment changed"); } } @@ -982,7 +1080,9 @@ export class AgentBrowserRuntime { await this.assertLiveInput(gestureId); const parsed = browserCursorSchema.safeParse(result.result.value); return parsed.success ? parsed.data : null; - } catch { return null; } + } catch { + return null; + } } async releaseHeldInput(page = this.page): Promise { @@ -1001,7 +1101,15 @@ export class AgentBrowserRuntime { this.heldTouches = false; this.activeTouches.clear(); await Promise.allSettled([ - ...(touchHeld ? [page.send("Input.dispatchTouchEvent", { type: "touchCancel", touchPoints: [] }, { mutation: true })] : []), + ...(touchHeld + ? [ + page.send( + "Input.dispatchTouchEvent", + { type: "touchCancel", touchPoints: [] }, + { mutation: true }, + ), + ] + : []), ...buttons.map((button) => page.send( "Input.dispatchMouseEvent", @@ -1309,7 +1417,9 @@ export class AgentBrowserRuntime { private onScreencastFrame(page: CdpSession, event: ScreencastFrame): void { // A late frame belongs to its emitting session, even after page reattachment. - void page.send("Page.screencastFrameAck", { sessionId: event.sessionId }).catch(() => undefined); + void page + .send("Page.screencastFrameAck", { sessionId: event.sessionId }) + .catch(() => undefined); if (page !== this.page || !this.screencastActive || this.emulationAppliedPage !== page) return; const dimensions = readJpegFrameDimensions(event.data); const viewport = this.viewport; @@ -1317,7 +1427,8 @@ export class AgentBrowserRuntime { // Reject those pixels and use the bounded screenshot fallback rather than mislabeling them. if (!viewport || !dimensions) return; const expectedPixels = captureDimensions(viewport, viewport.captureScale); - if (dimensions.width !== expectedPixels.width || dimensions.height !== expectedPixels.height) return; + if (dimensions.width !== expectedPixels.width || dimensions.height !== expectedPixels.height) + return; const { width, height } = dimensions; const frame: RuntimeFrame = { dataBase64: event.data, @@ -1329,9 +1440,10 @@ export class AgentBrowserRuntime { }; const now = performance.now(); const timestamp = event.metadata.timestamp; - const identity = typeof timestamp === "number" && Number.isFinite(timestamp) - ? `provider:${timestamp}` - : `event:${++this.streamEventSequence}`; + const identity = + typeof timestamp === "number" && Number.isFinite(timestamp) + ? `provider:${timestamp}` + : `event:${++this.streamEventSequence}`; this.capturePolicy.observeStream(identity, now); this.screencastFrame = frame; this.screencastFrameReceivedAt = now; @@ -1361,8 +1473,13 @@ export class AgentBrowserRuntime { private assertViewport(viewport: BrowserViewport): void { const captureScale = viewport.captureScale ?? 1; const pixels = captureDimensions(viewport, captureScale); - if (!Number.isFinite(captureScale) || captureScale < 1 || captureScale > 2 - || pixels.width > MAX_VIEWPORT.width || pixels.height > MAX_VIEWPORT.height) { + if ( + !Number.isFinite(captureScale) || + captureScale < 1 || + captureScale > 2 || + pixels.width > MAX_VIEWPORT.width || + pixels.height > MAX_VIEWPORT.height + ) { throw new RangeError("Capture density exceeds the supported image bounds"); } if ( diff --git a/paseo-shared-browser/server/browser-gesture.ts b/paseo-shared-browser/server/browser-gesture.ts index 6e2697f7..83b6629d 100644 --- a/paseo-shared-browser/server/browser-gesture.ts +++ b/paseo-shared-browser/server/browser-gesture.ts @@ -63,7 +63,9 @@ export class BrowserGesture { throw new Error("Release held mouse buttons before leaving"); if (event.kind === "key") { if (event.type === "down" && this.keys.has(event.code) !== event.repeat) { - throw new Error(event.repeat ? "Key repeat has no matching press" : "Key is already pressed"); + throw new Error( + event.repeat ? "Key repeat has no matching press" : "Key is already pressed", + ); } if (event.type === "up" && !this.keys.has(event.code)) { throw new Error("Key release has no matching press"); diff --git a/paseo-shared-browser/server/browser-policy.ts b/paseo-shared-browser/server/browser-policy.ts index 7010c60d..a6b3e2bd 100644 --- a/paseo-shared-browser/server/browser-policy.ts +++ b/paseo-shared-browser/server/browser-policy.ts @@ -3,30 +3,35 @@ import type { RpcInput, RpcOutput } from "@getpaseo/plugin"; import { type applyDevicePresetRpc, type BrowserFrame, - type BrowserInputEvent, type BrowserGestureEvent, + type BrowserInputEvent, type BrowserState, + type beginBrowserGestureRpc, + browserCursorSchema, + browserGestureEventSchema, DEFAULT_VIEWPORT, DEVICE_PRESETS, type DevicePresetId, + type endBrowserGestureRpc, FRAME_MAX_BYTES, MAX_VIEWPORT, MIN_VIEWPORT, mapDisplayedPoint, - type beginBrowserGestureRpc, - type updateBrowserGestureRpc, - type endBrowserGestureRpc, - browserGestureEventSchema, - browserCursorSchema, type navigateBrowserRpc, type resizeBrowserRpc, type sendBrowserInputRpc, + type updateBrowserGestureRpc, type Viewport, } from "../shared/browser"; -import type { JsonValue } from "./runtime-protocol"; -import { DEFAULT_CAPTURE_QUALITY, DEFAULT_JPEG_QUALITY, JPEG_QUALITY, captureDimensions } from "../shared/capture-settings"; +import { + captureDimensions, + DEFAULT_CAPTURE_QUALITY, + DEFAULT_JPEG_QUALITY, + JPEG_QUALITY, +} from "../shared/capture-settings"; import { BrowserGesture } from "./browser-gesture"; import { sameRuntimeInputAttachment } from "./input-generation"; +import type { JsonValue } from "./runtime-protocol"; export type WorkspaceValidator = (workspaceId: string) => Promise; type NavigateInput = RpcInput; type ResizeInput = RpcInput; @@ -34,7 +39,9 @@ type ApplyDevicePresetInput = RpcInput; type SendInput = RpcInput; type BeginGestureInput = RpcInput; type UpdateGestureInput = RpcInput; -type UpdateGestureReply = Omit, "state"> & { state: BrowserState }; +type UpdateGestureReply = Omit, "state"> & { + state: BrowserState; +}; type EndGestureInput = RpcInput; type CaptureQuality = "low" | "medium" | "high"; type InputTarget = SendInput["target"]; @@ -96,12 +103,15 @@ interface BrowserSession { viewers: Map; controller: Controller | null; mutationTail: Promise; - frameCache: Map; - }>; + frameCache: Map< + CaptureQuality, + { + frame: BrowserFrame; + cachedAt: number; + /** Exact runtime receipt, distinct from its normalized public frame fields. */ + rawFrame: Record; + } + >; recentFrames: Map< string, { navigationGeneration: number; viewportGeneration: number; expiresAt: number } @@ -459,12 +469,21 @@ export class SessionManager { // may obtain another decoded frame before admitting its still-unsent input. const state = await this.snapshotState(session, input.viewerToken); this.requireMutationAccess(session, input); - if (state.status !== "ready") throw new Error(state.error ?? "Browser runtime is unavailable"); + if (state.status !== "ready") + throw new Error(state.error ?? "Browser runtime is unavailable"); return { state, admission: "stale-frame" as const }; } await this.cancelGesture(session); - const gesture = new BrowserGesture(this.issueUniqueToken(), input.viewerToken, input.controlToken, - input.pointerKind, { ...input.expected }, { ...session.viewport }, this.now(), session.inputGeneration); + const gesture = new BrowserGesture( + this.issueUniqueToken(), + input.viewerToken, + input.controlToken, + input.pointerKind, + { ...input.expected }, + { ...session.viewport }, + this.now(), + session.inputGeneration, + ); session.gesture = gesture; try { await this.request(session, "input.begin", { gestureId: gesture.id }); @@ -494,32 +513,56 @@ export class SessionManager { gesture.assertSequence(input.sequence, this.now()); await this.assertGestureCurrent(session, gesture); gesture.validate(event); - if (event.kind === "down" || (event.kind === "touch" && event.type === "start" && gesture.touches.size === 0)) { + if ( + event.kind === "down" || + (event.kind === "touch" && event.type === "start" && gesture.touches.size === 0) + ) { if (!input.target) throw new Error("A current frame is required for a new press"); this.requireRecentFrame(session, input.target); } const hover = gesture.isHover(event); if (event.kind === "key") { - await this.request(session, "input.key", { gestureId: gesture.id, event: { - kind: event.kind, type: event.type, key: event.key, code: event.code, - modifiers: event.modifiers, repeat: event.repeat, - ...(event.text !== undefined ? { text: event.text } : {}), - } }); + await this.request(session, "input.key", { + gestureId: gesture.id, + event: { + kind: event.kind, + type: event.type, + key: event.key, + code: event.code, + modifiers: event.modifiers, + repeat: event.repeat, + ...(event.text !== undefined ? { text: event.text } : {}), + }, + }); } else if (event.kind === "text") { await this.request(session, "input.text", { gestureId: gesture.id, text: event.text }); } else if (event.kind === "leave") { await this.request(session, "mouse.leave", { gestureId: gesture.id }); gesture.lastPoint = null; } else if (event.kind === "touch") { - const points = event.points.map(point => ({ ...gesture.mapPoint(point), id: point.id })); + const points = event.points.map((point) => ({ + ...gesture.mapPoint(point), + id: point.id, + })); await this.request(session, "touch", { gestureId: gesture.id, type: event.type, points }); } else { const point = gesture.mapPoint(event.point); - const operation = event.kind === "move" ? "mouse.move" : event.kind === "down" ? "mouse.down" - : event.kind === "up" ? "mouse.up" : "mouse.wheel"; - await this.request(session, operation, { ...point, gestureId: gesture.id, - ...(event.kind === "down" || event.kind === "up" ? { button: event.button, clickCount: event.clickCount } : {}), - ...(event.kind === "scroll" ? { deltaX: event.deltaX, deltaY: event.deltaY } : {}) }); + const operation = + event.kind === "move" + ? "mouse.move" + : event.kind === "down" + ? "mouse.down" + : event.kind === "up" + ? "mouse.up" + : "mouse.wheel"; + await this.request(session, operation, { + ...point, + gestureId: gesture.id, + ...(event.kind === "down" || event.kind === "up" + ? { button: event.button, clickCount: event.clickCount } + : {}), + ...(event.kind === "scroll" ? { deltaX: event.deltaX, deltaY: event.deltaY } : {}), + }); } publishedEvent = event; await this.assertGestureCurrent(session, gesture); @@ -537,7 +580,12 @@ export class SessionManager { return { state, gestureId: gesture.id, nextSequence: gesture.nextSequence, cursor }; } catch (error) { if (publishedEvent) { - const completion = await this.completeNavigatingInput(session, gesture, publishedEvent, eventAcknowledged).catch(() => null); + const completion = await this.completeNavigatingInput( + session, + gesture, + publishedEvent, + eventAcknowledged, + ).catch(() => null); if (completion) return completion; } await this.cancelGesture(session, gesture); @@ -547,13 +595,19 @@ export class SessionManager { } /** Return a fresh viewing state after acknowledged input navigated, without admitting another input. */ - private async completeNavigatingInput(session: BrowserSession, gesture: BrowserGesture, event: BrowserGestureEvent, eventAcknowledged: boolean) { + private async completeNavigatingInput( + session: BrowserSession, + gesture: BrowserGesture, + event: BrowserGestureEvent, + eventAcknowledged: boolean, + ) { const state = await this.snapshotState(session, gesture.viewerToken); if (state.navigationGeneration === gesture.expected.navigationGeneration) return null; // Runtime inputGeneration is the private native attachment:document tuple. // A target replacement can also bump policy navigationGeneration, but must // never be accepted as same-attachment navigation after an acknowledged input. - if (!sameRuntimeInputAttachment(gesture.runtimeInputGeneration, session.inputGeneration)) return null; + if (!sameRuntimeInputAttachment(gesture.runtimeInputGeneration, session.inputGeneration)) + return null; // Only the old document constraint may change after successful publication. // Takeover, viewport changes and runtime/bridge fences are still refused. @@ -565,7 +619,8 @@ export class SessionManager { if (!eventAcknowledged) gesture.acknowledge(event, this.now()); await this.cancelGesture(session, gesture); const finalState = await this.snapshotState(session, gesture.viewerToken); - if (!sameRuntimeInputAttachment(gesture.runtimeInputGeneration, session.inputGeneration)) return null; + if (!sameRuntimeInputAttachment(gesture.runtimeInputGeneration, session.inputGeneration)) + return null; this.requireMutationAccess(session, { viewerToken: gesture.viewerToken, controlToken: gesture.controlToken, @@ -573,7 +628,13 @@ export class SessionManager { }); this.invalidateFrames(session); this.renewController(session, gesture.viewerToken); - return { state: finalState, gestureId: gesture.id, nextSequence: gesture.nextSequence, cursor: null, completion: "navigation" as const }; + return { + state: finalState, + gestureId: gesture.id, + nextSequence: gesture.nextSequence, + cursor: null, + completion: "navigation" as const, + }; } /** Cancellation releases only this channel, even after a lost update reply or navigation drift. */ @@ -599,48 +660,93 @@ export class SessionManager { }); } - private requireGesture(session: BrowserSession, input: UpdateGestureInput | EndGestureInput): BrowserGesture { + private requireGesture( + session: BrowserSession, + input: UpdateGestureInput | EndGestureInput, + ): BrowserGesture { const gesture = session.gesture; - if (!gesture || gesture.id !== input.gestureId || gesture.viewerToken !== input.viewerToken - || gesture.controlToken !== input.controlToken) throw new Error("Browser gesture is unavailable"); + if ( + !gesture || + gesture.id !== input.gestureId || + gesture.viewerToken !== input.viewerToken || + gesture.controlToken !== input.controlToken + ) + throw new Error("Browser gesture is unavailable"); return gesture; } - private assertGesturePacketContext(gesture: BrowserGesture, expected: BeginGestureInput["expected"]): void { + private assertGesturePacketContext( + gesture: BrowserGesture, + expected: BeginGestureInput["expected"], + ): void { // JSON member order is irrelevant; identity fields must match individually. - for (const key of ["sessionId", "runtimeId", "bridgeEpoch", "navigationGeneration", "viewportGeneration"] as const) { - if (gesture.expected[key] !== expected[key]) throw new Error("Browser gesture context changed"); + for (const key of [ + "sessionId", + "runtimeId", + "bridgeEpoch", + "navigationGeneration", + "viewportGeneration", + ] as const) { + if (gesture.expected[key] !== expected[key]) + throw new Error("Browser gesture context changed"); } } - private async assertGestureCurrent(session: BrowserSession, gesture: BrowserGesture): Promise { - this.requireMutationAccess(session, { viewerToken: gesture.viewerToken, controlToken: gesture.controlToken, expected: gesture.expected }); - if (session.gesture !== gesture || this.now() >= gesture.idleUntil || this.now() >= gesture.expiresAt) throw new Error("Browser gesture expired"); + private async assertGestureCurrent( + session: BrowserSession, + gesture: BrowserGesture, + ): Promise { + this.requireMutationAccess(session, { + viewerToken: gesture.viewerToken, + controlToken: gesture.controlToken, + expected: gesture.expected, + }); + if ( + session.gesture !== gesture || + this.now() >= gesture.idleUntil || + this.now() >= gesture.expiresAt + ) + throw new Error("Browser gesture expired"); // Runtime check pins the CDP document/attachment without repeating a full // metadata read for every drag point. snapshotState supplies final metadata. await this.request(session, "input.check", { gestureId: gesture.id }); - this.requireMutationAccess(session, { viewerToken: gesture.viewerToken, controlToken: gesture.controlToken, expected: gesture.expected }); + this.requireMutationAccess(session, { + viewerToken: gesture.viewerToken, + controlToken: gesture.controlToken, + expected: gesture.expected, + }); if (session.gesture !== gesture) throw new Error("Browser gesture was cancelled"); } private async gestureCursor(session: BrowserSession, gesture: BrowserGesture) { if (!gesture.lastPoint || gesture.pointerKind !== "mouse") return null; try { - const raw = await this.request(session, "cursor", { ...gesture.lastPoint, gestureId: gesture.id }); + const raw = await this.request(session, "cursor", { + ...gesture.lastPoint, + gestureId: gesture.id, + }); const parsed = browserCursorSchema.safeParse(raw); return parsed.success ? parsed.data : null; - } catch { return null; } + } catch { + return null; + } } private scheduleGestureTimeout(session: BrowserSession, gesture: BrowserGesture): void { if (gesture.timer) clearTimeout(gesture.timer); - gesture.timer = setTimeout(() => { - void this.serialize(session, async () => { - if (session.gesture === gesture && this.now() >= Math.min(gesture.idleUntil, gesture.expiresAt)) { - await this.cancelGesture(session, gesture); - } - }).catch(() => undefined); - }, Math.max(1, Math.min(gesture.idleUntil, gesture.expiresAt) - this.now())); + gesture.timer = setTimeout( + () => { + void this.serialize(session, async () => { + if ( + session.gesture === gesture && + this.now() >= Math.min(gesture.idleUntil, gesture.expiresAt) + ) { + await this.cancelGesture(session, gesture); + } + }).catch(() => undefined); + }, + Math.max(1, Math.min(gesture.idleUntil, gesture.expiresAt) - this.now()), + ); gesture.timer.unref?.(); } @@ -651,8 +757,11 @@ export class SessionManager { if (gesture.timer) clearTimeout(gesture.timer); // Cleanup targets the runtime's original attachment, never a newly navigated // or replaced page. Failure is best-effort and cannot replay the input. - try { await this.request(session, "input.end", { gestureId: gesture.id }); } - catch { /* Best-effort old-attachment cleanup must not escape teardown. */ } + try { + await this.request(session, "input.end", { gestureId: gesture.id }); + } catch { + /* Best-effort old-attachment cleanup must not escape teardown. */ + } } reset(): void { @@ -849,11 +958,13 @@ export class SessionManager { ): boolean { this.pruneRecentFrames(session); const frame = session.recentFrames.get(target.frameId); - return Boolean(frame && - target.navigationGeneration === session.navigationGeneration && - target.viewportGeneration === session.viewportGeneration && - frame.navigationGeneration === session.navigationGeneration && - frame.viewportGeneration === session.viewportGeneration); + return Boolean( + frame && + target.navigationGeneration === session.navigationGeneration && + target.viewportGeneration === session.viewportGeneration && + frame.navigationGeneration === session.navigationGeneration && + frame.viewportGeneration === session.viewportGeneration, + ); } private renewController(session: BrowserSession, token: string): void { this.heartbeatViewer(session, token); @@ -965,8 +1076,9 @@ export class SessionManager { cached.frame.captureEpoch === generation.captureEpoch && cached.frame.navigationGeneration === generation.navigationGeneration && cached.frame.viewportGeneration === generation.viewportGeneration && - (["dataBase64", "byteLength", "width", "height", "capturedAt", "transport"] as const) - .every(key => cached.rawFrame[key] === raw[key]) + (["dataBase64", "byteLength", "width", "height", "capturedAt", "transport"] as const).every( + (key) => cached.rawFrame[key] === raw[key], + ) ) { cached.cachedAt = this.now(); return cached.frame; @@ -1077,7 +1189,8 @@ export class SessionManager { const raw = asRecord(await this.request(session, "state", null)); const url = boundedText(String(raw.url ?? ""), 8192); const inputGeneration = typeof raw.inputGeneration === "string" ? raw.inputGeneration : null; - const documentChanged = session.inputGeneration !== null && inputGeneration !== session.inputGeneration; + const documentChanged = + session.inputGeneration !== null && inputGeneration !== session.inputGeneration; if ((session.lastUrl && session.lastUrl !== url) || documentChanged) { await this.cancelGesture(session); session.navigationGeneration += 1; diff --git a/paseo-shared-browser/server/browser.ts b/paseo-shared-browser/server/browser.ts index f2200218..001afdae 100644 --- a/paseo-shared-browser/server/browser.ts +++ b/paseo-shared-browser/server/browser.ts @@ -9,16 +9,16 @@ import type { acquireControlRpc, applyDevicePresetRpc, attachBrowserRpc, + beginBrowserGestureRpc, captureBrowserRpc, detachBrowserRpc, + endBrowserGestureRpc, listOpenBrowserWorkspacesRpc, navigateBrowserRpc, releaseControlRpc, resizeBrowserRpc, sendBrowserInputRpc, - beginBrowserGestureRpc, updateBrowserGestureRpc, - endBrowserGestureRpc, } from "../shared/browser"; import { resolveBrowserRuntimeRoot } from "./runtime-path"; import type { JsonValue } from "./runtime-protocol"; @@ -315,11 +315,15 @@ export async function handleSendBrowserInput(input: SendInput): Promise { +export async function handleBeginBrowserGesture( + input: BeginGestureInput, +): Promise { return (await getProductionManager()).beginGesture(input); } -export async function handleUpdateBrowserGesture(input: UpdateGestureInput): Promise { +export async function handleUpdateBrowserGesture( + input: UpdateGestureInput, +): Promise { return (await getProductionManager()).updateGesture(input); } diff --git a/paseo-shared-browser/server/capture-transport-policy.test.ts b/paseo-shared-browser/server/capture-transport-policy.test.ts index 9c58ec2c..713e715c 100644 --- a/paseo-shared-browser/server/capture-transport-policy.test.ts +++ b/paseo-shared-browser/server/capture-transport-policy.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { createCaptureTransportPolicy } from "./capture-transport-policy"; import type { RuntimeFrame } from "./agent-browser-runtime"; +import { createCaptureTransportPolicy } from "./capture-transport-policy"; const frame: RuntimeFrame = { dataBase64: "Zml4dHVyZQ==", diff --git a/paseo-shared-browser/server/capture-transport-runtime.test.ts b/paseo-shared-browser/server/capture-transport-runtime.test.ts index 1a747f07..6a728de9 100644 --- a/paseo-shared-browser/server/capture-transport-runtime.test.ts +++ b/paseo-shared-browser/server/capture-transport-runtime.test.ts @@ -44,7 +44,9 @@ function fixture() { session: "test", }); const calls: string[] = []; - let capture: (params: Record) => Promise<{ data: string }> = async () => ({ data: jpeg() }); + let capture: (params: Record) => Promise<{ data: string }> = async () => ({ + data: jpeg(), + }); const page = { send: async (method: string, params: Record = {}) => { calls.push(method); @@ -87,7 +89,7 @@ function fixture() { describe("runtime capture hysteresis", () => { it("captures only the visible viewport without Chromium's touch-resetting beyond-viewport mode", async () => { const { runtime, setCapture } = fixture(); - setCapture(async params => { + setCapture(async (params) => { expect(params.captureBeyondViewport).toBe(false); expect(params.clip).toEqual({ x: 0, y: 0, width: 1280, height: 800, scale: 1 }); return { data: jpeg() }; @@ -171,11 +173,10 @@ describe("runtime capture hysteresis", () => { }); }); - it("honors low/medium quality independently while high viewers keep the shared stream", async () => { const { runtime, calls, emit, setCapture } = fixture(); const qualities: unknown[] = []; - setCapture(async params => { + setCapture(async (params) => { qualities.push(params.quality); return { data: jpeg() }; }); @@ -195,7 +196,7 @@ it("honors low/medium quality independently while high viewers keep the shared s it("reduces quality only to satisfy the byte cap and never substitutes a cached high-quality frame", async () => { const { runtime, setCapture } = fixture(); const qualities: unknown[] = []; - setCapture(async params => { + setCapture(async (params) => { qualities.push(params.quality); return { data: params.quality === 70 ? jpeg() + "AAAA".repeat(50) : jpeg() }; }); diff --git a/paseo-shared-browser/server/emulation-restoration.test.ts b/paseo-shared-browser/server/emulation-restoration.test.ts index 3fa2551c..b07322db 100644 --- a/paseo-shared-browser/server/emulation-restoration.test.ts +++ b/paseo-shared-browser/server/emulation-restoration.test.ts @@ -1,7 +1,7 @@ import { EventEmitter } from "node:events"; import { describe, expect, it } from "vitest"; import { AgentBrowserRuntime, type DeviceEmulation } from "./agent-browser-runtime"; -import { CdpSession, type CdpConnection } from "./cdp"; +import { type CdpConnection, CdpSession } from "./cdp"; /** Protocol fixture exercises real attachment/configuration methods without a browser process. */ class ConnectionFixture extends EventEmitter { diff --git a/paseo-shared-browser/server/input-generation.test.ts b/paseo-shared-browser/server/input-generation.test.ts index de699ab4..8b6f6d3e 100644 --- a/paseo-shared-browser/server/input-generation.test.ts +++ b/paseo-shared-browser/server/input-generation.test.ts @@ -3,15 +3,29 @@ import { formatRuntimeInputGeneration, sameRuntimeInputAttachment } from "./inpu describe("private native input generation codec", () => { it("admits document changes only within the same exact native attachment", () => { - expect(sameRuntimeInputAttachment(formatRuntimeInputGeneration(3,4), formatRuntimeInputGeneration(3,5))).toBe(true); + expect( + sameRuntimeInputAttachment( + formatRuntimeInputGeneration(3, 4), + formatRuntimeInputGeneration(3, 5), + ), + ).toBe(true); expect(sameRuntimeInputAttachment("3:4", "4:4")).toBe(false); }); it("refuses missing, opaque, truncated and unbounded generation formats", () => { - for (const invalid of [null,"opaque","3","3:","03:4","3:-1","3:9007199254740992","3:4:5"]) { - expect(sameRuntimeInputAttachment("3:4",invalid)).toBe(false); - expect(sameRuntimeInputAttachment(invalid,"3:4")).toBe(false); + for (const invalid of [ + null, + "opaque", + "3", + "3:", + "03:4", + "3:-1", + "3:9007199254740992", + "3:4:5", + ]) { + expect(sameRuntimeInputAttachment("3:4", invalid)).toBe(false); + expect(sameRuntimeInputAttachment(invalid, "3:4")).toBe(false); } - expect(()=>formatRuntimeInputGeneration(-1,0)).toThrow("Invalid"); + expect(() => formatRuntimeInputGeneration(-1, 0)).toThrow("Invalid"); }); }); diff --git a/paseo-shared-browser/server/input-generation.ts b/paseo-shared-browser/server/input-generation.ts index d640adac..0e7c139c 100644 --- a/paseo-shared-browser/server/input-generation.ts +++ b/paseo-shared-browser/server/input-generation.ts @@ -6,14 +6,22 @@ /** Serialize native counters without treating document navigation as target replacement. */ export function formatRuntimeInputGeneration(attachment: number, document: number): string { - if (!Number.isSafeInteger(attachment) || attachment < 0 || !Number.isSafeInteger(document) || document < 0) { + if ( + !Number.isSafeInteger(attachment) || + attachment < 0 || + !Number.isSafeInteger(document) || + document < 0 + ) { throw new Error("Invalid native input generation"); } return `${attachment}:${document}`; } /** Compare only independently serialized attachment identity; malformed or absent metadata refuses completion. */ -export function sameRuntimeInputAttachment(original: string | null, current: string | null): boolean { +export function sameRuntimeInputAttachment( + original: string | null, + current: string | null, +): boolean { const before = readNativeCounters(original); const after = readNativeCounters(current); return before !== null && after !== null && before.attachment === after.attachment; diff --git a/paseo-shared-browser/server/jpeg-frame.ts b/paseo-shared-browser/server/jpeg-frame.ts index cf5cc856..e816668e 100644 --- a/paseo-shared-browser/server/jpeg-frame.ts +++ b/paseo-shared-browser/server/jpeg-frame.ts @@ -1,5 +1,7 @@ /** Read actual JPEG dimensions so CDP viewport metadata cannot authorize input on cropped pixels. */ -export function readJpegFrameDimensions(dataBase64: string): { width: number; height: number } | null { +export function readJpegFrameDimensions( + dataBase64: string, +): { width: number; height: number } | null { const bytes = Buffer.from(dataBase64, "base64"); if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8) return null; diff --git a/paseo-shared-browser/server/keyboard-input.test.ts b/paseo-shared-browser/server/keyboard-input.test.ts index a103c88c..a1d6a426 100644 --- a/paseo-shared-browser/server/keyboard-input.test.ts +++ b/paseo-shared-browser/server/keyboard-input.test.ts @@ -8,26 +8,54 @@ const key = (key: string, code: string, modifiers = 0) => describe("native keyboard envelopes", () => { it("preserves Unicode and space while Enter and Tab retain native meanings", () => { expect(nativeKeyEvent(key("😀", "KeyA"))).toMatchObject({ type: "keyDown", text: "😀" }); - expect(nativeKeyEvent(key(" ", "Space"))).toMatchObject({ text: " ", windowsVirtualKeyCode: 32 }); - expect(nativeKeyEvent(key("Enter", "Enter"))).toMatchObject({ text: "\r", windowsVirtualKeyCode: 13 }); - expect(nativeKeyEvent(key("Tab", "Tab"))).toMatchObject({ type: "rawKeyDown", windowsVirtualKeyCode: 9 }); + expect(nativeKeyEvent(key(" ", "Space"))).toMatchObject({ + text: " ", + windowsVirtualKeyCode: 32, + }); + expect(nativeKeyEvent(key("Enter", "Enter"))).toMatchObject({ + text: "\r", + windowsVirtualKeyCode: 13, + }); + expect(nativeKeyEvent(key("Tab", "Tab"))).toMatchObject({ + type: "rawKeyDown", + windowsVirtualKeyCode: 9, + }); expect(nativeKeyEvent(key("Tab", "Tab"))).not.toHaveProperty("text"); - expect(nativeKeyEvent(key("ArrowLeft", "ArrowLeft"))).toMatchObject({ windowsVirtualKeyCode: 37 }); + expect(nativeKeyEvent(key("ArrowLeft", "ArrowLeft"))).toMatchObject({ + windowsVirtualKeyCode: 37, + }); expect(nativeKeyEvent(key("A", "KeyA", 8))).toMatchObject({ text: "A", modifiers: 8 }); }); it("forwards shortcuts and repeat without synthesizing printable characters", () => { - expect(nativeKeyEvent(key("a", "KeyA", 2))).toMatchObject({ type: "rawKeyDown", windowsVirtualKeyCode: 65, modifiers: 2 }); + expect(nativeKeyEvent(key("a", "KeyA", 2))).toMatchObject({ + type: "rawKeyDown", + windowsVirtualKeyCode: 65, + modifiers: 2, + }); expect(nativeKeyEvent(key("a", "KeyA", 2))).not.toHaveProperty("text"); - expect(nativeKeyEvent({ ...key("a", "KeyA"), repeat: true })).toHaveProperty("autoRepeat", true); + expect(nativeKeyEvent({ ...key("a", "KeyA"), repeat: true })).toHaveProperty( + "autoRepeat", + true, + ); expect(heldKeyModifiers([{ key: "Control" }, { key: "Shift" }])).toBe(10); }); it("rejects shortcut text, keyup text/repeat and oversized committed text", () => { - expect(browserGestureKeySchema.safeParse({ ...key("a", "KeyA", 2), text: "a" }).success).toBe(false); - expect(browserGestureKeySchema.safeParse({ ...key("a", "KeyA"), type: "up", text: "a" }).success).toBe(false); - expect(browserGestureKeySchema.safeParse({ ...key("a", "KeyA"), type: "up", repeat: true }).success).toBe(false); - expect(browserGestureEventSchema.safeParse({ kind: "text", text: "漢字\n😀" }).success).toBe(true); - expect(browserGestureEventSchema.safeParse({ kind: "text", text: "x".repeat(16001) }).success).toBe(false); + expect(browserGestureKeySchema.safeParse({ ...key("a", "KeyA", 2), text: "a" }).success).toBe( + false, + ); + expect( + browserGestureKeySchema.safeParse({ ...key("a", "KeyA"), type: "up", text: "a" }).success, + ).toBe(false); + expect( + browserGestureKeySchema.safeParse({ ...key("a", "KeyA"), type: "up", repeat: true }).success, + ).toBe(false); + expect(browserGestureEventSchema.safeParse({ kind: "text", text: "漢字\n😀" }).success).toBe( + true, + ); + expect( + browserGestureEventSchema.safeParse({ kind: "text", text: "x".repeat(16001) }).success, + ).toBe(false); }); }); diff --git a/paseo-shared-browser/server/keyboard-input.ts b/paseo-shared-browser/server/keyboard-input.ts index 1455ebc7..7eb10c6e 100644 --- a/paseo-shared-browser/server/keyboard-input.ts +++ b/paseo-shared-browser/server/keyboard-input.ts @@ -6,15 +6,51 @@ import type { BrowserGestureKeyEvent } from "../shared/browser"; const SPECIAL_CODES: Record = { - Backspace: 8, Tab: 9, Enter: 13, NumpadEnter: 13, ShiftLeft: 16, ShiftRight: 16, - ControlLeft: 17, ControlRight: 17, AltLeft: 18, AltRight: 18, Pause: 19, - CapsLock: 20, Escape: 27, Space: 32, PageUp: 33, PageDown: 34, End: 35, - Home: 36, ArrowLeft: 37, ArrowUp: 38, ArrowRight: 39, ArrowDown: 40, - Insert: 45, Delete: 46, MetaLeft: 91, MetaRight: 92, ContextMenu: 93, - NumpadMultiply: 106, NumpadAdd: 107, NumpadSubtract: 109, NumpadDecimal: 110, - NumpadDivide: 111, NumLock: 144, ScrollLock: 145, Semicolon: 186, Equal: 187, - Comma: 188, Minus: 189, Period: 190, Slash: 191, Backquote: 192, - BracketLeft: 219, Backslash: 220, BracketRight: 221, Quote: 222, + Backspace: 8, + Tab: 9, + Enter: 13, + NumpadEnter: 13, + ShiftLeft: 16, + ShiftRight: 16, + ControlLeft: 17, + ControlRight: 17, + AltLeft: 18, + AltRight: 18, + Pause: 19, + CapsLock: 20, + Escape: 27, + Space: 32, + PageUp: 33, + PageDown: 34, + End: 35, + Home: 36, + ArrowLeft: 37, + ArrowUp: 38, + ArrowRight: 39, + ArrowDown: 40, + Insert: 45, + Delete: 46, + MetaLeft: 91, + MetaRight: 92, + ContextMenu: 93, + NumpadMultiply: 106, + NumpadAdd: 107, + NumpadSubtract: 109, + NumpadDecimal: 110, + NumpadDivide: 111, + NumLock: 144, + ScrollLock: 145, + Semicolon: 186, + Equal: 187, + Comma: 188, + Minus: 189, + Period: 190, + Slash: 191, + Backquote: 192, + BracketLeft: 219, + Backslash: 220, + BracketRight: 221, + Quote: 222, }; /** Physical code chooses the accelerator code independently of keyboard layout text. */ diff --git a/paseo-shared-browser/server/live-input-policy.test.ts b/paseo-shared-browser/server/live-input-policy.test.ts index 520109fc..b8656e79 100644 --- a/paseo-shared-browser/server/live-input-policy.test.ts +++ b/paseo-shared-browser/server/live-input-policy.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { SessionManager, type BrowserRuntimeClient } from "./browser-policy"; import type { BrowserGestureEvent } from "../shared/browser"; +import { type BrowserRuntimeClient, SessionManager } from "./browser-policy"; /** Exercise the real policy with a deterministic owned runtime; no browser or services are started. */ async function fixture() { @@ -103,7 +103,9 @@ async function fixture() { reloadSameUrl: () => { inputGeneration = "0:1"; }, - replaceTarget: () => { inputGeneration = "1:0"; }, + replaceTarget: () => { + inputGeneration = "1:0"; + }, intercept: (fn: typeof intercept) => { intercept = fn; }, @@ -145,33 +147,62 @@ describe("owned ordered live input", () => { try { await state.begin(); await state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true); - state.intercept(async operation => { + state.intercept(async (operation) => { if (operation === "mouse.up") state.changeUrl("https://fixture.invalid/after"); }); - const reply = await state.update({ kind: "up", button: "left", clickCount: 1, point: point() }); + const reply = await state.update({ + kind: "up", + button: "left", + clickCount: 1, + point: point(), + }); expect(reply.state.url).toBe("https://fixture.invalid/after"); - expect(reply.state.navigationGeneration).toBe(state.context.expected.navigationGeneration + 1); + expect(reply.state.navigationGeneration).toBe( + state.context.expected.navigationGeneration + 1, + ); expect(reply.cursor).toBeNull(); expect(reply.completion).toBe("navigation"); - expect(state.calls.filter(call => call.operation === "mouse.up")).toHaveLength(1); + expect(state.calls.filter((call) => call.operation === "mouse.up")).toHaveLength(1); await expect(state.update({ kind: "move", point: point() })).rejects.toThrow("unavailable"); - } finally { state.manager.disconnect(); } + } finally { + state.manager.disconnect(); + } }); it("acknowledges a form-submit key before same-URL document replacement without replay", async () => { const state = await fixture(); try { await state.begin(); - state.intercept(async operation => { + state.intercept(async (operation) => { if (operation === "input.key") state.reloadSameUrl(); }); - const reply = await state.update({kind:"key",type:"down",key:"Enter",code:"Enter",modifiers:0,repeat:false}); + const reply = await state.update({ + kind: "key", + type: "down", + key: "Enter", + code: "Enter", + modifiers: 0, + repeat: false, + }); expect(reply.completion).toBe("navigation"); - expect(reply.state.navigationGeneration).toBe(state.context.expected.navigationGeneration + 1); - expect(state.calls.filter(call => call.operation === "input.key")).toHaveLength(1); - expect(state.calls.filter(call => call.operation === "input.end")).toHaveLength(1); - await expect(state.update({kind:"key",type:"up",key:"Enter",code:"Enter",modifiers:0,repeat:false})).rejects.toThrow("unavailable"); - } finally { state.manager.disconnect(); } + expect(reply.state.navigationGeneration).toBe( + state.context.expected.navigationGeneration + 1, + ); + expect(state.calls.filter((call) => call.operation === "input.key")).toHaveLength(1); + expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); + await expect( + state.update({ + kind: "key", + type: "up", + key: "Enter", + code: "Enter", + modifiers: 0, + repeat: false, + }), + ).rejects.toThrow("unavailable"); + } finally { + state.manager.disconnect(); + } }); it("keeps pre-publication document drift and uncertain publication failures rejected", async () => { @@ -180,14 +211,21 @@ describe("owned ordered live input", () => { try { await state.begin(); if (scenario === "before") state.reloadSameUrl(); - else state.intercept(async operation => { - if (operation !== "mouse.down") return; - state.changeUrl("https://fixture.invalid/after"); - throw new Error("Mutation outcome is unknown"); - }); - await expect(state.update({kind:"down",button:"left",clickCount:1,point:point()},true)).rejects.toThrow(scenario === "before" ? "attachment" : "unknown"); - expect(state.calls.filter(call => call.operation === "mouse.down")).toHaveLength(scenario === "before" ? 0 : 1); - } finally { state.manager.disconnect(); } + else + state.intercept(async (operation) => { + if (operation !== "mouse.down") return; + state.changeUrl("https://fixture.invalid/after"); + throw new Error("Mutation outcome is unknown"); + }); + await expect( + state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true), + ).rejects.toThrow(scenario === "before" ? "attachment" : "unknown"); + expect(state.calls.filter((call) => call.operation === "mouse.down")).toHaveLength( + scenario === "before" ? 0 : 1, + ); + } finally { + state.manager.disconnect(); + } } }); @@ -195,14 +233,18 @@ describe("owned ordered live input", () => { const state = await fixture(); try { await state.begin(); - state.intercept(async operation => { + state.intercept(async (operation) => { if (operation !== "mouse.down") return; state.changeUrl("https://fixture.invalid/after"); state.manager.setBridgeEpoch(2); }); - await expect(state.update({kind:"down",button:"left",clickCount:1,point:point()},true)).rejects.toThrow("bridge"); - expect(state.calls.filter(call => call.operation === "mouse.down")).toHaveLength(1); - } finally { state.manager.disconnect(); } + await expect( + state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true), + ).rejects.toThrow("bridge"); + expect(state.calls.filter((call) => call.operation === "mouse.down")).toHaveLength(1); + } finally { + state.manager.disconnect(); + } }); it("refuses target replacement after a successful native input acknowledgment", async () => { @@ -210,14 +252,18 @@ describe("owned ordered live input", () => { try { await state.begin(); let published = false; - state.intercept(async operation => { + state.intercept(async (operation) => { if (operation === "mouse.down") published = true; if (operation === "input.check" && published) state.replaceTarget(); }); - await expect(state.update({kind:"down",button:"left",clickCount:1,point:point()},true)).rejects.toThrow("attachment"); - expect(state.calls.filter(call => call.operation === "mouse.down")).toHaveLength(1); - expect(state.calls.filter(call => call.operation === "input.end")).toHaveLength(1); - } finally { state.manager.disconnect(); } + await expect( + state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true), + ).rejects.toThrow("attachment"); + expect(state.calls.filter((call) => call.operation === "mouse.down")).toHaveLength(1); + expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); + } finally { + state.manager.disconnect(); + } }); it("retains a bounded decoded frame after hover, then streams every pressed drag move without requiring new frames", async () => { @@ -243,17 +289,27 @@ describe("owned ordered live input", () => { await state.begin(); await state.update({ kind: "scroll", point: point(), deltaX: 0, deltaY: 40 }); const before = state.calls.length; - const reply = await state.manager.beginGesture({ ...state.context, target: state.target, pointerKind: "mouse" }); + const reply = await state.manager.beginGesture({ + ...state.context, + target: state.target, + pointerKind: "mouse", + }); expect(reply).toHaveProperty("admission", "stale-frame"); expect(reply).not.toHaveProperty("gestureId"); - expect(state.calls.slice(before).map(call => call.operation)).not.toContain("input.begin"); - expect(state.calls.slice(before).map(call => call.operation)).not.toContain("input.end"); + expect(state.calls.slice(before).map((call) => call.operation)).not.toContain("input.begin"); + expect(state.calls.slice(before).map((call) => call.operation)).not.toContain("input.end"); await state.update({ kind: "scroll", point: point(), deltaX: 0, deltaY: 40 }); - await expect(state.manager.beginGesture({ ...state.context, controlToken: "wrong", target: state.target, pointerKind: "mouse" })).rejects.toThrow(); + await expect( + state.manager.beginGesture({ + ...state.context, + controlToken: "wrong", + target: state.target, + pointerKind: "mouse", + }), + ).rejects.toThrow(); state.manager.disconnect(); }); - it("does not return stale admission if its final metadata read fails or discovers replacement navigation", async () => { for (const failure of ["runtime", "navigation"] as const) { const state = await fixture(); @@ -261,15 +317,25 @@ describe("owned ordered live input", () => { await state.begin(); await state.update({ kind: "scroll", point: point(), deltaX: 0, deltaY: 40 }); let reads = 0; - state.intercept(async operation => { + state.intercept(async (operation) => { if (operation !== "state" || ++reads !== 2) return; if (failure === "runtime") throw new Error("Runtime read unavailable"); state.changeUrl("https://replacement.invalid/"); }); const before = state.calls.length; - await expect(state.manager.beginGesture({ ...state.context, target: state.target, pointerKind: "mouse" })).rejects.toThrow(failure === "runtime" ? "Runtime read unavailable" : "navigation"); - expect(state.calls.slice(before).map(call => call.operation)).not.toContain("input.begin"); - } finally { state.manager.disconnect(); } + await expect( + state.manager.beginGesture({ + ...state.context, + target: state.target, + pointerKind: "mouse", + }), + ).rejects.toThrow(failure === "runtime" ? "Runtime read unavailable" : "navigation"); + expect(state.calls.slice(before).map((call) => call.operation)).not.toContain( + "input.begin", + ); + } finally { + state.manager.disconnect(); + } } }); @@ -399,7 +465,10 @@ describe("owned ordered live input", () => { state.intercept(async (op) => { if (op === "mouse.down") state.reloadSameUrl(); }); - const reply = await state.update({ kind: "down", button: "left", clickCount: 1, point: point() }, true); + const reply = await state.update( + { kind: "down", button: "left", clickCount: 1, point: point() }, + true, + ); expect(reply.state.navigationGeneration).toBe(state.context.expected.navigationGeneration + 1); await expect(state.update({ kind: "move", point: point() })).rejects.toThrow("unavailable"); expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); @@ -443,24 +512,45 @@ describe("owned ordered live input", () => { it("streams keyboard edges and committed text without a fresh frame per keystroke", async () => { const state = await fixture(); await state.begin(); - const key = { kind: "key", type: "down", key: "a", code: "KeyA", modifiers: 0, repeat: false } as const; + const key = { + kind: "key", + type: "down", + key: "a", + code: "KeyA", + modifiers: 0, + repeat: false, + } as const; await state.update(key); await state.update({ ...key, repeat: true }); await state.update({ ...key, type: "up" }); await state.update({ kind: "text", text: "漢字\n😀" }); await state.end(); expect(state.calls.filter((call) => call.operation === "input.key")).toHaveLength(3); - expect(state.calls.filter((call) => call.operation === "input.text")).toEqual([{ operation: "input.text", input: expect.objectContaining({ text: "漢字\n😀" }) }]); + expect(state.calls.filter((call) => call.operation === "input.text")).toEqual([ + { operation: "input.text", input: expect.objectContaining({ text: "漢字\n😀" }) }, + ]); }); it("cancels a held gesture on device mode changes without reloading or inventing navigation", async () => { const state = await fixture(); await state.begin(); - await state.update({ kind: "key", type: "down", key: "Control", code: "ControlLeft", modifiers: 2, repeat: false }); + await state.update({ + kind: "key", + type: "down", + key: "Control", + code: "ControlLeft", + modifiers: 2, + repeat: false, + }); const reply = await state.manager.applyDevicePreset({ ...state.context, presetId: "pixel-7" }); expect(reply.state.navigationGeneration).toBe(state.context.expected.navigationGeneration); expect(reply.state.viewportGeneration).toBe(state.context.expected.viewportGeneration + 1); expect(state.calls.filter((call) => call.operation === "input.end")).toHaveLength(1); - expect(state.calls.filter((call) => call.operation === "reload" || call.operation === "navigate")).toEqual([]); - expect(state.calls.filter((call) => call.operation === "emulate").at(-1)?.input).toMatchObject({ mobile: true, touch: true }); + expect( + state.calls.filter((call) => call.operation === "reload" || call.operation === "navigate"), + ).toEqual([]); + expect(state.calls.filter((call) => call.operation === "emulate").at(-1)?.input).toMatchObject({ + mobile: true, + touch: true, + }); }); diff --git a/paseo-shared-browser/server/live-input-runtime.test.ts b/paseo-shared-browser/server/live-input-runtime.test.ts index 8fdb2503..fe5e1f8e 100644 --- a/paseo-shared-browser/server/live-input-runtime.test.ts +++ b/paseo-shared-browser/server/live-input-runtime.test.ts @@ -50,14 +50,20 @@ describe("runtime-owned live input", () => { it("completes acknowledged input that navigates while closing original held-input cleanup", async () => { const state = fixture(); await state.runtime.beginLiveInput("owned"); - state.intercept(async method => { - if (method !== "Input.dispatchKeyEvent" || state.calls.at(-1)?.params.type !== "keyDown") return; + state.intercept(async (method) => { + if (method !== "Input.dispatchKeyEvent" || state.calls.at(-1)?.params.type !== "keyDown") + return; state.control.documentGeneration++; await state.runtime.endLiveInput("owned"); }); - await expect(state.runtime.dispatchKey({kind:"key",type:"down",key:"Enter",code:"Enter",modifiers:0,repeat:false},"owned")).resolves.toBeUndefined(); - expect(state.calls.filter(call => call.params.type === "keyDown")).toHaveLength(1); - expect(state.calls.filter(call => call.params.type === "keyUp")).toHaveLength(1); + await expect( + state.runtime.dispatchKey( + { kind: "key", type: "down", key: "Enter", code: "Enter", modifiers: 0, repeat: false }, + "owned", + ), + ).resolves.toBeUndefined(); + expect(state.calls.filter((call) => call.params.type === "keyDown")).toHaveLength(1); + expect(state.calls.filter((call) => call.params.type === "keyUp")).toHaveLength(1); await expect(state.runtime.assertLiveInput("owned")).rejects.toThrow("attachment"); }); @@ -65,13 +71,15 @@ describe("runtime-owned live input", () => { for (const scenario of ["replacement", "uncertain"] as const) { const state = fixture(); await state.runtime.beginLiveInput("owned"); - state.intercept(async method => { + state.intercept(async (method) => { if (method !== "Input.dispatchMouseEvent") return; state.control.documentGeneration++; if (scenario === "uncertain") throw new Error("Mutation outcome is unknown"); - state.control.page = {send:async()=>({})}; + state.control.page = { send: async () => ({}) }; }); - await expect(state.runtime.mouseDown(20,30,"left",1,"owned")).rejects.toThrow(scenario === "replacement" ? "attachment" : "unknown"); + await expect(state.runtime.mouseDown(20, 30, "left", 1, "owned")).rejects.toThrow( + scenario === "replacement" ? "attachment" : "unknown", + ); state.intercept(async () => {}); await state.runtime.endLiveInput("owned"); } @@ -80,14 +88,15 @@ describe("runtime-owned live input", () => { it("does not restore held touches after an acknowledged touch start navigates", async () => { const state = fixture(); await state.runtime.beginLiveInput("owned"); - state.intercept(async method => { - if (method !== "Input.dispatchTouchEvent" || state.calls.at(-1)?.params.type !== "touchStart") return; + state.intercept(async (method) => { + if (method !== "Input.dispatchTouchEvent" || state.calls.at(-1)?.params.type !== "touchStart") + return; state.control.documentGeneration++; await state.runtime.endLiveInput("owned"); }); - await state.runtime.touch("touchStart",[{x:20,y:30,id:7}],"owned"); + await state.runtime.touch("touchStart", [{ x: 20, y: 30, id: 7 }], "owned"); await state.runtime.releaseHeldInput(); - expect(state.calls.filter(call => call.params.type === "touchCancel")).toHaveLength(1); + expect(state.calls.filter((call) => call.params.type === "touchCancel")).toHaveLength(1); await expect(state.runtime.assertLiveInput("owned")).rejects.toThrow("attachment"); }); @@ -236,14 +245,28 @@ describe("runtime-owned live input", () => { ).rejects.toThrow(); } expect(runtime.touch).toHaveBeenCalledTimes(1); - const event = { kind: "key", type: "down", key: "Enter", code: "Enter", modifiers: 0, repeat: false }; + const event = { + kind: "key", + type: "down", + key: "Enter", + code: "Enter", + modifiers: 0, + repeat: false, + }; await owner.request(owned, "input.key", { gestureId: "gesture", event }); await owner.request(owned, "input.text", { gestureId: "gesture", text: "漢字😀" }); expect(runtime.dispatchKey).toHaveBeenCalledWith(event, "gesture"); expect(runtime.insertText).toHaveBeenCalledWith("漢字😀", "gesture"); await expect(owner.request(owned, "input.key", { event })).rejects.toThrow(); - await expect(owner.request(owned, "input.key", { gestureId: "gesture", event: { ...event, modifiers: 99 } })).rejects.toThrow(); - await expect(owner.request(owned, "input.text", { gestureId: "gesture", text: "x".repeat(16001) })).rejects.toThrow(); + await expect( + owner.request(owned, "input.key", { + gestureId: "gesture", + event: { ...event, modifiers: 99 }, + }), + ).rejects.toThrow(); + await expect( + owner.request(owned, "input.text", { gestureId: "gesture", text: "x".repeat(16001) }), + ).rejects.toThrow(); expect(runtime.dispatchKey).toHaveBeenCalledTimes(1); expect(runtime.insertText).toHaveBeenCalledTimes(1); }); @@ -275,9 +298,7 @@ describe("runtime-owned live input", () => { await state.runtime.mouseUp(20, 30, button, 1, "owned"); await state.runtime.mouseMove(30, 40, "owned"); expect( - state.calls - .filter((call) => call.params.type === "mouseMoved") - .map((call) => call.params), + state.calls.filter((call) => call.params.type === "mouseMoved").map((call) => call.params), ).toEqual([ { type: "mouseMoved", x: 20, y: 30, button, buttons: mask }, { type: "mouseMoved", x: 30, y: 40, button: "none", buttons: 0 }, @@ -313,8 +334,7 @@ describe("runtime-owned live input", () => { ["none", 0], ]); expect( - state.calls.filter((call) => call.params.type === "mouseReleased").at(-1) - ?.params.button, + state.calls.filter((call) => call.params.type === "mouseReleased").at(-1)?.params.button, ).toBe("middle"); await state.runtime.endLiveInput("next"); }); @@ -323,31 +343,62 @@ describe("runtime-owned live input", () => { it("publishes ordered native Unicode/repeat/shortcut keys and committed text on the pinned attachment", async () => { const state = fixture(); await state.runtime.beginLiveInput("keys"); - await state.runtime.dispatchKey({ kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: false }, "keys"); - await state.runtime.dispatchKey({ kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: true }, "keys"); - await state.runtime.dispatchKey({ kind: "key", type: "up", key: "é", code: "KeyE", modifiers: 0, repeat: false }, "keys"); - await state.runtime.dispatchKey({ kind: "key", type: "down", key: "Control", code: "ControlLeft", modifiers: 2, repeat: false }, "keys"); + await state.runtime.dispatchKey( + { kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: false }, + "keys", + ); + await state.runtime.dispatchKey( + { kind: "key", type: "down", key: "é", code: "KeyE", modifiers: 0, repeat: true }, + "keys", + ); + await state.runtime.dispatchKey( + { kind: "key", type: "up", key: "é", code: "KeyE", modifiers: 0, repeat: false }, + "keys", + ); + await state.runtime.dispatchKey( + { kind: "key", type: "down", key: "Control", code: "ControlLeft", modifiers: 2, repeat: false }, + "keys", + ); await state.runtime.mouseDown(10, 20, "left", 1, "keys"); await state.runtime.insertText("漢字\n😀", "keys"); - expect(state.calls.filter((call) => call.method === "Input.dispatchKeyEvent").map((call) => call.params)).toMatchObject([ + expect( + state.calls + .filter((call) => call.method === "Input.dispatchKeyEvent") + .map((call) => call.params), + ).toMatchObject([ { type: "keyDown", text: "é", code: "KeyE", autoRepeat: false }, { type: "keyDown", text: "é", autoRepeat: true }, { type: "keyUp", code: "KeyE" }, { type: "rawKeyDown", code: "ControlLeft", modifiers: 2 }, ]); expect(state.calls.find((call) => call.params.type === "mousePressed")?.params.modifiers).toBe(2); - expect(state.calls.find((call) => call.method === "Input.insertText")?.params).toEqual({ text: "漢字\n😀" }); + expect(state.calls.find((call) => call.method === "Input.insertText")?.params).toEqual({ + text: "漢字\n😀", + }); await state.runtime.endLiveInput("keys"); - expect(state.calls.at(-1)?.params).toMatchObject({ type: "keyUp", key: "Control", code: "ControlLeft" }); + expect(state.calls.at(-1)?.params).toMatchObject({ + type: "keyUp", + key: "Control", + code: "ControlLeft", + }); }); it("cleans possibly published keys on the original page and rejects unmatched repeat/up", async () => { const state = fixture(); await state.runtime.beginLiveInput("keys"); - const down = { kind: "key", type: "down", key: "Shift", code: "ShiftRight", modifiers: 8, repeat: false } as const; + const down = { + kind: "key", + type: "down", + key: "Shift", + code: "ShiftRight", + modifiers: 8, + repeat: false, + } as const; await expect(state.runtime.dispatchKey({ ...down, repeat: true }, "keys")).rejects.toThrow(); await expect(state.runtime.dispatchKey({ ...down, type: "up" }, "keys")).rejects.toThrow(); - state.intercept(async (method) => { if (method === "Input.dispatchKeyEvent") throw new Error("Lost key acknowledgment"); }); + state.intercept(async (method) => { + if (method === "Input.dispatchKeyEvent") throw new Error("Lost key acknowledgment"); + }); await expect(state.runtime.dispatchKey(down, "keys")).rejects.toThrow("Lost key"); state.control.page = { send: async () => ({}) }; await state.runtime.endLiveInput("keys"); @@ -358,7 +409,10 @@ it("expires held keyboard modifiers without another client request", async () => vi.useFakeTimers(); const state = fixture(); await state.runtime.beginLiveInput("keys"); - await state.runtime.dispatchKey({ kind: "key", type: "down", key: "Alt", code: "AltLeft", modifiers: 1, repeat: false }, "keys"); + await state.runtime.dispatchKey( + { kind: "key", type: "down", key: "Alt", code: "AltLeft", modifiers: 1, repeat: false }, + "keys", + ); await vi.advanceTimersByTimeAsync(5001); expect(state.calls.filter((call) => call.params.type === "keyUp")).toHaveLength(1); await expect(state.runtime.insertText("late", "keys")).rejects.toThrow("attachment"); diff --git a/paseo-shared-browser/server/live-input-supervisor.test.ts b/paseo-shared-browser/server/live-input-supervisor.test.ts index 606b68cb..6c5e23a3 100644 --- a/paseo-shared-browser/server/live-input-supervisor.test.ts +++ b/paseo-shared-browser/server/live-input-supervisor.test.ts @@ -1,7 +1,7 @@ import { expect, it } from "vitest"; -import { RuntimeSupervisor } from "./supervisor"; -import type { JsonValue, RuntimeRequest } from "./runtime-protocol"; import { browserStateSchema } from "../shared/browser"; +import type { JsonValue, RuntimeRequest } from "./runtime-protocol"; +import { RuntimeSupervisor } from "./supervisor"; /** Exercise the immutable supervisor's human RPC path without a plugin or browser process. */ it("routes validated human gestures, fences replaced plugin bridges and never exposes live gesture operations to agent tickets", async () => { @@ -90,10 +90,19 @@ it("routes validated human gestures, fences replaced plugin bridges and never ex ...context, gestureId: begun.gestureId, sequence: 1, - event: { kind: "key", type: "down", key: "Control", code: "ControlLeft", modifiers: 2, repeat: false }, + event: { + kind: "key", + type: "down", + key: "Control", + code: "ControlLeft", + modifiers: 2, + repeat: false, + }, }); expect(calls.filter((value) => value === "input.key")).toHaveLength(1); - const afterKey = (await request("capture", { viewerToken: attached.viewerToken })) as { frame: { frameId: string } }; + const afterKey = (await request("capture", { viewerToken: attached.viewerToken })) as { + frame: { frameId: string }; + }; await request("gesture.update", { ...context, gestureId: begun.gestureId, diff --git a/paseo-shared-browser/server/mcp-entry.ts b/paseo-shared-browser/server/mcp-entry.ts index 9a6a7706..e1d3c1b4 100644 --- a/paseo-shared-browser/server/mcp-entry.ts +++ b/paseo-shared-browser/server/mcp-entry.ts @@ -3,9 +3,9 @@ import { join } from "node:path"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { z } from "zod"; -import { MIN_VIEWPORT, MAX_VIEWPORT } from "../shared/viewport-limits"; import { DEFAULT_CAPTURE_QUALITY } from "../shared/capture-settings"; import { DEVICE_PRESET_IDS } from "../shared/device-presets"; +import { MAX_VIEWPORT, MIN_VIEWPORT } from "../shared/viewport-limits"; import type { JsonValue } from "./runtime-protocol"; import { resolveSupervisorPaths } from "./supervisor"; import { AgentSupervisorClient } from "./supervisor-client"; diff --git a/paseo-shared-browser/server/runtime-frame-reuse.test.ts b/paseo-shared-browser/server/runtime-frame-reuse.test.ts index e948ac69..ec77dfa5 100644 --- a/paseo-shared-browser/server/runtime-frame-reuse.test.ts +++ b/paseo-shared-browser/server/runtime-frame-reuse.test.ts @@ -1,7 +1,7 @@ /** Exact fresh runtime receipts avoid resending cached JPEGs without reviving invalidated input. */ import { describe, expect, it } from "vitest"; -import { SessionManager, type BrowserRuntimeClient } from "./browser-policy"; import type { BrowserFrame, BrowserState } from "../shared/browser"; +import { type BrowserRuntimeClient, SessionManager } from "./browser-policy"; import type { JsonValue } from "./runtime-protocol"; async function fixture() { diff --git a/paseo-shared-browser/server/runtime-launch-environment.test.ts b/paseo-shared-browser/server/runtime-launch-environment.test.ts index 1258e1ec..e0d3c569 100644 --- a/paseo-shared-browser/server/runtime-launch-environment.test.ts +++ b/paseo-shared-browser/server/runtime-launch-environment.test.ts @@ -1,7 +1,8 @@ /** Per-instance display context reaches native CLI children without changing host or session authority. */ -import { afterEach, describe, expect, it } from "vitest"; + import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; import { AgentBrowserRuntime } from "./agent-browser-runtime"; const temporary: string[] = []; @@ -30,9 +31,7 @@ async function fixture(launchEnvironment?: Readonly>) { }); const native = runtime as unknown as { assertVersion(): Promise; - invoke( - args: string[], - ): Promise<{ + invoke(args: string[]): Promise<{ display: string | null; authority: string | null; wayland: string | null; diff --git a/paseo-shared-browser/server/stale-wheel-admission.test.ts b/paseo-shared-browser/server/stale-wheel-admission.test.ts index f9e319e3..961661d9 100644 --- a/paseo-shared-browser/server/stale-wheel-admission.test.ts +++ b/paseo-shared-browser/server/stale-wheel-admission.test.ts @@ -1,10 +1,10 @@ import assert from "node:assert/strict"; import { it } from "vitest"; -import { SessionManager, type BrowserRuntimeClient } from "./browser-policy"; import { - createBrowserInputQueue, type BrowserGestureAuthority, + createBrowserInputQueue, } from "../client/browser-input-queue"; +import { type BrowserRuntimeClient, SessionManager } from "./browser-policy"; it("recovers a captured-before-wheel frame that decodes after invalidation without repeating the first wheel", async () => { // No native process or profile: actual queue + policy, with delayed decoder and deterministic clock. diff --git a/paseo-shared-browser/server/supervisor.ts b/paseo-shared-browser/server/supervisor.ts index a4e74259..7b58662d 100644 --- a/paseo-shared-browser/server/supervisor.ts +++ b/paseo-shared-browser/server/supervisor.ts @@ -3,10 +3,20 @@ import { chmod, mkdir, open, readFile, rm, writeFile } from "node:fs/promises"; import { createServer, type Socket } from "node:net"; import { homedir, tmpdir } from "node:os"; import { dirname, join } from "node:path"; -import type { BrowserFrame, BrowserInputEvent, BrowserState, Viewport, DevicePresetId } from "../shared/browser"; -import { beginBrowserGestureRpc, updateBrowserGestureRpc, endBrowserGestureRpc } from "../shared/browser"; -import { SessionManager } from "./browser-policy"; +import type { + BrowserFrame, + BrowserInputEvent, + BrowserState, + DevicePresetId, + Viewport, +} from "../shared/browser"; +import { + beginBrowserGestureRpc, + endBrowserGestureRpc, + updateBrowserGestureRpc, +} from "../shared/browser"; import { DEFAULT_CAPTURE_QUALITY } from "../shared/capture-settings"; +import { SessionManager } from "./browser-policy"; import { CdpUnknownOutcomeError } from "./cdp"; import { type BridgeLease, @@ -299,7 +309,9 @@ export class RuntimeSupervisor; +export type BrowserDisplayPreferences = z.output; /** One revisioned document per plugin host, shared across that host's panels. */ export const browserDisplayPreferences = defineSettings({ diff --git a/paseo-shared-browser/shared/browser.ts b/paseo-shared-browser/shared/browser.ts index 20e7894b..f608004f 100644 --- a/paseo-shared-browser/shared/browser.ts +++ b/paseo-shared-browser/shared/browser.ts @@ -1,13 +1,23 @@ import { defineRpc } from "@getpaseo/plugin"; import { z } from "zod"; -import { MIN_VIEWPORT, MAX_VIEWPORT } from "./viewport-limits"; +import { + DEFAULT_CAPTURE_QUALITY, + FRAME_MAX_BASE64_CHARS, + FRAME_MAX_BYTES, +} from "./capture-settings"; import { DEVICE_PRESET_IDS } from "./device-presets"; -import { DEFAULT_CAPTURE_QUALITY, FRAME_MAX_BYTES, FRAME_MAX_BASE64_CHARS } from "./capture-settings"; -export { DEFAULT_CAPTURE_QUALITY, FRAME_MAX_BYTES, FRAME_MAX_BASE64_CHARS } from "./capture-settings"; -export { DEVICE_PRESETS, DEVICE_PRESET_IDS, type DevicePresetId } from "./device-presets"; +import { MAX_VIEWPORT, MIN_VIEWPORT } from "./viewport-limits"; + +export { + DEFAULT_CAPTURE_QUALITY, + FRAME_MAX_BASE64_CHARS, + FRAME_MAX_BYTES, +} from "./capture-settings"; +export { DEVICE_PRESET_IDS, DEVICE_PRESETS, type DevicePresetId } from "./device-presets"; export const DEFAULT_VIEWPORT = { width: 1280, height: 800 } as const; -export { MIN_VIEWPORT, MAX_VIEWPORT } from "./viewport-limits"; +export { MAX_VIEWPORT, MIN_VIEWPORT } from "./viewport-limits"; + const devicePresetIdSchema = z.enum(DEVICE_PRESET_IDS); const workspaceIdSchema = z @@ -259,33 +269,70 @@ export const sendBrowserInputRpc = defineRpc({ /** Safe CSS cursor names only. Custom cursor URLs never reach the local client. */ export const browserCursorSchema = z.enum([ - "default", "none", "pointer", "text", "vertical-text", "crosshair", "move", - "grab", "grabbing", "wait", "progress", "help", "not-allowed", "no-drop", - "copy", "alias", "context-menu", "cell", "all-scroll", "col-resize", "row-resize", - "n-resize", "s-resize", "e-resize", "w-resize", "ne-resize", "nw-resize", - "se-resize", "sw-resize", "ew-resize", "ns-resize", "nesw-resize", "nwse-resize", - "zoom-in", "zoom-out", + "default", + "none", + "pointer", + "text", + "vertical-text", + "crosshair", + "move", + "grab", + "grabbing", + "wait", + "progress", + "help", + "not-allowed", + "no-drop", + "copy", + "alias", + "context-menu", + "cell", + "all-scroll", + "col-resize", + "row-resize", + "n-resize", + "s-resize", + "e-resize", + "w-resize", + "ne-resize", + "nw-resize", + "se-resize", + "sw-resize", + "ew-resize", + "ns-resize", + "nesw-resize", + "nwse-resize", + "zoom-in", + "zoom-out", ]); export type BrowserCursor = z.output; -const touchPointSchema = displayedPointSchema.extend({ id: z.number().int().min(0).max(2_147_483_647) }); -/** Native keyboard modifiers use CDP's bits: Alt 1, Control 2, Meta 4, Shift 8. */ -export const browserGestureKeySchema = z.object({ - kind: z.literal("key"), - type: z.enum(["down", "up"]), - key: z.string().min(1).max(64), - code: z.string().min(1).max(64).regex(/^[A-Za-z][A-Za-z0-9]*$/), - modifiers: z.number().int().min(0).max(15), - repeat: z.boolean().default(false), - text: z.string().min(1).max(32).optional(), -}).superRefine((event, context) => { - if (event.type === "up" && (event.repeat || event.text !== undefined)) { - context.addIssue({ code: "custom", message: "Key release cannot repeat or insert text" }); - } - if (event.text !== undefined && (event.modifiers & 7) !== 0) { - context.addIssue({ code: "custom", message: "Shortcut keys cannot insert printable text" }); - } +const touchPointSchema = displayedPointSchema.extend({ + id: z.number().int().min(0).max(2_147_483_647), }); +/** Native keyboard modifiers use CDP's bits: Alt 1, Control 2, Meta 4, Shift 8. */ +export const browserGestureKeySchema = z + .object({ + kind: z.literal("key"), + type: z.enum(["down", "up"]), + key: z.string().min(1).max(64), + code: z + .string() + .min(1) + .max(64) + .regex(/^[A-Za-z][A-Za-z0-9]*$/), + modifiers: z.number().int().min(0).max(15), + repeat: z.boolean().default(false), + text: z.string().min(1).max(32).optional(), + }) + .superRefine((event, context) => { + if (event.type === "up" && (event.repeat || event.text !== undefined)) { + context.addIssue({ code: "custom", message: "Key release cannot repeat or insert text" }); + } + if (event.text !== undefined && (event.modifiers & 7) !== 0) { + context.addIssue({ code: "custom", message: "Shortcut keys cannot insert printable text" }); + } + }); export type BrowserGestureKeyEvent = z.output; /** @@ -298,35 +345,66 @@ export const browserGestureEventSchema = z.discriminatedUnion("kind", [ z.object({ kind: z.literal("text"), text: z.string().min(1).max(16_000) }), z.object({ kind: z.literal("leave") }), z.object({ kind: z.literal("move"), point: displayedPointSchema }), - z.object({ kind: z.literal("down"), point: displayedPointSchema, - button: z.enum(["left", "right", "middle"]).default("left"), clickCount: z.union([z.literal(1), z.literal(2)]).default(1) }), - z.object({ kind: z.literal("up"), point: displayedPointSchema, - button: z.enum(["left", "right", "middle"]).default("left"), clickCount: z.union([z.literal(1), z.literal(2)]).default(1) }), - z.object({ kind: z.literal("scroll"), point: displayedPointSchema, - deltaX: z.number().finite().min(-4_000).max(4_000), deltaY: z.number().finite().min(-4_000).max(4_000) }), - z.object({ kind: z.literal("touch"), type: z.enum(["start", "move", "end", "cancel"]), - points: z.array(touchPointSchema).max(5) }).superRefine((event, context) => { + z.object({ + kind: z.literal("down"), + point: displayedPointSchema, + button: z.enum(["left", "right", "middle"]).default("left"), + clickCount: z.union([z.literal(1), z.literal(2)]).default(1), + }), + z.object({ + kind: z.literal("up"), + point: displayedPointSchema, + button: z.enum(["left", "right", "middle"]).default("left"), + clickCount: z.union([z.literal(1), z.literal(2)]).default(1), + }), + z.object({ + kind: z.literal("scroll"), + point: displayedPointSchema, + deltaX: z.number().finite().min(-4_000).max(4_000), + deltaY: z.number().finite().min(-4_000).max(4_000), + }), + z + .object({ + kind: z.literal("touch"), + type: z.enum(["start", "move", "end", "cancel"]), + points: z.array(touchPointSchema).max(5), + }) + .superRefine((event, context) => { if ((event.type === "end" || event.type === "cancel") !== (event.points.length === 0)) { - context.addIssue({ code: "custom", message: "Touch start/move require points; end/cancel require none" }); + context.addIssue({ + code: "custom", + message: "Touch start/move require points; end/cancel require none", + }); } - if (new Set(event.points.map(point => point.id)).size !== event.points.length) { + if (new Set(event.points.map((point) => point.id)).size !== event.points.length) { context.addIssue({ code: "custom", message: "Touch identifiers must be unique" }); } }), ]); export type BrowserGestureEvent = z.output; -const gestureContextSchema = z.object({ viewerToken: opaqueTokenSchema, controlToken: opaqueTokenSchema, - expected: expectedStateSchema.extend({ runtimeId: runtimeIdSchema, bridgeEpoch: epochSchema }) }); +const gestureContextSchema = z.object({ + viewerToken: opaqueTokenSchema, + controlToken: opaqueTokenSchema, + expected: expectedStateSchema.extend({ runtimeId: runtimeIdSchema, bridgeEpoch: epochSchema }), +}); const gestureContinuationSchema = gestureContextSchema.extend({ - gestureId: opaqueTokenSchema, sequence: z.number().int().positive().max(100_000), + gestureId: opaqueTokenSchema, + sequence: z.number().int().positive().max(100_000), }); /** Begin pins a decoded recent frame but sends no physical input. One live channel per controller. Keyboard events may share either pointer channel. */ export const beginBrowserGestureRpc = defineRpc({ name: "shared-browser.gesture.begin", - input: gestureContextSchema.extend({ target: targetFrameSchema, pointerKind: z.enum(["mouse", "touch"]) }), + input: gestureContextSchema.extend({ + target: targetFrameSchema, + pointerKind: z.enum(["mouse", "touch"]), + }), output: z.union([ - z.object({ state: browserStateSchema, gestureId: opaqueTokenSchema, nextSequence: z.number().int().positive() }), + z.object({ + state: browserStateSchema, + gestureId: opaqueTokenSchema, + nextSequence: z.number().int().positive(), + }), // This receipt exists only before input.begin, so no physical action or channel is replayed. z.object({ state: browserStateSchema, admission: z.literal("stale-frame") }), ]), @@ -338,10 +416,19 @@ export const beginBrowserGestureRpc = defineRpc({ */ export const updateBrowserGestureRpc = defineRpc({ name: "shared-browser.gesture.update", - input: gestureContinuationSchema.extend({ event: browserGestureEventSchema, target: targetFrameSchema.optional() }), + input: gestureContinuationSchema.extend({ + event: browserGestureEventSchema, + target: targetFrameSchema.optional(), + }), // The native input was acknowledged and navigated on its original attachment. // This closes the old channel; it never authorizes a continuation or a replay. - output: z.object({ state: browserStateSchema, gestureId: opaqueTokenSchema, nextSequence: z.number().int().positive(), cursor: browserCursorSchema.nullable(), completion: z.literal("navigation").optional() }), + output: z.object({ + state: browserStateSchema, + gestureId: opaqueTokenSchema, + nextSequence: z.number().int().positive(), + cursor: browserCursorSchema.nullable(), + completion: z.literal("navigation").optional(), + }), }); /** * End/cancel releases the original held input without retrying an uncertain action. diff --git a/paseo-shared-browser/shared/resolution-menu.test.ts b/paseo-shared-browser/shared/resolution-menu.test.ts index 831d0b8d..71f4e00c 100644 --- a/paseo-shared-browser/shared/resolution-menu.test.ts +++ b/paseo-shared-browser/shared/resolution-menu.test.ts @@ -1,24 +1,19 @@ import { describe, expect, it } from "vitest"; +import { + browserDisplayPreferences, + browserDisplayPreferencesSchema, +} from "./browser-display-preferences"; import { DEVICE_PRESETS } from "./device-presets"; import { groupResolutionPresets, normalizeResolutionFavorites, toggleResolutionFavorite, } from "./resolution-menu"; -import { - browserDisplayPreferences, - browserDisplayPreferencesSchema, -} from "./browser-display-preferences"; describe("resolution choices and host preferences", () => { it("groups the whole catalogue by viewport ratio, sorts width and retains mobile ties", () => { const groups = groupResolutionPresets(); - expect(groups.map((group) => group.label)).toEqual([ - "16:9", - "16:10", - "1:1", - "Mobile", - ]); + expect(groups.map((group) => group.label)).toEqual(["16:9", "16:10", "1:1", "Mobile"]); expect( groups .flatMap((group) => group.presets) @@ -27,9 +22,7 @@ describe("resolution choices and host preferences", () => { ).toEqual(DEVICE_PRESETS.map((preset) => preset.id).sort()); for (const group of groups) { expect(group.presets.map((preset) => preset.viewport.width)).toEqual( - group.presets - .map((preset) => preset.viewport.width) - .sort((a, b) => a - b), + group.presets.map((preset) => preset.viewport.width).sort((a, b) => a - b), ); for (const preset of group.presets) { if (group.id === "mobile") expect(preset.isMobile).toBe(true); @@ -59,30 +52,17 @@ describe("resolution choices and host preferences", () => { "unknown", ]), ).toEqual(["pixel-7-sharp", "desktop-chrome"]); - expect(normalizeResolutionFavorites(Array(100).fill("pixel-7"))).toEqual([ - "pixel-7", - ]); + expect(normalizeResolutionFavorites(Array(100).fill("pixel-7"))).toEqual(["pixel-7"]); expect( - normalizeResolutionFavorites( - DEVICE_PRESETS.flatMap((preset) => [preset.id, preset.id]), - ), + normalizeResolutionFavorites(DEVICE_PRESETS.flatMap((preset) => [preset.id, preset.id])), ).toHaveLength(DEVICE_PRESETS.length); }); it("toggles only catalogue entries and does not mutate stored input", () => { - const input = Object.freeze([ - "retired", - "desktop-chrome", - "desktop-chrome", - ]); - expect(toggleResolutionFavorite(input, "pixel-7")).toEqual([ - "desktop-chrome", - "pixel-7", - ]); + const input = Object.freeze(["retired", "desktop-chrome", "desktop-chrome"]); + expect(toggleResolutionFavorite(input, "pixel-7")).toEqual(["desktop-chrome", "pixel-7"]); expect(toggleResolutionFavorite(input, "desktop-chrome")).toEqual([]); - expect(toggleResolutionFavorite(input, "unknown")).toEqual([ - "desktop-chrome", - ]); + expect(toggleResolutionFavorite(input, "unknown")).toEqual(["desktop-chrome"]); expect(input).toEqual(["retired", "desktop-chrome", "desktop-chrome"]); }); @@ -93,13 +73,11 @@ describe("resolution choices and host preferences", () => { captureQuality: "high", }); expect( - browserDisplayPreferencesSchema.parse({ favoritePresetIds: ["retired"] }) - .favoritePresetIds, + browserDisplayPreferencesSchema.parse({ favoritePresetIds: ["retired"] }).favoritePresetIds, ).toEqual(["retired"]); - expect( - browserDisplayPreferencesSchema.safeParse({ captureQuality: "ultra" }) - .success, - ).toBe(false); + expect(browserDisplayPreferencesSchema.safeParse({ captureQuality: "ultra" }).success).toBe( + false, + ); expect( browserDisplayPreferencesSchema.safeParse({ favoritePresetIds: Array(DEVICE_PRESETS.length + 1).fill("pixel-7"), diff --git a/paseo-shared-browser/shared/resolution-menu.ts b/paseo-shared-browser/shared/resolution-menu.ts index d2a8c55c..da399e3d 100644 --- a/paseo-shared-browser/shared/resolution-menu.ts +++ b/paseo-shared-browser/shared/resolution-menu.ts @@ -1,9 +1,5 @@ /** Catalogue grouping and stable-ID favorites for every resolution picker surface. */ -import { - DEVICE_PRESETS, - DEVICE_PRESET_IDS, - type DevicePresetId, -} from "./device-presets"; +import { DEVICE_PRESET_IDS, DEVICE_PRESETS, type DevicePresetId } from "./device-presets"; export type ResolutionPreset = (typeof DEVICE_PRESETS)[number]; export type ResolutionGroupId = "16:9" | "16:10" | "1:1" | "mobile"; @@ -34,26 +30,19 @@ export function groupResolutionPresets(): ResolutionGroup[] { else if (width === height) id = "1:1"; else if (width * 9 === height * 16) id = "16:9"; else if (width * 10 === height * 16) id = "16:10"; - else - throw new Error( - `Resolution preset ${preset.id} requires an aspect-ratio group`, - ); + else throw new Error(`Resolution preset ${preset.id} requires an aspect-ratio group`); const group = groups.find((candidate) => candidate.id === id); if (!group) throw new Error(`Resolution group ${id} is missing`); group.presets = [...group.presets, preset]; } return groups.map((group) => ({ ...group, - presets: [...group.presets].sort( - (a, b) => a.viewport.width - b.viewport.width, - ), + presets: [...group.presets].sort((a, b) => a.viewport.width - b.viewport.width), })); } /** Drop removed/unknown IDs and duplicates, retaining the user's favorite order. */ -export function normalizeResolutionFavorites( - ids: readonly string[], -): DevicePresetId[] { +export function normalizeResolutionFavorites(ids: readonly string[]): DevicePresetId[] { const favorites: DevicePresetId[] = []; const seen = new Set(); for (const id of ids) { @@ -66,13 +55,9 @@ export function normalizeResolutionFavorites( } /** Toggle only a current preset. Unknown IDs cannot become persisted shortcuts. */ -export function toggleResolutionFavorite( - ids: readonly string[], - id: string, -): DevicePresetId[] { +export function toggleResolutionFavorite(ids: readonly string[], id: string): DevicePresetId[] { const current = normalizeResolutionFavorites(ids); if (!KNOWN_PRESET_IDS.has(id)) return current; - if (current.includes(id as DevicePresetId)) - return current.filter((existing) => existing !== id); + if (current.includes(id as DevicePresetId)) return current.filter((existing) => existing !== id); return [...current, id as DevicePresetId]; } diff --git a/paseo-shared-browser/tests/browser.smoke.test.ts b/paseo-shared-browser/tests/browser.smoke.test.ts index e05c186c..74be58fa 100644 --- a/paseo-shared-browser/tests/browser.smoke.test.ts +++ b/paseo-shared-browser/tests/browser.smoke.test.ts @@ -9,6 +9,7 @@ import { resolveBrowserRuntimeRoot } from "../server/runtime-path"; import { resolveSupervisorPaths, startSupervisorServer } from "../server/supervisor"; import { SupervisorClient } from "../server/supervisor-client"; import type { BrowserFrame, BrowserState } from "../shared/browser"; +import { DEFAULT_CAPTURE_QUALITY } from "../shared/capture-settings"; const roots: string[] = []; afterEach(async () => { @@ -153,7 +154,15 @@ it("shares and persists a production agent-browser runtime across supervisor cli action: { kind: "goto", url: `${origin}/?delayed=1` }, }); expect(navigated.state.url).toBe(`${origin}/?delayed=1`); - expect(navigated.state.title).toBe("Shared Browser Smoke"); + // URL actions acknowledge native navigation; this fixture separately waits + // for its delayed DOM before exercising the remote input elements below. + await vi.waitFor( + async () => { + const current = await manager.status(first.viewerToken); + expect(current.state.title).toBe("Shared Browser Smoke"); + }, + { timeout: 5_000 }, + ); console.log("browser-smoke: navigated"); const firstCapture = await manager.capture(first.viewerToken, "medium", null); @@ -245,7 +254,13 @@ it("shares and persists a production agent-browser runtime across supervisor cli }); await vi.waitFor(() => expect(lastUserAgent).toContain("Pixel 7")); await vi.waitFor(async () => { - const resumedCapture = await manager.capture(resumed.viewerToken, "medium", null); + // Only the default quality requests the shared CDP stream. Other viewer + // qualities intentionally use screenshots without restarting that stream. + const resumedCapture = await manager.capture( + resumed.viewerToken, + DEFAULT_CAPTURE_QUALITY, + null, + ); expect(resumedCapture.frame?.transport).toBe("cdp-screencast"); }); console.log("browser-smoke: emulated"); diff --git a/paseo-shared-browser/tests/viewport-resync.test.ts b/paseo-shared-browser/tests/viewport-resync.test.ts index 20ea2b45..f3495335 100644 --- a/paseo-shared-browser/tests/viewport-resync.test.ts +++ b/paseo-shared-browser/tests/viewport-resync.test.ts @@ -26,10 +26,17 @@ function jpeg(width: number, height: number): string { ]).toString("base64"); } -function fakeRuntime(heights: number[], viewport = { width: 1280, height: 800 }, captureScale = 1, dpr = 1) { +function fakeRuntime( + heights: number[], + viewport = { width: 1280, height: 800 }, + captureScale = 1, + dpr = 1, +) { const calls: string[] = []; const page = { - async send(method: string) { + async send( + method: string, + ): Promise<{ data?: string; cssVisualViewport?: { pageX: number; pageY: number } }> { calls.push(method); if (method === "Page.captureScreenshot") { const height = heights.shift() ?? 800; @@ -46,14 +53,15 @@ function fakeRuntime(heights: number[], viewport = { width: 1280, height: 800 }, ipcDirectory: "/tmp/i", session: "fake", }); + const connection = { isOpen: true }; Object.assign(runtime, { page, emulationAppliedPage: page, - connection: { isOpen: true }, + connection, viewport: { ...viewport, deviceScaleFactor: dpr, captureScale, mobile: false, touch: false }, }); const control = runtime as unknown as { requirePage: () => Promise }; - Object.assign(page, { connection: (runtime as any).connection }); + Object.assign(page, { connection }); control.requirePage = async () => page; return { runtime, calls, page }; } @@ -89,7 +97,6 @@ describe("fallback frame viewport resync", () => { }); }); - it("recovers to captureScale pixels independently of mobile DPR", async () => { const { runtime, calls } = fakeRuntime([600, 1600], { width: 1280, height: 800 }, 2, 3); expect(await runtime.frame(1000000, 65, 1)).toMatchObject({ width: 2560, height: 1600 }); @@ -99,9 +106,10 @@ it("recovers to captureScale pixels independently of mobile DPR", async () => { it("rejects malformed JPEG without using it to trigger viewport recovery", async () => { const { runtime, calls, page } = fakeRuntime([800]); const original = page.send; - page.send = async (method: string) => method === "Page.captureScreenshot" - ? { data: Buffer.from("not a jpeg").toString("base64") } as any - : original(method); + page.send = async (method: string) => + method === "Page.captureScreenshot" + ? { data: Buffer.from("not a jpeg").toString("base64") } + : original(method); await expect(runtime.frame(1000000, 65, 1)).rejects.toThrow("malformed JPEG"); expect(calls).not.toContain("Emulation.setDeviceMetricsOverride"); }); @@ -112,7 +120,7 @@ it("drops viewport recovery after a mutation supersedes capture", async () => { page.send = async (method: string) => { const result = await original(method); if (method === "Emulation.setDeviceMetricsOverride") { - (runtime as any).invalidateScreencastFrame(); + (runtime as unknown as { invalidateScreencastFrame(): void }).invalidateScreencastFrame(); } return result; }; From 98dfd465960687416c1759240fbd36b07231129b Mon Sep 17 00:00:00 2001 From: Ryan Ewen Date: Sun, 4 Oct 2026 12:46:10 -0400 Subject: [PATCH 3/5] fix(shared-browser): resolve text cursors during hover Resolve CSS auto cursors over selectable glyphs and editable surfaces while preserving explicit cursor styles and existing control authority. Cover native text, padding, vertical text and shadow content with isolated Chromium checks. --- paseo-shared-browser/README.md | 2 + .../server/agent-browser-runtime.ts | 3 +- .../server/browser-cursor.test.ts | 64 +++++++++++++++++ paseo-shared-browser/server/browser-cursor.ts | 69 +++++++++++++++++++ .../tests/browser.smoke.test.ts | 67 +++++++++++++++++- 5 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 paseo-shared-browser/server/browser-cursor.test.ts create mode 100644 paseo-shared-browser/server/browser-cursor.ts diff --git a/paseo-shared-browser/README.md b/paseo-shared-browser/README.md index 82802395..fe1d405c 100644 --- a/paseo-shared-browser/README.md +++ b/paseo-shared-browser/README.md @@ -187,6 +187,8 @@ User-supplied `AGENT_BROWSER_*` variables are deliberately ignored. - Human control: **Take control**, **Release**, and **Take over** for explicit handoff. Agent MCP calls have no forced-takeover operation. - With control, mouse movement forwards real hover effects and standard browser cursor changes. + Automatic cursors resolve selectable text to an I-beam while respecting explicit + cursor styles and non-selectable areas. Wheel scrolling stays inside the browser canvas; held drags update before release. Leaving the canvas clears remote hover, while dragging beyond its edges still releases the held button. - Hidden Linux browsers use an authenticated private virtual display when `/usr/bin/Xvfb` diff --git a/paseo-shared-browser/server/agent-browser-runtime.ts b/paseo-shared-browser/server/agent-browser-runtime.ts index 03914aec..2c6ff127 100644 --- a/paseo-shared-browser/server/agent-browser-runtime.ts +++ b/paseo-shared-browser/server/agent-browser-runtime.ts @@ -24,6 +24,7 @@ import { browserCursorSchema, browserGestureKeySchema, } from "../shared/browser"; +import { browserCursorExpression } from "./browser-cursor"; import { GESTURE_IDLE_MS, GESTURE_LIFETIME_MS } from "./browser-gesture"; import { CAPTURE_MAX_AGE_MS, createCaptureTransportPolicy } from "./capture-transport-policy"; import { formatRuntimeInputGeneration } from "./input-generation"; @@ -1074,7 +1075,7 @@ export class AgentBrowserRuntime { await this.assertLiveInput(gestureId); const page = this.liveInput!.page; const result = await page.send<{ result: { value?: unknown } }>("Runtime.evaluate", { - expression: `(() => { const e = document.elementFromPoint(${x}, ${y}); return e ? getComputedStyle(e).cursor : null; })()`, + expression: browserCursorExpression(x, y), returnByValue: true, }); await this.assertLiveInput(gestureId); diff --git a/paseo-shared-browser/server/browser-cursor.test.ts b/paseo-shared-browser/server/browser-cursor.test.ts new file mode 100644 index 00000000..5095ed0b --- /dev/null +++ b/paseo-shared-browser/server/browser-cursor.test.ts @@ -0,0 +1,64 @@ +/** Exercise the actual page program with bounded DOM fixtures; native geometry is covered by smoke. */ +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; +import { browserCursorExpression } from "./browser-cursor"; + +function fixture({ + cursor = "auto", + userSelect = "auto", + writingMode = "horizontal-tb", + editable = false, + contains = true, + legacy = false, +} = {}) { + const node = { nodeType: 3, length: 5 }; + const element = { isContentEditable: editable, contains: () => contains }; + const range = { + setStart() {}, + setEnd() {}, + getClientRects: () => [{ left: 10, right: 40, top: 10, bottom: 25 }], + }; + const position = { offsetNode: node, offset: 2 }; + const document = { + elementFromPoint: () => element, + createRange: () => range, + ...(legacy + ? { caretRangeFromPoint: () => ({ startContainer: node, startOffset: 2 }) } + : { caretPositionFromPoint: () => position }), + }; + return { + document, + sample: (x = 20, y = 15) => + runInNewContext(browserCursorExpression(x, y), { + document, + getComputedStyle: () => ({ cursor, userSelect, writingMode }), + }), + }; +} + +it("resolves auto over selectable text, including the legacy caret API", () => { + expect(fixture().sample()).toBe("text"); + expect(fixture({ legacy: true }).sample()).toBe("text"); +}); + +it("preserves explicit cursor choices even over selectable text", () => { + expect(fixture({ cursor: "default" }).sample()).toBe("default"); + expect(fixture({ cursor: "pointer" }).sample()).toBe("pointer"); +}); + +it("does not mistake nearby text, overlays or non-selectable content for text", () => { + expect(fixture().sample(90, 15)).toBe("default"); + expect(fixture({ contains: false }).sample()).toBe("default"); + expect(fixture({ userSelect: "none" }).sample()).toBe("default"); +}); + +it("resolves empty editing surfaces and vertical text without moving selection", () => { + expect(fixture({ editable: true }).sample(90, 15)).toBe("text"); + expect(fixture({ writingMode: "vertical-rl" }).sample()).toBe("vertical-text"); +}); + +it("rejects invalid coordinates before constructing page code", () => { + for (const x of [Number.NaN, Number.POSITIVE_INFINITY, -1]) { + expect(() => browserCursorExpression(x, 10)).toThrow("finite non-negative"); + } +}); diff --git a/paseo-shared-browser/server/browser-cursor.ts b/paseo-shared-browser/server/browser-cursor.ts new file mode 100644 index 00000000..b459e900 --- /dev/null +++ b/paseo-shared-browser/server/browser-cursor.ts @@ -0,0 +1,69 @@ +/** + * Read-only cursor resolution for the guarded native page in agent-browser-runtime. + * CSS keeps `auto` as its computed value; resolve selectable glyphs/editing surfaces + * without exporting page text, changing selection, or inspecting a whole subtree. + * The page program handles the top document and exposed shadow roots. Opaque + * frames remain subject to the existing top-document hit-test limitation. + */ +const CURSOR_HIT_TEST = `(x, y) => { + let element = document.elementFromPoint(x, y); + if (!element) return null; + + const shadowRoots = []; + // Bound descent and retain roots for Chromium's caret hit-test options. + for (let depth = 0; depth < 16 && element.shadowRoot; depth++) { + const root = element.shadowRoot; + const inner = root.elementFromPoint(x, y); + if (!inner || inner === element) break; + shadowRoots.push(root); + element = inner; + } + + const style = getComputedStyle(element); + if (style.cursor !== 'auto') return style.cursor; + if (style.userSelect === 'none') return 'default'; + + const textCursor = style.writingMode.startsWith('vertical') || + style.writingMode.startsWith('sideways') ? 'vertical-text' : 'text'; + if (element.isContentEditable) return textCursor; + + // Explicit auto on text controls must also work over empty editable space. + const textInputTypes = ['text', 'search', 'url', 'tel', 'email', 'password', 'number']; + const textControl = element.tagName === 'TEXTAREA' || + (element.tagName === 'INPUT' && textInputTypes.includes(element.type)); + if (textControl && !element.disabled) return textCursor; + + let node; + let offset; + if (document.caretPositionFromPoint) { + const position = document.caretPositionFromPoint(x, y, { shadowRoots }); + node = position?.offsetNode; + offset = position?.offset; + } else if (document.caretRangeFromPoint) { + const position = document.caretRangeFromPoint(x, y); + node = position?.startContainer; + offset = position?.startOffset; + } + if (!node || node.nodeType !== 3 || !element.contains(node) || + !Number.isInteger(offset) || node.length === 0) return 'default'; + + // Caret APIs can snap to nearby text even over padding. Check only adjacent + // glyph rectangles so blank areas and overlays never inherit a text cursor. + const range = document.createRange(); + range.setStart(node, Math.max(0, offset - 1)); + range.setEnd(node, Math.min(node.length, offset + 1)); + for (const rect of range.getClientRects()) { + if (x >= rect.left && x < rect.right && y >= rect.top && y < rect.bottom) { + return textCursor; + } + } + return 'default'; +}`; + +/** Build the fixed page program with numeric CSS coordinates, never executable caller input. */ +export function browserCursorExpression(x: number, y: number): string { + if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0) { + throw new Error("Cursor coordinates must be finite non-negative CSS pixels"); + } + return `(${CURSOR_HIT_TEST})(${x}, ${y})`; +} diff --git a/paseo-shared-browser/tests/browser.smoke.test.ts b/paseo-shared-browser/tests/browser.smoke.test.ts index 74be58fa..0cc8e224 100644 --- a/paseo-shared-browser/tests/browser.smoke.test.ts +++ b/paseo-shared-browser/tests/browser.smoke.test.ts @@ -88,9 +88,23 @@ it("shares and persists a production agent-browser runtime across supervisor cli Shared Browser Smoke +
Selectable text
+
Non-selectable
+
Explicit arrow
+
+
Vertical text
+ Link +
+ `; response.setHeader("Content-Type", "text/html; charset=utf-8"); if (url.searchParams.has("delayed")) { @@ -165,12 +179,63 @@ it("shares and persists a production agent-browser runtime across supervisor cli ); console.log("browser-smoke: navigated"); - const firstCapture = await manager.capture(first.viewerToken, "medium", null); + let firstCapture = await manager.capture(first.viewerToken, "medium", null); expect(firstCapture.frame?.byteLength).toBeLessThanOrEqual(800_000); const secondCapture = await manager.capture(second.viewerToken, "medium", null); expect(secondCapture.state.sessionId).toBe(firstCapture.state.sessionId); expect(secondCapture.state.controller).toBe("other"); + // Exercise actual glyph geometry through the same guarded human RPC path. + const { runtimeId, bridgeEpoch } = firstCapture.state; + if (typeof runtimeId !== "string" || typeof bridgeEpoch !== "number") { + throw new Error("Native cursor fixture requires current runtime identity"); + } + const cursorContext = { + viewerToken: first.viewerToken, + controlToken: firstControl.controlToken, + expected: { + ...expected(firstCapture.state), + runtimeId, + bridgeEpoch, + }, + }; + const channel = await manager.beginGesture({ + ...cursorContext, + target: target(firstCapture.frame!), + pointerKind: "mouse", + }); + if (!("gestureId" in channel) || typeof channel.nextSequence !== "number") { + throw new Error("Fresh cursor fixture was not admitted"); + } + let sequence = channel.nextSequence; + for (const [x, y, cursor] of [ + [45, 115, "text"], + [390, 115, "default"], + [45, 165, "default"], + [45, 215, "default"], + [390, 265, "text"], + [55, 305, "vertical-text"], + [505, 108, "pointer"], + [505, 165, "text"], + [150, 46, "text"], + ] as const) { + const update = await manager.updateGesture({ + ...cursorContext, + gestureId: channel.gestureId, + sequence, + event: { kind: "move", point: { x, y, width: 1280, height: 800 } }, + }); + expect(update.cursor, `Cursor at ${x},${y}`).toBe(cursor); + sequence = update.nextSequence; + } + await manager.endGesture({ + ...cursorContext, + gestureId: channel.gestureId, + sequence, + cancel: false, + }); + firstCapture = await manager.capture(first.viewerToken, "medium", null); + await manager.sendInput({ viewerToken: first.viewerToken, controlToken: firstControl.controlToken, From 5545821b642dc8a76a1d060bcbaac5e2ff6d4b26 Mon Sep 17 00:00:00 2001 From: Ryan Ewen Date: Sun, 4 Oct 2026 12:46:10 -0400 Subject: [PATCH 4/5] fix(shared-browser): name presets and order resolution favorites Use recognizable height labels, add 1440-wide desktop choices, and project favorites in picker order without rewriting saved IDs. Include shared contract tests in the normal regression command. --- paseo-shared-browser/README.md | 6 ++++- .../client/use-resolution-favorites.ts | 8 ++++-- paseo-shared-browser/package.json | 4 +-- paseo-shared-browser/shared/device-presets.ts | 23 +++++----------- .../shared/resolution-menu.test.ts | 26 +++++++++++++++++++ .../shared/resolution-menu.ts | 9 +++++++ 6 files changed, 55 insertions(+), 21 deletions(-) diff --git a/paseo-shared-browser/README.md b/paseo-shared-browser/README.md index fe1d405c..758ad3ab 100644 --- a/paseo-shared-browser/README.md +++ b/paseo-shared-browser/README.md @@ -146,9 +146,13 @@ User-supplied `AGENT_BROWSER_*` variables are deliberately ignored. with a sharper 824 × 1678 capture. - Desktop presets include 1920 × 1080 and 2560 × 1440 (16:9), 1920 × 1200 and 2560 × 1600 (16:10), and 1920 × 1920 and 2560 × 2560 (1:1). +- Desktop choices use height names such as 720p, 1080p, 1200p and 1440p, with + dimensions shown alongside. Square choices name their width explicitly. + The 1440-wide choices are 1440 × 810, 1440 × 900 and 1440 × 1440. - The Resolution and quality menu groups desktop choices by 16:9, 16:10 and 1:1, then mobile. Each group sorts by width. Filled stars mark favorites; the monitor - menu provides favorites and access to the full resolution and quality list. + menu provides favorites in that same grouped order and access to the full + resolution and quality list. - Monitor and browser actions open compact anchored menus rather than dialogs. Menus stay within the pane, scroll long lists, and dismiss with an outside press, Escape or native Back. - The monitor menu offers **Fit to panel** (default) and **Actual size (100%)**. Actual size diff --git a/paseo-shared-browser/client/use-resolution-favorites.ts b/paseo-shared-browser/client/use-resolution-favorites.ts index 2602bd44..1e2db9c2 100644 --- a/paseo-shared-browser/client/use-resolution-favorites.ts +++ b/paseo-shared-browser/client/use-resolution-favorites.ts @@ -11,7 +11,11 @@ import { } from "../shared/browser-display-preferences"; import { DEFAULT_CAPTURE_QUALITY } from "../shared/capture-settings"; import type { DevicePresetId } from "../shared/device-presets"; -import { normalizeResolutionFavorites, toggleResolutionFavorite } from "../shared/resolution-menu"; +import { + normalizeResolutionFavorites, + orderedResolutionFavorites, + toggleResolutionFavorite, +} from "../shared/resolution-menu"; /** Call with the surface's host.id; persistence itself is scoped by Paseo. */ export function useResolutionFavorites(hostId: string) { @@ -24,7 +28,7 @@ export function useResolutionFavorites(hostId: string) { const disabled = settings.status !== "ready" || saving; const favoritePresetIds = settings.status === "ready" - ? normalizeResolutionFavorites(settings.values.favoritePresetIds) + ? orderedResolutionFavorites(settings.values.favoritePresetIds) : []; const error = settings.status === "error" || settings.status === "invalid" diff --git a/paseo-shared-browser/package.json b/paseo-shared-browser/package.json index ca15651d..6e7eb54c 100644 --- a/paseo-shared-browser/package.json +++ b/paseo-shared-browser/package.json @@ -43,8 +43,8 @@ "check:write": "biome check --write .", "typecheck": "tsc --noEmit", "prepare:runtime": "node scripts/prepare-runtime.mjs", - "test": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts client server", - "test:unit": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts client server", + "test": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts client server shared", + "test:unit": "vitest run tests/contracts.test.ts tests/lease.test.ts tests/runtime-lifecycle.test.ts tests/agent-access.test.ts tests/composer-pills.test.ts tests/sidebar.test.ts tests/pointer-mapping.test.ts tests/frame-lifecycle.test.ts tests/viewport-resync.test.ts client server shared", "test:smoke": "vitest run tests/browser.smoke.test.ts --testTimeout=30000" }, "dependencies": { diff --git a/paseo-shared-browser/shared/device-presets.ts b/paseo-shared-browser/shared/device-presets.ts index 49980dab..73faac96 100644 --- a/paseo-shared-browser/shared/device-presets.ts +++ b/paseo-shared-browser/shared/device-presets.ts @@ -11,19 +11,13 @@ const DESKTOP_CHROME_EMULATION = { /** Larger desktop presets change resolution while retaining the same input and browser behavior. */ function createDesktopPreset(id: Id, width: number, height: number) { - const dimensions = `${width} × ${height}`; - let ratio = "16:9"; - if (width === height) { - ratio = "1:1"; - } else if (width * 10 === height * 16) { - ratio = "16:10"; - } + const name = width === height ? `${width} square` : `${height}p`; return { ...DESKTOP_CHROME_EMULATION, id, - label: `Desktop ${dimensions} (${ratio})`, - shortLabel: dimensions, + label: name, + shortLabel: name, viewport: { width, height }, } as const; } @@ -43,15 +37,12 @@ const PIXEL_7_EMULATION = { } as const; export const DEVICE_PRESETS = [ - { - ...DESKTOP_CHROME_EMULATION, - id: "desktop-chrome", - label: "Desktop Chrome", - shortLabel: "Desktop", - viewport: { width: 1280, height: 720 }, - }, + createDesktopPreset("desktop-chrome", 1280, 720), createDesktopPreset("desktop-1280x800", 1280, 800), createDesktopPreset("desktop-1280x1280", 1280, 1280), + createDesktopPreset("desktop-1440x810", 1440, 810), + createDesktopPreset("desktop-1440x900", 1440, 900), + createDesktopPreset("desktop-1440x1440", 1440, 1440), createDesktopPreset("desktop-1920x1080", 1920, 1080), createDesktopPreset("desktop-2560x1440", 2560, 1440), createDesktopPreset("desktop-1920x1200", 1920, 1200), diff --git a/paseo-shared-browser/shared/resolution-menu.test.ts b/paseo-shared-browser/shared/resolution-menu.test.ts index 71f4e00c..bd5cea9e 100644 --- a/paseo-shared-browser/shared/resolution-menu.test.ts +++ b/paseo-shared-browser/shared/resolution-menu.test.ts @@ -7,6 +7,7 @@ import { DEVICE_PRESETS } from "./device-presets"; import { groupResolutionPresets, normalizeResolutionFavorites, + orderedResolutionFavorites, toggleResolutionFavorite, } from "./resolution-menu"; @@ -42,6 +43,31 @@ describe("resolution choices and host preferences", () => { ]); }); + it("projects scrambled saved favorites in the grouped picker order without rewriting them", () => { + const saved = Object.freeze([ + "pixel-7-sharp", + "desktop-1440x1440", + "desktop-1920x1200", + "desktop-2560x1440", + "desktop-1440x900", + "desktop-chrome", + "desktop-1440x810", + "desktop-chrome", + "retired", + ]); + expect(orderedResolutionFavorites(saved)).toEqual([ + "desktop-chrome", + "desktop-1440x810", + "desktop-2560x1440", + "desktop-1440x900", + "desktop-1920x1200", + "desktop-1440x1440", + "pixel-7-sharp", + ]); + expect(saved[0]).toBe("pixel-7-sharp"); + expect(saved).toHaveLength(9); + }); + it("prunes removed favorites and duplicates without changing stable favorite order", () => { expect( normalizeResolutionFavorites([ diff --git a/paseo-shared-browser/shared/resolution-menu.ts b/paseo-shared-browser/shared/resolution-menu.ts index da399e3d..c2b58db8 100644 --- a/paseo-shared-browser/shared/resolution-menu.ts +++ b/paseo-shared-browser/shared/resolution-menu.ts @@ -54,6 +54,15 @@ export function normalizeResolutionFavorites(ids: readonly string[]): DevicePres return favorites; } +/** Project stored favorites in picker order without rewriting the user's saved settings. */ +export function orderedResolutionFavorites(ids: readonly string[]): DevicePresetId[] { + const favorites = new Set(normalizeResolutionFavorites(ids)); + return groupResolutionPresets() + .flatMap((group) => group.presets) + .filter((preset) => favorites.has(preset.id)) + .map((preset) => preset.id); +} + /** Toggle only a current preset. Unknown IDs cannot become persisted shortcuts. */ export function toggleResolutionFavorite(ids: readonly string[], id: string): DevicePresetId[] { const current = normalizeResolutionFavorites(ids); From 0349825ba4ad786bcf67d359862fb053192b71d4 Mon Sep 17 00:00:00 2001 From: Ryan Ewen Date: Sun, 4 Oct 2026 12:46:10 -0400 Subject: [PATCH 5/5] fix(shared-browser): show tooltips on browser toolbar icons Use each icon control accessibility label as its native web hover title, including contextual mode labels. Keep native accessibility and shared toolbar behavior intact. --- paseo-shared-browser/README.md | 3 ++- .../client/browser-control-tooltip-web.ts | 10 ++++++++++ paseo-shared-browser/client/browser.tsx | 3 +++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 paseo-shared-browser/client/browser-control-tooltip-web.ts diff --git a/paseo-shared-browser/README.md b/paseo-shared-browser/README.md index 758ad3ab..3887d858 100644 --- a/paseo-shared-browser/README.md +++ b/paseo-shared-browser/README.md @@ -138,7 +138,8 @@ User-supplied `AGENT_BROWSER_*` variables are deliberately ignored. ## Controls - Toolbar: back, forward, reload, address bar, a combined monitor menu, mobile emulation toggle - and a vertical-dots browser actions menu. + and a vertical-dots browser actions menu. Icon controls expose their action name + as a hover tooltip on desktop/web and retain native accessibility labels. - Custom viewports support 320 to 2560 pixels wide and 480 to 2560 pixels high. Invalid sizes show their error inside the device dialog; a successful Apply closes it. - Desktop presets also include 1280 × 800 (16:10) and 1280 × 1280 (1:1). diff --git a/paseo-shared-browser/client/browser-control-tooltip-web.ts b/paseo-shared-browser/client/browser-control-tooltip-web.ts new file mode 100644 index 00000000..29c54c5f --- /dev/null +++ b/paseo-shared-browser/client/browser-control-tooltip-web.ts @@ -0,0 +1,10 @@ +/** Native browser hover titles for public RN web control refs; native clients keep accessibility labels. */ +import { Platform } from "react-native"; + +/** Attach the control's current accessible name without creating overlays or remote input handlers. */ +export function setBrowserControlTooltip(node: unknown, label: string): void { + if (Platform.OS !== "web" || !node || typeof node !== "object") return; + if ("setAttribute" in node && typeof node.setAttribute === "function") { + node.setAttribute("title", label); + } +} diff --git a/paseo-shared-browser/client/browser.tsx b/paseo-shared-browser/client/browser.tsx index 619292db..49c2824f 100644 --- a/paseo-shared-browser/client/browser.tsx +++ b/paseo-shared-browser/client/browser.tsx @@ -47,6 +47,7 @@ import { groupResolutionPresets } from "../shared/resolution-menu"; import { type BrowserCanvasDisplayMode, getBrowserCanvasLayout } from "./browser-canvas-layout"; import { BrowserCanvasViewport } from "./browser-canvas-viewport"; import { browserCaptureInterval } from "./browser-capture-cadence"; +import { setBrowserControlTooltip } from "./browser-control-tooltip-web"; import { type EmulationSelection, matchingResolutionPresetId } from "./browser-emulation-mode"; import type { FrameCandidate } from "./browser-frame-buffer"; import { BrowserFrameImage } from "./browser-frame-image"; @@ -588,8 +589,10 @@ function ChromeIconButton({ onPress(): void; }) { const [hovered, setHovered] = useState(false); + const tooltipRef = useCallback((node: unknown) => setBrowserControlTooltip(node, label), [label]); return (