diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 1a926718f..4ae95662f 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -956,7 +956,8 @@ interface CursorTelemetryPoint { | "double-click" | "right-click" | "middle-click" - | "mouseup"; + | "mouseup" + | "keydown"; cursorType?: | "arrow" | "text" diff --git a/electron/ipc/cursor/interaction.test.ts b/electron/ipc/cursor/interaction.test.ts index 4ea662414..56d2c6c19 100644 --- a/electron/ipc/cursor/interaction.test.ts +++ b/electron/ipc/cursor/interaction.test.ts @@ -12,10 +12,21 @@ vi.mock("electron", () => ({ })); import { + isCandidateTypingKeyEvent, repairBundledUiohookBinaryForCurrentArch, shouldStartGlobalInteractionHook, } from "./interaction"; +describe("isCandidateTypingKeyEvent", () => { + it("accepts ordinary keys but filters modifiers and Ctrl/Meta shortcuts", () => { + expect(isCandidateTypingKeyEvent({ keycode: 0x001e })).toBe(true); + expect(isCandidateTypingKeyEvent({ keycode: 0x002a })).toBe(false); // Shift + expect(isCandidateTypingKeyEvent({ keycode: 0x003a })).toBe(false); // Caps Lock + expect(isCandidateTypingKeyEvent({ keycode: 0x001e, ctrlKey: true })).toBe(false); + expect(isCandidateTypingKeyEvent({ keycode: 0x001e, metaKey: true })).toBe(false); + }); +}); + describe("shouldStartGlobalInteractionHook", () => { it("does not start the synchronous uiohook event tap on macOS", () => { expect(shouldStartGlobalInteractionHook("darwin")).toBe(false); diff --git a/electron/ipc/cursor/interaction.ts b/electron/ipc/cursor/interaction.ts index 47c42437f..1080fb898 100644 --- a/electron/ipc/cursor/interaction.ts +++ b/electron/ipc/cursor/interaction.ts @@ -13,6 +13,7 @@ import { } from "../state"; import type { CursorInteractionType, + HookKeyboardEvent, HookMouseEvent, UiohookLike, UiohookModuleNamespace, @@ -233,6 +234,47 @@ export function recordCursorMouseUp() { pushCursorSample(point.cx, point.cy, getCursorCaptureElapsedMs(), "mouseup"); } +/** uiohook virtual key codes for non-typing modifier/toggle keys. */ +const MODIFIER_ONLY_KEY_CODES = new Set([ + 0x002a, // left Shift + 0x0036, // right Shift + 0x001d, // left Control + 0x0e1d, // right Control + 0x0038, // left Alt + 0x0e38, // right Alt + 0x0e5b, // left Meta / Command + 0x0e5c, // right Meta / Command + 0x003a, // Caps Lock + 0x0045, // Num Lock + 0x0046, // Scroll Lock +]); + +/** + * Keeps keyboard telemetry privacy-preserving: this decision uses only a + * modifier state and key code to decide whether to record an anonymous + * timestamp + cursor position. It never reads or stores key characters. + */ +export function isCandidateTypingKeyEvent(event: HookKeyboardEvent): boolean { + if (event.ctrlKey || event.metaKey) { + return false; + } + + return typeof event.keycode === "number" && !MODIFIER_ONLY_KEY_CODES.has(event.keycode); +} + +export function recordCursorKeyDown() { + if (!isCursorCaptureActive || isCursorCapturePaused()) { + return; + } + + const point = getNormalizedCursorPoint(); + if (!point) { + return; + } + + pushCursorSample(point.cx, point.cy, getCursorCaptureElapsedMs(), "keydown"); +} + export async function startInteractionCapture() { if (!isCursorCaptureActive) { return; @@ -276,6 +318,12 @@ export async function startInteractionCapture() { recordCursorMouseUp(); }; + const onKeyDown = (event: HookKeyboardEvent) => { + if (isCandidateTypingKeyEvent(event)) { + recordCursorKeyDown(); + } + }; + const onMouseMove = (event: HookMouseEvent) => { if (process.platform !== "linux" || !isCursorCaptureActive || isCursorCapturePaused()) { return; @@ -291,6 +339,7 @@ export async function startInteractionCapture() { hook.on("mousedown", onMouseDown); hook.on("mouseup", onMouseUp); + hook.on("keydown", onKeyDown); if (process.platform === "linux") { hook.on("mousemove", onMouseMove); } @@ -300,12 +349,14 @@ export async function startInteractionCapture() { if (typeof hook.off === "function") { hook.off("mousedown", onMouseDown); hook.off("mouseup", onMouseUp); + hook.off("keydown", onKeyDown); if (process.platform === "linux") { hook.off("mousemove", onMouseMove); } } else if (typeof hook.removeListener === "function") { hook.removeListener("mousedown", onMouseDown); hook.removeListener("mouseup", onMouseUp); + hook.removeListener("keydown", onKeyDown); if (process.platform === "linux") { hook.removeListener("mousemove", onMouseMove); } diff --git a/electron/ipc/cursor/monitor.ts b/electron/ipc/cursor/monitor.ts index 8a507d56d..819ef2fc4 100644 --- a/electron/ipc/cursor/monitor.ts +++ b/electron/ipc/cursor/monitor.ts @@ -12,7 +12,7 @@ import { setNativeCursorMonitorProcess, } from "../state"; import type { CursorVisualType } from "../types"; -import { recordCursorMouseDown, recordCursorMouseUp } from "./interaction"; +import { recordCursorKeyDown, recordCursorMouseDown, recordCursorMouseUp } from "./interaction"; export function emitCursorStateChanged(cursorType: CursorVisualType) { BrowserWindow.getAllWindows().forEach((window) => { @@ -28,9 +28,11 @@ export function handleCursorMonitorStdout(chunk: Buffer) { setNativeCursorMonitorOutputBuffer(lines.pop() ?? ""); for (const line of lines) { - const interactionMatch = line.match(/^INTERACTION:(mousedown|mouseup)(?::([123]))?$/); + const interactionMatch = line.match(/^INTERACTION:(mousedown|mouseup|keydown)(?::([123]))?$/); if (interactionMatch) { - if (interactionMatch[1] === "mouseup") { + if (interactionMatch[1] === "keydown") { + recordCursorKeyDown(); + } else if (interactionMatch[1] === "mouseup") { recordCursorMouseUp(); } else { const button = Number(interactionMatch[2]); diff --git a/electron/ipc/cursor/telemetry.test.ts b/electron/ipc/cursor/telemetry.test.ts index 03613859b..0c2ac06cd 100644 --- a/electron/ipc/cursor/telemetry.test.ts +++ b/electron/ipc/cursor/telemetry.test.ts @@ -113,6 +113,16 @@ describe("cursor telemetry pause clock", () => { expect(rm).not.toHaveBeenCalled(); }); + it("preserves anonymous keydown timing telemetry", () => { + const samples = normalizeCursorTelemetrySamples([ + { timeMs: 42, cx: 0.25, cy: 0.75, interactionType: "keydown" }, + ]); + + expect(samples).toEqual([ + { timeMs: 42, cx: 0.25, cy: 0.75, interactionType: "keydown", cursorType: undefined }, + ]); + }); + it("removes the sidecar when saving an empty cursor telemetry payload", async () => { await writeCursorTelemetry("/tmp/recording.mp4", []); diff --git a/electron/ipc/cursor/telemetry.ts b/electron/ipc/cursor/telemetry.ts index 73f62714e..c937cedde 100644 --- a/electron/ipc/cursor/telemetry.ts +++ b/electron/ipc/cursor/telemetry.ts @@ -60,7 +60,8 @@ export function normalizeCursorTelemetrySamples(rawSamples: unknown): CursorTele point.interactionType === "right-click" || point.interactionType === "middle-click" || point.interactionType === "move" || - point.interactionType === "mouseup" + point.interactionType === "mouseup" || + point.interactionType === "keydown" ? point.interactionType : undefined, cursorType: diff --git a/electron/ipc/types.ts b/electron/ipc/types.ts index 58f5425bd..450d6de49 100644 --- a/electron/ipc/types.ts +++ b/electron/ipc/types.ts @@ -96,7 +96,8 @@ export type CursorInteractionType = | "double-click" | "right-click" | "middle-click" - | "mouseup"; + | "mouseup" + | "keydown"; export interface CursorTelemetryPoint { timeMs: number; @@ -120,7 +121,8 @@ export type NativeMacWindowSource = { height?: number; }; -export type HookEventName = "mousedown" | "mouseup" | "mousemove"; +export type HookMouseEventName = "mousedown" | "mouseup" | "mousemove"; +export type HookEventName = HookMouseEventName | "keydown"; export type HookMouseEvent = { button?: number; @@ -139,12 +141,23 @@ export type HookMouseEvent = { }; }; -export type HookEventListener = (event: HookMouseEvent) => void; +/** + * Deliberately excludes key characters and text. Keyboard capture is used only + * to identify the timing of likely typing activity. + */ +export type HookKeyboardEvent = { + keycode?: number; + ctrlKey?: boolean; + metaKey?: boolean; +}; export type UiohookLike = { - on: (eventName: HookEventName, listener: HookEventListener) => void; - off?: (eventName: HookEventName, listener: HookEventListener) => void; - removeListener?: (eventName: HookEventName, listener: HookEventListener) => void; + on(eventName: HookMouseEventName, listener: (event: HookMouseEvent) => void): void; + on(eventName: "keydown", listener: (event: HookKeyboardEvent) => void): void; + off?(eventName: HookMouseEventName, listener: (event: HookMouseEvent) => void): void; + off?(eventName: "keydown", listener: (event: HookKeyboardEvent) => void): void; + removeListener?(eventName: HookMouseEventName, listener: (event: HookMouseEvent) => void): void; + removeListener?(eventName: "keydown", listener: (event: HookKeyboardEvent) => void): void; start: () => void; stop?: () => void; }; diff --git a/electron/native/NativeCursorMonitor.swift b/electron/native/NativeCursorMonitor.swift index 18c644150..37de6d2a6 100644 --- a/electron/native/NativeCursorMonitor.swift +++ b/electron/native/NativeCursorMonitor.swift @@ -415,6 +415,16 @@ func mouseInteractionCallback( let action: String let button: Int switch type { + case .keyDown: + // Timing-only telemetry: never emit the key code, character, or text. + // Command/Control combinations are app shortcuts rather than typing. + let flags = event.flags + guard !flags.contains(.maskCommand), !flags.contains(.maskControl) else { + return Unmanaged.passUnretained(event) + } + print("INTERACTION:keydown") + fflush(stdout) + return Unmanaged.passUnretained(event) case .leftMouseDown: action = "mousedown" button = 1 @@ -455,6 +465,7 @@ let mouseEventTypes: [CGEventType] = [ .rightMouseUp, .otherMouseDown, .otherMouseUp, + .keyDown, ] let mouseEventMask = mouseEventTypes.reduce(CGEventMask(0)) { mask, type in mask | (CGEventMask(1) << type.rawValue) diff --git a/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor index 192f8719f..b390d5b5d 100755 Binary files a/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor and b/electron/native/bin/darwin-arm64/recordly-native-cursor-monitor differ diff --git a/electron/native/bin/darwin-x64/recordly-native-cursor-monitor b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor index b375827a8..b9849a4de 100755 Binary files a/electron/native/bin/darwin-x64/recordly-native-cursor-monitor and b/electron/native/bin/darwin-x64/recordly-native-cursor-monitor differ diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts index 5f237d5eb..45e28dc3a 100644 --- a/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.test.ts @@ -20,6 +20,10 @@ function makeMove(timeMs: number, cx = 0.5, cy = 0.5): CursorTelemetryPoint { return { timeMs, cx, cy, interactionType: "move" }; } +function makeKeyDown(timeMs: number, cx = 0.5, cy = 0.5): CursorTelemetryPoint { + return { timeMs, cx, cy, interactionType: "keydown" }; +} + /** Wraps click samples with surrounding move events to mimic real mixed telemetry. */ function withMoves(clicks: CursorTelemetryPoint[], totalMs: number): CursorTelemetryPoint[] { return [makeMove(0), ...clicks, makeMove(totalMs)]; @@ -48,6 +52,105 @@ describe("shouldAutoApplyFreshRecordingZoomsForSource", () => { }); describe("buildInteractionZoomSuggestions (click-cluster logic)", () => { + it("extends a text-field click through a valid typing burst", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: withMoves( + [ + makeClick(5_000, 0.3, 0.4), + makeKeyDown(5_200, 0.3, 0.4), + makeKeyDown(5_350, 0.3, 0.4), + makeKeyDown(5_500, 0.3, 0.4), + makeKeyDown(5_650, 0.3, 0.4), + ], + TOTAL_MS, + ), + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toEqual([ + { start: 4_500, end: 6_150, focus: { cx: 0.3, cy: 0.4 } }, + ]); + }); + + it("keeps a normal click window for isolated key activity", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: withMoves([makeClick(5_000), makeKeyDown(5_200)], TOTAL_MS), + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.suggestions).toEqual([ + { start: 4_500, end: 5_500, focus: { cx: 0.5, cy: 0.5 } }, + ]); + }); + + it("merges nearby typing bursts but splits bursts separated by a long pause", () => { + const shortPause = buildInteractionZoomSuggestions({ + cursorTelemetry: withMoves( + [ + makeClick(1_000), + makeKeyDown(1_100), makeKeyDown(1_200), makeKeyDown(1_300), + makeKeyDown(3_500), makeKeyDown(3_600), makeKeyDown(3_700), + ], + TOTAL_MS, + ), + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + expect(shortPause.suggestions).toHaveLength(1); + expect(shortPause.suggestions[0]).toMatchObject({ start: 500, end: 4_200 }); + + const longPause = buildInteractionZoomSuggestions({ + cursorTelemetry: withMoves( + [ + makeClick(1_000), + makeKeyDown(1_100), makeKeyDown(1_200), makeKeyDown(1_300), + makeKeyDown(4_000), makeKeyDown(4_100), makeKeyDown(4_200), + ], + TOTAL_MS, + ), + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + expect(longPause.suggestions).toHaveLength(2); + expect(longPause.suggestions.map(({ start, end }) => ({ start, end }))).toEqual([ + { start: 500, end: 1_800 }, + { start: 3_500, end: 4_700 }, + ]); + }); + + it("does not let a later burst inherit text context from an earlier burst", () => { + const result = buildInteractionZoomSuggestions({ + cursorTelemetry: withMoves( + [ + makeClick(2_600, 0.3, 0.4), + makeKeyDown(2_700, 0.3, 0.4), + makeKeyDown(2_800, 0.3, 0.4), + makeKeyDown(2_900, 0.3, 0.4), + // This later click lands within the first burst's text-cursor + // annotation window, but its typing happens away from that field. + makeClick(3_300, 0.7, 0.7), + makeMove(3_450, 0.75, 0.75), + makeMove(3_500, 0.8, 0.7), + makeMove(3_550, 0.75, 0.8), + makeKeyDown(3_600, 0.8, 0.8), + makeKeyDown(3_700, 0.8, 0.8), + makeKeyDown(3_800, 0.8, 0.8), + ], + TOTAL_MS, + ), + totalMs: TOTAL_MS, + defaultDurationMs: 3_000, + }); + + expect(result.status).toBe("ok"); + expect(result.suggestions).toEqual([ + { start: 2_100, end: 3_800, focus: { cx: 0.3, cy: 0.4 } }, + ]); + }); + it("creates one zoom track for a single isolated click with 500ms padding", () => { const telemetry = withMoves([makeClick(5_000)], TOTAL_MS); diff --git a/src/components/video-editor/timeline/zoomSuggestionUtils.ts b/src/components/video-editor/timeline/zoomSuggestionUtils.ts index fbdc5a68b..f93552320 100644 --- a/src/components/video-editor/timeline/zoomSuggestionUtils.ts +++ b/src/components/video-editor/timeline/zoomSuggestionUtils.ts @@ -12,6 +12,8 @@ export interface ZoomDwellCandidate { } export interface CursorInteractionCandidate extends ZoomDwellCandidate { + startMs?: number; + endMs?: number; kind: | "dwell" | "click-like" @@ -19,7 +21,8 @@ export interface CursorInteractionCandidate extends ZoomDwellCandidate { | "text-focus-like" | "dropdown-open" | "text-selection" - | "text-field-click"; + | "text-field-click" + | "typing-burst"; source: "explicit" | "heuristic"; } @@ -71,6 +74,10 @@ export function shouldAutoApplyFreshRecordingZoomsForSource( export const CLICK_CLUSTER_MERGE_GAP_MS = 2500; /** Padding added before the first click and after the last click in a cluster. */ export const CLICK_CLUSTER_PAD_MS = 500; +export const TYPING_BURST_MAX_IDLE_MS = 2000; +export const MIN_TYPING_BURST_KEY_COUNT = 3; +export const MAX_CLICK_TO_TYPING_DELAY_MS = 2000; +export const MAX_CLICK_TO_TYPING_DISTANCE = 0.08; const EXPLICIT_CLICK_TYPES = new Set>([ "click", "double-click", @@ -137,6 +144,16 @@ export function normalizeCursorTelemetry( continue; } + if (candidate.kind === "typing-burst") { + applyCursorTypeInRange( + normalized, + (candidate.startMs ?? candidate.centerTimeMs) - 100, + (candidate.endMs ?? candidate.centerTimeMs) + 500, + "text", + ); + continue; + } + if (candidate.kind === "text-field-click" || candidate.kind === "text-focus-like") { applyCursorTypeInRange( normalized, @@ -224,6 +241,62 @@ export function detectZoomDwellCandidates(samples: CursorTelemetryPoint[]): Zoom return dwellCandidates; } +type TypingBurst = { startMs: number; endMs: number }; + +function findTypingBurstsAfterClick( + samples: CursorTelemetryPoint[], + clickSample: CursorTelemetryPoint, +): TypingBurst[] { + const nextClickTime = samples.find( + (sample) => isExplicitClickType(sample.interactionType) && sample.timeMs > clickSample.timeMs, + )?.timeMs; + const keydowns = samples.filter( + (sample) => + sample.interactionType === "keydown" && + sample.timeMs > clickSample.timeMs && + (nextClickTime === undefined || sample.timeMs < nextClickTime), + ); + const bursts: TypingBurst[] = []; + let current: CursorTelemetryPoint[] = []; + + const commit = () => { + if (current.length < MIN_TYPING_BURST_KEY_COUNT) return; + const first = current[0]; + const last = current[current.length - 1]; + const startsSoonEnough = + bursts.length > 0 || first.timeMs - clickSample.timeMs <= MAX_CLICK_TO_TYPING_DELAY_MS; + const relevantSamples = samples.filter( + (sample) => sample.timeMs >= clickSample.timeMs && sample.timeMs <= last.timeMs, + ); + const burstSamples = samples.filter( + (sample) => sample.timeMs >= first.timeMs && sample.timeMs <= last.timeMs, + ); + const stayedNearField = relevantSamples.every( + (sample) => + Math.hypot(sample.cx - clickSample.cx, sample.cy - clickSample.cy) <= + MAX_CLICK_TO_TYPING_DISTANCE, + ); + const hasTextContext = burstSamples.some((sample) => sample.cursorType === "text"); + + if (startsSoonEnough && (stayedNearField || hasTextContext)) { + bursts.push({ startMs: first.timeMs, endMs: last.timeMs }); + } + }; + + for (const keydown of keydowns) { + if ( + current.length > 0 && + keydown.timeMs - current[current.length - 1].timeMs > TYPING_BURST_MAX_IDLE_MS + ) { + commit(); + current = []; + } + current.push(keydown); + } + commit(); + return bursts; +} + export function detectInteractionCandidates( samples: CursorTelemetryPoint[], ): CursorInteractionCandidate[] { @@ -254,6 +327,18 @@ export function detectInteractionCandidates( kind, source: "explicit", }); + + for (const [index, burst] of findTypingBurstsAfterClick(samples, clickSample).entries()) { + explicitInteractionCandidates.push({ + centerTimeMs: Math.round((burst.startMs + burst.endMs) / 2), + focus: { cx: clickSample.cx, cy: clickSample.cy }, + strength: 1400, + startMs: index === 0 ? clickSample.timeMs : burst.startMs, + endMs: burst.endMs, + kind: "typing-burst", + source: "explicit", + }); + } } // --- Phase 2: Dwell-based heuristic candidates --- @@ -315,8 +400,8 @@ function buildClickClusters( const sorted = [...clicks].sort((a, b) => a.centerTimeMs - b.centerTimeMs); const clusters: Array<{ firstMs: number; lastMs: number; focus: ZoomFocus }> = []; - let clusterStart = sorted[0].centerTimeMs; - let clusterEnd = sorted[0].centerTimeMs; + let clusterStart = sorted[0].startMs ?? sorted[0].centerTimeMs; + let clusterEnd = sorted[0].endMs ?? sorted[0].centerTimeMs; let bestStrength = sorted[0].strength; let bestFocus = sorted[0].focus; let sumCx = sorted[0].focus.cx; @@ -325,11 +410,13 @@ function buildClickClusters( for (let i = 1; i < sorted.length; i++) { const click = sorted[i]; - const gap = click.centerTimeMs - clusterEnd; + const clickStart = click.startMs ?? click.centerTimeMs; + const clickEnd = click.endMs ?? click.centerTimeMs; + const gap = clickStart - clusterEnd; if (gap <= mergeGapMs) { // Extend current cluster - clusterEnd = Math.max(clusterEnd, click.centerTimeMs); + clusterEnd = Math.max(clusterEnd, clickEnd); if (click.strength > bestStrength) { bestStrength = click.strength; bestFocus = click.focus; @@ -344,8 +431,8 @@ function buildClickClusters( lastMs: clusterEnd, focus: bestFocus ?? { cx: sumCx / count, cy: sumCy / count }, }); - clusterStart = click.centerTimeMs; - clusterEnd = click.centerTimeMs; + clusterStart = clickStart; + clusterEnd = clickEnd; bestStrength = click.strength; bestFocus = click.focus; sumCx = click.focus.cx; diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 55de007bb..92892c1b8 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -27,7 +27,8 @@ export interface CursorTelemetryPoint { | "double-click" | "right-click" | "middle-click" - | "mouseup"; + | "mouseup" + | "keydown"; cursorType?: | "arrow" | "text"