feat: add typing-aware auto zoom suggestions

electron/electron-env.d.ts

@@ -956,7 +956,8 @@ interface CursorTelemetryPoint {
 		| "double-click"
 		| "right-click"
 		| "middle-click"
-		| "mouseup";
+		| "mouseup"
+		| "keydown";
 	cursorType?:
 		| "arrow"
 		| "text"

electron/ipc/cursor/interaction.test.ts

@@ -12,10 +12,21 @@ vi.mock("electron", () => ({
 }));
 
 import {
+	isCandidateTypingKeyEvent,
 	repairBundledUiohookBinaryForCurrentArch,
 	shouldStartGlobalInteractionHook,
 } from "./interaction";
 
+describe("isCandidateTypingKeyEvent", () => {
+	it("accepts ordinary keys but filters modifiers and Ctrl/Meta shortcuts", () => {
+		expect(isCandidateTypingKeyEvent({ keycode: 0x001e })).toBe(true);
+		expect(isCandidateTypingKeyEvent({ keycode: 0x002a })).toBe(false); // Shift
+		expect(isCandidateTypingKeyEvent({ keycode: 0x003a })).toBe(false); // Caps Lock
+		expect(isCandidateTypingKeyEvent({ keycode: 0x001e, ctrlKey: true })).toBe(false);
+		expect(isCandidateTypingKeyEvent({ keycode: 0x001e, metaKey: true })).toBe(false);
+	});
+});
+
 describe("shouldStartGlobalInteractionHook", () => {
 	it("does not start the synchronous uiohook event tap on macOS", () => {
 		expect(shouldStartGlobalInteractionHook("darwin")).toBe(false);

electron/ipc/cursor/interaction.ts

@@ -13,6 +13,7 @@ import {
 } from "../state";
 import type {
 	CursorInteractionType,
+	HookKeyboardEvent,
 	HookMouseEvent,
 	UiohookLike,
 	UiohookModuleNamespace,
@@ -233,6 +234,47 @@ export function recordCursorMouseUp() {
 	pushCursorSample(point.cx, point.cy, getCursorCaptureElapsedMs(), "mouseup");
 }
 
+/** uiohook virtual key codes for non-typing modifier/toggle keys. */
+const MODIFIER_ONLY_KEY_CODES = new Set([
+	0x002a, // left Shift
+	0x0036, // right Shift
+	0x001d, // left Control
+	0x0e1d, // right Control
+	0x0038, // left Alt
+	0x0e38, // right Alt
+	0x0e5b, // left Meta / Command
+	0x0e5c, // right Meta / Command
+	0x003a, // Caps Lock
+	0x0045, // Num Lock
+	0x0046, // Scroll Lock
+]);
+
+/**
+ * Keeps keyboard telemetry privacy-preserving: this decision uses only a
+ * modifier state and key code to decide whether to record an anonymous
+ * timestamp + cursor position. It never reads or stores key characters.
+ */
+export function isCandidateTypingKeyEvent(event: HookKeyboardEvent): boolean {
+	if (event.ctrlKey || event.metaKey) {
+		return false;
+	}
+
+	return typeof event.keycode === "number" && !MODIFIER_ONLY_KEY_CODES.has(event.keycode);
+}
+
+export function recordCursorKeyDown() {
+	if (!isCursorCaptureActive || isCursorCapturePaused()) {
+		return;
+	}
+
+	const point = getNormalizedCursorPoint();
+	if (!point) {
+		return;
+	}
+
+	pushCursorSample(point.cx, point.cy, getCursorCaptureElapsedMs(), "keydown");
+}
+
 export async function startInteractionCapture() {
 	if (!isCursorCaptureActive) {
 		return;
@@ -276,6 +318,12 @@ export async function startInteractionCapture() {
 			recordCursorMouseUp();
 		};
 
+		const onKeyDown = (event: HookKeyboardEvent) => {
+			if (isCandidateTypingKeyEvent(event)) {
+				recordCursorKeyDown();
+			}
+		};
+
 		const onMouseMove = (event: HookMouseEvent) => {
 			if (process.platform !== "linux" || !isCursorCaptureActive || isCursorCapturePaused()) {
 				return;
@@ -291,6 +339,7 @@ export async function startInteractionCapture() {
 
 		hook.on("mousedown", onMouseDown);
 		hook.on("mouseup", onMouseUp);
+		hook.on("keydown", onKeyDown);
 		if (process.platform === "linux") {
 			hook.on("mousemove", onMouseMove);
 		}
@@ -300,12 +349,14 @@ export async function startInteractionCapture() {
 				if (typeof hook.off === "function") {
 					hook.off("mousedown", onMouseDown);
 					hook.off("mouseup", onMouseUp);
+					hook.off("keydown", onKeyDown);
 					if (process.platform === "linux") {
 						hook.off("mousemove", onMouseMove);
 					}
 				} else if (typeof hook.removeListener === "function") {
 					hook.removeListener("mousedown", onMouseDown);
 					hook.removeListener("mouseup", onMouseUp);
+					hook.removeListener("keydown", onKeyDown);
 					if (process.platform === "linux") {
 						hook.removeListener("mousemove", onMouseMove);
 					}

electron/ipc/cursor/monitor.ts

@@ -12,7 +12,7 @@ import {
 	setNativeCursorMonitorProcess,
 } from "../state";
 import type { CursorVisualType } from "../types";
-import { recordCursorMouseDown, recordCursorMouseUp } from "./interaction";
+import { recordCursorKeyDown, recordCursorMouseDown, recordCursorMouseUp } from "./interaction";
 
 export function emitCursorStateChanged(cursorType: CursorVisualType) {
 	BrowserWindow.getAllWindows().forEach((window) => {
@@ -28,9 +28,11 @@ export function handleCursorMonitorStdout(chunk: Buffer) {
 	setNativeCursorMonitorOutputBuffer(lines.pop() ?? "");
 
 	for (const line of lines) {
-		const interactionMatch = line.match(/^INTERACTION:(mousedown|mouseup)(?::([123]))?$/);
+		const interactionMatch = line.match(/^INTERACTION:(mousedown|mouseup|keydown)(?::([123]))?$/);
 		if (interactionMatch) {
-			if (interactionMatch[1] === "mouseup") {
+			if (interactionMatch[1] === "keydown") {
+				recordCursorKeyDown();
+			} else if (interactionMatch[1] === "mouseup") {
 				recordCursorMouseUp();
 			} else {
 				const button = Number(interactionMatch[2]);

electron/ipc/cursor/telemetry.test.ts

@@ -113,6 +113,16 @@ describe("cursor telemetry pause clock", () => {
 		expect(rm).not.toHaveBeenCalled();
 	});
 
+	it("preserves anonymous keydown timing telemetry", () => {
+		const samples = normalizeCursorTelemetrySamples([
+			{ timeMs: 42, cx: 0.25, cy: 0.75, interactionType: "keydown" },
+		]);
+
+		expect(samples).toEqual([
+			{ timeMs: 42, cx: 0.25, cy: 0.75, interactionType: "keydown", cursorType: undefined },
+		]);
+	});
+
 	it("removes the sidecar when saving an empty cursor telemetry payload", async () => {
 		await writeCursorTelemetry("/tmp/recording.mp4", []);
 

electron/ipc/cursor/telemetry.ts

@@ -60,7 +60,8 @@ export function normalizeCursorTelemetrySamples(rawSamples: unknown): CursorTele
 					point.interactionType === "right-click" ||
 					point.interactionType === "middle-click" ||
 					point.interactionType === "move" ||
-					point.interactionType === "mouseup"
+					point.interactionType === "mouseup" ||
+					point.interactionType === "keydown"
 						? point.interactionType
 						: undefined,
 				cursorType:

electron/ipc/types.ts

@@ -96,7 +96,8 @@ export type CursorInteractionType =
 	| "double-click"
 	| "right-click"
 	| "middle-click"
-	| "mouseup";
+	| "mouseup"
+	| "keydown";
 
 export interface CursorTelemetryPoint {
 	timeMs: number;
@@ -120,7 +121,8 @@ export type NativeMacWindowSource = {
 	height?: number;
 };
 
-export type HookEventName = "mousedown" | "mouseup" | "mousemove";
+export type HookMouseEventName = "mousedown" | "mouseup" | "mousemove";
+export type HookEventName = HookMouseEventName | "keydown";
 
 export type HookMouseEvent = {
 	button?: number;
@@ -139,12 +141,23 @@ export type HookMouseEvent = {
 	};
 };
 
-export type HookEventListener = (event: HookMouseEvent) => void;
+/**
+ * Deliberately excludes key characters and text. Keyboard capture is used only
+ * to identify the timing of likely typing activity.
+ */
+export type HookKeyboardEvent = {
+	keycode?: number;
+	ctrlKey?: boolean;
+	metaKey?: boolean;
+};
 
 export type UiohookLike = {
-	on: (eventName: HookEventName, listener: HookEventListener) => void;
-	off?: (eventName: HookEventName, listener: HookEventListener) => void;
-	removeListener?: (eventName: HookEventName, listener: HookEventListener) => void;
+	on(eventName: HookMouseEventName, listener: (event: HookMouseEvent) => void): void;
+	on(eventName: "keydown", listener: (event: HookKeyboardEvent) => void): void;
+	off?(eventName: HookMouseEventName, listener: (event: HookMouseEvent) => void): void;
+	off?(eventName: "keydown", listener: (event: HookKeyboardEvent) => void): void;
+	removeListener?(eventName: HookMouseEventName, listener: (event: HookMouseEvent) => void): void;
+	removeListener?(eventName: "keydown", listener: (event: HookKeyboardEvent) => void): void;
 	start: () => void;
 	stop?: () => void;
 };

electron/native/NativeCursorMonitor.swift

@@ -415,6 +415,16 @@ func mouseInteractionCallback(
 	let action: String
 	let button: Int
 	switch type {
+	case .keyDown:
+		// Timing-only telemetry: never emit the key code, character, or text.
+		// Command/Control combinations are app shortcuts rather than typing.
+		let flags = event.flags
+		guard !flags.contains(.maskCommand), !flags.contains(.maskControl) else {
+			return Unmanaged.passUnretained(event)
+		}
+		print("INTERACTION:keydown")
+		fflush(stdout)
+		return Unmanaged.passUnretained(event)
 	case .leftMouseDown:
 		action = "mousedown"
 		button = 1
@@ -455,6 +465,7 @@ let mouseEventTypes: [CGEventType] = [
 	.rightMouseUp,
 	.otherMouseDown,
 	.otherMouseUp,
+	.keyDown,
 ]
 let mouseEventMask = mouseEventTypes.reduce(CGEventMask(0)) { mask, type in
 	mask | (CGEventMask(1) << type.rawValue)

src/components/video-editor/timeline/zoomSuggestionUtils.test.ts

@@ -20,6 +20,10 @@ function makeMove(timeMs: number, cx = 0.5, cy = 0.5): CursorTelemetryPoint {
 	return { timeMs, cx, cy, interactionType: "move" };
 }
 
+function makeKeyDown(timeMs: number, cx = 0.5, cy = 0.5): CursorTelemetryPoint {
+	return { timeMs, cx, cy, interactionType: "keydown" };
+}
+
 /** Wraps click samples with surrounding move events to mimic real mixed telemetry. */
 function withMoves(clicks: CursorTelemetryPoint[], totalMs: number): CursorTelemetryPoint[] {
 	return [makeMove(0), ...clicks, makeMove(totalMs)];
@@ -48,6 +52,105 @@ describe("shouldAutoApplyFreshRecordingZoomsForSource", () => {
 });
 
 describe("buildInteractionZoomSuggestions (click-cluster logic)", () => {
+	it("extends a text-field click through a valid typing burst", () => {
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: withMoves(
+				[
+					makeClick(5_000, 0.3, 0.4),
+					makeKeyDown(5_200, 0.3, 0.4),
+					makeKeyDown(5_350, 0.3, 0.4),
+					makeKeyDown(5_500, 0.3, 0.4),
+					makeKeyDown(5_650, 0.3, 0.4),
+				],
+				TOTAL_MS,
+			),
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("ok");
+		expect(result.suggestions).toEqual([
+			{ start: 4_500, end: 6_150, focus: { cx: 0.3, cy: 0.4 } },
+		]);
+	});
+
+	it("keeps a normal click window for isolated key activity", () => {
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: withMoves([makeClick(5_000), makeKeyDown(5_200)], TOTAL_MS),
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.suggestions).toEqual([
+			{ start: 4_500, end: 5_500, focus: { cx: 0.5, cy: 0.5 } },
+		]);
+	});
+
+	it("merges nearby typing bursts but splits bursts separated by a long pause", () => {
+		const shortPause = buildInteractionZoomSuggestions({
+			cursorTelemetry: withMoves(
+				[
+					makeClick(1_000),
+					makeKeyDown(1_100), makeKeyDown(1_200), makeKeyDown(1_300),
+					makeKeyDown(3_500), makeKeyDown(3_600), makeKeyDown(3_700),
+				],
+				TOTAL_MS,
+			),
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+		expect(shortPause.suggestions).toHaveLength(1);
+		expect(shortPause.suggestions[0]).toMatchObject({ start: 500, end: 4_200 });
+
+		const longPause = buildInteractionZoomSuggestions({
+			cursorTelemetry: withMoves(
+				[
+					makeClick(1_000),
+					makeKeyDown(1_100), makeKeyDown(1_200), makeKeyDown(1_300),
+					makeKeyDown(4_000), makeKeyDown(4_100), makeKeyDown(4_200),
+				],
+				TOTAL_MS,
+			),
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+		expect(longPause.suggestions).toHaveLength(2);
+		expect(longPause.suggestions.map(({ start, end }) => ({ start, end }))).toEqual([
+			{ start: 500, end: 1_800 },
+			{ start: 3_500, end: 4_700 },
+		]);
+	});
+
+	it("does not let a later burst inherit text context from an earlier burst", () => {
+		const result = buildInteractionZoomSuggestions({
+			cursorTelemetry: withMoves(
+				[
+					makeClick(2_600, 0.3, 0.4),
+					makeKeyDown(2_700, 0.3, 0.4),
+					makeKeyDown(2_800, 0.3, 0.4),
+					makeKeyDown(2_900, 0.3, 0.4),
+					// This later click lands within the first burst's text-cursor
+					// annotation window, but its typing happens away from that field.
+					makeClick(3_300, 0.7, 0.7),
+					makeMove(3_450, 0.75, 0.75),
+					makeMove(3_500, 0.8, 0.7),
+					makeMove(3_550, 0.75, 0.8),
+					makeKeyDown(3_600, 0.8, 0.8),
+					makeKeyDown(3_700, 0.8, 0.8),
+					makeKeyDown(3_800, 0.8, 0.8),
+				],
+				TOTAL_MS,
+			),
+			totalMs: TOTAL_MS,
+			defaultDurationMs: 3_000,
+		});
+
+		expect(result.status).toBe("ok");
+		expect(result.suggestions).toEqual([
+			{ start: 2_100, end: 3_800, focus: { cx: 0.3, cy: 0.4 } },
+		]);
+	});
+
 	it("creates one zoom track for a single isolated click with 500ms padding", () => {
 		const telemetry = withMoves([makeClick(5_000)], TOTAL_MS);