From 0e857416349c3228e211a1d6a3bc235a214b17c3 Mon Sep 17 00:00:00 2001 From: NoahSeeger Date: Tue, 22 Sep 2026 23:40:33 +0200 Subject: [PATCH] fix: ignore hidden cursor telemetry in auto-zoom --- electron/media/cursorSidecar.test.ts | 16 +++++- electron/media/cursorSidecar.ts | 1 + src/components/video-editor/types.ts | 1 + .../timeline/zoom-suggestions.test.ts | 54 +++++++++++++++++++ .../ai-edition/timeline/zoom-suggestions.ts | 13 ++++- src/native/contracts.ts | 1 + 6 files changed, 83 insertions(+), 3 deletions(-) diff --git a/electron/media/cursorSidecar.test.ts b/electron/media/cursorSidecar.test.ts index 1c0e46a4d..88c7ed0da 100644 --- a/electron/media/cursorSidecar.test.ts +++ b/electron/media/cursorSidecar.test.ts @@ -149,7 +149,9 @@ describe("readCursorTelemetryFile", () => { const result = await readCursorTelemetryFile(video, {}); expect(result.success).toBe(true); - expect(result.samples).toEqual([{ timeMs: 10, cx: 0.5, cy: 0.5, interactionType: "click" }]); + expect(result.samples).toEqual([ + { timeMs: 10, cx: 0.5, cy: 0.5, visible: true, interactionType: "click" }, + ]); // This projection feeds the ai-edition auto-zoom detector, whose click // candidates ARE the recorded interactions (issue #699). It used to strip // the field — locked by a test, even — and the detector never saw a click @@ -158,6 +160,18 @@ describe("readCursorTelemetryFile", () => { // the extra field. }); + it("keeps hidden cursor state for auto-zoom", async () => { + const video = path.join(dir, "hidden.mp4"); + await writeSidecar(video, { + samples: [{ timeMs: 10, cx: 0.5, cy: 0.5, visible: false, interactionType: "click" }], + }); + + const result = await readCursorTelemetryFile(video, {}); + + expect(result.success).toBe(true); + expect(result.samples[0].visible).toBe(false); + }); + it("keeps every click kind, not only the plain left click", async () => { const video = path.join(dir, "kinds.mp4"); await writeSidecar(video, { diff --git a/electron/media/cursorSidecar.ts b/electron/media/cursorSidecar.ts index a47f63f5b..0a588e0ae 100644 --- a/electron/media/cursorSidecar.ts +++ b/electron/media/cursorSidecar.ts @@ -242,6 +242,7 @@ export async function readCursorTelemetryFile( timeMs: sample.timeMs, cx: sample.cx, cy: sample.cy, + visible: sample.visible, interactionType: sample.interactionType, })), }; diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index 426d531d1..2fcc72f95 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -251,6 +251,7 @@ export interface CursorTelemetryPoint { timeMs: number; cx: number; cy: number; + visible?: boolean; interactionType?: "move" | "click" | "double-click" | "right-click" | "middle-click" | "mouseup"; cursorType?: | "arrow" diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts index f36b1cc32..cc263a0c5 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.test.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.test.ts @@ -52,6 +52,60 @@ describe("detectZoomDwellCandidates", () => { }); describe("buildAutoZoomSuggestions", () => { + it("ignores a click made while the cursor is hidden", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [{ ...click(2000, 0.5, 0.5), visible: false }], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("does not join visible samples across a hidden interval into a dwell", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + { timeMs: 1000, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1200, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 1400, cx: 0.5, cy: 0.5, visible: false }, + { timeMs: 1800, cx: 0.5, cy: 0.5, visible: true }, + { timeMs: 2000, cx: 0.5, cy: 0.5, visible: true }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("ignores a dwell recorded while the cursor is hidden", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: dwell(2000, 0.5, 0.5).map((sample) => ({ + ...sample, + visible: false, + })), + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toEqual([]); + }); + + it("keeps a visible dwell beside hidden cursor samples", () => { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry: [ + ...dwell(1000, 0.2, 0.8), + { timeMs: 2000, cx: 0.2, cy: 0.8, visible: false }, + ], + totalMs: 5000, + existingRegions: [], + defaultDurationMs: 2000, + }); + expect(suggestions).toHaveLength(1); + expect(suggestions[0].focus.cx).toBeCloseTo(0.2); + expect(suggestions[0].focus.cy).toBeCloseTo(0.8); + }); + it("returns a centered span around each accepted dwell", () => { const telemetry = dwell(2000, 0.5, 0.5); const suggestions = buildAutoZoomSuggestions({ diff --git a/src/lib/ai-edition/timeline/zoom-suggestions.ts b/src/lib/ai-edition/timeline/zoom-suggestions.ts index b82872267..47d7f2a96 100644 --- a/src/lib/ai-edition/timeline/zoom-suggestions.ts +++ b/src/lib/ai-edition/timeline/zoom-suggestions.ts @@ -30,6 +30,7 @@ function normalizeTelemetrySample( timeMs: Math.max(0, Math.min(sample.timeMs, totalMs)), cx: Math.max(0, Math.min(sample.cx, 1)), cy: Math.max(0, Math.min(sample.cy, 1)), + visible: sample.visible, interactionType: sample.interactionType, }; } @@ -92,7 +93,13 @@ export function detectZoomDwellCandidates( }); }; - for (let index = 1; index < samples.length; index += 1) { + for (let index = 0; index < samples.length; index += 1) { + if (samples[index].visible === false) { + pushRunIfDwell(runStart, index); + runStart = index + 1; + continue; + } + if (index <= runStart) continue; const prev = samples[index - 1]; const curr = samples[index]; const distance = Math.hypot(curr.cx - prev.cx, curr.cy - prev.cy); @@ -122,7 +129,9 @@ function detectZoomClickCandidates(samples: CursorTelemetryPoint[]): ZoomDwellCa return samples .filter( (sample) => - sample.interactionType !== undefined && CLICK_INTERACTION_TYPES.has(sample.interactionType), + sample.visible !== false && + sample.interactionType !== undefined && + CLICK_INTERACTION_TYPES.has(sample.interactionType), ) .map((sample) => ({ centerTimeMs: sample.timeMs, diff --git a/src/native/contracts.ts b/src/native/contracts.ts index 5c40ddcb0..b9bbb7e1b 100644 --- a/src/native/contracts.ts +++ b/src/native/contracts.ts @@ -27,6 +27,7 @@ export interface CursorTelemetryPoint { timeMs: number; cx: number; cy: number; + visible?: boolean; } export interface CursorRecordingSample extends CursorTelemetryPoint {