diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 1a926718f..ed80fafb9 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -602,6 +602,21 @@ interface Window { message?: string; error?: string; }>; + getKeystrokeTelemetry: (videoPath?: string) => Promise<{ + success: boolean; + samples: KeystrokeSample[]; + message?: string; + error?: string; + }>; + setKeystrokeTelemetry: ( + videoPath: string | undefined, + samples: KeystrokeSample[], + ) => Promise<{ + success: boolean; + samples: KeystrokeSample[]; + message?: string; + error?: string; + }>; getSystemCursorAssets: () => Promise<{ success: boolean; cursors: Record; @@ -631,8 +646,17 @@ interface Window { success: boolean; trusted: boolean; prompted: boolean; + clientName?: string; + error?: string; + }>; + requestKeystrokeCapturePermission: () => Promise<{ + success: boolean; + trusted: boolean; + tapOk?: boolean; + clientName?: string; error?: string; }>; + stopKeystrokeTap: () => Promise<{ success: boolean }>; getScreenRecordingPermissionStatus: () => Promise<{ success: boolean; status: string; @@ -640,6 +664,7 @@ interface Window { }>; openScreenRecordingPreferences: () => Promise<{ success: boolean; error?: string }>; openAccessibilityPreferences: () => Promise<{ success: boolean; error?: string }>; + openInputMonitoringPreferences: () => Promise<{ success: boolean; error?: string }>; saveExportedVideo: ( videoData: ArrayBuffer, fileName: string, @@ -969,6 +994,17 @@ interface CursorTelemetryPoint { | "not-allowed"; } +interface KeystrokeSample { + timeMs: number; + key: string; + code: string; + ctrl: boolean; + alt: boolean; + shift: boolean; + meta: boolean; + repeat?: boolean; +} + interface SystemCursorAsset { dataUrl: string; hotspotX: number; diff --git a/electron/ipc/cursor/interaction.ts b/electron/ipc/cursor/interaction.ts index 47c42437f..2c2a9128a 100644 --- a/electron/ipc/cursor/interaction.ts +++ b/electron/ipc/cursor/interaction.ts @@ -5,6 +5,7 @@ import { hasLoggedInteractionHookFailure, interactionCaptureCleanup, isCursorCaptureActive, + isKeystrokeCaptureEnabled, lastLeftClick, setHasLoggedInteractionHookFailure, setInteractionCaptureCleanup, @@ -24,6 +25,7 @@ import { isCursorCapturePaused, pushCursorSample, } from "./telemetry"; +import { recordKeystrokeFromHookEvent } from "./keystrokes"; const nodeRequire = createRequire(import.meta.url); @@ -289,10 +291,20 @@ export async function startInteractionCapture() { setLinuxCursorScreenPoint({ x: point.x, y: point.y, updatedAt: Date.now() }); }; + const onKeyDown = (event: HookMouseEvent) => { + if (process.platform !== "linux" || !isKeystrokeCaptureEnabled) { + return; + } + recordKeystrokeFromHookEvent(event); + }; + hook.on("mousedown", onMouseDown); hook.on("mouseup", onMouseUp); if (process.platform === "linux") { hook.on("mousemove", onMouseMove); + if (isKeystrokeCaptureEnabled) { + hook.on("keydown", onKeyDown); + } } setInteractionCaptureCleanup(() => { @@ -302,12 +314,14 @@ export async function startInteractionCapture() { hook.off("mouseup", onMouseUp); if (process.platform === "linux") { hook.off("mousemove", onMouseMove); + hook.off("keydown", onKeyDown); } } else if (typeof hook.removeListener === "function") { hook.removeListener("mousedown", onMouseDown); hook.removeListener("mouseup", onMouseUp); if (process.platform === "linux") { hook.removeListener("mousemove", onMouseMove); + hook.removeListener("keydown", onKeyDown); } } } catch { diff --git a/electron/ipc/cursor/keystrokes.test.ts b/electron/ipc/cursor/keystrokes.test.ts new file mode 100644 index 000000000..1471aec64 --- /dev/null +++ b/electron/ipc/cursor/keystrokes.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { normalizeKeystrokeSamples, writeKeystrokeTelemetry } from "./keystrokes"; +import { beforeEach, vi } from "vitest"; + +const { writeFile, rm } = vi.hoisted(() => ({ + writeFile: vi.fn(), + rm: vi.fn(), +})); + +vi.mock("node:fs/promises", () => ({ + default: { + writeFile, + rm, + }, +})); + +vi.mock("electron", () => ({ + app: { + getPath: vi.fn(() => "/tmp"), + }, +})); + +vi.mock("../utils", () => ({ + getKeystrokePathForVideo: vi.fn(() => "/tmp/recording.keys.json"), +})); + +vi.mock("./telemetry", () => ({ + getCursorCaptureElapsedMs: vi.fn(() => 0), + isCursorCapturePaused: vi.fn(() => false), +})); + +vi.mock("../state", () => ({ + activeKeystrokeSamples: [], + pendingKeystrokeSamples: [], + isCursorCaptureActive: false, + isKeystrokeCaptureEnabled: false, + setActiveKeystrokeSamples: vi.fn(), + setIsKeystrokeCaptureEnabled: vi.fn(), + setPendingKeystrokeSamples: vi.fn(), +})); + +vi.mock("../../appSettingsStore", () => ({ + readAppSetting: vi.fn(() => false), +})); + +describe("keystroke sidecar", () => { + beforeEach(() => { + writeFile.mockReset(); + rm.mockReset(); + }); + + it("writes normalized samples", async () => { + const samples = await writeKeystrokeTelemetry("/tmp/recording.mp4", [ + { timeMs: 12, key: "Enter", ctrl: false }, + ]); + expect(samples[0]?.key).toBe("Enter"); + expect(writeFile).toHaveBeenCalled(); + }); + + it("removes the sidecar when empty", async () => { + await writeKeystrokeTelemetry("/tmp/recording.mp4", []); + expect(rm).toHaveBeenCalledWith("/tmp/recording.keys.json", { force: true }); + }); + + it("normalizes unknown payloads", () => { + expect(normalizeKeystrokeSamples("nope")).toEqual([]); + }); +}); diff --git a/electron/ipc/cursor/keystrokes.ts b/electron/ipc/cursor/keystrokes.ts new file mode 100644 index 000000000..a9ffff072 --- /dev/null +++ b/electron/ipc/cursor/keystrokes.ts @@ -0,0 +1,255 @@ +import fs from "node:fs/promises"; +import { + KEYSTROKE_OVERLAY_CAPTURE_SETTING, + KEYSTROKE_TELEMETRY_VERSION, + type KeystrokeSample, + normalizeKeystrokeSamples, + parseKeyMonitorLine, + shouldStoreCapturedKeystroke, +} from "../../../src/lib/keystrokeOverlay"; +import { readAppSetting } from "../../appSettingsStore"; +import { MAX_CURSOR_SAMPLES } from "../constants"; +import { + activeKeystrokeSamples, + isCursorCaptureActive, + pendingKeystrokeSamples, + setActiveKeystrokeSamples, + setIsKeystrokeCaptureEnabled, + setPendingKeystrokeSamples, + isKeystrokeCaptureEnabled, +} from "../state"; +import { getKeystrokePathForVideo } from "../utils"; +import { getCursorCaptureElapsedMs, isCursorCapturePaused } from "./telemetry"; + +const MAX_KEYSTROKE_SAMPLES = MAX_CURSOR_SAMPLES; + +export function isKeystrokeOverlayCaptureSettingEnabled() { + return readAppSetting(KEYSTROKE_OVERLAY_CAPTURE_SETTING) === true; +} + +export function syncKeystrokeCaptureEnabledFromSettings() { + setIsKeystrokeCaptureEnabled(isKeystrokeOverlayCaptureSettingEnabled()); + return isKeystrokeCaptureEnabled; +} + +export function resetKeystrokeCapture() { + setActiveKeystrokeSamples([]); + setPendingKeystrokeSamples([]); +} + +function inspectLinuxFocusedFieldPasswordState(): boolean | "unknown" { + return "unknown"; +} + +export function pushKeystrokeSample( + sample: KeystrokeSample, + isPasswordField: boolean | "unknown" = "unknown", +) { + if (!isCursorCaptureActive || isCursorCapturePaused() || !isKeystrokeCaptureEnabled) { + return; + } + + if ( + !shouldStoreCapturedKeystroke(sample, { + platform: process.platform, + isPasswordField, + }) + ) { + return; + } + + activeKeystrokeSamples.push(sample); + if (activeKeystrokeSamples.length > MAX_KEYSTROKE_SAMPLES) { + activeKeystrokeSamples.shift(); + } +} + +export function recordKeystrokeFromMonitorLine(line: string) { + const parsed = parseKeyMonitorLine(line); + if (!parsed || parsed.action !== "down") { + return; + } + + pushKeystrokeSample( + { + timeMs: getCursorCaptureElapsedMs(), + key: parsed.key, + code: parsed.code, + ctrl: parsed.ctrl, + alt: parsed.alt, + shift: parsed.shift, + meta: parsed.meta, + repeat: parsed.repeat || undefined, + }, + false, + ); +} + +export function recordKeystrokeFromHookEvent(event: { + keycode?: number; + rawcode?: number; + keychar?: number; + altKey?: boolean; + ctrlKey?: boolean; + metaKey?: boolean; + shiftKey?: boolean; + repeat?: boolean; +} | null) { + if (!event) { + return; + } + + const key = mapUiohookKey(event); + if (!key) { + return; + } + + const isPasswordField = + process.platform === "linux" ? inspectLinuxFocusedFieldPasswordState() : "unknown"; + + pushKeystrokeSample( + { + timeMs: getCursorCaptureElapsedMs(), + key, + code: key, + ctrl: event.ctrlKey === true, + alt: event.altKey === true, + shift: event.shiftKey === true, + meta: event.metaKey === true, + repeat: event.repeat === true ? true : undefined, + }, + isPasswordField, + ); +} + +const UIOHOOK_SPECIAL: Record = { + 1: "Escape", + 14: "Backspace", + 15: "Tab", + 28: "Enter", + 57: "Space", + 3655: "Home", + 3663: "End", + 3657: "PageUp", + 3665: "PageDown", + 3666: "Insert", + 3667: "Delete", + 57416: "ArrowUp", + 57419: "ArrowLeft", + 57421: "ArrowRight", + 57424: "ArrowDown", + 3675: "Meta", + 3676: "Meta", + 56: "Alt", + 3640: "Alt", + 29: "Control", + 3613: "Control", + 42: "Shift", + 54: "Shift", +}; + +function mapUiohookKey(event: { keycode?: number; rawcode?: number; keychar?: number }) { + const keycode = event.keycode ?? 0; + if (UIOHOOK_SPECIAL[keycode]) { + return UIOHOOK_SPECIAL[keycode]; + } + if (keycode >= 59 && keycode <= 68) { + return `F${keycode - 58}`; + } + if (keycode >= 87 && keycode <= 88) { + return `F${keycode - 76}`; + } + if (typeof event.keychar === "number" && event.keychar >= 32 && event.keychar <= 126) { + return String.fromCharCode(event.keychar); + } + + const letterByCode: Record = { + 16: "Q", + 17: "W", + 18: "E", + 19: "R", + 20: "T", + 21: "Y", + 22: "U", + 23: "I", + 24: "O", + 25: "P", + 30: "A", + 31: "S", + 32: "D", + 33: "F", + 34: "G", + 35: "H", + 36: "J", + 37: "K", + 38: "L", + 44: "Z", + 45: "X", + 46: "C", + 47: "V", + 48: "B", + 49: "N", + 50: "M", + 2: "1", + 3: "2", + 4: "3", + 5: "4", + 6: "5", + 7: "6", + 8: "7", + 9: "8", + 10: "9", + 11: "0", + }; + return letterByCode[keycode] ?? (keycode ? `Key${keycode}` : ""); +} + +export async function writeKeystrokeTelemetry(videoPath: string, samples: unknown) { + const telemetryPath = getKeystrokePathForVideo(videoPath); + const normalizedSamples = normalizeKeystrokeSamples(samples); + + if (normalizedSamples.length === 0) { + await fs.rm(telemetryPath, { force: true }); + return normalizedSamples; + } + + await fs.writeFile( + telemetryPath, + JSON.stringify({ version: KEYSTROKE_TELEMETRY_VERSION, samples: normalizedSamples }, null, 2), + "utf-8", + ); + + return normalizedSamples; +} + +export async function persistPendingKeystrokeTelemetry(videoPath: string) { + if (pendingKeystrokeSamples.length === 0 && activeKeystrokeSamples.length === 0) { + return; + } + snapshotKeystrokeTelemetryForPersistence(); + await writeKeystrokeTelemetry(videoPath, pendingKeystrokeSamples); + setPendingKeystrokeSamples([]); +} + +export function snapshotKeystrokeTelemetryForPersistence() { + if (activeKeystrokeSamples.length === 0) { + return; + } + + if (pendingKeystrokeSamples.length === 0) { + setPendingKeystrokeSamples([...activeKeystrokeSamples]); + return; + } + + const pendingRefs = new Set(pendingKeystrokeSamples); + setPendingKeystrokeSamples([ + ...pendingKeystrokeSamples, + ...activeKeystrokeSamples.filter((sample) => !pendingRefs.has(sample)), + ]); +} + +export function dropKeystrokesAfterElapsedMs(elapsedMs: number) { + setActiveKeystrokeSamples(activeKeystrokeSamples.filter((sample) => sample.timeMs <= elapsedMs)); +} + +export { normalizeKeystrokeSamples }; diff --git a/electron/ipc/cursor/macKeystrokeTap.ts b/electron/ipc/cursor/macKeystrokeTap.ts new file mode 100644 index 000000000..3c7ef66d7 --- /dev/null +++ b/electron/ipc/cursor/macKeystrokeTap.ts @@ -0,0 +1,42 @@ +import { createRequire } from "node:module"; +import { ensureKeystrokeTapBinary } from "../paths/binaries"; +import { recordKeystrokeFromMonitorLine } from "./keystrokes"; + +const nodeRequire = createRequire(import.meta.url); + +type KeystrokeTapAddon = { + start: (onLine: (line: string) => void) => boolean; + stop: () => boolean; +}; + +let loadedAddon: KeystrokeTapAddon | null = null; +let loadedAddonPath: string | null = null; + +async function loadAddon() { + const addonPath = await ensureKeystrokeTapBinary(); + if (loadedAddon && loadedAddonPath === addonPath) { + return loadedAddon; + } + delete nodeRequire.cache[addonPath]; + loadedAddon = nodeRequire(addonPath) as KeystrokeTapAddon; + loadedAddonPath = addonPath; + return loadedAddon; +} + +export async function startInProcessKeystrokeTap() { + const addon = await loadAddon(); + addon.stop(); + return addon.start((line) => { + if (line.startsWith("KEY:")) { + recordKeystrokeFromMonitorLine(line); + } + }); +} + +export function stopInProcessKeystrokeTap() { + loadedAddon?.stop(); +} + +export async function probeInProcessKeystrokeTap() { + return await startInProcessKeystrokeTap(); +} diff --git a/electron/ipc/cursor/monitor.ts b/electron/ipc/cursor/monitor.ts index 8a507d56d..e116a4aca 100644 --- a/electron/ipc/cursor/monitor.ts +++ b/electron/ipc/cursor/monitor.ts @@ -2,9 +2,11 @@ import { spawn } from "node:child_process"; import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import { BrowserWindow } from "electron"; -import { ensureNativeCursorMonitorBinary, getCursorMonitorExePath } from "../paths/binaries"; +import { ensureNativeCursorMonitorBinary, getCursorMonitorExePath, getPrebundledNativeHelperPath } from "../paths/binaries"; import { currentCursorVisualType, + isCursorCaptureActive, + isKeystrokeCaptureEnabled, nativeCursorMonitorOutputBuffer, nativeCursorMonitorProcess, setCurrentCursorVisualType, @@ -13,6 +15,8 @@ import { } from "../state"; import type { CursorVisualType } from "../types"; import { recordCursorMouseDown, recordCursorMouseUp } from "./interaction"; +import { recordKeystrokeFromMonitorLine } from "./keystrokes"; +import { startInProcessKeystrokeTap } from "./macKeystrokeTap"; export function emitCursorStateChanged(cursorType: CursorVisualType) { BrowserWindow.getAllWindows().forEach((window) => { @@ -39,6 +43,11 @@ export function handleCursorMonitorStdout(chunk: Buffer) { continue; } + if (line.startsWith("KEY:")) { + recordKeystrokeFromMonitorLine(line); + continue; + } + const match = line.match(/^STATE:(.+)$/); if (!match) continue; const next = match[1].trim() as CursorVisualType; @@ -55,7 +64,6 @@ export function handleCursorMonitorStdout(chunk: Buffer) { ) { if (currentCursorVisualType !== next) { setCurrentCursorVisualType(next); - // sampleCursorStateChange is called from cursor/telemetry.ts via the handler emitCursorStateChanged(next); } } @@ -84,8 +92,21 @@ export function stopNativeCursorMonitor() { setNativeCursorMonitorOutputBuffer(""); } +async function startMacKeystrokeTap() { + if (process.platform !== "darwin" || !isKeystrokeCaptureEnabled) { + return; + } + + try { + await startInProcessKeystrokeTap(); + } catch (error) { + console.warn("Failed to start keystroke tap:", error); + } +} + export async function startNativeCursorMonitor() { stopNativeCursorMonitor(); + void startMacKeystrokeTap(); if (process.platform !== "darwin" && process.platform !== "win32") { setCurrentCursorVisualType("arrow"); @@ -97,7 +118,6 @@ export async function startNativeCursorMonitor() { if (process.platform === "win32") { helperPath = getCursorMonitorExePath(); try { - // Use F_OK on Windows — X_OK is meaningless and can give false positives await fs.access(helperPath, fsConstants.F_OK); } catch { console.warn("Windows cursor monitor helper missing:", helperPath); @@ -105,7 +125,17 @@ export async function startNativeCursorMonitor() { return; } } else { - helperPath = await ensureNativeCursorMonitorBinary(); + const prebundledPath = getPrebundledNativeHelperPath("recordly-native-cursor-monitor"); + try { + await fs.access(prebundledPath, fsConstants.X_OK); + helperPath = prebundledPath; + } catch { + helperPath = await ensureNativeCursorMonitorBinary(); + } + } + + if (!isCursorCaptureActive) { + return; } setNativeCursorMonitorOutputBuffer(""); @@ -113,7 +143,9 @@ export async function startNativeCursorMonitor() { let proc: ReturnType | null; try { - proc = spawn(helperPath, [], { + const args = + process.platform === "win32" && isKeystrokeCaptureEnabled ? ["--capture-keys"] : []; + proc = spawn(helperPath, args, { stdio: ["pipe", "pipe", "pipe"], }); } catch (spawnError) { @@ -142,8 +174,11 @@ export async function startNativeCursorMonitor() { if (spawned.stdout) spawned.stdout.on("data", handleCursorMonitorStdout); if (spawned.stderr) { - spawned.stderr.on("data", () => { - // Drain stderr so helper logging cannot block the process. + spawned.stderr.on("data", (chunk: Buffer) => { + const message = chunk.toString().trim(); + if (message) { + console.warn("Native cursor monitor:", message); + } }); } diff --git a/electron/ipc/paths/binaries.test.ts b/electron/ipc/paths/binaries.test.ts index 361861739..4d8a07ccf 100644 --- a/electron/ipc/paths/binaries.test.ts +++ b/electron/ipc/paths/binaries.test.ts @@ -81,3 +81,47 @@ describe("Windows native helper path resolution", () => { expect(getWindowsCaptureExePath()).toBe(buildOutputPath); }); }); + +describe("packaged keystroke tap helper path", () => { + let tempRoot: string; + let appPath: string; + + beforeEach(async () => { + tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "recordly-keystroke-tap-")); + appPath = path.join(tempRoot, "App.asar"); + await fs.mkdir(appPath, { recursive: true }); + + vi.resetModules(); + vi.doMock("electron", () => ({ + app: { + isPackaged: true, + getAppPath: () => appPath, + getPath: () => path.join(tempRoot, "userData"), + }, + })); + }); + + afterEach(async () => { + vi.resetModules(); + vi.doUnmock("electron"); + await fs.rm(tempRoot, { recursive: true, force: true }); + }); + + it("selects the architecture-specific packaged node binary", async () => { + const { getKeystrokeTapSourcePath, getNativeArchTag, ensureKeystrokeTapBinary } = + await import("./binaries"); + const packagedPath = path.join( + appPath.replace(/\.asar$/, ".asar.unpacked"), + "electron", + "native", + "bin", + getNativeArchTag(), + "recordly-keystroke-tap.node", + ); + await fs.mkdir(path.dirname(packagedPath), { recursive: true }); + await fs.writeFile(packagedPath, "packaged-tap"); + + expect(getKeystrokeTapSourcePath()).toBe(packagedPath); + await expect(ensureKeystrokeTapBinary()).resolves.toBe(packagedPath); + }); +}); diff --git a/electron/ipc/paths/binaries.ts b/electron/ipc/paths/binaries.ts index 77c8dd35f..e3e9895dc 100644 --- a/electron/ipc/paths/binaries.ts +++ b/electron/ipc/paths/binaries.ts @@ -1,6 +1,7 @@ import { execFile } from "node:child_process"; import { existsSync, constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; import { app } from "electron"; @@ -109,6 +110,17 @@ export function getNativeCursorMonitorBinaryPath(): string { return path.join(app.getPath("userData"), "native-tools", "recordly-native-cursor-monitor"); } +export function getKeystrokeTapSourcePath(): string { + if (app.isPackaged) { + return getPrebundledNativeHelperPath("recordly-keystroke-tap.node"); + } + return resolveUnpackedAppPath("electron", "native", "KeystrokeEventTap.c"); +} + +export function getKeystrokeTapBinaryPath(): string { + return path.join(app.getPath("userData"), "native-tools", "recordly-keystroke-tap.node"); +} + export function getNativeWindowListSourcePath(): string { return resolveUnpackedAppPath("electron", "native", "ScreenCaptureKitWindowList.swift"); } @@ -228,12 +240,28 @@ export async function ensureSwiftHelperBinary( return binaryPath; } + const moduleCacheRoot = path.join(os.tmpdir(), "recordly-swift-module-cache"); try { await execFileAsync("swiftc", ["-O", sourcePath, "-o", binaryPath], { encoding: "utf8", timeout: 120000, + env: { + ...process.env, + CLANG_MODULE_CACHE_PATH: path.join(moduleCacheRoot, "clang"), + SWIFT_MODULECACHE_PATH: path.join(moduleCacheRoot, "swift"), + }, }); } catch (error) { + if (prebundledBinaryName) { + const fallbackPath = getPrebundledNativeHelperPath(prebundledBinaryName); + try { + await fs.access(fallbackPath, fsConstants.X_OK); + console.warn(`Failed to compile ${label}; using prebundled helper`); + return fallbackPath; + } catch { + // Fall through to the original compile error. + } + } const err = error as NodeJS.ErrnoException & { stdout?: string; stderr?: string }; const details = [err.stderr, err.stdout].filter(Boolean).join("\n").trim(); throw new Error(details || `Failed to compile ${label}`); @@ -242,6 +270,79 @@ export async function ensureSwiftHelperBinary( return binaryPath; } +export async function ensureKeystrokeTapBinary(): Promise { + if (app.isPackaged) { + const packagedPath = getKeystrokeTapSourcePath(); + try { + await fs.access(packagedPath); + return packagedPath; + } catch { + throw new Error( + `Keystroke tap helper is missing from this app build (${packagedPath}). Reinstall or update the app.`, + ); + } + } + + const sourcePath = getKeystrokeTapSourcePath(); + const binaryPath = getKeystrokeTapBinaryPath(); + await fs.mkdir(path.dirname(binaryPath), { recursive: true }); + + const [sourceStat, binaryStat] = await Promise.all([ + fs.stat(sourcePath), + fs.stat(binaryPath).catch(() => null), + ]); + if (binaryStat && binaryStat.mtimeMs >= sourceStat.mtimeMs) { + return binaryPath; + } + + const includeCandidates = [ + path.join(os.homedir(), ".nvm", "versions", "node", `v${process.versions.node}`, "include", "node"), + "/usr/local/include/node", + path.join(os.homedir(), ".nvm", "versions", "node", "v24.4.0", "include", "node"), + ]; + let includeDir: string | null = null; + for (const candidate of includeCandidates) { + try { + await fs.access(path.join(candidate, "node_api.h")); + includeDir = candidate; + break; + } catch { + // try next + } + } + if (!includeDir) { + throw new Error("node_api.h not found; cannot compile in-process keystroke tap"); + } + + try { + await execFileAsync( + "clang", + [ + "-bundle", + "-undefined", + "dynamic_lookup", + "-Os", + `-I${includeDir}`, + sourcePath, + "-framework", + "ApplicationServices", + "-o", + binaryPath, + ], + { + encoding: "utf8", + timeout: 60000, + }, + ); + await fs.chmod(binaryPath, 0o755); + return binaryPath; + } catch (error) { + const err = error as NodeJS.ErrnoException & { stdout?: string; stderr?: string }; + const details = [err.stderr, err.stdout].filter(Boolean).join("\n").trim(); + throw new Error(details || "Failed to compile native keystroke tap helper"); + } +} + export async function ensureNativeCaptureHelperBinary(): Promise { await ensureNativeHelperMigration(); return ensureSwiftHelperBinary( diff --git a/electron/ipc/recording/mac.ts b/electron/ipc/recording/mac.ts index 57a9e6881..d5b3e23ad 100644 --- a/electron/ipc/recording/mac.ts +++ b/electron/ipc/recording/mac.ts @@ -5,6 +5,10 @@ import { persistPendingCursorTelemetry, snapshotCursorTelemetryForPersistence, } from "../cursor/telemetry"; +import { + persistPendingKeystrokeTelemetry, + snapshotKeystrokeTelemetryForPersistence, +} from "../cursor/keystrokes"; import { lastNativeCaptureDiagnostics, nativeCaptureMicrophonePath, @@ -263,6 +267,7 @@ export async function finalizeStoredVideo(videoPath: string) { } snapshotCursorTelemetryForPersistence(); + snapshotKeystrokeTelemetryForPersistence(); setCurrentVideoPath(videoPath); setCurrentProjectPath(null); try { @@ -270,6 +275,11 @@ export async function finalizeStoredVideo(videoPath: string) { } catch (error) { console.warn("[mac-stop] Failed to persist cursor telemetry:", error); } + try { + await persistPendingKeystrokeTelemetry(videoPath); + } catch (error) { + console.warn("[mac-stop] Failed to persist keystroke telemetry:", error); + } if (isAutoRecordingPath(videoPath)) { await pruneAutoRecordings([videoPath]); } diff --git a/electron/ipc/register/permissions.ts b/electron/ipc/register/permissions.ts index f3b8b86f1..28894031b 100644 --- a/electron/ipc/register/permissions.ts +++ b/electron/ipc/register/permissions.ts @@ -1,6 +1,14 @@ -import { ipcMain, shell, systemPreferences } from "electron"; +import { app, ipcMain, shell, systemPreferences } from "electron"; +import { probeInProcessKeystrokeTap, stopInProcessKeystrokeTap } from "../cursor/macKeystrokeTap"; import { getMacPrivacySettingsUrl } from "../utils"; +function accessibilityClientName() { + if (process.execPath.includes("Electron.app")) { + return "Electron"; + } + return app.name || "Recordly"; +} + export function registerPermissionHandlers() { ipcMain.handle("open-external-url", async (_, url: string) => { try { @@ -26,18 +34,50 @@ export function registerPermissionHandlers() { success: true, trusted: systemPreferences.isTrustedAccessibilityClient(false), prompted: false, + clientName: accessibilityClientName(), }; }); ipcMain.handle("request-accessibility-permission", () => { if (process.platform !== "darwin") { - return { success: true, trusted: true, prompted: false }; + return { success: true, trusted: true, prompted: false, clientName: accessibilityClientName() }; } return { success: true, trusted: systemPreferences.isTrustedAccessibilityClient(true), prompted: true, + clientName: accessibilityClientName(), + }; + }); + + ipcMain.handle("stop-keystroke-tap", () => { + stopInProcessKeystrokeTap(); + return { success: true }; + }); + + ipcMain.handle("request-keystroke-capture-permission", async () => { + if (process.platform !== "darwin") { + return { success: true, trusted: true, clientName: accessibilityClientName() }; + } + + const trusted = systemPreferences.isTrustedAccessibilityClient(true); + let helperReady = false; + try { + const started = await probeInProcessKeystrokeTap(); + helperReady = started; + if (!helperReady) { + console.warn("Unable to prepare keystroke tap helper: in-process CGEventTapCreate failed"); + } + } catch (error) { + console.warn("Unable to prepare keystroke tap helper:", error); + } + + return { + success: true, + trusted, + tapOk: helperReady, + clientName: accessibilityClientName(), }; }); @@ -84,4 +124,18 @@ export function registerPermissionHandlers() { return { success: false, error: String(error) }; } }); + + ipcMain.handle("open-input-monitoring-preferences", async () => { + if (process.platform !== "darwin") { + return { success: true }; + } + + try { + await shell.openExternal(getMacPrivacySettingsUrl("input-monitoring")); + return { success: true }; + } catch (error) { + console.error("Failed to open Input Monitoring preferences:", error); + return { success: false, error: String(error) }; + } + }); } diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index 2c50e3976..dd25fa0dc 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -21,8 +21,19 @@ import { import { ALLOW_RECORDLY_WINDOW_CAPTURE } from "../constants"; import { startWindowBoundsCapture, stopWindowBoundsCapture } from "../cursor/bounds"; import { startInteractionCapture, stopInteractionCapture } from "../cursor/interaction"; +import { + dropKeystrokesAfterElapsedMs, + normalizeKeystrokeSamples, + persistPendingKeystrokeTelemetry, + resetKeystrokeCapture, + snapshotKeystrokeTelemetryForPersistence, + syncKeystrokeCaptureEnabledFromSettings, + writeKeystrokeTelemetry, +} from "../cursor/keystrokes"; +import { stopInProcessKeystrokeTap } from "../cursor/macKeystrokeTap"; import { startNativeCursorMonitor, stopNativeCursorMonitor } from "../cursor/monitor"; import { + getCursorCaptureElapsedMs, normalizeCursorTelemetrySamples, pauseCursorCaptureAtBoundary, persistPendingCursorTelemetry, @@ -44,7 +55,7 @@ import { getSystemCursorHelperSourcePath, getWindowsCaptureExePath, } from "../paths/binaries"; -import { rememberApprovedLocalReadPath } from "../project/manager"; +import { rememberApprovedLocalReadPath, isAllowedLocalReadPath } from "../project/manager"; import { getBrowserMicSidecarFilters, shouldKeepRecordingAudioSidecars, @@ -104,6 +115,7 @@ import { nativeScreenRecordingActive, selectedSource, setActiveCursorSamples, + setActiveKeystrokeSamples, setCachedSystemCursorAssets, setCachedSystemCursorAssetsSourceMtimeMs, setCursorCaptureStartTimeMs, @@ -143,11 +155,13 @@ import { windowsPendingVideoPath, windowsSystemAudioPath, } from "../state"; +import type { KeystrokeSample } from "../../../src/lib/keystrokeOverlay"; import type { CursorTelemetryPoint, NativeMacRecordingOptions, SelectedSource } from "../types"; import { getMacPrivacySettingsUrl, getRecordingsDir, getScreen, + getKeystrokePathForVideo, getTelemetryPathForVideo, moveFileWithOverwrite, normalizeVideoSourcePath, @@ -159,6 +173,49 @@ import { bringSelectedWindowForward } from "./sources"; const execFileAsync = promisify(execFile); +async function persistCaptureTelemetryForVideo(videoPath: string) { + snapshotCursorTelemetryForPersistence(); + snapshotKeystrokeTelemetryForPersistence(); + try { + await persistPendingCursorTelemetry(videoPath); + } catch (error) { + console.warn("Failed to persist cursor telemetry during native stop:", error); + } + try { + await persistPendingKeystrokeTelemetry(videoPath); + } catch (error) { + console.warn("Failed to persist keystroke telemetry during native stop:", error); + } +} + +function isAuthorizedIpcSender(event: Electron.IpcMainInvokeEvent) { + if (!event.sender || event.sender.isDestroyed()) { + return false; + } + const window = BrowserWindow.fromWebContents(event.sender); + return Boolean(window && !window.isDestroyed()); +} + +function resolveAuthorizedKeystrokeVideoPath( + event: Electron.IpcMainInvokeEvent, + videoPath?: string, +) { + if (!isAuthorizedIpcSender(event)) { + return null; + } + + const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath); + if (!targetVideoPath) { + return null; + } + + if (!isAllowedLocalReadPath(path.resolve(targetVideoPath))) { + return null; + } + + return targetVideoPath; +} + async function writeWindowsRecordingDiagnostics( videoPath: string | null | undefined, snapshot: Omit, @@ -1030,15 +1087,7 @@ export function registerRecordingHandlers( }); // Persist cursor telemetry before returning so the editor can find it immediately - snapshotCursorTelemetryForPersistence(); - try { - await persistPendingCursorTelemetry(finalVideoPath); - } catch (error) { - console.warn( - "Failed to persist cursor telemetry during native stop:", - error, - ); - } + await persistCaptureTelemetryForVideo(finalVideoPath); return { success: true, path: finalVideoPath }; } catch (error) { @@ -1097,6 +1146,7 @@ export function registerRecordingHandlers( recoveredAfterStopFailure: true, }, }); + await persistCaptureTelemetryForVideo(fallbackPath); return { success: true, path: fallbackPath }; } catch { // File is absent or failed validation. @@ -1868,6 +1918,8 @@ export function registerRecordingHandlers( stopCursorCapture(); stopInteractionCapture(); startWindowBoundsCapture(); + syncKeystrokeCaptureEnabledFromSettings(); + resetKeystrokeCapture(); void startNativeCursorMonitor(); setIsCursorCaptureActive(true); setActiveCursorSamples([]); @@ -1889,7 +1941,10 @@ export function registerRecordingHandlers( setLinuxCursorScreenPoint(null); resetCursorCaptureClock(); snapshotCursorTelemetryForPersistence(); + snapshotKeystrokeTelemetryForPersistence(); + stopInProcessKeystrokeTap(); setActiveCursorSamples([]); + setActiveKeystrokeSamples([]); } const source = selectedSource || { name: "Screen" }; @@ -1909,6 +1964,7 @@ export function registerRecordingHandlers( ipcMain.handle("pause-cursor-capture", (_, pausedAtMs?: unknown) => { pauseCursorCaptureAtBoundary(normalizeRendererTimestampMs(pausedAtMs)); + dropKeystrokesAfterElapsedMs(getCursorCaptureElapsedMs()); return { success: true }; }); @@ -1929,7 +1985,6 @@ export function registerRecordingHandlers( const content = await fs.readFile(telemetryPath, "utf-8"); const parsed = parseJsonWithByteOrderMark(content); const samples = normalizeCursorTelemetrySamples(parsed); - return { success: true, samples }; } catch (error) { const nodeError = error as NodeJS.ErrnoException; @@ -1973,4 +2028,59 @@ export function registerRecordingHandlers( } }, ); + + ipcMain.handle("get-keystroke-telemetry", async (event, videoPath?: string) => { + const targetVideoPath = resolveAuthorizedKeystrokeVideoPath(event, videoPath); + if (!targetVideoPath) { + return { success: true, samples: [] }; + } + + const telemetryPath = getKeystrokePathForVideo(targetVideoPath); + try { + const content = await fs.readFile(telemetryPath, "utf-8"); + const parsed = parseJsonWithByteOrderMark(content); + const samples = normalizeKeystrokeSamples(parsed); + return { success: true, samples }; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === "ENOENT") { + return { success: true, samples: [] }; + } + console.error("Failed to load keystroke telemetry:", error); + return { + success: false, + message: "Failed to load keystroke telemetry", + error: String(error), + samples: [], + }; + } + }); + + ipcMain.handle( + "set-keystroke-telemetry", + async (event, videoPath: string | undefined, samples: KeystrokeSample[]) => { + const targetVideoPath = resolveAuthorizedKeystrokeVideoPath(event, videoPath); + if (!targetVideoPath) { + return { + success: false, + samples: [], + message: "No video path available for keystroke telemetry", + error: "Missing video path", + }; + } + + try { + const normalizedSamples = await writeKeystrokeTelemetry(targetVideoPath, samples); + return { success: true, samples: normalizedSamples }; + } catch (error) { + console.error("Failed to save keystroke telemetry:", error); + return { + success: false, + samples: [], + message: "Failed to save keystroke telemetry", + error: String(error), + }; + } + }, + ); } diff --git a/electron/ipc/state.ts b/electron/ipc/state.ts index a0a41744e..eb2cd5adc 100644 --- a/electron/ipc/state.ts +++ b/electron/ipc/state.ts @@ -1,4 +1,5 @@ import type { ChildProcessWithoutNullStreams } from "node:child_process"; +import type { KeystrokeSample } from "../../src/lib/keystrokeOverlay"; import type { CursorInteractionType, CursorTelemetryPoint, @@ -80,7 +81,10 @@ export let cursorCaptureAccumulatedPausedMs = 0; export let cursorCapturePauseStartedAtMs: number | null = null; export let activeCursorSamples: CursorTelemetryPoint[] = []; export let pendingCursorSamples: CursorTelemetryPoint[] = []; +export let activeKeystrokeSamples: KeystrokeSample[] = []; +export let pendingKeystrokeSamples: KeystrokeSample[] = []; export let isCursorCaptureActive = false; +export let isKeystrokeCaptureEnabled = false; export let interactionCaptureCleanup: (() => void) | null = null; export let hasLoggedInteractionHookFailure = false; export let lastLeftClick: { timeMs: number; cx: number; cy: number } | null = null; @@ -251,6 +255,15 @@ export function setActiveCursorSamples(v: CursorTelemetryPoint[]) { export function setPendingCursorSamples(v: CursorTelemetryPoint[]) { pendingCursorSamples = v; } +export function setActiveKeystrokeSamples(v: KeystrokeSample[]) { + activeKeystrokeSamples = v; +} +export function setPendingKeystrokeSamples(v: KeystrokeSample[]) { + pendingKeystrokeSamples = v; +} +export function setIsKeystrokeCaptureEnabled(v: boolean) { + isKeystrokeCaptureEnabled = v; +} export function setIsCursorCaptureActive(v: boolean) { isCursorCaptureActive = v; } diff --git a/electron/ipc/types.ts b/electron/ipc/types.ts index 58f5425bd..898f1b9b0 100644 --- a/electron/ipc/types.ts +++ b/electron/ipc/types.ts @@ -120,7 +120,7 @@ export type NativeMacWindowSource = { height?: number; }; -export type HookEventName = "mousedown" | "mouseup" | "mousemove"; +export type HookEventName = "mousedown" | "mouseup" | "mousemove" | "keydown" | "keyup"; export type HookMouseEvent = { button?: number; @@ -129,6 +129,13 @@ export type HookMouseEvent = { y?: number; screenX?: number; screenY?: number; + keycode?: number; + rawcode?: number; + keychar?: number; + altKey?: boolean; + ctrlKey?: boolean; + metaKey?: boolean; + shiftKey?: boolean; data?: { button?: number; mouseButton?: number; diff --git a/electron/ipc/utils.ts b/electron/ipc/utils.ts index 23960f209..2053e1606 100644 --- a/electron/ipc/utils.ts +++ b/electron/ipc/utils.ts @@ -67,6 +67,10 @@ export function getTelemetryPathForVideo(videoPath: string) { return `${videoPath}.cursor.json`; } +export function getKeystrokePathForVideo(videoPath: string) { + return `${videoPath}.keys.json`; +} + export function isAutoRecordingPath(filePath: string) { return path.basename(filePath).startsWith(AUTO_RECORDING_PREFIX); } @@ -113,11 +117,15 @@ export async function getRecordingsDir() { return targetDir; } -export function getMacPrivacySettingsUrl(pane: "screen" | "accessibility" | "microphone"): string { +export function getMacPrivacySettingsUrl( + pane: "screen" | "accessibility" | "microphone" | "input-monitoring", +): string { if (pane === "screen") return "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture"; if (pane === "microphone") return "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone"; + if (pane === "input-monitoring") + return "x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent"; return "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility"; } diff --git a/electron/native/KeystrokeEventTap.c b/electron/native/KeystrokeEventTap.c new file mode 100644 index 000000000..ed00994a2 --- /dev/null +++ b/electron/native/KeystrokeEventTap.c @@ -0,0 +1,533 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static napi_threadsafe_function g_tsfn = NULL; +static CFMachPortRef g_tap = NULL; +static CFRunLoopSourceRef g_source = NULL; +static CGEventFlags g_modifier_flags = 0; + +static void emit_key_line(const char *line) { + if (!line) { + return; + } + if (g_tsfn) { + char *copy = strdup(line); + if (copy) { + napi_call_threadsafe_function(g_tsfn, copy, napi_tsfn_nonblocking); + } + return; + } + printf("%s\n", line); + fflush(stdout); +} + +static bool string_contains_ci(CFStringRef value, CFStringRef needle) { + if (!value || !needle) { + return false; + } + CFRange range = CFStringFind(value, needle, kCFCompareCaseInsensitive); + return range.location != kCFNotFound; +} + +typedef enum { + FOCUSED_FIELD_NON_SECURE = 0, + FOCUSED_FIELD_SECURE = 1, + FOCUSED_FIELD_UNKNOWN = 2, +} focused_field_kind; + +static CFStringRef copy_ax_string(AXUIElementRef element, CFStringRef attribute) { + CFTypeRef value = NULL; + if (AXUIElementCopyAttributeValue(element, attribute, &value) != kAXErrorSuccess || !value) { + return NULL; + } + if (CFGetTypeID(value) == CFStringGetTypeID()) { + return (CFStringRef)value; + } + CFRelease(value); + return NULL; +} + +static focused_field_kind element_security_kind(AXUIElementRef element) { + CFStringRef role = copy_ax_string(element, kAXRoleAttribute); + CFStringRef subrole = copy_ax_string(element, kAXSubroleAttribute); + if (!role && !subrole) { + return FOCUSED_FIELD_UNKNOWN; + } + const bool secure = + string_contains_ci(subrole, CFSTR("secure")) || + string_contains_ci(subrole, CFSTR("password")) || + string_contains_ci(role, CFSTR("secure")) || + string_contains_ci(role, CFSTR("password")); + if (role) { + CFRelease(role); + } + if (subrole) { + CFRelease(subrole); + } + return secure ? FOCUSED_FIELD_SECURE : FOCUSED_FIELD_NON_SECURE; +} + +static focused_field_kind focused_element_security_kind(void) { + AXUIElementRef system_wide = AXUIElementCreateSystemWide(); + if (!system_wide) { + return FOCUSED_FIELD_UNKNOWN; + } + + CFTypeRef focused = NULL; + AXError focused_error = AXUIElementCopyAttributeValue( + system_wide, + kAXFocusedUIElementAttribute, + &focused + ); + CFRelease(system_wide); + + if (focused_error != kAXErrorSuccess || !focused || CFGetTypeID(focused) != AXUIElementGetTypeID()) { + if (focused) { + CFRelease(focused); + } + return FOCUSED_FIELD_UNKNOWN; + } + + AXUIElementRef current = (AXUIElementRef)focused; + focused_field_kind result = FOCUSED_FIELD_NON_SECURE; + for (int depth = 0; current && depth < 6; depth += 1) { + focused_field_kind kind = element_security_kind(current); + if (kind != FOCUSED_FIELD_NON_SECURE) { + result = kind; + break; + } + CFTypeRef parent = NULL; + AXError parent_error = AXUIElementCopyAttributeValue(current, kAXParentAttribute, &parent); + if (parent_error != kAXErrorSuccess || !parent || CFGetTypeID(parent) != AXUIElementGetTypeID()) { + if (parent) { + CFRelease(parent); + } + if (parent_error != kAXErrorSuccess && + parent_error != kAXErrorNoValue && + parent_error != kAXErrorAttributeUnsupported) { + result = FOCUSED_FIELD_UNKNOWN; + } + break; + } + CFRelease(current); + current = (AXUIElementRef)parent; + } + if (current) { + CFRelease(current); + } + return result; +} + +static const char *key_name(CGKeyCode key_code) { + switch (key_code) { + case 36: + case 76: + return "Enter"; + case 53: + return "Escape"; + case 48: + return "Tab"; + case 51: + return "Backspace"; + case 117: + return "Delete"; + case 49: + return "Space"; + case 126: + return "ArrowUp"; + case 125: + return "ArrowDown"; + case 123: + return "ArrowLeft"; + case 124: + return "ArrowRight"; + case 115: + return "Home"; + case 119: + return "End"; + case 116: + return "PageUp"; + case 121: + return "PageDown"; + case 114: + return "Insert"; + case 122: + return "F1"; + case 120: + return "F2"; + case 99: + return "F3"; + case 118: + return "F4"; + case 96: + return "F5"; + case 97: + return "F6"; + case 98: + return "F7"; + case 100: + return "F8"; + case 101: + return "F9"; + case 109: + return "F10"; + case 103: + return "F11"; + case 111: + return "F12"; + case 0: + return "A"; + case 11: + return "B"; + case 8: + return "C"; + case 2: + return "D"; + case 14: + return "E"; + case 3: + return "F"; + case 5: + return "G"; + case 4: + return "H"; + case 34: + return "I"; + case 38: + return "J"; + case 40: + return "K"; + case 37: + return "L"; + case 46: + return "M"; + case 45: + return "N"; + case 31: + return "O"; + case 35: + return "P"; + case 12: + return "Q"; + case 15: + return "R"; + case 1: + return "S"; + case 17: + return "T"; + case 32: + return "U"; + case 9: + return "V"; + case 13: + return "W"; + case 7: + return "X"; + case 16: + return "Y"; + case 6: + return "Z"; + case 18: + return "1"; + case 19: + return "2"; + case 20: + return "3"; + case 21: + return "4"; + case 22: + return "5"; + case 23: + return "6"; + case 26: + return "7"; + case 28: + return "8"; + case 25: + return "9"; + case 29: + return "0"; + case 24: + return "Equal"; + case 27: + return "Minus"; + case 33: + return "["; + case 30: + return "]"; + case 42: + return "\\"; + case 41: + return ";"; + case 39: + return "'"; + case 43: + return ","; + case 47: + return "."; + case 44: + return "/"; + case 50: + return "`"; + case 105: + return "F13"; + case 107: + return "F14"; + case 113: + return "F15"; + case 106: + return "F16"; + case 64: + return "F17"; + case 79: + return "F18"; + case 80: + return "F19"; + case 56: + case 60: + return "Shift"; + case 59: + case 62: + return "Control"; + case 58: + case 61: + return "Alt"; + case 55: + case 54: + return "Meta"; + default: + return NULL; + } +} + +static CGEventRef tap_callback( + CGEventTapProxy proxy, + CGEventType type, + CGEventRef event, + void *user_info +) { + (void)proxy; + (void)user_info; + + if (type == kCGEventTapDisabledByTimeout || type == kCGEventTapDisabledByUserInput) { + if (g_tap) { + CGEventTapEnable(g_tap, true); + } + return event; + } + if (type == kCGEventFlagsChanged) { + g_modifier_flags = CGEventGetFlags(event); + return event; + } + if (type != kCGEventKeyDown) { + return event; + } + const focused_field_kind field_kind = focused_element_security_kind(); + if (field_kind == FOCUSED_FIELD_SECURE) { + return event; + } + + const CGKeyCode key_code = (CGKeyCode)CGEventGetIntegerValueField(event, kCGKeyboardEventKeycode); + const CGEventFlags flags = CGEventGetFlags(event) | g_modifier_flags; + char fallback[32]; + const char *name = key_name(key_code); + if (!name) { + snprintf(fallback, sizeof(fallback), "Key%u", (unsigned)key_code); + name = fallback; + } + + char modifiers[64] = {0}; + if (flags & kCGEventFlagMaskControl) { + strcat(modifiers, modifiers[0] ? ",ctrl" : "ctrl"); + } + if (flags & kCGEventFlagMaskAlternate) { + strcat(modifiers, modifiers[0] ? ",alt" : "alt"); + } + if (flags & kCGEventFlagMaskShift) { + strcat(modifiers, modifiers[0] ? ",shift" : "shift"); + } + if (flags & kCGEventFlagMaskCommand) { + strcat(modifiers, modifiers[0] ? ",meta" : "meta"); + } + if (CGEventGetIntegerValueField(event, kCGKeyboardEventAutorepeat) != 0) { + strcat(modifiers, modifiers[0] ? ",repeat" : "repeat"); + } + + char line[160]; + if (modifiers[0]) { + snprintf(line, sizeof(line), "KEY:down:%s:%s", name, modifiers); + } else { + snprintf(line, sizeof(line), "KEY:down:%s", name); + } + emit_key_line(line); + return event; +} + +static void *watch_stdin(void *arg) { + (void)arg; + char line[32]; + while (fgets(line, sizeof(line), stdin)) { + if (strncmp(line, "stop", 4) == 0) { + _exit(0); + } + } + _exit(0); + return NULL; +} + +int main(int argc, const char **argv) { + const int probe = argc > 1 && strcmp(argv[1], "--probe") == 0; + const CGEventMask mask = CGEventMaskBit(kCGEventKeyDown) | CGEventMaskBit(kCGEventFlagsChanged); + CFMachPortRef tap = CGEventTapCreate( + kCGSessionEventTap, + kCGHeadInsertEventTap, + kCGEventTapOptionListenOnly, + mask, + tap_callback, + NULL + ); + if (!tap) { + fputs("Keystroke event tap unavailable\n", stderr); + if (probe) { + printf("TAP:fail\n"); + fflush(stdout); + } + return 1; + } + if (probe) { + printf("TAP:ok\n"); + fflush(stdout); + CFRelease(tap); + return 0; + } + + CFRunLoopSourceRef source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0); + CFRunLoopAddSource(CFRunLoopGetCurrent(), source, kCFRunLoopCommonModes); + CGEventTapEnable(tap, true); + + pthread_t stdin_thread; + pthread_create(&stdin_thread, NULL, watch_stdin, NULL); + pthread_detach(stdin_thread); + + CFRunLoopRun(); + CFRelease(source); + CFRelease(tap); + return 0; +} + +static void call_js_key_line(napi_env env, napi_value js_callback, void *context, void *data) { + (void)context; + char *line = (char *)data; + if (env == NULL || js_callback == NULL) { + free(line); + return; + } + napi_value argv[1]; + napi_create_string_utf8(env, line ? line : "", NAPI_AUTO_LENGTH, &argv[0]); + napi_value global; + napi_get_global(env, &global); + napi_call_function(env, global, js_callback, 1, argv, NULL); + free(line); +} + +static void native_stop_tap(void) { + g_modifier_flags = 0; + if (g_tap) { + CGEventTapEnable(g_tap, false); + } + if (g_source) { + CFRunLoopRemoveSource(CFRunLoopGetMain(), g_source, kCFRunLoopCommonModes); + CFRelease(g_source); + g_source = NULL; + } + if (g_tap) { + CFRelease(g_tap); + g_tap = NULL; + } + if (g_tsfn) { + napi_release_threadsafe_function(g_tsfn, napi_tsfn_release); + g_tsfn = NULL; + } +} + +static napi_value Start(napi_env env, napi_callback_info info) { + size_t argc = 1; + napi_value args[1]; + napi_get_cb_info(env, info, &argc, args, NULL, NULL); + + napi_value result; + if (argc < 1) { + napi_get_boolean(env, false, &result); + return result; + } + + native_stop_tap(); + + napi_value resource_name; + napi_create_string_utf8(env, "recordly-keystroke-tap", NAPI_AUTO_LENGTH, &resource_name); + napi_status tsfn_status = napi_create_threadsafe_function( + env, + args[0], + NULL, + resource_name, + 0, + 1, + NULL, + NULL, + NULL, + call_js_key_line, + &g_tsfn + ); + if (tsfn_status != napi_ok) { + napi_get_boolean(env, false, &result); + return result; + } + + const CGEventMask mask = CGEventMaskBit(kCGEventKeyDown) | CGEventMaskBit(kCGEventFlagsChanged); + g_tap = CGEventTapCreate( + kCGSessionEventTap, + kCGHeadInsertEventTap, + kCGEventTapOptionListenOnly, + mask, + tap_callback, + NULL + ); + if (!g_tap) { + native_stop_tap(); + napi_get_boolean(env, false, &result); + return result; + } + + g_source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, g_tap, 0); + CFRunLoopAddSource(CFRunLoopGetMain(), g_source, kCFRunLoopCommonModes); + CGEventTapEnable(g_tap, true); + napi_get_boolean(env, true, &result); + return result; +} + +static napi_value Stop(napi_env env, napi_callback_info info) { + (void)info; + native_stop_tap(); + napi_value result; + napi_get_boolean(env, true, &result); + return result; +} + +static napi_value Init(napi_env env, napi_value exports) { + napi_value start_fn; + napi_value stop_fn; + napi_create_function(env, "start", NAPI_AUTO_LENGTH, Start, NULL, &start_fn); + napi_create_function(env, "stop", NAPI_AUTO_LENGTH, Stop, NULL, &stop_fn); + napi_set_named_property(env, exports, "start", start_fn); + napi_set_named_property(env, exports, "stop", stop_fn); + return exports; +} + +NAPI_MODULE(recordly_keystroke_tap, Init) diff --git a/electron/native/NativeCursorMonitor.swift b/electron/native/NativeCursorMonitor.swift index 18c644150..bb71010bb 100644 --- a/electron/native/NativeCursorMonitor.swift +++ b/electron/native/NativeCursorMonitor.swift @@ -406,12 +406,135 @@ if CommandLine.arguments.contains("--export-images") { exit(0) } +func elementLooksSecure(_ element: AXUIElement) -> Bool { + let role = attributeString(element, kAXRoleAttribute)?.lowercased() ?? "" + let subrole = attributeString(element, kAXSubroleAttribute)?.lowercased() ?? "" + let metadata = metadataString(for: element) + if subrole.contains("secure") || subrole.contains("password") { + return true + } + if role.contains("secure") || metadata.contains("secure text") || metadata.contains("password") { + return true + } + return false +} + +func focusedElementLooksSecure() -> Bool { + for element in ancestorChain(startingAt: focusedElement(), maxDepth: 6) { + if elementLooksSecure(element) { + return true + } + } + return false +} + +func keyName(for keyCode: Int64) -> String { + switch keyCode { + case 36, 76: return "Enter" + case 53: return "Escape" + case 48: return "Tab" + case 51: return "Backspace" + case 117: return "Delete" + case 49: return "Space" + case 126: return "ArrowUp" + case 125: return "ArrowDown" + case 123: return "ArrowLeft" + case 124: return "ArrowRight" + case 115: return "Home" + case 119: return "End" + case 116: return "PageUp" + case 121: return "PageDown" + case 114: return "Insert" + case 122: return "F1" + case 120: return "F2" + case 99: return "F3" + case 118: return "F4" + case 96: return "F5" + case 97: return "F6" + case 98: return "F7" + case 100: return "F8" + case 101: return "F9" + case 109: return "F10" + case 103: return "F11" + case 111: return "F12" + case 0: return "A" + case 11: return "B" + case 8: return "C" + case 2: return "D" + case 14: return "E" + case 3: return "F" + case 5: return "G" + case 4: return "H" + case 34: return "I" + case 38: return "J" + case 40: return "K" + case 37: return "L" + case 46: return "M" + case 45: return "N" + case 31: return "O" + case 35: return "P" + case 12: return "Q" + case 15: return "R" + case 1: return "S" + case 17: return "T" + case 32: return "U" + case 9: return "V" + case 13: return "W" + case 7: return "X" + case 16: return "Y" + case 6: return "Z" + case 18: return "1" + case 19: return "2" + case 20: return "3" + case 21: return "4" + case 22: return "5" + case 23: return "6" + case 24: return "7" + case 25: return "8" + case 26: return "9" + case 29: return "0" + case 56, 60: return "Shift" + case 59, 62: return "Control" + case 58, 61: return "Alt" + case 55, 54: return "Meta" + default: return "Key\(keyCode)" + } +} + +func emitKeyEvent(_ event: CGEvent, action: String) { + if focusedElementLooksSecure() { + return + } + + let keyCode = event.getIntegerValueField(.keyboardEventKeycode) + let flags = event.flags + var modifiers: [String] = [] + if flags.contains(.maskControl) { modifiers.append("ctrl") } + if flags.contains(.maskAlternate) { modifiers.append("alt") } + if flags.contains(.maskShift) { modifiers.append("shift") } + if flags.contains(.maskCommand) { modifiers.append("meta") } + if event.getIntegerValueField(.keyboardEventAutorepeat) != 0 { modifiers.append("repeat") } + + let encodedKey = keyName(for: keyCode).addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? "Key" + let suffix = modifiers.isEmpty ? "" : ":\(modifiers.joined(separator: ","))" + print("KEY:\(action):\(encodedKey)\(suffix)") + fflush(stdout) +} + func mouseInteractionCallback( proxy: CGEventTapProxy, type: CGEventType, event: CGEvent, refcon: UnsafeMutableRawPointer? ) -> Unmanaged? { + if type == .keyDown { + emitKeyEvent(event, action: "down") + return Unmanaged.passUnretained(event) + } + if type == .keyUp { + return Unmanaged.passUnretained(event) + } + let action: String let button: Int switch type { @@ -448,7 +571,8 @@ func mouseInteractionCallback( return Unmanaged.passUnretained(event) } -let mouseEventTypes: [CGEventType] = [ +let captureKeys = CommandLine.arguments.contains("--capture-keys") +var mouseEventTypes: [CGEventType] = [ .leftMouseDown, .leftMouseUp, .rightMouseDown, @@ -456,6 +580,10 @@ let mouseEventTypes: [CGEventType] = [ .otherMouseDown, .otherMouseUp, ] +if captureKeys { + mouseEventTypes.append(.keyDown) + mouseEventTypes.append(.keyUp) +} let mouseEventMask = mouseEventTypes.reduce(CGEventMask(0)) { mask, type in mask | (CGEventMask(1) << type.rawValue) } diff --git a/electron/native/cursor-monitor/CMakeLists.txt b/electron/native/cursor-monitor/CMakeLists.txt index 8155aed95..caa1178ab 100644 --- a/electron/native/cursor-monitor/CMakeLists.txt +++ b/electron/native/cursor-monitor/CMakeLists.txt @@ -6,4 +6,4 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) add_executable(cursor-monitor src/main.cpp) -target_link_libraries(cursor-monitor PRIVATE user32) +target_link_libraries(cursor-monitor PRIVATE user32 ole32 oleaut32 UIAutomationCore) diff --git a/electron/native/cursor-monitor/src/main.cpp b/electron/native/cursor-monitor/src/main.cpp index 9a811f7be..d7d924d30 100644 --- a/electron/native/cursor-monitor/src/main.cpp +++ b/electron/native/cursor-monitor/src/main.cpp @@ -5,8 +5,16 @@ #include #include #include +#include +#include + +#include +#include static std::atomic g_running{true}; +static std::atomic g_captureKeys{false}; +static HHOOK g_keyboardHook = nullptr; +static std::unordered_set g_downKeys; static void stdinListener() { std::string line; @@ -19,8 +27,197 @@ static void stdinListener() { g_running.store(false); } -int main() { +static HWND focusedWin32Window() { + HWND foreground = GetForegroundWindow(); + if (!foreground) { + return nullptr; + } + + DWORD threadId = GetWindowThreadProcessId(foreground, nullptr); + GUITHREADINFO info = {}; + info.cbSize = sizeof(info); + HWND focus = foreground; + if (GetGUIThreadInfo(threadId, &info) && info.hwndFocus) { + focus = info.hwndFocus; + } + return focus; +} + +enum class FocusedFieldKind { + NonSecure, + Secure, + Unknown +}; + +static FocusedFieldKind win32FocusedPasswordState() { + HWND focus = focusedWin32Window(); + if (!focus) { + return FocusedFieldKind::Unknown; + } + + LONG_PTR style = GetWindowLongPtr(focus, GWL_STYLE); + if (style & ES_PASSWORD) { + return FocusedFieldKind::Secure; + } + + DWORD_PTR passwordChar = 0; + const LRESULT sent = SendMessageTimeoutW( + focus, + EM_GETPASSWORDCHAR, + 0, + 0, + SMTO_ABORTIFHUNG | SMTO_BLOCK, + 50, + &passwordChar + ); + if (sent == 0) { + return FocusedFieldKind::Secure; + } + if (passwordChar != 0) { + return FocusedFieldKind::Secure; + } + + wchar_t className[256] = {}; + if (GetClassNameW(focus, className, 256) > 0) { + std::wstring cls(className); + if (cls.find(L"Password") != std::wstring::npos) { + return FocusedFieldKind::Secure; + } + return FocusedFieldKind::NonSecure; + } + return FocusedFieldKind::Unknown; +} + +static FocusedFieldKind uiaFocusedPasswordState() { + IUIAutomation* automation = nullptr; + HRESULT created = CoCreateInstance( + CLSID_CUIAutomation, + nullptr, + CLSCTX_INPROC_SERVER, + IID_IUIAutomation, + reinterpret_cast(&automation) + ); + if (FAILED(created) || !automation) { + return FocusedFieldKind::Unknown; + } + + IUIAutomationElement* focused = nullptr; + HRESULT focusResult = automation->GetFocusedElement(&focused); + if (FAILED(focusResult) || !focused) { + automation->Release(); + return FocusedFieldKind::Unknown; + } + + BOOL password = FALSE; + HRESULT passwordResult = focused->get_CurrentIsPassword(&password); + focused->Release(); + automation->Release(); + if (FAILED(passwordResult)) { + return FocusedFieldKind::Unknown; + } + return password ? FocusedFieldKind::Secure : FocusedFieldKind::NonSecure; +} + +static FocusedFieldKind focusedPasswordState() { + const FocusedFieldKind win32State = win32FocusedPasswordState(); + if (win32State == FocusedFieldKind::Secure) { + return FocusedFieldKind::Secure; + } + + const FocusedFieldKind uiaState = uiaFocusedPasswordState(); + if (uiaState == FocusedFieldKind::Secure) { + return FocusedFieldKind::Secure; + } + if (win32State == FocusedFieldKind::NonSecure && uiaState == FocusedFieldKind::NonSecure) { + return FocusedFieldKind::NonSecure; + } + return FocusedFieldKind::Unknown; +} + +static std::string keyNameFromVk(DWORD vk) { + switch (vk) { + case VK_RETURN: return "Enter"; + case VK_ESCAPE: return "Escape"; + case VK_TAB: return "Tab"; + case VK_BACK: return "Backspace"; + case VK_DELETE: return "Delete"; + case VK_SPACE: return "Space"; + case VK_UP: return "ArrowUp"; + case VK_DOWN: return "ArrowDown"; + case VK_LEFT: return "ArrowLeft"; + case VK_RIGHT: return "ArrowRight"; + case VK_HOME: return "Home"; + case VK_END: return "End"; + case VK_PRIOR: return "PageUp"; + case VK_NEXT: return "PageDown"; + case VK_INSERT: return "Insert"; + case VK_SHIFT: + case VK_LSHIFT: + case VK_RSHIFT: return "Shift"; + case VK_CONTROL: + case VK_LCONTROL: + case VK_RCONTROL: return "Control"; + case VK_MENU: + case VK_LMENU: + case VK_RMENU: return "Alt"; + case VK_LWIN: + case VK_RWIN: return "Meta"; + default: + break; + } + if (vk >= VK_F1 && vk <= VK_F24) { + return "F" + std::to_string(vk - VK_F1 + 1); + } + if (vk >= 0x30 && vk <= 0x39) { + return std::string(1, static_cast(vk)); + } + if (vk >= 0x41 && vk <= 0x5A) { + return std::string(1, static_cast(vk)); + } + return "Key" + std::to_string(vk); +} + +static LRESULT CALLBACK keyboardProc(int nCode, WPARAM wParam, LPARAM lParam) { + if (nCode == HC_ACTION && g_captureKeys.load()) { + const auto* info = reinterpret_cast(lParam); + if (info) { + if (wParam == WM_KEYUP || wParam == WM_SYSKEYUP) { + g_downKeys.erase(info->vkCode); + } else if (wParam == WM_KEYDOWN || wParam == WM_SYSKEYDOWN) { + const bool repeat = !g_downKeys.insert(info->vkCode).second; + if (!repeat && focusedPasswordState() == FocusedFieldKind::NonSecure) { + std::vector modifiers; + if (GetAsyncKeyState(VK_CONTROL) & 0x8000) modifiers.emplace_back("ctrl"); + if (GetAsyncKeyState(VK_MENU) & 0x8000) modifiers.emplace_back("alt"); + if (GetAsyncKeyState(VK_SHIFT) & 0x8000) modifiers.emplace_back("shift"); + if ((GetAsyncKeyState(VK_LWIN) & 0x8000) || (GetAsyncKeyState(VK_RWIN) & 0x8000)) { + modifiers.emplace_back("meta"); + } + std::string suffix; + if (!modifiers.empty()) { + suffix = ":"; + for (size_t i = 0; i < modifiers.size(); ++i) { + if (i > 0) suffix += ","; + suffix += modifiers[i]; + } + } + std::cout << "KEY:down:" << keyNameFromVk(info->vkCode) << suffix << std::endl; + } + } + } + } + return CallNextHookEx(g_keyboardHook, nCode, wParam, lParam); +} + +int main(int argc, char** argv) { std::setvbuf(stdout, nullptr, _IONBF, 0); + CoInitializeEx(nullptr, COINIT_MULTITHREADED); + + for (int i = 1; i < argc; ++i) { + if (std::string(argv[i]) == "--capture-keys") { + g_captureKeys.store(true); + } + } std::unordered_map cursorMap; cursorMap[LoadCursor(NULL, IDC_ARROW)] = "arrow"; @@ -37,6 +234,13 @@ int main() { std::thread listener(stdinListener); listener.detach(); + if (g_captureKeys.load()) { + g_keyboardHook = SetWindowsHookExW(WH_KEYBOARD_LL, keyboardProc, GetModuleHandleW(nullptr), 0); + if (!g_keyboardHook) { + std::cerr << "Keyboard hook unavailable; keystroke overlay capture disabled" << std::endl; + } + } + std::string lastType; while (g_running.load()) { @@ -53,8 +257,19 @@ int main() { } } + MSG msg; + while (PeekMessageW(&msg, nullptr, 0, 0, PM_REMOVE)) { + TranslateMessage(&msg); + DispatchMessageW(&msg); + } + Sleep(50); } + if (g_keyboardHook) { + UnhookWindowsHookEx(g_keyboardHook); + g_keyboardHook = nullptr; + } + CoUninitialize(); return 0; } diff --git a/electron/preload.ts b/electron/preload.ts index 990ee7a8f..4c86b1c2a 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -614,6 +614,12 @@ contextBridge.exposeInMainWorld("electronAPI", { setCursorTelemetry: (videoPath: string | undefined, samples: CursorTelemetryPoint[]) => { return ipcRenderer.invoke("set-cursor-telemetry", videoPath, samples); }, + getKeystrokeTelemetry: (videoPath?: string) => { + return ipcRenderer.invoke("get-keystroke-telemetry", videoPath); + }, + setKeystrokeTelemetry: (videoPath: string | undefined, samples: KeystrokeSample[]) => { + return ipcRenderer.invoke("set-keystroke-telemetry", videoPath, samples); + }, getSystemCursorAssets: () => { return ipcRenderer.invoke("get-system-cursor-assets"); }, @@ -659,6 +665,12 @@ contextBridge.exposeInMainWorld("electronAPI", { requestAccessibilityPermission: () => { return ipcRenderer.invoke("request-accessibility-permission"); }, + requestKeystrokeCapturePermission: () => { + return ipcRenderer.invoke("request-keystroke-capture-permission"); + }, + stopKeystrokeTap: () => { + return ipcRenderer.invoke("stop-keystroke-tap"); + }, getScreenRecordingPermissionStatus: () => { return ipcRenderer.invoke("get-screen-recording-permission-status"); }, @@ -668,6 +680,9 @@ contextBridge.exposeInMainWorld("electronAPI", { openAccessibilityPreferences: () => { return ipcRenderer.invoke("open-accessibility-preferences"); }, + openInputMonitoringPreferences: () => { + return ipcRenderer.invoke("open-input-monitoring-preferences"); + }, saveExportedVideo: ( videoData: ArrayBuffer, fileName: string, diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index 66cbe608b..2316cbad1 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -112,8 +112,10 @@ function LaunchWindowContent() { platform, appVersion, hideHudFromCapture, + captureKeystrokes, chooseRecordingsDirectory, toggleHudCaptureProtection, + toggleCaptureKeystrokes, } = useLaunchWindowSystemState(preparePermissions); const hudCaptureProtectionSupported = supportsHudCaptureProtection(platform ?? ""); @@ -378,6 +380,10 @@ function LaunchWindowContent() { onToggleHudCaptureProtection={() => { void toggleHudCaptureProtection(); }} + captureKeystrokes={captureKeystrokes} + onToggleCaptureKeystrokes={() => { + void toggleCaptureKeystrokes(); + }} onChooseRecordingsDirectory={() => { void chooseRecordingsDirectory(); }} diff --git a/src/components/launch/hooks/useLaunchWindowSystemState.ts b/src/components/launch/hooks/useLaunchWindowSystemState.ts index 56793fa36..80217abe3 100644 --- a/src/components/launch/hooks/useLaunchWindowSystemState.ts +++ b/src/components/launch/hooks/useLaunchWindowSystemState.ts @@ -1,8 +1,12 @@ import { useCallback, useEffect, useState } from "react"; +import { useScopedT } from "@/contexts/I18nContext"; +import { loadAppSetting, saveAppSetting } from "../../../lib/appSettings"; +import { KEYSTROKE_OVERLAY_CAPTURE_SETTING } from "../../../lib/keystrokeOverlay"; export function useLaunchWindowSystemState( preparePermissions: (args: { startup?: boolean }) => Promise, ) { + const t = useScopedT("launch"); const [recordingsDirectory, setRecordingsDirectory] = useState(null); const [hudOverlayMousePassthroughSupported, setHudOverlayMousePassthroughSupported] = useState< boolean | null @@ -10,6 +14,9 @@ export function useLaunchWindowSystemState( const [platform, setPlatform] = useState(null); const [appVersion, setAppVersion] = useState(null); const [hideHudFromCapture, setHideHudFromCapture] = useState(true); + const [captureKeystrokes, setCaptureKeystrokes] = useState( + () => loadAppSetting(KEYSTROKE_OVERLAY_CAPTURE_SETTING) === true, + ); useEffect(() => { window.electronAPI?.hudOverlayRendererReady?.(); @@ -65,6 +72,32 @@ export function useLaunchWindowSystemState( }; }, []); + useEffect(() => { + let cancelled = false; + const syncKeystrokePermission = async () => { + if (!captureKeystrokes) { + return; + } + try { + const nextPlatform = platform ?? (await window.electronAPI.getPlatform()); + if (nextPlatform !== "darwin") { + return; + } + const status = await window.electronAPI.getAccessibilityPermissionStatus(); + if (!cancelled && status?.success && status.trusted === false) { + setCaptureKeystrokes(false); + saveAppSetting(KEYSTROKE_OVERLAY_CAPTURE_SETTING, false); + } + } catch { + // Keep the stored preference if permission status cannot be read. + } + }; + void syncKeystrokePermission(); + return () => { + cancelled = true; + }; + }, [captureKeystrokes, platform]); + useEffect(() => { void preparePermissions({ startup: true }); }, [preparePermissions]); @@ -129,6 +162,43 @@ export function useLaunchWindowSystemState( } }, [hideHudFromCapture]); + const toggleCaptureKeystrokes = useCallback(async () => { + if (captureKeystrokes) { + setCaptureKeystrokes(false); + saveAppSetting(KEYSTROKE_OVERLAY_CAPTURE_SETTING, false); + void window.electronAPI.stopKeystrokeTap?.(); + return; + } + + const nextPlatform = platform ?? (await window.electronAPI.getPlatform().catch(() => null)); + if (nextPlatform === "darwin") { + try { + const permission = await window.electronAPI.requestKeystrokeCapturePermission?.(); + if (!permission?.trusted || permission.tapOk === false) { + await window.electronAPI.openAccessibilityPreferences(); + await window.electronAPI.openInputMonitoringPreferences?.(); + const clientName = permission?.clientName || "Electron"; + alert( + t( + "recording.captureKeystrokesNeedAccessibility", + "System Settings opened. Under Privacy & Security, enable {{name}} in both Accessibility and Input Monitoring. Not Cursor. Then click Show keys in recording again.", + { name: clientName }, + ), + ); + return; + } + } catch (error) { + console.warn("Unable to request key overlay permissions:", error); + setCaptureKeystrokes(false); + saveAppSetting(KEYSTROKE_OVERLAY_CAPTURE_SETTING, false); + return; + } + } + + setCaptureKeystrokes(true); + saveAppSetting(KEYSTROKE_OVERLAY_CAPTURE_SETTING, true); + }, [captureKeystrokes, platform, t]); + return { recordingsDirectory, hudOverlayMousePassthroughSupported, @@ -136,7 +206,9 @@ export function useLaunchWindowSystemState( appVersion, hideHudFromCapture, setHideHudFromCapture, + captureKeystrokes, chooseRecordingsDirectory, toggleHudCaptureProtection, + toggleCaptureKeystrokes, }; } diff --git a/src/components/launch/popovers/MorePopover.tsx b/src/components/launch/popovers/MorePopover.tsx index 9a5a52905..2e6f70339 100644 --- a/src/components/launch/popovers/MorePopover.tsx +++ b/src/components/launch/popovers/MorePopover.tsx @@ -8,6 +8,7 @@ import { SunIcon, MoonIcon, DesktopIcon, + Keyboard, } from "@phosphor-icons/react"; import type { ReactElement } from "react"; import { useI18n } from "@/contexts/I18nContext"; @@ -38,6 +39,8 @@ export function MorePopover({ supportsHudCaptureProtection, hideHudFromCapture, onToggleHudCaptureProtection, + captureKeystrokes, + onToggleCaptureKeystrokes, onChooseRecordingsDirectory, onOpenVideoFile, onOpenProjectBrowser, @@ -49,6 +52,8 @@ export function MorePopover({ supportsHudCaptureProtection: boolean; hideHudFromCapture: boolean; onToggleHudCaptureProtection: () => void; + captureKeystrokes: boolean; + onToggleCaptureKeystrokes: () => void; onChooseRecordingsDirectory: () => void; onOpenVideoFile: () => void; onOpenProjectBrowser: () => void; @@ -86,6 +91,15 @@ export function MorePopover({ : t("recording.showHudInVideo")} )} + } + selected={captureKeystrokes} + onClick={onToggleCaptureKeystrokes} + > + {captureKeystrokes + ? t("recording.stopCapturingKeystrokes") + : t("recording.captureKeystrokes")} + } onClick={() => { diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 8b289e3f5..975c93ca5 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -56,6 +56,7 @@ import type { CursorStyle, EditorEffectSection, FigureData, + KeystrokeOverlaySettings, Padding, WebcamOverlaySettings, WebcamPositionPreset, @@ -77,6 +78,7 @@ import { DEFAULT_CURSOR_SIZE, DEFAULT_CURSOR_STYLE, DEFAULT_CURSOR_SWAY, + DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, DEFAULT_PADDING, DEFAULT_WEBCAM_MARGIN, DEFAULT_WEBCAM_POSITION_PRESET, @@ -603,6 +605,8 @@ interface SettingsPanelProps { onCursorClickBounceDurationChange?: (duration: number) => void; cursorSway?: number; onCursorSwayChange?: (amount: number) => void; + keystrokeOverlaySettings?: KeystrokeOverlaySettings; + onKeystrokeOverlaySettingsChange?: (settings: KeystrokeOverlaySettings) => void; borderRadius?: number; onBorderRadiusChange?: (radius: number) => void; webcam?: WebcamOverlaySettings; @@ -1051,6 +1055,8 @@ export function SettingsPanel({ onCursorClickBounceDurationChange, cursorSway = DEFAULT_CURSOR_SWAY, onCursorSwayChange, + keystrokeOverlaySettings = DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, + onKeystrokeOverlaySettingsChange, borderRadius = getDefaultBorderRadiusPercent(), onBorderRadiusChange, webcam, @@ -1585,6 +1591,7 @@ export function SettingsPanel({ onCursorClickBounceChange?.(initialEditorPreferences.cursorClickBounce); onCursorClickBounceDurationChange?.(initialEditorPreferences.cursorClickBounceDuration); onCursorSwayChange?.(initialEditorPreferences.cursorSway); + onKeystrokeOverlaySettingsChange?.(DEFAULT_KEYSTROKE_OVERLAY_SETTINGS); }; const activeMotionPresetId = useMemo(() => { @@ -3473,6 +3480,116 @@ export function SettingsPanel({ return parseFloat(text.replace(/×$/, "")); }} /> +
+
+
+ {tSettings("effects.keystrokeOverlay.title", "Key overlay")} +
+ + onKeystrokeOverlaySettingsChange?.({ + ...keystrokeOverlaySettings, + enabled, + }) + } + aria-label={tSettings( + "effects.keystrokeOverlay.title", + "Key overlay", + )} + className="data-[state=checked]:bg-[#2563EB] scale-75" + /> +
+
+
+ {tSettings( + "effects.keystrokeOverlay.mode", + "Shortcuts only", + )} +
+ + onKeystrokeOverlaySettingsChange?.({ + ...keystrokeOverlaySettings, + mode: shortcutsOnly ? "shortcuts" : "all", + }) + } + aria-label={tSettings( + "effects.keystrokeOverlay.mode", + "Shortcuts only", + )} + className="data-[state=checked]:bg-[#2563EB] scale-75" + /> +
+
+
+ {tSettings( + "effects.keystrokeOverlay.position", + "Position", + )} +
+ +
+ + onKeystrokeOverlaySettingsChange?.({ + ...keystrokeOverlaySettings, + fontSize, + }) + } + formatValue={(v) => `${Math.round(v)}`} + parseInput={(text) => parseFloat(text)} + /> + + onKeystrokeOverlaySettingsChange?.({ + ...keystrokeOverlaySettings, + bottomOffset, + }) + } + formatValue={(v) => `${Math.round(v)}%`} + parseInput={(text) => parseFloat(text.replace(/%$/, ""))} + /> +
{showDevMotionControls ? (
diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 842343dc3..4c4fe229f 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -12,6 +12,12 @@ import { useState, } from "react"; import { getAssetPath, getRenderableAssetUrl, getRenderableVideoUrl } from "@/lib/assetPath"; +import { + DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, + formatKeystrokeLabel, + getKeystrokeOverlayOpacity, + getVisibleKeystroke, +} from "@/lib/keystrokeOverlay"; import { getWebcamShadowFilter } from "@/lib/exporter/shadowProfile"; import { getSquircleSvgPath } from "@/lib/geometry/squircle"; import { @@ -255,6 +261,8 @@ interface VideoPlaybackProps { autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; onEditAutoCaption?: (target: CaptionEditTarget, text: string) => void; + keystrokeTelemetry?: import("@/lib/keystrokeOverlay").KeystrokeSample[]; + keystrokeOverlaySettings?: import("@/lib/keystrokeOverlay").KeystrokeOverlaySettings; selectedAnnotationId?: string | null; onSelectAnnotation?: (id: string | null) => void; onAnnotationPositionChange?: (id: string, position: { x: number; y: number }) => void; @@ -340,6 +348,8 @@ const VideoPlayback = forwardRef( autoCaptions = [], autoCaptionSettings, onEditAutoCaption, + keystrokeTelemetry = [], + keystrokeOverlaySettings, selectedAnnotationId, onSelectAnnotation, onAnnotationPositionChange, @@ -2668,6 +2678,54 @@ const VideoPlayback = forwardRef(
) : null} + {!isGap + ? (() => { + const overlaySettings = + keystrokeOverlaySettings ?? + DEFAULT_KEYSTROKE_OVERLAY_SETTINGS; + const timeMs = Math.round(currentTime * 1000); + const visible = getVisibleKeystroke( + keystrokeTelemetry, + timeMs, + overlaySettings, + ); + const opacity = getKeystrokeOverlayOpacity(visible, timeMs); + if (!visible || opacity <= 0) { + return null; + } + const isMac = + typeof navigator !== "undefined" && + /Mac|iPhone|iPad/.test(navigator.platform); + const label = formatKeystrokeLabel(visible, isMac); + const offset = `${overlaySettings.bottomOffset}%`; + return ( +
+
+ {label} +
+
+ ); + })() + : null}
{ it("migrates legacy pixels once and marks the stored unit", () => { @@ -390,6 +390,7 @@ describe("editorPreferences", () => { ...DEFAULT_EDITOR_PREFERENCES, cropRegion: DEFAULT_CROP_REGION, autoCaptionSettings: DEFAULT_AUTO_CAPTION_SETTINGS, + keystrokeOverlaySettings: DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, }, }, ]), @@ -418,6 +419,7 @@ describe("editorPreferences", () => { ...DEFAULT_EDITOR_PREFERENCES, cropRegion: DEFAULT_CROP_REGION, autoCaptionSettings: DEFAULT_AUTO_CAPTION_SETTINGS, + keystrokeOverlaySettings: DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, }, }, ]), @@ -450,6 +452,7 @@ describe("editorPreferences", () => { ...DEFAULT_EDITOR_PREFERENCES, cropRegion: DEFAULT_CROP_REGION, autoCaptionSettings: DEFAULT_AUTO_CAPTION_SETTINGS, + keystrokeOverlaySettings: DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, }, }, ]), @@ -478,6 +481,7 @@ describe("editorPreferences", () => { ...DEFAULT_EDITOR_PREFERENCES, cropRegion: { x: 0.08, y: 0.12, width: 0.8, height: 0.7 }, autoCaptionSettings: DEFAULT_AUTO_CAPTION_SETTINGS, + keystrokeOverlaySettings: DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, }, }, ]), diff --git a/src/components/video-editor/editorPreferences.ts b/src/components/video-editor/editorPreferences.ts index bca8b266c..3ce17b09b 100644 --- a/src/components/video-editor/editorPreferences.ts +++ b/src/components/video-editor/editorPreferences.ts @@ -72,6 +72,7 @@ export interface EditorPresetSnapshot extends Omit; + pendingFreshRecordingAutoZoomPathRef: MutableRefObject; + autoSuggestedVideoPathRef: MutableRefObject; +}; + +export function useKeystrokeTelemetry({ + videoPath, + videoSourcePath, + timeline, + pendingFreshRecordingAutoZoomPathRef, + autoSuggestedVideoPathRef, +}: Input) { + const pendingRetryTimeoutRef = useRef(null); + const { setKeystrokeTelemetry } = timeline; + + useEffect(() => { + let mounted = true; + let retryAttempts = 0; + const scheduleRetry = () => { + if ( + pendingFreshRecordingAutoZoomPathRef.current !== videoPath || + autoSuggestedVideoPathRef.current === videoPath || + retryAttempts >= 12 + ) { + return; + } + retryAttempts += 1; + pendingRetryTimeoutRef.current = window.setTimeout(() => { + pendingRetryTimeoutRef.current = null; + if (mounted) void load(); + }, 350); + }; + async function load() { + if (!videoPath || !videoSourcePath) { + if (mounted) { + setKeystrokeTelemetry([]); + } + return; + } + setKeystrokeTelemetry([]); + try { + const result = await window.electronAPI.getKeystrokeTelemetry?.(videoSourcePath); + if (!mounted) return; + setKeystrokeTelemetry(result?.success ? result.samples : []); + if (!result?.success || result.samples.length === 0) scheduleRetry(); + } catch (error) { + console.warn("Unable to load keystroke telemetry:", error); + if (!mounted) return; + setKeystrokeTelemetry([]); + scheduleRetry(); + } + } + + if (pendingRetryTimeoutRef.current !== null) { + window.clearTimeout(pendingRetryTimeoutRef.current); + pendingRetryTimeoutRef.current = null; + } + void load(); + return () => { + mounted = false; + if (pendingRetryTimeoutRef.current !== null) { + window.clearTimeout(pendingRetryTimeoutRef.current); + pendingRetryTimeoutRef.current = null; + } + }; + }, [ + videoPath, + videoSourcePath, + setKeystrokeTelemetry, + pendingFreshRecordingAutoZoomPathRef, + autoSuggestedVideoPathRef, + ]); +} diff --git a/src/components/video-editor/hooks/useTimelineEditingController.ts b/src/components/video-editor/hooks/useTimelineEditingController.ts index 083340f9a..80dc9bbe6 100644 --- a/src/components/video-editor/hooks/useTimelineEditingController.ts +++ b/src/components/video-editor/hooks/useTimelineEditingController.ts @@ -18,6 +18,7 @@ import { useCursorTelemetry } from "./useCursorTelemetry"; import { useEditorGlobalInteractions } from "./useEditorGlobalInteractions"; import { useEditorPlaybackControls } from "./useEditorPlaybackControls"; import { useFreshRecordingAutoZoom } from "./useFreshRecordingAutoZoom"; +import { useKeystrokeTelemetry } from "./useKeystrokeTelemetry"; import { useTimelineProjection } from "./useTimelineProjection"; import { useZoomRegionCommands } from "./useZoomRegionCommands"; @@ -74,6 +75,13 @@ export function useTimelineEditingController(input: Input) { pendingFreshRecordingAutoZoomPathRef: input.pendingFreshRecordingAutoZoomPathRef, autoSuggestedVideoPathRef: input.autoSuggestedVideoPathRef, }); + useKeystrokeTelemetry({ + videoPath: input.videoPath, + videoSourcePath: input.videoSourcePath, + timeline, + pendingFreshRecordingAutoZoomPathRef: input.pendingFreshRecordingAutoZoomPathRef, + autoSuggestedVideoPathRef: input.autoSuggestedVideoPathRef, + }); const projection = useTimelineProjection({ timeline, duration: input.duration, diff --git a/src/components/video-editor/layout/EditorVideoPreview.tsx b/src/components/video-editor/layout/EditorVideoPreview.tsx index ae0fe3e96..ba7f5fd4c 100644 --- a/src/components/video-editor/layout/EditorVideoPreview.tsx +++ b/src/components/video-editor/layout/EditorVideoPreview.tsx @@ -101,6 +101,8 @@ export function EditorVideoPreview({ annotationRegions={timeline.annotationRegions} autoCaptions={timeline.autoCaptions} autoCaptionSettings={timeline.autoCaptionSettings} + keystrokeTelemetry={timeline.keystrokeTelemetry} + keystrokeOverlaySettings={timeline.keystrokeOverlaySettings} selectedAnnotationId={timeline.selectedAnnotationId} cursorTelemetry={effectiveCursorTelemetry} showCursor={effectiveShowCursor} diff --git a/src/components/video-editor/layout/useEditorSettingsPanelProps.ts b/src/components/video-editor/layout/useEditorSettingsPanelProps.ts index 5ea46b498..4ff3886a3 100644 --- a/src/components/video-editor/layout/useEditorSettingsPanelProps.ts +++ b/src/components/video-editor/layout/useEditorSettingsPanelProps.ts @@ -176,6 +176,8 @@ export function useEditorSettingsPanelProps(input: Input): ComponentProps; @@ -958,6 +962,10 @@ export function normalizeProjectEditor(editor: Partial): Pro audioRegions: normalizedAudioRegions, autoCaptions: normalizedAutoCaptions, autoCaptionSettings: normalizedAutoCaptionSettings, + keystrokeOverlaySettings: normalizeKeystrokeOverlaySettings( + (editor as Partial).keystrokeOverlaySettings, + DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, + ), webcam: { enabled: typeof webcam.enabled === "boolean" diff --git a/src/components/video-editor/state/useTimelineState.ts b/src/components/video-editor/state/useTimelineState.ts index 155d9a54f..1aee9865e 100644 --- a/src/components/video-editor/state/useTimelineState.ts +++ b/src/components/video-editor/state/useTimelineState.ts @@ -7,16 +7,21 @@ import type { CaptionCue, ClipRegion, CursorTelemetryPoint, + KeystrokeOverlaySettings, + KeystrokeSample, SpeedRegion, TrimRegion, ZoomRegion, } from "../types"; -import { DEFAULT_AUTO_CAPTION_SETTINGS } from "../types"; +import { DEFAULT_AUTO_CAPTION_SETTINGS, DEFAULT_KEYSTROKE_OVERLAY_SETTINGS } from "../types"; export function useTimelineState() { const [zoomRegions, setZoomRegions] = useState([]); const [cursorTelemetry, setCursorTelemetry] = useState([]); const [cursorTelemetrySourcePath, setCursorTelemetrySourcePath] = useState(null); + const [keystrokeTelemetry, setKeystrokeTelemetry] = useState([]); + const [keystrokeOverlaySettings, setKeystrokeOverlaySettings] = + useState(DEFAULT_KEYSTROKE_OVERLAY_SETTINGS); const [selectedZoomId, setSelectedZoomId] = useState(null); const [trimRegions, setTrimRegions] = useState([]); const [clipRegions, setClipRegions] = useState([]); @@ -46,6 +51,10 @@ export function useTimelineState() { setCursorTelemetry, cursorTelemetrySourcePath, setCursorTelemetrySourcePath, + keystrokeTelemetry, + setKeystrokeTelemetry, + keystrokeOverlaySettings, + setKeystrokeOverlaySettings, selectedZoomId, setSelectedZoomId, trimRegions, diff --git a/src/components/video-editor/types.ts b/src/components/video-editor/types.ts index d5dc8714f..a903521d9 100644 --- a/src/components/video-editor/types.ts +++ b/src/components/video-editor/types.ts @@ -611,6 +611,17 @@ export const DEFAULT_AUTO_CAPTION_SETTINGS: AutoCaptionSettings = { backgroundOpacity: 0.9, }; +export type { + KeystrokeOverlayMode, + KeystrokeOverlayPosition, + KeystrokeOverlaySettings, + KeystrokeSample, +} from "@/lib/keystrokeOverlay"; +export { + DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, + normalizeKeystrokeOverlaySettings, +} from "@/lib/keystrokeOverlay"; + export type PlaybackSpeed = 0.25 | 0.5 | 0.75 | 1.25 | 1.5 | 1.75 | 2; export interface SpeedRegion { diff --git a/src/i18n/locales/de/launch.json b/src/i18n/locales/de/launch.json index bae512379..dc7f7cfb2 100644 --- a/src/i18n/locales/de/launch.json +++ b/src/i18n/locales/de/launch.json @@ -17,6 +17,9 @@ "openProject": "Projekt öffnen", "hideHudFromVideo": "HUD in der Aufnahme ausblenden", "showHudInVideo": "HUD in der Aufnahme anzeigen", + "captureKeystrokes": "Tasten in der Aufnahme anzeigen", + "stopCapturingKeystrokes": "Tasten in der Aufnahme ausblenden", + "captureKeystrokesNeedAccessibility": "Die Systemeinstellungen wurden geöffnet. Aktivieren Sie unter Datenschutz & Sicherheit {{name}} sowohl bei Bedienungshilfen als auch bei Eingabeüberwachung. Nicht Cursor. Klicken Sie danach erneut auf Tasten in der Aufnahme anzeigen.", "hideHud": "HUD ausblenden", "closeApp": "App schließen", "screens": "Bildschirme", diff --git a/src/i18n/locales/de/settings.json b/src/i18n/locales/de/settings.json index d0837fae2..cbe9a0d28 100644 --- a/src/i18n/locales/de/settings.json +++ b/src/i18n/locales/de/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "Cursor-Klick-Rückprall", "cursorClickBounceDuration": "Sprunggeschwindigkeit", "cursorSway": "Cursor-Schwankung", + "keystrokeOverlay": { + "title": "Tasten-Overlay", + "mode": "Nur Tastenkürzel", + "position": "Position", + "bottom": "Unten", + "top": "Oben", + "size": "Größe", + "offset": "Abstand" + }, "webcam": "Webcam-Overlay", "webcamFootage": "Webcam-Aufnahmen", "webcamFootageDescription": "Zu diesem Video sind keine Webcam-Aufnahmen verknüpft", diff --git a/src/i18n/locales/en/launch.json b/src/i18n/locales/en/launch.json index d4f7aba94..d6db25928 100644 --- a/src/i18n/locales/en/launch.json +++ b/src/i18n/locales/en/launch.json @@ -17,6 +17,9 @@ "openProject": "Open project", "hideHudFromVideo": "Hide HUD from recording", "showHudInVideo": "Show HUD in recording", + "captureKeystrokes": "Show keys in recording", + "stopCapturingKeystrokes": "Hide keys from recording", + "captureKeystrokesNeedAccessibility": "System Settings opened. Under Privacy & Security, enable {{name}} in both Accessibility and Input Monitoring. Not Cursor. Then click Show keys in recording again.", "hideHud": "Hide HUD", "closeApp": "Close App", "screens": "Screens", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 596109cb3..4a37995d5 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -130,6 +130,15 @@ "cursorClickBounce": "Cursor Click Bounce", "cursorClickBounceDuration": "Bounce Speed", "cursorSway": "Cursor Sway", + "keystrokeOverlay": { + "title": "Key overlay", + "mode": "Shortcuts only", + "position": "Position", + "bottom": "Bottom", + "top": "Top", + "size": "Size", + "offset": "Offset" + }, "webcam": "Webcam Overlay", "webcamFootage": "Webcam Footage", "webcamFootageDescription": "No webcam footage linked to this video", diff --git a/src/i18n/locales/es/launch.json b/src/i18n/locales/es/launch.json index 4edc2d9c6..f728222cf 100644 --- a/src/i18n/locales/es/launch.json +++ b/src/i18n/locales/es/launch.json @@ -17,6 +17,9 @@ "openProject": "Abrir proyecto", "hideHudFromVideo": "Ocultar HUD de la grabación", "showHudInVideo": "Mostrar HUD en la grabación", + "captureKeystrokes": "Mostrar teclas en la grabación", + "stopCapturingKeystrokes": "Ocultar teclas de la grabación", + "captureKeystrokesNeedAccessibility": "Se abrieron Ajustes del Sistema. En Privacidad y seguridad, activa {{name}} tanto en Accesibilidad como en Monitorización de entrada. No Cursor. Luego vuelve a hacer clic en Mostrar teclas en la grabación.", "hideHud": "Ocultar HUD", "closeApp": "Cerrar aplicación", "screens": "Pantallas", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 3510d6bbb..4050d8c61 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "Rebote de clic del cursor", "cursorClickBounceDuration": "Velocidad del rebote", "cursorSway": "Balanceo del cursor", + "keystrokeOverlay": { + "title": "Superposición de teclas", + "mode": "Solo atajos", + "position": "Posición", + "bottom": "Abajo", + "top": "Arriba", + "size": "Tamaño", + "offset": "Desplazamiento" + }, "webcam": "Superposición de cámara", "webcamFootage": "Metraje de cámara", "webcamFootageDescription": "No hay metraje de cámara vinculado a este video", diff --git a/src/i18n/locales/fr/launch.json b/src/i18n/locales/fr/launch.json index 2969355fa..356543401 100644 --- a/src/i18n/locales/fr/launch.json +++ b/src/i18n/locales/fr/launch.json @@ -17,6 +17,9 @@ "openProject": "Ouvrir le projet", "hideHudFromVideo": "Masquer le HUD dans l’enregistrement", "showHudInVideo": "Afficher le HUD dans l’enregistrement", + "captureKeystrokes": "Afficher les touches dans l’enregistrement", + "stopCapturingKeystrokes": "Masquer les touches de l’enregistrement", + "captureKeystrokesNeedAccessibility": "Les Réglages Système ont été ouverts. Dans Confidentialité et sécurité, activez {{name}} à la fois dans Accessibilité et dans Surveillance des entrées. Pas Cursor. Cliquez ensuite de nouveau sur Afficher les touches dans l’enregistrement.", "hideHud": "Masquer le HUD", "closeApp": "Fermer l’application", "screens": "Écrans", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index d7e4e6753..2e67556a9 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "Rebond au clic du curseur", "cursorClickBounceDuration": "Vitesse du rebond", "cursorSway": "Oscillation du curseur", + "keystrokeOverlay": { + "title": "Superposition des touches", + "mode": "Raccourcis uniquement", + "position": "Position", + "bottom": "Bas", + "top": "Haut", + "size": "Taille", + "offset": "Décalage" + }, "webcam": "Incrustation webcam", "webcamFootage": "Vidéo de la webcam", "webcamFootageDescription": "Aucune vidéo de webcam liée à cette vidéo", diff --git a/src/i18n/locales/it/launch.json b/src/i18n/locales/it/launch.json index 0dbdbea34..b6e42b665 100644 --- a/src/i18n/locales/it/launch.json +++ b/src/i18n/locales/it/launch.json @@ -17,6 +17,9 @@ "openProject": "Apri progetto", "hideHudFromVideo": "Nascondi HUD dalla registrazione", "showHudInVideo": "Mostra HUD nella registrazione", + "captureKeystrokes": "Mostra i tasti nella registrazione", + "stopCapturingKeystrokes": "Nascondi i tasti dalla registrazione", + "captureKeystrokesNeedAccessibility": "Sono state aperte Impostazioni di Sistema. In Privacy e sicurezza, abilita {{name}} sia in Accessibilità sia in Monitoraggio input. Non Cursor. Poi fai di nuovo clic su Mostra i tasti nella registrazione.", "hideHud": "Nascondi HUD", "closeApp": "Chiudi app", "screens": "Schermi", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index 8027ac02c..9c4f6a04f 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "Rimbalzo al clic del cursore", "cursorClickBounceDuration": "Velocità rimbalzo", "cursorSway": "Oscillazione cursore", + "keystrokeOverlay": { + "title": "Overlay tasti", + "mode": "Solo scorciatoie", + "position": "Posizione", + "bottom": "In basso", + "top": "In alto", + "size": "Dimensione", + "offset": "Offset" + }, "webcam": "Overlay webcam", "webcamFootage": "Filmato webcam", "webcamFootageDescription": "Nessun filmato webcam collegato a questo video", diff --git a/src/i18n/locales/ko/launch.json b/src/i18n/locales/ko/launch.json index 2e4683a56..82eb0846c 100644 --- a/src/i18n/locales/ko/launch.json +++ b/src/i18n/locales/ko/launch.json @@ -17,6 +17,9 @@ "openProject": "프로젝트 열기", "hideHudFromVideo": "녹화에서 HUD 숨기기", "showHudInVideo": "녹화에 HUD 표시", + "captureKeystrokes": "녹화에 키 표시", + "stopCapturingKeystrokes": "녹화에서 키 숨기기", + "captureKeystrokesNeedAccessibility": "시스템 설정이 열렸습니다. 개인 정보 보호 및 보안에서 손쉬운 사용과 입력 모니터링 모두에 {{name}}을(를) 허용하세요. Cursor가 아닙니다. 그런 다음 녹화에 키 표시를 다시 클릭하세요.", "hideHud": "HUD 숨기기", "closeApp": "앱 닫기", "screens": "화면", diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json index b2063381b..8e8a6ef28 100644 --- a/src/i18n/locales/ko/settings.json +++ b/src/i18n/locales/ko/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "커서 클릭 바운스", "cursorClickBounceDuration": "바운스 속도", "cursorSway": "커서 흔들림", + "keystrokeOverlay": { + "title": "키 오버레이", + "mode": "단축키만", + "position": "위치", + "bottom": "아래", + "top": "위", + "size": "크기", + "offset": "간격" + }, "webcam": "웹캠 오버레이", "webcamFootage": "웹캠 영상", "webcamFootageDescription": "이 비디오에 연결된 웹캠 영상이 없습니다", diff --git a/src/i18n/locales/nl/launch.json b/src/i18n/locales/nl/launch.json index 774c7335a..ce970cd37 100644 --- a/src/i18n/locales/nl/launch.json +++ b/src/i18n/locales/nl/launch.json @@ -17,6 +17,9 @@ "openProject": "Project openen", "hideHudFromVideo": "HUD verbergen in opname", "showHudInVideo": "HUD tonen in opname", + "captureKeystrokes": "Toetsen in opname tonen", + "stopCapturingKeystrokes": "Toetsen in opname verbergen", + "captureKeystrokesNeedAccessibility": "Systeeminstellingen is geopend. Schakel onder Privacy en beveiliging {{name}} in bij zowel Toegankelijkheid als Invoermonitoring. Niet Cursor. Klik daarna opnieuw op Toetsen in opname tonen.", "hideHud": "HUD verbergen", "closeApp": "App sluiten", "screens": "Schermen", diff --git a/src/i18n/locales/nl/settings.json b/src/i18n/locales/nl/settings.json index 4fda91805..74a9e627c 100644 --- a/src/i18n/locales/nl/settings.json +++ b/src/i18n/locales/nl/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "Cursorklikstuit", "cursorClickBounceDuration": "Stuitsnelheid", "cursorSway": "Cursorbeweging", + "keystrokeOverlay": { + "title": "Toetsenoverlay", + "mode": "Alleen sneltoetsen", + "position": "Positie", + "bottom": "Onder", + "top": "Boven", + "size": "Grootte", + "offset": "Verschuiving" + }, "webcam": "Webcam-overlay", "webcamFootage": "Webcambeelden", "webcamFootageDescription": "Geen webcambeelden gekoppeld aan deze video", diff --git a/src/i18n/locales/pt-BR/launch.json b/src/i18n/locales/pt-BR/launch.json index cc8c35c7e..e9c9bdbe1 100644 --- a/src/i18n/locales/pt-BR/launch.json +++ b/src/i18n/locales/pt-BR/launch.json @@ -17,6 +17,9 @@ "openProject": "Abrir projeto", "hideHudFromVideo": "Ocultar HUD na gravação", "showHudInVideo": "Mostrar HUD na gravação", + "captureKeystrokes": "Mostrar teclas na gravação", + "stopCapturingKeystrokes": "Ocultar teclas da gravação", + "captureKeystrokesNeedAccessibility": "Os Ajustes do Sistema foram abertos. Em Privacidade e segurança, ative {{name}} em Acessibilidade e em Monitoramento de entrada. Não o Cursor. Depois clique de novo em Mostrar teclas na gravação.", "hideHud": "Ocultar HUD", "closeApp": "Fechar app", "screens": "Telas", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index 1cb1b72ee..cca0ec02a 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "Salto de clique do cursor", "cursorClickBounceDuration": "Velocidade do salto", "cursorSway": "Balanço do cursor", + "keystrokeOverlay": { + "title": "Sobreposição de teclas", + "mode": "Somente atalhos", + "position": "Posição", + "bottom": "Inferior", + "top": "Superior", + "size": "Tamanho", + "offset": "Deslocamento" + }, "webcam": "Sobreposição da webcam", "webcamFootage": "Filmagem da webcam", "webcamFootageDescription": "Nenhuma filmagem de webcam vinculada a este vídeo", diff --git a/src/i18n/locales/ru/launch.json b/src/i18n/locales/ru/launch.json index 47531cfa1..03925fbe2 100644 --- a/src/i18n/locales/ru/launch.json +++ b/src/i18n/locales/ru/launch.json @@ -17,6 +17,9 @@ "openProject": "Открыть проект", "hideHudFromVideo": "Скрыть интерфейс во время записи", "showHudInVideo": "Показать интерфейс во время записи", + "captureKeystrokes": "Показывать клавиши в записи", + "stopCapturingKeystrokes": "Скрыть клавиши в записи", + "captureKeystrokesNeedAccessibility": "Открыты системные настройки. В разделе «Конфиденциальность и безопасность» включите {{name}} и в Универсальном доступе, и в мониторинге ввода. Не Cursor. Затем снова нажмите «Показывать клавиши в записи».", "hideHud": "Скрыть интерфейс", "closeApp": "Закрыть приложение", "screens": "Экраны", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 7ec21ff1c..efdeef0bc 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "Отскок при клике", "cursorClickBounceDuration": "Скорость отскока", "cursorSway": "Покачивание курсора", + "keystrokeOverlay": { + "title": "Оверлей клавиш", + "mode": "Только сочетания", + "position": "Положение", + "bottom": "Снизу", + "top": "Сверху", + "size": "Размер", + "offset": "Отступ" + }, "webcam": "Наложение веб-камеры", "webcamFootage": "Запись с веб-камеры", "webcamFootageDescription": "Видео не добавлено", diff --git a/src/i18n/locales/zh-CN/launch.json b/src/i18n/locales/zh-CN/launch.json index 2b88c6d70..0ed40b205 100644 --- a/src/i18n/locales/zh-CN/launch.json +++ b/src/i18n/locales/zh-CN/launch.json @@ -17,6 +17,9 @@ "openProject": "打开项目", "hideHudFromVideo": "从录制中隐藏 HUD", "showHudInVideo": "在录制中显示 HUD", + "captureKeystrokes": "在录制中显示按键", + "stopCapturingKeystrokes": "在录制中隐藏按键", + "captureKeystrokesNeedAccessibility": "已打开系统设置。请在“隐私与安全性”中，于“辅助功能”和“输入监控”里同时启用 {{name}}。不是 Cursor。然后再次点击“在录制中显示按键”。", "hideHud": "隐藏 HUD", "closeApp": "关闭应用", "screens": "屏幕", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index 053d10951..3edf8e82d 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "光标点击弹跳", "cursorClickBounceDuration": "弹跳速度", "cursorSway": "光标摆动", + "keystrokeOverlay": { + "title": "按键叠加", + "mode": "仅快捷键", + "position": "位置", + "bottom": "底部", + "top": "顶部", + "size": "大小", + "offset": "偏移" + }, "webcam": "摄像头叠加", "webcamFootage": "摄像头素材", "webcamFootageDescription": "此视频尚未关联摄像头素材", diff --git a/src/i18n/locales/zh-TW/launch.json b/src/i18n/locales/zh-TW/launch.json index 9d44d3218..f397e9fa1 100644 --- a/src/i18n/locales/zh-TW/launch.json +++ b/src/i18n/locales/zh-TW/launch.json @@ -17,6 +17,9 @@ "openProject": "開啟專案", "hideHudFromVideo": "錄製時隱藏 HUD", "showHudInVideo": "錄製時顯示 HUD", + "captureKeystrokes": "在錄製中顯示按鍵", + "stopCapturingKeystrokes": "在錄製中隱藏按鍵", + "captureKeystrokesNeedAccessibility": "已開啟系統設定。請在「隱私權與安全性」中，於「輔助使用」和「輸入監控」同時啟用 {{name}}。不是 Cursor。然後再按一次「在錄製中顯示按鍵」。", "hideHud": "隱藏 HUD", "closeApp": "關閉應用程式", "screens": "螢幕", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index 7de885204..67d8a4560 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -124,6 +124,15 @@ "cursorClickBounce": "點擊彈跳", "cursorClickBounceDuration": "彈跳速度", "cursorSway": "游標擺動", + "keystrokeOverlay": { + "title": "按鍵疊加", + "mode": "僅快捷鍵", + "position": "位置", + "bottom": "底部", + "top": "頂部", + "size": "大小", + "offset": "偏移" + }, "webcam": "網路攝影機疊加", "webcamFootage": "網路攝影機素材", "webcamFootageDescription": "此影片尚未連結網路攝影機素材", diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 640bd39aa..c104f2c09 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -9,6 +9,8 @@ import type { CursorClickEffectStyle, CursorStyle, CursorTelemetryPoint, + KeystrokeOverlaySettings, + KeystrokeSample, Padding, SpeedRegion, WebcamOverlaySettings, @@ -73,6 +75,7 @@ import { import { isVideoWallpaperSource } from "@/lib/wallpapers"; import { renderAnnotations } from "./annotationRenderer"; import { renderCaptions } from "./captionRenderer"; +import { detectKeystrokeOverlayIsMac, renderKeystrokeOverlay } from "./keystrokeRenderer"; import { ForwardFrameSource } from "./forwardFrameSource"; import { resolveMediaElementSource } from "./localMediaSource"; @@ -108,6 +111,8 @@ interface FrameRenderConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: KeystrokeSample[]; + keystrokeOverlaySettings?: KeystrokeOverlaySettings; speedRegions?: SpeedRegion[]; previewWidth?: number; previewHeight?: number; @@ -249,6 +254,7 @@ export class FrameRenderer { private motionBlurState: MotionBlurState; private layoutCache: LayoutCache | null = null; private currentVideoTime = 0; + private readonly keystrokeOverlayIsMac = detectKeystrokeOverlayIsMac(); private springScale: SpringState; private springX: SpringState; private springY: SpringState; @@ -1528,6 +1534,18 @@ export class FrameRenderer { timestamp / 1000, ); } + + if (this.compositeCtx) { + renderKeystrokeOverlay( + this.compositeCtx, + this.config.keystrokeTelemetry ?? [], + this.config.keystrokeOverlaySettings, + this.config.width, + this.config.height, + timeMs, + this.keystrokeOverlayIsMac, + ); + } } private updateLayout(): void { diff --git a/src/lib/exporter/gifExporter.ts b/src/lib/exporter/gifExporter.ts index 2df730000..5ff93f83c 100644 --- a/src/lib/exporter/gifExporter.ts +++ b/src/lib/exporter/gifExporter.ts @@ -64,6 +64,8 @@ interface GifExporterConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: import("@/lib/keystrokeOverlay").KeystrokeSample[]; + keystrokeOverlaySettings?: import("@/lib/keystrokeOverlay").KeystrokeOverlaySettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; cursorStyle?: CursorStyle; @@ -165,6 +167,8 @@ export function buildGifFrameRendererConfig( annotationRegions: config.annotationRegions, autoCaptions: config.autoCaptions, autoCaptionSettings: config.autoCaptionSettings, + keystrokeTelemetry: config.keystrokeTelemetry, + keystrokeOverlaySettings: config.keystrokeOverlaySettings, speedRegions: config.speedRegions, previewWidth: config.previewWidth, previewHeight: config.previewHeight, diff --git a/src/lib/exporter/keystrokeRenderer.ts b/src/lib/exporter/keystrokeRenderer.ts new file mode 100644 index 000000000..2a546c837 --- /dev/null +++ b/src/lib/exporter/keystrokeRenderer.ts @@ -0,0 +1,66 @@ +import { + DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, + formatKeystrokeLabel, + getKeystrokeOverlayOpacity, + getVisibleKeystroke, + type KeystrokeOverlaySettings, + type KeystrokeSample, +} from "@/lib/keystrokeOverlay"; + +export function detectKeystrokeOverlayIsMac( + platformHint = + typeof navigator === "undefined" ? "" : `${navigator.platform} ${navigator.userAgent}`, +) { + return /mac|iphone|ipad|ipod/i.test(platformHint); +} + +export function renderKeystrokeOverlay( + ctx: CanvasRenderingContext2D, + samples: KeystrokeSample[], + settings: KeystrokeOverlaySettings | undefined, + width: number, + height: number, + timeMs: number, + isMac = false, +) { + const overlaySettings = settings ?? DEFAULT_KEYSTROKE_OVERLAY_SETTINGS; + const visible = getVisibleKeystroke(samples, timeMs, overlaySettings); + const opacity = getKeystrokeOverlayOpacity(visible, timeMs); + if (!visible || opacity <= 0) { + return; + } + + const label = formatKeystrokeLabel(visible, isMac); + const fontSize = Math.max(12, overlaySettings.fontSize * (width / 1920)); + ctx.save(); + ctx.globalAlpha = opacity; + ctx.font = `600 ${fontSize}px ui-sans-serif, system-ui, sans-serif`; + const paddingX = fontSize * 0.7; + const paddingY = fontSize * 0.42; + const metrics = ctx.measureText(label); + const boxWidth = metrics.width + paddingX * 2; + const boxHeight = fontSize + paddingY * 2; + const centerX = width / 2; + const offset = (overlaySettings.bottomOffset / 100) * height; + const centerY = + overlaySettings.position === "top" ? offset + boxHeight / 2 : height - offset - boxHeight / 2; + + ctx.fillStyle = "rgba(0, 0, 0, 0.82)"; + const x = centerX - boxWidth / 2; + const y = centerY - boxHeight / 2; + const radius = Math.min(12, boxHeight / 2); + ctx.beginPath(); + ctx.moveTo(x + radius, y); + ctx.arcTo(x + boxWidth, y, x + boxWidth, y + boxHeight, radius); + ctx.arcTo(x + boxWidth, y + boxHeight, x, y + boxHeight, radius); + ctx.arcTo(x, y + boxHeight, x, y, radius); + ctx.arcTo(x, y, x + boxWidth, y, radius); + ctx.closePath(); + ctx.fill(); + + ctx.fillStyle = "#ffffff"; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + ctx.fillText(label, centerX, centerY); + ctx.restore(); +} diff --git a/src/lib/exporter/modernFrameRenderer.ts b/src/lib/exporter/modernFrameRenderer.ts index 629f25210..0347ecb5a 100644 --- a/src/lib/exporter/modernFrameRenderer.ts +++ b/src/lib/exporter/modernFrameRenderer.ts @@ -19,6 +19,8 @@ import type { CursorClickEffectStyle, CursorStyle, CursorTelemetryPoint, + KeystrokeOverlaySettings, + KeystrokeSample, Padding, SpeedRegion, WebcamOverlaySettings, @@ -87,6 +89,7 @@ import { renderAnnotationToCanvas, } from "./annotationRenderer"; import { ForwardFrameSource } from "./forwardFrameSource"; +import { detectKeystrokeOverlayIsMac, renderKeystrokeOverlay } from "./keystrokeRenderer"; import { resolveMediaElementSource } from "./localMediaSource"; import { getShadowFilterPadding, @@ -127,6 +130,8 @@ interface FrameRenderConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: KeystrokeSample[]; + keystrokeOverlaySettings?: KeystrokeOverlaySettings; speedRegions?: SpeedRegion[]; previewWidth?: number; previewHeight?: number; @@ -428,6 +433,7 @@ export class FrameRenderer { private lastContentTimeMs: number | null = null; private layoutCache: LayoutCache | null = null; private currentVideoTime = 0; + private readonly keystrokeOverlayIsMac = detectKeystrokeOverlayIsMac(); private cursorOverlay: PixiCursorOverlay | null = null; private lastSyncedWebcamTime: number | null = null; private webcamRenderMode: "hidden" | "live" | "cached" = "hidden"; @@ -1510,6 +1516,15 @@ export class FrameRenderer { ); this.drawCaptionOverlay(context); + renderKeystrokeOverlay( + context, + this.config.keystrokeTelemetry ?? [], + this.config.keystrokeOverlaySettings, + this.config.width, + this.config.height, + this.currentVideoTime * 1000, + this.keystrokeOverlayIsMac, + ); this.outputCanvasOverride = canvas; } @@ -2982,8 +2997,31 @@ export class FrameRenderer { return; } - this.outputCanvasOverride = null; this.app!.render(); + if ( + this.config.keystrokeOverlaySettings?.enabled && + (this.config.keystrokeTelemetry?.length ?? 0) > 0 + ) { + const compositeState = this.ensureExportCompositeCanvas(); + if (compositeState) { + const { canvas, context } = compositeState; + context.clearRect(0, 0, canvas.width, canvas.height); + context.drawImage(this.app!.canvas as HTMLCanvasElement, 0, 0); + renderKeystrokeOverlay( + context, + this.config.keystrokeTelemetry ?? [], + this.config.keystrokeOverlaySettings, + this.config.width, + this.config.height, + this.currentVideoTime * 1000, + this.keystrokeOverlayIsMac, + ); + this.outputCanvasOverride = canvas; + return; + } + } + + this.outputCanvasOverride = null; } private updateLayout(): void { diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 81137b267..29f4c1feb 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -122,6 +122,8 @@ interface VideoExporterConfig extends ExportConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: import("@/lib/keystrokeOverlay").KeystrokeSample[]; + keystrokeOverlaySettings?: import("@/lib/keystrokeOverlay").KeystrokeOverlaySettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; cursorStyle?: CursorStyle; @@ -631,6 +633,8 @@ export class ModernVideoExporter { annotationRegions: this.config.annotationRegions, autoCaptions: this.config.autoCaptions, autoCaptionSettings: this.config.autoCaptionSettings, + keystrokeTelemetry: this.config.keystrokeTelemetry, + keystrokeOverlaySettings: this.config.keystrokeOverlaySettings, speedRegions: this.config.speedRegions, previewWidth: this.config.previewWidth, previewHeight: this.config.previewHeight, @@ -1746,6 +1750,12 @@ export class ModernVideoExporter { if ((this.config.autoCaptions ?? []).length > 0) { reasons.push("unsupported-caption-overlay"); } + if ( + this.config.keystrokeOverlaySettings?.enabled && + (this.config.keystrokeTelemetry?.length ?? 0) > 0 + ) { + reasons.push("unsupported-keystroke-overlay"); + } if (this.config.webcam?.enabled) { // Native GPU compositors use a different corner and shadow model. // Keep webcam exports on the shared renderer used by preview. diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index d58642c72..912a4e738 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -73,6 +73,8 @@ interface VideoExporterConfig extends ExportConfig { annotationRegions?: AnnotationRegion[]; autoCaptions?: CaptionCue[]; autoCaptionSettings?: AutoCaptionSettings; + keystrokeTelemetry?: import("@/lib/keystrokeOverlay").KeystrokeSample[]; + keystrokeOverlaySettings?: import("@/lib/keystrokeOverlay").KeystrokeOverlaySettings; cursorTelemetry?: CursorTelemetryPoint[]; showCursor?: boolean; cursorStyle?: CursorStyle; @@ -244,6 +246,8 @@ export class VideoExporter { annotationRegions: this.config.annotationRegions, autoCaptions: this.config.autoCaptions, autoCaptionSettings: this.config.autoCaptionSettings, + keystrokeTelemetry: this.config.keystrokeTelemetry, + keystrokeOverlaySettings: this.config.keystrokeOverlaySettings, speedRegions: this.config.speedRegions, previewWidth: this.config.previewWidth, previewHeight: this.config.previewHeight, diff --git a/src/lib/keystrokeOverlay.test.ts b/src/lib/keystrokeOverlay.test.ts new file mode 100644 index 000000000..a61582cd9 --- /dev/null +++ b/src/lib/keystrokeOverlay.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from "vitest"; +import { + filterKeystrokesForDisplay, + formatKeystrokeLabel, + getVisibleKeystroke, + normalizeKeystrokeSamples, + parseKeyMonitorLine, + shouldStoreCapturedKeystroke, + type KeystrokeSample, + DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, +} from "./keystrokeOverlay"; + +function sample(partial: Partial): KeystrokeSample { + return { + timeMs: 0, + key: "C", + code: "C", + ctrl: false, + alt: false, + shift: false, + meta: false, + ...partial, + }; +} + +describe("keystroke overlay helpers", () => { + it("normalizes sidecar samples and drops empty keys", () => { + expect( + normalizeKeystrokeSamples({ + samples: [ + { timeMs: 40, key: "C", ctrl: true }, + { timeMs: "bad", key: "" }, + { timeMs: 10, code: "Enter" }, + ], + }), + ).toEqual([ + { + timeMs: 10, + key: "Enter", + code: "Enter", + ctrl: false, + alt: false, + shift: false, + meta: false, + repeat: undefined, + }, + { + timeMs: 40, + key: "C", + code: "C", + ctrl: true, + alt: false, + shift: false, + meta: false, + repeat: undefined, + }, + ]); + }); + + it("never stores password-field keystrokes", () => { + expect( + shouldStoreCapturedKeystroke(sample({ key: "a" }), { + platform: "darwin", + isPasswordField: true, + }), + ).toBe(false); + expect( + shouldStoreCapturedKeystroke(sample({ key: "C", ctrl: true }), { + platform: "linux", + isPasswordField: "unknown", + }), + ).toBe(false); + }); + + it("stores shortcuts only on Linux", () => { + expect( + shouldStoreCapturedKeystroke(sample({ key: "a" }), { + platform: "linux", + isPasswordField: false, + }), + ).toBe(false); + expect( + shouldStoreCapturedKeystroke(sample({ key: "C", ctrl: true }), { + platform: "linux", + isPasswordField: false, + }), + ).toBe(true); + }); + + it("skips key-repeat and modifier-only events", () => { + expect( + shouldStoreCapturedKeystroke(sample({ key: "A", repeat: true }), { + isPasswordField: false, + }), + ).toBe(false); + expect( + shouldStoreCapturedKeystroke(sample({ key: "Shift", shift: true }), { + isPasswordField: false, + }), + ).toBe(false); + }); + + it("filters display to shortcuts by default", () => { + const samples = [ + sample({ timeMs: 100, key: "a" }), + sample({ timeMs: 200, key: "C", ctrl: true }), + sample({ timeMs: 300, key: "Enter" }), + ]; + expect(filterKeystrokesForDisplay(samples, DEFAULT_KEYSTROKE_OVERLAY_SETTINGS)).toEqual([ + samples[1], + samples[2], + ]); + expect( + filterKeystrokesForDisplay(samples, { + ...DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, + mode: "all", + }), + ).toEqual(samples); + }); + + it("returns the latest visible overlay label", () => { + const samples = [ + sample({ timeMs: 100, key: "Enter" }), + sample({ timeMs: 400, key: "C", ctrl: true }), + ]; + expect(getVisibleKeystroke(samples, 150, DEFAULT_KEYSTROKE_OVERLAY_SETTINGS)?.key).toBe( + "Enter", + ); + expect(getVisibleKeystroke(samples, 450, DEFAULT_KEYSTROKE_OVERLAY_SETTINGS)?.key).toBe("C"); + expect(getVisibleKeystroke(samples, 2500, DEFAULT_KEYSTROKE_OVERLAY_SETTINGS)).toBeNull(); + expect(getVisibleKeystroke(samples, 150, DEFAULT_KEYSTROKE_OVERLAY_SETTINGS)?.key).toBe( + "Enter", + ); + }); + + it("treats command combos including punctuation as shortcuts", () => { + expect( + filterKeystrokesForDisplay( + [ + sample({ key: "C", meta: true }), + sample({ key: ",", meta: true }), + sample({ key: "/", meta: true }), + sample({ key: "Z", meta: true, shift: true }), + sample({ key: "V", meta: true }), + sample({ key: "a" }), + ], + DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, + ).map((item) => item.key), + ).toEqual(["C", ",", "/", "Z", "V"]); + }); + + it("formats shortcut labels", () => { + expect(formatKeystrokeLabel(sample({ key: "C", ctrl: true }), false)).toBe("Ctrl+C"); + expect(formatKeystrokeLabel(sample({ key: "C", meta: true }), true)).toBe("⌘C"); + expect(formatKeystrokeLabel(sample({ key: ",", meta: true }), true)).toBe("⌘,"); + }); + + it("parses native KEY monitor lines", () => { + expect(parseKeyMonitorLine("KEY:down:C:ctrl,shift")).toEqual({ + action: "down", + key: "C", + code: "C", + ctrl: true, + alt: false, + shift: true, + meta: false, + repeat: false, + }); + }); +}); diff --git a/src/lib/keystrokeOverlay.ts b/src/lib/keystrokeOverlay.ts new file mode 100644 index 000000000..6bd337e92 --- /dev/null +++ b/src/lib/keystrokeOverlay.ts @@ -0,0 +1,451 @@ +export const KEYSTROKE_OVERLAY_CAPTURE_SETTING = "keystrokeOverlayCaptureEnabled"; +export const KEYSTROKE_TELEMETRY_VERSION = 1; +export const KEYSTROKE_OVERLAY_DISPLAY_MS = 1000; + +export type KeystrokeOverlayMode = "shortcuts" | "all"; +export type KeystrokeOverlayPosition = "top" | "bottom"; + +export interface KeystrokeSample { + timeMs: number; + key: string; + code: string; + ctrl: boolean; + alt: boolean; + shift: boolean; + meta: boolean; + repeat?: boolean; +} + +export interface KeystrokeOverlaySettings { + enabled: boolean; + mode: KeystrokeOverlayMode; + position: KeystrokeOverlayPosition; + fontSize: number; + bottomOffset: number; +} + +export const DEFAULT_KEYSTROKE_OVERLAY_SETTINGS: KeystrokeOverlaySettings = { + enabled: true, + mode: "shortcuts", + position: "bottom", + fontSize: 22, + bottomOffset: 8, +}; + +const SPECIAL_KEYS = new Set([ + "enter", + "return", + "escape", + "esc", + "tab", + "backspace", + "delete", + "space", + "arrowup", + "arrowdown", + "arrowleft", + "arrowright", + "home", + "end", + "pageup", + "pagedown", + "insert", + "capslock", +]); + +const MODIFIER_KEYS = new Set([ + "shift", + "control", + "ctrl", + "alt", + "option", + "meta", + "cmd", + "command", + "super", + "win", + "windows", +]); + +export function clampNumber(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +export function normalizeKeystrokeOverlaySettings( + value: unknown, + fallback: KeystrokeOverlaySettings = DEFAULT_KEYSTROKE_OVERLAY_SETTINGS, +): KeystrokeOverlaySettings { + const source = value && typeof value === "object" ? (value as Partial) : {}; + return { + enabled: typeof source.enabled === "boolean" ? source.enabled : fallback.enabled, + mode: source.mode === "all" || source.mode === "shortcuts" ? source.mode : fallback.mode, + position: + source.position === "top" || source.position === "bottom" + ? source.position + : fallback.position, + fontSize: + typeof source.fontSize === "number" && Number.isFinite(source.fontSize) + ? clampNumber(source.fontSize, 12, 64) + : fallback.fontSize, + bottomOffset: + typeof source.bottomOffset === "number" && Number.isFinite(source.bottomOffset) + ? clampNumber(source.bottomOffset, 0, 40) + : fallback.bottomOffset, + }; +} + +function normalizeKeyName(value: unknown): string { + if (typeof value !== "string") { + return ""; + } + return value.trim(); +} + +export function normalizeKeystrokeSamples(rawSamples: unknown): KeystrokeSample[] { + const samples = Array.isArray(rawSamples) + ? rawSamples + : Array.isArray((rawSamples as { samples?: unknown[] } | null | undefined)?.samples) + ? ((rawSamples as { samples: unknown[] }).samples ?? []) + : []; + + return samples + .filter((sample): sample is Record => Boolean(sample && typeof sample === "object")) + .map((sample) => { + const key = normalizeKeyName(sample.key) || normalizeKeyName(sample.code); + const code = normalizeKeyName(sample.code) || key; + return { + timeMs: + typeof sample.timeMs === "number" && Number.isFinite(sample.timeMs) + ? Math.max(0, sample.timeMs) + : 0, + key, + code, + ctrl: sample.ctrl === true, + alt: sample.alt === true, + shift: sample.shift === true, + meta: sample.meta === true, + repeat: sample.repeat === true ? true : undefined, + } satisfies KeystrokeSample; + }) + .filter((sample) => sample.key.length > 0) + .sort((left, right) => left.timeMs - right.timeMs); +} + +export function hasModifier(sample: Pick) { + return sample.ctrl || sample.alt || sample.shift || sample.meta; +} + +function canonicalKey(sample: KeystrokeSample) { + return (sample.key || sample.code).trim().toLowerCase(); +} + +export function isModifierOnlyKeystroke(sample: KeystrokeSample) { + return MODIFIER_KEYS.has(canonicalKey(sample)) && !hasOtherNonModifierIdentity(sample); +} + +function hasOtherNonModifierIdentity(sample: KeystrokeSample) { + const key = canonicalKey(sample); + return key.length > 0 && !MODIFIER_KEYS.has(key); +} + +export function isSpecialKeystroke(sample: KeystrokeSample) { + const key = canonicalKey(sample); + if (SPECIAL_KEYS.has(key)) { + return true; + } + if (/^f([1-9]|1[0-9]|2[0-4])$/.test(key)) { + return true; + } + if (key.startsWith("arrow")) { + return true; + } + return false; +} + +export function isShortcutKeystroke(sample: KeystrokeSample) { + if (isModifierOnlyKeystroke(sample)) { + return false; + } + if (isSpecialKeystroke(sample)) { + return true; + } + return hasModifier(sample) && hasOtherNonModifierIdentity(sample); +} + +export function shouldStoreCapturedKeystroke( + sample: KeystrokeSample, + options: { platform?: string; isPasswordField: boolean | "unknown" }, +) { + if (options.isPasswordField !== false) { + return false; + } + if (sample.repeat) { + return false; + } + if (isModifierOnlyKeystroke(sample)) { + return false; + } + if ((options.platform ?? "") === "linux") { + return isShortcutKeystroke(sample); + } + return true; +} + +export function filterKeystrokesForDisplay( + samples: KeystrokeSample[], + settings: KeystrokeOverlaySettings, +) { + if (!settings.enabled) { + return []; + } + return samples.filter((sample) => + settings.mode === "all" ? !isModifierOnlyKeystroke(sample) : isShortcutKeystroke(sample), + ); +} + +const KEY_LABELS: Record = { + enter: "Enter", + return: "Enter", + escape: "Esc", + esc: "Esc", + tab: "Tab", + backspace: "Backspace", + delete: "Delete", + space: "Space", + arrowup: "↑", + arrowdown: "↓", + arrowleft: "←", + arrowright: "→", + pageup: "Page Up", + pagedown: "Page Down", + home: "Home", + end: "End", + insert: "Insert", + capslock: "Caps Lock", + equal: "=", + minus: "-", + ",": ",", + ".": ".", + "/": "/", + control: "Ctrl", + ctrl: "Ctrl", + alt: "Alt", + option: "Alt", + shift: "Shift", + meta: "⌘", + cmd: "⌘", + command: "⌘", + super: "Super", + win: "Win", + windows: "Win", +}; + +function formatKeyCap(raw: string) { + const key = raw.trim(); + if (!key) { + return ""; + } + const lower = key.toLowerCase(); + if (KEY_LABELS[lower]) { + return KEY_LABELS[lower]; + } + if (/^f([1-9]|1[0-9]|2[0-4])$/i.test(key)) { + return key.toUpperCase(); + } + if (key.length === 1) { + return key.toUpperCase(); + } + if (lower.startsWith("key") && lower.length === 4) { + return key.slice(3).toUpperCase(); + } + if (lower.startsWith("digit") && lower.length === 6) { + return key.slice(5); + } + return key.charAt(0).toUpperCase() + key.slice(1); +} + +export function formatKeystrokeLabel(sample: KeystrokeSample, isMac = false) { + const parts: string[] = []; + if (sample.ctrl) { + parts.push(isMac ? "⌃" : "Ctrl"); + } + if (sample.alt) { + parts.push(isMac ? "⌥" : "Alt"); + } + if (sample.shift) { + parts.push(isMac ? "⇧" : "Shift"); + } + if (sample.meta) { + parts.push(isMac ? "⌘" : "Win"); + } + parts.push(formatKeyCap(sample.key || sample.code)); + return parts.filter(Boolean).join(isMac ? "" : "+"); +} + +type VisibleKeystrokeCache = { + samples: KeystrokeSample[]; + enabled: boolean; + mode: KeystrokeOverlaySettings["mode"]; + filtered: KeystrokeSample[]; + lastTimeMs: number; + lastIndex: number; +}; + +let visibleKeystrokeCache: VisibleKeystrokeCache | null = null; + +function findRightmostSampleAtOrBefore(samples: KeystrokeSample[], timeMs: number): number { + let low = 0; + let high = samples.length - 1; + let found = -1; + while (low <= high) { + const mid = (low + high) >> 1; + if (samples[mid].timeMs <= timeMs) { + found = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return found; +} + +function getCachedVisibleKeystrokes( + samples: KeystrokeSample[], + settings: KeystrokeOverlaySettings, +): VisibleKeystrokeCache { + if ( + visibleKeystrokeCache && + visibleKeystrokeCache.samples === samples && + visibleKeystrokeCache.enabled === settings.enabled && + visibleKeystrokeCache.mode === settings.mode + ) { + return visibleKeystrokeCache; + } + + visibleKeystrokeCache = { + samples, + enabled: settings.enabled, + mode: settings.mode, + filtered: filterKeystrokesForDisplay(samples, settings), + lastTimeMs: Number.NEGATIVE_INFINITY, + lastIndex: -1, + }; + return visibleKeystrokeCache; +} + +export function getVisibleKeystroke( + samples: KeystrokeSample[], + timeMs: number, + settings: KeystrokeOverlaySettings, + displayMs = KEYSTROKE_OVERLAY_DISPLAY_MS, +): KeystrokeSample | null { + const cache = getCachedVisibleKeystrokes(samples, settings); + const visible = cache.filtered; + if (visible.length === 0) { + cache.lastTimeMs = timeMs; + cache.lastIndex = -1; + return null; + } + + let index: number; + if (timeMs >= cache.lastTimeMs && cache.lastIndex >= -1) { + index = cache.lastIndex; + if (index < 0) { + index = visible[0].timeMs <= timeMs ? 0 : -1; + } + while (index + 1 < visible.length && visible[index + 1].timeMs <= timeMs) { + index += 1; + } + } else { + index = findRightmostSampleAtOrBefore(visible, timeMs); + } + + cache.lastTimeMs = timeMs; + cache.lastIndex = index; + if (index < 0) { + return null; + } + + const latest = visible[index]; + if (timeMs - latest.timeMs <= displayMs) { + return latest; + } + return null; +} + +export function getKeystrokeOverlayOpacity( + sample: KeystrokeSample | null, + timeMs: number, + displayMs = KEYSTROKE_OVERLAY_DISPLAY_MS, +) { + if (!sample) { + return 0; + } + const elapsed = timeMs - sample.timeMs; + if (elapsed < 0 || elapsed > displayMs) { + return 0; + } + const fadeStart = displayMs * 0.65; + if (elapsed <= fadeStart) { + return 1; + } + return Math.max(0, 1 - (elapsed - fadeStart) / (displayMs - fadeStart)); +} + +export interface ParsedKeyMonitorLine { + action: "down" | "up"; + key: string; + code: string; + ctrl: boolean; + alt: boolean; + shift: boolean; + meta: boolean; + repeat: boolean; +} + +export function parseKeyMonitorLine(line: string): ParsedKeyMonitorLine | null { + const match = line.trim().match(/^KEY:(down|up):([^:]+):?(.*)$/i); + if (!match) { + return null; + } + const modifiers = new Set( + (match[3] ?? "") + .split(",") + .map((part) => part.trim().toLowerCase()) + .filter(Boolean), + ); + const identity = decodeURIComponent(match[2] ?? "").trim(); + if (!identity) { + return null; + } + return { + action: match[1] === "up" ? "up" : "down", + key: identity, + code: identity, + ctrl: modifiers.has("ctrl") || modifiers.has("control"), + alt: modifiers.has("alt") || modifiers.has("option"), + shift: modifiers.has("shift"), + meta: modifiers.has("meta") || modifiers.has("cmd") || modifiers.has("command") || modifiers.has("win"), + repeat: modifiers.has("repeat"), + }; +} + +export function encodeKeyMonitorLine(sample: { + action?: "down" | "up"; + key: string; + ctrl?: boolean; + alt?: boolean; + shift?: boolean; + meta?: boolean; + repeat?: boolean; +}) { + const modifiers = [ + sample.ctrl ? "ctrl" : "", + sample.alt ? "alt" : "", + sample.shift ? "shift" : "", + sample.meta ? "meta" : "", + sample.repeat ? "repeat" : "", + ].filter(Boolean); + const identity = encodeURIComponent(sample.key); + return `KEY:${sample.action ?? "down"}:${identity}${modifiers.length ? `:${modifiers.join(",")}` : ""}`; +}