From d0e4543fdd81a93148873e55231b90a1f63a647a Mon Sep 17 00:00:00 2001 From: x1xhlol <185671340+x1xhlol@users.noreply.github.com> Date: Thu, 6 Aug 2026 13:55:30 +0000 Subject: [PATCH 1/9] Add global OCR text capture with area selector, clipboard copy, and settings --- .../desktop/src-tauri/src/general_settings.rs | 6 + apps/desktop/src-tauri/src/hotkeys.rs | 54 ++++++++- apps/desktop/src-tauri/src/lib.rs | 1 + apps/desktop/src-tauri/src/recording.rs | 110 ++++++++++++++---- .../src-tauri/src/recording_settings.rs | 1 + .../src-tauri/src/screenshot_editor.rs | 6 + apps/desktop/src-tauri/src/windows.rs | 1 + .../(window-chrome)/settings/general.tsx | 29 +++++ .../(window-chrome)/settings/hotkeys.tsx | 2 + .../src/routes/target-select-overlay.tsx | 98 +++++++++++++--- apps/desktop/src/utils/tauri.ts | 18 ++- 11 files changed, 280 insertions(+), 46 deletions(-) diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index 8ea2ce2fd9c..7b3d7456621 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -247,6 +247,10 @@ pub struct GeneralSettingsStore { pub camera_blur_disabled_by_crash: Option, #[serde(default)] pub update_channel: UpdateChannel, + #[serde(default)] + pub ocr_keep_screenshot: bool, + #[serde(default)] + pub ocr_show_notification: bool, } fn default_enable_native_camera_preview() -> bool { @@ -350,6 +354,8 @@ impl Default for GeneralSettingsStore { previous_recordings_paths: Vec::new(), camera_blur_disabled_by_crash: None, update_channel: UpdateChannel::Stable, + ocr_keep_screenshot: false, + ocr_show_notification: false, } } } diff --git a/apps/desktop/src-tauri/src/hotkeys.rs b/apps/desktop/src-tauri/src/hotkeys.rs index 2c9a6eed18c..1d028d94ec2 100644 --- a/apps/desktop/src-tauri/src/hotkeys.rs +++ b/apps/desktop/src-tauri/src/hotkeys.rs @@ -66,6 +66,7 @@ pub enum HotkeyAction { ScreenshotDisplay, ScreenshotWindow, ScreenshotArea, + OcrArea, #[serde(other)] Other, } @@ -73,6 +74,8 @@ pub enum HotkeyAction { #[derive(Serialize, Deserialize, Type, Default)] pub struct HotkeysStore { hotkeys: HashMap, + #[serde(default)] + seeded: Vec, } impl HotkeysStore { @@ -208,7 +211,7 @@ pub fn init(app: &AppHandle) { ) .unwrap(); - let store = match HotkeysStore::get(app) { + let mut store = match HotkeysStore::get(app) { Ok(Some(s)) => s, Ok(None) => HotkeysStore::default(), Err(e) => { @@ -217,6 +220,8 @@ pub fn init(app: &AppHandle) { } }; + seed_default_hotkeys(app, &mut store); + let global_shortcut = app.global_shortcut(); for hotkey in store.hotkeys.values() { global_shortcut.register(Shortcut::from(*hotkey)).ok(); @@ -225,6 +230,46 @@ pub fn init(app: &AppHandle) { app.manage(Mutex::new(store)); } +fn default_hotkey(action: HotkeyAction) -> Option { + match action { + HotkeyAction::OcrArea => Some(Hotkey { + code: Code::KeyT, + meta: cfg!(target_os = "macos"), + ctrl: !cfg!(target_os = "macos"), + alt: false, + shift: true, + }), + _ => None, + } +} + +fn seed_default_hotkeys(app: &AppHandle, store: &mut HotkeysStore) { + let mut changed = false; + + for action in [HotkeyAction::OcrArea] { + if store.seeded.contains(&action) || store.hotkeys.contains_key(&action) { + continue; + } + + let Some(hotkey) = default_hotkey(action) else { + continue; + }; + + if !store.hotkeys.values().any(|h| h == &hotkey) { + store.hotkeys.insert(action, hotkey); + } + store.seeded.push(action); + changed = true; + } + + if changed && let Ok(s) = app.store("store") { + s.set("hotkeys", serde_json::json!(&*store)); + if let Err(e) = s.save() { + eprintln!("Failed to save hotkeys store: {e}"); + } + } +} + async fn handle_hotkey(app: AppHandle, action: HotkeyAction) -> Result<(), String> { match action { HotkeyAction::StartStudioRecording => { @@ -332,6 +377,13 @@ async fn handle_hotkey(app: AppHandle, action: HotkeyAction) -> Result<(), Strin .emit(&app); Ok(()) } + HotkeyAction::OcrArea => { + let _ = RequestOpenRecordingPicker { + target_mode: Some(RecordingTargetMode::Ocr), + } + .emit(&app); + Ok(()) + } HotkeyAction::Other => Ok(()), } } diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 6a57a662496..90fe90c3f9b 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -4853,6 +4853,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { recording::restart_recording, recording::delete_recording, recording::take_screenshot, + recording::capture_ocr_text, recording::import_current_desktop_background, recording::list_cameras, recording::get_camera_formats, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index a12f05431d6..3e610acf411 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -2803,25 +2803,68 @@ pub async fn take_screenshot( app: AppHandle, target: ScreenCaptureTarget, ) -> Result { - use crate::NewScreenshotAdded; - use crate::notifications; - use crate::{PendingScreenshot, PendingScreenshots}; - use cap_recording::screenshot::capture_screenshot; - use image::ImageEncoder; - use std::time::Instant; + let image = capture_screen_image(&app, target.clone()).await?; - let general_settings = GeneralSettingsStore::get(&app).ok().flatten(); - let general_settings = general_settings.as_ref(); + AppSounds::Notification.play(); - let project_name = format_project_name( - general_settings - .and_then(|s| s.default_project_name_template.clone()) - .as_deref(), - target.title().as_deref().unwrap_or("Unknown"), - target.kind_str(), - RecordingMode::Screenshot, - None, - ); + save_screenshot_project(&app, image, &target) +} + +#[tauri::command(async)] +#[specta::specta] +#[tracing::instrument(name = "capture_ocr_text", skip(app))] +pub async fn capture_ocr_text( + app: AppHandle, + target: ScreenCaptureTarget, +) -> Result { + use tauri_plugin_clipboard_manager::ClipboardExt; + + let settings = GeneralSettingsStore::get(&app) + .ok() + .flatten() + .unwrap_or_default(); + + let image = capture_screen_image(&app, target.clone()).await?; + + let text = crate::screenshot_editor::recognize_text_from_dynamic_image(&image).await?; + let text = text.trim().to_string(); + + if text.is_empty() { + return Err("No text was found in the selected area".to_string()); + } + + app.clipboard() + .write_text(text.clone()) + .map_err(|e| format!("Failed to copy text to clipboard: {e}"))?; + + AppSounds::Notification.play(); + + if settings.ocr_keep_screenshot + && let Err(e) = save_screenshot_project(&app, image, &target) + { + error!("Failed to save OCR screenshot: {e}"); + } + + if settings.ocr_show_notification { + use tauri_plugin_notification::NotificationExt; + + let preview: String = text.chars().take(120).collect(); + app.notification() + .builder() + .title("Text copied to clipboard") + .body(preview) + .show() + .ok(); + } + + Ok(text) +} + +async fn capture_screen_image( + app: &AppHandle, + target: ScreenCaptureTarget, +) -> Result { + use cap_recording::screenshot::capture_screenshot; let mut hid_any = false; for (label, window) in app.webview_windows() { @@ -2844,13 +2887,36 @@ pub async fn take_screenshot( tokio::time::sleep(std::time::Duration::from_millis(150)).await; } - let automation_target = target.clone(); - - let image = capture_screenshot(target) + capture_screenshot(target) .await - .map_err(|e| format!("Failed to capture screenshot: {e}"))?; + .map_err(|e| format!("Failed to capture screenshot: {e}")) +} - AppSounds::Notification.play(); +fn save_screenshot_project( + app: &AppHandle, + image: image::DynamicImage, + target: &ScreenCaptureTarget, +) -> Result { + use crate::NewScreenshotAdded; + use crate::notifications; + use crate::{PendingScreenshot, PendingScreenshots}; + use image::ImageEncoder; + use std::time::Instant; + + let general_settings = GeneralSettingsStore::get(app).ok().flatten(); + let general_settings = general_settings.as_ref(); + + let project_name = format_project_name( + general_settings + .and_then(|s| s.default_project_name_template.clone()) + .as_deref(), + target.title().as_deref().unwrap_or("Unknown"), + target.kind_str(), + RecordingMode::Screenshot, + None, + ); + + let automation_target = target.clone(); let image_width = image.width(); let image_height = image.height(); diff --git a/apps/desktop/src-tauri/src/recording_settings.rs b/apps/desktop/src-tauri/src/recording_settings.rs index c8935289818..d6c9dc4c7c0 100644 --- a/apps/desktop/src-tauri/src/recording_settings.rs +++ b/apps/desktop/src-tauri/src/recording_settings.rs @@ -19,6 +19,7 @@ pub enum RecordingTargetMode { Window, Area, Camera, + Ocr, } #[derive(serde::Serialize, serde::Deserialize, specta::Type, Debug, Clone, Default)] diff --git a/apps/desktop/src-tauri/src/screenshot_editor.rs b/apps/desktop/src-tauri/src/screenshot_editor.rs index 222da938b07..47d62302b6a 100644 --- a/apps/desktop/src-tauri/src/screenshot_editor.rs +++ b/apps/desktop/src-tauri/src/screenshot_editor.rs @@ -986,6 +986,12 @@ pub async fn recognize_screenshot_text( pub async fn recognize_text_from_image_path(path: &std::path::Path) -> Result { let dynamic = image::open(path).map_err(|e| format!("Failed to open image for OCR: {e}"))?; + recognize_text_from_dynamic_image(&dynamic).await +} + +pub async fn recognize_text_from_dynamic_image( + dynamic: &image::DynamicImage, +) -> Result { let rgba = dynamic.to_rgba8(); let width = rgba.width(); let height = rgba.height(); diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 4dec972b3ac..4f08c1800b4 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -1707,6 +1707,7 @@ impl ShowCapWindow { Some(RecordingTargetMode::Window) => "&targetMode=window", Some(RecordingTargetMode::Area) => "&targetMode=area", Some(RecordingTargetMode::Camera) => "&targetMode=camera", + Some(RecordingTargetMode::Ocr) => "&targetMode=ocr", None => "", }; diff --git a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx index 46e7b205259..fade02a0acf 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx @@ -572,6 +572,35 @@ function Inner(props: { } /> +
+ + handleChange("ocrKeepScreenshot", value)} + /> + { + if (value) { + const permissionGranted = await isPermissionGranted(); + if (!permissionGranted) { + const permission = await requestPermission(); + if (permission !== "granted") return; + } + } + handleChange("ocrShowNotification", value); + }} + /> + +
+
(); const [options, setOptions] = useOptions(); const [areaSelectionPreferences, setAreaSelectionPreferences] = makePersisted( @@ -293,7 +293,16 @@ function Inner() { }); createEffect( - (prevMode: "display" | "window" | "area" | "camera" | null | undefined) => { + ( + prevMode: + | "display" + | "window" + | "area" + | "camera" + | "ocr" + | null + | undefined, + ) => { const mode = options.targetMode ?? null; if (prevMode === "area" && mode !== "area") { const target = pendingAreaTarget(); @@ -779,8 +788,16 @@ function Inner() { ); }} - + {(displayId) => { + const isOcr = () => options.targetMode === "ocr"; + const isImmediateCapture = () => + isOcr() || options.mode === "screenshot"; let controlsEl: HTMLDivElement | undefined; let cropperRef: CropperRef | undefined; @@ -812,19 +829,19 @@ function Inner() { const [screenshotSnapToRatio, setScreenshotSnapToRatio] = createSignal(true); const minSize = () => - options.mode === "screenshot" ? MIN_SCREENSHOT_SIZE : MIN_SIZE; + isImmediateCapture() ? MIN_SCREENSHOT_SIZE : MIN_SIZE; const currentAspect = () => - options.mode === "screenshot" + isImmediateCapture() ? screenshotAspect() : areaSelectionPreferences.aspectRatio; const currentSnapToRatio = () => - options.mode === "screenshot" + isImmediateCapture() ? screenshotSnapToRatio() : areaSelectionPreferences.snapToRatio; const effectiveInitialAreaBounds = createMemo(() => { const explicitBounds = initialAreaBounds(); if (explicitBounds) return explicitBounds; - if (options.mode === "screenshot") return undefined; + if (isImmediateCapture()) return undefined; return getLockedAreaBounds( areaSelectionPreferences, displayId(), @@ -855,7 +872,7 @@ function Inner() { }); const isSelectionLocked = createMemo( () => - options.mode !== "screenshot" && + !isImmediateCapture() && getLockedAreaBounds( areaSelectionPreferences, displayId(), @@ -864,7 +881,7 @@ function Inner() { ); function setAspect(aspect: Ratio | null) { - if (options.mode === "screenshot") { + if (isImmediateCapture()) { setScreenshotAspect(aspect); return; } @@ -875,7 +892,7 @@ function Inner() { } function setSnapToRatio(enabled: boolean) { - if (options.mode === "screenshot") { + if (isImmediateCapture()) { setScreenshotSnapToRatio(enabled); return; } @@ -884,7 +901,7 @@ function Inner() { function persistLockedSelection() { if ( - options.mode === "screenshot" || + isImmediateCapture() || !areaSelectionPreferences.locked || areaSelectionPreferences.screenId !== displayId() || !isValid() @@ -920,7 +937,7 @@ function Inner() { } if ( isInteracting() || - options.mode === "screenshot" || + isImmediateCapture() || !areaSelectionPreferences.locked || areaSelectionPreferences.screenId !== displayId() || !isValid() || @@ -989,7 +1006,7 @@ function Inner() { }); createEffect(async () => { - if (options.mode === "screenshot") return; + if (isImmediateCapture()) return; const bounds = crop(); const interacting = isInteracting(); const displayInfo = areaDisplayInfo.data; @@ -1252,6 +1269,51 @@ function Inner() { if (was && !interacting) { persistLockedSelection(); + if (isOcr() && isValid()) { + const cropBounds = crop(); + const target: ScreenCaptureTarget = { + variant: "area", + screen: displayId(), + bounds: { + position: { + x: cropBounds.x, + y: cropBounds.y, + }, + size: { + width: cropBounds.width, + height: cropBounds.height, + }, + }, + }; + + const overlayWindows = (await WebviewWindow.getAll()).filter( + (win) => win.label.startsWith("target-select-overlay-"), + ); + + try { + for (const win of overlayWindows) { + await win.setIgnoreCursorEvents(true); + await win.hide(); + } + await new Promise((resolve) => setTimeout(resolve, 50)); + + await commands.captureOcrText(target); + setOptions({ + targetMode: null, + targetModeDismissal: "screenshot", + }); + await commands.closeTargetSelectOverlays(); + } catch (e) { + for (const win of overlayWindows) { + await win.setIgnoreCursorEvents(false); + await win.show(); + } + const message = e instanceof Error ? e.message : String(e); + toast.error(`Failed to copy text: ${message}`); + console.error("Failed to copy text", e); + } + return; + } if (options.mode === "screenshot" && isValid()) { const cropBounds = crop(); const displayInfo = areaDisplayInfo.data; @@ -1330,7 +1392,9 @@ function Inner() {
{isValid() ? `${Math.round(crop().width)} × ${Math.round(crop().height)}` - : "Draw an area"} + : isOcr() + ? "Draw an area to copy text" + : "Draw an area"}
@@ -1402,7 +1466,7 @@ function Inner() { > - +