diff --git a/Cargo.lock b/Cargo.lock index 638cb957a7c..a0c3fc19791 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -398,6 +398,15 @@ dependencies = [ "security-framework 3.7.0", ] +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -1499,6 +1508,7 @@ dependencies = [ "relative-path", "reqwest", "rodio", + "rxing", "sanitize-filename", "scap-direct3d", "scap-screencapturekit", @@ -2185,6 +2195,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "cidre" version = "0.11.0" @@ -2339,6 +2359,15 @@ dependencies = [ "objc", ] +[[package]] +name = "codepage-437" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40c1169585d8d08e5675a39f2fc056cd19a258fc4cba5e3bbf4a9c1026de535" +dependencies = [ + "csv", +] + [[package]] name = "codespan-reporting" version = "0.12.0" @@ -2852,6 +2881,27 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctor" version = "0.2.9" @@ -3568,6 +3618,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -4953,6 +5014,24 @@ dependencies = [ "quick-error", ] +[[package]] +name = "imageproc" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602b4e8a4cc3e98372b766cd184ab532999bc0e839b7469e759511ccabc65d77" +dependencies = [ + "ab_glyph", + "approx", + "getrandom 0.2.16", + "image 0.25.8", + "itertools 0.12.1", + "nalgebra", + "num", + "rand 0.8.5", + "rand_distr", + "rayon", +] + [[package]] name = "imagequant" version = "4.4.1" @@ -6149,6 +6228,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "nalgebra" +version = "0.32.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5c17de023a86f59ed79891b2e5d5a94c705dbe904a5b5c9c952ea6221b03e4" +dependencies = [ + "approx", + "matrixmultiply", + "num-complex", + "num-rational", + "num-traits", + "simba", + "typenum", +] + [[package]] name = "nanorand" version = "0.7.0" @@ -7505,6 +7599,15 @@ dependencies = [ "phf_shared 0.11.3", ] +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + [[package]] name = "phf_codegen" version = "0.8.0" @@ -7609,6 +7712,15 @@ dependencies = [ "siphasher 1.0.1", ] +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher 1.0.1", +] + [[package]] name = "pico-args" version = "0.5.0" @@ -8174,6 +8286,16 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -8418,9 +8540,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -8430,9 +8552,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -8447,9 +8569,9 @@ checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" -version = "0.8.6" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "relative-path" @@ -8855,12 +8977,55 @@ dependencies = [ "unicode-script", ] +[[package]] +name = "rxing" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6609a7ccb6435312dd3a2bff9924cdbb1c96050510fff30676c3d7b8b0045739" +dependencies = [ + "chrono", + "chrono-tz", + "codepage-437", + "encoding_rs", + "fancy-regex", + "image 0.25.8", + "imageproc", + "num", + "once_cell", + "regex", + "rxing-one-d-proc-derive", + "serde", + "thiserror 2.0.16", + "unicode-segmentation", + "uriparse", + "urlencoding", +] + +[[package]] +name = "rxing-one-d-proc-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee3a551320da932c705835119f5f9a69a144966d18780d35e3b394585c22ce3" +dependencies = [ + "quote", + "syn 3.0.3", +] + [[package]] name = "ryu" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -9604,6 +9769,19 @@ dependencies = [ "libc", ] +[[package]] +name = "simba" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061507c94fc6ab4ba1c9a0305018408e312e17c041eb63bef8aa726fa33aceae" +dependencies = [ + "approx", + "num-complex", + "num-traits", + "paste", + "wide", +] + [[package]] name = "simd-adler32" version = "0.3.7" @@ -10142,6 +10320,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -11828,9 +12017,9 @@ checksum = "9fb421b350c9aff471779e262955939f565ec18b86c15364e6bdf0d662ca7c1f" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-vo" @@ -11905,6 +12094,16 @@ dependencies = [ "log", ] +[[package]] +name = "uriparse" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0200d0fc04d809396c2ad43f3c95da3582a2556eba8d453c1087f4120ee352ff" +dependencies = [ + "fnv", + "lazy_static", +] + [[package]] name = "url" version = "2.5.7" @@ -11917,6 +12116,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "urlpattern" version = "0.3.0" @@ -12612,6 +12817,16 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "wild" version = "2.2.1" diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index d457438e14c..1d87ebe020c 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -51,6 +51,7 @@ specta-typescript = "0.0.7" tokio = { workspace = true, features = ["signal"] } uuid = { version = "1.10.0", features = ["v4"] } image = "0.25.2" +rxing = "0.9.2" futures-intrusive = "0.5.0" anyhow.workspace = true futures = { workspace = true } diff --git a/apps/desktop/src-tauri/src/barcode.rs b/apps/desktop/src-tauri/src/barcode.rs new file mode 100644 index 00000000000..e9473cab8ea --- /dev/null +++ b/apps/desktop/src-tauri/src/barcode.rs @@ -0,0 +1,14 @@ +use rxing::{DecodeHintValue, DecodeHints}; + +pub fn decode_barcode(image: &image::DynamicImage) -> Option { + let luma = image.to_luma8(); + let width = luma.width(); + let height = luma.height(); + + let mut hints = DecodeHints::default().with(DecodeHintValue::TryHarder(true)); + + rxing::helpers::detect_in_luma_with_hints(luma.into_raw(), width, height, None, &mut hints) + .ok() + .map(|result| result.getText().to_string()) + .filter(|text| !text.is_empty()) +} diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index 8ea2ce2fd9c..7b3d7456621 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -247,6 +247,10 @@ pub struct GeneralSettingsStore { pub camera_blur_disabled_by_crash: Option, #[serde(default)] pub update_channel: UpdateChannel, + #[serde(default)] + pub ocr_keep_screenshot: bool, + #[serde(default)] + pub ocr_show_notification: bool, } fn default_enable_native_camera_preview() -> bool { @@ -350,6 +354,8 @@ impl Default for GeneralSettingsStore { previous_recordings_paths: Vec::new(), camera_blur_disabled_by_crash: None, update_channel: UpdateChannel::Stable, + ocr_keep_screenshot: false, + ocr_show_notification: false, } } } diff --git a/apps/desktop/src-tauri/src/hotkeys.rs b/apps/desktop/src-tauri/src/hotkeys.rs index 2c9a6eed18c..cb8aea45b5c 100644 --- a/apps/desktop/src-tauri/src/hotkeys.rs +++ b/apps/desktop/src-tauri/src/hotkeys.rs @@ -66,6 +66,7 @@ pub enum HotkeyAction { ScreenshotDisplay, ScreenshotWindow, ScreenshotArea, + OcrArea, #[serde(other)] Other, } @@ -73,6 +74,8 @@ pub enum HotkeyAction { #[derive(Serialize, Deserialize, Type, Default)] pub struct HotkeysStore { hotkeys: HashMap, + #[serde(default)] + seeded: Vec, } impl HotkeysStore { @@ -208,15 +211,21 @@ pub fn init(app: &AppHandle) { ) .unwrap(); - let store = match HotkeysStore::get(app) { - Ok(Some(s)) => s, - Ok(None) => HotkeysStore::default(), + let mut store = match HotkeysStore::get(app) { + Ok(Some(s)) => Some(s), + Ok(None) => Some(HotkeysStore::default()), Err(e) => { eprintln!("Failed to load hotkeys store: {e}"); - HotkeysStore::default() + None } }; + if let Some(store) = &mut store { + seed_default_hotkeys(app, store); + } + + let store = store.unwrap_or_default(); + let global_shortcut = app.global_shortcut(); for hotkey in store.hotkeys.values() { global_shortcut.register(Shortcut::from(*hotkey)).ok(); @@ -225,6 +234,46 @@ pub fn init(app: &AppHandle) { app.manage(Mutex::new(store)); } +fn default_hotkey(action: HotkeyAction) -> Option { + match action { + HotkeyAction::OcrArea => Some(Hotkey { + code: Code::KeyT, + meta: cfg!(target_os = "macos"), + ctrl: !cfg!(target_os = "macos"), + alt: false, + shift: true, + }), + _ => None, + } +} + +fn seed_default_hotkeys(app: &AppHandle, store: &mut HotkeysStore) { + let mut changed = false; + + for action in [HotkeyAction::OcrArea] { + if store.seeded.contains(&action) || store.hotkeys.contains_key(&action) { + continue; + } + + let Some(hotkey) = default_hotkey(action) else { + continue; + }; + + if !store.hotkeys.values().any(|h| h == &hotkey) { + store.hotkeys.insert(action, hotkey); + } + store.seeded.push(action); + changed = true; + } + + if changed && let Ok(s) = app.store("store") { + s.set("hotkeys", serde_json::json!(&*store)); + if let Err(e) = s.save() { + eprintln!("Failed to save hotkeys store: {e}"); + } + } +} + async fn handle_hotkey(app: AppHandle, action: HotkeyAction) -> Result<(), String> { match action { HotkeyAction::StartStudioRecording => { @@ -332,6 +381,13 @@ async fn handle_hotkey(app: AppHandle, action: HotkeyAction) -> Result<(), Strin .emit(&app); Ok(()) } + HotkeyAction::OcrArea => { + let _ = RequestOpenRecordingPicker { + target_mode: Some(RecordingTargetMode::Ocr), + } + .emit(&app); + Ok(()) + } HotkeyAction::Other => Ok(()), } } diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 6a57a662496..2c8c49b541f 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -6,6 +6,7 @@ mod audio_library; mod audio_meter; mod auth; mod automation; +mod barcode; mod camera; mod camera_legacy; #[cfg(target_os = "macos")] @@ -4853,6 +4854,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { recording::restart_recording, recording::delete_recording, recording::take_screenshot, + recording::capture_ocr_text, recording::import_current_desktop_background, recording::list_cameras, recording::get_camera_formats, @@ -4935,6 +4937,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { clip_thumbnails::get_clip_thumbnail, windows::position_traffic_lights, windows::set_theme, + windows::show_window_without_activating, windows::set_teleprompter_window_level, windows::set_teleprompter_window_opacity, windows::apply_macos_liquid_glass_background, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index a12f05431d6..3c8fda11cff 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -2803,26 +2803,87 @@ pub async fn take_screenshot( app: AppHandle, target: ScreenCaptureTarget, ) -> Result { - use crate::NewScreenshotAdded; - use crate::notifications; - use crate::{PendingScreenshot, PendingScreenshots}; - use cap_recording::screenshot::capture_screenshot; - use image::ImageEncoder; - use std::time::Instant; + let image = capture_screen_image(&app, target.clone()).await?; - let general_settings = GeneralSettingsStore::get(&app).ok().flatten(); - let general_settings = general_settings.as_ref(); + AppSounds::Notification.play(); - let project_name = format_project_name( - general_settings - .and_then(|s| s.default_project_name_template.clone()) - .as_deref(), - target.title().as_deref().unwrap_or("Unknown"), - target.kind_str(), - RecordingMode::Screenshot, - None, - ); + save_screenshot_project(&app, image, &target, true) +} + +#[tauri::command(async)] +#[specta::specta] +#[tracing::instrument(name = "capture_ocr_text", skip(app))] +pub async fn capture_ocr_text( + app: AppHandle, + target: ScreenCaptureTarget, +) -> Result { + use tauri_plugin_clipboard_manager::ClipboardExt; + + let settings = GeneralSettingsStore::get(&app) + .ok() + .flatten() + .unwrap_or_default(); + + let image = capture_screen_image(&app, target.clone()).await?; + + // A QR code or barcode in the selection takes priority over OCR: its + // decoded payload is what the user is after, and OCR would only pick up + // surrounding text. + let barcode = { + let image = image.clone(); + tokio::task::spawn_blocking(move || crate::barcode::decode_barcode(&image)) + .await + .ok() + .flatten() + }; + + let text = match barcode { + Some(payload) => payload, + None => { + let text = crate::screenshot_editor::recognize_text_from_dynamic_image(&image).await?; + text.trim().to_string() + } + }; + + if text.is_empty() { + return Err("No text was found in the selected area".to_string()); + } + + app.clipboard() + .write_text(text.clone()) + .map_err(|e| format!("Failed to copy text to clipboard: {e}"))?; + + AppSounds::Notification.play(); + + if settings.ocr_keep_screenshot + && let Err(e) = save_screenshot_project(&app, image, &target, false) + { + error!("Failed to save OCR screenshot: {e}"); + } + + if settings.enable_notifications && settings.ocr_show_notification { + use tauri_plugin_notification::NotificationExt; + let preview: String = text.chars().take(120).collect(); + app.notification() + .builder() + .title("Text copied to clipboard") + .body(preview) + .show() + .ok(); + } + + Ok(text) +} + +async fn capture_screen_image( + app: &AppHandle, + target: ScreenCaptureTarget, +) -> Result { + use crate::windows::show_overlay; + use cap_recording::screenshot::capture_screenshot; + + let mut hidden_windows = Vec::new(); let mut hid_any = false; for (label, window) in app.webview_windows() { if let Ok(id) = CapWindowId::from_str(&label) @@ -2835,8 +2896,18 @@ pub async fn take_screenshot( | CapWindowId::RecordingsOverlay ) { + let was_visible = window.is_visible().unwrap_or(false); hide_overlay(&window); hid_any = true; + // The target-select overlay's lifecycle is owned by the frontend, + // which hides it before invoking a capture and closes or restores + // it afterwards; re-showing it here would fight that. The occluder + // must keep ignoring cursor events, so it is re-shown without the + // show_overlay cursor-event reset. + if was_visible && !matches!(id, CapWindowId::TargetSelectOverlay { .. }) { + let ignores_cursor = matches!(id, CapWindowId::WindowCaptureOccluder { .. }); + hidden_windows.push((window, ignores_cursor)); + } } } @@ -2844,13 +2915,51 @@ pub async fn take_screenshot( tokio::time::sleep(std::time::Duration::from_millis(150)).await; } - let automation_target = target.clone(); - - let image = capture_screenshot(target) + let result = capture_screenshot(target) .await - .map_err(|e| format!("Failed to capture screenshot: {e}"))?; + .map_err(|e| format!("Failed to capture screenshot: {e}")); - AppSounds::Notification.play(); + for (window, ignores_cursor) in hidden_windows { + if ignores_cursor { + let _ = window.show(); + } else { + show_overlay(&window); + } + } + + result +} + +/// `run_side_effects` controls whether screenshot automations and the +/// "screenshot saved" notification fire once the project is written. OCR +/// captures skip them: an automation like copy-to-clipboard would overwrite +/// the text that was just copied. +fn save_screenshot_project( + app: &AppHandle, + image: image::DynamicImage, + target: &ScreenCaptureTarget, + run_side_effects: bool, +) -> Result { + use crate::NewScreenshotAdded; + use crate::notifications; + use crate::{PendingScreenshot, PendingScreenshots}; + use image::ImageEncoder; + use std::time::Instant; + + let general_settings = GeneralSettingsStore::get(app).ok().flatten(); + let general_settings = general_settings.as_ref(); + + let project_name = format_project_name( + general_settings + .and_then(|s| s.default_project_name_template.clone()) + .as_deref(), + target.title().as_deref().unwrap_or("Unknown"), + target.kind_str(), + RecordingMode::Screenshot, + None, + ); + + let automation_target = target.clone(); let image_width = image.width(); let image_height = image.height(); @@ -2975,16 +3084,18 @@ pub async fn take_screenshot( } .emit(&app_handle); - crate::automation::run_screenshot_automations( - app_handle.clone(), - image_path_for_emit.clone(), - &automation_target, - ); + if run_side_effects { + crate::automation::run_screenshot_automations( + app_handle.clone(), + image_path_for_emit.clone(), + &automation_target, + ); - notifications::send_notification( - &app_handle, - notifications::NotificationType::ScreenshotSaved, - ); + notifications::send_notification( + &app_handle, + notifications::NotificationType::ScreenshotSaved, + ); + } } Ok(Err(e)) => { error!("Failed to encode PNG: {e}"); diff --git a/apps/desktop/src-tauri/src/recording_settings.rs b/apps/desktop/src-tauri/src/recording_settings.rs index c8935289818..d6c9dc4c7c0 100644 --- a/apps/desktop/src-tauri/src/recording_settings.rs +++ b/apps/desktop/src-tauri/src/recording_settings.rs @@ -19,6 +19,7 @@ pub enum RecordingTargetMode { Window, Area, Camera, + Ocr, } #[derive(serde::Serialize, serde::Deserialize, specta::Type, Debug, Clone, Default)] diff --git a/apps/desktop/src-tauri/src/screenshot_editor.rs b/apps/desktop/src-tauri/src/screenshot_editor.rs index 222da938b07..47d62302b6a 100644 --- a/apps/desktop/src-tauri/src/screenshot_editor.rs +++ b/apps/desktop/src-tauri/src/screenshot_editor.rs @@ -986,6 +986,12 @@ pub async fn recognize_screenshot_text( pub async fn recognize_text_from_image_path(path: &std::path::Path) -> Result { let dynamic = image::open(path).map_err(|e| format!("Failed to open image for OCR: {e}"))?; + recognize_text_from_dynamic_image(&dynamic).await +} + +pub async fn recognize_text_from_dynamic_image( + dynamic: &image::DynamicImage, +) -> Result { let rgba = dynamic.to_rgba8(); let width = rgba.width(); let height = rgba.height(); diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 4dec972b3ac..e1e3b16e81c 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -130,6 +130,27 @@ pub fn show_overlay(window: &WebviewWindow) { let _ = window.show(); } +/// Shows the calling window without activating it, so the foreground app +/// keeps keyboard focus. +#[tauri::command] +#[specta::specta] +pub fn show_window_without_activating(window: WebviewWindow) { + #[cfg(windows)] + if let Ok(hwnd) = window.hwnd() { + use ::windows::Win32::UI::WindowsAndMessaging::{SW_SHOWNOACTIVATE, ShowWindow}; + + unsafe { + let _ = ShowWindow( + ::windows::Win32::Foundation::HWND(hwnd.0), + SW_SHOWNOACTIVATE, + ); + } + return; + } + + let _ = window.show(); +} + fn emit_app_event(app: &AppHandle, event: E) where E: Event + serde::Serialize + Clone, @@ -1707,6 +1728,7 @@ impl ShowCapWindow { Some(RecordingTargetMode::Window) => "&targetMode=window", Some(RecordingTargetMode::Area) => "&targetMode=area", Some(RecordingTargetMode::Camera) => "&targetMode=camera", + Some(RecordingTargetMode::Ocr) => "&targetMode=ocr", None => "", }; diff --git a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx index cc270e2b049..96a06e1e3b9 100644 --- a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx +++ b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx @@ -2035,11 +2035,18 @@ function Page() { const dismissalReveals = dismissal === "cancelled" || dismissal === "screenshot" || + dismissal === "ocr" || dismissal === "recordingInstant"; if (shouldRevealMainWindow && dismissalReveals) { - const currentWindow = getCurrentWindow(); - void currentWindow.show(); - void currentWindow.setFocus(); + // An OCR capture is a background copy-to-clipboard gesture; bring + // the window back without stealing focus from the user's app. + if (dismissal === "ocr") { + void commands.showWindowWithoutActivating(); + } else { + const currentWindow = getCurrentWindow(); + void currentWindow.show(); + void currentWindow.setFocus(); + } } } }); diff --git a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx index 46e7b205259..fade02a0acf 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx @@ -572,6 +572,35 @@ function Inner(props: { } /> +
+ + handleChange("ocrKeepScreenshot", value)} + /> + { + if (value) { + const permissionGranted = await isPermissionGranted(); + if (!permissionGranted) { + const permission = await requestPermission(); + if (permission !== "granted") return; + } + } + handleChange("ocrShowNotification", value); + }} + /> + +
+
(); const [options, setOptions] = useOptions(); const [areaSelectionPreferences, setAreaSelectionPreferences] = makePersisted( @@ -293,7 +293,16 @@ function Inner() { }); createEffect( - (prevMode: "display" | "window" | "area" | "camera" | null | undefined) => { + ( + prevMode: + | "display" + | "window" + | "area" + | "camera" + | "ocr" + | null + | undefined, + ) => { const mode = options.targetMode ?? null; if (prevMode === "area" && mode !== "area") { const target = pendingAreaTarget(); @@ -779,8 +788,16 @@ function Inner() { ); }} - + {(displayId) => { + const isOcr = () => options.targetMode === "ocr"; + const isImmediateCapture = () => + isOcr() || options.mode === "screenshot"; let controlsEl: HTMLDivElement | undefined; let cropperRef: CropperRef | undefined; @@ -812,19 +829,19 @@ function Inner() { const [screenshotSnapToRatio, setScreenshotSnapToRatio] = createSignal(true); const minSize = () => - options.mode === "screenshot" ? MIN_SCREENSHOT_SIZE : MIN_SIZE; + isImmediateCapture() ? MIN_SCREENSHOT_SIZE : MIN_SIZE; const currentAspect = () => - options.mode === "screenshot" + isImmediateCapture() ? screenshotAspect() : areaSelectionPreferences.aspectRatio; const currentSnapToRatio = () => - options.mode === "screenshot" + isImmediateCapture() ? screenshotSnapToRatio() : areaSelectionPreferences.snapToRatio; const effectiveInitialAreaBounds = createMemo(() => { const explicitBounds = initialAreaBounds(); if (explicitBounds) return explicitBounds; - if (options.mode === "screenshot") return undefined; + if (isImmediateCapture()) return undefined; return getLockedAreaBounds( areaSelectionPreferences, displayId(), @@ -855,7 +872,7 @@ function Inner() { }); const isSelectionLocked = createMemo( () => - options.mode !== "screenshot" && + !isImmediateCapture() && getLockedAreaBounds( areaSelectionPreferences, displayId(), @@ -864,7 +881,7 @@ function Inner() { ); function setAspect(aspect: Ratio | null) { - if (options.mode === "screenshot") { + if (isImmediateCapture()) { setScreenshotAspect(aspect); return; } @@ -875,7 +892,7 @@ function Inner() { } function setSnapToRatio(enabled: boolean) { - if (options.mode === "screenshot") { + if (isImmediateCapture()) { setScreenshotSnapToRatio(enabled); return; } @@ -884,7 +901,7 @@ function Inner() { function persistLockedSelection() { if ( - options.mode === "screenshot" || + isImmediateCapture() || !areaSelectionPreferences.locked || areaSelectionPreferences.screenId !== displayId() || !isValid() @@ -920,7 +937,7 @@ function Inner() { } if ( isInteracting() || - options.mode === "screenshot" || + isImmediateCapture() || !areaSelectionPreferences.locked || areaSelectionPreferences.screenId !== displayId() || !isValid() || @@ -989,7 +1006,7 @@ function Inner() { }); createEffect(async () => { - if (options.mode === "screenshot") return; + if (isImmediateCapture()) return; const bounds = crop(); const interacting = isInteracting(); const displayInfo = areaDisplayInfo.data; @@ -1252,6 +1269,51 @@ function Inner() { if (was && !interacting) { persistLockedSelection(); + if (isOcr() && isValid()) { + const cropBounds = crop(); + const target: ScreenCaptureTarget = { + variant: "area", + screen: displayId(), + bounds: { + position: { + x: cropBounds.x, + y: cropBounds.y, + }, + size: { + width: cropBounds.width, + height: cropBounds.height, + }, + }, + }; + + const overlayWindows = (await WebviewWindow.getAll()).filter( + (win) => win.label.startsWith("target-select-overlay-"), + ); + + try { + for (const win of overlayWindows) { + await win.setIgnoreCursorEvents(true); + await win.hide(); + } + await new Promise((resolve) => setTimeout(resolve, 50)); + + await commands.captureOcrText(target); + setOptions({ + targetMode: null, + targetModeDismissal: "ocr", + }); + await commands.closeTargetSelectOverlays(); + } catch (e) { + for (const win of overlayWindows) { + await win.setIgnoreCursorEvents(false); + await win.show(); + } + const message = e instanceof Error ? e.message : String(e); + toast.error(`Failed to copy text: ${message}`); + console.error("Failed to copy text", e); + } + return; + } if (options.mode === "screenshot" && isValid()) { const cropBounds = crop(); const displayInfo = areaDisplayInfo.data; @@ -1330,7 +1392,9 @@ function Inner() {
{isValid() ? `${Math.round(crop().width)} × ${Math.round(crop().height)}` - : "Draw an area"} + : isOcr() + ? "Draw an area to copy text" + : "Draw an area"}
@@ -1402,7 +1466,7 @@ function Inner() { > - +