diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index d457438e14c..d96c92a4856 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -167,6 +167,7 @@ windows = { workspace = true, features = [ "Win32_System_Power", "Win32_System_Threading", "Win32_System_WinRT", + "Win32_UI_Input_KeyboardAndMouse", "Win32_UI_WindowsAndMessaging", "Win32_Graphics_Gdi", ] } diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index 8ea2ce2fd9c..7b3d7456621 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -247,6 +247,10 @@ pub struct GeneralSettingsStore { pub camera_blur_disabled_by_crash: Option, #[serde(default)] pub update_channel: UpdateChannel, + #[serde(default)] + pub ocr_keep_screenshot: bool, + #[serde(default)] + pub ocr_show_notification: bool, } fn default_enable_native_camera_preview() -> bool { @@ -350,6 +354,8 @@ impl Default for GeneralSettingsStore { previous_recordings_paths: Vec::new(), camera_blur_disabled_by_crash: None, update_channel: UpdateChannel::Stable, + ocr_keep_screenshot: false, + ocr_show_notification: false, } } } diff --git a/apps/desktop/src-tauri/src/hotkeys.rs b/apps/desktop/src-tauri/src/hotkeys.rs index 2c9a6eed18c..a081fbc9ef5 100644 --- a/apps/desktop/src-tauri/src/hotkeys.rs +++ b/apps/desktop/src-tauri/src/hotkeys.rs @@ -66,6 +66,8 @@ pub enum HotkeyAction { ScreenshotDisplay, ScreenshotWindow, ScreenshotArea, + OcrArea, + ScrollingCaptureWindow, #[serde(other)] Other, } @@ -73,6 +75,8 @@ pub enum HotkeyAction { #[derive(Serialize, Deserialize, Type, Default)] pub struct HotkeysStore { hotkeys: HashMap, + #[serde(default)] + seeded: Vec, } impl HotkeysStore { @@ -208,15 +212,21 @@ pub fn init(app: &AppHandle) { ) .unwrap(); - let store = match HotkeysStore::get(app) { - Ok(Some(s)) => s, - Ok(None) => HotkeysStore::default(), + let mut store = match HotkeysStore::get(app) { + Ok(Some(s)) => Some(s), + Ok(None) => Some(HotkeysStore::default()), Err(e) => { eprintln!("Failed to load hotkeys store: {e}"); - HotkeysStore::default() + None } }; + if let Some(store) = &mut store { + seed_default_hotkeys(app, store); + } + + let store = store.unwrap_or_default(); + let global_shortcut = app.global_shortcut(); for hotkey in store.hotkeys.values() { global_shortcut.register(Shortcut::from(*hotkey)).ok(); @@ -225,6 +235,46 @@ pub fn init(app: &AppHandle) { app.manage(Mutex::new(store)); } +fn default_hotkey(action: HotkeyAction) -> Option { + match action { + HotkeyAction::OcrArea => Some(Hotkey { + code: Code::KeyT, + meta: cfg!(target_os = "macos"), + ctrl: !cfg!(target_os = "macos"), + alt: false, + shift: true, + }), + _ => None, + } +} + +fn seed_default_hotkeys(app: &AppHandle, store: &mut HotkeysStore) { + let mut changed = false; + + for action in [HotkeyAction::OcrArea] { + if store.seeded.contains(&action) || store.hotkeys.contains_key(&action) { + continue; + } + + let Some(hotkey) = default_hotkey(action) else { + continue; + }; + + if !store.hotkeys.values().any(|h| h == &hotkey) { + store.hotkeys.insert(action, hotkey); + } + store.seeded.push(action); + changed = true; + } + + if changed && let Ok(s) = app.store("store") { + s.set("hotkeys", serde_json::json!(&*store)); + if let Err(e) = s.save() { + eprintln!("Failed to save hotkeys store: {e}"); + } + } +} + async fn handle_hotkey(app: AppHandle, action: HotkeyAction) -> Result<(), String> { match action { HotkeyAction::StartStudioRecording => { @@ -332,6 +382,33 @@ async fn handle_hotkey(app: AppHandle, action: HotkeyAction) -> Result<(), Strin .emit(&app); Ok(()) } + HotkeyAction::OcrArea => { + let _ = RequestOpenRecordingPicker { + target_mode: Some(RecordingTargetMode::Ocr), + } + .emit(&app); + Ok(()) + } + HotkeyAction::ScrollingCaptureWindow => { + use scap_targets::Window; + + let window_id = Window::get_topmost_at_cursor() + .ok_or_else(|| "No window found under cursor".to_string())? + .id(); + let target = ScreenCaptureTarget::Window { + id: window_id.clone(), + }; + + match crate::scrolling_capture::capture_scrolling_window(app.clone(), window_id).await { + Ok(path) => { + if crate::automation::should_open_screenshot_editor(&app, &target) { + let _ = ShowCapWindow::ScreenshotEditor { path }.show(&app).await; + } + Ok(()) + } + Err(e) => Err(format!("Failed to capture scrolling window: {e}")), + } + } HotkeyAction::Other => Ok(()), } } diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 6a57a662496..43e41a7df29 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -38,6 +38,7 @@ mod recording_telemetry; mod recordings_locations; mod recovery; mod screenshot_editor; +mod scrolling_capture; mod target_select_overlay; mod telemetry; mod thumbnails; @@ -4853,6 +4854,8 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { recording::restart_recording, recording::delete_recording, recording::take_screenshot, + recording::capture_ocr_text, + scrolling_capture::capture_scrolling_window, recording::import_current_desktop_background, recording::list_cameras, recording::get_camera_formats, @@ -4935,6 +4938,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { clip_thumbnails::get_clip_thumbnail, windows::position_traffic_lights, windows::set_theme, + windows::show_window_without_activating, windows::set_teleprompter_window_level, windows::set_teleprompter_window_opacity, windows::apply_macos_liquid_glass_background, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index a12f05431d6..d597a7f235c 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -2803,26 +2803,71 @@ pub async fn take_screenshot( app: AppHandle, target: ScreenCaptureTarget, ) -> Result { - use crate::NewScreenshotAdded; - use crate::notifications; - use crate::{PendingScreenshot, PendingScreenshots}; - use cap_recording::screenshot::capture_screenshot; - use image::ImageEncoder; - use std::time::Instant; + let image = capture_screen_image(&app, target.clone()).await?; - let general_settings = GeneralSettingsStore::get(&app).ok().flatten(); - let general_settings = general_settings.as_ref(); + AppSounds::Notification.play(); - let project_name = format_project_name( - general_settings - .and_then(|s| s.default_project_name_template.clone()) - .as_deref(), - target.title().as_deref().unwrap_or("Unknown"), - target.kind_str(), - RecordingMode::Screenshot, - None, - ); + save_screenshot_project(&app, image, &target, true) +} + +#[tauri::command(async)] +#[specta::specta] +#[tracing::instrument(name = "capture_ocr_text", skip(app))] +pub async fn capture_ocr_text( + app: AppHandle, + target: ScreenCaptureTarget, +) -> Result { + use tauri_plugin_clipboard_manager::ClipboardExt; + + let settings = GeneralSettingsStore::get(&app) + .ok() + .flatten() + .unwrap_or_default(); + + let image = capture_screen_image(&app, target.clone()).await?; + + let text = crate::screenshot_editor::recognize_text_from_dynamic_image(&image).await?; + let text = text.trim().to_string(); + + if text.is_empty() { + return Err("No text was found in the selected area".to_string()); + } + + app.clipboard() + .write_text(text.clone()) + .map_err(|e| format!("Failed to copy text to clipboard: {e}"))?; + + AppSounds::Notification.play(); + + if settings.ocr_keep_screenshot + && let Err(e) = save_screenshot_project(&app, image, &target, false) + { + error!("Failed to save OCR screenshot: {e}"); + } + + if settings.enable_notifications && settings.ocr_show_notification { + use tauri_plugin_notification::NotificationExt; + + let preview: String = text.chars().take(120).collect(); + app.notification() + .builder() + .title("Text copied to clipboard") + .body(preview) + .show() + .ok(); + } + + Ok(text) +} +pub async fn capture_screen_image( + app: &AppHandle, + target: ScreenCaptureTarget, +) -> Result { + use crate::windows::show_overlay; + use cap_recording::screenshot::capture_screenshot; + + let mut hidden_windows = Vec::new(); let mut hid_any = false; for (label, window) in app.webview_windows() { if let Ok(id) = CapWindowId::from_str(&label) @@ -2835,8 +2880,18 @@ pub async fn take_screenshot( | CapWindowId::RecordingsOverlay ) { + let was_visible = window.is_visible().unwrap_or(false); hide_overlay(&window); hid_any = true; + // The target-select overlay's lifecycle is owned by the frontend, + // which hides it before invoking a capture and closes or restores + // it afterwards; re-showing it here would fight that. The occluder + // must keep ignoring cursor events, so it is re-shown without the + // show_overlay cursor-event reset. + if was_visible && !matches!(id, CapWindowId::TargetSelectOverlay { .. }) { + let ignores_cursor = matches!(id, CapWindowId::WindowCaptureOccluder { .. }); + hidden_windows.push((window, ignores_cursor)); + } } } @@ -2844,13 +2899,51 @@ pub async fn take_screenshot( tokio::time::sleep(std::time::Duration::from_millis(150)).await; } - let automation_target = target.clone(); - - let image = capture_screenshot(target) + let result = capture_screenshot(target) .await - .map_err(|e| format!("Failed to capture screenshot: {e}"))?; + .map_err(|e| format!("Failed to capture screenshot: {e}")); - AppSounds::Notification.play(); + for (window, ignores_cursor) in hidden_windows { + if ignores_cursor { + let _ = window.show(); + } else { + show_overlay(&window); + } + } + + result +} + +/// `run_side_effects` controls whether screenshot automations and the +/// "screenshot saved" notification fire once the project is written. OCR +/// captures skip them: an automation like copy-to-clipboard would overwrite +/// the text that was just copied. +pub fn save_screenshot_project( + app: &AppHandle, + image: image::DynamicImage, + target: &ScreenCaptureTarget, + run_side_effects: bool, +) -> Result { + use crate::NewScreenshotAdded; + use crate::notifications; + use crate::{PendingScreenshot, PendingScreenshots}; + use image::ImageEncoder; + use std::time::Instant; + + let general_settings = GeneralSettingsStore::get(app).ok().flatten(); + let general_settings = general_settings.as_ref(); + + let project_name = format_project_name( + general_settings + .and_then(|s| s.default_project_name_template.clone()) + .as_deref(), + target.title().as_deref().unwrap_or("Unknown"), + target.kind_str(), + RecordingMode::Screenshot, + None, + ); + + let automation_target = target.clone(); let image_width = image.width(); let image_height = image.height(); @@ -2975,16 +3068,18 @@ pub async fn take_screenshot( } .emit(&app_handle); - crate::automation::run_screenshot_automations( - app_handle.clone(), - image_path_for_emit.clone(), - &automation_target, - ); + if run_side_effects { + crate::automation::run_screenshot_automations( + app_handle.clone(), + image_path_for_emit.clone(), + &automation_target, + ); - notifications::send_notification( - &app_handle, - notifications::NotificationType::ScreenshotSaved, - ); + notifications::send_notification( + &app_handle, + notifications::NotificationType::ScreenshotSaved, + ); + } } Ok(Err(e)) => { error!("Failed to encode PNG: {e}"); diff --git a/apps/desktop/src-tauri/src/recording_settings.rs b/apps/desktop/src-tauri/src/recording_settings.rs index c8935289818..d6c9dc4c7c0 100644 --- a/apps/desktop/src-tauri/src/recording_settings.rs +++ b/apps/desktop/src-tauri/src/recording_settings.rs @@ -19,6 +19,7 @@ pub enum RecordingTargetMode { Window, Area, Camera, + Ocr, } #[derive(serde::Serialize, serde::Deserialize, specta::Type, Debug, Clone, Default)] diff --git a/apps/desktop/src-tauri/src/screenshot_editor.rs b/apps/desktop/src-tauri/src/screenshot_editor.rs index 222da938b07..47d62302b6a 100644 --- a/apps/desktop/src-tauri/src/screenshot_editor.rs +++ b/apps/desktop/src-tauri/src/screenshot_editor.rs @@ -986,6 +986,12 @@ pub async fn recognize_screenshot_text( pub async fn recognize_text_from_image_path(path: &std::path::Path) -> Result { let dynamic = image::open(path).map_err(|e| format!("Failed to open image for OCR: {e}"))?; + recognize_text_from_dynamic_image(&dynamic).await +} + +pub async fn recognize_text_from_dynamic_image( + dynamic: &image::DynamicImage, +) -> Result { let rgba = dynamic.to_rgba8(); let width = rgba.width(); let height = rgba.height(); diff --git a/apps/desktop/src-tauri/src/scrolling_capture.rs b/apps/desktop/src-tauri/src/scrolling_capture.rs new file mode 100644 index 00000000000..161c2378552 --- /dev/null +++ b/apps/desktop/src-tauri/src/scrolling_capture.rs @@ -0,0 +1,237 @@ +use cap_recording::screen_capture::ScreenCaptureTarget; +use image::DynamicImage; +use scap_targets::WindowId; +use tauri::AppHandle; +use tracing::debug; + +/// Maximum number of scroll-and-capture iterations. +const MAX_FRAMES: usize = 60; +/// Maximum height of the stitched image, in pixels. +const MAX_STITCHED_HEIGHT: u32 = 16_000; +/// Wheel notches injected per scroll step. Kept small so consecutive frames +/// overlap enough for reliable offset matching (one notch scrolls ~3 lines, +/// roughly 100px in most applications). +const SCROLL_NOTCHES: i32 = 2; +/// Delay after injecting a scroll before capturing the next frame, long +/// enough for smooth-scrolling animations to finish. +const SETTLE_MS: u64 = 600; +/// Fraction of the frame height ignored at the top when matching frames, so +/// sticky headers and toolbars don't pin the detected offset to zero. +const HEADER_SKIP_FRACTION: f32 = 0.2; +/// Maximum mean absolute pixel difference for an overlap to count as a match. +const MATCH_THRESHOLD: f64 = 12.0; +/// Minimum fraction of the matchable rows that must overlap for a candidate +/// offset to be considered; small overlaps produce noisy scores. +const MIN_OVERLAP_FRACTION: f32 = 0.25; + +/// Captures a window while scrolling it, stitching the frames into one tall +/// image. The window must be under the cursor so the injected wheel events +/// reach it. +#[tauri::command(async)] +#[specta::specta] +#[tracing::instrument(name = "capture_scrolling_window", skip(app))] +pub async fn capture_scrolling_window( + app: AppHandle, + window_id: WindowId, +) -> Result { + if !cfg!(any(windows, target_os = "macos")) { + return Err("Scrolling capture is not supported on this platform yet".to_string()); + } + + let target = ScreenCaptureTarget::Window { + id: window_id.clone(), + }; + + let first = crate::recording::capture_screen_image(&app, target.clone()).await?; + let frame_width = first.width(); + let frame_height = first.height(); + + if frame_width == 0 || frame_height == 0 { + return Err("Captured window is empty".to_string()); + } + + let mut stitched: Vec = vec![first.clone()]; + let mut stitched_height = frame_height; + let mut prev = first.to_luma8(); + + for frame_index in 0..MAX_FRAMES { + inject_scroll(-SCROLL_NOTCHES); + tokio::time::sleep(std::time::Duration::from_millis(SETTLE_MS)).await; + + let frame = crate::recording::capture_screen_image(&app, target.clone()).await?; + if frame.width() != frame_width || frame.height() != frame_height { + debug!("Window resized during scrolling capture; stopping"); + break; + } + + let cur = frame.to_luma8(); + let Some(offset) = find_scroll_offset(&prev, &cur) else { + debug!("No overlap match on frame {frame_index}; stopping"); + break; + }; + + if offset == 0 { + debug!("Content stopped scrolling on frame {frame_index}; done"); + break; + } + + let remaining_rows = MAX_STITCHED_HEIGHT.saturating_sub(stitched_height); + let new_rows = offset.min(frame_height).min(remaining_rows); + if new_rows == 0 { + debug!("Stitched image reached the height cap; stopping"); + break; + } + + let strip = frame.crop_imm(0, frame_height - new_rows, frame_width, new_rows); + stitched.push(strip); + stitched_height += new_rows; + prev = cur; + + if stitched_height >= MAX_STITCHED_HEIGHT { + debug!("Stitched image reached the height cap; stopping"); + break; + } + } + + let mut canvas = image::RgbaImage::new(frame_width, stitched_height); + let mut y = 0u32; + for part in &stitched { + image::imageops::replace(&mut canvas, &part.to_rgba8(), 0, i64::from(y)); + y += part.height(); + } + + crate::audio::AppSounds::Notification.play(); + + crate::recording::save_screenshot_project(&app, DynamicImage::ImageRgba8(canvas), &target, true) +} + +/// Finds how many pixels the content moved up between `prev` and `cur` by +/// locating the vertical shift with the smallest mean absolute difference +/// over the overlapping rows. Returns `None` when even the best candidate is +/// a poor match (e.g. the window repainted entirely). +fn find_scroll_offset(prev: &image::GrayImage, cur: &image::GrayImage) -> Option { + let width = prev.width() as usize; + let height = prev.height() as usize; + let skip_top = (height as f32 * HEADER_SKIP_FRACTION) as usize; + + let prev_raw = prev.as_raw(); + let cur_raw = cur.as_raw(); + + let col_step = (width / 64).max(1); + let row_step = 4usize; + + let mut best_offset = 0usize; + let mut best_score = f64::MAX; + + let matchable_rows = height - skip_top; + let min_overlap = ((matchable_rows as f32 * MIN_OVERLAP_FRACTION) as usize).max(row_step); + let max_offset = matchable_rows - min_overlap; + for offset in 0..=max_offset { + let overlap_rows = height - skip_top - offset; + let mut sum = 0u64; + let mut count = 0u64; + + for row in (0..overlap_rows).step_by(row_step) { + let prev_row = skip_top + offset + row; + let cur_row = skip_top + row; + let prev_base = prev_row * width; + let cur_base = cur_row * width; + + for col in (0..width).step_by(col_step) { + let a = prev_raw[prev_base + col] as i32; + let b = cur_raw[cur_base + col] as i32; + sum += a.abs_diff(b) as u64; + count += 1; + } + } + + if count == 0 { + continue; + } + + let score = sum as f64 / count as f64; + if score < best_score { + best_score = score; + best_offset = offset; + } + + if score < 0.5 && offset > 0 { + break; + } + } + + (best_score <= MATCH_THRESHOLD).then_some(best_offset as u32) +} + +/// Injects vertical mouse-wheel scrolling at the current cursor position. +/// Negative `notches` scrolls the content up (revealing content below). +#[allow(unused_variables)] +fn inject_scroll(notches: i32) { + #[cfg(windows)] + { + use ::windows::Win32::UI::Input::KeyboardAndMouse::{ + INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_WHEEL, MOUSEINPUT, SendInput, + }; + + const WHEEL_DELTA: i32 = 120; + + let input = INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dx: 0, + dy: 0, + mouseData: (notches * WHEEL_DELTA) as u32, + dwFlags: MOUSEEVENTF_WHEEL, + time: 0, + dwExtraInfo: 0, + }, + }, + }; + + unsafe { + SendInput(&[input], std::mem::size_of::() as i32); + } + } + + #[cfg(target_os = "macos")] + { + use std::ffi::c_void; + + // kCGScrollEventUnitLine + const UNITS_LINE: u32 = 1; + // kCGHIDEventTap + const HID_EVENT_TAP: u32 = 0; + // Lines scrolled per wheel notch. + const LINES_PER_NOTCH: i32 = 3; + + #[link(name = "CoreGraphics", kind = "framework")] + unsafe extern "C" { + fn CGEventCreateScrollWheelEvent2( + source: *const c_void, + units: u32, + wheel_count: u32, + wheel1: i32, + wheel2: i32, + wheel3: i32, + ) -> *mut c_void; + fn CGEventPost(tap: u32, event: *mut c_void); + fn CFRelease(cf: *const c_void); + } + + unsafe { + let event = CGEventCreateScrollWheelEvent2( + std::ptr::null(), + UNITS_LINE, + 1, + notches * LINES_PER_NOTCH, + 0, + 0, + ); + if !event.is_null() { + CGEventPost(HID_EVENT_TAP, event); + CFRelease(event); + } + } + } +} diff --git a/apps/desktop/src-tauri/src/windows.rs b/apps/desktop/src-tauri/src/windows.rs index 4dec972b3ac..e1e3b16e81c 100644 --- a/apps/desktop/src-tauri/src/windows.rs +++ b/apps/desktop/src-tauri/src/windows.rs @@ -130,6 +130,27 @@ pub fn show_overlay(window: &WebviewWindow) { let _ = window.show(); } +/// Shows the calling window without activating it, so the foreground app +/// keeps keyboard focus. +#[tauri::command] +#[specta::specta] +pub fn show_window_without_activating(window: WebviewWindow) { + #[cfg(windows)] + if let Ok(hwnd) = window.hwnd() { + use ::windows::Win32::UI::WindowsAndMessaging::{SW_SHOWNOACTIVATE, ShowWindow}; + + unsafe { + let _ = ShowWindow( + ::windows::Win32::Foundation::HWND(hwnd.0), + SW_SHOWNOACTIVATE, + ); + } + return; + } + + let _ = window.show(); +} + fn emit_app_event(app: &AppHandle, event: E) where E: Event + serde::Serialize + Clone, @@ -1707,6 +1728,7 @@ impl ShowCapWindow { Some(RecordingTargetMode::Window) => "&targetMode=window", Some(RecordingTargetMode::Area) => "&targetMode=area", Some(RecordingTargetMode::Camera) => "&targetMode=camera", + Some(RecordingTargetMode::Ocr) => "&targetMode=ocr", None => "", }; diff --git a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx index cc270e2b049..96a06e1e3b9 100644 --- a/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx +++ b/apps/desktop/src/routes/(window-chrome)/new-main/index.tsx @@ -2035,11 +2035,18 @@ function Page() { const dismissalReveals = dismissal === "cancelled" || dismissal === "screenshot" || + dismissal === "ocr" || dismissal === "recordingInstant"; if (shouldRevealMainWindow && dismissalReveals) { - const currentWindow = getCurrentWindow(); - void currentWindow.show(); - void currentWindow.setFocus(); + // An OCR capture is a background copy-to-clipboard gesture; bring + // the window back without stealing focus from the user's app. + if (dismissal === "ocr") { + void commands.showWindowWithoutActivating(); + } else { + const currentWindow = getCurrentWindow(); + void currentWindow.show(); + void currentWindow.setFocus(); + } } } }); diff --git a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx index 46e7b205259..fade02a0acf 100644 --- a/apps/desktop/src/routes/(window-chrome)/settings/general.tsx +++ b/apps/desktop/src/routes/(window-chrome)/settings/general.tsx @@ -572,6 +572,35 @@ function Inner(props: { } /> +
+ + handleChange("ocrKeepScreenshot", value)} + /> + { + if (value) { + const permissionGranted = await isPermissionGranted(); + if (!permissionGranted) { + const permission = await requestPermission(); + if (permission !== "granted") return; + } + } + handleChange("ocrShowNotification", value); + }} + /> + +
+
(); const [options, setOptions] = useOptions(); const [areaSelectionPreferences, setAreaSelectionPreferences] = makePersisted( @@ -293,7 +293,16 @@ function Inner() { }); createEffect( - (prevMode: "display" | "window" | "area" | "camera" | null | undefined) => { + ( + prevMode: + | "display" + | "window" + | "area" + | "camera" + | "ocr" + | null + | undefined, + ) => { const mode = options.targetMode ?? null; if (prevMode === "area" && mode !== "area") { const target = pendingAreaTarget(); @@ -779,8 +788,16 @@ function Inner() { ); }} - + {(displayId) => { + const isOcr = () => options.targetMode === "ocr"; + const isImmediateCapture = () => + isOcr() || options.mode === "screenshot"; let controlsEl: HTMLDivElement | undefined; let cropperRef: CropperRef | undefined; @@ -812,19 +829,19 @@ function Inner() { const [screenshotSnapToRatio, setScreenshotSnapToRatio] = createSignal(true); const minSize = () => - options.mode === "screenshot" ? MIN_SCREENSHOT_SIZE : MIN_SIZE; + isImmediateCapture() ? MIN_SCREENSHOT_SIZE : MIN_SIZE; const currentAspect = () => - options.mode === "screenshot" + isImmediateCapture() ? screenshotAspect() : areaSelectionPreferences.aspectRatio; const currentSnapToRatio = () => - options.mode === "screenshot" + isImmediateCapture() ? screenshotSnapToRatio() : areaSelectionPreferences.snapToRatio; const effectiveInitialAreaBounds = createMemo(() => { const explicitBounds = initialAreaBounds(); if (explicitBounds) return explicitBounds; - if (options.mode === "screenshot") return undefined; + if (isImmediateCapture()) return undefined; return getLockedAreaBounds( areaSelectionPreferences, displayId(), @@ -855,7 +872,7 @@ function Inner() { }); const isSelectionLocked = createMemo( () => - options.mode !== "screenshot" && + !isImmediateCapture() && getLockedAreaBounds( areaSelectionPreferences, displayId(), @@ -864,7 +881,7 @@ function Inner() { ); function setAspect(aspect: Ratio | null) { - if (options.mode === "screenshot") { + if (isImmediateCapture()) { setScreenshotAspect(aspect); return; } @@ -875,7 +892,7 @@ function Inner() { } function setSnapToRatio(enabled: boolean) { - if (options.mode === "screenshot") { + if (isImmediateCapture()) { setScreenshotSnapToRatio(enabled); return; } @@ -884,7 +901,7 @@ function Inner() { function persistLockedSelection() { if ( - options.mode === "screenshot" || + isImmediateCapture() || !areaSelectionPreferences.locked || areaSelectionPreferences.screenId !== displayId() || !isValid() @@ -920,7 +937,7 @@ function Inner() { } if ( isInteracting() || - options.mode === "screenshot" || + isImmediateCapture() || !areaSelectionPreferences.locked || areaSelectionPreferences.screenId !== displayId() || !isValid() || @@ -989,7 +1006,7 @@ function Inner() { }); createEffect(async () => { - if (options.mode === "screenshot") return; + if (isImmediateCapture()) return; const bounds = crop(); const interacting = isInteracting(); const displayInfo = areaDisplayInfo.data; @@ -1252,6 +1269,51 @@ function Inner() { if (was && !interacting) { persistLockedSelection(); + if (isOcr() && isValid()) { + const cropBounds = crop(); + const target: ScreenCaptureTarget = { + variant: "area", + screen: displayId(), + bounds: { + position: { + x: cropBounds.x, + y: cropBounds.y, + }, + size: { + width: cropBounds.width, + height: cropBounds.height, + }, + }, + }; + + const overlayWindows = (await WebviewWindow.getAll()).filter( + (win) => win.label.startsWith("target-select-overlay-"), + ); + + try { + for (const win of overlayWindows) { + await win.setIgnoreCursorEvents(true); + await win.hide(); + } + await new Promise((resolve) => setTimeout(resolve, 50)); + + await commands.captureOcrText(target); + setOptions({ + targetMode: null, + targetModeDismissal: "ocr", + }); + await commands.closeTargetSelectOverlays(); + } catch (e) { + for (const win of overlayWindows) { + await win.setIgnoreCursorEvents(false); + await win.show(); + } + const message = e instanceof Error ? e.message : String(e); + toast.error(`Failed to copy text: ${message}`); + console.error("Failed to copy text", e); + } + return; + } if (options.mode === "screenshot" && isValid()) { const cropBounds = crop(); const displayInfo = areaDisplayInfo.data; @@ -1330,7 +1392,9 @@ function Inner() {
{isValid() ? `${Math.round(crop().width)} × ${Math.round(crop().height)}` - : "Draw an area"} + : isOcr() + ? "Draw an area to copy text" + : "Draw an area"}
@@ -1402,7 +1466,7 @@ function Inner() { > - +