fix(foundry): keep local ASR warm and stream transcripts

docs/linux-egui-command-event-baseline.json

@@ -63,6 +63,7 @@
     "foundry_local_asr_release",
     "foundry_local_asr_reveal_model_dir",
     "foundry_local_asr_set_language_hint",
+    "foundry_local_asr_set_keep_loaded_secs",
     "foundry_local_asr_set_model",
     "foundry_local_asr_set_runtime_source",
     "foundry_local_asr_status",

openless-all/app/crates/openless-core/src/lib.rs

@@ -317,6 +317,7 @@ pub use settings::*;
 pub use shared_types::{
     CapsulePayload, CapsuleState, CapsuleStyle, CredentialsStatus, HotkeyMode, HotkeyStatus,
     PendingCorrection, PlatformCapabilities, SelectionPolishOutputMode, UserPreferences,
+    LOCAL_ASR_KEEP_LOADED_FOREVER_SECS,
 };
 pub use shortcut_types::{
     binding_from_legacy_trigger, binding_requires_side_aware_hook, bindings_overlap,

openless-all/app/crates/openless-core/src/shared_types.rs

@@ -19,6 +19,9 @@ pub use crate::android_types::{
 
 pub use crate::types::{HistorySource, PolishMode};
 
+/// 本地 ASR 保持加载设置的兼容值：不自动释放，仅由显式操作或进程退出卸载。
+pub const LOCAL_ASR_KEEP_LOADED_FOREVER_SECS: u32 = 86_400;
+
 /// 识别管线模式（issue #902）：`traditional` = 两段式 ASR + LLM 润色；
 /// `multimodal` = 单个多模态模型一步完成「音频 + 提示词 → 最终文本」。
 /// 两套配置在凭据库中完全隔离，运行时只读当前模式，切换不删除另一套配置。
@@ -530,7 +533,7 @@ pub struct UserPreferences {
     #[serde(default = "default_local_asr_mirror")]
     pub local_asr_mirror: String,
     /// 本地 ASR 引擎在内存中的保留时长（秒）。0 = 说完话即释放；
-    /// 较大值 = 上次使用后驻留 N 秒再释放；86400 = 一天 ≈ 永不释放。
+    /// 较大值 = 上次使用后驻留 N 秒再释放；86400 = 永不自动释放。
     /// 默认 300（5 分钟）：兼顾连续听写不重加载、长时间不用释放 1.2GB+ RAM。
     #[serde(default = "default_local_asr_keep_loaded_secs")]
     pub local_asr_keep_loaded_secs: u32,

openless-all/app/crates/openless-core/tests/local_asr_contract.rs

@@ -1458,6 +1458,8 @@ async fn backend_local_asr_service_owns_preferences_and_change_events() {
     assert_eq!(preferences.sherpa_onnx_language_hint, "zh-hans");
     assert_eq!(preferences.foundry_local_runtime_source, "ort-nightly");
     assert_eq!(preferences.foundry_local_asr_keep_loaded_secs, 42);
+    assert_eq!(preferences.local_asr_keep_loaded_secs, 300);
+    assert_eq!(preferences.sherpa_onnx_keep_loaded_secs, 300);
     assert_eq!(
         runtime.invalidated.lock().unwrap().as_slice(),
         [LocalAsrRuntime::Foundry, LocalAsrRuntime::Foundry]

openless-all/app/scripts/local-asr-polling-contract.test.mjs

@@ -81,6 +81,22 @@ for (const contract of [
   }
 }
 
+for (const contract of [
+  'setFoundryLocalAsrKeepLoadedSecs',
+  'foundryStatus?.keepLoadedSecs ?? 300',
+]) {
+  if (!source.includes(contract)) {
+    throw new Error(`Foundry keep-loaded UI contract is missing: ${contract}`);
+  }
+}
+
+const keepLoadedOptionUses = source.match(/options=\{keepLoadedOptions\}/g) ?? [];
+if (keepLoadedOptionUses.length !== 2) {
+  throw new Error(
+    `Generic and Foundry keep-loaded selectors must share the options, found ${keepLoadedOptionUses.length}`,
+  );
+}
+
 console.log(
   'LocalAsr keeps one refresh poller, pauses it for the download dialog, and preserves stable JSX component types',
 );

openless-all/app/src-tauri/src/asr/local/foundry_runtime.rs

@@ -130,9 +130,12 @@ mod imp {
     }
 
     use anyhow::{Context, Result};
-    use foundry_local_sdk::{DeviceType, FoundryLocalConfig, FoundryLocalManager, Model};
+    use foundry_local_sdk::{
+        AudioTranscriptionResponse, DeviceType, FoundryLocalConfig, FoundryLocalManager, Model,
+    };
+    use futures_util::{Stream, StreamExt};
     use parking_lot::Mutex;
-    use tokio::sync::Mutex as AsyncMutex;
+    use tokio::sync::{Mutex as AsyncMutex, OnceCell};
 
     use super::{
         FoundryCpuFallbackTerminalError, FoundryFallbackNotice, FoundryFallbackNoticeCallback,
@@ -493,6 +496,19 @@ mod imp {
             .ok_or_else(|| anyhow::anyhow!("Foundry Local Whisper total timeout exhausted"))
     }
 
+    async fn collect_foundry_transcription_text<S, E>(
+        mut stream: S,
+    ) -> std::result::Result<String, E>
+    where
+        S: Stream<Item = std::result::Result<AudioTranscriptionResponse, E>> + Unpin,
+    {
+        let mut text = String::new();
+        while let Some(chunk) = stream.next().await {
+            text.push_str(&chunk?.text);
+        }
+        Ok(text)
+    }
+
     struct FoundrySdkExecution<'a> {
         runtime: &'a FoundryLocalRuntime,
         manager: &'static FoundryLocalManager,
@@ -546,16 +562,19 @@ mod imp {
                 client = client.language(language_hint);
             }
             let model_id = self.loaded.model_id.clone();
-            let result = tokio::time::timeout(timeout, client.transcribe(audio_path))
-                .await
-                .with_context(|| {
-                    format!(
-                        "transcribe audio with Foundry model {model_id} timed out after {} seconds",
-                        timeout.as_secs()
-                    )
-                })?
-                .with_context(|| format!("transcribe audio with Foundry model {model_id}"))?;
-            Ok(result.text)
+            let result = tokio::time::timeout(timeout, async {
+                let stream = client.transcribe_streaming(audio_path).await?;
+                collect_foundry_transcription_text(stream).await
+            })
+            .await
+            .with_context(|| {
+                format!(
+                    "transcribe audio with Foundry model {model_id} timed out after {} seconds",
+                    timeout.as_secs()
+                )
+            })?
+            .with_context(|| format!("transcribe audio with Foundry model {model_id}"))?;
+            Ok(result)
         }
 
         async fn switch_to_cpu(
@@ -692,6 +711,8 @@ mod imp {
         /// 仍可中断（`cancel_prepare` + `check_prepare_cancelled`）。若未来要缩小粒度，
         /// 可让下载阶段不持锁、下载完成后重新校验 route epoch 再持锁加载/推理。
         lifecycle: AsyncMutex<()>,
+        /// EP 注册会使 SDK 的模型目录缓存失效；成功后本进程不再重复注册。
+        execution_providers_ready: OnceCell<()>,
         cancel_prepare: Arc<AtomicBool>,
         temporary_cpu_fallback_sequence: AtomicU64,
         route_epoch: AtomicU64,
@@ -708,6 +729,7 @@ mod imp {
         pub fn new() -> Self {
             Self {
                 lifecycle: AsyncMutex::new(()),
+                execution_providers_ready: OnceCell::new(),
                 cancel_prepare: Arc::new(AtomicBool::new(false)),
                 temporary_cpu_fallback_sequence: AtomicU64::new(0),
                 route_epoch: AtomicU64::new(0),
@@ -1084,24 +1106,33 @@ mod imp {
             ));
             let runtime_progress = Arc::clone(&progress);
             let runtime_alias = alias.to_string();
-            manager
-                .download_and_register_eps_with_progress(
-                    None,
-                    move |ep_name: &str, percent: f64| {
-                        let label = if ep_name.trim().is_empty() {
-                            "Foundry Local runtime components".to_string()
-                        } else {
-                            format!("Foundry Local runtime component: {ep_name}")
-                        };
-                        runtime_progress.as_ref()(FoundryPrepareProgressPayload::runtime(
-                            runtime_alias.clone(),
-                            label,
-                            percent,
-                        ));
-                    },
-                )
-                .await
-                .context("download/register Foundry execution providers")?;
+            let cancel_prepare = Arc::clone(&self.cancel_prepare);
+            self.execution_providers_ready
+                .get_or_try_init(|| async move {
+                    manager
+                        .download_and_register_eps_with_progress(
+                            None,
+                            move |ep_name: &str, percent: f64| {
+                                let label = if ep_name.trim().is_empty() {
+                                    "Foundry Local runtime components".to_string()
+                                } else {
+                                    format!("Foundry Local runtime component: {ep_name}")
+                                };
+                                runtime_progress.as_ref()(FoundryPrepareProgressPayload::runtime(
+                                    runtime_alias.clone(),
+                                    label,
+                                    percent,
+                                ));
+                            },
+                        )
+                        .await
+                        .context("download/register Foundry execution providers")?;
+                    if cancel_prepare.load(Ordering::SeqCst) {
+                        anyhow::bail!("Foundry Local Whisper prepare cancelled");
+                    }
+                    Ok::<(), anyhow::Error>(())
+                })
+                .await?;
             progress.as_ref()(FoundryPrepareProgressPayload::runtime(
                 alias,
                 "Foundry Local runtime components",
@@ -1662,15 +1693,16 @@ mod imp {
         }
 
         use super::{
-            cpu_load_completion, foundry_native_dir_candidates, is_cuda_cudnn_failure,
-            is_cuda_fallback_candidate, may_reuse_loaded_model, normalized_language_hint,
-            select_cpu_variant_id, select_foundry_native_dir,
+            collect_foundry_transcription_text, cpu_load_completion, foundry_native_dir_candidates,
+            is_cuda_cudnn_failure, is_cuda_fallback_candidate, may_reuse_loaded_model,
+            normalized_language_hint, select_cpu_variant_id, select_foundry_native_dir,
             should_release_temporary_cpu_fallback, transcribe_recording_with_adapter,
             FoundryCpuLoadCompletion, FoundryCpuSwitch, FoundryExecutionAdapter,
             FoundryExecutionDevice, FoundryFallbackNotice, FoundryFallbackNoticeCallback,
             FoundryLocalRuntime, FoundryVariantDescriptor,
         };
         use anyhow::Result;
+        use foundry_local_sdk::AudioTranscriptionResponse;
         use std::{
             collections::VecDeque,
             fs,
@@ -1823,6 +1855,45 @@ mod imp {
             (callback, received)
         }
 
+        fn transcription_response(text: &str) -> AudioTranscriptionResponse {
+            AudioTranscriptionResponse {
+                text: text.to_string(),
+                language: None,
+                duration: None,
+                segments: None,
+                words: None,
+            }
+        }
+
+        #[tokio::test]
+        async fn foundry_streaming_transcription_preserves_ordered_unicode_chunks() {
+            let stream = futures_util::stream::iter([
+                Ok::<_, &'static str>(transcription_response("中文")),
+                Ok(transcription_response("")),
+                Ok(transcription_response("转写完成")),
+            ]);
+
+            assert_eq!(
+                collect_foundry_transcription_text(stream).await.unwrap(),
+                "中文转写完成"
+            );
+        }
+
+        #[tokio::test]
+        async fn foundry_streaming_transcription_propagates_chunk_errors() {
+            let stream = futures_util::stream::iter([
+                Ok(transcription_response("partial")),
+                Err("stream failed"),
+            ]);
+
+            assert_eq!(
+                collect_foundry_transcription_text(stream)
+                    .await
+                    .unwrap_err(),
+                "stream failed"
+            );
+        }
+
         #[tokio::test]
         async fn cuda_failure_retries_the_failed_chunk_once_on_cpu_and_keeps_cpu_for_later_chunks()
         {

openless-all/app/src-tauri/src/commands/foundry_asr.rs

@@ -38,6 +38,7 @@ pub struct FoundryStatusWire {
     pub runtime_source: String,
     pub active_model: String,
     pub loaded_model_id: Option<String>,
+    pub keep_loaded_secs: u32,
     pub endpoint: Option<String>,
     pub error: Option<String>,
 }
@@ -51,6 +52,7 @@ impl From<openless_core::LocalAsrRuntimeStatus> for FoundryStatusWire {
             runtime_source: status.runtime_source.unwrap_or_default().as_str().into(),
             active_model: status.active_model,
             loaded_model_id: status.model_id,
+            keep_loaded_secs: status.keep_loaded_secs,
             endpoint: status.endpoint,
             error: status.error,
         }
@@ -140,6 +142,19 @@ pub async fn foundry_local_asr_set_runtime_source(
         .map_err(core_error)
 }
 
+#[tauri::command]
+pub async fn foundry_local_asr_set_keep_loaded_secs(
+    backend: CoreState<'_>,
+    seconds: u32,
+) -> Result<(), String> {
+    backend
+        .services()
+        .local_asr
+        .set_keep_loaded_secs(LocalAsrRuntime::Foundry, seconds)
+        .await
+        .map_err(core_error)
+}
+
 #[tauri::command]
 pub async fn foundry_local_asr_prepare(
     backend: CoreState<'_>,
@@ -248,6 +263,7 @@ mod wire_contract_tests {
                 "runtimeSource": "ort-nightly",
                 "activeModel": "whisper-small",
                 "loadedModelId": null,
+                "keepLoadedSecs": 300,
                 "endpoint": null,
                 "error": null,
             })

openless-all/app/src-tauri/src/core_adapters.rs

@@ -2256,6 +2256,12 @@ fn pcm_duration_ms(bytes: &[u8]) -> u64 {
     (bytes.len() as u64 / 2).saturating_mul(1_000) / 16_000
 }
 
+#[cfg(any(target_os = "windows", target_os = "macos", target_os = "linux", test))]
+fn local_asr_release_delay(keep_loaded_secs: u32) -> Option<std::time::Duration> {
+    (keep_loaded_secs != openless_core::LOCAL_ASR_KEEP_LOADED_FOREVER_SECS)
+        .then(|| std::time::Duration::from_secs(keep_loaded_secs as u64))
+}
+
 #[cfg(target_os = "windows")]
 fn schedule_foundry_release(
     runtime: Arc<crate::asr::local::FoundryLocalRuntime>,
@@ -2279,8 +2285,11 @@ fn schedule_foundry_release(
                 }
             }
         }
-        if keep_loaded_secs > 0 {
-            tokio::time::sleep(std::time::Duration::from_secs(keep_loaded_secs as u64)).await;
+        let Some(delay) = local_asr_release_delay(keep_loaded_secs) else {
+            return;
+        };
+        if !delay.is_zero() {
+            tokio::time::sleep(delay).await;
         }
         if current_generation.load(Ordering::Acquire) != generation {
             return;
@@ -2308,9 +2317,12 @@ fn schedule_sherpa_release(
     generation: u64,
     current_generation: Arc<AtomicU64>,
 ) {
+    let Some(delay) = local_asr_release_delay(keep_loaded_secs) else {
+        return;
+    };
     tauri::async_runtime::spawn(async move {
-        if keep_loaded_secs > 0 {
-            tokio::time::sleep(std::time::Duration::from_secs(keep_loaded_secs as u64)).await;
+        if !delay.is_zero() {
+            tokio::time::sleep(delay).await;
         }
         if current_generation.load(Ordering::Acquire) == generation {
             if let Err(error) = runtime
@@ -2329,8 +2341,10 @@ fn schedule_qwen_release(
     engine: std::sync::Weak<crate::asr::local::LocalQwenEngine>,
     keep_loaded_secs: u32,
 ) {
+    let Some(threshold) = local_asr_release_delay(keep_loaded_secs) else {
+        return;
+    };
     tauri::async_runtime::spawn(async move {
-        let threshold = std::time::Duration::from_secs(keep_loaded_secs as u64);
         if !threshold.is_zero() {
             tokio::time::sleep(threshold).await;
         }
@@ -2344,8 +2358,10 @@ fn schedule_whisper_release(
     engine: std::sync::Weak<crate::asr::local::WhisperEngine>,
     keep_loaded_secs: u32,
 ) {
+    let Some(threshold) = local_asr_release_delay(keep_loaded_secs) else {
+        return;
+    };
     tauri::async_runtime::spawn(async move {
-        let threshold = std::time::Duration::from_secs(keep_loaded_secs as u64);
         if !threshold.is_zero() {
             tokio::time::sleep(threshold).await;
         }
@@ -3354,6 +3370,22 @@ mod tests {
 
     struct IgnoreTextStreamSink;
 
+    #[test]
+    fn local_asr_keep_loaded_delay_distinguishes_immediate_finite_and_forever() {
+        assert_eq!(
+            super::local_asr_release_delay(0),
+            Some(std::time::Duration::ZERO)
+        );
+        assert_eq!(
+            super::local_asr_release_delay(300),
+            Some(std::time::Duration::from_secs(300))
+        );
+        assert_eq!(
+            super::local_asr_release_delay(openless_core::LOCAL_ASR_KEEP_LOADED_FOREVER_SECS),
+            None
+        );
+    }
+
     #[cfg(target_os = "windows")]
     #[tokio::test]
     async fn windows_preload_requires_the_requested_model_to_be_prepared() {