fix(foundry): keep local ASR warm and stream transcripts

docs/linux-egui-command-event-baseline.json

@@ -63,6 +63,7 @@
     "foundry_local_asr_release",
     "foundry_local_asr_reveal_model_dir",
     "foundry_local_asr_set_language_hint",
+    "foundry_local_asr_set_keep_loaded_secs",
     "foundry_local_asr_set_model",
     "foundry_local_asr_set_runtime_source",
     "foundry_local_asr_status",

openless-all/app/crates/openless-core/src/lib.rs

@@ -319,6 +319,7 @@ pub use settings::*;
 pub use shared_types::{
     CapsulePayload, CapsuleState, CapsuleStyle, CredentialsStatus, HotkeyMode, HotkeyStatus,
     PendingCorrection, PlatformCapabilities, SelectionPolishOutputMode, UserPreferences,
+    LOCAL_ASR_KEEP_LOADED_FOREVER_SECS,
 };
 pub use shortcut_types::{
     binding_from_legacy_trigger, binding_requires_side_aware_hook, bindings_overlap,

openless-all/app/crates/openless-core/src/local_asr_service.rs

@@ -178,6 +178,8 @@ pub trait ModelRuntimeAdapter: Send + Sync {
         self.test_model(target, model_dir)
     }
 
+    fn invalidate_scheduled_release(&self, _runtime: LocalAsrRuntime) {}
+
     fn invalidate_route(&self, _runtime: LocalAsrRuntime) {}
 }
 
@@ -1151,6 +1153,12 @@ impl LocalAsrApi for LocalAsrService {
             LocalAsrRuntime::Foundry => preferences.foundry_local_asr_keep_loaded_secs = seconds,
             LocalAsrRuntime::SherpaOnnx => preferences.sherpa_onnx_keep_loaded_secs = seconds,
         });
+        if result.is_ok()
+            && runtime == LocalAsrRuntime::Foundry
+            && seconds == crate::LOCAL_ASR_KEEP_LOADED_FOREVER_SECS
+        {
+            self.runtime.invalidate_scheduled_release(runtime);
+        }
         let preferences = Arc::clone(&self.preferences);
         let adapter = Arc::clone(&self.runtime);
         let model_store = Arc::clone(&self.model_store);

openless-all/app/crates/openless-core/src/shared_types.rs

@@ -19,6 +19,9 @@ pub use crate::android_types::{
 
 pub use crate::types::{HistorySource, PolishMode};
 
+/// 本地 ASR 保持加载设置的兼容值：不自动释放，仅由显式操作或进程退出卸载。
+pub const LOCAL_ASR_KEEP_LOADED_FOREVER_SECS: u32 = 86_400;
+
 /// 识别管线模式（issue #902）：`traditional` = 两段式 ASR + LLM 润色；
 /// `multimodal` = 单个多模态模型一步完成「音频 + 提示词 → 最终文本」。
 /// 两套配置在凭据库中完全隔离，运行时只读当前模式，切换不删除另一套配置。
@@ -530,7 +533,7 @@ pub struct UserPreferences {
     #[serde(default = "default_local_asr_mirror")]
     pub local_asr_mirror: String,
     /// 本地 ASR 引擎在内存中的保留时长（秒）。0 = 说完话即释放；
-    /// 较大值 = 上次使用后驻留 N 秒再释放；86400 = 一天 ≈ 永不释放。
+    /// 较大值 = 上次使用后驻留 N 秒再释放；86400 = 永不自动释放。
     /// 默认 300（5 分钟）：兼顾连续听写不重加载、长时间不用释放 1.2GB+ RAM。
     #[serde(default = "default_local_asr_keep_loaded_secs")]
     pub local_asr_keep_loaded_secs: u32,

openless-all/app/crates/openless-core/tests/local_asr_contract.rs

@@ -69,6 +69,7 @@ fn public_local_asr_preferences_keep_legacy_normalization_semantics() {
 #[derive(Default)]
 struct RecordingLocalAsrRuntime {
     invalidated: Mutex<Vec<LocalAsrRuntime>>,
+    invalidated_release_schedules: Mutex<Vec<LocalAsrRuntime>>,
     fail_release: std::sync::atomic::AtomicBool,
     fail_prepare: std::sync::atomic::AtomicBool,
     fail_preload: std::sync::atomic::AtomicBool,
@@ -359,6 +360,13 @@ impl ModelRuntimeAdapter for RecordingLocalAsrRuntime {
     fn invalidate_route(&self, runtime: LocalAsrRuntime) {
         self.invalidated.lock().unwrap().push(runtime);
     }
+
+    fn invalidate_scheduled_release(&self, runtime: LocalAsrRuntime) {
+        self.invalidated_release_schedules
+            .lock()
+            .unwrap()
+            .push(runtime);
+    }
 }
 
 #[derive(Default)]
@@ -1541,6 +1549,8 @@ async fn backend_local_asr_service_owns_preferences_and_change_events() {
     assert_eq!(preferences.sherpa_onnx_language_hint, "zh-hans");
     assert_eq!(preferences.foundry_local_runtime_source, "ort-nightly");
     assert_eq!(preferences.foundry_local_asr_keep_loaded_secs, 42);
+    assert_eq!(preferences.local_asr_keep_loaded_secs, 300);
+    assert_eq!(preferences.sherpa_onnx_keep_loaded_secs, 300);
     assert_eq!(
         runtime.invalidated.lock().unwrap().as_slice(),
         [LocalAsrRuntime::Foundry, LocalAsrRuntime::Foundry]
@@ -1554,6 +1564,53 @@ async fn backend_local_asr_service_owns_preferences_and_change_events() {
     let _ = std::fs::remove_dir_all(data_dir);
 }
 
+#[tokio::test]
+async fn foundry_never_release_invalidates_only_the_pending_finite_schedule() {
+    let (data_dir, runtime, backend) = local_asr_backend();
+
+    for seconds in [0, 60, 300, 1_800] {
+        backend
+            .services()
+            .local_asr
+            .set_keep_loaded_secs(LocalAsrRuntime::Foundry, seconds)
+            .await
+            .unwrap();
+    }
+    assert!(runtime
+        .invalidated_release_schedules
+        .lock()
+        .unwrap()
+        .is_empty());
+
+    backend
+        .services()
+        .local_asr
+        .set_keep_loaded_secs(
+            LocalAsrRuntime::Foundry,
+            openless_core::LOCAL_ASR_KEEP_LOADED_FOREVER_SECS,
+        )
+        .await
+        .unwrap();
+
+    let preferences = backend.get_preferences();
+    assert_eq!(
+        preferences.foundry_local_asr_keep_loaded_secs,
+        openless_core::LOCAL_ASR_KEEP_LOADED_FOREVER_SECS
+    );
+    assert_eq!(preferences.local_asr_keep_loaded_secs, 300);
+    assert_eq!(preferences.sherpa_onnx_keep_loaded_secs, 300);
+    assert_eq!(
+        runtime
+            .invalidated_release_schedules
+            .lock()
+            .unwrap()
+            .as_slice(),
+        [LocalAsrRuntime::Foundry]
+    );
+
+    let _ = std::fs::remove_dir_all(data_dir);
+}
+
 #[tokio::test]
 async fn backend_local_asr_storage_change_commits_only_after_runtime_quiesces() {
     let (data_dir, runtime, backend) = local_asr_backend();

openless-all/app/scripts/local-asr-polling-contract.test.mjs

@@ -81,6 +81,22 @@ for (const contract of [
   }
 }
 
+for (const contract of [
+  'setFoundryLocalAsrKeepLoadedSecs',
+  'foundryStatus?.keepLoadedSecs ?? 300',
+]) {
+  if (!source.includes(contract)) {
+    throw new Error(`Foundry keep-loaded UI contract is missing: ${contract}`);
+  }
+}
+
+const keepLoadedOptionUses = source.match(/options=\{keepLoadedOptions\}/g) ?? [];
+if (keepLoadedOptionUses.length !== 2) {
+  throw new Error(
+    `Generic and Foundry keep-loaded selectors must share the options, found ${keepLoadedOptionUses.length}`,
+  );
+}
+
 console.log(
   'LocalAsr keeps one refresh poller, pauses it for the download dialog, and preserves stable JSX component types',
 );

openless-all/app/src-tauri/src/asr/local/foundry_runtime.rs

@@ -130,9 +130,12 @@ mod imp {
     }
 
     use anyhow::{Context, Result};
-    use foundry_local_sdk::{DeviceType, FoundryLocalConfig, FoundryLocalManager, Model};
+    use foundry_local_sdk::{
+        AudioTranscriptionResponse, DeviceType, FoundryLocalConfig, FoundryLocalManager, Model,
+    };
+    use futures_util::{Stream, StreamExt};
     use parking_lot::Mutex;
-    use tokio::sync::Mutex as AsyncMutex;
+    use tokio::sync::{Mutex as AsyncMutex, OnceCell};
 
     use super::{
         FoundryCpuFallbackTerminalError, FoundryFallbackNotice, FoundryFallbackNoticeCallback,
@@ -493,6 +496,19 @@ mod imp {
             .ok_or_else(|| anyhow::anyhow!("Foundry Local Whisper total timeout exhausted"))
     }
 
+    async fn collect_foundry_transcription_text<S, E>(
+        mut stream: S,
+    ) -> std::result::Result<String, E>
+    where
+        S: Stream<Item = std::result::Result<AudioTranscriptionResponse, E>> + Unpin,
+    {
+        let mut text = String::new();
+        while let Some(chunk) = stream.next().await {
+            text.push_str(&chunk?.text);
+        }
+        Ok(text)
+    }
+
     struct FoundrySdkExecution<'a> {
         runtime: &'a FoundryLocalRuntime,
         manager: &'static FoundryLocalManager,
@@ -546,16 +562,19 @@ mod imp {
                 client = client.language(language_hint);
             }
             let model_id = self.loaded.model_id.clone();
-            let result = tokio::time::timeout(timeout, client.transcribe(audio_path))
-                .await
-                .with_context(|| {
-                    format!(
-                        "transcribe audio with Foundry model {model_id} timed out after {} seconds",
-                        timeout.as_secs()
-                    )
-                })?
-                .with_context(|| format!("transcribe audio with Foundry model {model_id}"))?;
-            Ok(result.text)
+            let result = tokio::time::timeout(timeout, async {
+                let stream = client.transcribe_streaming(audio_path).await?;
+                collect_foundry_transcription_text(stream).await
+            })
+            .await
+            .with_context(|| {
+                format!(
+                    "transcribe audio with Foundry model {model_id} timed out after {} seconds",
+                    timeout.as_secs()
+                )
+            })?
+            .with_context(|| format!("transcribe audio with Foundry model {model_id}"))?;
+            Ok(result)
         }
 
         async fn switch_to_cpu(
@@ -692,6 +711,8 @@ mod imp {
         /// 仍可中断（`cancel_prepare` + `check_prepare_cancelled`）。若未来要缩小粒度，
         /// 可让下载阶段不持锁、下载完成后重新校验 route epoch 再持锁加载/推理。
         lifecycle: AsyncMutex<()>,
+        /// EP 注册会使 SDK 的模型目录缓存失效；成功后本进程不再重复注册。
+        execution_providers_ready: OnceCell<()>,
         cancel_prepare: Arc<AtomicBool>,
         temporary_cpu_fallback_sequence: AtomicU64,
         route_epoch: AtomicU64,
@@ -708,6 +729,7 @@ mod imp {
         pub fn new() -> Self {
             Self {
                 lifecycle: AsyncMutex::new(()),
+                execution_providers_ready: OnceCell::new(),
                 cancel_prepare: Arc::new(AtomicBool::new(false)),
                 temporary_cpu_fallback_sequence: AtomicU64::new(0),
                 route_epoch: AtomicU64::new(0),
@@ -1084,24 +1106,33 @@ mod imp {
             ));
             let runtime_progress = Arc::clone(&progress);
             let runtime_alias = alias.to_string();
-            manager
-                .download_and_register_eps_with_progress(
-                    None,
-                    move |ep_name: &str, percent: f64| {
-                        let label = if ep_name.trim().is_empty() {
-                            "Foundry Local runtime components".to_string()
-                        } else {
-                            format!("Foundry Local runtime component: {ep_name}")
-                        };
-                        runtime_progress.as_ref()(FoundryPrepareProgressPayload::runtime(
-                            runtime_alias.clone(),
-                            label,
-                            percent,
-                        ));
-                    },
-                )
-                .await
-                .context("download/register Foundry execution providers")?;
+            let cancel_prepare = Arc::clone(&self.cancel_prepare);
+            self.execution_providers_ready
+                .get_or_try_init(|| async move {
+                    manager
+                        .download_and_register_eps_with_progress(
+                            None,
+                            move |ep_name: &str, percent: f64| {
+                                let label = if ep_name.trim().is_empty() {
+                                    "Foundry Local runtime components".to_string()
+                                } else {
+                                    format!("Foundry Local runtime component: {ep_name}")
+                                };
+                                runtime_progress.as_ref()(FoundryPrepareProgressPayload::runtime(
+                                    runtime_alias.clone(),
+                                    label,
+                                    percent,
+                                ));
+                            },
+                        )
+                        .await
+                        .context("download/register Foundry execution providers")?;
+                    if cancel_prepare.load(Ordering::SeqCst) {
+                        anyhow::bail!("Foundry Local Whisper prepare cancelled");
+                    }
+                    Ok::<(), anyhow::Error>(())
+                })
+                .await?;
             progress.as_ref()(FoundryPrepareProgressPayload::runtime(
                 alias,
                 "Foundry Local runtime components",
@@ -1662,15 +1693,16 @@ mod imp {
         }
 
         use super::{
-            cpu_load_completion, foundry_native_dir_candidates, is_cuda_cudnn_failure,
-            is_cuda_fallback_candidate, may_reuse_loaded_model, normalized_language_hint,
-            select_cpu_variant_id, select_foundry_native_dir,
+            collect_foundry_transcription_text, cpu_load_completion, foundry_native_dir_candidates,
+            is_cuda_cudnn_failure, is_cuda_fallback_candidate, may_reuse_loaded_model,
+            normalized_language_hint, select_cpu_variant_id, select_foundry_native_dir,
             should_release_temporary_cpu_fallback, transcribe_recording_with_adapter,
             FoundryCpuLoadCompletion, FoundryCpuSwitch, FoundryExecutionAdapter,
             FoundryExecutionDevice, FoundryFallbackNotice, FoundryFallbackNoticeCallback,
             FoundryLocalRuntime, FoundryVariantDescriptor,
         };
         use anyhow::Result;
+        use foundry_local_sdk::AudioTranscriptionResponse;
         use std::{
             collections::VecDeque,
             fs,
@@ -1823,6 +1855,45 @@ mod imp {
             (callback, received)
         }
 
+        fn transcription_response(text: &str) -> AudioTranscriptionResponse {
+            AudioTranscriptionResponse {
+                text: text.to_string(),
+                language: None,
+                duration: None,
+                segments: None,
+                words: None,
+            }
+        }
+
+        #[tokio::test]
+        async fn foundry_streaming_transcription_concatenates_ordered_responses() {
+            let stream = futures_util::stream::iter([
+                Ok::<_, &'static str>(transcription_response("中文")),
+                Ok(transcription_response("")),
+                Ok(transcription_response("转写完成")),
+            ]);
+
+            assert_eq!(
+                collect_foundry_transcription_text(stream).await.unwrap(),
+                "中文转写完成"
+            );
+        }
+
+        #[tokio::test]
+        async fn foundry_streaming_transcription_propagates_chunk_errors() {
+            let stream = futures_util::stream::iter([
+                Ok(transcription_response("partial")),
+                Err("stream failed"),
+            ]);
+
+            assert_eq!(
+                collect_foundry_transcription_text(stream)
+                    .await
+                    .unwrap_err(),
+                "stream failed"
+            );
+        }
+
         #[tokio::test]
         async fn cuda_failure_retries_the_failed_chunk_once_on_cpu_and_keeps_cpu_for_later_chunks()
         {