From 75c53185191c1d3e13ef2bf0bf6641f64ad60c5e Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 28 Sep 2026 01:29:08 +0800 Subject: [PATCH 1/2] fix(omni): let selection/less voice respect multimodal pipeline (#1119) Stop forcing AsrOnly/Traditional so multimodal capture uses PCM+Omni for instruction text while intent/EditPlan stay on LLM. --- .../app/crates/openless-core/src/api.rs | 501 ++++++++++++++---- .../openless-core/src/cloud_providers.rs | 21 + .../app/crates/openless-core/src/prompts.rs | 18 + .../src/selection_voice_service.rs | 13 +- .../tests/auxiliary_voice_lifecycle.rs | 29 +- 5 files changed, 476 insertions(+), 106 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 408e7e704..558628e55 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -11,7 +11,7 @@ use crate::config::{BackendConfig, BackendDependencies, Clock, SystemClock, Task use crate::correction::apply_correction_rules; use crate::credentials::{ ChannelKind, ChannelMutation, ChannelMutationResult, ChannelSummary, CredentialKey, - ProviderSlot, SecretValue, + CredentialStore, ProviderSlot, SecretValue, }; use crate::dictation_context::{ DictationAudioSource, DictationContext, DictationOutputTarget, DictationProviderInvocations, @@ -108,7 +108,10 @@ pub struct DictationHotkeyDispatchOptions { #[derive(Clone, Copy, PartialEq, Eq)] enum DictationContextPurpose { Dictation, - AsrOnly, + /// Selection Voice + Less Computer: respect `effective_pipeline_mode`, but + /// skip resolving an unused LLM at capture time (intent/EditPlan/Agent run + /// later). Replaces the old `AsrOnly` force-Traditional gate (#1119 B). + AuxiliaryVoice, QaText, QaVoice, } @@ -166,13 +169,26 @@ pub enum LessComputerVoiceFinish { pub struct VoiceTranscriptionSession { session_id: SessionId, - transcription: Arc, + /// Traditional ASR session. Absent when multimodal Omni will produce text. + transcription: Option>, + /// Multimodal PCM buffer filled by `start_audio_capture` (or host feed). + pcm: Option>, + /// Credentials + context + prompt for Omni finish. Present with `pcm`. + omni: Option, recording: Mutex>>, partials: Arc, lifecycle: Arc, task_spawner: Arc, } +/// Shared Omni finish inputs for Selection Voice / Less Computer multimodal. +#[derive(Clone)] +struct AuxiliaryOmniFinish { + credentials: Arc, + context: Arc, + system_prompt: String, +} + pub struct QaVoiceCaptureSession { context: Arc, recording: Mutex>>, @@ -647,7 +663,9 @@ struct VoiceTranscriptSink { } struct VoiceCaptureControl { - transcription: Arc, + transcription: Option>, + pcm: Option>, + omni: Option, recording: Mutex>>, closed: std::sync::atomic::AtomicBool, resources: Mutex>>, @@ -943,7 +961,10 @@ impl VoiceCaptureControl { Some(recording) => recording.stop().await, None => Ok(()), }; - let transcription = control.transcription.cancel().await; + let transcription = match control.transcription.as_ref() { + Some(transcription) => transcription.cancel().await, + None => Ok(()), + }; control .feedback .lock() @@ -1063,7 +1084,12 @@ impl LessComputerVoiceSession { "Less Computer PCM must be non-empty and contain complete 16-bit samples", )); } - self.control.transcription.consume_pcm_chunk(pcm); + if let Some(transcription) = self.control.transcription.as_ref() { + transcription.consume_pcm_chunk(pcm); + } + if let Some(buffer) = self.control.pcm.as_ref() { + buffer.consume_pcm_chunk(pcm); + } Ok(()) } @@ -1122,7 +1148,9 @@ impl LessComputerVoiceSession { } let control = Arc::clone(&self.control); let controls = Arc::clone(&self.controls); - let transcription = Arc::clone(&control.transcription); + let transcription = control.transcription.clone(); + let pcm = control.pcm.clone(); + let omni = control.omni.clone(); let recording = control.take_recording(); let archive = recording.as_ref().and_then(|recording| recording.archive()); let archive_successful_recording = self.archive_successful_recording; @@ -1162,7 +1190,9 @@ impl LessComputerVoiceSession { if let Some(recording) = recording { let _ = recording.stop().await; } - let _ = transcription.cancel().await; + if let Some(transcription) = transcription.as_ref() { + let _ = transcription.cancel().await; + } let _ = less_computer.abort_capture(session_id); return Err(BackendError::new( BackendErrorCode::Cancelled, @@ -1171,7 +1201,9 @@ impl LessComputerVoiceSession { } if let Some(recording) = recording { if let Err(error) = recording.stop().await { - let _ = transcription.cancel().await; + if let Some(transcription) = transcription.as_ref() { + let _ = transcription.cancel().await; + } return Err(fail_less_voice_finish( &less_computer, session_id, @@ -1182,23 +1214,75 @@ impl LessComputerVoiceSession { .await); } } - let transcript = match transcription.finish().await { - Ok(output) => output.text, - Err(error) => { - let _ = transcription.cancel().await; + let transcript = match (&transcription, &pcm, &omni) { + (Some(transcription), _, _) => match transcription.finish().await { + Ok(output) => output.text, + Err(error) => { + let _ = transcription.cancel().await; + return Err(fail_less_voice_finish( + &less_computer, + session_id, + &feedback, + mode, + error, + ) + .await); + } + }, + (None, Some(pcm), Some(omni)) => { + let wav = match crate::audio::encode_dictation_wav(&pcm.snapshot()) { + Ok(wav) => wav, + Err(error) => { + return Err(fail_less_voice_finish( + &less_computer, + session_id, + &feedback, + mode, + error, + ) + .await); + } + }; + match crate::cloud_providers::complete_omni_instruction_from_wav( + omni.credentials.as_ref(), + omni.context.as_ref(), + &wav, + &omni.system_prompt, + ) + .await + { + Ok(text) => text, + Err(error) => { + return Err(fail_less_voice_finish( + &less_computer, + session_id, + &feedback, + mode, + error, + ) + .await); + } + } + } + _ => { return Err(fail_less_voice_finish( &less_computer, session_id, &feedback, mode, - error, + BackendError::new( + BackendErrorCode::Internal, + "Less Computer voice capture has an invalid pipeline shape", + ), ) .await); } }; if less_computer.capture_cancelled(session_id) { feedback.settle(LessComputerVoiceOutcome::Cancelled, None); - let _ = transcription.cancel().await; + if let Some(transcription) = transcription.as_ref() { + let _ = transcription.cancel().await; + } let _ = less_computer.abort_capture(session_id); return Err(BackendError::new( BackendErrorCode::Cancelled, @@ -1343,7 +1427,9 @@ impl VoiceTranscriptionSession { .lock() .expect("voice transcription recording lock poisoned") .take(); - let transcription = Arc::clone(&self.transcription); + let transcription = self.transcription.clone(); + let pcm = self.pcm.clone(); + let omni = self.omni.clone(); let partials = Arc::clone(&self.partials); let lifecycle = Arc::clone(&self.lifecycle); let resources = lifecycle.resources(); @@ -1358,7 +1444,29 @@ impl VoiceTranscriptionSession { if lifecycle.is_cancelled() { return Err(VoiceCaptureLifecycle::cancelled_error()); } - let transcript = transcription.finish().await?.text.trim().to_string(); + let transcript = match (&transcription, &pcm, &omni) { + (Some(transcription), _, _) => { + transcription.finish().await?.text.trim().to_string() + } + (None, Some(pcm), Some(omni)) => { + // #1119 B: multimodal first stage — Omni audio→instruction text. + // Host keeps finish() -> String so selection_voice_session stays thin. + let wav = crate::audio::encode_dictation_wav(&pcm.snapshot())?; + crate::cloud_providers::complete_omni_instruction_from_wav( + omni.credentials.as_ref(), + omni.context.as_ref(), + &wav, + &omni.system_prompt, + ) + .await? + } + _ => { + return Err(BackendError::new( + BackendErrorCode::Internal, + "voice transcription capture has an invalid pipeline shape", + )); + } + }; if transcript.is_empty() { return Err(BackendError::new( BackendErrorCode::Provider, @@ -1370,7 +1478,9 @@ impl VoiceTranscriptionSession { .await; let (result, cancel_provider) = lifecycle.settle(result); if cancel_provider { - let _ = transcription.cancel().await; + if let Some(transcription) = transcription.as_ref() { + let _ = transcription.cancel().await; + } } lifecycle.release_resources(); let transcript = result?; @@ -1389,7 +1499,7 @@ impl VoiceTranscriptionSession { .lock() .expect("voice transcription recording lock poisoned") .take(); - let transcription = Arc::clone(&self.transcription); + let transcription = self.transcription.clone(); let lifecycle = Arc::clone(&self.lifecycle); let resources = lifecycle.resources(); own_voice_effect( @@ -1403,7 +1513,12 @@ impl VoiceTranscriptionSession { None => Ok(()), } }, - transcription.cancel(), + async move { + match transcription { + Some(transcription) => transcription.cancel().await, + None => Ok(()), + } + }, ) .await; lifecycle.release_resources(); @@ -2912,7 +3027,7 @@ impl OpenLessBackend { let context = match self .capture_dictation_context( &DictationStartOptions::default(), - DictationContextPurpose::AsrOnly, + DictationContextPurpose::AuxiliaryVoice, ) .await { @@ -2944,48 +3059,102 @@ impl OpenLessBackend { })), feedback: Arc::clone(&feedback), }); - let voice_capture = own_voice_start( - &self.deps.task_spawner, - Arc::clone(&resources), - self.deps.dictation_engine.start_voice_capture( - session_id, - Arc::clone(&context), - Arc::clone(&partials) as Arc, - Arc::clone(&recording_progress) as Arc, - resources.cancel.clone(), - ), - discard_voice_capture, - ) - .await; - let (transcription, recording) = match voice_capture { - Ok(capture) => (capture.transcription, Some(capture.recording)), - Err(error) if error.code == BackendErrorCode::Unsupported => { - ensure_capture()?; - match own_voice_start( - &self.deps.task_spawner, - Arc::clone(&resources), - Arc::clone(&self.deps.dictation_engine).start_transcription_with_progress( - Arc::clone(&self.deps.task_spawner), - session_id, - Arc::clone(&context), - Arc::clone(&partials) as Arc, - Arc::clone(&recording_progress) - as Arc, - ), - |transcription| transcription.cancel(), - ) - .await - { - Ok(session) => (session, None), - Err(error) => { - let _ = self.deps.services.less_computer.abort_capture(session_id); - return Err(error); - } + let omni_finish = AuxiliaryOmniFinish { + credentials: Arc::clone(&self.deps.credential_store), + context: Arc::clone(&context), + system_prompt: crate::prompts::auxiliary_voice_omni_instruction_prompt(), + }; + let (transcription, recording, pcm, omni) = if context.pipeline_mode + == crate::shared_types::PipelineMode::Multimodal + { + let audio_capture = own_voice_start( + &self.deps.task_spawner, + Arc::clone(&resources), + self.deps.dictation_engine.start_audio_capture( + session_id, + Arc::clone(&context), + Arc::clone(&recording_progress) + as Arc, + resources.cancel.clone(), + ), + |capture| { + Box::pin(async move { stop_and_discard_recording(capture.recording).await }) + }, + ) + .await; + match audio_capture { + Ok(capture) => ( + None, + Some(capture.recording), + Some(capture.pcm), + Some(omni_finish), + ), + Err(error) if error.code == BackendErrorCode::Unsupported => { + // Host/external PCM feed path (same seam as ASR Unsupported). + ensure_capture()?; + ( + None, + None, + Some(Arc::new(CapturedPcm::default())), + Some(omni_finish), + ) + } + Err(error) => { + let _ = self.deps.services.less_computer.abort_capture(session_id); + return Err(error); } } - Err(error) => { - let _ = self.deps.services.less_computer.abort_capture(session_id); - return Err(error); + } else { + let voice_capture = own_voice_start( + &self.deps.task_spawner, + Arc::clone(&resources), + self.deps.dictation_engine.start_voice_capture( + session_id, + Arc::clone(&context), + Arc::clone(&partials) as Arc, + Arc::clone(&recording_progress) + as Arc, + resources.cancel.clone(), + ), + discard_voice_capture, + ) + .await; + match voice_capture { + Ok(capture) => ( + Some(capture.transcription), + Some(capture.recording), + None, + None, + ), + Err(error) if error.code == BackendErrorCode::Unsupported => { + ensure_capture()?; + match own_voice_start( + &self.deps.task_spawner, + Arc::clone(&resources), + Arc::clone(&self.deps.dictation_engine) + .start_transcription_with_progress( + Arc::clone(&self.deps.task_spawner), + session_id, + Arc::clone(&context), + Arc::clone(&partials) as Arc, + Arc::clone(&recording_progress) + as Arc, + ), + |transcription| transcription.cancel(), + ) + .await + { + Ok(session) => (Some(session), None, None, None), + Err(error) => { + let _ = self.deps.services.less_computer.abort_capture(session_id); + return Err(error); + } + } + } + Err(error) => { + let _ = self.deps.services.less_computer.abort_capture(session_id); + return Err(error); + } } }; let request = if self.less_computer_capture_cancelled(session_id) { @@ -3000,6 +3169,7 @@ impl OpenLessBackend { Ok(request) => request, Err(error) => { let less_computer = Arc::clone(&self.deps.services.less_computer); + let transcription = transcription.clone(); let _ = own_voice_effect( &self.deps.task_spawner, Box::pin(async move { @@ -3007,7 +3177,9 @@ impl OpenLessBackend { if let Some(recording) = recording { let _ = recording.stop().await; } - let _ = transcription.cancel().await; + if let Some(transcription) = transcription { + let _ = transcription.cancel().await; + } less_computer.abort_capture(session_id) }), ) @@ -3017,6 +3189,8 @@ impl OpenLessBackend { }; let control = Arc::new(VoiceCaptureControl { transcription, + pcm, + omni, recording: Mutex::new(recording), closed: std::sync::atomic::AtomicBool::new(false), resources: Mutex::new(Some(resources)), @@ -3095,7 +3269,7 @@ impl OpenLessBackend { let mut context = self .capture_dictation_context( &DictationStartOptions::default(), - DictationContextPurpose::AsrOnly, + DictationContextPurpose::AuxiliaryVoice, ) .await?; context.recording.archive_enabled = false; @@ -3113,34 +3287,68 @@ impl OpenLessBackend { transcript: Mutex::new(crate::types::TranscriptAccumulator::default()), feedback: None, }); - let capture = own_voice_start( - &self.deps.task_spawner, - Arc::clone(&resources), - self.deps.dictation_engine.start_voice_capture( - session_id, - context, - Arc::clone(&partials) as Arc, - Arc::new(SelectionVoiceRecordingProgress { - session_id, - selection_voice: Arc::clone(&self.deps.services.selection_voice), - control, - task_spawner: Arc::clone(&self.deps.task_spawner), - started_at, - silence: Mutex::new(silence), - }), - resources.cancel.clone(), - ), - discard_voice_capture, - ) - .await?; - Ok(VoiceTranscriptionSession { + let recording_progress = Arc::new(SelectionVoiceRecordingProgress { session_id, - transcription: capture.transcription, - recording: Mutex::new(Some(capture.recording)), - partials, - lifecycle: Arc::new(VoiceCaptureLifecycle::with_resources(resources)), + selection_voice: Arc::clone(&self.deps.services.selection_voice), + control, task_spawner: Arc::clone(&self.deps.task_spawner), - }) + started_at, + silence: Mutex::new(silence), + }); + if context.pipeline_mode == crate::shared_types::PipelineMode::Multimodal { + let capture = own_voice_start( + &self.deps.task_spawner, + Arc::clone(&resources), + self.deps.dictation_engine.start_audio_capture( + session_id, + Arc::clone(&context), + Arc::clone(&recording_progress) as Arc, + resources.cancel.clone(), + ), + |capture| { + Box::pin(async move { stop_and_discard_recording(capture.recording).await }) + }, + ) + .await?; + Ok(VoiceTranscriptionSession { + session_id, + transcription: None, + pcm: Some(capture.pcm), + omni: Some(AuxiliaryOmniFinish { + credentials: Arc::clone(&self.deps.credential_store), + context, + system_prompt: crate::prompts::auxiliary_voice_omni_instruction_prompt(), + }), + recording: Mutex::new(Some(capture.recording)), + partials, + lifecycle: Arc::new(VoiceCaptureLifecycle::with_resources(resources)), + task_spawner: Arc::clone(&self.deps.task_spawner), + }) + } else { + let capture = own_voice_start( + &self.deps.task_spawner, + Arc::clone(&resources), + self.deps.dictation_engine.start_voice_capture( + session_id, + Arc::clone(&context), + Arc::clone(&partials) as Arc, + Arc::clone(&recording_progress) as Arc, + resources.cancel.clone(), + ), + discard_voice_capture, + ) + .await?; + Ok(VoiceTranscriptionSession { + session_id, + transcription: Some(capture.transcription), + pcm: None, + omni: None, + recording: Mutex::new(Some(capture.recording)), + partials, + lifecycle: Arc::new(VoiceCaptureLifecycle::with_resources(resources)), + task_spawner: Arc::clone(&self.deps.task_spawner), + }) + } } #[doc(hidden)] @@ -6503,17 +6711,14 @@ impl OpenLessBackend { .filter(|entry| entry.enabled) .map(|entry| entry.phrase) .collect(); - // Less/Selection audio always uses ASR. QA text needs no microphone - // provider; Omni needs neither traditional channel. An unused channel - // must not turn a valid route into a startup failure. - let pipeline_mode = if purpose == DictationContextPurpose::AsrOnly { - crate::shared_types::PipelineMode::Traditional - } else { - crate::shared_types::effective_pipeline_mode( - preferences.multimodal_pipeline_enabled, - preferences.pipeline_mode, - ) - }; + // Auxiliary Voice (Selection / Less) and QA Voice respect multimodal + // preferences. Skip unused traditional channels so a valid Omni route + // is not blocked by missing ASR/LLM. QA text needs no microphone + // provider. + let pipeline_mode = crate::shared_types::effective_pipeline_mode( + preferences.multimodal_pipeline_enabled, + preferences.pipeline_mode, + ); let traditional = pipeline_mode == crate::shared_types::PipelineMode::Traditional; let active_asr_provider = if traditional && purpose != DictationContextPurpose::QaText { self.resolve_session_provider(ProviderSlot::Asr, &preferences.active_asr_provider) @@ -6524,7 +6729,8 @@ impl OpenLessBackend { ) }; let mut deferred_llm_error = None; - let active_llm_provider = if traditional && purpose != DictationContextPurpose::AsrOnly { + let skip_llm_at_capture = purpose == DictationContextPurpose::AuxiliaryVoice; + let active_llm_provider = if traditional && !skip_llm_at_capture { match self .resolve_session_provider(ProviderSlot::Llm, &preferences.active_llm_provider) .await @@ -8108,6 +8314,95 @@ mod tests { assert_eq!(recorder.stop_count(), 2); } + #[tokio::test] + async fn selection_voice_capture_respects_pipeline_mode() { + let data_dir = TestDataDir::new("selection-voice-pipeline-mode"); + let recorder = Arc::new(crate::testing::FixtureAudioRecorder::new( + vec![vec![1, 0, 2, 0]], + vec![], + )); + let transcription = Arc::new(crate::testing::FixtureTranscriptionEngine::successful( + "edit this", + 120, + )); + let engine = crate::PipelineDictationEngine::new( + recorder.clone(), + transcription.clone(), + Arc::new(crate::testing::FixtureTextPolisher::successful("unused")), + ); + let backend = OpenLessBackend::new( + BackendConfig { + data_dir: data_dir.path().to_path_buf(), + ..BackendConfig::default() + }, + BackendDependencies { + dictation_engine: Arc::new(engine), + ..BackendDependencies::unsupported() + }, + ) + .unwrap(); + let mut preferences = backend.get_preferences(); + preferences.selection_voice_enabled = true; + backend.set_preferences(preferences).unwrap(); + + let traditional_id = backend + .services() + .selection_voice + .begin(crate::domains::SelectionCapture { + text: "draft".into(), + source_app: None, + }) + .await + .unwrap(); + let traditional = backend + .start_selection_voice_capture( + traditional_id, + Arc::new(FakeRecordingControl::default()), + ) + .await + .unwrap(); + assert_eq!(traditional.finish().await.unwrap(), "edit this"); + assert_eq!(transcription.pcm(), vec![1, 0, 2, 0]); + backend + .services() + .selection_voice + .cancel(Some(traditional_id)) + .await + .unwrap(); + + let mut preferences = backend.get_preferences(); + preferences.multimodal_pipeline_enabled = true; + preferences.pipeline_mode = crate::shared_types::PipelineMode::Multimodal; + backend.set_preferences(preferences).unwrap(); + let asr_pcm_before = transcription.pcm().len(); + let multimodal_id = backend + .services() + .selection_voice + .begin(crate::domains::SelectionCapture { + text: "draft".into(), + source_app: None, + }) + .await + .unwrap(); + let multimodal = backend + .start_selection_voice_capture( + multimodal_id, + Arc::new(FakeRecordingControl::default()), + ) + .await + .unwrap(); + // Multimodal finish needs Omni credentials; capture must not feed ASR. + assert!(multimodal.finish().await.is_err()); + assert_eq!(transcription.pcm().len(), asr_pcm_before); + assert_eq!(recorder.stop_count(), 2); + backend + .services() + .selection_voice + .cancel(Some(multimodal_id)) + .await + .unwrap(); + } + #[tokio::test] async fn qa_and_selection_voice_capture_can_cancel_during_transcription_finish() { struct PendingTranscription { @@ -8193,7 +8488,9 @@ mod tests { }); let capture = VoiceTranscriptionSession { session_id: SessionId::new(), - transcription: transcription.clone(), + transcription: Some(transcription.clone()), + pcm: None, + omni: None, recording: Mutex::new(None), partials: Arc::new(VoiceTranscriptSink { publisher: backend.event_publisher(), diff --git a/openless-all/app/crates/openless-core/src/cloud_providers.rs b/openless-all/app/crates/openless-core/src/cloud_providers.rs index 5a52a88ad..3be0040f9 100644 --- a/openless-all/app/crates/openless-core/src/cloud_providers.rs +++ b/openless-all/app/crates/openless-core/src/cloud_providers.rs @@ -1708,6 +1708,27 @@ pub async fn validate_shared_omni_provider( Ok(()) } +/// First-stage Omni for Selection Voice / Less Computer (#1119 product B): +/// audio + prompt → instruction/command text. Downstream intent / EditPlan / +/// Agent submit still use their existing LLM (or Agent) paths. +/// +/// Empty Omni output is returned as `Ok("")` so callers can apply product +/// policy (e.g. Less Computer Dictate → `Empty` outcome without a chat error; +/// Selection Voice / Agent still treat emptiness as a provider failure). +pub(crate) async fn complete_omni_instruction_from_wav( + credentials: &dyn CredentialStore, + context: &DictationContext, + wav_bytes: &[u8], + system_prompt: &str, +) -> Result { + let provider = build_omni_provider(credentials, context).await?; + let result = provider + .complete(system_prompt, "", Some(wav_bytes)) + .await + .map_err(map_omni_error)?; + Ok(result.trim().to_string()) +} + fn cancelled_omni_error() -> BackendError { BackendError::new(BackendErrorCode::Cancelled, "Omni request cancelled") } diff --git a/openless-all/app/crates/openless-core/src/prompts.rs b/openless-all/app/crates/openless-core/src/prompts.rs index 57828a5d1..1170c5198 100644 --- a/openless-all/app/crates/openless-core/src/prompts.rs +++ b/openless-all/app/crates/openless-core/src/prompts.rs @@ -270,6 +270,24 @@ pub fn selection_voice_instruction_polish_prompt() -> String { .to_string() } +/// 选区语音编辑 / Less Computer：多模态第一段（音频 → 指令文本，issue #1119 B）。 +/// 意图分类与 EditPlan 仍走下游 LLM；本提示只产出可提交的口述指令。 +pub fn auxiliary_voice_omni_instruction_prompt() -> String { + "# 任务（口述指令转写）\n\ + 用户通过语音描述想做什么（编辑选区、提问，或交给编程助手的指令）。\n\ + 输入是用户口述音频。\n\ + \n\ + ## 要求\n\ + - 转写并整理为一条简洁、可直接交给下游系统的指令句。\n\ + - 保留具体目标（格式、替换规则、翻译方向、提问焦点、编程任务）。\n\ + - 删除无意义口头禅，补全必要标点。\n\ + - 不要臆造输入中没有的内容。\n\ + \n\ + ## 输出\n\ + 只输出指令正文，不要解释、不要标题。" + .to_string() +} + /// 选区语音编辑：EditPlan 路径的对抗式防御（draft / instruction 是数据）。 pub fn voice_edit_injection_defense() -> &'static str { "# 安全约定（务必遵守）\n\ diff --git a/openless-all/app/crates/openless-core/src/selection_voice_service.rs b/openless-all/app/crates/openless-core/src/selection_voice_service.rs index c7faea4f8..20e90616d 100644 --- a/openless-all/app/crates/openless-core/src/selection_voice_service.rs +++ b/openless-all/app/crates/openless-core/src/selection_voice_service.rs @@ -325,6 +325,10 @@ impl SelectionVoiceWorkflow { "selection voice model runtime is not configured", ) })?; + // #1119 product B: first-stage instruction text may already come from Omni + // (audio→text in VoiceTranscriptionSession). Intent classification and + // EditPlan still use this traditional LLM polisher; SharedAuxiliaryTextPolisher + // is intentionally not required for correctness of capture alignment. let llm = crate::provider_resolution::resolve_session_provider( &self.credential_store, ProviderSlot::Llm, @@ -590,6 +594,13 @@ impl SelectionVoicePersistence { SelectionVoiceApplyOutcome::Failed => return, }; let final_chars = ticket.replacement_text.chars().count() as u64; + let pipeline_mode = match crate::shared_types::effective_pipeline_mode( + preferences.multimodal_pipeline_enabled, + preferences.pipeline_mode, + ) { + crate::shared_types::PipelineMode::Traditional => "traditional", + crate::shared_types::PipelineMode::Multimodal => "multimodal", + }; let session = DictationSession { id: ticket.session_id.to_string(), created_at: self.clock.now_utc().to_rfc3339(), @@ -612,7 +623,7 @@ impl SelectionVoicePersistence { asr_model: None, llm_provider: None, llm_model: None, - pipeline_mode: None, + pipeline_mode: Some(pipeline_mode.to_string()), asr_ms: None, polish_ms: None, }; diff --git a/openless-all/app/crates/openless-core/tests/auxiliary_voice_lifecycle.rs b/openless-all/app/crates/openless-core/tests/auxiliary_voice_lifecycle.rs index 222fb1377..3580fb6a5 100644 --- a/openless-all/app/crates/openless-core/tests/auxiliary_voice_lifecycle.rs +++ b/openless-all/app/crates/openless-core/tests/auxiliary_voice_lifecycle.rs @@ -342,7 +342,7 @@ fn backend( #[tokio::test] async fn qa_and_selection_voice_never_request_disk_archives() { - for entry in ["qa", "qa-omni", "selection", "dictation", "less"] { + for entry in ["qa", "qa-omni", "selection", "selection-omni", "dictation", "less"] { let plans = Arc::new(Mutex::new(Vec::new())); let recorder = testing::FixtureAudioRecorder::new(vec![vec![0; 320]], Vec::new()); let (backend, path) = backend( @@ -357,10 +357,10 @@ async fn qa_and_selection_voice_never_request_disk_archives() { Arc::new(QaRuntime::default()), ); // The main-path debug switch must not opt private QA/Selection audio - // into disk retention. Traditional and Omni QA share this boundary. + // into disk retention. Traditional and Omni QA/Selection share this boundary. let mut prefs = backend.get_preferences(); prefs.record_audio_for_debug = true; - if entry == "qa-omni" { + if matches!(entry, "qa-omni" | "selection-omni") { prefs.multimodal_pipeline_enabled = true; prefs.pipeline_mode = shared_types::PipelineMode::Multimodal; } @@ -426,6 +426,29 @@ async fn qa_and_selection_voice_never_request_disk_archives() { .await .unwrap(); } + "selection-omni" => { + let id = backend + .services() + .selection_voice + .begin(SelectionCapture { + text: "selection".into(), + source_app: None, + }) + .await + .unwrap(); + let capture = backend + .start_selection_voice_capture(id, Arc::new(Control)) + .await + .unwrap(); + // Finish would call Omni; cancel is enough to prove archive policy. + let _ = capture.cancel().await; + backend + .services() + .selection_voice + .cancel(Some(id)) + .await + .unwrap(); + } "dictation" => { backend .start_dictation_with_options(DictationStartOptions { From c9e686e1bb877236204751a37b29abd63cebd540 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 28 Sep 2026 02:11:03 +0800 Subject: [PATCH 2/2] test(core): stop FixedClock history fixture from age-pruning itself Retention uses wall-clock Utc::now(), so a 2026-08-28 stamp falls out of a 30-day window and flakes CI. --- openless-all/app/crates/openless-core/src/api.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index 558628e55..1d17580e4 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -11323,7 +11323,10 @@ mod tests { let mut preferences = backend.get_preferences(); preferences.translation_target_language = "English".to_string(); preferences.working_languages = vec!["简体中文".to_string()]; - preferences.history_retention_days = 30; + // Age prune uses wall-clock Utc::now(), while this fixture stamps + // created_at via FixedClock (2026-08-28). Non-zero retention eventually + // deletes the just-written entry once wall time drifts past the window. + preferences.history_retention_days = 0; preferences.history_max_entries = Some(20); backend.set_preferences(preferences).unwrap(); backend.start().await.unwrap();