From 9574f0f23dd68f7552a425557e01be9d71302908 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 19:45:19 +0000 Subject: [PATCH 01/18] feat(voice): add OpenAI text-to-speech --- src-tauri/Cargo.lock | 79 +- src-tauri/Cargo.toml | 1 + src-tauri/src/commands/mod.rs | 1 + src-tauri/src/commands/openai_audio.rs | 867 ++++++++++++++++++ src-tauri/src/lib.rs | 8 + src/app/AppShell.tsx | 5 + src/features/chat/ui/ChatView.tsx | 5 + .../voice-conversation/api/openAiVoice.ts | 71 ++ .../hooks/useOpenAiVoiceSetup.ts | 30 + .../lib/nativeAssistantSpeech.test.ts | 199 ++-- .../lib/nativeAssistantSpeech.ts | 158 +--- .../lib/voiceOutputPreference.ts | 3 +- .../lib/voiceSetupReadiness.test.ts | 10 + .../lib/voiceSetupReadiness.ts | 4 + .../ui/VoiceSettings.test.tsx | 22 + .../voice-conversation/ui/VoiceSettings.tsx | 79 +- src/shared/i18n/locales/en/settings.json | 6 + src/shared/i18n/locales/es/settings.json | 6 + 18 files changed, 1307 insertions(+), 247 deletions(-) create mode 100644 src-tauri/src/commands/openai_audio.rs create mode 100644 src/features/voice-conversation/api/openAiVoice.ts create mode 100644 src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 153965f09..fba337c1a 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -28,6 +28,7 @@ dependencies = [ "hex", "ignore", "infer", + "keyring", "libc", "log", "mime_guess", @@ -45,6 +46,7 @@ dependencies = [ "reqwest 0.13.4", "rodio", "rubato", + "rustls", "semver", "serde", "serde_json", @@ -72,6 +74,7 @@ dependencies = [ "tempfile", "time", "tokio", + "tokio-tungstenite", "toml 1.1.4+spec-1.1.0", "url", "uuid", @@ -1404,6 +1407,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "dbus" version = "0.9.12" @@ -3174,6 +3183,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "keyring" +version = "3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" +dependencies = [ + "log", + "security-framework 2.11.1", + "security-framework 3.7.0", + "zeroize", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -5262,7 +5283,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.7.0", ] [[package]] @@ -5290,7 +5311,7 @@ dependencies = [ "rustls-native-certs", "rustls-platform-verifier-android", "rustls-webpki", - "security-framework", + "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", "windows-sys 0.61.2", @@ -5401,6 +5422,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -7088,6 +7122,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", + "webpki-roots 0.26.11", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -7348,6 +7398,25 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 2.0.20", + "utf-8", +] + [[package]] name = "typeid" version = "1.0.3" @@ -7557,6 +7626,12 @@ dependencies = [ "url", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8-zero" version = "0.8.1" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 9cfaef08c..48a25cea9 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -118,6 +118,7 @@ objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2 objc2-core-audio = "0.3.2" objc2-foundation = { version = "0.3.2", features = ["NSDictionary", "NSError", "NSFileManager", "NSObject", "NSProcessInfo", "NSString", "NSURL"] } objc2-user-notifications = "0.3.2" +keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } rodio = { version = "0.22", default-features = false, features = ["playback"] } diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 74b4a1b4f..04e7ac784 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -37,6 +37,7 @@ pub mod model_setup; mod native_input_mute; pub mod native_voice; pub mod notifications; +pub mod openai_audio; #[cfg(feature = "block-voice-dictation")] pub mod openai_realtime; pub mod path_resolver; diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs new file mode 100644 index 000000000..b59318f7a --- /dev/null +++ b/src-tauri/src/commands/openai_audio.rs @@ -0,0 +1,867 @@ +//! OpenAI streaming speech playback for voice conversations. + +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, Mutex, + }, + time::{Duration, Instant}, +}; + +use futures_util::StreamExt; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE}; +use serde::Serialize; +use serde_json::json; +use tauri::{AppHandle, Emitter, State}; + +#[cfg(target_os = "macos")] +use super::{ + native_voice::{InterruptionSensitivity, NativeVoiceState}, + pocket_audio_player::PocketAudioPlayer, + pocket_voice::{effective_output_device_name, should_suppress_capture, VoiceInterruptionMode}, +}; + +const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; +const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; +const DEFAULT_TTS_VOICE: &str = "marin"; +const TTS_SAMPLE_RATE: u32 = 24_000; +const TTS_EVENT: &str = "openai-voice:stream-event"; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); +const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +const MAX_TTS_INPUT_CHARS: usize = 4096; + +#[derive(Clone, Debug, Default)] +pub struct OpenAiVoiceState { + playback: Arc>, +} + +#[derive(Debug)] +struct PlaybackRuntime { + active: Option>, + stream: Option, + speed: f32, +} + +impl Default for PlaybackRuntime { + fn default() -> Self { + Self { + active: None, + stream: None, + speed: stored_playback_speed(), + } + } +} + +#[derive(Debug)] +struct ActiveOpenAiStream { + id: String, + sender: mpsc::Sender, +} + +#[derive(Debug)] +enum OpenAiStreamCommand { + Append(String), + Flush, + Finish, + Stop, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OpenAiVoiceStatus { + configured: bool, + speech_model: String, + speech_voice: String, + playback_speed: f32, + tts_available: bool, + unavailable_reason: Option, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct OpenAiVoiceStreamEvent { + stream_id: String, + state: OpenAiStreamEventState, + error: Option, + delivery: Option, +} + +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +enum OpenAiStreamEventState { + Started, + Progress, + Completed, + Interrupted, + Failed, +} + +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct VoiceDeliveryProgress { + sample_rate: u32, + segments: Vec, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct VoiceDeliverySegment { + text: String, + played_frames: u64, + total_frames: u64, + synthesis_complete: bool, +} + +fn env_trimmed(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +fn goose_openai_api_key() -> Result, String> { + if let Some(value) = env_trimmed("OPENAI_API_KEY") { + return Ok(Some(value)); + } + #[cfg(target_os = "macos")] + { + let entry = keyring::Entry::new("goose", "secrets").map_err(|error| { + format!("Could not access Goose's secure credential store: {error}") + })?; + match entry.get_password() { + Ok(payload) => { + let secrets: serde_json::Value = + serde_json::from_str(&payload).map_err(|error| { + format!("Goose's secure credential store is not valid JSON: {error}") + })?; + return Ok(secrets + .get("OPENAI_API_KEY") + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string)); + } + Err(keyring::Error::NoEntry) => {} + Err(error) => { + return Err(format!( + "Could not read Goose's OpenAI credential from secure storage: {error}" + )); + } + } + } + let config_path = crate::services::goose_config::config_path()?; + let secrets_path = config_path + .parent() + .ok_or_else(|| "Could not resolve Goose's credential directory".to_string())? + .join("secrets.yaml"); + if !secrets_path.exists() { + return Ok(None); + } + let payload = std::fs::read_to_string(&secrets_path) + .map_err(|error| format!("Could not read Goose's credential file: {error}"))?; + let secrets: serde_json::Value = yaml_serde::from_str(&payload) + .map_err(|error| format!("Goose's credential file is invalid: {error}"))?; + Ok(secrets + .get("OPENAI_API_KEY") + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string)) +} + +pub(crate) fn openai_api_key_available() -> Result { + goose_openai_api_key().map(|key| key.is_some()) +} + +pub(crate) fn api_key() -> Result { + goose_openai_api_key()?.ok_or_else(|| { + "OpenAI voice is not configured. Configure the OpenAI provider in Berd, then try again." + .to_string() + }) +} + +fn base_url() -> String { + env_trimmed("OPENAI_BASE_URL").unwrap_or_else(|| DEFAULT_BASE_URL.to_string()) +} + +fn speech_model() -> String { + env_trimmed("OPENAI_TTS_MODEL").unwrap_or_else(|| DEFAULT_TTS_MODEL.to_string()) +} + +fn speech_voice() -> String { + env_trimmed("OPENAI_TTS_VOICE").unwrap_or_else(|| DEFAULT_TTS_VOICE.to_string()) +} + +fn endpoint(path: &str) -> Result { + let mut url = base_url(); + while url.ends_with('/') { + url.pop(); + } + let path = path.trim_start_matches('/'); + let full = format!("{url}/{path}"); + reqwest::Url::parse(&full) + .map(|_| full) + .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}")) +} + +fn authorized_headers(key: &str) -> Result { + let mut headers = HeaderMap::new(); + let bearer = format!("Bearer {key}"); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&bearer).map_err(|_| "OpenAI API key is not a valid header value")?, + ); + Ok(headers) +} + +fn speed_settings_path() -> Result { + Ok(crate::services::goose_config::config_path()? + .parent() + .ok_or_else(|| "Could not resolve Goose's configuration directory".to_string())? + .join("openai-voice-settings.json")) +} + +fn stored_playback_speed() -> f32 { + speed_settings_path() + .ok() + .and_then(|path| std::fs::read(path).ok()) + .and_then(|data| serde_json::from_slice::(&data).ok()) + .and_then(|value| value.get("playbackSpeed")?.as_f64()) + .map(|speed| speed as f32) + .filter(|speed| speed.is_finite() && (0.75..=2.0).contains(speed)) + .unwrap_or(1.0) +} + +fn persist_playback_speed(speed: f32) -> Result<(), String> { + let path = speed_settings_path()?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| format!("create OpenAI voice settings directory: {error}"))?; + } + std::fs::write( + &path, + serde_json::to_vec_pretty(&json!({ "playbackSpeed": speed })).unwrap(), + ) + .map_err(|error| format!("write OpenAI voice settings: {error}")) +} + +fn client() -> Result { + reqwest::Client::builder() + .connect_timeout(CONNECT_TIMEOUT) + .timeout(REQUEST_TIMEOUT) + .build() + .map_err(|error| format!("create OpenAI HTTP client: {error}")) +} + +#[tauri::command] +pub fn get_openai_voice_status( + state: State<'_, OpenAiVoiceState>, +) -> Result { + let configured = openai_api_key_available()?; + let playback_speed = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? + .speed; + let tts_available = cfg!(target_os = "macos"); + Ok(OpenAiVoiceStatus { + configured, + speech_model: speech_model(), + speech_voice: speech_voice(), + playback_speed, + tts_available, + unavailable_reason: if !configured { + Some("Configure the OpenAI provider in Berd to use OpenAI voice.".to_string()) + } else if !tts_available { + Some("OpenAI voice playback is currently supported on macOS only.".to_string()) + } else { + None + }, + }) +} + +#[tauri::command] +pub fn start_openai_voice_stream( + app: AppHandle, + state: State<'_, OpenAiVoiceState>, + native_voice: State<'_, NativeVoiceState>, + stream_id: String, + interruption_mode: VoiceInterruptionMode, + interruption_sensitivity: InterruptionSensitivity, +) -> Result<(), String> { + #[cfg(not(target_os = "macos"))] + { + let _ = ( + app, + state, + native_voice, + stream_id, + interruption_mode, + interruption_sensitivity, + ); + Err("OpenAI voice playback is currently supported on macOS only".to_string()) + } + + #[cfg(target_os = "macos")] + { + if stream_id.trim().is_empty() { + return Err("OpenAI voice stream id cannot be empty".to_string()); + } + let key = api_key()?; + let (sender, receiver) = mpsc::channel(); + let active = Arc::new(AtomicBool::new(true)); + { + let mut playback = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())?; + if playback.active.is_some() { + return Err("OpenAI voice playback is already active".to_string()); + } + playback.active = Some(active.clone()); + playback.stream = Some(ActiveOpenAiStream { + id: stream_id.clone(), + sender, + }); + } + let speed = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? + .speed; + let playback = state.playback.clone(); + let native_voice = native_voice.inner().clone(); + tauri::async_runtime::spawn_blocking(move || { + let result = run_openai_voice_stream( + &app, + &stream_id, + key, + active.clone(), + receiver, + native_voice, + interruption_mode, + interruption_sensitivity, + speed, + ); + if let Ok(mut playback) = playback.lock() { + playback.active = None; + playback.stream = None; + } + let (state, error, delivery) = match result { + Ok(outcome) => (outcome.state, None, outcome.delivery), + Err(failure) if !active.load(Ordering::SeqCst) => { + (OpenAiStreamEventState::Interrupted, None, failure.delivery) + } + Err(failure) => ( + OpenAiStreamEventState::Failed, + Some(failure.error), + failure.delivery, + ), + }; + emit_openai_stream_event(&app, &stream_id, state, error, delivery); + }); + Ok(()) + } +} + +#[tauri::command] +pub fn append_openai_voice_stream( + state: State<'_, OpenAiVoiceState>, + stream_id: String, + text: String, +) -> Result<(), String> { + if text.is_empty() { + return Ok(()); + } + send_stream_command(&state, &stream_id, OpenAiStreamCommand::Append(text)) +} + +#[tauri::command] +pub fn flush_openai_voice_stream( + state: State<'_, OpenAiVoiceState>, + stream_id: String, +) -> Result<(), String> { + send_stream_command(&state, &stream_id, OpenAiStreamCommand::Flush) +} + +#[tauri::command] +pub fn finish_openai_voice_stream( + state: State<'_, OpenAiVoiceState>, + stream_id: String, +) -> Result<(), String> { + send_stream_command(&state, &stream_id, OpenAiStreamCommand::Finish) +} + +#[tauri::command] +pub fn set_openai_playback_speed( + state: State<'_, OpenAiVoiceState>, + speed: f32, +) -> Result<(), String> { + if !speed.is_finite() || !(0.75..=2.0).contains(&speed) { + return Err("OpenAI playback speed must be between 0.75 and 2.0".to_string()); + } + persist_playback_speed(speed)?; + state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? + .speed = speed; + Ok(()) +} + +pub(crate) fn stop_openai_voice_inner(state: &OpenAiVoiceState) -> Result { + let playback = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())?; + let Some(active) = playback.active.as_ref() else { + return Ok(false); + }; + active.store(false, Ordering::SeqCst); + if let Some(stream) = playback.stream.as_ref() { + let _ = stream.sender.send(OpenAiStreamCommand::Stop); + } + Ok(true) +} + +#[tauri::command] +pub fn stop_openai_voice(state: State<'_, OpenAiVoiceState>) -> Result { + stop_openai_voice_inner(&state) +} + +fn send_stream_command( + state: &OpenAiVoiceState, + stream_id: &str, + command: OpenAiStreamCommand, +) -> Result<(), String> { + let playback = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())?; + let stream = playback + .stream + .as_ref() + .filter(|stream| stream.id == stream_id) + .ok_or_else(|| format!("OpenAI voice stream is not active: {stream_id}"))?; + stream + .sender + .send(command) + .map_err(|_| format!("OpenAI voice stream worker stopped: {stream_id}")) +} + +#[cfg(target_os = "macos")] +struct StreamOutcome { + state: OpenAiStreamEventState, + delivery: Option, +} + +#[cfg(target_os = "macos")] +struct StreamFailure { + error: String, + delivery: Option, +} + +#[cfg(target_os = "macos")] +impl From for StreamFailure { + fn from(error: String) -> Self { + Self { + error, + delivery: None, + } + } +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] // Stream worker keeps lifecycle and playback policy explicit. +fn run_openai_voice_stream( + app: &AppHandle, + stream_id: &str, + key: String, + active: Arc, + receiver: mpsc::Receiver, + native_voice: NativeVoiceState, + interruption_mode: VoiceInterruptionMode, + interruption_sensitivity: InterruptionSensitivity, + speed: f32, +) -> Result { + run_openai_voice_stream_inner( + app, + stream_id, + key, + active, + receiver, + native_voice, + interruption_mode, + interruption_sensitivity, + speed, + ) +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn run_openai_voice_stream_inner( + app: &AppHandle, + stream_id: &str, + key: String, + active: Arc, + receiver: mpsc::Receiver, + native_voice: NativeVoiceState, + interruption_mode: VoiceInterruptionMode, + interruption_sensitivity: InterruptionSensitivity, + speed: f32, +) -> Result { + let client = client()?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("Could not initialize OpenAI speech runtime: {error}"))?; + let player = PocketAudioPlayer::new(TTS_SAMPLE_RATE, 1.0, None)?; + let output_device = effective_output_device_name(None); + let suppress_capture = should_suppress_capture(interruption_mode, output_device.as_deref()); + let _assistant_speech = + native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture); + let mut pending = String::new(); + let mut delivery = VoiceDeliveryProgress { + sample_rate: TTS_SAMPLE_RATE, + segments: Vec::new(), + }; + let mut started = false; + let mut last_progress = Instant::now(); + + loop { + if !active.load(Ordering::SeqCst) { + player.stop(); + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Interrupted, + delivery: Some(snapshot_delivery(&delivery, &player)), + }); + } + match receiver.recv_timeout(Duration::from_millis(20)) { + Ok(OpenAiStreamCommand::Append(text)) => { + pending.push_str(&text); + if pending.len() >= 24 && pending.trim_end().ends_with(['.', '!', '?', '\n']) { + speak_pending( + &runtime, + app, + stream_id, + &client, + &key, + &active, + &player, + &mut pending, + &mut delivery, + &mut started, + speed, + ) + .map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; + } + } + Ok(OpenAiStreamCommand::Flush) => { + speak_pending( + &runtime, + app, + stream_id, + &client, + &key, + &active, + &player, + &mut pending, + &mut delivery, + &mut started, + speed, + ) + .map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; + } + Ok(OpenAiStreamCommand::Finish) => { + speak_pending( + &runtime, + app, + stream_id, + &client, + &key, + &active, + &player, + &mut pending, + &mut delivery, + &mut started, + speed, + )?; + while active.load(Ordering::SeqCst) && !player.is_empty() { + player.ensure_healthy()?; + if last_progress.elapsed() >= Duration::from_millis(100) { + emit_openai_stream_event( + app, + stream_id, + OpenAiStreamEventState::Progress, + None, + Some(snapshot_delivery(&delivery, &player)), + ); + last_progress = Instant::now(); + } + std::thread::sleep(Duration::from_millis(20)); + } + if !active.load(Ordering::SeqCst) { + player.stop(); + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Interrupted, + delivery: Some(snapshot_delivery(&delivery, &player)), + }); + } + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Completed, + delivery: None, + }); + } + Ok(OpenAiStreamCommand::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => { + active.store(false, Ordering::SeqCst); + player.stop(); + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Interrupted, + delivery: Some(snapshot_delivery(&delivery, &player)), + }); + } + Err(mpsc::RecvTimeoutError::Timeout) => { + if started && last_progress.elapsed() >= Duration::from_millis(100) { + emit_openai_stream_event( + app, + stream_id, + OpenAiStreamEventState::Progress, + None, + Some(snapshot_delivery(&delivery, &player)), + ); + last_progress = Instant::now(); + } + } + } + } +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn speak_pending( + runtime: &tokio::runtime::Runtime, + app: &AppHandle, + stream_id: &str, + client: &reqwest::Client, + key: &str, + active: &AtomicBool, + player: &PocketAudioPlayer, + pending: &mut String, + delivery: &mut VoiceDeliveryProgress, + started: &mut bool, + speed: f32, +) -> Result<(), String> { + let text = std::mem::take(pending).trim().to_string(); + if text.is_empty() { + return Ok(()); + } + for chunk in chunk_text(&text, MAX_TTS_INPUT_CHARS) { + if !active.load(Ordering::SeqCst) { + return Ok(()); + } + let mut segment_frames = 0_u64; + delivery.segments.push(VoiceDeliverySegment { + text: chunk.to_string(), + played_frames: 0, + total_frames: 0, + synthesis_complete: false, + }); + let mut bytes = + runtime.block_on(openai_speech_stream(client, key, chunk.to_string(), speed))?; + let mut pcm_remainder = Vec::::new(); + loop { + if !active.load(Ordering::SeqCst) { + return Ok(()); + } + let item = runtime.block_on(async { + tokio::time::timeout(Duration::from_millis(50), bytes.next()).await + }); + let Some(item) = (match item { + Ok(item) => item, + Err(_) => continue, + }) else { + break; + }; + let item = + item.map_err(|error| format_openai_request_error("stream speech audio", error))?; + if !active.load(Ordering::SeqCst) { + return Ok(()); + } + pcm_remainder.extend_from_slice(&item); + let sample_bytes = pcm_remainder.len() / 2 * 2; + let samples = pcm16le_to_f32(&pcm_remainder[..sample_bytes]); + pcm_remainder.drain(..sample_bytes); + player.enqueue(&samples)?; + segment_frames = segment_frames.saturating_add(samples.len() as u64); + upsert_delivery_segment(delivery, chunk, segment_frames, false); + if !*started && segment_frames > 0 { + *started = true; + emit_openai_stream_event( + app, + stream_id, + OpenAiStreamEventState::Started, + None, + None, + ); + } + } + if !pcm_remainder.is_empty() { + return Err("OpenAI speech returned an incomplete PCM sample".to_string()); + } + upsert_delivery_segment(delivery, chunk, segment_frames, true); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +async fn openai_speech_stream( + client: &reqwest::Client, + key: &str, + input: String, + speed: f32, +) -> Result>, String> { + let response = client + .post(endpoint("audio/speech")?) + .headers(authorized_headers(key)?) + .header(CONTENT_TYPE, "application/json") + .json(&json!({ + "model": speech_model(), + "voice": speech_voice(), + "input": input, + "speed": speed, + "response_format": "pcm", + "stream_format": "audio" + })) + .send() + .await + .map_err(|error| format_openai_request_error("start speech audio", error))?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(format_openai_response_error( + "start speech audio", + status, + &body, + )); + } + Ok(response.bytes_stream()) +} + +#[cfg(target_os = "macos")] +fn chunk_text(text: &str, max_chars: usize) -> Vec<&str> { + let mut chunks = Vec::new(); + let mut start = 0; + while start < text.len() { + let mut end = (start + max_chars).min(text.len()); + while end > start && !text.is_char_boundary(end) { + end -= 1; + } + if end == start { + end = text.len(); + } + chunks.push(text[start..end].trim()); + start = end; + } + chunks + .into_iter() + .filter(|chunk| !chunk.is_empty()) + .collect() +} + +#[cfg(target_os = "macos")] +fn pcm16le_to_f32(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]]) as f32 / i16::MAX as f32) + .collect() +} + +#[cfg(target_os = "macos")] +fn upsert_delivery_segment( + delivery: &mut VoiceDeliveryProgress, + _text: &str, + total_frames: u64, + synthesis_complete: bool, +) { + if let Some(segment) = delivery.segments.last_mut() { + segment.total_frames = total_frames; + segment.synthesis_complete = synthesis_complete; + } +} + +#[cfg(target_os = "macos")] +fn snapshot_delivery( + delivery: &VoiceDeliveryProgress, + player: &PocketAudioPlayer, +) -> VoiceDeliveryProgress { + let mut remaining_played = player.played_frames(); + let segments = delivery + .segments + .iter() + .map(|segment| { + let played_frames = remaining_played.min(segment.total_frames); + remaining_played = remaining_played.saturating_sub(played_frames); + VoiceDeliverySegment { + text: segment.text.clone(), + played_frames, + total_frames: segment.total_frames, + synthesis_complete: segment.synthesis_complete, + } + }) + .collect(); + VoiceDeliveryProgress { + sample_rate: delivery.sample_rate, + segments, + } +} + +fn emit_openai_stream_event( + app: &AppHandle, + stream_id: &str, + state: OpenAiStreamEventState, + error: Option, + delivery: Option, +) { + let _ = app.emit( + TTS_EVENT, + OpenAiVoiceStreamEvent { + stream_id: stream_id.to_string(), + state, + error, + delivery, + }, + ); +} + +fn format_openai_request_error(action: &str, error: reqwest::Error) -> String { + if error.is_timeout() { + format!("OpenAI voice could not {action}: the request timed out") + } else if error.is_connect() { + format!("OpenAI voice could not {action}: check your network connection") + } else { + format!("OpenAI voice could not {action}: {error}") + } +} + +fn format_openai_response_error(action: &str, status: reqwest::StatusCode, body: &str) -> String { + let preview: String = body.chars().take(500).collect(); + format!("OpenAI voice could not {action}: HTTP {status}: {preview}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(target_os = "macos")] + #[test] + fn chunks_tts_text_on_char_boundaries() { + assert_eq!(chunk_text("hello", 10), vec!["hello"]); + assert_eq!(chunk_text("ééé", 3), vec!["é", "é", "é"]); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 87355e496..44165a993 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -233,6 +233,7 @@ pub fn run() { app.manage(commands::model_setup::ModelSetupRegistry::default()); app.manage(commands::pocket_voice::PocketVoiceState::default()); app.manage(commands::siri_voice::SiriVoiceState::default()); + app.manage(commands::openai_audio::OpenAiVoiceState::default()); app.manage(commands::native_voice::NativeVoiceState::default()); app.manage(commands::voice_capture::VoiceCaptureState::default()); app.manage(commands::telemetry::TelemetryAuthState::new( @@ -654,6 +655,13 @@ pub fn run() { commands::pocket_voice::finish_pocket_voice_stream, commands::pocket_voice::stop_pocket_voice, commands::pocket_voice::remove_voice_model, + commands::openai_audio::get_openai_voice_status, + commands::openai_audio::start_openai_voice_stream, + commands::openai_audio::append_openai_voice_stream, + commands::openai_audio::flush_openai_voice_stream, + commands::openai_audio::finish_openai_voice_stream, + commands::openai_audio::stop_openai_voice, + commands::openai_audio::set_openai_playback_speed, commands::siri_voice::get_siri_voice_status, commands::siri_voice::select_siri_voice, commands::siri_voice::download_siri_voice, diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index b83e323e8..2face1bbf 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -236,6 +236,7 @@ import { } from "@/features/voice-conversation/api/voiceConversation"; import { usePocketVoiceSetup } from "@/features/voice-conversation/hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "@/features/voice-conversation/hooks/useMacSpeechSetup"; +import { useOpenAiVoiceSetup } from "@/features/voice-conversation/hooks/useOpenAiVoiceSetup"; import { useSiriVoiceSetup } from "@/features/voice-conversation/hooks/useSiriVoiceSetup"; import { isMacSpeechAvailable, @@ -743,6 +744,9 @@ export function AppShell({ ), ); const globalVoiceOutput = useVoiceOutputPreference(); + const globalOpenAiVoiceSetup = useOpenAiVoiceSetup( + capabilities.voiceConversation && globalVoiceOutput.backend === "openai", + ); const globalSiriVoiceSetup = useSiriVoiceSetup( capabilities.voiceConversation && globalVoiceOutput.backend === "siri", ); @@ -752,6 +756,7 @@ export function AppShell({ globalSiriVoiceSetup.status, globalVoiceInput.backend, globalVoiceOutput.backend, + globalOpenAiVoiceSetup.status, ); const voiceConversationWasEnabledRef = useRef(capabilities.voiceConversation); useEffect(() => { diff --git a/src/features/chat/ui/ChatView.tsx b/src/features/chat/ui/ChatView.tsx index a7d643ea2..883db84c0 100644 --- a/src/features/chat/ui/ChatView.tsx +++ b/src/features/chat/ui/ChatView.tsx @@ -65,6 +65,7 @@ import type { GlobalComposerHandoffRect } from "@/shared/ui/GlobalComposerPill"; import { useVoiceConversationController } from "@/features/voice-conversation/hooks/useVoiceConversationController"; import { usePocketVoiceSetup } from "@/features/voice-conversation/hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "@/features/voice-conversation/hooks/useMacSpeechSetup"; +import { useOpenAiVoiceSetup } from "@/features/voice-conversation/hooks/useOpenAiVoiceSetup"; import { useSiriVoiceSetup } from "@/features/voice-conversation/hooks/useSiriVoiceSetup"; import { isMacSpeechAvailable, @@ -234,6 +235,9 @@ export function ChatView({ isMacSpeechAvailable(macSpeechSetup.status, macSpeechSetup.loading), ); const voiceOutput = useVoiceOutputPreference(); + const openAiVoiceSetup = useOpenAiVoiceSetup( + capabilities.voiceConversation && voiceOutput.backend === "openai", + ); const siriVoiceSetup = useSiriVoiceSetup( capabilities.voiceConversation && voiceOutput.backend === "siri", ); @@ -243,6 +247,7 @@ export function ChatView({ siriVoiceSetup.status, voiceInput.backend, voiceOutput.backend, + openAiVoiceSetup.status, ); const voiceConversation = useVoiceConversationController({ sessionId, diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts new file mode 100644 index 000000000..774a3f895 --- /dev/null +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -0,0 +1,71 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen, type UnlistenFn } from "@tauri-apps/api/event"; +import type { VoiceDeliveryProgress } from "./pocketVoice"; +import type { + VoiceInterruptionMode, + VoiceInterruptionSensitivity, +} from "../lib/voiceInterruptionPreference"; + +export interface OpenAiVoiceStatus { + configured: boolean; + transcriptionModel: string; + speechModel: string; + speechVoice: string; + playbackSpeed: number; + ttsAvailable: boolean; + unavailableReason: string | null; +} + +export interface OpenAiVoiceStreamEvent { + streamId: string; + state: "started" | "progress" | "completed" | "interrupted" | "failed"; + error: string | null; + delivery?: VoiceDeliveryProgress | null; +} + +export function getOpenAiVoiceStatus(): Promise { + return invoke("get_openai_voice_status"); +} + +export function startOpenAiVoiceStream( + streamId: string, + interruptionMode: VoiceInterruptionMode, + interruptionSensitivity: VoiceInterruptionSensitivity, +): Promise { + return invoke("start_openai_voice_stream", { + streamId, + interruptionMode, + interruptionSensitivity, + }); +} + +export function appendOpenAiVoiceStream( + streamId: string, + text: string, +): Promise { + return invoke("append_openai_voice_stream", { streamId, text }); +} + +export function flushOpenAiVoiceStream(streamId: string): Promise { + return invoke("flush_openai_voice_stream", { streamId }); +} + +export function finishOpenAiVoiceStream(streamId: string): Promise { + return invoke("finish_openai_voice_stream", { streamId }); +} + +export function stopOpenAiVoice(): Promise { + return invoke("stop_openai_voice"); +} + +export function setOpenAiPlaybackSpeed(speed: number): Promise { + return invoke("set_openai_playback_speed", { speed }); +} + +export function listenToOpenAiVoiceStream( + onEvent: (event: OpenAiVoiceStreamEvent) => void, +): Promise { + return listen("openai-voice:stream-event", (event) => + onEvent(event.payload), + ); +} diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts new file mode 100644 index 000000000..77bfa5879 --- /dev/null +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts @@ -0,0 +1,30 @@ +import { useEffect, useState } from "react"; +import { + getOpenAiVoiceStatus, + type OpenAiVoiceStatus, +} from "../api/openAiVoice"; + +export function useOpenAiVoiceSetup(enabled = true) { + const [status, setStatus] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + if (!enabled) return; + let active = true; + void getOpenAiVoiceStatus().then( + (next) => { + if (active) setStatus(next); + }, + (cause) => { + if (active) { + setError(cause instanceof Error ? cause.message : String(cause)); + } + }, + ); + return () => { + active = false; + }; + }, [enabled]); + + return { status, error }; +} diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 04e395325..9cd66e2d4 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -5,7 +5,7 @@ import { useVoiceConversationStore } from "../stores/voiceConversationStore"; import type { PocketVoiceStreamEvent } from "../api/pocketVoice"; const mocks = vi.hoisted(() => ({ - backend: "pocket" as "pocket" | "siri", + backend: "pocket" as "pocket" | "siri" | "openai", interruptionMode: "automatic" as | "automatic" | "allowInterruptions" @@ -51,6 +51,22 @@ const mocks = vi.hoisted(() => ({ siriFinish: vi.fn<(streamId: string) => Promise>(), siriStop: vi.fn<() => Promise>(), siriStreamHandler: null as ((event: PocketVoiceStreamEvent) => void) | null, + openAiStart: + vi.fn< + ( + streamId: string, + interruptionMode: + | "automatic" + | "allowInterruptions" + | "preventFeedback", + interruptionSensitivity: "less" | "balanced" | "more", + ) => Promise + >(), + openAiAppend: vi.fn<(streamId: string, text: string) => Promise>(), + openAiFlush: vi.fn<(streamId: string) => Promise>(), + openAiFinish: vi.fn<(streamId: string) => Promise>(), + openAiStop: vi.fn<() => Promise>(), + openAiStreamHandler: null as ((event: PocketVoiceStreamEvent) => void) | null, })); vi.mock("../api/voiceConversation", () => ({ setVoiceConversationAssistantSpeaking: mocks.setAssistantSpeaking, @@ -75,6 +91,25 @@ vi.mock("../api/pocketVoice", () => ({ }, })); +vi.mock("../api/openAiVoice", () => ({ + startOpenAiVoiceStream: ( + streamId: string, + interruptionMode: typeof mocks.interruptionMode, + interruptionSensitivity: "less" | "balanced" | "more", + ) => mocks.openAiStart(streamId, interruptionMode, interruptionSensitivity), + appendOpenAiVoiceStream: (streamId: string, text: string) => + mocks.openAiAppend(streamId, text), + flushOpenAiVoiceStream: (streamId: string) => mocks.openAiFlush(streamId), + finishOpenAiVoiceStream: (streamId: string) => mocks.openAiFinish(streamId), + stopOpenAiVoice: () => mocks.openAiStop(), + listenToOpenAiVoiceStream: async ( + handler: (event: PocketVoiceStreamEvent) => void, + ) => { + mocks.openAiStreamHandler = handler; + return vi.fn(); + }, +})); + vi.mock("../api/siriVoice", () => ({ startSiriVoiceStream: ( streamId: string, @@ -181,6 +216,12 @@ describe("native assistant speech stream", () => { mocks.siriFinish.mockReset().mockResolvedValue(); mocks.siriStop.mockReset().mockResolvedValue(true); mocks.siriStreamHandler = null; + mocks.openAiStart.mockReset().mockResolvedValue(); + mocks.openAiAppend.mockReset().mockResolvedValue(); + mocks.openAiFlush.mockReset().mockResolvedValue(); + mocks.openAiFinish.mockReset().mockResolvedValue(); + mocks.openAiStop.mockReset().mockResolvedValue(true); + mocks.openAiStreamHandler = null; useChatStore.setState({ messagesBySession: {}, sessionStateById: {}, @@ -242,6 +283,34 @@ describe("native assistant speech stream", () => { }); }); + it("routes ordering and cancellation through OpenAI when selected", async () => { + mocks.backend = "openai"; + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Hello from OpenAI." }], "completed"), + ]); + + await vi.waitFor(() => { + expect(mocks.openAiStart).toHaveBeenCalledWith( + expect.any(String), + "automatic", + "less", + ); + expect(mocks.openAiAppend).toHaveBeenCalledWith( + mocks.openAiStart.mock.calls[0]?.[0], + "Hello from OpenAI.", + ); + expect(mocks.openAiFinish).toHaveBeenCalledTimes(1); + }); + expect(mocks.openAiStart.mock.invocationCallOrder[0]).toBeLessThan( + mocks.openAiAppend.mock.invocationCallOrder[0] ?? 0, + ); + stopNativeAssistantSpeech(); + await vi.waitFor(() => expect(mocks.openAiStop).toHaveBeenCalled()); + }); + it("routes the complete utterance stream through Siri when selected", async () => { mocks.backend = "siri"; mocks.interruptionMode = "allowInterruptions"; @@ -810,89 +879,6 @@ describe("native assistant speech stream", () => { } }); - it("holds an interruption past VAD idle until delayed final transcript arrives", async () => { - vi.useFakeTimers(); - try { - startNativeAssistantSpeech("session-1", vi.fn()); - useChatStore - .getState() - .setMessages("session-1", [ - assistant([{ type: "text", text: "Interrupted reply." }]), - ]); - await vi.runAllTimersAsync(); - await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); - const firstStreamId = mocks.start.mock.calls[0]?.[0] as string; - emit("started"); - - useVoiceConversationStore.setState({ userSpeaking: true }); - await vi.runAllTimersAsync(); - expect(mocks.stop).toHaveBeenCalled(); - mocks.streamHandler?.({ - streamId: firstStreamId, - state: "interrupted", - error: null, - delivery: { segments: [] }, - }); - - useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(300); - expect(mocks.start).toHaveBeenCalledTimes(1); - - finalizeVoiceTranscript("delayed-final"); - useChatStore - .getState() - .setMessages("session-1", [ - assistant( - [{ type: "text", text: "Interrupted reply." }], - "completed", - "assistant-1", - ), - voiceUser("delayed-final"), - ]); - await vi.runAllTimersAsync(); - - expect(mocks.start).toHaveBeenCalledTimes(1); - expect(takeVoicePlaybackNotices("session-1")).toContain( - "Original text: Interrupted reply.", - ); - } finally { - vi.useRealTimers(); - } - }); - - it("resumes a no-result interruption after the recognition segment timeout", async () => { - vi.useFakeTimers(); - try { - startNativeAssistantSpeech("session-1", vi.fn()); - useChatStore - .getState() - .setMessages("session-1", [ - assistant( - [{ type: "text", text: "False alarm reply." }], - "completed", - ), - ]); - useVoiceConversationStore.setState({ userSpeaking: true }); - await vi.runAllTimersAsync(); - expect(mocks.start).not.toHaveBeenCalled(); - - useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(300); - expect(mocks.start).not.toHaveBeenCalled(); - - await vi.advanceTimersByTimeAsync(200); - await vi.runAllTimersAsync(); - expect(mocks.start).toHaveBeenCalledTimes(1); - expect(mocks.append).toHaveBeenCalledWith( - mocks.start.mock.calls[0]?.[0], - "False alarm reply.", - ); - expect(takeVoicePlaybackNotices("session-1")).toBeNull(); - } finally { - vi.useRealTimers(); - } - }); - it("ignores a late started event after interruption is requested", async () => { let resolveStop: ((stopped: boolean) => void) | undefined; startNativeAssistantSpeech("session-1", vi.fn()); @@ -2514,7 +2500,7 @@ describe("native assistant speech stream", () => { }, }); useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(500); + await vi.advanceTimersByTimeAsync(250); await vi.runAllTimersAsync(); const secondStreamId = mocks.start.mock.calls[1]?.[0] as string; @@ -2550,7 +2536,7 @@ describe("native assistant speech stream", () => { }, }); useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(500); + await vi.advanceTimersByTimeAsync(250); await vi.runAllTimersAsync(); const thirdStreamId = mocks.start.mock.calls[2]?.[0] as string; @@ -2618,7 +2604,7 @@ describe("native assistant speech stream", () => { ]); useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(500); + await vi.advanceTimersByTimeAsync(250); await vi.runAllTimersAsync(); const resumedStreamId = mocks.start.mock.calls[1]?.[0] as string; @@ -2764,7 +2750,7 @@ describe("native assistant speech stream", () => { refreshedSiriVoice, ); useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(500); + await vi.advanceTimersByTimeAsync(250); await vi.runAllTimersAsync(); expect(mocks.siriStart).toHaveBeenCalledTimes(2); @@ -2957,43 +2943,6 @@ describe("native assistant speech stream", () => { }); }); - it("preserves the recognition deadline across repeated VAD edges", async () => { - vi.useFakeTimers(); - try { - startNativeAssistantSpeech("session-1", vi.fn()); - useChatStore - .getState() - .setMessages("session-1", [ - assistant([{ type: "text", text: "Interrupted reply." }]), - ]); - await vi.runAllTimersAsync(); - const firstStreamId = mocks.start.mock.calls[0]?.[0] as string; - - useVoiceConversationStore.setState({ userSpeaking: true }); - mocks.streamHandler?.({ - streamId: firstStreamId, - state: "interrupted", - error: null, - delivery: { segments: [] }, - }); - useVoiceConversationStore.setState({ userSpeaking: false }); - - await vi.advanceTimersByTimeAsync(300); - expect(mocks.start).toHaveBeenCalledTimes(1); - - useVoiceConversationStore.setState({ userSpeaking: true }); - useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(199); - expect(mocks.start).toHaveBeenCalledTimes(1); - - await vi.advanceTimersByTimeAsync(1); - expect(mocks.start).toHaveBeenCalledTimes(2); - await vi.runAllTimersAsync(); - } finally { - vi.useRealTimers(); - } - }); - it("never starts a held reply when a newer finalized voice transcript arrives", async () => { useVoiceConversationStore.setState({ userSpeaking: true }); startNativeAssistantSpeech("session-1", vi.fn()); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index f374b07b6..901adc352 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -14,6 +14,15 @@ import { type VoiceDeliveryProgress, type PocketVoiceStreamEvent, } from "../api/pocketVoice"; +import { + appendOpenAiVoiceStream, + finishOpenAiVoiceStream, + flushOpenAiVoiceStream, + listenToOpenAiVoiceStream, + startOpenAiVoiceStream, + stopOpenAiVoice, + type OpenAiVoiceStreamEvent, +} from "../api/openAiVoice"; import { appendSiriVoiceStream, finishSiriVoiceStream, @@ -134,7 +143,6 @@ function boundedDeliveryText( } const MALFORMED_VOICE_TRANSCRIPT_KEY = "\0malformed-voice-transcript"; const USER_IDLE_TRANSCRIPT_SETTLE_MS = 250; -const USER_RECOGNITION_SEGMENT_TIMEOUT_MS = 500; function voiceTranscriptKeyForMessage( sessionId: string, @@ -671,7 +679,7 @@ function queueStreamCommand( } function handleStreamEvent( - event: PocketVoiceStreamEvent | SiriVoiceStreamEvent, + event: PocketVoiceStreamEvent | SiriVoiceStreamEvent | OpenAiVoiceStreamEvent, ) { const utterance = activeUtterance; if (!utterance || utterance.id !== event.streamId) return; @@ -865,8 +873,9 @@ export function startNativeAssistantSpeech( activeSpeechSessionId = sessionId; activeSpeechRevision = useVoiceConversationStore.getState().status.revision; const activeGeneration = generation; + const outputBackend = getVoiceOutputBackend(); const streamBackend = - getVoiceOutputBackend() === "siri" + outputBackend === "siri" ? { start: ( streamId: string, @@ -891,14 +900,23 @@ export function startNativeAssistantSpeech( stop: stopSiriVoice, listen: listenToSiriVoiceStream, } - : { - start: startPocketVoiceStream, - append: appendPocketVoiceStream, - flush: flushPocketVoiceStream, - finish: finishPocketVoiceStream, - stop: stopPocketVoice, - listen: listenToPocketVoiceStream, - }; + : outputBackend === "openai" + ? { + start: startOpenAiVoiceStream, + append: appendOpenAiVoiceStream, + flush: flushOpenAiVoiceStream, + finish: finishOpenAiVoiceStream, + stop: stopOpenAiVoice, + listen: listenToOpenAiVoiceStream, + } + : { + start: startPocketVoiceStream, + append: appendPocketVoiceStream, + flush: flushPocketVoiceStream, + finish: finishPocketVoiceStream, + stop: stopPocketVoice, + listen: listenToPocketVoiceStream, + }; stopActiveVoice = streamBackend.stop; streamListenerReady = streamBackend .listen(handleStreamEvent) @@ -964,8 +982,7 @@ export function startNativeAssistantSpeech( let resumableInterruption: ResumableInterruption | null = null; let heldReleaseReady = false; let interruptionReleaseReady = false; - let pendingUserRecognitionSegment = false; - let recognitionSegmentTimer: number | null = null; + let idleSettling = false; let heldReleaseTimer: number | null = null; const cacheCausalTranscriptKeys = ( @@ -1270,15 +1287,9 @@ export function startNativeAssistantSpeech( return; } const messages = useChatStore.getState().messagesBySession[sessionId] ?? []; - if ( - heldSpeech || - voice.userSpeaking || - pendingUserRecognitionSegment || - heldReleaseTimer !== null - ) { - // Text can keep streaming while recognition resolves the user's - // interruption segment. Refresh the held snapshot before a finalized - // voice message can invalidate it. + if (heldSpeech || voice.userSpeaking || idleSettling) { + // Text can keep streaming during the idle settling turn. Refresh the + // held snapshot before a finalized voice message can invalidate it. holdAssistantChanges(messages); } const finalizedTranscriptKey = voice.latestFinalizedTranscriptKey; @@ -1299,7 +1310,7 @@ export function startNativeAssistantSpeech( interruptActiveUtterance(true, "userSpeaking"); } - if (voice.userSpeaking || pendingUserRecognitionSegment) return; + if (voice.userSpeaking || idleSettling) return; if (heldSpeech && !heldReleaseReady) return; if (resumableInterruption) return; if (activeUtterance?.interruptionRequested) return; @@ -1534,19 +1545,6 @@ export function startNativeAssistantSpeech( } let wasUserSpeaking = initialVoice.userSpeaking; let wasMicrophoneMuted = initialVoice.microphoneMuted; - let latestObservedFinalizedTranscriptKey = - useVoiceConversationStore.getState().latestFinalizedTranscriptKey; - const resolvePendingRecognitionSegment = (releaseSpeech = true) => { - if (recognitionSegmentTimer !== null) { - window.clearTimeout(recognitionSegmentTimer); - recognitionSegmentTimer = null; - } - pendingUserRecognitionSegment = false; - if (!releaseSpeech) return; - heldReleaseReady = heldSpeech !== null; - interruptionReleaseReady = true; - releaseResumableInterruption(); - }; const unsubscribeVoice = useVoiceConversationStore.subscribe((voice) => { const runningForSession = voice.status.lifecycle === "running" && @@ -1567,90 +1565,34 @@ export function startNativeAssistantSpeech( const becameUserSpeaking = voice.userSpeaking && !wasUserSpeaking; const becameUserIdle = !voice.userSpeaking && wasUserSpeaking; const becameMicrophoneMuted = voice.microphoneMuted && !wasMicrophoneMuted; - const finalizedTranscriptChanged = - voice.latestFinalizedTranscriptKey !== - latestObservedFinalizedTranscriptKey; - const hasInterruptedPlaybackHold = - activeUtterance?.interruptionRequested || resumableInterruption !== null; wasUserSpeaking = voice.userSpeaking; wasMicrophoneMuted = voice.microphoneMuted; - latestObservedFinalizedTranscriptKey = voice.latestFinalizedTranscriptKey; if (activeGeneration !== generation) return; if (becameMicrophoneMuted) { - resolvePendingRecognitionSegment(false); + if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer); + heldReleaseTimer = null; + idleSettling = false; interruptionReleaseReady = false; discardHeldAndResumableSpeech(); inspect(); return; } - if (finalizedTranscriptChanged && !pendingUserRecognitionSegment) { - if (heldReleaseTimer !== null) { - window.clearTimeout(heldReleaseTimer); - heldReleaseTimer = null; - } - holdAssistantChanges( - useChatStore.getState().messagesBySession[sessionId] ?? [], - ); - discardInvalidHeldSpeech(voice.latestFinalizedTranscriptKey); - inspect(); - return; - } if (becameUserSpeaking) { - if (heldReleaseTimer !== null) { - window.clearTimeout(heldReleaseTimer); - heldReleaseTimer = null; - } - const interrupted = interruptActiveUtterance(true, "userSpeaking"); - if (interrupted && recognitionSegmentTimer !== null) { - window.clearTimeout(recognitionSegmentTimer); - recognitionSegmentTimer = null; - } - pendingUserRecognitionSegment ||= interrupted; + if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer); + heldReleaseTimer = null; + idleSettling = false; heldReleaseReady = false; interruptionReleaseReady = false; - inspect(); - return; - } - if (finalizedTranscriptChanged && pendingUserRecognitionSegment) { - holdAssistantChanges( - useChatStore.getState().messagesBySession[sessionId] ?? [], - ); - const hadResumableInterruption = resumableInterruption !== null; - resolvePendingRecognitionSegment(false); - if (hadResumableInterruption) { - discardResumableInterruption(); - discardInvalidHeldSpeech(voice.latestFinalizedTranscriptKey); - } else { - discardHeldAndResumableSpeech(); - } + interruptActiveUtterance(true, "userSpeaking"); inspect(); return; } if (becameUserIdle) { if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer); - if (pendingUserRecognitionSegment || hasInterruptedPlaybackHold) { - pendingUserRecognitionSegment = true; - // VAD silence does not imply recognition is idle. Keep interrupted - // playback held until a final transcript arrives or the unresolved - // user-recognition segment hits a conservative bound. - recognitionSegmentTimer ??= window.setTimeout(() => { - recognitionSegmentTimer = null; - const current = useVoiceConversationStore.getState(); - if ( - activeGeneration !== generation || - current.userSpeaking || - current.status.lifecycle !== "running" || - current.status.sessionId !== sessionId || - !pendingUserRecognitionSegment - ) { - return; - } - resolvePendingRecognitionSegment(true); - inspect(); - }, USER_RECOGNITION_SEGMENT_TIMEOUT_MS); - inspect(); - return; - } + idleSettling = true; + // VAD can report silence shortly before the recognizer commits its final + // transcript. Give that transcript a bounded opportunity to invalidate + // causally stale speech before releasing the held reply. heldReleaseTimer = window.setTimeout(() => { heldReleaseTimer = null; const current = useVoiceConversationStore.getState(); @@ -1662,6 +1604,7 @@ export function startNativeAssistantSpeech( ) { return; } + idleSettling = false; heldReleaseReady = heldSpeech !== null; interruptionReleaseReady = true; releaseResumableInterruption(); @@ -1672,15 +1615,8 @@ export function startNativeAssistantSpeech( inspect(); }); stopVoiceSubscription = () => { - if (heldReleaseTimer !== null) { - window.clearTimeout(heldReleaseTimer); - } - if (recognitionSegmentTimer !== null) { - window.clearTimeout(recognitionSegmentTimer); - } + if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer); heldReleaseTimer = null; - recognitionSegmentTimer = null; - pendingUserRecognitionSegment = false; discardHeldAndResumableSpeech(); unsubscribeVoice(); }; diff --git a/src/features/voice-conversation/lib/voiceOutputPreference.ts b/src/features/voice-conversation/lib/voiceOutputPreference.ts index 061ad75a7..f3a575a9c 100644 --- a/src/features/voice-conversation/lib/voiceOutputPreference.ts +++ b/src/features/voice-conversation/lib/voiceOutputPreference.ts @@ -1,7 +1,7 @@ import { useCallback, useSyncExternalStore } from "react"; import { getPlatform } from "@/shared/lib/platform"; -export type VoiceOutputBackend = "pocket" | "siri"; +export type VoiceOutputBackend = "pocket" | "siri" | "openai"; const STORAGE_KEY = "goose:voice-output-backend"; const CHANGED_EVENT = "goose:voice-output-backend-changed"; @@ -14,6 +14,7 @@ function normalize(value: unknown): VoiceOutputBackend { if (value === "siri") { return getPlatform() === "mac" ? "siri" : "pocket"; } + if (value === "openai") return "openai"; return value === "pocket" ? value : getDefaultVoiceOutputBackend(); } diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts index bcafc3dc3..76bdb2cb3 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts @@ -53,6 +53,16 @@ describe("voice setup readiness", () => { ).toBe(false); }); + it("requires Berd's configured OpenAI credential for OpenAI output", () => { + const configured = { configured: true, ttsAvailable: true } as never; + expect( + isVoiceSetupReady(pocket, null, null, "parakeet", "openai", configured), + ).toBe(true); + expect(isVoiceSetupReady(pocket, null, null, "parakeet", "openai")).toBe( + false, + ); + }); + it("uses native macOS speech readiness instead of Parakeet when selected", () => { expect( isVoiceSetupReady( diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.ts index f8f11226c..0ae9ca757 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.ts @@ -2,6 +2,7 @@ import type { PocketVoiceStatus } from "../api/pocketVoice"; import type { SiriVoiceStatus } from "../api/siriVoice"; import type { MacSpeechStatus } from "../api/macSpeech"; import type { VoiceInputBackend } from "./voiceInputPreference"; +import type { OpenAiVoiceStatus } from "../api/openAiVoice"; import type { VoiceOutputBackend } from "./voiceOutputPreference"; export function isVoiceSetupReady( @@ -10,6 +11,7 @@ export function isVoiceSetupReady( siri: SiriVoiceStatus | null, inputBackend: VoiceInputBackend | null, outputBackend: VoiceOutputBackend, + openAi: OpenAiVoiceStatus | null = null, ): boolean { if (inputBackend === null) return false; const inputReady = @@ -21,6 +23,8 @@ export function isVoiceSetupReady( ) : Boolean(pocket?.parakeetInstalled); if (!inputReady) return false; + if (outputBackend === "openai") + return Boolean(openAi?.configured && openAi.ttsAvailable); if (outputBackend === "pocket") return Boolean(pocket?.pocketInstalled); return Boolean( siri?.supported && siri.selectedVoice && siri.selectedVoiceInstalled, diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 1c772f918..121da1064 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -51,6 +51,17 @@ const microphonePermissionState = vi.hoisted(() => ({ openSettings: vi.fn(), })); +vi.mock("../api/openAiVoice", () => ({ + getOpenAiVoiceStatus: vi.fn().mockResolvedValue({ + configured: true, + transcriptionModel: "gpt-live-transcribe", + speechModel: "gpt-4o-mini-tts", + speechVoice: "marin", + playbackSpeed: 1, + ttsAvailable: true, + unavailableReason: null, + }), +})); vi.mock("../hooks/usePocketVoiceSetup", () => ({ usePocketVoiceSetup: () => setupState.current, })); @@ -193,6 +204,17 @@ describe("VoiceSettings", () => { siriSetupState.current = siriSetup(); }); + it("renders selected OpenAI output settings", async () => { + outputState.backend = "openai"; + setupState.current = setup(pocketStatus()); + renderWithProviders(); + + expect( + await screen.findByText(/gpt-4o-mini-tts.*marin voice/), + ).toBeInTheDocument(); + expect(screen.getByText("Playback speed")).toBeInTheDocument(); + }); + it("shows interruption modes without VAD controls", () => { setupState.current = setup(pocketStatus()); renderWithProviders(); diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index f7322143e..cafde4bf6 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -14,6 +14,12 @@ import { SelectTrigger, SelectValue, } from "@/shared/ui/select"; +import { useEffect, useState } from "react"; +import { + getOpenAiVoiceStatus, + setOpenAiPlaybackSpeed, + type OpenAiVoiceStatus, +} from "../api/openAiVoice"; import { usePocketVoiceSetup } from "../hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "../hooks/useMacSpeechSetup"; import { useMicrophonePermission } from "../hooks/useMicrophonePermission"; @@ -30,6 +36,7 @@ import { useVoiceOutputPreference } from "../lib/voiceOutputPreference"; import { PocketVoiceSetupContent } from "./PocketVoiceSetupContent"; import { MacSpeechSettings } from "./MacSpeechSettings"; import { SiriVoiceSettings } from "./SiriVoiceSettings"; +import { PlaybackSpeedRow } from "./PlaybackSpeedRow"; const INTERRUPTION_MODES: VoiceInterruptionMode[] = [ "automatic", @@ -45,6 +52,9 @@ function readinessDescriptionKey( ): string | null { if (inputReady && outputReady) return null; if (!inputReady && !outputReady) { + if (backend === "openai") { + return "voice.notReadyOpenAi"; + } if (inputBackend === "macos") { return backend === "siri" ? "voice.notReadyMacInputAndSiriOutput" @@ -68,6 +78,31 @@ export function VoiceSettings() { const { t } = useTranslation("settings"); const setup = usePocketVoiceSetup(); const macSpeechSetup = useMacSpeechSetup(); + const [openAiStatus, setOpenAiStatus] = useState( + null, + ); + const [openAiError, setOpenAiError] = useState(null); + const [openAiSpeed, setOpenAiSpeed] = useState(1); + useEffect(() => { + let active = true; + void getOpenAiVoiceStatus().then( + (status) => { + if (active) { + setOpenAiStatus(status); + setOpenAiSpeed(status.playbackSpeed); + } + }, + (error) => { + if (active) + setOpenAiError( + error instanceof Error ? error.message : String(error), + ); + }, + ); + return () => { + active = false; + }; + }, []); const input = useVoiceInputPreference( isMacSpeechAvailable(macSpeechSetup.status, macSpeechSetup.loading), ); @@ -91,13 +126,15 @@ export function VoiceSettings() { ) : (setup.status?.parakeetInstalled ?? false); const outputReady = - output.backend === "siri" - ? Boolean( - siriSetup.status?.supported && - siriSetup.status.selectedVoice && - siriSetup.status.selectedVoiceInstalled, - ) - : (setup.status?.pocketInstalled ?? false); + output.backend === "openai" + ? Boolean(openAiStatus?.configured && openAiStatus.ttsAvailable) + : output.backend === "siri" + ? Boolean( + siriSetup.status?.supported && + siriSetup.status.selectedVoice && + siriSetup.status.selectedVoiceInstalled, + ) + : (setup.status?.pocketInstalled ?? false); const siriOutputLoaded = siriSetup.status !== null && siriSetup.statusError === null; const pocketStatusLoaded = @@ -230,6 +267,11 @@ export function VoiceSettings() { {t("voice.backendPocket")} + {openAiStatus?.ttsAvailable ? ( + + {t("voice.backendOpenAiTts")} + + ) : null} {siriSupported ? ( {t("voice.backendSiri")} ) : null} @@ -237,7 +279,28 @@ export function VoiceSettings() { )} details={ - output.backend === "siri" ? ( + output.backend === "openai" ? ( +
+

+ {openAiError ?? + openAiStatus?.unavailableReason ?? + (openAiStatus + ? t("voice.openAiTtsConfigured", { + model: openAiStatus.speechModel, + voice: openAiStatus.speechVoice, + }) + : t("voice.openAiChecking"))} +

+ { + await setOpenAiPlaybackSpeed(speed); + setOpenAiSpeed(speed); + }} + /> +
+ ) : output.backend === "siri" ? ( ) : ( diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index b80eba39c..6c966123e 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -870,6 +870,8 @@ }, "voice": { "backendMacSpeech": "Apple speech recognition", + "backendOpenAiStt": "OpenAI speech-to-text", + "backendOpenAiTts": "OpenAI text-to-speech", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", "backendSiri": "Siri voices", @@ -917,6 +919,7 @@ "notReadyMacInput": "Apple's on-device dictation model is not installed. Download it below to use Voice Conversation.", "notReadyMacInputAndPocketOutput": "Apple's on-device dictation model and Pocket TTS are not installed. Complete both steps below to use Voice Conversation.", "notReadyMacInputAndSiriOutput": "Apple's on-device dictation model is not installed, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", + "notReadyOpenAi": "OpenAI voice is not ready. Configure the OpenAI provider in Berd, then try again.", "notReadyPocketOutput": "Pocket TTS is not installed. Download it below to use Voice Conversation.", "notReadySiriOutput": "No installed Siri voice is selected. Download or select one below to use Voice Conversation.", "notReadyTitle": "Voice Conversation isn't ready", @@ -925,6 +928,9 @@ "noVoiceSelected": "No voice selected", "openMicrophoneSettings": "Open Microphone Settings", "openMicrophoneSettingsError": "Couldn't open Microphone Settings. Open System Settings and select Privacy & Security > Microphone.", + "openAiChecking": "Checking the OpenAI provider configured in Berd…", + "openAiSttConfigured": "Uses Berd’s configured OpenAI credential with {{model}}.", + "openAiTtsConfigured": "Uses Berd’s configured OpenAI credential with {{model}} and the {{voice}} voice. OpenAI voices are AI-generated.", "outputBackendDescription": "Choose how Berd speaks assistant responses.", "playbackSpeed": "Playback speed", "playing": "Playing", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 2044e1675..eddfa699f 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -873,6 +873,8 @@ }, "voice": { "backendMacSpeech": "Reconocimiento de voz de Apple", + "backendOpenAiStt": "Voz a texto de OpenAI", + "backendOpenAiTts": "Texto a voz de OpenAI", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", "backendSiri": "Voces de Siri", @@ -920,6 +922,7 @@ "notReadyMacInput": "El modelo de dictado de Apple no está instalado. Descárgalo abajo para usar la conversación por voz.", "notReadyMacInputAndPocketOutput": "El modelo de dictado de Apple y Pocket TTS no están instalados. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInputAndSiriOutput": "El modelo de dictado de Apple no está instalado y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyOpenAi": "La voz de OpenAI no está lista. Configura el proveedor de OpenAI en Berd e inténtalo de nuevo.", "notReadyPocketOutput": "Pocket TTS no está instalado. Descárgalo abajo para usar la conversación por voz.", "notReadySiriOutput": "No hay ninguna voz de Siri instalada seleccionada. Descarga o selecciona una abajo para usar la conversación por voz.", "notReadyTitle": "La conversación por voz no está lista", @@ -928,6 +931,9 @@ "noVoiceSelected": "No hay ninguna voz seleccionada", "openMicrophoneSettings": "Abrir ajustes del micrófono", "openMicrophoneSettingsError": "No se pudieron abrir los ajustes del micrófono. Abre Ajustes del Sistema y selecciona Privacidad y seguridad > Micrófono.", + "openAiChecking": "Comprobando el proveedor de OpenAI configurado en Berd…", + "openAiSttConfigured": "Usa la credencial de OpenAI configurada en Berd con {{model}}.", + "openAiTtsConfigured": "Usa la credencial de OpenAI configurada en Berd con {{model}} y la voz {{voice}}. Las voces de OpenAI son generadas por IA.", "outputBackendDescription": "Elige cómo Berd reproduce las respuestas del asistente.", "playbackSpeed": "Velocidad de reproducción", "playing": "Reproduciendo", From 8e4a8807c40ebafcb3a9b681252c78c9d1ec1d75 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 20:02:11 +0000 Subject: [PATCH 02/18] fix(voice): smooth OpenAI speech startup --- src-tauri/Cargo.lock | 49 ---------------- src-tauri/Cargo.toml | 1 - src-tauri/src/commands/openai_audio.rs | 56 ++++++------------- .../voice-conversation/api/openAiVoice.ts | 9 ++- 4 files changed, 24 insertions(+), 91 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index fba337c1a..1de42d10a 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -46,7 +46,6 @@ dependencies = [ "reqwest 0.13.4", "rodio", "rubato", - "rustls", "semver", "serde", "serde_json", @@ -74,7 +73,6 @@ dependencies = [ "tempfile", "time", "tokio", - "tokio-tungstenite", "toml 1.1.4+spec-1.1.0", "url", "uuid", @@ -1407,12 +1405,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" -[[package]] -name = "data-encoding" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" - [[package]] name = "dbus" version = "0.9.12" @@ -7122,22 +7114,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-tungstenite" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" -dependencies = [ - "futures-util", - "log", - "rustls", - "rustls-pki-types", - "tokio", - "tokio-rustls", - "tungstenite", - "webpki-roots 0.26.11", -] - [[package]] name = "tokio-util" version = "0.7.19" @@ -7398,25 +7374,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tungstenite" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" -dependencies = [ - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.9.5", - "rustls", - "rustls-pki-types", - "sha1", - "thiserror 2.0.20", - "utf-8", -] - [[package]] name = "typeid" version = "1.0.3" @@ -7626,12 +7583,6 @@ dependencies = [ "url", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8-zero" version = "0.8.1" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 48a25cea9..9cfaef08c 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -118,7 +118,6 @@ objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2 objc2-core-audio = "0.3.2" objc2-foundation = { version = "0.3.2", features = ["NSDictionary", "NSError", "NSFileManager", "NSObject", "NSProcessInfo", "NSString", "NSURL"] } objc2-user-notifications = "0.3.2" -keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } rodio = { version = "0.22", default-features = false, features = ["playback"] } diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index b59318f7a..2fd71f4b5 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -25,6 +25,9 @@ const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; const DEFAULT_TTS_VOICE: &str = "marin"; const TTS_SAMPLE_RATE: u32 = 24_000; +// Avoid starting the audio device from a tiny first network chunk that can drain +// before subsequent streamed PCM arrives. +const INITIAL_PLAYBACK_BUFFER_FRAMES: usize = TTS_SAMPLE_RATE as usize / 2; const TTS_EVENT: &str = "openai-voice:stream-event"; const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); @@ -123,32 +126,6 @@ fn goose_openai_api_key() -> Result, String> { if let Some(value) = env_trimmed("OPENAI_API_KEY") { return Ok(Some(value)); } - #[cfg(target_os = "macos")] - { - let entry = keyring::Entry::new("goose", "secrets").map_err(|error| { - format!("Could not access Goose's secure credential store: {error}") - })?; - match entry.get_password() { - Ok(payload) => { - let secrets: serde_json::Value = - serde_json::from_str(&payload).map_err(|error| { - format!("Goose's secure credential store is not valid JSON: {error}") - })?; - return Ok(secrets - .get("OPENAI_API_KEY") - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToString::to_string)); - } - Err(keyring::Error::NoEntry) => {} - Err(error) => { - return Err(format!( - "Could not read Goose's OpenAI credential from secure storage: {error}" - )); - } - } - } let config_path = crate::services::goose_config::config_path()?; let secrets_path = config_path .parent() @@ -169,17 +146,6 @@ fn goose_openai_api_key() -> Result, String> { .map(ToString::to_string)) } -pub(crate) fn openai_api_key_available() -> Result { - goose_openai_api_key().map(|key| key.is_some()) -} - -pub(crate) fn api_key() -> Result { - goose_openai_api_key()?.ok_or_else(|| { - "OpenAI voice is not configured. Configure the OpenAI provider in Berd, then try again." - .to_string() - }) -} - fn base_url() -> String { env_trimmed("OPENAI_BASE_URL").unwrap_or_else(|| DEFAULT_BASE_URL.to_string()) } @@ -256,8 +222,8 @@ fn client() -> Result { #[tauri::command] pub fn get_openai_voice_status( state: State<'_, OpenAiVoiceState>, + configured: bool, ) -> Result { - let configured = openai_api_key_available()?; let playback_speed = state .playback .lock() @@ -675,6 +641,7 @@ fn speak_pending( let mut bytes = runtime.block_on(openai_speech_stream(client, key, chunk.to_string(), speed))?; let mut pcm_remainder = Vec::::new(); + let mut initial_samples = Vec::::new(); loop { if !active.load(Ordering::SeqCst) { return Ok(()); @@ -697,7 +664,15 @@ fn speak_pending( let sample_bytes = pcm_remainder.len() / 2 * 2; let samples = pcm16le_to_f32(&pcm_remainder[..sample_bytes]); pcm_remainder.drain(..sample_bytes); - player.enqueue(&samples)?; + if *started { + player.enqueue(&samples)?; + } else { + initial_samples.extend_from_slice(&samples); + if initial_samples.len() >= INITIAL_PLAYBACK_BUFFER_FRAMES { + player.enqueue(&initial_samples)?; + initial_samples.clear(); + } + } segment_frames = segment_frames.saturating_add(samples.len() as u64); upsert_delivery_segment(delivery, chunk, segment_frames, false); if !*started && segment_frames > 0 { @@ -714,6 +689,9 @@ fn speak_pending( if !pcm_remainder.is_empty() { return Err("OpenAI speech returned an incomplete PCM sample".to_string()); } + if !initial_samples.is_empty() { + player.enqueue(&initial_samples)?; + } upsert_delivery_segment(delivery, chunk, segment_frames, true); } Ok(()) diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index 774a3f895..310623d46 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -1,5 +1,6 @@ import { invoke } from "@tauri-apps/api/core"; import { listen, type UnlistenFn } from "@tauri-apps/api/event"; +import { getProviderConfig } from "@/features/providers/api/credentials"; import type { VoiceDeliveryProgress } from "./pocketVoice"; import type { VoiceInterruptionMode, @@ -23,8 +24,12 @@ export interface OpenAiVoiceStreamEvent { delivery?: VoiceDeliveryProgress | null; } -export function getOpenAiVoiceStatus(): Promise { - return invoke("get_openai_voice_status"); +export async function getOpenAiVoiceStatus(): Promise { + const fields = await getProviderConfig("openai"); + const configured = fields.some( + (field) => field.key === "OPENAI_API_KEY" && field.isSecret && field.isSet, + ); + return invoke("get_openai_voice_status", { configured }); } export function startOpenAiVoiceStream( From 9e9ba5b009bf34c6d622f1016a08c9f7e24c6ea9 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 20:03:47 +0000 Subject: [PATCH 03/18] fix(voice): defer secure credential access --- src-tauri/Cargo.toml | 1 + src-tauri/src/commands/openai_audio.rs | 48 ++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 9cfaef08c..8250e0747 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -42,6 +42,7 @@ hex = "0.4" ignore = "0.4.25" fern = "0.7" infer = "0.19.0" +keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } libc = "0.2" log = "0.4.29" mime_guess = "2" diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 2fd71f4b5..66118f2b4 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -126,6 +126,32 @@ fn goose_openai_api_key() -> Result, String> { if let Some(value) = env_trimmed("OPENAI_API_KEY") { return Ok(Some(value)); } + #[cfg(target_os = "macos")] + { + let entry = keyring::Entry::new("goose", "secrets").map_err(|error| { + format!("Could not access Goose's secure credential store: {error}") + })?; + match entry.get_password() { + Ok(payload) => { + let secrets: serde_json::Value = + serde_json::from_str(&payload).map_err(|error| { + format!("Goose's secure credential store is not valid JSON: {error}") + })?; + return Ok(secrets + .get("OPENAI_API_KEY") + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string)); + } + Err(keyring::Error::NoEntry) => {} + Err(error) => { + return Err(format!( + "Could not read Goose's OpenAI credential from secure storage: {error}" + )); + } + } + } let config_path = crate::services::goose_config::config_path()?; let secrets_path = config_path .parent() @@ -671,11 +697,25 @@ fn speak_pending( if initial_samples.len() >= INITIAL_PLAYBACK_BUFFER_FRAMES { player.enqueue(&initial_samples)?; initial_samples.clear(); + *started = true; + emit_openai_stream_event( + app, + stream_id, + OpenAiStreamEventState::Started, + None, + None, + ); } } segment_frames = segment_frames.saturating_add(samples.len() as u64); upsert_delivery_segment(delivery, chunk, segment_frames, false); - if !*started && segment_frames > 0 { + } + if !pcm_remainder.is_empty() { + return Err("OpenAI speech returned an incomplete PCM sample".to_string()); + } + if !initial_samples.is_empty() { + player.enqueue(&initial_samples)?; + if !*started { *started = true; emit_openai_stream_event( app, @@ -686,12 +726,6 @@ fn speak_pending( ); } } - if !pcm_remainder.is_empty() { - return Err("OpenAI speech returned an incomplete PCM sample".to_string()); - } - if !initial_samples.is_empty() { - player.enqueue(&initial_samples)?; - } upsert_delivery_segment(delivery, chunk, segment_frames, true); } Ok(()) From 5f51297b544c0f2f76c51cf1b56bbaa5ff48f5b5 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 20:08:02 +0000 Subject: [PATCH 04/18] fix(voice): replace stale OpenAI playback --- src-tauri/src/commands/openai_audio.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 66118f2b4..be9196252 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -27,7 +27,7 @@ const DEFAULT_TTS_VOICE: &str = "marin"; const TTS_SAMPLE_RATE: u32 = 24_000; // Avoid starting the audio device from a tiny first network chunk that can drain // before subsequent streamed PCM arrives. -const INITIAL_PLAYBACK_BUFFER_FRAMES: usize = TTS_SAMPLE_RATE as usize / 2; +const INITIAL_PLAYBACK_BUFFER_FRAMES: usize = TTS_SAMPLE_RATE as usize / 5; const TTS_EVENT: &str = "openai-voice:stream-event"; const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); @@ -307,8 +307,11 @@ pub fn start_openai_voice_stream( .playback .lock() .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())?; - if playback.active.is_some() { - return Err("OpenAI voice playback is already active".to_string()); + if let Some(previous) = playback.active.as_ref() { + previous.store(false, Ordering::SeqCst); + } + if let Some(previous) = playback.stream.as_ref() { + let _ = previous.sender.send(OpenAiStreamCommand::Stop); } playback.active = Some(active.clone()); playback.stream = Some(ActiveOpenAiStream { @@ -336,8 +339,14 @@ pub fn start_openai_voice_stream( speed, ); if let Ok(mut playback) = playback.lock() { - playback.active = None; - playback.stream = None; + let still_owns_playback = playback + .active + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &active)); + if still_owns_playback { + playback.active = None; + playback.stream = None; + } } let (state, error, delivery) = match result { Ok(outcome) => (outcome.state, None, outcome.delivery), From df6d163cf99f2dbbd06a8b74693a4579cdd517df Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 16:26:57 -0400 Subject: [PATCH 05/18] fix(voice): keep OpenAI speech portable --- src-tauri/src/commands/openai_audio.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index be9196252..41a2f3e9c 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -5,21 +5,29 @@ use std::{ atomic::{AtomicBool, Ordering}, mpsc, Arc, Mutex, }, - time::{Duration, Instant}, + time::Duration, }; +#[cfg(target_os = "macos")] use futures_util::StreamExt; -use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE}; +#[cfg(target_os = "macos")] +use reqwest::header::CONTENT_TYPE; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; use serde::Serialize; use serde_json::json; use tauri::{AppHandle, Emitter, State}; -#[cfg(target_os = "macos")] use super::{ native_voice::{InterruptionSensitivity, NativeVoiceState}, + pocket_voice::VoiceInterruptionMode, +}; +#[cfg(target_os = "macos")] +use super::{ pocket_audio_player::PocketAudioPlayer, - pocket_voice::{effective_output_device_name, should_suppress_capture, VoiceInterruptionMode}, + pocket_voice::{effective_output_device_name, should_suppress_capture}, }; +#[cfg(target_os = "macos")] +use std::time::Instant; const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; @@ -172,6 +180,13 @@ fn goose_openai_api_key() -> Result, String> { .map(ToString::to_string)) } +pub(crate) fn api_key() -> Result { + goose_openai_api_key()?.ok_or_else(|| { + "OpenAI voice is not configured. Configure the OpenAI provider in Berd, then try again." + .to_string() + }) +} + fn base_url() -> String { env_trimmed("OPENAI_BASE_URL").unwrap_or_else(|| DEFAULT_BASE_URL.to_string()) } From 06e3c18159a7d559d6b10a68303f563850bc002e Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 17:00:31 -0400 Subject: [PATCH 06/18] fix(voice): harden OpenAI speech lifecycle --- src-tauri/src/commands/native_voice.rs | 2 + src-tauri/src/commands/openai_audio.rs | 228 +++++++++++++----- .../providers/api/credentials.test.ts | 15 ++ src/features/providers/api/credentials.ts | 17 ++ .../hooks/useOpenAiVoiceSetup.ts | 31 ++- .../lib/nativeAssistantSpeech.test.ts | 128 +++++++++- .../lib/nativeAssistantSpeech.ts | 119 +++++++-- .../ui/VoiceSettings.test.tsx | 43 +++- .../voice-conversation/ui/VoiceSettings.tsx | 1 + src/shared/i18n/locales/en/settings.json | 2 - src/shared/i18n/locales/es/settings.json | 2 - 11 files changed, 493 insertions(+), 95 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 4fa5e637e..2337baaf5 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -2178,6 +2178,8 @@ pub fn handle_voice_owner_window_destroyed(app: &AppHandle, window_label: &str) } app.state::() .stop_for_window_destroyed(window_label); + app.state::() + .stop_for_window_destroyed(window_label); } fn software_microphone_mute(native_microphone_mute_control: bool, muted: bool) -> bool { diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 41a2f3e9c..581f6c6f7 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -17,15 +17,16 @@ use serde::Serialize; use serde_json::json; use tauri::{AppHandle, Emitter, State}; -use super::{ - native_voice::{InterruptionSensitivity, NativeVoiceState}, - pocket_voice::VoiceInterruptionMode, -}; #[cfg(target_os = "macos")] use super::{ + native_voice::AssistantSpeechGuard, pocket_audio_player::PocketAudioPlayer, pocket_voice::{effective_output_device_name, should_suppress_capture}, }; +use super::{ + native_voice::{InterruptionSensitivity, NativeVoiceState}, + pocket_voice::VoiceInterruptionMode, +}; #[cfg(target_os = "macos")] use std::time::Instant; @@ -66,6 +67,7 @@ impl Default for PlaybackRuntime { #[derive(Debug)] struct ActiveOpenAiStream { id: String, + owner_window: String, sender: mpsc::Sender, } @@ -135,27 +137,33 @@ fn goose_openai_api_key() -> Result, String> { return Ok(Some(value)); } #[cfg(target_os = "macos")] + let mut secure_store_error = None; + #[cfg(target_os = "macos")] { - let entry = keyring::Entry::new("goose", "secrets").map_err(|error| { - format!("Could not access Goose's secure credential store: {error}") - })?; - match entry.get_password() { - Ok(payload) => { - let secrets: serde_json::Value = - serde_json::from_str(&payload).map_err(|error| { - format!("Goose's secure credential store is not valid JSON: {error}") - })?; - return Ok(secrets - .get("OPENAI_API_KEY") - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToString::to_string)); - } - Err(keyring::Error::NoEntry) => {} + match keyring::Entry::new("goose", "secrets") { + Ok(entry) => match entry.get_password() { + Ok(payload) => { + let secrets: serde_json::Value = + serde_json::from_str(&payload).map_err(|error| { + format!("Goose's secure credential store is not valid JSON: {error}") + })?; + return Ok(secrets + .get("OPENAI_API_KEY") + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string)); + } + Err(keyring::Error::NoEntry) => {} + Err(error) => { + secure_store_error = Some(format!( + "Could not read Goose's OpenAI credential from secure storage: {error}" + )); + } + }, Err(error) => { - return Err(format!( - "Could not read Goose's OpenAI credential from secure storage: {error}" + secure_store_error = Some(format!( + "Could not access Goose's secure credential store: {error}" )); } } @@ -166,6 +174,10 @@ fn goose_openai_api_key() -> Result, String> { .ok_or_else(|| "Could not resolve Goose's credential directory".to_string())? .join("secrets.yaml"); if !secrets_path.exists() { + #[cfg(target_os = "macos")] + if let Some(error) = secure_store_error { + return Err(error); + } return Ok(None); } let payload = std::fs::read_to_string(&secrets_path) @@ -290,6 +302,7 @@ pub fn get_openai_voice_status( #[tauri::command] pub fn start_openai_voice_stream( app: AppHandle, + webview_window: tauri::WebviewWindow, state: State<'_, OpenAiVoiceState>, native_voice: State<'_, NativeVoiceState>, stream_id: String, @@ -300,6 +313,7 @@ pub fn start_openai_voice_stream( { let _ = ( app, + webview_window, state, native_voice, stream_id, @@ -331,6 +345,7 @@ pub fn start_openai_voice_stream( playback.active = Some(active.clone()); playback.stream = Some(ActiveOpenAiStream { id: stream_id.clone(), + owner_window: webview_window.label().to_string(), sender, }); } @@ -425,11 +440,22 @@ pub fn set_openai_playback_speed( Ok(()) } -pub(crate) fn stop_openai_voice_inner(state: &OpenAiVoiceState) -> Result { +fn stop_openai_voice_for_owner( + state: &OpenAiVoiceState, + owner_window: Option<&str>, +) -> Result { let playback = state .playback .lock() .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())?; + if owner_window.is_some_and(|owner| { + playback + .stream + .as_ref() + .is_none_or(|stream| stream.owner_window != owner) + }) { + return Ok(false); + } let Some(active) = playback.active.as_ref() else { return Ok(false); }; @@ -440,6 +466,19 @@ pub(crate) fn stop_openai_voice_inner(state: &OpenAiVoiceState) -> Result Result { + stop_openai_voice_for_owner(state, None) +} + +impl OpenAiVoiceState { + pub(crate) fn stop_for_window_destroyed(&self, window_label: &str) -> bool { + stop_openai_voice_for_owner(self, Some(window_label)).unwrap_or_else(|error| { + log::warn!("Failed to stop OpenAI playback for a destroyed window: {error}"); + false + }) + } +} + #[tauri::command] pub fn stop_openai_voice(state: State<'_, OpenAiVoiceState>) -> Result { stop_openai_voice_inner(&state) @@ -499,32 +538,6 @@ fn run_openai_voice_stream( interruption_mode: VoiceInterruptionMode, interruption_sensitivity: InterruptionSensitivity, speed: f32, -) -> Result { - run_openai_voice_stream_inner( - app, - stream_id, - key, - active, - receiver, - native_voice, - interruption_mode, - interruption_sensitivity, - speed, - ) -} - -#[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -fn run_openai_voice_stream_inner( - app: &AppHandle, - stream_id: &str, - key: String, - active: Arc, - receiver: mpsc::Receiver, - native_voice: NativeVoiceState, - interruption_mode: VoiceInterruptionMode, - interruption_sensitivity: InterruptionSensitivity, - speed: f32, ) -> Result { let client = client()?; let runtime = tokio::runtime::Builder::new_current_thread() @@ -534,8 +547,7 @@ fn run_openai_voice_stream_inner( let player = PocketAudioPlayer::new(TTS_SAMPLE_RATE, 1.0, None)?; let output_device = effective_output_device_name(None); let suppress_capture = should_suppress_capture(interruption_mode, output_device.as_deref()); - let _assistant_speech = - native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture); + let mut assistant_speech = None::; let mut pending = String::new(); let mut delivery = VoiceDeliveryProgress { sample_rate: TTS_SAMPLE_RATE, @@ -567,6 +579,10 @@ fn run_openai_voice_stream_inner( &mut pending, &mut delivery, &mut started, + &native_voice, + interruption_sensitivity, + suppress_capture, + &mut assistant_speech, speed, ) .map_err(|error| StreamFailure { @@ -587,6 +603,10 @@ fn run_openai_voice_stream_inner( &mut pending, &mut delivery, &mut started, + &native_voice, + interruption_sensitivity, + suppress_capture, + &mut assistant_speech, speed, ) .map_err(|error| StreamFailure { @@ -606,6 +626,10 @@ fn run_openai_voice_stream_inner( &mut pending, &mut delivery, &mut started, + &native_voice, + interruption_sensitivity, + suppress_capture, + &mut assistant_speech, speed, )?; while active.load(Ordering::SeqCst) && !player.is_empty() { @@ -671,6 +695,10 @@ fn speak_pending( pending: &mut String, delivery: &mut VoiceDeliveryProgress, started: &mut bool, + native_voice: &NativeVoiceState, + interruption_sensitivity: InterruptionSensitivity, + suppress_capture: bool, + assistant_speech: &mut Option, speed: f32, ) -> Result<(), String> { let text = std::mem::take(pending).trim().to_string(); @@ -688,8 +716,16 @@ fn speak_pending( total_frames: 0, synthesis_complete: false, }); - let mut bytes = - runtime.block_on(openai_speech_stream(client, key, chunk.to_string(), speed))?; + let Some(mut bytes) = runtime.block_on(openai_speech_stream_cancellable( + client, + key, + chunk.to_string(), + speed, + active, + ))? + else { + return Ok(()); + }; let mut pcm_remainder = Vec::::new(); let mut initial_samples = Vec::::new(); loop { @@ -719,6 +755,10 @@ fn speak_pending( } else { initial_samples.extend_from_slice(&samples); if initial_samples.len() >= INITIAL_PLAYBACK_BUFFER_FRAMES { + assistant_speech.get_or_insert_with(|| { + native_voice + .begin_assistant_speech(interruption_sensitivity, suppress_capture) + }); player.enqueue(&initial_samples)?; initial_samples.clear(); *started = true; @@ -738,6 +778,9 @@ fn speak_pending( return Err("OpenAI speech returned an incomplete PCM sample".to_string()); } if !initial_samples.is_empty() { + assistant_speech.get_or_insert_with(|| { + native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture) + }); player.enqueue(&initial_samples)?; if !*started { *started = true; @@ -755,6 +798,39 @@ fn speak_pending( Ok(()) } +#[cfg(target_os = "macos")] +async fn run_while_active( + future: F, + active: &AtomicBool, +) -> Option { + tokio::pin!(future); + loop { + tokio::select! { + result = &mut future => return Some(result), + _ = tokio::time::sleep(Duration::from_millis(20)) => { + if !active.load(Ordering::SeqCst) { + return None; + } + } + } + } +} + +#[cfg(target_os = "macos")] +async fn openai_speech_stream_cancellable( + client: &reqwest::Client, + key: &str, + input: String, + speed: f32, + active: &AtomicBool, +) -> Result>>, String> +{ + match run_while_active(openai_speech_stream(client, key, input, speed), active).await { + Some(result) => result.map(Some), + None => Ok(None), + } +} + #[cfg(target_os = "macos")] async fn openai_speech_stream( client: &reqwest::Client, @@ -894,10 +970,54 @@ fn format_openai_response_error(action: &str, status: reqwest::StatusCode, body: mod tests { use super::*; + #[test] + fn destroyed_window_only_stops_its_openai_stream() { + let state = OpenAiVoiceState::default(); + let active = Arc::new(AtomicBool::new(true)); + let (sender, _receiver) = mpsc::channel(); + { + let mut playback = state.playback.lock().expect("playback state"); + playback.active = Some(active.clone()); + playback.stream = Some(ActiveOpenAiStream { + id: "stream-1".to_string(), + owner_window: "session-window".to_string(), + sender, + }); + } + + assert!(!state.stop_for_window_destroyed("other-window")); + assert!(active.load(Ordering::SeqCst)); + assert!(state.stop_for_window_destroyed("session-window")); + assert!(!active.load(Ordering::SeqCst)); + } + #[cfg(target_os = "macos")] #[test] fn chunks_tts_text_on_char_boundaries() { assert_eq!(chunk_text("hello", 10), vec!["hello"]); assert_eq!(chunk_text("ééé", 3), vec!["é", "é", "é"]); } + + #[cfg(target_os = "macos")] + #[test] + fn cancels_a_stalled_speech_request() { + let active = Arc::new(AtomicBool::new(true)); + let active_for_thread = active.clone(); + let cancellation = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(30)); + active_for_thread.store(false, Ordering::SeqCst); + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("runtime"); + + let result = runtime.block_on(run_while_active( + std::future::pending::<()>(), + active.as_ref(), + )); + + cancellation.join().expect("cancellation thread"); + assert_eq!(result, None); + } } diff --git a/src/features/providers/api/credentials.test.ts b/src/features/providers/api/credentials.test.ts index 39b30c508..b2b5fccf2 100644 --- a/src/features/providers/api/credentials.test.ts +++ b/src/features/providers/api/credentials.test.ts @@ -4,6 +4,7 @@ import { checkAllProviderStatus, deleteProviderConfig, getProviderConfig, + onProviderConfigChanged, saveProviderConfig, } from "./credentials"; @@ -86,6 +87,20 @@ describe("provider credential API", () => { }); }); + it("notifies subscribers after provider credentials change", async () => { + const listener = vi.fn(); + const unsubscribe = onProviderConfigChanged(listener); + mocks.configSave.mockResolvedValue({ + status: { providerId: "openai", isConfigured: true }, + refresh: { started: [], skipped: [] }, + }); + + await saveProviderConfig("openai", []); + + expect(listener).toHaveBeenCalledWith("openai"); + unsubscribe(); + }); + it("deletes provider config through ACP", async () => { const response = { status: { diff --git a/src/features/providers/api/credentials.ts b/src/features/providers/api/credentials.ts index 85299370d..8b07f321a 100644 --- a/src/features/providers/api/credentials.ts +++ b/src/features/providers/api/credentials.ts @@ -11,6 +11,20 @@ import { shareInFlight } from "@/shared/lib/shareInFlight"; export type ProviderStatus = ProviderConfigStatusDto; export type ProviderFieldSaveInput = ProviderConfigFieldUpdate; +type ProviderConfigChangedListener = (providerId: string) => void; +const providerConfigChangedListeners = new Set(); + +function notifyProviderConfigChanged(providerId: string) { + for (const listener of providerConfigChangedListeners) listener(providerId); +} + +export function onProviderConfigChanged( + listener: ProviderConfigChangedListener, +): () => void { + providerConfigChangedListeners.add(listener); + return () => providerConfigChangedListeners.delete(listener); +} + export async function getProviderConfig( providerId: string, ): Promise { @@ -30,6 +44,7 @@ export async function saveProviderConfig( providerId, fields, }); + notifyProviderConfigChanged(providerId); return response; } @@ -40,6 +55,7 @@ export async function authenticateProviderConfig( const response = await client.goose.GooseUnstableProvidersConfigAuthenticate({ providerId, }); + notifyProviderConfigChanged(providerId); return response; } @@ -50,6 +66,7 @@ export async function deleteProviderConfig( const response = await client.goose.GooseUnstableProvidersConfigDelete({ providerId, }); + notifyProviderConfigChanged(providerId); return response; } diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts index 77bfa5879..facdf429d 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts @@ -3,6 +3,7 @@ import { getOpenAiVoiceStatus, type OpenAiVoiceStatus, } from "../api/openAiVoice"; +import { onProviderConfigChanged } from "@/features/providers/api/credentials"; export function useOpenAiVoiceSetup(enabled = true) { const [status, setStatus] = useState(null); @@ -11,18 +12,28 @@ export function useOpenAiVoiceSetup(enabled = true) { useEffect(() => { if (!enabled) return; let active = true; - void getOpenAiVoiceStatus().then( - (next) => { - if (active) setStatus(next); - }, - (cause) => { - if (active) { - setError(cause instanceof Error ? cause.message : String(cause)); - } - }, - ); + const refresh = () => { + void getOpenAiVoiceStatus().then( + (next) => { + if (active) { + setStatus(next); + setError(null); + } + }, + (cause) => { + if (active) { + setError(cause instanceof Error ? cause.message : String(cause)); + } + }, + ); + }; + refresh(); + const unsubscribe = onProviderConfigChanged((providerId) => { + if (providerId === "openai") refresh(); + }); return () => { active = false; + unsubscribe(); }; }, [enabled]); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 9cd66e2d4..8a04b6b27 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -879,6 +879,89 @@ describe("native assistant speech stream", () => { } }); + it("holds an interruption past VAD idle until delayed final transcript arrives", async () => { + vi.useFakeTimers(); + try { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Interrupted reply." }]), + ]); + await vi.runAllTimersAsync(); + await vi.waitFor(() => expect(mocks.append).toHaveBeenCalled()); + const firstStreamId = mocks.start.mock.calls[0]?.[0] as string; + emit("started"); + + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.runAllTimersAsync(); + expect(mocks.stop).toHaveBeenCalled(); + mocks.streamHandler?.({ + streamId: firstStreamId, + state: "interrupted", + error: null, + delivery: { segments: [] }, + }); + + useVoiceConversationStore.setState({ userSpeaking: false }); + await vi.advanceTimersByTimeAsync(300); + expect(mocks.start).toHaveBeenCalledTimes(1); + + finalizeVoiceTranscript("delayed-final"); + useChatStore + .getState() + .setMessages("session-1", [ + assistant( + [{ type: "text", text: "Interrupted reply." }], + "completed", + "assistant-1", + ), + voiceUser("delayed-final"), + ]); + await vi.runAllTimersAsync(); + + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(takeVoicePlaybackNotices("session-1")).toContain( + "Original text: Interrupted reply.", + ); + } finally { + vi.useRealTimers(); + } + }); + + it("resumes a no-result interruption after the recognition segment timeout", async () => { + vi.useFakeTimers(); + try { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant( + [{ type: "text", text: "False alarm reply." }], + "completed", + ), + ]); + useVoiceConversationStore.setState({ userSpeaking: true }); + await vi.runAllTimersAsync(); + expect(mocks.start).not.toHaveBeenCalled(); + + useVoiceConversationStore.setState({ userSpeaking: false }); + await vi.advanceTimersByTimeAsync(300); + expect(mocks.start).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(200); + await vi.runAllTimersAsync(); + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(mocks.append).toHaveBeenCalledWith( + mocks.start.mock.calls[0]?.[0], + "False alarm reply.", + ); + expect(takeVoicePlaybackNotices("session-1")).toBeNull(); + } finally { + vi.useRealTimers(); + } + }); + it("ignores a late started event after interruption is requested", async () => { let resolveStop: ((stopped: boolean) => void) | undefined; startNativeAssistantSpeech("session-1", vi.fn()); @@ -2500,7 +2583,7 @@ describe("native assistant speech stream", () => { }, }); useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(250); + await vi.advanceTimersByTimeAsync(500); await vi.runAllTimersAsync(); const secondStreamId = mocks.start.mock.calls[1]?.[0] as string; @@ -2536,7 +2619,7 @@ describe("native assistant speech stream", () => { }, }); useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(250); + await vi.advanceTimersByTimeAsync(500); await vi.runAllTimersAsync(); const thirdStreamId = mocks.start.mock.calls[2]?.[0] as string; @@ -2604,7 +2687,7 @@ describe("native assistant speech stream", () => { ]); useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(250); + await vi.advanceTimersByTimeAsync(500); await vi.runAllTimersAsync(); const resumedStreamId = mocks.start.mock.calls[1]?.[0] as string; @@ -2750,7 +2833,7 @@ describe("native assistant speech stream", () => { refreshedSiriVoice, ); useVoiceConversationStore.setState({ userSpeaking: false }); - await vi.advanceTimersByTimeAsync(250); + await vi.advanceTimersByTimeAsync(500); await vi.runAllTimersAsync(); expect(mocks.siriStart).toHaveBeenCalledTimes(2); @@ -2943,6 +3026,43 @@ describe("native assistant speech stream", () => { }); }); + it("preserves the recognition deadline across repeated VAD edges", async () => { + vi.useFakeTimers(); + try { + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Interrupted reply." }]), + ]); + await vi.runAllTimersAsync(); + const firstStreamId = mocks.start.mock.calls[0]?.[0] as string; + + useVoiceConversationStore.setState({ userSpeaking: true }); + mocks.streamHandler?.({ + streamId: firstStreamId, + state: "interrupted", + error: null, + delivery: { segments: [] }, + }); + useVoiceConversationStore.setState({ userSpeaking: false }); + + await vi.advanceTimersByTimeAsync(300); + expect(mocks.start).toHaveBeenCalledTimes(1); + + useVoiceConversationStore.setState({ userSpeaking: true }); + useVoiceConversationStore.setState({ userSpeaking: false }); + await vi.advanceTimersByTimeAsync(199); + expect(mocks.start).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(1); + expect(mocks.start).toHaveBeenCalledTimes(2); + await vi.runAllTimersAsync(); + } finally { + vi.useRealTimers(); + } + }); + it("never starts a held reply when a newer finalized voice transcript arrives", async () => { useVoiceConversationStore.setState({ userSpeaking: true }); startNativeAssistantSpeech("session-1", vi.fn()); diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index 901adc352..ba5c44c98 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -143,6 +143,7 @@ function boundedDeliveryText( } const MALFORMED_VOICE_TRANSCRIPT_KEY = "\0malformed-voice-transcript"; const USER_IDLE_TRANSCRIPT_SETTLE_MS = 250; +const USER_RECOGNITION_SEGMENT_TIMEOUT_MS = 500; function voiceTranscriptKeyForMessage( sessionId: string, @@ -982,7 +983,8 @@ export function startNativeAssistantSpeech( let resumableInterruption: ResumableInterruption | null = null; let heldReleaseReady = false; let interruptionReleaseReady = false; - let idleSettling = false; + let pendingUserRecognitionSegment = false; + let recognitionSegmentTimer: number | null = null; let heldReleaseTimer: number | null = null; const cacheCausalTranscriptKeys = ( @@ -1287,9 +1289,15 @@ export function startNativeAssistantSpeech( return; } const messages = useChatStore.getState().messagesBySession[sessionId] ?? []; - if (heldSpeech || voice.userSpeaking || idleSettling) { - // Text can keep streaming during the idle settling turn. Refresh the - // held snapshot before a finalized voice message can invalidate it. + if ( + heldSpeech || + voice.userSpeaking || + pendingUserRecognitionSegment || + heldReleaseTimer !== null + ) { + // Text can keep streaming while recognition resolves the user's + // interruption segment. Refresh the held snapshot before a finalized + // voice message can invalidate it. holdAssistantChanges(messages); } const finalizedTranscriptKey = voice.latestFinalizedTranscriptKey; @@ -1310,7 +1318,7 @@ export function startNativeAssistantSpeech( interruptActiveUtterance(true, "userSpeaking"); } - if (voice.userSpeaking || idleSettling) return; + if (voice.userSpeaking || pendingUserRecognitionSegment) return; if (heldSpeech && !heldReleaseReady) return; if (resumableInterruption) return; if (activeUtterance?.interruptionRequested) return; @@ -1545,6 +1553,19 @@ export function startNativeAssistantSpeech( } let wasUserSpeaking = initialVoice.userSpeaking; let wasMicrophoneMuted = initialVoice.microphoneMuted; + let latestObservedFinalizedTranscriptKey = + useVoiceConversationStore.getState().latestFinalizedTranscriptKey; + const resolvePendingRecognitionSegment = (releaseSpeech = true) => { + if (recognitionSegmentTimer !== null) { + window.clearTimeout(recognitionSegmentTimer); + recognitionSegmentTimer = null; + } + pendingUserRecognitionSegment = false; + if (!releaseSpeech) return; + heldReleaseReady = heldSpeech !== null; + interruptionReleaseReady = true; + releaseResumableInterruption(); + }; const unsubscribeVoice = useVoiceConversationStore.subscribe((voice) => { const runningForSession = voice.status.lifecycle === "running" && @@ -1565,34 +1586,90 @@ export function startNativeAssistantSpeech( const becameUserSpeaking = voice.userSpeaking && !wasUserSpeaking; const becameUserIdle = !voice.userSpeaking && wasUserSpeaking; const becameMicrophoneMuted = voice.microphoneMuted && !wasMicrophoneMuted; + const finalizedTranscriptChanged = + voice.latestFinalizedTranscriptKey !== + latestObservedFinalizedTranscriptKey; + const hasInterruptedPlaybackHold = + activeUtterance?.interruptionRequested || resumableInterruption !== null; wasUserSpeaking = voice.userSpeaking; wasMicrophoneMuted = voice.microphoneMuted; + latestObservedFinalizedTranscriptKey = voice.latestFinalizedTranscriptKey; if (activeGeneration !== generation) return; if (becameMicrophoneMuted) { - if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer); - heldReleaseTimer = null; - idleSettling = false; + resolvePendingRecognitionSegment(false); interruptionReleaseReady = false; discardHeldAndResumableSpeech(); inspect(); return; } + if (finalizedTranscriptChanged && !pendingUserRecognitionSegment) { + if (heldReleaseTimer !== null) { + window.clearTimeout(heldReleaseTimer); + heldReleaseTimer = null; + } + holdAssistantChanges( + useChatStore.getState().messagesBySession[sessionId] ?? [], + ); + discardInvalidHeldSpeech(voice.latestFinalizedTranscriptKey); + inspect(); + return; + } if (becameUserSpeaking) { - if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer); - heldReleaseTimer = null; - idleSettling = false; + if (heldReleaseTimer !== null) { + window.clearTimeout(heldReleaseTimer); + heldReleaseTimer = null; + } + const interrupted = interruptActiveUtterance(true, "userSpeaking"); + if (interrupted && recognitionSegmentTimer !== null) { + window.clearTimeout(recognitionSegmentTimer); + recognitionSegmentTimer = null; + } + pendingUserRecognitionSegment ||= interrupted; heldReleaseReady = false; interruptionReleaseReady = false; - interruptActiveUtterance(true, "userSpeaking"); + inspect(); + return; + } + if (finalizedTranscriptChanged && pendingUserRecognitionSegment) { + holdAssistantChanges( + useChatStore.getState().messagesBySession[sessionId] ?? [], + ); + const hadResumableInterruption = resumableInterruption !== null; + resolvePendingRecognitionSegment(false); + if (hadResumableInterruption) { + discardResumableInterruption(); + discardInvalidHeldSpeech(voice.latestFinalizedTranscriptKey); + } else { + discardHeldAndResumableSpeech(); + } inspect(); return; } if (becameUserIdle) { if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer); - idleSettling = true; - // VAD can report silence shortly before the recognizer commits its final - // transcript. Give that transcript a bounded opportunity to invalidate - // causally stale speech before releasing the held reply. + if (pendingUserRecognitionSegment || hasInterruptedPlaybackHold) { + pendingUserRecognitionSegment = true; + // VAD silence does not imply recognition is idle. Keep interrupted + // playback held until a final transcript arrives or the unresolved + // user-recognition segment hits a conservative bound. + recognitionSegmentTimer ??= window.setTimeout(() => { + recognitionSegmentTimer = null; + const current = useVoiceConversationStore.getState(); + if ( + activeGeneration !== generation || + current.userSpeaking || + current.status.lifecycle !== "running" || + current.status.sessionId !== sessionId || + !pendingUserRecognitionSegment + ) { + return; + } + resolvePendingRecognitionSegment(true); + inspect(); + }, USER_RECOGNITION_SEGMENT_TIMEOUT_MS); + inspect(); + return; + } heldReleaseTimer = window.setTimeout(() => { heldReleaseTimer = null; const current = useVoiceConversationStore.getState(); @@ -1604,7 +1681,6 @@ export function startNativeAssistantSpeech( ) { return; } - idleSettling = false; heldReleaseReady = heldSpeech !== null; interruptionReleaseReady = true; releaseResumableInterruption(); @@ -1615,8 +1691,15 @@ export function startNativeAssistantSpeech( inspect(); }); stopVoiceSubscription = () => { - if (heldReleaseTimer !== null) window.clearTimeout(heldReleaseTimer); + if (heldReleaseTimer !== null) { + window.clearTimeout(heldReleaseTimer); + } + if (recognitionSegmentTimer !== null) { + window.clearTimeout(recognitionSegmentTimer); + } heldReleaseTimer = null; + recognitionSegmentTimer = null; + pendingUserRecognitionSegment = false; discardHeldAndResumableSpeech(); unsubscribeVoice(); }; diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 121da1064..36c641655 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -50,17 +50,20 @@ const microphonePermissionState = vi.hoisted(() => ({ openSettingsError: false, openSettings: vi.fn(), })); - -vi.mock("../api/openAiVoice", () => ({ - getOpenAiVoiceStatus: vi.fn().mockResolvedValue({ +const openAiStatusState = vi.hoisted(() => ({ + current: { configured: true, transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, ttsAvailable: true, - unavailableReason: null, - }), + unavailableReason: null as string | null, + }, +})); + +vi.mock("../api/openAiVoice", () => ({ + getOpenAiVoiceStatus: vi.fn(() => Promise.resolve(openAiStatusState.current)), })); vi.mock("../hooks/usePocketVoiceSetup", () => ({ usePocketVoiceSetup: () => setupState.current, @@ -202,6 +205,15 @@ describe("VoiceSettings", () => { }; interruptionState.mode = "automatic"; siriSetupState.current = siriSetup(); + openAiStatusState.current = { + configured: true, + transcriptionModel: "gpt-live-transcribe", + speechModel: "gpt-4o-mini-tts", + speechVoice: "marin", + playbackSpeed: 1, + ttsAvailable: true, + unavailableReason: null, + }; }); it("renders selected OpenAI output settings", async () => { @@ -215,6 +227,27 @@ describe("VoiceSettings", () => { expect(screen.getByText("Playback speed")).toBeInTheDocument(); }); + it("uses OpenAI guidance when the selected OpenAI output is not ready", async () => { + outputState.backend = "openai"; + openAiStatusState.current = { + ...openAiStatusState.current, + configured: false, + unavailableReason: + "Configure the OpenAI provider in Berd to use OpenAI voice.", + }; + setupState.current = setup(pocketStatus({ parakeetInstalled: true })); + renderWithProviders(); + + expect( + await screen.findByText( + "OpenAI voice is not ready. Configure the OpenAI provider in Berd, then try again.", + ), + ).toBeInTheDocument(); + expect( + screen.queryByText(/Pocket TTS is not installed/), + ).not.toBeInTheDocument(); + }); + it("shows interruption modes without VAD controls", () => { setupState.current = setup(pocketStatus()); renderWithProviders(); diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index cafde4bf6..78a552c0a 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -69,6 +69,7 @@ function readinessDescriptionKey( ? "voice.notReadyMacInput" : "voice.notReadyInput"; } + if (backend === "openai") return "voice.notReadyOpenAi"; return backend === "siri" ? "voice.notReadySiriOutput" : "voice.notReadyPocketOutput"; diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index 6c966123e..fc2256d32 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -870,7 +870,6 @@ }, "voice": { "backendMacSpeech": "Apple speech recognition", - "backendOpenAiStt": "OpenAI speech-to-text", "backendOpenAiTts": "OpenAI text-to-speech", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", @@ -929,7 +928,6 @@ "openMicrophoneSettings": "Open Microphone Settings", "openMicrophoneSettingsError": "Couldn't open Microphone Settings. Open System Settings and select Privacy & Security > Microphone.", "openAiChecking": "Checking the OpenAI provider configured in Berd…", - "openAiSttConfigured": "Uses Berd’s configured OpenAI credential with {{model}}.", "openAiTtsConfigured": "Uses Berd’s configured OpenAI credential with {{model}} and the {{voice}} voice. OpenAI voices are AI-generated.", "outputBackendDescription": "Choose how Berd speaks assistant responses.", "playbackSpeed": "Playback speed", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index eddfa699f..3163c7e8e 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -873,7 +873,6 @@ }, "voice": { "backendMacSpeech": "Reconocimiento de voz de Apple", - "backendOpenAiStt": "Voz a texto de OpenAI", "backendOpenAiTts": "Texto a voz de OpenAI", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", @@ -932,7 +931,6 @@ "openMicrophoneSettings": "Abrir ajustes del micrófono", "openMicrophoneSettingsError": "No se pudieron abrir los ajustes del micrófono. Abre Ajustes del Sistema y selecciona Privacidad y seguridad > Micrófono.", "openAiChecking": "Comprobando el proveedor de OpenAI configurado en Berd…", - "openAiSttConfigured": "Usa la credencial de OpenAI configurada en Berd con {{model}}.", "openAiTtsConfigured": "Usa la credencial de OpenAI configurada en Berd con {{model}} y la voz {{voice}}. Las voces de OpenAI son generadas por IA.", "outputBackendDescription": "Elige cómo Berd reproduce las respuestas del asistente.", "playbackSpeed": "Velocidad de reproducción", From e64516cbc9bd68639c84d7bfb0b3a41feb074e14 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 17:24:20 -0400 Subject: [PATCH 07/18] fix(voice): follow Goose OpenAI configuration --- src-tauri/src/commands/openai_audio.rs | 205 +++++++++++++++--- .../providers/api/credentials.test.ts | 23 +- src/features/providers/api/credentials.ts | 24 +- .../hooks/useOpenAiVoiceSetup.test.tsx | 66 ++++++ .../hooks/useOpenAiVoiceSetup.ts | 14 +- 5 files changed, 282 insertions(+), 50 deletions(-) create mode 100644 src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 581f6c6f7..4581f70c9 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -21,13 +21,15 @@ use tauri::{AppHandle, Emitter, State}; use super::{ native_voice::AssistantSpeechGuard, pocket_audio_player::PocketAudioPlayer, - pocket_voice::{effective_output_device_name, should_suppress_capture}, + pocket_voice::{ + effective_output_device_name, playback_latency_safety_duration, should_suppress_capture, + }, }; use super::{ native_voice::{InterruptionSensitivity, NativeVoiceState}, pocket_voice::VoiceInterruptionMode, }; -#[cfg(target_os = "macos")] +#[cfg(any(test, target_os = "macos"))] use std::time::Instant; const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; @@ -132,6 +134,22 @@ fn env_trimmed(name: &str) -> Option { .filter(|value| !value.is_empty()) } +fn goose_yaml_value(path: &std::path::Path, name: &str) -> Result, String> { + if !path.exists() { + return Ok(None); + } + let payload = std::fs::read_to_string(path) + .map_err(|error| format!("Could not read Goose configuration: {error}"))?; + let values: serde_json::Value = yaml_serde::from_str(&payload) + .map_err(|error| format!("Goose configuration is invalid: {error}"))?; + Ok(values + .get(name) + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string)) +} + fn goose_openai_api_key() -> Result, String> { if let Some(value) = env_trimmed("OPENAI_API_KEY") { return Ok(Some(value)); @@ -142,18 +160,23 @@ fn goose_openai_api_key() -> Result, String> { { match keyring::Entry::new("goose", "secrets") { Ok(entry) => match entry.get_password() { - Ok(payload) => { - let secrets: serde_json::Value = - serde_json::from_str(&payload).map_err(|error| { - format!("Goose's secure credential store is not valid JSON: {error}") - })?; - return Ok(secrets - .get("OPENAI_API_KEY") - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToString::to_string)); - } + Ok(payload) => match serde_json::from_str::(&payload) { + Ok(secrets) => { + if let Some(key) = secrets + .get("OPENAI_API_KEY") + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + { + return Ok(Some(key.to_string())); + } + } + Err(error) => { + secure_store_error = Some(format!( + "Goose's secure credential store is not valid JSON: {error}" + )); + } + }, Err(keyring::Error::NoEntry) => {} Err(error) => { secure_store_error = Some(format!( @@ -173,23 +196,14 @@ fn goose_openai_api_key() -> Result, String> { .parent() .ok_or_else(|| "Could not resolve Goose's credential directory".to_string())? .join("secrets.yaml"); - if !secrets_path.exists() { - #[cfg(target_os = "macos")] - if let Some(error) = secure_store_error { - return Err(error); - } - return Ok(None); + if let Some(key) = goose_yaml_value(&secrets_path, "OPENAI_API_KEY")? { + return Ok(Some(key)); } - let payload = std::fs::read_to_string(&secrets_path) - .map_err(|error| format!("Could not read Goose's credential file: {error}"))?; - let secrets: serde_json::Value = yaml_serde::from_str(&payload) - .map_err(|error| format!("Goose's credential file is invalid: {error}"))?; - Ok(secrets - .get("OPENAI_API_KEY") - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToString::to_string)) + #[cfg(target_os = "macos")] + if let Some(error) = secure_store_error { + return Err(error); + } + Ok(None) } pub(crate) fn api_key() -> Result { @@ -199,8 +213,30 @@ pub(crate) fn api_key() -> Result { }) } -fn base_url() -> String { - env_trimmed("OPENAI_BASE_URL").unwrap_or_else(|| DEFAULT_BASE_URL.to_string()) +fn openai_host_base_url(host: String) -> String { + let host = host.trim_end_matches('/'); + if host.ends_with("/v1") { + host.to_string() + } else { + format!("{host}/v1") + } +} + +fn base_url() -> Result { + if let Some(host) = env_trimmed("OPENAI_HOST") { + return Ok(openai_host_base_url(host)); + } + if let Some(base_url) = env_trimmed("OPENAI_BASE_URL") { + return Ok(base_url); + } + let config_path = crate::services::goose_config::config_path()?; + if let Some(base_url) = goose_yaml_value(&config_path, "OPENAI_BASE_URL")? { + return Ok(base_url); + } + if let Some(host) = goose_yaml_value(&config_path, "OPENAI_HOST")? { + return Ok(openai_host_base_url(host)); + } + Ok(DEFAULT_BASE_URL.to_string()) } fn speech_model() -> String { @@ -212,7 +248,7 @@ fn speech_voice() -> String { } fn endpoint(path: &str) -> Result { - let mut url = base_url(); + let mut url = base_url()?; while url.ends_with('/') { url.pop(); } @@ -547,7 +583,9 @@ fn run_openai_voice_stream( let player = PocketAudioPlayer::new(TTS_SAMPLE_RATE, 1.0, None)?; let output_device = effective_output_device_name(None); let suppress_capture = should_suppress_capture(interruption_mode, output_device.as_deref()); + let output_latency_grace = playback_latency_safety_duration(output_device.as_deref()); let mut assistant_speech = None::; + let mut playback_drained_at = None::; let mut pending = String::new(); let mut delivery = VoiceDeliveryProgress { sample_rate: TTS_SAMPLE_RATE, @@ -557,6 +595,13 @@ fn run_openai_voice_stream( let mut last_progress = Instant::now(); loop { + update_openai_assistant_speech( + player.is_empty(), + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + Instant::now(), + ); if !active.load(Ordering::SeqCst) { player.stop(); return Ok(StreamOutcome { @@ -583,6 +628,8 @@ fn run_openai_voice_stream( interruption_sensitivity, suppress_capture, &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, speed, ) .map_err(|error| StreamFailure { @@ -607,6 +654,8 @@ fn run_openai_voice_stream( interruption_sensitivity, suppress_capture, &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, speed, ) .map_err(|error| StreamFailure { @@ -630,6 +679,8 @@ fn run_openai_voice_stream( interruption_sensitivity, suppress_capture, &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, speed, )?; while active.load(Ordering::SeqCst) && !player.is_empty() { @@ -699,6 +750,8 @@ fn speak_pending( interruption_sensitivity: InterruptionSensitivity, suppress_capture: bool, assistant_speech: &mut Option, + playback_drained_at: &mut Option, + output_latency_grace: Duration, speed: f32, ) -> Result<(), String> { let text = std::mem::take(pending).trim().to_string(); @@ -729,6 +782,13 @@ fn speak_pending( let mut pcm_remainder = Vec::::new(); let mut initial_samples = Vec::::new(); loop { + update_openai_assistant_speech( + player.is_empty(), + assistant_speech, + playback_drained_at, + output_latency_grace, + Instant::now(), + ); if !active.load(Ordering::SeqCst) { return Ok(()); } @@ -759,6 +819,7 @@ fn speak_pending( native_voice .begin_assistant_speech(interruption_sensitivity, suppress_capture) }); + *playback_drained_at = None; player.enqueue(&initial_samples)?; initial_samples.clear(); *started = true; @@ -781,6 +842,7 @@ fn speak_pending( assistant_speech.get_or_insert_with(|| { native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture) }); + *playback_drained_at = None; player.enqueue(&initial_samples)?; if !*started { *started = true; @@ -894,6 +956,41 @@ fn pcm16le_to_f32(bytes: &[u8]) -> Vec { .collect() } +#[cfg(any(test, target_os = "macos"))] +fn openai_assistant_speech_grace_elapsed( + playback_drained: bool, + guard_active: bool, + playback_drained_at: &mut Option, + output_latency_grace: Duration, + now: Instant, +) -> bool { + if !guard_active || !playback_drained { + *playback_drained_at = None; + return false; + } + let drained_at = *playback_drained_at.get_or_insert(now); + now.saturating_duration_since(drained_at) >= output_latency_grace +} + +#[cfg(target_os = "macos")] +fn update_openai_assistant_speech( + playback_drained: bool, + assistant_speech: &mut Option, + playback_drained_at: &mut Option, + output_latency_grace: Duration, + now: Instant, +) { + if openai_assistant_speech_grace_elapsed( + playback_drained, + assistant_speech.is_some(), + playback_drained_at, + output_latency_grace, + now, + ) { + assistant_speech.take(); + } +} + #[cfg(target_os = "macos")] fn upsert_delivery_segment( delivery: &mut VoiceDeliveryProgress, @@ -991,6 +1088,48 @@ mod tests { assert!(!active.load(Ordering::SeqCst)); } + #[test] + fn openai_host_configuration_resolves_to_the_v1_api_root() { + assert_eq!( + openai_host_base_url("https://proxy.example".to_string()), + "https://proxy.example/v1" + ); + assert_eq!( + openai_host_base_url("https://proxy.example/v1/".to_string()), + "https://proxy.example/v1" + ); + } + + #[test] + fn capture_suppression_ends_after_playback_drain_grace() { + let started = Instant::now(); + let mut drained_at = None; + let grace = Duration::from_millis(100); + + assert!(!openai_assistant_speech_grace_elapsed( + true, + true, + &mut drained_at, + grace, + started, + )); + assert!(openai_assistant_speech_grace_elapsed( + true, + true, + &mut drained_at, + grace, + started + grace, + )); + assert!(!openai_assistant_speech_grace_elapsed( + false, + true, + &mut drained_at, + grace, + started + grace, + )); + assert_eq!(drained_at, None); + } + #[cfg(target_os = "macos")] #[test] fn chunks_tts_text_on_char_boundaries() { diff --git a/src/features/providers/api/credentials.test.ts b/src/features/providers/api/credentials.test.ts index b2b5fccf2..c0c2c2e23 100644 --- a/src/features/providers/api/credentials.test.ts +++ b/src/features/providers/api/credentials.test.ts @@ -15,15 +15,30 @@ const mocks = vi.hoisted(() => ({ configDelete: vi.fn(), configStatus: vi.fn(), getClient: vi.fn(), + emit: vi.fn(), + listen: vi.fn(), + providerConfigHandler: null as + | ((event: { payload: { providerId: string } }) => void) + | null, })); vi.mock("@/shared/api/acpConnection", () => ({ getClient: () => mocks.getClient(), })); +vi.mock("@tauri-apps/api/event", () => ({ + emit: (...args: unknown[]) => mocks.emit(...args), + listen: (event: string, handler: typeof mocks.providerConfigHandler) => { + mocks.listen(event, handler); + mocks.providerConfigHandler = handler; + return Promise.resolve(vi.fn()); + }, +})); describe("provider credential API", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.emit.mockResolvedValue(undefined); + mocks.providerConfigHandler = null; mocks.getClient.mockResolvedValue({ goose: { GooseUnstableProvidersConfigRead: mocks.configRead, @@ -87,9 +102,9 @@ describe("provider credential API", () => { }); }); - it("notifies subscribers after provider credentials change", async () => { + it("broadcasts provider credential changes across renderer windows", async () => { const listener = vi.fn(); - const unsubscribe = onProviderConfigChanged(listener); + const unsubscribe = await onProviderConfigChanged(listener); mocks.configSave.mockResolvedValue({ status: { providerId: "openai", isConfigured: true }, refresh: { started: [], skipped: [] }, @@ -97,6 +112,10 @@ describe("provider credential API", () => { await saveProviderConfig("openai", []); + expect(mocks.emit).toHaveBeenCalledWith("provider-config:changed", { + providerId: "openai", + }); + mocks.providerConfigHandler?.({ payload: { providerId: "openai" } }); expect(listener).toHaveBeenCalledWith("openai"); unsubscribe(); }); diff --git a/src/features/providers/api/credentials.ts b/src/features/providers/api/credentials.ts index 8b07f321a..d595f926b 100644 --- a/src/features/providers/api/credentials.ts +++ b/src/features/providers/api/credentials.ts @@ -4,6 +4,7 @@ import type { ProviderConfigStatusDto, ProviderSecretDto, } from "@aaif/goose-sdk"; +import { emit, listen, type UnlistenFn } from "@tauri-apps/api/event"; import type { ProviderFieldValue } from "@/shared/types/providers"; import { getClient } from "@/shared/api/acpConnection"; import { shareInFlight } from "@/shared/lib/shareInFlight"; @@ -11,18 +12,19 @@ import { shareInFlight } from "@/shared/lib/shareInFlight"; export type ProviderStatus = ProviderConfigStatusDto; export type ProviderFieldSaveInput = ProviderConfigFieldUpdate; -type ProviderConfigChangedListener = (providerId: string) => void; -const providerConfigChangedListeners = new Set(); +const PROVIDER_CONFIG_CHANGED_EVENT = "provider-config:changed"; -function notifyProviderConfigChanged(providerId: string) { - for (const listener of providerConfigChangedListeners) listener(providerId); +async function notifyProviderConfigChanged(providerId: string) { + await emit(PROVIDER_CONFIG_CHANGED_EVENT, { providerId }); } export function onProviderConfigChanged( - listener: ProviderConfigChangedListener, -): () => void { - providerConfigChangedListeners.add(listener); - return () => providerConfigChangedListeners.delete(listener); + listener: (providerId: string) => void, +): Promise { + return listen<{ providerId: string }>( + PROVIDER_CONFIG_CHANGED_EVENT, + (event) => listener(event.payload.providerId), + ); } export async function getProviderConfig( @@ -44,7 +46,7 @@ export async function saveProviderConfig( providerId, fields, }); - notifyProviderConfigChanged(providerId); + await notifyProviderConfigChanged(providerId); return response; } @@ -55,7 +57,7 @@ export async function authenticateProviderConfig( const response = await client.goose.GooseUnstableProvidersConfigAuthenticate({ providerId, }); - notifyProviderConfigChanged(providerId); + await notifyProviderConfigChanged(providerId); return response; } @@ -66,7 +68,7 @@ export async function deleteProviderConfig( const response = await client.goose.GooseUnstableProvidersConfigDelete({ providerId, }); - notifyProviderConfigChanged(providerId); + await notifyProviderConfigChanged(providerId); return response; } diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx new file mode 100644 index 000000000..ffb481ab9 --- /dev/null +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -0,0 +1,66 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenAiVoiceStatus } from "../api/openAiVoice"; +import { useOpenAiVoiceSetup } from "./useOpenAiVoiceSetup"; + +const mocks = vi.hoisted(() => ({ + getStatus: vi.fn<() => Promise>(), + configChanged: null as ((providerId: string) => void) | null, +})); + +vi.mock("../api/openAiVoice", () => ({ + getOpenAiVoiceStatus: () => mocks.getStatus(), +})); + +vi.mock("@/features/providers/api/credentials", () => ({ + onProviderConfigChanged: (listener: (providerId: string) => void) => { + mocks.configChanged = listener; + return Promise.resolve(vi.fn()); + }, +})); + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((next) => { + resolve = next; + }); + return { promise, resolve }; +} + +function status(configured: boolean): OpenAiVoiceStatus { + return { + configured, + transcriptionModel: "gpt-live-transcribe", + speechModel: "gpt-4o-mini-tts", + speechVoice: "marin", + playbackSpeed: 1, + ttsAvailable: true, + unavailableReason: configured ? null : "Configure OpenAI.", + }; +} + +describe("useOpenAiVoiceSetup", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.configChanged = null; + }); + + it("keeps the latest credential refresh when responses resolve out of order", async () => { + const initial = deferred(); + const refreshed = deferred(); + mocks.getStatus + .mockReturnValueOnce(initial.promise) + .mockReturnValueOnce(refreshed.promise); + const { result } = renderHook(() => useOpenAiVoiceSetup()); + await waitFor(() => expect(mocks.configChanged).not.toBeNull()); + + act(() => mocks.configChanged?.("openai")); + refreshed.resolve(status(true)); + await waitFor(() => expect(result.current.status?.configured).toBe(true)); + + initial.resolve(status(false)); + await act(async () => Promise.resolve()); + + expect(result.current.status?.configured).toBe(true); + }); +}); diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts index facdf429d..c452b4b0e 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts @@ -12,28 +12,34 @@ export function useOpenAiVoiceSetup(enabled = true) { useEffect(() => { if (!enabled) return; let active = true; + let refreshGeneration = 0; + let unsubscribe: (() => void) | null = null; const refresh = () => { + const generation = ++refreshGeneration; void getOpenAiVoiceStatus().then( (next) => { - if (active) { + if (active && generation === refreshGeneration) { setStatus(next); setError(null); } }, (cause) => { - if (active) { + if (active && generation === refreshGeneration) { setError(cause instanceof Error ? cause.message : String(cause)); } }, ); }; refresh(); - const unsubscribe = onProviderConfigChanged((providerId) => { + void onProviderConfigChanged((providerId) => { if (providerId === "openai") refresh(); + }).then((nextUnsubscribe) => { + if (active) unsubscribe = nextUnsubscribe; + else nextUnsubscribe(); }); return () => { active = false; - unsubscribe(); + unsubscribe?.(); }; }, [enabled]); From 1eb21676fb87e1d4ee20b6ef333f93d010857566 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 17:45:51 -0400 Subject: [PATCH 08/18] fix(voice): keep OpenAI setup and playback synchronized --- src-tauri/src/commands/openai_audio.rs | 102 ++++++++++++++---- .../voice-conversation/api/openAiVoice.ts | 6 +- .../hooks/useOpenAiVoiceSetup.test.tsx | 20 +++- .../hooks/useOpenAiVoiceSetup.ts | 7 +- .../ui/VoiceSettings.test.tsx | 8 +- .../voice-conversation/ui/VoiceSettings.tsx | 33 +----- 6 files changed, 119 insertions(+), 57 deletions(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 4581f70c9..c2e24696c 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -213,28 +213,40 @@ pub(crate) fn api_key() -> Result { }) } -fn openai_host_base_url(host: String) -> String { - let host = host.trim_end_matches('/'); - if host.ends_with("/v1") { - host.to_string() +fn normalize_openai_base_url(raw_url: String, assume_v1: bool) -> Result { + let mut url = reqwest::Url::parse(&raw_url) + .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; + if !matches!(url.scheme(), "http" | "https") { + return Err("OpenAI voice endpoint must use HTTP or HTTPS".to_string()); + } + let path = url.path().trim_end_matches('/').to_string(); + if assume_v1 || path.is_empty() { + let path = if path.ends_with("/v1") { + path + } else { + format!("{path}/v1") + }; + url.set_path(&path); } else { - format!("{host}/v1") + url.set_path(&path); } + url.set_fragment(None); + Ok(url.to_string().trim_end_matches('/').to_string()) } fn base_url() -> Result { if let Some(host) = env_trimmed("OPENAI_HOST") { - return Ok(openai_host_base_url(host)); + return normalize_openai_base_url(host, true); } if let Some(base_url) = env_trimmed("OPENAI_BASE_URL") { - return Ok(base_url); + return normalize_openai_base_url(base_url, false); } let config_path = crate::services::goose_config::config_path()?; if let Some(base_url) = goose_yaml_value(&config_path, "OPENAI_BASE_URL")? { - return Ok(base_url); + return normalize_openai_base_url(base_url, false); } if let Some(host) = goose_yaml_value(&config_path, "OPENAI_HOST")? { - return Ok(openai_host_base_url(host)); + return normalize_openai_base_url(host, true); } Ok(DEFAULT_BASE_URL.to_string()) } @@ -248,15 +260,19 @@ fn speech_voice() -> String { } fn endpoint(path: &str) -> Result { - let mut url = base_url()?; - while url.ends_with('/') { - url.pop(); - } - let path = path.trim_start_matches('/'); - let full = format!("{url}/{path}"); - reqwest::Url::parse(&full) - .map(|_| full) - .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}")) + endpoint_for_base_url(&base_url()?, path) +} + +fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { + let mut url = reqwest::Url::parse(base_url) + .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; + let base_path = url.path().trim_end_matches('/'); + url.set_path(&format!("{base_path}/{}", path.trim_start_matches('/'))); + Ok(url.to_string()) +} + +fn openai_voice_configured(provider_configured: bool, environment_key: Option<&str>) -> bool { + provider_configured || environment_key.is_some_and(|key| !key.trim().is_empty()) } fn authorized_headers(key: &str) -> Result { @@ -311,8 +327,13 @@ fn client() -> Result { #[tauri::command] pub fn get_openai_voice_status( state: State<'_, OpenAiVoiceState>, - configured: bool, + provider_configured: bool, ) -> Result { + // Provider metadata avoids a passive Keychain read; the environment is + // safe to inspect directly and has the same highest-priority semantics as + // the credential resolver used when a stream starts. + let environment_key = env_trimmed("OPENAI_API_KEY"); + let configured = openai_voice_configured(provider_configured, environment_key.as_deref()); let playback_speed = state .playback .lock() @@ -683,8 +704,17 @@ fn run_openai_voice_stream( output_latency_grace, speed, )?; - while active.load(Ordering::SeqCst) && !player.is_empty() { + while active.load(Ordering::SeqCst) + && (!player.is_empty() || assistant_speech.is_some()) + { player.ensure_healthy()?; + update_openai_assistant_speech( + player.is_empty(), + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + Instant::now(), + ); if last_progress.elapsed() >= Duration::from_millis(100) { emit_openai_stream_event( app, @@ -811,6 +841,10 @@ fn speak_pending( let samples = pcm16le_to_f32(&pcm_remainder[..sample_bytes]); pcm_remainder.drain(..sample_bytes); if *started { + assistant_speech.get_or_insert_with(|| { + native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture) + }); + *playback_drained_at = None; player.enqueue(&samples)?; } else { initial_samples.extend_from_slice(&samples); @@ -1091,13 +1125,37 @@ mod tests { #[test] fn openai_host_configuration_resolves_to_the_v1_api_root() { assert_eq!( - openai_host_base_url("https://proxy.example".to_string()), + normalize_openai_base_url("https://proxy.example".to_string(), true).unwrap(), + "https://proxy.example/v1" + ); + assert_eq!( + normalize_openai_base_url("https://proxy.example/v1/".to_string(), true).unwrap(), "https://proxy.example/v1" ); + } + + #[test] + fn openai_base_url_preserves_custom_paths_and_query_parameters() { assert_eq!( - openai_host_base_url("https://proxy.example/v1/".to_string()), + normalize_openai_base_url("https://proxy.example".to_string(), false).unwrap(), "https://proxy.example/v1" ); + let base = normalize_openai_base_url( + "https://proxy.example/openai?api-version=2026-01-01".to_string(), + false, + ) + .unwrap(); + assert_eq!( + endpoint_for_base_url(&base, "audio/speech").unwrap(), + "https://proxy.example/openai/audio/speech?api-version=2026-01-01" + ); + } + + #[test] + fn environment_credential_makes_openai_voice_ready() { + assert!(openai_voice_configured(false, Some("environment-key"))); + assert!(!openai_voice_configured(false, None)); + assert!(!openai_voice_configured(false, Some(" "))); } #[test] diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index 310623d46..08baabcc5 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -26,10 +26,12 @@ export interface OpenAiVoiceStreamEvent { export async function getOpenAiVoiceStatus(): Promise { const fields = await getProviderConfig("openai"); - const configured = fields.some( + const providerConfigured = fields.some( (field) => field.key === "OPENAI_API_KEY" && field.isSecret && field.isSet, ); - return invoke("get_openai_voice_status", { configured }); + return invoke("get_openai_voice_status", { + providerConfigured, + }); } export function startOpenAiVoiceStream( diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index ffb481ab9..f0f4009e6 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -6,6 +6,7 @@ import { useOpenAiVoiceSetup } from "./useOpenAiVoiceSetup"; const mocks = vi.hoisted(() => ({ getStatus: vi.fn<() => Promise>(), configChanged: null as ((providerId: string) => void) | null, + finishListening: null as (() => void) | null, })); vi.mock("../api/openAiVoice", () => ({ @@ -15,7 +16,9 @@ vi.mock("../api/openAiVoice", () => ({ vi.mock("@/features/providers/api/credentials", () => ({ onProviderConfigChanged: (listener: (providerId: string) => void) => { mocks.configChanged = listener; - return Promise.resolve(vi.fn()); + return new Promise<() => void>((resolve) => { + mocks.finishListening = () => resolve(() => undefined); + }); }, })); @@ -43,6 +46,7 @@ describe("useOpenAiVoiceSetup", () => { beforeEach(() => { vi.clearAllMocks(); mocks.configChanged = null; + mocks.finishListening = null; }); it("keeps the latest credential refresh when responses resolve out of order", async () => { @@ -53,6 +57,8 @@ describe("useOpenAiVoiceSetup", () => { .mockReturnValueOnce(refreshed.promise); const { result } = renderHook(() => useOpenAiVoiceSetup()); await waitFor(() => expect(mocks.configChanged).not.toBeNull()); + act(() => mocks.finishListening?.()); + await waitFor(() => expect(mocks.getStatus).toHaveBeenCalledTimes(1)); act(() => mocks.configChanged?.("openai")); refreshed.resolve(status(true)); @@ -63,4 +69,16 @@ describe("useOpenAiVoiceSetup", () => { expect(result.current.status?.configured).toBe(true); }); + + it("refreshes after listener registration captures credential changes", async () => { + mocks.getStatus.mockResolvedValue(status(true)); + const { result } = renderHook(() => useOpenAiVoiceSetup()); + + await waitFor(() => expect(mocks.finishListening).not.toBeNull()); + expect(mocks.getStatus).not.toHaveBeenCalled(); + + act(() => mocks.finishListening?.()); + + await waitFor(() => expect(result.current.status?.configured).toBe(true)); + }); }); diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts index c452b4b0e..a155cb32a 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts @@ -30,12 +30,13 @@ export function useOpenAiVoiceSetup(enabled = true) { }, ); }; - refresh(); void onProviderConfigChanged((providerId) => { if (providerId === "openai") refresh(); }).then((nextUnsubscribe) => { - if (active) unsubscribe = nextUnsubscribe; - else nextUnsubscribe(); + if (active) { + unsubscribe = nextUnsubscribe; + refresh(); + } else nextUnsubscribe(); }); return () => { active = false; diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 36c641655..2d129ab01 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -63,7 +63,13 @@ const openAiStatusState = vi.hoisted(() => ({ })); vi.mock("../api/openAiVoice", () => ({ - getOpenAiVoiceStatus: vi.fn(() => Promise.resolve(openAiStatusState.current)), + setOpenAiPlaybackSpeed: vi.fn(() => Promise.resolve()), +})); +vi.mock("../hooks/useOpenAiVoiceSetup", () => ({ + useOpenAiVoiceSetup: () => ({ + status: openAiStatusState.current, + error: null, + }), })); vi.mock("../hooks/usePocketVoiceSetup", () => ({ usePocketVoiceSetup: () => setupState.current, diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 78a552c0a..b7d2b244a 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -15,11 +15,7 @@ import { SelectValue, } from "@/shared/ui/select"; import { useEffect, useState } from "react"; -import { - getOpenAiVoiceStatus, - setOpenAiPlaybackSpeed, - type OpenAiVoiceStatus, -} from "../api/openAiVoice"; +import { setOpenAiPlaybackSpeed } from "../api/openAiVoice"; import { usePocketVoiceSetup } from "../hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "../hooks/useMacSpeechSetup"; import { useMicrophonePermission } from "../hooks/useMicrophonePermission"; @@ -37,6 +33,7 @@ import { PocketVoiceSetupContent } from "./PocketVoiceSetupContent"; import { MacSpeechSettings } from "./MacSpeechSettings"; import { SiriVoiceSettings } from "./SiriVoiceSettings"; import { PlaybackSpeedRow } from "./PlaybackSpeedRow"; +import { useOpenAiVoiceSetup } from "../hooks/useOpenAiVoiceSetup"; const INTERRUPTION_MODES: VoiceInterruptionMode[] = [ "automatic", @@ -79,31 +76,11 @@ export function VoiceSettings() { const { t } = useTranslation("settings"); const setup = usePocketVoiceSetup(); const macSpeechSetup = useMacSpeechSetup(); - const [openAiStatus, setOpenAiStatus] = useState( - null, - ); - const [openAiError, setOpenAiError] = useState(null); + const { status: openAiStatus, error: openAiError } = useOpenAiVoiceSetup(); const [openAiSpeed, setOpenAiSpeed] = useState(1); useEffect(() => { - let active = true; - void getOpenAiVoiceStatus().then( - (status) => { - if (active) { - setOpenAiStatus(status); - setOpenAiSpeed(status.playbackSpeed); - } - }, - (error) => { - if (active) - setOpenAiError( - error instanceof Error ? error.message : String(error), - ); - }, - ); - return () => { - active = false; - }; - }, []); + if (openAiStatus) setOpenAiSpeed(openAiStatus.playbackSpeed); + }, [openAiStatus]); const input = useVoiceInputPreference( isMacSpeechAvailable(macSpeechSetup.status, macSpeechSetup.loading), ); From 2d898b4f6387d505c4ba0610a6c90c114ea143a4 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 17:50:59 -0400 Subject: [PATCH 09/18] fix(voice): load OpenAI status without event listener --- .../hooks/useOpenAiVoiceSetup.test.tsx | 12 ++++++++++++ .../hooks/useOpenAiVoiceSetup.ts | 17 +++++++++++------ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index f0f4009e6..418ec3180 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -7,6 +7,7 @@ const mocks = vi.hoisted(() => ({ getStatus: vi.fn<() => Promise>(), configChanged: null as ((providerId: string) => void) | null, finishListening: null as (() => void) | null, + listenerError: null as Error | null, })); vi.mock("../api/openAiVoice", () => ({ @@ -16,6 +17,7 @@ vi.mock("../api/openAiVoice", () => ({ vi.mock("@/features/providers/api/credentials", () => ({ onProviderConfigChanged: (listener: (providerId: string) => void) => { mocks.configChanged = listener; + if (mocks.listenerError) return Promise.reject(mocks.listenerError); return new Promise<() => void>((resolve) => { mocks.finishListening = () => resolve(() => undefined); }); @@ -47,6 +49,7 @@ describe("useOpenAiVoiceSetup", () => { vi.clearAllMocks(); mocks.configChanged = null; mocks.finishListening = null; + mocks.listenerError = null; }); it("keeps the latest credential refresh when responses resolve out of order", async () => { @@ -81,4 +84,13 @@ describe("useOpenAiVoiceSetup", () => { await waitFor(() => expect(result.current.status?.configured).toBe(true)); }); + + it("still loads status when listener registration fails", async () => { + mocks.listenerError = new Error("listener unavailable"); + mocks.getStatus.mockResolvedValue(status(true)); + + const { result } = renderHook(() => useOpenAiVoiceSetup()); + + await waitFor(() => expect(result.current.status?.configured).toBe(true)); + }); }); diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts index a155cb32a..4c10964a7 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts @@ -32,12 +32,17 @@ export function useOpenAiVoiceSetup(enabled = true) { }; void onProviderConfigChanged((providerId) => { if (providerId === "openai") refresh(); - }).then((nextUnsubscribe) => { - if (active) { - unsubscribe = nextUnsubscribe; - refresh(); - } else nextUnsubscribe(); - }); + }).then( + (nextUnsubscribe) => { + if (active) { + unsubscribe = nextUnsubscribe; + refresh(); + } else nextUnsubscribe(); + }, + () => { + if (active) refresh(); + }, + ); return () => { active = false; unsubscribe?.(); From f112a4ef078e40e92c524ef000ebae219e31a73a Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 17:54:27 -0400 Subject: [PATCH 10/18] fix(voice): keep transcription status in STT layer --- src/features/voice-conversation/api/openAiVoice.ts | 1 - .../voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx | 1 - src/features/voice-conversation/ui/VoiceSettings.test.tsx | 2 -- 3 files changed, 4 deletions(-) diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index 08baabcc5..a16fe2771 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -9,7 +9,6 @@ import type { export interface OpenAiVoiceStatus { configured: boolean; - transcriptionModel: string; speechModel: string; speechVoice: string; playbackSpeed: number; diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index 418ec3180..49e28a70c 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -35,7 +35,6 @@ function deferred() { function status(configured: boolean): OpenAiVoiceStatus { return { configured, - transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 2d129ab01..799e1e07a 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -53,7 +53,6 @@ const microphonePermissionState = vi.hoisted(() => ({ const openAiStatusState = vi.hoisted(() => ({ current: { configured: true, - transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -213,7 +212,6 @@ describe("VoiceSettings", () => { siriSetupState.current = siriSetup(); openAiStatusState.current = { configured: true, - transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, From 98c931d8f0f99e64d892030626ad21b64b1ddc78 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 18:06:42 -0400 Subject: [PATCH 11/18] fix(voice): gate macOS speech playback internals --- src-tauri/src/commands/openai_audio.rs | 40 +++++++++++++++++++++----- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index c2e24696c..c6c2e81dc 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -1,21 +1,23 @@ //! OpenAI streaming speech playback for voice conversations. -use std::{ - sync::{ - atomic::{AtomicBool, Ordering}, - mpsc, Arc, Mutex, - }, - time::Duration, +use std::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, Mutex, }; +#[cfg(any(test, target_os = "macos"))] +use std::time::Duration; #[cfg(target_os = "macos")] use futures_util::StreamExt; #[cfg(target_os = "macos")] use reqwest::header::CONTENT_TYPE; +#[cfg(target_os = "macos")] use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; use serde::Serialize; use serde_json::json; -use tauri::{AppHandle, Emitter, State}; +#[cfg(target_os = "macos")] +use tauri::Emitter; +use tauri::{AppHandle, State}; #[cfg(target_os = "macos")] use super::{ @@ -32,16 +34,23 @@ use super::{ #[cfg(any(test, target_os = "macos"))] use std::time::Instant; +#[cfg(target_os = "macos")] const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; const DEFAULT_TTS_VOICE: &str = "marin"; +#[cfg(target_os = "macos")] const TTS_SAMPLE_RATE: u32 = 24_000; // Avoid starting the audio device from a tiny first network chunk that can drain // before subsequent streamed PCM arrives. +#[cfg(target_os = "macos")] const INITIAL_PLAYBACK_BUFFER_FRAMES: usize = TTS_SAMPLE_RATE as usize / 5; +#[cfg(target_os = "macos")] const TTS_EVENT: &str = "openai-voice:stream-event"; +#[cfg(target_os = "macos")] const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); +#[cfg(target_os = "macos")] const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +#[cfg(target_os = "macos")] const MAX_TTS_INPUT_CHARS: usize = 4096; #[derive(Clone, Debug, Default)] @@ -73,6 +82,7 @@ struct ActiveOpenAiStream { sender: mpsc::Sender, } +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] #[derive(Debug)] enum OpenAiStreamCommand { Append(String), @@ -81,6 +91,7 @@ enum OpenAiStreamCommand { Stop, } +#[cfg(target_os = "macos")] #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct OpenAiVoiceStatus { @@ -101,6 +112,7 @@ struct OpenAiVoiceStreamEvent { delivery: Option, } +#[cfg(target_os = "macos")] #[derive(Clone, Copy, Serialize)] #[serde(rename_all = "camelCase")] enum OpenAiStreamEventState { @@ -111,6 +123,7 @@ enum OpenAiStreamEventState { Failed, } +#[cfg(target_os = "macos")] #[derive(Clone, Debug, Default, Serialize)] #[serde(rename_all = "camelCase")] struct VoiceDeliveryProgress { @@ -118,6 +131,7 @@ struct VoiceDeliveryProgress { segments: Vec, } +#[cfg(target_os = "macos")] #[derive(Clone, Debug, Serialize)] #[serde(rename_all = "camelCase")] struct VoiceDeliverySegment { @@ -134,6 +148,7 @@ fn env_trimmed(name: &str) -> Option { .filter(|value| !value.is_empty()) } +#[cfg(target_os = "macos")] fn goose_yaml_value(path: &std::path::Path, name: &str) -> Result, String> { if !path.exists() { return Ok(None); @@ -150,6 +165,7 @@ fn goose_yaml_value(path: &std::path::Path, name: &str) -> Result .map(ToString::to_string)) } +#[cfg(target_os = "macos")] fn goose_openai_api_key() -> Result, String> { if let Some(value) = env_trimmed("OPENAI_API_KEY") { return Ok(Some(value)); @@ -206,6 +222,7 @@ fn goose_openai_api_key() -> Result, String> { Ok(None) } +#[cfg(target_os = "macos")] pub(crate) fn api_key() -> Result { goose_openai_api_key()?.ok_or_else(|| { "OpenAI voice is not configured. Configure the OpenAI provider in Berd, then try again." @@ -213,6 +230,7 @@ pub(crate) fn api_key() -> Result { }) } +#[cfg(any(test, target_os = "macos"))] fn normalize_openai_base_url(raw_url: String, assume_v1: bool) -> Result { let mut url = reqwest::Url::parse(&raw_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; @@ -234,6 +252,7 @@ fn normalize_openai_base_url(raw_url: String, assume_v1: bool) -> Result Result { if let Some(host) = env_trimmed("OPENAI_HOST") { return normalize_openai_base_url(host, true); @@ -259,10 +278,12 @@ fn speech_voice() -> String { env_trimmed("OPENAI_TTS_VOICE").unwrap_or_else(|| DEFAULT_TTS_VOICE.to_string()) } +#[cfg(target_os = "macos")] fn endpoint(path: &str) -> Result { endpoint_for_base_url(&base_url()?, path) } +#[cfg(any(test, target_os = "macos"))] fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { let mut url = reqwest::Url::parse(base_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; @@ -275,6 +296,7 @@ fn openai_voice_configured(provider_configured: bool, environment_key: Option<&s provider_configured || environment_key.is_some_and(|key| !key.trim().is_empty()) } +#[cfg(target_os = "macos")] fn authorized_headers(key: &str) -> Result { let mut headers = HeaderMap::new(); let bearer = format!("Bearer {key}"); @@ -316,6 +338,7 @@ fn persist_playback_speed(speed: f32) -> Result<(), String> { .map_err(|error| format!("write OpenAI voice settings: {error}")) } +#[cfg(target_os = "macos")] fn client() -> Result { reqwest::Client::builder() .connect_timeout(CONNECT_TIMEOUT) @@ -1064,6 +1087,7 @@ fn snapshot_delivery( } } +#[cfg(target_os = "macos")] fn emit_openai_stream_event( app: &AppHandle, stream_id: &str, @@ -1082,6 +1106,7 @@ fn emit_openai_stream_event( ); } +#[cfg(target_os = "macos")] fn format_openai_request_error(action: &str, error: reqwest::Error) -> String { if error.is_timeout() { format!("OpenAI voice could not {action}: the request timed out") @@ -1092,6 +1117,7 @@ fn format_openai_request_error(action: &str, error: reqwest::Error) -> String { } } +#[cfg(target_os = "macos")] fn format_openai_response_error(action: &str, status: reqwest::StatusCode, body: &str) -> String { let preview: String = body.chars().take(500).collect(); format!("OpenAI voice could not {action}: HTTP {status}: {preview}") From ff49e26fb8622d9d0aada8a1c10152a63b360470 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 26 Aug 2026 18:07:47 -0400 Subject: [PATCH 12/18] fix(voice): keep OpenAI status cross-platform --- src-tauri/src/commands/openai_audio.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index c6c2e81dc..c69956006 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -91,7 +91,6 @@ enum OpenAiStreamCommand { Stop, } -#[cfg(target_os = "macos")] #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct OpenAiVoiceStatus { @@ -103,6 +102,7 @@ pub struct OpenAiVoiceStatus { unavailable_reason: Option, } +#[cfg(target_os = "macos")] #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] struct OpenAiVoiceStreamEvent { From 1cfc4ab1430a4b118bb8db4b6dabe0a566c6ed73 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 28 Aug 2026 13:46:49 -0400 Subject: [PATCH 13/18] feat(voice): add a dedicated OpenAI TTS key --- src-tauri/src/commands/openai_audio.rs | 148 +++++++++--------- src-tauri/src/lib.rs | 2 + .../voice-conversation/api/openAiVoice.ts | 25 +-- .../hooks/useOpenAiVoiceSetup.test.tsx | 15 +- .../hooks/useOpenAiVoiceSetup.ts | 6 +- .../lib/voiceSetupReadiness.test.ts | 2 +- .../ui/OpenAiApiKeyField.tsx | 100 ++++++++++++ .../ui/VoiceSettings.test.tsx | 27 +++- .../voice-conversation/ui/VoiceSettings.tsx | 13 +- src/shared/i18n/locales/en/settings.json | 12 +- src/shared/i18n/locales/es/settings.json | 12 +- 11 files changed, 255 insertions(+), 107 deletions(-) create mode 100644 src/features/voice-conversation/ui/OpenAiApiKeyField.tsx diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index c69956006..0a12f892d 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -15,7 +15,6 @@ use reqwest::header::CONTENT_TYPE; use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; use serde::Serialize; use serde_json::json; -#[cfg(target_os = "macos")] use tauri::Emitter; use tauri::{AppHandle, State}; @@ -38,6 +37,9 @@ use std::time::Instant; const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; const DEFAULT_TTS_VOICE: &str = "marin"; +const KEYRING_SERVICE: &str = "berd-openai-voice"; +const TTS_KEYRING_ACCOUNT: &str = "tts-api-key"; +const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; #[cfg(target_os = "macos")] const TTS_SAMPLE_RATE: u32 = 24_000; // Avoid starting the audio device from a tiny first network chunk that can drain @@ -166,66 +168,58 @@ fn goose_yaml_value(path: &std::path::Path, name: &str) -> Result } #[cfg(target_os = "macos")] -fn goose_openai_api_key() -> Result, String> { - if let Some(value) = env_trimmed("OPENAI_API_KEY") { - return Ok(Some(value)); - } - #[cfg(target_os = "macos")] - let mut secure_store_error = None; - #[cfg(target_os = "macos")] - { - match keyring::Entry::new("goose", "secrets") { - Ok(entry) => match entry.get_password() { - Ok(payload) => match serde_json::from_str::(&payload) { - Ok(secrets) => { - if let Some(key) = secrets - .get("OPENAI_API_KEY") - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - { - return Ok(Some(key.to_string())); - } - } - Err(error) => { - secure_store_error = Some(format!( - "Goose's secure credential store is not valid JSON: {error}" - )); - } - }, - Err(keyring::Error::NoEntry) => {} - Err(error) => { - secure_store_error = Some(format!( - "Could not read Goose's OpenAI credential from secure storage: {error}" - )); - } - }, - Err(error) => { - secure_store_error = Some(format!( - "Could not access Goose's secure credential store: {error}" - )); - } - } - } - let config_path = crate::services::goose_config::config_path()?; - let secrets_path = config_path - .parent() - .ok_or_else(|| "Could not resolve Goose's credential directory".to_string())? - .join("secrets.yaml"); - if let Some(key) = goose_yaml_value(&secrets_path, "OPENAI_API_KEY")? { - return Ok(Some(key)); - } - #[cfg(target_os = "macos")] - if let Some(error) = secure_store_error { - return Err(error); +fn stored_api_key(account: &str) -> Result, String> { + let entry = keyring::Entry::new(KEYRING_SERVICE, account) + .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}"))?; + match entry.get_password() { + Ok(value) => Ok(Some(value)), + Err(keyring::Error::NoEntry) => Ok(None), + Err(error) => Err(format!( + "Could not read Berd's OpenAI voice credential: {error}" + )), } +} + +#[cfg(not(target_os = "macos"))] +fn stored_api_key(_account: &str) -> Result, String> { Ok(None) } #[cfg(target_os = "macos")] -pub(crate) fn api_key() -> Result { - goose_openai_api_key()?.ok_or_else(|| { - "OpenAI voice is not configured. Configure the OpenAI provider in Berd, then try again." +fn store_api_key(account: &str, api_key: &str) -> Result<(), String> { + let entry = keyring::Entry::new(KEYRING_SERVICE, account) + .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}"))?; + entry + .set_password(api_key) + .map_err(|error| format!("Could not save Berd's OpenAI voice credential: {error}")) +} + +#[cfg(not(target_os = "macos"))] +fn store_api_key(_account: &str, _api_key: &str) -> Result<(), String> { + Err("OpenAI voice credentials are unsupported on this platform".to_string()) +} + +#[cfg(target_os = "macos")] +fn clear_api_key(account: &str) -> Result<(), String> { + let entry = keyring::Entry::new(KEYRING_SERVICE, account) + .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}"))?; + match entry.delete_credential() { + Ok(()) | Err(keyring::Error::NoEntry) => Ok(()), + Err(error) => Err(format!( + "Could not remove Berd's OpenAI voice credential: {error}" + )), + } +} + +#[cfg(not(target_os = "macos"))] +fn clear_api_key(_account: &str) -> Result<(), String> { + Err("OpenAI voice credentials are unsupported on this platform".to_string()) +} + +#[cfg(target_os = "macos")] +fn tts_api_key() -> Result { + stored_api_key(TTS_KEYRING_ACCOUNT)?.ok_or_else(|| { + "OpenAI text-to-speech is not configured. Add its API key in Voice settings, then try again." .to_string() }) } @@ -292,10 +286,6 @@ fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { Ok(url.to_string()) } -fn openai_voice_configured(provider_configured: bool, environment_key: Option<&str>) -> bool { - provider_configured || environment_key.is_some_and(|key| !key.trim().is_empty()) -} - #[cfg(target_os = "macos")] fn authorized_headers(key: &str) -> Result { let mut headers = HeaderMap::new(); @@ -350,13 +340,8 @@ fn client() -> Result { #[tauri::command] pub fn get_openai_voice_status( state: State<'_, OpenAiVoiceState>, - provider_configured: bool, ) -> Result { - // Provider metadata avoids a passive Keychain read; the environment is - // safe to inspect directly and has the same highest-priority semantics as - // the credential resolver used when a stream starts. - let environment_key = env_trimmed("OPENAI_API_KEY"); - let configured = openai_voice_configured(provider_configured, environment_key.as_deref()); + let configured = stored_api_key(TTS_KEYRING_ACCOUNT)?.is_some(); let playback_speed = state .playback .lock() @@ -370,7 +355,7 @@ pub fn get_openai_voice_status( playback_speed, tts_available, unavailable_reason: if !configured { - Some("Configure the OpenAI provider in Berd to use OpenAI voice.".to_string()) + Some("Add an OpenAI text-to-speech API key in Voice settings.".to_string()) } else if !tts_available { Some("OpenAI voice playback is currently supported on macOS only.".to_string()) } else { @@ -379,6 +364,26 @@ pub fn get_openai_voice_status( }) } +#[tauri::command] +pub fn set_openai_tts_api_key(app: AppHandle, api_key: String) -> Result<(), String> { + let api_key = api_key.trim(); + if api_key.is_empty() { + return Err("OpenAI text-to-speech API key cannot be empty".to_string()); + } + store_api_key(TTS_KEYRING_ACCOUNT, api_key)?; + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; + Ok(()) +} + +#[tauri::command] +pub fn clear_openai_tts_api_key(app: AppHandle) -> Result<(), String> { + clear_api_key(TTS_KEYRING_ACCOUNT)?; + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; + Ok(()) +} + #[tauri::command] pub fn start_openai_voice_stream( app: AppHandle, @@ -408,7 +413,7 @@ pub fn start_openai_voice_stream( if stream_id.trim().is_empty() { return Err("OpenAI voice stream id cannot be empty".to_string()); } - let key = api_key()?; + let key = tts_api_key()?; let (sender, receiver) = mpsc::channel(); let active = Arc::new(AtomicBool::new(true)); { @@ -1177,13 +1182,6 @@ mod tests { ); } - #[test] - fn environment_credential_makes_openai_voice_ready() { - assert!(openai_voice_configured(false, Some("environment-key"))); - assert!(!openai_voice_configured(false, None)); - assert!(!openai_voice_configured(false, Some(" "))); - } - #[test] fn capture_suppression_ends_after_playback_drain_grace() { let started = Instant::now(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 44165a993..2e41e9a23 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -656,6 +656,8 @@ pub fn run() { commands::pocket_voice::stop_pocket_voice, commands::pocket_voice::remove_voice_model, commands::openai_audio::get_openai_voice_status, + commands::openai_audio::set_openai_tts_api_key, + commands::openai_audio::clear_openai_tts_api_key, commands::openai_audio::start_openai_voice_stream, commands::openai_audio::append_openai_voice_stream, commands::openai_audio::flush_openai_voice_stream, diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index a16fe2771..71a18d541 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -1,6 +1,5 @@ import { invoke } from "@tauri-apps/api/core"; import { listen, type UnlistenFn } from "@tauri-apps/api/event"; -import { getProviderConfig } from "@/features/providers/api/credentials"; import type { VoiceDeliveryProgress } from "./pocketVoice"; import type { VoiceInterruptionMode, @@ -23,14 +22,22 @@ export interface OpenAiVoiceStreamEvent { delivery?: VoiceDeliveryProgress | null; } -export async function getOpenAiVoiceStatus(): Promise { - const fields = await getProviderConfig("openai"); - const providerConfigured = fields.some( - (field) => field.key === "OPENAI_API_KEY" && field.isSecret && field.isSet, - ); - return invoke("get_openai_voice_status", { - providerConfigured, - }); +export function getOpenAiVoiceStatus(): Promise { + return invoke("get_openai_voice_status"); +} + +export function setOpenAiTtsApiKey(apiKey: string): Promise { + return invoke("set_openai_tts_api_key", { apiKey }); +} + +export function clearOpenAiTtsApiKey(): Promise { + return invoke("clear_openai_tts_api_key"); +} + +export function listenToOpenAiVoiceSettings( + onChanged: () => void, +): Promise { + return listen("openai-voice:settings-changed", onChanged); } export function startOpenAiVoiceStream( diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index 49e28a70c..e77463709 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -5,18 +5,15 @@ import { useOpenAiVoiceSetup } from "./useOpenAiVoiceSetup"; const mocks = vi.hoisted(() => ({ getStatus: vi.fn<() => Promise>(), - configChanged: null as ((providerId: string) => void) | null, + settingsChanged: null as (() => void) | null, finishListening: null as (() => void) | null, listenerError: null as Error | null, })); vi.mock("../api/openAiVoice", () => ({ getOpenAiVoiceStatus: () => mocks.getStatus(), -})); - -vi.mock("@/features/providers/api/credentials", () => ({ - onProviderConfigChanged: (listener: (providerId: string) => void) => { - mocks.configChanged = listener; + listenToOpenAiVoiceSettings: (listener: () => void) => { + mocks.settingsChanged = listener; if (mocks.listenerError) return Promise.reject(mocks.listenerError); return new Promise<() => void>((resolve) => { mocks.finishListening = () => resolve(() => undefined); @@ -46,7 +43,7 @@ function status(configured: boolean): OpenAiVoiceStatus { describe("useOpenAiVoiceSetup", () => { beforeEach(() => { vi.clearAllMocks(); - mocks.configChanged = null; + mocks.settingsChanged = null; mocks.finishListening = null; mocks.listenerError = null; }); @@ -58,11 +55,11 @@ describe("useOpenAiVoiceSetup", () => { .mockReturnValueOnce(initial.promise) .mockReturnValueOnce(refreshed.promise); const { result } = renderHook(() => useOpenAiVoiceSetup()); - await waitFor(() => expect(mocks.configChanged).not.toBeNull()); + await waitFor(() => expect(mocks.settingsChanged).not.toBeNull()); act(() => mocks.finishListening?.()); await waitFor(() => expect(mocks.getStatus).toHaveBeenCalledTimes(1)); - act(() => mocks.configChanged?.("openai")); + act(() => mocks.settingsChanged?.()); refreshed.resolve(status(true)); await waitFor(() => expect(result.current.status?.configured).toBe(true)); diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts index 4c10964a7..33c9d8e98 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts @@ -1,9 +1,9 @@ import { useEffect, useState } from "react"; import { getOpenAiVoiceStatus, + listenToOpenAiVoiceSettings, type OpenAiVoiceStatus, } from "../api/openAiVoice"; -import { onProviderConfigChanged } from "@/features/providers/api/credentials"; export function useOpenAiVoiceSetup(enabled = true) { const [status, setStatus] = useState(null); @@ -30,9 +30,7 @@ export function useOpenAiVoiceSetup(enabled = true) { }, ); }; - void onProviderConfigChanged((providerId) => { - if (providerId === "openai") refresh(); - }).then( + void listenToOpenAiVoiceSettings(refresh).then( (nextUnsubscribe) => { if (active) { unsubscribe = nextUnsubscribe; diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts index 76bdb2cb3..04406b9c7 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts @@ -53,7 +53,7 @@ describe("voice setup readiness", () => { ).toBe(false); }); - it("requires Berd's configured OpenAI credential for OpenAI output", () => { + it("requires the dedicated OpenAI text-to-speech key for OpenAI output", () => { const configured = { configured: true, ttsAvailable: true } as never; expect( isVoiceSetupReady(pocket, null, null, "parakeet", "openai", configured), diff --git a/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx b/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx new file mode 100644 index 000000000..f91df7c90 --- /dev/null +++ b/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx @@ -0,0 +1,100 @@ +import { useId, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/shared/ui/button"; +import { Input } from "@/shared/ui/input"; + +interface OpenAiApiKeyFieldProps { + kind: "stt" | "tts"; + configured: boolean; + onSave: (apiKey: string) => Promise; + onClear: () => Promise; +} + +export function OpenAiApiKeyField({ + kind, + configured, + onSave, + onClear, +}: OpenAiApiKeyFieldProps) { + const { t } = useTranslation("settings"); + const inputId = useId(); + const [apiKey, setApiKey] = useState(""); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + const label = + kind === "stt" ? t("voice.openAiSttApiKey") : t("voice.openAiTtsApiKey"); + + const save = async () => { + setSaving(true); + setError(null); + try { + await onSave(apiKey); + setApiKey(""); + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setSaving(false); + } + }; + + const clear = async () => { + setSaving(true); + setError(null); + try { + await onClear(); + setApiKey(""); + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setSaving(false); + } + }; + + return ( +
+ +
+ setApiKey(event.target.value)} + placeholder={configured ? t("voice.openAiApiKeySaved") : "sk-…"} + autoComplete="off" + spellCheck={false} + /> + + {configured ? ( + + ) : null} +
+

+ {configured + ? t("voice.openAiApiKeyConfigured") + : t("voice.openAiApiKeyNotConfigured")} +

+ {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 799e1e07a..4b3bd867a 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -60,9 +60,15 @@ const openAiStatusState = vi.hoisted(() => ({ unavailableReason: null as string | null, }, })); +const openAiApiMocks = vi.hoisted(() => ({ + setTtsApiKey: vi.fn(() => Promise.resolve()), + clearTtsApiKey: vi.fn(() => Promise.resolve()), +})); vi.mock("../api/openAiVoice", () => ({ setOpenAiPlaybackSpeed: vi.fn(() => Promise.resolve()), + setOpenAiTtsApiKey: openAiApiMocks.setTtsApiKey, + clearOpenAiTtsApiKey: openAiApiMocks.clearTtsApiKey, })); vi.mock("../hooks/useOpenAiVoiceSetup", () => ({ useOpenAiVoiceSetup: () => ({ @@ -218,6 +224,8 @@ describe("VoiceSettings", () => { ttsAvailable: true, unavailableReason: null, }; + openAiApiMocks.setTtsApiKey.mockClear(); + openAiApiMocks.clearTtsApiKey.mockClear(); }); it("renders selected OpenAI output settings", async () => { @@ -231,20 +239,35 @@ describe("VoiceSettings", () => { expect(screen.getByText("Playback speed")).toBeInTheDocument(); }); + it("saves a dedicated OpenAI text-to-speech API key", async () => { + outputState.backend = "openai"; + setupState.current = setup(pocketStatus({ parakeetInstalled: true })); + renderWithProviders(); + + const user = userEvent.setup(); + await user.type( + screen.getByLabelText("OpenAI text-to-speech API key"), + "tts-secret", + ); + await user.click(screen.getByRole("button", { name: "Save key" })); + + expect(openAiApiMocks.setTtsApiKey).toHaveBeenCalledWith("tts-secret"); + }); + it("uses OpenAI guidance when the selected OpenAI output is not ready", async () => { outputState.backend = "openai"; openAiStatusState.current = { ...openAiStatusState.current, configured: false, unavailableReason: - "Configure the OpenAI provider in Berd to use OpenAI voice.", + "Add an OpenAI text-to-speech API key in Voice settings.", }; setupState.current = setup(pocketStatus({ parakeetInstalled: true })); renderWithProviders(); expect( await screen.findByText( - "OpenAI voice is not ready. Configure the OpenAI provider in Berd, then try again.", + "OpenAI voice is not ready. Add the required API key below, then try again.", ), ).toBeInTheDocument(); expect( diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index b7d2b244a..4b3576c70 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -15,7 +15,11 @@ import { SelectValue, } from "@/shared/ui/select"; import { useEffect, useState } from "react"; -import { setOpenAiPlaybackSpeed } from "../api/openAiVoice"; +import { + clearOpenAiTtsApiKey, + setOpenAiPlaybackSpeed, + setOpenAiTtsApiKey, +} from "../api/openAiVoice"; import { usePocketVoiceSetup } from "../hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "../hooks/useMacSpeechSetup"; import { useMicrophonePermission } from "../hooks/useMicrophonePermission"; @@ -34,6 +38,7 @@ import { MacSpeechSettings } from "./MacSpeechSettings"; import { SiriVoiceSettings } from "./SiriVoiceSettings"; import { PlaybackSpeedRow } from "./PlaybackSpeedRow"; import { useOpenAiVoiceSetup } from "../hooks/useOpenAiVoiceSetup"; +import { OpenAiApiKeyField } from "./OpenAiApiKeyField"; const INTERRUPTION_MODES: VoiceInterruptionMode[] = [ "automatic", @@ -259,6 +264,12 @@ export function VoiceSettings() { details={ output.backend === "openai" ? (
+

{openAiError ?? openAiStatus?.unavailableReason ?? diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index fc2256d32..91170585c 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -918,7 +918,7 @@ "notReadyMacInput": "Apple's on-device dictation model is not installed. Download it below to use Voice Conversation.", "notReadyMacInputAndPocketOutput": "Apple's on-device dictation model and Pocket TTS are not installed. Complete both steps below to use Voice Conversation.", "notReadyMacInputAndSiriOutput": "Apple's on-device dictation model is not installed, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", - "notReadyOpenAi": "OpenAI voice is not ready. Configure the OpenAI provider in Berd, then try again.", + "notReadyOpenAi": "OpenAI voice is not ready. Add the required API key below, then try again.", "notReadyPocketOutput": "Pocket TTS is not installed. Download it below to use Voice Conversation.", "notReadySiriOutput": "No installed Siri voice is selected. Download or select one below to use Voice Conversation.", "notReadyTitle": "Voice Conversation isn't ready", @@ -927,8 +927,12 @@ "noVoiceSelected": "No voice selected", "openMicrophoneSettings": "Open Microphone Settings", "openMicrophoneSettingsError": "Couldn't open Microphone Settings. Open System Settings and select Privacy & Security > Microphone.", - "openAiChecking": "Checking the OpenAI provider configured in Berd…", - "openAiTtsConfigured": "Uses Berd’s configured OpenAI credential with {{model}} and the {{voice}} voice. OpenAI voices are AI-generated.", + "openAiApiKeyConfigured": "Saved securely for this voice service.", + "openAiApiKeyNotConfigured": "This key is separate from provider credentials used by Goose.", + "openAiApiKeySaved": "API key saved", + "openAiChecking": "Checking OpenAI voice settings…", + "openAiTtsApiKey": "OpenAI text-to-speech API key", + "openAiTtsConfigured": "Uses {{model}} and the {{voice}} voice. OpenAI voices are AI-generated.", "outputBackendDescription": "Choose how Berd speaks assistant responses.", "playbackSpeed": "Playback speed", "playing": "Playing", @@ -936,11 +940,13 @@ "preview": "Preview", "previewVoice": "Preview {{voice}}", "removeModel": "Remove model", + "removeApiKey": "Remove", "removeModelDescription": "Voice Conversation will stop before {{model}} is removed. You can download the model again later.", "removeModelQueued": "Queued for removal…", "removeModelTitle": "Remove {{model}}?", "removingModel": "Removing model…", "retryDownload": "Retry model download", + "saveApiKey": "Save key", "settingsDescription": "Choose how Berd speaks, install speech recognition, and preview available voices.", "siriLanguage": "Language", "siriLoading": "Loading Siri voices…", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 3163c7e8e..a64265189 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -921,7 +921,7 @@ "notReadyMacInput": "El modelo de dictado de Apple no está instalado. Descárgalo abajo para usar la conversación por voz.", "notReadyMacInputAndPocketOutput": "El modelo de dictado de Apple y Pocket TTS no están instalados. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInputAndSiriOutput": "El modelo de dictado de Apple no está instalado y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", - "notReadyOpenAi": "La voz de OpenAI no está lista. Configura el proveedor de OpenAI en Berd e inténtalo de nuevo.", + "notReadyOpenAi": "La voz de OpenAI no está lista. Añade abajo la clave API necesaria e inténtalo de nuevo.", "notReadyPocketOutput": "Pocket TTS no está instalado. Descárgalo abajo para usar la conversación por voz.", "notReadySiriOutput": "No hay ninguna voz de Siri instalada seleccionada. Descarga o selecciona una abajo para usar la conversación por voz.", "notReadyTitle": "La conversación por voz no está lista", @@ -930,8 +930,12 @@ "noVoiceSelected": "No hay ninguna voz seleccionada", "openMicrophoneSettings": "Abrir ajustes del micrófono", "openMicrophoneSettingsError": "No se pudieron abrir los ajustes del micrófono. Abre Ajustes del Sistema y selecciona Privacidad y seguridad > Micrófono.", - "openAiChecking": "Comprobando el proveedor de OpenAI configurado en Berd…", - "openAiTtsConfigured": "Usa la credencial de OpenAI configurada en Berd con {{model}} y la voz {{voice}}. Las voces de OpenAI son generadas por IA.", + "openAiApiKeyConfigured": "Guardada de forma segura para este servicio de voz.", + "openAiApiKeyNotConfigured": "Esta clave es independiente de las credenciales de proveedor que usa Goose.", + "openAiApiKeySaved": "Clave API guardada", + "openAiChecking": "Comprobando los ajustes de voz de OpenAI…", + "openAiTtsApiKey": "Clave API de texto a voz de OpenAI", + "openAiTtsConfigured": "Usa {{model}} y la voz {{voice}}. Las voces de OpenAI son generadas por IA.", "outputBackendDescription": "Elige cómo Berd reproduce las respuestas del asistente.", "playbackSpeed": "Velocidad de reproducción", "playing": "Reproduciendo", @@ -939,11 +943,13 @@ "preview": "Escuchar", "previewVoice": "Escuchar {{voice}}", "removeModel": "Eliminar modelo", + "removeApiKey": "Eliminar", "removeModelDescription": "La conversación por voz se detendrá antes de eliminar {{model}}. Puedes volver a descargar el modelo más tarde.", "removeModelQueued": "En cola para eliminar…", "removeModelTitle": "¿Eliminar {{model}}?", "removingModel": "Eliminando modelo…", "retryDownload": "Reintentar descarga del modelo", + "saveApiKey": "Guardar clave", "settingsDescription": "Elige cómo habla Berd, instala el reconocimiento de voz y escucha las voces disponibles.", "siriLanguage": "Idioma", "siriLoading": "Cargando voces de Siri…", From 3b82a50df17ead9dea65bd49c7fe0919bf07be01 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 28 Aug 2026 13:57:26 -0400 Subject: [PATCH 14/18] refactor(voice): isolate OpenAI voice credentials --- .../providers/api/credentials.test.ts | 35 ------------------- src/features/providers/api/credentials.ts | 19 ---------- 2 files changed, 54 deletions(-) diff --git a/src/features/providers/api/credentials.test.ts b/src/features/providers/api/credentials.test.ts index c0c2c2e23..a5e6ce942 100644 --- a/src/features/providers/api/credentials.test.ts +++ b/src/features/providers/api/credentials.test.ts @@ -4,7 +4,6 @@ import { checkAllProviderStatus, deleteProviderConfig, getProviderConfig, - onProviderConfigChanged, saveProviderConfig, } from "./credentials"; @@ -15,30 +14,14 @@ const mocks = vi.hoisted(() => ({ configDelete: vi.fn(), configStatus: vi.fn(), getClient: vi.fn(), - emit: vi.fn(), - listen: vi.fn(), - providerConfigHandler: null as - | ((event: { payload: { providerId: string } }) => void) - | null, })); vi.mock("@/shared/api/acpConnection", () => ({ getClient: () => mocks.getClient(), })); -vi.mock("@tauri-apps/api/event", () => ({ - emit: (...args: unknown[]) => mocks.emit(...args), - listen: (event: string, handler: typeof mocks.providerConfigHandler) => { - mocks.listen(event, handler); - mocks.providerConfigHandler = handler; - return Promise.resolve(vi.fn()); - }, -})); - describe("provider credential API", () => { beforeEach(() => { vi.clearAllMocks(); - mocks.emit.mockResolvedValue(undefined); - mocks.providerConfigHandler = null; mocks.getClient.mockResolvedValue({ goose: { GooseUnstableProvidersConfigRead: mocks.configRead, @@ -102,24 +85,6 @@ describe("provider credential API", () => { }); }); - it("broadcasts provider credential changes across renderer windows", async () => { - const listener = vi.fn(); - const unsubscribe = await onProviderConfigChanged(listener); - mocks.configSave.mockResolvedValue({ - status: { providerId: "openai", isConfigured: true }, - refresh: { started: [], skipped: [] }, - }); - - await saveProviderConfig("openai", []); - - expect(mocks.emit).toHaveBeenCalledWith("provider-config:changed", { - providerId: "openai", - }); - mocks.providerConfigHandler?.({ payload: { providerId: "openai" } }); - expect(listener).toHaveBeenCalledWith("openai"); - unsubscribe(); - }); - it("deletes provider config through ACP", async () => { const response = { status: { diff --git a/src/features/providers/api/credentials.ts b/src/features/providers/api/credentials.ts index d595f926b..85299370d 100644 --- a/src/features/providers/api/credentials.ts +++ b/src/features/providers/api/credentials.ts @@ -4,7 +4,6 @@ import type { ProviderConfigStatusDto, ProviderSecretDto, } from "@aaif/goose-sdk"; -import { emit, listen, type UnlistenFn } from "@tauri-apps/api/event"; import type { ProviderFieldValue } from "@/shared/types/providers"; import { getClient } from "@/shared/api/acpConnection"; import { shareInFlight } from "@/shared/lib/shareInFlight"; @@ -12,21 +11,6 @@ import { shareInFlight } from "@/shared/lib/shareInFlight"; export type ProviderStatus = ProviderConfigStatusDto; export type ProviderFieldSaveInput = ProviderConfigFieldUpdate; -const PROVIDER_CONFIG_CHANGED_EVENT = "provider-config:changed"; - -async function notifyProviderConfigChanged(providerId: string) { - await emit(PROVIDER_CONFIG_CHANGED_EVENT, { providerId }); -} - -export function onProviderConfigChanged( - listener: (providerId: string) => void, -): Promise { - return listen<{ providerId: string }>( - PROVIDER_CONFIG_CHANGED_EVENT, - (event) => listener(event.payload.providerId), - ); -} - export async function getProviderConfig( providerId: string, ): Promise { @@ -46,7 +30,6 @@ export async function saveProviderConfig( providerId, fields, }); - await notifyProviderConfigChanged(providerId); return response; } @@ -57,7 +40,6 @@ export async function authenticateProviderConfig( const response = await client.goose.GooseUnstableProvidersConfigAuthenticate({ providerId, }); - await notifyProviderConfigChanged(providerId); return response; } @@ -68,7 +50,6 @@ export async function deleteProviderConfig( const response = await client.goose.GooseUnstableProvidersConfigDelete({ providerId, }); - await notifyProviderConfigChanged(providerId); return response; } From e9f01d848146e1a10cecee8747c5c67a0967c7bb Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 28 Aug 2026 14:25:58 -0400 Subject: [PATCH 15/18] fix(voice): harden OpenAI speech playback --- src-tauri/src/commands/openai_audio.rs | 73 ++++++++++++++++--- src-tauri/src/commands/pocket_voice.rs | 2 +- .../hooks/useOpenAiVoiceSetup.test.tsx | 23 +++++- .../hooks/useOpenAiVoiceSetup.ts | 1 + .../hooks/useVoiceConversationController.ts | 4 +- .../ui/OpenAiApiKeyField.tsx | 6 +- .../voice-conversation/ui/VoiceSettings.tsx | 19 ++++- 7 files changed, 107 insertions(+), 21 deletions(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 0a12f892d..58da3724b 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -23,7 +23,8 @@ use super::{ native_voice::AssistantSpeechGuard, pocket_audio_player::PocketAudioPlayer, pocket_voice::{ - effective_output_device_name, playback_latency_safety_duration, should_suppress_capture, + effective_output_device_name, playback_latency_safety_duration, selected_output_device, + should_suppress_capture, }, }; use super::{ @@ -54,6 +55,8 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); #[cfg(target_os = "macos")] const MAX_TTS_INPUT_CHARS: usize = 4096; +#[cfg(target_os = "macos")] +const MAX_FINAL_PLAYBACK_DRAIN: Duration = Duration::from_secs(600); #[derive(Clone, Debug, Default)] pub struct OpenAiVoiceState { @@ -228,8 +231,8 @@ fn tts_api_key() -> Result { fn normalize_openai_base_url(raw_url: String, assume_v1: bool) -> Result { let mut url = reqwest::Url::parse(&raw_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; - if !matches!(url.scheme(), "http" | "https") { - return Err("OpenAI voice endpoint must use HTTP or HTTPS".to_string()); + if url.scheme() != "https" { + return Err("OpenAI voice endpoint must use HTTPS".to_string()); } let path = url.path().trim_end_matches('/').to_string(); if assume_v1 || path.is_empty() { @@ -338,16 +341,23 @@ fn client() -> Result { } #[tauri::command] -pub fn get_openai_voice_status( +pub async fn get_openai_voice_status( state: State<'_, OpenAiVoiceState>, ) -> Result { - let configured = stored_api_key(TTS_KEYRING_ACCOUNT)?.is_some(); let playback_speed = state .playback .lock() .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? .speed; let tts_available = cfg!(target_os = "macos"); + let configured = if tts_available { + tauri::async_runtime::spawn_blocking(|| stored_api_key(TTS_KEYRING_ACCOUNT)) + .await + .map_err(|error| format!("Could not check OpenAI voice credentials: {error}"))?? + .is_some() + } else { + false + }; Ok(OpenAiVoiceStatus { configured, speech_model: speech_model(), @@ -629,8 +639,9 @@ fn run_openai_voice_stream( .enable_all() .build() .map_err(|error| format!("Could not initialize OpenAI speech runtime: {error}"))?; - let player = PocketAudioPlayer::new(TTS_SAMPLE_RATE, 1.0, None)?; - let output_device = effective_output_device_name(None); + let configured_output_device = selected_output_device(); + let player = PocketAudioPlayer::new(TTS_SAMPLE_RATE, 1.0, configured_output_device.as_deref())?; + let output_device = effective_output_device_name(configured_output_device.as_deref()); let suppress_capture = should_suppress_capture(interruption_mode, output_device.as_deref()); let output_latency_grace = playback_latency_safety_duration(output_device.as_deref()); let mut assistant_speech = None::; @@ -644,6 +655,12 @@ fn run_openai_voice_stream( let mut last_progress = Instant::now(); loop { + if started { + player.ensure_healthy().map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; + } update_openai_assistant_speech( player.is_empty(), &mut assistant_speech, @@ -731,11 +748,27 @@ fn run_openai_voice_stream( &mut playback_drained_at, output_latency_grace, speed, - )?; + ) + .map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; + let drain_started = Instant::now(); while active.load(Ordering::SeqCst) && (!player.is_empty() || assistant_speech.is_some()) { - player.ensure_healthy()?; + if drain_started.elapsed() >= MAX_FINAL_PLAYBACK_DRAIN { + player.stop(); + return Err(StreamFailure { + error: "OpenAI voice playback did not finish within 10 minutes" + .to_string(), + delivery: Some(snapshot_delivery(&delivery, &player)), + }); + } + player.ensure_healthy().map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; update_openai_assistant_speech( player.is_empty(), &mut assistant_speech, @@ -1001,6 +1034,15 @@ fn chunk_text(text: &str, max_chars: usize) -> Vec<&str> { if end == start { end = text.len(); } + if end < text.len() { + if let Some((offset, _)) = text[start..end] + .char_indices() + .rev() + .find(|(offset, character)| *offset > 0 && character.is_whitespace()) + { + end = start + offset; + } + } chunks.push(text[start..end].trim()); start = end; } @@ -1165,6 +1207,15 @@ mod tests { ); } + #[test] + fn openai_voice_endpoints_require_https() { + assert_eq!( + normalize_openai_base_url("http://proxy.example".to_string(), true) + .expect_err("plaintext endpoint must be rejected"), + "OpenAI voice endpoint must use HTTPS" + ); + } + #[test] fn openai_base_url_preserves_custom_paths_and_query_parameters() { assert_eq!( @@ -1217,6 +1268,10 @@ mod tests { fn chunks_tts_text_on_char_boundaries() { assert_eq!(chunk_text("hello", 10), vec!["hello"]); assert_eq!(chunk_text("ééé", 3), vec!["é", "é", "é"]); + assert_eq!( + chunk_text("hello wide world", 8), + vec!["hello", "wide", "world"] + ); } #[cfg(target_os = "macos")] diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index 5e1a21c86..68228d50b 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -514,7 +514,7 @@ fn playback_speed(base: &Path) -> f32 { settings(base).playback_speed.clamp(0.75, 2.0) } -fn selected_output_device() -> Option { +pub(crate) fn selected_output_device() -> Option { std::env::var("VOICE_CONVERSATION_OUTPUT_DEVICE") .ok() .filter(|value| !value.is_empty()) diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index e77463709..fe14edd3b 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -23,10 +23,12 @@ vi.mock("../api/openAiVoice", () => ({ function deferred() { let resolve!: (value: T) => void; - const promise = new Promise((next) => { + let reject!: (reason?: unknown) => void; + const promise = new Promise((next, fail) => { resolve = next; + reject = fail; }); - return { promise, resolve }; + return { promise, reject, resolve }; } function status(configured: boolean): OpenAiVoiceStatus { @@ -89,4 +91,21 @@ describe("useOpenAiVoiceSetup", () => { await waitFor(() => expect(result.current.status?.configured).toBe(true)); }); + + it("clears stale readiness when a credential refresh fails", async () => { + const refresh = deferred(); + mocks.getStatus + .mockResolvedValueOnce(status(true)) + .mockReturnValueOnce(refresh.promise); + const { result } = renderHook(() => useOpenAiVoiceSetup()); + await waitFor(() => expect(mocks.finishListening).not.toBeNull()); + act(() => mocks.finishListening?.()); + await waitFor(() => expect(result.current.status?.configured).toBe(true)); + + act(() => mocks.settingsChanged?.()); + refresh.reject(new Error("Keychain unavailable")); + + await waitFor(() => expect(result.current.status).toBeNull()); + expect(result.current.error).toBe("Keychain unavailable"); + }); }); diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts index 33c9d8e98..f31c7b801 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts @@ -25,6 +25,7 @@ export function useOpenAiVoiceSetup(enabled = true) { }, (cause) => { if (active && generation === refreshGeneration) { + setStatus(null); setError(cause instanceof Error ? cause.message : String(cause)); } }, diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index cf94cb821..fbd386338 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -772,11 +772,11 @@ export function useVoiceConversationController({ const onFailure = (text: string, playbackError: unknown) => { addErrorNotification( sessionId, - `Pocket TTS could not speak the assistant response: ${errorText( + `Voice playback could not speak the assistant response: ${errorText( playbackError, )}`, ); - console.error("Native Pocket playback failed", { + console.error("Native voice playback failed", { sessionId, textLength: text.length, error: playbackError, diff --git a/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx b/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx index f91df7c90..d39830e89 100644 --- a/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx +++ b/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx @@ -4,14 +4,14 @@ import { Button } from "@/shared/ui/button"; import { Input } from "@/shared/ui/input"; interface OpenAiApiKeyFieldProps { - kind: "stt" | "tts"; + label: string; configured: boolean; onSave: (apiKey: string) => Promise; onClear: () => Promise; } export function OpenAiApiKeyField({ - kind, + label, configured, onSave, onClear, @@ -21,8 +21,6 @@ export function OpenAiApiKeyField({ const [apiKey, setApiKey] = useState(""); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); - const label = - kind === "stt" ? t("voice.openAiSttApiKey") : t("voice.openAiTtsApiKey"); const save = async () => { setSaving(true); diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 4b3576c70..46287941b 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -83,6 +83,7 @@ export function VoiceSettings() { const macSpeechSetup = useMacSpeechSetup(); const { status: openAiStatus, error: openAiError } = useOpenAiVoiceSetup(); const [openAiSpeed, setOpenAiSpeed] = useState(1); + const [openAiSpeedError, setOpenAiSpeedError] = useState(null); useEffect(() => { if (openAiStatus) setOpenAiSpeed(openAiStatus.playbackSpeed); }, [openAiStatus]); @@ -265,7 +266,7 @@ export function VoiceSettings() { output.backend === "openai" ? (

{ - await setOpenAiPlaybackSpeed(speed); - setOpenAiSpeed(speed); + setOpenAiSpeedError(null); + try { + await setOpenAiPlaybackSpeed(speed); + setOpenAiSpeed(speed); + } catch (cause) { + setOpenAiSpeedError( + cause instanceof Error ? cause.message : String(cause), + ); + } }} /> + {openAiSpeedError ? ( +

+ {openAiSpeedError} +

+ ) : null}
) : output.backend === "siri" ? ( From 2aca321bcdb67c18b612c15628e77fba180e6dcd Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 28 Aug 2026 14:35:58 -0400 Subject: [PATCH 16/18] refactor(voice): name OpenAI TTS readiness explicitly --- src-tauri/src/commands/openai_audio.rs | 8 ++++---- .../voice-conversation/api/openAiVoice.ts | 2 +- .../hooks/useOpenAiVoiceSetup.test.tsx | 20 +++++++++++++------ .../lib/voiceSetupReadiness.test.ts | 2 +- .../lib/voiceSetupReadiness.ts | 2 +- .../ui/VoiceSettings.test.tsx | 6 +++--- .../voice-conversation/ui/VoiceSettings.tsx | 4 ++-- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 58da3724b..3887306d9 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -99,7 +99,7 @@ enum OpenAiStreamCommand { #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct OpenAiVoiceStatus { - configured: bool, + tts_configured: bool, speech_model: String, speech_voice: String, playback_speed: f32, @@ -350,7 +350,7 @@ pub async fn get_openai_voice_status( .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? .speed; let tts_available = cfg!(target_os = "macos"); - let configured = if tts_available { + let tts_configured = if tts_available { tauri::async_runtime::spawn_blocking(|| stored_api_key(TTS_KEYRING_ACCOUNT)) .await .map_err(|error| format!("Could not check OpenAI voice credentials: {error}"))?? @@ -359,12 +359,12 @@ pub async fn get_openai_voice_status( false }; Ok(OpenAiVoiceStatus { - configured, + tts_configured, speech_model: speech_model(), speech_voice: speech_voice(), playback_speed, tts_available, - unavailable_reason: if !configured { + unavailable_reason: if !tts_configured { Some("Add an OpenAI text-to-speech API key in Voice settings.".to_string()) } else if !tts_available { Some("OpenAI voice playback is currently supported on macOS only.".to_string()) diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index 71a18d541..b0677da6c 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -7,7 +7,7 @@ import type { } from "../lib/voiceInterruptionPreference"; export interface OpenAiVoiceStatus { - configured: boolean; + ttsConfigured: boolean; speechModel: string; speechVoice: string; playbackSpeed: number; diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index fe14edd3b..9203e3f1b 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -33,7 +33,7 @@ function deferred() { function status(configured: boolean): OpenAiVoiceStatus { return { - configured, + ttsConfigured: configured, speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -63,12 +63,14 @@ describe("useOpenAiVoiceSetup", () => { act(() => mocks.settingsChanged?.()); refreshed.resolve(status(true)); - await waitFor(() => expect(result.current.status?.configured).toBe(true)); + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); initial.resolve(status(false)); await act(async () => Promise.resolve()); - expect(result.current.status?.configured).toBe(true); + expect(result.current.status?.ttsConfigured).toBe(true); }); it("refreshes after listener registration captures credential changes", async () => { @@ -80,7 +82,9 @@ describe("useOpenAiVoiceSetup", () => { act(() => mocks.finishListening?.()); - await waitFor(() => expect(result.current.status?.configured).toBe(true)); + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); }); it("still loads status when listener registration fails", async () => { @@ -89,7 +93,9 @@ describe("useOpenAiVoiceSetup", () => { const { result } = renderHook(() => useOpenAiVoiceSetup()); - await waitFor(() => expect(result.current.status?.configured).toBe(true)); + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); }); it("clears stale readiness when a credential refresh fails", async () => { @@ -100,7 +106,9 @@ describe("useOpenAiVoiceSetup", () => { const { result } = renderHook(() => useOpenAiVoiceSetup()); await waitFor(() => expect(mocks.finishListening).not.toBeNull()); act(() => mocks.finishListening?.()); - await waitFor(() => expect(result.current.status?.configured).toBe(true)); + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); act(() => mocks.settingsChanged?.()); refresh.reject(new Error("Keychain unavailable")); diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts index 04406b9c7..6a35f946e 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts @@ -54,7 +54,7 @@ describe("voice setup readiness", () => { }); it("requires the dedicated OpenAI text-to-speech key for OpenAI output", () => { - const configured = { configured: true, ttsAvailable: true } as never; + const configured = { ttsConfigured: true, ttsAvailable: true } as never; expect( isVoiceSetupReady(pocket, null, null, "parakeet", "openai", configured), ).toBe(true); diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.ts index 0ae9ca757..196c853d9 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.ts @@ -24,7 +24,7 @@ export function isVoiceSetupReady( : Boolean(pocket?.parakeetInstalled); if (!inputReady) return false; if (outputBackend === "openai") - return Boolean(openAi?.configured && openAi.ttsAvailable); + return Boolean(openAi?.ttsConfigured && openAi.ttsAvailable); if (outputBackend === "pocket") return Boolean(pocket?.pocketInstalled); return Boolean( siri?.supported && siri.selectedVoice && siri.selectedVoiceInstalled, diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 4b3bd867a..df674ba9d 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -52,7 +52,7 @@ const microphonePermissionState = vi.hoisted(() => ({ })); const openAiStatusState = vi.hoisted(() => ({ current: { - configured: true, + ttsConfigured: true, speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -217,7 +217,7 @@ describe("VoiceSettings", () => { interruptionState.mode = "automatic"; siriSetupState.current = siriSetup(); openAiStatusState.current = { - configured: true, + ttsConfigured: true, speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -258,7 +258,7 @@ describe("VoiceSettings", () => { outputState.backend = "openai"; openAiStatusState.current = { ...openAiStatusState.current, - configured: false, + ttsConfigured: false, unavailableReason: "Add an OpenAI text-to-speech API key in Voice settings.", }; diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 46287941b..569fec213 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -111,7 +111,7 @@ export function VoiceSettings() { : (setup.status?.parakeetInstalled ?? false); const outputReady = output.backend === "openai" - ? Boolean(openAiStatus?.configured && openAiStatus.ttsAvailable) + ? Boolean(openAiStatus?.ttsConfigured && openAiStatus.ttsAvailable) : output.backend === "siri" ? Boolean( siriSetup.status?.supported && @@ -267,7 +267,7 @@ export function VoiceSettings() {
From 2497d6280c91b6c030ce52d1273f12a7375f9190 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 28 Aug 2026 15:04:04 -0400 Subject: [PATCH 17/18] fix(voice): harden OpenAI TTS configuration --- src-tauri/src/commands/openai_audio.rs | 51 +++++++++++++++---- .../voice-conversation/api/openAiVoice.ts | 2 +- .../hooks/useOpenAiVoiceSetup.test.tsx | 18 ++++++- .../hooks/useOpenAiVoiceSetup.ts | 5 +- .../voice-conversation/ui/VoiceSettings.tsx | 21 +++++--- src/shared/i18n/locales/en/settings.json | 4 ++ src/shared/i18n/locales/es/settings.json | 4 ++ 7 files changed, 84 insertions(+), 21 deletions(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 3887306d9..d14a98c8c 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -50,7 +50,7 @@ const INITIAL_PLAYBACK_BUFFER_FRAMES: usize = TTS_SAMPLE_RATE as usize / 5; #[cfg(target_os = "macos")] const TTS_EVENT: &str = "openai-voice:stream-event"; #[cfg(target_os = "macos")] -const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); +const STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(120); #[cfg(target_os = "macos")] const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); #[cfg(target_os = "macos")] @@ -335,7 +335,6 @@ fn persist_playback_speed(speed: f32) -> Result<(), String> { fn client() -> Result { reqwest::Client::builder() .connect_timeout(CONNECT_TIMEOUT) - .timeout(REQUEST_TIMEOUT) .build() .map_err(|error| format!("create OpenAI HTTP client: {error}")) } @@ -364,10 +363,10 @@ pub async fn get_openai_voice_status( speech_voice: speech_voice(), playback_speed, tts_available, - unavailable_reason: if !tts_configured { - Some("Add an OpenAI text-to-speech API key in Voice settings.".to_string()) - } else if !tts_available { - Some("OpenAI voice playback is currently supported on macOS only.".to_string()) + unavailable_reason: if !tts_available { + Some("unsupportedPlatform".to_string()) + } else if !tts_configured { + Some("missingApiKey".to_string()) } else { None }, @@ -375,11 +374,16 @@ pub async fn get_openai_voice_status( } #[tauri::command] -pub fn set_openai_tts_api_key(app: AppHandle, api_key: String) -> Result<(), String> { +pub fn set_openai_tts_api_key( + app: AppHandle, + state: State<'_, OpenAiVoiceState>, + api_key: String, +) -> Result<(), String> { let api_key = api_key.trim(); if api_key.is_empty() { return Err("OpenAI text-to-speech API key cannot be empty".to_string()); } + stop_openai_voice_inner(&state)?; store_api_key(TTS_KEYRING_ACCOUNT, api_key)?; app.emit(SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; @@ -387,7 +391,11 @@ pub fn set_openai_tts_api_key(app: AppHandle, api_key: String) -> Result<(), Str } #[tauri::command] -pub fn clear_openai_tts_api_key(app: AppHandle) -> Result<(), String> { +pub fn clear_openai_tts_api_key( + app: AppHandle, + state: State<'_, OpenAiVoiceState>, +) -> Result<(), String> { + stop_openai_voice_inner(&state)?; clear_api_key(TTS_KEYRING_ACCOUNT)?; app.emit(SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; @@ -678,7 +686,7 @@ fn run_openai_voice_stream( match receiver.recv_timeout(Duration::from_millis(20)) { Ok(OpenAiStreamCommand::Append(text)) => { pending.push_str(&text); - if pending.len() >= 24 && pending.trim_end().ends_with(['.', '!', '?', '\n']) { + if pending.len() >= 24 && ends_sentence_boundary(&pending) { speak_pending( &runtime, app, @@ -872,6 +880,7 @@ fn speak_pending( }; let mut pcm_remainder = Vec::::new(); let mut initial_samples = Vec::::new(); + let mut last_network_data = Instant::now(); loop { update_openai_assistant_speech( player.is_empty(), @@ -888,12 +897,14 @@ fn speak_pending( }); let Some(item) = (match item { Ok(item) => item, - Err(_) => continue, + Err(_) if last_network_data.elapsed() < STREAM_IDLE_TIMEOUT => continue, + Err(_) => return Err("OpenAI speech audio stream timed out".to_string()), }) else { break; }; let item = item.map_err(|error| format_openai_request_error("stream speech audio", error))?; + last_network_data = Instant::now(); if !active.load(Ordering::SeqCst) { return Ok(()); } @@ -955,6 +966,17 @@ fn speak_pending( Ok(()) } +#[cfg(target_os = "macos")] +fn ends_sentence_boundary(text: &str) -> bool { + if text.trim_end_matches([' ', '\t', '\r']).ends_with('\n') { + return true; + } + let trimmed = text.trim_end(); + trimmed + .trim_end_matches(['"', '\'', '”', '’', ')', ']', '}']) + .ends_with(['.', '!', '?']) +} + #[cfg(target_os = "macos")] async fn run_while_active( future: F, @@ -1274,6 +1296,15 @@ mod tests { ); } + #[cfg(target_os = "macos")] + #[test] + fn recognizes_sentence_boundaries_before_streaming() { + assert!(ends_sentence_boundary("Hello world.\n")); + assert!(ends_sentence_boundary("Did it work?” ")); + assert!(ends_sentence_boundary("It did!)")); + assert!(!ends_sentence_boundary("Still speaking,")); + } + #[cfg(target_os = "macos")] #[test] fn cancels_a_stalled_speech_request() { diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index b0677da6c..45bf46b33 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -12,7 +12,7 @@ export interface OpenAiVoiceStatus { speechVoice: string; playbackSpeed: number; ttsAvailable: boolean; - unavailableReason: string | null; + unavailableReason: "missingApiKey" | "unsupportedPlatform" | null; } export interface OpenAiVoiceStreamEvent { diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index 9203e3f1b..c1a87c9cb 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -38,7 +38,7 @@ function status(configured: boolean): OpenAiVoiceStatus { speechVoice: "marin", playbackSpeed: 1, ttsAvailable: true, - unavailableReason: configured ? null : "Configure OpenAI.", + unavailableReason: configured ? null : "missingApiKey", }; } @@ -116,4 +116,20 @@ describe("useOpenAiVoiceSetup", () => { await waitFor(() => expect(result.current.status).toBeNull()); expect(result.current.error).toBe("Keychain unavailable"); }); + + it("does not expose cached readiness while disabled", async () => { + mocks.listenerError = new Error("listener unavailable"); + mocks.getStatus.mockResolvedValue(status(true)); + const { result, rerender } = renderHook( + ({ enabled }) => useOpenAiVoiceSetup(enabled), + { initialProps: { enabled: true } }, + ); + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); + + rerender({ enabled: false }); + + expect(result.current).toEqual({ status: null, error: null }); + }); }); diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts index f31c7b801..f53d75090 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts @@ -48,5 +48,8 @@ export function useOpenAiVoiceSetup(enabled = true) { }; }, [enabled]); - return { status, error }; + return { + status: enabled ? status : null, + error: enabled ? error : null, + }; } diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 569fec213..f6239f4a5 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -55,7 +55,9 @@ function readinessDescriptionKey( if (inputReady && outputReady) return null; if (!inputReady && !outputReady) { if (backend === "openai") { - return "voice.notReadyOpenAi"; + return inputBackend === "macos" + ? "voice.notReadyMacInputAndOpenAiOutput" + : "voice.notReadyInputAndOpenAiOutput"; } if (inputBackend === "macos") { return backend === "siri" @@ -273,13 +275,16 @@ export function VoiceSettings() { />

{openAiError ?? - openAiStatus?.unavailableReason ?? - (openAiStatus - ? t("voice.openAiTtsConfigured", { - model: openAiStatus.speechModel, - voice: openAiStatus.speechVoice, - }) - : t("voice.openAiChecking"))} + (openAiStatus?.unavailableReason === "unsupportedPlatform" + ? t("voice.openAiTtsUnsupportedPlatform") + : openAiStatus?.unavailableReason === "missingApiKey" + ? t("voice.openAiTtsNeedsKey") + : openAiStatus + ? t("voice.openAiTtsConfigured", { + model: openAiStatus.speechModel, + voice: openAiStatus.speechVoice, + }) + : t("voice.openAiChecking"))}

Date: Fri, 28 Aug 2026 15:40:08 -0400 Subject: [PATCH 18/18] fix(voice): guard macOS keyring constant --- src-tauri/src/commands/openai_audio.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index d14a98c8c..a0709de77 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -38,6 +38,7 @@ use std::time::Instant; const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; const DEFAULT_TTS_VOICE: &str = "marin"; +#[cfg(target_os = "macos")] const KEYRING_SERVICE: &str = "berd-openai-voice"; const TTS_KEYRING_ACCOUNT: &str = "tts-api-key"; const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed";