diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 153965f09..1de42d10a 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -28,6 +28,7 @@ dependencies = [ "hex", "ignore", "infer", + "keyring", "libc", "log", "mime_guess", @@ -3174,6 +3175,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "keyring" +version = "3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" +dependencies = [ + "log", + "security-framework 2.11.1", + "security-framework 3.7.0", + "zeroize", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -5262,7 +5275,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.7.0", ] [[package]] @@ -5290,7 +5303,7 @@ dependencies = [ "rustls-native-certs", "rustls-platform-verifier-android", "rustls-webpki", - "security-framework", + "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", "windows-sys 0.61.2", @@ -5401,6 +5414,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.7.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 9cfaef08c..8250e0747 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -42,6 +42,7 @@ hex = "0.4" ignore = "0.4.25" fern = "0.7" infer = "0.19.0" +keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } libc = "0.2" log = "0.4.29" mime_guess = "2" diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 74b4a1b4f..04e7ac784 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -37,6 +37,7 @@ pub mod model_setup; mod native_input_mute; pub mod native_voice; pub mod notifications; +pub mod openai_audio; #[cfg(feature = "block-voice-dictation")] pub mod openai_realtime; pub mod path_resolver; diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 4fa5e637e..2337baaf5 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -2178,6 +2178,8 @@ pub fn handle_voice_owner_window_destroyed(app: &AppHandle, window_label: &str) } app.state::() .stop_for_window_destroyed(window_label); + app.state::() + .stop_for_window_destroyed(window_label); } fn software_microphone_mute(native_microphone_mute_control: bool, muted: bool) -> bool { diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs new file mode 100644 index 000000000..a0709de77 --- /dev/null +++ b/src-tauri/src/commands/openai_audio.rs @@ -0,0 +1,1331 @@ +//! OpenAI streaming speech playback for voice conversations. + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, Mutex, +}; +#[cfg(any(test, target_os = "macos"))] +use std::time::Duration; + +#[cfg(target_os = "macos")] +use futures_util::StreamExt; +#[cfg(target_os = "macos")] +use reqwest::header::CONTENT_TYPE; +#[cfg(target_os = "macos")] +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; +use serde::Serialize; +use serde_json::json; +use tauri::Emitter; +use tauri::{AppHandle, State}; + +#[cfg(target_os = "macos")] +use super::{ + native_voice::AssistantSpeechGuard, + pocket_audio_player::PocketAudioPlayer, + pocket_voice::{ + effective_output_device_name, playback_latency_safety_duration, selected_output_device, + should_suppress_capture, + }, +}; +use super::{ + native_voice::{InterruptionSensitivity, NativeVoiceState}, + pocket_voice::VoiceInterruptionMode, +}; +#[cfg(any(test, target_os = "macos"))] +use std::time::Instant; + +#[cfg(target_os = "macos")] +const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; +const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; +const DEFAULT_TTS_VOICE: &str = "marin"; +#[cfg(target_os = "macos")] +const KEYRING_SERVICE: &str = "berd-openai-voice"; +const TTS_KEYRING_ACCOUNT: &str = "tts-api-key"; +const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; +#[cfg(target_os = "macos")] +const TTS_SAMPLE_RATE: u32 = 24_000; +// Avoid starting the audio device from a tiny first network chunk that can drain +// before subsequent streamed PCM arrives. +#[cfg(target_os = "macos")] +const INITIAL_PLAYBACK_BUFFER_FRAMES: usize = TTS_SAMPLE_RATE as usize / 5; +#[cfg(target_os = "macos")] +const TTS_EVENT: &str = "openai-voice:stream-event"; +#[cfg(target_os = "macos")] +const STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(120); +#[cfg(target_os = "macos")] +const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +#[cfg(target_os = "macos")] +const MAX_TTS_INPUT_CHARS: usize = 4096; +#[cfg(target_os = "macos")] +const MAX_FINAL_PLAYBACK_DRAIN: Duration = Duration::from_secs(600); + +#[derive(Clone, Debug, Default)] +pub struct OpenAiVoiceState { + playback: Arc>, +} + +#[derive(Debug)] +struct PlaybackRuntime { + active: Option>, + stream: Option, + speed: f32, +} + +impl Default for PlaybackRuntime { + fn default() -> Self { + Self { + active: None, + stream: None, + speed: stored_playback_speed(), + } + } +} + +#[derive(Debug)] +struct ActiveOpenAiStream { + id: String, + owner_window: String, + sender: mpsc::Sender, +} + +#[cfg_attr(not(target_os = "macos"), allow(dead_code))] +#[derive(Debug)] +enum OpenAiStreamCommand { + Append(String), + Flush, + Finish, + Stop, +} + +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct OpenAiVoiceStatus { + tts_configured: bool, + speech_model: String, + speech_voice: String, + playback_speed: f32, + tts_available: bool, + unavailable_reason: Option, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct OpenAiVoiceStreamEvent { + stream_id: String, + state: OpenAiStreamEventState, + error: Option, + delivery: Option, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +enum OpenAiStreamEventState { + Started, + Progress, + Completed, + Interrupted, + Failed, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct VoiceDeliveryProgress { + sample_rate: u32, + segments: Vec, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct VoiceDeliverySegment { + text: String, + played_frames: u64, + total_frames: u64, + synthesis_complete: bool, +} + +fn env_trimmed(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +#[cfg(target_os = "macos")] +fn goose_yaml_value(path: &std::path::Path, name: &str) -> Result, String> { + if !path.exists() { + return Ok(None); + } + let payload = std::fs::read_to_string(path) + .map_err(|error| format!("Could not read Goose configuration: {error}"))?; + let values: serde_json::Value = yaml_serde::from_str(&payload) + .map_err(|error| format!("Goose configuration is invalid: {error}"))?; + Ok(values + .get(name) + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string)) +} + +#[cfg(target_os = "macos")] +fn stored_api_key(account: &str) -> Result, String> { + let entry = keyring::Entry::new(KEYRING_SERVICE, account) + .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}"))?; + match entry.get_password() { + Ok(value) => Ok(Some(value)), + Err(keyring::Error::NoEntry) => Ok(None), + Err(error) => Err(format!( + "Could not read Berd's OpenAI voice credential: {error}" + )), + } +} + +#[cfg(not(target_os = "macos"))] +fn stored_api_key(_account: &str) -> Result, String> { + Ok(None) +} + +#[cfg(target_os = "macos")] +fn store_api_key(account: &str, api_key: &str) -> Result<(), String> { + let entry = keyring::Entry::new(KEYRING_SERVICE, account) + .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}"))?; + entry + .set_password(api_key) + .map_err(|error| format!("Could not save Berd's OpenAI voice credential: {error}")) +} + +#[cfg(not(target_os = "macos"))] +fn store_api_key(_account: &str, _api_key: &str) -> Result<(), String> { + Err("OpenAI voice credentials are unsupported on this platform".to_string()) +} + +#[cfg(target_os = "macos")] +fn clear_api_key(account: &str) -> Result<(), String> { + let entry = keyring::Entry::new(KEYRING_SERVICE, account) + .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}"))?; + match entry.delete_credential() { + Ok(()) | Err(keyring::Error::NoEntry) => Ok(()), + Err(error) => Err(format!( + "Could not remove Berd's OpenAI voice credential: {error}" + )), + } +} + +#[cfg(not(target_os = "macos"))] +fn clear_api_key(_account: &str) -> Result<(), String> { + Err("OpenAI voice credentials are unsupported on this platform".to_string()) +} + +#[cfg(target_os = "macos")] +fn tts_api_key() -> Result { + stored_api_key(TTS_KEYRING_ACCOUNT)?.ok_or_else(|| { + "OpenAI text-to-speech is not configured. Add its API key in Voice settings, then try again." + .to_string() + }) +} + +#[cfg(any(test, target_os = "macos"))] +fn normalize_openai_base_url(raw_url: String, assume_v1: bool) -> Result { + let mut url = reqwest::Url::parse(&raw_url) + .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; + if url.scheme() != "https" { + return Err("OpenAI voice endpoint must use HTTPS".to_string()); + } + let path = url.path().trim_end_matches('/').to_string(); + if assume_v1 || path.is_empty() { + let path = if path.ends_with("/v1") { + path + } else { + format!("{path}/v1") + }; + url.set_path(&path); + } else { + url.set_path(&path); + } + url.set_fragment(None); + Ok(url.to_string().trim_end_matches('/').to_string()) +} + +#[cfg(target_os = "macos")] +fn base_url() -> Result { + if let Some(host) = env_trimmed("OPENAI_HOST") { + return normalize_openai_base_url(host, true); + } + if let Some(base_url) = env_trimmed("OPENAI_BASE_URL") { + return normalize_openai_base_url(base_url, false); + } + let config_path = crate::services::goose_config::config_path()?; + if let Some(base_url) = goose_yaml_value(&config_path, "OPENAI_BASE_URL")? { + return normalize_openai_base_url(base_url, false); + } + if let Some(host) = goose_yaml_value(&config_path, "OPENAI_HOST")? { + return normalize_openai_base_url(host, true); + } + Ok(DEFAULT_BASE_URL.to_string()) +} + +fn speech_model() -> String { + env_trimmed("OPENAI_TTS_MODEL").unwrap_or_else(|| DEFAULT_TTS_MODEL.to_string()) +} + +fn speech_voice() -> String { + env_trimmed("OPENAI_TTS_VOICE").unwrap_or_else(|| DEFAULT_TTS_VOICE.to_string()) +} + +#[cfg(target_os = "macos")] +fn endpoint(path: &str) -> Result { + endpoint_for_base_url(&base_url()?, path) +} + +#[cfg(any(test, target_os = "macos"))] +fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { + let mut url = reqwest::Url::parse(base_url) + .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; + let base_path = url.path().trim_end_matches('/'); + url.set_path(&format!("{base_path}/{}", path.trim_start_matches('/'))); + Ok(url.to_string()) +} + +#[cfg(target_os = "macos")] +fn authorized_headers(key: &str) -> Result { + let mut headers = HeaderMap::new(); + let bearer = format!("Bearer {key}"); + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&bearer).map_err(|_| "OpenAI API key is not a valid header value")?, + ); + Ok(headers) +} + +fn speed_settings_path() -> Result { + Ok(crate::services::goose_config::config_path()? + .parent() + .ok_or_else(|| "Could not resolve Goose's configuration directory".to_string())? + .join("openai-voice-settings.json")) +} + +fn stored_playback_speed() -> f32 { + speed_settings_path() + .ok() + .and_then(|path| std::fs::read(path).ok()) + .and_then(|data| serde_json::from_slice::(&data).ok()) + .and_then(|value| value.get("playbackSpeed")?.as_f64()) + .map(|speed| speed as f32) + .filter(|speed| speed.is_finite() && (0.75..=2.0).contains(speed)) + .unwrap_or(1.0) +} + +fn persist_playback_speed(speed: f32) -> Result<(), String> { + let path = speed_settings_path()?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| format!("create OpenAI voice settings directory: {error}"))?; + } + std::fs::write( + &path, + serde_json::to_vec_pretty(&json!({ "playbackSpeed": speed })).unwrap(), + ) + .map_err(|error| format!("write OpenAI voice settings: {error}")) +} + +#[cfg(target_os = "macos")] +fn client() -> Result { + reqwest::Client::builder() + .connect_timeout(CONNECT_TIMEOUT) + .build() + .map_err(|error| format!("create OpenAI HTTP client: {error}")) +} + +#[tauri::command] +pub async fn get_openai_voice_status( + state: State<'_, OpenAiVoiceState>, +) -> Result { + let playback_speed = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? + .speed; + let tts_available = cfg!(target_os = "macos"); + let tts_configured = if tts_available { + tauri::async_runtime::spawn_blocking(|| stored_api_key(TTS_KEYRING_ACCOUNT)) + .await + .map_err(|error| format!("Could not check OpenAI voice credentials: {error}"))?? + .is_some() + } else { + false + }; + Ok(OpenAiVoiceStatus { + tts_configured, + speech_model: speech_model(), + speech_voice: speech_voice(), + playback_speed, + tts_available, + unavailable_reason: if !tts_available { + Some("unsupportedPlatform".to_string()) + } else if !tts_configured { + Some("missingApiKey".to_string()) + } else { + None + }, + }) +} + +#[tauri::command] +pub fn set_openai_tts_api_key( + app: AppHandle, + state: State<'_, OpenAiVoiceState>, + api_key: String, +) -> Result<(), String> { + let api_key = api_key.trim(); + if api_key.is_empty() { + return Err("OpenAI text-to-speech API key cannot be empty".to_string()); + } + stop_openai_voice_inner(&state)?; + store_api_key(TTS_KEYRING_ACCOUNT, api_key)?; + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; + Ok(()) +} + +#[tauri::command] +pub fn clear_openai_tts_api_key( + app: AppHandle, + state: State<'_, OpenAiVoiceState>, +) -> Result<(), String> { + stop_openai_voice_inner(&state)?; + clear_api_key(TTS_KEYRING_ACCOUNT)?; + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; + Ok(()) +} + +#[tauri::command] +pub fn start_openai_voice_stream( + app: AppHandle, + webview_window: tauri::WebviewWindow, + state: State<'_, OpenAiVoiceState>, + native_voice: State<'_, NativeVoiceState>, + stream_id: String, + interruption_mode: VoiceInterruptionMode, + interruption_sensitivity: InterruptionSensitivity, +) -> Result<(), String> { + #[cfg(not(target_os = "macos"))] + { + let _ = ( + app, + webview_window, + state, + native_voice, + stream_id, + interruption_mode, + interruption_sensitivity, + ); + Err("OpenAI voice playback is currently supported on macOS only".to_string()) + } + + #[cfg(target_os = "macos")] + { + if stream_id.trim().is_empty() { + return Err("OpenAI voice stream id cannot be empty".to_string()); + } + let key = tts_api_key()?; + let (sender, receiver) = mpsc::channel(); + let active = Arc::new(AtomicBool::new(true)); + { + let mut playback = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())?; + if let Some(previous) = playback.active.as_ref() { + previous.store(false, Ordering::SeqCst); + } + if let Some(previous) = playback.stream.as_ref() { + let _ = previous.sender.send(OpenAiStreamCommand::Stop); + } + playback.active = Some(active.clone()); + playback.stream = Some(ActiveOpenAiStream { + id: stream_id.clone(), + owner_window: webview_window.label().to_string(), + sender, + }); + } + let speed = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? + .speed; + let playback = state.playback.clone(); + let native_voice = native_voice.inner().clone(); + tauri::async_runtime::spawn_blocking(move || { + let result = run_openai_voice_stream( + &app, + &stream_id, + key, + active.clone(), + receiver, + native_voice, + interruption_mode, + interruption_sensitivity, + speed, + ); + if let Ok(mut playback) = playback.lock() { + let still_owns_playback = playback + .active + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &active)); + if still_owns_playback { + playback.active = None; + playback.stream = None; + } + } + let (state, error, delivery) = match result { + Ok(outcome) => (outcome.state, None, outcome.delivery), + Err(failure) if !active.load(Ordering::SeqCst) => { + (OpenAiStreamEventState::Interrupted, None, failure.delivery) + } + Err(failure) => ( + OpenAiStreamEventState::Failed, + Some(failure.error), + failure.delivery, + ), + }; + emit_openai_stream_event(&app, &stream_id, state, error, delivery); + }); + Ok(()) + } +} + +#[tauri::command] +pub fn append_openai_voice_stream( + state: State<'_, OpenAiVoiceState>, + stream_id: String, + text: String, +) -> Result<(), String> { + if text.is_empty() { + return Ok(()); + } + send_stream_command(&state, &stream_id, OpenAiStreamCommand::Append(text)) +} + +#[tauri::command] +pub fn flush_openai_voice_stream( + state: State<'_, OpenAiVoiceState>, + stream_id: String, +) -> Result<(), String> { + send_stream_command(&state, &stream_id, OpenAiStreamCommand::Flush) +} + +#[tauri::command] +pub fn finish_openai_voice_stream( + state: State<'_, OpenAiVoiceState>, + stream_id: String, +) -> Result<(), String> { + send_stream_command(&state, &stream_id, OpenAiStreamCommand::Finish) +} + +#[tauri::command] +pub fn set_openai_playback_speed( + state: State<'_, OpenAiVoiceState>, + speed: f32, +) -> Result<(), String> { + if !speed.is_finite() || !(0.75..=2.0).contains(&speed) { + return Err("OpenAI playback speed must be between 0.75 and 2.0".to_string()); + } + persist_playback_speed(speed)?; + state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? + .speed = speed; + Ok(()) +} + +fn stop_openai_voice_for_owner( + state: &OpenAiVoiceState, + owner_window: Option<&str>, +) -> Result { + let playback = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())?; + if owner_window.is_some_and(|owner| { + playback + .stream + .as_ref() + .is_none_or(|stream| stream.owner_window != owner) + }) { + return Ok(false); + } + let Some(active) = playback.active.as_ref() else { + return Ok(false); + }; + active.store(false, Ordering::SeqCst); + if let Some(stream) = playback.stream.as_ref() { + let _ = stream.sender.send(OpenAiStreamCommand::Stop); + } + Ok(true) +} + +pub(crate) fn stop_openai_voice_inner(state: &OpenAiVoiceState) -> Result { + stop_openai_voice_for_owner(state, None) +} + +impl OpenAiVoiceState { + pub(crate) fn stop_for_window_destroyed(&self, window_label: &str) -> bool { + stop_openai_voice_for_owner(self, Some(window_label)).unwrap_or_else(|error| { + log::warn!("Failed to stop OpenAI playback for a destroyed window: {error}"); + false + }) + } +} + +#[tauri::command] +pub fn stop_openai_voice(state: State<'_, OpenAiVoiceState>) -> Result { + stop_openai_voice_inner(&state) +} + +fn send_stream_command( + state: &OpenAiVoiceState, + stream_id: &str, + command: OpenAiStreamCommand, +) -> Result<(), String> { + let playback = state + .playback + .lock() + .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())?; + let stream = playback + .stream + .as_ref() + .filter(|stream| stream.id == stream_id) + .ok_or_else(|| format!("OpenAI voice stream is not active: {stream_id}"))?; + stream + .sender + .send(command) + .map_err(|_| format!("OpenAI voice stream worker stopped: {stream_id}")) +} + +#[cfg(target_os = "macos")] +struct StreamOutcome { + state: OpenAiStreamEventState, + delivery: Option, +} + +#[cfg(target_os = "macos")] +struct StreamFailure { + error: String, + delivery: Option, +} + +#[cfg(target_os = "macos")] +impl From for StreamFailure { + fn from(error: String) -> Self { + Self { + error, + delivery: None, + } + } +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] // Stream worker keeps lifecycle and playback policy explicit. +fn run_openai_voice_stream( + app: &AppHandle, + stream_id: &str, + key: String, + active: Arc, + receiver: mpsc::Receiver, + native_voice: NativeVoiceState, + interruption_mode: VoiceInterruptionMode, + interruption_sensitivity: InterruptionSensitivity, + speed: f32, +) -> Result { + let client = client()?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("Could not initialize OpenAI speech runtime: {error}"))?; + let configured_output_device = selected_output_device(); + let player = PocketAudioPlayer::new(TTS_SAMPLE_RATE, 1.0, configured_output_device.as_deref())?; + let output_device = effective_output_device_name(configured_output_device.as_deref()); + let suppress_capture = should_suppress_capture(interruption_mode, output_device.as_deref()); + let output_latency_grace = playback_latency_safety_duration(output_device.as_deref()); + let mut assistant_speech = None::; + let mut playback_drained_at = None::; + let mut pending = String::new(); + let mut delivery = VoiceDeliveryProgress { + sample_rate: TTS_SAMPLE_RATE, + segments: Vec::new(), + }; + let mut started = false; + let mut last_progress = Instant::now(); + + loop { + if started { + player.ensure_healthy().map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; + } + update_openai_assistant_speech( + player.is_empty(), + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + Instant::now(), + ); + if !active.load(Ordering::SeqCst) { + player.stop(); + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Interrupted, + delivery: Some(snapshot_delivery(&delivery, &player)), + }); + } + match receiver.recv_timeout(Duration::from_millis(20)) { + Ok(OpenAiStreamCommand::Append(text)) => { + pending.push_str(&text); + if pending.len() >= 24 && ends_sentence_boundary(&pending) { + speak_pending( + &runtime, + app, + stream_id, + &client, + &key, + &active, + &player, + &mut pending, + &mut delivery, + &mut started, + &native_voice, + interruption_sensitivity, + suppress_capture, + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + speed, + ) + .map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; + } + } + Ok(OpenAiStreamCommand::Flush) => { + speak_pending( + &runtime, + app, + stream_id, + &client, + &key, + &active, + &player, + &mut pending, + &mut delivery, + &mut started, + &native_voice, + interruption_sensitivity, + suppress_capture, + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + speed, + ) + .map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; + } + Ok(OpenAiStreamCommand::Finish) => { + speak_pending( + &runtime, + app, + stream_id, + &client, + &key, + &active, + &player, + &mut pending, + &mut delivery, + &mut started, + &native_voice, + interruption_sensitivity, + suppress_capture, + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + speed, + ) + .map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; + let drain_started = Instant::now(); + while active.load(Ordering::SeqCst) + && (!player.is_empty() || assistant_speech.is_some()) + { + if drain_started.elapsed() >= MAX_FINAL_PLAYBACK_DRAIN { + player.stop(); + return Err(StreamFailure { + error: "OpenAI voice playback did not finish within 10 minutes" + .to_string(), + delivery: Some(snapshot_delivery(&delivery, &player)), + }); + } + player.ensure_healthy().map_err(|error| StreamFailure { + error, + delivery: Some(snapshot_delivery(&delivery, &player)), + })?; + update_openai_assistant_speech( + player.is_empty(), + &mut assistant_speech, + &mut playback_drained_at, + output_latency_grace, + Instant::now(), + ); + if last_progress.elapsed() >= Duration::from_millis(100) { + emit_openai_stream_event( + app, + stream_id, + OpenAiStreamEventState::Progress, + None, + Some(snapshot_delivery(&delivery, &player)), + ); + last_progress = Instant::now(); + } + std::thread::sleep(Duration::from_millis(20)); + } + if !active.load(Ordering::SeqCst) { + player.stop(); + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Interrupted, + delivery: Some(snapshot_delivery(&delivery, &player)), + }); + } + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Completed, + delivery: None, + }); + } + Ok(OpenAiStreamCommand::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => { + active.store(false, Ordering::SeqCst); + player.stop(); + return Ok(StreamOutcome { + state: OpenAiStreamEventState::Interrupted, + delivery: Some(snapshot_delivery(&delivery, &player)), + }); + } + Err(mpsc::RecvTimeoutError::Timeout) => { + if started && last_progress.elapsed() >= Duration::from_millis(100) { + emit_openai_stream_event( + app, + stream_id, + OpenAiStreamEventState::Progress, + None, + Some(snapshot_delivery(&delivery, &player)), + ); + last_progress = Instant::now(); + } + } + } + } +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn speak_pending( + runtime: &tokio::runtime::Runtime, + app: &AppHandle, + stream_id: &str, + client: &reqwest::Client, + key: &str, + active: &AtomicBool, + player: &PocketAudioPlayer, + pending: &mut String, + delivery: &mut VoiceDeliveryProgress, + started: &mut bool, + native_voice: &NativeVoiceState, + interruption_sensitivity: InterruptionSensitivity, + suppress_capture: bool, + assistant_speech: &mut Option, + playback_drained_at: &mut Option, + output_latency_grace: Duration, + speed: f32, +) -> Result<(), String> { + let text = std::mem::take(pending).trim().to_string(); + if text.is_empty() { + return Ok(()); + } + for chunk in chunk_text(&text, MAX_TTS_INPUT_CHARS) { + if !active.load(Ordering::SeqCst) { + return Ok(()); + } + let mut segment_frames = 0_u64; + delivery.segments.push(VoiceDeliverySegment { + text: chunk.to_string(), + played_frames: 0, + total_frames: 0, + synthesis_complete: false, + }); + let Some(mut bytes) = runtime.block_on(openai_speech_stream_cancellable( + client, + key, + chunk.to_string(), + speed, + active, + ))? + else { + return Ok(()); + }; + let mut pcm_remainder = Vec::::new(); + let mut initial_samples = Vec::::new(); + let mut last_network_data = Instant::now(); + loop { + update_openai_assistant_speech( + player.is_empty(), + assistant_speech, + playback_drained_at, + output_latency_grace, + Instant::now(), + ); + if !active.load(Ordering::SeqCst) { + return Ok(()); + } + let item = runtime.block_on(async { + tokio::time::timeout(Duration::from_millis(50), bytes.next()).await + }); + let Some(item) = (match item { + Ok(item) => item, + Err(_) if last_network_data.elapsed() < STREAM_IDLE_TIMEOUT => continue, + Err(_) => return Err("OpenAI speech audio stream timed out".to_string()), + }) else { + break; + }; + let item = + item.map_err(|error| format_openai_request_error("stream speech audio", error))?; + last_network_data = Instant::now(); + if !active.load(Ordering::SeqCst) { + return Ok(()); + } + pcm_remainder.extend_from_slice(&item); + let sample_bytes = pcm_remainder.len() / 2 * 2; + let samples = pcm16le_to_f32(&pcm_remainder[..sample_bytes]); + pcm_remainder.drain(..sample_bytes); + if *started { + assistant_speech.get_or_insert_with(|| { + native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture) + }); + *playback_drained_at = None; + player.enqueue(&samples)?; + } else { + initial_samples.extend_from_slice(&samples); + if initial_samples.len() >= INITIAL_PLAYBACK_BUFFER_FRAMES { + assistant_speech.get_or_insert_with(|| { + native_voice + .begin_assistant_speech(interruption_sensitivity, suppress_capture) + }); + *playback_drained_at = None; + player.enqueue(&initial_samples)?; + initial_samples.clear(); + *started = true; + emit_openai_stream_event( + app, + stream_id, + OpenAiStreamEventState::Started, + None, + None, + ); + } + } + segment_frames = segment_frames.saturating_add(samples.len() as u64); + upsert_delivery_segment(delivery, chunk, segment_frames, false); + } + if !pcm_remainder.is_empty() { + return Err("OpenAI speech returned an incomplete PCM sample".to_string()); + } + if !initial_samples.is_empty() { + assistant_speech.get_or_insert_with(|| { + native_voice.begin_assistant_speech(interruption_sensitivity, suppress_capture) + }); + *playback_drained_at = None; + player.enqueue(&initial_samples)?; + if !*started { + *started = true; + emit_openai_stream_event( + app, + stream_id, + OpenAiStreamEventState::Started, + None, + None, + ); + } + } + upsert_delivery_segment(delivery, chunk, segment_frames, true); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn ends_sentence_boundary(text: &str) -> bool { + if text.trim_end_matches([' ', '\t', '\r']).ends_with('\n') { + return true; + } + let trimmed = text.trim_end(); + trimmed + .trim_end_matches(['"', '\'', '”', '’', ')', ']', '}']) + .ends_with(['.', '!', '?']) +} + +#[cfg(target_os = "macos")] +async fn run_while_active( + future: F, + active: &AtomicBool, +) -> Option { + tokio::pin!(future); + loop { + tokio::select! { + result = &mut future => return Some(result), + _ = tokio::time::sleep(Duration::from_millis(20)) => { + if !active.load(Ordering::SeqCst) { + return None; + } + } + } + } +} + +#[cfg(target_os = "macos")] +async fn openai_speech_stream_cancellable( + client: &reqwest::Client, + key: &str, + input: String, + speed: f32, + active: &AtomicBool, +) -> Result>>, String> +{ + match run_while_active(openai_speech_stream(client, key, input, speed), active).await { + Some(result) => result.map(Some), + None => Ok(None), + } +} + +#[cfg(target_os = "macos")] +async fn openai_speech_stream( + client: &reqwest::Client, + key: &str, + input: String, + speed: f32, +) -> Result>, String> { + let response = client + .post(endpoint("audio/speech")?) + .headers(authorized_headers(key)?) + .header(CONTENT_TYPE, "application/json") + .json(&json!({ + "model": speech_model(), + "voice": speech_voice(), + "input": input, + "speed": speed, + "response_format": "pcm", + "stream_format": "audio" + })) + .send() + .await + .map_err(|error| format_openai_request_error("start speech audio", error))?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(format_openai_response_error( + "start speech audio", + status, + &body, + )); + } + Ok(response.bytes_stream()) +} + +#[cfg(target_os = "macos")] +fn chunk_text(text: &str, max_chars: usize) -> Vec<&str> { + let mut chunks = Vec::new(); + let mut start = 0; + while start < text.len() { + let mut end = (start + max_chars).min(text.len()); + while end > start && !text.is_char_boundary(end) { + end -= 1; + } + if end == start { + end = text.len(); + } + if end < text.len() { + if let Some((offset, _)) = text[start..end] + .char_indices() + .rev() + .find(|(offset, character)| *offset > 0 && character.is_whitespace()) + { + end = start + offset; + } + } + chunks.push(text[start..end].trim()); + start = end; + } + chunks + .into_iter() + .filter(|chunk| !chunk.is_empty()) + .collect() +} + +#[cfg(target_os = "macos")] +fn pcm16le_to_f32(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(2) + .map(|sample| i16::from_le_bytes([sample[0], sample[1]]) as f32 / i16::MAX as f32) + .collect() +} + +#[cfg(any(test, target_os = "macos"))] +fn openai_assistant_speech_grace_elapsed( + playback_drained: bool, + guard_active: bool, + playback_drained_at: &mut Option, + output_latency_grace: Duration, + now: Instant, +) -> bool { + if !guard_active || !playback_drained { + *playback_drained_at = None; + return false; + } + let drained_at = *playback_drained_at.get_or_insert(now); + now.saturating_duration_since(drained_at) >= output_latency_grace +} + +#[cfg(target_os = "macos")] +fn update_openai_assistant_speech( + playback_drained: bool, + assistant_speech: &mut Option, + playback_drained_at: &mut Option, + output_latency_grace: Duration, + now: Instant, +) { + if openai_assistant_speech_grace_elapsed( + playback_drained, + assistant_speech.is_some(), + playback_drained_at, + output_latency_grace, + now, + ) { + assistant_speech.take(); + } +} + +#[cfg(target_os = "macos")] +fn upsert_delivery_segment( + delivery: &mut VoiceDeliveryProgress, + _text: &str, + total_frames: u64, + synthesis_complete: bool, +) { + if let Some(segment) = delivery.segments.last_mut() { + segment.total_frames = total_frames; + segment.synthesis_complete = synthesis_complete; + } +} + +#[cfg(target_os = "macos")] +fn snapshot_delivery( + delivery: &VoiceDeliveryProgress, + player: &PocketAudioPlayer, +) -> VoiceDeliveryProgress { + let mut remaining_played = player.played_frames(); + let segments = delivery + .segments + .iter() + .map(|segment| { + let played_frames = remaining_played.min(segment.total_frames); + remaining_played = remaining_played.saturating_sub(played_frames); + VoiceDeliverySegment { + text: segment.text.clone(), + played_frames, + total_frames: segment.total_frames, + synthesis_complete: segment.synthesis_complete, + } + }) + .collect(); + VoiceDeliveryProgress { + sample_rate: delivery.sample_rate, + segments, + } +} + +#[cfg(target_os = "macos")] +fn emit_openai_stream_event( + app: &AppHandle, + stream_id: &str, + state: OpenAiStreamEventState, + error: Option, + delivery: Option, +) { + let _ = app.emit( + TTS_EVENT, + OpenAiVoiceStreamEvent { + stream_id: stream_id.to_string(), + state, + error, + delivery, + }, + ); +} + +#[cfg(target_os = "macos")] +fn format_openai_request_error(action: &str, error: reqwest::Error) -> String { + if error.is_timeout() { + format!("OpenAI voice could not {action}: the request timed out") + } else if error.is_connect() { + format!("OpenAI voice could not {action}: check your network connection") + } else { + format!("OpenAI voice could not {action}: {error}") + } +} + +#[cfg(target_os = "macos")] +fn format_openai_response_error(action: &str, status: reqwest::StatusCode, body: &str) -> String { + let preview: String = body.chars().take(500).collect(); + format!("OpenAI voice could not {action}: HTTP {status}: {preview}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn destroyed_window_only_stops_its_openai_stream() { + let state = OpenAiVoiceState::default(); + let active = Arc::new(AtomicBool::new(true)); + let (sender, _receiver) = mpsc::channel(); + { + let mut playback = state.playback.lock().expect("playback state"); + playback.active = Some(active.clone()); + playback.stream = Some(ActiveOpenAiStream { + id: "stream-1".to_string(), + owner_window: "session-window".to_string(), + sender, + }); + } + + assert!(!state.stop_for_window_destroyed("other-window")); + assert!(active.load(Ordering::SeqCst)); + assert!(state.stop_for_window_destroyed("session-window")); + assert!(!active.load(Ordering::SeqCst)); + } + + #[test] + fn openai_host_configuration_resolves_to_the_v1_api_root() { + assert_eq!( + normalize_openai_base_url("https://proxy.example".to_string(), true).unwrap(), + "https://proxy.example/v1" + ); + assert_eq!( + normalize_openai_base_url("https://proxy.example/v1/".to_string(), true).unwrap(), + "https://proxy.example/v1" + ); + } + + #[test] + fn openai_voice_endpoints_require_https() { + assert_eq!( + normalize_openai_base_url("http://proxy.example".to_string(), true) + .expect_err("plaintext endpoint must be rejected"), + "OpenAI voice endpoint must use HTTPS" + ); + } + + #[test] + fn openai_base_url_preserves_custom_paths_and_query_parameters() { + assert_eq!( + normalize_openai_base_url("https://proxy.example".to_string(), false).unwrap(), + "https://proxy.example/v1" + ); + let base = normalize_openai_base_url( + "https://proxy.example/openai?api-version=2026-01-01".to_string(), + false, + ) + .unwrap(); + assert_eq!( + endpoint_for_base_url(&base, "audio/speech").unwrap(), + "https://proxy.example/openai/audio/speech?api-version=2026-01-01" + ); + } + + #[test] + fn capture_suppression_ends_after_playback_drain_grace() { + let started = Instant::now(); + let mut drained_at = None; + let grace = Duration::from_millis(100); + + assert!(!openai_assistant_speech_grace_elapsed( + true, + true, + &mut drained_at, + grace, + started, + )); + assert!(openai_assistant_speech_grace_elapsed( + true, + true, + &mut drained_at, + grace, + started + grace, + )); + assert!(!openai_assistant_speech_grace_elapsed( + false, + true, + &mut drained_at, + grace, + started + grace, + )); + assert_eq!(drained_at, None); + } + + #[cfg(target_os = "macos")] + #[test] + fn chunks_tts_text_on_char_boundaries() { + assert_eq!(chunk_text("hello", 10), vec!["hello"]); + assert_eq!(chunk_text("ééé", 3), vec!["é", "é", "é"]); + assert_eq!( + chunk_text("hello wide world", 8), + vec!["hello", "wide", "world"] + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn recognizes_sentence_boundaries_before_streaming() { + assert!(ends_sentence_boundary("Hello world.\n")); + assert!(ends_sentence_boundary("Did it work?” ")); + assert!(ends_sentence_boundary("It did!)")); + assert!(!ends_sentence_boundary("Still speaking,")); + } + + #[cfg(target_os = "macos")] + #[test] + fn cancels_a_stalled_speech_request() { + let active = Arc::new(AtomicBool::new(true)); + let active_for_thread = active.clone(); + let cancellation = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(30)); + active_for_thread.store(false, Ordering::SeqCst); + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("runtime"); + + let result = runtime.block_on(run_while_active( + std::future::pending::<()>(), + active.as_ref(), + )); + + cancellation.join().expect("cancellation thread"); + assert_eq!(result, None); + } +} diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index 5e1a21c86..68228d50b 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -514,7 +514,7 @@ fn playback_speed(base: &Path) -> f32 { settings(base).playback_speed.clamp(0.75, 2.0) } -fn selected_output_device() -> Option { +pub(crate) fn selected_output_device() -> Option { std::env::var("VOICE_CONVERSATION_OUTPUT_DEVICE") .ok() .filter(|value| !value.is_empty()) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 87355e496..2e41e9a23 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -233,6 +233,7 @@ pub fn run() { app.manage(commands::model_setup::ModelSetupRegistry::default()); app.manage(commands::pocket_voice::PocketVoiceState::default()); app.manage(commands::siri_voice::SiriVoiceState::default()); + app.manage(commands::openai_audio::OpenAiVoiceState::default()); app.manage(commands::native_voice::NativeVoiceState::default()); app.manage(commands::voice_capture::VoiceCaptureState::default()); app.manage(commands::telemetry::TelemetryAuthState::new( @@ -654,6 +655,15 @@ pub fn run() { commands::pocket_voice::finish_pocket_voice_stream, commands::pocket_voice::stop_pocket_voice, commands::pocket_voice::remove_voice_model, + commands::openai_audio::get_openai_voice_status, + commands::openai_audio::set_openai_tts_api_key, + commands::openai_audio::clear_openai_tts_api_key, + commands::openai_audio::start_openai_voice_stream, + commands::openai_audio::append_openai_voice_stream, + commands::openai_audio::flush_openai_voice_stream, + commands::openai_audio::finish_openai_voice_stream, + commands::openai_audio::stop_openai_voice, + commands::openai_audio::set_openai_playback_speed, commands::siri_voice::get_siri_voice_status, commands::siri_voice::select_siri_voice, commands::siri_voice::download_siri_voice, diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index b83e323e8..2face1bbf 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -236,6 +236,7 @@ import { } from "@/features/voice-conversation/api/voiceConversation"; import { usePocketVoiceSetup } from "@/features/voice-conversation/hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "@/features/voice-conversation/hooks/useMacSpeechSetup"; +import { useOpenAiVoiceSetup } from "@/features/voice-conversation/hooks/useOpenAiVoiceSetup"; import { useSiriVoiceSetup } from "@/features/voice-conversation/hooks/useSiriVoiceSetup"; import { isMacSpeechAvailable, @@ -743,6 +744,9 @@ export function AppShell({ ), ); const globalVoiceOutput = useVoiceOutputPreference(); + const globalOpenAiVoiceSetup = useOpenAiVoiceSetup( + capabilities.voiceConversation && globalVoiceOutput.backend === "openai", + ); const globalSiriVoiceSetup = useSiriVoiceSetup( capabilities.voiceConversation && globalVoiceOutput.backend === "siri", ); @@ -752,6 +756,7 @@ export function AppShell({ globalSiriVoiceSetup.status, globalVoiceInput.backend, globalVoiceOutput.backend, + globalOpenAiVoiceSetup.status, ); const voiceConversationWasEnabledRef = useRef(capabilities.voiceConversation); useEffect(() => { diff --git a/src/features/chat/ui/ChatView.tsx b/src/features/chat/ui/ChatView.tsx index a7d643ea2..883db84c0 100644 --- a/src/features/chat/ui/ChatView.tsx +++ b/src/features/chat/ui/ChatView.tsx @@ -65,6 +65,7 @@ import type { GlobalComposerHandoffRect } from "@/shared/ui/GlobalComposerPill"; import { useVoiceConversationController } from "@/features/voice-conversation/hooks/useVoiceConversationController"; import { usePocketVoiceSetup } from "@/features/voice-conversation/hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "@/features/voice-conversation/hooks/useMacSpeechSetup"; +import { useOpenAiVoiceSetup } from "@/features/voice-conversation/hooks/useOpenAiVoiceSetup"; import { useSiriVoiceSetup } from "@/features/voice-conversation/hooks/useSiriVoiceSetup"; import { isMacSpeechAvailable, @@ -234,6 +235,9 @@ export function ChatView({ isMacSpeechAvailable(macSpeechSetup.status, macSpeechSetup.loading), ); const voiceOutput = useVoiceOutputPreference(); + const openAiVoiceSetup = useOpenAiVoiceSetup( + capabilities.voiceConversation && voiceOutput.backend === "openai", + ); const siriVoiceSetup = useSiriVoiceSetup( capabilities.voiceConversation && voiceOutput.backend === "siri", ); @@ -243,6 +247,7 @@ export function ChatView({ siriVoiceSetup.status, voiceInput.backend, voiceOutput.backend, + openAiVoiceSetup.status, ); const voiceConversation = useVoiceConversationController({ sessionId, diff --git a/src/features/providers/api/credentials.test.ts b/src/features/providers/api/credentials.test.ts index 39b30c508..a5e6ce942 100644 --- a/src/features/providers/api/credentials.test.ts +++ b/src/features/providers/api/credentials.test.ts @@ -19,7 +19,6 @@ const mocks = vi.hoisted(() => ({ vi.mock("@/shared/api/acpConnection", () => ({ getClient: () => mocks.getClient(), })); - describe("provider credential API", () => { beforeEach(() => { vi.clearAllMocks(); diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts new file mode 100644 index 000000000..45bf46b33 --- /dev/null +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -0,0 +1,84 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen, type UnlistenFn } from "@tauri-apps/api/event"; +import type { VoiceDeliveryProgress } from "./pocketVoice"; +import type { + VoiceInterruptionMode, + VoiceInterruptionSensitivity, +} from "../lib/voiceInterruptionPreference"; + +export interface OpenAiVoiceStatus { + ttsConfigured: boolean; + speechModel: string; + speechVoice: string; + playbackSpeed: number; + ttsAvailable: boolean; + unavailableReason: "missingApiKey" | "unsupportedPlatform" | null; +} + +export interface OpenAiVoiceStreamEvent { + streamId: string; + state: "started" | "progress" | "completed" | "interrupted" | "failed"; + error: string | null; + delivery?: VoiceDeliveryProgress | null; +} + +export function getOpenAiVoiceStatus(): Promise { + return invoke("get_openai_voice_status"); +} + +export function setOpenAiTtsApiKey(apiKey: string): Promise { + return invoke("set_openai_tts_api_key", { apiKey }); +} + +export function clearOpenAiTtsApiKey(): Promise { + return invoke("clear_openai_tts_api_key"); +} + +export function listenToOpenAiVoiceSettings( + onChanged: () => void, +): Promise { + return listen("openai-voice:settings-changed", onChanged); +} + +export function startOpenAiVoiceStream( + streamId: string, + interruptionMode: VoiceInterruptionMode, + interruptionSensitivity: VoiceInterruptionSensitivity, +): Promise { + return invoke("start_openai_voice_stream", { + streamId, + interruptionMode, + interruptionSensitivity, + }); +} + +export function appendOpenAiVoiceStream( + streamId: string, + text: string, +): Promise { + return invoke("append_openai_voice_stream", { streamId, text }); +} + +export function flushOpenAiVoiceStream(streamId: string): Promise { + return invoke("flush_openai_voice_stream", { streamId }); +} + +export function finishOpenAiVoiceStream(streamId: string): Promise { + return invoke("finish_openai_voice_stream", { streamId }); +} + +export function stopOpenAiVoice(): Promise { + return invoke("stop_openai_voice"); +} + +export function setOpenAiPlaybackSpeed(speed: number): Promise { + return invoke("set_openai_playback_speed", { speed }); +} + +export function listenToOpenAiVoiceStream( + onEvent: (event: OpenAiVoiceStreamEvent) => void, +): Promise { + return listen("openai-voice:stream-event", (event) => + onEvent(event.payload), + ); +} diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx new file mode 100644 index 000000000..c1a87c9cb --- /dev/null +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -0,0 +1,135 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenAiVoiceStatus } from "../api/openAiVoice"; +import { useOpenAiVoiceSetup } from "./useOpenAiVoiceSetup"; + +const mocks = vi.hoisted(() => ({ + getStatus: vi.fn<() => Promise>(), + settingsChanged: null as (() => void) | null, + finishListening: null as (() => void) | null, + listenerError: null as Error | null, +})); + +vi.mock("../api/openAiVoice", () => ({ + getOpenAiVoiceStatus: () => mocks.getStatus(), + listenToOpenAiVoiceSettings: (listener: () => void) => { + mocks.settingsChanged = listener; + if (mocks.listenerError) return Promise.reject(mocks.listenerError); + return new Promise<() => void>((resolve) => { + mocks.finishListening = () => resolve(() => undefined); + }); + }, +})); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((next, fail) => { + resolve = next; + reject = fail; + }); + return { promise, reject, resolve }; +} + +function status(configured: boolean): OpenAiVoiceStatus { + return { + ttsConfigured: configured, + speechModel: "gpt-4o-mini-tts", + speechVoice: "marin", + playbackSpeed: 1, + ttsAvailable: true, + unavailableReason: configured ? null : "missingApiKey", + }; +} + +describe("useOpenAiVoiceSetup", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.settingsChanged = null; + mocks.finishListening = null; + mocks.listenerError = null; + }); + + it("keeps the latest credential refresh when responses resolve out of order", async () => { + const initial = deferred(); + const refreshed = deferred(); + mocks.getStatus + .mockReturnValueOnce(initial.promise) + .mockReturnValueOnce(refreshed.promise); + const { result } = renderHook(() => useOpenAiVoiceSetup()); + await waitFor(() => expect(mocks.settingsChanged).not.toBeNull()); + act(() => mocks.finishListening?.()); + await waitFor(() => expect(mocks.getStatus).toHaveBeenCalledTimes(1)); + + act(() => mocks.settingsChanged?.()); + refreshed.resolve(status(true)); + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); + + initial.resolve(status(false)); + await act(async () => Promise.resolve()); + + expect(result.current.status?.ttsConfigured).toBe(true); + }); + + it("refreshes after listener registration captures credential changes", async () => { + mocks.getStatus.mockResolvedValue(status(true)); + const { result } = renderHook(() => useOpenAiVoiceSetup()); + + await waitFor(() => expect(mocks.finishListening).not.toBeNull()); + expect(mocks.getStatus).not.toHaveBeenCalled(); + + act(() => mocks.finishListening?.()); + + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); + }); + + it("still loads status when listener registration fails", async () => { + mocks.listenerError = new Error("listener unavailable"); + mocks.getStatus.mockResolvedValue(status(true)); + + const { result } = renderHook(() => useOpenAiVoiceSetup()); + + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); + }); + + it("clears stale readiness when a credential refresh fails", async () => { + const refresh = deferred(); + mocks.getStatus + .mockResolvedValueOnce(status(true)) + .mockReturnValueOnce(refresh.promise); + const { result } = renderHook(() => useOpenAiVoiceSetup()); + await waitFor(() => expect(mocks.finishListening).not.toBeNull()); + act(() => mocks.finishListening?.()); + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); + + act(() => mocks.settingsChanged?.()); + refresh.reject(new Error("Keychain unavailable")); + + await waitFor(() => expect(result.current.status).toBeNull()); + expect(result.current.error).toBe("Keychain unavailable"); + }); + + it("does not expose cached readiness while disabled", async () => { + mocks.listenerError = new Error("listener unavailable"); + mocks.getStatus.mockResolvedValue(status(true)); + const { result, rerender } = renderHook( + ({ enabled }) => useOpenAiVoiceSetup(enabled), + { initialProps: { enabled: true } }, + ); + await waitFor(() => + expect(result.current.status?.ttsConfigured).toBe(true), + ); + + rerender({ enabled: false }); + + expect(result.current).toEqual({ status: null, error: null }); + }); +}); diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts new file mode 100644 index 000000000..f53d75090 --- /dev/null +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.ts @@ -0,0 +1,55 @@ +import { useEffect, useState } from "react"; +import { + getOpenAiVoiceStatus, + listenToOpenAiVoiceSettings, + type OpenAiVoiceStatus, +} from "../api/openAiVoice"; + +export function useOpenAiVoiceSetup(enabled = true) { + const [status, setStatus] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + if (!enabled) return; + let active = true; + let refreshGeneration = 0; + let unsubscribe: (() => void) | null = null; + const refresh = () => { + const generation = ++refreshGeneration; + void getOpenAiVoiceStatus().then( + (next) => { + if (active && generation === refreshGeneration) { + setStatus(next); + setError(null); + } + }, + (cause) => { + if (active && generation === refreshGeneration) { + setStatus(null); + setError(cause instanceof Error ? cause.message : String(cause)); + } + }, + ); + }; + void listenToOpenAiVoiceSettings(refresh).then( + (nextUnsubscribe) => { + if (active) { + unsubscribe = nextUnsubscribe; + refresh(); + } else nextUnsubscribe(); + }, + () => { + if (active) refresh(); + }, + ); + return () => { + active = false; + unsubscribe?.(); + }; + }, [enabled]); + + return { + status: enabled ? status : null, + error: enabled ? error : null, + }; +} diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index cf94cb821..fbd386338 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -772,11 +772,11 @@ export function useVoiceConversationController({ const onFailure = (text: string, playbackError: unknown) => { addErrorNotification( sessionId, - `Pocket TTS could not speak the assistant response: ${errorText( + `Voice playback could not speak the assistant response: ${errorText( playbackError, )}`, ); - console.error("Native Pocket playback failed", { + console.error("Native voice playback failed", { sessionId, textLength: text.length, error: playbackError, diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts index 04e395325..8a04b6b27 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.test.ts @@ -5,7 +5,7 @@ import { useVoiceConversationStore } from "../stores/voiceConversationStore"; import type { PocketVoiceStreamEvent } from "../api/pocketVoice"; const mocks = vi.hoisted(() => ({ - backend: "pocket" as "pocket" | "siri", + backend: "pocket" as "pocket" | "siri" | "openai", interruptionMode: "automatic" as | "automatic" | "allowInterruptions" @@ -51,6 +51,22 @@ const mocks = vi.hoisted(() => ({ siriFinish: vi.fn<(streamId: string) => Promise>(), siriStop: vi.fn<() => Promise>(), siriStreamHandler: null as ((event: PocketVoiceStreamEvent) => void) | null, + openAiStart: + vi.fn< + ( + streamId: string, + interruptionMode: + | "automatic" + | "allowInterruptions" + | "preventFeedback", + interruptionSensitivity: "less" | "balanced" | "more", + ) => Promise + >(), + openAiAppend: vi.fn<(streamId: string, text: string) => Promise>(), + openAiFlush: vi.fn<(streamId: string) => Promise>(), + openAiFinish: vi.fn<(streamId: string) => Promise>(), + openAiStop: vi.fn<() => Promise>(), + openAiStreamHandler: null as ((event: PocketVoiceStreamEvent) => void) | null, })); vi.mock("../api/voiceConversation", () => ({ setVoiceConversationAssistantSpeaking: mocks.setAssistantSpeaking, @@ -75,6 +91,25 @@ vi.mock("../api/pocketVoice", () => ({ }, })); +vi.mock("../api/openAiVoice", () => ({ + startOpenAiVoiceStream: ( + streamId: string, + interruptionMode: typeof mocks.interruptionMode, + interruptionSensitivity: "less" | "balanced" | "more", + ) => mocks.openAiStart(streamId, interruptionMode, interruptionSensitivity), + appendOpenAiVoiceStream: (streamId: string, text: string) => + mocks.openAiAppend(streamId, text), + flushOpenAiVoiceStream: (streamId: string) => mocks.openAiFlush(streamId), + finishOpenAiVoiceStream: (streamId: string) => mocks.openAiFinish(streamId), + stopOpenAiVoice: () => mocks.openAiStop(), + listenToOpenAiVoiceStream: async ( + handler: (event: PocketVoiceStreamEvent) => void, + ) => { + mocks.openAiStreamHandler = handler; + return vi.fn(); + }, +})); + vi.mock("../api/siriVoice", () => ({ startSiriVoiceStream: ( streamId: string, @@ -181,6 +216,12 @@ describe("native assistant speech stream", () => { mocks.siriFinish.mockReset().mockResolvedValue(); mocks.siriStop.mockReset().mockResolvedValue(true); mocks.siriStreamHandler = null; + mocks.openAiStart.mockReset().mockResolvedValue(); + mocks.openAiAppend.mockReset().mockResolvedValue(); + mocks.openAiFlush.mockReset().mockResolvedValue(); + mocks.openAiFinish.mockReset().mockResolvedValue(); + mocks.openAiStop.mockReset().mockResolvedValue(true); + mocks.openAiStreamHandler = null; useChatStore.setState({ messagesBySession: {}, sessionStateById: {}, @@ -242,6 +283,34 @@ describe("native assistant speech stream", () => { }); }); + it("routes ordering and cancellation through OpenAI when selected", async () => { + mocks.backend = "openai"; + startNativeAssistantSpeech("session-1", vi.fn()); + useChatStore + .getState() + .setMessages("session-1", [ + assistant([{ type: "text", text: "Hello from OpenAI." }], "completed"), + ]); + + await vi.waitFor(() => { + expect(mocks.openAiStart).toHaveBeenCalledWith( + expect.any(String), + "automatic", + "less", + ); + expect(mocks.openAiAppend).toHaveBeenCalledWith( + mocks.openAiStart.mock.calls[0]?.[0], + "Hello from OpenAI.", + ); + expect(mocks.openAiFinish).toHaveBeenCalledTimes(1); + }); + expect(mocks.openAiStart.mock.invocationCallOrder[0]).toBeLessThan( + mocks.openAiAppend.mock.invocationCallOrder[0] ?? 0, + ); + stopNativeAssistantSpeech(); + await vi.waitFor(() => expect(mocks.openAiStop).toHaveBeenCalled()); + }); + it("routes the complete utterance stream through Siri when selected", async () => { mocks.backend = "siri"; mocks.interruptionMode = "allowInterruptions"; diff --git a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts index f374b07b6..ba5c44c98 100644 --- a/src/features/voice-conversation/lib/nativeAssistantSpeech.ts +++ b/src/features/voice-conversation/lib/nativeAssistantSpeech.ts @@ -14,6 +14,15 @@ import { type VoiceDeliveryProgress, type PocketVoiceStreamEvent, } from "../api/pocketVoice"; +import { + appendOpenAiVoiceStream, + finishOpenAiVoiceStream, + flushOpenAiVoiceStream, + listenToOpenAiVoiceStream, + startOpenAiVoiceStream, + stopOpenAiVoice, + type OpenAiVoiceStreamEvent, +} from "../api/openAiVoice"; import { appendSiriVoiceStream, finishSiriVoiceStream, @@ -671,7 +680,7 @@ function queueStreamCommand( } function handleStreamEvent( - event: PocketVoiceStreamEvent | SiriVoiceStreamEvent, + event: PocketVoiceStreamEvent | SiriVoiceStreamEvent | OpenAiVoiceStreamEvent, ) { const utterance = activeUtterance; if (!utterance || utterance.id !== event.streamId) return; @@ -865,8 +874,9 @@ export function startNativeAssistantSpeech( activeSpeechSessionId = sessionId; activeSpeechRevision = useVoiceConversationStore.getState().status.revision; const activeGeneration = generation; + const outputBackend = getVoiceOutputBackend(); const streamBackend = - getVoiceOutputBackend() === "siri" + outputBackend === "siri" ? { start: ( streamId: string, @@ -891,14 +901,23 @@ export function startNativeAssistantSpeech( stop: stopSiriVoice, listen: listenToSiriVoiceStream, } - : { - start: startPocketVoiceStream, - append: appendPocketVoiceStream, - flush: flushPocketVoiceStream, - finish: finishPocketVoiceStream, - stop: stopPocketVoice, - listen: listenToPocketVoiceStream, - }; + : outputBackend === "openai" + ? { + start: startOpenAiVoiceStream, + append: appendOpenAiVoiceStream, + flush: flushOpenAiVoiceStream, + finish: finishOpenAiVoiceStream, + stop: stopOpenAiVoice, + listen: listenToOpenAiVoiceStream, + } + : { + start: startPocketVoiceStream, + append: appendPocketVoiceStream, + flush: flushPocketVoiceStream, + finish: finishPocketVoiceStream, + stop: stopPocketVoice, + listen: listenToPocketVoiceStream, + }; stopActiveVoice = streamBackend.stop; streamListenerReady = streamBackend .listen(handleStreamEvent) diff --git a/src/features/voice-conversation/lib/voiceOutputPreference.ts b/src/features/voice-conversation/lib/voiceOutputPreference.ts index 061ad75a7..f3a575a9c 100644 --- a/src/features/voice-conversation/lib/voiceOutputPreference.ts +++ b/src/features/voice-conversation/lib/voiceOutputPreference.ts @@ -1,7 +1,7 @@ import { useCallback, useSyncExternalStore } from "react"; import { getPlatform } from "@/shared/lib/platform"; -export type VoiceOutputBackend = "pocket" | "siri"; +export type VoiceOutputBackend = "pocket" | "siri" | "openai"; const STORAGE_KEY = "goose:voice-output-backend"; const CHANGED_EVENT = "goose:voice-output-backend-changed"; @@ -14,6 +14,7 @@ function normalize(value: unknown): VoiceOutputBackend { if (value === "siri") { return getPlatform() === "mac" ? "siri" : "pocket"; } + if (value === "openai") return "openai"; return value === "pocket" ? value : getDefaultVoiceOutputBackend(); } diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts index bcafc3dc3..6a35f946e 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts @@ -53,6 +53,16 @@ describe("voice setup readiness", () => { ).toBe(false); }); + it("requires the dedicated OpenAI text-to-speech key for OpenAI output", () => { + const configured = { ttsConfigured: true, ttsAvailable: true } as never; + expect( + isVoiceSetupReady(pocket, null, null, "parakeet", "openai", configured), + ).toBe(true); + expect(isVoiceSetupReady(pocket, null, null, "parakeet", "openai")).toBe( + false, + ); + }); + it("uses native macOS speech readiness instead of Parakeet when selected", () => { expect( isVoiceSetupReady( diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.ts index f8f11226c..196c853d9 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.ts @@ -2,6 +2,7 @@ import type { PocketVoiceStatus } from "../api/pocketVoice"; import type { SiriVoiceStatus } from "../api/siriVoice"; import type { MacSpeechStatus } from "../api/macSpeech"; import type { VoiceInputBackend } from "./voiceInputPreference"; +import type { OpenAiVoiceStatus } from "../api/openAiVoice"; import type { VoiceOutputBackend } from "./voiceOutputPreference"; export function isVoiceSetupReady( @@ -10,6 +11,7 @@ export function isVoiceSetupReady( siri: SiriVoiceStatus | null, inputBackend: VoiceInputBackend | null, outputBackend: VoiceOutputBackend, + openAi: OpenAiVoiceStatus | null = null, ): boolean { if (inputBackend === null) return false; const inputReady = @@ -21,6 +23,8 @@ export function isVoiceSetupReady( ) : Boolean(pocket?.parakeetInstalled); if (!inputReady) return false; + if (outputBackend === "openai") + return Boolean(openAi?.ttsConfigured && openAi.ttsAvailable); if (outputBackend === "pocket") return Boolean(pocket?.pocketInstalled); return Boolean( siri?.supported && siri.selectedVoice && siri.selectedVoiceInstalled, diff --git a/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx b/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx new file mode 100644 index 000000000..d39830e89 --- /dev/null +++ b/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx @@ -0,0 +1,98 @@ +import { useId, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/shared/ui/button"; +import { Input } from "@/shared/ui/input"; + +interface OpenAiApiKeyFieldProps { + label: string; + configured: boolean; + onSave: (apiKey: string) => Promise; + onClear: () => Promise; +} + +export function OpenAiApiKeyField({ + label, + configured, + onSave, + onClear, +}: OpenAiApiKeyFieldProps) { + const { t } = useTranslation("settings"); + const inputId = useId(); + const [apiKey, setApiKey] = useState(""); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + + const save = async () => { + setSaving(true); + setError(null); + try { + await onSave(apiKey); + setApiKey(""); + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setSaving(false); + } + }; + + const clear = async () => { + setSaving(true); + setError(null); + try { + await onClear(); + setApiKey(""); + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setSaving(false); + } + }; + + return ( +
+ +
+ setApiKey(event.target.value)} + placeholder={configured ? t("voice.openAiApiKeySaved") : "sk-…"} + autoComplete="off" + spellCheck={false} + /> + + {configured ? ( + + ) : null} +
+

+ {configured + ? t("voice.openAiApiKeyConfigured") + : t("voice.openAiApiKeyNotConfigured")} +

+ {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 1c772f918..df674ba9d 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -50,7 +50,32 @@ const microphonePermissionState = vi.hoisted(() => ({ openSettingsError: false, openSettings: vi.fn(), })); +const openAiStatusState = vi.hoisted(() => ({ + current: { + ttsConfigured: true, + speechModel: "gpt-4o-mini-tts", + speechVoice: "marin", + playbackSpeed: 1, + ttsAvailable: true, + unavailableReason: null as string | null, + }, +})); +const openAiApiMocks = vi.hoisted(() => ({ + setTtsApiKey: vi.fn(() => Promise.resolve()), + clearTtsApiKey: vi.fn(() => Promise.resolve()), +})); +vi.mock("../api/openAiVoice", () => ({ + setOpenAiPlaybackSpeed: vi.fn(() => Promise.resolve()), + setOpenAiTtsApiKey: openAiApiMocks.setTtsApiKey, + clearOpenAiTtsApiKey: openAiApiMocks.clearTtsApiKey, +})); +vi.mock("../hooks/useOpenAiVoiceSetup", () => ({ + useOpenAiVoiceSetup: () => ({ + status: openAiStatusState.current, + error: null, + }), +})); vi.mock("../hooks/usePocketVoiceSetup", () => ({ usePocketVoiceSetup: () => setupState.current, })); @@ -191,6 +216,63 @@ describe("VoiceSettings", () => { }; interruptionState.mode = "automatic"; siriSetupState.current = siriSetup(); + openAiStatusState.current = { + ttsConfigured: true, + speechModel: "gpt-4o-mini-tts", + speechVoice: "marin", + playbackSpeed: 1, + ttsAvailable: true, + unavailableReason: null, + }; + openAiApiMocks.setTtsApiKey.mockClear(); + openAiApiMocks.clearTtsApiKey.mockClear(); + }); + + it("renders selected OpenAI output settings", async () => { + outputState.backend = "openai"; + setupState.current = setup(pocketStatus()); + renderWithProviders(); + + expect( + await screen.findByText(/gpt-4o-mini-tts.*marin voice/), + ).toBeInTheDocument(); + expect(screen.getByText("Playback speed")).toBeInTheDocument(); + }); + + it("saves a dedicated OpenAI text-to-speech API key", async () => { + outputState.backend = "openai"; + setupState.current = setup(pocketStatus({ parakeetInstalled: true })); + renderWithProviders(); + + const user = userEvent.setup(); + await user.type( + screen.getByLabelText("OpenAI text-to-speech API key"), + "tts-secret", + ); + await user.click(screen.getByRole("button", { name: "Save key" })); + + expect(openAiApiMocks.setTtsApiKey).toHaveBeenCalledWith("tts-secret"); + }); + + it("uses OpenAI guidance when the selected OpenAI output is not ready", async () => { + outputState.backend = "openai"; + openAiStatusState.current = { + ...openAiStatusState.current, + ttsConfigured: false, + unavailableReason: + "Add an OpenAI text-to-speech API key in Voice settings.", + }; + setupState.current = setup(pocketStatus({ parakeetInstalled: true })); + renderWithProviders(); + + expect( + await screen.findByText( + "OpenAI voice is not ready. Add the required API key below, then try again.", + ), + ).toBeInTheDocument(); + expect( + screen.queryByText(/Pocket TTS is not installed/), + ).not.toBeInTheDocument(); }); it("shows interruption modes without VAD controls", () => { diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index f7322143e..f6239f4a5 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -14,6 +14,12 @@ import { SelectTrigger, SelectValue, } from "@/shared/ui/select"; +import { useEffect, useState } from "react"; +import { + clearOpenAiTtsApiKey, + setOpenAiPlaybackSpeed, + setOpenAiTtsApiKey, +} from "../api/openAiVoice"; import { usePocketVoiceSetup } from "../hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "../hooks/useMacSpeechSetup"; import { useMicrophonePermission } from "../hooks/useMicrophonePermission"; @@ -30,6 +36,9 @@ import { useVoiceOutputPreference } from "../lib/voiceOutputPreference"; import { PocketVoiceSetupContent } from "./PocketVoiceSetupContent"; import { MacSpeechSettings } from "./MacSpeechSettings"; import { SiriVoiceSettings } from "./SiriVoiceSettings"; +import { PlaybackSpeedRow } from "./PlaybackSpeedRow"; +import { useOpenAiVoiceSetup } from "../hooks/useOpenAiVoiceSetup"; +import { OpenAiApiKeyField } from "./OpenAiApiKeyField"; const INTERRUPTION_MODES: VoiceInterruptionMode[] = [ "automatic", @@ -45,6 +54,11 @@ function readinessDescriptionKey( ): string | null { if (inputReady && outputReady) return null; if (!inputReady && !outputReady) { + if (backend === "openai") { + return inputBackend === "macos" + ? "voice.notReadyMacInputAndOpenAiOutput" + : "voice.notReadyInputAndOpenAiOutput"; + } if (inputBackend === "macos") { return backend === "siri" ? "voice.notReadyMacInputAndSiriOutput" @@ -59,6 +73,7 @@ function readinessDescriptionKey( ? "voice.notReadyMacInput" : "voice.notReadyInput"; } + if (backend === "openai") return "voice.notReadyOpenAi"; return backend === "siri" ? "voice.notReadySiriOutput" : "voice.notReadyPocketOutput"; @@ -68,6 +83,12 @@ export function VoiceSettings() { const { t } = useTranslation("settings"); const setup = usePocketVoiceSetup(); const macSpeechSetup = useMacSpeechSetup(); + const { status: openAiStatus, error: openAiError } = useOpenAiVoiceSetup(); + const [openAiSpeed, setOpenAiSpeed] = useState(1); + const [openAiSpeedError, setOpenAiSpeedError] = useState(null); + useEffect(() => { + if (openAiStatus) setOpenAiSpeed(openAiStatus.playbackSpeed); + }, [openAiStatus]); const input = useVoiceInputPreference( isMacSpeechAvailable(macSpeechSetup.status, macSpeechSetup.loading), ); @@ -91,13 +112,15 @@ export function VoiceSettings() { ) : (setup.status?.parakeetInstalled ?? false); const outputReady = - output.backend === "siri" - ? Boolean( - siriSetup.status?.supported && - siriSetup.status.selectedVoice && - siriSetup.status.selectedVoiceInstalled, - ) - : (setup.status?.pocketInstalled ?? false); + output.backend === "openai" + ? Boolean(openAiStatus?.ttsConfigured && openAiStatus.ttsAvailable) + : output.backend === "siri" + ? Boolean( + siriSetup.status?.supported && + siriSetup.status.selectedVoice && + siriSetup.status.selectedVoiceInstalled, + ) + : (setup.status?.pocketInstalled ?? false); const siriOutputLoaded = siriSetup.status !== null && siriSetup.statusError === null; const pocketStatusLoaded = @@ -230,6 +253,11 @@ export function VoiceSettings() { {t("voice.backendPocket")} + {openAiStatus?.ttsAvailable ? ( + + {t("voice.backendOpenAiTts")} + + ) : null} {siriSupported ? ( {t("voice.backendSiri")} ) : null} @@ -237,7 +265,49 @@ export function VoiceSettings() { )} details={ - output.backend === "siri" ? ( + output.backend === "openai" ? ( +
+ +

+ {openAiError ?? + (openAiStatus?.unavailableReason === "unsupportedPlatform" + ? t("voice.openAiTtsUnsupportedPlatform") + : openAiStatus?.unavailableReason === "missingApiKey" + ? t("voice.openAiTtsNeedsKey") + : openAiStatus + ? t("voice.openAiTtsConfigured", { + model: openAiStatus.speechModel, + voice: openAiStatus.speechVoice, + }) + : t("voice.openAiChecking"))} +

+ { + setOpenAiSpeedError(null); + try { + await setOpenAiPlaybackSpeed(speed); + setOpenAiSpeed(speed); + } catch (cause) { + setOpenAiSpeedError( + cause instanceof Error ? cause.message : String(cause), + ); + } + }} + /> + {openAiSpeedError ? ( +

+ {openAiSpeedError} +

+ ) : null} +
+ ) : output.backend === "siri" ? ( ) : ( diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index b80eba39c..47f61ac63 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -870,6 +870,7 @@ }, "voice": { "backendMacSpeech": "Apple speech recognition", + "backendOpenAiTts": "OpenAI text-to-speech", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", "backendSiri": "Siri voices", @@ -912,11 +913,14 @@ "modelMissingSize": "Not installed · {{size}} download", "modelNotInstalled": "Not installed", "notReadyInput": "Parakeet STT is not installed. Download it below to use Voice Conversation.", + "notReadyInputAndOpenAiOutput": "Parakeet STT is not installed, and the OpenAI text-to-speech key is missing. Complete both steps below to use Voice Conversation.", "notReadyInputAndPocketOutput": "Parakeet STT and Pocket TTS are not installed. Download both below to use Voice Conversation.", "notReadyInputAndSiriOutput": "Parakeet STT is not installed, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", "notReadyMacInput": "Apple's on-device dictation model is not installed. Download it below to use Voice Conversation.", + "notReadyMacInputAndOpenAiOutput": "Apple's on-device dictation model is not installed, and the OpenAI text-to-speech key is missing. Complete both steps below to use Voice Conversation.", "notReadyMacInputAndPocketOutput": "Apple's on-device dictation model and Pocket TTS are not installed. Complete both steps below to use Voice Conversation.", "notReadyMacInputAndSiriOutput": "Apple's on-device dictation model is not installed, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", + "notReadyOpenAi": "OpenAI voice is not ready. Add the required API key below, then try again.", "notReadyPocketOutput": "Pocket TTS is not installed. Download it below to use Voice Conversation.", "notReadySiriOutput": "No installed Siri voice is selected. Download or select one below to use Voice Conversation.", "notReadyTitle": "Voice Conversation isn't ready", @@ -925,6 +929,14 @@ "noVoiceSelected": "No voice selected", "openMicrophoneSettings": "Open Microphone Settings", "openMicrophoneSettingsError": "Couldn't open Microphone Settings. Open System Settings and select Privacy & Security > Microphone.", + "openAiApiKeyConfigured": "Saved securely for this voice service.", + "openAiApiKeyNotConfigured": "This key is separate from provider credentials used by Goose.", + "openAiApiKeySaved": "API key saved", + "openAiChecking": "Checking OpenAI voice settings…", + "openAiTtsApiKey": "OpenAI text-to-speech API key", + "openAiTtsConfigured": "Uses {{model}} and the {{voice}} voice. OpenAI voices are AI-generated.", + "openAiTtsNeedsKey": "Add an OpenAI text-to-speech API key to use this voice.", + "openAiTtsUnsupportedPlatform": "OpenAI voice playback is currently supported on macOS only.", "outputBackendDescription": "Choose how Berd speaks assistant responses.", "playbackSpeed": "Playback speed", "playing": "Playing", @@ -932,11 +944,13 @@ "preview": "Preview", "previewVoice": "Preview {{voice}}", "removeModel": "Remove model", + "removeApiKey": "Remove", "removeModelDescription": "Voice Conversation will stop before {{model}} is removed. You can download the model again later.", "removeModelQueued": "Queued for removal…", "removeModelTitle": "Remove {{model}}?", "removingModel": "Removing model…", "retryDownload": "Retry model download", + "saveApiKey": "Save key", "settingsDescription": "Choose how Berd speaks, install speech recognition, and preview available voices.", "siriLanguage": "Language", "siriLoading": "Loading Siri voices…", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 2044e1675..06b6650de 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -873,6 +873,7 @@ }, "voice": { "backendMacSpeech": "Reconocimiento de voz de Apple", + "backendOpenAiTts": "Texto a voz de OpenAI", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", "backendSiri": "Voces de Siri", @@ -915,11 +916,14 @@ "modelMissingSize": "No instalado · descarga de {{size}}", "modelNotInstalled": "No instalado", "notReadyInput": "Parakeet STT no está instalado. Descárgalo abajo para usar la conversación por voz.", + "notReadyInputAndOpenAiOutput": "Parakeet STT no está instalado y falta la clave de texto a voz de OpenAI. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyInputAndPocketOutput": "Parakeet STT y Pocket TTS no están instalados. Descarga ambos abajo para usar la conversación por voz.", "notReadyInputAndSiriOutput": "Parakeet STT no está instalado y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInput": "El modelo de dictado de Apple no está instalado. Descárgalo abajo para usar la conversación por voz.", + "notReadyMacInputAndOpenAiOutput": "El modelo de dictado de Apple no está instalado y falta la clave de texto a voz de OpenAI. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInputAndPocketOutput": "El modelo de dictado de Apple y Pocket TTS no están instalados. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInputAndSiriOutput": "El modelo de dictado de Apple no está instalado y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyOpenAi": "La voz de OpenAI no está lista. Añade abajo la clave API necesaria e inténtalo de nuevo.", "notReadyPocketOutput": "Pocket TTS no está instalado. Descárgalo abajo para usar la conversación por voz.", "notReadySiriOutput": "No hay ninguna voz de Siri instalada seleccionada. Descarga o selecciona una abajo para usar la conversación por voz.", "notReadyTitle": "La conversación por voz no está lista", @@ -928,6 +932,14 @@ "noVoiceSelected": "No hay ninguna voz seleccionada", "openMicrophoneSettings": "Abrir ajustes del micrófono", "openMicrophoneSettingsError": "No se pudieron abrir los ajustes del micrófono. Abre Ajustes del Sistema y selecciona Privacidad y seguridad > Micrófono.", + "openAiApiKeyConfigured": "Guardada de forma segura para este servicio de voz.", + "openAiApiKeyNotConfigured": "Esta clave es independiente de las credenciales de proveedor que usa Goose.", + "openAiApiKeySaved": "Clave API guardada", + "openAiChecking": "Comprobando los ajustes de voz de OpenAI…", + "openAiTtsApiKey": "Clave API de texto a voz de OpenAI", + "openAiTtsConfigured": "Usa {{model}} y la voz {{voice}}. Las voces de OpenAI son generadas por IA.", + "openAiTtsNeedsKey": "Añade una clave API de texto a voz de OpenAI para usar esta voz.", + "openAiTtsUnsupportedPlatform": "La reproducción de voz de OpenAI solo es compatible actualmente con macOS.", "outputBackendDescription": "Elige cómo Berd reproduce las respuestas del asistente.", "playbackSpeed": "Velocidad de reproducción", "playing": "Reproduciendo", @@ -935,11 +947,13 @@ "preview": "Escuchar", "previewVoice": "Escuchar {{voice}}", "removeModel": "Eliminar modelo", + "removeApiKey": "Eliminar", "removeModelDescription": "La conversación por voz se detendrá antes de eliminar {{model}}. Puedes volver a descargar el modelo más tarde.", "removeModelQueued": "En cola para eliminar…", "removeModelTitle": "¿Eliminar {{model}}?", "removingModel": "Eliminando modelo…", "retryDownload": "Reintentar descarga del modelo", + "saveApiKey": "Guardar clave", "settingsDescription": "Elige cómo habla Berd, instala el reconocimiento de voz y escucha las voces disponibles.", "siriLanguage": "Idioma", "siriLoading": "Cargando voces de Siri…",