diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 1de42d10a..85b7c1a70 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -46,6 +46,7 @@ dependencies = [ "reqwest 0.13.4", "rodio", "rubato", + "rustls", "semver", "serde", "serde_json", @@ -73,6 +74,7 @@ dependencies = [ "tempfile", "time", "tokio", + "tokio-tungstenite", "toml 1.1.4+spec-1.1.0", "url", "uuid", @@ -1122,7 +1124,7 @@ dependencies = [ "bitflags 2.13.1", "core-foundation 0.10.1", "core-graphics-types", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -1405,6 +1407,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "dbus" version = "0.9.12" @@ -1416,6 +1424,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "dbus-secret-service" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708b509edf7889e53d7efb0ffadd994cc6c2345ccb62f55cfd6b0682165e4fa6" +dependencies = [ + "dbus", + "openssl", + "zeroize", +] + [[package]] name = "defmt" version = "1.1.1" @@ -2004,6 +2023,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -2011,7 +2039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -2025,6 +2053,12 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -3181,9 +3215,13 @@ version = "3.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eebcc3aff044e5944a8fbaf69eb277d11986064cba30c468730e8b9909fb551c" dependencies = [ + "byteorder", + "dbus-secret-service", "log", + "openssl", "security-framework 2.11.1", "security-framework 3.7.0", + "windows-sys 0.60.2", "zeroize", ] @@ -3238,6 +3276,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" dependencies = [ + "cc", "pkg-config", ] @@ -4142,12 +4181,59 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -7114,6 +7200,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489a59b6730eda1b0171fcfda8b121f4bee2b35cba8645ca35c5f7ba3eb736c1" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tungstenite", + "webpki-roots 0.26.11", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -7374,6 +7476,25 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.5", + "rustls", + "rustls-pki-types", + "sha1", + "thiserror 2.0.20", + "utf-8", +] + [[package]] name = "typeid" version = "1.0.3" @@ -7583,6 +7704,12 @@ dependencies = [ "url", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8-zero" version = "0.8.1" @@ -8965,6 +9092,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 8250e0747..1f025558a 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -42,7 +42,6 @@ hex = "0.4" ignore = "0.4.25" fern = "0.7" infer = "0.19.0" -keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } libc = "0.2" log = "0.4.29" mime_guess = "2" @@ -96,11 +95,14 @@ tauri-plugin-updater = "2" tauri-plugin-window-state = "2" time = { version = "0.3", features = ["formatting"] } tokio = { version = "1.50.0", features = ["full"] } +tokio-tungstenite = { version = "0.27", features = ["rustls-tls-webpki-roots"] } +rustls = { version = "0.23", default-features = false, features = ["aws_lc_rs", "std", "tls12"] } url = "2" uuid = { version = "1", features = ["v4", "serde"] } zip = { version = "2", default-features = false, features = ["deflate"] } [target.'cfg(windows)'.dependencies] +keyring = { version = "3.6.3", default-features = false, features = ["windows-native"] } windows-sys = { version = "0.59", features = [ "Win32_Foundation", "Win32_Globalization", @@ -113,6 +115,7 @@ windows-sys = { version = "0.59", features = [ [target.'cfg(target_os = "macos")'.dependencies] block2 = "0.6" coreaudio-rs = "0.14.2" +keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } objc2 = "0.6" objc2-app-kit = { version = "0.3.2", features = ["NSApplication", "NSRunningApplication", "NSImage", "NSMenu", "NSMenuItem", "NSAlert", "NSButton", "NSControl", "NSCell", "NSResponder", "NSView"] } objc2-avf-audio = { version = "0.3.2", features = ["AVAudioApplication", "block2"] } @@ -121,6 +124,9 @@ objc2-foundation = { version = "0.3.2", features = ["NSDictionary", "NSError", " objc2-user-notifications = "0.3.2" rodio = { version = "0.22", default-features = false, features = ["playback"] } +[target.'cfg(target_os = "linux")'.dependencies] +keyring = { version = "3.6.3", default-features = false, features = ["sync-secret-service", "vendored"] } + [features] default = [] diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 2337baaf5..58200288b 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -41,6 +41,10 @@ const SILENCE_FLUSH_FRAMES: usize = 75; const FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS: u64 = 5; const FINAL_TRANSCRIPT_DELIVERY_TIMEOUT: Duration = Duration::from_secs(FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS); +const OPENAI_NETWORK_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +const OPENAI_FINAL_WRITE_TIMEOUT: Duration = Duration::from_secs(1); +const OPENAI_STARTUP_TIMEOUT: Duration = Duration::from_secs(12); +const OPENAI_PRE_ROLL_CHUNKS: usize = 15; const STT_WORKER_SHUTDOWN_TIMEOUT_SECONDS: u64 = mac_speech::RECOGNITION_FINISH_TIMEOUT_SECONDS + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT_SECONDS + 1; @@ -49,6 +53,7 @@ const STT_WORKER_SHUTDOWN_TIMEOUT_SECONDS: u64 = pub enum VoiceInputBackend { Parakeet, Macos, + Openai, } fn active_vad_threshold_for_speech( @@ -603,10 +608,12 @@ impl NativeVoiceState { } // Native input mute is authoritative when installed so a hardware // unmute cannot be masked by a stale renderer fallback latch. - self.microphone_muted.store( - software_microphone_mute(native_microphone_mute_control, muted), - Ordering::SeqCst, - ); + let software_muted = software_microphone_mute(native_microphone_mute_control, muted); + let previous_software_muted = + self.microphone_muted.swap(software_muted, Ordering::SeqCst); + if !native_microphone_mute_control && previous_software_muted != software_muted { + self.input_mute_epoch.fetch_add(1, Ordering::AcqRel); + } owner_window_label }; Ok(Some(owner_window_label)) @@ -827,6 +834,60 @@ impl SttPipeline { )) } + fn new_openai( + api_key: String, + input_muted: Arc, + input_mute_epoch: Arc, + assistant_speaking: Arc, + assistant_vad_threshold: Arc, + speech_vad_threshold: f32, + ) -> Result<(Self, tokio_mpsc::Receiver, mpsc::Receiver<()>), String> { + let (audio_tx, audio_rx) = mpsc::sync_channel(AUDIO_QUEUE_DEPTH); + let (event_tx, event_rx) = tokio_mpsc::channel(64); + let (startup_tx, startup_rx) = mpsc::sync_channel(0); + let shutdown = Arc::new(AtomicBool::new(false)); + let discard_on_shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_mute_epoch = Arc::new(AtomicU64::new(0)); + let worker_shutdown = Arc::clone(&shutdown); + let worker_discard_on_shutdown = Arc::clone(&discard_on_shutdown); + let worker_input_muted = Arc::clone(&input_muted); + let worker_input_mute_epoch = Arc::clone(&input_mute_epoch); + let worker_shutdown_mute_epoch = Arc::clone(&shutdown_mute_epoch); + let thread = thread::Builder::new() + .name("berd-openai-stt".into()) + .spawn(move || { + openai_stt_worker( + api_key, + audio_rx, + event_tx, + worker_shutdown, + worker_discard_on_shutdown, + worker_input_muted, + worker_input_mute_epoch, + worker_shutdown_mute_epoch, + assistant_speaking, + assistant_vad_threshold, + speech_vad_threshold, + startup_tx, + ) + }) + .map_err(|error| format!("start OpenAI speech recognition: {error}"))?; + Ok(( + Self { + audio_tx, + audio_seen: AtomicBool::new(false), + shutdown, + discard_on_shutdown, + input_muted, + input_mute_epoch, + shutdown_mute_epoch, + thread: Some(thread), + }, + event_rx, + startup_rx, + )) + } + fn new_macos( input_muted: Arc, input_mute_epoch: Arc, @@ -1073,7 +1134,10 @@ where } async fn status(app: &AppHandle, state: &NativeVoiceState) -> NativeVoiceStatus { - let parakeet_available = parakeet_model_dir(app).is_ok(); + let parakeet_available = parakeet_model_dir(app).is_ok() + || app + .state::() + .is_configured(); #[cfg(target_os = "macos")] let macos_status = || async { mac_speech::status_async() @@ -1231,6 +1295,19 @@ pub async fn start_native_voice_conversation( "Download the macOS speech recognition model before starting a call.".to_string(), ); } + let openai_api_key = if input_backend == VoiceInputBackend::Openai { + Some(super::openai_audio::stt_api_key()?) + } else { + None + }; + if openai_api_key.is_some() { + let deadline = tokio::time::Instant::now() + Duration::from_secs(1); + while !webview_window.is_focused().unwrap_or(false) + && tokio::time::Instant::now() < deadline + { + tokio::time::sleep(Duration::from_millis(25)).await; + } + } let window_label = webview_window.label().to_string(); let owner_id = native_owner_id(&session_id); let lifecycle_guard = state @@ -1271,6 +1348,36 @@ pub async fn start_native_voice_conversation( Arc::clone(&state.assistant_vad_threshold), speech_vad_threshold, ), + VoiceInputBackend::Openai => { + let startup = SttPipeline::new_openai( + openai_api_key.expect("OpenAI key resolved for OpenAI input"), + Arc::clone(&state.input_muted), + Arc::clone(&state.input_mute_epoch), + Arc::clone(&state.assistant_speaking), + Arc::clone(&state.assistant_vad_threshold), + speech_vad_threshold, + ); + match startup { + Err(error) => Err(error), + Ok((pipeline, events, startup)) => { + let startup_result = tokio::task::spawn_blocking(move || { + startup.recv_timeout(OPENAI_STARTUP_TIMEOUT) + }) + .await; + match startup_result { + Ok(Ok(())) => Ok((pipeline, events)), + Ok(Err(error)) => { + drop(pipeline); + Err(format!("OpenAI transcription did not start: {error}")) + } + Err(error) => { + drop(pipeline); + Err(format!("wait for OpenAI transcription startup: {error}")) + } + } + } + } + } }; let (pipeline, mut events) = match pipeline { Ok(result) => result, @@ -2473,6 +2580,613 @@ fn macos_stt_worker( let _ = forward_macos_events(&mut recognition_events, &event_tx, Some(delivery_deadline)); } +#[derive(Debug, PartialEq, Eq)] +struct OpenAiCommittedTurn { + item_id: String, + mute_epoch: u64, +} + +fn block_on_openai_operation( + runtime: &tokio::runtime::Runtime, + shutdown: &AtomicBool, + future: F, + action: &str, +) -> Result, String> +where + F: std::future::Future>, + E: std::fmt::Display, +{ + runtime.block_on(async { + let wait_for_shutdown = async { + while !shutdown.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + }; + tokio::select! { + result = tokio::time::timeout(OPENAI_NETWORK_OPERATION_TIMEOUT, future) => { + match result { + Ok(Ok(value)) => Ok(Some(value)), + Ok(Err(error)) => Err(format!("{action}: {error}")), + Err(_) => Err(format!("{action}: operation timed out")), + } + } + () = wait_for_shutdown => Ok(None), + } + }) +} + +fn push_openai_pre_roll(pre_roll: &mut VecDeque>, pcm: Vec) { + pre_roll.push_back(pcm); + while pre_roll.len() > OPENAI_PRE_ROLL_CHUNKS { + pre_roll.pop_front(); + } +} + +fn openai_turn_reached_limit(samples_16k: usize) -> bool { + samples_16k >= MAX_SPEECH_SAMPLES +} + +fn record_openai_transcription_event( + value: &serde_json::Value, + current_mute_epoch: u64, + pending_commit_epochs: &mut VecDeque, + committed: &mut VecDeque, + completed: &mut HashMap, +) -> Result, String> { + let mut newly_committed = None; + match value.get("type").and_then(|value| value.as_str()) { + Some("input_audio_buffer.committed") => { + if let Some(item_id) = value.get("item_id").and_then(|value| value.as_str()) { + let turn = OpenAiCommittedTurn { + item_id: item_id.to_string(), + mute_epoch: pending_commit_epochs + .pop_front() + .unwrap_or(current_mute_epoch), + }; + if turn.mute_epoch == current_mute_epoch { + committed.push_back(OpenAiCommittedTurn { + item_id: turn.item_id.clone(), + mute_epoch: turn.mute_epoch, + }); + } + newly_committed = Some(turn); + } + } + Some("conversation.item.input_audio_transcription.completed") => { + if let (Some(item_id), Some(transcript)) = ( + value.get("item_id").and_then(|value| value.as_str()), + value.get("transcript").and_then(|value| value.as_str()), + ) { + if committed + .iter() + .any(|turn| turn.item_id == item_id && turn.mute_epoch == current_mute_epoch) + { + completed.insert(item_id.to_string(), transcript.trim().to_string()); + } + } + } + Some("conversation.item.input_audio_transcription.failed") => { + if value + .get("item_id") + .and_then(|value| value.as_str()) + .is_some_and(|item_id| { + !committed.iter().any(|turn| { + turn.item_id == item_id && turn.mute_epoch == current_mute_epoch + }) + }) + { + return Ok(None); + } + return Err(value + .pointer("/error/message") + .and_then(|value| value.as_str()) + .unwrap_or("OpenAI realtime transcription failed.") + .to_string()); + } + Some("error") => { + return Err(value + .pointer("/error/message") + .and_then(|value| value.as_str()) + .unwrap_or("OpenAI realtime transcription failed.") + .to_string()); + } + _ => {} + } + Ok(newly_committed) +} + +fn deliver_completed_openai_turns( + committed: &mut VecDeque, + completed: &mut HashMap, + event_tx: &tokio_mpsc::Sender, + current_mute_epoch: u64, + final_item_id: Option<&str>, + final_delivery: &mut Option>, +) { + while committed + .front() + .is_some_and(|turn| completed.contains_key(&turn.item_id)) + { + let turn = committed.pop_front().expect("checked front"); + let text = completed.remove(&turn.item_id).unwrap_or_default(); + if turn.mute_epoch != current_mute_epoch { + continue; + } + let delivered = (Some(turn.item_id.as_str()) == final_item_id) + .then(|| final_delivery.take()) + .flatten(); + deliver_recognition_result(text, event_tx, delivered); + } +} + +#[allow(clippy::too_many_arguments)] // Worker boundary keeps channel and mute lifecycle inputs explicit. +fn openai_stt_worker( + key: String, + audio_rx: Receiver, + event_tx: tokio_mpsc::Sender, + shutdown: Arc, + discard_on_shutdown: Arc, + input_muted: Arc, + input_mute_epoch: Arc, + shutdown_mute_epoch: Arc, + assistant_speaking: Arc, + assistant_vad_threshold: Arc, + speech_vad_threshold: f32, + startup_tx: SyncSender<()>, +) { + use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; + use futures_util::{SinkExt, StreamExt}; + use rubato::{Fft, FixedSync, Resampler}; + use tokio_tungstenite::tungstenite::{client::IntoClientRequest, Message}; + + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(format!( + "Could not initialize OpenAI realtime transcription: {error}" + ))); + return; + } + }; + let model = super::openai_audio::transcription_model(); + let endpoint = match super::openai_audio::realtime_endpoint() { + Ok(endpoint) => endpoint, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + }; + if let Err(existing) = rustls::crypto::aws_lc_rs::default_provider().install_default() { + // Another dependency may have installed the same process-wide provider first. + drop(existing); + } + let mut request = match endpoint.into_client_request() { + Ok(request) => request, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(format!( + "prepare OpenAI realtime connection: {error}" + ))); + return; + } + }; + let authorization = match format!("Bearer {key}").parse() { + Ok(authorization) => authorization, + Err(_) => { + let _ = event_tx.blocking_send(SttMessage::Failed( + "OpenAI API key is not a valid header value".to_string(), + )); + return; + } + }; + request.headers_mut().insert("Authorization", authorization); + let connection = block_on_openai_operation( + &runtime, + &shutdown, + tokio_tungstenite::connect_async(request), + "connect to OpenAI realtime transcription", + ); + let mut socket = match connection { + Ok(Some((socket, _))) => socket, + Ok(None) => return, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + }; + + macro_rules! send_openai_startup_message { + ($message:expr, $action:literal) => { + match block_on_openai_operation(&runtime, &shutdown, socket.send($message), $action) { + Ok(Some(())) => {} + Ok(None) => return, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + } + }; + } + let session_update = serde_json::json!({ + "type": "session.update", + "session": { + "type": "transcription", + "audio": { "input": { + "format": { "type": "audio/pcm", "rate": 24000 }, + "transcription": { "model": model, "delay": "low" }, + "turn_detection": null + }} + } + }); + send_openai_startup_message!( + Message::Text(session_update.to_string().into()), + "configure OpenAI realtime transcription" + ); + + let mut resampler = match Fft::::new(48_000, 24_000, 960, 2, 1, FixedSync::Input) { + Ok(resampler) => resampler, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(format!( + "Could not initialize OpenAI audio resampling: {error}" + ))); + return; + } + }; + if startup_tx.send(()).is_err() { + return; + } + let chunk_in = resampler.input_frames_next(); + let mut vad = earshot::Detector::new(earshot::DefaultPredictor::new()); + let mut input_48k = Vec::new(); + let mut vad_16k = Vec::new(); + let mut silence_frames = 0_usize; + let mut in_speech = false; + let mut turn_has_audio = false; + let mut turn_samples_16k = 0_usize; + let mut pre_roll = VecDeque::>::new(); + let mut observed_mute_epoch = input_mute_epoch.load(Ordering::Acquire); + let mut pending_commit_epochs = VecDeque::::new(); + let mut committed_turns = VecDeque::::new(); + let mut completed_turns = HashMap::::new(); + + macro_rules! send_openai_stream_message { + ($message:expr, $action:literal, $on_shutdown:block) => { + match block_on_openai_operation(&runtime, &shutdown, socket.send($message), $action) { + Ok(Some(())) => {} + Ok(None) => $on_shutdown + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + } + }; + } + + 'worker: loop { + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); + if shutting_down { + break; + } + if current_mute_epoch != observed_mute_epoch { + observed_mute_epoch = current_mute_epoch; + input_48k.clear(); + vad_16k.clear(); + silence_frames = 0; + turn_has_audio = false; + turn_samples_16k = 0; + pre_roll.clear(); + committed_turns.clear(); + completed_turns.clear(); + if std::mem::take(&mut in_speech) { + let _ = event_tx.blocking_send(SttMessage::Speaking(false)); + } + send_openai_stream_message!( + Message::Text( + serde_json::json!({"type":"input_audio_buffer.clear"}) + .to_string() + .into() + ), + "clear muted OpenAI transcription audio", + { break 'worker } + ); + } + while let Some(event) = runtime.block_on(async { + tokio::time::timeout(Duration::from_millis(1), socket.next()) + .await + .ok() + .flatten() + }) { + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); + if shutting_down { + break 'worker; + } + if current_mute_epoch != observed_mute_epoch { + continue 'worker; + } + let message = match event { + Ok(Message::Text(text)) => text, + Ok(Message::Close(_)) => { + let _ = event_tx.blocking_send(SttMessage::Failed( + "OpenAI realtime transcription disconnected.".to_string(), + )); + return; + } + Ok(_) => continue, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(format!( + "OpenAI realtime transcription failed: {error}" + ))); + return; + } + }; + let Ok(value) = serde_json::from_str::(&message) else { + continue; + }; + let recorded_turn = match record_openai_transcription_event( + &value, + observed_mute_epoch, + &mut pending_commit_epochs, + &mut committed_turns, + &mut completed_turns, + ) { + Ok(turn) => turn, + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + return; + } + }; + if let Some(turn) = recorded_turn.filter(|turn| turn.mute_epoch != observed_mute_epoch) + { + send_openai_stream_message!( + Message::Text( + serde_json::json!({ + "type": "conversation.item.delete", + "item_id": turn.item_id, + }) + .to_string() + .into() + ), + "discard muted OpenAI transcription turn", + { break 'worker } + ); + } + deliver_completed_openai_turns( + &mut committed_turns, + &mut completed_turns, + &event_tx, + input_mute_epoch.load(Ordering::Acquire), + None, + &mut None, + ); + } + + let batch = match audio_rx.recv_timeout(Duration::from_millis(20)) { + Ok(batch) => Some(batch), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + let (shutting_down, current_mute_epoch) = + sample_effective_mute_epoch(&input_mute_epoch, &shutdown, &shutdown_mute_epoch); + // Stop consuming queued microphone batches as soon as shutdown begins. + // The bounded finalization below commits only audio already accepted by + // this worker, so network backpressure cannot extend the owner timeout. + if shutting_down { + break; + } + if current_mute_epoch != observed_mute_epoch { + continue 'worker; + } + if input_muted.load(Ordering::Acquire) { + continue; + } + let Some(batch) = batch else { continue }; + if batch.mute_epoch != observed_mute_epoch { + continue; + } + input_48k.extend( + batch + .bytes + .chunks_exact(4) + .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])), + ); + while input_48k.len() >= chunk_in { + let chunk: Vec = input_48k.drain(..chunk_in).collect(); + let pcm_24k = resample(&mut resampler, &chunk); + let pcm_bytes: Vec = pcm_24k + .iter() + .flat_map(|sample| { + ((sample.clamp(-1.0, 1.0) * i16::MAX as f32).round() as i16).to_le_bytes() + }) + .collect(); + // Earshot requires 16 kHz; use every third 48 kHz source sample for activity only. + vad_16k.extend(chunk.iter().step_by(3).copied()); + let mut speech_started = false; + let mut should_commit = false; + while vad_16k.len() >= VAD_FRAME_SAMPLES { + let frame: Vec = vad_16k.drain(..VAD_FRAME_SAMPLES).collect(); + let threshold = active_vad_threshold_for_speech( + &assistant_speaking, + &assistant_vad_threshold, + speech_vad_threshold, + ); + if vad.predict_f32(&frame) > threshold { + silence_frames = 0; + if !in_speech { + in_speech = true; + speech_started = true; + let _ = event_tx.blocking_send(SttMessage::Speaking(true)); + } + } else if in_speech { + silence_frames += 1; + if silence_frames >= SILENCE_FLUSH_FRAMES { + should_commit = true; + silence_frames = 0; + in_speech = false; + } + } + } + + if speech_started { + while let Some(pre_roll_bytes) = pre_roll.pop_front() { + send_openai_stream_message!( + Message::Text( + serde_json::json!({ + "type": "input_audio_buffer.append", + "audio": BASE64.encode(pre_roll_bytes) + }) + .to_string() + .into() + ), + "stream OpenAI transcription pre-roll", + { break 'worker } + ); + } + } + + if speech_started || in_speech || should_commit { + send_openai_stream_message!( + Message::Text( + serde_json::json!({ + "type": "input_audio_buffer.append", + "audio": BASE64.encode(pcm_bytes) + }) + .to_string() + .into() + ), + "stream audio to OpenAI realtime transcription", + { break 'worker } + ); + turn_has_audio = true; + turn_samples_16k = turn_samples_16k.saturating_add(chunk.len() / 3); + } else { + push_openai_pre_roll(&mut pre_roll, pcm_bytes); + } + + if should_commit || openai_turn_reached_limit(turn_samples_16k) { + send_openai_stream_message!( + Message::Text( + serde_json::json!({"type":"input_audio_buffer.commit"}) + .to_string() + .into() + ), + "commit OpenAI transcription turn", + { break 'worker } + ); + pending_commit_epochs.push_back(observed_mute_epoch); + turn_has_audio = false; + turn_samples_16k = 0; + silence_frames = 0; + if std::mem::take(&mut in_speech) || should_commit { + let _ = event_tx.blocking_send(SttMessage::Speaking(false)); + } + } + } + } + if discard_on_shutdown.load(Ordering::Acquire) { + return; + } + let mut final_item_id = None::; + let mut final_delivery = None::>; + let mut final_delivered = None::>; + if turn_has_audio { + let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); + let commit = serde_json::json!({"type":"input_audio_buffer.commit"}); + let final_write = runtime.block_on(tokio::time::timeout( + OPENAI_FINAL_WRITE_TIMEOUT, + socket.send(Message::Text(commit.to_string().into())), + )); + if matches!(final_write, Ok(Ok(()))) { + pending_commit_epochs.push_back(observed_mute_epoch); + final_delivery = Some(delivered_tx); + final_delivered = Some(delivered_rx); + } + } + if final_delivered.is_none() + && (!committed_turns.is_empty() || !pending_commit_epochs.is_empty()) + { + let (delivered_tx, delivered_rx) = mpsc::sync_channel(0); + final_item_id = pending_commit_epochs + .is_empty() + .then(|| committed_turns.back().map(|turn| turn.item_id.clone())) + .flatten(); + final_delivery = Some(delivered_tx); + final_delivered = Some(delivered_rx); + } + + // Keep receiving until the shutdown commit itself completes. Earlier turns + // are delivered from the queue first, and already committed transcripts are + // drained even when there was no partial turn to commit at shutdown. + let deadline = std::time::Instant::now() + FINAL_TRANSCRIPT_DELIVERY_TIMEOUT; + while std::time::Instant::now() < deadline { + if final_delivered + .as_ref() + .is_some_and(|receiver| receiver.try_recv().is_ok()) + || (final_delivered.is_none() + && committed_turns.is_empty() + && pending_commit_epochs.is_empty()) + { + break; + } + let Some(message) = runtime.block_on(async { + tokio::time::timeout(Duration::from_millis(50), socket.next()) + .await + .ok() + .flatten() + }) else { + continue; + }; + let Ok(Message::Text(text)) = message else { + continue; + }; + let Ok(value) = serde_json::from_str::(&text) else { + continue; + }; + let event_type = value.get("type").and_then(|value| value.as_str()); + match record_openai_transcription_event( + &value, + observed_mute_epoch, + &mut pending_commit_epochs, + &mut committed_turns, + &mut completed_turns, + ) { + Ok(Some(turn)) + if turn.mute_epoch == observed_mute_epoch + && pending_commit_epochs.is_empty() + && final_delivered.is_some() => + { + final_item_id = Some(turn.item_id); + } + Ok(_) => {} + Err(error) => { + let _ = event_tx.blocking_send(SttMessage::Failed(error)); + break; + } + } + deliver_completed_openai_turns( + &mut committed_turns, + &mut completed_turns, + &event_tx, + observed_mute_epoch, + final_item_id.as_deref(), + &mut final_delivery, + ); + if event_type == Some("conversation.item.input_audio_transcription.completed") + && final_item_id.as_deref() == value.get("item_id").and_then(|value| value.as_str()) + && final_delivery.is_none() + { + break; + } + } + // Do not await the peer's WebSocket close handshake here. OpenAI may leave + // it pending beyond the bounded voice-stop window; dropping the socket + // closes the connection after final transcript delivery has been drained. + drop(socket); +} + #[allow(clippy::too_many_arguments)] // Worker boundary keeps channel and mute lifecycle inputs explicit. fn stt_worker( model_dir: PathBuf, @@ -3308,6 +4022,30 @@ mod tests { assert!(state.microphone_is_muted()); } + #[test] + fn software_microphone_mute_advances_the_audio_epoch() { + let state = NativeVoiceState::default(); + { + let mut runtime = state.runtime.lock().expect("lock native runtime"); + runtime.session_id = Some("session-1".to_string()); + runtime.revision = 4; + runtime.owner = Some(RuntimeOwner { + window_label: "main".to_string(), + }); + runtime.native_microphone_mute_control = false; + } + + assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 0); + state + .set_microphone_muted_target("main", "session-1", 4, true) + .expect("mute"); + assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 1); + state + .set_microphone_muted_target("main", "session-1", 4, false) + .expect("unmute"); + assert_eq!(state.input_mute_epoch.load(Ordering::Acquire), 2); + } + #[test] fn owner_stop_authorization_is_lifecycle_bound() { let state = NativeVoiceState::default(); @@ -3971,6 +4709,210 @@ mod tests { assert!(delivered_rx.try_recv().is_ok()); } + #[test] + fn openai_transcripts_are_delivered_in_commit_order() { + let mut pending_epochs = VecDeque::from([7, 7]); + let mut committed = VecDeque::new(); + let mut completed = HashMap::new(); + for item_id in ["first", "second"] { + record_openai_transcription_event( + &serde_json::json!({ + "type": "input_audio_buffer.committed", + "item_id": item_id, + }), + 7, + &mut pending_epochs, + &mut committed, + &mut completed, + ) + .expect("commit event"); + } + for (item_id, transcript) in [("second", "two"), ("first", "one")] { + record_openai_transcription_event( + &serde_json::json!({ + "type": "conversation.item.input_audio_transcription.completed", + "item_id": item_id, + "transcript": transcript, + }), + 7, + &mut pending_epochs, + &mut committed, + &mut completed, + ) + .expect("completion event"); + } + let (event_tx, mut event_rx) = tokio_mpsc::channel(4); + + deliver_completed_openai_turns( + &mut committed, + &mut completed, + &event_tx, + 7, + None, + &mut None, + ); + + let texts = [event_rx.try_recv(), event_rx.try_recv()].map(|event| match event { + Ok(SttMessage::Final { text, .. }) => text, + _ => panic!("expected finalized transcript"), + }); + assert_eq!(texts, ["one", "two"]); + } + + #[test] + fn openai_transcription_ignores_commits_from_stale_mute_epochs() { + let mut pending_epochs = VecDeque::from([1]); + let mut committed = VecDeque::new(); + let mut completed = HashMap::new(); + + let turn = record_openai_transcription_event( + &serde_json::json!({ + "type": "input_audio_buffer.committed", + "item_id": "stale", + }), + 2, + &mut pending_epochs, + &mut committed, + &mut completed, + ) + .expect("commit event") + .expect("recorded commit"); + + assert_eq!(turn.mute_epoch, 1); + assert!(committed.is_empty()); + } + + #[test] + fn openai_transcription_discards_completed_turns_after_mute_changes() { + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: "stale".to_string(), + mute_epoch: 1, + }]); + let mut completed = HashMap::from([("stale".to_string(), "ignore me".to_string())]); + let (event_tx, mut event_rx) = tokio_mpsc::channel(1); + + deliver_completed_openai_turns( + &mut committed, + &mut completed, + &event_tx, + 2, + None, + &mut None, + ); + + assert!(committed.is_empty()); + assert!(completed.is_empty()); + assert!(event_rx.try_recv().is_err()); + } + + #[test] + fn openai_transcription_surfaces_protocol_failures() { + let error = record_openai_transcription_event( + &serde_json::json!({ + "type": "conversation.item.input_audio_transcription.failed", + "error": { "message": "bad audio" }, + }), + 0, + &mut VecDeque::new(), + &mut VecDeque::new(), + &mut HashMap::new(), + ) + .expect_err("failure event"); + + assert_eq!(error, "bad audio"); + } + + #[test] + fn openai_transcription_ignores_failures_for_discarded_turns() { + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: "current".to_string(), + mute_epoch: 2, + }]); + + let result = record_openai_transcription_event( + &serde_json::json!({ + "type": "conversation.item.input_audio_transcription.failed", + "item_id": "stale", + "error": { "message": "discarded turn failed" }, + }), + 2, + &mut VecDeque::new(), + &mut committed, + &mut HashMap::new(), + ); + + assert_eq!(result, Ok(None)); + } + + #[test] + fn empty_openai_final_acknowledges_shutdown_delivery() { + let mut committed = VecDeque::from([OpenAiCommittedTurn { + item_id: "final".to_string(), + mute_epoch: 3, + }]); + let mut completed = HashMap::from([("final".to_string(), String::new())]); + let (event_tx, mut event_rx) = tokio_mpsc::channel(1); + let (delivered_tx, delivered_rx) = mpsc::sync_channel(1); + let mut final_delivery = Some(delivered_tx); + + deliver_completed_openai_turns( + &mut committed, + &mut completed, + &event_tx, + 3, + Some("final"), + &mut final_delivery, + ); + + assert!(delivered_rx.try_recv().is_ok()); + assert!(event_rx.try_recv().is_err()); + } + + #[test] + fn stalled_openai_operation_observes_shutdown() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("runtime"); + let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_for_thread = shutdown.clone(); + let signal = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(30)); + shutdown_for_thread.store(true, Ordering::Release); + }); + + let result = block_on_openai_operation( + &runtime, + shutdown.as_ref(), + std::future::pending::>(), + "stalled operation", + ) + .expect("shutdown is not an error"); + + signal.join().expect("shutdown signal"); + assert_eq!(result, None); + } + + #[test] + fn openai_idle_audio_keeps_only_bounded_pre_roll() { + let mut pre_roll = VecDeque::new(); + for index in 0..(OPENAI_PRE_ROLL_CHUNKS * 4) { + push_openai_pre_roll(&mut pre_roll, vec![index as u8]); + } + + assert_eq!(pre_roll.len(), OPENAI_PRE_ROLL_CHUNKS); + assert_eq!( + pre_roll.front(), + Some(&vec![(OPENAI_PRE_ROLL_CHUNKS * 3) as u8]) + ); + } + + #[test] + fn openai_continuous_speech_has_a_turn_limit() { + assert!(!openai_turn_reached_limit(MAX_SPEECH_SAMPLES - 1)); + assert!(openai_turn_reached_limit(MAX_SPEECH_SAMPLES)); + } + #[test] fn native_voice_events_use_renderer_field_names() { let event = NativeVoiceEvent::User { @@ -4017,5 +4959,10 @@ mod tests { .expect("deserialize macOS backend"), VoiceInputBackend::Macos, ); + assert_eq!( + serde_json::from_str::("\"openai\"") + .expect("deserialize OpenAI backend"), + VoiceInputBackend::Openai, + ); } } diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index a0709de77..58775f358 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -1,7 +1,7 @@ -//! OpenAI streaming speech playback for voice conversations. +//! OpenAI realtime transcription configuration and streaming speech playback. use std::sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, mpsc, Arc, Mutex, }; #[cfg(any(test, target_os = "macos"))] @@ -34,12 +34,12 @@ use super::{ #[cfg(any(test, target_os = "macos"))] use std::time::Instant; -#[cfg(target_os = "macos")] const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; +const DEFAULT_TRANSCRIPTION_MODEL: &str = "gpt-live-transcribe"; const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; const DEFAULT_TTS_VOICE: &str = "marin"; -#[cfg(target_os = "macos")] const KEYRING_SERVICE: &str = "berd-openai-voice"; +const STT_KEYRING_ACCOUNT: &str = "stt-api-key"; const TTS_KEYRING_ACCOUNT: &str = "tts-api-key"; const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; #[cfg(target_os = "macos")] @@ -62,6 +62,14 @@ const MAX_FINAL_PLAYBACK_DRAIN: Duration = Duration::from_secs(600); #[derive(Clone, Debug, Default)] pub struct OpenAiVoiceState { playback: Arc>, + configured: Arc, + credential_revision: Arc, +} + +impl OpenAiVoiceState { + pub(crate) fn is_configured(&self) -> bool { + self.configured.load(Ordering::Acquire) + } } #[derive(Debug)] @@ -100,7 +108,11 @@ enum OpenAiStreamCommand { #[derive(Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct OpenAiVoiceStatus { + stt_configured: bool, tts_configured: bool, + stt_unavailable_reason: Option, + tts_unavailable_reason: Option, + transcription_model: String, speech_model: String, speech_voice: String, playback_speed: f32, @@ -154,7 +166,6 @@ fn env_trimmed(name: &str) -> Option { .filter(|value| !value.is_empty()) } -#[cfg(target_os = "macos")] fn goose_yaml_value(path: &std::path::Path, name: &str) -> Result, String> { if !path.exists() { return Ok(None); @@ -171,7 +182,6 @@ fn goose_yaml_value(path: &std::path::Path, name: &str) -> Result .map(ToString::to_string)) } -#[cfg(target_os = "macos")] fn stored_api_key(account: &str) -> Result, String> { let entry = keyring::Entry::new(KEYRING_SERVICE, account) .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}"))?; @@ -184,12 +194,6 @@ fn stored_api_key(account: &str) -> Result, String> { } } -#[cfg(not(target_os = "macos"))] -fn stored_api_key(_account: &str) -> Result, String> { - Ok(None) -} - -#[cfg(target_os = "macos")] fn store_api_key(account: &str, api_key: &str) -> Result<(), String> { let entry = keyring::Entry::new(KEYRING_SERVICE, account) .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}"))?; @@ -198,12 +202,6 @@ fn store_api_key(account: &str, api_key: &str) -> Result<(), String> { .map_err(|error| format!("Could not save Berd's OpenAI voice credential: {error}")) } -#[cfg(not(target_os = "macos"))] -fn store_api_key(_account: &str, _api_key: &str) -> Result<(), String> { - Err("OpenAI voice credentials are unsupported on this platform".to_string()) -} - -#[cfg(target_os = "macos")] fn clear_api_key(account: &str) -> Result<(), String> { let entry = keyring::Entry::new(KEYRING_SERVICE, account) .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}"))?; @@ -215,11 +213,6 @@ fn clear_api_key(account: &str) -> Result<(), String> { } } -#[cfg(not(target_os = "macos"))] -fn clear_api_key(_account: &str) -> Result<(), String> { - Err("OpenAI voice credentials are unsupported on this platform".to_string()) -} - #[cfg(target_os = "macos")] fn tts_api_key() -> Result { stored_api_key(TTS_KEYRING_ACCOUNT)?.ok_or_else(|| { @@ -228,7 +221,13 @@ fn tts_api_key() -> Result { }) } -#[cfg(any(test, target_os = "macos"))] +pub(crate) fn stt_api_key() -> Result { + stored_api_key(STT_KEYRING_ACCOUNT)?.ok_or_else(|| { + "OpenAI speech-to-text is not configured. Add its API key in Voice settings, then try again." + .to_string() + }) +} + fn normalize_openai_base_url(raw_url: String, assume_v1: bool) -> Result { let mut url = reqwest::Url::parse(&raw_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; @@ -250,7 +249,6 @@ fn normalize_openai_base_url(raw_url: String, assume_v1: bool) -> Result Result { if let Some(host) = env_trimmed("OPENAI_HOST") { return normalize_openai_base_url(host, true); @@ -268,6 +266,29 @@ fn base_url() -> Result { Ok(DEFAULT_BASE_URL.to_string()) } +pub(crate) fn realtime_endpoint() -> Result { + let mut url = reqwest::Url::parse(&endpoint("realtime")?) + .map_err(|error| format!("OpenAI realtime endpoint is invalid: {error}"))?; + url.query_pairs_mut().append_pair("intent", "transcription"); + match url.scheme() { + "http" => url.set_scheme("ws").expect("compatible scheme"), + "https" => url.set_scheme("wss").expect("compatible scheme"), + "ws" | "wss" => {} + scheme => { + return Err(format!( + "OpenAI realtime endpoint has unsupported scheme: {scheme}" + )) + } + } + Ok(url.to_string()) +} + +pub(crate) fn transcription_model() -> String { + env_trimmed("OPENAI_TRANSCRIPTION_MODEL") + .or_else(|| env_trimmed("OPENAI_STT_MODEL")) + .unwrap_or_else(|| DEFAULT_TRANSCRIPTION_MODEL.to_string()) +} + fn speech_model() -> String { env_trimmed("OPENAI_TTS_MODEL").unwrap_or_else(|| DEFAULT_TTS_MODEL.to_string()) } @@ -276,12 +297,10 @@ fn speech_voice() -> String { env_trimmed("OPENAI_TTS_VOICE").unwrap_or_else(|| DEFAULT_TTS_VOICE.to_string()) } -#[cfg(target_os = "macos")] fn endpoint(path: &str) -> Result { endpoint_for_base_url(&base_url()?, path) } -#[cfg(any(test, target_os = "macos"))] fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { let mut url = reqwest::Url::parse(base_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; @@ -350,16 +369,30 @@ pub async fn get_openai_voice_status( .map_err(|_| "OpenAI voice playback state lock was poisoned".to_string())? .speed; let tts_available = cfg!(target_os = "macos"); - let tts_configured = if tts_available { - tauri::async_runtime::spawn_blocking(|| stored_api_key(TTS_KEYRING_ACCOUNT)) - .await - .map_err(|error| format!("Could not check OpenAI voice credentials: {error}"))?? - .is_some() - } else { - false - }; + let credential_revision = state.credential_revision.load(Ordering::Acquire); + let (stt_result, tts_result) = tauri::async_runtime::spawn_blocking(move || { + let tts_result = if tts_available { + stored_api_key(TTS_KEYRING_ACCOUNT) + } else { + Ok(None) + }; + (stored_api_key(STT_KEYRING_ACCOUNT), tts_result) + }) + .await + .map_err(|error| format!("Could not check OpenAI voice credentials: {error}"))?; + let stt_error = stt_result.as_ref().err().cloned(); + let tts_error = tts_result.as_ref().err().cloned(); + let stt_configured = stt_result.unwrap_or(None).is_some(); + let tts_configured = tts_result.unwrap_or(None).is_some(); + if state.credential_revision.load(Ordering::Acquire) == credential_revision { + state.configured.store(stt_configured, Ordering::Release); + } Ok(OpenAiVoiceStatus { + stt_configured, tts_configured, + stt_unavailable_reason: stt_error, + tts_unavailable_reason: tts_error, + transcription_model: transcription_model(), speech_model: speech_model(), speech_voice: speech_voice(), playback_speed, @@ -374,6 +407,37 @@ pub async fn get_openai_voice_status( }) } +#[tauri::command] +pub fn set_openai_stt_api_key( + app: AppHandle, + state: State<'_, OpenAiVoiceState>, + api_key: String, +) -> Result<(), String> { + let api_key = api_key.trim(); + if api_key.is_empty() { + return Err("OpenAI speech-to-text API key cannot be empty".to_string()); + } + store_api_key(STT_KEYRING_ACCOUNT, api_key)?; + state.credential_revision.fetch_add(1, Ordering::AcqRel); + state.configured.store(true, Ordering::Release); + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; + Ok(()) +} + +#[tauri::command] +pub fn clear_openai_stt_api_key( + app: AppHandle, + state: State<'_, OpenAiVoiceState>, +) -> Result<(), String> { + clear_api_key(STT_KEYRING_ACCOUNT)?; + state.credential_revision.fetch_add(1, Ordering::AcqRel); + state.configured.store(false, Ordering::Release); + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}"))?; + Ok(()) +} + #[tauri::command] pub fn set_openai_tts_api_key( app: AppHandle, @@ -1286,6 +1350,16 @@ mod tests { assert_eq!(drained_at, None); } + #[test] + fn configured_readiness_does_not_require_reading_the_secret() { + let state = OpenAiVoiceState::default(); + assert!(!state.is_configured()); + + state.configured.store(true, Ordering::Release); + + assert!(state.is_configured()); + } + #[cfg(target_os = "macos")] #[test] fn chunks_tts_text_on_char_boundaries() { diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 2e41e9a23..b95c0b06a 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -656,6 +656,8 @@ pub fn run() { commands::pocket_voice::stop_pocket_voice, commands::pocket_voice::remove_voice_model, commands::openai_audio::get_openai_voice_status, + commands::openai_audio::set_openai_stt_api_key, + commands::openai_audio::clear_openai_stt_api_key, commands::openai_audio::set_openai_tts_api_key, commands::openai_audio::clear_openai_tts_api_key, commands::openai_audio::start_openai_voice_stream, diff --git a/src/app/AppShell.tsx b/src/app/AppShell.tsx index 2face1bbf..a73c2d299 100644 --- a/src/app/AppShell.tsx +++ b/src/app/AppShell.tsx @@ -745,7 +745,9 @@ export function AppShell({ ); const globalVoiceOutput = useVoiceOutputPreference(); const globalOpenAiVoiceSetup = useOpenAiVoiceSetup( - capabilities.voiceConversation && globalVoiceOutput.backend === "openai", + capabilities.voiceConversation && + (globalVoiceInput.backend === "openai" || + globalVoiceOutput.backend === "openai"), ); const globalSiriVoiceSetup = useSiriVoiceSetup( capabilities.voiceConversation && globalVoiceOutput.backend === "siri", diff --git a/src/features/chat/ui/ChatView.tsx b/src/features/chat/ui/ChatView.tsx index 883db84c0..2c199553a 100644 --- a/src/features/chat/ui/ChatView.tsx +++ b/src/features/chat/ui/ChatView.tsx @@ -236,7 +236,8 @@ export function ChatView({ ); const voiceOutput = useVoiceOutputPreference(); const openAiVoiceSetup = useOpenAiVoiceSetup( - capabilities.voiceConversation && voiceOutput.backend === "openai", + capabilities.voiceConversation && + (voiceInput.backend === "openai" || voiceOutput.backend === "openai"), ); const siriVoiceSetup = useSiriVoiceSetup( capabilities.voiceConversation && voiceOutput.backend === "siri", diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index 45bf46b33..06bb2af6d 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -7,7 +7,11 @@ import type { } from "../lib/voiceInterruptionPreference"; export interface OpenAiVoiceStatus { + sttConfigured: boolean; ttsConfigured: boolean; + sttUnavailableReason: string | null; + ttsUnavailableReason: string | null; + transcriptionModel: string; speechModel: string; speechVoice: string; playbackSpeed: number; @@ -30,6 +34,14 @@ export function setOpenAiTtsApiKey(apiKey: string): Promise { return invoke("set_openai_tts_api_key", { apiKey }); } +export function setOpenAiSttApiKey(apiKey: string): Promise { + return invoke("set_openai_stt_api_key", { apiKey }); +} + +export function clearOpenAiSttApiKey(): Promise { + return invoke("clear_openai_stt_api_key"); +} + export function clearOpenAiTtsApiKey(): Promise { return invoke("clear_openai_tts_api_key"); } diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index df1520925..9c4846053 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -526,7 +526,7 @@ export function rejectVoiceConversationTranscript( export async function startVoiceConversation( sessionId: string, - inputBackend: "parakeet" | "macos" = "parakeet", + inputBackend: "parakeet" | "macos" | "openai" = "parakeet", foregroundGeneration = 0, ): Promise { resetMicrophoneMuteState(); diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index c1a87c9cb..c134b7b64 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -33,7 +33,11 @@ function deferred() { function status(configured: boolean): OpenAiVoiceStatus { return { + sttConfigured: configured, ttsConfigured: configured, + sttUnavailableReason: null, + ttsUnavailableReason: null, + transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -64,13 +68,13 @@ describe("useOpenAiVoiceSetup", () => { act(() => mocks.settingsChanged?.()); refreshed.resolve(status(true)); await waitFor(() => - expect(result.current.status?.ttsConfigured).toBe(true), + expect(result.current.status?.sttConfigured).toBe(true), ); initial.resolve(status(false)); await act(async () => Promise.resolve()); - expect(result.current.status?.ttsConfigured).toBe(true); + expect(result.current.status?.sttConfigured).toBe(true); }); it("refreshes after listener registration captures credential changes", async () => { @@ -83,7 +87,7 @@ describe("useOpenAiVoiceSetup", () => { act(() => mocks.finishListening?.()); await waitFor(() => - expect(result.current.status?.ttsConfigured).toBe(true), + expect(result.current.status?.sttConfigured).toBe(true), ); }); @@ -94,7 +98,7 @@ describe("useOpenAiVoiceSetup", () => { const { result } = renderHook(() => useOpenAiVoiceSetup()); await waitFor(() => - expect(result.current.status?.ttsConfigured).toBe(true), + expect(result.current.status?.sttConfigured).toBe(true), ); }); diff --git a/src/features/voice-conversation/lib/voiceInputPreference.ts b/src/features/voice-conversation/lib/voiceInputPreference.ts index 4743634b0..c47fe7f28 100644 --- a/src/features/voice-conversation/lib/voiceInputPreference.ts +++ b/src/features/voice-conversation/lib/voiceInputPreference.ts @@ -1,14 +1,16 @@ import { useCallback, useSyncExternalStore } from "react"; import type { MacSpeechStatus } from "../api/macSpeech"; -export type VoiceInputBackend = "parakeet" | "macos"; +export type VoiceInputBackend = "parakeet" | "macos" | "openai"; const STORAGE_KEY = "goose:voice-input-backend"; const CHANGED_EVENT = "goose:voice-input-backend-changed"; let inMemoryBackend: VoiceInputBackend | null = null; function normalizeStored(value: unknown): VoiceInputBackend | null { - return value === "parakeet" || value === "macos" ? value : null; + return value === "parakeet" || value === "macos" || value === "openai" + ? value + : null; } export function getStoredVoiceInputBackend(): VoiceInputBackend | null { @@ -28,6 +30,7 @@ export function resolveVoiceInputBackend( ): VoiceInputBackend | null { if (macSpeechAvailable === null) return null; if (stored === "parakeet") return "parakeet"; + if (stored === "openai") return "openai"; if (stored === "macos" && macSpeechAvailable) return "macos"; return macSpeechAvailable ? "macos" : "parakeet"; } diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts index 6a35f946e..d373ae189 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.test.ts @@ -53,7 +53,7 @@ describe("voice setup readiness", () => { ).toBe(false); }); - it("requires the dedicated OpenAI text-to-speech key for OpenAI output", () => { + it("requires the configured OpenAI TTS key and TTS availability for OpenAI output", () => { const configured = { ttsConfigured: true, ttsAvailable: true } as never; expect( isVoiceSetupReady(pocket, null, null, "parakeet", "openai", configured), @@ -63,6 +63,16 @@ describe("voice setup readiness", () => { ); }); + it("allows configured OpenAI input when TTS is unavailable", () => { + const configuredStt = { sttConfigured: true, ttsAvailable: false } as never; + expect( + isVoiceSetupReady(pocket, null, null, "openai", "pocket", configuredStt), + ).toBe(true); + expect(isVoiceSetupReady(pocket, null, null, "openai", "pocket")).toBe( + false, + ); + }); + it("uses native macOS speech readiness instead of Parakeet when selected", () => { expect( isVoiceSetupReady( diff --git a/src/features/voice-conversation/lib/voiceSetupReadiness.ts b/src/features/voice-conversation/lib/voiceSetupReadiness.ts index 196c853d9..fae141b14 100644 --- a/src/features/voice-conversation/lib/voiceSetupReadiness.ts +++ b/src/features/voice-conversation/lib/voiceSetupReadiness.ts @@ -15,13 +15,15 @@ export function isVoiceSetupReady( ): boolean { if (inputBackend === null) return false; const inputReady = - inputBackend === "macos" - ? Boolean( - macSpeech?.supported && - macSpeech.localeSupported && - macSpeech.modelInstalled, - ) - : Boolean(pocket?.parakeetInstalled); + inputBackend === "openai" + ? Boolean(openAi?.sttConfigured) + : inputBackend === "macos" + ? Boolean( + macSpeech?.supported && + macSpeech.localeSupported && + macSpeech.modelInstalled, + ) + : Boolean(pocket?.parakeetInstalled); if (!inputReady) return false; if (outputBackend === "openai") return Boolean(openAi?.ttsConfigured && openAi.ttsAvailable); diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index df674ba9d..214143f6a 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -52,7 +52,11 @@ const microphonePermissionState = vi.hoisted(() => ({ })); const openAiStatusState = vi.hoisted(() => ({ current: { + sttConfigured: true, ttsConfigured: true, + sttUnavailableReason: null, + ttsUnavailableReason: null, + transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -61,12 +65,16 @@ const openAiStatusState = vi.hoisted(() => ({ }, })); const openAiApiMocks = vi.hoisted(() => ({ + setSttApiKey: vi.fn(() => Promise.resolve()), + clearSttApiKey: vi.fn(() => Promise.resolve()), setTtsApiKey: vi.fn(() => Promise.resolve()), clearTtsApiKey: vi.fn(() => Promise.resolve()), })); vi.mock("../api/openAiVoice", () => ({ setOpenAiPlaybackSpeed: vi.fn(() => Promise.resolve()), + setOpenAiSttApiKey: openAiApiMocks.setSttApiKey, + clearOpenAiSttApiKey: openAiApiMocks.clearSttApiKey, setOpenAiTtsApiKey: openAiApiMocks.setTtsApiKey, clearOpenAiTtsApiKey: openAiApiMocks.clearTtsApiKey, })); @@ -217,7 +225,11 @@ describe("VoiceSettings", () => { interruptionState.mode = "automatic"; siriSetupState.current = siriSetup(); openAiStatusState.current = { + sttConfigured: true, ttsConfigured: true, + sttUnavailableReason: null, + ttsUnavailableReason: null, + transcriptionModel: "gpt-live-transcribe", speechModel: "gpt-4o-mini-tts", speechVoice: "marin", playbackSpeed: 1, @@ -226,19 +238,40 @@ describe("VoiceSettings", () => { }; openAiApiMocks.setTtsApiKey.mockClear(); openAiApiMocks.clearTtsApiKey.mockClear(); + openAiApiMocks.setSttApiKey.mockClear(); + openAiApiMocks.clearSttApiKey.mockClear(); }); - it("renders selected OpenAI output settings", async () => { + it("renders independently selected OpenAI input and output settings", async () => { + inputState.backend = "openai"; outputState.backend = "openai"; setupState.current = setup(pocketStatus()); renderWithProviders(); expect( - await screen.findByText(/gpt-4o-mini-tts.*marin voice/), + await screen.findByText("Uses gpt-live-transcribe."), + ).toBeInTheDocument(); + expect( + screen.getByText(/gpt-4o-mini-tts.*marin voice/), ).toBeInTheDocument(); expect(screen.getByText("Playback speed")).toBeInTheDocument(); }); + it("saves a dedicated OpenAI speech-to-text API key", async () => { + inputState.backend = "openai"; + setupState.current = setup(pocketStatus({ pocketInstalled: true })); + renderWithProviders(); + + const user = userEvent.setup(); + await user.type( + screen.getByLabelText("OpenAI speech-to-text API key"), + "stt-secret", + ); + await user.click(screen.getAllByRole("button", { name: "Save key" })[0]); + + expect(openAiApiMocks.setSttApiKey).toHaveBeenCalledWith("stt-secret"); + }); + it("saves a dedicated OpenAI text-to-speech API key", async () => { outputState.backend = "openai"; setupState.current = setup(pocketStatus({ parakeetInstalled: true })); @@ -254,20 +287,57 @@ describe("VoiceSettings", () => { expect(openAiApiMocks.setTtsApiKey).toHaveBeenCalledWith("tts-secret"); }); + it("uses OpenAI guidance when only the selected OpenAI input is not ready", async () => { + inputState.backend = "openai"; + outputState.backend = "pocket"; + openAiStatusState.current = { + ...openAiStatusState.current, + sttConfigured: false, + }; + setupState.current = setup(pocketStatus({ pocketInstalled: true })); + renderWithProviders(); + + expect( + await screen.findByText( + "OpenAI transcription is not ready. Add its speech-to-text API key below, then try again.", + ), + ).toBeInTheDocument(); + expect( + screen.queryByText(/Parakeet STT is not installed/), + ).not.toBeInTheDocument(); + }); + + it("does not show the TTS platform restriction for configured OpenAI input", async () => { + inputState.backend = "openai"; + outputState.backend = "pocket"; + openAiStatusState.current = { + ...openAiStatusState.current, + ttsAvailable: false, + unavailableReason: "unsupportedPlatform", + }; + setupState.current = setup(pocketStatus({ pocketInstalled: true })); + renderWithProviders(); + + expect( + await screen.findByText("Uses gpt-live-transcribe."), + ).toBeInTheDocument(); + expect( + screen.queryByText(/playback is currently supported on macOS only/), + ).not.toBeInTheDocument(); + }); + it("uses OpenAI guidance when the selected OpenAI output is not ready", async () => { outputState.backend = "openai"; openAiStatusState.current = { ...openAiStatusState.current, ttsConfigured: false, - unavailableReason: - "Add an OpenAI text-to-speech API key in Voice settings.", }; setupState.current = setup(pocketStatus({ parakeetInstalled: true })); renderWithProviders(); expect( await screen.findByText( - "OpenAI voice is not ready. Add the required API key below, then try again.", + "OpenAI voice playback is not ready. Add its text-to-speech API key below, then try again.", ), ).toBeInTheDocument(); expect( diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index f6239f4a5..0898756fd 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -16,7 +16,9 @@ import { } from "@/shared/ui/select"; import { useEffect, useState } from "react"; import { + clearOpenAiSttApiKey, clearOpenAiTtsApiKey, + setOpenAiSttApiKey, setOpenAiPlaybackSpeed, setOpenAiTtsApiKey, } from "../api/openAiVoice"; @@ -54,6 +56,11 @@ function readinessDescriptionKey( ): string | null { if (inputReady && outputReady) return null; if (!inputReady && !outputReady) { + if (inputBackend === "openai") { + return backend === "openai" + ? "voice.notReadyOpenAiSttAndTts" + : "voice.notReadyOpenAi"; + } if (backend === "openai") { return inputBackend === "macos" ? "voice.notReadyMacInputAndOpenAiOutput" @@ -69,11 +76,12 @@ function readinessDescriptionKey( : "voice.notReadyInputAndPocketOutput"; } if (!inputReady) { + if (inputBackend === "openai") return "voice.notReadyOpenAiStt"; return inputBackend === "macos" ? "voice.notReadyMacInput" : "voice.notReadyInput"; } - if (backend === "openai") return "voice.notReadyOpenAi"; + if (backend === "openai") return "voice.notReadyOpenAiTts"; return backend === "siri" ? "voice.notReadySiriOutput" : "voice.notReadyPocketOutput"; @@ -104,13 +112,15 @@ export function VoiceSettings() { const interruptionHeadingId = useId(); const interruptionDescriptionId = useId(); const inputReady = - input.backend === "macos" - ? Boolean( - macSpeechSetup.status?.supported && - macSpeechSetup.status.localeSupported && - macSpeechSetup.status.modelInstalled, - ) - : (setup.status?.parakeetInstalled ?? false); + input.backend === "openai" + ? (openAiStatus?.sttConfigured ?? false) + : input.backend === "macos" + ? Boolean( + macSpeechSetup.status?.supported && + macSpeechSetup.status.localeSupported && + macSpeechSetup.status.modelInstalled, + ) + : (setup.status?.parakeetInstalled ?? false); const outputReady = output.backend === "openai" ? Boolean(openAiStatus?.ttsConfigured && openAiStatus.ttsAvailable) @@ -204,6 +214,9 @@ export function VoiceSettings() { {t("voice.backendParakeet")} + + {t("voice.backendOpenAiStt")} + {macSpeechSetup.status?.supported && macSpeechSetup.status.localeSupported ? ( @@ -214,7 +227,27 @@ export function VoiceSettings() { )} details={ - input.backend === "macos" ? ( + input.backend === "openai" ? ( +
+ +

+ {openAiError ?? + openAiStatus?.sttUnavailableReason ?? + (openAiStatus + ? openAiStatus.sttConfigured + ? t("voice.openAiSttConfigured", { + model: openAiStatus.transcriptionModel, + }) + : t("voice.openAiSttNotConfigured") + : t("voice.openAiChecking"))} +

+
+ ) : input.backend === "macos" ? ( ) : input.backend === "parakeet" ? (

{openAiError ?? + openAiStatus?.ttsUnavailableReason ?? (openAiStatus?.unavailableReason === "unsupportedPlatform" ? t("voice.openAiTtsUnsupportedPlatform") : openAiStatus?.unavailableReason === "missingApiKey" diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index 47f61ac63..5b2072506 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -870,6 +870,7 @@ }, "voice": { "backendMacSpeech": "Apple speech recognition", + "backendOpenAiStt": "OpenAI speech-to-text", "backendOpenAiTts": "OpenAI text-to-speech", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", @@ -921,6 +922,9 @@ "notReadyMacInputAndPocketOutput": "Apple's on-device dictation model and Pocket TTS are not installed. Complete both steps below to use Voice Conversation.", "notReadyMacInputAndSiriOutput": "Apple's on-device dictation model is not installed, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", "notReadyOpenAi": "OpenAI voice is not ready. Add the required API key below, then try again.", + "notReadyOpenAiStt": "OpenAI transcription is not ready. Add its speech-to-text API key below, then try again.", + "notReadyOpenAiSttAndTts": "OpenAI transcription and voice playback are not ready. Add both of their separate API keys below, then try again.", + "notReadyOpenAiTts": "OpenAI voice playback is not ready. Add its text-to-speech API key below, then try again.", "notReadyPocketOutput": "Pocket TTS is not installed. Download it below to use Voice Conversation.", "notReadySiriOutput": "No installed Siri voice is selected. Download or select one below to use Voice Conversation.", "notReadyTitle": "Voice Conversation isn't ready", @@ -933,6 +937,9 @@ "openAiApiKeyNotConfigured": "This key is separate from provider credentials used by Goose.", "openAiApiKeySaved": "API key saved", "openAiChecking": "Checking OpenAI voice settings…", + "openAiSttApiKey": "OpenAI speech-to-text API key", + "openAiSttConfigured": "Uses {{model}}.", + "openAiSttNotConfigured": "Add a speech-to-text API key to use OpenAI transcription.", "openAiTtsApiKey": "OpenAI text-to-speech API key", "openAiTtsConfigured": "Uses {{model}} and the {{voice}} voice. OpenAI voices are AI-generated.", "openAiTtsNeedsKey": "Add an OpenAI text-to-speech API key to use this voice.", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 06b6650de..9590adf14 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -873,6 +873,7 @@ }, "voice": { "backendMacSpeech": "Reconocimiento de voz de Apple", + "backendOpenAiStt": "Voz a texto de OpenAI", "backendOpenAiTts": "Texto a voz de OpenAI", "backendParakeet": "Parakeet STT", "backendPocket": "Pocket TTS", @@ -924,6 +925,9 @@ "notReadyMacInputAndPocketOutput": "El modelo de dictado de Apple y Pocket TTS no están instalados. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInputAndSiriOutput": "El modelo de dictado de Apple no está instalado y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyOpenAi": "La voz de OpenAI no está lista. Añade abajo la clave API necesaria e inténtalo de nuevo.", + "notReadyOpenAiStt": "La transcripción de OpenAI no está lista. Añade abajo su clave API de voz a texto e inténtalo de nuevo.", + "notReadyOpenAiSttAndTts": "La transcripción y la reproducción de voz de OpenAI no están listas. Añade abajo sus dos claves API independientes e inténtalo de nuevo.", + "notReadyOpenAiTts": "La reproducción de voz de OpenAI no está lista. Añade abajo su clave API de texto a voz e inténtalo de nuevo.", "notReadyPocketOutput": "Pocket TTS no está instalado. Descárgalo abajo para usar la conversación por voz.", "notReadySiriOutput": "No hay ninguna voz de Siri instalada seleccionada. Descarga o selecciona una abajo para usar la conversación por voz.", "notReadyTitle": "La conversación por voz no está lista", @@ -936,6 +940,9 @@ "openAiApiKeyNotConfigured": "Esta clave es independiente de las credenciales de proveedor que usa Goose.", "openAiApiKeySaved": "Clave API guardada", "openAiChecking": "Comprobando los ajustes de voz de OpenAI…", + "openAiSttApiKey": "Clave API de voz a texto de OpenAI", + "openAiSttConfigured": "Usa {{model}}.", + "openAiSttNotConfigured": "Añade una clave API de voz a texto para usar la transcripción de OpenAI.", "openAiTtsApiKey": "Clave API de texto a voz de OpenAI", "openAiTtsConfigured": "Usa {{model}} y la voz {{voice}}. Las voces de OpenAI son generadas por IA.", "openAiTtsNeedsKey": "Añade una clave API de texto a voz de OpenAI para usar esta voz.",