diff --git a/memory-graph.md b/memory-graph.md new file mode 100644 index 000000000..c06c1819b --- /dev/null +++ b/memory-graph.md @@ -0,0 +1,34 @@ +# Memory Graph — OpenLess + +- slug: openless +- path: F:/编程/openless +- updated: 2026-09-20 + +## Summary + +Tauri 2 跨平台 AI 语音输入与选区助手，前端使用 React / TypeScript，原生层使用 Rust。 + +## Entities + +- OpenLess (Project): 唯一工作副本为 F:/编程/openless。 +- LiveTranscriptPill (Module): 共享实时转写胶囊，控制文字位置、入场和外框尺寸。 +- insertTextAnimation (Module): 字素差分、传播延迟和胶囊目标几何的纯函数模型。 + +## Relations + +- Capsule / TypelessCapsule --renders--> LiveTranscriptPill。 +- LiveTranscriptPill --uses--> insertTextAnimation / framer-motion。 + +## Facts + +- 文字使用稳定字素 key；水平位置以右边缘为锚，独立于垂直入场动画。 +- 新字从下方淡入并轻微上弹；字距 1.4px，旧字按新增批次分为 2–4 字组向左传播，组间 26ms，传播延迟上限 96ms。 +- 胶囊回中比左扩晚 140ms 启动，允许中途偏左，最终居中。 +- 文本可视区域跟随实时胶囊宽度，长文本保留右侧并裁剪左侧。 +- scripts/insert-text-motion-h5.mjs 在独立无头 Chrome 中验证轨迹、回中、快速纠错、长文本和减少动态效果。 + +## Decisions + +- 流式原文默认开启，字号默认14px、范围12–20px；通过UserPreferences保存并由prefs:changed即时同步。 +- 原文仅接收transcript_delta，忽略润色输出；关闭后保留波形与状态，原生窗口尺寸不变。 +- 浏览器验证不能替代打包后的原生窗口体验验证。 diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index d75302e3c..aa9b457b1 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -2542,6 +2542,7 @@ impl OpenLessBackend { &self.deps.task_spawner, Arc::clone(&resources), Arc::clone(&self.deps.dictation_engine).start_transcription_with_progress( + Arc::clone(&self.deps.task_spawner), session_id, Arc::clone(&context), Arc::clone(&partials) as Arc, diff --git a/openless-all/app/crates/openless-core/src/asr/bailian.rs b/openless-all/app/crates/openless-core/src/asr/bailian.rs index 9e2850e32..e5b01b872 100644 --- a/openless-all/app/crates/openless-core/src/asr/bailian.rs +++ b/openless-all/app/crates/openless-core/src/asr/bailian.rs @@ -46,14 +46,15 @@ const PER_ADDR_TCP_TIMEOUT: Duration = Duration::from_millis(1500); fn default_port_for_request( request: &tokio_tungstenite::tungstenite::handshake::client::Request, -) -> Result { +) -> Result> { let default_port = match request.uri().scheme_str() { Some("ws") => 80, Some("wss") => 443, _ => { return Err(WsError::Url( tokio_tungstenite::tungstenite::error::UrlError::UnsupportedUrlScheme, - )) + ) + .into()) } }; Ok(request.uri().port_u16().unwrap_or(default_port)) @@ -72,13 +73,14 @@ async fn connect_ws_to_addrs( WsStream, tokio_tungstenite::tungstenite::handshake::client::Response, ), - WsError, + Box, > { if addrs.is_empty() { return Err(WsError::Io(std::io::Error::new( std::io::ErrorKind::NotFound, "no addresses for websocket endpoint", - ))); + )) + .into()); } let mut last_err = None; @@ -101,12 +103,14 @@ async fn connect_ws_to_addrs( } } - Err(last_err.unwrap_or_else(|| { - WsError::Io(std::io::Error::new( - std::io::ErrorKind::NotConnected, - "no tcp candidate", - )) - })) + Err(last_err + .unwrap_or_else(|| { + WsError::Io(std::io::Error::new( + std::io::ErrorKind::NotConnected, + "no tcp candidate", + )) + }) + .into()) } async fn connect_ws_prefer_ipv4( @@ -116,7 +120,7 @@ async fn connect_ws_prefer_ipv4( WsStream, tokio_tungstenite::tungstenite::handshake::client::Response, ), - WsError, + Box, > { let port = default_port_for_request(&request)?; let host = request.uri().host().unwrap_or("").to_string(); @@ -528,36 +532,24 @@ impl BailianRealtimeASR { .and_then(Value::as_i64) .unwrap_or(0); - let mut delta: Option = None; - { + let snapshot = { let mut st = self.state.lock(); st.last_result_text = trimmed.to_string(); - if is_sentence_final { - // 所有 final 结果（含 sentence_id == 0）都存入 final_segments。 - // BTreeMap 覆盖语义保证同一 sentence_id 不会重复追加。 st.final_segments.insert(sentence_id, trimmed.to_string()); st.partial_segments.remove(&sentence_id); } else { - let previous = st - .partial_segments - .get(&sentence_id) - .map(String::as_str) - .unwrap_or(""); - delta = trimmed - .strip_prefix(previous) - .filter(|suffix| !suffix.is_empty()) - .map(str::to_string); st.partial_segments.insert(sentence_id, trimmed.to_string()); } - } - if let Some(delta) = delta { - if let Some(sink) = self.partial_sink.lock().clone() { - let _ = sink.publish(TextStreamChunk { - text: delta, - offset: 0, - }); - } + let mut segments = st.partial_segments.clone(); + segments.extend(st.final_segments.clone()); + merge_segments(&segments.into_values().collect::>()) + }; + if let Some(sink) = self.partial_sink.lock().clone() { + let _ = sink.publish(TextStreamChunk { + text: snapshot, + offset: 0, + }); } } @@ -813,6 +805,33 @@ async fn close_writer(writer: &SharedWriter) -> Result<(), BailianASRError> { mod tests { use super::*; + #[test] + fn transcript_snapshots_keep_prefix_and_recognition_corrections() { + let asr = create_test_asr(); + let sink = Arc::new(super::super::TranscriptCapture::default()); + asr.set_partial_sink(sink.clone()); + for (id, text, final_result) in [ + (0, "你", false), + (0, "你好", false), + (0, "您好", false), + (0, "您好。", true), + (1, "世", false), + (1, "世界", false), + (1, "世界！", true), + ] { + asr.record_result(&make_result_event(id, text, final_result)); + } + sink.assert_snapshots(&[ + "你", + "你好", + "您好", + "您好。", + "您好。世", + "您好。世界", + "您好。世界！", + ]); + } + // ---- helpers ---- fn make_result_event(sentence_id: i64, text: &str, is_final: bool) -> Value { @@ -869,13 +888,17 @@ mod tests { let request = "https://localhost/path".into_client_request().unwrap(); let explicit_port = "https://localhost:443/path".into_client_request().unwrap(); assert!(matches!( - default_port_for_request(&request), + default_port_for_request(&request) + .as_ref() + .map_err(|error| error.as_ref()), Err(WsError::Url( tokio_tungstenite::tungstenite::error::UrlError::UnsupportedUrlScheme )) )); assert!(matches!( - default_port_for_request(&explicit_port), + default_port_for_request(&explicit_port) + .as_ref() + .map_err(|error| error.as_ref()), Err(WsError::Url( tokio_tungstenite::tungstenite::error::UrlError::UnsupportedUrlScheme )) @@ -943,7 +966,7 @@ mod tests { let request = "ws://localhost/path".into_client_request().unwrap(); let result = connect_ws_to_addrs(request, Vec::new()).await; assert!( - matches!(result, Err(WsError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound) + matches!(result.as_ref().map_err(|error| error.as_ref()), Err(WsError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound) ); } diff --git a/openless-all/app/crates/openless-core/src/asr/mod.rs b/openless-all/app/crates/openless-core/src/asr/mod.rs index a8177bfea..63952c8e7 100644 --- a/openless-all/app/crates/openless-core/src/asr/mod.rs +++ b/openless-all/app/crates/openless-core/src/asr/mod.rs @@ -35,3 +35,28 @@ pub struct DictionaryHotword { pub phrase: String, pub enabled: bool, } + +#[cfg(test)] +#[derive(Default)] +pub(crate) struct TranscriptCapture(std::sync::Mutex>); +#[cfg(test)] +impl crate::ports::TextStreamSink for TranscriptCapture { + fn publish(&self, chunk: crate::ports::TextStreamChunk) -> Result<(), crate::BackendError> { + self.0.lock().unwrap().push(chunk); + Ok(()) + } +} +#[cfg(test)] +impl TranscriptCapture { + pub(crate) fn assert_snapshots(&self, expected: &[&str]) { + let chunks = self.0.lock().unwrap(); + assert!(chunks.iter().all(|chunk| chunk.offset == 0)); + assert_eq!( + chunks + .iter() + .map(|chunk| chunk.text.as_str()) + .collect::>(), + expected + ); + } +} diff --git a/openless-all/app/crates/openless-core/src/asr/qwen_realtime.rs b/openless-all/app/crates/openless-core/src/asr/qwen_realtime.rs index 7d8fb4c11..9af708573 100644 --- a/openless-all/app/crates/openless-core/src/asr/qwen_realtime.rs +++ b/openless-all/app/crates/openless-core/src/asr/qwen_realtime.rs @@ -131,6 +131,8 @@ struct SyncState { send_tx: Option>, /// VAD 断句后按到达顺序累积的已完成句段（completed.transcript）。 completed_segments: Vec, + // Per connection: ignore updates/repeats for completed item IDs. + completed_item_ids: std::collections::HashSet, /// 当前未完成句段的最新 interim 文本；completed 到达后清空。 /// 服务端在句段开放期把累积文本放 `stash`、精修期放 `text`，取非空者。 partial_text: String, @@ -473,22 +475,26 @@ impl Qwen3RealtimeASR { }); if let Some(text) = text { let text = text.trim(); - let delta = { + let snapshot = { let mut state = self.state.lock(); - let delta = text - .strip_prefix(&state.partial_text) - .unwrap_or("") - .to_string(); + if state.session_finished + || value + .get("item_id") + .and_then(Value::as_str) + .is_some_and(|id| state.completed_item_ids.contains(id)) + { + return; + } state.partial_text = text.to_string(); - delta + let mut segments = state.completed_segments.clone(); + segments.push(state.partial_text.clone()); + join_segments(&segments) }; - if !delta.is_empty() { - if let Some(sink) = self.partial_sink.lock().clone() { - let _ = sink.publish(TextStreamChunk { - text: delta, - offset: 0, - }); - } + if let Some(sink) = self.partial_sink.lock().clone() { + let _ = sink.publish(TextStreamChunk { + text: snapshot, + offset: 0, + }); } } } @@ -499,10 +505,26 @@ impl Qwen3RealtimeASR { }; let trimmed = transcript.trim(); let mut st = self.state.lock(); + if st.session_finished { + return; + } + if let Some(id) = value.get("item_id").and_then(Value::as_str) { + if !st.completed_item_ids.insert(id.to_owned()) { + return; + } + } if !trimmed.is_empty() { st.completed_segments.push(trimmed.to_string()); } st.partial_text.clear(); + let snapshot = join_segments(&st.completed_segments); + drop(st); + if let Some(sink) = self.partial_sink.lock().clone() { + let _ = sink.publish(TextStreamChunk { + text: snapshot, + offset: 0, + }); + } } fn finish_success(&self) { @@ -697,6 +719,33 @@ async fn close_writer(writer: &SharedWriter) -> Result<(), Qwen3ASRError> { mod tests { use super::*; + #[test] + fn transcript_snapshots_keep_prefix_and_recognition_corrections() { + let asr = create_test_asr(); + let sink = Arc::new(super::super::TranscriptCapture::default()); + asr.set_partial_sink(sink.clone()); + for text in ["你", "你好", "您好"] { + asr.record_partial(&serde_json::json!({"text": text})); + } + asr.record_completed(&serde_json::json!({"transcript": "您好。"})); + asr.record_partial(&serde_json::json!({"stash": "世界"})); + asr.record_completed(&serde_json::json!({"transcript": "世界！"})); + sink.assert_snapshots(&["你", "你好", "您好", "您好。", "您好。世界", "您好。世界！"]); + } + + #[test] + fn completed_item_ignores_late_partial_and_duplicate_but_accepts_next_item() { + let asr = create_test_asr(); + let sink = Arc::new(super::super::TranscriptCapture::default()); + asr.set_partial_sink(sink.clone()); + asr.record_partial(&serde_json::json!({"item_id":"a", "text":"你好"})); + asr.record_completed(&serde_json::json!({"item_id":"a", "transcript":"你好。"})); + asr.record_partial(&serde_json::json!({"item_id":"a", "text":"旧文字"})); + asr.record_completed(&serde_json::json!({"item_id":"a", "transcript":"你好。"})); + asr.record_partial(&serde_json::json!({"item_id":"b", "text":"世界"})); + sink.assert_snapshots(&["你好", "你好。", "你好。世界"]); + } + fn create_test_asr() -> Qwen3RealtimeASR { Qwen3RealtimeASR::new(Qwen3RealtimeCredentials { api_key: "sk-test".to_string(), diff --git a/openless-all/app/crates/openless-core/src/asr/stepfun_realtime.rs b/openless-all/app/crates/openless-core/src/asr/stepfun_realtime.rs index f40892713..db66fc251 100644 --- a/openless-all/app/crates/openless-core/src/asr/stepfun_realtime.rs +++ b/openless-all/app/crates/openless-core/src/asr/stepfun_realtime.rs @@ -170,6 +170,8 @@ struct SyncState { send_tx: Option>, /// VAD 断句后按到达顺序累积的已完成句段（completed.transcript）。 completed_segments: Vec, + // Per connection: ignore updates/repeats for completed item IDs. + completed_item_ids: std::collections::HashSet, /// 当前开放句段的 interim 全文 = delta.text（已确定前缀）+ delta.stash /// （未定尾巴）；completed 到达后清空。 partial_text: String, @@ -603,22 +605,26 @@ impl StepfunRealtimeASR { let combined = format!("{confirmed}{stash}"); let combined = combined.trim(); if !combined.is_empty() { - let delta = { + let snapshot = { let mut state = self.state.lock(); - let delta = combined - .strip_prefix(&state.partial_text) - .unwrap_or("") - .to_string(); + if state.session_finished + || value + .get("item_id") + .and_then(Value::as_str) + .is_some_and(|id| state.completed_item_ids.contains(id)) + { + return; + } state.partial_text = combined.to_string(); - delta + let mut segments = state.completed_segments.clone(); + segments.push(state.partial_text.clone()); + join_segments(&segments) }; - if !delta.is_empty() { - if let Some(sink) = self.partial_sink.lock().clone() { - let _ = sink.publish(TextStreamChunk { - text: delta, - offset: 0, - }); - } + if let Some(sink) = self.partial_sink.lock().clone() { + let _ = sink.publish(TextStreamChunk { + text: snapshot, + offset: 0, + }); } } } @@ -629,8 +635,16 @@ impl StepfunRealtimeASR { .get("transcript") .and_then(Value::as_str) .unwrap_or(""); - let should_finish = { + let (should_finish, snapshot) = { let mut st = self.state.lock(); + if st.session_finished { + return !st.session_finished; + } + if let Some(id) = value.get("item_id").and_then(Value::as_str) { + if !st.completed_item_ids.insert(id.to_owned()) { + return !st.session_finished; + } + } let trimmed = transcript.trim(); if !trimmed.is_empty() { st.completed_segments.push(trimmed.to_string()); @@ -638,11 +652,20 @@ impl StepfunRealtimeASR { st.partial_text.clear(); st.open_segments = st.open_segments.saturating_sub(1); st.last_completed_at = Some(Instant::now()); - st.finishing - && !st.tail_write_pending - && !has_audio_after_last_completed(&st) - && st.open_segments == 0 + ( + st.finishing + && !st.tail_write_pending + && !has_audio_after_last_completed(&st) + && st.open_segments == 0, + join_segments(&st.completed_segments), + ) }; + if let Some(sink) = self.partial_sink.lock().clone() { + let _ = sink.publish(TextStreamChunk { + text: snapshot, + offset: 0, + }); + } if should_finish { self.finish_success(); return false; @@ -884,8 +907,36 @@ async fn close_writer(writer: &SharedWriter) -> Result<(), StepfunASRError> { #[cfg(test)] mod tests { use super::*; + + #[test] + fn transcript_snapshots_keep_prefix_and_recognition_corrections() { + let asr = create_test_asr(); + let sink = Arc::new(super::super::TranscriptCapture::default()); + asr.set_partial_sink(sink.clone()); + for text in ["你", "你好", "您好"] { + asr.record_partial(&serde_json::json!({"text": text})); + } + asr.record_completed(&serde_json::json!({"transcript": "您好。"})); + asr.record_partial(&serde_json::json!({"stash": "世界"})); + asr.record_completed(&serde_json::json!({"transcript": "世界！"})); + sink.assert_snapshots(&["你", "你好", "您好", "您好。", "您好。世界", "您好。世界！"]); + } + use futures_util::{SinkExt, StreamExt}; + #[test] + fn completed_item_ignores_late_partial_and_duplicate_but_accepts_next_item() { + let asr = create_test_asr(); + let sink = Arc::new(super::super::TranscriptCapture::default()); + asr.set_partial_sink(sink.clone()); + asr.record_partial(&serde_json::json!({"item_id":"a", "text":"你好"})); + asr.record_completed(&serde_json::json!({"item_id":"a", "transcript":"你好。"})); + asr.record_partial(&serde_json::json!({"item_id":"a", "text":"旧文字"})); + asr.record_completed(&serde_json::json!({"item_id":"a", "transcript":"你好。"})); + asr.record_partial(&serde_json::json!({"item_id":"b", "text":"世界"})); + sink.assert_snapshots(&["你好", "你好。", "你好。世界"]); + } + fn create_test_asr() -> StepfunRealtimeASR { StepfunRealtimeASR::new(StepfunRealtimeCredentials { api_key: "sk-test".to_string(), diff --git a/openless-all/app/crates/openless-core/src/asr/tencent_cloud.rs b/openless-all/app/crates/openless-core/src/asr/tencent_cloud.rs index 4832843df..880dae0e9 100644 --- a/openless-all/app/crates/openless-core/src/asr/tencent_cloud.rs +++ b/openless-all/app/crates/openless-core/src/asr/tencent_cloud.rs @@ -409,32 +409,24 @@ impl TencentCloudStreamingASR { .get("slice_type") .and_then(Value::as_i64) .unwrap_or(1); - let mut state = self.state.lock(); - state.last_result_text = text.to_string(); - let mut delta = None; - if slice_type == 2 { - state.final_segments.insert(index, text.to_string()); - state.partial_segments.remove(&index); - } else { - let previous = state - .partial_segments - .get(&index) - .map(String::as_str) - .unwrap_or(""); - delta = text - .strip_prefix(previous) - .filter(|suffix| !suffix.is_empty()) - .map(str::to_string); - state.partial_segments.insert(index, text.to_string()); - } - drop(state); - if let Some(delta) = delta { - if let Some(sink) = self.partial_sink.lock().clone() { - let _ = sink.publish(TextStreamChunk { - text: delta, - offset: 0, - }); + let snapshot = { + let mut state = self.state.lock(); + state.last_result_text = text.to_string(); + if slice_type == 2 { + state.final_segments.insert(index, text.to_string()); + state.partial_segments.remove(&index); + } else { + state.partial_segments.insert(index, text.to_string()); } + let mut segments = state.partial_segments.clone(); + segments.extend(state.final_segments.clone()); + super::mimo::join_transcript_chunks(&segments.into_values().collect::>()) + }; + if let Some(sink) = self.partial_sink.lock().clone() { + let _ = sink.publish(TextStreamChunk { + text: snapshot, + offset: 0, + }); } } @@ -651,6 +643,34 @@ fn classify_server_error(code: i64) -> TencentCloudASRError { #[cfg(test)] mod tests { use super::*; + + #[test] + fn transcript_snapshots_keep_prefix_and_recognition_corrections() { + let asr = TencentCloudStreamingASR::new(credentials()); + let sink = Arc::new(super::super::TranscriptCapture::default()); + asr.set_partial_sink(sink.clone()); + for (id, text, final_result) in [ + (0, "你", false), + (0, "你好", false), + (0, "您好", false), + (0, "您好。", true), + (1, "世", false), + (1, "世界", false), + (1, "世界！", true), + ] { + asr.record_result(&serde_json::json!({"index": id, "voice_text_str": text, "slice_type": if final_result {2} else {1}})); + } + sink.assert_snapshots(&[ + "你", + "你好", + "您好", + "您好。", + "您好。世", + "您好。世界", + "您好。世界！", + ]); + } + use std::sync::atomic::{AtomicUsize, Ordering}; struct DelayedFirstSpawner { diff --git a/openless-all/app/crates/openless-core/src/asr/volcengine.rs b/openless-all/app/crates/openless-core/src/asr/volcengine.rs index 3beb8f576..c4dec7b95 100644 --- a/openless-all/app/crates/openless-core/src/asr/volcengine.rs +++ b/openless-all/app/crates/openless-core/src/asr/volcengine.rs @@ -754,25 +754,17 @@ impl VolcengineStreamingASR { } } - // 缓存最新的 partial transcript：服务端在 final 帧前断连时 fallback 用。 - // 仅在非空且不是 final 时更新（final 走另一条路径）。 - if !has_final && !full_text.is_empty() { - let delta = { - let mut state = self.state.lock(); - let delta = full_text - .strip_prefix(&state.last_partial_text) - .unwrap_or("") - .to_string(); - state.last_partial_text = full_text.clone(); - delta - }; - if !delta.is_empty() { - if let Some(sink) = self.partial_sink.lock().clone() { - let _ = sink.publish(TextStreamChunk { - text: delta, - offset: 0, - }); - } + // Offset zero replaces the current transcript, so publish the full snapshot, + // including recognition corrections rather than only an appended suffix. + if !full_text.is_empty() { + if !has_final { + self.state.lock().last_partial_text = full_text.clone(); + } + if let Some(sink) = self.partial_sink.lock().clone() { + let _ = sink.publish(TextStreamChunk { + text: full_text.clone(), + offset: 0, + }); } } @@ -982,6 +974,35 @@ fn hotword_context(entries: &[DictionaryHotword]) -> Option { mod tests { use super::*; + #[test] + fn transcript_snapshots_keep_prefix_and_recognition_corrections() { + let asr = VolcengineStreamingASR::new( + VolcengineCredentials { + service: VolcengineService::Standard, + auth_mode: VolcengineAuthMode::AppIdToken, + app_id: "test".into(), + access_token: "test".into(), + resource_id: VolcengineCredentials::default_resource_id().into(), + }, + Vec::new(), + ); + let sink = Arc::new(super::super::TranscriptCapture::default()); + asr.set_partial_sink(sink.clone()); + for text in ["你", "你好", "您好", "您好。世界"] { + let payload = + serde_json::to_vec(&serde_json::json!({"result": {"text": text}})).unwrap(); + let bytes = frame::build( + MessageType::FullServerResponse, + frame::Flags::None, + frame::Serialization::Json, + &payload, + None, + ); + assert!(asr.handle_frame(&bytes)); + } + sink.assert_snapshots(&["你", "你好", "您好", "您好。世界"]); + } + #[test] fn hotword_context_dedupes_case_insensitively_and_caps() { let mut entries = vec![ diff --git a/openless-all/app/crates/openless-core/src/asr/xfyun.rs b/openless-all/app/crates/openless-core/src/asr/xfyun.rs index e2587f7eb..3c612aceb 100644 --- a/openless-all/app/crates/openless-core/src/asr/xfyun.rs +++ b/openless-all/app/crates/openless-core/src/asr/xfyun.rs @@ -447,33 +447,24 @@ impl XfyunStreamingASR { return; } - let mut state = self.state.lock(); - state.last_result_text = trimmed.to_string(); - let mut delta = None; - if is_final { - // 最终结果：以 seg_id 去重覆盖，收尾按 seg_id 顺序拼接。 - state.final_segments.insert(seg_id, trimmed.to_string()); - state.partial_segments.remove(&seg_id); - } else { - let previous = state - .partial_segments - .get(&seg_id) - .map(String::as_str) - .unwrap_or(""); - delta = trimmed - .strip_prefix(previous) - .filter(|suffix| !suffix.is_empty()) - .map(str::to_string); - state.partial_segments.insert(seg_id, trimmed.to_string()); - } - drop(state); - if let Some(delta) = delta { - if let Some(sink) = self.partial_sink.lock().clone() { - let _ = sink.publish(TextStreamChunk { - text: delta, - offset: 0, - }); + let snapshot = { + let mut state = self.state.lock(); + state.last_result_text = trimmed.to_string(); + if is_final { + state.final_segments.insert(seg_id, trimmed.to_string()); + state.partial_segments.remove(&seg_id); + } else { + state.partial_segments.insert(seg_id, trimmed.to_string()); } + let mut segments = state.partial_segments.clone(); + segments.extend(state.final_segments.clone()); + super::mimo::join_transcript_chunks(&segments.into_values().collect::>()) + }; + if let Some(sink) = self.partial_sink.lock().clone() { + let _ = sink.publish(TextStreamChunk { + text: snapshot, + offset: 0, + }); } } @@ -690,6 +681,37 @@ fn classify_server_error(code: &str, desc: &str) -> XfyunASRError { mod tests { use super::*; + #[test] + fn transcript_snapshots_keep_prefix_and_recognition_corrections() { + let asr = XfyunStreamingASR::new(XfyunCredentials { + app_id: "test".into(), + api_key: "test".into(), + }); + let sink = Arc::new(super::super::TranscriptCapture::default()); + asr.set_partial_sink(sink.clone()); + for (id, text, final_result) in [ + (0, "你", false), + (0, "你好", false), + (0, "您好", false), + (0, "您好。", true), + (1, "世", false), + (1, "世界", false), + (1, "世界！", true), + ] { + let data = serde_json::json!({"seg_id": id, "cn": {"st": {"type": if final_result {"0"} else {"1"}, "rt": [{"ws": [{"cw": [{"w": text}]}]}]}}}); + asr.record_result(&serde_json::json!({"data": data.to_string()})); + } + sink.assert_snapshots(&[ + "你", + "你好", + "您好", + "您好。", + "您好。世", + "您好。世界", + "您好。世界！", + ]); + } + #[test] fn signa_matches_official_documentation_example() { // 官方文档示例：appid=595f23df，ts=1512041814，apiKey=d9f4aa7ea6d94faca62cd88a28fd5234 diff --git a/openless-all/app/crates/openless-core/src/cloud_sync.rs b/openless-all/app/crates/openless-core/src/cloud_sync.rs index b6d9eb264..7b1865671 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync.rs @@ -162,7 +162,7 @@ impl CloudSyncService { .parse::() .is_ok_and(|ip| ip.is_loopback()) }); - if (!matches!(url.scheme(), "https") && !(url.scheme() == "http" && loopback)) + if !(matches!(url.scheme(), "https") || url.scheme() == "http" && loopback) || !url.username().is_empty() || url.password().is_some() { diff --git a/openless-all/app/crates/openless-core/src/dictation_engine.rs b/openless-all/app/crates/openless-core/src/dictation_engine.rs index 4e081370e..d72836062 100644 --- a/openless-all/app/crates/openless-core/src/dictation_engine.rs +++ b/openless-all/app/crates/openless-core/src/dictation_engine.rs @@ -1011,6 +1011,7 @@ pub(crate) fn buffered_transcription_session( context: Arc, partials: Arc, progress: Arc, + task_spawner: Arc, ) -> Arc { let partials = if context.recording.transcribe_after_stop { Arc::new(DiscardTextStream) as Arc @@ -1021,7 +1022,7 @@ pub(crate) fn buffered_transcription_session( prepared, partials, progress, )); if !context.recording.transcribe_after_stop { - buffered.attach_in_background(); + buffered.attach_in_background(task_spawner); } buffered } @@ -1149,13 +1150,13 @@ impl BufferedTranscriptionSession { }) } - fn attach_in_background(&self) { + fn attach_in_background(&self, task_spawner: Arc) { let attaching = self.attach(); - tokio::spawn(async move { + task_spawner.spawn(Box::pin(async move { if let Err(error) = attaching.await { log::warn!("provider-only transcription startup failed: {error}"); } - }); + })); } fn prepared(&self) -> Arc { @@ -1909,6 +1910,53 @@ mod tests { } } + #[tokio::test] + async fn provider_only_capture_uses_host_spawner_and_stable_mode_defers_start() { + #[derive(Default)] + struct QueuedSpawner(Mutex>>); + impl crate::TaskSpawner for QueuedSpawner { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.0.lock().unwrap().push(task); + } + } + for stable in [false, true] { + let fixture = fixture_engine( + false, + Ok(crate::ports::PolishOutput::text("unused")), + None, + None, + ); + let spawner = Arc::new(QueuedSpawner::default()); + let mut context = DictationContext::default(); + context.recording.transcribe_after_stop = stable; + let session = Arc::new(fixture.engine) + .start_transcription_with_progress( + spawner.clone(), + SessionId::new(), + Arc::new(context), + Arc::new(DiscardTextStream), + Arc::new(NoopRecordingProgress), + ) + .await + .unwrap(); + session.consume_pcm_chunk(&[1, 0, 2, 0]); + assert_eq!(fixture.transcription_starts.load(Ordering::SeqCst), 0); + assert!(fixture.pcm.lock().unwrap().is_empty()); + let tasks = std::mem::take(&mut *spawner.0.lock().unwrap()); + assert_eq!(tasks.len(), usize::from(!stable)); + for task in tasks { + task.await; + } + if !stable { + assert_eq!(fixture.transcription_starts.load(Ordering::SeqCst), 1); + assert_eq!(*fixture.pcm.lock().unwrap(), vec![1, 0, 2, 0]); + } + assert_eq!(session.finish().await.unwrap().text, "raw text"); + assert_eq!(fixture.transcription_starts.load(Ordering::SeqCst), 1); + assert_eq!(*fixture.pcm.lock().unwrap(), vec![1, 0, 2, 0]); + } + } + #[tokio::test] async fn pipeline_streams_pcm_progress_and_terminal_deltas() { let fixture = fixture_engine( diff --git a/openless-all/app/crates/openless-core/src/domains.rs b/openless-all/app/crates/openless-core/src/domains.rs index e8d9ee790..45e1060ee 100644 --- a/openless-all/app/crates/openless-core/src/domains.rs +++ b/openless-all/app/crates/openless-core/src/domains.rs @@ -282,7 +282,7 @@ pub trait LocalAsrApi: Send + Sync { fn delete_model(&self, target: LocalAsrTarget) -> BoxFuture<'static, Result<(), BackendError>>; fn cleanup_incomplete( &self, - target: LocalAsrTarget, + _target: LocalAsrTarget, ) -> BoxFuture<'static, Result<(), BackendError>> { unsupported("local ASR incomplete download cleanup") } diff --git a/openless-all/app/crates/openless-core/src/external_audio.rs b/openless-all/app/crates/openless-core/src/external_audio.rs index 24d0cbf8d..977ba1d44 100644 --- a/openless-all/app/crates/openless-core/src/external_audio.rs +++ b/openless-all/app/crates/openless-core/src/external_audio.rs @@ -299,8 +299,10 @@ mod tests { std::env::temp_dir().join(format!("openless-remote-archive-{}", uuid::Uuid::new_v4())); let recorder = ExternalAudioRecorder::with_recordings_directory(directory.clone()); let id = SessionId::new(); - let mut context = DictationContext::default(); - context.audio_source = DictationAudioSource::External; + let mut context = DictationContext { + audio_source: DictationAudioSource::External, + ..DictationContext::default() + }; context.recording.archive_enabled = true; let consumer = Arc::new(RecordingConsumer::default()); let recording = recorder @@ -356,8 +358,10 @@ mod tests { } let recorder = ExternalAudioRecorder::with_recordings_directory(directory.clone()); let id = SessionId::new(); - let mut context = DictationContext::default(); - context.audio_source = DictationAudioSource::External; + let mut context = DictationContext { + audio_source: DictationAudioSource::External, + ..DictationContext::default() + }; context.recording.archive_enabled = enabled; let consumer = Arc::new(RecordingConsumer::default()); let recording = recorder @@ -389,8 +393,10 @@ mod tests { std::fs::create_dir_all(&directory).unwrap(); std::fs::write(directory.join("user.wav"), b"keep").unwrap(); let recorder = ExternalAudioRecorder::with_recordings_directory(directory.clone()); - let mut context = DictationContext::default(); - context.audio_source = DictationAudioSource::External; + let mut context = DictationContext { + audio_source: DictationAudioSource::External, + ..DictationContext::default() + }; context.recording.archive_enabled = true; context.recording.max_entries = Some(2); for _ in 0..4 { diff --git a/openless-all/app/crates/openless-core/src/external_audio/archive.rs b/openless-all/app/crates/openless-core/src/external_audio/archive.rs index 14a786195..0f6681204 100644 --- a/openless-all/app/crates/openless-core/src/external_audio/archive.rs +++ b/openless-all/app/crates/openless-core/src/external_audio/archive.rs @@ -50,7 +50,7 @@ impl ExternalRecordingArchive { entries.push((path, modified)); } } - entries.sort_by(|left, right| right.1.cmp(&left.1)); + entries.sort_by_key(|entry| std::cmp::Reverse(entry.1)); let cap = plan .max_entries .map(|count| (count as usize).clamp(1, crate::history::HISTORY_CAP)) diff --git a/openless-all/app/crates/openless-core/src/history.rs b/openless-all/app/crates/openless-core/src/history.rs index c72218037..8973570a4 100644 --- a/openless-all/app/crates/openless-core/src/history.rs +++ b/openless-all/app/crates/openless-core/src/history.rs @@ -153,6 +153,37 @@ mod tests { } } + #[test] + fn legacy_quick_note_history_survives_read_and_append() { + let path = std::env::temp_dir().join(format!( + "openless-legacy-history-{}.json", + uuid::Uuid::new_v4() + )); + let mut legacy = + serde_json::to_value(session("legacy", chrono::Utc::now().to_rfc3339())).unwrap(); + legacy["source"] = "quick_note".into(); + let bytes = serde_json::to_vec(&vec![legacy]).unwrap(); + std::fs::write(&path, &bytes).unwrap(); + let store = HistoryStore::at_path(path.clone()); + assert_eq!(store.list().unwrap()[0].source, HistorySource::QuickNote); + assert_eq!( + std::fs::read(&path).unwrap(), + bytes, + "reading must not rewrite history" + ); + store + .append_with_retention(session("new", chrono::Utc::now().to_rfc3339()), 0, None) + .unwrap(); + let entries = store.list().unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[1].source, HistorySource::QuickNote); + assert_eq!(entries[1].raw_transcript, "raw"); + let persisted: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + assert_eq!(persisted[1]["source"], "quick_note"); + std::fs::remove_file(path).unwrap(); + } + #[test] fn append_orders_caps_and_filters_retention() { let path = std::env::temp_dir().join(format!( diff --git a/openless-all/app/crates/openless-core/src/ports.rs b/openless-all/app/crates/openless-core/src/ports.rs index 0ba837b9d..7a78e7ed8 100644 --- a/openless-all/app/crates/openless-core/src/ports.rs +++ b/openless-all/app/crates/openless-core/src/ports.rs @@ -277,6 +277,7 @@ pub trait DictationEngine: Send + Sync + 'static { /// It uses the same buffered lifecycle as normal voice capture. fn start_transcription_with_progress( self: Arc, + task_spawner: Arc, session_id: SessionId, context: Arc, partials: Arc, @@ -286,7 +287,11 @@ pub trait DictationEngine: Send + Sync + 'static { Box::pin(async move { let prepared = preparation.await?; Ok(crate::dictation_engine::buffered_transcription_session( - prepared, context, partials, progress, + prepared, + context, + partials, + progress, + task_spawner, )) }) } diff --git a/openless-all/app/crates/openless-core/src/provider_service.rs b/openless-all/app/crates/openless-core/src/provider_service.rs index 47ab7db4b..cc5eac8e6 100644 --- a/openless-all/app/crates/openless-core/src/provider_service.rs +++ b/openless-all/app/crates/openless-core/src/provider_service.rs @@ -517,8 +517,8 @@ fn validate_provider_endpoint(endpoint: &str, allow_websocket: bool) -> Result<( let url = url::Url::parse(endpoint).map_err(|_| invalid_request("provider endpoint is invalid"))?; if url.host_str().is_none() - || !matches!(url.scheme(), "http" | "https") - && !(allow_websocket && matches!(url.scheme(), "ws" | "wss")) + || !(matches!(url.scheme(), "http" | "https") + || allow_websocket && matches!(url.scheme(), "ws" | "wss")) { return Err(invalid_request("provider endpoint is invalid")); } diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index 87c334826..a03e8f8c8 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -336,6 +336,10 @@ pub struct UserPreferences { /// 录音胶囊外观。偏好事件同步到各窗口，录音状态同时携带当前样式。 #[serde(default)] pub capsule_style: CapsuleStyle, + #[serde(default = "default_true")] + pub capsule_transcript_enabled: bool, + #[serde(default = "default_capsule_transcript_font_size")] + pub capsule_transcript_font_size: u8, /// 录音期间临时静音系统输出，停止/取消/出错后恢复原静音状态。 #[serde(default)] pub mute_during_recording: bool, @@ -782,6 +786,10 @@ struct UserPreferencesWire { show_capsule: bool, #[serde(default)] capsule_style: CapsuleStyle, + #[serde(default = "default_true")] + capsule_transcript_enabled: bool, + #[serde(default = "default_capsule_transcript_font_size")] + capsule_transcript_font_size: u8, #[serde(default)] mute_during_recording: bool, #[serde(default)] @@ -1019,6 +1027,8 @@ impl Default for UserPreferencesWire { launch_at_login: prefs.launch_at_login, show_capsule: prefs.show_capsule, capsule_style: prefs.capsule_style, + capsule_transcript_enabled: prefs.capsule_transcript_enabled, + capsule_transcript_font_size: prefs.capsule_transcript_font_size, mute_during_recording: prefs.mute_during_recording, stable_transcription_enabled: prefs.stable_transcription_enabled, audio_cue_on_record: prefs.audio_cue_on_record, @@ -1178,6 +1188,8 @@ impl<'de> Deserialize<'de> for UserPreferences { launch_at_login: wire.launch_at_login, show_capsule: wire.show_capsule, capsule_style: wire.capsule_style, + capsule_transcript_enabled: wire.capsule_transcript_enabled, + capsule_transcript_font_size: wire.capsule_transcript_font_size.clamp(12, 20), mute_during_recording: wire.mute_during_recording, stable_transcription_enabled: wire.stable_transcription_enabled, audio_cue_on_record: wire.audio_cue_on_record, @@ -1537,6 +1549,8 @@ impl Default for UserPreferences { launch_at_login: false, show_capsule: true, capsule_style: CapsuleStyle::Siri, + capsule_transcript_enabled: true, + capsule_transcript_font_size: default_capsule_transcript_font_size(), mute_during_recording: false, stable_transcription_enabled: false, audio_cue_on_record: true, @@ -3398,3 +3412,42 @@ mod tests { assert_eq!(json["polishMs"], 1450); } } + +fn default_capsule_transcript_font_size() -> u8 { + 14 +} + +#[cfg(test)] +mod capsule_transcript_preferences_tests { + use super::*; + #[test] + fn capsule_transcript_defaults_and_roundtrip() { + let mut value = serde_json::to_value(UserPreferences::default()).unwrap(); + value + .as_object_mut() + .unwrap() + .remove("capsuleTranscriptEnabled"); + value + .as_object_mut() + .unwrap() + .remove("capsuleTranscriptFontSize"); + let old: UserPreferences = serde_json::from_value(value).unwrap(); + assert!(old.capsule_transcript_enabled); + assert_eq!(old.capsule_transcript_font_size, 14); + let mut prefs = old; + prefs.capsule_transcript_enabled = false; + prefs.capsule_transcript_font_size = 20; + let restored: UserPreferences = + serde_json::from_slice(&serde_json::to_vec(&prefs).unwrap()).unwrap(); + assert!(!restored.capsule_transcript_enabled); + assert_eq!(restored.capsule_transcript_font_size, 20); + let mut value = serde_json::to_value(prefs).unwrap(); + value["capsuleTranscriptFontSize"] = 0.into(); + assert_eq!( + serde_json::from_value::(value) + .unwrap() + .capsule_transcript_font_size, + 12 + ); + } +} diff --git a/openless-all/app/crates/openless-core/src/types.rs b/openless-all/app/crates/openless-core/src/types.rs index 8b33cc455..78b54c347 100644 --- a/openless-all/app/crates/openless-core/src/types.rs +++ b/openless-all/app/crates/openless-core/src/types.rs @@ -58,6 +58,8 @@ pub enum HistorySource { Voice, SelectionPolish, SelectionVoiceEdit, + /// Kept for history written by the legacy quick-note feature. + QuickNote, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] diff --git a/openless-all/app/motion-preview.html b/openless-all/app/motion-preview.html new file mode 100644 index 000000000..cd858fff6 --- /dev/null +++ b/openless-all/app/motion-preview.html @@ -0,0 +1,33 @@ +
diff --git a/openless-all/app/scripts/insert-text-motion-h5.mjs b/openless-all/app/scripts/insert-text-motion-h5.mjs new file mode 100644 index 000000000..7d698f8ce --- /dev/null +++ b/openless-all/app/scripts/insert-text-motion-h5.mjs @@ -0,0 +1,307 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdirSync, writeFileSync, unlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, resolve } from 'node:path'; +import { createServer } from 'vite'; + +const out = process.env.OPENLESS_MOTION_ARTIFACT_DIR || join(tmpdir(), 'openless-motion-h5'); +mkdirSync(out, { recursive: true }); +const server = await createServer({ + server: { host: '127.0.0.1', port: 1438, strictPort: true, watch: null }, +}); +const fixture = resolve(`.insert-motion-h5-${process.pid}-${Date.now()}.html`); +writeFileSync( + fixture, + `
`, +); +await server.listen(); +const pageUrl = 'http://127.0.0.1:1438/' + basename(fixture); +const chrome = spawn( + process.env.CHROME_PATH || 'C:/Program Files/Google/Chrome/Application/chrome.exe', + [ + '--headless=new', + '--no-first-run', + '--no-default-browser-check', + '--remote-debugging-port=9437', + '--window-size=900,500', + '--user-data-dir=' + join(tmpdir(), 'openless-motion-' + Date.now()), + pageUrl, + ], + { stdio: 'ignore', windowsHide: true }, +); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +let socket; +try { + let target; + for (let i = 0; i < 100; i++) { + try { + target = (await (await fetch('http://127.0.0.1:9437/json/list')).json()).find( + (t) => t.type === 'page' && t.url === pageUrl, + ); + } catch {} + if (target) break; + await sleep(100); + } + assert(target, 'Chrome page target'); + socket = new WebSocket(target.webSocketDebuggerUrl); + await new Promise((r) => socket.addEventListener('open', r, { once: true })); + let id = 0; + const pending = new Map(); + socket.addEventListener('message', (e) => { + const m = JSON.parse(String(e.data)); + if (m.id) { + const p = pending.get(m.id); + pending.delete(m.id); + m.error ? p.reject(m.error) : p.resolve(m.result); + } + }); + const send = (method, params = {}) => + new Promise((resolve, reject) => { + pending.set(++id, { resolve, reject }); + socket.send(JSON.stringify({ id, method, params })); + }); + const evaluate = async (expression) => { + const r = await send('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + }); + if (r.exceptionDetails) + throw Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text); + return r.result.value; + }; + for (let i = 0; i < 150; i++) { + if (await evaluate('Boolean(window.ready)')) break; + await sleep(100); + } + assert(await evaluate('Boolean(window.ready)'), 'React fixture ready'); + await sleep(800); + const samples = await evaluate(`new Promise(resolve=>{ + const frames=[]; const start=performance.now();window.show('帮我查找一下'); + function tick(now){const shell=document.querySelector('.ol-live-transcript-pill').getBoundingClientRect(); + const glyphs=[...document.querySelectorAll('.ol-live-transcript-position')].map(e=>{const c=e.firstElementChild;const s=getComputedStyle(c);const r=e.getBoundingClientRect();return {text:e.textContent,x:r.x,y:new DOMMatrix(s.transform).m42,opacity:+s.opacity};}); + frames.push({t:now-start,left:shell.x,width:shell.width,glyphs});if(now-start<1200)requestAnimationFrame(tick);else resolve(frames);}requestAnimationFrame(tick); + })`); + writeFileSync(join(out, 'trajectory.json'), JSON.stringify(samples, null, 2)); + assert(samples.length > 15, 'animation sampled across frames'); + const settledGap = samples.at(-1).glyphs[2].x - samples.at(-1).glyphs[1].x; + assert( + samples + .filter((frame) => frame.glyphs[2].opacity > 0.25) + .every((frame) => frame.glyphs[2].x - frame.glyphs[1].x > settledGap * 0.7), + 'new glyph cannot appear on top of the retained tail', + ); + assert( + samples.some((f) => f.glyphs[2].y > 1), + 'new glyph rises from below', + ); + assert( + samples.some((f) => f.glyphs[2].y < -0.05), + 'new glyph gently overshoots baseline', + ); + const settled = samples.at(-1); + assert(Math.abs(settled.glyphs[2].y) < 0.1, 'glyph settles'); + assert( + settled.glyphs.every((g) => g.opacity > 0.99), + 'glyphs opaque', + ); + const center = await evaluate( + `(()=>{const r=document.getElementById('root').getBoundingClientRect();return r.x+r.width/2})()`, + ); + assert(Math.abs(settled.left + settled.width / 2 - center) < 0.2, 'shell recenters'); + assert( + samples.some((frame) => frame.left + frame.width / 2 < center - 8), + 'shell visibly leans left before recentering', + ); + await evaluate(`window.show('帮我查找上海的天气')`); + await sleep(1200); + const wave = await evaluate(`new Promise(resolve => { + const read = () => [...document.querySelectorAll('.ol-live-transcript-position')].slice(0,9).map(e => new DOMMatrix(getComputedStyle(e).transform).m41); + const before=read();const start=performance.now();const frames=[]; + window.show('帮我查找上海的天气预报吧'); + function tick(now){frames.push({t:now-start,delta:read().map((x,i)=>x-before[i])}); + if(now-start<800)requestAnimationFrame(tick);else resolve(frames);} + requestAnimationFrame(tick); + })`); + const onset = (index) => wave.find((frame) => frame.delta[index] < -0.4)?.t; + assert(onset(5) > onset(8) + 12, 'next group begins after the rightmost group'); + assert(Math.abs(onset(6) - onset(8)) < 16, 'three glyphs begin together'); + assert( + wave.some((frame) => frame.delta[8] < -8 && frame.delta[5] < -2 && frame.delta[0] < -0.5), + 'multiple groups move concurrently', + ); + writeFileSync(join(out, 'wave.json'), JSON.stringify(wave, null, 2)); + await evaluate(`window.show('帮我查找一下10六号')`); + await sleep(45); + await evaluate(`window.show('帮我查找一下10六号的天气')`); + await sleep(45); + await evaluate(`window.show('帮我查找一下十六号的天气')`); + await sleep(900); + assert.equal( + await evaluate(`document.querySelector('[role=status]').textContent`), + '帮我查找一下十六号的天气', + ); + await evaluate(`window.show('长文本测试'.repeat(25),true,'dark')`); + await sleep(1000); + const clip = await evaluate( + `(()=>{const t=document.querySelector('.ol-live-transcript-track');const p=document.querySelector('.ol-live-transcript-pill');const r=t.getBoundingClientRect();const b=p.getBoundingClientRect();return {overflow:getComputedStyle(t).overflow,width:b.width,left:r.left-b.left,right:b.right-r.right,count:document.querySelectorAll('button').length}})()`, + ); + assert.equal(clip.overflow, 'hidden'); + assert(clip.width <= 440.2); + assert(clip.left >= 45 && clip.right >= 45); + assert.equal(clip.count, 2); + await send('Emulation.setEmulatedMedia', { + features: [{ name: 'prefers-reduced-motion', value: 'reduce' }], + }); + await send('Page.reload'); + await sleep(1000); + await evaluate(`window.show('减少动态效果验证')`); + await sleep(80); + assert( + await evaluate( + `[...document.querySelectorAll('.ol-live-transcript-char')].every(e=>+getComputedStyle(e).opacity===1&&Math.abs(new DOMMatrix(getComputedStyle(e).transform).m42)<0.01)`, + ), + 'reduced motion immediate', + ); + await send('Emulation.setEmulatedMedia', { features: [] }); + await send('Page.reload'); + await sleep(900); + await evaluate(`window.show('帮我查找一下十六号的天气')`); + await sleep(1000); + // Optional real browser filmstrip; timestamps preserve capture cadence. + if (process.env.OPENLESS_MOTION_RECORD === '1') { + const frames = []; + socket.addEventListener('message', (event) => { + const message = JSON.parse(String(event.data)); + if (message.method !== 'Page.screencastFrame') return; + const file = `motion-${String(frames.length).padStart(4, '0')}.jpg`; + writeFileSync(join(out, file), Buffer.from(message.params.data, 'base64')); + frames.push({ file, time: message.params.metadata.timestamp }); + void send('Page.screencastFrameAck', { sessionId: message.params.sessionId }); + }); + await evaluate(`window.show('')`); + await sleep(600); + await send('Page.startScreencast', { format: 'jpeg', quality: 90, everyNthFrame: 1 }); + for (const [text, pause] of [ + ['帮我', 900], + ['帮我查找一', 650], + ['帮我查找一下', 650], + ['帮我查找一下10六号', 800], + ['帮我查找一下，16号', 1000], + ]) { + await evaluate(`window.show(${JSON.stringify(text)})`); + await sleep(pause); + } + await send('Page.stopScreencast'); + writeFileSync( + join(out, 'motion.ffconcat'), + 'ffconcat version 1.0\n' + + frames + .map( + (frame, i) => + `file '${frame.file}'\nduration ${Math.max(0.001, (frames[i + 1]?.time ?? frame.time + 0.5) - frame.time)}\n`, + ) + .join('') + + `file '${frames.at(-1).file}'\n`, + ); + } + const shot = await send('Page.captureScreenshot', { format: 'png' }); + writeFileSync(join(out, 'settled.png'), Buffer.from(shot.data, 'base64')); + await send('Page.navigate', { url: pageUrl }); + await sleep(1200); + await evaluate(`window.show('10六号')`); + await sleep(1000); + const correction = await evaluate(`new Promise(resolve=>{ + const frames=[];const start=performance.now();window.show('十六号'); + function tick(now){const glyphs=[...document.querySelectorAll('.ol-live-transcript-position')].map(e=>({x:e.getBoundingClientRect().x,width:e.getBoundingClientRect().width,opacity:+getComputedStyle(e.firstElementChild).opacity}));frames.push(glyphs); + if(now-start<600)requestAnimationFrame(tick);else resolve(frames);} + requestAnimationFrame(tick); + })`); + assert( + correction.every( + (frame) => frame[0].opacity < 0.1 || frame[0].x + frame[0].width <= frame[1].x + 1, + ), + 'middle correction never crosses retained suffix', + ); + // Exercise the production Capsule route, not just the isolated animation. + for (const style of ['siri', 'classic', 'typeless']) { + for (const enabled of [true, false]) { + await send('Emulation.setDeviceMetricsOverride', { + width: style === 'typeless' ? 206 : 460, + height: style === 'typeless' ? 57 : style === 'classic' ? 100 : 180, + deviceScaleFactor: 1, + mobile: false, + }); + await send('Page.navigate', { + url: `http://127.0.0.1:1438/?window=capsule&os=win&style=${style}&insertDemo=1&transcript=${enabled ? 1 : 0}&fontSize=20`, + }); + // Production demo simulates five recognition batches ending after 2440ms. + for (let attempt = 0; attempt < 150; attempt += 1) { + const mounted = await evaluate( + `Boolean(document.querySelector('.ol-live-transcript-pill, .ol-typeless-capsule, .ol-capsule-pill, canvas'))`, + ); + if (mounted) break; + await sleep(100); + } + await sleep(3200); + const result = await evaluate(`(() => { + const pills=[...document.querySelectorAll('.ol-live-transcript-pill')]; + const chars=[...document.querySelectorAll('.ol-live-transcript-char')]; + const rect=pills[0]?.getBoundingClientRect(); + return {pills:pills.length,chars:chars.length,width:rect?.width,font:chars[0]?parseFloat(getComputedStyle(chars[0]).fontSize):0,buttons:document.querySelectorAll('button').length}; + })()`); + if (result.pills !== (enabled ? 1 : 0)) { + writeFileSync( + join(out, 'integration-failure.txt'), + await evaluate('document.documentElement.outerHTML'), + ); + } + assert.equal(result.pills, enabled ? 1 : 0, `${style} display preference`); + if (enabled) { + assert(result.chars > 0, `${style} original text`); + assert(result.width <= (style === 'typeless' ? 206 : 460), `${style} bounded width`); + assert( + Math.abs(result.font * (style === 'typeless' ? 0.447 : 1) - 20) < 0.1, + `${style} visible font size`, + ); + } + if (style !== 'siri') assert(result.buttons >= 2, `${style} keeps controls`); + const shot = await send('Page.captureScreenshot', { format: 'png' }); + writeFileSync( + join(out, `${style}-${enabled ? 'text' : 'wave'}.png`), + Buffer.from(shot.data, 'base64'), + ); + } + } + console.log( + JSON.stringify( + { + passed: true, + frames: samples.length, + minY: Math.min(...samples.map((f) => f.glyphs[2].y)), + clip, + artifacts: resolve(out), + }, + null, + 2, + ), + ); +} finally { + socket?.close(); + chrome.kill(); + await server.close(); + unlinkSync(fixture); +} diff --git a/openless-all/app/src/components/Capsule.tsx b/openless-all/app/src/components/Capsule.tsx index e377f6961..40d21ef28 100644 --- a/openless-all/app/src/components/Capsule.tsx +++ b/openless-all/app/src/components/Capsule.tsx @@ -5,6 +5,8 @@ import { detectOS, type OS } from './WindowChrome'; import { warmUpSiriShaders } from './SiriGL'; import { VoiceOrbStage } from './VoiceOrbStage'; import { TypelessCapsule } from './TypelessCapsule'; +import { LiveTranscriptPill } from './LiveTranscriptPill'; +import { capsuleTranscriptFontSize, visibleCapsuleTranscript } from '../lib/capsuleTranscript'; import { getSettings } from '../lib/ipc/settings'; import { cancelDictation, stopDictation } from '../lib/ipc/dictation'; import { @@ -20,6 +22,7 @@ import type { CapsuleStyle, InsertFallbackCardPayload, PendingCorrection, + UserPreferences, } from '../lib/types'; import { VocabSuggestionCard } from './VocabSuggestionCard'; import { InsertFallbackCard } from './InsertFallbackCard'; @@ -500,6 +503,8 @@ interface ClassicCapsuleProps { level: number; insertedChars: number; message?: string; + transcript?: string; + transcriptFontSize?: number; operating?: boolean; translation: boolean; } @@ -514,6 +519,8 @@ function ClassicCapsule({ level, insertedChars, message, + transcript, + transcriptFontSize = 14, operating, translation, }: ClassicCapsuleProps) { @@ -526,6 +533,9 @@ function ClassicCapsule({ const onConfirm = useCallback(() => { void stopDictation(); }, []); + const liveText = transcript?.trim() ?? ''; + const recording = state === 'recording'; + const processing = state === 'transcribing' || state === 'polishing'; return ( <> @@ -571,16 +581,35 @@ function ClassicCapsule({ {t('capsule.translating')} - + {liveText ? ( + + ) : ( + + )} ); } @@ -613,6 +642,16 @@ const CAPSULE_PREVIEW_STATES: CapsuleState[] = [ 'error', ]; +/** 参考视频里的插入节奏：分片到达、全部并发、远处更晚启动。 */ +const INSERT_DEMO_CHUNKS = [ + '帮我', + '帮我查找一', + '帮我查找一下', + '帮我查找一下10', + '帮我查找一下10六号', +]; +const INSERT_DEMO_GAPS_MS = [420, 560, 640, 540, 280]; + function getPreviewCapsulePayload() { if (isTauri || typeof window === 'undefined') { return { @@ -623,6 +662,7 @@ function getPreviewCapsulePayload() { warming: false, selectionPolish: false, style: 'siri' as CapsuleStyle, + insertDemo: false, }; } @@ -632,14 +672,16 @@ function getPreviewCapsulePayload() { ? (stateParam as CapsuleState) : 'recording'; const previewLevel = Number(params.get('level') ?? 0.6); + const insertDemo = params.get('insertDemo') === '1'; return { - state: previewState, + state: insertDemo ? ('recording' as CapsuleState) : previewState, level: Number.isFinite(previewLevel) ? Math.min(1, Math.max(0, previewLevel)) : 0.6, message: params.get('message') ?? undefined, translation: params.get('translation') === '1', warming: params.get('warming') === '1', selectionPolish: params.get('selectionPolish') === '1', style: parseCapsuleStyle(params.get('style')) ?? 'siri', + insertDemo, }; } @@ -656,11 +698,22 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { const [level, setLevel] = useState(preview.level); const [message, setMessage] = useState(preview.message); const [localAsrText, setLocalAsrText] = useState(''); + const [transcriptEnabled, setTranscriptEnabled] = useState( + () => isTauri || new URLSearchParams(window.location.search).get('transcript') !== '0', + ); + const [transcriptFontSize, setTranscriptFontSize] = useState(() => + capsuleTranscriptFontSize( + isTauri + ? undefined + : Number(new URLSearchParams(window.location.search).get('fontSize') || 14), + ), + ); const transcriptViewRef = useRef({ sessionId: null, sequence: 0, text: '', }); + const capsuleStateRef = useRef(preview.state); const [translation, setTranslation] = useState(preview.translation); const [selectionPolish, setSelectionPolish] = useState(preview.selectionPolish); // 偏好事件即时换肤；录音状态携带同一个样式，保证首次显示也能正确呈现。 @@ -729,7 +782,16 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { setState(p.state); setLevel(p.level ?? 0); setMessage(p.message ?? undefined); - if (p.state === 'recording') setLocalAsrText(''); + const previous = capsuleStateRef.current; + capsuleStateRef.current = p.state; + if ( + p.state === 'recording' && + previous !== 'recording' && + previous !== 'transcribing' && + previous !== 'polishing' + ) { + setLocalAsrText(''); + } setTranslation(p.translation === true); setWarming(p.warming === true); setSelectionPolish(p.selectionPolish === true); @@ -780,7 +842,9 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { let preferenceRevision = 0; (async () => { const { listen } = await import('@tauri-apps/api/event'); - const handle = await listen<{ capsuleStyle?: CapsuleStyle }>('prefs:changed', (event) => { + const handle = await listen('prefs:changed', (event) => { + setTranscriptEnabled(event.payload.capsuleTranscriptEnabled ?? true); + setTranscriptFontSize(capsuleTranscriptFontSize(event.payload.capsuleTranscriptFontSize)); preferenceRevision += 1; const next = parseCapsuleStyle(event.payload?.capsuleStyle); if (next) { @@ -797,6 +861,8 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { const preferences = await getSettings(); const next = parseCapsuleStyle(preferences.capsuleStyle); if (!cancelled && revisionAtRead === preferenceRevision && next) { + setTranscriptEnabled(preferences.capsuleTranscriptEnabled ?? true); + setTranscriptFontSize(capsuleTranscriptFontSize(preferences.capsuleTranscriptFontSize)); stylePreferenceReadyRef.current = true; setCapsuleStyle(next); } @@ -864,6 +930,25 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { }); }, [warming]); + useEffect(() => { + if (!preview.insertDemo) return undefined; + let cancelled = false; + let elapsed = 0; + const timers: number[] = []; + INSERT_DEMO_CHUNKS.forEach((chunk, index) => { + elapsed += INSERT_DEMO_GAPS_MS[index] ?? 480; + timers.push( + window.setTimeout(() => { + if (!cancelled) setLocalAsrText(chunk); + }, elapsed), + ); + }); + return () => { + cancelled = true; + for (const timer of timers) window.clearTimeout(timer); + }; + }, [preview.insertDemo]); + // 兜底卡片排在最前：它是在会话收尾那一刻弹的，那一帧胶囊还在渲染 Done/Error 终态， // 而这次会话的结果恰恰是「没落进去」—— 让终态盖在上面等于报了个假的成功。 // 后端那边同步让路：卡片可见时 idle 隐藏不收窗口（见 capsule_focus.rs）。 @@ -891,6 +976,13 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { : state === 'transcribing' && localAsrText ? localAsrText : message; + const liveTranscript = visibleCapsuleTranscript( + localAsrText, + transcriptEnabled, + renderedState, + renderedSelectionPolish, + ); + const showLiveTranscript = liveTranscript.length > 0; return (
@@ -932,6 +1028,8 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { level={leaving ? 0 : level} insertedChars={insertedCharsRef.current} message={renderedMessage} + transcript={liveTranscript} + transcriptFontSize={transcriptFontSize} operating={operatingRef.current} translation={translation} warming={!leaving && warming} @@ -985,14 +1083,41 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { {t('capsule.translating')}
- +
+ +
+ {showLiveTranscript && ( +
+ +
+ )} ))} {renderedSelectionPolish && ( diff --git a/openless-all/app/src/components/CapsuleStyles.css b/openless-all/app/src/components/CapsuleStyles.css index d8fe3f444..b4311504c 100644 --- a/openless-all/app/src/components/CapsuleStyles.css +++ b/openless-all/app/src/components/CapsuleStyles.css @@ -242,6 +242,128 @@ opacity: 0.55; } } +.ol-live-transcript-stage { + position: relative; + pointer-events: none; + overflow: visible; +} + +.ol-live-transcript-pill { + position: absolute; + top: 0; + display: flex; + align-items: center; + justify-content: space-between; + gap: 6px; + height: 40px; + padding: 4px 6px; + border-radius: 999px; + box-sizing: border-box; + will-change: left, width; +} + +.ol-live-transcript-pill[data-tone='frost'] { + background: var(--ol-capsule-pill-bg); + border: 1px solid var(--ol-capsule-pill-border); + box-shadow: var(--ol-capsule-pill-shadow), var(--ol-capsule-pill-inset); +} + +.ol-live-transcript-pill[data-tone='dark'] { + background: var(--ol-typeless-bg, #18181b); + border: 1px solid var(--ol-typeless-border, #52525b); + box-shadow: + 0 5px 14px rgb(0 0 0 / 16%), + inset 0 1px 0 rgb(255 255 255 / 5%); +} + +.ol-live-transcript-track { + position: absolute; + top: 0; + height: 40px; + display: flex; + align-items: center; + justify-content: flex-end; + overflow: hidden; + font-family: var(--ol-font-sans); + font-size: 16px; + font-weight: 500; + line-height: 1.2; + letter-spacing: 0; + white-space: pre; + color: var(--ol-capsule-center-ink); + pointer-events: none; +} + +.ol-live-transcript-stage[data-tone='dark'] .ol-live-transcript-track { + color: var(--ol-typeless-ink, #fafafa); +} + +.ol-live-transcript-track[data-overflow='true'] { + mask-image: linear-gradient(to right, transparent, #000 22px); +} + +.ol-live-transcript-control { + display: inline-flex; + flex: 0 0 auto; + align-items: center; + justify-content: center; + padding: 0; + border: 0; + border-radius: 50%; + cursor: pointer; + color: var(--ol-capsule-btn-ink); + background: var(--ol-capsule-btn-bg); +} + +.ol-live-transcript-stage[data-tone='dark'] .ol-live-transcript-control { + color: var(--ol-typeless-ink, #fafafa); + background: var(--ol-typeless-button-bg, #3f3f46); +} + +.ol-live-transcript-confirm { + background: var(--ol-capsule-btn-bg-confirm); +} + +.ol-live-transcript-stage[data-tone='dark'] .ol-live-transcript-confirm { + color: var(--ol-typeless-confirm-ink, #18181b); + background: var(--ol-typeless-confirm-bg, #fafafa); +} + +.ol-live-transcript-control:disabled { + opacity: 0.42; + cursor: default; +} + +.ol-live-transcript-control:focus-visible { + outline: 2px solid #79b0ff; + outline-offset: 2px; +} + +.ol-live-transcript-position { + position: absolute; + left: 100%; + flex: none; + will-change: transform; +} + +.ol-live-transcript-char { + display: inline-block; + transform-origin: 50% 85%; + will-change: transform, opacity, filter; +} + +.ol-live-transcript-sr { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + @media (prefers-reduced-motion: reduce) { .ol-typeless-capsule, .ol-typeless-recording, @@ -252,4 +374,9 @@ .ol-typeless-capsule[data-processing='true'] .ol-typeless-status > span { animation: none; } + .ol-live-transcript-pill, + .ol-live-transcript-char, + .ol-live-transcript-position { + will-change: auto; + } } diff --git a/openless-all/app/src/components/LiveTranscriptPill.tsx b/openless-all/app/src/components/LiveTranscriptPill.tsx new file mode 100644 index 000000000..913dd03ae --- /dev/null +++ b/openless-all/app/src/components/LiveTranscriptPill.tsx @@ -0,0 +1,314 @@ +import { animate, motion, useMotionValue, useReducedMotion, useTransform } from 'framer-motion'; +import { useEffect, useMemo, useState } from 'react'; +import './CapsuleStyles.css'; +import { + INSERT_TEXT_MOTION, + appendedBirthAdvance, + clampInsertContentWidth, + diffInsertUnits, + firstBornIndex, + planCapsuleInsertMotion, + planCharDelays, + prefixWidths, + rightAnchoredPositions, + type InsertUnit, +} from '../lib/insertTextAnimation'; +import { Icon } from './Icon'; + +export type LiveTranscriptTone = 'frost' | 'dark'; + +export interface LiveTranscriptPillProps { + text: string; + tone?: LiveTranscriptTone; + stageWidth: number; + maxWidth: number; + minWidth?: number; + height?: number; + onCancel?: () => void; + onConfirm?: () => void; + cancelEnabled?: boolean; + confirmEnabled?: boolean; + cancelLabel?: string; + confirmLabel?: string; + controlSize?: number; + fontSize?: number; +} + +const FONT_SIZE = 16; +const FONT_WEIGHT = 500; +const DEFAULT_PILL_HEIGHT = 40; + +let measureCanvas: HTMLCanvasElement | null = null; + +function getSansFont(): string { + if (typeof window === 'undefined') return 'system-ui, sans-serif'; + const token = getComputedStyle(document.documentElement) + .getPropertyValue('--ol-font-sans') + .trim(); + return token || 'system-ui, sans-serif'; +} + +function measureGlyphWidths(glyphs: string[], fontSize: number): number[] { + if (glyphs.length === 0) return []; + if (typeof document === 'undefined') { + return glyphs.map((glyph) => Math.max(fontSize * 0.92, glyph.length * fontSize)); + } + if (!measureCanvas) measureCanvas = document.createElement('canvas'); + const ctx = measureCanvas.getContext('2d'); + if (!ctx) return glyphs.map(() => fontSize); + ctx.font = `${FONT_WEIGHT} ${fontSize}px ${getSansFont()}`; + return glyphs.map((glyph) => { + const raw = ctx.measureText(glyph === ' ' ? '\u00a0' : glyph).width; + return Math.max(1, raw) + INSERT_TEXT_MOTION.glyphGapPx; + }); +} + +export function LiveTranscriptPill({ + text, + tone = 'frost', + stageWidth, + maxWidth, + minWidth = INSERT_TEXT_MOTION.minWidth, + height = DEFAULT_PILL_HEIGHT, + onCancel, + onConfirm, + cancelEnabled = true, + confirmEnabled = true, + cancelLabel, + confirmLabel, + controlSize = 28, + fontSize = FONT_SIZE, +}: LiveTranscriptPillProps) { + const reduceMotion = useReducedMotion(); + const hasControls = Boolean(onCancel || onConfirm); + const padX = INSERT_TEXT_MOTION.padX + (hasControls ? controlSize + 6 : 0); + const pillMinWidth = hasControls ? Math.max(minWidth, controlSize * 2 + 72) : minWidth; + const [units, setUnits] = useState(() => diffInsertUnits([], text)); + const [seenText, setSeenText] = useState(text); + if (text !== seenText) { + setSeenText(text); + setUnits((prev) => diffInsertUnits(prev, text)); + } + + const widths = useMemo( + () => + measureGlyphWidths( + units.map((unit) => unit.text), + fontSize, + ), + [units, fontSize], + ); + const contentWidth = useMemo( + () => + clampInsertContentWidth( + widths.reduce((sum, width) => sum + width, 0), + maxWidth, + padX, + ), + [widths, maxWidth, padX], + ); + const positions = useMemo(() => rightAnchoredPositions(widths), [widths]); + // Start appended glyphs beyond the old tail, so they never appear on top of + // retained text while the displacement spring is still gathering speed. + const birthAdvance = appendedBirthAdvance(units, widths); + const originIndex = firstBornIndex(units); + const starts = useMemo(() => prefixWidths(widths), [widths]); + const originX = (starts[originIndex] ?? 0) + (widths[originIndex] ?? 0) / 2; + const insertedCount = units.filter((unit) => unit.born).length; + const delays = useMemo( + () => planCharDelays(widths, originIndex, insertedCount), + [widths, originIndex, insertedCount], + ); + const plan = useMemo( + () => + planCapsuleInsertMotion({ + stageWidth, + contentWidth, + originX, + minWidth: pillMinWidth, + maxWidth, + padX, + }), + [stageWidth, contentWidth, originX, pillMinWidth, maxWidth, padX], + ); + + const widthMv = useMotionValue(pillMinWidth); + const rightEdgeMv = useMotionValue(stageWidth / 2 + pillMinWidth / 2); + // Delayed recentering may otherwise push a large batch beyond the native + // WebView's left edge. Constrain the rendered frame, not spring momentum. + const visibleRightMv = useTransform([rightEdgeMv, widthMv], ([right, width]) => + Math.max(Number(width), Math.min(stageWidth, Number(right))), + ); + const leftMv = useTransform( + [visibleRightMv, widthMv], + ([right, width]) => Number(right) - Number(width), + ); + const trackRightMv = useTransform(visibleRightMv, (right) => stageWidth - Number(right) + padX); + + const trackWidthMv = useTransform(widthMv, (width) => Math.max(1, width - padX * 2)); + + useEffect(() => { + if (reduceMotion) { + widthMv.set(plan.width); + rightEdgeMv.set(plan.rightEdge); + return undefined; + } + const widthAnim = animate(widthMv, plan.width, { + type: 'spring', + ...INSERT_TEXT_MOTION.widthSpring, + delay: plan.widthDelayMs / 1000, + }); + const rightAnim = animate(rightEdgeMv, plan.rightEdge, { + type: 'spring', + ...INSERT_TEXT_MOTION.recenterSpring, + delay: plan.recenterDelayMs / 1000, + }); + return () => { + widthAnim.stop(); + rightAnim.stop(); + }; + }, [ + plan.width, + plan.rightEdge, + plan.widthDelayMs, + plan.recenterDelayMs, + reduceMotion, + rightEdgeMv, + widthMv, + ]); + + const overflowing = contentWidth + padX * 2 >= maxWidth - 0.5; + + return ( +
+ + {onCancel && ( + + )} + {onConfirm && ( + + )} + + + {units.map((unit, index) => ( + + ))} + + + {text} + +
+ ); +} + +/** Birth timing belongs to this keyed glyph, not to later recognition revisions. */ +function TranscriptGlyph({ + text, + delay, + reduceMotion, +}: { + text: string; + delay: number; + reduceMotion: boolean; +}) { + const [birthDelay] = useState(delay); + return ( + + {text === ' ' ? '\u00a0' : text} + + ); +} diff --git a/openless-all/app/src/components/TypelessCapsule.tsx b/openless-all/app/src/components/TypelessCapsule.tsx index b9d75f382..14fd9c241 100644 --- a/openless-all/app/src/components/TypelessCapsule.tsx +++ b/openless-all/app/src/components/TypelessCapsule.tsx @@ -4,12 +4,15 @@ import { cancelDictation, stopDictation } from '../lib/ipc/dictation'; import type { CapsuleState, CapsuleStyle } from '../lib/types'; import { Icon } from './Icon'; import { VoiceOrbStage } from './VoiceOrbStage'; +import { LiveTranscriptPill } from './LiveTranscriptPill'; import './CapsuleStyles.css'; interface TypelessCapsuleProps { state: CapsuleState; level: number; message?: string; + transcript?: string; + transcriptFontSize?: number; insertedChars?: number; operating?: boolean; translation?: boolean; @@ -36,6 +39,8 @@ export function TypelessCapsule({ state, level, message, + transcript, + transcriptFontSize = 14, insertedChars = 0, operating = false, translation = false, @@ -45,6 +50,7 @@ export function TypelessCapsule({ const { t } = useTranslation(); const recording = state === 'recording'; const processing = state === 'transcribing' || state === 'polishing'; + const liveText = transcript?.trim() ?? ''; const label = processing ? t(operating ? 'capsule.using' : 'capsule.thinking') : state === 'done' @@ -55,6 +61,30 @@ export function TypelessCapsule({ const cancel = useCallback(() => void cancelDictation(), []); const confirm = useCallback(() => void stopDictation(), []); + if (liveText) { + return ( +
+ {translation && {t('capsule.translating')}} + +
+ ); + } + return (
{translation && {t('capsule.translating')}} diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index ea3628ad5..324aadbf7 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -1116,6 +1116,10 @@ export const de: typeof zhCN = { capsuleDesc: 'Zeigt während der Aufnahme eine halbtransparente Kapsel am unteren Bildschirmrand.', capsuleStyleTypeless: 'Kompakter Typeless-Stil', + capsuleTranscriptLabel: 'Live-Transkript anzeigen', + capsuleTranscriptDesc: + 'Erkannten Originaltext statt Audiobalken anzeigen. Ohne Streaming erscheint der Text nach der Erkennung. Deaktivieren stellt die bisherige Anzeige wieder her.', + capsuleTranscriptFontSize: 'Transkript-Schriftgröße', capsuleStyleLabel: 'Kapselstil', capsuleStyleSiri: 'Schimmernder Siri-Stil', capsuleStyleClassic: 'OpenLess-Standardstil', diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index bf2d03b75..a8eed923b 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -1092,6 +1092,10 @@ export const en: typeof zhCN = { capsuleLabel: 'Recording capsule', capsuleDesc: 'Show a translucent capsule at the bottom of the screen while recording.', capsuleStyleTypeless: 'Typeless compact style', + capsuleTranscriptLabel: 'Show live transcript', + capsuleTranscriptDesc: + 'Show recognized speech in the capsule instead of audio bars. Turn off to restore the original display. Non-streaming recognition appears when results arrive.', + capsuleTranscriptFontSize: 'Transcript font size', capsuleStyleLabel: 'Capsule style', capsuleStyleSiri: 'Shimmer Siri style', capsuleStyleClassic: 'OpenLess default style', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index 0c71ec6f2..de967466b 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -1111,6 +1111,10 @@ export const es: typeof zhCN = { capsuleLabel: 'Cápsula de grabación', capsuleDesc: 'Muestra una cápsula en la parte inferior de la pantalla mientras grabas.', capsuleStyleTypeless: 'Estilo compacto Typeless', + capsuleTranscriptLabel: 'Mostrar transcripción en directo', + capsuleTranscriptDesc: + 'Muestra el texto reconocido en lugar de las barras de audio. Sin streaming, aparece al recibir el resultado. Desactiva para restaurar la vista original.', + capsuleTranscriptFontSize: 'Tamaño de la transcripción', capsuleStyleLabel: 'Estilo de cápsula', capsuleStyleSiri: 'Estilo luminoso Siri', capsuleStyleClassic: 'Estilo predeterminado de OpenLess', diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index 692b1e9d2..da0c74c9c 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -1123,6 +1123,10 @@ export const fr: typeof zhCN = { capsuleLabel: 'Capsule d’enregistrement', capsuleDesc: 'Affiche une capsule en bas de l’écran pendant l’enregistrement.', capsuleStyleTypeless: 'Style compact Typeless', + capsuleTranscriptLabel: 'Afficher la transcription en direct', + capsuleTranscriptDesc: + 'Affiche le texte reconnu à la place des barres audio. Sans streaming, le texte apparaît à réception du résultat. Désactivez pour retrouver l’affichage initial.', + capsuleTranscriptFontSize: 'Taille de la transcription', capsuleStyleLabel: 'Style de capsule', capsuleStyleSiri: 'Style lumineux Siri', capsuleStyleClassic: 'Style par défaut d’OpenLess', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 7e5440262..656553451 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -1080,6 +1080,10 @@ export const ja: typeof zhCN = { capsuleLabel: '録音カプセル', capsuleDesc: '録音 / 転写中、画面下部に半透明のカプセルを表示。', capsuleStyleTypeless: 'Typeless コンパクトスタイル', + capsuleTranscriptLabel: '文字起こしをリアルタイム表示', + capsuleTranscriptDesc: + '音声バーの代わりに認識した原文を表示します。オフにすると元の表示に戻ります。非ストリーミング認識は結果の受信後に表示します。', + capsuleTranscriptFontSize: '文字起こしの文字サイズ', capsuleStyleLabel: 'カプセルスタイル', capsuleStyleSiri: '光条 Siri スタイル', capsuleStyleClassic: 'Openless デフォルトスタイル', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 6dc745a55..71c523691 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -1072,6 +1072,10 @@ export const ko: typeof zhCN = { capsuleLabel: '녹음 캡슐', capsuleDesc: '녹음 / 전사 중 화면 하단에 반투명 캡슐을 표시합니다.', capsuleStyleTypeless: 'Typeless 컴팩트 스타일', + capsuleTranscriptLabel: '실시간 원문 표시', + capsuleTranscriptDesc: + '오디오 막대 대신 인식된 원문을 표시합니다. 끄면 기존 표시로 돌아갑니다. 비스트리밍 인식은 결과 수신 후 표시됩니다.', + capsuleTranscriptFontSize: '원문 글자 크기', capsuleStyleLabel: '캡슐 스타일', capsuleStyleSiri: '시리 광선 스타일', capsuleStyleClassic: 'Openless 기본 스타일', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index f503982eb..084badb9c 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -1044,6 +1044,10 @@ export const zhCN = { capsuleLabel: '录音胶囊', capsuleDesc: '录音 / 转写时显示屏幕底部胶囊。', capsuleStyleTypeless: 'Typeless 传统风格', + capsuleTranscriptLabel: '显示流式原文', + capsuleTranscriptDesc: + '在胶囊中显示语音识别原文，替换音频条；关闭后恢复原有显示。非流式识别将在结果返回后显示。', + capsuleTranscriptFontSize: '原文字号', capsuleStyleLabel: '胶囊样式', capsuleStyleSiri: '流光 Siri 风格', capsuleStyleClassic: 'Openless 默认风格', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index e2b3cff71..f5ea070c7 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -1057,6 +1057,10 @@ export const zhTW: typeof zhCN = { capsuleLabel: '錄音膠囊', capsuleDesc: '錄音 / 轉寫時在屏幕底部顯示半透明膠囊。', capsuleStyleTypeless: 'Typeless 傳統風格', + capsuleTranscriptLabel: '顯示串流原文', + capsuleTranscriptDesc: + '在膠囊中顯示語音辨識原文，取代音訊條；關閉後恢復原有顯示。非串流辨識將於結果傳回後顯示。', + capsuleTranscriptFontSize: '原文字級', capsuleStyleLabel: '膠囊樣式', capsuleStyleSiri: '流光 Siri 風格', capsuleStyleClassic: 'Openless 預設風格', diff --git a/openless-all/app/src/lib/backendEvent.test.ts b/openless-all/app/src/lib/backendEvent.test.ts index 35c7e4f4b..4962a3201 100644 --- a/openless-all/app/src/lib/backendEvent.test.ts +++ b/openless-all/app/src/lib/backendEvent.test.ts @@ -38,3 +38,38 @@ state = applyTranscriptEvent(state, { assertState(state, '你好🙂', 2); console.log('backendEvent.test.ts passed'); + +state = applyTranscriptEvent(state, { + sequence: 4, + sessionId: 'a', + kind: { type: 'polish_delta', payload: { text: '润色结果', offset: 0 } }, +}); +assertState(state, '你好🙂', 4); +state = applyTranscriptEvent(state, { + sequence: 5, + sessionId: 'b', + kind: { type: 'dictation_state_changed', payload: { phase: 'starting', sessionId: 'b' } }, +}); +assertState(state, '', 5); +state = applyTranscriptEvent(state, { + sequence: 6, + sessionId: 'a', + kind: { type: 'transcript_delta', payload: { text: '旧会话', offset: 0, isFinal: true } }, +}); +assertState(state, '', 5); +state = applyTranscriptEvent(state, { + sequence: 7, + sessionId: 'b', + kind: { type: 'transcript_delta', payload: { text: '新会话', offset: 0, isFinal: false } }, +}); +assertState(state, '新会话', 7); + +// Cloud ASR snapshots must retain all preceding words and apply corrections. +let cloudState: TranscriptViewState = { sessionId: null, sequence: 0, text: '' }; +for (const [index, text] of ['你', '你好', '您好', '您好。', '您好。世界'].entries()) { + cloudState = applyTranscriptEvent(cloudState, { + sequence: index + 1, sessionId: 'cloud', + kind: { type: 'transcript_delta', payload: { text, offset: 0, isFinal: false } }, + }); + assertState(cloudState, text, index + 1); +} diff --git a/openless-all/app/src/lib/capsuleTranscript.test.ts b/openless-all/app/src/lib/capsuleTranscript.test.ts new file mode 100644 index 000000000..440f44839 --- /dev/null +++ b/openless-all/app/src/lib/capsuleTranscript.test.ts @@ -0,0 +1,16 @@ +const assert = { + equal(actual: unknown, expected: unknown) { + if (actual !== expected) throw new Error(`Expected ${String(expected)}, got ${String(actual)}`); + }, +}; +import { capsuleTranscriptFontSize, visibleCapsuleTranscript } from './capsuleTranscript'; +assert.equal(capsuleTranscriptFontSize(undefined), 14); +assert.equal(capsuleTranscriptFontSize(NaN), 14); +assert.equal(capsuleTranscriptFontSize(50), 20); +assert.equal(capsuleTranscriptFontSize(0), 12); +assert.equal(visibleCapsuleTranscript('原文', false, 'recording', false), ''); +assert.equal(visibleCapsuleTranscript('原文', true, 'recording', false), '原文'); +assert.equal(visibleCapsuleTranscript('原文', true, 'polishing', false), '原文'); +assert.equal(visibleCapsuleTranscript('原文', true, 'done', false), ''); +assert.equal(visibleCapsuleTranscript('原文', true, 'recording', true), ''); +console.log('capsuleTranscript tests passed'); diff --git a/openless-all/app/src/lib/capsuleTranscript.ts b/openless-all/app/src/lib/capsuleTranscript.ts new file mode 100644 index 000000000..856845c35 --- /dev/null +++ b/openless-all/app/src/lib/capsuleTranscript.ts @@ -0,0 +1,18 @@ +import type { CapsuleState } from './types'; + +export function capsuleTranscriptFontSize(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) + ? Math.min(20, Math.max(12, value)) + : 14; +} + +export function visibleCapsuleTranscript( + text: string, + enabled: boolean, + state: CapsuleState, + selectionPolish: boolean, +): string { + return enabled && !selectionPolish && ['recording', 'transcribing', 'polishing'].includes(state) + ? text.trim() + : ''; +} diff --git a/openless-all/app/src/lib/insertTextAnimation.test.ts b/openless-all/app/src/lib/insertTextAnimation.test.ts new file mode 100644 index 000000000..000d1ce52 --- /dev/null +++ b/openless-all/app/src/lib/insertTextAnimation.test.ts @@ -0,0 +1,182 @@ +import { + INSERT_TEXT_MOTION, + appendedBirthAdvance, + clampInsertContentWidth, + diffInsertUnits, + firstBornIndex, + planCapsuleInsertMotion, + planCharDelays, + prefixWidths, + propagationDelayMs, + resetInsertUnitKeys, + rightAnchoredPositions, + segmentInsertUnits, + type InsertUnit, +} from './insertTextAnimation'; + +function assert(condition: boolean, message: string) { + if (!condition) throw new Error(message); +} + +function assertEqual(actual: T, expected: T, name: string) { + if (actual !== expected) { + throw new Error(`${name}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); + } +} + +function assertClose(actual: number, expected: number, name: string, epsilon = 0.01) { + if (Math.abs(actual - expected) > epsilon) { + throw new Error(`${name}: expected ~${expected}, got ${actual}`); + } +} + +resetInsertUnitKeys(); +assertEqual(segmentInsertUnits('帮我').join(''), '帮我', 'CJK stays intact'); +assertEqual(segmentInsertUnits('10六号').join(''), '10六号', 'ASCII + CJK mix'); +assert( + segmentInsertUnits('🙂a').length === 2, + `emoji is one unit, got ${segmentInsertUnits('🙂a').length}`, +); + +let units: InsertUnit[] = []; +units = diffInsertUnits(units, '帮我'); +assertEqual(units.map((u) => u.text).join(''), '帮我', 'first insert text'); +assert( + units.every((u) => u.born), + 'first insert marks every glyph born', +); +assertEqual(firstBornIndex(units), 0, 'all-new origin is the first glyph'); + +const afterFirst = units; +units = diffInsertUnits(units, '帮我查找一'); +assertEqual(units.map((u) => u.text).join(''), '帮我查找一', 'append keeps prefix'); +assertEqual(units[0].key, afterFirst[0].key, 'prefix keys stay stable'); +assertEqual(units[1].key, afterFirst[1].key, 'second prefix key stays stable'); +assert( + units.slice(0, 2).every((u) => !u.born), + 'prefix is no longer born', +); +assert( + units.slice(2).every((u) => u.born), + 'appended glyphs are born', +); +assertEqual(firstBornIndex(units), 2, 'origin sits at the first new glyph'); + +const beforeReplace = units; +units = diffInsertUnits(units, '帮我查找一下'); +assert( + units.slice(0, 5).every((u) => !u.born), + 'longer prefix stays settled', +); +assertEqual(units[5].text, '下', 'suffix append'); +assert(units[5].born, 'new trailing glyph is born'); +assertEqual(units[0].key, beforeReplace[0].key, 'keys survive another append'); + +units = diffInsertUnits(units, '帮我X一下'); +assertEqual( + units.map((u) => u.text).join(''), + '帮我X一下', + 'middle replace keeps prefix and suffix', +); +assertEqual(units[0].key, beforeReplace[0].key, 'prefix key survives replace'); +assert(units[2].born, 'replacement glyph is born'); +assertEqual(units[2].text, 'X', 'middle is the replacement'); +assert(!units[3].born && units[3].text === '一', 'suffix stays settled'); + +units = diffInsertUnits(units, ''); +assertEqual(units.length, 0, 'empty text clears units'); + +assertEqual(propagationDelayMs(0), 0, 'zero distance is immediate'); +assertEqual(propagationDelayMs(-8), 0, 'negative distance is immediate'); +assertClose( + propagationDelayMs(INSERT_TEXT_MOTION.waveSpeedPxPerMs * 40), + 40, + 'delay is distance / wave speed', +); +assertEqual( + propagationDelayMs(10_000), + INSERT_TEXT_MOTION.maxDelayMs, + 'delay is capped so far glyphs still feel concurrent', +); + +const widths = [16, 16, 16, 16, 16]; +const starts = prefixWidths(widths); +assertEqual(starts.join(','), '0,16,32,48,64', 'prefix widths accumulate'); +const delays = planCharDelays(widths, 3); +assert( + delays[3] < delays[2] && delays[2] < delays[1] && delays[1] < delays[0], + 'wave travels left', +); +assert(delays[4] > delays[3], 'new glyphs to the right of origin stagger slightly'); + +const plan = planCapsuleInsertMotion({ + stageWidth: 460, + contentWidth: 120, + originX: 96, + minWidth: 72, + maxWidth: 440, + padX: 18, +}); +assertEqual(plan.width, 156, 'pill hugs text plus horizontal padding'); +assertClose(plan.left, (460 - 156) / 2, 'final left is centered'); +assertClose(plan.rightEdge, plan.left + plan.width, 'right edge is left + width'); +assert(plan.widthDelayMs > INSERT_TEXT_MOTION.widthDelayMinMs, 'left edge waits for the wave'); +assert( + plan.recenterDelayMs > plan.widthDelayMs, + 'recenter lags the leftward expansion, like a second-order correction', +); + +const clamped = clampInsertContentWidth(400, 156, 18); +assertEqual(clamped, 120, 'content width cannot exceed inner max'); + +const tiny = planCapsuleInsertMotion({ + stageWidth: 460, + contentWidth: 8, + originX: 0, + minWidth: 72, +}); +assertEqual(tiny.width, 72, 'empty-ish text still has a readable pill'); +assertClose(tiny.left + tiny.width / 2, 230, 'tiny pill stays centered'); + +console.log('insertTextAnimation.test.ts passed'); + +const oldPositions = rightAnchoredPositions([16, 16]); +const newPositions = rightAnchoredPositions([16, 16, 16]); +assertClose(newPositions[0] - oldPositions[0], -16, 'append pushes existing glyph left'); +assertClose(newPositions[1] - oldPositions[1], -16, 'retained glyph destination delta'); +assertClose(newPositions[2], -16, 'new glyph ends at right anchor'); +assert( + Math.max(...planCharDelays(Array(200).fill(16), 0)) <= INSERT_TEXT_MOTION.maxDelayMs, + 'batch stagger bounded', +); +const emojiUnits = diffInsertUnits([], 'a👨‍👩‍👧‍👦é'); +assertEqual(emojiUnits.length, 3, 'combined emoji and accent intact'); +const corrected = diffInsertUnits(emojiUnits, 'b👨‍👩‍👧‍👦é'); +assertEqual(corrected[1].key, emojiUnits[1].key, 'correction preserves emoji'); +assertEqual(corrected[2].key, emojiUnits[2].key, 'correction preserves suffix'); + +const paired = planCharDelays(Array(10).fill(17.4), 8, 2); +assert(paired[7] < paired[5] && paired[5] < paired[3], 'pairs propagate from right to left'); +assert(paired[6] - paired[7] < 5, 'two-glyph group moves together'); +assert(paired[5] - paired[6] > 20, 'next group follows with a perceptible delay'); +const triples = planCharDelays(Array(12).fill(17.4), 9, 3); +assert(triples[6] - triples[8] < 5, 'three-glyph batch pushes a three-glyph group'); +assert(triples[5] - triples[6] > 20, 'next triple follows'); +assert( + plan.recenterDelayMs - plan.widthDelayMs >= 140, + 'shell is allowed to lean left before recentering', +); + +const originalNumber = diffInsertUnits([], '10六号'); +const correctedNumber = diffInsertUnits(originalNumber, '十六号'); +assertEqual( + appendedBirthAdvance(correctedNumber, [16, 16, 16]), + 0, + 'correction cannot enter over retained suffix', +); +const appendedNumber = diffInsertUnits(originalNumber, '10六号天气'); +assertEqual( + appendedBirthAdvance(appendedNumber, [16, 16, 16, 16, 16, 16]), + 32, + 'append enters beyond old tail', +); diff --git a/openless-all/app/src/lib/insertTextAnimation.ts b/openless-all/app/src/lib/insertTextAnimation.ts new file mode 100644 index 000000000..a817ba519 --- /dev/null +++ b/openless-all/app/src/lib/insertTextAnimation.ts @@ -0,0 +1,210 @@ +/** + * 直播转写插入动画的纯函数模型。 + * + * 坐标直觉（与参考视频一致）： + * - 新字出现在右侧插入点：淡入 + 上弹回落 + * - 既有文字被挤向左侧，延迟随离插入点的距离增加（声波传播） + * - 胶囊先以右缘为锚向左拓宽，再整体右移回到居中 + * + * 这些量只描述「何时、往哪」；真正的弹簧由 LiveTranscriptPill 交给 framer-motion。 + */ + +export const INSERT_TEXT_MOTION = { + waveSpeedPxPerMs: 2.35, + glyphGapPx: 1.4, + groupDelayMs: 26, + groupRippleMs: 2, + bornFadeMs: 180, + bornY: 9, + bornBlurPx: 1.8, + bornStaggerMs: 8, + widthDelayMinMs: 12, + recenterExtraMs: 140, + maxDelayMs: 96, + charSpring: { stiffness: 460, damping: 23, mass: 0.8 }, + shiftSpring: { stiffness: 260, damping: 30, mass: 1 }, + widthSpring: { stiffness: 330, damping: 32, mass: 0.9 }, + recenterSpring: { stiffness: 180, damping: 27, mass: 1 }, + padX: 18, + minWidth: 72, +} as const; + +export interface InsertUnit { + key: string; + text: string; + born: boolean; +} + +let insertUnitSeq = 0; + +export function resetInsertUnitKeys() { + insertUnitSeq = 0; +} + +function allocInsertKey() { + insertUnitSeq += 1; + return `ins-${insertUnitSeq}`; +} + +export function segmentInsertUnits(text: string): string[] { + const SegmenterCtor = ( + Intl as typeof Intl & { + Segmenter?: new ( + locales?: string | string[], + options?: { granularity?: 'grapheme' | 'word' | 'sentence' }, + ) => { segment(input: string): Iterable<{ segment: string }> }; + } + ).Segmenter; + if (typeof SegmenterCtor === 'function') { + return Array.from( + new SegmenterCtor(undefined, { granularity: 'grapheme' }).segment(text), + (part) => part.segment, + ); + } + return Array.from(text); +} + +/** + * 以最长公共前缀 + 后缀对齐，给稳定字素分配稳定 key。 + * 中间新增的字素 marked born，下一轮会被压成 born: false。 + */ +export function diffInsertUnits(prev: InsertUnit[], nextText: string): InsertUnit[] { + const next = segmentInsertUnits(nextText); + if (next.length === 0) return []; + + const prevTexts = prev.map((unit) => unit.text); + let prefix = 0; + while (prefix < prevTexts.length && prefix < next.length && prevTexts[prefix] === next[prefix]) { + prefix += 1; + } + let suffix = 0; + while ( + suffix < prevTexts.length - prefix && + suffix < next.length - prefix && + prevTexts[prevTexts.length - 1 - suffix] === next[next.length - 1 - suffix] + ) { + suffix += 1; + } + + const units: InsertUnit[] = []; + for (let i = 0; i < prefix; i += 1) { + units.push({ ...prev[i], born: false }); + } + const middle = next.slice(prefix, next.length - suffix); + for (const glyph of middle) { + units.push({ key: allocInsertKey(), text: glyph, born: true }); + } + if (suffix > 0) { + const suffixStart = prev.length - suffix; + for (let i = 0; i < suffix; i += 1) { + units.push({ ...prev[suffixStart + i], born: false }); + } + } + return units; +} + +export function firstBornIndex(units: InsertUnit[]): number { + const index = units.findIndex((unit) => unit.born); + if (index >= 0) return index; + return Math.max(0, units.length - 1); +} + +export function prefixWidths(widths: number[]): number[] { + const starts: number[] = []; + let cursor = 0; + for (const width of widths) { + starts.push(cursor); + cursor += width; + } + return starts; +} + +export function propagationDelayMs( + distancePx: number, + waveSpeedPxPerMs = INSERT_TEXT_MOTION.waveSpeedPxPerMs, +): number { + if (!Number.isFinite(distancePx) || distancePx <= 0) return 0; + if (!Number.isFinite(waveSpeedPxPerMs) || waveSpeedPxPerMs <= 0) return 0; + return Math.min(INSERT_TEXT_MOTION.maxDelayMs, distancePx / waveSpeedPxPerMs); +} + +export function planCharDelays(widths: number[], originIndex: number, insertedCount = 1): number[] { + if (widths.length === 0) return []; + const starts = prefixWidths(widths); + const safeOrigin = Math.min(Math.max(0, originIndex), widths.length - 1); + const originX = starts[safeOrigin] + widths[safeOrigin] / 2; + // Recognition batches carry the impulse: 2/3/4 inserted glyphs move in + // corresponding groups. A single-glyph update still pushes a 3-glyph cluster. + const groupSize = insertedCount <= 1 ? 3 : Math.min(4, insertedCount); + return widths.map((width, index) => { + if (index < safeOrigin) { + const distance = safeOrigin - 1 - index; + return Math.min( + INSERT_TEXT_MOTION.maxDelayMs, + 4 + + Math.floor(distance / groupSize) * INSERT_TEXT_MOTION.groupDelayMs + + (distance % groupSize) * INSERT_TEXT_MOTION.groupRippleMs, + ); + } + const center = starts[index] + width / 2; + const bornBoost = + index >= safeOrigin ? (index - safeOrigin) * INSERT_TEXT_MOTION.bornStaggerMs : 0; + return Math.min( + INSERT_TEXT_MOTION.maxDelayMs, + propagationDelayMs(Math.abs(center - originX)) + bornBoost, + ); + }); +} + +export interface CapsuleInsertPlan { + width: number; + left: number; + rightEdge: number; + widthDelayMs: number; + recenterDelayMs: number; +} + +export function planCapsuleInsertMotion(args: { + stageWidth: number; + contentWidth: number; + originX: number; + minWidth?: number; + maxWidth?: number; + padX?: number; +}): CapsuleInsertPlan { + const padX = args.padX ?? INSERT_TEXT_MOTION.padX; + const minWidth = args.minWidth ?? INSERT_TEXT_MOTION.minWidth; + const maxWidth = args.maxWidth ?? Number.POSITIVE_INFINITY; + const hugged = Math.max(minWidth, args.contentWidth + padX * 2); + const width = Math.min(maxWidth, hugged); + const left = (args.stageWidth - width) / 2; + const rightEdge = left + width; + const originFromLeft = Math.max(0, Math.min(args.contentWidth, args.originX)); + const widthDelayMs = INSERT_TEXT_MOTION.widthDelayMinMs + propagationDelayMs(originFromLeft); + const recenterDelayMs = widthDelayMs + INSERT_TEXT_MOTION.recenterExtraMs; + return { width, left, rightEdge, widthDelayMs, recenterDelayMs }; +} + +export function clampInsertContentWidth( + contentWidth: number, + maxWidth: number, + padX?: number, +): number { + const pad = padX ?? INSERT_TEXT_MOTION.padX; + const inner = Math.max(0, maxWidth - pad * 2); + return Math.min(contentWidth, inner); +} + +/** Right-anchored positions remain independent of the animated shell width. */ +export function rightAnchoredPositions(widths: number[]): number[] { + const total = widths.reduce((sum, width) => sum + width, 0); + return prefixWidths(widths).map((start) => start - total); +} + +/** Only an appended suffix enters beyond the old tail. A correction already has + * a destination gap and must not sweep across its retained suffix. */ +export function appendedBirthAdvance(units: InsertUnit[], widths: number[]): number { + const origin = units.findIndex((unit) => unit.born); + if (origin <= 0 || units.slice(origin).some((unit) => !unit.born)) return 0; + return widths.slice(origin).reduce((sum, width) => sum + width, 0); +} diff --git a/openless-all/app/src/lib/ipc/mock-data.ts b/openless-all/app/src/lib/ipc/mock-data.ts index 0298d0e01..6e8267df7 100644 --- a/openless-all/app/src/lib/ipc/mock-data.ts +++ b/openless-all/app/src/lib/ipc/mock-data.ts @@ -47,6 +47,8 @@ export let mockSettings: UserPreferences = { launchAtLogin: false, showCapsule: true, capsuleStyle: 'siri', + capsuleTranscriptEnabled: true, + capsuleTranscriptFontSize: 14, muteDuringRecording: false, stableTranscriptionEnabled: false, audioCueOnRecord: true, diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index 25ecc2cd2..3cf1c9d27 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -343,6 +343,8 @@ export interface UserPreferences { showCapsule: boolean; /** 录音胶囊外观；保存后同步到胶囊窗口。 */ capsuleStyle: CapsuleStyle; + capsuleTranscriptEnabled: boolean; + capsuleTranscriptFontSize: number; /** 录音期间临时静音系统输出，停止/取消/出错后恢复原静音状态。 */ muteDuringRecording: boolean; /** 先完整录音，停止后再连接当前 ASR 并提交整段音频。默认关闭。 */ diff --git a/openless-all/app/src/pages/settings/RecordingInputSection.tsx b/openless-all/app/src/pages/settings/RecordingInputSection.tsx index 6a7f94abc..c71d0a14e 100644 --- a/openless-all/app/src/pages/settings/RecordingInputSection.tsx +++ b/openless-all/app/src/pages/settings/RecordingInputSection.tsx @@ -448,6 +448,34 @@ export function RecordingInputSection() {
)} + {os !== 'linux' && !isAndroid && ( + <> + + savePrefs({ ...prefs, capsuleTranscriptEnabled: next })} + /> + + {(prefs.capsuleTranscriptEnabled ?? true) && ( + + + savePrefs({ ...prefs, capsuleTranscriptFontSize: Number(next) }) + } + options={[12, 14, 16, 18, 20].map((size) => ({ + value: String(size), + label: `${size}px`, + }))} + ariaLabel={t('settings.recording.capsuleTranscriptFontSize')} + /> + + )} + + )} ({ : host ? { protocol: 'ws', host, port: 1421 } : undefined, - watch: { ignored: ['**/src-tauri/**'] }, + watch: { ignored: ['**/src-tauri/**', '**/target/**'] }, proxy: { // Browser-only preview parity: the native app calls OrcaRouter directly, // while Vite needs a same-origin bridge because /models does not advertise CORS.