From 61a29de12fde93ee60102a345a1a6667dd619432 Mon Sep 17 00:00:00 2001 From: Francis Date: Sun, 4 Oct 2026 21:52:29 +0900 Subject: [PATCH] =?UTF-8?q?feat(remote-input):=20=E8=BF=9C=E7=A8=8B?= =?UTF-8?q?=E8=BE=93=E5=85=A5=E5=8F=AF=E9=80=89=E8=BD=AC=E5=8F=91=E6=9C=AC?= =?UTF-8?q?=E4=BC=9A=E8=AF=9D=E7=9A=84=E5=AE=9E=E6=97=B6=E8=AF=86=E5=88=AB?= =?UTF-8?q?=E6=96=87=E5=AD=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 客户端在 hello 帧中发送 "partials": true 后，本连接自己开启的会话的 TranscriptDelta 会作为 {"type":"partial","text","offset","final"} 转发； 未请求的客户端（包括自带手机页面）消息不变，所有权规则不变。 --- .../app/src-tauri/src/remote_server/mod.rs | 77 +++++++++++++++++-- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/openless-all/app/src-tauri/src/remote_server/mod.rs b/openless-all/app/src-tauri/src/remote_server/mod.rs index 346e83aba..342fc29b7 100644 --- a/openless-all/app/src-tauri/src/remote_server/mod.rs +++ b/openless-all/app/src-tauri/src/remote_server/mod.rs @@ -421,9 +421,14 @@ fn send_json(value: &T) -> Message { /// Phones only receive the Core session opened by their own connection. Global capsule / /// plain-text broadcasts have no owner and would send local PC dictation or another phone's /// content to all paired connections; they must not be a network egress. +/// +/// `wants_partials`: the client asked for live text in its hello frame. The owned session's +/// streaming transcript (`TranscriptDelta`) is then forwarded as `partial` frames; it is the +/// same session-scoped text the desktop capsule shows, and never another session's. fn backend_event_to_phone( event: &openless_core::BackendEvent, remote_session_id: &mut Option, + wants_partials: bool, ) -> Vec { use openless_core::{BackendEventKind, DictationPhase}; if remote_session_id.is_none() || event.session_id != *remote_session_id { @@ -468,6 +473,13 @@ fn backend_event_to_phone( } messages } + BackendEventKind::TranscriptDelta(delta) if wants_partials => { + // Same contract as TranscriptAccumulator::apply: replace from `offset` (chars). + vec![serde_json::json!({ + "type":"partial", "text":delta.text, "offset":delta.offset, "final":delta.is_final, + }) + .to_string()] + } _ => Vec::new(), } } @@ -481,9 +493,11 @@ type PendingRemoteStop = async fn handle_ws(mut socket: WebSocket, state: Arc, peer_ip: IpAddr) { // 1) Handshake: wait for the first hello + PIN frame. let connection_id = openless_core::SessionId::new(); + let mut wants_partials = false; let authed = match tokio::time::timeout(Duration::from_secs(15), socket.recv()).await { Ok(Some(Ok(Message::Text(txt)))) => { let candidate = parse_hello_pin(&txt); + wants_partials = parse_hello_partials(&txt); match state .backend .services() @@ -609,7 +623,9 @@ async fn handle_ws(mut socket: WebSocket, state: Arc, peer_ip: IpAddr) continue; } last_event_sequence = event.sequence; - for msg in backend_event_to_phone(&event, &mut remote_session_id) { + let messages = + backend_event_to_phone(&event, &mut remote_session_id, wants_partials); + for msg in messages { if socket.send(Message::Text(msg)).await.is_err() { break 'connection; } @@ -816,6 +832,16 @@ fn parse_hello_pin(txt: &str) -> openless_core::SecretValue { openless_core::SecretValue::new(pin) } +/// Optional `"partials": true` in the hello frame opts this connection into live text. +/// Clients that do not send it (the bundled phone page) keep the previous message set. +fn parse_hello_partials(txt: &str) -> bool { + serde_json::from_str::(txt) + .ok() + .filter(|value| value.get("type").and_then(serde_json::Value::as_str) == Some("hello")) + .and_then(|value| value.get("partials").and_then(serde_json::Value::as_bool)) + .unwrap_or(false) +} + fn parse_audio_frame( frame: &[u8], ) -> Result<(openless_core::SessionId, u64, Vec), openless_core::BackendError> { @@ -831,7 +857,10 @@ mod tests { RemoteInputConfig, RemoteInputService, SessionId, }; - use super::{apply_remote_control, backend_event_to_phone, parse_audio_frame, parse_hello_pin}; + use super::{ + apply_remote_control, backend_event_to_phone, parse_audio_frame, parse_hello_partials, + parse_hello_pin, + }; fn backend() -> ( OpenLessBackend, @@ -1050,11 +1079,11 @@ mod tests { }), }; assert!( - backend_event_to_phone(&result(SessionId::new()), &mut owner).is_empty(), + backend_event_to_phone(&result(SessionId::new()), &mut owner, false).is_empty(), "其他手机/本机结果不可转发" ); assert_eq!(owner, Some(session)); - let messages = backend_event_to_phone(&result(session), &mut owner); + let messages = backend_event_to_phone(&result(session), &mut owner, false); assert_eq!(messages.len(), 2); assert_eq!( serde_json::from_str::(&messages[1]).unwrap(), @@ -1062,11 +1091,49 @@ mod tests { ); assert_eq!(owner, None); assert!( - backend_event_to_phone(&result(session), &mut owner).is_empty(), + backend_event_to_phone(&result(session), &mut owner, false).is_empty(), "取消/终态之后的迟到结果不可转发" ); } + #[test] + fn websocket_forwards_live_transcript_only_when_asked_and_only_for_the_owned_session() { + use openless_core::{BackendEvent, BackendEventKind, TranscriptDelta}; + let session = SessionId::new(); + let mut owner = Some(session); + let delta = |id, text: &str, offset| BackendEvent { + sequence: 1, + session_id: Some(id), + kind: BackendEventKind::TranscriptDelta(TranscriptDelta { + text: text.into(), + offset, + is_final: false, + }), + }; + assert!( + backend_event_to_phone(&delta(session, "你好", 0), &mut owner, false).is_empty(), + "未在 hello 中请求实时文字的客户端保持原有消息集" + ); + assert!( + backend_event_to_phone(&delta(SessionId::new(), "别人的", 0), &mut owner, true) + .is_empty(), + "其他会话的实时文字不可转发" + ); + let messages = backend_event_to_phone(&delta(session, "您好", 1), &mut owner, true); + assert_eq!(messages.len(), 1); + assert_eq!( + serde_json::from_str::(&messages[0]).unwrap(), + serde_json::json!({"type":"partial", "text":"您好", "offset":1, "final":false}) + ); + assert_eq!(owner, Some(session), "实时文字不结束会话所有权"); + + assert!(parse_hello_partials( + r#"{"type":"hello","pin":"123456","partials":true}"# + )); + assert!(!parse_hello_partials(r#"{"type":"hello","pin":"123456"}"#)); + assert!(!parse_hello_partials(r#"{"type":"other","partials":true}"#)); + } + #[test] fn websocket_wire_parser_requires_contract_2_frames() { let session = SessionId::new();