From daa1bfa0c11eb3def53f274c5d79a46a7f1fd59e Mon Sep 17 00:00:00 2001 From: DepengWang <2818245+DepengWang@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:55:33 +0800 Subject: [PATCH 1/6] feat(windows): add UIA cursor-context reader Adds a Windows UI Automation implementation of read_around_cursor, reusing the existing HostContextAdapter -> cursor_context_input() pipeline instead of a parallel path. Prefers TextPattern2::GetCaretRange (active caret only), falls back to TextPattern::GetSelection for collapsed selections only, and never guesses a caret from ValuePattern. The text window is built by expanding a Range from the caret (never reading the whole document), then sliced to budget by Core's existing window_around_cursor/plan_window. Gated by the same password-field (CurrentIsPassword) and process-blocklist checks as the existing edit watcher, plus a focus-consistency recheck before returning a result so an Alt+Tab mid-read can't mix two apps' text. Runs on spawn_blocking with a 1s outer timeout; any failure degrades to no context, never affects dictation. Adds budget-split/redistribution/emoji-boundary tests to openless-core's window.rs (previously untested) and a pure is_blocked_process_name helper with tests in windows.rs. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011h9QUkoFtCoDLsJjRDB6dX --- .../openless-core/src/host_document/window.rs | 56 ++++ .../app/src-tauri/src/host_document/mod.rs | 78 ++++- .../src-tauri/src/host_document/windows.rs | 284 ++++++++++++++++-- 3 files changed, 391 insertions(+), 27 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/host_document/window.rs b/openless-all/app/crates/openless-core/src/host_document/window.rs index 996206a2b..b372ecdb2 100644 --- a/openless-all/app/crates/openless-core/src/host_document/window.rs +++ b/openless-all/app/crates/openless-core/src/host_document/window.rs @@ -44,3 +44,59 @@ pub fn utf16_offset_to_char_offset(text: &str, utf16_offset: usize) -> usize { } text.chars().count() } + +#[cfg(test)] +mod tests { + use super::*; + + /// Windows UIA 光标上下文方案（`windows_cursor_context开发方案.md` §6）的验收用例: + /// budget=600 时 before<=480、after<=120，且 before+after<=600。 + #[test] + fn budget_splits_roughly_eighty_twenty() { + let long = "字".repeat(2000); + let span = plan_window(long.chars().count(), 1000, 600); + assert!(span.cursor_in_span <= 480, "before 不应超过预算的 80%"); + assert!(span.len - span.cursor_in_span <= 120, "after 不应超过预算的 20%"); + assert!(span.len <= 600); + } + + /// 左侧文本不足时，剩余预算应让给右侧（方案 §6 "如果左侧不足 480 字，可以把剩余预算让给右侧"）。 + #[test] + fn insufficient_left_text_gives_remaining_budget_to_the_right() { + let text = "字".repeat(1000); + // cursor 只有 10 个字在左边，右边还有 990 个字可读。 + let span = plan_window(text.chars().count(), 10, 600); + assert_eq!(span.cursor_in_span, 10, "左边全部给出，不应凭空截断"); + assert_eq!(span.len, 600, "右侧应吃满剩余预算以补满 600"); + } + + /// 右侧文本不足时，剩余预算应让给左侧。 + #[test] + fn insufficient_right_text_gives_remaining_budget_to_the_left() { + let text = "字".repeat(1000); + // cursor 在第 990 个字，右边只剩 10 个字。 + let span = plan_window(text.chars().count(), 990, 600); + assert_eq!(span.len - span.cursor_in_span, 10, "右边全部给出"); + assert_eq!(span.len, 600, "左侧应补满剩余预算到 600"); + } + + /// emoji / surrogate pair 不应被按 UTF-16 长度误判为 2 个 char 而切坏字符。 + #[test] + fn window_around_cursor_does_not_split_emoji_or_surrogate_pairs() { + let text = "前文🙂😀后文"; + // 光标紧跟在两个 emoji 之后（按 char 计数，不按 UTF-16 code unit 计数）。 + let cursor_char = "前文🙂😀".chars().count(); + let window = window_around_cursor(text, cursor_char, 600); + assert_eq!(window.before(), "前文🙂😀"); + assert_eq!(window.after(), "后文"); + // 两个 emoji 字符仍然完整，没有产生孤立的 surrogate / 替换字符。 + assert!(window.text.chars().all(|c| c != '\u{FFFD}')); + } + + #[test] + fn zero_budget_yields_an_empty_window_at_the_cursor() { + let span = plan_window(100, 50, 0); + assert_eq!(span.len, 0); + assert_eq!(span.cursor_in_span, 0); + } +} diff --git a/openless-all/app/src-tauri/src/host_document/mod.rs b/openless-all/app/src-tauri/src/host_document/mod.rs index b3b4c32a0..d14c0762a 100644 --- a/openless-all/app/src-tauri/src/host_document/mod.rs +++ b/openless-all/app/src-tauri/src/host_document/mod.rs @@ -6,9 +6,9 @@ //! //! ## Boundary //! -//! Cursor context is available on macOS. Edit learning has separate local consent: -//! macOS AX, Windows UIA and Android accessibility provide bounded observation. -//! Linux retains explicit vocabulary entry. +//! Cursor context is available on macOS and Windows (UI Automation). Edit learning has +//! separate local consent: macOS AX, Windows UIA and Android accessibility provide bounded +//! observation. Linux retains explicit vocabulary entry. //! //! ## Three hard constraints (new code must not violate these, even though the old AX //! code in this repo does) @@ -68,6 +68,12 @@ const AX_MESSAGING_TIMEOUT_SECS: f32 = 0.2; #[cfg(target_os = "macos")] const READ_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(1200); +/// Outer timeout for one Windows UIA cursor-context read. UIA calls themselves are capped at +/// 200ms each (`SetConnectionTimeout` / `SetTransactionTimeout` in `windows.rs`); this bounds +/// the whole read (a handful of round-trips) per `windows_cursor_context开发方案.md` §11. +#[cfg(target_os = "windows")] +const WINDOWS_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(1000); + /// Outcome of one read. Every variant beyond `Ok` must explain why nothing was read — /// during installation verification this distinguishes "blocked" from "AX unsupported". #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] @@ -250,12 +256,16 @@ pub async fn probe_around_cursor(budget_chars: usize) -> HostDocumentReadResult { macos_probe(budget_chars).await } - #[cfg(not(target_os = "macos"))] + #[cfg(target_os = "windows")] + { + windows_probe(budget_chars).await + } + #[cfg(not(any(target_os = "macos", target_os = "windows")))] { let _ = budget_chars; HostDocumentReadResult::new( HostDocumentStatus::Unsupported, - Some("cursor context is macOS-only for now".to_string()), + Some("cursor context is unsupported on this platform".to_string()), ) } } @@ -316,6 +326,50 @@ fn blocked_result(reason: BlockReason) -> HostDocumentReadResult { ) } +/// Windows counterpart of [`macos_probe`]. UIA is a synchronous COM API, so the real read runs +/// on `spawn_blocking`; the gate itself (password field, process blocklist) lives in +/// `windows::read_around_cursor_blocking` because it needs the live focused element, unlike +/// macOS where the bundle-id-only pre-check can run before leaving the async worker. +#[cfg(target_os = "windows")] +async fn windows_probe(budget_chars: usize) -> HostDocumentReadResult { + let started = std::time::Instant::now(); + let (app_name, _) = crate::selection::current_front_app_parts(); + + let finish = |mut result: HostDocumentReadResult| { + result.app_name = app_name.clone(); + result.elapsed_ms = started.elapsed().as_millis() as u64; + result + }; + + let handle = tokio::task::spawn_blocking(move || windows::read_around_cursor_blocking(budget_chars)); + + match tokio::time::timeout(WINDOWS_READ_TIMEOUT, handle).await { + Ok(Ok(ReadOutcome::Window(window))) => finish(HostDocumentReadResult { + window: Some(window), + ..HostDocumentReadResult::new(HostDocumentStatus::Ok, None) + }), + Ok(Ok(ReadOutcome::Blocked(reason))) => finish(HostDocumentReadResult::new( + HostDocumentStatus::Blocked, + Some(reason.as_str().to_string()), + )), + Ok(Ok(ReadOutcome::Unavailable(reason))) => finish(HostDocumentReadResult::new( + HostDocumentStatus::Unavailable, + Some(reason.to_string()), + )), + Ok(Err(join_error)) => finish(HostDocumentReadResult::new( + HostDocumentStatus::Unavailable, + Some(format!("blocking task failed: {join_error}")), + )), + Err(_) => finish(HostDocumentReadResult::new( + HostDocumentStatus::Timeout, + Some(format!( + "no response within {}ms", + WINDOWS_READ_TIMEOUT.as_millis() + )), + )), + } +} + // ═══════════════════════════════════════════════════════════════════════════ // Edit watcher // ═══════════════════════════════════════════════════════════════════════════ @@ -570,10 +624,20 @@ mod tests { } #[tokio::test] - #[cfg(not(target_os = "macos"))] - async fn non_macos_reports_unsupported_without_touching_anything() { + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + async fn non_macos_non_windows_reports_unsupported_without_touching_anything() { let result = probe_around_cursor(DEFAULT_BUDGET_CHARS).await; assert_eq!(result.status, HostDocumentStatus::Unsupported); assert!(result.window.is_none()); } + + /// On a CI/build machine with no focused editable control, Windows UIA should degrade to + /// `Unavailable` or `Blocked`, never panic or hang past the outer timeout — this is the + /// same contract `debug_read_cursor_context` relies on for install verification. + #[tokio::test] + #[cfg(target_os = "windows")] + async fn windows_probe_never_reports_unsupported() { + let result = probe_around_cursor(DEFAULT_BUDGET_CHARS).await; + assert_ne!(result.status, HostDocumentStatus::Unsupported); + } } diff --git a/openless-all/app/src-tauri/src/host_document/windows.rs b/openless-all/app/src-tauri/src/host_document/windows.rs index 8acef471f..e087818e5 100644 --- a/openless-all/app/src-tauri/src/host_document/windows.rs +++ b/openless-all/app/src-tauri/src/host_document/windows.rs @@ -9,7 +9,7 @@ use std::sync::{ use std::time::{Duration, Instant}; use windows::core::{implement, Interface, Result, PWSTR, VARIANT}; use windows::Win32::{ - Foundation::{CloseHandle, HWND}, + Foundation::{CloseHandle, BOOL, HWND}, System::{ Com::{ CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, @@ -167,6 +167,37 @@ pub(super) fn spawn_edit_watcher( Some(stop) } +/// Sensitive process name fragments (password managers, terminals). Matched against the +/// lowercased executable filename only (not the full path), substring match. +/// +/// Pure function — no COM, so it is unit-testable without a live UIA element. The live lookup +/// is [`allowed_process`], shared by the edit watcher and the cursor-context reader so both +/// paths answer identically (see `openless-core`'s equivalent `SENSITIVE_BUNDLE_PREFIXES` note +/// on macOS: one ungated path means no gate). +const BLOCKED_PROCESS_NAME_FRAGMENTS: &[&str] = &[ + "keepass", + "1password", + "bitwarden", + "lastpass", + "dashlane", + "windowsterminal", + "powershell", + "pwsh", + "cmd.exe", + "conhost", + "mintty", + "wezterm", + "alacritty", + "putty", +]; + +fn is_blocked_process_name(executable_name: &str) -> bool { + let lowered = executable_name.to_lowercase(); + BLOCKED_PROCESS_NAME_FRAGMENTS + .iter() + .any(|blocked| lowered.contains(blocked)) +} + unsafe fn allowed_process(element: &IUIAutomationElement) -> Result { let pid = element.CurrentProcessId()? as u32; if pid == std::process::id() { @@ -183,26 +214,9 @@ unsafe fn allowed_process(element: &IUIAutomationElement) -> Result { ); let _ = CloseHandle(process); result?; - let path = String::from_utf16_lossy(&buffer[..len as usize]).to_lowercase(); + let path = String::from_utf16_lossy(&buffer[..len as usize]); let name = path.rsplit(['/', '\\']).next().unwrap_or(""); - Ok(![ - "keepass", - "1password", - "bitwarden", - "lastpass", - "dashlane", - "windowsterminal", - "powershell", - "pwsh", - "cmd.exe", - "conhost", - "mintty", - "wezterm", - "alacritty", - "putty", - ] - .iter() - .any(|blocked| name.contains(blocked))) + Ok(!is_blocked_process_name(name)) } unsafe fn read_text(element: &IUIAutomationElement) -> Result { @@ -313,3 +327,233 @@ unsafe fn observe(request: &Request) -> Result<()> { } result } + +// ═══════════════════════════════════════════════════════════════════════════ +// Cursor context reader +// ═══════════════════════════════════════════════════════════════════════════ +// +// A separate, read-only path from the edit watcher above: `read_text()` / `observe()` +// serve insertion-delivery checks and local edit learning (full document, never leaves the +// device). This path serves LLM cursor context — it only ever fetches a bounded span around +// the caret, and the result may be sent to the configured LLM provider, so it goes through +// its own safety gate ([`allowed_process`] + `CurrentIsPassword`) before a single UIA text +// call is made. See `windows_cursor_context开发方案.md` §4. + +/// Outcome of locating a usable caret [`IUIAutomationTextRange`]. +enum CaretLookup { + /// A zero-length (collapsed) range at the insertion point. + Found(IUIAutomationTextRange), + /// `TextPattern::GetSelection()` fallback found a real (non-collapsed) selection. Per + /// §8 of the design doc, the first version never guesses which end is the caret — + /// "selection rewrite" is a separate future feature, not folded into dictation context. + NonCollapsedSelection, + Unavailable(&'static str), +} + +/// Finds the caret as a zero-length text range. **Only callable in a `spawn_blocking` +/// context** (every call here is a synchronous COM round-trip). +/// +/// Priority, matching §5/§8 of the design doc: +/// 1. `IUIAutomationTextPattern2::GetCaretRange()`, only when `isActive == TRUE` — an inactive +/// caret is unusable and short-circuits to `Unavailable` without trying the fallback (an +/// inactive caret on this element is unlikely to become valid by asking a different pattern). +/// 2. `IUIAutomationTextPattern::GetSelection()`, used only when it is exactly one collapsed +/// range — this is the compatibility path for controls without `TextPattern2`. +/// 3. Neither pattern exists (including controls that only expose `ValuePattern`, see §9): +/// `Unavailable`. **Never** derived from `ValuePattern::CurrentValue()` — that pattern +/// cannot report a caret offset, and guessing one (e.g. end-of-value) would silently inject +/// wrong context when the user is editing mid-document. +unsafe fn find_caret_range(element: &IUIAutomationElement) -> CaretLookup { + if let Ok(pattern2) = element.GetCurrentPatternAs::(UIA_TextPattern2Id) + { + let mut is_active = BOOL(0); + match pattern2.GetCaretRange(&mut is_active) { + Ok(range) if is_active.as_bool() => return CaretLookup::Found(range), + Ok(_) => return CaretLookup::Unavailable("caret is not active"), + // GetCaretRange itself failed even though the pattern exists; fall through and + // try the TextPattern/GetSelection compatibility path below. + Err(_) => {} + } + } + + let Ok(pattern) = element.GetCurrentPatternAs::(UIA_TextPatternId) + else { + let has_value_only = element + .GetCurrentPatternAs::(UIA_ValuePatternId) + .is_ok(); + return CaretLookup::Unavailable(if has_value_only { + "focused control exposes value but no caret text range" + } else { + "no caret-capable text pattern on focused element" + }); + }; + let Ok(selection) = pattern.GetSelection() else { + return CaretLookup::Unavailable("GetSelection failed"); + }; + let Ok(count) = selection.Length() else { + return CaretLookup::Unavailable("selection length unavailable"); + }; + if count != 1 { + // 0 => no selection/caret reported at all; >1 => discontiguous selection. Neither is + // "a caret"; stay conservative rather than picking one range heuristically. + return CaretLookup::Unavailable("no single caret-equivalent selection range"); + } + let Ok(range) = selection.GetElement(0) else { + return CaretLookup::Unavailable("selection range unavailable"); + }; + match range.CompareEndpoints( + TextPatternRangeEndpoint_Start, + &range, + TextPatternRangeEndpoint_End, + ) { + Ok(0) => CaretLookup::Found(range), + Ok(_) => CaretLookup::NonCollapsedSelection, + Err(_) => CaretLookup::Unavailable("selection endpoint comparison failed"), + } +} + +/// Reads the document window around the caret. **Only callable in a `spawn_blocking` +/// context.** Never reads the full document (§7 of the design doc): the range is expanded +/// from the caret by `MoveEndpointByUnit`, not sliced out of `DocumentRange()`. +unsafe fn read_document(element: &IUIAutomationElement, budget_chars: usize) -> super::ReadOutcome { + let caret = match find_caret_range(element) { + CaretLookup::Found(range) => range, + CaretLookup::NonCollapsedSelection => { + return super::ReadOutcome::Unavailable("non-collapsed selection") + } + CaretLookup::Unavailable(reason) => return super::ReadOutcome::Unavailable(reason), + }; + + // Over-fetch relative to the final budget: `TextUnit_Character` is defined by the text + // provider, which for some controls counts UTF-16 code units rather than Unicode scalars + // (surrogate pairs, e.g. emoji, would then move fewer "characters" than Rust chars). Core's + // `window_around_cursor` below does the exact, char-precise 80/20 split and redistribution + // (design doc §6); this local fetch only needs to be generous enough to feed it, not exact. + let over_fetch = (budget_chars.saturating_mul(2)).max(1) as i32; + let before_want = over_fetch * 4 / 5; + let after_want = over_fetch - before_want; + // Defensive cap on the cross-process text copy; the range itself is already bounded by the + // Move calls above, this only guards against a provider returning more than requested. + let text_cap = over_fetch.saturating_add(64); + + let Ok(before_range) = caret.Clone() else { + return super::ReadOutcome::Unavailable("caret range clone failed"); + }; + let _ = before_range.MoveEndpointByUnit( + TextPatternRangeEndpoint_Start, + TextUnit_Character, + -before_want, + ); + let Ok(before_text) = before_range.GetText(text_cap) else { + return super::ReadOutcome::Unavailable("caret range text unavailable"); + }; + + let Ok(after_range) = caret.Clone() else { + return super::ReadOutcome::Unavailable("caret range clone failed"); + }; + let _ = + after_range.MoveEndpointByUnit(TextPatternRangeEndpoint_End, TextUnit_Character, after_want); + let Ok(after_text) = after_range.GetText(text_cap) else { + return super::ReadOutcome::Unavailable("caret range text unavailable"); + }; + + let mut text = before_text.to_string(); + let cursor = text.chars().count(); + text.push_str(&after_text.to_string()); + + super::ReadOutcome::Window(super::window_around_cursor(&text, cursor, budget_chars)) +} + +/// Synchronously reads the document around the cursor. **Only callable in a `spawn_blocking` +/// context** — see [`super::windows_probe`] for the async/timeout wrapper. +/// +/// Gate order follows §15-17 of the design doc: password field and process blocklist are +/// checked immediately after acquiring the focused element, before any text pattern is +/// touched. Focus consistency (§12) is re-checked right before returning a successful window, +/// so a focus change mid-read (Alt+Tab, OpenLess's own UI stealing focus, …) discards the +/// result instead of mixing one app's text with another's dictation. +pub(super) fn read_around_cursor_blocking(budget_chars: usize) -> super::ReadOutcome { + unsafe { + if CoInitializeEx(None, COINIT_MULTITHREADED).is_err() { + return super::ReadOutcome::Unavailable("COM initialization failed"); + } + struct ComGuard; + impl Drop for ComGuard { + fn drop(&mut self) { + unsafe { CoUninitialize() } + } + } + let _com = ComGuard; + + let outcome = (|| -> Result { + let uia: IUIAutomation = CoCreateInstance(&CUIAutomation8, None, CLSCTX_INPROC_SERVER)?; + let timeouts: IUIAutomation2 = uia.cast()?; + timeouts.SetConnectionTimeout(200)?; + timeouts.SetTransactionTimeout(200)?; + + let initial_window = GetForegroundWindow(); + let element = uia.GetFocusedElement()?; + + if element.CurrentIsPassword()?.as_bool() { + return Ok(super::ReadOutcome::Blocked(super::BlockReason::SecureTextField)); + } + if !allowed_process(&element)? { + return Ok(super::ReadOutcome::Blocked(super::BlockReason::BlockedApp)); + } + + let outcome = read_document(&element, budget_chars); + if !matches!(outcome, super::ReadOutcome::Window(_)) { + return Ok(outcome); + } + + // Focus must not have moved during the read above — otherwise this window's "before" + // text could belong to a different app than the dictation that is about to use it. + if GetForegroundWindow() != initial_window + || !uia + .CompareElements(&element, &uia.GetFocusedElement()?)? + .as_bool() + { + return Ok(super::ReadOutcome::Unavailable("focus changed during capture")); + } + + Ok(outcome) + })(); + + outcome.unwrap_or(super::ReadOutcome::Unavailable("UIA call failed")) + } +} + +#[cfg(test)] +mod cursor_context_tests { + use super::*; + + #[test] + fn password_managers_and_terminals_are_blocked_by_name() { + for name in [ + "KeePassXC.exe", + "1Password.exe", + "Bitwarden.exe", + "powershell.exe", + "WindowsTerminal.exe", + "cmd.exe", + "conhost.exe", + ] { + assert!(is_blocked_process_name(name), "{name} should be blocked"); + } + } + + #[test] + fn ordinary_editors_are_not_blocked_by_name() { + for name in ["notepad.exe", "WINWORD.EXE", "Code.exe", "chrome.exe"] { + assert!(!is_blocked_process_name(name), "{name} should not be blocked"); + } + } + + #[test] + fn a_name_that_merely_contains_a_blocked_fragment_is_still_blocked() { + // Substring match is intentional here (unlike the macOS bundle-id prefix match): Windows + // executable names have no reverse-DNS namespace to anchor a prefix check against, and + // helper/updater binaries commonly append suffixes to the vendor name. + assert!(is_blocked_process_name("1password-updater.exe")); + } +} From f63a1644938fbca5b2744c02a302829dd2132dcf Mon Sep 17 00:00:00 2001 From: DepengWang <2818245+DepengWang@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:55:42 +0800 Subject: [PATCH 2/6] feat(settings): enable cursor context on Windows Shows the Cursor context toggle on Windows as well as macOS (Linux still has no implementation, so it stays hidden there). Updates the cursorContextEnabled doc comment in types.ts and all 8 locale descriptions to drop the "macOS only" wording and mention password fields/terminals are excluded on both platforms. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011h9QUkoFtCoDLsJjRDB6dX --- openless-all/app/src/i18n/de.ts | 2 +- openless-all/app/src/i18n/en.ts | 2 +- openless-all/app/src/i18n/es.ts | 2 +- openless-all/app/src/i18n/fr.ts | 2 +- openless-all/app/src/i18n/ja.ts | 2 +- openless-all/app/src/i18n/ko.ts | 2 +- openless-all/app/src/i18n/zh-CN.ts | 2 +- openless-all/app/src/i18n/zh-TW.ts | 2 +- openless-all/app/src/lib/types.ts | 6 ++++-- openless-all/app/src/pages/settings/DataStorageSection.tsx | 4 ++-- 10 files changed, 14 insertions(+), 12 deletions(-) diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index ed9c03e2e..f976df14c 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -1298,7 +1298,7 @@ export const de: typeof zhCN = { desc: 'Gesprächsverlauf und Kontext, die auf diesem Gerät gespeichert werden.', cursorContextLabel: 'Cursorkontext (experimentell)', cursorContextDesc: - 'Text rund um den Cursor zur Überarbeitung an das Modell senden (nur macOS). Diese Einstellung ist vom lokalen Lernen getrennt. Passwortfelder und bekannte sensible Apps sind ausgeschlossen.', + 'Text rund um den Cursor zur Überarbeitung an das Modell senden (macOS / Windows). Diese Einstellung ist vom lokalen Lernen getrennt. Passwortfelder, Terminals und bekannte sensible Apps sind ausgeschlossen.', }, codingConsole: { title: 'Claude-Konsole', diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 970ab81bc..58c484ee1 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -1276,7 +1276,7 @@ export const en: typeof zhCN = { desc: 'Conversation history and context kept on this device.', cursorContextLabel: 'Cursor context (experimental)', cursorContextDesc: - 'Send nearby document text with polish requests (macOS only). This switch is separate from local vocabulary learning. Password fields and known sensitive apps are excluded.', + 'Send nearby document text with polish requests (macOS / Windows). This switch is separate from local vocabulary learning. Password fields, terminals and known sensitive apps are excluded.', }, codingConsole: { title: 'Claude Console', diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index c3ecec806..ed760dfce 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -1292,7 +1292,7 @@ export const es: typeof zhCN = { desc: 'Historial de conversaciones y contexto guardados en este dispositivo.', cursorContextLabel: 'Contexto del cursor (experimental)', cursorContextDesc: - 'Envía el texto cercano al cursor al modelo para pulirlo (solo macOS). Es independiente del aprendizaje local. Se excluyen contraseñas y aplicaciones sensibles conocidas.', + 'Envía el texto cercano al cursor al modelo para pulirlo (macOS / Windows). Es independiente del aprendizaje local. Se excluyen contraseñas, terminales y aplicaciones sensibles conocidas.', }, codingConsole: { title: 'Consola de Claude', diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index c9b2bc894..abfeb81b9 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -1313,7 +1313,7 @@ export const fr: typeof zhCN = { desc: 'Historique des conversations et contexte conservés sur cet appareil.', cursorContextLabel: 'Contexte du curseur (expérimental)', cursorContextDesc: - 'Envoyer le texte autour du curseur au modèle pour la reformulation (macOS uniquement). Ce réglage est indépendant de l’apprentissage local. Les champs de mot de passe et les applications sensibles connues sont exclus.', + 'Envoyer le texte autour du curseur au modèle pour la reformulation (macOS / Windows). Ce réglage est indépendant de l’apprentissage local. Les champs de mot de passe, les terminaux et les applications sensibles connues sont exclus.', }, codingConsole: { title: 'Console Claude', diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 64e9e0363..884c79d0d 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -1262,7 +1262,7 @@ export const ja: typeof zhCN = { desc: 'この端末に保存される会話履歴とコンテキスト。', cursorContextLabel: 'カーソル文脈（実験的）', cursorContextDesc: - '推敲時にカーソル付近の文章をモデルへ送信します（macOSのみ）。端末内の単語学習とは独立した設定です。パスワード欄と既知の機密アプリは除外します。', + '推敲時にカーソル付近の文章をモデルへ送信します（macOS / Windows）。端末内の単語学習とは独立した設定です。パスワード欄、ターミナル、既知の機密アプリは除外します。', }, codingConsole: { title: 'Claude コンソール', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 371cc7e2d..a5345c6bf 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -1254,7 +1254,7 @@ export const ko: typeof zhCN = { desc: '이 기기에 보관되는 대화 기록과 컨텍스트.', cursorContextLabel: '커서 문맥 (실험적)', cursorContextDesc: - '다듬기 요청 시 커서 주변 텍스트를 모델에 보냅니다(macOS 전용). 기기 내 단어 학습과 별도 설정입니다. 비밀번호 입력란과 알려진 민감한 앱은 제외됩니다.', + '다듬기 요청 시 커서 주변 텍스트를 모델에 보냅니다(macOS / Windows). 기기 내 단어 학습과 별도 설정입니다. 비밀번호 입력란, 터미널, 알려진 민감한 앱은 제외됩니다.', }, codingConsole: { title: 'Claude 콘솔', diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index ee582d872..8cf67f384 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -1221,7 +1221,7 @@ export const zhCN = { desc: '本机保留的历史会话与对话上下文。', cursorContextLabel: '光标上下文（实验）', cursorContextDesc: - '润色时将光标附近文本发给模型（仅 macOS）。此开关与本地手改学词独立；排除密码框和已知敏感应用。', + '润色时将光标附近文本发给模型（macOS / Windows）。此开关与本地手改学词独立；排除密码框、终端及已知敏感应用。', }, codingConsole: { title: 'Claude 控制台', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 5b7cc7b3e..4ad7d7ffc 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -1221,7 +1221,7 @@ export const zhTW: typeof zhCN = { desc: '本機保留的歷史會話與對話上下文。', cursorContextLabel: '遊標上下文（實驗）', cursorContextDesc: - '潤色時將游標附近文字傳給模型（僅 macOS）。此開關與本機手改學詞獨立；排除密碼欄位和已知敏感應用程式。', + '潤色時將游標附近文字傳給模型（macOS / Windows）。此開關與本機手改學詞獨立；排除密碼欄位、終端機及已知敏感應用程式。', }, codingConsole: { title: 'Claude 主控臺', diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index d9cf11ab8..0d10082ed 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -508,8 +508,10 @@ export interface UserPreferences { * When on, Cmd+V can re-paste that output, matching the one-shot path. Default true. */ streamingInsertSaveClipboard: boolean; /** Whether to send the text near the cursor in the document the user is writing to LLM polish as context. - * Default false — when on, every dictation reads the foreground app's body text and sends part of it to the LLM provider. - * macOS only; password fields / Secure Input / password managers / terminals are always hard-blocked. */ + * Default false — when on, every dictation reads the foreground app's body text and the context + * may be sent to the configured LLM provider. macOS and Windows only; password fields / Secure + * Input (macOS) / UIA password controls (Windows) / known password managers / terminals are + * always hard-blocked. */ cursorContextEnabled: boolean; vocabularyLearningEnabled: boolean; vocabularyLearningSettings: { diff --git a/openless-all/app/src/pages/settings/DataStorageSection.tsx b/openless-all/app/src/pages/settings/DataStorageSection.tsx index 50574856f..86db97e66 100644 --- a/openless-all/app/src/pages/settings/DataStorageSection.tsx +++ b/openless-all/app/src/pages/settings/DataStorageSection.tsx @@ -83,9 +83,9 @@ export function DataStorageSection() { {/* Cursor context. Placing it under "Privacy" rather than "Polish" is deliberate: this toggle's real cost is not tokens but "sending text from other apps to the LLM provider". - Shown on macOS only — other platforms have no implementation, and a switch that changes + Shown on macOS and Windows — Linux has no implementation, and a switch that changes nothing would just mislead. */} - {detectOS() === 'mac' && ( + {(detectOS() === 'mac' || detectOS() === 'win') && ( Date: Fri, 2 Oct 2026 18:55:50 +0800 Subject: [PATCH 3/6] docs(cursor-context): document Windows UIA support Updates the Cursor Context sections in README.md and README.zh.md to cover Windows: TextPattern2 caret-range priority, TextPattern selection fallback, and the "skip rather than guess" rule when the caret can't be reliably located. Password-field exclusion wording now also mentions the Windows UIA password-control flag alongside macOS Secure Input. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011h9QUkoFtCoDLsJjRDB6dX --- README.md | 8 +++++--- README.zh.md | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9844e9551..3d489df8a 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ That is what **authorizing the infrastructure once, at launch** means: on first Capabilities that sediment yet more of the coordination you used to repeat every day into defaults: -- 📖 **A dictionary that learns.** Until now the dictionary only knew what you typed into it by hand. Now, when you correct a word OpenLess just wrote, it asks — once, on a small card — whether to remember it, and one click puts it in. Paired with **cursor context** (opt-in, macOS), which lets the polish model read what you are writing around your cursor, OpenLess stops being a transcriber that guesses at homophones and starts being an input method that knows your words. Every suggestion is reviewed by you; nothing is learned silently. +- 📖 **A dictionary that learns.** Until now the dictionary only knew what you typed into it by hand. Now, when you correct a word OpenLess just wrote, it asks — once, on a small card — whether to remember it, and one click puts it in. Paired with **cursor context** (opt-in, macOS / Windows), which lets the polish model read what you are writing around your cursor, OpenLess stops being a transcriber that guesses at homophones and starts being an input method that knows your words. Every suggestion is reviewed by you; nothing is learned silently. - 🎨 **Style Pack Marketplace.** OpenLess no longer ships a single fixed "polish" voice. Build your own **style packs** with custom system prompts, switch between them with a hotkey, and **install community packs in one click** — or publish your own to share. When a style is tuned to your exact task (cold emails, commit messages, 小红书 posts, formal reports, your team's tone), the output is not merely cleaner — it is *noticeably better*, because the model is finally writing the way you intend. - ⚡ **Streaming insertion.** Text now flows to your cursor **character by character** as it is polished, rather than making you wait for the complete result. Perceived latency drops sharply, so dictation feels nearly as fast as thinking — and it automatically falls back to a one-shot paste when an application cannot accept streamed keystrokes. @@ -376,13 +376,15 @@ The dictionary handles your proper nouns, product names, names of people, and ne - **Learn from corrections (experimental).** Open Settings → Experiments & extensions → Learn from corrections to enable it and configure observation duration (10–60 seconds, default 60), suggestion duration (5–60 seconds, default 10), and maximum automatic phrase length (2–32 characters, default 12). It defaults to off and is not enabled by cursor context or cloud sync. On macOS, Windows and Android, supported editors can be observed after insertion. Suggestions require confirmation before expiry to enter the dictionary. Changing parameters stops the current observation and clears pending suggestions; new values apply to the next dictation. Android requires accessibility. When observation is unavailable, use **Remember a word** in history details; Android IME result editing also offers an unchecked dictionary option. Every path requires explicit confirmation and does not create global replacement rules. - **Entries that earn their keep get priority.** The hotword budget sent to ASR providers is finite (a few hundred characters). Entries are ranked by hit count, with a few reserved seats for words you just added by hand, so the terms you actually use keep their place instead of being pushed out by whatever you added most recently. -### Cursor context (opt-in, macOS) +### Cursor context (opt-in, macOS / Windows) Settings → Privacy → Data storage → **Cursor context**. Off by default. When on, each dictation reads a few hundred characters around your cursor **in the app you are writing in** and sends them with the polish request, so the model knows what you are writing about. Chinese homophones (接口/借口, 大鱼/大禹) are indistinguishable to an acoustic model but obvious from context. Local vocabulary learning has a separate switch and does not require cursor context. Observed text is not sent to a model; words explicitly added to the dictionary participate in future ASR and polish requests as described above. -Cursor context excludes password fields, macOS Secure Input, known password managers and terminals. Turning it off stops reading cursor context for polishing; other authorized accessibility features, including local vocabulary learning and insertion, work independently. Turning vocabulary learning off stops observation and clears pending suggestions. +On Windows, cursor context is read via UI Automation from the focused text control: it prefers `TextPattern2`'s caret range, with a `TextPattern` selection fallback for controls that only support collapsed selections. When the caret cannot be reliably located, it is skipped rather than guessed — wrong context is worse than none, and dictation continues normally either way. + +Cursor context excludes password fields, macOS Secure Input (or the Windows UIA password-control flag), known password managers and terminals. Turning it off stops reading cursor context for polishing; other authorized accessibility features, including local vocabulary learning and insertion, work independently. Turning vocabulary learning off stops observation and clears pending suggestions. The main window is organized as Home / History / Dictionary / Settings. The Dictionary tab opens a separate editor window when you click "New". The Home tab shows total dictation time, total characters, average characters per minute, estimated time saved, and dictionary participation statistics. diff --git a/README.zh.md b/README.zh.md index dbddbc371..aeddfeff1 100644 --- a/README.zh.md +++ b/README.zh.md @@ -132,7 +132,7 @@ OpenLess 做的不是“更快的听写”,而是**消灭“想法 → 干净文 下面这些能力,把过去每天都要重复的协调,进一步沉降成了默认规则: -- 📖 **会自己长的词典。** 在此之前,词典里只有你亲手敲进去的东西。现在,当你改掉 OpenLess 刚写出来的某个词,它会在屏幕角落弹一张小卡片问一声要不要记住,点一下就进去了。配合**光标上下文**(需手动开启,仅 macOS)——让润色模型读得到你光标周围正在写的内容——OpenLess 不再是一个靠猜同音词的转写工具,而开始成为**一个认得你的词的输入法**。每一条建议都由你过目,没有任何东西是悄悄学走的。 +- 📖 **会自己长的词典。** 在此之前,词典里只有你亲手敲进去的东西。现在,当你改掉 OpenLess 刚写出来的某个词,它会在屏幕角落弹一张小卡片问一声要不要记住,点一下就进去了。配合**光标上下文**(需手动开启,macOS / Windows)——让润色模型读得到你光标周围正在写的内容——OpenLess 不再是一个靠猜同音词的转写工具,而开始成为**一个认得你的词的输入法**。每一条建议都由你过目,没有任何东西是悄悄学走的。 - 🎨 **风格包市场(Style Pack Marketplace)。** OpenLess 不再只内置一种固定的“润色”语气。你可以用自定义系统提示词构建自己的**风格包**,用快捷键在它们之间切换,并**一键安装社区分享的风格包**——也可以发布自己的与他人分享。当风格与你的具体任务高度契合(冷启动邮件、commit message、小红书文案、正式报告、团队语气)时,产出的文本不只是更干净,而是*明显更好*,因为模型终于在按你真正想要的方式写作。 - ⚡ **流式插入。** 文本现在会随润色**逐字符**写入光标,而不必等待完整结果生成。感知延迟大幅下降,听写几乎和思考一样快——当某个应用无法接受流式按键时,它会自动回退为一次性粘贴。 @@ -383,13 +383,15 @@ OpenLess 的润色模型只重塑文本。它不回答问题、不执行任务 - **手改学词（实验）。** 在「设置 → 实验与扩展 → 手改学词」进入独立配置页，设置观察时长（10–60 秒，默认 60）、建议保留时长（5–60 秒，默认 10）及自动建议最大词长（2–32 个字符，默认 12）。功能默认关闭，不跟随光标上下文或云同步授权。macOS、Windows 和 Android 可在支持的编辑器中观察插入后的修改，候选需在有效期内逐条确认才加入词典。修改参数会结束当前观察并清空待确认建议，下次听写生效。Android 需要无障碍服务。无法观察时，可在历史详情点击「记住词汇」手动输入正确词；Android 输入法编辑结果也提供默认不勾选的加入词典选项。所有入口均需明确确认，不自动创建全局替换规则。 - **真正在用的词优先。** 发给 ASR 的热词预算是有限的(几百字符)。条目按命中次数排序,并给刚手动添加的词留几个保底席位——这样你天天在用的那些词不会被「最近刚加的」挤出去。 -### 光标上下文(需手动开启,仅 macOS) +### 光标上下文(需手动开启,macOS / Windows) 设置 → 隐私 → 数据存储 → **光标上下文**。默认关闭。 开启后,每次听写会读取**你正在写的那个应用里**光标附近的几百个字,随润色请求一起发出,让模型知道你在写什么。中文同音词(接口/借口、大鱼/大禹)声学模型分不出来,但上下文能分。手改学词是独立的本地功能，不需要开启此设置。观察文本本身不会发送给模型；确认加入词典的词会按词典规则参与后续 ASR／润色。 -光标上下文排除密码输入框、macOS Secure Input、已知密码管理器和终端。关闭此开关后不会为润色读取光标上下文；其他已授权的辅助功能（如手改学词或插入）独立工作。手改学词关闭后会停止观察并清空待确认建议。 +Windows 使用 UI Automation 读取当前焦点文本控件的光标附近内容：优先使用 `TextPattern2` 的 caret range；控件不支持 `TextPattern2` 时，兼容使用 `TextPattern` 的 collapsed selection。无法可靠定位 caret 时直接跳过，不会猜测光标位置，也不会影响正常听写。 + +光标上下文排除密码输入框、macOS Secure Input（Windows 上为 UIA 密码控件标记）、已知密码管理器和终端。关闭此开关后不会为润色读取光标上下文；其他已授权的辅助功能（如手改学词或插入）独立工作。手改学词关闭后会停止观察并清空待确认建议。 主窗口组织为 首页 / 历史 / 词典 / 设置。点击“新建”时,词典页会打开一个独立的编辑窗口。首页展示总听写时长、总字数、平均每分钟字数、估算节省的时间,以及词典参与统计。 From a5fed40e5c3d429597f0535a42d97b359fdbed7a Mon Sep 17 00:00:00 2001 From: DepengWang <2818245+DepengWang@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:08:53 +0800 Subject: [PATCH 4/6] feat(cursor-context): log capture metadata on the real dictation path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TauriHostContextAdapter::capture previously logged nothing on success — only debug_read_cursor_context exercised the status/chars_before/ chars_after/elapsed_ms/app log line. Extends the same metadata-only log (no document body, per design doc §18/§19) to every real dictation that has cursor context enabled, so whether a capture actually happened is visible from the normal log file without a devtools round-trip. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011h9QUkoFtCoDLsJjRDB6dX --- .../app/src-tauri/src/core_adapters.rs | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index dd900c4ac..dbd80e9dc 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -3691,13 +3691,34 @@ impl openless_core::HostContextAdapter for TauriHostContextAdapter { Box::pin(async move { let front_app = crate::coordinator::capture_frontmost_app(); let cursor_context = if include_cursor { - crate::host_document::read_around_cursor(crate::host_document::DEFAULT_BUDGET_CHARS) - .await - .map(|window| { - let before = window.text.chars().take(window.cursor).collect::(); - let after = window.text.chars().skip(window.cursor).collect::(); - openless_core::prompts::cursor_context_input(&before, &after) - }) + let started = std::time::Instant::now(); + let window = crate::host_document::read_around_cursor( + crate::host_document::DEFAULT_BUDGET_CHARS, + ) + .await; + // Metadata only — never the document body (design doc §18/§19). This is the + // same shape debug_read_cursor_context already logs; extending it to the real + // dictation path means "did this capture actually happen" is visible from the + // normal log file, with no devtools round-trip needed. + match &window { + Some(window) => log::info!( + "[cursor-context] status=ok chars_before={} chars_after={} elapsed_ms={} app={:?}", + window.before().chars().count(), + window.after().chars().count(), + started.elapsed().as_millis(), + front_app, + ), + None => log::info!( + "[cursor-context] status=none elapsed_ms={} app={:?}", + started.elapsed().as_millis(), + front_app, + ), + } + window.map(|window| { + let before = window.text.chars().take(window.cursor).collect::(); + let after = window.text.chars().skip(window.cursor).collect::(); + openless_core::prompts::cursor_context_input(&before, &after) + }) } else { None }; From 50a02da23c8ff5c981bacf57f7011436db836f0d Mon Sep 17 00:00:00 2001 From: DepengWang <2818245+DepengWang@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:09:00 +0800 Subject: [PATCH 5/6] fix(windows): pace SendInput keystrokes to stop dropped characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows SendInput text injection sent characters within a chunk back to back with zero delay, only pausing every 16 characters. Found while testing cursor context (windowsInsertionMode=sendInput): dictating mixed Chinese/English text dropped the leading character(s) of a Latin-script word right after the CJK-to-ASCII transition, e.g. "open" arrived as "pen" and "OpenLess" vanished entirely — well before the 16-char chunk boundary, so the existing chunk-level pause never applied. macOS already works around the same class of problem (its INTER_KEYSTROKE_DELAY comment: "Chromium / Electron / Tauri themselves drop characters when keyDown/keyUp have no delay"); Windows had no equivalent. Adds a 1ms gap after every character, not just at chunk boundaries — confirmed on hardware to stop the drops, with no perceptible typing latency. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011h9QUkoFtCoDLsJjRDB6dX --- .../app/src-tauri/src/unicode_keystroke.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/openless-all/app/src-tauri/src/unicode_keystroke.rs b/openless-all/app/src-tauri/src/unicode_keystroke.rs index 9eac0ff2a..f21b8a36d 100644 --- a/openless-all/app/src-tauri/src/unicode_keystroke.rs +++ b/openless-all/app/src-tauri/src/unicode_keystroke.rs @@ -433,6 +433,17 @@ mod windows_impl { const SENDINPUT_CHUNK_CHARS: usize = 16; const SENDINPUT_CHUNK_DELAY: Duration = Duration::from_millis(12); + /// Gap after every synthetic keystroke, not just at chunk boundaries. + /// + /// Mirrors macOS's `INTER_KEYSTROKE_DELAY` (see that module's comment: "Chromium / + /// Electron / Tauri themselves drop characters when keyDown/keyUp have no delay"). Windows + /// had no such gap — characters within a chunk were sent back-to-back with zero spacing, + /// only pausing every [`SENDINPUT_CHUNK_CHARS`]. Observed on hardware: dictating mixed + /// Chinese/English text ("我要把这个 open list...") dropped the leading character(s) of the + /// Latin-script word right after the CJK→ASCII transition (e.g. "open" arrived as "pen"). + /// 1ms is inaudible/invisible to the user but gives the target app's message loop room to + /// process each keystroke before the next one lands. + const INTER_KEYSTROKE_DELAY: Duration = Duration::from_millis(1); /// Windows 上没有 input source 概念，token 留空。Send/Sync 自动派生。 pub struct PreviousInputSource; @@ -491,6 +502,9 @@ mod windows_impl { } typed_chars += 1; sent_in_chunk += 1; + if chars.peek().is_some() { + std::thread::sleep(INTER_KEYSTROKE_DELAY); + } if sent_in_chunk >= SENDINPUT_CHUNK_CHARS && chars.peek().is_some() { std::thread::sleep(SENDINPUT_CHUNK_DELAY); From c0788fbd10a2c4d60f7dacabf2ab9ce6bab472b6 Mon Sep 17 00:00:00 2001 From: DepengWang <2818245+DepengWang@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:35:15 +0800 Subject: [PATCH 6/6] docs(cursor-context): note VS Code accessibility support requirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirmed on hardware: Chromium-based editors (VS Code) only keep their accessibility tree's caret position in sync once they detect an active accessibility client, so with default settings the position UI Automation reads back can be stale even though the read itself succeeds. Setting editor.accessibilitySupport to "on" (plus a window reload) fixed it. Documents this in both READMEs so users don't mistake it for a bug. Also adds a metadata-only diagnostic log (source pattern + requested vs. actual Move distances, no document content) to host_document/windows.rs, added while narrowing this down between VS Code/Chrome — kept because it is cheap, useful for future reports of "wrong context" on a given app, and fully compliant with the no-document-body logging rule. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_011h9QUkoFtCoDLsJjRDB6dX --- README.md | 2 + README.zh.md | 2 + .../src-tauri/src/host_document/windows.rs | 50 +++++++++++++------ 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 3d489df8a..28bbd0575 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,8 @@ When on, each dictation reads a few hundred characters around your cursor **in t On Windows, cursor context is read via UI Automation from the focused text control: it prefers `TextPattern2`'s caret range, with a `TextPattern` selection fallback for controls that only support collapsed selections. When the caret cannot be reliably located, it is skipped rather than guessed — wrong context is worse than none, and dictation continues normally either way. +Chromium-based editors (VS Code, Electron apps) only keep their accessibility tree's caret position in sync once they detect an active accessibility client, so with default settings the position UI Automation reads back can be stale. In VS Code, set `editor.accessibilitySupport` to `on` (Settings → search "accessibility support"; a window reload may be needed) to get accurate cursor context; the same applies to other Electron-based editors with similar settings. Plain Chrome/Edge text fields do not need this. + Cursor context excludes password fields, macOS Secure Input (or the Windows UIA password-control flag), known password managers and terminals. Turning it off stops reading cursor context for polishing; other authorized accessibility features, including local vocabulary learning and insertion, work independently. Turning vocabulary learning off stops observation and clears pending suggestions. The main window is organized as Home / History / Dictionary / Settings. The Dictionary tab opens a separate editor window when you click "New". The Home tab shows total dictation time, total characters, average characters per minute, estimated time saved, and dictionary participation statistics. diff --git a/README.zh.md b/README.zh.md index aeddfeff1..41e70fdf8 100644 --- a/README.zh.md +++ b/README.zh.md @@ -391,6 +391,8 @@ OpenLess 的润色模型只重塑文本。它不回答问题、不执行任务 Windows 使用 UI Automation 读取当前焦点文本控件的光标附近内容：优先使用 `TextPattern2` 的 caret range；控件不支持 `TextPattern2` 时，兼容使用 `TextPattern` 的 collapsed selection。无法可靠定位 caret 时直接跳过，不会猜测光标位置，也不会影响正常听写。 +基于 Chromium 的编辑器（VS Code、Electron 应用）只有在检测到有无障碍客户端在用时，才会实时同步无障碍树里的光标位置，默认设置下 UI Automation 读到的位置可能是过时的。VS Code 里需要把 `editor.accessibilitySupport` 设为 `on`（设置里搜"accessibility support"，改完可能需要重新加载窗口）才能拿到准确的光标上下文；其他基于 Electron 的编辑器一般也有类似设置。普通的 Chrome/Edge 文本框不需要这一步。 + 光标上下文排除密码输入框、macOS Secure Input（Windows 上为 UIA 密码控件标记）、已知密码管理器和终端。关闭此开关后不会为润色读取光标上下文；其他已授权的辅助功能（如手改学词或插入）独立工作。手改学词关闭后会停止观察并清空待确认建议。 主窗口组织为 首页 / 历史 / 词典 / 设置。点击“新建”时,词典页会打开一个独立的编辑窗口。首页展示总听写时长、总字数、平均每分钟字数、估算节省的时间,以及词典参与统计。 diff --git a/openless-all/app/src-tauri/src/host_document/windows.rs b/openless-all/app/src-tauri/src/host_document/windows.rs index e087818e5..35f2d7f11 100644 --- a/openless-all/app/src-tauri/src/host_document/windows.rs +++ b/openless-all/app/src-tauri/src/host_document/windows.rs @@ -341,8 +341,9 @@ unsafe fn observe(request: &Request) -> Result<()> { /// Outcome of locating a usable caret [`IUIAutomationTextRange`]. enum CaretLookup { - /// A zero-length (collapsed) range at the insertion point. - Found(IUIAutomationTextRange), + /// A zero-length (collapsed) range at the insertion point, tagged with which pattern + /// produced it (diagnostic only — never logged with document content). + Found(IUIAutomationTextRange, &'static str), /// `TextPattern::GetSelection()` fallback found a real (non-collapsed) selection. Per /// §8 of the design doc, the first version never guesses which end is the caret — /// "selection rewrite" is a separate future feature, not folded into dictation context. @@ -368,7 +369,9 @@ unsafe fn find_caret_range(element: &IUIAutomationElement) -> CaretLookup { { let mut is_active = BOOL(0); match pattern2.GetCaretRange(&mut is_active) { - Ok(range) if is_active.as_bool() => return CaretLookup::Found(range), + Ok(range) if is_active.as_bool() => { + return CaretLookup::Found(range, "textPattern2") + } Ok(_) => return CaretLookup::Unavailable("caret is not active"), // GetCaretRange itself failed even though the pattern exists; fall through and // try the TextPattern/GetSelection compatibility path below. @@ -406,7 +409,7 @@ unsafe fn find_caret_range(element: &IUIAutomationElement) -> CaretLookup { &range, TextPatternRangeEndpoint_End, ) { - Ok(0) => CaretLookup::Found(range), + Ok(0) => CaretLookup::Found(range, "textPattern"), Ok(_) => CaretLookup::NonCollapsedSelection, Err(_) => CaretLookup::Unavailable("selection endpoint comparison failed"), } @@ -416,8 +419,8 @@ unsafe fn find_caret_range(element: &IUIAutomationElement) -> CaretLookup { /// context.** Never reads the full document (§7 of the design doc): the range is expanded /// from the caret by `MoveEndpointByUnit`, not sliced out of `DocumentRange()`. unsafe fn read_document(element: &IUIAutomationElement, budget_chars: usize) -> super::ReadOutcome { - let caret = match find_caret_range(element) { - CaretLookup::Found(range) => range, + let (caret, source) = match find_caret_range(element) { + CaretLookup::Found(range, source) => (range, source), CaretLookup::NonCollapsedSelection => { return super::ReadOutcome::Unavailable("non-collapsed selection") } @@ -439,11 +442,13 @@ unsafe fn read_document(element: &IUIAutomationElement, budget_chars: usize) -> let Ok(before_range) = caret.Clone() else { return super::ReadOutcome::Unavailable("caret range clone failed"); }; - let _ = before_range.MoveEndpointByUnit( - TextPatternRangeEndpoint_Start, - TextUnit_Character, - -before_want, - ); + let before_moved = before_range + .MoveEndpointByUnit( + TextPatternRangeEndpoint_Start, + TextUnit_Character, + -before_want, + ) + .unwrap_or(0); let Ok(before_text) = before_range.GetText(text_cap) else { return super::ReadOutcome::Unavailable("caret range text unavailable"); }; @@ -451,15 +456,30 @@ unsafe fn read_document(element: &IUIAutomationElement, budget_chars: usize) -> let Ok(after_range) = caret.Clone() else { return super::ReadOutcome::Unavailable("caret range clone failed"); }; - let _ = - after_range.MoveEndpointByUnit(TextPatternRangeEndpoint_End, TextUnit_Character, after_want); + let after_moved = after_range + .MoveEndpointByUnit(TextPatternRangeEndpoint_End, TextUnit_Character, after_want) + .unwrap_or(0); let Ok(after_text) = after_range.GetText(text_cap) else { return super::ReadOutcome::Unavailable("caret range text unavailable"); }; - let mut text = before_text.to_string(); + let before_text = before_text.to_string(); + let after_text = after_text.to_string(); + // Diagnostic only: how far the provider actually let each endpoint move vs. what was + // requested, and how many chars that text turned out to hold. No document content. + // `moved` vs the resulting char count diverging a lot (e.g. moved=-1200 but + // before_text has only a handful of chars) points at a provider whose "character" unit + // or caret tracking doesn't match the visible cursor (seen on Chromium-based UIA: VS + // Code, Chrome) rather than a bug in this redistribution math. + log::info!( + "[cursor-context] source={source} before_want={before_want} before_moved={before_moved} before_chars={} after_want={after_want} after_moved={after_moved} after_chars={}", + before_text.chars().count(), + after_text.chars().count(), + ); + + let mut text = before_text; let cursor = text.chars().count(); - text.push_str(&after_text.to_string()); + text.push_str(&after_text); super::ReadOutcome::Window(super::window_around_cursor(&text, cursor, budget_chars)) }