From 59823c91b11a2f9a210434a6fb45ad7e4b9a5537 Mon Sep 17 00:00:00 2001 From: Cassie Date: Fri, 2 Oct 2026 11:39:18 +0800 Subject: [PATCH 1/2] =?UTF-8?q?docs(dictation):=20=E6=8F=90=E8=AE=AE?= =?UTF-8?q?=E9=95=BF=E6=8C=89=E5=AE=9E=E6=97=B6=E8=BE=93=E5=85=A5=E4=B8=8E?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E5=8C=96=E7=BB=93=E6=9E=9C=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/live-dictation-proposal.zh.md | 83 ++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 docs/live-dictation-proposal.zh.md diff --git a/docs/live-dictation-proposal.zh.md b/docs/live-dictation-proposal.zh.md new file mode 100644 index 000000000..e6200ba2b --- /dev/null +++ b/docs/live-dictation-proposal.zh.md @@ -0,0 +1,83 @@ +# 提案：按手势区分实时输入与完整输出 + +状态：设计提案，尚未实现实时写入或自动替换。 +基线：`beta` / `6f9af895`（OpenLess 2.0.0-Beta.4）。 + +## 要解决的体验 + +短按与长按应提供两种明确体验： + +| 手势 | 录音期间 | 停止录音后 | +| --- | --- | --- | +| 短按锁存 | 继续录音，目标应用不写入中间文本；再次按键停止 | 整段润色、校验后，一次性写入 | +| 长按 | 按住说话时，在目标输入框实时显示 ASR 原文 | 松手停止，润色并校验后，仅替换本次口述占用的文本范围 | + +Auto 模式沿用现有 350 ms 分类阈值。短按仍是“按一下开始、再按一下结束”，不是一次按键就完成录音；Hold 模式按下录音、松手停止。Toggle/DoubleClick 保持现有录音启停语义并使用完整输出。按钮、CLI、Android 手势和远端输入必须明确各自策略，不能只根据全局默认快捷键模式推断来源。 + +本提案改变的是输入行为，不是强行将所有模型请求改成相同传输方式。短按可以在内部接收 SSE 并累积结果，但不把中间结果打进目标应用。长按实时阶段显示 ASR 原文，不声称这些文字已经经过 LLM 润色。 + +## 现有实现与差距 + +- `openless-all/app/crates/openless-core/src/hotkey_interpreter.rs` 已实现 Hold、Toggle、DoubleClick 和 Auto；Auto 使用原生事件时间戳识别长短按。 +- `dictation_context.rs` 捕获独立的输入设置；快捷键语义与模型润色流式输入目前没有绑定。 +- `api.rs` 的 `BackendEngineProgress::publish` 将 `TranscriptDelta` 累积到状态并发布事件，只有 `PolishDelta` 被送入 `ActiveTextInsertion`。因此“实时 ASR 可用”不等于“说话时就输入”。 +- `asr/volcengine.rs` 已通过 `TextStreamChunk { offset: 0, text: full_text }` 发出全量快照。新识别结果可能改写已经识别的前缀，不能简单追加 delta。其他支持 partial sink 的服务也必须分别确认事件语义。 +- `streaming_insert.rs` 的 `reconcile_final` 只支持补尾和完整插入。最终正文不以已输入文本开头时会返回 `CopyFallback`，不支持安全替换。 +- `ports.rs` 的 `TextInsertionSession` 只有 write/copy/finish/cancel，需要增加能力明确、默认不支持的受保护文本范围替换接口。 + +短回复可能等待很久后在几个 SSE 事件中集中返回。仅凭设置里的流式开关不能判断视觉上是否逐字出现；也不能仅凭这种表现判断客户端强制采用了非流式协议。 + +## 拟议实现 + +### 1. 会话级输入策略 + +在 Core 捕获 `CompleteAfterStop`、`LiveTranscriptThenPolish` 和 Auto 尚未分类状态。保留原生时间戳、press_id 和取消代际保护：延迟的启动握手不能把 50 ms 短按误判成长按。350 ms 前不得写入中间文本；短按松手后锁存的录音也不得进入实时输入。 + +录音事件读取、手势分类与异步启动应解耦，避免原有串行 bridge 在等待 ASR/录音启动时延迟处理 Released。停录音、Esc、组合键取消、窗口关闭和新会话必须阻止旧会话继续写入。 + +### 2. ASR 能力与实时快照 + +显式声明服务是否提供录音期间的转写快照。实时接口保留完整快照和修订范围，按用户可感知的短间隔合并频繁更新。批量 ASR 保持停止后完整输出，并解释该服务暂不支持实时显示；不自动更换用户的服务，也不通过反复上传重叠音频伪装实时能力。 + +### 3. 原文写入与最终替换 + +首次输入时冻结目标应用、控件和插入范围。平台适配器必须证明能够查询和更新该范围，才启用实时原文；不支持的控件退回完整输出。 + +所有 ASR 修订和最终润色都走同一个受保护范围更新操作： + +1. 确认仍是原控件，并验证当前范围内容与客户端上次确认的内容相同。 +2. 确认光标/选区仍符合本次口述操作，且用户没有插入、删除、粘贴或移动光标。 +3. 只修改本次口述的范围，保留前后正文；正确处理 UTF-16、emoji、换行及中文。 +4. 验证平台真正接受了更新，不能把“事件已经发送”当作“文本已写入”。 + +无法确认所有权时立即停止自动修改，保留用户编辑，提示完整润色结果可复制。禁止按字符串长度盲目发送 Backspace，也禁止无条件覆盖控件全文。取消时加入仍在执行的原生操作，之后才释放输入源和会话资源。 + +macOS AX 操作需保留已有安全门禁、输入字段限制和消息超时，运行在阻塞线程；用于保护替换的文档基线只留在本地，不作为新增 LLM 上下文上传。Windows TSF/UIA、Linux 输入法及 Android 需分别实现能力握手；首个实现可限定 macOS 可验证的文本控件，其余平台明确回退。 + +### 4. 结构化结果校验 + +在首次完整插入或最终替换前，对内置结构化风格执行同一套有限校验： + +- 同层级列表连续编号，识别重复编号和 `2. 2.` 双编号。 +- 不把 `(a)` 子项重新当作顶层事项；每组子项独立编号。 +- 保留年份、版本号、命令、路径、URL、引用、代码块及用户自定义格式。 +- 最终插入内容与历史记录保持一致。 + +`output_cleaning.rs` 目前不检查列表编号；OpenAI 兼容流式路径直接传出文本，非流式响应解析会调用 `clean_polish_output`。需统一润色输出清洗语义，并考虑思考标签、套话、围栏横跨 token 的情况。QA 正文、翻译正文和自定义风格不能复用未经限定的编号替换规则。 + +报告中的 ` `、`2\.` 可能产生于模型、Markdown 序列化或目标编辑器转换。先比较 ASR 文本、模型原始正文、清洗后的正文及目标应用实际内容，确定责任层；不能全局解码实体或删除反斜线。诊断只保留用户明确授权的文本样本，不新增默认全文日志。 + +## 验收标准 + +- Auto 349 ms 短按不输入中间正文；350 ms 长按在支持实时 ASR/控件时，松手前可看到原文。 +- 慢启动、排队释放、重复按键、过期 press_id 不改变手势分类。 +- 短按停止后仅插入一次完整结果；长按停止后只替换口述范围，不追加重复正文。 +- ASR 前缀改写、重复快照、offset 修订不产生重复或丢字。 +- 中途手动编辑、移动光标、切换控件、控件销毁或权限丢失时，用户文本保持不变。 +- Esc、服务错误和关闭流程阻止后续输入，最终操作与新会话互不穿插。 +- 测试重复编号、子项重置、版本号、年份、围栏、多层列表、跨 token 标签和 Unicode 边界。 +- 在真实可编辑控件验证原文出现、ASR 修订、松手润色、用户编辑保护和批量 ASR 回退。 + +## 与麦克风修复的关系 + +设备原生格式、启动恢复和空/全零录音拦截由独立麦克风 PR 处理，本提案不依赖修改用户的首选设备。这里同样不能把设备存在、流回调存活、非零音频、有效语音识别四件事混为一谈。 From cc69638aed0798f5add71229afe6689ed2fae0e9 Mon Sep 17 00:00:00 2001 From: Cassie Date: Sat, 3 Oct 2026 00:25:11 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix(dictation):=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E9=95=BF=E6=8C=89=E5=8E=9F=E6=96=87=E8=BE=93=E5=85=A5=E3=80=81?= =?UTF-8?q?=E8=8C=83=E5=9B=B4=E6=9B=BF=E6=8D=A2=E4=B8=8E=E7=BC=96=E5=8F=B7?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/live-dictation-proposal.zh.md | 93 ++-- .../app/crates/openless-core/src/api.rs | 503 +++++++++++++++++- .../openless-core/src/dictation_context.rs | 20 + .../app/crates/openless-core/src/lib.rs | 2 +- .../openless-core/src/output_cleaning.rs | 123 +++++ .../app/crates/openless-core/src/ports.rs | 15 + .../src-tauri/src/coordinator/hotkey_loops.rs | 24 + .../app/src-tauri/src/core_adapters.rs | 87 ++- .../app/src-tauri/src/host_document/macos.rs | 257 +++++++++ .../app/src-tauri/src/host_document/mod.rs | 2 +- openless-all/app/src-tauri/src/selection.rs | 12 + 11 files changed, 1066 insertions(+), 72 deletions(-) diff --git a/docs/live-dictation-proposal.zh.md b/docs/live-dictation-proposal.zh.md index e6200ba2b..a99a63a12 100644 --- a/docs/live-dictation-proposal.zh.md +++ b/docs/live-dictation-proposal.zh.md @@ -1,83 +1,54 @@ -# 提案：按手势区分实时输入与完整输出 +# 按手势区分实时输入与完整输出 -状态：设计提案，尚未实现实时写入或自动替换。 +状态：已实现首版，等待审核与 macOS 控件实测。 基线：`beta` / `6f9af895`（OpenLess 2.0.0-Beta.4）。 -## 要解决的体验 +## 手势与输出 -短按与长按应提供两种明确体验： - -| 手势 | 录音期间 | 停止录音后 | +| 模式 | 录音期间 | 停止录音后 | | --- | --- | --- | -| 短按锁存 | 继续录音，目标应用不写入中间文本；再次按键停止 | 整段润色、校验后，一次性写入 | -| 长按 | 按住说话时，在目标输入框实时显示 ASR 原文 | 松手停止，润色并校验后，仅替换本次口述占用的文本范围 | - -Auto 模式沿用现有 350 ms 分类阈值。短按仍是“按一下开始、再按一下结束”，不是一次按键就完成录音；Hold 模式按下录音、松手停止。Toggle/DoubleClick 保持现有录音启停语义并使用完整输出。按钮、CLI、Android 手势和远端输入必须明确各自策略，不能只根据全局默认快捷键模式推断来源。 - -本提案改变的是输入行为，不是强行将所有模型请求改成相同传输方式。短按可以在内部接收 SSE 并累积结果，但不把中间结果打进目标应用。长按实时阶段显示 ASR 原文，不声称这些文字已经经过 LLM 润色。 - -## 现有实现与差距 - -- `openless-all/app/crates/openless-core/src/hotkey_interpreter.rs` 已实现 Hold、Toggle、DoubleClick 和 Auto；Auto 使用原生事件时间戳识别长短按。 -- `dictation_context.rs` 捕获独立的输入设置；快捷键语义与模型润色流式输入目前没有绑定。 -- `api.rs` 的 `BackendEngineProgress::publish` 将 `TranscriptDelta` 累积到状态并发布事件，只有 `PolishDelta` 被送入 `ActiveTextInsertion`。因此“实时 ASR 可用”不等于“说话时就输入”。 -- `asr/volcengine.rs` 已通过 `TextStreamChunk { offset: 0, text: full_text }` 发出全量快照。新识别结果可能改写已经识别的前缀，不能简单追加 delta。其他支持 partial sink 的服务也必须分别确认事件语义。 -- `streaming_insert.rs` 的 `reconcile_final` 只支持补尾和完整插入。最终正文不以已输入文本开头时会返回 `CopyFallback`，不支持安全替换。 -- `ports.rs` 的 `TextInsertionSession` 只有 write/copy/finish/cancel，需要增加能力明确、默认不支持的受保护文本范围替换接口。 - -短回复可能等待很久后在几个 SSE 事件中集中返回。仅凭设置里的流式开关不能判断视觉上是否逐字出现；也不能仅凭这种表现判断客户端强制采用了非流式协议。 - -## 拟议实现 - -### 1. 会话级输入策略 - -在 Core 捕获 `CompleteAfterStop`、`LiveTranscriptThenPolish` 和 Auto 尚未分类状态。保留原生时间戳、press_id 和取消代际保护：延迟的启动握手不能把 50 ms 短按误判成长按。350 ms 前不得写入中间文本；短按松手后锁存的录音也不得进入实时输入。 - -录音事件读取、手势分类与异步启动应解耦，避免原有串行 bridge 在等待 ASR/录音启动时延迟处理 Released。停录音、Esc、组合键取消、窗口关闭和新会话必须阻止旧会话继续写入。 +| Auto 短按（不足 350 ms） | 锁存录音，不向目标应用写入中间文本；再次按键停止 | 完整结果一次性写入 | +| Auto 长按（至少 350 ms） | 支持实时 ASR、且目标控件通过能力检查时，显示 ASR 原文 | 松手停止，润色后替换本次口述范围 | +| Hold | 按住录音，在支持的目标中显示原文 | 松手停止并润色替换 | +| Toggle / DoubleClick | 保留既有录音启停语义，不写入中间正文 | 完整输出 | -### 2. ASR 能力与实时快照 +使用者需选择 Auto 才能用同一个快捷键区分长短按。本改动不修改保存的快捷键模式、麦克风或识别服务。按钮、CLI、Android 与远端输入保留调用时的原有输出设置；内置结构化风格会等待完整结果校验后再输出。 -显式声明服务是否提供录音期间的转写快照。实时接口保留完整快照和修订范围，按用户可感知的短间隔合并频繁更新。批量 ASR 保持停止后完整输出，并解释该服务暂不支持实时显示；不自动更换用户的服务，也不通过反复上传重叠音频伪装实时能力。 +实时显示依赖服务在录音期间发布 `TranscriptDelta`。稳定转写模式、批量 ASR 或只返回最终正文的服务仍在停止后完整输出；不自动换服务、不重复上传重叠音频模拟实时识别。 -### 3. 原文写入与最终替换 +## 实现边界 -首次输入时冻结目标应用、控件和插入范围。平台适配器必须证明能够查询和更新该范围，才启用实时原文；不支持的控件退回完整输出。 +- Core 捕获会话级 `DictationDelivery`，将热点键模式与服务的 SSE 传输方式分开。短按允许内部累计流式响应，但不向目标控件发送中间结果。 +- macOS 原生桥仍按 FIFO 顺序执行异步启停。独立入口即时记录物理释放时间，避免启动握手延迟把短按误判成长按。Auto 阈值前保留最新原文快照，释放后丢弃延迟快照。 +- `TranscriptAccumulator` 先应用 ASR 的 offset 修订，再把完整快照送入受保护范围更新；50 ms 合并更新。不会把全量快照当成追加片段。 +- 新 `TextInsertionSession::replace_owned` 默认不支持。当前只有 macOS 的可验证 `AXTextField` / `AXTextArea` 适配器实现；Windows、Linux、Android 和不支持 AX 选区写入的控件回退完整输出。 +- 本地基线通过现有 Secure Input、密码管理器、终端及字段门禁，AX 单次消息有超时，调用运行在阻塞线程。基线只用于本地保护，不作为新增 LLM 上下文上传。 -所有 ASR 修订和最终润色都走同一个受保护范围更新操作： +## 文本所有权与取消 -1. 确认仍是原控件，并验证当前范围内容与客户端上次确认的内容相同。 -2. 确认光标/选区仍符合本次口述操作，且用户没有插入、删除、粘贴或移动光标。 -3. 只修改本次口述的范围，保留前后正文；正确处理 UTF-16、emoji、换行及中文。 -4. 验证平台真正接受了更新，不能把“事件已经发送”当作“文本已写入”。 +原文首次写入前捕获控件、空选区光标和有限长度基线（最多 20,000 UTF-16 单元）。已有选中文本时不启用实时输入。每次修订或最终替换前检查： -无法确认所有权时立即停止自动修改，保留用户编辑，提示完整润色结果可复制。禁止按字符串长度盲目发送 Backspace，也禁止无条件覆盖控件全文。取消时加入仍在执行的原生操作，之后才释放输入源和会话资源。 +1. 仍聚焦同一可访问控件，仍通过安全门禁。 +2. 光标在本次口述末尾、没有额外选区，全文与上次确认的本地基线相同。 +3. 通过 `AXSelectedTextRange` 与 `AXSelectedText` 更新本次口述范围；选区设置后再次检查基线与选区。 +4. 读取结果与光标确认送达；正确转换 UTF-16，拒绝切开 emoji 的代理对。 -macOS AX 操作需保留已有安全门禁、输入字段限制和消息超时，运行在阻塞线程；用于保护替换的文档基线只留在本地，不作为新增 LLM 上下文上传。Windows TSF/UIA、Linux 输入法及 Android 需分别实现能力握手；首个实现可限定 macOS 可验证的文本控件，其余平台明确回退。 +不发送按字符数猜测的 Backspace，也不设置控件全文 `AXValue`。发现编辑、控件切换、权限丢失或送达不确定后，停止自动写入，保留当前正文，将最终结果复制到剪贴板并走既有复制回退提示。已经尝试原生写入的会话不会再进行可能重复正文的完整粘贴。 -### 4. 结构化结果校验 +取消会清除排队快照，并等待正在执行的原生调用结束后释放会话。已确认写入的原文不会在取消时盲目删除。 -在首次完整插入或最终替换前，对内置结构化风格执行同一套有限校验： +**平台限制：** macOS AX 没有跨进程原子“比较并替换”操作；这些检查无法消除用户恰在最后一次验证与 AX 写入之间操作的竞态。应在真实控件持续输入、移动光标、切换应用和取消场景中验证；本 PR 不声称提供事务级隔离。原生调用不以丢弃阻塞任务来处理超时，避免超时后的旧写入穿插到新会话。 -- 同层级列表连续编号，识别重复编号和 `2. 2.` 双编号。 -- 不把 `(a)` 子项重新当作顶层事项；每组子项独立编号。 -- 保留年份、版本号、命令、路径、URL、引用、代码块及用户自定义格式。 -- 最终插入内容与历史记录保持一致。 +## 输出格式 -`output_cleaning.rs` 目前不检查列表编号；OpenAI 兼容流式路径直接传出文本，非流式响应解析会调用 `clean_polish_output`。需统一润色输出清洗语义，并考虑思考标签、套话、围栏横跨 token 的情况。QA 正文、翻译正文和自定义风格不能复用未经限定的编号替换规则。 +内置 `builtin.structured` 的成功润色结果在插入和历史存储前走同一校验：同级连续数字列表重新编号，移除相邻重复的同一编号（如 `2. 2.`），父项变化时重置子列表。其他风格、翻译及润色失败的原文回退不执行编号校验。 -报告中的 ` `、`2\.` 可能产生于模型、Markdown 序列化或目标编辑器转换。先比较 ASR 文本、模型原始正文、清洗后的正文及目标应用实际内容，确定责任层；不能全局解码实体或删除反斜线。诊断只保留用户明确授权的文本样本，不新增默认全文日志。 +代码围栏、引用、年份、版本号、URL、字母子项和单独数字行保持原样。结构化结果不先流式写入未校验编号。清洗模型思考标签与套话时保留未发生实质清洗的原有外围空白。 -## 验收标准 +报告中的 ` ` 和 `2\.` 仍需对照模型原始正文与目标编辑器内容确定来源；本改动不全局解码实体或删除反斜线，也不新增默认全文日志。 -- Auto 349 ms 短按不输入中间正文；350 ms 长按在支持实时 ASR/控件时，松手前可看到原文。 -- 慢启动、排队释放、重复按键、过期 press_id 不改变手势分类。 -- 短按停止后仅插入一次完整结果；长按停止后只替换口述范围，不追加重复正文。 -- ASR 前缀改写、重复快照、offset 修订不产生重复或丢字。 -- 中途手动编辑、移动光标、切换控件、控件销毁或权限丢失时，用户文本保持不变。 -- Esc、服务错误和关闭流程阻止后续输入，最终操作与新会话互不穿插。 -- 测试重复编号、子项重置、版本号、年份、围栏、多层列表、跨 token 标签和 Unicode 边界。 -- 在真实可编辑控件验证原文出现、ASR 修订、松手润色、用户编辑保护和批量 ASR 回退。 +## 验证与待验证 -## 与麦克风修复的关系 +核心回归覆盖原文前缀修订、最终替换、用户编辑后的复制回退、349 ms 短按、阈值前快照缓存、取消等待正在执行的替换、完整输出、编号与历史一致性。UTF-16 范围测试覆盖中文、emoji、换行、文末插入和代理对边界。 -设备原生格式、启动恢复和空/全零录音拦截由独立麦克风 PR 处理，本提案不依赖修改用户的首选设备。这里同样不能把设备存在、流回调存活、非零音频、有效语音识别四件事混为一谈。 +还需维护者/使用者在真实 macOS 文本控件验证 AX 能力、实时 ASR 出字、输入法、控件自动格式化、手动编辑、焦点变化与取消。未验证平台不启用受保护替换。麦克风格式与空录音修复是独立 PR #1135。 diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index a96c1cbe9..01b03cdba 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -1761,6 +1761,10 @@ type TextInsertionPreparation = futures_util::future::Shared< >; struct ActiveTextInsertion { + delivery: crate::dictation_context::DictationDelivery, + owned_replacement: bool, + physical_releases: + Mutex>>>>, platform: Arc, streaming: bool, script: crate::shared_types::ChineseScriptPreference, @@ -1781,6 +1785,8 @@ struct ActiveTextInsertion { struct ActiveTextInsertionState { stream: crate::streaming_insert::StreamingInsertState, scheduled: bool, + live_snapshot: Option, + live_touched: bool, } impl ActiveTextInsertion { @@ -1794,9 +1800,20 @@ impl ActiveTextInsertion { && context.insertion.windows_insertion_mode != crate::shared_types::WindowsInsertionMode::SendInput; let platform_streaming = platform.supports_streaming(); + let delivery = context.insertion.delivery; + let owned_replacement = matches!( + delivery, + crate::dictation_context::DictationDelivery::Hold { .. } + | crate::dictation_context::DictationDelivery::Auto { .. } + ) && platform.supports_owned_replacement(); Arc::new(Self { + delivery, + owned_replacement, + physical_releases: Mutex::new(None), platform, - streaming: context.uses_llm_polisher() + streaming: delivery == crate::dictation_context::DictationDelivery::Preferences + && context.polish.style_pack_id != "builtin.structured" + && context.uses_llm_polisher() && platform_streaming && crate::streaming_insert::streaming_insert_eligible( context.insertion.streaming, @@ -1816,6 +1833,119 @@ impl ActiveTextInsertion { }) } + fn live_allowed(&self) -> bool { + use crate::dictation_context::DictationDelivery; + if !self.owned_replacement || self.terminal.load(Ordering::Acquire) != 0 { + return false; + } + let (press_id, pressed_at, threshold) = match self.delivery { + DictationDelivery::Hold { + press_id, + pressed_at, + } => (press_id, pressed_at, std::time::Duration::ZERO), + DictationDelivery::Auto { + press_id, + pressed_at, + } => (press_id, pressed_at, std::time::Duration::from_millis(350)), + _ => return false, + }; + let ledger = self + .physical_releases + .lock() + .expect("release ledger lock poisoned"); + // A later physical release also closes this held gesture. This fails + // closed when a startup backlog evicts its exact id from the ledger. + let released = ledger.as_ref().is_some_and(|ledger| { + ledger + .lock() + .expect("physical release lock poisoned") + .iter() + .any(|(id, at)| *id == press_id || *at >= pressed_at) + }); + !released && pressed_at.elapsed() >= threshold + } + + fn push_transcript(self: &Arc, snapshot: String) { + if !self.owned_replacement || self.terminal.load(Ordering::Acquire) != 0 { + return; + } + let should_spawn = { + let mut state = self.state.lock().expect("text insertion lock poisoned"); + if state.stream.failed.is_some() { + return; + } + let snapshot = + crate::streaming_insert::apply_chinese_script_preference(&snapshot, self.script); + if state.live_snapshot.is_none() && state.stream.typed_text == snapshot { + return; + } + state.live_snapshot = Some(snapshot); + if state.scheduled { + false + } else { + state.scheduled = true; + true + } + }; + if should_spawn { + let insertion = Arc::clone(self); + self.task_spawner + .spawn(Box::pin(async move { insertion.live_flush_loop().await })); + } + } + + async fn live_flush_loop(self: Arc) { + loop { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let snapshot = { + let mut state = self.state.lock().expect("text insertion lock poisoned"); + if state.stream.failed.is_some() || self.terminal.load(Ordering::Acquire) != 0 { + state.scheduled = false; + self.drained.notify_waiters(); + return; + } + if !self.live_allowed() { + if let crate::dictation_context::DictationDelivery::Auto { + pressed_at, .. + } = self.delivery + { + if pressed_at.elapsed() < std::time::Duration::from_millis(350) { + // Keep the first snapshot until classification; release + // evidence is checked again before any native effect. + continue; + } + } + state.scheduled = false; + self.drained.notify_waiters(); + return; + } + state.live_snapshot.take() + }; + let Some(snapshot) = snapshot else { + self.state + .lock() + .expect("text insertion lock poisoned") + .scheduled = false; + self.drained.notify_waiters(); + return; + }; + let result = self.platform.replace_owned(snapshot.clone()).await; + let mut state = self.state.lock().expect("text insertion lock poisoned"); + // Any attempted native edit makes subsequent final paste unsafe, + // including delivery failures where the outcome may be unknown. + state.live_touched = true; + match result { + Ok(()) => state.stream.typed_text = snapshot, + Err(error) => state.stream.failed = Some(error.to_string()), + } + if state.live_snapshot.is_none() || state.stream.failed.is_some() { + state.scheduled = false; + self.drained.notify_waiters(); + return; + } + } + } + fn push(self: &Arc, delta: &crate::types::PolishDelta) { if !self.streaming || delta.is_final { return; @@ -1941,6 +2071,27 @@ impl ActiveTextInsertion { } async fn finish_committed(&self, final_text: String) -> Result { + let (live_touched, failed) = { + let state = self.state.lock().expect("text insertion lock poisoned"); + (state.live_touched, state.stream.failed.is_some()) + }; + if live_touched { + if failed + || self + .platform + .replace_owned(final_text.clone()) + .await + .is_err() + { + return self.finish_with_fallback(final_text).await; + } + if self.save_streamed_text_to_clipboard { + if let Err(error) = self.platform.copy(final_text).await { + log::warn!("failed to copy live result: {error}"); + } + } + return self.platform.finish(String::new()).await; + } let reconciliation = self .state .lock() @@ -1989,6 +2140,7 @@ impl ActiveTextInsertion { { let mut state = self.state.lock().expect("text insertion lock poisoned"); state.stream.pending.clear(); + state.live_snapshot = None; state.stream.failed = Some("text insertion session was cancelled".to_string()); } if self @@ -2176,7 +2328,30 @@ impl EngineProgressSink for BackendEngineProgress { .entry(session_id) .or_default() .apply(&delta)?; + let snapshot = state + .transcripts + .get(&session_id) + .expect("transcript applied") + .text() + .to_owned(); + let recording = matches!( + state.dictation.phase, + DictationPhase::Starting | DictationPhase::Recording + ); drop(state); + if recording { + if let Some(insertion) = self + .text_insertions + .lock() + .expect("text insertion registry lock poisoned") + .get(&session_id) + .and_then(|preparation| preparation.peek()) + .and_then(|result| result.as_ref().ok()) + .cloned() + { + insertion.push_transcript(snapshot); + } + } self.events .publish(Some(session_id), BackendEventKind::TranscriptDelta(delta)); } @@ -2219,6 +2394,7 @@ pub struct OpenLessBackend { phase_changed: Arc, hotkey: Mutex, hotkey_dispatch_gate: tokio::sync::Mutex<()>, + physical_releases: Arc>>, less_computer_hotkey_press_at: Mutex>, vocabulary: Arc, correction_rules: Arc, @@ -2816,6 +2992,7 @@ impl OpenLessBackend { })), phase_changed: Arc::new(tokio::sync::Notify::new()), hotkey: Mutex::new(crate::hotkey_interpreter::HotkeyInterpreter::default()), + physical_releases: Arc::new(Mutex::new(std::collections::VecDeque::new())), hotkey_dispatch_gate: tokio::sync::Mutex::new(()), less_computer_hotkey_press_at: Mutex::new(None), vocabulary: repositories.vocabulary, @@ -4038,6 +4215,21 @@ impl OpenLessBackend { } } + /// Native bridges report releases before queueing their ordered async work. + /// This prevents a short tap during slow startup from enabling live input. + pub fn observe_dictation_hotkey_release(&self, press_id: u64, at: std::time::Instant) { + let mut releases = self + .physical_releases + .lock() + .expect("physical release lock poisoned"); + if !releases.iter().any(|(id, _)| *id == press_id) { + releases.push_back((press_id, at)); + if releases.len() > 64 { + releases.pop_front(); + } + } + } + /// Apply physical dictation-key edges using the shared hotkey-mode rules. /// /// Native listeners provide a stable physical-press id plus monotonic event @@ -4086,6 +4278,9 @@ impl OpenLessBackend { ) -> Result { use crate::hotkey_interpreter::HotkeyIntent; + if let DictationHotkeyEdge::Released { press_id, at } = edge { + self.observe_dictation_hotkey_release(press_id, at); + } // Pressed and Released stay FIFO even though start/finalize await native // work. Combined deliberately bypasses this gate: it has a dedicated // low-latency host bridge and must be able to cancel a start in flight. @@ -4095,8 +4290,26 @@ impl OpenLessBackend { } else { Some(self.hotkey_dispatch_gate.lock().await) }; + let mut options = options; let preferences = self.get_preferences(); let mode = preferences.hotkey.mode; + if let DictationHotkeyEdge::Pressed { press_id, at } = edge { + options.start.delivery = match mode { + crate::shared_types::HotkeyMode::Hold => { + crate::dictation_context::DictationDelivery::Hold { + press_id, + pressed_at: at, + } + } + crate::shared_types::HotkeyMode::Auto => { + crate::dictation_context::DictationDelivery::Auto { + press_id, + pressed_at: at, + } + } + _ => crate::dictation_context::DictationDelivery::Complete, + }; + } let modifier_only = crate::shortcut_types::is_modifier_chord_binding(&preferences.dictation_hotkey) || crate::hotkey_interpreter::modifier_arbitration_required( @@ -5686,6 +5899,7 @@ impl OpenLessBackend { } if context.insertion.enabled { let insertion_context = Arc::clone(&context); + let physical_releases = Arc::clone(&self.physical_releases); let task_spawner = Arc::clone(&self.deps.task_spawner); let resources = Arc::clone(&starting_resources); let preparing: futures_util::future::BoxFuture< @@ -5695,12 +5909,13 @@ impl OpenLessBackend { let platform = inserter .begin(session_id, Arc::clone(&insertion_context)) .await?; - Ok(ActiveTextInsertion::new( - platform, - &insertion_context, - task_spawner, - resources, - )) + let insertion = + ActiveTextInsertion::new(platform, &insertion_context, task_spawner, resources); + *insertion + .physical_releases + .lock() + .expect("release ledger lock poisoned") = Some(physical_releases); + Ok(insertion) }); let preparation = futures_util::FutureExt::shared(preparing); { @@ -6120,6 +6335,19 @@ impl OpenLessBackend { return Err(error); } + if context.polish.style_pack_id == "builtin.structured" + && !context.polish.translation_active + && !engine_result.polish_failed + { + let cleaned = crate::output_cleaning::clean_polish_output(&engine_result.polished_text); + if cleaned != engine_result.polished_text.trim() { + engine_result.polished_text = cleaned; + } + engine_result.polished_text = crate::output_cleaning::normalize_structured_numbering( + &engine_result.polished_text, + ); + } + engine_result.polished_text = crate::streaming_insert::apply_chinese_script_preference( &engine_result.polished_text, context.polish.chinese_script_preference, @@ -6140,7 +6368,7 @@ impl OpenLessBackend { .get(&session_id) .and_then(|preparation| preparation.peek()) .and_then(|result| result.as_ref().ok()) - .is_some_and(|insertion| insertion.has_written_text()); + .is_some_and(|insertion| !insertion.owned_replacement && insertion.has_written_text()); if !correction_rules.is_empty() && !streamed_text_is_visible { engine_result.polished_text = apply_correction_rules(&engine_result.polished_text, &correction_rules); @@ -12685,6 +12913,9 @@ mod tests { ) .unwrap(); let mut preferences = backend.get_preferences(); + // This fixture verifies the legacy streaming contract; structured output + // is finalized as a whole so numbering can be validated first. + preferences.active_style_pack_id = "builtin.light".into(); preferences.streaming_insert = true; preferences.streaming_insert_save_clipboard = false; preferences.windows_insertion_mode = crate::shared_types::WindowsInsertionMode::SendInput; @@ -12855,6 +13086,9 @@ mod tests { .unwrap(), ); let mut preferences = backend.get_preferences(); + // This fixture verifies the legacy streaming contract; structured output + // is finalized as a whole so numbering can be validated first. + preferences.active_style_pack_id = "builtin.light".into(); preferences.streaming_insert = streaming; preferences.windows_insertion_mode = crate::shared_types::WindowsInsertionMode::SendInput; preferences.translation_target_language = "English".into(); @@ -13039,6 +13273,9 @@ mod tests { ) .unwrap(); let mut preferences = backend.get_preferences(); + // This fixture verifies the legacy streaming contract; structured output + // is finalized as a whole so numbering can be validated first. + preferences.active_style_pack_id = "builtin.light".into(); preferences.streaming_insert = true; preferences.streaming_insert_save_clipboard = false; preferences.windows_insertion_mode = crate::shared_types::WindowsInsertionMode::SendInput; @@ -14917,4 +15154,254 @@ mod tests { ); assert!(!data_dir.path().exists()); } + #[derive(Default)] + struct OwnedRangeFixture { + actions: Mutex>, + edited: std::sync::atomic::AtomicBool, + entered: Option>, + release: Option>, + } + impl TextInsertionSession for OwnedRangeFixture { + fn supports_owned_replacement(&self) -> bool { + true + } + fn replace_owned(&self, text: String) -> BoxFuture<'static, Result<(), BackendError>> { + let valid = !self.edited.load(Ordering::Acquire); + self.actions.lock().unwrap().push(format!("replace:{text}")); + let entered = self.entered.clone(); + let release = self.release.clone(); + boxed(async move { + if let Some(entered) = entered { + entered.add_permits(1); + } + if let Some(release) = release { + release.acquire().await.unwrap().forget(); + } + if valid { + Ok(()) + } else { + Err(BackendError::new(BackendErrorCode::Platform, "user edit")) + } + }) + } + fn write(&self, _: String) -> BoxFuture<'static, Result> { + panic!("raw text must use owned replacement") + } + fn copy(&self, text: String) -> BoxFuture<'static, Result<(), BackendError>> { + self.actions.lock().unwrap().push(format!("copy:{text}")); + boxed(async { Ok(()) }) + } + fn finish(&self, text: String) -> BoxFuture<'static, Result> { + self.actions.lock().unwrap().push(format!("finish:{text}")); + boxed(async { Ok(InsertOutcome::Inserted) }) + } + fn cancel(&self) -> BoxFuture<'static, Result<(), BackendError>> { + self.actions.lock().unwrap().push("cancel".into()); + boxed(async { Ok(()) }) + } + } + fn owned_insertion( + fixture: Arc, + delivery: crate::dictation_context::DictationDelivery, + ) -> Arc { + let gate = Arc::new(crate::voice_session::VoiceSessionGate::default()); + let session = SessionId::new(); + gate.acquire(session, crate::voice_session::VoiceSessionKind::Dictation) + .unwrap(); + let mut context = DictationContext::default(); + context.insertion.delivery = delivery; + context.insertion.save_streamed_text_to_clipboard = false; + ActiveTextInsertion::new( + fixture, + &context, + Arc::new(TokioTaskSpawner), + gate.hold_resources(session).unwrap(), + ) + } + #[tokio::test] + async fn live_snapshots_revise_prefix_then_replace_with_polished_text() { + let fixture = Arc::new(OwnedRangeFixture::default()); + let insertion = owned_insertion( + fixture.clone(), + crate::dictation_context::DictationDelivery::Hold { + press_id: 7, + pressed_at: std::time::Instant::now(), + }, + ); + insertion.push_transcript("你号".into()); + insertion.wait_for_stream_drain().await; + insertion.push_transcript("你好世界🙂".into()); + insertion.wait_for_stream_drain().await; + assert_eq!( + insertion.finish("你好，世界🙂！".into()).await.unwrap(), + InsertOutcome::Inserted + ); + assert_eq!( + *fixture.actions.lock().unwrap(), + [ + "replace:你号", + "replace:你好世界🙂", + "replace:你好，世界🙂！", + "finish:" + ] + ); + } + #[tokio::test] + async fn live_user_edit_keeps_document_and_copies_final_result() { + let fixture = Arc::new(OwnedRangeFixture::default()); + let insertion = owned_insertion( + fixture.clone(), + crate::dictation_context::DictationDelivery::Hold { + press_id: 7, + pressed_at: std::time::Instant::now(), + }, + ); + insertion.push_transcript("原文".into()); + insertion.wait_for_stream_drain().await; + fixture.edited.store(true, Ordering::Release); + assert_eq!( + insertion.finish("润色".into()).await.unwrap(), + InsertOutcome::CopiedFallback + ); + assert_eq!( + *fixture.actions.lock().unwrap(), + ["replace:原文", "replace:润色", "copy:润色", "finish:"] + ); + } + #[tokio::test] + async fn live_auto_short_release_during_startup_never_writes_raw_text() { + let fixture = Arc::new(OwnedRangeFixture::default()); + let pressed = std::time::Instant::now() - std::time::Duration::from_secs(1); + let insertion = owned_insertion( + fixture.clone(), + crate::dictation_context::DictationDelivery::Auto { + press_id: 8, + pressed_at: pressed, + }, + ); + *insertion.physical_releases.lock().unwrap() = + Some(Arc::new(Mutex::new(std::collections::VecDeque::from([( + 8, + pressed + std::time::Duration::from_millis(349), + )])))); + insertion.push_transcript("原文".into()); + insertion.wait_for_stream_drain().await; + insertion.finish("完整结果".into()).await.unwrap(); + assert_eq!(*fixture.actions.lock().unwrap(), ["finish:完整结果"]); + } + #[tokio::test] + async fn live_auto_holds_first_snapshot_until_threshold_and_ignores_polish_deltas() { + let fixture = Arc::new(OwnedRangeFixture::default()); + let insertion = owned_insertion( + fixture.clone(), + crate::dictation_context::DictationDelivery::Auto { + press_id: 8, + pressed_at: std::time::Instant::now(), + }, + ); + insertion.push_transcript("原文".into()); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert!(fixture.actions.lock().unwrap().is_empty()); + insertion.wait_for_stream_drain().await; + insertion.push(&crate::types::PolishDelta { + offset: 0, + text: "润色碎片".into(), + is_final: false, + }); + insertion.finish("润色结果".into()).await.unwrap(); + assert_eq!( + *fixture.actions.lock().unwrap(), + ["replace:原文", "replace:润色结果", "finish:"] + ); + } + #[tokio::test] + async fn live_cancel_joins_native_replacement_and_discards_queued_snapshots() { + let entered = Arc::new(tokio::sync::Semaphore::new(0)); + let release = Arc::new(tokio::sync::Semaphore::new(0)); + let fixture = Arc::new(OwnedRangeFixture { + entered: Some(entered.clone()), + release: Some(release.clone()), + ..Default::default() + }); + let insertion = owned_insertion( + fixture.clone(), + crate::dictation_context::DictationDelivery::Hold { + press_id: 7, + pressed_at: std::time::Instant::now(), + }, + ); + insertion.push_transcript("原文".into()); + entered.acquire().await.unwrap().forget(); + insertion.push_transcript("排队原文".into()); + let cancelling = insertion.clone(); + let task = tokio::spawn(async move { cancelling.cancel().await }); + tokio::task::yield_now().await; + assert!(!task.is_finished()); + release.add_permits(1); + task.await.unwrap().unwrap(); + insertion.push_transcript("过期原文".into()); + assert_eq!(*fixture.actions.lock().unwrap(), ["replace:原文", "cancel"]); + } + #[tokio::test] + async fn complete_delivery_ignores_intermediate_raw_and_polish_text() { + let fixture = Arc::new(OwnedRangeFixture::default()); + let insertion = owned_insertion( + fixture.clone(), + crate::dictation_context::DictationDelivery::Complete, + ); + insertion.push_transcript("原文".into()); + insertion.push(&crate::types::PolishDelta { + offset: 0, + text: "中间结果".into(), + is_final: false, + }); + insertion.finish("完整结果".into()).await.unwrap(); + assert_eq!(*fixture.actions.lock().unwrap(), ["finish:完整结果"]); + } + #[tokio::test] + async fn structured_numbering_is_validated_before_insertion_and_history() { + use crate::testing::{FixtureDictationEngine, FixtureInsertionAction, FixtureTextInserter}; + let data_dir = TestDataDir::new("structured-final-numbering"); + let inserter = FixtureTextInserter::with_outcome(InsertOutcome::Inserted); + let mut deps = BackendDependencies::unsupported(); + deps.host_actions = Arc::new(FakeHost::default()); + deps.credential_store = Arc::new(crate::credentials::InMemoryCredentialStore::default()); + deps.text_inserter = Arc::new(inserter.clone()); + deps.task_spawner = Arc::new(TokioTaskSpawner); + deps.dictation_engine = Arc::new( + FixtureDictationEngine::successful("raw", "1. A\n2. 2. B\n2. C").with_polish_deltas( + vec![crate::types::PolishDelta { + text: "1. A\n2. 2. B".into(), + offset: 0, + is_final: false, + }], + ), + ); + let backend = OpenLessBackend::new( + BackendConfig { + data_dir: data_dir.path().to_path_buf(), + ..BackendConfig::default() + }, + deps, + ) + .unwrap(); + backend.start().await.unwrap(); + let session_id = backend.start_dictation().await.unwrap(); + let result = backend.stop_dictation().await.unwrap(); + assert_eq!(result.polished_text, "1. A\n2. B\n3. C"); + assert_eq!( + backend.list_history().unwrap()[0].final_text, + result.polished_text + ); + assert_eq!( + inserter.actions(), + vec![ + FixtureInsertionAction::Prepare(session_id), + FixtureInsertionAction::Insert { + session_id, + text: result.polished_text + } + ] + ); + } } diff --git a/openless-all/app/crates/openless-core/src/dictation_context.rs b/openless-all/app/crates/openless-core/src/dictation_context.rs index 37fb85c46..29fe0ffd2 100644 --- a/openless-all/app/crates/openless-core/src/dictation_context.rs +++ b/openless-all/app/crates/openless-core/src/dictation_context.rs @@ -37,8 +37,25 @@ pub enum DictationOutputTarget { Qa, } +/// Delivery is captured per invocation; non-hotkey callers retain their settings. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum DictationDelivery { + #[default] + Preferences, + Complete, + Hold { + press_id: u64, + pressed_at: std::time::Instant, + }, + Auto { + press_id: u64, + pressed_at: std::time::Instant, + }, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct DictationStartOptions { + pub delivery: DictationDelivery, pub translation_requested: bool, pub audio_source: DictationAudioSource, pub insert_text: bool, @@ -51,6 +68,7 @@ pub struct DictationStartOptions { impl Default for DictationStartOptions { fn default() -> Self { Self { + delivery: DictationDelivery::Preferences, translation_requested: false, audio_source: DictationAudioSource::Microphone, insert_text: true, @@ -147,6 +165,7 @@ pub struct PolishHistoryTurn { #[derive(Debug, Clone, PartialEq, Eq)] pub struct DictationInsertionContext { + pub delivery: DictationDelivery, pub enabled: bool, pub observe_edits: bool, pub streaming: bool, @@ -332,6 +351,7 @@ impl DictationContext { prior_turns, }, insertion: DictationInsertionContext { + delivery: options.delivery, enabled: options.insert_text && !matches!( options.output_target, diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index 7881a0eb0..04c0a3e63 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -243,7 +243,7 @@ pub use credentials::{ }; pub use dictation_context::{ build_asr_prompt, eligible_polish_context_turns, DictationAudioSource, DictationContext, - DictationInsertionContext, DictationOutputTarget, DictationPolishContext, + DictationDelivery, DictationInsertionContext, DictationOutputTarget, DictationPolishContext, DictationStartOptions, DictationStopOptions, PolishHistoryTurn, ProviderInvocation, RecordingPlan, ASR_PROMPT_CHAR_BUDGET, }; diff --git a/openless-all/app/crates/openless-core/src/output_cleaning.rs b/openless-all/app/crates/openless-core/src/output_cleaning.rs index 3df090714..ce7d37ab7 100644 --- a/openless-all/app/crates/openless-core/src/output_cleaning.rs +++ b/openless-all/app/crates/openless-core/src/output_cleaning.rs @@ -280,3 +280,126 @@ mod tests { ); } } + +/// Normalize only ordinary Markdown ordered-list runs in the built-in structured +/// style. Code, quotes, dates, versions and serialized Markdown stay literal. +pub(crate) fn normalize_structured_numbering(text: &str) -> String { + let mut lines: Vec = text.split('\n').map(str::to_owned).collect(); + let mut groups: Vec<(usize, Vec)> = Vec::new(); + let mut fence: Option = None; + let mut completed = Vec::new(); + for (index, line) in lines.iter().enumerate() { + let trimmed = line.trim_start(); + let marker = if trimmed.starts_with("```") { + Some('`') + } else if trimmed.starts_with("~~~") { + Some('~') + } else { + None + }; + if let Some(marker) = marker { + if fence == Some(marker) { + fence = None; + } else if fence.is_none() { + fence = Some(marker); + } + completed.extend(groups.drain(..)); + continue; + } + if fence.is_some() { + continue; + } + if let Some((indent, _, _, _)) = ordered_marker(line) { + // An indented code block outside a list is literal Markdown. + if indent >= 4 && groups.is_empty() { + continue; + } + while groups.last().is_some_and(|(level, _)| *level > indent) { + completed.push(groups.pop().unwrap()); + } + if let Some((level, indices)) = groups.last_mut().filter(|(level, _)| *level == indent) + { + let _ = level; + indices.push(index); + } else { + groups.push((indent, vec![index])); + } + } else if !trimmed.is_empty() { + let indent = line.len() - trimmed.len(); + while groups.last().is_some_and(|(level, _)| *level >= indent) { + completed.push(groups.pop().unwrap()); + } + } + } + completed.extend(groups); + for (_, indices) in completed { + if indices.len() < 2 { + continue; + } + for (number, index) in indices.into_iter().enumerate() { + let line = &lines[index]; + let (indent, old, delimiter, body) = ordered_marker(line).unwrap(); + let mut body = &line[body..]; + // Strip only a duplicated identical marker, not arbitrary nested text. + if let Some((0, repeated, repeat_delimiter, repeated_body)) = ordered_marker(body) { + if repeated == old && repeat_delimiter == delimiter { + body = &body[repeated_body..]; + } + } + lines[index] = format!("{}{}{delimiter} {body}", &line[..indent], number + 1); + } + } + lines.join("\n") +} + +fn ordered_marker(line: &str) -> Option<(usize, usize, char, usize)> { + let indent = line.len() - line.trim_start_matches([' ', '\t']).len(); + let bytes = line.as_bytes(); + let mut end = indent; + while bytes.get(end).is_some_and(u8::is_ascii_digit) { + end += 1; + } + if end == indent || end - indent > 2 { + return None; + } + let number = line[indent..end].parse().ok()?; + let delimiter = char::from(*bytes.get(end)?); + if !matches!(delimiter, '.' | ')') || !bytes.get(end + 1)?.is_ascii_whitespace() { + return None; + } + let body = + end + 1 + line[end + 1..].len() - line[end + 1..].trim_start_matches([' ', '\t']).len(); + Some((indent, number, delimiter, body)) +} + +#[cfg(test)] +mod numbering_tests { + use super::*; + #[test] + fn duplicate_numbers_and_nested_runs_are_normalized() { + assert_eq!( + normalize_structured_numbering( + "1. A\n2. 2. B\n 1. child\n 1. child2\n2. C\n3. D\n3. E" + ), + "1. A\n2. B\n 1. child\n 2. child2\n3. C\n4. D\n5. E" + ); + } + #[test] + fn separate_lists_and_parent_items_reset_children() { + assert_eq!( + normalize_structured_numbering( + "1. A\n 1. a\n 1. b\n2. B\n 1. c\n 1. d\n\nHeading\n1. E\n1. F" + ), + "1. A\n 1. a\n 2. b\n2. B\n 1. c\n 2. d\n\nHeading\n1. E\n2. F" + ); + } + #[test] + fn code_versions_dates_quotes_entities_and_alphabetic_children_remain_literal() { + assert_eq!( + normalize_structured_numbering(" 1. command\n 1. command"), + " 1. command\n 1. command" + ); + let text = "2026. year\n2.1 version\n> 1. quote\n> 1. quote\n```\n1. code\n1. code\n```\n1. parent\n (a) child\n2. parent\n2\\. escaped \nhttps://example.com/1.2"; + assert_eq!(normalize_structured_numbering(text), text); + } +} diff --git a/openless-all/app/crates/openless-core/src/ports.rs b/openless-all/app/crates/openless-core/src/ports.rs index fcb669339..6c42bb017 100644 --- a/openless-all/app/crates/openless-core/src/ports.rs +++ b/openless-all/app/crates/openless-core/src/ports.rs @@ -613,6 +613,21 @@ pub trait TextInserter: Send + Sync { } pub trait TextInsertionSession: Send + Sync { + /// Replaces only the range owned by this session. Adapters must verify focus, + /// caret and unchanged text, then confirm delivery. Unsupported by default. + fn supports_owned_replacement(&self) -> bool { + false + } + + fn replace_owned(&self, _text: String) -> BoxFuture<'static, Result<(), BackendError>> { + Box::pin(async { + Err(BackendError::new( + BackendErrorCode::Unsupported, + "protected text replacement is unavailable", + )) + }) + } + /// Native preparation can decline streaming while retaining final paste /// support (for example when macOS cannot switch the keyboard input source). /// Core owns the fallback decision; adapters must never acknowledge chunks diff --git a/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs b/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs index 6803ac60a..7006a85da 100644 --- a/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs +++ b/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs @@ -2151,6 +2151,30 @@ pub(super) async fn arm_translation_if_effective(inner: &Arc) -> bool { } pub(super) fn hotkey_bridge_loop(inner: Arc, rx: mpsc::Receiver) { + let (ordered_tx, ordered_rx) = mpsc::channel(); + let worker = Arc::clone(&inner); + let Ok(join) = std::thread::Builder::new() + .name("dictation-ordered-edges".into()) + .spawn(move || hotkey_ordered_bridge_loop(worker, ordered_rx)) + else { + log::error!("[coord] cannot start ordered dictation edge worker"); + return; + }; + while let Ok(evt) = rx.recv() { + if let HotkeyEvent::Released { press_id, at } = &evt { + inner + .backend + .observe_dictation_hotkey_release(*press_id, *at); + } + if ordered_tx.send(evt).is_err() { + break; + } + } + drop(ordered_tx); + let _ = join.join(); +} + +fn hotkey_ordered_bridge_loop(inner: Arc, rx: mpsc::Receiver) { while let Ok(evt) = rx.recv() { if inner.shortcut_recording_active.load(Ordering::SeqCst) { // Recording mode: forward only the "record Fn" event to the diff --git a/openless-all/app/src-tauri/src/core_adapters.rs b/openless-all/app/src-tauri/src/core_adapters.rs index dd900c4ac..4afad62d5 100644 --- a/openless-all/app/src-tauri/src/core_adapters.rs +++ b/openless-all/app/src-tauri/src/core_adapters.rs @@ -2978,6 +2978,30 @@ impl CoreTextInserter for TauriTextInserter { // whichever application happens to be focused at the end. let insertion_target = insertion_target .unwrap_or_else(crate::selection::capture_selection_insertion_target); + #[cfg(target_os = "macos")] + let owned_range = if matches!( + context.insertion.delivery, + openless_core::dictation_context::DictationDelivery::Hold { .. } + | openless_core::dictation_context::DictationDelivery::Auto { .. } + ) { + let target = insertion_target.clone(); + tauri::async_runtime::spawn_blocking(move || { + if !crate::selection::live_insertion_app_matches(&target) { + return None; + } + crate::host_document::OwnedTextRange::capture() + .map(|range| Arc::new(Mutex::new(range))) + }) + .await + .map_err(|error| { + BackendError::new( + BackendErrorCode::Internal, + format!("join live text capability probe: {error}"), + ) + })? + } else { + None + }; #[cfg(target_os = "windows")] let prepared = if context.insertion.windows_insertion_mode == openless_core::shared_types::WindowsInsertionMode::Tsf @@ -3005,7 +3029,9 @@ impl CoreTextInserter for TauriTextInserter { ) })?; let (previous, streaming_ready) = prepare_streaming_input_source( - context.uses_llm_polisher() + context.insertion.delivery == openless_core::dictation_context::DictationDelivery::Preferences + && context.polish.style_pack_id != "builtin.structured" + && context.uses_llm_polisher() && openless_core::streaming_insert::streaming_insert_eligible( context.insertion.streaming, context.polish.translation_active, @@ -3069,6 +3095,8 @@ impl CoreTextInserter for TauriTextInserter { #[cfg(target_os = "macos")] streaming_worker, #[cfg(target_os = "macos")] + owned_range, + #[cfg(target_os = "macos")] cancel_requested, #[cfg(target_os = "macos")] terminal: Arc::new(MacInsertionTerminal::default()), @@ -3079,6 +3107,8 @@ impl CoreTextInserter for TauriTextInserter { #[derive(Clone)] struct TauriTextInsertionSession { + #[cfg(target_os = "macos")] + owned_range: Option>>, session_id: SessionId, backend: BackendSlot, context: Arc, @@ -3370,6 +3400,61 @@ impl TauriTextInsertionSession { } impl TextInsertionSession for TauriTextInsertionSession { + fn supports_owned_replacement(&self) -> bool { + #[cfg(target_os = "macos")] + { + self.owned_range.is_some() + } + #[cfg(not(target_os = "macos"))] + { + false + } + } + + fn replace_owned(&self, text: String) -> BoxFuture<'static, Result<(), BackendError>> { + let session = self.clone(); + Box::pin(async move { + #[cfg(target_os = "macos")] + { + let range = session.owned_range.clone().ok_or_else(|| { + BackendError::new( + BackendErrorCode::Unsupported, + "protected text range unavailable", + ) + })?; + // Join the blocking effect even on cancellation; abandoning a timed + // out write would allow it to overwrite the next voice session. + tauri::async_runtime::spawn_blocking(move || { + if session.finished.load(Ordering::Acquire) { + return Err(BackendError::new( + BackendErrorCode::Cancelled, + "text insertion session closed", + )); + } + range + .lock() + .replace(&text, &session.cancel_requested) + .map_err(|error| BackendError::new(BackendErrorCode::Platform, error)) + }) + .await + .map_err(|error| { + BackendError::new( + BackendErrorCode::Internal, + format!("join protected replacement: {error}"), + ) + })? + } + #[cfg(not(target_os = "macos"))] + { + let _ = (session, text); + Err(BackendError::new( + BackendErrorCode::Unsupported, + "protected replacement unavailable", + )) + } + }) + } + fn supports_streaming(&self) -> bool { #[cfg(target_os = "macos")] { diff --git a/openless-all/app/src-tauri/src/host_document/macos.rs b/openless-all/app/src-tauri/src/host_document/macos.rs index 3d173dbd2..283efaa1c 100644 --- a/openless-all/app/src-tauri/src/host_document/macos.rs +++ b/openless-all/app/src-tauri/src/host_document/macos.rs @@ -162,6 +162,16 @@ extern "C" { parameter: CFTypeRef, value: *mut CFTypeRef, ) -> AxError; + fn AXUIElementIsAttributeSettable( + element: AxUiElementRef, + attribute: CFStringRef, + settable: *mut u8, + ) -> AxError; + fn AXUIElementSetAttributeValue( + element: AxUiElementRef, + attribute: CFStringRef, + value: CFTypeRef, + ) -> AxError; fn AXValueGetValue(value: AxValueRef, value_type: i32, out: *mut c_void) -> u8; fn AXValueCreate(value_type: i32, value_ptr: *const c_void) -> AxValueRef; } @@ -1456,3 +1466,250 @@ mod tests { assert_eq!(caret_offset_from_location(isize::MIN), None); } } + +/// A bounded, local baseline for one verified text field. Only selected-text +/// writes are allowed; never set AXValue for the entire document. +pub(crate) struct OwnedTextRange { + element: SendableElement, + baseline: String, + start: usize, + owned: String, + failed: bool, +} + +impl OwnedTextRange { + /// Must run on a blocking thread, including the AX capability probe. + pub(crate) fn capture() -> Option { + unsafe { + let GatedElement::Ready(element) = focused_element_passing_the_gate(live_gate()) else { + return None; + }; + let owned = SendableElement(element as usize); + if !matches!( + copy_string_attr(element, b"AXRole\0").as_deref(), + Some("AXTextField" | "AXTextArea") + ) { + return None; + } + let range = copy_selected_range(element)?; + if range.location < 0 || range.length != 0 { + return None; + } + if !matches!( + classify_document_length( + copy_index_attr(element, b"AXNumberOfCharacters\0"), + FULL_TEXT_MAX_UTF16 + ), + DocumentLength::WithinLimit(_) + ) { + return None; + } + let baseline = copy_string_attr(element, b"AXValue\0")?; + if baseline.encode_utf16().count() > FULL_TEXT_MAX_UTF16 { + return None; + } + let start = range.location as usize; + if start > baseline.encode_utf16().count() { + return None; + } + for attr in [ + b"AXSelectedTextRange\0".as_slice(), + b"AXSelectedText\0".as_slice(), + ] { + let name = cfstring_from_static(attr)?; + let mut settable = 0; + let error = AXUIElementIsAttributeSettable(element, name, &mut settable); + CFRelease(name); + if error != AX_ERROR_SUCCESS || settable == 0 { + return None; + } + } + Some(Self { + element: owned, + baseline, + start, + owned: String::new(), + failed: false, + }) + } + } + + pub(crate) fn replace( + &mut self, + text: &str, + cancelled: &AtomicBool, + ) -> Result<(), &'static str> { + if self.failed || cancelled.load(Ordering::Acquire) { + return Err("live insertion is closed"); + } + let result = unsafe { self.replace_checked(text, cancelled) }; + if result.is_err() { + self.failed = true; + } + result + } + + unsafe fn replace_checked( + &mut self, + text: &str, + cancelled: &AtomicBool, + ) -> Result<(), &'static str> { + let GatedElement::Ready(focused) = focused_element_passing_the_gate(live_gate()) else { + return Err("target focus is unavailable"); + }; + let focused = SendableElement(focused as usize); + let element = self.element.as_ref(); + if CFEqual(focused.as_ref() as CFTypeRef, element as CFTypeRef) == 0 { + return Err("target control changed"); + } + let range = copy_selected_range(element).ok_or("target caret unavailable")?; + let expected_caret = self.start + self.owned.encode_utf16().count(); + if range.location != expected_caret as isize || range.length != 0 { + return Err("user moved the caret"); + } + if !live_baseline_matches(element, &self.baseline) { + return Err("user edited the document"); + } + if text == self.owned { + return Ok(()); + } + let next = replace_utf16_span( + &self.baseline, + self.start, + self.owned.encode_utf16().count(), + text, + ) + .ok_or("invalid UTF-16 range")?; + if next.encode_utf16().count() > FULL_TEXT_MAX_UTF16 { + return Err("live text exceeds the document limit"); + } + let selected = CFRange { + location: self.start as isize, + length: self.owned.encode_utf16().count() as isize, + }; + let value = AXValueCreate( + K_AX_VALUE_CF_RANGE_TYPE, + &selected as *const _ as *const c_void, + ); + if value.is_null() { + return Err("could not create selected range"); + } + let selected_result = set_live_attr(element, b"AXSelectedTextRange\0", value); + CFRelease(value); + selected_result?; + // Recheck after selecting: never overwrite a concurrent document edit. + let actual = copy_selected_range(element).ok_or("selected range unavailable")?; + if actual.location != selected.location + || actual.length != selected.length + || !live_baseline_matches(element, &self.baseline) + || cancelled.load(Ordering::Acquire) + { + return Err("document changed while selecting the owned range"); + } + let ctext = std::ffi::CString::new(text).map_err(|_| "text contains a NUL")?; + let value = + CFStringCreateWithCString(std::ptr::null(), ctext.as_ptr(), K_CF_STRING_ENCODING_UTF8); + if value.is_null() { + return Err("could not create replacement text"); + } + let result = set_live_attr(element, b"AXSelectedText\0", value); + CFRelease(value); + result?; + if copy_string_attr(element, b"AXValue\0").as_deref() != Some(&next) { + return Err("replacement delivery is unconfirmed"); + } + // Standard text fields collapse selection after AXSelectedText writes. + // If they do not, decline future updates instead of moving the user's caret. + let range = copy_selected_range(element).ok_or("replacement caret unavailable")?; + if range.location != (self.start + text.encode_utf16().count()) as isize + || range.length != 0 + { + return Err("replacement caret is unconfirmed"); + } + self.baseline = next; + self.owned = text.to_owned(); + Ok(()) + } +} + +unsafe fn live_baseline_matches(element: AxUiElementRef, baseline: &str) -> bool { + matches!( + classify_document_length( + copy_index_attr(element, b"AXNumberOfCharacters\0"), + FULL_TEXT_MAX_UTF16 + ), + DocumentLength::WithinLimit(_) + ) && copy_string_attr(element, b"AXValue\0").as_deref() == Some(baseline) +} + +fn live_gate() -> GateInputs { + GateInputs { + secure_input: crate::unicode_keystroke::is_secure_input_enabled(), + bundle_id: crate::selection::current_front_app_parts().1, + ..GateInputs::default() + } +} + +unsafe fn set_live_attr( + element: AxUiElementRef, + attr: &'static [u8], + value: CFTypeRef, +) -> Result<(), &'static str> { + let name = cfstring_from_static(attr).ok_or("AX attribute unavailable")?; + let error = AXUIElementSetAttributeValue(element, name, value); + CFRelease(name); + if error == AX_ERROR_SUCCESS { + Ok(()) + } else { + Err("AX selected-text update failed") + } +} + +fn replace_utf16_span( + text: &str, + start: usize, + length: usize, + replacement: &str, +) -> Option { + let end = start.checked_add(length)?; + let mut units = 0; + let mut start_byte = None; + let mut end_byte = None; + for (byte, ch) in text + .char_indices() + .chain(std::iter::once((text.len(), '\0'))) + { + if units == start { + start_byte = Some(byte); + } + if units == end { + end_byte = Some(byte); + break; + } + units += ch.len_utf16(); + } + Some(format!( + "{}{}{}", + &text[..start_byte?], + replacement, + &text[end_byte?..] + )) +} + +#[cfg(test)] +mod live_range_tests { + use super::*; + #[test] + fn replacement_preserves_surroundings_and_rejects_split_surrogates() { + assert_eq!( + replace_utf16_span("前🙂原文\n后", 3, 2, "润色😀"), + Some("前🙂润色😀\n后".to_owned()) + ); + assert_eq!(replace_utf16_span("🙂后", 1, 0, "x"), None); + assert_eq!(replace_utf16_span("前🙂", 1, 1, "x"), None); + assert_eq!( + replace_utf16_span("前🙂", 3, 0, "尾"), + Some("前🙂尾".to_owned()) + ); + } +} diff --git a/openless-all/app/src-tauri/src/host_document/mod.rs b/openless-all/app/src-tauri/src/host_document/mod.rs index b3b4c32a0..54856172c 100644 --- a/openless-all/app/src-tauri/src/host_document/mod.rs +++ b/openless-all/app/src-tauri/src/host_document/mod.rs @@ -39,7 +39,7 @@ mod windows; pub(crate) use windows::insert_with_delivery_check; #[cfg(target_os = "macos")] -pub(crate) use macos::{KeyboardDelivery, KeyboardDeliveryOutcome}; +pub(crate) use macos::{KeyboardDelivery, KeyboardDeliveryOutcome, OwnedTextRange}; // `minimal_edit` is currently used only by the macOS observation callback; no consumer on non-macOS builds. #[allow(unused_imports)] diff --git a/openless-all/app/src-tauri/src/selection.rs b/openless-all/app/src-tauri/src/selection.rs index c4223ddf3..fcad57888 100644 --- a/openless-all/app/src-tauri/src/selection.rs +++ b/openless-all/app/src-tauri/src/selection.rs @@ -1529,3 +1529,15 @@ mod tests { )); } } + +/// Check identity without reactivating another app for live dictation. +#[cfg(target_os = "macos")] +pub(crate) fn live_insertion_app_matches(target: &SelectionInsertionTarget) -> bool { + let current = capture_selection_insertion_target(); + match (&target.macos, ¤t.macos) { + (Some(expected), Some(current)) => { + expected.front_app_pid.is_some() && expected.front_app_pid == current.front_app_pid + } + _ => false, + } +}