diff --git a/.gitignore b/.gitignore index 8761e1dcf..97861b955 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ node_modules/ !benchmark/lib/** **/dist/ **/dist-firefox/ +.artifacts/ diff --git a/README.i18n.yaml b/README.i18n.yaml index 8a63c95eb..02268f125 100644 --- a/README.i18n.yaml +++ b/README.i18n.yaml @@ -1,5 +1,5 @@ # Bilingual consistency record for the repository README files. # README.md is the default English page; README.zh.md is its Chinese counterpart. # Values are git blob hashes from the last confirmed-consistent review. -README.md: 1163cc9c9a7332a95f8b2448461243f0813ffdf2 -README.zh.md: 6ce7a8314f10dcd45316906f79991e0604b0814c +README.md: bd8959e81cbc1bb1016d1f686ade8b59bb6c32f4 +README.zh.md: ba858f21da58c435cdf21098b2a683e6b54e54b4 diff --git a/README.md b/README.md index 3f743c5b9..bd8959e81 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ Connect [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) to t `dsh` is DeepSeek AI's open-source, plugin-based agent harness. This repository provides a companion browser bridge plugin and Chrome/Firefox MV3 extension as one standalone pnpm workspace. -Browser operation remains text-only: pages become structured text with a numbered inventory of interactive elements, and the model addresses those elements by number. dsh 0.2.0 multimodal chat is separate from that page channel—the side panel accepts PNG, JPEG, WebP, and GIF attachments when the host advertises image support, while browser tools still never capture screenshots. +Pages become structured text with a numbered inventory of interactive elements. Tools address elements by number or selector; `browser_screenshot` captures the controlled tab's viewport after approval. The side panel also accepts PNG, JPEG, WebP, and GIF attachments when the host advertises image support. > [!IMPORTANT] -> The workspace pins dsh 0.2.0-rc.2, the minimum supported runtime. Older DSH releases are not supported. +> The workspace targets the latest public dsh release, currently pinned to 0.2.1-rc.5. Older DSH releases are not supported. ## Quick install @@ -49,20 +49,24 @@ The paired Playwright / extension duration ratio was **1.24** (95% CI **1.16–1 | Capability | Tool | Notes | |---|---|---| -| Read page | `browser_snapshot` | Structured text snapshot: title, URL, main text, numbered controls, and masked form fields; `delta: true` returns only changes | -| Click element | `browser_click` | Click links, buttons, checkboxes, and other controls by inventory number | -| Fill forms | `browser_type` | React/Vue-compatible input; `replace` clears the field first | -| Press keys | `browser_press` | Keyboard events such as Enter, Tab, Escape, and arrow keys | -| Scroll | `browser_scroll` | Viewport scrolling: up, down, top, and bottom | +| Read page | `browser_snapshot` | Structured text snapshot: title, URL, main text, numbered controls, and masked form fields; `delta: true` returns only changes, `frame` reads one iframe, and `query` locates any element by text/selector/role and returns its selector | +| Click | `browser_click` | Click or double-click by index or selector; supports mouse buttons and modifiers | +| Text | `browser_type` | React/Vue-compatible input; `replace` clears the field first, and `mode` selects caret insertion or per-character key events | +| Press keys | `browser_press` | Single keys or chords (`modifiers`); `action: down/up` holds a key across calls and `repeat` repeats it | +| Scroll | `browser_scroll` | Scroll the page or selected iframe up, down, to the top, or to the bottom | +| Clipboard | `browser_clipboard` | `write` puts text/HTML on the system clipboard; `paste` delivers it as a paste event or a real platform shortcut | +| Files | `browser_upload` | Attach local files to an `input[type=file]` (Chrome) | | Navigate | `browser_navigate` / `browser_open_tab` / `browser_back` / `browser_forward` / `browser_reload` | Navigation inside the controlled tab, or open a URL in a new tab and follow it (`active:false` keeps the current tab in front) | | List tabs | `browser_list_tabs` | List accessible tabs with stable IDs, titles, URLs, window/index metadata, and active/controlled state | -| Follow tab | `browser_follow_tab` | Bind later browser tools to a tab returned by `browser_list_tabs` without activating it | +| Follow tab | `browser_follow_tab` | Bind later browser tools to a tab returned by `browser_list_tabs`; it stays in the background unless `activate: true` | | Close tab | `browser_close_tab` | Close a tab returned by `browser_list_tabs` | | Read region | `browser_get_text` | Lazy-loaded or partial page text | | Wait for stability | `browser_wait` | Page-load and render-settle detection | | Send images | `session.prompt` / `session.attachment` | Host-capability-gated image drafts, image-only prompts, and durable history previews | | Quote a selection | side panel composer | Text you highlight in the page appears in the composer and is sent with your next message as fenced, attributed page content | +Clicks, text entry, and scrolling run in the selected document, including cross-origin iframes. Chrome uses its debugger for native keyboard events, caret insertion, system paste, and file uploads. Screenshots require an active tab in a focused window; `browser_follow_tab { activate: true }` brings the controlled page to the front. + ## Repository layout ``` @@ -83,7 +87,7 @@ scripts/install.ps1 ## Detailed installation and usage -Requirements: Node.js `^22.19` or `>=24`, Corepack/pnpm, and Chrome 116+ or Firefox 140+. Windows additionally needs Windows PowerShell 5.1, which ships with Windows, or PowerShell 7+. +Requirements: Node.js `>=24`, Corepack/pnpm, and Chrome 125+ or Firefox 140+. Windows additionally needs Windows PowerShell 5.1, which ships with Windows, or PowerShell 7+. ### Install or update @@ -137,7 +141,7 @@ cd ~/.dsh/dsh-browser && pnpm start From a source checkout, run `pnpm start` in the repository root. The exact supported public runtime is: ```sh -npx @deepseek-ai/dsh@0.2.0-rc.2 web +npx @deepseek-ai/dsh@0.2.1-rc.5 web ``` Local Chrome use requires no configuration; Firefox requires the local bridge token described above. Open a page, click the DeepSeek whale icon, and wait for **Connected**. Existing HTTP(S) tabs are instrumented on the first action. On browser-protected pages and extension stores, the model can read tab metadata and use browser-level HTTP(S) navigation, back, forward, and reload, but it cannot inspect or operate the protected page DOM. @@ -148,7 +152,8 @@ Local Chrome use requires no configuration; Firefox requires the local bridge to - Make sure dsh web is running locally (default `http://127.0.0.1:3080`). - Verify the bridge is loaded: open `http://127.0.0.1:3080/ext/bridge-config`. It should return JSON such as `{"wsUrl":"ws://127.0.0.1:3080/ext/bridge"}`. If it returns a web page instead of JSON, the running dsh predates the bridge registration — restart dsh and refresh the page; the extension reconnects on its own. -- The extension probes ports 3080, 3081, 3090, and 14389 automatically. If dsh runs on another port — or you use a remote `--host 0.0.0.0` deployment — set the address (and bridge token) in the panel settings. Firefox always requires the token. +- Discovery runs once when the panel opens. A filled-in address is tried first; if a **loopback** address does not answer, the panel makes one automatic pass (last working URL → `dsh web` defaults → the plugin's fixed beacon ports 43189–43198). When that pass finds nothing the status reads **No service found** and nothing retries until you reopen the panel or press **Detect**. +- DSH Desktop assigns a random local Web port on every launch, which is why the bridge plugin also listens on a fixed loopback beacon port. If the panel cannot find it, open **Settings → Detect** to list every local instance and click the one to use; the address can still be typed by hand. Remote `--host` deployments need the address and token filled in manually; Firefox always requires the token. ## Development @@ -178,6 +183,8 @@ Notes: `check:runtime` checks the resolved DSH dependencies and lockfile; `test:smoke` starts the real web host in a temporary DSH home and verifies the bridge and session reads after a restart, without model credentials. CI runs these checks after a clean installation. +After building, run `pnpm test:browser` with a Playwright Chromium installation, or set `CHROMIUM_PATH` to a Chromium executable that supports loading unpacked extensions. This starts real DSH and Chromium instances to verify sharing controls, developer-tools consent, input, iframe uploads and targeting, and CSS edits. Site isolation is enabled and verified for cross-site frames; element clicks, input, and scrolling cover same-origin, cross-origin, and nested frames. It needs no model credentials and saves results under `.artifacts/browser-smoke/`. + If you encounter `cache.hydratePrepared is not a function`, update the repository, rerun `pnpm install --frozen-lockfile` and `pnpm run build`, then restart `pnpm start`. Session data and the global package cache can be kept. ## Security @@ -185,8 +192,8 @@ If you encounter `cache.hydratePrepared is not a function`, update the repositor - The bridge path sits outside the `/api` trust boundary and performs its own bearer-token authentication. - Local Chrome extension origins retain zero-configuration loopback access; Firefox origins are per-install UUIDs and must present the bearer token. - Privileged gateway methods such as `settings.*`, `credentials.*`, and `host.open*` reject non-loopback sources. -- The browser-page pipeline is text-only and never captures screenshots; explicitly attached chat images use dsh's durable attachment service. Password and payment-card values never leave the page. -- When work begins, the assistant binds to the active tab (at prompt submission, or at the first direct browser-tool call). If you switch tabs manually, later browser actions pause and the side panel asks whether the assistant should continue on the original tab or follow the new one. Choosing the original tab permits background operation; the extension never silently retargets or changes your visible tab. Closing the controlled tab also pauses tools until you explicitly select the current page. +- Text snapshots mask password and payment-card values. Screenshots capture visible pixels without field masking and require approval unless unrestricted control is enabled; page sharing set to `off` blocks screenshots under restricted control. Screenshots and attached chat images use dsh's durable attachment service. Developer tools have a separate high-privilege consent gate. +- When work begins, the assistant binds to the active tab (at prompt submission, or at the first direct browser-tool call). If you switch tabs manually, later browser actions pause and the side panel asks whether the assistant should stay on the original page or follow the current one, and offers "jump back to the assistant page" to move your own tab back to the assistant's page. Choosing the original page permits background operation; apart from that button the extension never silently retargets or changes your visible tab. Closing the controlled tab also pauses tools until you explicitly select the current page. - Text you highlight is captured only while a side panel is open and page sharing is not `off`, and never from password or payment-card fields. It stays inside the extension until you send the message, is dropped when you dismiss it or its page navigates or closes, and reaches the model inside the same untrusted-content boundary as page snapshots — including its source title and URL, which the page also controls. - Page-authored text is wrapped as untrusted input. The default `auto` mode reads only the controlled tab without an extra prompt; privacy-sensitive users can select `ask` for per-read confirmation or `off` to block reads entirely. In `ask` mode, the read dialog can allow one read or persistently switch back to `auto`; this can be reversed in Settings. Read page text is sent to the selected model. - Click, type, keypress, navigation, history, and reload calls fail closed until the user approves them. An origin may be trusted for the current side-panel session (cleared when the last panel closes or the service worker restarts), while permanent trust is managed explicitly in Settings. Explicit cross-origin `browser_navigate` calls and unknown history destinations always prompt again. diff --git a/README.zh.md b/README.zh.md index fc8fe4d45..ba858f21d 100644 --- a/README.zh.md +++ b/README.zh.md @@ -8,10 +8,10 @@ `dsh` 是由 DeepSeek AI 开发的开源、插件化 agent harness（智能体框架）。本仓库将配套的浏览器桥插件与 Chrome/Firefox MV3 扩展组成一个独立的 pnpm workspace。 -浏览器操作仍采用纯文本设计：页面会转换为结构化文本和带编号的交互元素清单，模型通过编号定位元素。dsh 0.2.0 的多模态对话走独立通道——宿主声明图片能力时，侧栏可发送 PNG、JPEG、WebP 和 GIF；浏览器工具本身仍不会截取页面截图。 +页面会转换为结构化文本和带编号的交互元素清单。工具通过编号或选择器定位元素；`browser_screenshot` 经批准后截取受控标签页的可视区域。宿主声明图片能力时，侧栏也可发送 PNG、JPEG、WebP 和 GIF 附件。 > [!IMPORTANT] -> 当前工作区固定使用 dsh 0.2.0-rc.2，也是最低支持版本；不再支持旧版 DSH。 +> 工作区针对最新公开发布的 dsh，目前固定使用 0.2.1-rc.5；不支持旧版 DSH。 ## 快速安装 @@ -49,20 +49,24 @@ Playwright / 扩展的配对耗时比为 **1.24**（95% CI **1.16–1.34**）： | 能力 | 工具 | 说明 | |---|---|---| -| 读取页面 | `browser_snapshot` | 结构化文本快照：标题/URL/正文/编号交互清单/表单字段（敏感值掩码）；`delta: true` 只返回变化 | -| 点击元素 | `browser_click` | 按编号点击链接/按钮/复选框等 | -| 填写表单 | `browser_type` | 输入文本（React/Vue 受控组件兼容），`replace` 清空重填 | -| 按键 | `browser_press` | 键盘事件（Enter/Tab/Escape/方向键…） | -| 滚动 | `browser_scroll` | 视口滚动（up/down/top/bottom） | +| 读取页面 | `browser_snapshot` | 结构化文本快照：标题/URL/正文/编号交互清单/表单字段（敏感值掩码）；`delta: true` 只返回变化，`frame` 单读一个 iframe，`query` 按文本/选择器/角色定位任意元素并给出选择器 | +| 点击 | `browser_click` | 按编号或选择器点击、双击；支持鼠标按键和修饰键 | +| 文本 | `browser_type` | 输入文本（React/Vue 受控组件兼容），`replace` 清空重填；`mode` 可选插入或逐键输入（自动补全类字段） | +| 按键 | `browser_press` | 单键或组合键（`modifiers`），`action: down/up` 跨调用按住，`repeat` 重复 | +| 滚动 | `browser_scroll` | 向上、向下滚动页面或指定 iframe，或滚动到顶部、底部 | +| 剪贴板 | `browser_clipboard` | `write` 写入系统剪贴板（文本/HTML）；`paste` 以粘贴事件或真实快捷键粘贴到页面 | +| 文件 | `browser_upload` | 把本机文件附加到 `input[type=file]`（Chrome） | | 页面导航 | `browser_navigate` / `browser_open_tab` / `browser_back` / `browser_forward` / `browser_reload` | 受控标签页内导航，或新开标签页并跟随（`active:false` 时保持当前页在前台） | | 列出标签页 | `browser_list_tabs` | 列出可访问标签页的稳定 ID、标题、URL、窗口/顺序以及活动/受控状态 | -| 跟随标签页 | `browser_follow_tab` | 将后续浏览器工具绑定到 `browser_list_tabs` 返回的标签页，而不激活该标签页 | +| 跟随标签页 | `browser_follow_tab` | 将后续浏览器工具绑定到 `browser_list_tabs` 返回的标签页；默认不激活，`activate: true` 才切到前台 | | 关闭标签页 | `browser_close_tab` | 关闭 `browser_list_tabs` 返回的标签页 | | 读取区域 | `browser_get_text` | 懒加载内容 / 局部文本 | | 等待稳定 | `browser_wait` | 页面加载与渲染稳定检测 | | 发送图片 | `session.prompt` / `session.attachment` | 按宿主能力启用图片草稿、纯图片消息和持久历史预览 | | 引用选中内容 | 侧栏输入框 | 在页面里划选的文字会出现在输入框，随下一条消息一起发送，并带上来源与不可信内容边界 | +点击、文本输入和滚动操作在指定文档中执行，支持跨域 iframe。Chrome 通过调试器执行原生键盘事件、光标位置插入、系统粘贴和文件上传。截图要求受控标签页处于活动状态且窗口聚焦，可以使用 `browser_follow_tab { activate: true }` 将页面切到前台。 + ## 组成 ``` @@ -83,7 +87,7 @@ scripts/install.ps1 ## 详细安装与使用 -前置要求：Node.js `^22.19` 或 `>=24`、Corepack/pnpm，以及 Chrome 116+ 或 Firefox 140+。Windows 还需要系统自带的 Windows PowerShell 5.1，或 PowerShell 7+。 +前置要求：Node.js `>=24`、Corepack/pnpm，以及 Chrome 125+ 或 Firefox 140+。Windows 还需要系统自带的 Windows PowerShell 5.1，或 PowerShell 7+。 ### 安装或更新 @@ -137,7 +141,7 @@ cd ~/.dsh/dsh-browser && pnpm start 使用源码 checkout 时，请在仓库根目录运行 `pnpm start`。受支持的精确公开版本为： ```sh -npx @deepseek-ai/dsh@0.2.0-rc.2 web +npx @deepseek-ai/dsh@0.2.1-rc.5 web ``` Chrome 本机使用无需配置；Firefox 需要填写上述本地桥 token。打开页面，点击 DeepSeek 鲸鱼图标，等待侧边栏显示**已连接**。已有 HTTP(S) 标签页会在第一次操作时自动加载。在浏览器受保护页面和扩展商店中，模型可以读取标签页元数据，并通过浏览器级能力导航到 HTTP(S)、后退、前进和刷新，但不能读取或操作受保护页面的 DOM。 @@ -148,7 +152,8 @@ Chrome 本机使用无需配置；Firefox 需要填写上述本地桥 token。 - 确认本机 dsh web 正在运行（默认 `http://127.0.0.1:3080`）。 - 确认桥接已加载：浏览器打开 `http://127.0.0.1:3080/ext/bridge-config`，应返回类似 `{"wsUrl":"ws://127.0.0.1:3080/ext/bridge"}` 的 JSON。如果返回的是网页而不是 JSON，说明当前运行的 dsh 早于桥接注册——重启 dsh 并刷新页面即可，扩展会自动重连。 -- 扩展会自动探测 3080/3081/3090/14389 端口。若 dsh 运行在其它端口，或使用 `--host 0.0.0.0` 远程部署，请在面板设置中填写地址与桥接 token。Firefox 始终需要 token。 +- 面板打开时探测一次：已填地址优先尝试；回环地址连不上时再自动探测一次（上次成功的地址 → `dsh web` 默认端口 → 插件的固定信标端口 43189–43198）。都没找到时状态显示「未找到服务」，在重新打开面板或点「自动检测」之前不会重试。 +- DSH Desktop 每次启动都会重新随机分配本地 Web 端口，所以桥接插件同时监听一个固定回环信标端口。面板找不到时，打开**设置 → 自动检测**列出本机所有实例，点击其中一条即可填入地址；地址仍可手动填写。远程 `--host` 部署需要手动填写地址和 token；Firefox 始终需要 token。 ## 开发 @@ -178,6 +183,8 @@ pnpm --filter dsh-browser-extension run test `check:runtime` 检查实际解析的 DSH 依赖和锁文件；`test:smoke` 使用临时 DSH home 启动真实 web 宿主，验证桥接和重启后的会话读取，无需模型密钥。CI 在干净安装后运行这些检查。 +构建后运行 `pnpm test:browser`，需要已安装 Playwright Chromium，或通过 `CHROMIUM_PATH` 指定支持加载未打包扩展的 Chromium 可执行文件。测试会启动真实 DSH 和 Chromium，验证页面共享、开发者工具授权、输入、iframe 上传与定位，以及 CSS 修改。测试开启并确认跨站 iframe 的进程隔离，覆盖同源、跨源和嵌套 iframe 中的元素点击、输入与滚动。测试无需模型密钥，结果保存在 `.artifacts/browser-smoke/`。 + 如果遇到 `cache.hydratePrepared is not a function`，更新仓库后重新运行 `pnpm install --frozen-lockfile` 和 `pnpm run build`，再重启 `pnpm start`。无需删除会话数据或清空全局缓存。 ## 安全 @@ -185,8 +192,8 @@ pnpm --filter dsh-browser-extension run test - 桥路径在 `/api` 信任栅栏之外，自带 bearer token 认证。 - Chrome 扩展的本地 Origin 保留零配置回环访问；Firefox Origin 是每次安装生成的 UUID，必须携带 bearer token。 - 特权网关方法（`settings.*`/`credentials.*`/`host.open*`）对非回环来源一律拒绝。 -- 单活动连接；浏览器页面管线为纯文本且不截图；用户主动添加的对话图片交给 dsh 持久附件服务，密码和卡号值永不回传。 -- 助手开始工作时会绑定当时的活动标签页（提交提示时绑定；直接调用浏览器工具时则在首次调用绑定）。用户手动切页后，后续浏览器操作会暂停，侧栏会询问让助手继续原页面还是跟随新页面；选择原页面后允许在后台继续，但扩展绝不静默改绑或切换用户正在看的页面。受控标签页关闭后也会暂停，直到用户显式选择当前页。 +- 文本快照会遮盖密码和卡号值。截图直接捕获可见像素，不会遮盖字段；除非启用了完全控制，否则截图需要审批，页面共享设为「关闭」时会阻止截图。截图和对话图片使用 dsh 持久附件服务。开发者工具有独立的高权限授权。 +- 助手开始工作时会绑定当时的活动标签页（提交提示时绑定；直接调用浏览器工具时则在首次调用绑定）。用户手动切页后，后续浏览器操作会暂停，侧栏会询问让助手留在原页面还是跟随当前页面，并提供「跳转回到助手页面」把你自己正在看的标签页切回助手页面；选择留在原页面后允许在后台继续，除该按钮外扩展不会切换用户正在看的页面、也不会静默改绑。受控标签页关闭后也会暂停，直到用户显式选择当前页面。 - 只有在侧栏打开、且页面共享不是「关闭」时才会捕获划选内容，密码和卡号字段永不读取。内容在发送之前始终留在扩展内部；移除、页面跳转或标签页关闭都会丢弃它；发送时与页面快照一样包在不可信内容边界内，来源标题和 URL 同样由页面提供，因此也放在边界之内。 - 网页文字会标记为不可信输入。默认「自动共享」只按需读取受控标签页且不额外弹窗；对隐私敏感时可选择「每次询问」，或用「关闭」完全阻断读取。在「每次询问」模式下，读取弹窗可以仅允许一次，也可以持久切回自动读取；之后仍可在设置中关闭。读取的页面文字会发送给当前选择的模型。 - 点击、输入、按键、导航、历史跳转和刷新默认失败关闭，必须由用户批准。可以只在当前侧栏会话中信任单个 origin（最后一个侧栏关闭或 Service Worker 重启即清空）；永久信任需在设置中显式管理。显式跨域 `browser_navigate` 和未知目标的历史跳转始终重新询问。 diff --git a/extensions/dsh-browser/README.i18n.yaml b/extensions/dsh-browser/README.i18n.yaml index c591a0b25..6f5a3ba27 100644 --- a/extensions/dsh-browser/README.i18n.yaml +++ b/extensions/dsh-browser/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write dsh-browser/extensions/dsh-browser/README.md -README.md: 6b6ea0b302a340fca44d6f461ebf84a7b5419c89 -README.zh.md: 36f25412f7e1431aa89a8ac41ef045c9b7676790 +README.md: b7616050fb86b97cdca40a50fdebb2c86aa3e797 +README.zh.md: 473461e54defd70e2c07389576f791e81feb8de8 diff --git a/extensions/dsh-browser/README.md b/extensions/dsh-browser/README.md index 711efd03a..b7616050f 100644 --- a/extensions/dsh-browser/README.md +++ b/extensions/dsh-browser/README.md @@ -4,23 +4,29 @@ English | [中文](README.zh.md) The **browser-operation end** of dsh: the model reads and operates the browser page you have open — extract content, click elements, fill forms, scroll, and navigate, all in the real page with your login state preserved. The side panel is the conversation entry. -**Two explicit channels**: browser pages are still rendered as structured text (a numbered interactive-element inventory), so browser tools never take screenshots. Separately, a dsh 0.1.2 host can advertise multimodal image limits; the side panel then accepts PNG, JPEG, WebP, and GIF attachments and renders their durable history references. +Browser pages are rendered as structured text with a numbered interactive-element inventory. `browser_screenshot` can also capture the controlled viewport after approval. When the DSH host advertises image support, the side panel accepts PNG, JPEG, WebP, and GIF attachments and renders their durable history references. + ## What the model can do | Capability | Action | Notes | |---|---|---| -| Read page | `browser_snapshot` | Title/URL/main text/numbered inventory/form fields (sensitive values masked); `delta: true` returns only changes | -| Click element | `browser_click` | Click by inventory number (links/buttons/checkboxes…), React/Vue compatible | -| Fill forms | `browser_type` | Type text; `replace` clears first | -| Keys | `browser_press` | Enter/Tab/Escape/arrows etc. | -| Scroll | `browser_scroll` | Viewport scrolling (up/down/top/bottom) | +| Read page | `browser_snapshot` | Title/URL/main text/numbered inventory/form fields (sensitive values masked); `delta: true` returns only changes, `frame` reads one iframe, `query` locates any element and returns its selector | +| Capture viewport | `browser_screenshot` | Captures visible pixels without masking; requires screenshot approval unless unrestricted control is enabled | +| Click | `browser_click` | Click or double-click by index or selector; supports mouse buttons and modifiers | +| Text | `browser_type` | Type text; `replace` clears first; `mode` chooses insertion or per-character keys | +| Keys | `browser_press` | Keys and chords, including held modifiers across calls | +| Scroll | `browser_scroll` | Scroll the page or selected iframe up, down, to the top, or to the bottom | +| Clipboard / files | `browser_clipboard` / `browser_upload` | Write or paste text and rich HTML; attach local files to a file input | | Navigate | `browser_navigate` / `browser_open_tab` / `browser_back` / `browser_forward` / `browser_reload` | Navigation inside the controlled tab, or open a URL in a new tab and follow it | | List tabs | `browser_list_tabs` | List accessible tabs with stable IDs, titles, URLs, and active/controlled state | -| Follow tab | `browser_follow_tab` | Bind later browser tools to a listed tab without activating it | +| Follow tab | `browser_follow_tab` | Bind later browser tools to a listed tab; stays in the background unless `activate: true` | | Close tab | `browser_close_tab` | Close a listed tab | | Read region | `browser_get_text` | Lazy-loaded content / partial text | | Wait | `browser_wait` | Page load and render-settle detection | +| Inspect elements / CSS | `browser_devtools_elements` / `browser_devtools_set_element_style` / `browser_devtools_set_element_attribute` | High privilege: computed styles, box model, matched rules, live CSS and attribute edits for debugging | +| Run page JavaScript | `browser_console_eval` / `browser_console_logs` | High privilege: evaluate in the page's own context and read recorded console output | +| Capture network | `browser_devtools_network` / `browser_devtools_list_requests` / `browser_devtools_get_request` / `browser_devtools_request_body` | High privilege: captured request list, per-request detail, and bodies (credential headers redacted by default) | | Chat with images | `session.prompt` / `session.attachment` | Host-gated image selection, image-only sends, and durable history previews | | Quote what you highlight | side panel composer | The text you select in the page becomes a quote in the composer and rides along with your next message | @@ -90,12 +96,12 @@ The recommended zero-configuration command does not require Git or a local clone Or use the exact supported public runtime: ```sh - npx @deepseek-ai/dsh@0.2.0-rc.2 web + npx @deepseek-ai/dsh@0.2.1-rc.5 web ``` Both commands load the same bundle from the local `web` profile. Port 3080 is used by default; append `--port ` when it is occupied. - **DSH Desktop users**: the Desktop app assigns a random local Web port by default (`dsh-desktop.port: 0`), so port-based auto-discovery cannot predict it. Pin the port to `43189` in Desktop settings (see the [deepseek-harness-desktop user guide](https://github.com/anywhere-labs/deepseek-harness-desktop/blob/master/docs/user-guide.en.md)); auto-discovery covers that port. Alternatively, enter `http://127.0.0.1:` manually in the side-panel settings. + **DSH Desktop users**: the Desktop app launches its host with `--port 0`, so the OS assigns a different local Web port on every start and no fixed port list can predict it. The bridge plugin therefore also listens on a fixed loopback discovery beacon (`43189`–`43198`, first free port) that advertises the real bridge URL. Discovery runs once when the panel opens: a filled-in address is tried first, and only a loopback address that does not answer falls back to one automatic pass. When nothing is found the status reads "No service found" and nothing retries; open **Settings → Detect** to list every local instance and pick one, or type `http://127.0.0.1:` by hand. To pin the port instead, override the `webserver` row in `~/.dsh/profiles/desktop/cordis.patch.yml` — patch `config` is replaced wholesale, so repeat `host` as well. Loading or reloading the extension is passive: it does not probe local ports or open a WebSocket until the side panel is opened. A healthy connection established by the user may remain available for background approvals after the panel closes, but it will not reconnect without an open panel if it drops or another browser replaces it. @@ -105,28 +111,75 @@ Pages that were already open before extension installation or reload are instrum For extension-only development, load `extensions/dsh-browser/dist/` from `chrome://extensions`, or run `build:firefox` and load `extensions/dsh-browser/dist-firefox/manifest.json` from `about:debugging#/runtime/this-firefox`. Rebuild and reload after code changes. -## Why browser operation stays text-only - -- **Snapshot as the view**: the model's entire view of the page is structured text (title/URL/main/numbered elements/forms), budgeted at 32k chars by default (plugin-configurable, negotiated to the extension via `hello.ok`). +## Composer controls: permission, model, commands, and skills + +The composer carries the same controls the Web GUI exposes, backed by Host +methods the bridge owns: + +- **Permission preset** (left chip): the deployment's preset table + (`read-only`, `workspace-write`, `danger-full-access`, plus `auto` when the + experimental Auto integration is mounted) with the Session's current value. + Selecting a preset runs the Host's own `/permission ` command, so the + `permission/preset` selection and both knob events land in the Session log + exactly as a Web GUI selection would. `Full access` and `Auto review` ask for + an explicit confirmation first; a derived `custom` state is shown but never + selectable, and the menu footnotes the default for future chats. Before the + first message creates the chat, the choice is remembered and applied at + materialization, before that first prompt is admitted. +- **Model and reasoning effort** (right chip): the Session's model directory + grouped by provider, plus the reasoning efforts the selected model declares + (`Default` included). A selection only shapes the **next** assembled turn, so + the chip stays usable while a turn is running. When the directory cannot be + read, the row says why and offers a retry instead of going silently grey; a + missed `turn/end` is repaired from the Host's `running` flag the next time the + panel becomes visible. +- **`/` menu**: typing `/` in an empty line opens the Host command directory + (`/compact`, `/permission`, `/plan`, `/goal`, `/feedback`, `/export`, …) and + the Session's user-invocable skills, including skills marked + `modelInvocable: false` (badged `User only`) that exist precisely for explicit + invocation. `↑`/`↓` move, `Enter`/`Tab` accept, `Esc` closes; picking a row + inserts its exact text so its input stays editable. On submit the panel + arbitrates exactly like the Host adapter: a command name runs through + `commands.execute` (its output never becomes a model message), a skill name is + sent as a literal prompt so the Host injects its instructions, and an + unrecognized `/…` line is **refused** with the draft preserved — start the + line with a space to send it as ordinary text. Commands take no attachments + here, and neither is a provisional chat's directory browsable: it answers + empty until the first message creates the real Session. + +## Page reading and control + +- **Text snapshots**: title, URL, main text, numbered elements, and forms are budgeted at 32k chars by default (plugin-configurable, negotiated to the extension via `hello.ok`). Screenshots provide an additional image view after approval. - **Page text is untrusted input**: snapshots and targeted text reads are enclosed in a fresh nonce-bound trust marker and explicitly tell the model never to treat page-authored commands as instructions. This is defense in depth; extension-side action approval is the enforcement boundary. - **Stable numbering**: element numbers persist across snapshots (WeakMap + `data-dsh-el`), so the model can say "click 7"; a large page change explicitly reports "numbers reindexed". - **Delta mode**: `browser_snapshot({delta:true})` returns only changed element numbers, saving tokens. -- **Privacy**: password/credit-card values always render as `••••` and never leave the page; accessible names never use a sensitive field's current value. -- **Tab affinity**: prompt submission binds the active tab before the model starts working; a direct browser-tool call also performs the initial bind when needed. A manual tab/window switch pauses later tools and asks whether the assistant should stay on the original tab or follow the newly visible one. Staying permits explicit background operation without changing the user's visible tab; following resets page-reference state. A closed controlled tab fails closed until the user selects the current page, and a switch withdraws any open action approval. +- **Privacy**: text snapshots render password/credit-card values as `••••`; accessible names never use a sensitive field's current value. Screenshots capture visible pixels without masking. Developer-tools access requires separate consent. +- **Tab affinity**: prompt submission binds the active tab before the model starts working; a direct browser-tool call also performs the initial bind when needed. A manual tab/window switch pauses later tools and asks whether the assistant should stay on the original page or follow the current one; that prompt also offers "jump back to the assistant page", which moves the user's own tab to the page the assistant is bound to. Staying permits explicit background operation without changing the user's visible tab; following resets page-reference state. A closed controlled tab fails closed until the user selects the current page, and a switch withdraws any open action approval. - **Proportional approval**: the default `auto` mode lets the model read the controlled tab without an extra prompt; `ask` restores per-read confirmation and `off` blocks reads. In `ask` mode, the read dialog can allow one read or persistently switch back to `auto`, which remains reversible in Settings. State-changing tools still fail closed and show their exact origin plus a redacted action summary. The user may deny, allow once, or trust one origin for the current side-panel session; temporary trust clears when the last panel closes or the service worker restarts. Permanent trust is managed explicitly in Settings. If the panel is closed, an approval remains pending for up to 60 seconds and, when enabled, a system notification opens the panel for review. The panel restores the requesting session before showing a session-scoped approval. Caller cancellation or bridge timeout withdraws any open approval before an action can run. - **Unrestricted control is explicit**: **Allow unrestricted browser control** becomes active only after the setting is saved successfully, then lets page reads, page actions, and tab list/follow/close operations run without confirmation. Calls capture their access mode when received, so enabling unrestricted control never retroactively elevates an existing restricted call. Disabling it takes effect immediately, cancels calls that have not dispatched an action, waits for already-dispatched browser operations to settle, and only then saves the restrictive setting. A rapid re-enable remains restricted until that revocation finishes, and concurrent saves persist in request order. Protected-page DOM content remains inaccessible. - **Conversation continuity**: reopening the panel resumes the most recently active browser conversation by default, falling back to the latest non-empty durable session before creating a new one. This can be disabled in Settings. +## Developer tools (opt-in, high privilege) + +The developer-tools group drives the controlled tab through the Chrome debugger protocol (`chrome.debugger`), so it can inspect and edit page elements and CSS, run JavaScript in the page's own context, and capture network traffic — including response bodies, which page-level interception cannot read reliably. + +- **Off by default**: developer-tools calls ask for approval until the user grants consent through Settings or the current side-panel session. The dialog offers allow once, trust this session, or deny. Session trust expires when the last panel closes or the worker restarts. The blanket **Allow unrestricted browser control** switch and the persistent origin allowlist never cover this group. +- **Persistent consent**: **Allow developer tools** in Settings authorizes the group across panel sessions. Turning it off detaches the debugger and cancels in-flight devtools calls, including calls allowed by session trust. Navigation cancels pending approvals for the replaced document. +- **One debugger per tab**: Chrome refuses a second debugger, so the browser's own DevTools cannot stay open on the same tab while capture is attached. The tool result explains this and tells you to close DevTools on that tab. +- **Data handling**: capture is in memory only, bounded to 200 requests and 500 console entries, and resets on attach and on every top-level navigation. `Authorization`, `Cookie`, `Set-Cookie`, and similar header values are shown as `[redacted]` unless a call explicitly opts in, and bodies of authentication-shaped URLs are redacted by default. +- **Platform**: Chrome only. The Firefox build does not request the `debugger` permission, so the host never registers the group there. + ## Permissions -Chrome uses `sidePanel`; Firefox uses `sidebar_action`. Both request `storage` (settings and recent-session continuity), `notifications` (optional reminders for approvals received while the panel is closed), `tabs` + `activeTab` + `scripting` (observe tab changes and inject/message the explicitly controlled tab, including lazy recovery for pages opened before install), `webNavigation` (enumerate and bind messages to that tab's frame documents), `alarms` (background keepalive), and `http/https` (content-script injection on normal pages). Firefox's AMO manifest declares the browsing activity, website content/activity, and personal communications that the add-on sends to the configured dsh/model service. The extension never changes the visible tab or silently follows a manual switch; background operation happens only after the user chooses to stay on the original tab. +Chrome uses `sidePanel`; Firefox uses `sidebar_action`. Both request `storage` (settings and recent sessions), `notifications` (optional approval reminders), `tabs` + `activeTab` + `scripting` (observe tab changes and instrument the controlled tab), `webNavigation` (bind operations to frame documents), `alarms` (background keepalive), and `` host access. Content scripts run on HTTP(S) pages. Chrome additionally requests `debugger` for trusted input and developer tools, plus `clipboardWrite` and `offscreen` for clipboard operations. Firefox omits these Chrome-specific capabilities. Firefox's AMO manifest declares the browsing activity, website content/activity, and personal communications sent to the configured dsh/model service. The extension can activate a tab through an explicit return-to-page action or `browser_follow_tab { activate: true }`; manual tab switches require a control-handoff decision. ## Known limitations - Only one extension connection at a time. An unopened browser profile never claims it; if another open panel replaces a live connection, the replaced client yields instead of starting a reconnect fight. +- Developer tools require Chrome 125+, a setting/session grant or per-call approval, and an available debugger connection. Chrome shows its debugging banner while the connection remains attached, including during network capture. - Tab affinity is global to that extension connection rather than per chat session. - Accessible cross-origin iframes are snapshotted and operated with stable `(frame, index)` addresses. Restricted or short-lived frames are reported as unavailable without failing the whole page snapshot. - Captcha/image-only controls cannot be handled — the tool result reports "elements with no accessible name" and asks the user to complete that step manually. - No automatic token rotation. -- Synthetic `browser_press` events do not trigger browser-native default actions such as Tab focus movement, arrow-key scrolling, or Enter activation; use manual input when a workflow depends on those defaults. +- Synthetic `browser_press` events do not trigger browser-native default actions such as Tab focus movement, arrow-key scrolling, or Enter activation. Chrome's trusted input channel supports these actions; Firefox uses synthetic input. - `browser_wait` considers page load plus a fixed quiet window, but does not observe continuously changing DOM state; a live-updating SPA may be reported as stable. diff --git a/extensions/dsh-browser/README.zh.md b/extensions/dsh-browser/README.zh.md index 20f886825..473461e54 100644 --- a/extensions/dsh-browser/README.zh.md +++ b/extensions/dsh-browser/README.zh.md @@ -4,23 +4,29 @@ dsh 的**浏览器操作端**：让模型直接读取并操作你在浏览器里打开的页面——抓取内容、点击元素、填写表单、滚动与导航，全部在真实页面执行、登录态保留。侧边栏面板是与模型对话的入口。 -**两条明确分离的通道**：浏览器页面仍以结构化文本呈现（带编号的交互元素清单），浏览器工具不会截图；另一方面，dsh 0.1.2 宿主可声明多模态图片限制，侧栏据此接收 PNG、JPEG、WebP 和 GIF，并渲染会话中的持久图片附件。 +浏览器页面以结构化文本和带编号的交互元素清单呈现。`browser_screenshot` 也可经批准后截取受控页面的可视区域。DSH 宿主声明图片能力时，侧栏接收 PNG、JPEG、WebP 和 GIF，并渲染会话中的持久图片附件。 + ## 模型能做什么 | 能力 | 动作 | 说明 | |---|---|---| -| 读取页面 | `browser_snapshot` | 标题/URL/正文/编号交互清单/表单字段（敏感值掩码）；`delta: true` 只返回变化，省 token | -| 点击元素 | `browser_click` | 按编号点击（链接/按钮/复选框…），React/Vue 组件兼容 | -| 填写表单 | `browser_type` | 输入文本，`replace` 清空重填 | -| 按键 | `browser_press` | Enter/Tab/Escape/方向键等 | -| 滚动 | `browser_scroll` | 视口滚动（up/down/top/bottom） | +| 读取页面 | `browser_snapshot` | 标题/URL/正文/编号交互清单/表单字段（敏感值掩码）；`delta: true` 只返回变化，`frame` 单读 iframe，`query` 定位任意元素并给出选择器 | +| 截取可视区域 | `browser_screenshot` | 捕获可见像素，不遮盖字段；除非启用了完全控制，否则需要截图审批 | +| 点击 | `browser_click` | 按编号或选择器点击、双击；支持鼠标按键和修饰键 | +| 文本 | `browser_type` | 输入文本，`replace` 清空重填；`mode` 选择光标插入或逐字符按键 | +| 按键 | `browser_press` | 单键与组合键，可跨调用按住修饰键 | +| 滚动 | `browser_scroll` | 向上、向下滚动页面或指定 iframe，或滚动到顶部、底部 | +| 剪贴板 / 文件 | `browser_clipboard` / `browser_upload` | 写入或粘贴文本与富文本；把本机文件附加到 file input | | 导航 | `browser_navigate` / `browser_open_tab` / `browser_back` / `browser_forward` / `browser_reload` | 受控标签页内跳转，或新开标签页并跟随 | | 列出标签页 | `browser_list_tabs` | 列出可访问标签页的稳定 ID、标题、URL 和活动/受控状态 | -| 跟随标签页 | `browser_follow_tab` | 将后续浏览器工具绑定到已列出的标签页，而不激活该标签页 | +| 跟随标签页 | `browser_follow_tab` | 将后续浏览器工具绑定到已列出的标签页；默认后台，`activate: true` 才切前台 | | 关闭标签页 | `browser_close_tab` | 关闭已列出的标签页 | | 读区域 | `browser_get_text` | 懒加载内容 / 局部文本 | | 等待 | `browser_wait` | 页面加载与渲染稳定检测 | +| 查看元素与 CSS | `browser_devtools_elements` / `browser_devtools_set_element_style` / `browser_devtools_set_element_attribute` | 高权限：计算样式、盒模型、命中规则，并可现场修改 CSS 与属性来调试 | +| 执行页面 JS | `browser_console_eval` / `browser_console_logs` | 高权限：在页面自身上下文执行代码，并读取记录的控制台输出 | +| 网络抓包 | `browser_devtools_network` / `browser_devtools_list_requests` / `browser_devtools_get_request` / `browser_devtools_request_body` | 高权限：已捕获请求列表、单条详情与正文（凭据类请求头默认脱敏） | | 图片对话 | `session.prompt` / `session.attachment` | 按宿主能力启用图片选择、纯图片发送和持久历史预览 | | 引用你划选的内容 | 侧栏输入框 | 你在页面里选中的文字会变成输入框里的引用，随下一条消息一起发送 | @@ -90,12 +96,12 @@ pnpm --filter dsh-browser-extension run test 或使用受支持的精确公开版本： ```sh - npx @deepseek-ai/dsh@0.2.0-rc.2 web + npx @deepseek-ai/dsh@0.2.1-rc.5 web ``` 两种命令都会从本机 `web` profile 加载同一个 bundle。默认端口为 3080；如被占用，可追加 `--port `。 - **DSH Desktop 用户**：桌面版默认让系统随机分配本地 Web 端口（`dsh-desktop.port: 0`），自动探测无法预知随机端口。请在桌面版设置中把端口固定为 `43189`（见 [deepseek-harness-desktop 用户指南](https://github.com/anywhere-labs/deepseek-harness-desktop/blob/master/docs/user-guide.md)），扩展的自动探测会覆盖该端口；或直接在侧栏设置中手动填写 `http://127.0.0.1:<端口>`。 + **DSH Desktop 用户**：桌面版启动 Host 时用 `--port 0`，本地 Web 端口每次启动都由系统重新分配，固定端口列表无法预知。因此桥接插件会同时在固定的回环信标端口（`43189`–`43198`，取第一个空闲端口）上通告真实桥地址。面板打开时探测一次：已填地址优先尝试，只有回环地址连不上时才自动探测一次。都没找到时状态显示「未找到服务」，不再重试；此时打开**设置 → 自动检测**列出本机所有实例并点选，或直接手填 `http://127.0.0.1:<端口>`。若想固定端口，可在 `~/.dsh/profiles/desktop/cordis.patch.yml` 覆盖 `webserver` 行——patch 的 `config` 是整体替换，务必把 `host` 一起写出。 加载或重新加载扩展本身是被动的：只有打开侧栏后，扩展才会探测本机端口并创建 WebSocket。用户已建立的健康连接可在侧栏关闭后继续用于后台审批；但连接一旦掉线或被另一浏览器替换，没有打开侧栏时就不会重连。 @@ -105,28 +111,47 @@ pnpm --filter dsh-browser-extension run test 如果只开发扩展，Chrome 从 `chrome://extensions` 加载 `extensions/dsh-browser/dist/`；Firefox 运行 `build:firefox` 后，从 `about:debugging#/runtime/this-firefox` 加载 `extensions/dsh-browser/dist-firefox/manifest.json`。代码更新后需重新构建并重新加载。 -## 为什么浏览器操作仍采用纯文本 +## 输入区控制：权限模式、模型、命令与 Skill + +输入区提供与 Web GUI 对齐的控件，全部由桥插件自有的 Host 方法支撑： + +- **权限模式**（左侧 chip）：部署的预设表（`read-only`、`workspace-write`、`danger-full-access`，实验 Auto integration 挂载时还有 `auto`）加上当前会话的取值。选择预设会执行宿主自己的 `/permission ` 命令，因此 `permission/preset` 选择与两个旋钮事件都会像 Web GUI 选择一样写入会话日志。「完全权限」与「Auto review」会先要求显式确认；推导态 `custom` 只作展示、不可选；菜单底部提示新会话的默认值。首次发送消息创建会话之前，选择会被记住，并在 materialize 时、第一条提示词准入之前应用。 +- **模型与推理强度**（右侧 chip）：按 provider 分组的模型目录，以及当前模型声明的推理强度（含「默认」）。选择只影响**下一轮**装配，因此回合运行中 chip 依然可用。目录读取失败时该行会说明原因并提供重试，而不是静默变灰；漏掉的 `turn/end` 会在面板下次变为可见时按宿主的 `running` 标记纠正。 +- **`/` 选单**：在空行输入 `/` 会打开宿主命令目录（`/compact`、`/permission`、`/plan`、`/goal`、`/feedback`、`/export` 等）与该会话的用户可调用 skill，其中包含标记为 `modelInvocable: false` 的 skill（带「仅用户可调用」角标）——它们正是为显式调用而存在。`↑`/`↓` 移动、`Enter`/`Tab` 接受、`Esc` 关闭；选中某行会插入其确切文本，方便继续补参数。提交时面板按与宿主适配器一致的规则裁决：命令名走 `commands.execute`（其输出永远不会变成模型消息），skill 名按字面文本发送（由宿主注入 skill 正文），无法识别的 `/…` 行会被**拒绝**且草稿保留——若想按普通文本发送，请在行首加一个空格。命令不支持附件；临时会话（尚未创建）的命令/skill 目录在第一条消息创建会话前返回空。 + +## 页面读取与控制 -- **快照即视图**：模型对页面的全部认知 = 结构化文本（标题/URL/正文/编号元素/表单），默认预算 32k 字符（插件可配，经 `hello.ok` 协商给扩展）。 +- **文本快照**：标题、URL、正文、编号元素和表单的默认预算为 32k 字符（插件可配，经 `hello.ok` 协商给扩展）。经过审批后，截图还可提供页面图像。 - **页面文字是不可信输入**：快照和局部文本读取会放进带随机 nonce 的信任边界，并明确要求模型不得把网页中的命令当成指令。这只是纵深防御；扩展侧的操作审批才是强制安全边界。 - **稳定编号**：元素编号跨快照保持（WeakMap + `data-dsh-el`），模型可以说"点 7 号"；页面大改时显式提示"编号已重排"。 - **delta 模式**：`browser_snapshot({delta:true})` 只返回变化元素的编号，省 token。 -- **隐私**：密码/卡号字段的值永远以 `••••` 呈现，绝不回传；可访问名称从不使用敏感字段的当前值。 -- **标签页绑定**：提交提示时会在模型开始工作前绑定活动标签页；如果直接调用浏览器工具，也会在需要时完成首次绑定。手动切换标签页或窗口后，后续工具会暂停，并询问助手继续原页面还是跟随当前页。选择原页面后允许后台操作，但不会改变用户正在看的页面；选择跟随后会重置页面引用状态。受控页关闭后失败关闭，直到用户选择当前页；切页还会撤销尚未完成的操作审批。 +- **隐私**：文本快照中的密码和卡号值以 `••••` 呈现，可访问名称不使用敏感字段的当前值。截图直接捕获可见像素，不会遮盖字段。开发者工具需要独立授权。 +- **标签页绑定**：提交提示时会在模型开始工作前绑定活动标签页；如果直接调用浏览器工具，也会在需要时完成首次绑定。手动切换标签页或窗口后，后续工具会暂停，并询问助手留在原页面还是跟随当前页面；此时还可以点「跳转回到助手页面」，把你自己正在看的标签页切回助手所在的页面。选择留在原页面后允许后台操作，但不会改变用户正在看的页面；选择跟随后会重置页面引用状态。受控页关闭后失败关闭，直到用户选择当前页面；切页还会撤销尚未完成的操作审批。 - **分级审批**：默认「自动共享」允许模型按需读取受控标签页而不额外弹窗；「每次询问」可恢复逐次读取确认，「关闭」会阻断读取。在「每次询问」模式下，读取弹窗可以仅允许一次，也可以持久切回自动读取，之后仍可在设置中关闭。状态变更工具仍然失败关闭，并显示实际 origin 和脱敏动作摘要；用户可拒绝、仅允许一次，或只在当前侧栏会话中信任单个 origin。最后一个侧栏关闭或 Service Worker 重启会清空临时信任；永久信任需在设置中显式管理。侧栏关闭时，审批最多保留 60 秒；启用通知后，系统通知可把用户带回侧栏。会话级审批只会在其所属会话恢复完成后显示。调用方取消或桥接超时时，会先撤销尚未完成的审批，过期动作不会继续执行。 - **完全控制需显式开启**：**允许模型完全控制浏览器**只有设置保存成功后才会生效，启用后页面读取、页面操作以及标签页列出/跟随/关闭均无需确认。调用在收到时固定其访问模式，因此开启完全控制不会追溯提升已经开始的受限调用。关闭会立即生效：取消尚未下发操作的调用，等待已经下发到浏览器的操作完成，再保存限制设置。快速重新开启也会在旧权限撤销完成前保持受限，并发保存会按请求顺序落盘。受保护页面的 DOM 内容始终不可访问。 - **会话续接**：重新打开侧栏时默认恢复最近活跃的浏览器会话；若该会话不可用，则恢复最新的非空持久会话，最后才创建新会话。可在设置中关闭。 +## 开发者工具（需显式开启，高权限） + +开发者工具组通过 Chrome 调试协议（`chrome.debugger`）驱动受控标签页，因此可以查看并修改页面元素与 CSS、在页面自身上下文执行 JavaScript、以及抓取网络请求（含页面层拦截拿不可靠的响应正文）。 + +- **默认关闭**：用户通过设置或当前侧栏会话授权之前，开发者工具调用会请求审批。弹窗提供仅允许一次、本次会话信任、拒绝。最后一个侧栏关闭或 worker 重启后，会话信任失效。**允许模型完全控制浏览器**开关与持久化 origin 白名单都不覆盖这一组。 +- **持久授权**：设置里的**允许开发者工具**跨侧栏会话授权这一组工具。关闭设置会断开调试器并取消进行中的开发者工具调用，包括通过会话信任允许的调用。页面导航会取消旧文档尚未完成的审批。 +- **每个标签页只允许一个调试器**：Chrome 会拒绝第二个调试器，因此附加期间浏览器自带的开发者工具无法在同一标签页打开。工具返回会说明原因并提示先关闭该标签页的 DevTools。 +- **数据处理**：抓包只存在内存中，上限为 200 条请求与 500 条控制台记录，附加时与每次顶层导航时都会重置。`Authorization`、`Cookie`、`Set-Cookie` 等请求头默认显示为 `[redacted]`，除非某次调用显式选择；疑似认证 URL 的正文默认脱敏。 +- **平台**：仅 Chrome。Firefox 构建不申请 `debugger` 权限，因此宿主也不会在该平台注册这一组工具。 + ## 权限说明 -Chrome 使用 `sidePanel`，Firefox 使用 `sidebar_action`。两者都申请 `storage`（设置与最近会话续接）、`notifications`（侧栏关闭时可选的审批提醒）、`tabs` + `activeTab` + `scripting`（观察切页，并向用户显式选择的受控标签页注入/发消息；安装前已打开的页面也会按需补注入）、`webNavigation`（枚举该标签页中的 frame，并把消息绑定到具体文档）、`alarms`（后台保活）和 `http/https`（内容脚本注入普通网页）。Firefox AMO manifest 如实声明扩展会把浏览活动、网页内容/操作和对话内容发送给用户配置的 dsh/模型服务。扩展绝不改变用户正在看的标签页，也不会静默跟随手动切页；只有用户选择继续原页面后，助手才会在后台操作。 +Chrome 使用 `sidePanel`，Firefox 使用 `sidebar_action`。两者都申请 `storage`（设置与最近会话）、`notifications`（可选的审批提醒）、`tabs` + `activeTab` + `scripting`（观察切页并向受控标签页注入脚本）、`webNavigation`（将操作绑定到 frame 文档）、`alarms`（后台保活）和 `` 主机权限。内容脚本在 HTTP(S) 页面运行。Chrome 还申请用于真实输入和开发者工具的 `debugger`，以及用于剪贴板操作的 `clipboardWrite` 和 `offscreen`。Firefox 不申请这些 Chrome 专用能力。Firefox AMO manifest 声明了发送给用户配置的 dsh/模型服务的浏览活动、网页内容与操作、对话内容。显式返回助手页面或调用 `browser_follow_tab { activate: true }` 可以激活标签页；手动切换标签页需要确认控制权交接。 ## 已知限制 - 同时只有一个扩展连接桥。未打开侧栏的浏览器 Profile 不会抢占连接；另一个已打开的侧栏顶替连接后，被替换的一端会主动让权，不再反复重连互踢。 +- 开发者工具需要 Chrome 125 以上、设置或会话授权或逐次审批，以及可用的调试连接。连接保持期间会显示 Chrome 的调试提示，包括网络抓包期间。 - 标签页绑定属于整个扩展连接，而不是单个对话会话。 - 可访问的跨源 iframe 会进入快照，并通过稳定的 `(frame, index)` 地址执行操作；受保护或已销毁的 frame 会标记为不可访问，不影响整页快照。 - 验证码/纯图片按钮无法处理——工具结果会标注"存在无文本可访问名的元素"，提示用户手动完成该步。 - 令牌无自动轮换。 -- `browser_press` 的合成按键不触发浏览器原生默认行为（Tab 焦点移动、方向键、Enter 激活等），仅用于框架内的键盘事件；依赖原生行为的场景请手动操作。 +- `browser_press` 的合成按键不触发浏览器原生默认行为，例如 Tab 焦点移动、方向键滚动或 Enter 激活。Chrome 的真实输入通道支持这些操作；Firefox 使用合成输入。 - `browser_wait` 以加载完成 + 固定静默窗口为准，不观察持续 DOM 更新（连续刷新的 SPA 可能被报为稳定）。 diff --git a/extensions/dsh-browser/manifest.firefox.json b/extensions/dsh-browser/manifest.firefox.json index 8d6dda156..3ae20857d 100644 --- a/extensions/dsh-browser/manifest.firefox.json +++ b/extensions/dsh-browser/manifest.firefox.json @@ -28,8 +28,7 @@ "notifications" ], "host_permissions": [ - "http://*/*", - "https://*/*" + "" ], "background": { "scripts": [ diff --git a/extensions/dsh-browser/manifest.json b/extensions/dsh-browser/manifest.json index d1311c27b..a14e5f697 100644 --- a/extensions/dsh-browser/manifest.json +++ b/extensions/dsh-browser/manifest.json @@ -4,7 +4,7 @@ "description": "__MSG_extensionDescription__", "default_locale": "en", "version": "0.1.4", - "minimum_chrome_version": "116", + "minimum_chrome_version": "125", "permissions": [ "sidePanel", "storage", @@ -13,11 +13,13 @@ "scripting", "webNavigation", "alarms", - "notifications" + "notifications", + "debugger", + "clipboardWrite", + "offscreen" ], "host_permissions": [ - "http://*/*", - "https://*/*" + "" ], "background": { "service_worker": "background.js", diff --git a/extensions/dsh-browser/offscreen/index.html b/extensions/dsh-browser/offscreen/index.html new file mode 100644 index 000000000..4da9c8288 --- /dev/null +++ b/extensions/dsh-browser/offscreen/index.html @@ -0,0 +1,10 @@ + + + + + dsh clipboard + + + + + diff --git a/extensions/dsh-browser/package.json b/extensions/dsh-browser/package.json index d7d949c5b..f7877e569 100644 --- a/extensions/dsh-browser/package.json +++ b/extensions/dsh-browser/package.json @@ -22,11 +22,11 @@ "tldts": "^6.1.86" }, "devDependencies": { - "@yuxianglin/dsh-bridge-browser": "workspace:^", "@types/chrome": "^0.0.268", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", "@vitejs/plugin-react": "^4.3.1", + "@yuxianglin/dsh-bridge-browser": "workspace:^", "jsdom": "^26.1.0", "typescript": "^5.6.2", "vite": "^6.0.0", diff --git a/extensions/dsh-browser/scripts/build.mjs b/extensions/dsh-browser/scripts/build.mjs index a4e5394bf..29fc3d2ea 100644 --- a/extensions/dsh-browser/scripts/build.mjs +++ b/extensions/dsh-browser/scripts/build.mjs @@ -20,6 +20,7 @@ const configs = [ 'vite.background.config.ts', 'vite.content.config.ts', 'vite.panel.config.ts', + 'vite.offscreen.config.ts', ] if (watch) { diff --git a/extensions/dsh-browser/src/background/authorization.ts b/extensions/dsh-browser/src/background/authorization.ts index d263a0587..8772f53d9 100644 --- a/extensions/dsh-browser/src/background/authorization.ts +++ b/extensions/dsh-browser/src/background/authorization.ts @@ -2,7 +2,7 @@ import type { ToolCall } from './tools.ts' import type { TabFrame } from './frames.ts' -import type { ApprovalPrompt } from '../security/approval.ts' +import type { ApprovalAuthorization, ApprovalPrompt } from '../security/approval.ts' import { getUiLocale, type UiLocale } from '../i18n.ts' const PAGE_READS = new Set(['browser_snapshot', 'browser_get_text']) @@ -10,6 +10,8 @@ const STATE_CHANGING_ACTIONS = new Set([ 'browser_click', 'browser_type', 'browser_press', + 'browser_clipboard', + 'browser_upload', 'browser_navigate', 'browser_open_tab', 'browser_back', @@ -17,6 +19,100 @@ const STATE_CHANGING_ACTIONS = new Set([ 'browser_reload', ]) +/** + * The developer-tools group. These calls drive the page through the debugger + * (CDP) instead of the content script, so they never consult the ordinary + * origin allowlist: only the explicit developer-tools setting, a session grant + * the user made in this browser, or a fresh per-call approval can pass. + */ +export const DEVTOOLS_TOOLS = new Set([ + 'browser_devtools_elements', + 'browser_devtools_set_element_style', + 'browser_devtools_set_element_attribute', + 'browser_console_eval', + 'browser_console_logs', + 'browser_devtools_network', + 'browser_devtools_list_requests', + 'browser_devtools_get_request', + 'browser_devtools_request_body', +]) + +/** Whether a tool belongs to the developer-tools group. */ +export function isDevToolsTool(name: string): boolean { + return DEVTOOLS_TOOLS.has(name) +} + +/** + * The approval prompt for one developer-tools call. + * + * Every devtools call requires its own consent or an explicit developer-tools + * setting/session grant. Origin trust and unrestricted access never cover it. Only the + * model-facing presentation differs between read-shaped and action-shaped calls. + * @param call - the tool call awaiting a decision. + * @param frames - the controlled tab's frames, for the origin list. + * @param locale - UI locale for the summary text. + * @returns the prompt handed to the side panel. + */ +export function devToolsApprovalForCall( + call: ToolCall, + frames: TabFrame[], + locale: UiLocale = getUiLocale(), +): ApprovalPrompt { + const destructive = call.name === 'browser_devtools_set_element_style' + || call.name === 'browser_devtools_set_element_attribute' + || call.name === 'browser_console_eval' + return { + kind: destructive ? 'action' : 'read', + action: call.name, + summary: summarizeDevToolsAction(call, locale), + origins: uniqueOrigins(frames.filter((frame) => frame.frameId === 0), frames), + // Session trust for devtools is granted through the dedicated prompt path, + // never as a blanket "trust this origin for every browser action". + canTrust: false, + devTools: true, + } +} + +function summarizeDevToolsAction(call: ToolCall, locale: UiLocale): string { + const detail = inlineArg(call) + switch (call.name) { + case 'browser_devtools_elements': + return localized(locale, `Inspect page element (${detail})`, `查看页面元素（${detail}）`) + case 'browser_devtools_set_element_style': + return localized(locale, `Modify live CSS on page element (${detail})`, `修改页面元素 CSS（${detail}）`) + case 'browser_devtools_set_element_attribute': + return localized(locale, `Change page element attribute (${detail})`, `修改页面元素属性（${detail}）`) + case 'browser_console_eval': + return localized( + locale, + `Run JavaScript in the page (${typeof call.args.expression === 'string' ? `${call.args.expression.length} characters, not shown here}` : 'expression not shown'})`, + `在页面中执行 JavaScript（${typeof call.args.expression === 'string' ? `${call.args.expression.length} 个字符，此处不显示内容` : '表达式内容不显示'}）`, + ) + case 'browser_console_logs': + return localized(locale, 'Read the page console output', '读取页面控制台输出') + case 'browser_devtools_network': + return localized(locale, `Control network capture (${safeInline(typeof call.args.action === 'string' ? call.args.action : 'status')})`, `控制网络抓包（${safeInline(typeof call.args.action === 'string' ? call.args.action : 'status')}）`) + case 'browser_devtools_list_requests': + return localized(locale, 'List captured network requests', '列出已捕获的网络请求') + case 'browser_devtools_get_request': + return localized(locale, `Read one captured request in full, including headers (index ${safeInline(String(call.args.index ?? '?'))})`, `查看单条网络请求的完整内容，含请求头（序号 ${safeInline(String(call.args.index ?? '?'))}）`) + case 'browser_devtools_request_body': + return localized(locale, `Read a captured request or response body (index ${safeInline(String(call.args.index ?? '?'))})`, `读取已捕获的请求或响应正文（序号 ${safeInline(String(call.args.index ?? '?'))}）`) + default: + return call.name + } +} + +/** A short, non-secret hint about which element a call targets. */ +function inlineArg(call: ToolCall): string { + if (typeof call.args.selector === 'string' && call.args.selector !== '') return safeInline(call.args.selector) + if (typeof call.args.index === 'number') { + const frame = typeof call.args.frame === 'number' && call.args.frame !== 0 ? `, frame ${call.args.frame}` : '' + return `snapshot index ${call.args.index}${frame}` + } + return 'no selector given' +} + /** Return an approval prompt, or undefined when this call needs no prompt. */ export function approvalPromptForCall( call: ToolCall, @@ -24,11 +120,23 @@ export function approvalPromptForCall( frames: TabFrame[], locale: UiLocale = getUiLocale(), ): ApprovalPrompt | undefined { + // A screenshot photographs exactly what the user sees — passwords, private + // messages, account balances included — so it always prompts, regardless of + // the page-content sharing preference. Only unrestricted access skips it. + if (call.name === 'browser_screenshot') { + return { + kind: 'read', + action: call.name, + summary: localized(locale, 'Capture a screenshot of the current page (may contain sensitive content)', '截取当前页面的可视区域截图（可能包含敏感信息）'), + origins: uniqueOrigins(frames, frames), + canTrust: false, + } + } if (PAGE_READS.has(call.name)) { if (sharePageContent !== 'ask') return undefined - const targetFrames = call.name === 'browser_snapshot' - ? frames - : frames.filter((frame) => frame.frameId === requestedFrame(call.args)) + const targetFrames = call.name === 'browser_get_text' + ? frames.filter((frame) => frame.frameId === requestedFrame(call.args)) + : frames return { kind: 'read', action: call.name, @@ -56,6 +164,9 @@ export function approvalPromptForCall( const target = frames.find((frame) => frame.frameId === frameId) ?? frames.find((frame) => frame.frameId === 0) const origins = uniqueOrigins(target === undefined ? [] : [target], frames) let canTrust = origins.length === 1 && call.name !== 'browser_back' && call.name !== 'browser_forward' + // Clipboard writes affect the user's system clipboard and uploads hand local + // files to a page; neither is an origin-scoped permission. + if (call.name === 'browser_clipboard' || call.name === 'browser_upload') canTrust = false if (call.name === 'browser_navigate') { const destination = originFromUrl(typeof call.args.url === 'string' ? call.args.url : '') if (destination !== undefined && !origins.includes(destination)) origins.push(destination) @@ -112,7 +223,16 @@ function summarizeAction(call: ToolCall, locale: UiLocale): string { : '' const index = typeof call.args.index === 'number' ? call.args.index : '?' switch (call.name) { - case 'browser_click': return localized(locale, `Click element [${index}]${frame}`, `点击元素 [${index}]${frame}`) + case 'browser_click': { + const target = typeof call.args.selector === 'string' + ? `selected element (${safeInline(call.args.selector)})` + : `element [${index}]` + const targetZh = typeof call.args.selector === 'string' + ? `选择器元素（${safeInline(call.args.selector)}）` + : `元素 [${index}]` + const gesture = typeof call.args.action === 'string' && call.args.action !== 'click' ? ` ${call.args.action}` : '' + return localized(locale, `${gesture.trim() === '' ? 'Click' : gesture.trim()} ${target}${frame}`, `${gesture === '' ? '点击' : gesture.trim()} ${targetZh}${frame}`) + } case 'browser_type': { const length = typeof call.args.text === 'string' ? call.args.text.length : 0 return localized( @@ -145,6 +265,21 @@ function summarizeAction(call: ToolCall, locale: UiLocale): string { `在新标签页打开 ${destination}`, ) } + case 'browser_clipboard': { + const writing = call.args.action === 'write' + const html = typeof call.args.html === 'string' && call.args.html !== '' + return writing + ? localized( + locale, + `Write ${html ? 'text and rich HTML' : 'text'} to the system clipboard`, + `向系统剪贴板写入${html ? '文本与富文本' : '文本'}`, + ) + : localized(locale, 'Paste the clipboard into the page', '把剪贴板内容粘贴到页面') + } + case 'browser_upload': { + const count = Array.isArray(call.args.paths) ? call.args.paths.length : 0 + return localized(locale, `Attach ${count} local file(s) to a page field`, `向页面字段附加 ${count} 个本地文件`) + } case 'browser_back': return localized(locale, 'Go back in browser history (destination domain unknown)', '返回浏览历史上一页（目标域名未知）') case 'browser_forward': return localized(locale, 'Go forward in browser history (destination domain unknown)', '前进到浏览历史下一页（目标域名未知）') case 'browser_reload': return localized(locale, 'Reload the current page', '重新加载当前页面') @@ -169,3 +304,36 @@ function safeInline(value: string, maxLength = 40): string { const inline = value.replace(/\s+/g, ' ').trim() return inline.length <= maxLength ? inline : `${inline.slice(0, maxLength - 1)}…` } + +/** Inputs the approval gate evaluates before a prompt is even sent. */ +export interface ApprovalGateInput { + prompt: ApprovalPrompt + /** Persisted "allow developer tools" setting. */ + devToolsAllowed: boolean + /** Ephemeral devtools grant the user made in this side-panel session. */ + devToolsSessionGrant: boolean + /** The user's blanket browser-control switch. */ + unrestrictedAccess: boolean + /** Result of the persistent + session origin allowlist for ordinary actions. */ + coveredByTrustedOrigins: boolean +} + +/** + * Decide whether an approval prompt can be skipped. + * + * Developer tools are their own consent surface: unrestricted access and the + * persistent origin allowlist deliberately do not cover them, so enabling + * either can never hand the model the debugger. Ordinary actions keep the + * existing shortcut semantics. + * + * @param input - the prompt plus the policy state it is evaluated against. + * @returns the authorization to apply, or `'prompt'` when the user must decide. + */ +export function resolveApprovalGate(input: ApprovalGateInput): ApprovalAuthorization | 'prompt' { + if (input.prompt.devTools === true) { + if (input.devToolsSessionGrant || input.devToolsAllowed) return 'approved' + return 'prompt' + } + if (input.unrestrictedAccess || input.coveredByTrustedOrigins) return 'approved' + return 'prompt' +} diff --git a/extensions/dsh-browser/src/background/bridge.ts b/extensions/dsh-browser/src/background/bridge.ts index 33c9da70a..f26e778bf 100644 --- a/extensions/dsh-browser/src/background/bridge.ts +++ b/extensions/dsh-browser/src/background/bridge.ts @@ -51,6 +51,13 @@ export class BridgeClient { private readonly probe: BridgeProbe = async () => true, /** Whether a disconnected client still has an active user-owned lease. */ private readonly shouldReconnect: () => boolean = () => true, + /** + * Capabilities this build can serve beyond the text snapshot contract. + * Advertised on every `hello`: the host registers the developer-tools group + * only for an extension that claims it, and the user's own setting still + * gates every call. + */ + private readonly capabilities: Pick = {}, ) {} /** Current coarse state (mirrors the last emitted sink value). */ @@ -152,7 +159,12 @@ export class BridgeClient { socket.send(JSON.stringify({ t: 'hello', token: this.token, - caps: { textOnly: true, snapshotMaxChars: DEFAULT_SNAPSHOT_MAX_CHARS, maxInteractiveItems: 60 }, + caps: { + textOnly: true, + snapshotMaxChars: DEFAULT_SNAPSHOT_MAX_CHARS, + maxInteractiveItems: 60, + ...this.capabilities, + }, } satisfies ClientFrame)) let authed = false diff --git a/extensions/dsh-browser/src/background/clipboard-write.ts b/extensions/dsh-browser/src/background/clipboard-write.ts new file mode 100644 index 000000000..928a9c8cc --- /dev/null +++ b/extensions/dsh-browser/src/background/clipboard-write.ts @@ -0,0 +1,71 @@ +/** + * System clipboard writes for `browser_clipboard { action: 'write' }`. + * + * Chrome runs the write in a packaged offscreen document: an MV3 service + * worker has no DOM and cannot touch the clipboard itself. Firefox has no + * offscreen API, but its event page still has a document, so the same payload + * is copied through a hidden editable host. When neither path is available the + * call fails with a stable code instead of silently doing nothing. + * + * @module + */ + +import { InputError } from './input-actions.ts' +import { copyClipboardInDocument } from '../shared/clipboard-document.ts' +import { + CLIPBOARD_WRITE_MESSAGE, + OFFSCREEN_CLIPBOARD_URL, + type ClipboardWriteOutcome, + type ClipboardWritePayload, +} from '../shared/clipboard.ts' + +let creating: Promise | undefined + +/** Create the offscreen document unless it already exists. */ +async function ensureOffscreenDocument(): Promise { + if (creating !== undefined) return creating + creating = (async () => { + if (await chrome.offscreen.hasDocument()) return + await chrome.offscreen.createDocument({ + url: OFFSCREEN_CLIPBOARD_URL, + reasons: ['CLIPBOARD' as chrome.offscreen.Reason], + justification: 'Write text or rich HTML to the clipboard at the user\'s request.', + }) + })() + try { + await creating + } finally { + creating = undefined + } +} + +/** + * Write text and optional rich HTML to the system clipboard. + * + * @param payload - clipboard content. + * @throws {InputError} with `input-unavailable` when the platform cannot copy. + */ +export async function writeClipboard(payload: ClipboardWritePayload): Promise { + if (payload.text === '' && payload.html === undefined) { + throw new InputError('bad-args', 'text or html is required for a clipboard write.') + } + if (chrome.offscreen === undefined) { + if (typeof document === 'undefined') throw new InputError('input-unavailable', 'The browser has no clipboard document.') + return copyClipboardInDocument(payload) + } + await ensureOffscreenDocument() + const response: unknown = await chrome.runtime.sendMessage({ type: CLIPBOARD_WRITE_MESSAGE, target: 'offscreen', payload }) + if (typeof response === 'object' && response !== null && (response as { ok?: unknown }).ok === true) { + const outcome = (response as { outcome?: unknown }).outcome + if (typeof outcome === 'object' && outcome !== null) { + const record = outcome as { rich?: unknown; plainDerived?: unknown; plainText?: unknown } + if (typeof record.rich === 'boolean' && typeof record.plainDerived === 'boolean' && typeof record.plainText === 'string') { + return { rich: record.rich, plainDerived: record.plainDerived, plainText: record.plainText } + } + } + } + if (typeof response === 'object' && response !== null && typeof (response as { error?: unknown }).error === 'string') { + throw new InputError('action-failed', (response as { error: string }).error) + } + throw new InputError('action-failed', 'The offscreen document did not confirm the clipboard write.') +} diff --git a/extensions/dsh-browser/src/background/debugger-broker.ts b/extensions/dsh-browser/src/background/debugger-broker.ts new file mode 100644 index 000000000..3643ad920 --- /dev/null +++ b/extensions/dsh-browser/src/background/debugger-broker.ts @@ -0,0 +1,305 @@ +/** + * Shared debugger broker. + * + * A tab can have exactly one debugger attachment, but two features need it: + * the developer-tools group (inspection, page JavaScript, network capture) and + * trusted input (keyboard, text delivery, and file uploads). The broker owns + * the per-tab session, counts the features that hold it, and decides when the + * attachment can be dropped — immediately when the developer tools stop, and + * after an idle window for input so a sequence of keystrokes does not re-attach + * between every call. + * + * @module + */ + +import { + DebuggerSession, + DevToolsError, + debuggerApi, + type CdpDomain, + type DebuggerApi, + type DevToolsSessionListeners, +} from './debugger-session.ts' + +/** Feature that holds an attachment. */ +export type DebuggerOwner = 'devtools' | 'input' + +/** One feature's hold on a tab's debugger. */ +export interface DebuggerLease { + readonly session: DebuggerSession + /** Drop this hold; the attachment follows the broker's idle policy. */ + release(options?: { immediate?: boolean }): Promise +} + +/** Attachment state the panel can report. */ +export interface DebuggerBrokerState { + attached: boolean + tabId?: number + owners: DebuggerOwner[] +} + +/** What one acquire needs. */ +export interface DebuggerAcquireOptions { + domains?: readonly CdpDomain[] + listeners?: DevToolsSessionListeners +} + +/** Broker surface used by the features and the background entry point. */ +export interface DebuggerBroker { + /** Attach (or join an attachment) for one feature. */ + acquire(tabId: number, owner: DebuggerOwner, options?: DebuggerAcquireOptions): Promise + /** Re-attach after an external detach, without changing ownership. */ + reattach(tabId: number, owner: DebuggerOwner, domains?: readonly CdpDomain[]): Promise + /** The attached session for a tab, when one exists. */ + attachedSession(tabId: number): DebuggerSession | undefined + /** Drop one feature's hold; the attachment survives while another owner needs it. */ + detachOwner(tabId: number, owner: DebuggerOwner): Promise + /** Drop every owner and detach (tab closed, settings revoked, worker shutdown). */ + detachTab(tabId: number): Promise + /** Drop every owner for every tab. */ + detachAll(): Promise + /** Current attachment state for the panel. */ + state(): DebuggerBrokerState + /** Observe attachment changes (attach, detach, error). */ + onChange(listener: () => void): () => void +} + +/** Broker construction options; the API is injectable for tests. */ +export interface DebuggerBrokerOptions { + api?: DebuggerApi + commandTimeoutMs?: number + /** How long an input-only attachment survives without calls. */ + idleDetachMs?: number + /** Notified on every attachment change. */ + onStateChange?: () => void +} + +/** Default idle window for an input-only attachment. */ +export const DEFAULT_INPUT_IDLE_DETACH_MS = 20_000 + +interface BrokerEntry { + tabId: number + session: DebuggerSession + owners: Map + listeners: Map + idleTimer?: ReturnType + queue: Promise +} + +/** Create the broker. */ +export function createDebuggerBroker(options: DebuggerBrokerOptions = {}): DebuggerBroker { + const api = options.api ?? debuggerApi() + const idleDetachMs = options.idleDetachMs ?? DEFAULT_INPUT_IDLE_DETACH_MS + const entries = new Map() + const observers = new Set<() => void>() + let lastTabId: number | undefined + + const notify = (): void => { + for (const observer of observers) { + try { observer() } catch { /* an observer must not break the broker */ } + } + options.onStateChange?.() + } + + const clearIdle = (entry: BrokerEntry): void => { + if (entry.idleTimer !== undefined) { + clearTimeout(entry.idleTimer) + entry.idleTimer = undefined + } + } + + /** Serialize attach/release per tab so concurrent calls cannot double-attach. */ + const enqueue = (entry: BrokerEntry, task: () => Promise): Promise => { + const run = entry.queue.catch(() => undefined).then(task) + entry.queue = run.then(() => undefined, () => undefined) + return run + } + + const getEntry = (tabId: number): BrokerEntry => { + const existing = entries.get(tabId) + if (existing !== undefined) return existing + if (api === undefined) { + throw new DevToolsError('devtools-unavailable', 'This browser build cannot attach a debugger.') + } + // The session fans events out to whichever owners are registered, so the + // entry can be created before those owners arrive. + const entry: BrokerEntry = { + tabId, + owners: new Map(), + listeners: new Map(), + queue: Promise.resolve(), + // Assigned immediately below; the closure only runs after creation. + session: undefined as unknown as DebuggerSession, + } + entry.session = new DebuggerSession( + tabId, + api, + { + onCdpEvent: (method, params) => { + for (const listener of entry.listeners.values()) listener.onCdpEvent?.(method, params) + }, + onDetached: (reason) => { + clearIdle(entry) + for (const listener of entry.listeners.values()) listener.onDetached?.(reason) + notify() + }, + }, + options.commandTimeoutMs, + ) + entries.set(tabId, entry) + return entry + } + + /** Attach, recovering from a stale attachment left by a worker restart. */ + const attachWithRecovery = async (session: DebuggerSession, domains: readonly CdpDomain[]): Promise => { + try { + await session.attach(domains) + } catch (error: unknown) { + if (!(error instanceof DevToolsError) || error.code !== 'devtools-conflict' || api === undefined) throw error + // After an MV3 worker restart the extension believes it is detached while + // Chrome still holds the attachment. Drop it and retry once; a genuine + // DevTools user keeps the tab and sees the actionable error. + try { + await api.detach({ tabId: session.tabId }) + await session.attach(domains) + } catch { + throw error + } + } + } + + const detachEntry = async (entry: BrokerEntry): Promise => { + clearIdle(entry) + entry.owners.clear() + entry.listeners.clear() + if (entry.session.isAttached) await entry.session.detach() + notify() + } + + const scheduleIdleDetach = (entry: BrokerEntry): void => { + clearIdle(entry) + if (idleDetachMs <= 0) { + void enqueue(entry, async () => { + if (entry.owners.size === 0) await detachEntry(entry) + }) + return + } + entry.idleTimer = setTimeout(() => { + entry.idleTimer = undefined + void enqueue(entry, async () => { + if (entry.owners.size === 0) await detachEntry(entry) + }) + }, idleDetachMs) + } + + const release = async (tabId: number, owner: DebuggerOwner, releaseOptions: { immediate?: boolean } = {}): Promise => { + const entry = entries.get(tabId) + if (entry === undefined) return + await enqueue(entry, async () => { + const remaining = (entry.owners.get(owner) ?? 1) - 1 + if (remaining > 0) { + entry.owners.set(owner, remaining) + return + } + entry.owners.delete(owner) + entry.listeners.delete(owner) + if (entry.owners.size > 0) return + if (releaseOptions.immediate === true) await detachEntry(entry) + else scheduleIdleDetach(entry) + }) + } + + return { + acquire: async (tabId, owner, acquireOptions = {}) => { + if (api === undefined) { + throw new DevToolsError('devtools-unavailable', 'This browser build cannot attach a debugger.') + } + const entry = getEntry(tabId) + return enqueue(entry, async () => { + clearIdle(entry) + if (acquireOptions.listeners !== undefined) entry.listeners.set(owner, acquireOptions.listeners) + entry.owners.set(owner, (entry.owners.get(owner) ?? 0) + 1) + try { + if (entry.session.isAttached) await entry.session.enableDomains(acquireOptions.domains ?? []) + else await attachWithRecovery(entry.session, acquireOptions.domains ?? []) + } catch (error: unknown) { + const remaining = (entry.owners.get(owner) ?? 1) - 1 + if (remaining > 0) entry.owners.set(owner, remaining) + else { + entry.owners.delete(owner) + entry.listeners.delete(owner) + } + throw error + } + lastTabId = tabId + notify() + return { + session: entry.session, + release: (leaseOptions) => release(tabId, owner, leaseOptions), + } + }) + }, + + reattach: async (tabId, owner, domains = []) => { + const entry = entries.get(tabId) + if (entry === undefined || !entry.owners.has(owner)) { + throw new DevToolsError('devtools-detached', 'This feature no longer holds the tab debugger.') + } + return enqueue(entry, async () => { + clearIdle(entry) + await attachWithRecovery(entry.session, domains) + lastTabId = tabId + notify() + return entry.session + }) + }, + + attachedSession: (tabId) => { + const entry = entries.get(tabId) + return entry?.session.isAttached === true ? entry.session : undefined + }, + + detachOwner: async (tabId, owner) => { + const entry = entries.get(tabId) + if (entry === undefined) return + await enqueue(entry, async () => { + if (!entry.owners.has(owner)) return + entry.owners.delete(owner) + entry.listeners.delete(owner) + if (entry.owners.size === 0) await detachEntry(entry) + }) + }, + + detachTab: async (tabId) => { + const entry = entries.get(tabId) + if (entry === undefined) return + entries.delete(tabId) + await enqueue(entry, async () => { await detachEntry(entry) }) + if (lastTabId === tabId) lastTabId = undefined + notify() + }, + + detachAll: async () => { + const all = [...entries.values()] + entries.clear() + await Promise.all(all.map((entry) => enqueue(entry, async () => { await detachEntry(entry) }))) + lastTabId = undefined + notify() + }, + + state: () => { + const entry = lastTabId === undefined ? undefined : entries.get(lastTabId) + if (entry === undefined) return { attached: false, owners: [] } + return { + attached: entry.session.isAttached, + tabId: entry.tabId, + owners: [...entry.owners.keys()], + } + }, + + onChange: (listener) => { + observers.add(listener) + return () => { observers.delete(listener) } + }, + } +} diff --git a/extensions/dsh-browser/src/background/debugger-frame.ts b/extensions/dsh-browser/src/background/debugger-frame.ts new file mode 100644 index 000000000..695efe742 --- /dev/null +++ b/extensions/dsh-browser/src/background/debugger-frame.ts @@ -0,0 +1,58 @@ +import { DevToolsError, type CdpClient, type FrameExecutionContext } from './debugger-session.ts' +import type { TabFrame } from './frames.ts' + +// 用目标文档上的一次性标记关联 Chrome frameId 与 CDP execution context。 +export async function resolveDebuggerFrame(client: CdpClient, tabId: number, frame: TabFrame): Promise { + if (client.executionContexts === undefined || frame.documentId === undefined) { + throw new DevToolsError('devtools-unavailable', 'The browser cannot resolve the target document for debugging.') + } + const contexts = await client.executionContexts() + const attribute = `data-dsh-frame-${crypto.randomUUID()}` + const target = { tabId, documentIds: [frame.documentId] } + await chrome.scripting.executeScript({ + target, args: [attribute], + func: (name: string) => { document.documentElement.setAttribute(name, '') }, + }) + try { + for (const context of contexts) { + const answer = await context.client.send('Runtime.evaluate', { + expression: `document.documentElement.hasAttribute(${JSON.stringify(attribute)})`, + uniqueContextId: context.uniqueId, returnByValue: true, + }) as { result?: { value?: unknown } } + if (answer.result?.value !== true) continue + const current = await chrome.webNavigation.getFrame({ tabId, frameId: frame.frameId }) + if (current?.documentId !== frame.documentId) { + throw new DevToolsError('devtools-unavailable', 'The target frame navigated before execution.') + } + return context + } + throw new DevToolsError('devtools-unavailable', `Frame ${frame.frameId} has no matching debugger context.`) + } finally { + await chrome.scripting.executeScript({ + target, args: [attribute], + func: (name: string) => { document.documentElement.removeAttribute(name) }, + }) + } +} + +export async function resolveFrameNode(client: CdpClient, selector: string, uniqueContextId: string): Promise { + await client.send('DOM.enable') + await client.send('DOM.getDocument', { depth: -1, pierce: true }) + const answer = await client.send('Runtime.evaluate', { + expression: `document.querySelector(${JSON.stringify(selector)})`, + uniqueContextId, returnByValue: false, + }) as { result?: { objectId?: string }; exceptionDetails?: unknown } + if (answer.exceptionDetails !== undefined || answer.result?.objectId === undefined) { + throw new DevToolsError('devtools-command-failed', `No element matched ${selector} in the target frame.`) + } + const objectId = answer.result.objectId + try { + const result = await client.send('DOM.requestNode', { objectId }) as { nodeId?: number } + if (result.nodeId === undefined || result.nodeId === 0) { + throw new DevToolsError('devtools-command-failed', 'The target element is no longer available.') + } + return result.nodeId + } finally { + await client.send('Runtime.releaseObject', { objectId }) + } +} diff --git a/extensions/dsh-browser/src/background/debugger-session.ts b/extensions/dsh-browser/src/background/debugger-session.ts new file mode 100644 index 000000000..5aadd9c58 --- /dev/null +++ b/extensions/dsh-browser/src/background/debugger-session.ts @@ -0,0 +1,351 @@ +/** + * Chrome debugger (CDP) session for one controlled tab. + * + * The developer-tools features are the only consumers: element/CSS inspection + * and edits, page-context JavaScript evaluation plus recorded console output, + * and network capture. One attachment per tab; every command is bounded by a + * timeout and settles on detach so a tool call can never hang on a dead + * session. + * + * Chrome refuses to attach while DevTools is open on the same tab; that case is + * reported as {@link DevToolsError} code `devtools-conflict` with an actionable + * message instead of a raw protocol error. + * + * @module + */ + +/** Stable failure codes the devtools tools map onto model-facing errors. */ +export type DevToolsErrorCode = + | 'devtools-unavailable' + | 'devtools-conflict' + | 'devtools-detached' + | 'devtools-timeout' + | 'devtools-command-failed' + +/** Typed devtools failure; `code` is the machine identity, `message` the human text. */ +export class DevToolsError extends Error { + constructor( + readonly code: DevToolsErrorCode, + message: string, + ) { + super(message) + this.name = 'DevToolsError' + } +} + +/** One `chrome.debugger.sendCommand` call, structurally typed for tests. */ +export type SendCommand = (target: { tabId: number; sessionId?: string }, method: string, params?: Record) => Promise + +/** The CDP command surface every developer-tools feature module depends on. */ +export interface CdpClient { + send(method: string, params?: Record): Promise + executionContexts?(): Promise +} + +export interface FrameExecutionContext { + frameId: string + uniqueId: string + client: CdpClient +} + +/** The subset of `chrome.debugger` this module uses. */ +export interface DebuggerApi { + attach(target: { tabId: number }, version: string): Promise + detach(target: { tabId: number }): Promise + sendCommand: SendCommand + onEvent: { + addListener(listener: (source: { tabId?: number; sessionId?: string }, method: string, params?: unknown) => void): void + removeListener(listener: (source: { tabId?: number; sessionId?: string }, method: string, params?: unknown) => void): void + } + onDetach: { + addListener(listener: (source: { tabId?: number }, reason: string) => void): void + removeListener(listener: (source: { tabId?: number }, reason: string) => void): void + } +} + +/** CDP protocol version negotiated by the attachment. */ +const CDP_VERSION = '1.3' + +/** Per-command budget; a stalled page must fail the tool call, not hang it. */ +const DEFAULT_COMMAND_TIMEOUT_MS = 10_000 + +/** Bounded network buffers so a long page session cannot exhaust extension memory. */ +const NETWORK_BUFFERS = { maxTotalBufferSize: 20_000_000, maxResourceBufferSize: 5_000_000, maxPostDataSize: 1_000_000 } + +/** CDP domains this extension can enable on an attachment. */ +export type CdpDomain = 'DOM' | 'CSS' | 'Runtime' | 'Network' + +/** Domains the developer-tools group needs. */ +export const DEVTOOLS_DOMAINS: readonly CdpDomain[] = ['DOM', 'CSS', 'Runtime', 'Network'] + +/** Observer hooks the devtools feature modules subscribe to. */ +export interface DevToolsSessionListeners { + onCdpEvent?(method: string, params: unknown): void + onDetached?(reason: string): void +} + +/** One dispatched command awaiting its single settle. */ +interface PendingCommand { + settle(error?: DevToolsError, value?: unknown): void + timer: ReturnType +} + +/** Read the browser's debugger API, or undefined when the platform lacks it. */ +export function debuggerApi(): DebuggerApi | undefined { + const candidate = (globalThis as { chrome?: { debugger?: unknown } }).chrome?.debugger as DebuggerApi | undefined + if (candidate === undefined) return undefined + return typeof candidate.attach === 'function' && typeof candidate.sendCommand === 'function' ? candidate : undefined +} + +/** Whether the running browser can serve the developer-tools group at all. */ +export function devToolsSupported(): boolean { + return debuggerApi() !== undefined +} + +/** + * One debugger attachment. Instances are not shared across tabs: the session + * owns exactly one tab and refuses commands after it detaches. + */ +export class DebuggerSession { + private pending = new Set() + private attached = false + private paused = false + private disposeEvents: (() => void) | undefined + private enabled: CdpDomain[] = [] + private frameContexts = new Map() + private childSessions = new Map>() + private autoAttach: Promise | undefined + + constructor( + readonly tabId: number, + private readonly api: DebuggerApi, + private readonly listeners: DevToolsSessionListeners = {}, + private readonly commandTimeoutMs: number = DEFAULT_COMMAND_TIMEOUT_MS, + ) {} + + /** @returns whether this session currently holds the tab's debugger. */ + get isAttached(): boolean { + return this.attached + } + + /** Mark the session paused (setting revoked): commands fail until it re-attaches. */ + pause(): void { + this.paused = true + } + + /** Allow commands again after a re-attach. */ + resume(): void { + this.paused = false + } + + /** + * Attach and enable the domains this feature set needs. Idempotent. + * @param domains - CDP domains to enable; defaults to the developer-tools set. + * @throws {DevToolsError} when another debugger owns the tab or the platform refuses. + */ + async attach(domains: readonly CdpDomain[] = DEVTOOLS_DOMAINS): Promise { + if (this.attached) { + await this.enableDomains(domains) + return + } + try { + await this.api.attach({ tabId: this.tabId }, CDP_VERSION) + } catch (error: unknown) { + throw mapAttachFailure(error) + } + this.attached = true + this.paused = false + this.bindEvents() + this.enabled = [] + try { + await this.enableDomains(domains) + } catch (error: unknown) { + // A partial attachment is worse than none: the caller retries cleanly. + await this.detach() + throw error + } + } + + /** + * Enable CDP domains that are not enabled yet. Trusted input attaches with no + * domains (it only dispatches events), while the developer tools need their + * inspection and capture domains; whichever caller attaches first, the other + * can add its own set without re-attaching. + * + * @param domains - domains to enable. + */ + async enableDomains(domains: readonly CdpDomain[]): Promise { + for (const domain of domains) { + if (this.enabled.includes(domain)) continue + if (domain === 'Network') await this.send('Network.enable', NETWORK_BUFFERS) + else await this.send(`${domain}.enable`, {}) + this.enabled.push(domain) + } + } + + /** CDP domains this session successfully enabled. */ + get domains(): readonly string[] { + return [...this.enabled] + } + + /** + * Send one CDP command. + * @param method - CDP method name. + * @param params - CDP parameters. + * @returns the command result. + * @throws {DevToolsError} on timeout, detach, or a CDP error response. + */ + send(method: string, params: Record = {}): Promise { + return this.sendToSession(undefined, method, params) + } + + // Runtime 的 uniqueId 在导航和进程切换后不会复用。 + async executionContexts(): Promise { + await this.enableDomains(['Runtime']) + this.autoAttach ??= this.send('Target.setAutoAttach', { + autoAttach: true, waitForDebuggerOnStart: false, flatten: true, + filter: [{ type: 'iframe', exclude: false }], + }).then(() => undefined) + await this.autoAttach + let count = -1 + while (count !== this.childSessions.size) { + count = this.childSessions.size + await Promise.all(this.childSessions.values()) + } + return [...this.frameContexts.values()] + } + + private sendToSession(sessionId: string | undefined, method: string, params: Record = {}): Promise { + if (!this.attached || this.paused) { + return Promise.reject(new DevToolsError('devtools-detached', 'The developer-tools session is not attached to the page.')) + } + return new Promise((resolve, reject) => { + let settled = false + const settle = (error?: DevToolsError, value?: unknown): void => { + if (settled) return + settled = true + clearTimeout(timer) + this.pending.delete(entry) + if (error === undefined) resolve(value) + else reject(error) + } + const timer = setTimeout(() => { + settle(new DevToolsError( + 'devtools-timeout', + `Developer-tools command ${method} did not answer within ${this.commandTimeoutMs}ms.`, + )) + }, this.commandTimeoutMs) + const entry: PendingCommand = { settle, timer } + this.pending.add(entry) + let invocation: Promise + try { + invocation = this.api.sendCommand({ tabId: this.tabId, ...(sessionId === undefined ? {} : { sessionId }) }, method, params) as Promise + } catch (error: unknown) { + settle(toDevToolsError(error, method)) + return + } + if (invocation === undefined || typeof invocation.then !== 'function') { + // A callback-only API surface cannot be awaited here, so fail fast + // instead of leaving the caller waiting for the timeout. + settle(new DevToolsError('devtools-command-failed', `Developer-tools command ${method} could not be dispatched.`)) + return + } + invocation.then( + (value) => { settle(undefined, value) }, + (error: unknown) => { settle(toDevToolsError(error, method)) }, + ) + }) + } + + /** Detach and release every pending command. Safe to call repeatedly. */ + async detach(): Promise { + if (!this.attached) return + this.attached = false + this.releasePending('The developer-tools session detached from the page.') + this.releaseEvents() + try { + await this.api.detach({ tabId: this.tabId }) + } catch { + // Already detached (tab closed, Chromium took it over): nothing to undo. + } + } + + private releasePending(reason: string): void { + const failure = new DevToolsError('devtools-detached', reason) + for (const entry of [...this.pending]) entry.settle(failure) + } + + private bindEvents(): void { + const onEvent = (source: { tabId?: number; sessionId?: string }, method: string, params?: unknown): void => { + if (source.tabId !== this.tabId) return + const prefix = `${source.sessionId ?? ''}:` + if (method === 'Runtime.executionContextCreated') { + const context = (params as { context: { id: number; uniqueId: string; auxData?: { frameId?: string; isDefault?: boolean } } }).context + if (context.auxData?.isDefault && context.auxData.frameId !== undefined) { + this.frameContexts.set(`${prefix}${context.id}`, { + frameId: context.auxData.frameId, uniqueId: context.uniqueId, + client: { send: (name, args) => this.sendToSession(source.sessionId, name, args) }, + }) + } + } else if (method === 'Runtime.executionContextDestroyed') { + this.frameContexts.delete(`${prefix}${(params as { executionContextId: number }).executionContextId}`) + } else if (method === 'Runtime.executionContextsCleared') { + for (const key of this.frameContexts.keys()) if (key.startsWith(prefix)) this.frameContexts.delete(key) + } else if (method === 'Target.attachedToTarget') { + const { sessionId } = params as { sessionId: string } + const ready = this.sendToSession(sessionId, 'Runtime.enable').then(async () => { + await this.sendToSession(sessionId, 'Target.setAutoAttach', { + autoAttach: true, waitForDebuggerOnStart: false, flatten: true, + filter: [{ type: 'iframe', exclude: false }], + }) + }) + this.childSessions.set(sessionId, ready) + } else if (method === 'Target.detachedFromTarget') { + const { sessionId } = params as { sessionId: string } + this.childSessions.delete(sessionId) + for (const key of this.frameContexts.keys()) if (key.startsWith(`${sessionId}:`)) this.frameContexts.delete(key) + } + if (source.sessionId === undefined) this.listeners.onCdpEvent?.(method, params) + } + const onDetach = (source: { tabId?: number }, reason: string): void => { + if (source.tabId !== this.tabId) return + this.attached = false + this.releasePending(`The developer-tools session detached from the page (${reason}).`) + this.releaseEvents() + this.listeners.onDetached?.(reason) + } + this.api.onEvent.addListener(onEvent) + this.api.onDetach.addListener(onDetach) + this.disposeEvents = () => { + this.api.onEvent.removeListener(onEvent) + this.api.onDetach.removeListener(onDetach) + } + } + + private releaseEvents(): void { + this.frameContexts.clear() + this.childSessions.clear() + this.autoAttach = undefined + this.disposeEvents?.() + this.disposeEvents = undefined + } +} + +/** Map an attach failure onto the actionable devtools error vocabulary. */ +export function mapAttachFailure(error: unknown): DevToolsError { + const detail = error instanceof Error ? error.message : String(error) + if (/another debugger|already attached|debugger is already/i.test(detail)) { + return new DevToolsError( + 'devtools-conflict', + 'Chrome DevTools (or another debugger) is currently attached to this tab. ' + + 'Close DevTools for this tab and retry, because a page can only be debugged by one client at a time.', + ) + } + return new DevToolsError('devtools-command-failed', `The debugger could not attach to this tab: ${detail}`) +} + +/** Map a CDP command rejection onto the devtools error vocabulary. */ +export function toDevToolsError(error: unknown, method: string): DevToolsError { + const detail = error instanceof Error ? error.message : String(error) + return new DevToolsError('devtools-command-failed', `Developer-tools command ${method} failed: ${detail}`) +} diff --git a/extensions/dsh-browser/src/background/devtools-controller.ts b/extensions/dsh-browser/src/background/devtools-controller.ts new file mode 100644 index 000000000..67b35a2ed --- /dev/null +++ b/extensions/dsh-browser/src/background/devtools-controller.ts @@ -0,0 +1,792 @@ +/** + * The developer-tools controller: owns the debugger sessions, folds CDP events + * into the network/console stores, and implements the nine devtools operations + * as text-producing handlers. + * + * Everything volatile (tabs, attachments, buffers) lives here, and the Chrome + * debugger API is injected, so the whole capability is testable against a + * scripted debugger surface. + * + * @module + */ + +import { + DebuggerSession, + DevToolsError, + DEVTOOLS_DOMAINS, + debuggerApi, + devToolsSupported, +} from './debugger-session.ts' +import { createDebuggerBroker, type DebuggerBroker, type DebuggerLease } from './debugger-broker.ts' +import { resolveDebuggerFrame } from './debugger-frame.ts' +import { listTabFrames } from './frames.ts' +import type { CdpClient } from './debugger-session.ts' +import { + NetworkCapture, + type NetworkEntry, +} from './devtools-network.ts' +import { + FrameRegistry, + attributeRisk, + attributeValue, + frameContextId, + inspectElement, + resolveNodeId, + setAttribute, + setInlineStyle, + setMatchedRuleStyle, + shortIdentity, + syncFrames, +} from './devtools-elements.ts' +import { + REDACTED, + formatBody, + formatEvaluation, + isSensitiveBodyUrl, + truncateText, +} from './devtools-serialization.ts' +import { + renderAttributeResult, + renderBody, + renderCaptureState, + renderConsoleLogs, + renderElement, + renderRequestDetail, + renderRequestList, + renderStyleEdit, +} from './devtools-format.ts' + +/** Default output budget for one developer-tools result. */ +export const DEFAULT_DEVTOOLS_OUTPUT_CHARS = 24_000 + +/** Response/request body bytes fetched per call; CDP buffers keep them bounded. */ +const BODY_CHAR_CAP = 65_536 + +/** One controlled tab the developer tools can attach to. */ +export interface DevToolsTarget { + tabId: number + url?: string + title?: string +} + +/** Runtime facts the controller needs from its owner (the background worker). */ +export interface DevToolsControllerOptions { + /** Resolve the tab the calling session is bound to. */ + resolveTarget: (sessionId?: string) => Promise + /** Debugger surface override for tests; defaults to the real `chrome.debugger`. */ + api?: import('./debugger-session.ts').DebuggerApi + /** Shared attachment owner; created from `api` when omitted. */ + broker?: DebuggerBroker + /** Output budget override. */ + maxChars?: number + /** Command timeout override (tests). */ + commandTimeoutMs?: number + /** Notified whenever attachment state changes, so the panel can reflect it. */ + onStateChange?: (state: DevToolsState) => void +} + +/** Attachment state reported to the user interfaces. */ +export interface DevToolsState { + supported: boolean + attached: boolean + tabId?: number + error?: string +} + +/** The controller surface the background dispatcher uses. */ +export interface DevToolsController { + /** + * Whether the platform can serve developer tools at all. + * + * User consent is deliberately NOT checked here: the background's approval + * gate decides (a per-call dialog, or the user's standing allowance), and a + * second hard gate would make that dialog unreachable. + */ + supported(): boolean + /** Current attachment state. */ + state(): DevToolsState + /** Handle one devtools tool call by name. */ + handle(name: string, args: Record, sessionId?: string, validate?: (target: DevToolsTarget) => Promise): Promise<{ text: string }> + /** Drop the attachment for a tab (tab closed, binding changed). */ + releaseTab(tabId: number): void + /** Detach everything and disable further commands (setting revoked). */ + revoke(): Promise + /** Detach every attachment while keeping the capability usable. */ + detachAll(): Promise + /** Allow commands again after the user re-enables the setting. */ + resume(): void + /** + * Attach to a tab ahead of any tool call so network capture starts before the + * page issues traffic. Only called once the user has enabled the allowance. + */ + prewarm(tabId: number): Promise +} + +interface TabContext { + tabId: number + session: DebuggerSession + client?: CdpClient + executionContext?: string + lease: DebuggerLease + capture: NetworkCapture + frames: FrameRegistry + detachedReason?: string + /** Per-method CDP event tally, so a silent capture can be explained. */ + cdpEvents: Record + /** CDP domains this session enabled successfully. */ + domains: string[] + /** Which `Network.*` event kinds this page emitted, and how often. */ + networkEvents: Record +} + +/** + * Browser-internal pages the debugger API refuses, by URL scheme. + * + * Detecting them up front turns an opaque `chrome.debugger` failure ("Cannot + * access a chrome:// URL") into an instruction the model and the user can act + * on: switch the controlled tab to a normal page. + */ +const UNDEBUGGABLE_SCHEME = /^(chrome|chrome-untrusted|devtools|edge|brave|about):/i + +/** + * Whether the controlled tab can be served by a debugger at all. + * @param url - the controlled tab's URL, when known. + * @returns undefined when debuggable, or the user-facing reason. + */ +export function undebuggableReason(url: string | undefined): string | undefined { + if (url === undefined || url === '') return undefined + const scheme = UNDEBUGGABLE_SCHEME.exec(url)?.[1] + if (scheme === undefined) return undefined + return `This page cannot be debugged: chrome.debugger refuses ${scheme}:// pages. ` + + 'Switch the controlled tab to a normal http(s) page and retry.' +} + +/** + * Create the developer-tools controller. + * @param options - runtime dependencies and limits. + * @returns the controller. + */ +export function createDevToolsController(options: DevToolsControllerOptions): DevToolsController { + const api = options.api ?? debuggerApi() + const maxChars = options.maxChars ?? DEFAULT_DEVTOOLS_OUTPUT_CHARS + const contexts = new Map() + const broker = options.broker ?? createDebuggerBroker({ + ...(api === undefined ? {} : { api }), + ...(options.commandTimeoutMs === undefined ? {} : { commandTimeoutMs: options.commandTimeoutMs }), + }) + let revoked = false + + const supported = (): boolean => api !== undefined && devToolsSupported() + + /** The last reported failure; attachment state itself comes from the broker. */ + let failureReason: string | undefined + + const state = (): DevToolsState => { + const shared = broker.state() + return { + supported: supported(), + attached: shared.attached, + ...(shared.tabId === undefined ? {} : { tabId: shared.tabId }), + ...(failureReason === undefined ? {} : { error: failureReason }), + } + } + + /** + * Replace the reported failure. A fresh attach attempt clears an older error + * instead of leaving it next to a switch the user has since enabled, while a + * refusal reports its own cause so the user knows what to change. + */ + const clearFailure = (reason?: string): void => { + failureReason = reason + } + + const notify = (): void => { options.onStateChange?.(state()) } + + /** Listener set this controller registers with the shared broker. */ + const listenersFor = (tabId: number): import('./debugger-session.ts').DevToolsSessionListeners => ({ + onCdpEvent: (method, params) => { handleCdpEvent(contexts.get(tabId), method, params) }, + onDetached: (reason) => { + const context = contexts.get(tabId) + if (context !== undefined) context.detachedReason = reason + notify() + }, + }) + + /** + * Capture state for a status question. The background can ask this after a + * `stop` dropped the tab context, so it falls back to the shared attachment. + */ + const statusSnapshot = (tabId?: number): { attached: boolean; tabId?: number; entries: number; generation: number } => { + const shared = broker.state() + const resolved = tabId ?? shared.tabId + const context = resolved === undefined ? undefined : contexts.get(resolved) + const capture = context?.capture.state() + return { + attached: context === undefined ? false : context.capture.state().attached && shared.attached, + ...(resolved === undefined ? {} : { tabId: resolved }), + entries: capture?.entries ?? 0, + generation: capture?.generation ?? 0, + } + } + + const contextFor = async (tabId: number): Promise => { + const existing = contexts.get(tabId) + if (existing !== undefined) { + if (!existing.session.isAttached) { + // Only a real re-attach invalidates the capture; an already attached + // session keeps accumulating without losing the model's indices. + await attach(existing, !existing.capture.state().attached) + } + return existing + } + let lease: DebuggerLease + try { + lease = await broker.acquire(tabId, 'devtools', { + domains: DEVTOOLS_DOMAINS, + listeners: listenersFor(tabId), + }) + } catch (error: unknown) { + failureReason = error instanceof Error ? error.message : String(error) + notify() + throw error + } + const context: TabContext = { + tabId, + session: lease.session, + lease, + capture: new NetworkCapture(), + frames: new FrameRegistry(), + cdpEvents: {}, + domains: [...lease.session.domains], + networkEvents: {}, + } + contexts.set(tabId, context) + try { + await afterAttach(context, true) + } catch (error: unknown) { + contexts.delete(tabId) + await lease.release({ immediate: true }) + failureReason = error instanceof Error ? error.message : String(error) + notify() + throw error + } + return context + } + + const attach = async (context: TabContext, resetCapture: boolean): Promise => { + // A new attempt supersedes whatever the previous one reported. + clearFailure() + try { + const session = await broker.reattach(context.tabId, 'devtools', DEVTOOLS_DOMAINS) + context.session = session + } catch (error: unknown) { + contexts.delete(context.tabId) + await context.lease.release({ immediate: true }) + failureReason = error instanceof Error ? error.message : String(error) + notify() + throw error + } + await afterAttach(context, resetCapture) + } + + /** Post-attach bookkeeping shared by a fresh context and a re-attach. */ + const afterAttach = async (context: TabContext, resetCapture: boolean): Promise => { + context.session.resume() + context.detachedReason = undefined + context.domains = [...context.session.domains] + if (resetCapture) { + context.capture.clear() + context.capture.start(context.tabId) + } + clearFailure() + // Frame numbering must be known before any index-based address resolves. + await syncFrames(context.session, context.frames).catch(() => undefined) + notify() + } + + const handleCdpEvent = (context: TabContext | undefined, method: string, params: unknown): void => { + if (context === undefined) return + context.cdpEvents[method] = (context.cdpEvents[method] ?? 0) + 1 + context.frames.observe(method, params) + if (method === 'Page.frameNavigated' && (params as { frame?: { parentId?: string } }).frame?.parentId === undefined) { + // A new top-level document makes every captured index stale. + context.capture.ingestNavigation() + void syncFrames(context.session, context.frames).catch(() => undefined) + } + if (method.startsWith('Network.')) { + // Which network event kinds this page actually emits. Chrome withholds + // the main request event on some pages, and a tally explains a thin + // capture without dumping payloads into normal output. + const kind = method.slice('Network.'.length) + context.networkEvents[kind] = (context.networkEvents[kind] ?? 0) + 1 + } + context.capture.ingest(method, params) + } + + const targetFor = async (sessionId?: string): Promise => { + const target = await options.resolveTarget(sessionId) + return target + } + + /** Resolve the controlled tab, or the given one when the caller knows it. */ + const resolveTarget = async (tabId?: number, sessionId?: string): Promise => { + const target = await targetFor(sessionId) + if (target === undefined) return undefined + return tabId === undefined || target.tabId === tabId ? target : { tabId } + } + + /** + * Resolve the tab context for one call, enforcing the user's allowance. + * @throws {DevToolsError} when unsupported, disabled, or the tab is unknown. + */ + const requireContext = async (sessionId?: string, allowAttach = true, validate?: (target: DevToolsTarget) => Promise): Promise<{ context?: TabContext; target?: DevToolsTarget }> => { + if (allowAttach) return requireAttachedContext(sessionId, validate) + if (!supported() || revoked) return {} + const target = await targetFor(sessionId) + const context = target === undefined ? undefined : contexts.get(target.tabId) + return { ...(context === undefined ? {} : { context }), ...(target === undefined ? {} : { target }) } + } + + const requireAttachedContext = async (sessionId?: string, validate?: (target: DevToolsTarget) => Promise): Promise<{ context: TabContext; target: DevToolsTarget }> => { + if (!supported()) { + throw new DevToolsError( + 'devtools-unavailable', + 'This browser build cannot provide developer tools (the debugger API is unavailable).', + ) + } + if (revoked) { + throw new DevToolsError('devtools-unavailable', 'The developer-tools session was revoked.') + } + const target = await targetFor(sessionId) + if (target === undefined) { + throw new DevToolsError('devtools-unavailable', 'No controlled tab is available for developer tools.') + } + await validate?.(target) + const blocked = undebuggableReason(target.url) + if (blocked !== undefined) { + clearFailure(blocked) + notify() + throw new DevToolsError('devtools-unavailable', blocked) + } + const context = await contextFor(target.tabId) + return { context, target } + } + + const finish = (text: string): { text: string } => ({ text: truncateText(text, maxChars) }) + + return { + supported, + state, + releaseTab: (tabId) => { + const context = contexts.get(tabId) + if (context === undefined) return + contexts.delete(tabId) + void broker.detachTab(tabId) + notify() + }, + revoke: async () => { + revoked = true + contexts.clear() + failureReason = undefined + await broker.detachAll() + notify() + }, + detachAll: async () => { + contexts.clear() + failureReason = undefined + await broker.detachAll() + notify() + }, + resume: () => { + revoked = false + notify() + }, + prewarm: async (tabId) => { + if (!supported() || revoked) { + throw new DevToolsError('devtools-unavailable', 'This browser build cannot provide developer tools.') + } + const target = await resolveTarget(tabId) + if (undebuggableReason(target?.url) !== undefined) { + // Enabling the switch is not itself a failed operation: an internal + // page simply has nothing to prewarm, and the first real call reports + // the reason if the user still has not moved to a debuggable tab. + clearFailure() + return + } + const context = await contextFor(tabId) + context.capture.start(tabId) + notify() + }, + handle: async (name, args, sessionId, validate) => { + // `status` must answer without resolving or attaching to a tab, so the + // panel can show capture state even when the debugger is detached. + const readOnly = name === 'browser_devtools_network' && args.action === 'status' + try { + const { context, target } = await requireContext(sessionId, !readOnly, validate) + if (target !== undefined) await validate?.(target) + if (context === undefined || target === undefined) { + // A status question is answerable from the shared state alone. + if (readOnly) return finish(renderCaptureState(statusSnapshot(target?.tabId), 'status')) + throw new DevToolsError('devtools-unavailable', 'No controlled tab is available for developer tools.') + } + const frameId = args.frame ?? 0 + if (typeof frameId !== 'number' || !Number.isInteger(frameId) || frameId < 0) { + throw new Error('frame must be a non-negative integer from browser_snapshot.') + } + if (frameId !== 0) { + const frames = await listTabFrames(target.tabId, target.url) + const frame = frames.find(candidate => candidate.frameId === frameId) + if (frame === undefined) throw new Error(`Frame ${frameId} does not exist in the current page.`) + const resolved = await resolveDebuggerFrame(context.session, target.tabId, frame) + context.frames.bindFrame(frameId, resolved.frameId) + await resolved.client.send('DOM.enable') + await resolved.client.send('CSS.enable') + await validate?.(target) + return finish(await dispatch({ ...context, client: resolved.client, executionContext: resolved.uniqueId }, target, name, { ...args, frame: 0 })) + } + return finish(await dispatch(context, target, name, args)) + } catch (error: unknown) { + throw toToolError(error) + } + }, + } + + /** Route one devtools operation. */ + async function dispatch( + context: TabContext, + target: DevToolsTarget, + name: string, + args: Record, + ): Promise { + switch (name) { + case 'browser_devtools_elements': + return elements(context, args) + case 'browser_devtools_set_element_style': + return setStyle(context, args) + case 'browser_devtools_set_element_attribute': + return setElementAttribute(context, args) + case 'browser_console_eval': + return consoleEval(context, args) + case 'browser_console_logs': + return consoleLogs(context, args) + case 'browser_devtools_network': + return networkControl(context, target, args) + case 'browser_devtools_list_requests': + return listRequests(context, args) + case 'browser_devtools_get_request': + return getRequest(context, args) + case 'browser_devtools_request_body': + return requestBody(context, args) + default: + throw new Error(`Unknown developer-tools operation ${name}.`) + } + } + + // ---- Elements ---- + + async function elements(context: TabContext, args: Record): Promise { + const client = context.client ?? context.session + const nodeId = await resolveNodeId(client, locatorOf(args), context.frames, context.executionContext) + const properties = stringList(args.properties) + const view = await inspectElement(client, nodeId, properties) + return renderElement(view) + } + + async function setStyle(context: TabContext, args: Record): Promise { + const client = context.client ?? context.session + const declarations = declarationList(args.properties) + const nodeId = await resolveNodeId(client, locatorOf(args), context.frames, context.executionContext) + const node = await client.send('DOM.describeNode', { nodeId, depth: 0 }) as { node?: { localName?: string; nodeName?: string; attributes?: string[] } } + const identity = shortIdentity(node.node ?? {}, pairsOf(node.node?.attributes)) + const mode = args.mode === 'rule' ? 'rule' : 'inline' + if (mode === 'rule') { + const edited = await setMatchedRuleStyle(client, nodeId, declarations) + return renderStyleEdit(identity, 'rule', edited.selector, edited.declarations) + } + const applied = await setInlineStyle(client, nodeId, declarations) + return renderStyleEdit(identity, 'inline', 'style attribute', applied) + } + + async function setElementAttribute(context: TabContext, args: Record): Promise { + const client = context.client ?? context.session + const name = typeof args.name === 'string' ? args.name.trim() : '' + if (name === '') throw new Error('name is required.') + const risk = attributeRisk(name) + if (risk === 'forbidden') throw new Error(`The attribute name "${name}" cannot be written through developer tools.`) + const remove = args.remove === true + const value = typeof args.value === 'string' ? args.value : undefined + if (risk === 'script' && !remove) { + throw new Error( + `Refusing to write the script-capable attribute "${name}". ` + + 'Inline handler and resource URL attributes change what the page executes; use browser_console_eval for that.', + ) + } + const nodeId = await resolveNodeId(client, locatorOf(args), context.frames, context.executionContext) + const before = await attributeValue(client, nodeId, name) + if (!remove && value === undefined) { + const node = await client.send('DOM.describeNode', { nodeId, depth: 0 }) as { node?: { localName?: string; nodeName?: string; attributes?: string[] } } + const identity = shortIdentity(node.node ?? {}, pairsOf(node.node?.attributes)) + return renderAttributeResult(identity, name, before, undefined, 'read') + } + await setAttribute(client, nodeId, name, value, remove) + const after = remove ? undefined : value + const node = await client.send('DOM.describeNode', { nodeId, depth: 0 }) as { node?: { localName?: string; nodeName?: string; attributes?: string[] } } + const identity = shortIdentity(node.node ?? {}, pairsOf(node.node?.attributes)) + return renderAttributeResult(identity, name, before, after, remove ? 'removed' : 'set') + } + + // ---- Console ---- + + async function consoleEval(context: TabContext, args: Record): Promise { + const expression = typeof args.expression === 'string' ? args.expression : '' + if (expression.trim() === '') throw new Error('expression is required.') + const frame = numberArg(args.frame) ?? 0 + const contextId = await frameContextId(context.session, frame, context.frames) + const budget = numberArg(args.maxChars) ?? Math.min(maxChars, 8_000) + let response: unknown + try { + response = await (context.client ?? context.session).send('Runtime.evaluate', { + expression, + ...(context.executionContext === undefined ? {} : { uniqueContextId: context.executionContext }), + ...(contextId === undefined ? {} : { contextId }), + awaitPromise: args.awaitPromise === true, + // CDP hands back an opaque "object Object" for plain objects unless the + // value is serialized; JSON-compatible results are what the model wants. + // Classes/functions/DOM nodes still fall back to description + preview. + returnByValue: true, + allowUnsafeEvalBlockedByCSP: false, + userGesture: true, + includeCommandLineAPI: true, + generatePreview: args.includePreview === true, + objectGroup: 'dsh-devtools', + }) + } catch (error: unknown) { + const detail = error instanceof Error ? error.message : String(error) + return `Evaluation failed:\n${detail}` + } + const formatted = formatEvaluation(response, { + maxChars: Math.max(200, budget), + includePreview: args.includePreview === true, + }) + return formatted.ok ? `Evaluation result:\n${formatted.text}` : formatted.text + } + + function consoleLogs(context: TabContext, args: Record): string { + const limit = numberArg(args.limit) ?? 50 + const sinceIndex = numberArg(args.sinceIndex) + const level = typeof args.level === 'string' ? args.level : undefined + const entries = context.capture.consoleLog() + // Entry indices start at 1, so "after 0" must not mean "after nothing": + // the window filter applies only when the caller actually asks for one. + .filter((entry) => sinceIndex === undefined || entry.index > sinceIndex) + .filter((entry) => level === undefined || entry.level === level || (level === 'error' && entry.level === 'exception')) + .slice(-limit) + return renderConsoleLogs(entries, sinceIndex ?? 0) + } + + // ---- Network ---- + + async function networkControl(context: TabContext, target: DevToolsTarget, args: Record): Promise { + const action = typeof args.action === 'string' ? args.action : 'status' + if (action === 'stop') { + context.capture.stop() + contexts.delete(context.tabId) + // Release only this feature's hold: the shared attachment stays alive for + // a trusted-input owner and is dropped when nobody needs it. + await context.lease.release({ immediate: true }) + notify() + return renderCaptureState({ ...context.capture.state(), attached: false, tabId: target.tabId }, 'stop') + } + if (action === 'start') { + if (!context.session.isAttached) await attach(context, true) + else if (context.capture.state().attached !== true) context.capture.start(context.tabId) + notify() + return renderCaptureState(context.capture.state(), 'start') + } + const state = context.capture.state() + return `${renderCaptureState(state, 'status')}\n${pipelineEvidence(context)}` + } + + function listRequests(context: TabContext, args: Record): string { + const limit = numberArg(args.limit) ?? 50 + const sinceIndex = numberArg(args.sinceIndex) + const urlPattern = matcher(args.urlPattern) + const method = typeof args.method === 'string' && args.method !== '' ? args.method.toUpperCase() : undefined + const resourceType = typeof args.resourceType === 'string' && args.resourceType !== '' ? args.resourceType.toLowerCase() : undefined + const statusMin = numberArg(args.statusMin) + const statusMax = numberArg(args.statusMax) + const entries = context.capture.list().filter((entry) => { + // Request indices begin at 1, so the window filter must be opt-in. + if (sinceIndex !== undefined && entry.index <= sinceIndex) return false + if (urlPattern !== undefined && !urlPattern(entry.url)) return false + if (method !== undefined && entry.method.toUpperCase() !== method) return false + if (resourceType !== undefined && (entry.resourceType ?? '').toLowerCase() !== resourceType) return false + if (statusMin !== undefined && (entry.status ?? 0) < statusMin) return false + if (statusMax !== undefined && (entry.status ?? 0) > statusMax) return false + if (args.failedOnly === true && entry.failed === undefined) return false + return true + }) + return renderRequestList(entries, context.capture.state().generation, limit) + } + + async function getRequest(context: TabContext, args: Record): Promise { + const entry = requireEntry(context, args) + const includeSensitiveHeaders = args.includeSensitiveHeaders === true + const includeSensitiveBody = args.includeSensitiveBody === true + let body: string | undefined + let truncated = false + if (args.includeBody === true) { + const fetched = await fetchBody(context, entry, 'response') + body = fetched.text + truncated = fetched.truncated + } + const detail = renderRequestDetail(entry, { includeSensitiveHeaders, includeSensitiveBody, ...(body === undefined ? {} : { body }), bodyTruncated: truncated }) + return truncateText(detail, numberArg(args.maxChars) ?? maxChars) + } + + async function requestBody(context: TabContext, args: Record): Promise { + const entry = requireEntry(context, args) + const kind = args.kind === 'request' ? 'request' : 'response' + const includeSensitiveBody = args.includeSensitiveBody === true + if (!includeSensitiveBody && isSensitiveBodyUrl(entry.url)) { + return renderBody(entry, kind, '', { truncated: false, json: false, redacted: true }) + } + const fetched = await fetchBody(context, entry, kind) + const limited = truncateText(fetched.text, numberArg(args.maxChars) ?? maxChars) + return renderBody(entry, kind, limited, { truncated: fetched.truncated, json: fetched.json, redacted: false }) + + (fetched.redactedHeaders === true ? `\n\n(Headers containing credentials were ${REDACTED} in the request detail view.)` : '') + } + + function requireEntry(context: TabContext, args: Record): NetworkEntry { + const index = numberArg(args.index) + if (index === undefined) throw new Error('index is required and must be a captured request index.') + const entry = context.capture.get(index) + if (entry === undefined) { + const state = context.capture.state() + throw new Error( + `No captured request has index ${index} in capture generation ${state.generation} (${state.entries} entries). ` + + 'Call browser_devtools_list_requests again for current indices.', + ) + } + return entry + } + + async function fetchBody( + context: TabContext, + entry: NetworkEntry, + kind: 'request' | 'response', + ): Promise<{ text: string; truncated: boolean; json: boolean; redactedHeaders?: boolean }> { + const command = kind === 'request' ? 'Network.getRequestPostData' : 'Network.getResponseBody' + let raw: { body?: string; postData?: string; base64Encoded?: boolean } + try { + raw = (await context.session.send(command, { requestId: entry.requestId })) as typeof raw + } catch (error: unknown) { + const detail = error instanceof Error ? error.message : String(error) + throw new Error( + kind === 'request' + ? `This request has no captured payload (${detail}).` + : `The response body is no longer available (${detail}). Chrome only keeps bodies for requests it buffered while the debugger was attached.`, + ) + } + const text = raw.body ?? raw.postData ?? '' + if (text === '') throw new Error(kind === 'request' ? 'This request has no payload.' : 'This response has an empty body.') + const decoded = raw.base64Encoded === true ? `[base64] ${text}` : text + const sliced = decoded.slice(0, BODY_CHAR_CAP) + const formatted = formatBody(sliced) + return { text: formatted.text, truncated: decoded.length > BODY_CHAR_CAP, json: formatted.json } + } + + /** One line of pipeline evidence attached to every capture status answer. */ + function pipelineEvidence(context: TabContext | undefined): string { + if (context === undefined) return 'Pipeline: no session has been created for this tab yet.' + const methods = Object.entries(context.cdpEvents) + .sort(([, left], [, right]) => right - left) + .slice(0, 6) + .map(([method, count]) => `${method}×${count}`) + const kinds = Object.entries(context.networkEvents) + .sort(([, left], [, right]) => right - left) + .map(([kind, count]) => `${kind}×${count}`) + return `Pipeline: attached=${context.session.isAttached} · domains=${context.domains.join('+') || 'none'}` + + ` · cdp events=${methods.length === 0 ? 'none received' : methods.join(', ')}` + + ` · network events=${kinds.length === 0 ? 'none' : kinds.join(', ')}` + } + + /** Turn any controller failure into the tool-facing error text. */ + function toToolError(error: unknown): Error { + if (error instanceof DevToolsError) return new Error(error.message) + if (error instanceof Error) return error + return new Error(String(error)) + } +} + +/** Extract the element locator from raw tool arguments. */ +function locatorOf(args: Record): { selector?: string; index?: number; frame?: number } { + const selector = typeof args.selector === 'string' && args.selector.trim() !== '' ? args.selector : undefined + const index = numberArg(args.index) + const frame = numberArg(args.frame) + return { + ...(selector === undefined ? {} : { selector }), + ...(index === undefined ? {} : { index }), + ...(frame === undefined ? {} : { frame }), + } +} + +/** Attribute pairs from a `DOM.describeNode` payload. */ +function pairsOf(attributes: string[] | undefined): { name: string; value: string }[] { + const pairs: { name: string; value: string }[] = [] + const list = attributes ?? [] + for (let index = 0; index + 1 < list.length; index += 2) { + pairs.push({ name: list[index]!, value: list[index + 1]! }) + } + return pairs +} + +function numberArg(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined +} + +/** Serialize the tool's declaration list, rejecting silently wrong shapes. */ +function declarationList(value: unknown): { name: string; value: string; important?: boolean }[] { + if (!Array.isArray(value) || value.length === 0) { + throw new Error('properties must be a non-empty array of { name, value } declarations.') + } + return value.map((entry) => { + const record = entry as { name?: unknown; value?: unknown; important?: unknown } + if (typeof record?.name !== 'string' || record.name.trim() === '' || typeof record.value !== 'string') { + throw new Error('Each property needs a CSS name and a string value.') + } + return { + name: record.name.trim(), + value: record.value, + ...(record.important === true ? { important: true } : {}), + } + }) +} + +/** A string list argument, ignoring non-strings. */ +function stringList(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined + const list = value.filter((entry): entry is string => typeof entry === 'string' && entry.trim() !== '') + return list.length === 0 ? undefined : list +} + +/** Build a URL matcher from a substring or `/regex/` pattern. */ +function matcher(pattern: unknown): ((url: string) => boolean) | undefined { + if (typeof pattern !== 'string' || pattern === '') return undefined + const slash = /^\/(.*)\/([a-z]*)$/i.exec(pattern) + if (slash !== null) { + try { + const regex = new RegExp(slash[1]!, slash[2]!.includes('i') ? 'i' : '') + return (url) => regex.test(url) + } catch { + // An invalid regex is just a substring. + } + } + return (url) => url.includes(pattern) +} + +/** Serialize one capture state for the panel. */ +export function captureStatusText(capture: NetworkCapture): string { + const state = capture.state() + return `Network capture ${state.attached ? 'active' : 'inactive'} · ${state.entries} entries · generation ${state.generation}${state.tabId === undefined ? '' : ` · tab ${state.tabId}`}` +} diff --git a/extensions/dsh-browser/src/background/devtools-elements.ts b/extensions/dsh-browser/src/background/devtools-elements.ts new file mode 100644 index 000000000..9810c810d --- /dev/null +++ b/extensions/dsh-browser/src/background/devtools-elements.ts @@ -0,0 +1,705 @@ +/** + * Element and CSS inspection plus live debugging edits, driven over CDP. + * + * Every helper resolves the target element the same way: a CSS selector through + * `DOM.querySelector` on the document root, or a `browser_snapshot` inventory + * index through the `data-dsh-el` attribute the content script writes (evaluated + * in the frame's own JavaScript context so iframe inventory numbers resolve too). + * + * The resolution layer is a small port so the presentation logic can be tested + * against a scripted CDP client without Chrome. + * + * @module + */ + +import type { CdpClient } from './debugger-session.ts' +import { resolveFrameNode } from './debugger-frame.ts' + +/** Computed-style properties returned when the caller names none. */ +export const DEFAULT_COMPUTED_PROPERTIES = [ + 'display', + 'position', + 'width', + 'height', + 'margin', + 'padding', + 'color', + 'background-color', + 'font-family', + 'font-size', + 'font-weight', + 'line-height', + 'border', + 'flex-direction', + 'gap', + 'overflow', + 'opacity', + 'z-index', + 'transform', +] as const + +/** Attribute names the model must never write through the debugger. */ +const FORBIDDEN_ATTRIBUTES = new Set(['srcdoc']) + +/** Attribute names that carry inline event handlers or resource URLs. */ +const SCRIPT_ATTRIBUTES = /^on[a-z]|^(src|href|xlink:href)$/i + +/** A locator: exactly one of selector/index is used. */ +export interface ElementLocator { + selector?: string + index?: number + frame?: number +} + +/** One attribute name/value pair. */ +export interface AttributeView { + name: string + value: string +} + +/** One matched CSS rule rendered for the model. */ +export interface MatchedRuleView { + selector: string + origin: string + declarations: string[] + media?: string +} + +/** The element inspector's full report. */ +export interface ElementView { + identity: string + domPath: string + inventoryIndex?: number + box?: { x: number; y: number; width: number; height: number } + boxModel?: { content: string; padding: string; border: string; margin: string } + computed: Record + inlineStyle: string[] + matchedRules: MatchedRuleView[] + attributes: AttributeView[] + childNodeCount: number +} + +/** CDP frame tree, explicit Chrome frame bindings, and default contexts. */ +export interface DebuggerFrames { + /** CDP traversal order, independent of Chrome frame IDs. */ + order(): readonly string[] + frameId(chromeFrameId: number): string | undefined + /** The page's default execution context for a frame, when one was observed. */ + defaultContext(frameId: string): number | undefined + /** Record one `Page.frameNavigated` / `Runtime.executionContextCreated` observation. */ + observe(method: string, params: unknown): void +} + +/** A `Runtime.executionContextCreated` payload. */ +interface ExecutionContextPayload { + context?: { id?: number; origin?: string; name?: string; auxData?: { frameId?: string; isDefault?: boolean } } +} + +/** A `Page.frameNavigated` payload. */ +interface FrameNavigatedPayload { + frame?: { id?: string; parentId?: string; url?: string } +} + +/** + * Track the frame tree and each frame's default execution context from the CDP + * event stream. `Page.getFrameTree` remains the source of truth for a freshly + * attached session; this registry covers the common case without an extra + * command and always picks the *page's own* context when one is known. + */ +export class FrameRegistry implements DebuggerFrames { + private frames: string[] = [] + private contexts = new Map() + private bindings = new Map() + + bindFrame(chromeFrameId: number, cdpFrameId: string): void { + this.bindings.set(chromeFrameId, cdpFrameId) + } + + frameId(chromeFrameId: number): string | undefined { + return this.bindings.get(chromeFrameId) + } + + /** @returns frames in traversal order (main frame first). */ + order(): readonly string[] { + return this.frames + } + + /** @returns the default execution context of a frame, when observed. */ + defaultContext(frameId: string): number | undefined { + return this.contexts.get(frameId) + } + + /** Replace the frame order from an authoritative `Page.getFrameTree` read. */ + setTree(ids: readonly string[]): void { + this.frames = [...ids] + } + + /** + * Fold one CDP event into the registry. + * @param method - CDP event name. + * @param params - CDP event payload. + */ + observe(method: string, params: unknown): void { + switch (method) { + case 'Runtime.executionContextCreated': { + const context = (params as ExecutionContextPayload).context + const frameId = context?.auxData?.frameId + if (frameId === undefined || context?.id === undefined) return + // Page contexts report `isDefault`; extension/isolated worlds do not. + if (context.auxData?.isDefault !== true) return + this.contexts.set(frameId, context.id) + break + } + case 'Runtime.executionContextDestroyed': { + const id = (params as { executionContextId?: number }).executionContextId + if (id === undefined) return + for (const [frameId, contextId] of this.contexts) { + if (contextId === id) this.contexts.delete(frameId) + } + break + } + case 'Runtime.executionContextsCleared': + this.contexts.clear() + this.bindings.clear() + break + case 'Page.frameNavigated': { + const frame = (params as FrameNavigatedPayload).frame + if (frame?.id === undefined) return + if (frame.parentId === undefined) { + // A new top-level document invalidates every previous frame and index. + this.frames = [frame.id] + this.contexts.clear() + this.bindings.clear() + break + } + if (!this.frames.includes(frame.id)) this.frames.push(frame.id) + break + } + case 'Page.frameDetached': { + const frameId = (params as { frameId?: string }).frameId + if (frameId === undefined) return + this.frames = this.frames.filter((id) => id !== frameId) + this.contexts.delete(frameId) + for (const [chromeId, cdpId] of this.bindings) if (cdpId === frameId) this.bindings.delete(chromeId) + break + } + default: + break + } + } +} + +/** + * Resolve one locator to a CDP node id. + * @param client - the CDP session. + * @param locator - selector or inventory index, plus an optional frame. + * @param frames - frame registry used for index lookups outside the top frame. + * @returns the DOM node id. + * @throws when no element matches or the locator is incomplete. + */ +export async function resolveNodeId( + client: CdpClient, + locator: ElementLocator, + frames?: DebuggerFrames, + uniqueContextId?: string, +): Promise { + if (uniqueContextId !== undefined) { + const selector = locator.selector ?? (typeof locator.index === 'number' && Number.isInteger(locator.index) && locator.index > 0 + ? `[data-dsh-el="${locator.index}"]` : undefined) + if (selector === undefined) throw new Error('Provide exactly one of selector or index to identify the element.') + return resolveFrameNode(client, selector, uniqueContextId) + } + if (typeof locator.selector === 'string' && locator.selector.trim() !== '') { + const root = (await client.send('DOM.getDocument', { depth: 0 })) as { root?: { nodeId?: number } } + const documentNodeId = root.root?.nodeId + if (documentNodeId === undefined) throw new Error('The page document is unavailable.') + const found = (await client.send('DOM.querySelector', { nodeId: documentNodeId, selector: locator.selector })) as { nodeId?: number } + if (found.nodeId === undefined || found.nodeId === 0) { + throw new Error(`No element matches the selector ${locator.selector}.`) + } + return found.nodeId + } + if (typeof locator.index === 'number' && Number.isInteger(locator.index) && locator.index > 0) { + return resolveInventoryIndex(client, locator.index, locator.frame ?? 0, frames) + } + throw new Error('Provide exactly one of selector or index to identify the element.') +} + +/** + * Resolve a `browser_snapshot` inventory index through the `data-dsh-el` + * attribute the content script writes on every inventoried element. + */ +async function resolveInventoryIndex( + client: CdpClient, + index: number, + frame: number, + frames: DebuggerFrames | undefined, +): Promise { + const contextId = await frameContextId(client, frame, frames) + const evaluated = (await client.send('Runtime.evaluate', { + expression: `document.querySelector('[data-dsh-el="${index}"]')`, + ...contextId === undefined ? {} : { contextId }, + returnByValue: false, + objectGroup: 'dsh-devtools', + includeCommandLineAPI: false, + })) as { result?: { objectId?: string; subtype?: string } } + const objectId = evaluated.result?.objectId + if (objectId === undefined || evaluated.result?.subtype === 'null') { + throw new Error(`Inventory index ${index} no longer matches an element${frame === 0 ? '' : ` in frame ${frame}`}. Call browser_snapshot again for fresh indices.`) + } + try { + const node = (await client.send('DOM.requestNode', { objectId })) as { nodeId?: number } + if (node.nodeId === undefined) throw new Error(`Inventory index ${index} could not be resolved to a DOM node.`) + return node.nodeId + } finally { + // The remote object is only an intermediate step towards the node id. + void client.send('Runtime.releaseObject', { objectId }).catch(() => undefined) + } +} + +/** Locate the JavaScript context of one frame; the main frame needs none. */ +export async function frameContextId( + _client: CdpClient, + frame: number, + frames: DebuggerFrames | undefined, +): Promise { + if (frame === 0) return undefined + const frameId = frames?.frameId(frame) + if (frameId === undefined) { + throw new Error(`Frame ${frame} does not exist in the current page.`) + } + const context = frames?.defaultContext(frameId) + if (context !== undefined) return context + throw new Error(`Frame ${frame} has no usable page JavaScript context.`) +} + +/** Frame id of one snapshot frame number, or undefined for the main frame. */ +export function frameIdAt(frames: DebuggerFrames | undefined, frame: number): string | undefined { + return frames?.frameId(frame) +} + +/** Refresh the frame registry from `Page.getFrameTree` (authoritative at attach). */ +export async function syncFrames(client: CdpClient, frames: FrameRegistry): Promise { + const tree = (await client.send('Page.getFrameTree', {})) as { frameTree?: FrameTreeNode } + frames.setTree(flattenFrameTree(tree.frameTree).map((frame) => frame.id ?? '').filter((id) => id !== '')) +} + +interface FrameTreeNode { + frame?: { id?: string; url?: string } + childFrames?: FrameTreeNode[] +} + +/** Depth-first frame order: main frame first, then children in document order. */ +function flattenFrameTree(node: FrameTreeNode | undefined): { id?: string; url?: string }[] { + if (node?.frame === undefined) return [] + const frames = [node.frame] + for (const child of node.childFrames ?? []) frames.push(...flattenFrameTree(child)) + return frames +} + +/** Fetch everything the element inspector reports for one node. */ +export async function inspectElement( + client: CdpClient, + nodeId: number, + properties?: readonly string[], +): Promise { + const described = (await client.send('DOM.describeNode', { nodeId, depth: 0, pierce: false })) as { node?: DescribedNode } + const node = described.node ?? {} + const attributes = attributePairs(node.attributes) + const computedResponse = (await client.send('CSS.getComputedStyleForNode', { nodeId })) as { + computedStyle?: { name: string; value: string }[] + } + const computedStyles = computedResponse.computedStyle ?? [] + const requested = properties === undefined || properties.length === 0 + ? [...DEFAULT_COMPUTED_PROPERTIES] + : properties + const computed: Record = {} + for (const name of requested) { + // A duplicate request must not add a second line to the report. + if (name in computed) continue + const found = computedStyles.find((entry) => entry.name.toLowerCase() === name.toLowerCase()) + computed[name] = found === undefined ? '(not set)' : found.value + } + const matched = await matchedRules(client, nodeId) + const boxModel = await boxModelOf(client, nodeId) + const inventory = attributes.find((attribute) => attribute.name === 'data-dsh-el') + const inventoryIndex = inventory === undefined ? undefined : Number(inventory.value) + return { + identity: shortIdentity(node, attributes), + domPath: await describePath(client, nodeId, described), + ...(inventoryIndex === undefined || !Number.isInteger(inventoryIndex) ? {} : { inventoryIndex }), + ...(boxModel === undefined ? {} : { box: boxModel.box, boxModel: boxModel.boxes }), + computed, + inlineStyle: matched.inline, + matchedRules: matched.rules, + attributes, + childNodeCount: node.childNodeCount ?? 0, + } +} + +/** The `DOM.describeNode` subset this module reads. */ +interface DescribedNode { + nodeName?: string + localName?: string + attributes?: string[] + childNodeCount?: number + parentId?: number +} + +/** Read one node description. */ +async function describeNode(client: CdpClient, nodeId: number): Promise { + const described = (await client.send('DOM.describeNode', { nodeId, depth: 0, pierce: false })) as { node?: DescribedNode } + return described.node ?? {} +} + +/** Flatten `CSS.getMatchedStylesForNode` into inline declarations and rules. */ +export async function matchedRules( + client: CdpClient, + nodeId: number, +): Promise<{ inline: string[]; rules: MatchedRuleView[] }> { + const response = (await client.send('CSS.getMatchedStylesForNode', { nodeId })) as { + inlineStyle?: StylePayload + matchedCSSRules?: MatchedRulePayload[] + inherited?: { matchedCSSRules?: MatchedRulePayload[] }[] + } + const rules: MatchedRuleView[] = (response.matchedCSSRules ?? []).map((entry) => toRuleView(entry, 'author')) + for (const inherited of response.inherited ?? []) { + for (const entry of inherited.matchedCSSRules ?? []) rules.push(toRuleView(entry, 'inherited')) + } + return { inline: declarationsOf(response.inlineStyle), rules } +} + +interface StylePayload { + cssProperties?: { name: string; value: string; important?: boolean; disabled?: boolean }[] + cssText?: string + /** Editable sheet id; absent for inline styles Chrome does not expose as a sheet. */ + styleSheetId?: string + /** Token range of the declaration block inside its stylesheet. */ + range?: RangePayload +} + +interface MatchedRulePayload { + rule?: { + selectorList?: { text?: string } + origin?: string + media?: { text?: string }[] + styleSheetId?: string + style?: StylePayload + } + matchingSelectors?: number[] +} + +function toRuleView(entry: MatchedRulePayload, origin: string): MatchedRuleView { + const rule = entry.rule ?? {} + const matching = entry.matchingSelectors ?? [] + const selector = matching.length === 0 + ? rule.selectorList?.text ?? '(unknown selector)' + : matching + .map((position) => rule.selectorList?.text?.split(',')[position]?.trim() ?? '') + .filter((part) => part !== '') + .join(', ') + const media = (rule.media ?? []).map((query) => query.text ?? '').filter((text) => text !== '').join(' and ') + return { + selector: selector === '' ? rule.selectorList?.text ?? '(unknown selector)' : selector, + origin: rule.origin ?? origin, + declarations: declarationsOf(rule.style), + ...(media === '' ? {} : { media }), + } +} + +/** Every longhand a given shorthand can expand into. */ +const LONGHANDS_OF: Record = { + margin: ['margin-top', 'margin-right', 'margin-bottom', 'margin-left'], + padding: ['padding-top', 'padding-right', 'padding-bottom', 'padding-left'], + gap: ['row-gap', 'column-gap'], + overflow: ['overflow-x', 'overflow-y'], + background: ['background-color', 'background-image', 'background-position', 'background-size', 'background-repeat', 'background-attachment'], + border: ['border-width', 'border-style', 'border-color'], + inset: ['top', 'right', 'bottom', 'left'], +} + +/** Which shorthand family a property belongs to; longhands belong to their own. */ +export function propertyFamily(name: string): string { + const normalized = name.trim().toLowerCase() + for (const [shorthand, longhands] of Object.entries(LONGHANDS_OF)) { + if (normalized === shorthand || longhands.includes(normalized)) return shorthand + } + return normalized +} + +/** + * Canonicalize one declaration block. + * + * CDP reports both a shorthand and every longhand it expands into, which is + * most of a style report's bulk. A longhand stays only when nothing in the same + * block covers it, or when the caller explicitly edited it (dropping an edit + * would silently discard what the user asked for). + * + * @param declarations - `name: value` lines in cascade order. + * @param edited - lower-cased property names the caller supplied. + * @returns the reduced lines, in their original order. + */ +export function canonicalizeDeclarations( + declarations: readonly string[], + edited: ReadonlySet = new Set(), +): string[] { + const present = new Set() + for (const declaration of declarations) { + const separator = declaration.indexOf(':') + if (separator < 0) continue + present.add(declaration.slice(0, separator).trim().toLowerCase()) + } + const covered = new Set() + for (const [shorthand, longhands] of Object.entries(LONGHANDS_OF)) { + if (!present.has(shorthand)) continue + for (const longhand of longhands) { + if (!edited.has(longhand)) covered.add(longhand) + } + } + return declarations.filter((declaration) => { + const separator = declaration.indexOf(':') + if (separator < 0) return true + return !covered.has(declaration.slice(0, separator).trim().toLowerCase()) + }) +} + +/** + * Render one CDP style payload as `name: value` lines. + * + * Custom properties are dropped: they are theme plumbing, not an answer about + * this element, and a single page can put hundreds of them on `body`. + */ +export function declarationsOf(style: StylePayload | undefined): string[] { + const seen = new Set() + const declarations: string[] = [] + for (const property of style?.cssProperties ?? []) { + if (property.disabled === true) continue + if (property.name.startsWith('--')) continue + const name = property.name.trim().toLowerCase() + if (name === '' || seen.has(name)) continue + seen.add(name) + declarations.push(`${property.name}: ${property.value}${property.important === true ? ' !important' : ''}`) + } + return canonicalizeDeclarations(declarations) +} + +/** Box model in compact numeric form plus per-edge lists. */ +async function boxModelOf( + client: CdpClient, + nodeId: number, +): Promise<{ box: { x: number; y: number; width: number; height: number }; boxes: { content: string; padding: string; border: string; margin: string } } | undefined> { + let model: { content?: number[]; padding?: number[]; border?: number[]; margin?: number[] } | undefined + try { + const response = (await client.send('DOM.getBoxModel', { nodeId })) as { model?: typeof model } + model = response.model + } catch { + // Elements without a layout box (display:none, detached) report no model. + return undefined + } + const content = model?.content + if (content === undefined || content.length < 8) return undefined + const left = Math.min(content[0]!, content[6]!) + const top = Math.min(content[1]!, content[3]!) + const right = Math.max(content[2]!, content[4]!) + const bottom = Math.max(content[5]!, content[7]!) + return { + box: { x: round(left), y: round(top), width: round(right - left), height: round(bottom - top) }, + boxes: { + content: `${round(right - left)}×${round(bottom - top)}`, + padding: quadToEdges(model?.padding), + border: quadToEdges(model?.border), + margin: quadToEdges(model?.margin), + }, + } +} + +/** Turn one CDP quad into `top right bottom left` pixel values. */ +function quadToEdges(quad: number[] | undefined): string { + if (quad === undefined || quad.length < 8) return '0px 0px 0px 0px' + const [x1, y1, x2, , , y3, x4, y4] = quad as [number, number, number, number, number, number, number, number] + return `${round(Math.abs(y1 - y3))}px ${round(Math.abs(x2 - x1))}px ${round(Math.abs(y4 - y1))}px ${round(Math.abs(x1 - x4))}px` +} + +function round(value: number): number { + return Math.round(value * 10) / 10 +} + +/** A short element identity, preferring id then the first class. */ +export function shortIdentity(node: { nodeName?: string; localName?: string }, attributes: AttributeView[]): string { + const tag = (node.localName ?? node.nodeName ?? 'node').toLowerCase() + const id = attributes.find((attribute) => attribute.name === 'id')?.value + if (id !== undefined && id !== '') return `${tag}#${id}` + const firstClass = attributes.find((attribute) => attribute.name === 'class')?.value?.split(/\s+/).find((entry) => entry !== '') + return firstClass === undefined ? tag : `${tag}.${firstClass}` +} + +/** `DOM.describeNode` reports attributes as a flat name/value list. */ +export function attributePairs(attributes: string[] | undefined): AttributeView[] { + const pairs: AttributeView[] = [] + const list = attributes ?? [] + for (let index = 0; index + 1 < list.length; index += 2) { + pairs.push({ name: list[index]!, value: list[index + 1]! }) + } + return pairs +} + +/** + * Build a readable ancestor path for one element. + * + * The already-described target is reused, and the walk stops at the first + * unordered or unknown ancestor so a deep DOM cannot turn one inspection into a + * long chain of commands. + */ +async function describePath( + client: CdpClient, + nodeId: number, + target?: { node?: DescribedNode }, +): Promise { + const origin = target?.node ?? await describeNode(client, nodeId) + const segments: string[] = [shortIdentity(origin, attributePairs(origin.attributes))] + let current = origin + for (let depth = 0; depth < 16; depth += 1) { + const parentId = current.parentId + if (parentId === undefined || parentId <= 0) break + let parent: DescribedNode + try { + parent = await describeNode(client, parentId) + } catch { + // Some nodes (documents, shadow roots) have no describable parent. + break + } + if (parent.nodeName === undefined) break + segments.unshift(shortIdentity(parent, attributePairs(parent.attributes))) + current = parent + } + return segments.join(' > ') +} + +/** + * Apply inline (element `style` attribute) declarations. + * + * `CSS.setStyleTexts` needs a `styleSheetId` plus the declaration block's + * `range`, which Chrome only reports for inline styles it exposes as an editable + * sheet (`isInline`). When that metadata is absent the edit is applied through + * the `style` attribute, which reaches the same result without depending on CDP + * internals. + */ +export async function setInlineStyle( + client: CdpClient, + nodeId: number, + properties: readonly { name: string; value: string; important?: boolean }[], +): Promise { + const matched = (await client.send('CSS.getMatchedStylesForNode', { nodeId })) as { inlineStyle?: StylePayload } + const inline = matched.inlineStyle + const merged = mergeDeclarations(inline, properties) + const text = merged.map((declaration) => `${declaration};`).join(' ') + const styleSheetId = inline?.styleSheetId + const range = inline?.range + if (styleSheetId !== undefined && range !== undefined) { + await client.send('CSS.setStyleTexts', { + edits: [{ styleSheetId, range, text }], + nodeForPropertySyntaxValidation: nodeId, + }) + return merged + } + await client.send('DOM.setAttributeValue', { nodeId, name: 'style', value: inlineText(merged) }) + return merged +} + +/** The `style` attribute text for a declaration list. */ +function inlineText(declarations: readonly string[]): string { + return declarations.length === 0 ? '' : `${declarations.map((declaration) => `${declaration};`).join(' ')}` +} + +/** + * Edit declarations of the first author rule that matches the element. + * + * Chrome edits the declaration range and preserves the surrounding stylesheet. + */ +export async function setMatchedRuleStyle( + client: CdpClient, + nodeId: number, + properties: readonly { name: string; value: string; important?: boolean }[], +): Promise<{ selector: string; declarations: string[] }> { + const response = (await client.send('CSS.getMatchedStylesForNode', { nodeId })) as { + matchedCSSRules?: MatchedRulePayload[] + } + const rule = (response.matchedCSSRules ?? []).map((entry) => entry.rule).find((candidate) => candidate?.styleSheetId !== undefined) + const styleSheetId = rule?.styleSheetId + if (rule === undefined || styleSheetId === undefined) { + throw new Error('This element has no matching author stylesheet rule to edit; use mode "inline" instead.') + } + const merged = mergeDeclarations(rule.style, properties) + const range = rule.style?.range + if (range === undefined) throw new Error('The matching stylesheet rule has no editable declaration range.') + const edited = await client.send('CSS.setStyleTexts', { + edits: [{ styleSheetId, range, text: inlineText(merged) }], + nodeForPropertySyntaxValidation: nodeId, + }) as { styles?: StylePayload[] } + const applied = edited.styles?.[0] + if (applied === undefined) throw new Error('Chrome did not return the edited stylesheet rule.') + return { selector: rule.selectorList?.text ?? '(unknown selector)', declarations: declarationsOf(applied) } +} + +interface RangePayload { + startLine: number + startColumn: number + endLine: number + endColumn: number +} + +/** Apply a property change set to an existing declaration list. */ +export function mergeDeclarations( + style: StylePayload | undefined, + properties: readonly { name: string; value: string; important?: boolean }[], +): string[] { + const edits = new Map() + for (const property of properties) edits.set(property.name.trim().toLowerCase(), property) + // 修改 shorthand 时替换其展开项；修改 longhand 时保留其他方向的声明。 + const covered = new Set([...edits.keys()].flatMap(name => LONGHANDS_OF[name] ?? [])) + const declarations: string[] = [] + const seen = new Set() + for (const property of style?.cssProperties ?? []) { + const key = property.name.trim().toLowerCase() + if (property.disabled === true || seen.has(key)) continue + seen.add(key) + if (edits.has(key) || covered.has(key)) continue + declarations.push(`${property.name}: ${property.value}${property.important === true ? ' !important' : ''}`) + } + for (const edit of edits.values()) { + if (edit.value === '') continue + declarations.push(`${edit.name}: ${edit.value}${edit.important === true ? ' !important' : ''}`) + } + return canonicalizeDeclarations(declarations, new Set(edits.keys())) +} + +/** Attribute names that require extra scrutiny before a write. */ +export function attributeRisk(name: string): 'forbidden' | 'script' | 'normal' { + const normalized = name.trim().toLowerCase() + if (normalized === '' || /[\s<>]/.test(normalized)) return 'forbidden' + if (FORBIDDEN_ATTRIBUTES.has(normalized)) return 'forbidden' + if (SCRIPT_ATTRIBUTES.test(normalized)) return 'script' + return 'normal' +} + +/** Set or remove one attribute. */ +export async function setAttribute( + client: CdpClient, + nodeId: number, + name: string, + value: string | undefined, + remove: boolean, +): Promise { + if (remove) { + await client.send('DOM.removeAttribute', { nodeId, name }) + return + } + if (value === undefined) return + await client.send('DOM.setAttributeValue', { nodeId, name, value }) +} + +/** Current value of one attribute, or undefined when absent. */ +export async function attributeValue(client: CdpClient, nodeId: number, name: string): Promise { + const found = (await client.send('DOM.getAttributes', { nodeId })) as { attributes?: string[] } + return attributePairs(found.attributes).find((attribute) => attribute.name.toLowerCase() === name.toLowerCase())?.value +} diff --git a/extensions/dsh-browser/src/background/devtools-format.ts b/extensions/dsh-browser/src/background/devtools-format.ts new file mode 100644 index 000000000..8ed13b272 --- /dev/null +++ b/extensions/dsh-browser/src/background/devtools-format.ts @@ -0,0 +1,211 @@ +/** + * Presentation layer for the developer-tools tools: turn CDP-derived view + * models into the single bounded text payload each tool returns. + * + * Page-authored values are passed through {@link wrapUntrustedContent} by the + * caller that owns the budget, so these renderers stay pure and test-free of + * Chrome APIs. + * + * @module + */ + +import { + formatBytes, + formatHeaders, + formatTiming, + isSensitiveBodyUrl, + REDACTED, + truncateText, +} from './devtools-serialization.ts' +import type { ConsoleEntry, ConsoleLevel, NetworkEntry } from './devtools-network.ts' +import type { ElementView } from './devtools-elements.ts' +/** Render one inspected element report. */ +export function renderElement(view: ElementView): string { + const lines: string[] = [`Element: ${view.identity}`] + lines.push(`DOM path: ${view.domPath}`) + if (view.inventoryIndex !== undefined) lines.push(`Snapshot inventory index: ${view.inventoryIndex}`) + lines.push(`Child nodes: ${view.childNodeCount}`) + if (view.box !== undefined) { + lines.push(`Box: x ${view.box.x} y ${view.box.y} ${view.box.width}×${view.box.height}`) + } + if (view.boxModel !== undefined) { + lines.push(`Box model (content / padding / border / margin): ${view.boxModel.content} / ${view.boxModel.padding} / ${view.boxModel.border} / ${view.boxModel.margin}`) + } + lines.push('', 'Computed style:') + for (const [name, value] of Object.entries(view.computed)) lines.push(` ${name}: ${value}`) + lines.push('', 'Inline style:') + lines.push(...view.inlineStyle.length === 0 ? [' (none)'] : view.inlineStyle.map((declaration) => ` ${declaration}`)) + lines.push('', 'Matched CSS rules (most specific first):') + if (view.matchedRules.length === 0) lines.push(' (no stylesheet rules matched)') + for (const rule of view.matchedRules) { + const media = rule.media === undefined ? '' : ` @media ${rule.media}` + lines.push(` ${rule.selector} [${rule.origin}]${media}`) + lines.push(...rule.declarations.length === 0 + ? [' (empty rule)'] + : rule.declarations.map((declaration) => ` ${declaration}`)) + } + lines.push('', 'Attributes:') + lines.push(...view.attributes.length === 0 + ? [' (none)'] + : view.attributes.map((attribute) => ` ${attribute.name}="${attribute.value}"`)) + return lines.join('\n') +} + +/** Render one element-list header, shared by inspection and edits. */ +export function renderElementHeader(identity: string, note?: string): string { + return note === undefined ? `Element: ${identity}` : `Element: ${identity}\n${note}` +} + +/** Render the result of a style edit. */ +export function renderStyleEdit( + identity: string, + mode: 'inline' | 'rule', + target: string, + declarations: readonly string[], +): string { + const lines = [renderElementHeader(identity, mode === 'inline' + ? 'Applied to the inline style attribute (a page reload reverts it).' + : `Applied to the matching stylesheet rule: ${target} (reverts when the page replaces its stylesheets).`)] + lines.push('', 'Declarations now applied:') + lines.push(...declarations.length === 0 ? [' (empty)' ] : declarations.map((declaration) => ` ${declaration}`)) + return lines.join('\n') +} + +/** Render an attribute read/write result. */ +export function renderAttributeResult( + identity: string, + name: string, + before: string | undefined, + after: string | undefined, + action: 'read' | 'set' | 'removed', +): string { + const lines = [`Element: ${identity}`, `Attribute: ${name}`] + lines.push(`Before: ${before === undefined ? '(absent)' : before}`) + if (action === 'read') return lines.join('\n') + lines.push(`After: ${after === undefined ? '(removed)' : after}`) + return lines.join('\n') +} + +/** Render one console-log query result. */ +export function renderConsoleLogs(entries: readonly ConsoleEntry[], sinceIndex: number): string { + if (entries.length === 0) { + return `No console entries recorded${sinceIndex > 0 ? ` after index ${sinceIndex}` : ''}. ` + + 'The extension records output from the moment the developer-tools session attached; reload the page to capture startup logs.' + } + const lines = [`Console entries (${entries.length}):`] + for (const entry of entries) { + lines.push(`[${entry.index}] ${entry.level}: ${entry.text}`) + } + return lines.join('\n') +} + +/** Render one evaluation result, already formatted by the serialization layer. */ +export function renderEvaluation(text: string, ok: boolean): string { + return `${ok ? 'Evaluation result:' : 'Evaluation failed:'}\n${text}` +} + +/** Render the capture control answer. */ +export function renderCaptureState(state: { attached: boolean; tabId?: number; entries: number; generation: number }, action: string): string { + const lines = [`Network capture ${state.attached ? 'active' : 'inactive'}.`] + if (state.tabId !== undefined) lines.push(`Tab: ${state.tabId}`) + lines.push(`Captured requests: ${state.entries}`) + lines.push(`Capture generation: ${state.generation} (increments whenever the log resets, for example after a navigation)`) + if (action === 'start') { + lines.push('', 'Trigger the requests you want to inspect, then call browser_devtools_list_requests.') + } + if (action === 'stop') { + lines.push('', 'The debugger detached; captured entries remain listed until the next capture or navigation.') + } + return lines.join('\n') +} + +/** One line per captured request, without bodies. */ +export function renderRequestList(entries: readonly NetworkEntry[], generation: number, limit: number): string { + if (entries.length === 0) { + return 'No captured requests matched. Start capture with browser_devtools_network action=start, then trigger the page traffic again.' + } + const shown = entries.slice(-limit) + const lines = [`Captured requests (${shown.length} of ${entries.length}, generation ${generation}):`] + for (const entry of shown) { + const status = entry.failed === undefined + ? (entry.status === undefined ? '(pending)' : String(entry.status)) + : `FAILED ${entry.failed}` + const duration = entry.durationMs === undefined ? '' : ` ${entry.durationMs}ms` + const size = entry.encodedDataLength === undefined ? '' : ` ${formatBytes(entry.encodedDataLength)}` + lines.push(`[${entry.index}] ${entry.method} ${entry.url}`) + lines.push(` ${status}${duration}${size} type=${entry.resourceType ?? '?'}${entry.fromDiskCache ? ' from-disk-cache' : ''}${entry.fromServiceWorker ? ' from-service-worker' : ''}`) + if (entry.pendingRequestEvent === true) { + lines.push(' (Chrome withheld the main event for this request; method and request headers may be incomplete)') + } + } + lines.push('', 'Use browser_devtools_get_request with an index for headers, timing, and (optionally) the response body.') + return lines.join('\n') +} + +/** Full detail of one captured request. */ +export function renderRequestDetail( + entry: NetworkEntry, + options: { includeSensitiveHeaders: boolean; includeSensitiveBody: boolean; body?: string; bodyTruncated: boolean }, +): string { + const lines: string[] = [] + lines.push(`[${entry.index}] ${entry.method} ${entry.url}`) + lines.push(`Type: ${entry.resourceType ?? 'unknown'}${entry.frameId === undefined ? '' : ` · frame ${entry.frameId}`}`) + if (entry.initiator !== undefined) lines.push(`Initiator: ${entry.initiator}`) + const status = entry.failed === undefined + ? `${entry.status ?? '(pending)'} ${entry.statusText ?? ''}`.trim() + : `FAILED: ${entry.failed}` + lines.push(`Status: ${status}${entry.blockedReason === undefined ? '' : ` (blocked: ${entry.blockedReason})`}`) + if (entry.mimeType !== undefined) lines.push(`MIME: ${entry.mimeType}`) + if (entry.durationMs !== undefined) lines.push(`Duration: ${entry.durationMs}ms`) + lines.push(`Transfer size: ${formatBytes(entry.encodedDataLength)}${entry.headerSize === undefined ? '' : ` (headers ${formatBytes(entry.headerSize)})`}`) + lines.push(`Timing: ${formatTiming(entry.timing, entry.durationMs)}`) + const cache: string[] = [] + if (entry.fromDiskCache) cache.push('disk cache') + if (entry.fromServiceWorker) cache.push('service worker') + if (cache.length > 0) lines.push(`Served from: ${cache.join(', ')}`) + if (entry.redirects.length > 0) { + lines.push('', 'Redirects:') + for (const hop of entry.redirects) lines.push(` ${hop.status} ${hop.url}`) + } + lines.push('', 'Request headers:') + lines.push(...formatHeaders(entry.requestHeaders, options.includeSensitiveHeaders)) + if (entry.postData !== undefined) { + lines.push('', 'Request payload:') + const sensitive = !options.includeSensitiveBody && isSensitiveBodyUrl(entry.url) + lines.push(sensitive ? REDACTED : truncateText(entry.postData, 4_000)) + } else if (entry.hasPostData) { + lines.push('', 'This request carries a payload; read it with browser_devtools_request_body kind=request.') + } + lines.push('', 'Response headers:') + lines.push(...formatHeaders(entry.responseHeaders, options.includeSensitiveHeaders)) + if (options.body !== undefined) { + const sensitive = !options.includeSensitiveBody && isSensitiveBodyUrl(entry.url) + lines.push('', 'Response body:') + lines.push(sensitive ? REDACTED : options.body) + if (options.bodyTruncated) lines.push('(body truncated)') + } + return lines.join('\n') +} + +/** Render one body fetch. */ +export function renderBody( + entry: NetworkEntry, + kind: 'request' | 'response', + body: string, + options: { truncated: boolean; json: boolean; redacted: boolean }, +): string { + const lines = [`[${entry.index}] ${kind} body of ${entry.method} ${entry.url}`] + if (options.redacted) { + lines.push('', REDACTED, 'This request looks authentication-related; pass includeSensitiveBody=true to read it.') + return lines.join('\n') + } + lines.push(`Format: ${options.json ? 'JSON (pretty-printed)' : 'text'}${options.truncated ? ' · truncated' : ''}`) + lines.push('', body) + if (options.truncated) lines.push('', '(body truncated to the configured budget)') + return lines.join('\n') +} + +/** Levels are exposed so the tool layer can validate its own filter input. */ +export function isConsoleLevel(value: string): value is ConsoleLevel { + return value === 'log' || value === 'info' || value === 'warning' || value === 'error' || value === 'exception' +} diff --git a/extensions/dsh-browser/src/background/devtools-network.ts b/extensions/dsh-browser/src/background/devtools-network.ts new file mode 100644 index 000000000..019ba59fe --- /dev/null +++ b/extensions/dsh-browser/src/background/devtools-network.ts @@ -0,0 +1,489 @@ +/** + * Network capture for the developer-tools group: folds `Network.*` CDP events + * into a bounded in-memory request log, plus the bounded console log the + * `Runtime` domain emits. + * + * The store is deliberately passive — it never issues commands — so the same + * instance can be fed from unit tests with synthetic event payloads. + * + * @module + */ + +import { + formatConsoleArguments, + formatException, + summarizeTiming, + type CdpHeader, + type TimingSummary, +} from './devtools-serialization.ts' + +/** + * Normalize a CDP header collection. + * + * `Network.requestWillBeSent` reports headers as an array of name/value pairs + * while the extra-info and response events report a plain map; both shapes reach + * this store, so the array form is the single internal representation. + */ +export function toHeaderList(value: unknown): CdpHeader[] { + if (Array.isArray(value)) return value as CdpHeader[] + if (typeof value === 'object' && value !== null) { + return Object.entries(value as Record).map(([name, header]) => ({ + name, + value: String(header), + })) + } + return [] +} + +/** Entries kept per attached tab; the oldest are dropped first. */ +export const NETWORK_BUFFER_LIMIT = 200 + +/** Console entries kept per attached tab. */ +export const CONSOLE_BUFFER_LIMIT = 500 + +/** Redirect hops kept per entry. */ +const REDIRECT_LIMIT = 10 + +/** One captured request/response pair. */ +export interface NetworkEntry { + /** Monotonic index inside the current capture generation; stable for the model. */ + index: number + requestId: string + url: string + method: string + /** CDP resource type (Document, XHR, Fetch, …). */ + resourceType?: string + requestHeaders: CdpHeader[] + /** Request payload when the browser reported it inline. */ + postData?: string + hasPostData: boolean + /** Wall-clock-ish monotonic timestamp of `requestWillBeSent` (seconds). */ + startTime: number + frameId?: string + initiator?: string + redirects: { url: string; status: number }[] + status?: number + statusText?: string + mimeType?: string + responseHeaders: CdpHeader[] + /** `Network.loadingFinished.encodedDataLength`. */ + encodedDataLength?: number + /** `Response.encodedDataLength` observed before the body finished loading. */ + headerSize?: number + timing?: TimingSummary + fromDiskCache: boolean + fromServiceWorker: boolean + /** `Network.loadingFailed` detail, when the request never completed. */ + failed?: string + blockedReason?: string + /** Duration in ms between request start and finish/failure. */ + durationMs?: number + finished: boolean + /** + * True while only the extra-info/response events have been seen, so the + * method and URL may still be stale. Reported to the model rather than + * silently presented as authoritative. + */ + pendingRequestEvent?: boolean +} + +/** Console event kinds the extension records. */ +export type ConsoleLevel = 'log' | 'info' | 'warning' | 'error' | 'exception' + +/** One recorded console entry. */ +export interface ConsoleEntry { + index: number + level: ConsoleLevel + timestamp: number + text: string +} + +/** Which capture generation a store currently holds; the model can detect resets. */ +export interface CaptureState { + attached: boolean + tabId?: number + entries: number + generation: number +} + +/** Either wire shape: the array form or the map form. */ +type HeaderList = readonly CdpHeader[] | Record | undefined + +interface RequestWillBeSentParams { + requestId?: string + request?: { url?: string; method?: string; headers?: HeaderList; postData?: string; hasPostData?: boolean } + type?: string + timestamp?: number + frameId?: string + initiator?: { type?: string; url?: string } + redirectResponse?: { url?: string; status?: number } +} + +/** + * `Network.requestWillBeSentExtraInfo`. + * + * Chrome emits this for requests whose main `Network.requestWillBeSent` event + * is withheld or arrives late (observed on live pages); it carries the request + * headers and the request id, which is enough to start tracking the request. + */ +interface RequestExtraInfoParams { + requestId?: string + headers?: HeaderList +} + +/** `Network.responseReceivedExtraInfo`: headers and status without a `Response`. */ +interface ResponseExtraInfoParams { + requestId?: string + headers?: HeaderList + statusCode?: number + headersText?: string +} + +interface ResponseReceivedParams { + requestId?: string + type?: string + response?: { + url?: string + status?: number + statusText?: string + mimeType?: string + headers?: HeaderList + encodedDataLength?: number + timing?: unknown + fromDiskCache?: boolean + fromServiceWorker?: boolean + } +} + +interface LoadingFinishedParams { + requestId?: string + timestamp?: number + encodedDataLength?: number +} + +interface LoadingFailedParams { + requestId?: string + timestamp?: number + errorText?: string + blockedReason?: string +} + +interface ConsoleApiParams { + type?: string + args?: unknown[] + timestamp?: number +} + +/** + * Bounded network + console capture for one attached tab. + * + * A generation counter increments whenever the log resets (attach or a + * top-level navigation), so the model can tell "no such index" from "the page + * navigated and the log started over". + */ +export class NetworkCapture { + private entries = new Map() + private order: string[] = [] + private sequence = 0 + private generation = 0 + private tabId: number | undefined + private attached = false + private consoleEntries: ConsoleEntry[] = [] + private consoleSequence = 0 + + /** Mark capture active for a tab and reset the log (a fresh attachment). */ + start(tabId: number): void { + this.tabId = tabId + this.attached = true + this.clear() + } + + /** Mark capture inactive; the log stays readable until the next start. */ + stop(): void { + this.attached = false + } + + /** Drop every captured entry and start a new generation. */ + clear(): void { + this.entries.clear() + this.order = [] + this.sequence = 0 + this.consoleEntries = [] + this.consoleSequence = 0 + this.generation += 1 + } + + /** Current capture state for status reporting. */ + state(): CaptureState { + return { + attached: this.attached, + ...(this.tabId === undefined ? {} : { tabId: this.tabId }), + entries: this.order.length, + generation: this.generation, + } + } + + /** Captured requests in arrival order. */ + list(): NetworkEntry[] { + return this.order.map((id) => this.entries.get(id)).filter((entry): entry is NetworkEntry => entry !== undefined) + } + + /** One captured request by its model-facing index. */ + get(index: number): NetworkEntry | undefined { + return this.order + .map((id) => this.entries.get(id)) + .find((entry) => entry !== undefined && entry.index === index) + } + + /** Recorded console entries in arrival order. */ + consoleLog(): ConsoleEntry[] { + return [...this.consoleEntries] + } + + /** + * Fold one CDP event into the store. Unknown methods are ignored so the + * session can forward everything it observes. + * @param method - CDP event name. + * @param params - CDP event payload. + */ + ingest(method: string, params: unknown): void { + switch (method) { + case 'Network.requestWillBeSent': + this.onRequestWillBeSent(params as RequestWillBeSentParams) + break + case 'Network.requestWillBeSentExtraInfo': + this.onRequestExtraInfo(params as RequestExtraInfoParams) + break + case 'Network.responseReceived': + this.onResponseReceived(params as ResponseReceivedParams) + break + case 'Network.responseReceivedExtraInfo': + this.onResponseExtraInfo(params as ResponseExtraInfoParams) + break + case 'Network.loadingFinished': + this.onLoadingFinished(params as LoadingFinishedParams) + break + case 'Network.loadingFailed': + this.onLoadingFailed(params as LoadingFailedParams) + break + case 'Runtime.consoleAPICalled': + this.onConsoleApi(params as ConsoleApiParams) + break + case 'Runtime.exceptionThrown': + this.onExceptionThrown(params) + break + default: + break + } + } + + /** A new top-level document means every previous index is stale. */ + ingestNavigation(): void { + this.clear() + } + + private onRequestWillBeSent(params: RequestWillBeSentParams): void { + const requestId = params.requestId + if (requestId === undefined) return + const existing = this.entries.get(requestId) + if (existing !== undefined && params.redirectResponse !== undefined) { + // A redirect reuses the request id: keep one entry and record each hop, + // including where the overwritten request had been heading. + if (existing.redirects.length < REDIRECT_LIMIT) { + existing.redirects.push({ url: existing.url, status: params.redirectResponse.status ?? 0 }) + } + existing.failed = undefined + delete existing.status + this.applyRequest(existing, params) + return + } + const entry = existing ?? this.createEntry(requestId, params) + entry.pendingRequestEvent = false + this.applyRequest(entry, params) + if (existing === undefined) this.push(entry) + } + + /** The one place entry indices are minted. */ + private createEntryFromId(requestId: string): NetworkEntry { + this.sequence += 1 + return { + index: this.sequence, + requestId, + url: '', + method: 'GET', + requestHeaders: [], + hasPostData: false, + startTime: 0, + redirects: [], + responseHeaders: [], + fromDiskCache: false, + fromServiceWorker: false, + finished: false, + } + } + + private createEntry(requestId: string, params: RequestWillBeSentParams): NetworkEntry { + const entry = this.createEntryFromId(requestId) + entry.url = params.request?.url ?? '' + entry.method = params.request?.method ?? 'GET' + entry.requestHeaders = toHeaderList(params.request?.headers) + entry.hasPostData = params.request?.hasPostData === true + entry.startTime = params.timestamp ?? 0 + return entry + } + + private applyRequest(entry: NetworkEntry, params: RequestWillBeSentParams): void { + if (params.request?.url !== undefined) entry.url = params.request.url + if (params.request?.method !== undefined) entry.method = params.request.method + if (params.request?.headers !== undefined) entry.requestHeaders = toHeaderList(params.request.headers) + if (params.request?.postData !== undefined) entry.postData = params.request.postData + if (params.request?.hasPostData === true) entry.hasPostData = true + if (params.type !== undefined) entry.resourceType = params.type + if (params.frameId !== undefined) entry.frameId = params.frameId + if (params.initiator !== undefined) { + entry.initiator = params.initiator.type === undefined + ? params.initiator.url + : `${params.initiator.type}${params.initiator.url === undefined ? '' : ` ${params.initiator.url}`}` + } + } + + /** + * Start tracking a request from its extra-info event when the main + * `requestWillBeSent` never arrives. Reporting the traffic matters more than + * insisting on the one event Chrome does not always send. + */ + private onRequestExtraInfo(params: RequestExtraInfoParams): void { + const requestId = params.requestId + if (requestId === undefined) return + if (this.entries.has(requestId)) { + const existing = this.entries.get(requestId)! + if (params.headers !== undefined && existing.requestHeaders.length === 0) { + existing.requestHeaders = toHeaderList(params.headers) + } + return + } + const entry = this.createEntryFromId(requestId) + entry.requestHeaders = toHeaderList(params.headers) + // Marked so the model knows why the request line has no method or URL yet. + entry.pendingRequestEvent = true + this.push(entry) + } + + private onResponseReceived(params: ResponseReceivedParams): void { + const requestId = params.requestId + if (requestId === undefined) return + const response = params.response + if (response === undefined) return + const entry = this.entries.get(requestId) ?? this.adoptResponse(requestId, response, params.type) + if (entry === undefined) return + if (response.url !== undefined) entry.url = response.url + entry.status = response.status + entry.statusText = response.statusText + entry.mimeType = response.mimeType + entry.responseHeaders = toHeaderList(response.headers) + entry.headerSize = response.encodedDataLength + entry.fromDiskCache = response.fromDiskCache === true + entry.fromServiceWorker = response.fromServiceWorker === true + if (params.type !== undefined) entry.resourceType = params.type + entry.timing = summarize(response.timing) + } + + /** Fold a headers-only response event into an existing (or new) entry. */ + private onResponseExtraInfo(params: ResponseExtraInfoParams): void { + const requestId = params.requestId + if (requestId === undefined) return + const entry = this.entries.get(requestId) ?? this.createEntryFromId(requestId) + if (!this.entries.has(requestId)) this.push(entry) + entry.responseHeaders = toHeaderList(params.headers) + if (typeof params.statusCode === 'number' && entry.status === undefined) entry.status = params.statusCode + } + + /** Build an entry from a response alone, when no request event was seen. */ + private adoptResponse( + requestId: string, + response: { url?: string; mimeType?: string }, + resourceType?: string, + ): NetworkEntry | undefined { + const entry = this.createEntryFromId(requestId) + if (response.url !== undefined) entry.url = response.url + if (response.mimeType !== undefined) entry.mimeType = response.mimeType + if (resourceType !== undefined) entry.resourceType = resourceType + entry.pendingRequestEvent = true + this.push(entry) + return entry + } + + private onLoadingFinished(params: LoadingFinishedParams): void { + const requestId = params.requestId + if (requestId === undefined) return + const entry = this.entries.get(requestId) + if (entry === undefined) return + entry.encodedDataLength = params.encodedDataLength + entry.finished = true + if (params.timestamp !== undefined && entry.startTime > 0) { + entry.durationMs = round(params.timestamp - entry.startTime) + } + } + + private onLoadingFailed(params: LoadingFailedParams): void { + const requestId = params.requestId + if (requestId === undefined) return + const entry = this.entries.get(requestId) + if (entry === undefined) return + entry.failed = params.errorText ?? 'request failed' + if (params.blockedReason !== undefined) entry.blockedReason = params.blockedReason + if (params.timestamp !== undefined && entry.startTime > 0) { + entry.durationMs = round(params.timestamp - entry.startTime) + } + } + + private onConsoleApi(params: ConsoleApiParams): void { + const level = consoleLevel(params.type) + if (level === undefined) return + this.pushConsole(level, params.timestamp ?? 0, params.args) + } + + private onExceptionThrown(params: unknown): void { + const payload = (params ?? {}) as { timestamp?: number; exceptionDetails?: unknown } + this.pushConsole('exception', payload.timestamp ?? 0, formatException(payload.exceptionDetails)) + } + + private pushConsole(level: ConsoleLevel, timestamp: number, args: unknown): void { + this.consoleSequence += 1 + const text = typeof args === 'string' ? args : formatConsoleArguments(args as readonly unknown[]) + this.consoleEntries.push({ index: this.consoleSequence, level, timestamp, text }) + if (this.consoleEntries.length > CONSOLE_BUFFER_LIMIT) { + this.consoleEntries.splice(0, this.consoleEntries.length - CONSOLE_BUFFER_LIMIT) + } + } + + private push(entry: NetworkEntry): void { + this.entries.set(entry.requestId, entry) + this.order.push(entry.requestId) + while (this.order.length > NETWORK_BUFFER_LIMIT) { + const dropped = this.order.shift() + if (dropped !== undefined) this.entries.delete(dropped) + } + } +} + +/** CDP console types the extension records; `debug`/`trace` stay out of the way. */ +function consoleLevel(type: string | undefined): ConsoleLevel | undefined { + switch (type) { + case 'log': return 'log' + case 'info': + case 'debug': return 'info' + case 'warning': return 'warning' + case 'error': + case 'assert': return 'error' + default: return undefined + } +} + +function summarize(timing: unknown): TimingSummary | undefined { + return summarizeTiming(timing) +} + +function round(value: number): number { + return Math.round(value * 1000 * 10) / 10 +} diff --git a/extensions/dsh-browser/src/background/devtools-serialization.ts b/extensions/dsh-browser/src/background/devtools-serialization.ts new file mode 100644 index 000000000..244dd5763 --- /dev/null +++ b/extensions/dsh-browser/src/background/devtools-serialization.ts @@ -0,0 +1,295 @@ +/** + * Pure formatting helpers for the developer-tools tools. + * + * Everything a CDP command or event returns is either page-authored, transport + * detail, or a protocol object; this module turns those into bounded, + * model-readable text. It is deliberately free of Chrome APIs and of I/O so the + * whole presentation layer is unit-testable. + * + * @module + */ + +/** Header names whose values never reach the model unless the user opts in per call. */ +const SENSITIVE_HEADERS = new Set([ + 'authorization', + 'cookie', + 'set-cookie', + 'proxy-authorization', + 'x-csrf-token', + 'x-xsrf-token', + 'x-api-key', + 'api-key', + 'x-auth-token', + 'x-access-token', + 'x-session-token', +]) + +/** URL shapes whose bodies are redacted by default (credentials travel there). */ +const SENSITIVE_BODY_URL = /(?:^|[^a-z])(auth|login|logout|signin|sign-in|signup|register|oauth|token|session|password|passwd|credential|apikey|api-key)(?:[^a-z]|$)/i + +/** One CDP header entry as `Network`/`Fetch` report it. */ +export interface CdpHeader { + name: string + value: string +} + +/** Redaction marker used in place of a hidden value. */ +export const REDACTED = '[redacted]' + +/** Whether a header name is treated as a credential carrier. */ +export function isSensitiveHeader(name: string): boolean { + return SENSITIVE_HEADERS.has(name.trim().toLowerCase()) +} + +/** Whether a request URL suggests a body that should be redacted by default. */ +export function isSensitiveBodyUrl(url: string): boolean { + return SENSITIVE_BODY_URL.test(url) +} + +/** + * Render a CDP header list as indented lines. + * @param headers - header entries in wire order. + * @param sensitive - when false (default) credential-bearing values are masked. + * @returns one line per header. + */ +export function formatHeaders(headers: readonly CdpHeader[] | undefined, sensitive = false): string[] { + if (headers === undefined || headers.length === 0) return [' (none)'] + return headers.map((header) => { + const value = !sensitive && isSensitiveHeader(header.name) ? REDACTED : header.value + return ` ${header.name}: ${value}` + }) +} + +/** Truncate text and state the original length when it was cut. */ +export function truncateText(text: string, maxChars: number): string { + if (text.length <= maxChars) return text + return `${text.slice(0, Math.max(0, maxChars))}\n…(truncated; ${text.length} characters total)` +} + +/** + * Pretty-print JSON when possible, otherwise return the text unchanged. + * @param text - candidate JSON text. + * @returns the formatted body and whether it parsed. + */ +export function formatBody(text: string): { text: string; json: boolean } { + const trimmed = text.trim() + if (trimmed === '' || (trimmed[0] !== '{' && trimmed[0] !== '[')) return { text, json: false } + try { + return { text: JSON.stringify(JSON.parse(trimmed), null, 2), json: true } + } catch { + return { text, json: false } + } +} + +/** Human-readable property list from a CDP `RemoteObject` preview. */ +export function previewProperties(value: unknown): string { + const preview = (value as { preview?: { properties?: { name: string; type: string; value?: string }[]; overflow?: boolean } } | null)?.preview + if (preview?.properties === undefined || preview.properties.length === 0) return '' + const parts = preview.properties.map((property) => { + const rendered = property.value === undefined ? property.type : property.value + return `${property.name}: ${rendered}` + }) + return `{${parts.join(', ')}${preview.overflow === true ? ', …' : ''}}` +} + +/** + * A short description of a CDP `RemoteObject` when no value could be returned. + * + * `description` already names the type (`Object`, `Map(2)`, a function source), + * so it is used alone when present; only a value-less object carries a `type` + * that reads like a label ("object", "function"). + */ +export function describeRemoteObject(value: unknown): string { + const remote = value as { type?: string; subtype?: string; className?: string; description?: string } | null + if (remote === null || remote === undefined) return 'undefined' + const description = remote.description ?? remote.className ?? '' + if (description !== '') return description + return remote.subtype ?? remote.type ?? 'value' +} + +/** + * Render the value of one `Runtime.evaluate` result. + * @param result - the command's `result` field. + * @param options - preview and character budget. + * @returns the model-facing text plus whether the value round-tripped by value. + */ +export function formatEvaluationResult( + result: unknown, + options: { maxChars: number; includePreview: boolean }, +): { text: string; byValue: boolean } { + const remote = result as { type?: string; subtype?: string; value?: unknown; unserializableValue?: string; description?: string } | null + if (remote === null || remote === undefined) return { text: 'undefined', byValue: true } + if (remote.type === 'undefined') return { text: 'undefined (the expression produced no value)', byValue: true } + if (remote.unserializableValue !== undefined) return { text: remote.unserializableValue, byValue: true } + if (remote.type === 'string') return { text: String(remote.value ?? remote.description ?? ''), byValue: true } + if (remote.type === 'boolean' || remote.type === 'number' || remote.type === 'bigint') { + return { text: String(remote.value ?? remote.description ?? ''), byValue: true } + } + if (remote.subtype === 'null') return { text: 'null', byValue: true } + + const lines: string[] = [] + if (remote.type === 'function') { + lines.push(`function: ${remote.description ?? '(anonymous)'}`) + } else if ('value' in remote && remote.value !== undefined) { + try { + lines.push(truncateText(JSON.stringify(remote.value, null, 2), options.maxChars)) + } catch { + lines.push(describeRemoteObject(remote)) + } + } else { + lines.push(describeRemoteObject(remote)) + } + if (options.includePreview) { + const preview = previewProperties(remote) + if (preview !== '') lines.push(preview) + } + return { text: lines.join('\n'), byValue: false } +} + +/** + * Render one `Runtime.evaluate` result including its exception, if any. + * @param response - the raw command result. + * @param options - preview and character budget. + * @returns the model-facing text and whether the evaluation completed. + */ +export function formatEvaluation( + response: unknown, + options: { maxChars: number; includePreview: boolean }, +): { text: string; ok: boolean } { + const payload = (response ?? {}) as { result?: unknown; exceptionDetails?: unknown } + if (payload.exceptionDetails !== undefined) { + return { text: formatException(payload.exceptionDetails), ok: false } + } + const formatted = formatEvaluationResult(payload.result, options) + return { text: formatted.text, ok: true } +} + +/** Render CDP `ExceptionDetails` (thrown value, text, stack) as text. */ +export function formatException(details: unknown): string { + const exception = details as { + text?: string + lineNumber?: number + columnNumber?: number + url?: string + exception?: { description?: string; value?: unknown } + stackTrace?: { callFrames?: { functionName?: string; url?: string; lineNumber?: number; columnNumber?: number }[] } + } | null + if (exception === null || exception === undefined) return 'Evaluation failed with no details.' + const lines: string[] = [] + const thrown = exception.exception?.description + ?? (exception.exception?.value === undefined ? undefined : safeStringify(exception.exception.value)) + lines.push(`Evaluation threw: ${thrown ?? exception.text ?? 'unknown error'}`) + const location = exception.url === undefined || exception.lineNumber === undefined + ? '' + : ` at ${exception.url}:${exception.lineNumber + 1}:${(exception.columnNumber ?? 0) + 1}` + if (location !== '') lines.push(`Thrown${location}`) + for (const frame of exception.stackTrace?.callFrames?.slice(0, 8) ?? []) { + lines.push(` at ${frame.functionName === '' ? '(anonymous)' : frame.functionName ?? '(anonymous)'} (${frame.url ?? '?'}:${(frame.lineNumber ?? 0) + 1}:${(frame.columnNumber ?? 0) + 1})`) + } + return lines.join('\n') +} + +/** + * Render the arguments of one `Runtime.consoleAPICalled` event. + * @param args - CDP `RemoteObject` arguments. + * @returns a single readable line body. + */ +export function formatConsoleArguments(args: readonly unknown[] | undefined): string { + if (args === undefined || args.length === 0) return '(no arguments)' + return args.map((argument) => { + const remote = argument as { type?: string; subtype?: string; value?: unknown; description?: string; unserializableValue?: string } | null + if (remote === null || remote === undefined) return 'undefined' + if (remote.type === 'string') return String(remote.value ?? '') + if (remote.unserializableValue !== undefined) return remote.unserializableValue + if (remote.value !== undefined && (remote.type === 'number' || remote.type === 'boolean')) return String(remote.value) + if (remote.description !== undefined) return remote.description + if (remote.value === undefined) return remote.type ?? 'value' + return safeStringify(remote.value) + }).join(' ') +} + +/** Serializable timing breakdown in milliseconds, relative to request start. */ +export interface TimingSummary { + dns: number + connect: number + tls: number + wait: number + receive: number + total: number +} + +/** + * Convert CDP `ResourceTiming` into relative millisecond phases. + * @param timing - the response timing object. + * @returns phase durations, or undefined when timing is unavailable. + */ +export function summarizeTiming(timing: unknown): TimingSummary | undefined { + const value = timing as Partial<{ + requestTime: number + dnsStart: number + dnsEnd: number + connectStart: number + connectEnd: number + sslStart: number + sslEnd: number + sendStart: number + sendEnd: number + receiveHeadersStart: number + receiveHeadersEnd: number + }> | null + if (value === null || value === undefined || typeof value.requestTime !== 'number') return undefined + const ms = (from: number | undefined, to: number | undefined): number => { + if (typeof from !== 'number' || typeof to !== 'number' || from < 0 || to < 0 || to < from) return 0 + return Math.round((to - from) * 1000 * 10) / 10 + } + const dns = ms(value.dnsStart, value.dnsEnd) + const connect = ms(value.connectStart, value.connectEnd) + const tls = ms(value.sslStart, value.sslEnd) + const receiveHeadersStart = typeof value.receiveHeadersStart === 'number' ? value.receiveHeadersStart : value.sendEnd + const wait = ms(receiveHeadersStart, value.receiveHeadersEnd) + // `requestTime` and the phase values share the monotonic clock, so the total + // is request-start to headers-received. Negative deltas (clock skew, missing + // phases) collapse to zero instead of reporting nonsense. + const totalMs = typeof value.receiveHeadersEnd === 'number' && value.receiveHeadersEnd >= value.requestTime + ? Math.round((value.receiveHeadersEnd - value.requestTime) * 1000 * 10) / 10 + : 0 + return { + dns, + connect, + tls, + wait, + receive: 0, + total: totalMs, + } +} + +/** + * Render a timing breakdown as one line. + * @param timing - phase breakdown from `ResourceTiming`. + * @param measuredMs - measured request duration, used when the phases carry no + * usable total (a cache hit reports zeros, which reads as "instant"). + */ +export function formatTiming(timing: TimingSummary | undefined, measuredMs?: number): string { + const measured = measuredMs === undefined ? '' : ` · measured ${measuredMs}ms` + if (timing === undefined) return `timing unavailable${measured}` + const total = timing.total > 0 ? timing.total : measuredMs + const renderedTotal = total === undefined || total === 0 ? 'unavailable' : `${total}ms` + return `dns ${timing.dns}ms · connect ${timing.connect}ms · tls ${timing.tls}ms · wait ${timing.wait}ms · total ${renderedTotal}` +} + +/** Format a byte count for display. */ +export function formatBytes(bytes: number | undefined): string { + if (bytes === undefined || !Number.isFinite(bytes) || bytes < 0) return '?' + if (bytes < 1024) return `${bytes} B` + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB` + return `${(bytes / (1024 * 1024)).toFixed(2)} MB` +} + +/** JSON.stringify that never throws on cycles or exotic values. */ +export function safeStringify(value: unknown): string { + try { + return JSON.stringify(value) ?? String(value) + } catch { + return Object.prototype.toString.call(value) + } +} diff --git a/extensions/dsh-browser/src/background/discovery.ts b/extensions/dsh-browser/src/background/discovery.ts new file mode 100644 index 000000000..f65a611b5 --- /dev/null +++ b/extensions/dsh-browser/src/background/discovery.ts @@ -0,0 +1,196 @@ +/** + * Bridge discovery for the side panel. + * + * DSH Desktop launches its host with `--port 0`, so the local Web port changes + * on every launch. Discovery therefore probes a bounded list of known ports — + * the URL that worked last time, the `dsh web` defaults, and the plugin's fixed + * discovery beacon (43189–43198), which every current host binds on its first + * free port. The platform's ephemeral range is never swept. + * + * The startup pass stops at the first answer; the settings page enumerates + * everything so the user can pick between several local instances. Nothing here + * schedules itself: the caller decides when a pass runs. + * + * @module + */ + +import { + BEACON_PORTS, + LEGACY_BRIDGE_PORTS, + BRIDGE_CONFIG_PATH, + parseBridgeConfigPayload, + type BridgeConfigPayload, +} from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' + +/** How long one known-port probe may take before it counts as absent. */ +export const GROUP_PROBE_TIMEOUT_MS = 1_500 + +/** Where a discovered instance came from. */ +export type InstanceSource = 'fixed' | 'beacon' + +/** One reachable dsh bridge. */ +export interface BridgeInstance { + /** Bridge WebSocket URL, as advertised by the host. */ + wsUrl: string + /** Port of that URL, for display and ordering. */ + port: number + /** Every path that found it; more than one means the answers agreed. */ + sources: InstanceSource[] + /** Host process id, when the plugin advertises it. */ + pid?: number + /** Host start time (ISO 8601), when the plugin advertises it. */ + startedAt?: string +} + +/** Progress of one discovering pass. */ +export interface DiscoveryProgress { + /** Ports probed so far. */ + done: number + /** Ports this pass intends to probe. */ + total: number + /** Instances found so far, in discovery order. */ + instances: BridgeInstance[] +} + +/** Side effects one pass needs; all injectable so the runner stays testable. */ +export interface DiscoveryEnv { + /** Read the discovery answer of one port, or `undefined` when nothing answers. */ + probe(port: number, timeoutMs: number): Promise + /** Whether the caller still wants results (panel lease, user cancellation). */ + shouldContinue(): boolean + /** Called once before the first probe and again after every group finishes. */ + onProgress?(progress: DiscoveryProgress): void +} + +/** One ordered batch of probes. */ +export interface DiscoveryGroup { + source: InstanceSource + ports: readonly number[] +} + +/** What one pass should probe, in order. */ +export interface DiscoveryPlan { + groups: DiscoveryGroup[] +} + +/** Inputs for {@link planDiscovery}. */ +export interface DiscoveryPlanOptions { + /** URL that worked last time; probed before the fixed list. */ + rememberedUrl?: string + /** URL already tried and known to fail, never probed again. */ + excludeUrl?: string +} + +/** Loopback URL of one port's discovery endpoint. */ +export function discoveryUrl(port: number): string { + return `http://127.0.0.1:${port}${BRIDGE_CONFIG_PATH}` +} + +/** Port of a bridge URL, or `undefined` when it does not parse. */ +export function portOfUrl(url: string | undefined): number | undefined { + if (url === undefined || url === '') return undefined + try { + const port = Number(new URL(url).port) + return Number.isInteger(port) && port > 0 ? port : undefined + } catch { + return undefined + } +} + +/** + * Read one discovery answer and validate it. + * @param port - loopback port to ask. + * @param timeoutMs - per-probe budget. + * @param fetchImpl - injectable fetch. + * @returns the advertised instance, or `undefined` when the port is not a dsh bridge. + */ +export async function readBridgeConfig( + port: number, + timeoutMs: number, + fetchImpl: typeof fetch = fetch, +): Promise { + try { + const response = await fetchImpl(discoveryUrl(port), { signal: AbortSignal.timeout(timeoutMs) }) + if (!response.ok) return undefined + return parseBridgeConfigPayload(await response.json()) + } catch { + return undefined + } +} + +/** + * Build the probe order for one pass. + * + * The remembered URL comes first (the common case: the next launch reused a + * port), then the `dsh web` defaults, then the beacon block. Ports appearing + * twice are probed once. + * @param options - remembered URL and an address to skip. + * @returns the ordered probe groups. + */ +export function planDiscovery(options: DiscoveryPlanOptions): DiscoveryPlan { + const probed = new Set() + const excluded = portOfUrl(options.excludeUrl) + if (excluded !== undefined) probed.add(excluded) + const groups: DiscoveryGroup[] = [] + const push = (source: InstanceSource, ports: readonly number[]): void => { + const fresh = ports.filter((port) => !probed.has(port)) + for (const port of fresh) probed.add(port) + if (fresh.length > 0) groups.push({ source, ports: fresh }) + } + const remembered = portOfUrl(options.rememberedUrl) + if (remembered !== undefined) push('fixed', [remembered]) + push('fixed', LEGACY_BRIDGE_PORTS) + push('beacon', BEACON_PORTS) + return { groups } +} + +/** + * Run one discovery pass. + * @param plan - probe order from {@link planDiscovery}. + * @param env - probe, cancellation, and progress hooks. + * @param options - `stopAtFirstHit` ends the pass as soon as a group answers (startup). + * @returns every instance found, in discovery order. + */ +export async function discoverInstances( + plan: DiscoveryPlan, + env: DiscoveryEnv, + options: { stopAtFirstHit: boolean }, +): Promise { + const found = new Map() + const record = (payload: BridgeConfigPayload, source: InstanceSource): void => { + const existing = found.get(payload.wsUrl) + if (existing === undefined) { + found.set(payload.wsUrl, { + wsUrl: payload.wsUrl, + port: portOfUrl(payload.wsUrl) ?? 0, + sources: [source], + ...(payload.pid === undefined ? {} : { pid: payload.pid }), + ...(payload.startedAt === undefined ? {} : { startedAt: payload.startedAt }), + }) + return + } + if (!existing.sources.includes(source)) existing.sources.push(source) + } + const snapshot = (): BridgeInstance[] => [...found.values()] + const total = plan.groups.reduce((sum, group) => sum + group.ports.length, 0) + let done = 0 + const report = (): void => { + env.onProgress?.({ done, total, instances: snapshot() }) + } + report() + + for (const group of plan.groups) { + if (!env.shouldContinue()) return snapshot() + const answers = await Promise.all(group.ports.map(async port => ({ + payload: await env.probe(port, GROUP_PROBE_TIMEOUT_MS), + }))) + for (const { payload } of answers) { + if (payload !== undefined) record(payload, group.source) + } + done += group.ports.length + report() + if (options.stopAtFirstHit && found.size > 0) return snapshot() + } + + return snapshot() +} diff --git a/extensions/dsh-browser/src/background/index.ts b/extensions/dsh-browser/src/background/index.ts index 2ca9ea6a3..531198664 100644 --- a/extensions/dsh-browser/src/background/index.ts +++ b/extensions/dsh-browser/src/background/index.ts @@ -10,17 +10,25 @@ * panel → bg: { type: 'rpc', id, method, payload } * panel → bg: { type: 'respond', id, rpcId, result } * panel → bg: { type: 'settings', id, settings: Partial } + * panel → bg: { type: 'devtools.detach', id } * panel → bg: { type: 'session.active', sessionId } * panel → bg: { type: 'approval.response', id, decision } * panel → bg: { type: 'tab-affinity.response', revision, decision, sessionId } * panel → bg: { type: 'tab-affinity.rebind', id } + * panel → bg: { type: 'tab-affinity.return', id, revision } * panel → bg: { type: 'panel.window', windowId } * panel → bg: { type: 'selection.clear', selection? } * panel → bg: { type: 'request-status' } + * panel → bg: { type: 'discovery.start', id } + * panel → bg: { type: 'discovery.cancel' } + * bg → panel: { type: 'discovery.progress', id, done, total, instances } + * bg → panel: { type: 'discovery.result', id, ok, instances? | error? } * bg → panel: { type: 'rpc.result', id, ok, result? | error? } * bg → panel: { type: 'respond.result', id, ok, result? | error? } * bg → panel: { type: 'settings.result', id, ok, error? } - * bg → panel: { type: 'status', state: BridgeState, caps? } + * bg → panel: { type: 'devtools.status', supported, enabled, attached, tabId?, error? } + * bg → panel: { type: 'devtools.detach.result', id, ok, error? } + * bg → panel: { type: 'status', state: BridgeState, caps?, discovery, address? } * bg → panel: { type: 'event', frame: ServerFrame } * bg → panel: { type: 'approval.request', request } * bg → panel: { type: 'approval.resolved', id } @@ -28,6 +36,7 @@ * bg → panel: { type: 'selection', selection } * bg → panel: { type: 'tab-affinity', state } * bg → panel: { type: 'tab-affinity.rebind.result', id, ok, error? } + * bg → panel: { type: 'tab-affinity.return.result', id, ok, error? } * * @module */ @@ -39,18 +48,23 @@ import { type RespondResult, } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' import type { ServerFrame } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' -import { BRIDGE_CONFIG_PATH, BRIDGE_PATH } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' +import { BRIDGE_CONFIG_PATH, BRIDGE_PATH, DEVTOOLS_CAPS_VERSION, parseBridgeConfigPayload } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' import { BridgeClient, type BridgeState } from './bridge.ts' import { createRpc } from './rpc.ts' +import { discoverInstances, planDiscovery, readBridgeConfig } from './discovery.ts' +import { frameDocumentKey, listTabFrames } from './frames.ts' import { dispatchOpenTab, dispatchToolCall, isNavigationCandidateTool, isTabManagementTool, + resetInputState, resetTabSnapshot, + type InputRuntime, type ToolAnswer, type ToolCall, } from './tools.ts' +import { createDebuggerBroker } from './debugger-broker.ts' import { isApprovalDecision, type ApprovalAuthorization, @@ -75,6 +89,9 @@ import { FocusedWindowTracker } from './focused-window.ts' import { SelectionTracker, type SelectionSource } from './selection.ts' import { parsePageSelection, parseSelectionCapture } from '../selection.ts' import { ApprovalCoordinator, type ApprovalRequestResult } from './approval-coordinator.ts' +import { devToolsSupported } from './debugger-session.ts' +import { createDevToolsController } from './devtools-controller.ts' +import { devToolsApprovalForCall, isDevToolsTool, resolveApprovalGate } from './authorization.ts' import { LEGACY_RECENT_SESSION_STORAGE_KEY, PAGE_SESSION_CONTEXT_STORAGE_KEY, @@ -88,6 +105,12 @@ export interface Settings { sharePageContent: 'ask' | 'auto' | 'off' /** Allow every browser operation without an approval prompt. */ unrestrictedBrowserAccess: boolean + /** + * High-privilege developer tools (element/CSS debugging, page JavaScript, + * network capture). Off by default: while it is off every devtools call + * raises its own approval prompt, and the origin allowlist never covers it. + */ + allowDevTools: boolean /** Origins whose state-changing actions may run without another prompt. */ trustedActionOrigins: string[] /** Show an OS notification when no side panel can display an approval. */ @@ -102,53 +125,137 @@ const SETTINGS_DEFAULTS: Settings = { token: '', sharePageContent: 'auto', unrestrictedBrowserAccess: false, + allowDevTools: false, trustedActionOrigins: [], approvalNotifications: true, autoResumeSession: true, } /** - * 自动探测的候选端口： - * - dsh web（CLI）默认 3080，端口被占时依次回退 3081 / 3090； - * - DSH Desktop 默认由系统随机分配本地 Web 端口（`dsh-desktop.port: 0`）， - * 用户指南推荐固定为 43189（见 deepseek-harness-desktop docs/user-guide）； - * - 14389 为历史桌面应用端口，保留兼容旧版。 + * 自动探测：DSH Desktop 启动 Host 时用 `--port 0`，本地 Web 端口每次启动都由系统 + * 随机分配，固定端口列表无法预知。因此按以下顺序探测： + * 上次成功的地址 → `dsh web` 默认端口 → 插件固定端口信标（43189–43198）。 + * 只在面板打开时跑一次、或用户点「自动检测」时跑；未找到即终态，不再重试。 */ -const DISCOVERY_PORTS = [3080, 3081, 3090, 14389, 43189] const LEGACY_LOCAL_URL = 'ws://127.0.0.1:3080' +const LAST_BRIDGE_URL_KEY = 'dshLastBridgeUrl' +const DISCOVERY_LATCH_KEY = 'dshDiscoveryLatch' + +/** 一次面板租约内的自动探测闸门；`not-found` 表示已确认没有服务，不再重试。 */ +type DiscoveryLatch = 'idle' | 'running' | 'not-found' + +/** 报给面板的探测状态，与传输层状态分开。 */ +type DiscoveryState = 'idle' | 'running' | 'found' | 'not-found' + +let discoveryLatch: DiscoveryLatch = 'idle' +let discoveryState: DiscoveryState = 'idle' +/** 上次成功连接的桥地址，下次探测时优先尝试。 */ +let lastBridgeUrl: string | undefined +/** 当前连接实际使用的地址（手动填写或探测得到）。 */ +let connectedAddress: string | undefined +/** 用户点「自动检测」发起的枚举，同一时间只跑一个。 */ +let activeScan: { id: string; controller: AbortController } | undefined +/** 回环地址判定：只有本机地址才在连接失败时回落到自动探测。 */ +function isLocalBridgeAddress(url: string): boolean { + try { + const host = new URL(url).hostname.toLowerCase() + return host === '127.0.0.1' || host === 'localhost' || host === '[::1]' || host === '::1' + } catch { + return false + } +} -/** 探测本机 dsh 的桥地址：fetch /ext/bridge-config 直到成功。 */ -async function discoverBridge(shouldContinue: () => boolean = () => true): Promise { - for (const port of DISCOVERY_PORTS) { - if (!shouldContinue()) return undefined - try { - const response = await fetch(`http://127.0.0.1:${port}/ext/bridge-config`, { - signal: AbortSignal.timeout(1_500), - }) - if (!shouldContinue()) return undefined - if (!response.ok) continue - const body = await response.json() as { wsUrl?: unknown } - if (typeof body.wsUrl === 'string' && body.wsUrl.startsWith('ws://')) return body.wsUrl - } catch { - // 该端口没有 dsh 或未挂桥：试下一个。 +/** 记下成功的地址，供下次面板打开时优先尝试。 */ +function rememberBridgeUrl(url: string): void { + if (lastBridgeUrl === url) return + lastBridgeUrl = url + void Promise.resolve(chrome.storage.local.set({ [LAST_BRIDGE_URL_KEY]: url })).catch(() => {}) +} + +/** 跨 service worker 重启保留「未找到」，避免后台重启后偷偷重试。 */ +function persistDiscoveryLatch(value: DiscoveryLatch): void { + const area = (chrome.storage as { session?: chrome.storage.StorageArea }).session + if (area === undefined) return + void Promise.resolve(area.set({ [DISCOVERY_LATCH_KEY]: value })).catch(() => {}) +} + +/** 恢复上次运行留下的闸门与最近地址。 */ +async function loadDiscoveryMemory(): Promise { + try { + const stored = await chrome.storage.local.get(LAST_BRIDGE_URL_KEY) + const remembered = stored?.[LAST_BRIDGE_URL_KEY] + if (typeof remembered === 'string' && remembered !== '') lastBridgeUrl = remembered + } catch { + // 记忆地址只是加速手段，读不到就按无记忆处理。 + } + const area = (chrome.storage as { session?: chrome.storage.StorageArea }).session + if (area === undefined) return + try { + const stored = await area.get(DISCOVERY_LATCH_KEY) + if (stored?.[DISCOVERY_LATCH_KEY] === 'not-found') { + discoveryLatch = 'not-found' + discoveryState = 'not-found' } + } catch { + // 同上：读不到按 idle 处理。 + } +} + +const discoveryMemory = loadDiscoveryMemory() + +/** 运行一次自动探测，返回首个命中的桥地址。 */ +async function discoverBridgeUrl( + shouldContinue: () => boolean, + excludeUrl?: string, +): Promise { + await discoveryMemory + if (!shouldContinue()) return undefined + const previousState = discoveryState + discoveryLatch = 'running' + discoveryState = 'running' + broadcastStatus() + try { + const instances = await discoverInstances(planDiscovery({ + ...(lastBridgeUrl === undefined ? {} : { rememberedUrl: lastBridgeUrl }), + ...(excludeUrl === undefined ? {} : { excludeUrl }), + }), { + probe: (port, timeoutMs) => readBridgeConfig(port, timeoutMs), + shouldContinue, + }, { stopAtFirstHit: true }) + // A pass that lost its lease (panel closed, newer settings) reports nothing + // and leaves the confirmed state of the lease that replaced it untouched. + if (!shouldContinue()) { + discoveryState = previousState + return undefined + } + const first = instances[0] + if (first === undefined) { + discoveryLatch = 'not-found' + discoveryState = 'not-found' + persistDiscoveryLatch('not-found') + return undefined + } + discoveryLatch = 'idle' + discoveryState = 'found' + rememberBridgeUrl(first.wsUrl) + return first.wsUrl + } finally { + if (discoveryLatch === 'running') discoveryLatch = 'idle' } - return undefined } /** Avoid opening a noisy loopback WebSocket until the local bridge responds. */ async function probeBridge(url: string): Promise { try { + if (!isLocalBridgeAddress(url)) return true const target = new URL(url) - if (target.hostname !== '127.0.0.1') return true target.protocol = target.protocol === 'wss:' ? 'https:' : 'http:' target.pathname = BRIDGE_CONFIG_PATH target.search = '' target.hash = '' const response = await fetch(target, { signal: AbortSignal.timeout(1_500) }) if (!response.ok) return false - const body = await response.json() as { wsUrl?: unknown } - return typeof body.wsUrl === 'string' && body.wsUrl.startsWith('ws://') + return parseBridgeConfigPayload(await response.json() as unknown) !== undefined } catch { return false } @@ -178,6 +285,52 @@ const transientEvents = new TransientEventCache() const tabAffinity = new TabAffinityController() const focusedWindow = new FocusedWindowTracker() const selections = new SelectionTracker() + +/** Whether this browser build can serve the developer-tools group at all. */ +const devToolsCapability = devToolsSupported() + +/** + * One debugger attachment per tab, shared by the developer tools and trusted + * input. Input holds it only around one call and lets it expire when idle. + */ +const debuggerBroker = createDebuggerBroker({ + onStateChange: () => { broadcastDevToolsState() }, +}) + +/** Trusted input channel handed to the tool dispatcher. */ +const inputRuntime: InputRuntime = { + supported: devToolsCapability, + async run(tabId, operation) { + const lease = await debuggerBroker.acquire(tabId, 'input') + try { + return await operation(lease.session) + } finally { + await lease.release() + } + }, +} + +/** + * High-privilege developer tools. The controller refuses every call unless the + * user's setting is on, and the background still routes each call through the + * approval gate first. + */ +const devTools = createDevToolsController({ + broker: debuggerBroker, + // Consent is enforced by the approval gate in `dispatchDevToolsTool`: with the + // setting off every call raises its own dialog, and unrelated switches never + // cover this group. The controller only owns capability and target policy. + resolveTarget: async (sessionId) => { + const target = await resolveToolTab(sessionId) + if ('ok' in target || target.id === undefined) return undefined + return { + tabId: target.id, + ...(target.url === undefined ? {} : { url: target.url }), + ...(target.title === undefined ? {} : { title: target.title }), + } + }, + onStateChange: () => { broadcastDevToolsState() }, +}) const pageSessionContexts = new PageSessionContextTracker({ read: async () => (await chrome.storage.session.get(PAGE_SESSION_CONTEXT_STORAGE_KEY))[PAGE_SESSION_CONTEXT_STORAGE_KEY], write: async (value) => { @@ -187,9 +340,18 @@ const pageSessionContexts = new PageSessionContextTracker({ void chrome.storage.session.remove(LEGACY_RECENT_SESSION_STORAGE_KEY).catch(() => {}) /** Ephemeral allowlist: cleared when the last side panel closes or this worker restarts. */ const sessionTrustedActionOrigins = new Set() +/** + * Ephemeral developer-tools grant for this panel lease. Set when the user + * chooses "trust this session" on a devtools prompt; never persisted, and + * never implied by the origin allowlist or unrestricted access. + */ +let sessionDevToolsGrant = false /** Tool calls that are either withdrawable or completing an already-dispatched action. */ interface ActiveToolCall { controller: AbortController + tabId?: number + /** Tool name, so a revocation can target the developer-tools group only. */ + name: string unrestrictedAccess: boolean committed: boolean revocationRequested: boolean @@ -221,6 +383,17 @@ class TabAffinityRebindError extends Error { } } +/** Stable failures of one explicit "jump back to the assistant page" request. */ +class TabAffinityReturnError extends Error { + constructor( + readonly code: 'stale-revision' | 'no-controlled-tab' | 'lost' | 'focus-failed' | 'activate-failed', + message: string, + ) { + super(message) + this.name = 'TabAffinityReturnError' + } +} + function rebindAbortReason(signal: AbortSignal): Error { return signal.reason instanceof Error ? signal.reason @@ -266,8 +439,11 @@ async function persistSettings(next: Partial): Promise { const accessRevision = changesUnrestrictedAccess ? ++unrestrictedAccessRevision : unrestrictedAccessRevision const updated = normalizeSettings({ ...settings, ...next }) const revokesUnrestrictedAccess = settings.unrestrictedBrowserAccess && !updated.unrestrictedBrowserAccess + const changesDevTools = typeof next.allowDevTools === 'boolean' + const revokesDevTools = changesDevTools && (settings.allowDevTools || sessionDevToolsGrant) && !updated.allowDevTools settings = updated if (!updated.unrestrictedBrowserAccess) unrestrictedAccessActive = false + if (revokesDevTools) sessionDevToolsGrant = false syncSelectionWatch() let accessTransition: Promise | undefined if (revokesUnrestrictedAccess) { @@ -297,6 +473,21 @@ async function persistSettings(next: Partial): Promise { unrestrictedAccessActive = true syncSelectionWatch() } + if (changesDevTools) { + // Turning the allowance off is a revocation: detach the debugger and abort + // every in-flight devtools call so the capability cannot outlive the choice. + if (revokesDevTools) { + cancelDevToolsCalls() + await devTools.revoke() + } else if (updated.allowDevTools) { + devTools.resume() + await prewarmDevTools() + } else { + // Still off: nothing to prewarm, and no consent was granted. + devTools.resume() + } + broadcastDevToolsState() + } } function normalizeSettings(candidate: Settings): Settings { @@ -310,6 +501,9 @@ function normalizeSettings(candidate: Settings): Settings { ...candidate, sharePageContent, unrestrictedBrowserAccess: candidate.unrestrictedBrowserAccess === true, + // A browser without a debugger API can never use the developer tools, so + // the persisted choice is kept but the runtime path stays disabled. + allowDevTools: candidate.allowDevTools === true && devToolsCapability, trustedActionOrigins: trusted, approvalNotifications: candidate.approvalNotifications !== false, autoResumeSession: candidate.autoResumeSession !== false, @@ -332,13 +526,50 @@ function disarmBridgeKeepalive(): void { void Promise.resolve(chrome.alarms.clear(BRIDGE_KEEPALIVE_ALARM)).catch(() => {}) } +/** One status snapshot: transport state, capabilities, and discovery state. */ +function statusPayload(): { + type: 'status' + state: BridgeState + caps: BridgeCaps | null + discovery: DiscoveryState + address?: string +} { + return { + type: 'status', + state: bridge?.state ?? ('stopped' as BridgeState), + caps, + discovery: discoveryState, + ...(connectedAddress === undefined ? {} : { address: connectedAddress }), + } +} + function broadcastStatus(): void { - const payload = { type: 'status', state: bridge?.state ?? ('stopped' as BridgeState), caps } + const payload = statusPayload() for (const port of panelPorts) { try { port.postMessage(payload) } catch { /* port already closed */ } } } +/** Broadcast developer-tools support, attachment state, and the user's choice. */ +function broadcastDevToolsState(): void { + for (const port of panelPorts) postDevToolsState(port) +} + +/** Report developer-tools state to one panel port. */ +function postDevToolsState(port: chrome.runtime.Port): void { + const state = devTools.state() + try { + port.postMessage({ + type: 'devtools.status', + supported: state.supported, + enabled: settingsLoaded && settings.allowDevTools, + attached: state.attached, + ...(state.tabId === undefined ? {} : { tabId: state.tabId }), + ...(state.error === undefined ? {} : { error: state.error }), + }) + } catch { /* port already closed */ } +} + function broadcastTabAffinity(): void { const payload = { type: 'tab-affinity', state: tabAffinity.snapshot() } for (const port of panelPorts) { @@ -405,6 +636,15 @@ function unrestrictedAccessEnabled(): boolean { return unrestrictedAccessActive && unrestrictedRevocation === undefined } +/** + * Whether the user has opted into developer tools without a per-call prompt. + * The ephemeral session grant is applied by the approval gate instead, so this + * reports only the persisted setting. + */ +function devToolsAllowed(): boolean { + return settingsLoaded && settings.allowDevTools +} + /** Page selections are captured only in a window with an open panel. */ function selectionWatchEnabled(windowId: number): boolean { return selectionSharingEnabled() && hasPanelInWindow(windowId) @@ -884,18 +1124,33 @@ async function authorizeToolCall( unrestrictedAccess: boolean = unrestrictedAccessEnabled(), ): Promise { if (signal.aborted) return 'cancelled' - if (unrestrictedAccess) return 'approved' - if (actionCoveredByTrustedOrigins( + const gate = resolveApprovalGate({ prompt, - sessionTrustedActionOrigins, - settings.trustedActionOrigins, - )) { - return 'approved' - } + devToolsAllowed: devToolsAllowed(), + devToolsSessionGrant: sessionDevToolsGrant, + unrestrictedAccess, + coveredByTrustedOrigins: actionCoveredByTrustedOrigins( + prompt, + sessionTrustedActionOrigins, + settings.trustedActionOrigins, + ), + }) + if (gate !== 'prompt') return gate const result: ApprovalRequestResult = await approvals.request(prompt, signal, windowId, sessionId) if (signal.aborted) return 'cancelled' if (result.status !== 'decision') return result.status const { decision } = result + if (prompt.devTools === true) { + // 授权持续到侧栏关闭、worker 终止或用户修改设置。 + if (decision === 'trust-session') { + if (panelPorts.size === 0) return 'cancelled' + sessionDevToolsGrant = true + devTools.resume() + return 'approved' + } + if (decision === 'allow-once') devTools.resume() + return decision === 'allow-once' ? 'approved' : 'denied' + } if (decision === 'always-allow-reads' && prompt.kind === 'read') { await persistSettings({ sharePageContent: 'auto' }) return 'approved' @@ -904,13 +1159,6 @@ async function authorizeToolCall( sessionTrustedActionOrigins.add(prompt.origins[0]!) return 'approved' } - // Retain wire compatibility with panels from the previous build. The new UI - // manages permanent trust explicitly in Settings instead of offering it in - // the action dialog. - if (decision === 'trust-origin' && prompt.kind === 'action' && prompt.canTrust && prompt.origins.length === 1) { - await persistSettings({ trustedActionOrigins: [...settings.trustedActionOrigins, prompt.origins[0]!] }) - return 'approved' - } return decision === 'allow-once' ? 'approved' : 'denied' } @@ -988,6 +1236,74 @@ async function resolveTabAffinityResponse(response: { } } +/** + * Bring the user's own visible tab back to the page the assistant is bound to. + * + * This is the only path that moves the user's visible tab, and it only runs + * after an explicit click. The binding never changes: the controlled tab is + * still exactly the page the user already approved, so observing it as the + * active tab resolves the handoff back to `following` without widening what any + * tool may touch. Every failure leaves the binding and the visible tab alone + * (except a vanished tab, which fails closed into `lost`). + */ +async function returnToControlledTab(revision: number, panelWindowId: number | undefined): Promise { + await affinityReady + const state = tabAffinity.snapshot() + const controlled = state.controlled + if (controlled === null) { + throw new TabAffinityReturnError('no-controlled-tab', getUiLocale() === 'zh' + ? '助手没有可返回的页面' + : 'The assistant has no page to return to') + } + if (state.revision !== revision) { + throw new TabAffinityReturnError('stale-revision', getUiLocale() === 'zh' + ? '标签页状态已变化，请重新选择' + : 'The tab state changed, so this choice no longer applies') + } + const tabId = controlled.tabId + let live: chrome.tabs.Tab + try { + // Read the tab itself rather than the snapshot: the user may have dragged it + // into another window since the binding was recorded. + live = await chrome.tabs.get(tabId) + } catch { + // The approved page is gone: fail closed and let every panel show `lost`. + const affectedSessions = tabAffinity.sessionIdsForTab(tabId) + if (tabAffinity.removeTab(tabId)) { + for (const sid of affectedSessions) { + activeFollowRefreshes.get(sid)?.controller.abort() + cancelPendingApprovals(sid) + } + persistTabAffinity() + broadcastTabAffinity() + } + throw new TabAffinityReturnError('lost', getUiLocale() === 'zh' + ? '受控标签页已关闭，无法返回' + : 'The controlled tab was closed, so there is no page to return to') + } + if (panelWindowId === undefined || live.windowId !== panelWindowId) { + try { + await chrome.windows.update(live.windowId, { focused: true }) + } catch { + throw new TabAffinityReturnError('focus-failed', getUiLocale() === 'zh' + ? '无法切换到助手页面所在的窗口' + : 'The window holding the assistant page could not be focused') + } + } + try { + await chrome.tabs.update(tabId, { active: true }) + } catch { + throw new TabAffinityReturnError('activate-failed', getUiLocale() === 'zh' + ? '无法切换到助手页面' + : 'The assistant page could not be activated') + } + // Chrome also emits tabs.onActivated, but that path only accepts activations + // from the focused window. Observing the tab directly keeps the state + // deterministic for the panel that is waiting on this acknowledgement. + const observed = await chrome.tabs.get(tabId).then(summarizeTab, () => null) + if (observed !== null) observeActiveSummary(observed) +} + /** Move browser control to the current tab only after a fresh, valid query. */ async function rebindTabAffinityToActive(signal: AbortSignal, sessionId?: string): Promise { await abortable(Promise.all([affinityReady, pageSessionContexts.ready]).then(() => undefined), signal) @@ -1063,6 +1379,7 @@ function routeToolCall(call: ToolCall): void { let settle!: () => void const activeCall: ActiveToolCall = { controller, + name: call.name, unrestrictedAccess, committed: false, revocationRequested: false, @@ -1109,6 +1426,7 @@ function routeToolCall(call: ToolCall): void { unrestrictedAccess, ...(controlledTabId === undefined ? {} : { controlledTabId }), followTab: (tab) => followModelSelectedTab(tab, call.sessionId), + input: inputRuntime, commitAction, rollbackActionCommit, }, @@ -1116,6 +1434,8 @@ function routeToolCall(call: ToolCall): void { } void (isTabManagementTool(call.name) ? managementDispatch() + : isDevToolsTool(call.name) + ? dispatchDevToolsTool(call, controller.signal) : call.name === 'browser_open_tab' ? resolveOpenTabWindow(call.sessionId).then((target) => 'ok' in target ? target @@ -1140,7 +1460,7 @@ function routeToolCall(call: ToolCall): void { controller.signal, target, () => target.id !== undefined && tabAffinity.allowsTarget(target.id, call.sessionId), - { unrestrictedAccess, commitAction, rollbackActionCommit }, + { unrestrictedAccess, input: inputRuntime, commitAction, rollbackActionCommit }, )) ).then( async (answer) => { @@ -1203,6 +1523,96 @@ function cancelToolCall(id: string): void { if (call !== undefined && !call.committed) call.controller.abort() } +/** + * Run one developer-tools call: approve it, then hand it to the debugger + * controller. Devtools calls take a dedicated path because they never touch the + * content script and must not be satisfied by the page-sharing relaxations. + */ +async function dispatchDevToolsTool(call: ToolCall, signal: AbortSignal): Promise { + if (signal.aborted) return devToolsCancelled() + const target = await resolveToolTab(call.sessionId) + if ('ok' in target) return target + const activeCall = activeToolCalls.get(call.id) + if (activeCall !== undefined) activeCall.tabId = target.id + if (signal.aborted) return devToolsCancelled() + const frames = await listTabFrames(target.id!, target.url) + const prompt = devToolsApprovalForCall(call, frames) + const authorization = await authorizeToolCall(prompt, signal, target.windowId ?? 0, call.sessionId) + if (signal.aborted) return devToolsCancelled() + if (authorization !== 'approved') { + return { + ok: false, + error: { code: authorization === 'timed-out' ? 'timeout' : 'action-failed', message: approvalFailureMessage(prompt, authorization) }, + } + } + try { + const validate = async (executionTarget: { tabId: number }): Promise => { + if (signal.aborted) throw new Error('The developer-tools call was cancelled.') + const current = await resolveToolTab(call.sessionId) + if ('ok' in current || current.id !== target.id || executionTarget.tabId !== target.id) { + throw new Error('The controlled tab changed while approval was pending. Request approval again.') + } + const currentFrames = await listTabFrames(target.id!, current.url) + if (signal.aborted || frames.length !== currentFrames.length || frames.some((frame) => { + const currentFrame = currentFrames.find((candidate) => candidate.frameId === frame.frameId) + return currentFrame === undefined || frameDocumentKey(frame) !== frameDocumentKey(currentFrame) + })) { + throw new Error('The page changed while approval was pending. Request approval again.') + } + } + return { ok: true, result: await devTools.handle(call.name, call.args, call.sessionId, validate) } + } catch (error: unknown) { + return { + ok: false, + error: { + code: signal.aborted ? 'bridge-closed' : 'action-failed', + message: error instanceof Error ? error.message : String(error), + }, + } + } +} + +function devToolsCancelled(): ToolAnswer { + return { ok: false, error: { code: 'bridge-closed', message: 'The browser tool call was cancelled.' } } +} + +/** Abort every in-flight developer-tools call (setting revoked). */ +function cancelDevToolsCalls(tabId?: number): void { + for (const call of activeToolCalls.values()) { + if (isDevToolsTool(call.name) && !call.committed && (tabId === undefined || call.tabId === tabId)) { + call.controller.abort() + } + } +} + +/** + * Attach the debugger to the controlled tab as soon as the user enables the + * allowance, so network capture sees traffic from the start instead of only + * after the first devtools call. + */ +async function prewarmDevTools(): Promise { + await affinityReady + const target = await resolveToolTab() + if ('ok' in target || target.id === undefined) return + await devTools.prewarm(target.id).catch(() => { + // Attach is lazy by design: a tab that cannot be debugged right now only + // surfaces an error when a devtools call actually needs it. + }) +} + +function approvalFailureMessage(prompt: ApprovalPrompt, authorization: Exclude): string { + switch (authorization) { + case 'denied': + return `The user denied the browser approval request for "${prompt.action}".` + case 'unavailable': + return `No browser side panel was available to receive or complete the approval request for "${prompt.action}".` + case 'timed-out': + return `The browser approval request for "${prompt.action}" timed out before the user responded.` + case 'cancelled': + return `The browser approval request for "${prompt.action}" was cancelled.` + } +} + async function revokeUnrestrictedAccess(): Promise { const calls = [...unsettledToolCalls].filter((call) => call.unrestrictedAccess) const refreshes = [...activeFollowRefreshes.values()].filter((refresh) => refresh.unrestrictedAccess) @@ -1224,21 +1634,42 @@ function cancelAllToolCalls(): void { activeToolCalls.clear() } -/** (Re)start the bridge with the current settings. 零配置：地址留空时自动探测；回环连接无需 token。 */ +/** + * (Re)start the bridge with the current settings. + * + * Order of preference: a filled-in address is tried first (it is the user's + * choice), and only a *loopback* address that fails its pre-flight falls back + * to one automatic discovery pass. With no address the pass runs directly. + * Nothing retries once that pass finds nothing. + */ async function startBridge(): Promise { - const revision = ++bridgeStartRevision if (panelPorts.size === 0) return + // One automatic pass per lease: a second panel in another window joins it + // instead of launching a competing scan. Checked before the revision bump so + // a duplicate trigger cannot invalidate the pass it is deferring to. + if (discoveryLatch === 'running') return + const revision = ++bridgeStartRevision + const lease = (): boolean => revision === bridgeStartRevision && panelPorts.size > 0 let url = settings.bridgeUrl - if (url === '') { - url = await discoverBridge(() => revision === bridgeStartRevision && panelPorts.size > 0) ?? '' + if (url !== '') { + if (isLocalBridgeAddress(url) && !await probeBridge(url)) { + if (!lease()) return + url = await discoverBridgeUrl(lease, url) ?? '' + } + } else if (discoveryLatch !== 'not-found') { + url = await discoverBridgeUrl(lease) ?? '' + } else { + // Already confirmed absent in this lease: report and wait for the user. + url = '' } // Discovery is asynchronous. A panel may have closed or a newer settings // update may have started while its fetches were in flight. - if (revision !== bridgeStartRevision || panelPorts.size === 0) return + if (!lease()) return if (url === '') { bridge?.stop() bridge = null rpc = null + connectedAddress = undefined broadcastStatus() return } @@ -1251,6 +1682,7 @@ async function startBridge(): Promise { } catch { // 非法 URL 原样交给 WebSocket 构造函数报错。 } + connectedAddress = url if (bridge === null) { const client = new BridgeClient({ onStateChange: (state) => { @@ -1278,13 +1710,89 @@ async function startBridge(): Promise { broadcastStatus() void pushBudgetToControlledTab(negotiated) }, - }, probeBridge, () => panelPorts.size > 0) + }, probeBridge, () => panelPorts.size > 0, { + // Advertised as soon as the platform can serve it, so the host registers + // the developer-tools group; the user's setting and per-call approval + // still gate every one of those calls. + ...devToolsCapability ? { devTools: { version: DEVTOOLS_CAPS_VERSION } } : {}, + }) bridge = client rpc = createRpc(client) } bridge.start(url, settings.token) } +/** + * Enumerate every local instance for the settings picker. + * + * Unlike the startup pass this never stops early: every fixed and beacon group + * is probed so the picker can list every local instance. + * Results stream back to the requesting panel; cancellation ends the pass. + * @param port - panel port that asked. + * @param id - request id echoed on every message. + */ +async function runPanelDiscovery(port: chrome.runtime.Port, id: string): Promise { + activeScan?.controller.abort() + const controller = new AbortController() + activeScan = { id, controller } + discoveryLatch = 'idle' + discoveryState = 'running' + persistDiscoveryLatch('idle') + broadcastStatus() + const reply = (message: unknown): void => { + try { port.postMessage(message) } catch { /* port closed */ } + } + try { + await discoveryMemory + const instances = await discoverInstances(planDiscovery({ + ...(lastBridgeUrl === undefined ? {} : { rememberedUrl: lastBridgeUrl }), + }), { + probe: (candidate, timeoutMs) => readBridgeConfig(candidate, timeoutMs), + shouldContinue: () => !controller.signal.aborted, + onProgress: (progress) => { + if (activeScan?.id !== id) return + reply({ + type: 'discovery.progress', + id, + done: progress.done, + total: progress.total, + instances: progress.instances, + }) + }, + }, { stopAtFirstHit: false }) + if (activeScan?.id !== id) return + if (controller.signal.aborted) { + // A cancelled scan keeps whatever state the bridge already had. + discoveryState = bridge?.state === 'connected' ? 'found' : 'idle' + broadcastStatus() + reply({ type: 'discovery.result', id, ok: false, error: { code: 'cancelled', message: 'discovery cancelled' } }) + return + } + if (instances.length > 0) { + discoveryLatch = 'idle' + discoveryState = 'found' + rememberBridgeUrl(instances[0]!.wsUrl) + } else { + discoveryLatch = 'not-found' + discoveryState = 'not-found' + persistDiscoveryLatch('not-found') + } + broadcastStatus() + reply({ type: 'discovery.result', id, ok: true, instances }) + } catch (error) { + if (activeScan?.id !== id) return + discoveryLatch = 'idle' + reply({ + type: 'discovery.result', + id, + ok: false, + error: { code: 'scan-failed', message: error instanceof Error ? error.message : String(error) }, + }) + } finally { + if (activeScan?.id === id) activeScan = undefined + } +} + /** Gateway RPC with a helpful error when the bridge is down. */ async function gatewayRpc(method: string, payload: unknown): Promise { if (rpc === null || bridge === null || !bridge.connected) { @@ -1326,14 +1834,21 @@ chrome.runtime.onConnect.addListener((port) => { controller: AbortController timer: ReturnType }>() + // In-flight "jump back to the assistant page" requests. Activation is + // idempotent, but a double click must not race two acknowledgements. + const tabAffinityReturns = new Set() panelPorts.add(port) if (wasIdle) armBridgeKeepalive() void settingsReady.then(syncSelectionWatch) void settingsReady.then(() => { if (!panelPorts.has(port)) return + // One automatic pass per panel lease: a confirmed miss stays terminal until + // the lease ends or the user asks again (settings save / 自动检测). + if (discoveryLatch === 'not-found') return if (bridge === null || bridge.state === 'stopped') return startBridge() }) - try { port.postMessage({ type: 'status', state: bridge?.state ?? ('stopped' as BridgeState), caps }) } catch { /* port closed */ } + try { port.postMessage(statusPayload()) } catch { /* port closed */ } + broadcastDevToolsState() void affinityReady.then(async () => { await syncActiveTab() try { port.postMessage({ type: 'tab-affinity', state: tabAffinity.snapshot() }) } catch { /* port closed */ } @@ -1402,6 +1917,11 @@ chrome.runtime.onConnect.addListener((port) => { await persistSettings(settingsMsg.settings) const connectionChanged = settings.bridgeUrl !== previousConnection.bridgeUrl || settings.token !== previousConnection.token + if (connectionChanged) { + // An explicit address supersedes a confirmed miss from this lease. + discoveryLatch = 'idle' + persistDiscoveryLatch('idle') + } if (panelPorts.size > 0) { if (connectionChanged) await startBridge() broadcastStatus() @@ -1448,6 +1968,42 @@ chrome.runtime.onConnect.addListener((port) => { void postResumeHint(port, registration.windowId) break } + case 'devtools.detach': { + // The user asked for an immediate detach from the settings page; keep + // the persisted allowance but drop the current attachment. + const request = message as { id?: unknown } + void devTools.detachAll().then( + () => { + broadcastDevToolsState() + if (typeof request.id === 'string') { + try { port.postMessage({ type: 'devtools.detach.result', id: request.id, ok: true }) } catch { /* port closed */ } + } + }, + (error: unknown) => { + if (typeof request.id !== 'string') return + try { + port.postMessage({ + type: 'devtools.detach.result', + id: request.id, + ok: false, + error: { message: error instanceof Error ? error.message : String(error) }, + }) + } catch { /* port closed */ } + }, + ) + break + } + case 'discovery.start': { + // Manual scan from the settings page: enumerate every local instance. + const request = message as { id?: unknown } + if (typeof request.id !== 'string') break + void runPanelDiscovery(port, request.id) + break + } + case 'discovery.cancel': { + activeScan?.controller.abort() + break + } case 'selection.clear': { // The user sent, dismissed, or explicitly abandoned this window's // quote. A send/dismiss names the value it acted on so a newer capture @@ -1560,9 +2116,49 @@ chrome.runtime.onConnect.addListener((port) => { }) break } + case 'tab-affinity.return': { + const request = message as { id?: unknown; revision?: unknown } + if (typeof request.id !== 'string') break + const requestId = request.id + if (tabAffinityReturns.has(requestId)) break + if (typeof request.revision !== 'number') { + try { + port.postMessage({ + type: 'tab-affinity.return.result', + id: requestId, + ok: false, + error: { code: 'bad-request', message: 'the tab affinity revision is missing' }, + }) + } catch { /* port closed */ } + break + } + tabAffinityReturns.add(requestId) + void returnToControlledTab(request.revision, panelWindows.get(port)).then( + () => { + try { port.postMessage({ type: 'tab-affinity.return.result', id: requestId, ok: true }) } catch { /* port closed */ } + }, + (error: unknown) => { + try { + port.postMessage({ + type: 'tab-affinity.return.result', + id: requestId, + ok: false, + error: { + code: error instanceof TabAffinityReturnError ? error.code : 'return-failed', + message: error instanceof Error ? error.message : String(error), + }, + }) + } catch { /* port closed */ } + }, + ).finally(() => { tabAffinityReturns.delete(requestId) }) + break + } case 'request-status': try { port.postMessage({ type: 'status', state: bridge?.state ?? ('stopped' as BridgeState), caps }) + // A push on connect cannot reach a panel that has not subscribed yet, + // so the status request is also the pull path for developer tools. + postDevToolsState(port) port.postMessage({ type: 'tab-affinity', state: tabAffinity.snapshot() }) const statusWindowId = panelWindows.get(port) if (statusWindowId !== undefined) { @@ -1587,6 +2183,7 @@ chrome.runtime.onConnect.addListener((port) => { : 'The background connection was lost, so tab binding was cancelled')) } tabAffinityRebinds.clear() + tabAffinityReturns.clear() const panelWindowId = panelWindows.get(port) const panelSessionId = panelActiveSessions.get(port) panelActiveSessions.delete(port) @@ -1599,10 +2196,18 @@ chrome.runtime.onConnect.addListener((port) => { } syncSelectionWatch() if (panelPorts.size === 0) { + sessionDevToolsGrant = false + cancelDevToolsCalls() bridgeStartRevision += 1 bridge?.suspendReconnect() sessionTrustedActionOrigins.clear() approvals.notifyPending() + // A fresh panel lease re-runs the automatic pass, so a miss is not final + // for the browser session — only for the lease it was observed in. + activeScan?.controller.abort() + discoveryLatch = 'idle' + discoveryState = 'idle' + persistDiscoveryLatch('idle') if (bridge?.state !== 'connected') disarmBridgeKeepalive() } }) @@ -1691,7 +2296,24 @@ chrome.tabs.onRemoved.addListener((tabId) => { // Matching the exact frame keeps an iframe's navigation from invalidating a // quote taken from its parent page, and vice versa. chrome.webNavigation.onCommitted.addListener(({ tabId, frameId }) => { + cancelDevToolsCalls(tabId) broadcastSelections(selections.clearTab(tabId, frameId)) + // A new top-level document drops any modifier or button held by a trusted + // input sequence; keeping it would leak state into the next page. + if (frameId === 0) { + resetInputState(tabId) + // Only the input hold is stale after a navigation. The developer-tools + // capture must survive navigation (it re-syncs frames and keeps buffering), + // so never detach the whole tab here. + void debuggerBroker.detachOwner(tabId, 'input').catch(() => undefined) + } +}) + +// A closed tab cannot stay debugged; release the attachment immediately so the +// "being debugged" banner never outlives its page. +chrome.tabs.onRemoved.addListener((tabId) => { + resetInputState(tabId) + devTools.releaseTab(tabId) }) // Ports are cleaned up by their own disconnect; only the window's quote is @@ -1710,6 +2332,9 @@ chrome.windows.onFocusChanged.addListener((windowId) => { chrome.alarms.onAlarm.addListener((alarm) => { if (alarm.name !== BRIDGE_KEEPALIVE_ALARM) return + // A confirmed miss is terminal for this lease: do not re-probe in the + // background, and do not wake the worker for it either. + if (discoveryLatch === 'not-found') return if (panelPorts.size === 0) { if (bridge === null || bridge.state !== 'connected') disarmBridgeKeepalive() return diff --git a/extensions/dsh-browser/src/background/input-actions.ts b/extensions/dsh-browser/src/background/input-actions.ts new file mode 100644 index 000000000..e520a1592 --- /dev/null +++ b/extensions/dsh-browser/src/background/input-actions.ts @@ -0,0 +1,261 @@ +/** + * Trusted input dispatch over CDP. + * + * Sends keyboard events, inserts text, and attaches files. Target selection, + * focus emulation, and approval are handled by the caller. + * + * @module + */ + +import type { CdpClient } from './debugger-session.ts' +import { resolveFrameNode } from './debugger-frame.ts' +import { + MODIFIER_BITS, + modifierMask, + strokeForKey, + strokeInsertsText, + type ModifierName, + type KeyStroke, +} from './keys.ts' +import type { InputState } from './input-state.ts' + +/** Stable failure codes for trusted input. */ +export type InputErrorCode = 'input-unavailable' | 'bad-args' | 'action-failed' + +/** One trusted-input failure. */ +export class InputError extends Error { + constructor( + readonly code: InputErrorCode, + message: string, + ) { + super(message) + this.name = 'InputError' + } +} + +/** Largest single `Input.insertText` payload; longer text is chunked. */ +const INSERT_TEXT_CHUNK = 8_192 + +/** One keyboard request. */ +export interface KeyRequest { + key: string + action?: 'press' | 'down' | 'up' + modifiers?: ModifierName[] + repeat?: number + /** Text this press inserts, overriding the key table's character (uppercase typing). */ + text?: string +} + +/** + * macOS editing shortcuts are application-menu commands, not renderer editing: + * a plain CDP key event never reaches them. Naming the command on the key-down + * is what Chrome's own DevTools does for these combinations. + */ +const MAC_EDITING_COMMANDS: Readonly> = { + a: 'selectAll', + c: 'copy', + v: 'paste', + x: 'cut', + z: 'undo', +} + +/** + * The editing command a key-down should name, when the platform needs one. + * + * @param key - pressed key (KeyboardEvent.key semantics). + * @param modifiers - active modifiers. + * @param agent - user agent of the browser running the page. + * @returns the command name, or undefined when a plain key event suffices. + */ +export function editingCommandFor(key: string, modifiers: readonly ModifierName[], agent: string): string | undefined { + if (!/mac|iphone|ipad|ipod/i.test(agent)) return undefined + if (!modifiers.includes('Meta')) return undefined + const lower = key.toLowerCase() + if (lower === 'z' && modifiers.includes('Shift')) return 'redo' + return MAC_EDITING_COMMANDS[lower] +} + +/** The platform string trusted input uses when the caller does not supply one. */ +function defaultAgent(): string { + return typeof navigator === 'undefined' ? '' : navigator.userAgent +} + +/** Resolve a key name or fail with the supported vocabulary. */ +function requireStroke(key: string): KeyStroke { + const stroke = strokeForKey(key) + if (stroke === undefined) { + throw new InputError('bad-args', `Unsupported key "${key}".`) + } + return stroke +} + +/** Union of held modifiers and the request's own modifiers. */ +function combinedMask(state: InputState, modifiers: readonly ModifierName[] | undefined): number { + return state.modifiersMask() | modifierMask(modifiers ?? []) +} + +/** Keys that are modifiers themselves, so press/release updates the state. */ +const MODIFIER_KEYS = new Set(['Alt', 'Control', 'Meta', 'Shift']) + +/** + * Send one keyboard event sequence. + * + * @param client - attached CDP client. + * @param state - per-tab input state. + * @param request - key, action, and modifiers. + */ +export async function dispatchKey( + client: CdpClient, + state: InputState, + request: KeyRequest, + agent: string = defaultAgent(), +): Promise { + const stroke = requireStroke(request.key) + const action = request.action ?? 'press' + const repeat = request.repeat ?? 1 + if (!Number.isInteger(repeat) || repeat < 1 || repeat > 100) { + throw new InputError('bad-args', 'repeat must be an integer between 1 and 100.') + } + const modifiers = request.modifiers ?? [] + const mask = combinedMask(state, modifiers) + if (MODIFIER_KEYS.has(stroke.key as ModifierName)) { + state.setModifier(stroke.key as ModifierName, action !== 'up') + } + const overridden = (mask & (MODIFIER_BITS.Control | MODIFIER_BITS.Meta | MODIFIER_BITS.Alt)) === 0 + ? request.text + : undefined + const text = overridden ?? (strokeInsertsText(stroke, mask) ? stroke.text : undefined) + const command = action === 'up' ? undefined : editingCommandFor(stroke.key, modifiers, agent) + for (let count = 0; count < repeat; count += 1) { + if (action !== 'up') { + await client.send('Input.dispatchKeyEvent', { + type: text === undefined ? 'rawKeyDown' : 'keyDown', + key: stroke.key, + code: stroke.code, + windowsVirtualKeyCode: stroke.windowsVirtualKeyCode, + modifiers: mask, + ...(text === undefined ? {} : { text }), + ...(command === undefined ? {} : { commands: [command] }), + ...(count > 0 ? { autoRepeat: true } : {}), + }) + } + if (action !== 'down') { + await client.send('Input.dispatchKeyEvent', { + type: 'keyUp', + key: stroke.key, + code: stroke.code, + windowsVirtualKeyCode: stroke.windowsVirtualKeyCode, + modifiers: mask, + }) + } + } +} + +/** + * Type text as a sequence of key events, one character at a time. + * + * Some pages react to `keydown`/`keyup` per character (autocomplete, shortcut + * hints) and ignore a single insert. Characters the key table cannot express + * (CJK, emoji, shifted punctuation) fall back to `insertText` one at a time. + * + * @param client - attached CDP client. + * @param state - per-tab input state. + * @param text - text to type. + */ +export async function typeAsKeystrokes(client: CdpClient, state: InputState, text: string): Promise { + for (const character of text) { + const upper = character.length === 1 && character !== character.toLowerCase() && /^[A-Z]$/.test(character) + const stroke = strokeForKey(character) + if (stroke === undefined || stroke.text === undefined) { + await insertTextAtCaret(client, character) + continue + } + if (upper) { + // Shift changes the physical key but CDP does not transform `text`; send + // the uppercase character explicitly so the insertion matches the key. + await dispatchKey(client, state, { key: character.toLowerCase(), modifiers: ['Shift'], text: character }) + continue + } + await dispatchKey(client, state, { key: character }) + } +} + +/** + * Insert text at the current caret, bypassing keyboard layout. + * + * @param client - attached CDP client. + * @param text - text to insert; long strings are chunked. + */ +export async function insertTextAtCaret(client: CdpClient, text: string): Promise { + for (let offset = 0; offset < text.length; offset += INSERT_TEXT_CHUNK) { + await client.send('Input.insertText', { text: text.slice(offset, offset + INSERT_TEXT_CHUNK) }) + } +} + +/** + * Whether this platform's paste shortcut uses Meta (macOS) or Control. + * + * @param agent - a `navigator.userAgent`-shaped string (or `navigator.platform`). + * @returns the modifier to hold while pressing `v`. + */ +export function pasteShortcutModifier(agent: string): ModifierName { + return /mac|iphone|ipad|ipod/i.test(agent) ? 'Meta' : 'Control' +} + +/** + * Send the platform's paste shortcut to the focused element. + * + * @param client - attached CDP client. + * @param state - per-tab input state. + * @param agent - platform string used to choose the modifier. + */ +export async function sendPasteShortcut(client: CdpClient, state: InputState, agent: string): Promise { + const modifier = pasteShortcutModifier(agent) + await dispatchKey(client, state, { key: 'v', modifiers: [modifier], repeat: 1 }) +} + +/** + * Toggle focus emulation around one trusted-input sequence. + * + * Background tabs report `document.hasFocus() === false`, and some editors + * gate typing on it. Emulation makes the page behave as focused without + * actually focusing the window; the switch is always restored. + * + * @param client - attached CDP client. + * @param run - the input sequence. + * @returns whatever `run` returns. + */ +export async function withFocusEmulation(client: CdpClient, run: () => Promise): Promise { + await client.send('Emulation.setFocusEmulationEnabled', { enabled: true }) + try { + return await run() + } finally { + await client.send('Emulation.setFocusEmulationEnabled', { enabled: false }) + } +} + +/** + * Assign files to a file input addressed by a unique selector. + * + * The browser process reads the paths; the extension never reads file content. + * + * @param client - attached CDP client. + * @param selector - selector produced by the content script's upload marker. + * @param paths - host paths to attach. + */ +export async function setInputFiles(client: CdpClient, selector: string, paths: readonly string[], uniqueContextId?: string): Promise { + if (paths.length === 0) throw new InputError('bad-args', 'paths must contain at least one local file path.') + if (uniqueContextId !== undefined) { + const nodeId = await resolveFrameNode(client, selector, uniqueContextId) + await client.send('DOM.setFileInputFiles', { files: [...paths], nodeId }) + return + } + const document = await client.send('DOM.getDocument', { depth: -1, pierce: true }) as { root?: { nodeId?: number } } + const rootNodeId = document.root?.nodeId + if (rootNodeId === undefined) throw new InputError('action-failed', 'The page document could not be resolved.') + const found = await client.send('DOM.querySelector', { nodeId: rootNodeId, selector }) as { nodeId?: number } + if (found.nodeId === undefined || found.nodeId === 0) { + throw new InputError('action-failed', `No file input matched ${selector}.`) + } + await client.send('DOM.setFileInputFiles', { files: [...paths], nodeId: found.nodeId }) +} diff --git a/extensions/dsh-browser/src/background/input-state.ts b/extensions/dsh-browser/src/background/input-state.ts new file mode 100644 index 000000000..855ed582e --- /dev/null +++ b/extensions/dsh-browser/src/background/input-state.ts @@ -0,0 +1,66 @@ +/** + * Tracks held keyboard modifiers per tab and resets them on navigation. + * + * @module + */ + +import { MODIFIER_BITS, modifierMask, type ModifierName } from './keys.ts' + +/** One tab's input state. */ +export class InputState { + private heldModifiers = new Set() + + /** Record a modifier key going down or up. */ + setModifier(modifier: ModifierName, pressed: boolean): void { + if (pressed) this.heldModifiers.add(modifier) + else this.heldModifiers.delete(modifier) + } + + /** @returns whether the modifier is currently held. */ + isModifierHeld(modifier: ModifierName): boolean { + return this.heldModifiers.has(modifier) + } + + /** @returns the held modifiers as a CDP bitmask. */ + modifiersMask(): number { + return modifierMask([...this.heldModifiers]) + } + + /** @returns held modifier names (for status text). */ + modifiers(): ModifierName[] { + return [...this.heldModifiers] + } + + /** Release held modifiers on navigation, detach, or tab close. */ + reset(): void { + this.heldModifiers.clear() + } +} + +/** One `InputState` per tab, created on demand. */ +export class InputStateStore { + private readonly states = new Map() + + /** @returns the state for a tab, creating an empty one when needed. */ + get(tabId: number): InputState { + let state = this.states.get(tabId) + if (state === undefined) { + state = new InputState() + this.states.set(tabId, state) + } + return state + } + + /** Reset and forget one tab's state. */ + reset(tabId: number): void { + this.states.delete(tabId) + } + + /** Reset and forget every tab's state. */ + clear(): void { + this.states.clear() + } +} + +/** Modifier bitmask constant re-exported for callers that build chords. */ +export { MODIFIER_BITS } diff --git a/extensions/dsh-browser/src/background/keys.ts b/extensions/dsh-browser/src/background/keys.ts new file mode 100644 index 000000000..884d6c90b --- /dev/null +++ b/extensions/dsh-browser/src/background/keys.ts @@ -0,0 +1,185 @@ +/** + * Keyboard mapping for trusted key dispatch. + * + * `Input.dispatchKeyEvent` needs a DOM `key`, a physical `code`, and a Windows + * virtual-key code; the model only names keys. This module is the single table + * that translates the model-facing vocabulary and reports the supported set + * when a name is unknown, so a typo fails with an actionable message instead of + * a silently wrong keystroke. + * + * @module + */ + +/** Modifier names as the model spells them (KeyboardEvent modifier keys). */ +export type ModifierName = 'Alt' | 'Control' | 'Meta' | 'Shift' + +/** CDP modifier bitmask values. */ +export const MODIFIER_BITS: Readonly> = { + Alt: 1, + Control: 2, + Meta: 4, + Shift: 8, +} + +/** One physical key description. */ +export interface KeyStroke { + key: string + code: string + windowsVirtualKeyCode: number + /** Text a non-command key press produces; absent for named/control keys. */ + text?: string +} + +/** Whether a string is one of the four modifier names. */ +export function isModifierName(value: string): value is ModifierName { + return value === 'Alt' || value === 'Control' || value === 'Meta' || value === 'Shift' +} + +/** + * Combine modifier names into the CDP bitmask. + * + * @param modifiers - model-supplied modifier list. + * @returns the modifier bitmask (0 when none). + */ +export function modifierMask(modifiers: readonly ModifierName[]): number { + let mask = 0 + for (const modifier of modifiers) mask |= MODIFIER_BITS[modifier] + return mask +} + +/** Aliases the model (or a keyboard layout) may use for the same key. */ +const KEY_ALIASES: Readonly> = { + esc: 'Escape', + escape: 'Escape', + return: 'Enter', + enter: 'Enter', + space: ' ', + spacebar: ' ', + del: 'Delete', + delete: 'Delete', + ins: 'Insert', + insert: 'Insert', + up: 'ArrowUp', + down: 'ArrowDown', + left: 'ArrowLeft', + right: 'ArrowRight', + arrowup: 'ArrowUp', + arrowdown: 'ArrowDown', + arrowleft: 'ArrowLeft', + arrowright: 'ArrowRight', + pageup: 'PageUp', + pagedown: 'PageDown', + home: 'Home', + end: 'End', + backspace: 'Backspace', + tab: 'Tab', +} + +/** Named (non-printable) keys. */ +const NAMED_KEYS: Readonly>> = { + Enter: { code: 'Enter', windowsVirtualKeyCode: 13 }, + Tab: { code: 'Tab', windowsVirtualKeyCode: 9 }, + Escape: { code: 'Escape', windowsVirtualKeyCode: 27 }, + Backspace: { code: 'Backspace', windowsVirtualKeyCode: 8 }, + Delete: { code: 'Delete', windowsVirtualKeyCode: 46 }, + ArrowUp: { code: 'ArrowUp', windowsVirtualKeyCode: 38 }, + ArrowDown: { code: 'ArrowDown', windowsVirtualKeyCode: 40 }, + ArrowLeft: { code: 'ArrowLeft', windowsVirtualKeyCode: 37 }, + ArrowRight: { code: 'ArrowRight', windowsVirtualKeyCode: 39 }, + Home: { code: 'Home', windowsVirtualKeyCode: 36 }, + End: { code: 'End', windowsVirtualKeyCode: 35 }, + PageUp: { code: 'PageUp', windowsVirtualKeyCode: 33 }, + PageDown: { code: 'PageDown', windowsVirtualKeyCode: 34 }, + Insert: { code: 'Insert', windowsVirtualKeyCode: 45 }, + Shift: { code: 'ShiftLeft', windowsVirtualKeyCode: 16 }, + Control: { code: 'ControlLeft', windowsVirtualKeyCode: 17 }, + Alt: { code: 'AltLeft', windowsVirtualKeyCode: 18 }, + Meta: { code: 'MetaLeft', windowsVirtualKeyCode: 91 }, +} + +/** Printable punctuation: key → physical code and virtual-key code. */ +const PUNCTUATION: Readonly> = { + '-': { code: 'Minus', vk: 189 }, + '=': { code: 'Equal', vk: 187 }, + '[': { code: 'BracketLeft', vk: 219 }, + ']': { code: 'BracketRight', vk: 221 }, + '\\': { code: 'Backslash', vk: 220 }, + ';': { code: 'Semicolon', vk: 186 }, + "'": { code: 'Quote', vk: 222 }, + ',': { code: 'Comma', vk: 188 }, + '.': { code: 'Period', vk: 190 }, + '/': { code: 'Slash', vk: 191 }, + '`': { code: 'Backquote', vk: 192 }, +} + +/** Keys the model may name, for an actionable `bad-args` message. */ +export const SUPPORTED_KEYS: readonly string[] = [ + ...'abcdefghijklmnopqrstuvwxyz'.split(''), + ...'0123456789'.split(''), + ' ', ...Object.keys(PUNCTUATION), + ...Object.keys(NAMED_KEYS), + ...Array.from({ length: 12 }, (_, index) => `F${index + 1}`), +].sort() + +/** + * Translate one model-facing key name into a CDP key stroke. + * + * Letters are accepted in either case (a capital letter does not imply Shift; + * the caller adds modifiers explicitly), and aliases such as `Esc` or `Space` + * resolve to their canonical names. + * + * @param key - key name from the tool call. + * @returns the stroke description, or undefined when the name is unknown. + */ +export function strokeForKey(key: string): KeyStroke | undefined { + if (key === '') return undefined + const alias = KEY_ALIASES[key.toLowerCase()] + const name = alias ?? key + const named = NAMED_KEYS[name] + if (named !== undefined) { + const printable = name === 'Tab' || name === 'Enter' + return { + key: name, + code: named.code, + windowsVirtualKeyCode: named.windowsVirtualKeyCode, + ...(printable ? { text: name === 'Enter' ? '\r' : '\t' } : {}), + } + } + if (/^F([1-9]|1[0-2])$/.test(name)) { + const number = Number(name.slice(1)) + return { key: name, code: name, windowsVirtualKeyCode: 111 + number } + } + if (name.length === 1) { + const lower = name.toLowerCase() + if (/^[a-z]$/.test(lower)) { + return { key: lower, code: `Key${lower.toUpperCase()}`, windowsVirtualKeyCode: lower.toUpperCase().charCodeAt(0), text: lower } + } + if (/^[0-9]$/.test(lower)) { + return { key: lower, code: `Digit${lower}`, windowsVirtualKeyCode: lower.charCodeAt(0), text: lower } + } + if (lower === ' ') { + return { key: ' ', code: 'Space', windowsVirtualKeyCode: 32, text: ' ' } + } + const punctuation = PUNCTUATION[lower] + if (punctuation !== undefined) { + return { key: lower, code: punctuation.code, windowsVirtualKeyCode: punctuation.vk, text: lower } + } + } + return undefined +} + +/** + * Whether a stroke inserts text; command chords (Control/Meta/Alt) must not. + * + * @param stroke - resolved stroke. + * @param mask - active modifier bitmask. + * @returns true when the stroke should carry `text`. + */ +export function strokeInsertsText(stroke: KeyStroke, mask: number): boolean { + if (stroke.text === undefined) return false + if ((mask & (MODIFIER_BITS.Control | MODIFIER_BITS.Meta | MODIFIER_BITS.Alt)) !== 0) return false + // Shift+Tab moves focus backwards and Shift+Enter is a modified newline; + // neither should insert the base key's text. + if (stroke.key === 'Tab' || stroke.key === 'Enter') return mask === 0 + return true +} diff --git a/extensions/dsh-browser/src/background/tools.ts b/extensions/dsh-browser/src/background/tools.ts index 0fc3069a1..e78ebb700 100644 --- a/extensions/dsh-browser/src/background/tools.ts +++ b/extensions/dsh-browser/src/background/tools.ts @@ -9,7 +9,7 @@ */ import { DEFAULT_SNAPSHOT_MAX_CHARS } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' -import type { ToolError } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' +import type { ToolError, ToolErrorCode } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' import { allocateFrameBudgets, frameDocumentKey, @@ -22,6 +22,22 @@ import { approvalPromptForCall, originFromUrl } from './authorization.ts' import { waitForNextDocumentReady } from './navigation.ts' import type { ApprovalAuthorization, ApprovalPrompt } from '../security/approval.ts' import { getUiLocale } from '../i18n.ts' +import { DevToolsError, type CdpClient } from './debugger-session.ts' +import { InputStateStore } from './input-state.ts' +import { + InputError, + dispatchKey, + insertTextAtCaret, + sendPasteShortcut, + typeAsKeystrokes, + setInputFiles, + withFocusEmulation, +} from './input-actions.ts' +import { isModifierName, type ModifierName } from './keys.ts' +import type { InputSetupRequest, InputSetupResult } from '../shared/input-setup.ts' +import { writeClipboard } from './clipboard-write.ts' +import type { ClipboardWriteOutcome } from '../shared/clipboard.ts' +import { resolveDebuggerFrame } from './debugger-frame.ts' /** A tool call from the bridge. */ export interface ToolCall { @@ -53,6 +69,7 @@ const ACTION_DELTA_TOOLS = new Set([ 'browser_type', 'browser_press', 'browser_scroll', + 'browser_clipboard', 'browser_wait', ]) const ACTION_DELTA_GUIDANCE = 'The page settled and its current changes are included below. Continue from this state; take another snapshot only when broader page context is needed.' @@ -76,6 +93,8 @@ const STATE_CHANGING_PAGE_TOOLS = new Set([ 'browser_type', 'browser_press', 'browser_scroll', + 'browser_clipboard', + 'browser_upload', 'browser_navigate', 'browser_back', 'browser_forward', @@ -96,6 +115,11 @@ export function resetTabSnapshot(tabId: number): void { snapshotDocumentsByTab.delete(tabId) } +/** Forget held modifiers for a tab that moved on. */ +export function resetInputState(tabId: number): void { + inputStates.reset(tabId) +} + /** Whether a successful tool may have changed the controlled tab's page URL. */ export function isNavigationCandidateTool(name: string): boolean { return NAVIGATION_CANDIDATE_TOOLS.has(name) @@ -106,6 +130,14 @@ export function isTabManagementTool(name: string): boolean { return TAB_MANAGEMENT_TOOL_NAMES.has(name) } +/** Trusted input channel owned by the background entry point (debugger broker). */ +export interface InputRuntime { + /** Whether the platform can dispatch real browser input. */ + supported: boolean + /** Run one operation against the tab's shared debugger attachment. */ + run(tabId: number, operation: (client: CdpClient) => Promise): Promise +} + /** Tab-affinity services owned by the background entry point. */ export interface TabManagementContext { /** Skip all browser approval prompts after the user enables unrestricted access. */ @@ -114,6 +146,8 @@ export interface TabManagementContext { controlledTabId?: number /** Rebind subsequent tools to one existing tab without activating it. */ followTab?: (tab: chrome.tabs.Tab) => void | Promise + /** Trusted input, when the platform and the user's browser allow it. */ + input?: InputRuntime /** Mark the point after which a state-changing operation cannot be withdrawn. */ commitAction?: () => void /** Restore withdrawability when a content-script action was not delivered. */ @@ -171,6 +205,18 @@ function unavailable(message: string): ToolAnswer { return { ok: false, error: { code: 'content-unavailable', message } } } +/** Map a debugger failure to the model-facing vocabulary instead of masking it. */ +function devToolsFailure(error: unknown): ToolAnswer | undefined { + if (!(error instanceof DevToolsError)) return undefined + return { + ok: false, + error: { + code: error.code === 'devtools-conflict' ? 'input-unavailable' : 'action-failed', + message: error.message, + }, + } +} + function tabMetadataSnapshot( tabId: number, windowId: number | undefined, @@ -337,13 +383,186 @@ function wrapActionDelta(status: string, pageContent: string, frame: TabFrame, m return `${prefix}${separator}${wrapUntrustedContent(framedContent, boundaryBudget)}` } +/** Per-tab held modifier state. */ +const inputStates = new InputStateStore() + +/** Failure of an internal content-script input setup request. */ +class InputSetupError extends Error { + constructor( + readonly code: ToolErrorCode, + message: string, + ) { + super(message) + this.name = 'InputSetupError' + } +} + +/** Content-script response envelope for one setup request. */ +interface InputSetupEnvelope { + ok?: boolean + result?: InputSetupResult + error?: { code?: string; message?: string } +} + +/** Ask one frame to measure, focus, mark, or report its frame chain. */ +async function sendInputSetup(tabId: number, frame: TabFrame, request: InputSetupRequest): Promise { + const response = await chrome.tabs.sendMessage( + tabId, + { type: 'DSH_INPUT_SETUP', request }, + frame.documentId === undefined ? { frameId: frame.frameId } : { documentId: frame.documentId }, + ) as InputSetupEnvelope | undefined + if (typeof response !== 'object' || response === null || response.ok !== true || response.result === undefined) { + const code = response?.error?.code + const stable: ToolErrorCode = code === 'bad-args' || code === 'input-unavailable' || code === 'action-failed' + ? code + : 'action-failed' + throw new InputSetupError(stable, response?.error?.message ?? 'The page did not answer the input setup request.') + } + return response.result +} + +/** Validate the model-supplied modifier list. */ +function requestedModifiers(args: Record): ModifierName[] { + if (args.modifiers === undefined) return [] + if (!Array.isArray(args.modifiers)) { + throw new InputError('bad-args', 'modifiers must be a list of Alt, Control, Meta, or Shift.') + } + const modifiers: ModifierName[] = [] + for (const value of args.modifiers) { + if (typeof value !== 'string' || !isModifierName(value)) { + throw new InputError('bad-args', 'modifiers must be a list of Alt, Control, Meta, or Shift.') + } + modifiers.push(value) + } + return modifiers +} + +/** Report what the clipboard actually holds, including derived plain text. */ +function describeClipboardWrite(outcome: ClipboardWriteOutcome): string { + if (!outcome.rich) return `Wrote ${outcome.plainText.length} character(s) to the system clipboard.` + if (outcome.plainDerived) { + return `Wrote rich HTML to the system clipboard; the plain-text flavor is derived from the HTML (${outcome.plainText.length} characters).` + } + return `Wrote rich HTML and ${outcome.plainText.length} plain-text character(s) to the system clipboard.` +} + +/** Describe the target in a status line without echoing page text. */ +function describeTarget(args: Record): string { + if (args.index !== undefined) return `element [${String(args.index)}]` + if (typeof args.selector === 'string') return `element (${args.selector})` + return 'the focused element' +} + +/** + * After trusted input, wait for the frame to settle and attach an authorized page delta. + * The `browser_wait` action already owns both behaviors for content-script + * actions; reusing it keeps the result shape identical for real input. + */ +async function settleAfterInput( + tabId: number, + frame: TabFrame, + status: string, + budget: ContentBudget, + includePageDelta: boolean, +): Promise { + try { + const response = await chrome.tabs.sendMessage( + tabId, + { type: 'DSH_ACTION', action: 'browser_wait', args: {}, budget, includePageDelta }, + frame.documentId === undefined ? { frameId: frame.frameId } : { documentId: frame.documentId }, + ) + if (isToolAnswer(response)) { + const pageContent = answerPageContent(response) + if (includePageDelta && pageContent !== undefined) { + return { ok: true, result: { text: wrapActionDelta(`${status} (trusted)`, pageContent, frame, budget.maxChars) } } + } + } + } catch { + // The page may be navigating; the trusted action itself already succeeded. + } + return { ok: true, result: { text: `${status} (trusted)` } } +} + +/** + * Classify how one call reaches the page. + * + * `content` stays in the frame, `trusted` prefers real input but degrades to + * the frame for calls that have a synthetic equivalent, and `required` has no + * synthetic equivalent at all. + */ +function inputRoute(call: ToolCall): 'content' | 'trusted' | 'required' | 'host' { + switch (call.name) { + case 'browser_press': + if (call.args.trusted === true) return 'required' + if (call.args.trusted === false) return 'content' + if (Array.isArray(call.args.modifiers) || call.args.action !== undefined) return 'trusted' + return 'content' + case 'browser_type': + if (call.args.trusted === true) return 'required' + if (call.args.trusted === false) return 'content' + if (call.args.mode === 'insert' || call.args.mode === 'keystrokes') return 'trusted' + return 'content' + case 'browser_clipboard': + if (call.args.action === 'write') return 'host' + return call.args.method === 'system' ? 'required' : 'content' + case 'browser_upload': + return 'required' + default: + return 'content' + } +} + +/** Session text for a trusted-input failure. */ +function trustedFailure(error: unknown): ToolAnswer | undefined { + if (error instanceof InputError) { + return { ok: false, error: { code: error.code, message: error.message } } + } + if (error instanceof InputSetupError) { + return { ok: false, error: { code: error.code, message: error.message } } + } + return undefined +} + +/** Map a page-side selector failure to a model-facing answer. */ +function inputUnavailable(message: string): ToolAnswer { + return { ok: false, error: { code: 'input-unavailable', message } } +} + +/** Whether a runtime is present and supported. */ +function usableInput(runtime: InputRuntime | undefined): runtime is InputRuntime { + return runtime !== undefined && runtime.supported +} + +/** A token that only marks one file input for one upload call. */ +function uploadToken(): string { + return crypto.randomUUID() +} + async function snapshotAllFrames( tabId: number, - frames: TabFrame[], + allFrames: TabFrame[], call: ToolCall, budget: ContentBudget, ): Promise { - const budgets = allocateFrameBudgets(frames, budget) + // `frame` narrows the snapshot to one iframe: a canvas or editor frame is + // usually bigger than the share it gets from a whole-page budget. + const requested = call.args.frame + let frames = allFrames + let budgets: Map + if (requested !== undefined) { + if (typeof requested !== 'number' || !Number.isInteger(requested) || requested < 0) { + return { ok: false, error: { code: 'bad-args', message: 'frame must be a non-negative integer.' } } + } + const single = allFrames.find((frame) => frame.frameId === requested) + if (single === undefined) { + return unavailable(`Frame ${requested} does not exist or has navigated. Call browser_snapshot again.`) + } + frames = [single] + budgets = new Map([[single.frameId, { maxItems: budget.maxItems, maxChars: budget.maxChars }]]) + } else { + budgets = allocateFrameBudgets(allFrames, budget) + } + const scoped = requested !== undefined const previous = snapshotDocumentsByTab.get(tabId) ?? new Map() const deltaRequested = call.args.delta === true @@ -380,17 +599,20 @@ async function snapshotAllFrames( continue } capturedDocuments.set(frame.frameId, frameDocumentKey(frame)) - if (frame.frameId === 0) sections.push(text) + if (frame.frameId === 0 && !scoped) sections.push(text) else sections.push(frameHeader(frame), text) } - if (deltaRequested) { + if (deltaRequested && !scoped) { const liveIds = new Set(frames.map((frame) => frame.frameId)) const removed = [...previous.keys()].filter((frameId) => frameId !== 0 && !liveIds.has(frameId)) if (removed.length > 0) sections.push(`\nRemoved iframes: ${removed.join(', ')}`) } - snapshotDocumentsByTab.set(tabId, capturedDocuments) + // A scoped snapshot must not erase the other frames' delta baselines. + const nextDocuments = scoped ? new Map(previous) : new Map() + for (const [frameId, key] of capturedDocuments) nextDocuments.set(frameId, key) + snapshotDocumentsByTab.set(tabId, nextDocuments) return { ok: true, result: { text: wrapUntrustedContent(sections.join('\n'), budget.maxChars) } } } @@ -425,6 +647,220 @@ async function snapshotAfterNavigation( return { ok: true, result: { text: `${prefix}\n\n${snapshot}` } } } +/** Require a usable trusted-input runtime, or fail with the stable code. */ +function requireInput(runtime: InputRuntime | undefined): InputRuntime { + if (!usableInput(runtime)) { + throw new InputError( + 'input-unavailable', + 'This browser has no trusted input channel (it needs the Chrome debugger). ' + + 'Target elements by index or selector instead, or install the Chrome build.', + ) + } + return runtime +} + +/** Focus (and optionally select) the addressed target before trusted input. */ +async function prepareInputTarget( + tabId: number, + frame: TabFrame, + args: Record, + select: 'none' | 'contents' | 'end', +): Promise { + if (args.index === undefined && typeof args.selector !== 'string') return + await sendInputSetup(tabId, frame, { + op: 'prepare', + ...(args.index === undefined ? {} : { index: args.index as number }), + ...(typeof args.selector === 'string' ? { selector: args.selector } : {}), + select, + }) +} + +/** Validated `paths` argument for `browser_upload`. */ +function requestedPaths(value: unknown): string[] { + if (!Array.isArray(value) || value.length === 0) { + throw new InputError('bad-args', 'paths must be a non-empty list of local file paths.') + } + const paths: string[] = [] + for (const entry of value) { + if (typeof entry !== 'string' || entry.trim() === '') { + throw new InputError('bad-args', 'paths must be a non-empty list of local file paths.') + } + paths.push(entry) + } + return paths +} + +/** Trusted keyboard: real key events, including held modifiers. */ +async function trustedPress( + tabId: number, + frame: TabFrame, + call: ToolCall, + budget: ContentBudget, + runtime: InputRuntime | undefined, + includePageDelta: boolean, +): Promise { + const args = call.args + const key = typeof args.key === 'string' && args.key !== '' ? args.key : '' + if (key === '') throw new InputError('bad-args', 'key must not be empty.') + if (args.action !== undefined && args.action !== 'press' && args.action !== 'down' && args.action !== 'up') { + throw new InputError('bad-args', 'action must be press, down, or up.') + } + const action = args.action === 'down' || args.action === 'up' ? args.action : 'press' + const modifiers = requestedModifiers(args) + const repeat = args.repeat === undefined ? 1 : args.repeat + if (typeof repeat !== 'number' || !Number.isInteger(repeat) || repeat < 1 || repeat > 100) { + throw new InputError('bad-args', 'repeat must be an integer between 1 and 100.') + } + const input = requireInput(runtime) + await prepareInputTarget(tabId, frame, args, 'none') + await input.run(tabId, async (client) => { + await withFocusEmulation(client, async () => { + await dispatchKey(client, inputStates.get(tabId), { key, action, modifiers, repeat }) + }) + }) + const chord = modifiers.length === 0 ? key : `${modifiers.join('+')}+${key}` + return settleAfterInput(tabId, frame, `sent key "${chord}"${repeat > 1 ? ` x${repeat}` : ''}`, budget, includePageDelta) +} + +/** Trusted text: a caret insertion or per-character key events. */ +async function trustedType( + tabId: number, + frame: TabFrame, + call: ToolCall, + budget: ContentBudget, + runtime: InputRuntime | undefined, + includePageDelta: boolean, +): Promise { + const args = call.args + const text = typeof args.text === 'string' ? args.text : '' + if (text === '') throw new InputError('bad-args', 'text must not be empty.') + const mode = args.mode === 'keystrokes' ? 'keystrokes' : 'insert' + const input = requireInput(runtime) + const selection = args.replace === true ? 'contents' : args.mode === 'insert' || args.mode === 'keystrokes' ? 'none' : 'end' + await prepareInputTarget(tabId, frame, args, selection) + await input.run(tabId, async (client) => { + await withFocusEmulation(client, async () => { + if (mode === 'keystrokes') await typeAsKeystrokes(client, inputStates.get(tabId), text) + else await insertTextAtCaret(client, text) + }) + }) + return settleAfterInput( + tabId, + frame, + `entered ${text.length} characters into ${describeTarget(args)} (${mode})`, + budget, + includePageDelta, + ) +} + +/** Clipboard write (host path) and system-clipboard paste (trusted path). */ +async function trustedClipboard( + tabId: number, + frame: TabFrame, + call: ToolCall, + budget: ContentBudget, + runtime: InputRuntime | undefined, + includePageDelta: boolean, +): Promise { + const args = call.args + const text = typeof args.text === 'string' ? args.text : '' + const html = typeof args.html === 'string' && args.html !== '' ? args.html : undefined + if (args.action === 'write') { + if (text === '' && html === undefined) throw new InputError('bad-args', 'text or html is required for a clipboard write.') + const outcome = await writeClipboard({ text, ...(html === undefined ? {} : { html }) }) + return { ok: true, result: { text: describeClipboardWrite(outcome) } } + } + const input = requireInput(runtime) + if (text !== '' || html !== undefined) { + await writeClipboard({ text, ...(html === undefined ? {} : { html }) }) + } + await prepareInputTarget(tabId, frame, args, 'none') + await input.run(tabId, async (client) => { + await withFocusEmulation(client, async () => { + await sendPasteShortcut(client, inputStates.get(tabId), navigator.userAgent) + }) + }) + return settleAfterInput(tabId, frame, `pasted the system clipboard into ${describeTarget(args)}`, budget, includePageDelta) +} + +/** File attachment: mark the input, let the browser process read the paths. */ +async function trustedUpload( + tabId: number, + frame: TabFrame, + call: ToolCall, + runtime: InputRuntime | undefined, +): Promise { + const args = call.args + if (args.index === undefined && typeof args.selector !== 'string') { + throw new InputError('bad-args', 'browser_upload needs index or selector for the file input.') + } + const paths = requestedPaths(args.paths) + const input = requireInput(runtime) + const token = uploadToken() + const mark = await sendInputSetup(tabId, frame, { + op: 'upload-mark', + token, + ...(args.index === undefined ? {} : { index: args.index as number }), + ...(typeof args.selector === 'string' ? { selector: args.selector } : {}), + }) + if (mark.op !== 'upload-mark') { + throw new InputSetupError('action-failed', 'The page returned an unexpected upload result.') + } + try { + await input.run(tabId, async (client) => { + if (frame.frameId === 0) { + await setInputFiles(client, mark.selector, paths) + return + } + const target = await resolveDebuggerFrame(client, tabId, frame) + await setInputFiles(target.client, mark.selector, paths, target.uniqueId) + }) + } finally { + void sendInputSetup(tabId, frame, { op: 'upload-clear', token }).catch(() => undefined) + } + return { + ok: true, + result: { text: `Attached ${paths.length} file(s) to ${describeTarget(args)}.` }, + } +} + +/** + * Dispatch one call through the trusted input channel. + * + * @returns the answer, or undefined when the call should fall back to the + * content script's synthetic path. + */ +async function dispatchTrustedInput( + tabId: number, + frame: TabFrame, + call: ToolCall, + budget: ContentBudget, + runtime: InputRuntime | undefined, + includePageDelta: boolean, +): Promise { + const route = inputRoute(call) + if (route === 'content') return undefined + if (route === 'trusted' && !usableInput(runtime)) return undefined + try { + switch (call.name) { + case 'browser_clipboard': + return await trustedClipboard(tabId, frame, call, budget, runtime, includePageDelta) + case 'browser_upload': + return await trustedUpload(tabId, frame, call, runtime) + case 'browser_press': + return await trustedPress(tabId, frame, call, budget, runtime, includePageDelta) + case 'browser_type': + return await trustedType(tabId, frame, call, budget, runtime, includePageDelta) + default: + return undefined + } + } catch (error: unknown) { + const mapped = trustedFailure(error) + if (mapped !== undefined) return mapped + throw error + } +} + async function dispatchOnce( tabId: number, frames: TabFrame[], @@ -435,6 +871,7 @@ async function dispatchOnce( includeActionDelta: boolean = false, commitAction?: () => void, rollbackActionCommit?: () => void, + inputRuntime?: InputRuntime, ): Promise { if (isCancelled(call, signal)) return cancelled() if (targetStillAllowed?.() === false) return targetChanged() @@ -446,11 +883,23 @@ async function dispatchOnce( if (frame === undefined) { return unavailable(`Frame ${frameId} does not exist or has navigated. Call browser_snapshot again.`) } + const route = inputRoute(call) + const syntheticFallback = route === 'trusted' + const hasSnapshotBaseline = snapshotDocumentsByTab.get(tabId)?.get(frameId) === frameDocumentKey(frame) + if (route !== 'content') { + // Commit before dispatching: a trusted gesture cannot be withdrawn once the + // browser has delivered it, so the deadline must not abort it mid-flight. + if (STATE_CHANGING_PAGE_TOOLS.has(call.name)) commitAction?.() + const trusted = await dispatchTrustedInput(tabId, frame, call, budget, inputRuntime, includeActionDelta && hasSnapshotBaseline) + if (trusted !== undefined) return trusted + if (!syntheticFallback) { + return inputUnavailable('This browser has no trusted input channel for this action.') + } + } // No await occurs between this guard and tabs.sendMessage, so an expired // approval cannot cross the final state-changing dispatch boundary. if (isCancelled(call, signal)) return cancelled() if (targetStillAllowed?.() === false) return targetChanged() - const hasSnapshotBaseline = snapshotDocumentsByTab.get(tabId)?.get(frameId) === frameDocumentKey(frame) const requestPageDelta = includeActionDelta && hasSnapshotBaseline && ACTION_DELTA_TOOLS.has(call.name) const navigationWait = includeActionDelta && isNavigationCandidateTool(call.name) ? waitForNextDocumentReady(tabId, frameId, frame.documentId, signal) @@ -479,11 +928,12 @@ async function dispatchOnce( navigationWait?.cancel() return unavailable('The page content script returned an invalid response.') } - const text = answerText(response) - if (text === undefined) { + const rawText = answerText(response) + if (rawText === undefined) { navigationWait?.cancel() return response } + const text = syntheticFallback ? `${rawText} (synthetic)` : rawText if (answerNavigationPending(response) && navigationWait !== undefined) { const ready = await navigationWait.ready if (isCancelled(call, signal)) return cancelled() @@ -551,7 +1001,9 @@ function tabManagementApproval(call: ToolCall, tab?: chrome.tabs.Tab): ApprovalP kind: 'action', action: call.name, summary: call.name === 'browser_follow_tab' - ? (locale === 'zh' ? `跟随标签页 ${tab?.id ?? '?'}：${display}` : `Follow tab ${tab?.id ?? '?'}: ${display}`) + ? (call.args.activate === true + ? (locale === 'zh' ? `跟随并切换到标签页 ${tab?.id ?? '?'}：${display}` : `Follow and activate tab ${tab?.id ?? '?'}: ${display}`) + : (locale === 'zh' ? `跟随标签页 ${tab?.id ?? '?'}：${display}` : `Follow tab ${tab?.id ?? '?'}: ${display}`)) : (locale === 'zh' ? `关闭标签页 ${tab?.id ?? '?'}：${display}` : `Close tab ${tab?.id ?? '?'}: ${display}`), origins: origin === undefined ? [] : [origin], canTrust: false, @@ -628,11 +1080,30 @@ async function dispatchTabManagementTool( if (call.name === 'browser_follow_tab') { if (context.followTab === undefined) return unavailable('The browser could not bind the selected tab in this session.') + const activate = call.args.activate === true context.commitAction?.() await context.followTab(current) + if (activate) { + if (current.windowId !== undefined) { + try { + await chrome.windows.update(current.windowId, { focused: true }) + } catch { + return unavailable(`Tab ${tabId} was followed but its window could not be focused.`) + } + } + try { + await chrome.tabs.update(tabId, { active: true }) + } catch { + return unavailable(`Tab ${tabId} was followed but could not be activated.`) + } + } return { ok: true, - result: { text: `Tab ${tabId} is now the controlled tab. Call browser_snapshot before operating its page.` }, + result: { + text: activate + ? `Tab ${tabId} is now the controlled and active tab. Call browser_snapshot before operating its page.` + : `Tab ${tabId} is now the controlled tab. Call browser_snapshot before operating its page.`, + }, } } if (call.name === 'browser_close_tab') { @@ -655,6 +1126,120 @@ async function dispatchTabManagementTool( return unavailable(`Unsupported tab-management tool: ${call.name}`) } +/** Parse the model-supplied screenshot capture options. */ +function screenshotCaptureOptions(call: ToolCall): { format: 'png' | 'jpeg'; quality: number } | ToolAnswer { + const format = call.args.format === undefined ? 'png' : call.args.format + if (format !== 'png' && format !== 'jpeg') { + return { ok: false, error: { code: 'action-failed', message: 'format must be "png" or "jpeg".' } } + } + const quality = call.args.quality === undefined ? 85 : call.args.quality + if (typeof quality !== 'number' || !Number.isInteger(quality) || quality < 1 || quality > 100) { + return { ok: false, error: { code: 'action-failed', message: 'quality must be an integer between 1 and 100.' } } + } + return { format, quality } +} + +/** Decode a captureVisibleTab data URL into base64 pixels and media type. */ +function decodeCaptureDataUrl(dataUrl: string): { data: string; mediaType: string } | undefined { + const comma = dataUrl.indexOf(',') + if (comma === -1) return undefined + const header = /^data:([^;,]+);base64$/.exec(dataUrl.slice(0, comma)) + if (header === null) return undefined + const data = dataUrl.slice(comma + 1) + return data === '' ? undefined : { data, mediaType: header[1] ?? '' } +} + +async function measureImageBytes(data: string, mediaType: string): Promise<{ width: number; height: number } | undefined> { + try { + const binary = atob(data) + const bytes = new Uint8Array(binary.length) + for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index) + const bitmap = await createImageBitmap(new Blob([bytes as BlobPart], { type: mediaType })) + try { + return { width: bitmap.width, height: bitmap.height } + } finally { + bitmap.close() + } + } catch { + // Dimensions are advisory metadata; the capture stays deliverable. + return undefined + } +} + +/** + * Capture the controlled tab's viewport. `captureVisibleTab` photographs the + * active tab of the window, so a controlled-but-inactive tab is a hard error + * rather than a silent capture of the wrong page. + */ +async function dispatchScreenshot( + tab: Pick & { active?: boolean; windowId?: number }, + call: ToolCall, + signal?: AbortSignal, + targetStillAllowed?: () => boolean, +): Promise { + if (isCancelled(call, signal)) return cancelled() + if (targetStillAllowed?.() === false) return targetChanged() + const options = screenshotCaptureOptions(call) + if ('ok' in options) return options + let active = tab.active + let windowId = tab.windowId + if (active === undefined || windowId === undefined) { + const fresh = await findTab(tab.id!) + if (fresh === undefined) return unavailable('The controlled tab closed before the screenshot was captured.') + active = fresh.active + windowId = fresh.windowId + } + if (isCancelled(call, signal)) return cancelled() + if (targetStillAllowed?.() === false) return targetChanged() + if (active !== true) { + return unavailable('The controlled tab is not the active tab of its window, so its viewport cannot be captured. Activate the tab first, then retry.') + } + if (windowId === undefined) { + return unavailable('The controlled tab has no window, so its viewport cannot be captured.') + } + // `captureVisibleTab` photographs the last composited frame. A window that is + // not focused can hand back a stale image that no longer matches the DOM, so + // fail clearly instead of presenting old pixels as the current state. + const windowsApi = (globalThis as { + chrome?: { windows?: { get?: (id: number) => Promise<{ focused?: boolean }> } } + }).chrome?.windows + if (typeof windowsApi?.get === 'function') { + try { + const win = await windowsApi.get(windowId) + if (win.focused === false) { + return unavailable('The browser window is not focused, so a captured frame could be stale. Bring it to the front (browser_follow_tab { activate: true }) and retry.') + } + } catch { + // A closed window is reported by the capture call itself. + } + } + let dataUrl: string + try { + dataUrl = await chrome.tabs.captureVisibleTab(windowId, { + format: options.format, + ...(options.format === 'jpeg' ? { quality: options.quality } : {}), + }) + } catch (error: unknown) { + const detail = error instanceof Error ? error.message : String(error) + return unavailable(`The screenshot could not be captured: ${detail}`) + } + if (isCancelled(call, signal)) return cancelled() + if (targetStillAllowed?.() === false) return targetChanged() + const captured = decodeCaptureDataUrl(dataUrl) + if (captured === undefined) return unavailable('The browser returned the screenshot in an unsupported format.') + const dimensions = await measureImageBytes(captured.data, captured.mediaType) + return { + ok: true, + result: { + image: { + data: captured.data, + mediaType: captured.mediaType, + ...dimensions === undefined ? {} : dimensions, + }, + }, + } +} + /** * Dispatch one tool call to the selected tab's content script. * @param call - the tool call to execute. @@ -689,7 +1274,7 @@ export async function dispatchToolCall( // Privacy boundary: with sharing off, no page content may leave the page. if (!tabManagement.unrestrictedAccess && sharePageContent === 'off' - && (call.name === 'browser_snapshot' || call.name === 'browser_get_text')) { + && (call.name === 'browser_snapshot' || call.name === 'browser_get_text' || call.name === 'browser_screenshot')) { return { ok: false, error: { code: 'action-failed', message: 'Page content sharing is disabled in Settings > Page content sharing.' } } } const tab = targetTab ?? (await chrome.tabs.query({ active: true, lastFocusedWindow: true }))[0] @@ -703,7 +1288,7 @@ export async function dispatchToolCall( if (targetStillAllowed?.() === false) return targetChanged() const frameError = validateFrameTarget(call, frames) if (frameError !== undefined) return frameError - if (!isInjectablePage(tab.url) && !TAB_NATIVE_TOOLS.has(call.name)) { + if (!isInjectablePage(tab.url) && !TAB_NATIVE_TOOLS.has(call.name) && call.name !== 'browser_screenshot') { return unavailable('The current page DOM is protected by the browser. Only snapshot metadata, navigate, back, forward, and reload are available on this page.') } const targetError = validateElementTarget(call, tab.id, frames) @@ -734,9 +1319,15 @@ export async function dispatchToolCall( if (refreshedTargetError !== undefined) return refreshedTargetError } if (!isInjectablePage(tab.url)) { + if (call.name === 'browser_screenshot') { + return unavailable('A screenshot cannot be captured on this page: the browser protects pages outside http and https.') + } return await dispatchTabNativeTool(tab.id, tab.url, tab.title, tab.windowId, call, effectiveBudget, signal, targetStillAllowed, tabManagement.commitAction) ?? unavailable('The current page DOM is protected by the browser.') } + if (call.name === 'browser_screenshot') { + return await dispatchScreenshot(tab, call, signal, targetStillAllowed) + } try { return await dispatchOnce( tab.id, @@ -748,11 +1339,15 @@ export async function dispatchToolCall( tabManagement.unrestrictedAccess || sharePageContent === 'auto', tabManagement.commitAction, tabManagement.rollbackActionCommit, + tabManagement.input, ) } catch (error: unknown) { if (isCancelled(call, signal)) return cancelled() + const trustedFailureAnswer = devToolsFailure(error) + if (trustedFailureAnswer !== undefined) return trustedFailureAnswer if (!contentScriptReceiverMissing(error)) { - return unavailable('The content script stopped responding after the operation was dispatched. Call browser_snapshot before continuing.') + const detail = error instanceof Error ? error.message : String(error) + return unavailable(`The browser operation failed after it was dispatched: ${detail}`) } // Manifest content scripts do not run retroactively in tabs that were // already open when an unpacked extension was installed or reloaded. @@ -789,9 +1384,13 @@ export async function dispatchToolCall( tabManagement.unrestrictedAccess || sharePageContent === 'auto', tabManagement.commitAction, tabManagement.rollbackActionCommit, + tabManagement.input, ) - } catch { - return unavailable('The content script did not answer after it was loaded. Call browser_snapshot again before retrying.') + } catch (error: unknown) { + const trustedFailureAnswer = devToolsFailure(error) + if (trustedFailureAnswer !== undefined) return trustedFailureAnswer + const detail = error instanceof Error ? error.message : String(error) + return unavailable(`The content script did not answer after it was loaded (${detail}). Call browser_snapshot again before retrying.`) } } } @@ -806,8 +1405,20 @@ function validateFrameTarget(call: ToolCall, frames: TabFrame[]): ToolAnswer | u return undefined } +const INDEX_TARGETED_TOOLS = new Set([ + 'browser_click', + 'browser_type', + 'browser_press', + 'browser_scroll', + 'browser_clipboard', + 'browser_upload', +]) + function validateElementTarget(call: ToolCall, tabId: number, frames: TabFrame[]): ToolAnswer | undefined { - if (call.name !== 'browser_click' && call.name !== 'browser_type') return undefined + // A selector or "the focused element" is self-contained; only + // a snapshot index needs a snapshot of the same document to resolve. + if (!INDEX_TARGETED_TOOLS.has(call.name)) return undefined + if (call.args.index === undefined) return undefined const frameId = requestedFrame(call.args) const frame = frames.find((candidate) => candidate.frameId === frameId) const snapshotted = snapshotDocumentsByTab.get(tabId)?.get(frameId) diff --git a/extensions/dsh-browser/src/content/actions.ts b/extensions/dsh-browser/src/content/actions.ts index 7cdf60b74..cbbe19873 100644 --- a/extensions/dsh-browser/src/content/actions.ts +++ b/extensions/dsh-browser/src/content/actions.ts @@ -11,7 +11,9 @@ */ import { pageText, truncate } from './extract.ts' +import { findElements, LocatorError, parseLocatorQuery, renderFound } from './find.ts' import type { ElementIds } from './ids.ts' +import { dispatchPaste } from './paste.ts' import type { SnapshotBudget } from './snapshot.ts' import { buildSnapshot, renderSnapshot } from './snapshot.ts' @@ -134,7 +136,7 @@ function elementOrThrow(ids: ElementIds, index: number): Element { /** Error carrying a stable wire code. */ export class ActionError extends Error { constructor( - readonly code: 'action-failed' | 'bad-args', + readonly code: 'action-failed' | 'bad-args' | 'input-unavailable', message: string, ) { super(message) @@ -142,6 +144,36 @@ export class ActionError extends Error { } } +/** + * Resolve the element a model action refers to: a stable inventory index or a + * CSS selector. Exactly one form is allowed; both and neither are caller bugs. + * + * @param args - raw tool arguments. + * @param ids - the frame's element registry. + * @returns the addressed element. + */ +export function resolveActionTarget(args: Record, ids: ElementIds): Element { + const hasIndex = args.index !== undefined + const selector = typeof args.selector === 'string' && args.selector !== '' ? args.selector : undefined + if (hasIndex && selector !== undefined) { + throw new ActionError('bad-args', 'Provide either index or selector, not both.') + } + if (!hasIndex && selector === undefined) { + throw new ActionError('bad-args', 'Provide index or selector.') + } + if (hasIndex) return elementOrThrow(ids, numberArg(args, 'index')) + let found: Element | null + try { + found = document.querySelector(selector as string) + } catch { + throw new ActionError('bad-args', `selector is not a valid CSS selector: ${String(selector)}`) + } + if (found === null) { + throw new ActionError('action-failed', `No element matched selector: ${String(selector)}`) + } + return found +} + /** React-compatible value write: native setter + input/change events. */ function setNativeValue(input: HTMLInputElement | HTMLTextAreaElement, value: string): void { const prototype = input instanceof HTMLTextAreaElement @@ -178,6 +210,11 @@ export async function runAction(action: string, args: Record, c return pressAction(args, ctx) case 'browser_scroll': return scrollAction(args, ctx) + case 'browser_clipboard': + if (args.action !== 'paste') { + throw new ActionError('bad-args', 'Only clipboard paste runs in the page; clipboard writes are dispatched by the browser background.') + } + return clipboardPasteAction(args, ctx) case 'browser_navigate': return navigateAction(args) case 'browser_back': @@ -196,12 +233,36 @@ export async function runAction(action: string, args: Record, c } function snapshotAction(args: Record, ctx: ActionContext): ActionResult { + if (args.query !== undefined) { + try { + const query = parseLocatorQuery(args.query) + const found = findElements(query) + return { text: renderFound(found, query, (el) => ctx.ids.indexOf(el)) } + } catch (error: unknown) { + if (error instanceof LocatorError) throw new ActionError('bad-args', error.message) + throw error instanceof Error ? error : new ActionError('action-failed', String(error)) + } + } const delta = args.delta === true const region = typeof args.region === 'string' && args.region !== '' ? args.region : undefined + // A region that matches nothing must not look like an empty page. + let regionMiss = false + if (region !== undefined) { + try { + regionMiss = document.querySelector(region) === null + } catch { + throw new ActionError('bad-args', `region is not a valid CSS selector: ${region}`) + } + } // 基线在每次快照后都更新：delta 调用才能相对上一次（无论是否 delta）比较。 const view = buildSnapshot(ctx.ids, { delta, region, budget: ctx.budget }, lastSnapshot) lastSnapshot = view - return { text: renderSnapshot(view, delta) } + const rendered = renderSnapshot(view, delta) + return { + text: regionMiss + ? `No element matched region selector "${region}"; showing the whole page.\n${rendered}` + : rendered, + } } /** Module-level last snapshot state for delta mode (content-script lifetime). */ @@ -223,11 +284,84 @@ function withPageDelta(text: string, ctx: ActionContext): ActionResult { } } +/** A short, model-facing description of the addressed element. */ +function targetLabel(args: Record, el: Element): string { + // Selector- and focus-targeted calls have no index at all; reading through + // `numberArg` here used to throw before the action even ran. + const index = args.index + if (typeof index === 'number' && Number.isInteger(index) && index >= 0) return `[${index}]` + const selector = typeof args.selector === 'string' ? args.selector : '' + return `${el.tagName.toLowerCase()} (${selector})` +} + +/** + * Dispatch the full pointer/mouse sequence at an element's center. + * + * Menus and custom controls bind `pointerdown`/`mousedown` rather than only + * `click`; a bare `el.click()` leaves them inert. Clicking with a synthetic + * `MouseEvent` still runs the browser's activation behavior for buttons, + * checkboxes and labels. + * + * @param el - target element. + * @param clicks - 1, 2, or 3 clicks; 2 also emits `dblclick`. + */ +function dispatchPointerClick(el: Element, clicks: number, button: number, modifiers: readonly string[]): void { + const rect = el.getBoundingClientRect() + const clientX = rect.left + rect.width / 2 + const clientY = rect.top + rect.height / 2 + const base = { + bubbles: true, cancelable: true, composed: true, clientX, clientY, button, + altKey: modifiers.includes('Alt'), ctrlKey: modifiers.includes('Control'), + metaKey: modifiers.includes('Meta'), shiftKey: modifiers.includes('Shift'), + } + const buttons = button === 0 ? 1 : button === 1 ? 4 : 2 + for (let count = 1; count <= clicks; count += 1) { + if (typeof PointerEvent === 'function') { + el.dispatchEvent(new PointerEvent('pointerdown', { + ...base, pointerId: 1, pointerType: 'mouse', isPrimary: true, buttons, + })) + } + el.dispatchEvent(new MouseEvent('mousedown', { ...base, buttons, detail: count })) + if (typeof PointerEvent === 'function') { + el.dispatchEvent(new PointerEvent('pointerup', { + ...base, pointerId: 1, pointerType: 'mouse', isPrimary: true, buttons: 0, + })) + } + el.dispatchEvent(new MouseEvent('mouseup', { ...base, buttons: 0, detail: count })) + if (button === 2) el.dispatchEvent(new MouseEvent('contextmenu', { ...base, buttons: 0, detail: count })) + el.dispatchEvent(new MouseEvent(button === 0 ? 'click' : 'auxclick', { ...base, buttons: 0, detail: count })) + } + if (clicks >= 2 && button === 0) el.dispatchEvent(new MouseEvent('dblclick', { ...base, buttons: 0, detail: 2 })) +} + async function clickAction(args: Record, ctx: ActionContext): Promise { - const index = numberArg(args, 'index') - const el = elementOrThrow(ctx.ids, index) + const action = args.action + if (action !== undefined && action !== 'click' && action !== 'dblclick') { + throw new ActionError( + 'bad-args', + 'action must be click or dblclick.', + ) + } + const el = resolveActionTarget(args, ctx.ids) + const label = targetLabel(args, el) + if (el.matches(':disabled') || el.closest('button:disabled') !== null) { + throw new ActionError('action-failed', `Element ${label} is disabled.`) + } + const buttonName = args.button ?? 'left' + if (buttonName !== 'left' && buttonName !== 'middle' && buttonName !== 'right') { + throw new ActionError('bad-args', 'button must be left, middle, or right.') + } + const button = buttonName === 'left' ? 0 : buttonName === 'middle' ? 1 : 2 + const modifiers = args.modifiers ?? [] + if (!Array.isArray(modifiers) || modifiers.some(value => typeof value !== 'string' || !PRESS_MODIFIERS.has(value))) { + throw new ActionError('bad-args', 'modifiers must be a list of Alt, Control, Meta, or Shift.') + } + const clicks = action === 'dblclick' ? 2 : args.clicks === undefined ? 1 : numberArg(args, 'clicks') + if (clicks < 1 || clicks > 3) { + throw new ActionError('bad-args', 'clicks must be 1, 2, or 3.') + } el.scrollIntoView({ block: 'center', behavior: 'instant' }) - if (el instanceof HTMLAnchorElement) { + if (el instanceof HTMLAnchorElement && clicks === 1 && button === 0 && modifiers.length === 0) { const target = el.target.trim().toLowerCase() const sameFrameTarget = target === '' || target === '_self' let href: URL | undefined @@ -249,7 +383,7 @@ async function clickAction(args: Record, ctx: ActionContext): P if (requiresNativeActivation) { setTimeout(() => { el.click() }, 0) return { - text: `Clicked link [${index}] using native browser activation. Call browser_snapshot to read the resulting state.`, + text: `Clicked link ${label} using native browser activation. Call browser_snapshot to read the resulting state.`, } } // Dispatch the click handlers without its default navigation so a @@ -261,7 +395,7 @@ async function clickAction(args: Record, ctx: ActionContext): P })) if (!shouldNavigate) { await waitForPageSettled(ACTION_SETTLE) - return withPageDelta(`Clicked link [${index}].`, ctx) + return withPageDelta(`Clicked link ${label}.`, ctx) } const sameDocument = href.origin === location.origin && href.pathname === location.pathname @@ -269,25 +403,28 @@ async function clickAction(args: Record, ctx: ActionContext): P if (sameDocument) { if (href.hash !== location.hash) location.hash = href.hash await waitForPageSettled(ACTION_SETTLE) - return withPageDelta(`Clicked link [${index}].`, ctx) + return withPageDelta(`Clicked link ${label}.`, ctx) } // A cross-document navigation can unload this content script before an // awaited response. Answer first and navigate in the next task. setTimeout(() => { location.href = href.href }, 0) return { - text: `Clicked link [${index}]. Call browser_snapshot again after navigation settles.`, + text: `Clicked link ${label}. Call browser_snapshot again after navigation settles.`, navigationPending: true, } } setTimeout(() => { el.click() }, 0) - return { text: `Clicked link [${index}]. The link may open outside the controlled frame.` } + return { text: `Clicked link ${label}. The link may open outside the controlled frame.` } } - if (el instanceof HTMLButtonElement && el.disabled) { - throw new ActionError('action-failed', `Button [${index}] is disabled.`) + dispatchPointerClick(el, clicks, button, modifiers) + // An untrusted click runs no focus default action; text fields and editors + // still expect a click to place the caret, which the next type/press relies on. + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement + || el instanceof HTMLSelectElement || (el instanceof HTMLElement && el.isContentEditable)) { + el.focus() } - ;(el as HTMLElement).click() await waitForPageSettled(ACTION_SETTLE) - return withPageDelta(`Clicked [${index}].`, ctx) + return withPageDelta(`Clicked ${label}.`, ctx) } /** @@ -335,13 +472,15 @@ function isEditable(el: Element): el is HTMLElement { * @param text - text to insert. * @param replace - whether to replace the host's current contents. */ -function typeIntoContentEditable(el: HTMLElement, text: string, replace: boolean): void { +function typeIntoContentEditable(el: HTMLElement, text: string, replace: boolean, insert: boolean): void { // `isEditable` already refused a disabled island, so a null host here means // the element is editable without the attribute; address it directly. const host = editingHost(el) ?? el host.focus() const selection = host.ownerDocument.getSelection() - if (selection !== null) { + const selectionInside = selection !== null && selection.rangeCount > 0 + && host.contains(selection.anchorNode) && host.contains(selection.focusNode) + if (selection !== null && (replace || !insert || !selectionInside)) { const range = host.ownerDocument.createRange() range.selectNodeContents(host) // A caret collapsed to the end appends; a full selection is replaced by insertText. @@ -363,34 +502,126 @@ function typeIntoContentEditable(el: HTMLElement, text: string, replace: boolean } async function typeAction(args: Record, ctx: ActionContext): Promise { - const index = numberArg(args, 'index') const text = typeof args.text === 'string' ? args.text : '' if (text === '') throw new ActionError('bad-args', 'text must not be empty.') + const mode = args.mode + if (mode !== undefined && mode !== 'auto' && mode !== 'insert') { + throw new ActionError( + 'input-unavailable', + `type mode "${String(mode)}" needs the trusted input channel, which the browser routes through its debugger.`, + ) + } const replace = args.replace === true - const el = elementOrThrow(ctx.ids, index) - if (isEditable(el)) { - typeIntoContentEditable(el, text, replace) - } else if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { - if (replace) setNativeValue(el, '') - setNativeValue(el, `${el.value}${text}`) + const focusedOnly = args.index === undefined && args.selector === undefined + const el = focusedOnly ? document.activeElement : resolveActionTarget(args, ctx.ids) + if (!(el instanceof Element)) { + throw new ActionError('action-failed', 'No element has focus; click the target first or pass index or selector.') + } + const label = focusedOnly ? 'the focused element' : targetLabel(args, el) + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { + if (mode === 'insert' && !replace) { + const start = el.selectionStart + const end = el.selectionEnd + if (start === null || end === null) throw new ActionError('action-failed', 'This input does not expose a text selection.') + setNativeValue(el, `${el.value.slice(0, start)}${text}${el.value.slice(end)}`) + el.setSelectionRange(start + text.length, start + text.length) + } else { + setNativeValue(el, replace ? text : `${el.value}${text}`) + } + } else if (isEditable(el)) { + typeIntoContentEditable(el, text, replace, mode === 'insert') } else { - throw new ActionError('action-failed', `Element [${index}] is not editable (${el.tagName.toLowerCase()}).`) + throw new ActionError('action-failed', `Element ${label} is not editable (${el.tagName.toLowerCase()}).`) } await waitForPageSettled(TYPE_SETTLE) - return withPageDelta(`Entered ${text.length} characters into [${index}].`, ctx) + return withPageDelta(`Entered ${text.length} characters into ${label}.`, ctx) } +/** Wire modifier names accepted by `browser_press`. */ +const PRESS_MODIFIERS = new Set(['Alt', 'Control', 'Meta', 'Shift']) + async function pressAction(args: Record, ctx: ActionContext): Promise { const key = typeof args.key === 'string' && args.key !== '' ? args.key : '' if (key === '') throw new ActionError('bad-args', 'key must not be empty.') + const action = args.action === 'down' || args.action === 'up' ? args.action : 'press' + if (args.action !== undefined && args.action !== 'press' && action !== args.action) { + throw new ActionError('bad-args', `action must be "press", "down", or "up"; received "${String(args.action)}".`) + } + const repeat = args.repeat === undefined ? 1 : args.repeat + if (typeof repeat !== 'number' || !Number.isInteger(repeat) || repeat < 1 || repeat > 100) { + throw new ActionError('bad-args', 'repeat must be an integer between 1 and 100.') + } + const modifiers: string[] = [] + if (Array.isArray(args.modifiers)) { + for (const value of args.modifiers) { + if (typeof value !== 'string' || !PRESS_MODIFIERS.has(value)) { + throw new ActionError('bad-args', 'modifiers must be a list of Alt, Control, Meta, or Shift.') + } + modifiers.push(value) + } + } + if (args.index !== undefined || args.selector !== undefined) { + const el = resolveActionTarget(args, ctx.ids) + if (el instanceof HTMLElement) el.focus() + } const target = document.activeElement instanceof HTMLElement ? document.activeElement : document.body - target.dispatchEvent(new KeyboardEvent('keydown', { key, bubbles: true, cancelable: true })) - target.dispatchEvent(new KeyboardEvent('keyup', { key, bubbles: true, cancelable: true })) - if (key === 'Enter' && target instanceof HTMLInputElement && target.form !== null) { + const init: KeyboardEventInit = { + key, + bubbles: true, + cancelable: true, + altKey: modifiers.includes('Alt'), + ctrlKey: modifiers.includes('Control'), + metaKey: modifiers.includes('Meta'), + shiftKey: modifiers.includes('Shift'), + } + for (let count = 0; count < repeat; count += 1) { + if (action !== 'up') target.dispatchEvent(new KeyboardEvent('keydown', init)) + if (action !== 'down') target.dispatchEvent(new KeyboardEvent('keyup', init)) + } + if (key === 'Enter' && action !== 'up' && target instanceof HTMLInputElement && target.form !== null) { target.form.dispatchEvent(new Event('submit', { bubbles: true, cancelable: true })) } + const chord = modifiers.length === 0 ? key : `${modifiers.join('+')}+${key}` await waitForPageSettled(ACTION_SETTLE) - return withPageDelta(`Sent key "${key}".`, ctx) + return withPageDelta(`Sent key "${chord}"${repeat > 1 ? ` ×${repeat}` : ''}.`, ctx) +} + +/** + * Dispatch a paste event carrying text and optional sanitized HTML. + * + * The page's own paste handler consumes the event; no system clipboard is + * touched, so this works in every supported browser and never overwrites what + * the user has copied. + */ +async function clipboardPasteAction(args: Record, ctx: ActionContext): Promise { + const text = typeof args.text === 'string' ? args.text : '' + if (text === '') throw new ActionError('bad-args', 'text must not be empty when pasting.') + const html = typeof args.html === 'string' && args.html !== '' ? args.html : undefined + const focusedOnly = args.index === undefined && args.selector === undefined + const el = focusedOnly ? document.activeElement : resolveActionTarget(args, ctx.ids) + if (!(el instanceof Element)) { + throw new ActionError('action-failed', 'No element has focus; click the paste target first or pass index or selector.') + } + if (el instanceof HTMLElement) el.focus() + const accepted = dispatchPaste(el, { text, ...(html === undefined ? {} : { html }) }) + const label = focusedOnly ? 'the focused element' : targetLabel(args, el) + // Plain form controls have no paste handler and an untrusted paste event has + // no default insertion; fall back to a caret insertion so the intent lands. + let fallback = false + if (!accepted && (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement)) { + const start = el.selectionStart ?? el.value.length + const end = el.selectionEnd ?? el.value.length + setNativeValue(el, `${el.value.slice(0, start)}${text}${el.value.slice(end)}`) + fallback = true + } + await waitForPageSettled(TYPE_SETTLE) + const detail = html === undefined ? 'text' : 'text+html' + const note = accepted + ? '' + : fallback + ? '; the page did not consume the paste event, so the text was inserted at the caret' + : '; the page did not consume the paste event' + return withPageDelta(`Pasted ${text.length} characters (${detail}) into ${label}${note}.`, ctx) } async function scrollAction(args: Record, ctx: ActionContext): Promise { @@ -460,7 +691,14 @@ function reloadAction(): ActionResult { async function getTextAction(args: Record): Promise { const selector = typeof args.selector === 'string' && args.selector !== '' ? args.selector : undefined - const source = selector !== undefined ? document.querySelector(selector) : null + let source: Element | null = null + if (selector !== undefined) { + try { + source = document.querySelector(selector) + } catch { + throw new ActionError('bad-args', `selector is not a valid CSS selector: ${selector}`) + } + } const text = source !== null ? pageText(source) : selector !== undefined ? `No element matched selector: ${selector}` : pageText() const truncated = truncate(text, 8_000) return { text: truncated.text + (truncated.truncated > 0 ? `\n(Truncated ${truncated.truncated} characters.)` : '') } diff --git a/extensions/dsh-browser/src/content/find.ts b/extensions/dsh-browser/src/content/find.ts new file mode 100644 index 000000000..17ea255ec --- /dev/null +++ b/extensions/dsh-browser/src/content/find.ts @@ -0,0 +1,328 @@ +/** + * Query-based element locator for the `browser_snapshot { query }` action. + * + * The snapshot inventory only lists semantically interactive elements, so rich + * editors, canvas apps and SVG surfaces keep their real controls out of reach. + * This module locates any element by text, selector or role and returns its + * selector for page actions. + * + * The locator is deliberately app-agnostic: it treats every element alike and + * never keys on a domain, component class or editor implementation. + * + * @module + */ + +import { accessibleName } from './extract.ts' + +/** One locator request as supplied by the model. */ +export interface LocatorQuery { + text?: string + selector?: string + role?: string + includeHidden?: boolean + limit?: number +} + +/** One located element with everything the model needs to act on it. */ +export interface FoundElement { + element: Element + selector: string + visible: boolean + editable: boolean + focusable: boolean + label: string + role: string +} + +/** Bounded scan so a giant page cannot stall one tool call. */ +const MAX_SCANNED_ELEMENTS = 20_000 + +/** Label cap; the snapshot uses the same width for inventory names. */ +const LABEL_MAX_CHARS = 80 + +const DEFAULT_LIMIT = 10 +const MAX_LIMIT = 30 + +/** Tags that never carry usable text or interaction of their own. */ +const SKIPPED_TAGS = new Set([ + 'html', + 'head', + 'body', + 'script', + 'style', + 'meta', + 'link', + 'title', + 'noscript', + 'template', +]) + +/** Explicit role, else the browser's implicit role for common form/content tags. */ +export function roleOfElement(el: Element): string { + const explicit = el.getAttribute('role') + if (explicit !== null && explicit.trim() !== '') return explicit.trim() + if (el instanceof HTMLAnchorElement) return el.hasAttribute('href') ? 'link' : 'generic' + if (el instanceof HTMLButtonElement) return 'button' + if (el instanceof HTMLInputElement) { + switch (el.type) { + case 'checkbox': return 'checkbox' + case 'radio': return 'radio' + case 'button': + case 'submit': + case 'reset': return 'button' + case 'range': return 'slider' + case 'search': return 'searchbox' + default: return 'textbox' + } + } + if (el instanceof HTMLTextAreaElement) return 'textbox' + if (el instanceof HTMLSelectElement) return 'combobox' + if (el instanceof HTMLElement && el.isContentEditable) return 'textbox' + if (/^h[1-6]$/i.test(el.tagName)) return 'heading' + if (el instanceof HTMLImageElement) return 'img' + return 'generic' +} + +/** Whether an element can take keyboard focus, even when it has no size (editor proxies). */ +export function isFocusableElement(el: Element): boolean { + if (!(el instanceof HTMLElement)) { + return el.hasAttribute('tabindex') || el.getAttribute('contenteditable') === 'true' + } + if (el.isContentEditable) return true + if (el instanceof HTMLInputElement) return !el.disabled + if (el instanceof HTMLTextAreaElement) return !el.disabled && !el.readOnly + if (el instanceof HTMLSelectElement || el instanceof HTMLButtonElement) return !el.disabled + if (el.getAttribute('tabindex') !== null) return Number(el.getAttribute('tabindex')) >= 0 + return el.hasAttribute('href') +} + +/** Whether text or HTML input would land inside this element. */ +export function isEditableElement(el: Element): boolean { + if (el instanceof HTMLInputElement) return !el.disabled && !el.readOnly && el.type !== 'checkbox' && el.type !== 'radio' + if (el instanceof HTMLTextAreaElement) return !el.disabled && !el.readOnly + return el instanceof HTMLElement && el.isContentEditable +} + +/** + * Whether the element renders with a box, regardless of its tag namespace. + * `HTMLElement`-only checks would exclude every SVG node. + */ +function isRendered(el: Element): boolean { + const style = getComputedStyle(el) + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return false + const rect = el.getBoundingClientRect() + return rect.width > 0 && rect.height > 0 +} + +/** Escape an identifier for a CSS selector. */ +function cssEscape(value: string): string { + if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') return CSS.escape(value) + return value.replace(/[^a-zA-Z0-9_-]/g, (ch) => `\\${ch}`) +} + +/** + * Build a selector that resolves back to this element. The walk uses + * `:nth-child` so it stays unique among siblings; an id short-circuits it. + * + * @param el - target element. + * @returns a CSS selector, or an empty string when the element has no parent chain. + */ +export function uniqueSelectorFor(el: Element): string { + if (el.id !== '') { + const byId = `#${cssEscape(el.id)}` + try { + if (document.querySelectorAll(byId).length === 1) return byId + } catch { + // Fall through to the structural path. + } + } + const parts: string[] = [] + let node: Element | null = el + while (node !== null) { + if (node === document.documentElement) { + parts.unshift('html') + break + } + if (node.id !== '') { + const byId = `#${cssEscape(node.id)}` + try { + if (document.querySelectorAll(byId).length === 1) { + parts.unshift(byId) + break + } + } catch { + // Keep walking. + } + } + const parent: Element | null = node.parentElement + if (parent === null) { + parts.unshift(node.tagName.toLowerCase()) + break + } + const position = [...parent.children].indexOf(node) + 1 + parts.unshift(`${node.tagName.toLowerCase()}:nth-child(${position})`) + node = parent + } + return parts.join(' > ') +} + +/** Normalized visible text, used for label matching and reporting. */ +function normalizedText(el: Element): string { + return (el.textContent ?? '').replace(/\s+/g, ' ').trim() +} + +/** Whether an element's own labels (not descendants' bulk text) match the needle. */ +function matchesText(el: Element, needle: string): boolean { + const labels = [ + el.getAttribute('aria-label') ?? '', + el.getAttribute('title') ?? '', + el.getAttribute('placeholder') ?? '', + el.getAttribute('alt') ?? '', + el instanceof HTMLInputElement && ['button', 'submit', 'reset'].includes(el.type) ? el.value : '', + ] + for (const label of labels) { + if (label.toLowerCase().includes(needle)) return true + } + const text = normalizedText(el) + // A container whose subtree text dwarfs the query is not the element the + // caller means; only short, self-descriptive nodes match on their own text. + return text.length > 0 && text.length <= 200 && text.toLowerCase().includes(needle) +} + +/** Thrown for malformed locator arguments so callers can map them to `bad-args`. */ +export class LocatorError extends Error { + constructor(message: string) { + super(message) + this.name = 'LocatorError' + } +} + +/** Read and validate the model-supplied query object. */ +export function parseLocatorQuery(value: unknown): LocatorQuery { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new LocatorError('query must be an object with text, selector, role, includeHidden, and limit.') + } + const record = value as Record + const text = typeof record.text === 'string' && record.text !== '' ? record.text : undefined + const selector = typeof record.selector === 'string' && record.selector !== '' ? record.selector : undefined + const role = typeof record.role === 'string' && record.role !== '' ? record.role : undefined + if (text !== undefined && selector !== undefined) { + throw new LocatorError('query.text and query.selector are mutually exclusive.') + } + if (text === undefined && selector === undefined && role === undefined) { + throw new LocatorError('query needs at least one of text, selector, or role.') + } + const limit = record.limit === undefined ? DEFAULT_LIMIT : record.limit + if (typeof limit !== 'number' || !Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) { + throw new LocatorError(`query.limit must be an integer between 1 and ${MAX_LIMIT}.`) + } + return { + ...(text === undefined ? {} : { text }), + ...(selector === undefined ? {} : { selector }), + ...(role === undefined ? {} : { role }), + ...(record.includeHidden === true ? { includeHidden: true } : {}), + limit, + } +} + +/** How many raw matches a scan keeps before visibility filtering decides. */ +const MAX_MATCHES = 200 + +/** Candidate elements for a query, in document order. */ +function candidates(query: LocatorQuery): Element[] { + if (query.selector !== undefined) { + try { + return [...document.querySelectorAll(query.selector)] + } catch { + throw new LocatorError(`query.selector is not a valid CSS selector: ${query.selector}`) + } + } + const matched: Element[] = [] + const needle = query.text?.toLowerCase() + let scanned = 0 + for (const el of document.querySelectorAll('*')) { + if (matched.length >= MAX_MATCHES || scanned >= MAX_SCANNED_ELEMENTS) break + scanned += 1 + if (SKIPPED_TAGS.has(el.tagName.toLowerCase())) continue + if (query.role !== undefined && roleOfElement(el) !== query.role) continue + if (needle !== undefined && !matchesText(el, needle)) continue + matched.push(el) + } + return matched +} + +/** + * Collapse nested matches onto the element a caller actually means. + * + * A container and its labelled child both satisfy a text query; returning both + * wastes the result budget and picks wrappers over controls. The innermost + * focusable/editable element wins, a wrapper of such a control is dropped, and + * otherwise the innermost text carrier wins. + * + * @param matches - raw matches in document order. + * @returns the matches worth reporting. + */ +function dedupeNested(matches: Element[]): Element[] { + const semantic = new Set(matches.filter((el) => isFocusableElement(el) || isEditableElement(el))) + return matches.filter((el) => { + const semanticDescendant = [...semantic].some((control) => control !== el && el.contains(control)) + if (semantic.has(el)) return !semanticDescendant + const semanticAncestor = [...semantic].some((control) => control !== el && control.contains(el)) + if (semanticAncestor) return false + return !matches.some((other) => other !== el && el.contains(other)) + }) +} + +/** + * Locate elements matching one query. + * + * @param query - parsed locator request. + * @returns up to `query.limit` matches, visible ones first, in document order. + */ +export function findElements(query: LocatorQuery): FoundElement[] { + const limit = query.limit ?? DEFAULT_LIMIT + const found: FoundElement[] = [] + for (const el of dedupeNested(candidates(query))) { + const visible = isRendered(el) + if (!visible && query.includeHidden !== true) continue + found.push({ + element: el, + selector: uniqueSelectorFor(el), + visible, + editable: isEditableElement(el), + focusable: isFocusableElement(el), + label: accessibleName(el), + role: roleOfElement(el), + }) + } + // The page may render an off-screen control first; what the user can see is + // the likelier target, and document order is stable within each group. + found.sort((left, right) => Number(right.visible) - Number(left.visible)) + return found.slice(0, limit) +} + +/** Build the model-facing query result text. */ +export function renderFound(found: FoundElement[], query: LocatorQuery, indexOf: (el: Element) => number | undefined): string { + const description = query.selector !== undefined + ? `selector "${query.selector}"` + : query.text !== undefined + ? `text "${query.text}"` + : `role "${query.role ?? ''}"` + if (found.length === 0) { + return `No elements matched ${description}. Call browser_snapshot without query for the page inventory, or loosen includeHidden.` + } + const lines = [`Found ${found.length} element(s) matching ${description}:`] + for (const item of found) { + const index = indexOf(item.element) + const identity = index === undefined ? '-' : `[${index}]` + const state = [ + item.visible ? undefined : 'hidden', + item.editable ? 'editable' : undefined, + item.focusable ? 'focusable' : undefined, + ].filter((value) => value !== undefined).join('/') + const label = item.label.length > LABEL_MAX_CHARS ? `${item.label.slice(0, LABEL_MAX_CHARS - 1)}…` : item.label + lines.push(` ${identity} ${item.role} "${label}" · selector=${item.selector}${state === '' ? '' : ` · ${state}`}`) + } + lines.push('Click by index when one is present, otherwise by selector.') + return lines.join('\n') +} diff --git a/extensions/dsh-browser/src/content/index.ts b/extensions/dsh-browser/src/content/index.ts index 1cf4b24eb..2cf264707 100644 --- a/extensions/dsh-browser/src/content/index.ts +++ b/extensions/dsh-browser/src/content/index.ts @@ -13,7 +13,9 @@ import { DEFAULT_SNAPSHOT_MAX_CHARS } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' import { runAction, ActionError } from './actions.ts' import { ElementIds } from './ids.ts' +import { handleInputSetup } from './input-support.ts' import { SelectionWatcher } from './selection.ts' +import type { InputSetupRequest } from '../shared/input-setup.ts' import type { SnapshotBudget } from './snapshot.ts' /** Negotiated snapshot budgets, patched in from the background via message. */ @@ -36,9 +38,30 @@ export interface ToolResult { error?: { code: string; message: string } } -function onMessage(message: unknown, _sender: chrome.runtime.MessageSender, sendResponse: (response: ToolResult) => void): true | undefined { +function onMessage(message: unknown, _sender: chrome.runtime.MessageSender, sendResponse: (response: unknown) => void): true | undefined { if (typeof message !== 'object' || message === null) return const msg = message as { type?: string } + if (msg.type === 'DSH_INPUT_SETUP') { + // Internal trusted-input plumbing: focus/selection/geometry for the + // background's debugger-driven pointer and keyboard. Never model-facing. + const request = (message as { request?: InputSetupRequest }).request + if (request === undefined) { + sendResponse({ ok: false, error: { code: 'bad-args', message: 'input setup request is missing.' } }) + return + } + try { + sendResponse({ ok: true, result: handleInputSetup(request, ids) }) + } catch (error: unknown) { + sendResponse({ + ok: false, + error: { + code: error instanceof ActionError ? error.code : 'action-failed', + message: error instanceof Error ? error.message : String(error), + }, + }) + } + return + } if (msg.type === 'DSH_BUDGET') { const incoming = (message as { budget?: Partial }).budget if (incoming !== undefined) { diff --git a/extensions/dsh-browser/src/content/input-support.ts b/extensions/dsh-browser/src/content/input-support.ts new file mode 100644 index 000000000..49d793b7a --- /dev/null +++ b/extensions/dsh-browser/src/content/input-support.ts @@ -0,0 +1,106 @@ +/** + * Content-script half of trusted input. + * + * The background dispatches keyboard events, but only the + * frame can focus a target or select its contents. These handlers + * are internal plumbing: they never reach the model and return structured data + * rather than text. + * + * @module + */ + +import type { InputSetupRequest, InputSetupResult } from '../shared/input-setup.ts' +import { ActionError, resolveActionTarget } from './actions.ts' +import type { ElementIds } from './ids.ts' + +/** Attribute used to hand one file input to the debugger without ambiguity. */ +const UPLOAD_ATTRIBUTE = 'data-dsh-upload' + +/** Focus one element and optionally select everything it contains. */ +function prepare( + args: { index?: number; selector?: string; select?: 'none' | 'contents' | 'end' }, + ids: ElementIds, +): { focused: boolean; selected: boolean } { + const focusedOnly = args.index === undefined && args.selector === undefined + const element = focusedOnly ? document.activeElement : resolveActionTarget(args, ids) + if (!(element instanceof Element)) { + throw new ActionError('action-failed', 'No element has focus; click the target first or pass index or selector.') + } + if (element instanceof HTMLElement) element.focus() + let selected = false + const selectAll = args.select === 'contents' + const selectEnd = args.select === 'end' + if (selectAll || selectEnd) { + if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) { + if (selectAll) { + element.select() + selected = element.selectionStart === 0 && element.selectionEnd === element.value.length + } else { + const end = element.value.length + element.setSelectionRange(end, end) + } + } else if (element instanceof HTMLElement && element.isContentEditable) { + // Selecting a child would replace only that child; the editable host is + // the element whose contents the caller means. + const host = element.closest('[contenteditable]') + const target = host instanceof HTMLElement ? host : element + const selection = target.ownerDocument.getSelection() + if (selection !== null) { + const range = target.ownerDocument.createRange() + range.selectNodeContents(target) + if (selectEnd) range.collapse(false) + selection.removeAllRanges() + selection.addRange(range) + selected = selectAll + } + } + } + const active = document.activeElement + return { + focused: active === element || (active !== null && (element.contains(active) || active.contains(element))), + selected, + } +} + +/** Tag a file input so the debugger can address it by a unique selector. */ +function markUpload(args: { token: string; index?: number; selector?: string }, ids: ElementIds): string { + const element = resolveActionTarget(args, ids) + if (!(element instanceof HTMLInputElement) || element.type !== 'file') { + throw new ActionError('action-failed', 'browser_upload needs an input element with type="file".') + } + const token = args.token.replace(/[^a-zA-Z0-9-]/g, '') + if (token === '') throw new ActionError('bad-args', 'upload token must not be empty.') + element.setAttribute(UPLOAD_ATTRIBUTE, token) + return `input[${UPLOAD_ATTRIBUTE}="${token}"]` +} + +/** Remove the temporary upload marker. */ +function clearUpload(token: string): boolean { + const safe = token.replace(/"/g, '\\"') + const element = document.querySelector(`input[${UPLOAD_ATTRIBUTE}="${safe}"]`) + if (element === null) return false + element.removeAttribute(UPLOAD_ATTRIBUTE) + return true +} + +/** + * Handle one internal trusted-input setup request. + * + * @param request - the operation and its target. + * @param ids - the frame's element registry. + * @returns the structured result for the background. + */ +export function handleInputSetup(request: InputSetupRequest, ids: ElementIds): InputSetupResult { + switch (request.op) { + case 'prepare': + return { op: 'prepare', ...prepare(request, ids) } + case 'upload-mark': + return { op: 'upload-mark', selector: markUpload(request, ids) } + case 'upload-clear': + return { op: 'upload-clear', cleared: clearUpload(request.token) } + default: { + const unknown = request as { op?: unknown } + throw new ActionError('bad-args', `Unknown input setup operation: ${String(unknown.op)}.`) + } + } +} diff --git a/extensions/dsh-browser/src/content/paste.ts b/extensions/dsh-browser/src/content/paste.ts new file mode 100644 index 000000000..6e4502326 --- /dev/null +++ b/extensions/dsh-browser/src/content/paste.ts @@ -0,0 +1,39 @@ +/** + * Rich clipboard paste for `browser_clipboard { action: 'paste' }`. + * + * The system clipboard is not touched here: the payload is placed on a + * `DataTransfer` and delivered as a `paste` event, which is exactly the event + * rich editors and canvas apps listen for. HTML is sanitized first so a page + * cannot be handed active content through the model's clipboard. + * + * @module + */ + +import { sanitizeClipboardHtml } from '../shared/clipboard.ts' + +/** What the caller wants to paste. */ +export interface PastePayload { + text: string + html?: string +} + +/** Build a cancelable paste event carrying the payload. */ +export function createPasteEvent(payload: PastePayload): ClipboardEvent { + const clipboardData = new DataTransfer() + clipboardData.setData('text/plain', payload.text) + if (payload.html !== undefined) clipboardData.setData('text/html', sanitizeClipboardHtml(payload.html)) + return new ClipboardEvent('paste', { bubbles: true, cancelable: true, clipboardData }) +} + +/** + * Dispatch a paste event at one element. + * + * @param target - element whose paste handler should run. + * @param payload - text plus optional rich HTML. + * @returns whether the page called `preventDefault` (a handler accepted it). + */ +export function dispatchPaste(target: Element, payload: PastePayload): boolean { + const event = createPasteEvent(payload) + target.dispatchEvent(event) + return event.defaultPrevented +} diff --git a/extensions/dsh-browser/src/offscreen/clipboard.ts b/extensions/dsh-browser/src/offscreen/clipboard.ts new file mode 100644 index 000000000..a1f4b6210 --- /dev/null +++ b/extensions/dsh-browser/src/offscreen/clipboard.ts @@ -0,0 +1,27 @@ +/** + * Offscreen clipboard writer. + * + * Chrome MV3 service workers have no DOM and no clipboard access, but an + * offscreen document with the `CLIPBOARD` reason can perform the copy on the + * extension's behalf. The background creates this document on demand and sends + * one message per write; the document never runs page code. + * + * @module + */ + +import { + CLIPBOARD_WRITE_MESSAGE, + type ClipboardWritePayload, +} from '../shared/clipboard.ts' +import { copyClipboardInDocument } from '../shared/clipboard-document.ts' + +chrome.runtime.onMessage.addListener((message: unknown, _sender, sendResponse: (response: unknown) => void) => { + if (typeof message !== 'object' || message === null) return + const request = message as { type?: unknown; target?: unknown; payload?: ClipboardWritePayload } + if (request.type !== CLIPBOARD_WRITE_MESSAGE || request.target !== 'offscreen' || request.payload === undefined) return + try { + sendResponse({ ok: true, outcome: copyClipboardInDocument(request.payload) }) + } catch (error: unknown) { + sendResponse({ ok: false, error: error instanceof Error ? error.message : String(error) }) + } +}) diff --git a/extensions/dsh-browser/src/panel/App.tsx b/extensions/dsh-browser/src/panel/App.tsx index 6d3d93602..c772c28f9 100644 --- a/extensions/dsh-browser/src/panel/App.tsx +++ b/extensions/dsh-browser/src/panel/App.tsx @@ -13,7 +13,7 @@ import type { BridgeCaps } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' import type { ServerFrame } from '@yuxianglin/dsh-bridge-browser/src/protocol.ts' import type { BridgeState } from '../background/bridge.ts' import type { AffinityTab, TabAffinityDecision, TabAffinityState } from '../background/tab-affinity.ts' -import { connectPanel, PanelRpcError, type PanelApi, type PanelSettings } from './api.ts' +import { connectPanel, PanelRpcError, type BridgeInstance, type DevToolsStatus, type DiscoveryStatus, type PanelApi, type PanelSettings } from './api.ts' import { renderMarkdown } from './markdown.ts' import whaleUrl from '../../assets/icons/deepseek-256.png' import type { ApprovalDecision, ApprovalRequest } from '../security/approval.ts' @@ -28,6 +28,7 @@ import { stepUiScale, uiScaleAtLimit, } from './ui-scale.ts' +import { canReturnToControlledTab } from './tab-handoff.ts' import { QuestionCard } from './QuestionCard.tsx' import { UpdateCard } from './UpdateCard.tsx' import { MessageImages } from './MessageImages.tsx' @@ -64,6 +65,29 @@ import { type ComposerDraft, } from './composer.ts' import { isNearScrollBottom } from './scroll.ts' +import { + isExperimentalPreset, + isSelectablePreset, + parseSessionPermissions, + permissionLabel, + requiresRiskConfirmation, + type SessionPermissionDirectory, +} from './permissions.ts' +import { + localSlashEntries, + parseCommandDirectory, + parseSkillDirectory, + rankSlashEntries, + resolveComposerLine, + slashEntries, + slashQuery, + type ComposerCatalogs, + type ComposerCommandDirectory, + type ComposerSkillDirectory, + type HostCommand, + type HostSkill, + type SlashEntry, +} from './commands.ts' import { latestSessionTitle, projectedSessionTitle, @@ -94,6 +118,32 @@ function normalizeWebOrigin(value: string): string | null { return normalizeTrustedOrigin(value) ?? null } +/** One-line reason for an unavailable composer control, without raw stack text. */ +function describeCatalogFailure(cause: unknown): string { + return cause instanceof Error ? cause.message : String(cause) +} + +/** + * The stable code a structured RPC failure carries, read structurally: the + * error class identity is not part of the contract, and reading it as + * `instanceof` would break any consumer that replaces the api module. + */ +function errorCodeOf(cause: unknown): string | undefined { + return typeof cause === 'object' && cause !== null && typeof (cause as { code?: unknown }).code === 'string' + ? (cause as { code: string }).code + : undefined +} + +/** True when the bridge predates a Host method the panel now calls. */ +function isUnavailableMethod(cause: unknown): boolean { + return errorCodeOf(cause) === 'not-found' || /is unavailable/iu.test(describeCatalogFailure(cause)) +} + +/** True when the Host refused a command because the Session is still deferred. */ +function isProvisionalFailure(cause: unknown): boolean { + return errorCodeOf(cause) === 'provisional-session' +} + /** One editable API-relay profile in the settings view. */ interface RelayProfileDraft { /** Stable llm-pi-ai provider-route key; empty until first save mints it. */ @@ -117,6 +167,8 @@ interface SessionModelOption { providerName: string id: string name: string + /** Reasoning efforts this model declares; empty when it declares none. */ + efforts: Array<{ id: string; name: string }> } /** Advisory directory snapshot used by the composer model switcher. */ @@ -124,6 +176,14 @@ interface SessionModelDirectory { current: { provider: string; model: string; reasoningEffort?: string } | null options: SessionModelOption[] routable: boolean | null + /** Set when the last directory read failed, so the chip can explain itself. */ + error?: string +} + +/** One pending risk confirmation before a preset switch is submitted. */ +interface PermissionConfirmation { + preset: string + label: string } /** @@ -185,6 +245,18 @@ function isAuthFailure(cause: unknown): boolean { && (cause.code.includes('credential') || /\b401\b|\b403\b|invalid[ _-]?key/i.test(cause.message)) } +/** Extract image files from a paste or drop payload; empty when it carries none. */ +export function transferredImageFiles(dataTransfer: DataTransfer | null): File[] { + if (dataTransfer === null) return [] + const files: File[] = [] + for (const item of dataTransfer.items) { + if (item.kind !== 'file' || !item.type.startsWith('image/')) continue + const file = item.getAsFile() + if (file !== null) files.push(file) + } + return files +} + async function discoverOnce( api: PanelApi, profile: RelayProfileDraft, @@ -390,10 +462,14 @@ function TabAffinityBanner({ state, copy, onDecision, + onReturn, + returnPending, }: { state: TabAffinityState | null copy: PanelCopy onDecision: (decision: TabAffinityDecision) => void + onReturn: () => void + returnPending: boolean }): React.JSX.Element | null { if (state === null || state.status === 'unbound' || state.status === 'following') return null const controlled = tabLabel(state.controlled, copy.tabHandoff.closedTab) @@ -432,6 +508,17 @@ function TabAffinityBanner({

{body}

+ {canReturnToControlledTab(state) && ( + + )} {handoff && } {state.active !== null && ( - {request.kind === 'read' && ( + {!devTools && request.kind === 'read' && ( )} - {request.kind === 'action' && request.canTrust && request.origins.length === 1 && ( + {!devTools && request.kind === 'action' && request.canTrust && request.origins.length === 1 && ( )} + {devTools && ( + + )}
- {request.kind === 'read' - ? copy.approval.readFootnote - : copy.approval.actionFootnote} + {devTools + ? copy.approval.devToolsFootnote + : request.kind === 'read' + ? copy.approval.readFootnote + : copy.approval.actionFootnote} ) } +/** + * One-line description of the current developer-tools state. + * + * Before the background reports anything, the truthful description is the + * default posture: the capability is off and every call asks for approval. + */ +/** Whether the reported state carries a failure worth emphasizing. */ +function hasDevToolsError(status: DevToolsStatus | null): boolean { + return status !== null && status.error !== undefined && status.error !== '' +} + +function devToolsStatusText(status: DevToolsStatus | null, copy: PanelCopy): string { + if (status === null) return copy.settings.devToolsOff + if (!status.supported) return copy.settings.devToolsUnsupported + if (status.error !== undefined && status.error !== '') return `${copy.settings.devToolsError} ${status.error}` + // An attachment can also come from trusted input while the developer-tools + // allowance is off; report the live attachment before the setting's default. + if (status.attached) return `${copy.settings.devToolsAttached}${status.tabId === undefined ? '' : ` (tab ${status.tabId})`}` + if (!status.enabled) return copy.settings.devToolsOff + return copy.settings.devToolsIdle +} + /** * A page quote: the highlighted text plus where it came from. The quote is * page-authored, so it renders as plain text and never as markdown. @@ -646,6 +763,7 @@ export function App(): React.JSX.Element { const [showTextSize, setShowTextSize] = useState(false) const [approvalQueue, setApprovalQueue] = useState([]) const [tabAffinity, setTabAffinity] = useState(null) + const [tabAffinityReturnPending, setTabAffinityReturnPending] = useState(false) const [trustedOriginInput, setTrustedOriginInput] = useState('') const [error, setError] = useState(null) const [showSessionPicker, setShowSessionPicker] = useState(false) @@ -660,6 +778,23 @@ export function App(): React.JSX.Element { const [modelSelecting, setModelSelecting] = useState(false) const [showModelPicker, setShowModelPicker] = useState(false) const modelPickerRef = useRef(null) + const [sessionPermissions, setSessionPermissions] = useState(null) + const [permissionBusy, setPermissionBusy] = useState(false) + const [showPermissionPicker, setShowPermissionPicker] = useState(false) + const [permissionConfirm, setPermissionConfirm] = useState(null) + const [permissionError, setPermissionError] = useState(null) + const permissionPickerRef = useRef(null) + const [commandDirectory, setCommandDirectory] = useState(null) + const [skillDirectory, setSkillDirectory] = useState(null) + const [showSlashMenu, setShowSlashMenu] = useState(false) + const [slashIndex, setSlashIndex] = useState(0) + const [notice, setNotice] = useState(null) + const slashMenuRef = useRef(null) + const caretRef = useRef(0) + const composerCatalogsRef = useRef<{ commands: HostCommand[]; skills: HostSkill[] }>({ + commands: [], + skills: [], + }) const [sessionTitle, setSessionTitle] = useState(null) const [resumeHint, setResumeHint] = useState<{ ready: boolean; sessionId: string | null }>({ ready: false, sessionId: null }) const [questions, setQuestions] = useState([]) @@ -795,6 +930,7 @@ export function App(): React.JSX.Element { token: raw?.token ?? '', sharePageContent: raw?.sharePageContent ?? 'auto', unrestrictedBrowserAccess: raw?.unrestrictedBrowserAccess ?? false, + allowDevTools: raw?.allowDevTools ?? false, trustedActionOrigins: raw?.trustedActionOrigins ?? [], approvalNotifications: raw?.approvalNotifications ?? true, autoResumeSession: raw?.autoResumeSession ?? true, @@ -802,15 +938,60 @@ export function App(): React.JSX.Element { }) }, []) + // Developer-tools state lives in the background: it owns the debugger + // attachment and reports every change here. + const [devToolsStatus, setDevToolsStatus] = useState(null) + useEffect(() => api.onDevToolsStatus(setDevToolsStatus), [api]) + + // Discovery state reported by the background. 'not-found' is terminal for the + // current panel lease, so the status chip can say so instead of "未连接". + const [discovery, setDiscovery] = useState('idle') + const [bridgeAddress, setBridgeAddress] = useState(null) + // Manual scan from the settings page: the only writer of this list. + const [detecting, setDetecting] = useState(false) + const [detected, setDetected] = useState([]) + const [detectAttempted, setDetectAttempted] = useState(false) + const [detectProgress, setDetectProgress] = useState<{ done: number; total: number } | null>(null) + const [detectError, setDetectError] = useState(null) + + useEffect(() => api.onDiscoveryProgress((progress) => { + setDetectProgress({ done: progress.done, total: progress.total }) + setDetected(progress.instances) + }), [api]) + + const runDetection = async (): Promise => { + if (detecting) { + await api.cancelDiscovery().catch(() => {}) + return + } + setDetecting(true) + setDetectError(null) + setDetected([]) + setDetectProgress(null) + try { + setDetected(await api.startDiscovery()) + } catch (cause) { + // Cancellation is a user action, not an error worth showing. + if (cause instanceof PanelRpcError && cause.code === 'cancelled') return + setDetectError(cause instanceof Error ? cause.message : String(cause)) + } finally { + setDetectAttempted(true) + setDetecting(false) + setDetectProgress(null) + } + } + // 每次连接重启（连接配置变更/断线重连）都新建会话。状态消息逐条监听： // React 会把 stopped/connecting 等瞬时状态合并进同一帧渲染，依赖渲染 // 状态无法可靠观察到"连接已重置"，因此在这里按消息粒度判定。 const [sessionEpoch, setSessionEpoch] = useState(0) const lastStateRef = useRef(null) useEffect(() => { - const offStatus = api.onStatus((next, nextCaps) => { + const offStatus = api.onStatus((next, nextCaps, nextDiscovery, nextAddress) => { setState(next) setCaps(nextCaps) + setDiscovery(nextDiscovery) + setBridgeAddress(nextAddress ?? null) const previous = lastStateRef.current lastStateRef.current = next if (previous !== null && next !== previous && next === 'stopped') { @@ -838,10 +1019,24 @@ export function App(): React.JSX.Element { setStopping(false) setSessionChanging(false) setShowSessionPicker(false) + setSessionPermissions(null) + setPermissionBusy(false) + setShowPermissionPicker(false) + setPermissionConfirm(null) + setPermissionError(null) + setCommandDirectory(null) + setSkillDirectory(null) + setShowSlashMenu(false) + setNotice(null) stoppingRef.current = false clearQuestions() setSessionEpoch((epoch) => epoch + 1) } + if (previous !== null && next !== previous && next === 'connected' && sessionRef.current !== null) { + // A reconnect can outlive a directory read; reload instead of trusting + // whatever the previous bridge generation answered. + void refreshComposerDirectories(sessionRef.current) + } }) const offEvent = api.onEvent((frame) => { void onFrame(frame) }) const offApproval = api.onApprovalRequest((request) => { @@ -924,6 +1119,55 @@ export function App(): React.JSX.Element { } }, [showModelPicker]) + // The permission menu is a sibling overlay, so it owns its own dismissal. + useEffect(() => { + if (!showPermissionPicker) return + const onPointerDown = (event: PointerEvent): void => { + const root = permissionPickerRef.current + if (root !== null && event.target instanceof Node && root.contains(event.target)) return + setShowPermissionPicker(false) + } + const onKeyDown = (event: KeyboardEvent): void => { + if (event.key === 'Escape') setShowPermissionPicker(false) + } + document.addEventListener('pointerdown', onPointerDown, true) + document.addEventListener('keydown', onKeyDown, true) + return () => { + document.removeEventListener('pointerdown', onPointerDown, true) + document.removeEventListener('keydown', onKeyDown, true) + } + }, [showPermissionPicker]) + + // The slash menu belongs to the composer: a pointer outside both closes it. + useEffect(() => { + if (!showSlashMenu) return + const onPointerDown = (event: PointerEvent): void => { + const menu = slashMenuRef.current + if (menu !== null && event.target instanceof Node && menu.contains(event.target)) return + if (event.target instanceof Element && event.target.closest('.composer-box') !== null) return + setShowSlashMenu(false) + } + document.addEventListener('pointerdown', onPointerDown, true) + return () => { document.removeEventListener('pointerdown', onPointerDown, true) } + }, [showSlashMenu]) + + // A missed `turn/end` (dropped frame, panel opened mid-turn, sleep) must not + // disable the composer forever: re-read the Host's running flag on return. + useEffect(() => { + const onVisibilityChange = (): void => { + if (document.visibilityState !== 'visible') return + const sessionId = sessionRef.current + if (sessionId === null) return + void loadSessionCatalog().then(({ items }) => { + if (sessionRef.current !== sessionId) return + const entry = items.find((item) => item.sessionId === sessionId) + if (entry !== undefined) setWorking(entry.running) + }).catch(() => {}) + } + document.addEventListener('visibilitychange', onVisibilityChange) + return () => { document.removeEventListener('visibilitychange', onVisibilityChange) } + }, []) + function applyImageProjection(sessionId: string, seq: number, value: unknown): void { if (sessionRef.current !== sessionId || !Number.isSafeInteger(seq)) return const previous = imageProjectionRef.current @@ -1040,6 +1284,12 @@ export function App(): React.JSX.Element { await refreshHistory(payload.sessionId) return } + if (payload.event.type === 'permission/preset') { + // Another panel (or the Web GUI) may have switched this Session's preset; + // the projection is the authority, so re-read it for every own Session. + if (payload.sessionId === sessionRef.current) void refreshSessionPermissions(payload.sessionId) + return + } if (payload.sessionId !== sessionRef.current) return const row = rowFromEvent(payload.event) if (row !== null) { @@ -1181,7 +1431,7 @@ export function App(): React.JSX.Element { setSessionTitle(null) sessionRuntimeRef.current.seedRunning(created.sessionId, false) applyHistory(created.sessionId, await readHistory(created.sessionId)) - await refreshSessionModels(created.sessionId) + await refreshComposerDirectories(created.sessionId) } /** Pull the advisory model directory for the active session into the composer menu. */ @@ -1196,7 +1446,11 @@ export function App(): React.JSX.Element { const result = await api.rpc<{ current?: { provider: string; model: string; reasoningEffort?: string } routable?: boolean - groups?: Array<{ id: string; name: string; models: Array<{ id: string; name: string }> }> + groups?: Array<{ + id: string + name: string + models: Array<{ id: string; name: string; reasoning?: { efforts?: Array<{ id?: unknown; name?: unknown }> } }> + }> }>('session.models', { sessionId }) if (sessionRef.current !== sessionId) return const sessionOptions = (result.groups ?? []).flatMap((group) => group.models.map((model) => ({ @@ -1204,6 +1458,7 @@ export function App(): React.JSX.Element { providerName: group.name, id: model.id, name: model.name, + efforts: parseReasoningEfforts(model.reasoning?.efforts), }))) const options = sessionOptions.length > 0 ? sessionOptions : (fromSettings?.options ?? []) setSessionModels({ @@ -1215,11 +1470,43 @@ export function App(): React.JSX.Element { routable: result.routable ?? fromSettings?.routable ?? null, options, }) - } catch { + } catch (cause) { if (sessionRef.current !== sessionId) return // Host-wide settings still list configured relays before a session exists. - setSessionModels(fromSettings) + // An empty fallback is reported instead of a silently grey chip. + setSessionModels(fromSettings === null + ? { + current: null, + options: [], + routable: false, + error: cause instanceof Error ? cause.message : String(cause), + } + : { ...fromSettings, error: describeCatalogFailure(cause) }) + } + } + + /** Refresh every composer-side directory for one Session in one pass. */ + async function refreshComposerDirectories(sessionId: string | null = sessionRef.current): Promise { + await Promise.all([ + refreshSessionModels(sessionId), + refreshSessionPermissions(sessionId), + refreshComposerCatalogs(sessionId), + ]) + } + + /** Read the declared reasoning efforts out of one model catalog entry. */ + function parseReasoningEfforts(value: unknown): Array<{ id: string; name: string }> { + if (!Array.isArray(value)) return [] + const efforts: Array<{ id: string; name: string }> = [] + for (const entry of value) { + if (typeof entry !== 'object' || entry === null) continue + const id = typeof (entry as { id?: unknown }).id === 'string' ? (entry as { id: string }).id.trim() : '' + const name = typeof (entry as { name?: unknown }).name === 'string' + ? (entry as { name: string }).name.trim() + : id + if (id !== '') efforts.push({ id, name: name === '' ? id : name }) } + return efforts } /** Build a composer directory from saved llm-pi-ai routes (works without a Host session). */ @@ -1230,7 +1517,7 @@ export function App(): React.JSX.Element { const providers = (described.namespaces?.find((candidate) => candidate.ns === 'llm-pi-ai') ?.value?.providers ?? {}) as Record + models?: Array<{ id: string; name?: string; reasoning?: { efforts?: unknown } }> }> const defaults = (described.namespaces?.find((candidate) => candidate.ns === 'agent-default-model') ?.value ?? {}) as { provider?: unknown; model?: unknown; reasoningEffort?: unknown } @@ -1242,6 +1529,7 @@ export function App(): React.JSX.Element { : provider, id: model.id, name: typeof model.name === 'string' && model.name !== '' ? model.name : model.id, + efforts: parseReasoningEfforts(model.reasoning?.efforts), })) )).filter((option) => option.id !== '') if (options.length === 0) return { current: null, options: [], routable: false } @@ -1266,7 +1554,9 @@ export function App(): React.JSX.Element { /** Switch the next assembled turn onto the chosen provider/model. */ async function selectSessionModel(option: SessionModelOption): Promise { const sessionId = sessionRef.current - if (sessionId === null || modelSelecting || busy || working) return + // A selection only shapes the NEXT assembled turn, so a running turn (or an + // in-flight prompt) never blocks it — matching the Web GUI model picker. + if (sessionId === null || modelSelecting) return setShowModelPicker(false) if (sessionModels?.current?.provider === option.provider && sessionModels.current.model === option.id) return @@ -1279,6 +1569,10 @@ export function App(): React.JSX.Element { sessionId, provider: option.provider, model: option.id, + // Keep the effort the user already chose when it applies to the new model. + ...(option.efforts.some((effort) => effort.id === sessionModels?.current?.reasoningEffort) + ? { reasoningEffort: sessionModels?.current?.reasoningEffort } + : {}), }) if (sessionRef.current !== sessionId) return setSessionModels((current) => current === null @@ -1301,6 +1595,151 @@ export function App(): React.JSX.Element { } } + /** Pull the Session's permission directory (catalog plus its current preset). */ + async function refreshSessionPermissions(sessionId: string | null = sessionRef.current): Promise { + if (sessionId === null) { + setSessionPermissions(null) + setShowPermissionPicker(false) + return + } + try { + const result = await api.rpc('session.permissions', { sessionId }) + if (sessionRef.current !== sessionId) return + const parsed = parseSessionPermissions(result) + setSessionPermissions(parsed) + if (parsed === null) setPermissionError(copy.app.permissionUnavailable) + } catch (cause) { + if (sessionRef.current !== sessionId) return + setSessionPermissions(null) + setPermissionError(isUnavailableMethod(cause) ? copy.app.permissionUnavailable : describeCatalogFailure(cause)) + } + } + + /** + * Pull the Host command and skill directories for the composer's `/` menu. + * A provisional Session answers empty with `provisional: true`; the panel + * then keeps the last real directory and says when it becomes authoritative. + */ + async function refreshComposerCatalogs(sessionId: string | null = sessionRef.current): Promise { + if (sessionId === null) { + setCommandDirectory(null) + setSkillDirectory(null) + setShowSlashMenu(false) + return + } + const [commands, skills] = await Promise.all([ + api.rpc('commands.list', { sessionId }).catch((cause: unknown) => cause), + api.rpc('skills.list', { sessionId }).catch((cause: unknown) => cause), + ]) + if (sessionRef.current !== sessionId) return + const commandDirectory = commands instanceof Error ? null : parseCommandDirectory(commands) + const skillDirectory = skills instanceof Error ? null : parseSkillDirectory(skills) + if (commandDirectory !== null) setCommandDirectory(commandDirectory) + if (skillDirectory !== null) setSkillDirectory(skillDirectory) + if (commandDirectory !== null && !commandDirectory.provisional) { + composerCatalogsRef.current = { ...composerCatalogsRef.current, commands: commandDirectory.commands } + } + if (skillDirectory !== null && !skillDirectory.provisional) { + composerCatalogsRef.current = { ...composerCatalogsRef.current, skills: skillDirectory.skills } + } + } + + /** Switch the Session's permission preset, confirming the risky ones first. */ + async function applyPermissionPreset(preset: string): Promise { + const sessionId = sessionRef.current + if (sessionId === null || permissionBusy) return + setShowPermissionPicker(false) + setPermissionConfirm(null) + setPermissionBusy(true) + setPermissionError(null) + setNotice(null) + try { + const result = await api.rpc<{ result?: { kind?: string; text?: string } }>('session.setPermission', { + sessionId, + preset, + }) + if (sessionRef.current !== sessionId) return + if (result?.result?.kind === 'error') { + setPermissionError(copy.app.permissionSwitchFailed(result.result.text ?? preset)) + return + } + await refreshSessionPermissions(sessionId) + } catch (cause) { + if (sessionRef.current !== sessionId) return + setPermissionError(copy.app.permissionSwitchFailed(describeCatalogFailure(cause))) + } finally { + setPermissionBusy(false) + } + } + + /** Open the confirmation for a risky preset, or switch directly. */ + function requestPermissionPreset(preset: string, label: string): void { + if (!isSelectablePreset(preset) || permissionBusy) return + if (sessionPermissions?.current === preset) { + setShowPermissionPicker(false) + return + } + if (requiresRiskConfirmation(preset)) { + setShowPermissionPicker(false) + setPermissionConfirm({ preset, label }) + return + } + void applyPermissionPreset(preset) + } + + /** Run one Host command line; command output never becomes a model message. */ + async function runCommandLine(line: string): Promise { + const sessionId = sessionRef.current + if (sessionId === null) return false + setNotice(null) + setError(null) + try { + const result = await api.rpc<{ result?: { kind?: string; text?: string } }>('commands.execute', { + sessionId, + line, + }) + if (sessionRef.current !== sessionId) return true + const failure = result?.result?.kind === 'error' ? result.result.text : undefined + if (failure !== undefined) setError(copy.app.commandFailed(failure)) + else setNotice(result?.result?.text ?? copy.app.commandSucceeded) + // A permission line changed the Session's knobs; keep the chip honest. + void refreshSessionPermissions(sessionId) + return true + } catch (cause) { + if (sessionRef.current !== sessionId) return true + const reason = describeCatalogFailure(cause) + setError(isProvisionalFailure(cause) ? copy.app.commandProvisional : copy.app.commandFailed(reason)) + return false + } + } + + /** Move the next assembled turn onto a different reasoning effort. */ + async function selectReasoningEffort(option: SessionModelOption, reasoningEffort: string | undefined): Promise { + const sessionId = sessionRef.current + if (sessionId === null || modelSelecting) return + setShowModelPicker(false) + setModelSelecting(true) + setError(null) + try { + const result = await api.rpc<{ + selected: { provider: string; model: string; reasoningEffort?: string } + }>('session.selectModel', { + sessionId, + provider: option.provider, + model: option.id, + ...(reasoningEffort === undefined ? {} : { reasoningEffort }), + }) + if (sessionRef.current !== sessionId) return + setSessionModels((current) => current === null + ? { current: result.selected, routable: true, options: [option] } + : { ...current, current: result.selected, routable: true }) + } catch (cause) { + if (sessionRef.current === sessionId) setError(describeCatalogFailure(cause)) + } finally { + setModelSelecting(false) + } + } + /** Load the raw host index plus workspace archive state once. */ async function loadSessionCatalog(): Promise<{ items: SessionPickerEntry[] @@ -1349,7 +1788,7 @@ export function App(): React.JSX.Element { await api.setActiveSession(hinted) setSessionTitle(projectedSessionTitle(entry) ?? sessionDisplayTitle(entry)) applyHistory(hinted, history) - await refreshSessionModels(hinted) + await refreshComposerDirectories(hinted) return } } catch { @@ -1401,7 +1840,7 @@ export function App(): React.JSX.Element { sessionRef.current = entry.sessionId setSessionTitle(projectedSessionTitle(entry) ?? sessionDisplayTitle(entry)) await refreshHistory(entry.sessionId) - await refreshSessionModels(entry.sessionId) + await refreshComposerDirectories(entry.sessionId) } catch (cause) { if (sessionTransitionRef.current === transition) { setError(cause instanceof Error ? cause.message : String(cause)) @@ -1450,7 +1889,7 @@ export function App(): React.JSX.Element { setSessionTitle(sessionId) if (history !== undefined) applyHistory(sessionId, history) else setError(historyError instanceof Error ? historyError.message : String(historyError)) - await refreshSessionModels(sessionId) + await refreshComposerDirectories(sessionId) } finally { finishSessionTransition(transition) } @@ -1545,8 +1984,47 @@ export function App(): React.JSX.Element { } } + /** Accept one `/` menu row: run a local action or insert its exact text. */ + function pickSlashEntry(entry: SlashEntry): void { + setShowSlashMenu(false) + setSlashIndex(0) + setError(null) + setNotice(null) + if (entry.kind === 'local') { + setDraft(emptyComposerDraft()) + if (entry.action === 'model') setShowModelPicker(true) + return + } + // Inserting (rather than executing) keeps the row's own input editable. + setDraft((current) => ({ ...current, text: entry.insert })) + } + + /** Same arbitration as Enter: run the highlighted row, or submit the draft. */ + function acceptSlashMenu(): void { + const picked = slashRows[slashActiveIndex] + if (picked === undefined) return + pickSlashEntry(picked) + } + + /** Directories the composer resolves one submitted line against. */ + function composerLineCatalogs(): ComposerCatalogs { + const cached = composerCatalogsRef.current + return { + // A provisional Session answers empty; the last real directory is a + // better guess than dropping the user's command on the floor. + commands: commandDirectory !== null && commandDirectory.commands.length > 0 + ? commandDirectory.commands + : cached.commands, + skills: skillDirectory !== null && skillDirectory.skills.length > 0 + ? skillDirectory.skills + : cached.skills, + local: localSlashEntries(copy), + } + } + async function send(textOverride?: string): Promise { - const text = (textOverride ?? input).trim() + const submitted = textOverride ?? input + const text = submitted.trim() const submittedImages = textOverride === undefined ? draftImages : [] // A quick-start prompt asks about the whole page, not about a quote. const submittedSelection = textOverride === undefined ? selection : null @@ -1554,6 +2032,36 @@ export function App(): React.JSX.Element { // busy state 是异步的：连续回车可能都通过 state 检查——用 ref 同步锁。 if ((text === '' && submittedImages.length === 0 && submittedSelection === null) || busy || addingImagesRef.current || sendingRef.current || sessionChangingRef.current || id === null) return + + // A line starting with `/` is a command or a skill, never an ordinary + // prompt: the Host adapter would reject an unknown one anyway, so the panel + // refuses it here and keeps the draft instead of sending something else. + const resolution = resolveComposerLine(submitted, composerLineCatalogs()) + if (resolution.kind === 'unknown-command') { + setError(copy.app.unknownCommand(resolution.name)) + return + } + if (resolution.kind === 'local') { + setDraft(emptyComposerDraft()) + setShowSlashMenu(false) + if (resolution.name === 'model') setShowModelPicker(true) + return + } + if (resolution.kind === 'command') { + if (submittedImages.length > 0 || submittedSelection !== null) { + setError(copy.app.commandAttachmentsUnsupported) + return + } + sendingRef.current = true + setDraft(emptyComposerDraft()) + setShowSlashMenu(false) + try { + await runCommandLine(resolution.line) + } finally { + sendingRef.current = false + } + return + } sendingRef.current = true const submittedDraft: ComposerDraft = { text, images: submittedImages } if (textOverride === undefined) { @@ -1575,7 +2083,7 @@ export function App(): React.JSX.Element { ...(clientTimeZone === undefined ? {} : { clientTimeZone }), }) // Deferred sessions materialize on the first prompt; refresh so chips appear. - void refreshSessionModels(id) + void refreshComposerDirectories(id) if (submittedSelection !== null) { // Keep the background authoritative while the prompt is in flight. // Conditional clearing cannot consume a newer highlight captured in @@ -1886,6 +2394,25 @@ export function App(): React.JSX.Element { } } + /** + * Ask the background to move this browser's visible tab back to the page the + * assistant is bound to. A stale revision means the user already switched + * again; the fresh state is on its way, so that case stays silent. + */ + async function returnToControlledTab(): Promise { + if (tabAffinity === null || tabAffinityReturnPending) return + setTabAffinityReturnPending(true) + try { + await api.returnToControlledTab(tabAffinity.revision) + } catch (cause) { + if (!(cause instanceof PanelRpcError && cause.code === 'stale-revision')) { + setError(cause instanceof Error ? cause.message : String(cause)) + } + } finally { + setTabAffinityReturnPending(false) + } + } + function addTrustedOrigin(): void { const origin = normalizeWebOrigin(trustedOriginInput) if (origin === null) return @@ -1902,7 +2429,10 @@ export function App(): React.JSX.Element { } // 状态栏只显示连接状态；快照上限是技术细节，在设置页说明（见 hint）。 - const statusText = copy.status[state] + // Discovery can end in a confirmed miss; that is more useful than "未连接". + const statusText = state === 'stopped' && discovery === 'not-found' + ? copy.statusNotFound + : copy.status[state] const sessionMenuTitle = sessionTitle ?? copy.app.newSession const currentModelOption = sessionModels?.options.find((option) => ( option.provider === sessionModels.current?.provider @@ -1928,9 +2458,81 @@ export function App(): React.JSX.Element { } return groups })() + const reasoningOptions = currentModelOption?.efforts ?? [] + const currentEffort = sessionModels?.current?.reasoningEffort + const currentEffortLabel = currentEffort === undefined + ? undefined + : (reasoningOptions.find((effort) => effort.id === currentEffort)?.name ?? currentEffort) + const modelChipLabel = `${currentModelLabel}${currentEffortLabel === undefined ? '' : ` · ${currentEffortLabel}`}` + const permissionOption = sessionPermissions?.options.find((option) => ( + option.value === sessionPermissions.current + )) + const permissionChipLabel = sessionPermissions === null + ? copy.app.permissionControl + : permissionLabel(sessionPermissions.current, permissionOption, copy) + const defaultPermissionLabel = sessionPermissions === null + ? null + : permissionLabel( + sessionPermissions.defaultPreset, + sessionPermissions.options.find((option) => option.value === sessionPermissions.defaultPreset), + copy, + ) + const slashQueryText = showSlashMenu ? slashQuery(input, caretRef.current) : null + const slashRows = (() => { + const entries = slashEntries( + commandDirectory?.commands ?? composerCatalogsRef.current.commands, + skillDirectory?.skills ?? composerCatalogsRef.current.skills, + localSlashEntries(copy), + ) + return rankSlashEntries(entries, slashQueryText ?? '') + })() + // No directory yet, or a deferred Session's empty answer: the menu lists what + // the last real Session knew and says so. + const slashDirectoryProvisional = commandDirectory === null || commandDirectory.provisional + const slashActiveIndex = slashRows.length === 0 ? -1 : Math.min(slashIndex, slashRows.length - 1) const approvalDialog = !approvalReadyForSession(queuedApproval, sessionRef.current, sessionChanging) ? null : + const permissionDialog = permissionConfirm === null + ? null + : ( +
+
+
+
+ ) if (showSettings) { return ( @@ -1953,6 +2555,52 @@ export function App(): React.JSX.Element { placeholder={copy.settings.bridgePlaceholder} /> +
+
+ + + {detectProgress === null + ? copy.settings.detectHelp + : copy.settings.detectProgress(detectProgress.done, detectProgress.total)} + +
+ {detectError === null ? null : {detectError}} + {detected.length > 0 ? ( +
    + {detected.map((instance) => { + const current = bridgeAddress === instance.wsUrl + return ( +
  • + +
    • + ) + })} +
+ ) : detectAttempted && !detecting ? ( + {copy.settings.detectEmpty} + ) : null} + {bridgeAddress === null || state !== 'connected' + ? null + : {copy.settings.detectConnected(bridgeAddress)}} +
+
+ + {devToolsStatus?.attached === true && ( + + )} +