diff --git a/README.i18n.yaml b/README.i18n.yaml index 8a63c95eb..3f26e5254 100644 --- a/README.i18n.yaml +++ b/README.i18n.yaml @@ -1,5 +1,5 @@ # Bilingual consistency record for the repository README files. # README.md is the default English page; README.zh.md is its Chinese counterpart. # Values are git blob hashes from the last confirmed-consistent review. -README.md: 1163cc9c9a7332a95f8b2448461243f0813ffdf2 -README.zh.md: 6ce7a8314f10dcd45316906f79991e0604b0814c +README.md: ebbf7f40daeaad7d88e8fbbb36432408d2e40698 +README.zh.md: c7a10e409a768cc84ea6988f884927f345b9425f diff --git a/README.md b/README.md index 3f743c5b9..ce82fe151 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,12 @@ The paired Playwright / extension duration ratio was **1.24** (95% CI **1.16–1 | Capability | Tool | Notes | |---|---|---| -| Read page | `browser_snapshot` | Structured text snapshot: title, URL, main text, numbered controls, and masked form fields; `delta: true` returns only changes | -| Click element | `browser_click` | Click links, buttons, checkboxes, and other controls by inventory number | -| Fill forms | `browser_type` | React/Vue-compatible input; `replace` clears the field first | -| Press keys | `browser_press` | Keyboard events such as Enter, Tab, Escape, and arrow keys | +| Read page | `browser_snapshot` | Structured text snapshot: title, URL, main text, numbered controls (including heuristic clickables), and masked form fields; `region` scopes text **and** inventory; `delta: true` returns only changes | +| Click element | `browser_click` | Click by snapshot index, CSS selector, or visible text (exactly one). Prefer higher `depth` for nested heuristics. Open picker panels expose `[overlay]` options; prefer `text:"2024"` / `text:"01"` for year/month cells. Dispatches pointerdown→mousedown→focus→mouseup→click | +| Fill forms | `browser_type` | Address by index or selector; React/Vue-compatible input; `replace` clears first; hidden date/select inputs are listed and writable. Writing a hidden input does **not** update controlled design-system pickers — click the panel cells instead | +| Focus element | `browser_focus` | Focus by index or selector before press/type | +| Upload file | `browser_upload` | Host reads an absolute local path (size/extension limits) into `input[type=file]`; requires approval | +| Press keys | `browser_press` | Sends a key to the focused element (Enter, Escape, arrows, Backspace, Delete). Does not move focus via Tab, does not produce IME text, and does not synthesize form submit while a picker/dropdown overlay is open | | Scroll | `browser_scroll` | Viewport scrolling: up, down, top, and bottom | | Navigate | `browser_navigate` / `browser_open_tab` / `browser_back` / `browser_forward` / `browser_reload` | Navigation inside the controlled tab, or open a URL in a new tab and follow it (`active:false` keeps the current tab in front) | | List tabs | `browser_list_tabs` | List accessible tabs with stable IDs, titles, URLs, window/index metadata, and active/controlled state | @@ -63,6 +65,13 @@ The paired Playwright / extension duration ratio was **1.24** (95% CI **1.16–1 | Send images | `session.prompt` / `session.attachment` | Host-capability-gated image drafts, image-only prompts, and durable history previews | | Quote a selection | side panel composer | Text you highlight in the page appears in the composer and is sent with your next message as fenced, attributed page content | +### Known limits + +- Closed Shadow DOM is not readable from the content script. +- `browser_press` synthesizes keyboard events on the focused element; it does not move focus via Tab, cannot drive IME composition, and does not synthesize form submit while a floating picker/dropdown is open (persistent in-flow panels do not count). Trusted-event-only flows need CDP / `chrome.debugger` (out of scope). +- Nested heuristic controls expose `depth`; when a shallow click does nothing, retry a higher-depth sibling on the same chain. +- Visually hidden inputs are listed (values always masked) so agents can address them; assigning them with `browser_type` will not open panels or update React/Vue picker state — click overlay cells instead. + ## Repository layout ``` @@ -78,7 +87,7 @@ scripts/install.ps1 - **Your real browser, not a headless copy**: the model works in the page you already have open, retaining logins, sessions, and cookies. - **A text-first page interface**: numbered controls, stable IDs across snapshots, delta updates, and masked sensitive values make pages operable without screenshots; user-attached chat images use dsh's separate multimodal message path. - **Pointing instead of describing**: highlight the passage you mean and the side panel quotes it, so "explain this" needs no page tour. The quote is captured only while a panel is open, and nothing is sent until you send the message. -- **A narrow privacy boundary**: passwords and payment-card values are always rendered as `••••` and never leave the page. +- **A narrow privacy boundary**: passwords, payment-card values, token/OTP-like fields, and CSS-hidden form inventory values are always rendered as `••••` and never leave the page. - **A guarded bridge**: authenticated handshakes protect remote connections, privileged gateway methods reject non-loopback callers, and the extension binds tools to one user-controlled tab. ## Detailed installation and usage diff --git a/README.zh.md b/README.zh.md index fc8fe4d45..056bc0e05 100644 --- a/README.zh.md +++ b/README.zh.md @@ -49,10 +49,12 @@ Playwright / 扩展的配对耗时比为 **1.24**（95% CI **1.16–1.34**）： | 能力 | 工具 | 说明 | |---|---|---| -| 读取页面 | `browser_snapshot` | 结构化文本快照：标题/URL/正文/编号交互清单/表单字段（敏感值掩码）；`delta: true` 只返回变化 | -| 点击元素 | `browser_click` | 按编号点击链接/按钮/复选框等 | -| 填写表单 | `browser_type` | 输入文本（React/Vue 受控组件兼容），`replace` 清空重填 | -| 按键 | `browser_press` | 键盘事件（Enter/Tab/Escape/方向键…） | +| 读取页面 | `browser_snapshot` | 结构化文本快照：标题/URL/正文/编号交互清单（含启发式可点击项）/表单字段（敏感值掩码）；`region` 同时限定正文与清单；`delta: true` 只返回变化 | +| 点击元素 | `browser_click` | 按编号、CSS 选择器或可见文本三选一点击。嵌套启发式优先更高 `depth`；打开的日期面板会露出 `[overlay]` 选项，年份/月份请用 `text:"2024"` / `text:"01"`。派发 pointerdown→mousedown→focus→mouseup→click | +| 填写表单 | `browser_type` | 按编号或选择器定位；兼容 React/Vue 受控输入；`replace` 清空重填；隐藏的日期/下拉真实 input 可列出并写入。**写入隐藏 input 不会驱动受控日期组件**——应点击面板单元格 | +| 聚焦元素 | `browser_focus` | 按编号或选择器聚焦，便于后续按键/输入 | +| 上传文件 | `browser_upload` | Host 读取本机绝对路径（有大小/扩展名限制）写入 `input[type=file]`；需审批 | +| 按键 | `browser_press` | 向当前焦点元素发送按键（Enter/Escape/方向键/Backspace/Delete）。不是真实 Tab 焦点遍历，不支持 IME；浮层打开时不会合成 form submit | | 滚动 | `browser_scroll` | 视口滚动（up/down/top/bottom） | | 页面导航 | `browser_navigate` / `browser_open_tab` / `browser_back` / `browser_forward` / `browser_reload` | 受控标签页内导航，或新开标签页并跟随（`active:false` 时保持当前页在前台） | | 列出标签页 | `browser_list_tabs` | 列出可访问标签页的稳定 ID、标题、URL、窗口/顺序以及活动/受控状态 | @@ -63,6 +65,13 @@ Playwright / 扩展的配对耗时比为 **1.24**（95% CI **1.16–1.34**）： | 发送图片 | `session.prompt` / `session.attachment` | 按宿主能力启用图片草稿、纯图片消息和持久历史预览 | | 引用选中内容 | 侧栏输入框 | 在页面里划选的文字会出现在输入框，随下一条消息一起发送，并带上来源与不可信内容边界 | +### 已知限制 + +- Closed Shadow DOM 对内容脚本不可读。 +- `browser_press` 只在焦点元素上合成键盘事件，不能用 Tab 真实移动焦点，也不能驱动 IME；需要可信事件的场景需 CDP / `chrome.debugger`（不在本仓库范围）。 +- 嵌套启发式控件带 `depth`；浅层点击无效时，应改点同链路上更高 depth 的项。 +- 视觉隐藏的 input 只反映受控 picker 的当前值；用 `browser_type` 写入它们不会打开面板，也不会更新 React/Vue 组件状态。 + ## 组成 ``` diff --git a/extensions/dsh-browser/manifest.firefox.json b/extensions/dsh-browser/manifest.firefox.json index 8d6dda156..5aedc1be3 100644 --- a/extensions/dsh-browser/manifest.firefox.json +++ b/extensions/dsh-browser/manifest.firefox.json @@ -3,7 +3,7 @@ "name": "__MSG_extensionName__", "description": "__MSG_extensionDescription__", "default_locale": "en", - "version": "0.1.4", + "version": "0.1.5", "browser_specific_settings": { "gecko": { "id": "dsh-browser@lum1104.github.io", diff --git a/extensions/dsh-browser/manifest.json b/extensions/dsh-browser/manifest.json index d1311c27b..9d4f1716e 100644 --- a/extensions/dsh-browser/manifest.json +++ b/extensions/dsh-browser/manifest.json @@ -3,7 +3,7 @@ "name": "__MSG_extensionName__", "description": "__MSG_extensionDescription__", "default_locale": "en", - "version": "0.1.4", + "version": "0.1.5", "minimum_chrome_version": "116", "permissions": [ "sidePanel", diff --git a/extensions/dsh-browser/package.json b/extensions/dsh-browser/package.json index d7d949c5b..7193897ae 100644 --- a/extensions/dsh-browser/package.json +++ b/extensions/dsh-browser/package.json @@ -1,7 +1,7 @@ { "name": "dsh-browser-extension", "description": "Chrome and Firefox MV3 extension: sidebar chat with a local dsh instance and text-only read/operate of a user-controlled tab through the dsh browser bridge", - "version": "0.1.4", + "version": "0.1.5", "author": "Yuxiang Lin", "license": "MIT", "private": true, diff --git a/extensions/dsh-browser/src/background/authorization.ts b/extensions/dsh-browser/src/background/authorization.ts index d263a0587..7f22997b5 100644 --- a/extensions/dsh-browser/src/background/authorization.ts +++ b/extensions/dsh-browser/src/background/authorization.ts @@ -9,6 +9,8 @@ const PAGE_READS = new Set(['browser_snapshot', 'browser_get_text']) const STATE_CHANGING_ACTIONS = new Set([ 'browser_click', 'browser_type', + 'browser_focus', + 'browser_upload', 'browser_press', 'browser_navigate', 'browser_open_tab', @@ -110,15 +112,33 @@ function summarizeAction(call: ToolCall, locale: UiLocale): string { const frame = typeof call.args.frame === 'number' && call.args.frame !== 0 ? localized(locale, `, iframe ${call.args.frame}`, `，iframe ${call.args.frame}`) : '' - const index = typeof call.args.index === 'number' ? call.args.index : '?' switch (call.name) { - case 'browser_click': return localized(locale, `Click element [${index}]${frame}`, `点击元素 [${index}]${frame}`) + case 'browser_click': return localized( + locale, + `Click ${describeTarget(call.args)}${frame}`, + `点击${describeTargetZh(call.args)}${frame}`, + ) case 'browser_type': { const length = typeof call.args.text === 'string' ? call.args.text.length : 0 return localized( locale, - `Enter ${length} characters in element [${index}]${frame} (the text is not shown in this dialog)`, - `向元素 [${index}] 输入 ${length} 个字符${frame}（文本内容不会显示在确认框）`, + `Enter ${length} characters in ${describeTarget(call.args)}${frame} (the text is not shown in this dialog)`, + `向${describeTargetZh(call.args)}输入 ${length} 个字符${frame}（文本内容不会显示在确认框）`, + ) + } + case 'browser_focus': return localized( + locale, + `Focus ${describeTarget(call.args)}${frame}`, + `聚焦${describeTargetZh(call.args)}${frame}`, + ) + case 'browser_upload': { + const name = typeof call.args.name === 'string' && call.args.name !== '' + ? call.args.name + : typeof call.args.path === 'string' ? basename(call.args.path) : 'file' + return localized( + locale, + `Upload “${safeInline(name)}” to ${describeTarget(call.args)}${frame}`, + `上传「${safeInline(name)}」到${describeTargetZh(call.args)}${frame}`, ) } case 'browser_press': return localized( @@ -152,6 +172,34 @@ function summarizeAction(call: ToolCall, locale: UiLocale): string { } } +function describeTarget(args: Record): string { + if (typeof args.selector === 'string' && args.selector !== '') { + return `element matching selector “${safeInline(args.selector, 80)}”` + } + if (typeof args.text === 'string' && args.text !== '' && args.index === undefined && args.selector === undefined) { + // click-by-text only; type uses text as payload + return `element with text “${safeInline(args.text)}”` + } + const index = typeof args.index === 'number' ? args.index : '?' + return `element [${index}]` +} + +function describeTargetZh(args: Record): string { + if (typeof args.selector === 'string' && args.selector !== '') { + return `匹配选择器「${safeInline(args.selector, 80)}」的元素` + } + if (typeof args.text === 'string' && args.text !== '' && args.index === undefined && args.selector === undefined) { + return `文本为「${safeInline(args.text)}」的元素` + } + const index = typeof args.index === 'number' ? args.index : '?' + return `元素 [${index}]` +} + +function basename(path: string): string { + const parts = path.split(/[/\\]/) + return parts[parts.length - 1] || path +} + function displayUrl(value: string, locale: UiLocale): string { try { const url = new URL(value) diff --git a/extensions/dsh-browser/src/background/tools.ts b/extensions/dsh-browser/src/background/tools.ts index 0fc3069a1..2a6376174 100644 --- a/extensions/dsh-browser/src/background/tools.ts +++ b/extensions/dsh-browser/src/background/tools.ts @@ -51,6 +51,8 @@ const CONTENT_SCRIPT_FILE = 'content.js' const ACTION_DELTA_TOOLS = new Set([ 'browser_click', 'browser_type', + 'browser_focus', + 'browser_upload', 'browser_press', 'browser_scroll', 'browser_wait', @@ -74,6 +76,8 @@ const TAB_NATIVE_TOOLS = new Set([ const STATE_CHANGING_PAGE_TOOLS = new Set([ 'browser_click', 'browser_type', + 'browser_focus', + 'browser_upload', 'browser_press', 'browser_scroll', 'browser_navigate', @@ -81,6 +85,12 @@ const STATE_CHANGING_PAGE_TOOLS = new Set([ 'browser_forward', 'browser_reload', ]) +const ELEMENT_TARGET_TOOLS = new Set([ + 'browser_click', + 'browser_type', + 'browser_focus', + 'browser_upload', +]) /** Tools that operate on the browser tab collection rather than one page document. */ export const TAB_MANAGEMENT_TOOL_NAMES = new Set([ 'browser_list_tabs', @@ -807,7 +817,7 @@ function validateFrameTarget(call: ToolCall, frames: TabFrame[]): ToolAnswer | u } function validateElementTarget(call: ToolCall, tabId: number, frames: TabFrame[]): ToolAnswer | undefined { - if (call.name !== 'browser_click' && call.name !== 'browser_type') return undefined + if (!ELEMENT_TARGET_TOOLS.has(call.name)) return undefined const frameId = requestedFrame(call.args) const frame = frames.find((candidate) => candidate.frameId === frameId) const snapshotted = snapshotDocumentsByTab.get(tabId)?.get(frameId) diff --git a/extensions/dsh-browser/src/content/actions.ts b/extensions/dsh-browser/src/content/actions.ts index 7cdf60b74..ddcc61f00 100644 --- a/extensions/dsh-browser/src/content/actions.ts +++ b/extensions/dsh-browser/src/content/actions.ts @@ -10,10 +10,25 @@ * @module */ -import { pageText, truncate } from './extract.ts' +import { + accessibleName, + hasPointerCursor, + isClickableTarget, + isVisible, + pageText, + querySelectorAllDeep, + documentHasOpenOverlay, + InvalidSelectorError, + truncate, +} from './extract.ts' import type { ElementIds } from './ids.ts' import type { SnapshotBudget } from './snapshot.ts' -import { buildSnapshot, renderSnapshot } from './snapshot.ts' +import { + buildSnapshot, + renderSnapshot, + SnapshotInvalidSelectorError, + SnapshotRegionError, +} from './snapshot.ts' /** A settled action result. */ export interface ActionResult { @@ -134,7 +149,7 @@ function elementOrThrow(ids: ElementIds, index: number): Element { /** Error carrying a stable wire code. */ export class ActionError extends Error { constructor( - readonly code: 'action-failed' | 'bad-args', + readonly code: 'action-failed' | 'bad-args' | 'no-match' | 'ambiguous-match', message: string, ) { super(message) @@ -142,21 +157,247 @@ export class ActionError extends Error { } } -/** React-compatible value write: native setter + input/change events. */ +/** React-compatible value write: native setter + beforeinput/input/change events. */ function setNativeValue(input: HTMLInputElement | HTMLTextAreaElement, value: string): void { + const previous = input.value const prototype = input instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype const setter = Object.getOwnPropertyDescriptor(prototype, 'value')?.set + dispatchBeforeInput(input, value, previous === '' ? 'insertText' : 'insertReplacementText') if (setter === undefined) { input.value = value } else { setter.call(input, value) } - input.dispatchEvent(new Event('input', { bubbles: true })) + input.dispatchEvent(new InputEvent('input', { bubbles: true, data: value, inputType: 'insertText' })) input.dispatchEvent(new Event('change', { bubbles: true })) } +function dispatchBeforeInput(target: EventTarget, data: string, inputType: string): void { + try { + target.dispatchEvent(new InputEvent('beforeinput', { + bubbles: true, + cancelable: true, + data, + inputType, + })) + } catch { + // jsdom / older engines may lack InputEvent; skip without failing the write. + } +} + +/** How an action addressed its target. */ +interface ResolvedTarget { + element: Element + label: string + /** Skip scrollIntoView for visually hidden form controls. */ + skipScroll: boolean +} + +/** + * Resolve `index` | `selector` | `text` (mutually exclusive) to one element. + * @param mode - `click` allows text targeting; `type`/`focus`/`upload` do not. + */ +function resolveTarget( + ids: ElementIds, + args: Record, + mode: 'click' | 'type' | 'focus' | 'upload', +): ResolvedTarget { + const hasIndex = args.index !== undefined + const hasSelector = typeof args.selector === 'string' && args.selector !== '' + // For type/upload, `text` is the payload / unused; only click uses text targeting. + const hasTextTarget = mode === 'click' && typeof args.text === 'string' && args.text !== '' + const modes = [hasIndex, hasSelector, hasTextTarget].filter(Boolean).length + if (modes === 0) { + throw new ActionError( + 'bad-args', + mode === 'click' + ? 'Provide exactly one of index, selector, or text to address the element.' + : 'Provide exactly one of index or selector to address the element.', + ) + } + if (modes > 1) { + throw new ActionError('bad-args', 'index, selector, and text addressing are mutually exclusive; provide only one.') + } + + if (hasIndex) { + const index = numberArg(args, 'index') + const el = elementOrThrow(ids, index) + return { + element: el, + label: `[${index}]`, + skipScroll: !isVisible(el), + } + } + + if (hasSelector) { + const selector = args.selector as string + const allowHidden = args.allowHidden === true + const nth = optionalNth(args) + let matches: Element[] + try { + matches = querySelectorAllDeep(document, selector) + } catch (error) { + if (error instanceof InvalidSelectorError) { + throw new ActionError('bad-args', error.message) + } + throw error + } + const candidates = allowHidden ? matches : matches.filter((el) => isVisible(el)) + if (candidates.length === 0) { + throw new ActionError( + 'no-match', + allowHidden + ? `No element matched selector: ${selector}` + : `No visible element matched selector: ${selector}. Pass allowHidden=true to target hidden matches.`, + ) + } + if (nth !== undefined) { + if (nth < 0 || nth >= candidates.length) { + throw new ActionError( + 'no-match', + `selector "${selector}" matched ${candidates.length} element(s); nth=${nth} is out of range (0..${candidates.length - 1}).`, + ) + } + const el = candidates[nth]! + return { element: el, label: `selector "${selector}" nth=${nth}`, skipScroll: !isVisible(el) } + } + if (candidates.length > 1) { + throw new ActionError('ambiguous-match', formatAmbiguous(selector, candidates)) + } + const el = candidates[0]! + return { element: el, label: `selector "${selector}"`, skipScroll: !isVisible(el) } + } + + const needle = (args.text as string).trim() + const exact = args.exact !== false + const nth = optionalNth(args) + const textMatches = findByVisibleText(needle, exact) + if (textMatches.length === 0) { + throw new ActionError('no-match', `No visible element matched text: "${needle}"`) + } + const chosen = nth !== undefined + ? (() => { + if (nth < 0 || nth >= textMatches.length) { + throw new ActionError( + 'no-match', + `text "${needle}" matched ${textMatches.length} element(s); nth=${nth} is out of range (0..${textMatches.length - 1}).`, + ) + } + return textMatches[nth]! + })() + : textMatches.length === 1 + ? textMatches[0]! + : pickDeepestSmallest(textMatches) + if (nth === undefined && textMatches.length > 1 && !sameClickableLift(textMatches, chosen)) { + // Multiple distinct clickable lifts — require nth. + const lifts = textMatches.map((el) => liftClickable(el)) + const unique = uniqueElements(lifts) + if (unique.length > 1) { + throw new ActionError('ambiguous-match', formatAmbiguous(`text "${needle}"`, unique)) + } + } + const target = liftClickable(chosen) + return { + element: target, + label: `text "${needle}"`, + skipScroll: false, + } +} + +function optionalNth(args: Record): number | undefined { + if (args.nth === undefined) return undefined + if (typeof args.nth !== 'number' || !Number.isInteger(args.nth) || args.nth < 0) { + throw new ActionError('bad-args', `nth must be a non-negative integer; received ${String(args.nth)}.`) + } + return args.nth +} + +function formatAmbiguous(query: string, candidates: Element[]): string { + const preview = candidates.slice(0, 5).map((el, i) => { + const name = accessibleName(el) + return ` #${i}: ${el.tagName.toLowerCase()} "${name}"` + }).join('\n') + const more = candidates.length > 5 ? `\n …and ${candidates.length - 5} more` : '' + return `Ambiguous match for ${query}: ${candidates.length} candidates. Pass nth to disambiguate.\n${preview}${more}` +} + +function findByVisibleText(needle: string, exact: boolean): Element[] { + const matches: Element[] = [] + const walk = (root: Document | Element | ShadowRoot): void => { + const nodes = root.querySelectorAll('*') + for (const el of nodes) { + if (!isVisible(el)) continue + const text = (el.textContent ?? '').replace(/\s+/g, ' ').trim() + const hit = exact ? text === needle : text.includes(needle) + if (hit) matches.push(el) + if (el.shadowRoot !== null) walk(el.shadowRoot) + } + } + walk(document) + // Keep leaf matches only: ancestors share the same textContent and would + // otherwise explode into ambiguous clickable lifts (html/body/…). + return matches.filter((el) => !matches.some((other) => other !== el && el.contains(other))) +} + +function pickDeepestSmallest(elements: Element[]): Element { + return elements.reduce((best, el) => { + const bestDepth = depthOf(best) + const depth = depthOf(el) + if (depth > bestDepth) return el + if (depth < bestDepth) return best + const bestArea = areaOf(best) + const area = areaOf(el) + return area < bestArea ? el : best + }) +} + +function depthOf(el: Element): number { + let depth = 0 + let node: Element | null = el + while (node !== null) { + depth += 1 + node = node.parentElement + } + return depth +} + +function areaOf(el: Element): number { + const rect = el.getBoundingClientRect() + return Math.max(0, rect.width) * Math.max(0, rect.height) +} + +/** Walk up at most 3 ancestors looking for a clickable host. */ +function liftClickable(el: Element): Element { + let current: Element | null = el + for (let i = 0; i < 4 && current !== null; i += 1) { + if (isClickableTarget(current, hasPointerCursor)) return current + current = current.parentElement + } + return el +} + +function sameClickableLift(elements: Element[], chosen: Element): boolean { + const lift = liftClickable(chosen) + return elements.every((el) => liftClickable(el) === lift) +} + +function uniqueElements(elements: Element[]): Element[] { + const seen = new Set() + const out: Element[] = [] + for (const el of elements) { + if (seen.has(el)) continue + seen.add(el) + out.push(el) + } + return out +} + +function describeTarget(target: ResolvedTarget): string { + return target.label +} + /** Action implementations; each returns a text result. */ export interface ActionContext { ids: ElementIds @@ -174,6 +415,10 @@ export async function runAction(action: string, args: Record, c return clickAction(args, ctx) case 'browser_type': return typeAction(args, ctx) + case 'browser_focus': + return focusAction(args, ctx) + case 'browser_upload': + return uploadAction(args, ctx) case 'browser_press': return pressAction(args, ctx) case 'browser_scroll': @@ -198,10 +443,20 @@ export async function runAction(action: string, args: Record, c function snapshotAction(args: Record, ctx: ActionContext): ActionResult { const delta = args.delta === true const region = typeof args.region === 'string' && args.region !== '' ? args.region : undefined - // 基线在每次快照后都更新：delta 调用才能相对上一次（无论是否 delta）比较。 - const view = buildSnapshot(ctx.ids, { delta, region, budget: ctx.budget }, lastSnapshot) - lastSnapshot = view - return { text: renderSnapshot(view, delta) } + try { + // 基线在每次快照后都更新：delta 调用才能相对上一次（无论是否 delta）比较。 + const view = buildSnapshot(ctx.ids, { delta, region, budget: ctx.budget }, lastSnapshot) + lastSnapshot = view + return { text: renderSnapshot(view, delta) } + } catch (error) { + if (error instanceof SnapshotRegionError) { + throw new ActionError('no-match', error.message) + } + if (error instanceof SnapshotInvalidSelectorError) { + throw new ActionError('bad-args', error.message) + } + throw error + } } /** Module-level last snapshot state for delta mode (content-script lifetime). */ @@ -224,12 +479,15 @@ function withPageDelta(text: string, ctx: ActionContext): ActionResult { } async function clickAction(args: Record, ctx: ActionContext): Promise { - const index = numberArg(args, 'index') - const el = elementOrThrow(ctx.ids, index) - el.scrollIntoView({ block: 'center', behavior: 'instant' }) + const target = resolveTarget(ctx.ids, args, 'click') + const el = target.element + const label = describeTarget(target) + if (!target.skipScroll) { + el.scrollIntoView({ block: 'center', behavior: 'instant' }) + } if (el instanceof HTMLAnchorElement) { - const target = el.target.trim().toLowerCase() - const sameFrameTarget = target === '' || target === '_self' + const targetAttr = el.target.trim().toLowerCase() + const sameFrameTarget = targetAttr === '' || targetAttr === '_self' let href: URL | undefined try { href = new URL(el.href) } catch { /* let the native click handle unusual links */ } const controlledNavigation = sameFrameTarget @@ -249,7 +507,7 @@ async function clickAction(args: Record, ctx: ActionContext): P if (requiresNativeActivation) { setTimeout(() => { el.click() }, 0) return { - text: `Clicked link [${index}] using native browser activation. Call browser_snapshot to read the resulting state.`, + text: `Clicked link ${label} using native browser activation. Call browser_snapshot to read the resulting state.`, } } // Dispatch the click handlers without its default navigation so a @@ -261,7 +519,7 @@ async function clickAction(args: Record, ctx: ActionContext): P })) if (!shouldNavigate) { await waitForPageSettled(ACTION_SETTLE) - return withPageDelta(`Clicked link [${index}].`, ctx) + return withPageDelta(`Clicked link ${label}.`, ctx) } const sameDocument = href.origin === location.origin && href.pathname === location.pathname @@ -269,25 +527,89 @@ async function clickAction(args: Record, ctx: ActionContext): P if (sameDocument) { if (href.hash !== location.hash) location.hash = href.hash await waitForPageSettled(ACTION_SETTLE) - return withPageDelta(`Clicked link [${index}].`, ctx) + return withPageDelta(`Clicked link ${label}.`, ctx) } // A cross-document navigation can unload this content script before an // awaited response. Answer first and navigate in the next task. setTimeout(() => { location.href = href.href }, 0) return { - text: `Clicked link [${index}]. Call browser_snapshot again after navigation settles.`, + text: `Clicked link ${label}. Call browser_snapshot again after navigation settles.`, navigationPending: true, } } setTimeout(() => { el.click() }, 0) - return { text: `Clicked link [${index}]. The link may open outside the controlled frame.` } + return { text: `Clicked link ${label}. The link may open outside the controlled frame.` } } if (el instanceof HTMLButtonElement && el.disabled) { - throw new ActionError('action-failed', `Button [${index}] is disabled.`) + throw new ActionError('action-failed', `Button ${label} is disabled.`) } - ;(el as HTMLElement).click() + synthesizePointerClick(el as HTMLElement) await waitForPageSettled(ACTION_SETTLE) - return withPageDelta(`Clicked [${index}].`, ctx) + return withPageDelta(`Clicked ${label}.`, ctx) +} + +/** + * Dispatch a pointer/mouse sequence that matches a real user press. + * + * Many design-system pickers open on `pointerdown`/`mousedown`/`focus`, not + * on the synthetic `HTMLElement.click()` event alone. + */ +function synthesizePointerClick(target: HTMLElement): void { + // Omit `view`: jsdom rejects `view: window` on MouseEvent/PointerEvent. + const downInit: MouseEventInit = { + bubbles: true, + cancelable: true, + composed: true, + button: 0, + buttons: 1, + } + dispatchPointer(target, 'pointerdown', downInit) + target.dispatchEvent(new MouseEvent('mousedown', downInit)) + const focusTarget = focusableWithin(target) + if (focusTarget !== null) { + try { + focusTarget.focus({ preventScroll: true }) + } catch { + focusTarget.focus() + } + } + const upInit: MouseEventInit = { ...downInit, buttons: 0 } + dispatchPointer(target, 'pointerup', upInit) + target.dispatchEvent(new MouseEvent('mouseup', upInit)) + target.dispatchEvent(new MouseEvent('click', { ...upInit, detail: 1 })) +} + +function dispatchPointer(target: HTMLElement, type: string, mouseInit: MouseEventInit): void { + try { + if (typeof PointerEvent === 'function') { + target.dispatchEvent(new PointerEvent(type, { + ...mouseInit, + pointerId: 1, + pointerType: 'mouse', + isPrimary: true, + })) + return + } + } catch { + // Fall through to a MouseEvent stand-in. + } + // Avoid double-firing mousedown/mouseup when PointerEvent is unavailable — + // the caller already dispatches those MouseEvents. + if (type === 'pointerdown' || type === 'pointerup') return + target.dispatchEvent(new MouseEvent(type, mouseInit)) +} + +/** Prefer the addressed element, else a nested focusable control (picker input). */ +function focusableWithin(el: HTMLElement): HTMLElement | null { + if (el.tabIndex >= 0) return el + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement) { + return el + } + if (el.isContentEditable) return el + const nested = el.querySelector( + 'input:not([type="hidden"]):not([disabled]), select:not([disabled]), textarea:not([disabled]), [tabindex]:not([tabindex="-1"]), [contenteditable="true"], [contenteditable=""]', + ) + return nested ?? el } /** @@ -357,27 +679,111 @@ function typeIntoContentEditable(el: HTMLElement, text: string, replace: boolean // Fall through to the direct-write path below. } } + // Direct writes do not synthesize beforeinput; emit it for controlled editors. + dispatchBeforeInput(host, text, replace ? 'insertReplacementText' : 'insertText') if (replace) host.textContent = '' host.textContent = `${host.textContent ?? ''}${text}` - host.dispatchEvent(new Event('input', { bubbles: true })) + host.dispatchEvent(new InputEvent('input', { bubbles: true, data: text, inputType: 'insertText' })) } async function typeAction(args: Record, ctx: ActionContext): Promise { - const index = numberArg(args, 'index') - const text = typeof args.text === 'string' ? args.text : '' + // Input payload lives in `text`; addressing uses index|selector only. + const addressing = { ...args } + const text = typeof addressing.text === 'string' ? addressing.text : '' if (text === '') throw new ActionError('bad-args', 'text must not be empty.') + // When addressing by selector, `text` is the value to type — strip it from + // the mutual-exclusion check by resolving with a type-mode copy that keeps + // selector/index only. + const resolveArgs: Record = { ...addressing } + if (typeof resolveArgs.selector === 'string' && resolveArgs.selector !== '') { + delete resolveArgs.text + } const replace = args.replace === true - const el = elementOrThrow(ctx.ids, index) + const target = resolveTarget(ctx.ids, resolveArgs, 'type') + const el = target.element + const label = describeTarget(target) + if (!target.skipScroll) { + el.scrollIntoView({ block: 'center', behavior: 'instant' }) + } if (isEditable(el)) { typeIntoContentEditable(el, text, replace) } else if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { + if (el instanceof HTMLInputElement && el.type === 'file') { + throw new ActionError('action-failed', `Element ${label} is a file input; use browser_upload instead of browser_type.`) + } + ;(el as HTMLElement).focus() if (replace) setNativeValue(el, '') setNativeValue(el, `${el.value}${text}`) + } else if (el instanceof HTMLSelectElement) { + el.focus() + el.value = text + el.dispatchEvent(new Event('input', { bubbles: true })) + el.dispatchEvent(new Event('change', { bubbles: true })) } else { - throw new ActionError('action-failed', `Element [${index}] is not editable (${el.tagName.toLowerCase()}).`) + throw new ActionError('action-failed', `Element ${label} is not editable (${el.tagName.toLowerCase()}).`) + } + await waitForPageSettled(TYPE_SETTLE) + return withPageDelta(`Entered ${text.length} characters into ${label}.`, ctx) +} + +async function focusAction(args: Record, ctx: ActionContext): Promise { + const target = resolveTarget(ctx.ids, args, 'focus') + const el = target.element + if (!(el instanceof HTMLElement)) { + throw new ActionError('action-failed', `Element ${describeTarget(target)} cannot receive focus.`) } + if (!target.skipScroll) el.scrollIntoView({ block: 'center', behavior: 'instant' }) + el.focus() await waitForPageSettled(TYPE_SETTLE) - return withPageDelta(`Entered ${text.length} characters into [${index}].`, ctx) + return withPageDelta(`Focused ${describeTarget(target)}.`, ctx) +} + +async function uploadAction(args: Record, ctx: ActionContext): Promise { + const name = typeof args.name === 'string' && args.name !== '' ? args.name : 'upload.bin' + const mimeType = typeof args.mimeType === 'string' && args.mimeType !== '' ? args.mimeType : 'application/octet-stream' + const dataBase64 = typeof args.dataBase64 === 'string' ? args.dataBase64 : '' + if (dataBase64 === '') throw new ActionError('bad-args', 'dataBase64 must not be empty.') + const target = resolveTarget(ctx.ids, args, 'upload') + const el = target.element + if (!(el instanceof HTMLInputElement) || el.type !== 'file') { + throw new ActionError('action-failed', `Element ${describeTarget(target)} is not an input[type=file].`) + } + let bytes: Uint8Array + try { + const binary = atob(dataBase64) + bytes = Uint8Array.from(binary, (ch) => ch.charCodeAt(0)) + } catch { + throw new ActionError('bad-args', 'dataBase64 is not valid base64.') + } + const copy = new Uint8Array(bytes.byteLength) + copy.set(bytes) + const file = new File([copy], name, { type: mimeType }) + assignInputFiles(el, file) + el.dispatchEvent(new Event('input', { bubbles: true })) + el.dispatchEvent(new Event('change', { bubbles: true })) + await waitForPageSettled(ACTION_SETTLE) + return withPageDelta(`Uploaded "${name}" (${bytes.length} bytes) to ${describeTarget(target)}.`, ctx) +} + +/** Assign a FileList onto an input, with a jsdom-friendly fallback. */ +function assignInputFiles(input: HTMLInputElement, file: File): void { + if (typeof DataTransfer !== 'undefined') { + const transfer = new DataTransfer() + transfer.items.add(file) + try { + input.files = transfer.files + if (input.files === transfer.files) return + } catch { + // Fall through to defineProperty. + } + } + const list = { + 0: file, + length: 1, + item: (index: number) => (index === 0 ? file : null), + *[Symbol.iterator]() { yield file }, + } as unknown as FileList + Object.defineProperty(input, 'files', { configurable: true, value: list }) } async function pressAction(args: Record, ctx: ActionContext): Promise { @@ -386,7 +792,14 @@ async function pressAction(args: Record, ctx: ActionContext): P const target = document.activeElement instanceof HTMLElement ? document.activeElement : document.body target.dispatchEvent(new KeyboardEvent('keydown', { key, bubbles: true, cancelable: true })) target.dispatchEvent(new KeyboardEvent('keyup', { key, bubbles: true, cancelable: true })) - if (key === 'Enter' && target instanceof HTMLInputElement && target.form !== null) { + // Never synthesize form submit while a picker/dropdown is open — Enter is often + // used to confirm a cell, and a synthetic submit can submit a job/checkout form. + if ( + key === 'Enter' + && target instanceof HTMLInputElement + && target.form !== null + && !documentHasOpenOverlay() + ) { target.form.dispatchEvent(new Event('submit', { bubbles: true, cancelable: true })) } await waitForPageSettled(ACTION_SETTLE) diff --git a/extensions/dsh-browser/src/content/extract.ts b/extensions/dsh-browser/src/content/extract.ts index 03e5965be..920bae9cc 100644 --- a/extensions/dsh-browser/src/content/extract.ts +++ b/extensions/dsh-browser/src/content/extract.ts @@ -10,7 +10,27 @@ * @module */ -/** Every element type the model may be asked to operate on. */ +/** How an inventory element was discovered. */ +export type InteractiveSource = 'selector' | 'heuristic' | 'overlay' + +/** One element discovered for the interactive inventory. */ +export interface CollectedInteractive { + element: Element + source: InteractiveSource + /** + * Nesting depth among heuristic pointer ancestors (1 = outermost kept peer). + * Selector/overlay hits omit this unless nested heuristics apply. + */ + depth?: number +} + +/** Cap on heuristic clickables so they cannot crowd out selector-matched controls. */ +export const DEFAULT_MAX_HEURISTIC_ITEMS = 20 + +/** Cap on short-text leaves collected inside floating picker/dropdown panels. */ +export const DEFAULT_MAX_OVERLAY_ITEMS = 40 + +/** Selector-matched interactive tags/roles (CSS whitelist). */ const INTERACTIVE_SELECTOR = [ 'a[href]', 'button', @@ -23,6 +43,20 @@ const INTERACTIVE_SELECTOR = [ '[role="radio"]', '[role="tab"]', '[role="menuitem"]', + '[role="menuitemcheckbox"]', + '[role="combobox"]', + '[role="listbox"]', + '[role="listitem"]', + '[role="option"]', + '[role="grid"]', + '[role="gridcell"]', + '[role="switch"]', + '[role="spinbutton"]', + '[role="dialog"]', + '[tabindex]', + '[aria-haspopup]', + '[aria-expanded]', + '[aria-controls]', 'summary', '[contenteditable="true"]', '[contenteditable=""]', @@ -31,6 +65,12 @@ const INTERACTIVE_SELECTOR = [ /** Default cap on one item's rendered name/state text. */ const MAX_ITEM_NAME_CHARS = 80 +/** Heuristic clickable labels longer than this are treated as containers. */ +const MAX_HEURISTIC_NAME_CHARS = 24 + +/** Predicate used by heuristic discovery; injectable for jsdom fixtures. */ +export type PointerCursorPredicate = (el: Element) => boolean + /** * Whether an element is visible to the user: not display/visibility/opacity * hidden and occupying layout space. @@ -83,6 +123,63 @@ export function truncate(text: string, max: number): { text: string; truncated: return { text: `${text.slice(0, max)}…`, truncated: text.length - max } } +/** CSS.escape with a fallback for environments that lack it (jsdom). */ +function cssEscape(value: string): string { + if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') return CSS.escape(value) + return value.replace(/[^a-zA-Z0-9_-]/g, (ch) => `\\${ch}`) +} + +/** + * Root that owns tree-scoped ID references for `el` (document or shadow root). + * Attribute ID refs and HTML label/control associations do not cross this boundary. + */ +function treeRoot(el: Element): Document | ShadowRoot { + const root = el.getRootNode() + return root instanceof ShadowRoot || root instanceof Document ? root : el.ownerDocument +} + +/** Resolve one id inside `root` only — never borrow from another tree. */ +function getElementByIdInTree(root: Document | ShadowRoot, id: string): Element | null { + if (id === '') return null + if (root instanceof Document) return root.getElementById(id) + return root.querySelector(`#${cssEscape(id)}`) +} + +/** + * Resolve `aria-labelledby` IDs in reference order within the control's own tree. + * Duplicate IDs after their first occurrence are ignored. No cross-shadow fallback. + */ +function labelledByText(el: Element): string | undefined { + const labelledBy = el.getAttribute('aria-labelledby') + if (labelledBy === null) return undefined + const root = treeRoot(el) + const seen = new Set() + const parts: string[] = [] + for (const rawId of labelledBy.trim().split(/\s+/)) { + if (rawId === '' || seen.has(rawId)) continue + seen.add(rawId) + const ref = getElementByIdInTree(root, rawId) + const refText = ref?.textContent + if (refText !== undefined && refText.trim() !== '') parts.push(clean(refText)) + } + if (parts.length === 0) return undefined + return parts.join(' ') +} + +/** + * Resolve an associated `