diff --git a/src/firefox/dom.ts b/src/firefox/dom.ts index 58a868d5..334171ac 100644 --- a/src/firefox/dom.ts +++ b/src/firefox/dom.ts @@ -5,6 +5,7 @@ import { Key } from 'selenium-webdriver'; import type { BrowsingContext, Input, Script } from 'webdriver-bidi-protocol'; import type { BiDiFacade } from './bidi'; +import type { ResolvedElement } from './snapshot/resolver.js'; /** * Key names accepted by press_key. Names are matched case-insensitively. @@ -159,7 +160,7 @@ export class DomInteractions { private resolveUid: ( context: BrowsingContext.BrowsingContext, uid: string - ) => Promise = () => { + ) => Promise = () => { throw new Error('Not implemented'); } ) {} @@ -248,9 +249,9 @@ export class DomInteractions { uid: string, dblClick = false ): Promise { - const el = await this.resolveUid(context, uid); - await this.waitForVisible(context, el); - const { x, y } = await this.scrollAndGetInViewCenterPoint(context, el); + const { context: frame, element: el } = await this.resolveUid(context, uid); + await this.waitForVisible(frame, el); + const { x, y } = await this.scrollAndGetInViewCenterPoint(frame, el); const actions: Input.PointerSourceAction[] = [ { type: 'pointerMove', x, y }, { type: 'pointerDown', button: 0 }, @@ -260,17 +261,17 @@ export class DomInteractions { actions.push({ type: 'pointerDown', button: 0 }); actions.push({ type: 'pointerUp', button: 0 }); } - await this.sendPointerActions(context, actions); + await this.sendPointerActions(frame, actions); } /** * Hover over element by UID */ async hoverByUid(context: BrowsingContext.BrowsingContext, uid: string): Promise { - const el = await this.resolveUid(context, uid); - await this.waitForVisible(context, el); - const { x, y } = await this.scrollAndGetInViewCenterPoint(context, el); - await this.sendPointerActions(context, [{ type: 'pointerMove', x, y }]); + const { context: frame, element: el } = await this.resolveUid(context, uid); + await this.waitForVisible(frame, el); + const { x, y } = await this.scrollAndGetInViewCenterPoint(frame, el); + await this.sendPointerActions(frame, [{ type: 'pointerMove', x, y }]); } /** @@ -281,13 +282,13 @@ export class DomInteractions { uid: string, value: string ): Promise { - const el = await this.resolveUid(context, uid); + const { context: frame, element: el } = await this.resolveUid(context, uid); // Select the old content after focusing and delete it with a key press, // instead of assigning the value directly: a page that writes its own value // back on focus (such as a controlled React input that re-renders) would // otherwise restore it, and the typed text would be appended to it. const hasContent = await this.bidi.callFunction( - context, + frame, ` (el) => { el.focus(); @@ -306,7 +307,7 @@ export class DomInteractions { ); const actions = hasContent ? comboKeyActions({ modifiers: [], key: Key.BACK_SPACE }) : []; actions.push(...textKeyActions(value)); - await this.sendKeyActions(context, actions); + await this.sendKeyActions(frame, actions); } /** @@ -318,13 +319,13 @@ export class DomInteractions { fromUid: string, toUid: string ): Promise { - const [sourceElement, targetElement] = await Promise.all([ + const [source, target] = await Promise.all([ this.resolveUid(context, fromUid), this.resolveUid(context, toUid), ]); // Use JS drag events fallback for compatibility (Actions DnD not used) await this.bidi.callFunction( - context, + source.context, ` (src, tgt) => { function dispatch(type, target, dt) { @@ -339,7 +340,7 @@ export class DomInteractions { dispatch('dragend', src, dt); } `, - [sourceElement, targetElement] + [source.element, target.element] ); } @@ -364,9 +365,9 @@ export class DomInteractions { uid: string, filePath: string ): Promise { - const el = await this.resolveUid(context, uid); + const { context: frame, element: el } = await this.resolveUid(context, uid); const files = filePath.split('\n').filter((path) => path !== ''); - await this.bidi.sendCommand('input.setFiles', { context, element: el, files }); + await this.bidi.sendCommand('input.setFiles', { context: frame, element: el, files }); } /** @@ -418,10 +419,10 @@ export class DomInteractions { * focused. */ private async focusByUid(context: BrowsingContext.BrowsingContext, uid: string): Promise { - const el = await this.resolveUid(context, uid); - await this.waitForVisible(context, el, 5000); + const { context: frame, element: el } = await this.resolveUid(context, uid); + await this.waitForVisible(frame, el, 5000); const focused = await this.bidi.callFunction( - context, + frame, '(el) => { el.focus(); return el.getRootNode().activeElement === el; }', [el] ); @@ -476,13 +477,13 @@ export class DomInteractions { context: BrowsingContext.BrowsingContext, uid: string ): Promise { - const el = await this.resolveUid(context, uid); - await this.waitForVisible(context, el); - await this.scrollAndGetInViewCenterPoint(context, el); + const { context: frame, element: el } = await this.resolveUid(context, uid); + await this.waitForVisible(frame, el); + await this.scrollAndGetInViewCenterPoint(frame, el); // Take screenshot of element (WebDriver BiDi automatically crops to element bounds) const result = await this.bidi.sendCommand('browsingContext.captureScreenshot', { - context, + context: frame, format: { type: 'png' }, clip: { type: 'element', element: el }, }); diff --git a/src/firefox/index.ts b/src/firefox/index.ts index 5d7a1997..ecc3799f 100644 --- a/src/firefox/index.ts +++ b/src/firefox/index.ts @@ -118,7 +118,7 @@ export class FirefoxClient { this.dom = new DomInteractions( this.bidi, (context: BrowsingContext.BrowsingContext, uid: string) => - this.snapshot!.resolveUidToElement(context, uid) + this.snapshot!.resolveUidToContextElement(context, uid) ); this.pages = new PageManagement( diff --git a/src/firefox/snapshot/injected/treeWalker.ts b/src/firefox/snapshot/injected/treeWalker.ts index 12774e8b..c4ac9f78 100644 --- a/src/firefox/snapshot/injected/treeWalker.ts +++ b/src/firefox/snapshot/injected/treeWalker.ts @@ -108,6 +108,8 @@ export function walkTree( iframeResult.node.isIframe = true; iframeResult.node.frameSrc = iframe.src; childResults.push(iframeResult.node); + } else { + childResults.push(...iframeResult.relevantChildren); } } } catch { diff --git a/src/firefox/snapshot/manager.ts b/src/firefox/snapshot/manager.ts index c64f0fd4..bbe708ca 100644 --- a/src/firefox/snapshot/manager.ts +++ b/src/firefox/snapshot/manager.ts @@ -12,7 +12,7 @@ import { nativeToLocalValue } from '../../utils/local-value.js'; import { logDebug } from '../../utils/logger.js'; import type { Snapshot, SnapshotJson, InjectedScriptResult } from './types.js'; import { formatSnapshotTree } from './formatter.js'; -import { UidResolver } from './resolver.js'; +import { UidResolver, type ResolvedElement } from './resolver.js'; export const SNAPSHOT_MANAGER_SANDBOX = 'snapshot-manager'; @@ -181,6 +181,16 @@ export class SnapshotManager { return await this.resolver.resolveUidToElement(context, uid); } + /** + * Resolve UID to its element and the browsing context of the frame that holds it + */ + async resolveUidToContextElement( + context: BrowsingContext.BrowsingContext, + uid: string + ): Promise { + return await this.resolver.resolveUidToContextElement(context, uid); + } + /** * Clear snapshot UIDs */ diff --git a/src/firefox/snapshot/resolver.ts b/src/firefox/snapshot/resolver.ts index 41c383d4..127746ea 100644 --- a/src/firefox/snapshot/resolver.ts +++ b/src/firefox/snapshot/resolver.ts @@ -13,6 +13,26 @@ import { logDebug } from '../../utils/logger.js'; const RESOLVE_SCRIPT = '(el) => window.__resolveUid ? window.__resolveUid(el) : null'; const SELECTOR_SCRIPT = '(el) => window.__uidToSelector ? window.__uidToSelector(el) : null'; const CLEAR_SCRIPT = 'if (window.__clearUidRegistry) { window.__clearUidRegistry(); }'; +// Index of each frame on the way from the top window down to the element, in document order like +// the children of a browsing context tree. +const FRAME_PATH_SCRIPT = `(uid) => { + const el = window.__resolveUid ? window.__resolveUid(uid) : null; + const path = []; + for (let win = el?.ownerDocument.defaultView; win && win !== window; win = win.parent) { + let index = 0; + while (win.parent[index] !== win) { + index++; + } + path.unshift(index); + } + return path; +}`; + +export interface ResolvedElement { + /** Browsing context of the document that holds the element */ + context: BrowsingContext.BrowsingContext; + element: Script.SharedReference; +} /** * UID Resolver class @@ -74,6 +94,36 @@ export class UidResolver { logDebug(`Resolved element for UID: ${uid}`); return { sharedId: element.sharedId }; } + + /** + * Resolve UID to its element together with the browsing context that owns it. A node from a + * same-origin iframe can only be used in the context of that iframe. + */ + async resolveUidToContextElement( + context: BrowsingContext.BrowsingContext, + uid: string + ): Promise { + const element = await this.resolveUidToElement(context, uid); + const path = await this.bidi.callFunction( + context, + FRAME_PATH_SCRIPT, + [nativeToLocalValue(uid)], + SNAPSHOT_MANAGER_SANDBOX + ); + if (path.length === 0) { + return { context, element }; + } + + const { contexts } = await this.bidi.sendCommand('browsingContext.getTree', { root: context }); + let info: BrowsingContext.Info | undefined = contexts[0]; + for (const index of path) { + info = info?.children?.[index]; + } + if (!info) { + throw new Error(notFoundMessage(uid)); + } + return { context: info.context, element }; + } } function notFoundMessage(uid: string): string { diff --git a/tests/integration/firefox/frames.test.ts b/tests/integration/firefox/frames.test.ts new file mode 100644 index 00000000..b80a4b25 --- /dev/null +++ b/tests/integration/firefox/frames.test.ts @@ -0,0 +1,81 @@ +/** + * Integration tests for UID input inside same-origin iframes + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createServer, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { createTestFirefox, closeFirefox, findNodeInSnapshot } from '../../helpers/firefox.js'; +import type { FirefoxClient } from '@/firefox/index.js'; + +const PAGES: Record = { + '/': '

Top

', + '/frame': + '' + + '', +}; + +describe('UID input in iframes', () => { + let firefox: FirefoxClient; + let server: Server; + let origin: string; + + beforeAll(async () => { + server = createServer((req, res) => { + const body = PAGES[req.url ?? '']; + res.writeHead(body === undefined ? 404 : 200, { 'Content-Type': 'text/html' }).end(body); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + + firefox = await createTestFirefox(); + }); + + afterAll(async () => { + await closeFirefox(firefox); + await new Promise((resolve) => server.close(resolve)); + }); + + async function openPage(): Promise { + await firefox.navigate(`${origin}/`); + await firefox.evaluate( + 'new Promise((r) => { const f = document.getElementById("frame"); f.contentDocument?.getElementById("inner") ? r() : f.addEventListener("load", r); })' + ); + } + + async function uidOf(id: string): Promise { + const snapshot = await firefox.takeSnapshot(); + return findNodeInSnapshot(snapshot.json.root, (node) => node.id === id)!.uid; + } + + function frameValue(expression: string): Promise { + return firefox.evaluate(`document.getElementById("frame").contentDocument.${expression}`); + } + + it('should include the elements of the iframe in a default snapshot', async () => { + await openPage(); + + const snapshot = await firefox.takeSnapshot(); + + expect(findNodeInSnapshot(snapshot.json.root, (node) => node.id === 'innerBtn')?.name).toBe( + 'Go' + ); + }, 20000); + + it('should fill and click elements inside the iframe by UID', async () => { + await openPage(); + + await firefox.fillByUid(await uidOf('inner'), 'abc'); + await firefox.clickByUid(await uidOf('innerBtn')); + + expect(await frameValue('getElementById("out").textContent')).toBe('abc'); + }, 20000); + + it('should focus an element inside the iframe for press_key', async () => { + await openPage(); + + await firefox.pressKey('x', await uidOf('inner')); + + expect(await frameValue('getElementById("inner").value')).toBe('x'); + }, 20000); +}); diff --git a/tests/unit/firefox/press-key.test.ts b/tests/unit/firefox/press-key.test.ts index 9c6ad633..0dc36688 100644 --- a/tests/unit/firefox/press-key.test.ts +++ b/tests/unit/firefox/press-key.test.ts @@ -75,7 +75,7 @@ describe('DomInteractions.pressKey', () => { it('should focus a uid before pressing the key', async () => { const { bidi, sendCommand, callFunction } = createBidi(); const sharedRef: Script.SharedReference = { sharedId: 'shared-1' }; - const resolveUid = vi.fn().mockResolvedValue(sharedRef); + const resolveUid = vi.fn().mockResolvedValue({ context: CONTEXT, element: sharedRef }); const dom = new DomInteractions(bidi, resolveUid); await dom.pressKey(CONTEXT, 'Enter', 'uid-1'); @@ -92,7 +92,9 @@ describe('DomInteractions.pressKey', () => { it('should reject a uid that cannot take focus instead of pressing elsewhere', async () => { const { bidi, sendCommand } = createBidi(false); - const resolveUid = vi.fn().mockResolvedValue({ sharedId: 'shared-2' }); + const resolveUid = vi + .fn() + .mockResolvedValue({ context: CONTEXT, element: { sharedId: 'shared-2' } }); const dom = new DomInteractions(bidi, resolveUid); await expect(dom.pressKey(CONTEXT, 'Escape', 'uid-2')).rejects.toThrow( diff --git a/tests/unit/firefox/type-text.test.ts b/tests/unit/firefox/type-text.test.ts index bd124f59..7be139b6 100644 --- a/tests/unit/firefox/type-text.test.ts +++ b/tests/unit/firefox/type-text.test.ts @@ -84,7 +84,7 @@ describe('DomInteractions.typeText', () => { it('should focus a uid first and still type through the actions keyboard', async () => { const { bidi, sendCommand, callFunction } = createBidi(); const sharedRef: Script.SharedReference = { sharedId: 'shared-1' }; - const resolveUid = vi.fn().mockResolvedValue(sharedRef); + const resolveUid = vi.fn().mockResolvedValue({ context: CONTEXT, element: sharedRef }); const dom = new DomInteractions(bidi, resolveUid); await dom.typeText(CONTEXT, 'hello', { uid: 'uid-1' }); @@ -98,7 +98,9 @@ describe('DomInteractions.typeText', () => { it('should reject a uid that cannot take focus before typing anything', async () => { const { bidi, sendCommand } = createBidi(false); - const resolveUid = vi.fn().mockResolvedValue({ sharedId: 'shared-2' }); + const resolveUid = vi + .fn() + .mockResolvedValue({ context: CONTEXT, element: { sharedId: 'shared-2' } }); const dom = new DomInteractions(bidi, resolveUid); await expect(dom.typeText(CONTEXT, 'hello', { uid: 'uid-2' })).rejects.toThrow(