Bug 2089770 - Act on UIDs inside same-origin iframes

src/firefox/dom.ts

@@ -5,6 +5,7 @@
 import { Key } from 'selenium-webdriver';
 import type { BrowsingContext, Input, Script } from 'webdriver-bidi-protocol';
 import type { BiDiFacade } from './bidi';
+import type { ResolvedElement } from './snapshot/resolver.js';
 
 /**
  * Key names accepted by press_key. Names are matched case-insensitively.
@@ -159,7 +160,7 @@ export class DomInteractions {
     private resolveUid: (
       context: BrowsingContext.BrowsingContext,
       uid: string
-    ) => Promise<Script.SharedReference> = () => {
+    ) => Promise<ResolvedElement> = () => {
       throw new Error('Not implemented');
     }
   ) {}
@@ -248,9 +249,9 @@ export class DomInteractions {
     uid: string,
     dblClick = false
   ): Promise<void> {
-    const el = await this.resolveUid(context, uid);
-    await this.waitForVisible(context, el);
-    const { x, y } = await this.scrollAndGetInViewCenterPoint(context, el);
+    const { context: frame, element: el } = await this.resolveUid(context, uid);
+    await this.waitForVisible(frame, el);
+    const { x, y } = await this.scrollAndGetInViewCenterPoint(frame, el);
     const actions: Input.PointerSourceAction[] = [
       { type: 'pointerMove', x, y },
       { type: 'pointerDown', button: 0 },
@@ -260,17 +261,17 @@ export class DomInteractions {
       actions.push({ type: 'pointerDown', button: 0 });
       actions.push({ type: 'pointerUp', button: 0 });
     }
-    await this.sendPointerActions(context, actions);
+    await this.sendPointerActions(frame, actions);
   }
 
   /**
    * Hover over element by UID
    */
   async hoverByUid(context: BrowsingContext.BrowsingContext, uid: string): Promise<void> {
-    const el = await this.resolveUid(context, uid);
-    await this.waitForVisible(context, el);
-    const { x, y } = await this.scrollAndGetInViewCenterPoint(context, el);
-    await this.sendPointerActions(context, [{ type: 'pointerMove', x, y }]);
+    const { context: frame, element: el } = await this.resolveUid(context, uid);
+    await this.waitForVisible(frame, el);
+    const { x, y } = await this.scrollAndGetInViewCenterPoint(frame, el);
+    await this.sendPointerActions(frame, [{ type: 'pointerMove', x, y }]);
   }
 
   /**
@@ -281,13 +282,13 @@ export class DomInteractions {
     uid: string,
     value: string
   ): Promise<void> {
-    const el = await this.resolveUid(context, uid);
+    const { context: frame, element: el } = await this.resolveUid(context, uid);
     // Select the old content after focusing and delete it with a key press,
     // instead of assigning the value directly: a page that writes its own value
     // back on focus (such as a controlled React input that re-renders) would
     // otherwise restore it, and the typed text would be appended to it.
     const hasContent = await this.bidi.callFunction<boolean>(
-      context,
+      frame,
       `
       (el) => {
         el.focus();
@@ -306,7 +307,7 @@ export class DomInteractions {
     );
     const actions = hasContent ? comboKeyActions({ modifiers: [], key: Key.BACK_SPACE }) : [];
     actions.push(...textKeyActions(value));
-    await this.sendKeyActions(context, actions);
+    await this.sendKeyActions(frame, actions);
   }
 
   /**
@@ -318,13 +319,13 @@ export class DomInteractions {
     fromUid: string,
     toUid: string
   ): Promise<void> {
-    const [sourceElement, targetElement] = await Promise.all([
+    const [source, target] = await Promise.all([
       this.resolveUid(context, fromUid),
       this.resolveUid(context, toUid),
     ]);
     // Use JS drag events fallback for compatibility (Actions DnD not used)
     await this.bidi.callFunction(
-      context,
+      source.context,
       `
       (src, tgt) => {
         function dispatch(type, target, dt) {
@@ -339,7 +340,7 @@ export class DomInteractions {
         dispatch('dragend', src, dt);
       }
       `,
-      [sourceElement, targetElement]
+      [source.element, target.element]
     );
   }
 
@@ -364,9 +365,9 @@ export class DomInteractions {
     uid: string,
     filePath: string
   ): Promise<void> {
-    const el = await this.resolveUid(context, uid);
+    const { context: frame, element: el } = await this.resolveUid(context, uid);
     const files = filePath.split('\n').filter((path) => path !== '');
-    await this.bidi.sendCommand('input.setFiles', { context, element: el, files });
+    await this.bidi.sendCommand('input.setFiles', { context: frame, element: el, files });
   }
 
   /**
@@ -418,10 +419,10 @@ export class DomInteractions {
    * focused.
    */
   private async focusByUid(context: BrowsingContext.BrowsingContext, uid: string): Promise<void> {
-    const el = await this.resolveUid(context, uid);
-    await this.waitForVisible(context, el, 5000);
+    const { context: frame, element: el } = await this.resolveUid(context, uid);
+    await this.waitForVisible(frame, el, 5000);
     const focused = await this.bidi.callFunction(
-      context,
+      frame,
       '(el) => { el.focus(); return el.getRootNode().activeElement === el; }',
       [el]
     );
@@ -476,13 +477,13 @@ export class DomInteractions {
     context: BrowsingContext.BrowsingContext,
     uid: string
   ): Promise<string> {
-    const el = await this.resolveUid(context, uid);
-    await this.waitForVisible(context, el);
-    await this.scrollAndGetInViewCenterPoint(context, el);
+    const { context: frame, element: el } = await this.resolveUid(context, uid);
+    await this.waitForVisible(frame, el);
+    await this.scrollAndGetInViewCenterPoint(frame, el);
 
     // Take screenshot of element (WebDriver BiDi automatically crops to element bounds)
     const result = await this.bidi.sendCommand('browsingContext.captureScreenshot', {
-      context,
+      context: frame,
       format: { type: 'png' },
       clip: { type: 'element', element: el },
     });

src/firefox/index.ts

@@ -118,7 +118,7 @@ export class FirefoxClient {
     this.dom = new DomInteractions(
       this.bidi,
       (context: BrowsingContext.BrowsingContext, uid: string) =>
-        this.snapshot!.resolveUidToElement(context, uid)
+        this.snapshot!.resolveUidToContextElement(context, uid)
     );
 
     this.pages = new PageManagement(

src/firefox/snapshot/injected/treeWalker.ts

@@ -108,6 +108,8 @@ export function walkTree(
             iframeResult.node.isIframe = true;
             iframeResult.node.frameSrc = iframe.src;
             childResults.push(iframeResult.node);
+          } else {
+            childResults.push(...iframeResult.relevantChildren);
           }
         }
       } catch {

src/firefox/snapshot/manager.ts

@@ -12,7 +12,7 @@ import { nativeToLocalValue } from '../../utils/local-value.js';
 import { logDebug } from '../../utils/logger.js';
 import type { Snapshot, SnapshotJson, InjectedScriptResult } from './types.js';
 import { formatSnapshotTree } from './formatter.js';
-import { UidResolver } from './resolver.js';
+import { UidResolver, type ResolvedElement } from './resolver.js';
 
 export const SNAPSHOT_MANAGER_SANDBOX = 'snapshot-manager';
 
@@ -181,6 +181,16 @@ export class SnapshotManager {
     return await this.resolver.resolveUidToElement(context, uid);
   }
 
+  /**
+   * Resolve UID to its element and the browsing context of the frame that holds it
+   */
+  async resolveUidToContextElement(
+    context: BrowsingContext.BrowsingContext,
+    uid: string
+  ): Promise<ResolvedElement> {
+    return await this.resolver.resolveUidToContextElement(context, uid);
+  }
+
   /**
    * Clear snapshot UIDs
    */

src/firefox/snapshot/resolver.ts

@@ -13,6 +13,26 @@ import { logDebug } from '../../utils/logger.js';
 const RESOLVE_SCRIPT = '(el) => window.__resolveUid ? window.__resolveUid(el) : null';
 const SELECTOR_SCRIPT = '(el) => window.__uidToSelector ? window.__uidToSelector(el) : null';
 const CLEAR_SCRIPT = 'if (window.__clearUidRegistry) { window.__clearUidRegistry(); }';
+// Index of each frame on the way from the top window down to the element, in document order like
+// the children of a browsing context tree.
+const FRAME_PATH_SCRIPT = `(uid) => {
+  const el = window.__resolveUid ? window.__resolveUid(uid) : null;
+  const path = [];
+  for (let win = el?.ownerDocument.defaultView; win && win !== window; win = win.parent) {
+    let index = 0;
+    while (win.parent[index] !== win) {
+      index++;
+    }
+    path.unshift(index);
+  }
+  return path;
+}`;
+
+export interface ResolvedElement {
+  /** Browsing context of the document that holds the element */
+  context: BrowsingContext.BrowsingContext;
+  element: Script.SharedReference;
+}
 
 /**
  * UID Resolver class
@@ -74,6 +94,36 @@ export class UidResolver {
     logDebug(`Resolved element for UID: ${uid}`);
     return { sharedId: element.sharedId };
   }
+
+  /**
+   * Resolve UID to its element together with the browsing context that owns it. A node from a
+   * same-origin iframe can only be used in the context of that iframe.
+   */
+  async resolveUidToContextElement(
+    context: BrowsingContext.BrowsingContext,
+    uid: string
+  ): Promise<ResolvedElement> {
+    const element = await this.resolveUidToElement(context, uid);
+    const path = await this.bidi.callFunction<number[]>(
+      context,
+      FRAME_PATH_SCRIPT,
+      [nativeToLocalValue(uid)],
+      SNAPSHOT_MANAGER_SANDBOX
+    );
+    if (path.length === 0) {
+      return { context, element };
+    }
+
+    const { contexts } = await this.bidi.sendCommand('browsingContext.getTree', { root: context });
+    let info: BrowsingContext.Info | undefined = contexts[0];
+    for (const index of path) {
+      info = info?.children?.[index];
+    }
+    if (!info) {
+      throw new Error(notFoundMessage(uid));
+    }
+    return { context: info.context, element };
+  }
 }
 
 function notFoundMessage(uid: string): string {

tests/integration/firefox/frames.test.ts

@@ -0,0 +1,81 @@
+/**
+ * Integration tests for UID input inside same-origin iframes
+ */
+
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { createServer, type Server } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { createTestFirefox, closeFirefox, findNodeInSnapshot } from '../../helpers/firefox.js';
+import type { FirefoxClient } from '@/firefox/index.js';
+
+const PAGES: Record<string, string> = {
+  '/': '<h1>Top</h1><iframe id=frame src=/frame style="margin-top:200px"></iframe>',
+  '/frame':
+    '<input id=inner><button id=innerBtn onclick="out.textContent = inner.value">Go</button>' +
+    '<output id=out></output>',
+};
+
+describe('UID input in iframes', () => {
+  let firefox: FirefoxClient;
+  let server: Server;
+  let origin: string;
+
+  beforeAll(async () => {
+    server = createServer((req, res) => {
+      const body = PAGES[req.url ?? ''];
+      res.writeHead(body === undefined ? 404 : 200, { 'Content-Type': 'text/html' }).end(body);
+    });
+    await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+    origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+
+    firefox = await createTestFirefox();
+  });
+
+  afterAll(async () => {
+    await closeFirefox(firefox);
+    await new Promise((resolve) => server.close(resolve));
+  });
+
+  async function openPage(): Promise<void> {
+    await firefox.navigate(`${origin}/`);
+    await firefox.evaluate(
+      'new Promise((r) => { const f = document.getElementById("frame"); f.contentDocument?.getElementById("inner") ? r() : f.addEventListener("load", r); })'
+    );
+  }
+
+  async function uidOf(id: string): Promise<string> {
+    const snapshot = await firefox.takeSnapshot();
+    return findNodeInSnapshot(snapshot.json.root, (node) => node.id === id)!.uid;
+  }
+
+  function frameValue(expression: string): Promise<unknown> {
+    return firefox.evaluate(`document.getElementById("frame").contentDocument.${expression}`);
+  }
+
+  it('should include the elements of the iframe in a default snapshot', async () => {
+    await openPage();
+
+    const snapshot = await firefox.takeSnapshot();
+
+    expect(findNodeInSnapshot(snapshot.json.root, (node) => node.id === 'innerBtn')?.name).toBe(
+      'Go'
+    );
+  }, 20000);
+
+  it('should fill and click elements inside the iframe by UID', async () => {
+    await openPage();
+
+    await firefox.fillByUid(await uidOf('inner'), 'abc');
+    await firefox.clickByUid(await uidOf('innerBtn'));
+
+    expect(await frameValue('getElementById("out").textContent')).toBe('abc');
+  }, 20000);
+
+  it('should focus an element inside the iframe for press_key', async () => {
+    await openPage();
+
+    await firefox.pressKey('x', await uidOf('inner'));
+
+    expect(await frameValue('getElementById("inner").value')).toBe('x');
+  }, 20000);
+});

tests/unit/firefox/press-key.test.ts

@@ -75,7 +75,7 @@ describe('DomInteractions.pressKey', () => {
   it('should focus a uid before pressing the key', async () => {
     const { bidi, sendCommand, callFunction } = createBidi();
     const sharedRef: Script.SharedReference = { sharedId: 'shared-1' };
-    const resolveUid = vi.fn().mockResolvedValue(sharedRef);
+    const resolveUid = vi.fn().mockResolvedValue({ context: CONTEXT, element: sharedRef });
     const dom = new DomInteractions(bidi, resolveUid);
 
     await dom.pressKey(CONTEXT, 'Enter', 'uid-1');
@@ -92,7 +92,9 @@ describe('DomInteractions.pressKey', () => {
 
   it('should reject a uid that cannot take focus instead of pressing elsewhere', async () => {
     const { bidi, sendCommand } = createBidi(false);
-    const resolveUid = vi.fn().mockResolvedValue({ sharedId: 'shared-2' });
+    const resolveUid = vi
+      .fn()
+      .mockResolvedValue({ context: CONTEXT, element: { sharedId: 'shared-2' } });
     const dom = new DomInteractions(bidi, resolveUid);
 
     await expect(dom.pressKey(CONTEXT, 'Escape', 'uid-2')).rejects.toThrow(

tests/unit/firefox/type-text.test.ts

@@ -84,7 +84,7 @@ describe('DomInteractions.typeText', () => {
   it('should focus a uid first and still type through the actions keyboard', async () => {
     const { bidi, sendCommand, callFunction } = createBidi();
     const sharedRef: Script.SharedReference = { sharedId: 'shared-1' };
-    const resolveUid = vi.fn().mockResolvedValue(sharedRef);
+    const resolveUid = vi.fn().mockResolvedValue({ context: CONTEXT, element: sharedRef });
     const dom = new DomInteractions(bidi, resolveUid);
 
     await dom.typeText(CONTEXT, 'hello', { uid: 'uid-1' });
@@ -98,7 +98,9 @@ describe('DomInteractions.typeText', () => {
 
   it('should reject a uid that cannot take focus before typing anything', async () => {
     const { bidi, sendCommand } = createBidi(false);
-    const resolveUid = vi.fn().mockResolvedValue({ sharedId: 'shared-2' });
+    const resolveUid = vi
+      .fn()
+      .mockResolvedValue({ context: CONTEXT, element: { sharedId: 'shared-2' } });
     const dom = new DomInteractions(bidi, resolveUid);
 
     await expect(dom.typeText(CONTEXT, 'hello', { uid: 'uid-2' })).rejects.toThrow(