From c064adc28185ff1344506234d559294379a09999 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 18:37:02 -0700 Subject: [PATCH 1/4] chore: claim browser media engine quest Co-Authored-By: GPT-6 From bcf47a71ea4de8ac420c4623bff887391e4ebc6c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 19:00:06 -0700 Subject: [PATCH 2/4] test(browser): run media QA with alternate player engines Co-Authored-By: GPT-6 --- .github/workflows/nightly.yml | 35 +++++ quest/m2/browser-media-qa-engines.md | 50 ++++-- test/interop/README.md | 22 ++- test/interop/clients/js/harness.test.ts | 29 ++++ test/interop/clients/js/harness.ts | 69 ++++++-- test/interop/clients/js/media.ts | 148 +++++++++--------- .../clients/js/src/capabilities.test.ts | 22 +++ test/interop/clients/js/src/capabilities.ts | 64 ++++++++ test/interop/clients/js/src/contract.ts | 7 + test/interop/clients/js/src/probe.ts | 14 +- test/interop/interop.sh | 31 +++- test/justfile | 2 +- 12 files changed, 379 insertions(+), 114 deletions(-) create mode 100644 test/interop/clients/js/harness.test.ts create mode 100644 test/interop/clients/js/src/capabilities.test.ts create mode 100644 test/interop/clients/js/src/capabilities.ts diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3b6b5606dcd..d5f0384c9d8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -177,6 +177,41 @@ jobs: NEXTEST_PROFILE: ci MOQ_STRICT: 1 + media-engines: + name: Media (${{ matrix.engine }} player) + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + engine: [chromium, firefox, webkit] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + - name: Rust cache + uses: ./.github/actions/rust-cache + - name: Install publisher and player engines + run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium ${{ matrix.engine }}" + - name: Media and negative controls + run: nix develop --command just test media --media-engines ${{ matrix.engine }} + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + - name: Preserve measurements and traces + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: media-${{ matrix.engine }} + path: ${{ runner.temp }}/moq-test-runs + if-no-files-found: error + # Full platform coverage without rebuilding after every main queue merge. platform: uses: ./.github/workflows/platform.yml diff --git a/quest/m2/browser-media-qa-engines.md b/quest/m2/browser-media-qa-engines.md index 135579bb715..1777486fbb7 100644 --- a/quest/m2/browser-media-qa-engines.md +++ b/quest/m2/browser-media-qa-engines.md @@ -11,17 +11,17 @@ measured and which APIs each one lacked. `just test media` measures presented frame progress, tone presence, and audio/video skew against a deterministic fixture, but `harness.ts` -launches only Playwright's Chromium and nothing installs another engine. Its -capability probe already enumerates what the player needs (WebTransport, -WebCodecs encode and decode, AudioWorklet, `MediaStreamTrackProcessor`, -`canvas.captureStream`) and fails naming what is missing. - -Neither Playwright Firefox nor Playwright WebKit ships WebTransport, and -WebKit also lacks the WebCodecs encoder, so neither can publish and neither -can play over QUIC. What they can do is play over the WebSocket fallback. -Real Firefox does use WebTransport; only Playwright's build lacks it, so a -fallback run covers the player, not Firefox's transport path (see -[#4460](https://github.com/moq-dev/moq/pull/4460), merged 2026-10-01). +supports pinned Chromium, Firefox, and WebKit players with a Chromium fixture. +Its capability probe enumerates WebTransport, WebSocket, WebCodecs encode and +decode, AudioWorklet, `MediaStreamTrackProcessor`, and `canvas.captureStream`, +then names what is missing separately for publisher and player roles. + +Playwright capabilities change with the pinned engine. A native probe of +Playwright 1.63 Firefox 155 found WebTransport and WebCodecs, but no +`MediaStreamTrackProcessor`, so it can play but cannot publish this fixture. +The alternate players explicitly use WebSocket to isolate player behavior; +this does not prove their QUIC transport. Probe publisher and player roles +separately instead of assuming API support from the engine name. - Split the harness's publisher and subscriber roles so an engine can take one side: a Chromium fixture with a Firefox or WebKit player over the @@ -36,3 +36,31 @@ fallback run covers the player, not Firefox's transport path (see - Real Firefox 155 still needs a manual run too: it reports a non-empty `WebTransport.protocol` and lands on lite-06 against the in-tree relay, while 153/154 take the draft-14 SETUP path. + +## Validation evidence (2026-10-15) + +The engine role split, shared graders, explicit WebSocket alternate players, +read-only pre-gesture observations, and nightly Chromium/Firefox/WebKit matrix +are implemented. This quest remains open until a complete alternate-engine +media run passes. + +Firefox 155.0 renders the fixture over WebSocket and passes the pre-gesture +gate, but its audio context remains suspended after the real click with +`userActivated: true`; `gesture starts audio` correctly fails. A standalone +AudioContext also remained suspended across pointerdown, pointerup, and click +resume attempts. The production cause is not established. Negative controls +that fail at this earlier gate are failures, not sensitivity proof. + +WebKit 2359 is installed locally but cannot launch because host runtime +libraries are absent. This is a runtime dependency failure, not evidence about +WebKit APIs. Nightly installs dependencies with Playwright `--with-deps`. +No tolerance or autoplay gate has been relaxed. + +The default Chromium path still enforces its original gate and tolerances. Its +full run fails cold-start tone presence at 65/80 samples against the unchanged +90% floor, matching the previously recorded class of cold-start refusal in +#5635. This does not establish a common cause with the Firefox startup failure. +The Chromium silent-audio, offset-audio, leaked-session, and lagging-latecomer +negative controls pass. Its frozen-video control instead fails publisher +readiness with a suspended fixture context, so that row is not sensitivity +proof in this run. No repeat was used to turn either failure into a pass. diff --git a/test/interop/README.md b/test/interop/README.md index 3a2fe8cbcde..58ce2baaaf9 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -231,6 +231,19 @@ refresh each reason before committing. The matrix asks "did bytes arrive and did a pixel light up". That passes on a frozen picture, on silence, and on audio a second out of step, so `just test media` measures the media itself, browser to browser. +Use `just test media --media-engines chromium,firefox,webkit` to run each pinned +Playwright player against a Chromium publisher. The harness installs the requested +engines and runs the same lifecycle cases, content tolerances, and five negative +controls per engine. A missing runtime or required API fails that engine by name. +Firefox and WebKit players explicitly use WebSocket; Chromium keeps normal +transport selection. This does not measure Firefox's QUIC path. + +The capability report names the engine version and separately lists missing +publisher and player APIs. Firefox 155 in Playwright 1.63 exposes WebTransport and +WebCodecs but lacks `MediaStreamTrackProcessor`, so it can play this fixture but +cannot publish it. Capabilities are measured on every run rather than assumed +from an engine name. Playwright WebKit is not Safari; Safari device/transport +bugs and real Firefox transport negotiation still require manual evidence. The publisher is a fixture, not a fake camera: a canvas painting a frame counter as black/white blocks, and a tone stepping through a fixed frequency table. Both @@ -242,7 +255,7 @@ samples, not that a machine played them. Each run covers, against a real local relay: -- **capabilities** - probes every platform API the player needs, and fails +- **capabilities** - probes native APIs separately for the publisher and player, and fails naming what is missing rather than skipping a case. - **cold start** - the publisher reports when it is announced and encoding, then a fresh page joins. No reload, unlike the matrix driver: a subscriber that @@ -251,8 +264,11 @@ Each run covers, against a real local relay: `--autoplay-policy=document-user-activation-required`, which applies to top-level Web Audio contexts. The fixture and player graphs must be suspended before either page is clicked, then both must carry audio afterwards. Harness - state probes use CDP with `userGesture: false`: Playwright's usual page reads - themselves grant activation and would invalidate this assertion. + state probes use CDP with `userGesture: false`. Firefox retains its default + Web Audio autoplay block. Firefox/WebKit observations consume the page's + existing sample events without executing scripts in the measured page: + Playwright's usual reads grant activation and would invalidate this assertion. + Capability probes run in a separate context before the measured page opens. - **capture permission** - the camera case uses a fake device for deterministic input, while Playwright denies and then grants permissions. Both source errors must be visible; neither a full nor a microphone-only denial may announce a diff --git a/test/interop/clients/js/harness.test.ts b/test/interop/clients/js/harness.test.ts new file mode 100644 index 00000000000..6250a64c638 --- /dev/null +++ b/test/interop/clients/js/harness.test.ts @@ -0,0 +1,29 @@ +import { expect, test } from "bun:test"; +import type { Browser, Page } from "playwright"; +import { open, readPlayerState, waitForWatch } from "./harness"; +import { SAMPLE_EVENT } from "./src/contract"; + +test("Firefox player observations never evaluate in the page before its gesture", async () => { + let consoleListener: (message: { text: () => string }) => void = () => {}; + const sample = { seq: 1, userActivated: false, centerPlayVisible: true, controlLabel: "Play" }; + const page = { + on(event: string, listener: typeof consoleListener) { + if (event === "console") consoleListener = listener; + }, + context: () => ({ browser: () => ({ browserType: () => ({ name: () => "firefox" }) }) }), + evaluate() { + throw new Error("evaluation would grant user activation"); + }, + goto() { + consoleListener({ text: () => SAMPLE_EVENT + JSON.stringify(sample) }); + }, + } as unknown as Page; + const browser = { newPage: () => page } as unknown as Browser; + await open(browser, "http://localhost"); + await waitForWatch(page); + const observed = await readPlayerState(page); + expect(observed.seq).toBe(1); + expect(observed.userActivated).toBe(false); + expect(observed.centerPlayVisible).toBe(true); + expect(observed.controlLabel).toBe("Play"); +}); diff --git a/test/interop/clients/js/harness.ts b/test/interop/clients/js/harness.ts index 23c02268950..b98c8566b18 100644 --- a/test/interop/clients/js/harness.ts +++ b/test/interop/clients/js/harness.ts @@ -1,5 +1,5 @@ /** - * Shared Playwright plumbing for the browser clients: serve the built page, launch Chromium, and + * Shared Playwright plumbing for the browser clients: serve the built page, launch an engine, and * read back the state each role mirrors onto the DOM. * * Used by `driver.ts` (the interop matrix) and `media.ts` (the media-output and lifecycle checks). @@ -14,9 +14,18 @@ import { type BrowserContextOptions, type CDPSession, chromium, + firefox, type Page, + webkit, } from "playwright"; -import { CONTROL, type FixtureState, type InteropControl, type Resources, type Sample } from "./src/contract"; +import { + CONTROL, + type FixtureState, + type InteropControl, + type Resources, + SAMPLE_EVENT, + type Sample, +} from "./src/contract"; /** * A failed check, named after the property it was measuring. @@ -103,19 +112,29 @@ export function pageUrl(origin: string, role: string, params: Record { +/** Browser engines the media harness can launch. */ +export const ENGINES = ["chromium", "firefox", "webkit"] as const; +/** One of the pinned Playwright engines. */ +export type Engine = (typeof ENGINES)[number]; + +/** Launch an engine with its normal gesture gate and caller-supplied Chromium arguments. */ +export function launch(args: string[] = [], engine: Engine = "chromium"): Promise { // Playwright closes the browser on SIGTERM by default, which ends its sessions without a // close. The drivers close it themselves; the harness reaps whatever a signal leaves behind. - return chromium.launch({ channel: "chromium", headless: true, args, handleSIGTERM: false }); + // Full Chromium includes WebTransport and WebCodecs; the headless shell does not. + if (engine === "chromium") + return chromium.launch({ channel: "chromium", headless: true, args, handleSIGTERM: false }); + if (engine === "firefox") + return firefox.launch({ + headless: true, + handleSIGTERM: false, + firefoxUserPrefs: { "media.autoplay.default": 1, "media.autoplay.block-webaudio": true }, + }); + return webkit.launch({ headless: true, handleSIGTERM: false }); } +const samples = new WeakMap(); + /** Contexts tracing this process, saved by {@link finishTraces} when the run fails. */ const traces: Array<{ context: BrowserContext; name: string }> = []; @@ -162,6 +181,10 @@ export async function open( const page = await browser.newPage(options); const errors: BrowserErrors = { page: [], console: [] }; page.on("console", (message) => { + if (message.text().startsWith(SAMPLE_EVENT)) { + samples.set(page, JSON.parse(message.text().slice(SAMPLE_EVENT.length)) as Sample); + return; + } console.error(`[${label}] ${message.text()}`); if (message.type() === "error") errors.console.push(message.text()); }); @@ -183,12 +206,16 @@ export function throwPageErrors(errors: BrowserErrors): void { if (messages.length > 0) throw new Error(messages.join("\n")); } -// Playwright's page.evaluate and locator reads grant user activation in Chromium. A state probe +// Playwright's page.evaluate and locator reads grant user activation. A state probe // before the deliberate click would therefore unlock Web Audio and invalidate the gesture test. const sessions = new WeakMap(); /** Inspect page state without granting user activation to the document. */ export async function inspect(page: Page, fn: (arg: A) => T, arg: A): Promise> { + // Non-Chromium reads here follow an explicit gesture. Pre-gesture player reads + // use the page's read-only sample events instead of evaluation. + if (page.context().browser()?.browserType().name() !== "chromium") + return (await page.evaluate(`(${fn.toString()})(${JSON.stringify(arg) ?? "undefined"})`)) as Awaited; let session = sessions.get(page); if (!session) { session = await page.context().newCDPSession(page); @@ -208,11 +235,14 @@ export async function inspect(page: Page, fn: (arg: A) => T, arg: A): Prom export async function waitForWatch(page: Page): Promise { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { - const ready = await inspect( - page, - (tag) => document.querySelector(`${tag}[data-interop-ready]`) !== null, - SELECTORS.watch, - ); + const ready = + page.context().browser()?.browserType().name() !== "chromium" + ? samples.has(page) + : await inspect( + page, + (tag) => document.querySelector(`${tag}[data-interop-ready]`) !== null, + SELECTORS.watch, + ); if (ready) return; await sleep(POLL_INTERVAL_MS); } @@ -221,6 +251,11 @@ export async function waitForWatch(page: Page): Promise { /** Read one sample plus the player chrome. Throws until the page has sampled at least once. */ export async function readPlayerState(page: Page): Promise { + if (page.context().browser()?.browserType().name() !== "chromium") { + const sample = samples.get(page); + if (!sample) throw new Error("the player has not published a sample"); + return { ...sample, centerPlayVisible: sample.centerPlayVisible ?? false }; + } const state = await inspect( page, (selectors) => { diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index e527cec2609..7a6003d7c7e 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -23,6 +23,8 @@ import { type BrowserErrors, check, command, + ENGINES, + type Engine, Failure, finishTraces, inspect, @@ -45,6 +47,7 @@ import { waitForState, waitForWatch, } from "./harness"; +import { type Capabilities, missing, probe, type Role } from "./src/capabilities"; import { type CaptureState, FAULTS, leakedPlayerStarted, reachesLive, SAMPLE_MS, SAMPLE_RATE } from "./src/contract"; import * as Pattern from "./src/pattern"; import { resumeSkew } from "./src/resume"; @@ -56,6 +59,7 @@ type Case = (typeof CASES)[number]; const { values } = parseArgs({ options: { url: { type: "string" }, + "player-engine": { type: "string", default: "chromium" }, timeout: { type: "string", default: "30" }, fault: { type: "string", default: "none" }, cases: { type: "string" }, @@ -69,16 +73,25 @@ const url = values.url; const timeoutMs = Number.parseFloat(values.timeout ?? "30") * 1000; const fault = values.fault ?? "none"; const expectFail = values["expect-fail"]; +const requestedEngine = values["player-engine"] ?? "chromium"; const selected = new Set( values.cases === undefined ? CASES : values.cases === "none" ? [] : values.cases.split(","), ); const unknown = [...selected].filter((name) => !CASES.some((c) => c === name)); -if (!url || !Number.isFinite(timeoutMs) || timeoutMs <= 0 || !FAULTS.some((f) => f === fault) || unknown.length > 0) { +if ( + !url || + !Number.isFinite(timeoutMs) || + timeoutMs <= 0 || + !FAULTS.some((f) => f === fault) || + unknown.length > 0 || + !ENGINES.some((engine) => engine === requestedEngine) +) { console.error( - `usage: media.ts --url U [--timeout S>0] [--fault ${FAULTS.join("|")}] [--cases none|${CASES.join(",")}] [--leak] [--lag] [--expect-fail TEXT]`, + `usage: media.ts --url U [--player-engine ${ENGINES.join("|")}] [--timeout S>0] [--fault ${FAULTS.join("|")}] [--cases none|${CASES.join(",")}] [--leak] [--lag] [--expect-fail TEXT]`, ); process.exit(2); } +const playerEngine = requestedEngine as Engine; const wants = (name: Case) => selected.has(name); // process.exit narrows `url` above, but not inside the function declarations below. @@ -373,67 +386,45 @@ function measure(samples: PlayerState[], label: string): void { ); } -/** Probe every platform API the player needs, rather than assuming this engine has them. */ -async function capabilities(page: Page): Promise { - // The fixture's own configuration, so the probe answers "can this browser run this fixture" - // rather than "can it run something like it". - const wanted = { sampleRate: SAMPLE_RATE, width: Pattern.WIDTH, height: Pattern.HEIGHT }; - const found = await inspect( - page, - async (wanted) => { - const has = (name: string) => typeof (globalThis as Record)[name] === "function"; - const probe = async (fn: () => Promise<{ supported?: boolean }>) => { - try { - return (await fn()).supported === true; - } catch { - return false; - } - }; - return { - WebTransport: has("WebTransport"), - VideoDecoder: has("VideoDecoder"), - VideoEncoder: has("VideoEncoder"), - AudioDecoder: has("AudioDecoder"), - AudioEncoder: has("AudioEncoder"), - AudioWorkletNode: has("AudioWorkletNode"), - MediaStreamTrackProcessor: has("MediaStreamTrackProcessor"), - "canvas.captureStream": typeof HTMLCanvasElement.prototype.captureStream === "function", - "encode avc1.42001f": await probe(() => - VideoEncoder.isConfigSupported({ - codec: "avc1.42001f", - width: wanted.width, - height: wanted.height, - }), - ), - "encode opus": await probe(() => - AudioEncoder.isConfigSupported({ - codec: "opus", - sampleRate: wanted.sampleRate, - numberOfChannels: 1, - }), - ), - "decode opus": await probe(() => - AudioDecoder.isConfigSupported({ - codec: "opus", - sampleRate: wanted.sampleRate, - numberOfChannels: 1, - }), - ), +// The capability page has its own context. Evaluation may grant activation there, +// but cannot unlock the measured fixture or player's audio graph. +const observed = new Map(); +async function capabilities(browser: Browser, engine: Engine, role: Role): Promise { + let record = observed.get(engine); + if (!record) { + const page = await browser.newPage(); + try { + await page.route("**/*", (route) => + route.fulfill({ contentType: "text/html", body: "Capabilities" }), + ); + await page.goto(server.origin); + record = { + version: browser.version(), + found: await page.evaluate(probe, { + sampleRate: SAMPLE_RATE, + width: Pattern.WIDTH, + height: Pattern.HEIGHT, + }), }; - }, - wanted, - ); - - for (const [name, ok] of Object.entries(found)) console.error(` ${ok ? "yes" : "NO "} ${name}`); - - const missing = Object.entries(found) - .filter(([, ok]) => !ok) - .map(([name]) => name); - const version = page.context().browser()?.version() ?? "unknown"; + observed.set(engine, record); + } finally { + await page.close(); + } + console.error(`=== engine capabilities: ${engine} ${record.version} ===`); + for (const [name, ok] of Object.entries(record.found)) console.error(` ${ok ? "yes" : "NO "} ${name}`); + console.error( + ` engine matrix: ${JSON.stringify({ engine, version: record.version, publisherMissing: missing(record.found, "publisher"), playerMissing: missing(record.found, "player") })}`, + ); + if (engine === "webkit") + console.error( + " Playwright WebKit is not Safari; Safari transport and device defects still need a manual run", + ); + } + const absent = missing(record.found, role); check( - missing.length === 0, + absent.length === 0, "capabilities", - () => `chromium ${version} lacks ${missing.join(", ")}; these checks cannot run against it`, + () => `${engine} ${record.version} cannot run ${role}: missing ${absent.join(", ")}`, ); } @@ -443,9 +434,15 @@ const browsers: Browser[] = []; // One browser per role. Chromium's default autoplay behavior varies with engagement and when the // graph is built, so require a gesture in this QA run. Pages in one browser share a renderer // scheduler, and the one that is not frontmost is throttled and reported hidden. -async function browserFor(args: string[] = []): Promise { - const browser = await launch(["--autoplay-policy=document-user-activation-required", ...args]); +async function browserFor(role: Role, args: string[] = []): Promise { + const engine = role === "publisher" ? "chromium" : playerEngine; + console.error(`=== launching ${engine} ${role} ===`); + const browser = await launch( + engine === "chromium" ? ["--autoplay-policy=document-user-activation-required", ...args] : [], + engine, + ); browsers.push(browser); + await capabilities(browser, engine, role); return browser; } @@ -466,9 +463,21 @@ async function subscriber( ): Promise<[Page, BrowserErrors]> { // visible="always" because the window is never frontmost in a headless run, and the default // policy would stop downloading video and leave the canvas black. - const params: Record = { url: relay, broadcast, visible: "always", muted: String(muted) }; + const address = new URL(relay); + if (playerEngine !== "chromium") address.protocol = address.protocol === "https:" ? "wss:" : "ws:"; + const params: Record = { + url: address.toString(), + broadcast, + visible: "always", + muted: String(muted), + }; if (delay !== undefined) params.delay = delay; - const [page, errors] = await open(await browserFor(), pageUrl(server.origin, "subscribe", params), label, trace); + const [page, errors] = await open( + await browserFor("player"), + pageUrl(server.origin, "subscribe", params), + label, + trace, + ); await waitForWatch(page); return [page, errors]; } @@ -491,7 +500,7 @@ async function captureDenial(broadcast: string): Promise { console.error("=== capture denial and grant ==="); const [viewer, viewerErrors] = await subscriber(broadcast, "capture viewer", { muted: true }); const [publisher, publisherErrors] = await open( - await browserFor(["--use-fake-device-for-media-stream"]), + await browserFor("publisher", ["--use-fake-device-for-media-stream"]), pageUrl(server.origin, "publish", { url: relay, broadcast }), "capture publisher", false, @@ -565,14 +574,11 @@ try { // ── publisher ──────────────────────────────────────────────────────────── const [publisher, publisherErrors] = await open( - await browserFor(), + await browserFor("publisher"), pageUrl(server.origin, "fixture", { url: relay, broadcast, fault }), "fixture", ); - console.error("=== capabilities ==="); - await capabilities(publisher); - console.error("=== publisher readiness ==="); const beforePublisherGesture = await readFixtureState(publisher); check( @@ -854,7 +860,9 @@ if (expectFail !== undefined) { console.error(`FAIL ${failure.message}`); code = 1; } else { - console.error("media: all checks passed"); + console.error( + `media: ${playerEngine} player over ${playerEngine === "chromium" ? "automatic transport selection" : "WebSocket fallback"} measured; all checks passed (Chromium publisher)`, + ); } await finishTraces(code !== 0); diff --git a/test/interop/clients/js/src/capabilities.test.ts b/test/interop/clients/js/src/capabilities.test.ts new file mode 100644 index 00000000000..37b3c889ddd --- /dev/null +++ b/test/interop/clients/js/src/capabilities.test.ts @@ -0,0 +1,22 @@ +import { expect, test } from "bun:test"; +import { type Capabilities, missing } from "./capabilities"; + +const player: Capabilities = { + WebSocket: true, + VideoDecoder: true, + AudioDecoder: true, + AudioWorkletNode: true, + "decode avc1.42001f": true, + "decode opus": true, +}; + +test("a fallback player does not require the publisher's capture or encoder APIs", () => { + expect(missing(player, "player")).toEqual([]); + expect(missing(player, "publisher")).toContain("MediaStreamTrackProcessor"); + expect(missing(player, "publisher")).toContain("WebTransport"); +}); + +test("unsupported native decode refuses the player with the missing API named", () => { + expect(missing({ ...player, "decode opus": false }, "player")).toEqual(["decode opus"]); + expect(missing({ ...player, VideoDecoder: false }, "player")).toEqual(["VideoDecoder"]); +}); diff --git a/test/interop/clients/js/src/capabilities.ts b/test/interop/clients/js/src/capabilities.ts new file mode 100644 index 00000000000..b5b410bb57f --- /dev/null +++ b/test/interop/clients/js/src/capabilities.ts @@ -0,0 +1,64 @@ +/** Native browser capabilities, kept separate from publisher and player requirements. @module */ + +/** Which side of the media fixture an engine runs. */ +export type Role = "publisher" | "player"; + +/** Every native capability the media harness observes. */ +export type Capabilities = Record; + +/** Probe native APIs on a secure page without importing the media implementation. */ +export async function probe(wanted: { sampleRate: number; width: number; height: number }): Promise { + const has = (name: string) => typeof (globalThis as unknown as Record)[name] === "function"; + const supports = async (fn: () => Promise<{ supported?: boolean }>) => { + try { + return (await fn()).supported === true; + } catch { + return false; + } + }; + return { + WebTransport: has("WebTransport"), + WebSocket: has("WebSocket"), + VideoDecoder: has("VideoDecoder"), + VideoEncoder: has("VideoEncoder"), + AudioDecoder: has("AudioDecoder"), + AudioEncoder: has("AudioEncoder"), + AudioWorkletNode: has("AudioWorkletNode"), + MediaStreamTrackProcessor: has("MediaStreamTrackProcessor"), + "canvas.captureStream": typeof HTMLCanvasElement.prototype.captureStream === "function", + "encode avc1.42001f": await supports(() => + VideoEncoder.isConfigSupported({ codec: "avc1.42001f", width: wanted.width, height: wanted.height }), + ), + "decode avc1.42001f": await supports(() => + VideoDecoder.isConfigSupported({ + codec: "avc1.42001f", + codedWidth: wanted.width, + codedHeight: wanted.height, + }), + ), + "encode opus": await supports(() => + AudioEncoder.isConfigSupported({ codec: "opus", sampleRate: wanted.sampleRate, numberOfChannels: 1 }), + ), + "decode opus": await supports(() => + AudioDecoder.isConfigSupported({ codec: "opus", sampleRate: wanted.sampleRate, numberOfChannels: 1 }), + ), + }; +} + +/** Name missing APIs for a role; players can use WebSocket without an encoder or WebTransport. */ +export function missing(found: Capabilities, role: Role): string[] { + const required = + role === "publisher" + ? [ + "WebTransport", + "VideoEncoder", + "AudioEncoder", + "AudioWorkletNode", + "MediaStreamTrackProcessor", + "canvas.captureStream", + "encode avc1.42001f", + "encode opus", + ] + : ["WebSocket", "VideoDecoder", "AudioDecoder", "AudioWorkletNode", "decode avc1.42001f", "decode opus"]; + return required.filter((name) => found[name] !== true); +} diff --git a/test/interop/clients/js/src/contract.ts b/test/interop/clients/js/src/contract.ts index 6f8aa05d3c9..aaae47dc3d1 100644 --- a/test/interop/clients/js/src/contract.ts +++ b/test/interop/clients/js/src/contract.ts @@ -145,6 +145,10 @@ export type Sample = { /** Root mean square of the graph root's output over the analyser window. */ rms?: number; + /** Current player chrome, sampled without a driver granting user activation. */ + controlLabel?: string; + centerPlayVisible?: boolean; + /** Whether the player is paused. */ paused: boolean; /** Whether the `paused` attribute is reflected onto the element. */ @@ -213,3 +217,6 @@ export const CONTROL = "__moqInterop"; export function publish(commands: Partial): void { (window as unknown as Record)[CONTROL] = commands; } + +/** Prefix for read-only player samples delivered to the driver through console events. */ +export const SAMPLE_EVENT = "__moqInteropSample:"; diff --git a/test/interop/clients/js/src/probe.ts b/test/interop/clients/js/src/probe.ts index b6e3c5cca60..ae944d6af5a 100644 --- a/test/interop/clients/js/src/probe.ts +++ b/test/interop/clients/js/src/probe.ts @@ -7,14 +7,14 @@ * Both are sampled in the same tick, which is what makes their difference a synchronization * measurement rather than two unrelated timelines. * - * Everything lands on the element's dataset. Playwright may evaluate in a different JavaScript - * world, where DOM nodes are shared but custom-element instance fields are not, so a dataset string - * is the only reading a driver can trust. + * Samples reach both the element's dataset and a console event. Playwright may evaluate in a + * different JavaScript world, where custom-element instance fields are not shared. Alternate + * engines use console observations because evaluating even a read grants user activation. * * @module */ import type MoqWatch from "@moq/watch/element"; -import { AUDIBLE_RMS, SAMPLE_MS, type Sample, TONE_FLOOR_DB } from "./contract"; +import { AUDIBLE_RMS, SAMPLE_EVENT, SAMPLE_MS, type Sample, TONE_FLOOR_DB } from "./contract"; import { resources } from "./instrument"; import * as Pattern from "./pattern"; @@ -124,6 +124,9 @@ export function attach(el: MoqWatch): () => void { } } + const ui = document.querySelector("moq-watch-ui")?.shadowRoot; + const control = ui?.querySelector("button.control[aria-label]"); + const centerPlay = ui?.querySelector("button.center-play"); const state: Sample = { seq: seq++, broadcastActive: el.broadcast.out.active.peek() !== undefined, @@ -140,12 +143,15 @@ export function attach(el: MoqWatch): () => void { audioStalled: el.audio.out.stalled.peek(), delay: el.sync.out.delay.peek(), ...(analyser ? analyze(analyser, spectrum, wave) : {}), + controlLabel: control?.getAttribute("aria-label") ?? undefined, + centerPlayVisible: centerPlay ? getComputedStyle(centerPlay).display !== "none" : false, paused: el.paused, pausedAttribute: el.hasAttribute("paused"), resources: resources(), }; el.dataset.interopState = JSON.stringify(state); + console.debug(SAMPLE_EVENT + JSON.stringify(state)); }; sample(); diff --git a/test/interop/interop.sh b/test/interop/interop.sh index c424835fa4d..96e953bf077 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -60,6 +60,7 @@ URL="" KEY="" # the HMAC key every cell's token is signed with (set once the relay's auth server starts) NEGATIVE=0 MEDIA=0 +MEDIA_ENGINES="chromium" TAIL_ONLY=0 # Cargo profile for the relay/cli/moq-c builds. Debug compiles faster, which is @@ -109,6 +110,12 @@ while [[ $# -gt 0 ]]; do MEDIA=1 shift ;; + --media-engines) + require_value "$@" + MEDIA_ENGINES="$2" + MEDIA=1 + shift 2 + ;; # The full matrix. --timeout 30 gives headless Chromium cold-start # headroom; flags after it still override. --all) @@ -159,6 +166,12 @@ if [[ "$MEDIA" -eq 1 ]]; then SUBSCRIBERS="js" fi +if [[ ! "$MEDIA_ENGINES" =~ ^(chromium|firefox|webkit)(,(chromium|firefox|webkit))*$ ]]; then + echo "error: media engines must be a comma-separated list of chromium,firefox,webkit" >&2 + exit 2 +fi +IFS=',' read -r -a MEDIA_ENGINE_LIST <<<"$MEDIA_ENGINES" + IFS=',' read -r -a PUB_LIST <<<"$PUBLISHERS" IFS=',' read -r -a SUB_LIST <<<"$SUBSCRIBERS" @@ -303,8 +316,8 @@ prepare_js() { fi if needs js; then # Nix provides Chromium via PLAYWRIGHT_BROWSERS_PATH; otherwise fetch it. - if [[ -z "${PLAYWRIGHT_BROWSERS_PATH:-}" ]] && ! (cd "$CLIENTS/js" && bunx playwright install chromium) >"$HARNESS_RUN/js-chromium.log" 2>&1; then - mark_broken js "playwright chromium install failed" + if [[ -z "${PLAYWRIGHT_BROWSERS_PATH:-}" ]] && ! (cd "$CLIENTS/js" && bunx playwright install chromium "${MEDIA_ENGINE_LIST[@]}") >"$HARNESS_RUN/js-chromium.log" 2>&1; then + mark_broken js "requested Playwright engine install failed" sed 's/^/ /' "$HARNESS_RUN/js-chromium.log" >&2 || true elif ! (cd "$CLIENTS/js" && bun run check) >"$HARNESS_RUN/js-check.log" 2>&1; then mark_broken js "type check failed" @@ -1087,12 +1100,14 @@ if [[ "$MEDIA" -eq 1 ]]; then overall=1 else echo "=== media output and lifecycle ===" - run_media "media output + lifecycle" - run_media "control: frozen video" --fault frozen-video --cases none --expect-fail "video progress" - run_media "control: silent audio" --fault silent-audio --cases none --expect-fail "audio tone" - run_media "control: offset audio" --fault audio-offset --cases none --expect-fail "audio/video sync" - run_media "control: leaked session" --leak --cases detach --expect-fail "resource baseline" - run_media "control: lagging latecomer" --lag --cases late-join --expect-fail "late join reaches live" + for engine in "${MEDIA_ENGINE_LIST[@]}"; do + run_media "$engine: media output + lifecycle" --player-engine "$engine" + run_media "$engine: control: frozen video" --player-engine "$engine" --fault frozen-video --cases none --expect-fail "video progress" + run_media "$engine: control: silent audio" --player-engine "$engine" --fault silent-audio --cases none --expect-fail "audio tone" + run_media "$engine: control: offset audio" --player-engine "$engine" --fault audio-offset --cases none --expect-fail "audio/video sync" + run_media "$engine: control: leaked session" --player-engine "$engine" --leak --cases detach --expect-fail "resource baseline" + run_media "$engine: control: lagging latecomer" --player-engine "$engine" --lag --cases late-join --expect-fail "late join reaches live" + done fi elif [[ "$NEGATIVE" -eq 1 ]]; then # Negative control: no publisher. Every subscriber must FAIL (time out with diff --git a/test/justfile b/test/justfile index 8b3860825bd..203dc18a83d 100644 --- a/test/justfile +++ b/test/justfile @@ -67,7 +67,7 @@ interop-negative *args: # republish, late join). Ends with negative controls that inject a frozen picture, silence, a # shifted tone, and a leaked session; each must fail the assertion it names. --timeout 30 gives -# headless Chromium cold-start headroom. +# headless Chromium cold-start headroom. --media-engines selects player engines; the fixture stays Chromium. [positional-arguments] media *args: ./interop/interop.sh --media --timeout 30 "$@" From 7db0631a19b25182234880db1de777cc80eeb09d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:50:46 -0700 Subject: [PATCH 3/4] test(browser): read player state only from logged samples Every engine now reads the player through the samples it logs, so the Chromium CDP read and the per-engine branches go away, and a navigation clears the previous document's sample. The Firefox player passes the full media run locally, so the quest is complete. Nightly drops its Chromium row, which interop.yml already runs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 4 +- quest/m2/README.md | 1 - quest/m2/browser-media-qa-engines.md | 66 ------------------------ test/interop/README.md | 12 ++--- test/interop/clients/js/driver.ts | 6 +-- test/interop/clients/js/harness.test.ts | 48 ++++++++++++------ test/interop/clients/js/harness.ts | 67 +++++-------------------- test/interop/clients/js/media.ts | 21 +++++--- test/interop/clients/js/src/contract.ts | 7 +-- test/interop/clients/js/src/probe.ts | 10 ++-- 10 files changed, 80 insertions(+), 162 deletions(-) delete mode 100644 quest/m2/browser-media-qa-engines.md diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d5f0384c9d8..1d0f8b19762 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -177,6 +177,8 @@ jobs: NEXTEST_PROFILE: ci MOQ_STRICT: 1 + # Media QA with non-Chromium players against a Chromium fixture. The Chromium + # player already runs in interop.yml, on PRs and nightly. media-engines: name: Media (${{ matrix.engine }} player) runs-on: ubuntu-24.04-arm @@ -184,7 +186,7 @@ jobs: strategy: fail-fast: false matrix: - engine: [chromium, firefox, webkit] + engine: [firefox, webkit] steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/quest/m2/README.md b/quest/m2/README.md index afa2c84fa54..e289e47b89f 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -146,7 +146,6 @@ ranked just below deterministic TS legs, which it requires. - [Socket close](/quest/m2/noq-socket-close.md) - moq-tokio's QUIC endpoint releases its socket on close, so it drops its wrapper - [TS health stats](/quest/m2/ts-health-stats.md) - the TS counters ride the stats plumbing beside the media counters - [Teleoperation](/quest/m2/teleop/README.md) - MoQ carries robot video down and control up on one session as a library capability -- [Media QA on other engines](/quest/m2/browser-media-qa-engines.md) - the media harness measures a Firefox or WebKit player over the fallback and names what each engine lacks - [Windows.Graphics.Capture](/quest/m2/capture-wgc.md) - the WGC display and window backend verified on real Windows hardware - [Windows capture parity](/quest/m2/capture-windows.md) - system audio and a settled app-capture policy - [Linux capture parity](/quest/m2/capture-linux.md) - Wayland window/system-audio capture with explicit display-selection and app-capture limits diff --git a/quest/m2/browser-media-qa-engines.md b/quest/m2/browser-media-qa-engines.md deleted file mode 100644 index 1777486fbb7..00000000000 --- a/quest/m2/browser-media-qa-engines.md +++ /dev/null @@ -1,66 +0,0 @@ -# [M] Run the media QA harness with a Firefox or WebKit player - -## Goal - -The media output and lifecycle checks measure a player on more than the -pinned Chromium, and an engine that cannot run a case says so by name -instead of passing by omission. A green run states which engines were -measured and which APIs each one lacked. - -## Plan - -`just test media` measures presented frame progress, tone presence, -and audio/video skew against a deterministic fixture, but `harness.ts` -supports pinned Chromium, Firefox, and WebKit players with a Chromium fixture. -Its capability probe enumerates WebTransport, WebSocket, WebCodecs encode and -decode, AudioWorklet, `MediaStreamTrackProcessor`, and `canvas.captureStream`, -then names what is missing separately for publisher and player roles. - -Playwright capabilities change with the pinned engine. A native probe of -Playwright 1.63 Firefox 155 found WebTransport and WebCodecs, but no -`MediaStreamTrackProcessor`, so it can play but cannot publish this fixture. -The alternate players explicitly use WebSocket to isolate player behavior; -this does not prove their QUIC transport. Probe publisher and player roles -separately instead of assuming API support from the engine name. - -- Split the harness's publisher and subscriber roles so an engine can take - one side: a Chromium fixture with a Firefox or WebKit player over the - fallback is the case worth having. -- Install the engines the run asks for and report a per-engine matrix. An - engine that cannot publish is not a skipped cell; it is a stated limit with - the missing API named, and a green run says which engines it measured. -- Keep the tolerances shared. A skew bound that only holds on one engine is - a measurement of that engine, not of the player. -- Playwright WebKit is not Safari. Say so in the report; a Safari defect such - as #2812 still needs a manual run. -- Real Firefox 155 still needs a manual run too: it reports a non-empty - `WebTransport.protocol` and lands on lite-06 against the in-tree relay, while - 153/154 take the draft-14 SETUP path. - -## Validation evidence (2026-10-15) - -The engine role split, shared graders, explicit WebSocket alternate players, -read-only pre-gesture observations, and nightly Chromium/Firefox/WebKit matrix -are implemented. This quest remains open until a complete alternate-engine -media run passes. - -Firefox 155.0 renders the fixture over WebSocket and passes the pre-gesture -gate, but its audio context remains suspended after the real click with -`userActivated: true`; `gesture starts audio` correctly fails. A standalone -AudioContext also remained suspended across pointerdown, pointerup, and click -resume attempts. The production cause is not established. Negative controls -that fail at this earlier gate are failures, not sensitivity proof. - -WebKit 2359 is installed locally but cannot launch because host runtime -libraries are absent. This is a runtime dependency failure, not evidence about -WebKit APIs. Nightly installs dependencies with Playwright `--with-deps`. -No tolerance or autoplay gate has been relaxed. - -The default Chromium path still enforces its original gate and tolerances. Its -full run fails cold-start tone presence at 65/80 samples against the unchanged -90% floor, matching the previously recorded class of cold-start refusal in -#5635. This does not establish a common cause with the Firefox startup failure. -The Chromium silent-audio, offset-audio, leaked-session, and lagging-latecomer -negative controls pass. Its frozen-video control instead fails publisher -readiness with a suspended fixture context, so that row is not sensitivity -proof in this run. No repeat was used to turn either failure into a pass. diff --git a/test/interop/README.md b/test/interop/README.md index 58ce2baaaf9..17424651b61 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -263,12 +263,12 @@ Each run covers, against a real local relay: - **user gesture** - Chromium is launched with `--autoplay-policy=document-user-activation-required`, which applies to top-level Web Audio contexts. The fixture and player graphs must be suspended - before either page is clicked, then both must carry audio afterwards. Harness - state probes use CDP with `userGesture: false`. Firefox retains its default - Web Audio autoplay block. Firefox/WebKit observations consume the page's - existing sample events without executing scripts in the measured page: - Playwright's usual reads grant activation and would invalidate this assertion. - Capability probes run in a separate context before the measured page opens. + before either page is clicked, then both must carry audio afterwards. Firefox + is launched with its Web Audio autoplay block enabled. Playwright's usual page + reads grant activation and would invalidate this assertion, so the player logs + each sample to the console and the harness never evaluates in a player page + before its click. The Chromium fixture is read over CDP with + `userGesture: false`, and capability probes run in a separate page. - **capture permission** - the camera case uses a fake device for deterministic input, while Playwright denies and then grants permissions. Both source errors must be visible; neither a full nor a microphone-only denial may announce a diff --git a/test/interop/clients/js/driver.ts b/test/interop/clients/js/driver.ts index 85d80e095a6..167fcc2e0f9 100644 --- a/test/interop/clients/js/driver.ts +++ b/test/interop/clients/js/driver.ts @@ -19,7 +19,6 @@ import { finishTraces, launch, open, - type PlayerState, POLL_INTERVAL_MS, pageUrl, pause, @@ -32,6 +31,7 @@ import { waitForState, waitForWatch, } from "./harness"; +import type { Sample } from "./src/contract"; /** Remove the page's players, which closes their sessions, and wait until every transport has. */ async function closeSessions(page: Page): Promise { @@ -78,7 +78,7 @@ if ( const PAUSE_STABILITY_MS = 750; -async function waitForStablePause(page: Page, errors: BrowserErrors, deadline: number): Promise { +async function waitForStablePause(page: Page, errors: BrowserErrors, deadline: number): Promise { let previous = await readPlayerState(page); let stableSince = Date.now(); while (Date.now() < deadline) { @@ -123,7 +123,7 @@ try { const start = Date.now(); const startupDeadline = start + timeoutMs; let reloaded = false; - let playing: PlayerState | undefined; + let playing: Sample | undefined; while (Date.now() < startupDeadline) { throwPageErrors(errors); const state = await readPlayerState(page); diff --git a/test/interop/clients/js/harness.test.ts b/test/interop/clients/js/harness.test.ts index 6250a64c638..d3cd8ba7a39 100644 --- a/test/interop/clients/js/harness.test.ts +++ b/test/interop/clients/js/harness.test.ts @@ -3,27 +3,45 @@ import type { Browser, Page } from "playwright"; import { open, readPlayerState, waitForWatch } from "./harness"; import { SAMPLE_EVENT } from "./src/contract"; -test("Firefox player observations never evaluate in the page before its gesture", async () => { - let consoleListener: (message: { text: () => string }) => void = () => {}; - const sample = { seq: 1, userActivated: false, centerPlayVisible: true, controlLabel: "Play" }; +// A page that logs samples on demand and refuses evaluation, since evaluating would grant +// user activation before the gesture checks. +function fakePage() { + const listeners = new Map void>(); + const frame = {}; const page = { - on(event: string, listener: typeof consoleListener) { - if (event === "console") consoleListener = listener; + on(event: string, listener: (arg: unknown) => void) { + listeners.set(event, listener); }, - context: () => ({ browser: () => ({ browserType: () => ({ name: () => "firefox" }) }) }), + mainFrame: () => frame, evaluate() { throw new Error("evaluation would grant user activation"); }, - goto() { - consoleListener({ text: () => SAMPLE_EVENT + JSON.stringify(sample) }); - }, + goto() {}, } as unknown as Page; - const browser = { newPage: () => page } as unknown as Browser; + const log = (sample: object) => listeners.get("console")?.({ text: () => SAMPLE_EVENT + JSON.stringify(sample) }); + const navigate = () => listeners.get("framenavigated")?.(frame); + return { page, log, navigate, browser: { newPage: () => page } as unknown as Browser }; +} + +test("player observations never evaluate in the page", async () => { + const { page, log, browser } = fakePage(); await open(browser, "http://localhost"); + log({ seq: 1, userActivated: false, centerPlayVisible: true, controlLabel: "Play" }); await waitForWatch(page); - const observed = await readPlayerState(page); - expect(observed.seq).toBe(1); - expect(observed.userActivated).toBe(false); - expect(observed.centerPlayVisible).toBe(true); - expect(observed.controlLabel).toBe("Play"); + expect(await readPlayerState(page)).toMatchObject({ + seq: 1, + userActivated: false, + centerPlayVisible: true, + controlLabel: "Play", + }); +}); + +test("a navigation discards the previous document's sample", async () => { + const { page, log, navigate, browser } = fakePage(); + await open(browser, "http://localhost"); + log({ seq: 7 }); + navigate(); + await expect(readPlayerState(page)).rejects.toThrow("has not published a sample"); + log({ seq: 0 }); + expect((await readPlayerState(page)).seq).toBe(0); }); diff --git a/test/interop/clients/js/harness.ts b/test/interop/clients/js/harness.ts index b98c8566b18..a7022c9ea99 100644 --- a/test/interop/clients/js/harness.ts +++ b/test/interop/clients/js/harness.ts @@ -48,14 +48,6 @@ export function check(ok: boolean, assertion: string, detail: () => string): voi if (!ok) throw new Failure(assertion, detail()); } -/** The page's own measurements plus the player chrome, which lives in a shadow root. */ -export type PlayerState = Sample & { - /** `aria-label` of the primary control button: "Pause" while playing, "Play" while paused. */ - controlLabel?: string; - /** Whether the big center play button is showing. */ - centerPlayVisible: boolean; -}; - /** Errors the page reported, collected so any wait can fail on them instead of timing out. */ export type BrowserErrors = { page: string[]; @@ -66,7 +58,6 @@ export type BrowserErrors = { export const SELECTORS = { watch: "moq-watch", ui: "moq-watch-ui", - control: "button.control[aria-label]", pauseControl: 'button.control[aria-label="Pause"]', centerPlay: "button.center-play", fixture: "#fixture", @@ -133,6 +124,8 @@ export function launch(args: string[] = [], engine: Engine = "chromium"): Promis return webkit.launch({ headless: true, handleSIGTERM: false }); } +// The latest sample each player page logged. Cleared on navigation so a reload cannot read the +// previous document's state. const samples = new WeakMap(); /** Contexts tracing this process, saved by {@link finishTraces} when the run fails. */ @@ -180,6 +173,9 @@ export async function open( ): Promise<[Page, BrowserErrors]> { const page = await browser.newPage(options); const errors: BrowserErrors = { page: [], console: [] }; + page.on("framenavigated", (frame) => { + if (frame === page.mainFrame()) samples.delete(page); + }); page.on("console", (message) => { if (message.text().startsWith(SAMPLE_EVENT)) { samples.set(page, JSON.parse(message.text().slice(SAMPLE_EVENT.length)) as Sample); @@ -210,12 +206,8 @@ export function throwPageErrors(errors: BrowserErrors): void { // before the deliberate click would therefore unlock Web Audio and invalidate the gesture test. const sessions = new WeakMap(); -/** Inspect page state without granting user activation to the document. */ +/** Inspect a Chromium page without granting user activation to the document. Players log samples instead. */ export async function inspect(page: Page, fn: (arg: A) => T, arg: A): Promise> { - // Non-Chromium reads here follow an explicit gesture. Pre-gesture player reads - // use the page's read-only sample events instead of evaluation. - if (page.context().browser()?.browserType().name() !== "chromium") - return (await page.evaluate(`(${fn.toString()})(${JSON.stringify(arg) ?? "undefined"})`)) as Awaited; let session = sessions.get(page); if (!session) { session = await page.context().newCDPSession(page); @@ -235,50 +227,17 @@ export async function inspect(page: Page, fn: (arg: A) => T, arg: A): Prom export async function waitForWatch(page: Page): Promise { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { - const ready = - page.context().browser()?.browserType().name() !== "chromium" - ? samples.has(page) - : await inspect( - page, - (tag) => document.querySelector(`${tag}[data-interop-ready]`) !== null, - SELECTORS.watch, - ); - if (ready) return; + if (samples.has(page)) return; await sleep(POLL_INTERVAL_MS); } throw new Error("the player did not publish its first sample"); } -/** Read one sample plus the player chrome. Throws until the page has sampled at least once. */ -export async function readPlayerState(page: Page): Promise { - if (page.context().browser()?.browserType().name() !== "chromium") { - const sample = samples.get(page); - if (!sample) throw new Error("the player has not published a sample"); - return { ...sample, centerPlayVisible: sample.centerPlayVisible ?? false }; - } - const state = await inspect( - page, - (selectors) => { - const watch = document.querySelector(selectors.watch); - const ui = document.querySelector(selectors.ui); - const control = ui?.shadowRoot?.querySelector(selectors.control); - const centerPlay = ui?.shadowRoot?.querySelector(selectors.centerPlay); - - return { - sample: watch?.dataset.interopState, - controlLabel: control?.getAttribute("aria-label") ?? undefined, - centerPlayVisible: centerPlay ? getComputedStyle(centerPlay).display !== "none" : false, - }; - }, - SELECTORS, - ); - - if (!state.sample) throw new Error("the player has not published a sample"); - return { - ...(JSON.parse(state.sample) as Sample), - controlLabel: state.controlLabel, - centerPlayVisible: state.centerPlayVisible, - }; +/** Read the latest player sample. Throws until the page has sampled at least once. */ +export async function readPlayerState(page: Page): Promise { + const sample = samples.get(page); + if (!sample) throw new Error("the player has not published a sample"); + return sample; } /** Read what the fixture publisher says about itself. Throws until it has published anything. */ @@ -344,7 +303,7 @@ export async function waitFor( } /** Poll the player until `predicate` holds. See {@link waitFor}. */ -export function waitForState(page: Page, errors: BrowserErrors, props: WaitProps): Promise { +export function waitForState(page: Page, errors: BrowserErrors, props: WaitProps): Promise { return waitFor(page, errors, readPlayerState, props); } diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index 7a6003d7c7e..0f4b71f3f06 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -30,7 +30,6 @@ import { inspect, launch, open, - type PlayerState, POLL_INTERVAL_MS, pageUrl, pause, @@ -48,7 +47,15 @@ import { waitForWatch, } from "./harness"; import { type Capabilities, missing, probe, type Role } from "./src/capabilities"; -import { type CaptureState, FAULTS, leakedPlayerStarted, reachesLive, SAMPLE_MS, SAMPLE_RATE } from "./src/contract"; +import { + type CaptureState, + FAULTS, + leakedPlayerStarted, + reachesLive, + SAMPLE_MS, + SAMPLE_RATE, + type Sample, +} from "./src/contract"; import * as Pattern from "./src/pattern"; import { resumeSkew } from "./src/resume"; @@ -131,8 +138,8 @@ const percentile = (values: number[], p: number) => { }; /** Collect distinct samples from the player for `ms`, so a window can be measured after the fact. */ -async function collect(page: Page, errors: BrowserErrors, ms: number): Promise { - const samples: PlayerState[] = []; +async function collect(page: Page, errors: BrowserErrors, ms: number): Promise { + const samples: Sample[] = []; const deadline = Date.now() + ms; while (Date.now() < deadline) { throwPageErrors(errors); @@ -291,7 +298,7 @@ async function measurePlayback(player: Page, playerErrors: BrowserErrors, label: // One sample as a line: elapsed time, the frame on the canvas, the tone step heard against the one // that frame belongs to, how far the tone stood above the noise floor, and whether audio was // arriving at all, which separates a silent player from a publisher that stopped sending. -function traceLine(sample: PlayerState, start: number): string { +function traceLine(sample: Sample, start: number): string { const step = sample.frameId === undefined ? "?" : Pattern.expectedStep(sample.frameId); const margin = sample.toneDb !== undefined && sample.noiseDb !== undefined ? sample.toneDb - sample.noiseDb : 0; return ( @@ -309,7 +316,7 @@ function traceLine(sample: PlayerState, start: number): string { * the tone off the graph root that feeds the speakers. A failure prints every reading in the window, * so the summary can be traced back to the readings behind it. */ -function assertMedia(samples: PlayerState[], label: string): void { +function assertMedia(samples: Sample[], label: string): void { try { measure(samples, label); } catch (err) { @@ -320,7 +327,7 @@ function assertMedia(samples: PlayerState[], label: string): void { } } -function measure(samples: PlayerState[], label: string): void { +function measure(samples: Sample[], label: string): void { check(samples.length >= 10, "sampling", () => `${label}: only ${samples.length} samples in the window`); const readable = samples.filter((s) => s.frameId !== undefined); diff --git a/test/interop/clients/js/src/contract.ts b/test/interop/clients/js/src/contract.ts index aaae47dc3d1..8429eec6e88 100644 --- a/test/interop/clients/js/src/contract.ts +++ b/test/interop/clients/js/src/contract.ts @@ -145,9 +145,10 @@ export type Sample = { /** Root mean square of the graph root's output over the analyser window. */ rms?: number; - /** Current player chrome, sampled without a driver granting user activation. */ + /** `aria-label` of the primary control button: "Pause" while playing, "Play" while paused. */ controlLabel?: string; - centerPlayVisible?: boolean; + /** Whether the big center play button is showing. */ + centerPlayVisible: boolean; /** Whether the player is paused. */ paused: boolean; @@ -218,5 +219,5 @@ export function publish(commands: Partial): void { (window as unknown as Record)[CONTROL] = commands; } -/** Prefix for read-only player samples delivered to the driver through console events. */ +/** Prefix of the console line carrying each player {@link Sample} to the driver. */ export const SAMPLE_EVENT = "__moqInteropSample:"; diff --git a/test/interop/clients/js/src/probe.ts b/test/interop/clients/js/src/probe.ts index ae944d6af5a..7667dcd9881 100644 --- a/test/interop/clients/js/src/probe.ts +++ b/test/interop/clients/js/src/probe.ts @@ -7,9 +7,9 @@ * Both are sampled in the same tick, which is what makes their difference a synchronization * measurement rather than two unrelated timelines. * - * Samples reach both the element's dataset and a console event. Playwright may evaluate in a - * different JavaScript world, where custom-element instance fields are not shared. Alternate - * engines use console observations because evaluating even a read grants user activation. + * Each sample is logged to the console for the driver. Reading the page instead would mean + * evaluating in it, and Playwright evaluation grants user activation, which would invalidate the + * gesture checks. * * @module */ @@ -100,7 +100,7 @@ export function watchResources(): void { } /** - * Start sampling a player onto `el.dataset.interopState`, and mark it ready for a driver to poll. + * Start sampling a player to the console, beginning immediately. * * Returns a function that stops sampling. */ @@ -150,12 +150,10 @@ export function attach(el: MoqWatch): () => void { resources: resources(), }; - el.dataset.interopState = JSON.stringify(state); console.debug(SAMPLE_EVENT + JSON.stringify(state)); }; sample(); - el.dataset.interopReady = ""; const timer = self.setInterval(sample, SAMPLE_MS); return () => self.clearInterval(timer); From c116f191ee78f6fddd32b5ddb5a461b6196342ff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 15:01:25 -0700 Subject: [PATCH 4/4] quest: drop the teleop entry main already removed Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/quest/m2/README.md b/quest/m2/README.md index 4a419894631..d25bf492a63 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -160,7 +160,6 @@ ranked just below deterministic TS legs, which it requires. - [Keep-alive from the idle timeout](/quest/m2/quic-keep-alive.md) - the keep-alive interval defaults to a fraction of the negotiated idle timeout; an explicit `quic.keep_alive` overrides - [Socket close](/quest/m2/noq-socket-close.md) - moq-tokio's QUIC endpoint releases its socket on close, so it drops its wrapper - [TS health stats](/quest/m2/ts-health-stats.md) - the TS counters ride the stats plumbing beside the media counters -- [Teleoperation](/quest/m2/teleop/README.md) - MoQ carries robot video down and control up on one session as a library capability - [Windows.Graphics.Capture](/quest/m2/capture-wgc.md) - the WGC display and window backend verified on real Windows hardware - [Windows capture parity](/quest/m2/capture-windows.md) - system audio and a settled app-capture policy - [Linux capture parity](/quest/m2/capture-linux.md) - Wayland window/system-audio capture with explicit display-selection and app-capture limits