diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0c6000da6a1..19c50d3c231 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -189,6 +189,43 @@ jobs: NEXTEST_PROFILE: ci MOQ_STRICT: 1 + # Media QA with non-Chromium players against a Chromium fixture. The Chromium + # player already runs in interop.yml, on PRs and nightly. + media-engines: + name: Media (${{ matrix.engine }} player) + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + engine: [firefox, webkit] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + - name: Rust cache + uses: ./.github/actions/rust-cache + - name: Install publisher and player engines + run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium ${{ matrix.engine }}" + - name: Media and negative controls + run: nix develop --command just test media --media-engines ${{ matrix.engine }} + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + - name: Preserve measurements and traces + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: media-${{ matrix.engine }} + path: ${{ runner.temp }}/moq-test-runs + if-no-files-found: error + # Full platform coverage without rebuilding after every main queue merge. platform: uses: ./.github/workflows/platform.yml diff --git a/quest/m2/README.md b/quest/m2/README.md index b955da7d258..d25bf492a63 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -160,7 +160,6 @@ ranked just below deterministic TS legs, which it requires. - [Keep-alive from the idle timeout](/quest/m2/quic-keep-alive.md) - the keep-alive interval defaults to a fraction of the negotiated idle timeout; an explicit `quic.keep_alive` overrides - [Socket close](/quest/m2/noq-socket-close.md) - moq-tokio's QUIC endpoint releases its socket on close, so it drops its wrapper - [TS health stats](/quest/m2/ts-health-stats.md) - the TS counters ride the stats plumbing beside the media counters -- [Media QA on other engines](/quest/m2/browser-media-qa-engines.md) - the media harness measures a Firefox or WebKit player over the fallback and names what each engine lacks - [Windows.Graphics.Capture](/quest/m2/capture-wgc.md) - the WGC display and window backend verified on real Windows hardware - [Windows capture parity](/quest/m2/capture-windows.md) - system audio and a settled app-capture policy - [Linux capture parity](/quest/m2/capture-linux.md) - Wayland window/system-audio capture with explicit display-selection and app-capture limits diff --git a/quest/m2/browser-media-qa-engines.md b/quest/m2/browser-media-qa-engines.md deleted file mode 100644 index 135579bb715..00000000000 --- a/quest/m2/browser-media-qa-engines.md +++ /dev/null @@ -1,38 +0,0 @@ -# [M] Run the media QA harness with a Firefox or WebKit player - -## Goal - -The media output and lifecycle checks measure a player on more than the -pinned Chromium, and an engine that cannot run a case says so by name -instead of passing by omission. A green run states which engines were -measured and which APIs each one lacked. - -## Plan - -`just test media` measures presented frame progress, tone presence, -and audio/video skew against a deterministic fixture, but `harness.ts` -launches only Playwright's Chromium and nothing installs another engine. Its -capability probe already enumerates what the player needs (WebTransport, -WebCodecs encode and decode, AudioWorklet, `MediaStreamTrackProcessor`, -`canvas.captureStream`) and fails naming what is missing. - -Neither Playwright Firefox nor Playwright WebKit ships WebTransport, and -WebKit also lacks the WebCodecs encoder, so neither can publish and neither -can play over QUIC. What they can do is play over the WebSocket fallback. -Real Firefox does use WebTransport; only Playwright's build lacks it, so a -fallback run covers the player, not Firefox's transport path (see -[#4460](https://github.com/moq-dev/moq/pull/4460), merged 2026-10-01). - -- Split the harness's publisher and subscriber roles so an engine can take - one side: a Chromium fixture with a Firefox or WebKit player over the - fallback is the case worth having. -- Install the engines the run asks for and report a per-engine matrix. An - engine that cannot publish is not a skipped cell; it is a stated limit with - the missing API named, and a green run says which engines it measured. -- Keep the tolerances shared. A skew bound that only holds on one engine is - a measurement of that engine, not of the player. -- Playwright WebKit is not Safari. Say so in the report; a Safari defect such - as #2812 still needs a manual run. -- Real Firefox 155 still needs a manual run too: it reports a non-empty - `WebTransport.protocol` and lands on lite-06 against the in-tree relay, while - 153/154 take the draft-14 SETUP path. diff --git a/test/interop/README.md b/test/interop/README.md index 3a2fe8cbcde..17424651b61 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -231,6 +231,19 @@ refresh each reason before committing. The matrix asks "did bytes arrive and did a pixel light up". That passes on a frozen picture, on silence, and on audio a second out of step, so `just test media` measures the media itself, browser to browser. +Use `just test media --media-engines chromium,firefox,webkit` to run each pinned +Playwright player against a Chromium publisher. The harness installs the requested +engines and runs the same lifecycle cases, content tolerances, and five negative +controls per engine. A missing runtime or required API fails that engine by name. +Firefox and WebKit players explicitly use WebSocket; Chromium keeps normal +transport selection. This does not measure Firefox's QUIC path. + +The capability report names the engine version and separately lists missing +publisher and player APIs. Firefox 155 in Playwright 1.63 exposes WebTransport and +WebCodecs but lacks `MediaStreamTrackProcessor`, so it can play this fixture but +cannot publish it. Capabilities are measured on every run rather than assumed +from an engine name. Playwright WebKit is not Safari; Safari device/transport +bugs and real Firefox transport negotiation still require manual evidence. The publisher is a fixture, not a fake camera: a canvas painting a frame counter as black/white blocks, and a tone stepping through a fixed frequency table. Both @@ -242,7 +255,7 @@ samples, not that a machine played them. Each run covers, against a real local relay: -- **capabilities** - probes every platform API the player needs, and fails +- **capabilities** - probes native APIs separately for the publisher and player, and fails naming what is missing rather than skipping a case. - **cold start** - the publisher reports when it is announced and encoding, then a fresh page joins. No reload, unlike the matrix driver: a subscriber that @@ -250,9 +263,12 @@ Each run covers, against a real local relay: - **user gesture** - Chromium is launched with `--autoplay-policy=document-user-activation-required`, which applies to top-level Web Audio contexts. The fixture and player graphs must be suspended - before either page is clicked, then both must carry audio afterwards. Harness - state probes use CDP with `userGesture: false`: Playwright's usual page reads - themselves grant activation and would invalidate this assertion. + before either page is clicked, then both must carry audio afterwards. Firefox + is launched with its Web Audio autoplay block enabled. Playwright's usual page + reads grant activation and would invalidate this assertion, so the player logs + each sample to the console and the harness never evaluates in a player page + before its click. The Chromium fixture is read over CDP with + `userGesture: false`, and capability probes run in a separate page. - **capture permission** - the camera case uses a fake device for deterministic input, while Playwright denies and then grants permissions. Both source errors must be visible; neither a full nor a microphone-only denial may announce a diff --git a/test/interop/clients/js/driver.ts b/test/interop/clients/js/driver.ts index 85d80e095a6..167fcc2e0f9 100644 --- a/test/interop/clients/js/driver.ts +++ b/test/interop/clients/js/driver.ts @@ -19,7 +19,6 @@ import { finishTraces, launch, open, - type PlayerState, POLL_INTERVAL_MS, pageUrl, pause, @@ -32,6 +31,7 @@ import { waitForState, waitForWatch, } from "./harness"; +import type { Sample } from "./src/contract"; /** Remove the page's players, which closes their sessions, and wait until every transport has. */ async function closeSessions(page: Page): Promise { @@ -78,7 +78,7 @@ if ( const PAUSE_STABILITY_MS = 750; -async function waitForStablePause(page: Page, errors: BrowserErrors, deadline: number): Promise { +async function waitForStablePause(page: Page, errors: BrowserErrors, deadline: number): Promise { let previous = await readPlayerState(page); let stableSince = Date.now(); while (Date.now() < deadline) { @@ -123,7 +123,7 @@ try { const start = Date.now(); const startupDeadline = start + timeoutMs; let reloaded = false; - let playing: PlayerState | undefined; + let playing: Sample | undefined; while (Date.now() < startupDeadline) { throwPageErrors(errors); const state = await readPlayerState(page); diff --git a/test/interop/clients/js/harness.test.ts b/test/interop/clients/js/harness.test.ts new file mode 100644 index 00000000000..d3cd8ba7a39 --- /dev/null +++ b/test/interop/clients/js/harness.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from "bun:test"; +import type { Browser, Page } from "playwright"; +import { open, readPlayerState, waitForWatch } from "./harness"; +import { SAMPLE_EVENT } from "./src/contract"; + +// A page that logs samples on demand and refuses evaluation, since evaluating would grant +// user activation before the gesture checks. +function fakePage() { + const listeners = new Map void>(); + const frame = {}; + const page = { + on(event: string, listener: (arg: unknown) => void) { + listeners.set(event, listener); + }, + mainFrame: () => frame, + evaluate() { + throw new Error("evaluation would grant user activation"); + }, + goto() {}, + } as unknown as Page; + const log = (sample: object) => listeners.get("console")?.({ text: () => SAMPLE_EVENT + JSON.stringify(sample) }); + const navigate = () => listeners.get("framenavigated")?.(frame); + return { page, log, navigate, browser: { newPage: () => page } as unknown as Browser }; +} + +test("player observations never evaluate in the page", async () => { + const { page, log, browser } = fakePage(); + await open(browser, "http://localhost"); + log({ seq: 1, userActivated: false, centerPlayVisible: true, controlLabel: "Play" }); + await waitForWatch(page); + expect(await readPlayerState(page)).toMatchObject({ + seq: 1, + userActivated: false, + centerPlayVisible: true, + controlLabel: "Play", + }); +}); + +test("a navigation discards the previous document's sample", async () => { + const { page, log, navigate, browser } = fakePage(); + await open(browser, "http://localhost"); + log({ seq: 7 }); + navigate(); + await expect(readPlayerState(page)).rejects.toThrow("has not published a sample"); + log({ seq: 0 }); + expect((await readPlayerState(page)).seq).toBe(0); +}); diff --git a/test/interop/clients/js/harness.ts b/test/interop/clients/js/harness.ts index 23c02268950..a7022c9ea99 100644 --- a/test/interop/clients/js/harness.ts +++ b/test/interop/clients/js/harness.ts @@ -1,5 +1,5 @@ /** - * Shared Playwright plumbing for the browser clients: serve the built page, launch Chromium, and + * Shared Playwright plumbing for the browser clients: serve the built page, launch an engine, and * read back the state each role mirrors onto the DOM. * * Used by `driver.ts` (the interop matrix) and `media.ts` (the media-output and lifecycle checks). @@ -14,9 +14,18 @@ import { type BrowserContextOptions, type CDPSession, chromium, + firefox, type Page, + webkit, } from "playwright"; -import { CONTROL, type FixtureState, type InteropControl, type Resources, type Sample } from "./src/contract"; +import { + CONTROL, + type FixtureState, + type InteropControl, + type Resources, + SAMPLE_EVENT, + type Sample, +} from "./src/contract"; /** * A failed check, named after the property it was measuring. @@ -39,14 +48,6 @@ export function check(ok: boolean, assertion: string, detail: () => string): voi if (!ok) throw new Failure(assertion, detail()); } -/** The page's own measurements plus the player chrome, which lives in a shadow root. */ -export type PlayerState = Sample & { - /** `aria-label` of the primary control button: "Pause" while playing, "Play" while paused. */ - controlLabel?: string; - /** Whether the big center play button is showing. */ - centerPlayVisible: boolean; -}; - /** Errors the page reported, collected so any wait can fail on them instead of timing out. */ export type BrowserErrors = { page: string[]; @@ -57,7 +58,6 @@ export type BrowserErrors = { export const SELECTORS = { watch: "moq-watch", ui: "moq-watch-ui", - control: "button.control[aria-label]", pauseControl: 'button.control[aria-label="Pause"]', centerPlay: "button.center-play", fixture: "#fixture", @@ -103,19 +103,31 @@ export function pageUrl(origin: string, role: string, params: Record { +/** Browser engines the media harness can launch. */ +export const ENGINES = ["chromium", "firefox", "webkit"] as const; +/** One of the pinned Playwright engines. */ +export type Engine = (typeof ENGINES)[number]; + +/** Launch an engine with its normal gesture gate and caller-supplied Chromium arguments. */ +export function launch(args: string[] = [], engine: Engine = "chromium"): Promise { // Playwright closes the browser on SIGTERM by default, which ends its sessions without a // close. The drivers close it themselves; the harness reaps whatever a signal leaves behind. - return chromium.launch({ channel: "chromium", headless: true, args, handleSIGTERM: false }); + // Full Chromium includes WebTransport and WebCodecs; the headless shell does not. + if (engine === "chromium") + return chromium.launch({ channel: "chromium", headless: true, args, handleSIGTERM: false }); + if (engine === "firefox") + return firefox.launch({ + headless: true, + handleSIGTERM: false, + firefoxUserPrefs: { "media.autoplay.default": 1, "media.autoplay.block-webaudio": true }, + }); + return webkit.launch({ headless: true, handleSIGTERM: false }); } +// The latest sample each player page logged. Cleared on navigation so a reload cannot read the +// previous document's state. +const samples = new WeakMap(); + /** Contexts tracing this process, saved by {@link finishTraces} when the run fails. */ const traces: Array<{ context: BrowserContext; name: string }> = []; @@ -161,7 +173,14 @@ export async function open( ): Promise<[Page, BrowserErrors]> { const page = await browser.newPage(options); const errors: BrowserErrors = { page: [], console: [] }; + page.on("framenavigated", (frame) => { + if (frame === page.mainFrame()) samples.delete(page); + }); page.on("console", (message) => { + if (message.text().startsWith(SAMPLE_EVENT)) { + samples.set(page, JSON.parse(message.text().slice(SAMPLE_EVENT.length)) as Sample); + return; + } console.error(`[${label}] ${message.text()}`); if (message.type() === "error") errors.console.push(message.text()); }); @@ -183,11 +202,11 @@ export function throwPageErrors(errors: BrowserErrors): void { if (messages.length > 0) throw new Error(messages.join("\n")); } -// Playwright's page.evaluate and locator reads grant user activation in Chromium. A state probe +// Playwright's page.evaluate and locator reads grant user activation. A state probe // before the deliberate click would therefore unlock Web Audio and invalidate the gesture test. const sessions = new WeakMap(); -/** Inspect page state without granting user activation to the document. */ +/** Inspect a Chromium page without granting user activation to the document. Players log samples instead. */ export async function inspect(page: Page, fn: (arg: A) => T, arg: A): Promise> { let session = sessions.get(page); if (!session) { @@ -208,42 +227,17 @@ export async function inspect(page: Page, fn: (arg: A) => T, arg: A): Prom export async function waitForWatch(page: Page): Promise { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { - const ready = await inspect( - page, - (tag) => document.querySelector(`${tag}[data-interop-ready]`) !== null, - SELECTORS.watch, - ); - if (ready) return; + if (samples.has(page)) return; await sleep(POLL_INTERVAL_MS); } throw new Error("the player did not publish its first sample"); } -/** Read one sample plus the player chrome. Throws until the page has sampled at least once. */ -export async function readPlayerState(page: Page): Promise { - const state = await inspect( - page, - (selectors) => { - const watch = document.querySelector(selectors.watch); - const ui = document.querySelector(selectors.ui); - const control = ui?.shadowRoot?.querySelector(selectors.control); - const centerPlay = ui?.shadowRoot?.querySelector(selectors.centerPlay); - - return { - sample: watch?.dataset.interopState, - controlLabel: control?.getAttribute("aria-label") ?? undefined, - centerPlayVisible: centerPlay ? getComputedStyle(centerPlay).display !== "none" : false, - }; - }, - SELECTORS, - ); - - if (!state.sample) throw new Error("the player has not published a sample"); - return { - ...(JSON.parse(state.sample) as Sample), - controlLabel: state.controlLabel, - centerPlayVisible: state.centerPlayVisible, - }; +/** Read the latest player sample. Throws until the page has sampled at least once. */ +export async function readPlayerState(page: Page): Promise { + const sample = samples.get(page); + if (!sample) throw new Error("the player has not published a sample"); + return sample; } /** Read what the fixture publisher says about itself. Throws until it has published anything. */ @@ -309,7 +303,7 @@ export async function waitFor( } /** Poll the player until `predicate` holds. See {@link waitFor}. */ -export function waitForState(page: Page, errors: BrowserErrors, props: WaitProps): Promise { +export function waitForState(page: Page, errors: BrowserErrors, props: WaitProps): Promise { return waitFor(page, errors, readPlayerState, props); } diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index e527cec2609..0f4b71f3f06 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -23,12 +23,13 @@ import { type BrowserErrors, check, command, + ENGINES, + type Engine, Failure, finishTraces, inspect, launch, open, - type PlayerState, POLL_INTERVAL_MS, pageUrl, pause, @@ -45,7 +46,16 @@ import { waitForState, waitForWatch, } from "./harness"; -import { type CaptureState, FAULTS, leakedPlayerStarted, reachesLive, SAMPLE_MS, SAMPLE_RATE } from "./src/contract"; +import { type Capabilities, missing, probe, type Role } from "./src/capabilities"; +import { + type CaptureState, + FAULTS, + leakedPlayerStarted, + reachesLive, + SAMPLE_MS, + SAMPLE_RATE, + type Sample, +} from "./src/contract"; import * as Pattern from "./src/pattern"; import { resumeSkew } from "./src/resume"; @@ -56,6 +66,7 @@ type Case = (typeof CASES)[number]; const { values } = parseArgs({ options: { url: { type: "string" }, + "player-engine": { type: "string", default: "chromium" }, timeout: { type: "string", default: "30" }, fault: { type: "string", default: "none" }, cases: { type: "string" }, @@ -69,16 +80,25 @@ const url = values.url; const timeoutMs = Number.parseFloat(values.timeout ?? "30") * 1000; const fault = values.fault ?? "none"; const expectFail = values["expect-fail"]; +const requestedEngine = values["player-engine"] ?? "chromium"; const selected = new Set( values.cases === undefined ? CASES : values.cases === "none" ? [] : values.cases.split(","), ); const unknown = [...selected].filter((name) => !CASES.some((c) => c === name)); -if (!url || !Number.isFinite(timeoutMs) || timeoutMs <= 0 || !FAULTS.some((f) => f === fault) || unknown.length > 0) { +if ( + !url || + !Number.isFinite(timeoutMs) || + timeoutMs <= 0 || + !FAULTS.some((f) => f === fault) || + unknown.length > 0 || + !ENGINES.some((engine) => engine === requestedEngine) +) { console.error( - `usage: media.ts --url U [--timeout S>0] [--fault ${FAULTS.join("|")}] [--cases none|${CASES.join(",")}] [--leak] [--lag] [--expect-fail TEXT]`, + `usage: media.ts --url U [--player-engine ${ENGINES.join("|")}] [--timeout S>0] [--fault ${FAULTS.join("|")}] [--cases none|${CASES.join(",")}] [--leak] [--lag] [--expect-fail TEXT]`, ); process.exit(2); } +const playerEngine = requestedEngine as Engine; const wants = (name: Case) => selected.has(name); // process.exit narrows `url` above, but not inside the function declarations below. @@ -118,8 +138,8 @@ const percentile = (values: number[], p: number) => { }; /** Collect distinct samples from the player for `ms`, so a window can be measured after the fact. */ -async function collect(page: Page, errors: BrowserErrors, ms: number): Promise { - const samples: PlayerState[] = []; +async function collect(page: Page, errors: BrowserErrors, ms: number): Promise { + const samples: Sample[] = []; const deadline = Date.now() + ms; while (Date.now() < deadline) { throwPageErrors(errors); @@ -278,7 +298,7 @@ async function measurePlayback(player: Page, playerErrors: BrowserErrors, label: // One sample as a line: elapsed time, the frame on the canvas, the tone step heard against the one // that frame belongs to, how far the tone stood above the noise floor, and whether audio was // arriving at all, which separates a silent player from a publisher that stopped sending. -function traceLine(sample: PlayerState, start: number): string { +function traceLine(sample: Sample, start: number): string { const step = sample.frameId === undefined ? "?" : Pattern.expectedStep(sample.frameId); const margin = sample.toneDb !== undefined && sample.noiseDb !== undefined ? sample.toneDb - sample.noiseDb : 0; return ( @@ -296,7 +316,7 @@ function traceLine(sample: PlayerState, start: number): string { * the tone off the graph root that feeds the speakers. A failure prints every reading in the window, * so the summary can be traced back to the readings behind it. */ -function assertMedia(samples: PlayerState[], label: string): void { +function assertMedia(samples: Sample[], label: string): void { try { measure(samples, label); } catch (err) { @@ -307,7 +327,7 @@ function assertMedia(samples: PlayerState[], label: string): void { } } -function measure(samples: PlayerState[], label: string): void { +function measure(samples: Sample[], label: string): void { check(samples.length >= 10, "sampling", () => `${label}: only ${samples.length} samples in the window`); const readable = samples.filter((s) => s.frameId !== undefined); @@ -373,67 +393,45 @@ function measure(samples: PlayerState[], label: string): void { ); } -/** Probe every platform API the player needs, rather than assuming this engine has them. */ -async function capabilities(page: Page): Promise { - // The fixture's own configuration, so the probe answers "can this browser run this fixture" - // rather than "can it run something like it". - const wanted = { sampleRate: SAMPLE_RATE, width: Pattern.WIDTH, height: Pattern.HEIGHT }; - const found = await inspect( - page, - async (wanted) => { - const has = (name: string) => typeof (globalThis as Record)[name] === "function"; - const probe = async (fn: () => Promise<{ supported?: boolean }>) => { - try { - return (await fn()).supported === true; - } catch { - return false; - } +// The capability page has its own context. Evaluation may grant activation there, +// but cannot unlock the measured fixture or player's audio graph. +const observed = new Map(); +async function capabilities(browser: Browser, engine: Engine, role: Role): Promise { + let record = observed.get(engine); + if (!record) { + const page = await browser.newPage(); + try { + await page.route("**/*", (route) => + route.fulfill({ contentType: "text/html", body: "Capabilities" }), + ); + await page.goto(server.origin); + record = { + version: browser.version(), + found: await page.evaluate(probe, { + sampleRate: SAMPLE_RATE, + width: Pattern.WIDTH, + height: Pattern.HEIGHT, + }), }; - return { - WebTransport: has("WebTransport"), - VideoDecoder: has("VideoDecoder"), - VideoEncoder: has("VideoEncoder"), - AudioDecoder: has("AudioDecoder"), - AudioEncoder: has("AudioEncoder"), - AudioWorkletNode: has("AudioWorkletNode"), - MediaStreamTrackProcessor: has("MediaStreamTrackProcessor"), - "canvas.captureStream": typeof HTMLCanvasElement.prototype.captureStream === "function", - "encode avc1.42001f": await probe(() => - VideoEncoder.isConfigSupported({ - codec: "avc1.42001f", - width: wanted.width, - height: wanted.height, - }), - ), - "encode opus": await probe(() => - AudioEncoder.isConfigSupported({ - codec: "opus", - sampleRate: wanted.sampleRate, - numberOfChannels: 1, - }), - ), - "decode opus": await probe(() => - AudioDecoder.isConfigSupported({ - codec: "opus", - sampleRate: wanted.sampleRate, - numberOfChannels: 1, - }), - ), - }; - }, - wanted, - ); - - for (const [name, ok] of Object.entries(found)) console.error(` ${ok ? "yes" : "NO "} ${name}`); - - const missing = Object.entries(found) - .filter(([, ok]) => !ok) - .map(([name]) => name); - const version = page.context().browser()?.version() ?? "unknown"; + observed.set(engine, record); + } finally { + await page.close(); + } + console.error(`=== engine capabilities: ${engine} ${record.version} ===`); + for (const [name, ok] of Object.entries(record.found)) console.error(` ${ok ? "yes" : "NO "} ${name}`); + console.error( + ` engine matrix: ${JSON.stringify({ engine, version: record.version, publisherMissing: missing(record.found, "publisher"), playerMissing: missing(record.found, "player") })}`, + ); + if (engine === "webkit") + console.error( + " Playwright WebKit is not Safari; Safari transport and device defects still need a manual run", + ); + } + const absent = missing(record.found, role); check( - missing.length === 0, + absent.length === 0, "capabilities", - () => `chromium ${version} lacks ${missing.join(", ")}; these checks cannot run against it`, + () => `${engine} ${record.version} cannot run ${role}: missing ${absent.join(", ")}`, ); } @@ -443,9 +441,15 @@ const browsers: Browser[] = []; // One browser per role. Chromium's default autoplay behavior varies with engagement and when the // graph is built, so require a gesture in this QA run. Pages in one browser share a renderer // scheduler, and the one that is not frontmost is throttled and reported hidden. -async function browserFor(args: string[] = []): Promise { - const browser = await launch(["--autoplay-policy=document-user-activation-required", ...args]); +async function browserFor(role: Role, args: string[] = []): Promise { + const engine = role === "publisher" ? "chromium" : playerEngine; + console.error(`=== launching ${engine} ${role} ===`); + const browser = await launch( + engine === "chromium" ? ["--autoplay-policy=document-user-activation-required", ...args] : [], + engine, + ); browsers.push(browser); + await capabilities(browser, engine, role); return browser; } @@ -466,9 +470,21 @@ async function subscriber( ): Promise<[Page, BrowserErrors]> { // visible="always" because the window is never frontmost in a headless run, and the default // policy would stop downloading video and leave the canvas black. - const params: Record = { url: relay, broadcast, visible: "always", muted: String(muted) }; + const address = new URL(relay); + if (playerEngine !== "chromium") address.protocol = address.protocol === "https:" ? "wss:" : "ws:"; + const params: Record = { + url: address.toString(), + broadcast, + visible: "always", + muted: String(muted), + }; if (delay !== undefined) params.delay = delay; - const [page, errors] = await open(await browserFor(), pageUrl(server.origin, "subscribe", params), label, trace); + const [page, errors] = await open( + await browserFor("player"), + pageUrl(server.origin, "subscribe", params), + label, + trace, + ); await waitForWatch(page); return [page, errors]; } @@ -491,7 +507,7 @@ async function captureDenial(broadcast: string): Promise { console.error("=== capture denial and grant ==="); const [viewer, viewerErrors] = await subscriber(broadcast, "capture viewer", { muted: true }); const [publisher, publisherErrors] = await open( - await browserFor(["--use-fake-device-for-media-stream"]), + await browserFor("publisher", ["--use-fake-device-for-media-stream"]), pageUrl(server.origin, "publish", { url: relay, broadcast }), "capture publisher", false, @@ -565,14 +581,11 @@ try { // ── publisher ──────────────────────────────────────────────────────────── const [publisher, publisherErrors] = await open( - await browserFor(), + await browserFor("publisher"), pageUrl(server.origin, "fixture", { url: relay, broadcast, fault }), "fixture", ); - console.error("=== capabilities ==="); - await capabilities(publisher); - console.error("=== publisher readiness ==="); const beforePublisherGesture = await readFixtureState(publisher); check( @@ -854,7 +867,9 @@ if (expectFail !== undefined) { console.error(`FAIL ${failure.message}`); code = 1; } else { - console.error("media: all checks passed"); + console.error( + `media: ${playerEngine} player over ${playerEngine === "chromium" ? "automatic transport selection" : "WebSocket fallback"} measured; all checks passed (Chromium publisher)`, + ); } await finishTraces(code !== 0); diff --git a/test/interop/clients/js/src/capabilities.test.ts b/test/interop/clients/js/src/capabilities.test.ts new file mode 100644 index 00000000000..37b3c889ddd --- /dev/null +++ b/test/interop/clients/js/src/capabilities.test.ts @@ -0,0 +1,22 @@ +import { expect, test } from "bun:test"; +import { type Capabilities, missing } from "./capabilities"; + +const player: Capabilities = { + WebSocket: true, + VideoDecoder: true, + AudioDecoder: true, + AudioWorkletNode: true, + "decode avc1.42001f": true, + "decode opus": true, +}; + +test("a fallback player does not require the publisher's capture or encoder APIs", () => { + expect(missing(player, "player")).toEqual([]); + expect(missing(player, "publisher")).toContain("MediaStreamTrackProcessor"); + expect(missing(player, "publisher")).toContain("WebTransport"); +}); + +test("unsupported native decode refuses the player with the missing API named", () => { + expect(missing({ ...player, "decode opus": false }, "player")).toEqual(["decode opus"]); + expect(missing({ ...player, VideoDecoder: false }, "player")).toEqual(["VideoDecoder"]); +}); diff --git a/test/interop/clients/js/src/capabilities.ts b/test/interop/clients/js/src/capabilities.ts new file mode 100644 index 00000000000..b5b410bb57f --- /dev/null +++ b/test/interop/clients/js/src/capabilities.ts @@ -0,0 +1,64 @@ +/** Native browser capabilities, kept separate from publisher and player requirements. @module */ + +/** Which side of the media fixture an engine runs. */ +export type Role = "publisher" | "player"; + +/** Every native capability the media harness observes. */ +export type Capabilities = Record; + +/** Probe native APIs on a secure page without importing the media implementation. */ +export async function probe(wanted: { sampleRate: number; width: number; height: number }): Promise { + const has = (name: string) => typeof (globalThis as unknown as Record)[name] === "function"; + const supports = async (fn: () => Promise<{ supported?: boolean }>) => { + try { + return (await fn()).supported === true; + } catch { + return false; + } + }; + return { + WebTransport: has("WebTransport"), + WebSocket: has("WebSocket"), + VideoDecoder: has("VideoDecoder"), + VideoEncoder: has("VideoEncoder"), + AudioDecoder: has("AudioDecoder"), + AudioEncoder: has("AudioEncoder"), + AudioWorkletNode: has("AudioWorkletNode"), + MediaStreamTrackProcessor: has("MediaStreamTrackProcessor"), + "canvas.captureStream": typeof HTMLCanvasElement.prototype.captureStream === "function", + "encode avc1.42001f": await supports(() => + VideoEncoder.isConfigSupported({ codec: "avc1.42001f", width: wanted.width, height: wanted.height }), + ), + "decode avc1.42001f": await supports(() => + VideoDecoder.isConfigSupported({ + codec: "avc1.42001f", + codedWidth: wanted.width, + codedHeight: wanted.height, + }), + ), + "encode opus": await supports(() => + AudioEncoder.isConfigSupported({ codec: "opus", sampleRate: wanted.sampleRate, numberOfChannels: 1 }), + ), + "decode opus": await supports(() => + AudioDecoder.isConfigSupported({ codec: "opus", sampleRate: wanted.sampleRate, numberOfChannels: 1 }), + ), + }; +} + +/** Name missing APIs for a role; players can use WebSocket without an encoder or WebTransport. */ +export function missing(found: Capabilities, role: Role): string[] { + const required = + role === "publisher" + ? [ + "WebTransport", + "VideoEncoder", + "AudioEncoder", + "AudioWorkletNode", + "MediaStreamTrackProcessor", + "canvas.captureStream", + "encode avc1.42001f", + "encode opus", + ] + : ["WebSocket", "VideoDecoder", "AudioDecoder", "AudioWorkletNode", "decode avc1.42001f", "decode opus"]; + return required.filter((name) => found[name] !== true); +} diff --git a/test/interop/clients/js/src/contract.ts b/test/interop/clients/js/src/contract.ts index 6f8aa05d3c9..8429eec6e88 100644 --- a/test/interop/clients/js/src/contract.ts +++ b/test/interop/clients/js/src/contract.ts @@ -145,6 +145,11 @@ export type Sample = { /** Root mean square of the graph root's output over the analyser window. */ rms?: number; + /** `aria-label` of the primary control button: "Pause" while playing, "Play" while paused. */ + controlLabel?: string; + /** Whether the big center play button is showing. */ + centerPlayVisible: boolean; + /** Whether the player is paused. */ paused: boolean; /** Whether the `paused` attribute is reflected onto the element. */ @@ -213,3 +218,6 @@ export const CONTROL = "__moqInterop"; export function publish(commands: Partial): void { (window as unknown as Record)[CONTROL] = commands; } + +/** Prefix of the console line carrying each player {@link Sample} to the driver. */ +export const SAMPLE_EVENT = "__moqInteropSample:"; diff --git a/test/interop/clients/js/src/probe.ts b/test/interop/clients/js/src/probe.ts index b6e3c5cca60..7667dcd9881 100644 --- a/test/interop/clients/js/src/probe.ts +++ b/test/interop/clients/js/src/probe.ts @@ -7,14 +7,14 @@ * Both are sampled in the same tick, which is what makes their difference a synchronization * measurement rather than two unrelated timelines. * - * Everything lands on the element's dataset. Playwright may evaluate in a different JavaScript - * world, where DOM nodes are shared but custom-element instance fields are not, so a dataset string - * is the only reading a driver can trust. + * Each sample is logged to the console for the driver. Reading the page instead would mean + * evaluating in it, and Playwright evaluation grants user activation, which would invalidate the + * gesture checks. * * @module */ import type MoqWatch from "@moq/watch/element"; -import { AUDIBLE_RMS, SAMPLE_MS, type Sample, TONE_FLOOR_DB } from "./contract"; +import { AUDIBLE_RMS, SAMPLE_EVENT, SAMPLE_MS, type Sample, TONE_FLOOR_DB } from "./contract"; import { resources } from "./instrument"; import * as Pattern from "./pattern"; @@ -100,7 +100,7 @@ export function watchResources(): void { } /** - * Start sampling a player onto `el.dataset.interopState`, and mark it ready for a driver to poll. + * Start sampling a player to the console, beginning immediately. * * Returns a function that stops sampling. */ @@ -124,6 +124,9 @@ export function attach(el: MoqWatch): () => void { } } + const ui = document.querySelector("moq-watch-ui")?.shadowRoot; + const control = ui?.querySelector("button.control[aria-label]"); + const centerPlay = ui?.querySelector("button.center-play"); const state: Sample = { seq: seq++, broadcastActive: el.broadcast.out.active.peek() !== undefined, @@ -140,16 +143,17 @@ export function attach(el: MoqWatch): () => void { audioStalled: el.audio.out.stalled.peek(), delay: el.sync.out.delay.peek(), ...(analyser ? analyze(analyser, spectrum, wave) : {}), + controlLabel: control?.getAttribute("aria-label") ?? undefined, + centerPlayVisible: centerPlay ? getComputedStyle(centerPlay).display !== "none" : false, paused: el.paused, pausedAttribute: el.hasAttribute("paused"), resources: resources(), }; - el.dataset.interopState = JSON.stringify(state); + console.debug(SAMPLE_EVENT + JSON.stringify(state)); }; sample(); - el.dataset.interopReady = ""; const timer = self.setInterval(sample, SAMPLE_MS); return () => self.clearInterval(timer); diff --git a/test/interop/interop.sh b/test/interop/interop.sh index c424835fa4d..96e953bf077 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -60,6 +60,7 @@ URL="" KEY="" # the HMAC key every cell's token is signed with (set once the relay's auth server starts) NEGATIVE=0 MEDIA=0 +MEDIA_ENGINES="chromium" TAIL_ONLY=0 # Cargo profile for the relay/cli/moq-c builds. Debug compiles faster, which is @@ -109,6 +110,12 @@ while [[ $# -gt 0 ]]; do MEDIA=1 shift ;; + --media-engines) + require_value "$@" + MEDIA_ENGINES="$2" + MEDIA=1 + shift 2 + ;; # The full matrix. --timeout 30 gives headless Chromium cold-start # headroom; flags after it still override. --all) @@ -159,6 +166,12 @@ if [[ "$MEDIA" -eq 1 ]]; then SUBSCRIBERS="js" fi +if [[ ! "$MEDIA_ENGINES" =~ ^(chromium|firefox|webkit)(,(chromium|firefox|webkit))*$ ]]; then + echo "error: media engines must be a comma-separated list of chromium,firefox,webkit" >&2 + exit 2 +fi +IFS=',' read -r -a MEDIA_ENGINE_LIST <<<"$MEDIA_ENGINES" + IFS=',' read -r -a PUB_LIST <<<"$PUBLISHERS" IFS=',' read -r -a SUB_LIST <<<"$SUBSCRIBERS" @@ -303,8 +316,8 @@ prepare_js() { fi if needs js; then # Nix provides Chromium via PLAYWRIGHT_BROWSERS_PATH; otherwise fetch it. - if [[ -z "${PLAYWRIGHT_BROWSERS_PATH:-}" ]] && ! (cd "$CLIENTS/js" && bunx playwright install chromium) >"$HARNESS_RUN/js-chromium.log" 2>&1; then - mark_broken js "playwright chromium install failed" + if [[ -z "${PLAYWRIGHT_BROWSERS_PATH:-}" ]] && ! (cd "$CLIENTS/js" && bunx playwright install chromium "${MEDIA_ENGINE_LIST[@]}") >"$HARNESS_RUN/js-chromium.log" 2>&1; then + mark_broken js "requested Playwright engine install failed" sed 's/^/ /' "$HARNESS_RUN/js-chromium.log" >&2 || true elif ! (cd "$CLIENTS/js" && bun run check) >"$HARNESS_RUN/js-check.log" 2>&1; then mark_broken js "type check failed" @@ -1087,12 +1100,14 @@ if [[ "$MEDIA" -eq 1 ]]; then overall=1 else echo "=== media output and lifecycle ===" - run_media "media output + lifecycle" - run_media "control: frozen video" --fault frozen-video --cases none --expect-fail "video progress" - run_media "control: silent audio" --fault silent-audio --cases none --expect-fail "audio tone" - run_media "control: offset audio" --fault audio-offset --cases none --expect-fail "audio/video sync" - run_media "control: leaked session" --leak --cases detach --expect-fail "resource baseline" - run_media "control: lagging latecomer" --lag --cases late-join --expect-fail "late join reaches live" + for engine in "${MEDIA_ENGINE_LIST[@]}"; do + run_media "$engine: media output + lifecycle" --player-engine "$engine" + run_media "$engine: control: frozen video" --player-engine "$engine" --fault frozen-video --cases none --expect-fail "video progress" + run_media "$engine: control: silent audio" --player-engine "$engine" --fault silent-audio --cases none --expect-fail "audio tone" + run_media "$engine: control: offset audio" --player-engine "$engine" --fault audio-offset --cases none --expect-fail "audio/video sync" + run_media "$engine: control: leaked session" --player-engine "$engine" --leak --cases detach --expect-fail "resource baseline" + run_media "$engine: control: lagging latecomer" --player-engine "$engine" --lag --cases late-join --expect-fail "late join reaches live" + done fi elif [[ "$NEGATIVE" -eq 1 ]]; then # Negative control: no publisher. Every subscriber must FAIL (time out with diff --git a/test/justfile b/test/justfile index 4a025c99774..37bc12b2d03 100644 --- a/test/justfile +++ b/test/justfile @@ -67,7 +67,7 @@ interop-negative *args: # republish, late join). Ends with negative controls that inject a frozen picture, silence, a # shifted tone, and a leaked session; each must fail the assertion it names. --timeout 30 gives -# headless Chromium cold-start headroom. +# headless Chromium cold-start headroom. --media-engines selects player engines; the fixture stays Chromium. [positional-arguments] media *args: ./interop/interop.sh --media --timeout 30 "$@"