diff --git a/quest/m1/README.md b/quest/m1/README.md index 781f2aee06b..b8419a2473a 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -116,7 +116,6 @@ tracks, so per-group stats stop costing cluster announces. - [Jitter estimator density](/quest/m1/jitter-density.md) - the estimator does O(1) work per frame however densely timestamps are packed, in JS and Rust - [AAC alias classification](/quest/m1/aac-he-alias.md) - every accepted AAC spelling gets the right frame duration from one classification - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles -- [Publish channel count](/quest/m1/publish-audio-channel-count.md) - forcing a channel count on an Audio.Capture stops costing the subscriber gaps of silence, and the media fixture's cold-start audio-tone flake is fixed at its cause - [Signals notify on every structural change](/quest/m1/signals-equality.md) - a renamed `undefined` key or a changed sparse array length notifies, checked by a fast-check model of the notification contract - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [qmux local lifecycle](/quest/m1/qmux-local-lifecycle.md) - an abort blocked on send credit resets at once, and a close behind a backed-up socket still sends APPLICATION_CLOSE diff --git a/quest/m1/publish-audio-channel-count.md b/quest/m1/publish-audio-channel-count.md deleted file mode 100644 index dc6598ac830..00000000000 --- a/quest/m1/publish-audio-channel-count.md +++ /dev/null @@ -1,53 +0,0 @@ -# [M] Forcing a channel count on Audio.Capture does not cost audio - -## Goal - -`new Audio.Capture({ source, channelCount: 1 })` publishes audio a subscriber -plays as smoothly as the same capture without the override, so pinning a -publisher's channel count stops being a way to degrade it. The browser media -fixture can then pin mono again. - -This is also the home for the media fixture's cold-start audio-tone flake -without the pin, which shows the same symptom. Fix it at its cause, not by -lowering the tone floor. - -`channelCount` keeps its meaning, "force the captured channel count". - -## Plan - -What is known: - -- The real-worklet matrix in - `test/interop/clients/js/capture-channel-count.browser.ts` (in - `just test harness`) finds nothing override-specific: no timestamp gaps, - the requested format, and audible output in every channel, on Linux and - macOS. It only asserts each channel is audible at least once. An uncommitted - per-quantum trace (#5270) found one short silent run during startup in - every case, the no-override baseline included, and none after. Under heavy - load the baseline also showed a later run, so a strict per-quantum check - flakes on the source. Tightening the matrix needs a baseline-relative - measure. -- The player stalls. In CI (#5270), the pinned fixture failed the `just test media` - audio-tone check in 2 of 3 runs, against ~5% of unpinned runs (5 of 108, - all cold start). Failures look the same either way: player samples go - `stalled` while received audio bytes keep growing, and the jitter delay jumps - from ~77 ms to 500-700 ms. - -The working hypothesis is that the explicit downmix (`channelCountMode: -"explicit"` in `js/publish/src/audio/capture.ts`) changes something the player -reacts to, such as frame cadence or burstiness, rather than dropping PCM. -Neither half is proven. Measure the -publisher's frame arrival timing pinned vs unpinned before picking a fix, and -check whether the player's re-stall behavior -([watch jitter target](/quest/m1/audio-jitter-target/watch.md)) is the -amplifier. A loaded local host fails the media fixture either way, so compare -on CI or a quiet machine. - -Not covered: an applied `getUserMedia` channelCount constraint also takes the -explicit path, but a destination-node track refuses that constraint, so the -harness cannot exercise it. - -## Related - -- [Watch jitter target](/quest/m1/audio-jitter-target/watch.md) - a ring that runs dry re-stalls and refills to the target -- [Test flakes, round 2](/quest/m1/test-flakes-2/README.md) - load flakes fixed at their cause diff --git a/quest/m1/test-flakes-2/README.md b/quest/m1/test-flakes-2/README.md index fd5086a45a0..7c065240e9d 100644 --- a/quest/m1/test-flakes-2/README.md +++ b/quest/m1/test-flakes-2/README.md @@ -45,4 +45,3 @@ Public API: none. Wire: none. ## Related - [Controlled time](/quest/m1/time/README.md) - the systemic fix these rules ask for: every crate on a controlled clock -- [Publish channel count](/quest/m1/publish-audio-channel-count.md) - tracks the media fixture's cold-start audio-tone flake, a player stall it shares with the mono pin diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index ebf2ee4d16b..0a73fc5b0b9 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -50,6 +50,8 @@ import { type Capabilities, missing, probe, type Role } from "./src/capabilities import { type CaptureState, FAULTS, + LONG_FRAMES, + type LongFrame, leakedPlayerStarted, reachesLive, SAMPLE_MS, @@ -294,7 +296,146 @@ async function measurePlayback(player: Page, playerErrors: BrowserErrors, label: description: `${label}: the audio buffer to fill to the player's delay and play`, predicate: (state) => state.audioContext === "running" && !state.audioStalled, }); - assertMedia(await collect(player, playerErrors, WINDOW_MS), label); + await (await observe(player, playerErrors, label)).grade(); +} + +/** + * Collect one window while the host is watched, so a window that fails `grade`, whether on a + * caller's own transition checks or on {@link assertMedia}, also prints what stalled the pages. + */ +async function observe( + player: Page, + playerErrors: BrowserErrors, + label: string, +): Promise<{ samples: Sample[]; grade: (checks?: () => void) => Promise }> { + const host = watchHost(label); + const start = Date.now(); + const samples = await collect(player, playerErrors, WINDOW_MS); + const bounds = { start, end: Date.now() }; + const grade = async (checks?: () => void) => { + try { + checks?.(); + assertMedia(samples, label); + } catch (err) { + await explainStalls(player, bounds, label, host); + throw err; + } finally { + host?.process.kill(); + } + }; + return { samples, grade }; +} + +/** + * How long a failed window waits for each page to report its long animation frames. Playwright puts + * no timeout on an evaluate, so a page whose main thread never returns would otherwise hold the + * failure, the other page's evidence, the host's and the browser cleanup until the job times out. + */ +const STALL_READ_MS = 2000; + +/** How often the host's busiest processes are read during a window. */ +const HOST_INTERVAL_S = 0.5; + +/** + * Read the host's busiest processes for the length of one window, on Linux, where `top` has a batch + * mode. A page's main thread stopping for hundreds of milliseconds is either the page's own work or a + * starved host, and only the host can say which. + */ +function watchHost( + label: string, +): { process: Bun.Subprocess<"ignore", "pipe", "ignore">; output: Promise } | undefined { + if (process.platform !== "linux") return undefined; + const iterations = Math.ceil(WINDOW_MS / 1000 / HOST_INTERVAL_S) + 1; + // Evidence only: a host without `top` loses this part of a failure's explanation, not the window. + try { + const top = Bun.spawn( + ["top", "-b", "-c", "-w", "512", "-d", String(HOST_INTERVAL_S), "-n", String(iterations)], + { stdin: "ignore", stdout: "pipe", stderr: "ignore" }, + ); + return { process: top, output: new Response(top.stdout).text() }; + } catch (err) { + console.error(` ${label}: host processes unavailable: ${err}`); + return undefined; + } +} + +// A process as `top -c` prints it, cut to what tells the processes apart: the binary, plus the type of +// a Chromium child (renderer, gpu-process, utility), whose binaries all share one name. +function processName(command: string): string { + const [binary = "", ...args] = command.split(/\s+/); + const type = args.find((arg) => arg.startsWith("--type="))?.slice("--type=".length); + const name = binary.split("/").pop() ?? binary; + return type ? `${name}:${type}` : name; +} + +/** + * Print what stopped the pages' main threads during a failed window: each page's long animation + * frames with the scripts that ran in them, and the host's busiest processes. + * + * Evidence only. A page that cannot be read leaves its part out rather than masking the failure. + */ +async function explainStalls( + player: Page, + bounds: { start: number; end: number }, + label: string, + host: ReturnType, +): Promise { + // The window has already failed, so a plain evaluate's user activation can no longer skew it, and + // unlike `inspect` it reads a non-Chromium player too. The bounds come from the driver's clock, + // which the pages' epoch timestamps share, so each page reports on its own and one that cannot be + // read, or does not answer within STALL_READ_MS, leaves the other's evidence intact. + const { start, end } = bounds; + for (const [name, page] of [ + ["player", player], + ["fixture", fixturePage], + ] as const) { + if (!page) continue; + let timer: ReturnType | undefined; + try { + const unanswered = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`no answer within ${STALL_READ_MS}ms, so its main thread is still blocked`)), + STALL_READ_MS, + ); + }); + const frames = await Promise.race([ + page.evaluate( + ({ key, from, to }) => + ((globalThis as unknown as Record)[key] ?? []).filter( + (frame) => frame.at + frame.duration >= from && frame.at <= to, + ), + { key: LONG_FRAMES, from: start, to: end }, + ), + unanswered, + ]); + console.error(` ${label}: ${name} long animation frames: ${frames.length}`); + for (const frame of frames) { + console.error( + ` +${((frame.at - start) / 1000).toFixed(2)}s ${frame.duration}ms (blocking ${frame.blocking}ms) ${frame.scripts}`, + ); + } + } catch (err) { + console.error(` ${label}: ${name} long animation frames unavailable: ${err}`); + } finally { + clearTimeout(timer); + } + } + + if (!host) return; + host.process.kill(); + const snapshots = (await host.output).split(/^top - /m).slice(1); + console.error(` ${label}: busiest host processes every ${HOST_INTERVAL_S}s from the window's start`); + for (const [index, snapshot] of snapshots.entries()) { + const lines = snapshot.split("\n"); + const idle = lines.find((line) => line.startsWith("%Cpu"))?.match(/([\d.]+) id/)?.[1]; + const header = lines.findIndex((line) => /^\s*PID\s/.test(line)); + const busiest = lines + .slice(header + 1, header + 7) + .map((line) => line.trim().split(/\s+/)) + .filter((fields) => fields.length >= 12) + .map((fields) => `${processName(fields.slice(11).join(" "))} ${fields[8]}%`); + console.error(` +${(index * HOST_INTERVAL_S).toFixed(1)}s idle ${idle ?? "?"}%: ${busiest.join(", ")}`); + } } // One sample as a line: elapsed time, the frame on the canvas, the tone step heard against the one @@ -439,6 +580,9 @@ async function capabilities(browser: Browser, engine: Engine, role: Role): Promi } const server = serve(); + +/** The fixture publisher, whose stalls {@link explainStalls} reports beside the player's. */ +let fixturePage: Page | undefined; const browsers: Browser[] = []; // One browser per role. Chromium's default autoplay behavior varies with engagement and when the @@ -592,6 +736,7 @@ try { pageUrl(server.origin, "fixture", { url: relay, broadcast, fault }), "fixture", ); + fixturePage = publisher; console.error("=== publisher readiness ==="); const beforePublisherGesture = await readFixtureState(publisher); @@ -673,25 +818,28 @@ try { console.error("=== codec identity and coded-size changes ==="); const before = await readPlayerState(player); await command(publisher, "changeVideoCodec"); - const changed = await collect(player, playerErrors, WINDOW_MS); + const change = await observe(player, playerErrors, "codec rename"); + const changed = change.samples; const first = changed.findIndex( (sample) => sample.generation === 1 && sample.videoCodec?.startsWith("vp8") && sample.videoTrack !== before.videoTrack, ); - check(first >= 0, "codec rename", () => `no renamed VP8 picture: ${JSON.stringify(changed.at(-1))}`); - check( - changed.slice(first).every((sample) => sample.frameId === undefined || sample.generation === 1), - "old track picture", - () => "a pre-switch picture appeared after the new codec's first picture", - ); - assertMedia(changed, "codec rename"); + await change.grade(() => { + check(first >= 0, "codec rename", () => `no renamed VP8 picture: ${JSON.stringify(changed.at(-1))}`); + check( + changed.slice(first).every((sample) => sample.frameId === undefined || sample.generation === 1), + "old track picture", + () => "a pre-switch picture appeared after the new codec's first picture", + ); + }); const renamed = changed[first].videoTrack; await measurePlayback(player, playerErrors, "after codec rename"); await command(publisher, "resizeVideo"); - const resized = await collect(player, playerErrors, WINDOW_MS); + const resize = await observe(player, playerErrors, "coded-size growth"); + const resized = resize.samples; const grown = resized.findIndex( (sample) => sample.generation === 2 && @@ -700,18 +848,23 @@ try { sample.videoCodedWidth === Pattern.WIDTH * 2 && sample.videoCodedHeight === Pattern.HEIGHT * 2, ); - check(grown >= 0, "coded-size growth", () => `no larger presented picture: ${JSON.stringify(resized.at(-1))}`); - check( - resized.every((sample) => sample.videoTrack === renamed && sample.videoCodec?.startsWith("vp8")), - "resize keeps identity", - () => `VP8 resize changed track: ${JSON.stringify(resized.at(-1))}`, - ); - check( - resized.slice(grown).every((sample) => sample.frameId === undefined || sample.generation === 2), - "old track picture", - () => "a pre-resize picture appeared after the first larger picture", - ); - assertMedia(resized, "coded-size growth"); + await resize.grade(() => { + check( + grown >= 0, + "coded-size growth", + () => `no larger presented picture: ${JSON.stringify(resized.at(-1))}`, + ); + check( + resized.every((sample) => sample.videoTrack === renamed && sample.videoCodec?.startsWith("vp8")), + "resize keeps identity", + () => `VP8 resize changed track: ${JSON.stringify(resized.at(-1))}`, + ); + check( + resized.slice(grown).every((sample) => sample.frameId === undefined || sample.generation === 2), + "old track picture", + () => "a pre-resize picture appeared after the first larger picture", + ); + }); await measurePlayback(player, playerErrors, "after coded-size growth"); } diff --git a/test/interop/clients/js/src/contract.ts b/test/interop/clients/js/src/contract.ts index 01e095c7272..a365fd8f1d2 100644 --- a/test/interop/clients/js/src/contract.ts +++ b/test/interop/clients/js/src/contract.ts @@ -184,6 +184,21 @@ export type Sample = { resources: Resources; }; +/** A main-thread stall a page logged: one long animation frame and the scripts that ran longest in it. */ +export type LongFrame = { + /** When the frame started, in epoch milliseconds, so two pages' logs line up. */ + at: number; + /** How long the frame took, in milliseconds. */ + duration: number; + /** How much of it blocked input, in milliseconds. */ + blocking: number; + /** The longest-running scripts in the frame, or "no script" when the time went elsewhere. */ + scripts: string; +}; + +/** The `window` property each page keeps its {@link LongFrame} log on. */ +export const LONG_FRAMES = "__moqLongFrames"; + /** Whether a changed presented frame belongs to playback within the live bound. */ export function reachesLive(observation: { previous?: number; diff --git a/test/interop/clients/js/src/fixture.ts b/test/interop/clients/js/src/fixture.ts index dfe8e6bdb64..79a28b1bee7 100644 --- a/test/interop/clients/js/src/fixture.ts +++ b/test/interop/clients/js/src/fixture.ts @@ -126,13 +126,12 @@ export class Fixture { // then requires an audio catalog after the click, which covers the capture resuming its own // suspended graph. // - // No sampleRate or channelCount override: the graph already runs at SAMPLE_RATE, and asking - // the capture for one channel puts the AudioWorklet behind an explicit Web Audio downmix that - // made the player stall far more often in CI, with no capture loss found in isolation; see - // /quest/m1/publish-audio-channel-count.md. Publishing the destination node's own format - // costs nothing here, since the tone is the same in both channels. + // Forced to mono, so every media check also covers the explicit Web Audio downmix a requested + // channel count puts in front of the AudioWorklet, the path a macOS microphone takes. The tone + // is the same in both channels, so the downmix changes nothing a check reads. const audioCapture = new Publish.Audio.Capture({ source: { track: audioTrack, kind: "music" }, + channelCount: 1, }); this.#signals.cleanup(() => audioCapture.close()); diff --git a/test/interop/clients/js/src/main.ts b/test/interop/clients/js/src/main.ts index 7254bb7548e..cc9a5331fb5 100644 --- a/test/interop/clients/js/src/main.ts +++ b/test/interop/clients/js/src/main.ts @@ -1,6 +1,7 @@ /** - * Browser entry point: count the platform resources the page holds, register the `@moq/publish` + - * `@moq/watch` custom elements from the workspace, then run the shared role logic. + * Browser entry point: count the platform resources the page holds, log its main-thread stalls, + * register the `@moq/publish` + `@moq/watch` custom elements from the workspace, then run the shared + * role logic. * * The instrumentation import comes first: it wraps constructors, so anything built before it runs * would go uncounted. @@ -8,6 +9,7 @@ * @module */ import "./instrument.ts"; +import "./stalls.ts"; import "@moq/publish/element"; import "@moq/watch/element"; import "@moq/watch/ui"; diff --git a/test/interop/clients/js/src/stalls.ts b/test/interop/clients/js/src/stalls.ts new file mode 100644 index 00000000000..f8ca46627cf --- /dev/null +++ b/test/interop/clients/js/src/stalls.ts @@ -0,0 +1,42 @@ +/** + * Logs every long animation frame the page renders, with the scripts that ran longest in it. + * + * A media window that fails because a page's main thread stopped for hundreds of milliseconds looks + * the same from its samples whatever stopped it: a script of ours, the browser, or a starved host. + * This log is what tells them apart. Importing the module starts it. + * + * @module + */ + +import { LONG_FRAMES, type LongFrame } from "./contract"; + +/** Enough for minutes of a struggling page; the driver only ever reads the last few seconds. */ +const KEEP = 500; + +const log: LongFrame[] = []; +(globalThis as unknown as Record)[LONG_FRAMES] = log; + +type Script = { duration: number; invoker: string; sourceFunctionName: string; sourceURL: string }; +type Entry = PerformanceEntry & { blockingDuration: number; scripts?: Script[] }; + +// Absent outside Chromium, where the log simply stays empty. +if (PerformanceObserver.supportedEntryTypes?.includes("long-animation-frame")) { + new PerformanceObserver((list) => { + for (const entry of list.getEntries() as Entry[]) { + const scripts = [...(entry.scripts ?? [])] + .sort((a, b) => b.duration - a.duration) + .slice(0, 2) + .map( + (s) => + `${s.invoker} ${s.sourceFunctionName || "?"}@${s.sourceURL.split("/").pop() ?? ""} ${Math.round(s.duration)}ms`, + ); + log.push({ + at: performance.timeOrigin + entry.startTime, + duration: Math.round(entry.duration), + blocking: Math.round(entry.blockingDuration), + scripts: scripts.join(", ") || "no script", + }); + if (log.length > KEEP) log.shift(); + } + }).observe({ type: "long-animation-frame", buffered: true }); +}