From 49d37f8e7f2d36ef6e348028d88ab5ac918e72c8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 16:59:28 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m0/interop-green/media-resume-delay.md Quest-Claim: f0e50c8c02d48f6e From 730c3ab36279b181b64e91b20d86acb9b06c9363 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 18:00:22 -0800 Subject: [PATCH 2/4] fix(watch): a resumed rendition continues its arrival estimate A pause or a hidden tab releases the rendition, and the resume subscribed again from the cold-start prior. The first outage after it then took a tenth of the histogram, so one late keyframe group held the shared delay near a second and froze audio and video. Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 9 ++++- js/watch/src/audio/decoder.test.ts | 34 +++++++++++++++- js/watch/src/audio/decoder.ts | 11 ++++-- js/watch/src/video/decoder.test.ts | 62 ++++++++++++++++++++++++++++++ js/watch/src/video/decoder.ts | 16 ++++++-- 5 files changed, 121 insertions(+), 11 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index b80f4e4a880..f884106e5a4 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -586,6 +586,13 @@ interval in progress and any open stall, folds into the continued histogram as if that interval closed at the switch, judged by the outgoing track's join: a switch most often follows an outage, and would otherwise drop the outage's one observation. +A subscription the viewer stops and starts again on the same path, as a pause or +a hidden tab does, continues the same way. The stop says nothing about the path, +as a [declared silence](#declared-silence) says nothing about it. Starting over +would give the first outage after the resume a tenth of the histogram, so one +late flush would hold the target near its own lateness for a half-life. A +continued histogram gives that outage only `1 - FORGET`. + ## At the speaker Audio is placed on the clock where it is heard, not where it is rendered. The @@ -687,7 +694,7 @@ that interval is [stalled](#one-observation-per-stall), whose stall owns them. the work: after 1024 folds the retained weight is about 2e-8, so the cap changes no output an `f64` can represent. -A receiver coming back from a pause is not cold: the evidence that survived +A subscription whose arrivals stop and come back is not cold: the evidence that survived keeps decaying at the base rate. Only the weight the prior regained is replaced quickly, the same way it is at cold start, since it stands for no evidence at all. diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 353b9b88921..f0aa8e02b4e 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -1588,7 +1588,8 @@ async function livePlayback(burst: number) { // Published before the decoder subscribes, since a broadcast with no request handler refuses // an unknown track. let track = producer.createTrack("audio", { timescale: Moq.Time.Timescale.MILLI, maxAge: Container.MAX_AGE }); - const decoder = new Decoder({ source, sync }); + const enabled = new Signal(true); + const decoder = new Decoder({ source, sync, enabled }); const samples = (48_000 * FRAME) / 1000; const output = [new Float32Array(samples), new Float32Array(samples)]; @@ -1601,6 +1602,7 @@ async function livePlayback(burst: number) { const ring = () => worklets.findLast((worklet) => worklet.ring && insert.mock.contexts.includes(worklet.ring)); return { sync, + enabled, error: decoder.out.error, // Lose one whole group without changing the following media timestamps or arrival cadence. // An open fill nobody serves stands in for a session's feed, which owes the group until the @@ -1700,7 +1702,7 @@ it("a missing live group re-primes without clearing decoded PCM or resetting Syn } }); -describe("Decoder across a rename", () => { +describe("Decoder across a rename or a pause", () => { // A codec change renames the audio track, and the replacement's consumer reads the same path. // Clearing the estimate at teardown and restarting it from the cold-start prior dropped the // shared "auto" delay and climbed back as the new track relearned the lateness, so the ring @@ -1734,6 +1736,34 @@ describe("Decoder across a rename", () => { } }); + // A pause stops the rendition and a resume subscribes again on the same path. Restarting its + // estimate from the cold-start prior gave the first outage after the resume a tenth of the + // histogram, so one late burst held the shared delay near a second. + it("keeps the estimate across a pause, so one late burst after the resume does not raise the delay", async () => { + const playback = await livePlayback(1); + try { + // Long enough for the estimate to settle on a path where every frame lands on time. + await playback.run(20_000); + const warm = playback.sync.out.delay.peek(); + + playback.enabled.set(false); + await playback.run(23_000); + playback.enabled.set(true); + // Past the join, so the outage below is measured as the path's. + await playback.run(25_000); + + // One outage: 800ms of frames held back and flushed at once. + playback.burst = 40; + await playback.run(25_800); + playback.burst = 1; + await playback.run(28_000); + + expect(playback.sync.out.delay.peek().asMillis()).toBeLessThan(warm.asMillis() + 100); + } finally { + await playback.close(); + } + }); + // Another path's arrivals say nothing about this one's, so a rendition read from elsewhere // learns its own estimate instead of inheriting the outgoing one. it("does not carry the delay to another path", async () => { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 30041a8cb2a..b8d6368720a 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -150,7 +150,9 @@ type Consumed = { path: Moq.Path.Valid; codec: string; consumer: Container.Consu // What the decoder does with the selected rendition. Only a running rendition counts a stall. type Phase = - | { kind: "stopped" } + // Not decoding. A stop keeps the last consumer, so a rendition resumed after a pause or a mute + // continues its estimate: the listener's stop says nothing about the path. + | { kind: "stopped"; consumer?: Consumed } // Decoding the rendition at `key` into the ring, which an underrun or a stall is counted against. | { kind: "running"; key: Key; consumer?: Consumed } // The source failed to decode, and stays failed until another source is selected. @@ -609,11 +611,11 @@ export class Decoder { // Closing this effect releases both its container reader and track subscription. const failure = new Signal(undefined); const rendition: Key = { path: broadcast.in.name.peek(), id: track }; - // A swap keeps the outgoing consumer, whose estimate the replacement may continue. + // A swap or a resume keeps the outgoing consumer, whose estimate the replacement may continue. this.#phase.set({ kind: "running", key: rendition, - consumer: phase.kind === "running" ? phase.consumer : undefined, + consumer: phase.kind === "failed" ? undefined : phase.consumer, }); let attempt: Effect | undefined; effect.run((inner) => { @@ -1038,8 +1040,9 @@ export class Decoder { // Leave the running phase, failed or stopped; a stop keeps an earlier failure. No consumer // replaces the outgoing one, so its estimate stops holding the buffer open. #retire(failure?: Failure): void { + const phase = this.#phase.peek(); if (failure) this.#phase.set({ kind: "failed", failure }); - else if (this.#phase.peek().kind === "running") this.#phase.set({ kind: "stopped" }); + else if (phase.kind === "running") this.#phase.set({ kind: "stopped", consumer: phase.consumer }); this.#measured.set(undefined); // Its arrivals stop holding the clock's reference. this.#track.reset(); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 1cbaf5147cb..7f99848bf73 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1209,6 +1209,68 @@ it("a renamed video track keeps the delay, so audio neither skips nor goes silen } }); +// A pause releases the rendition and a resume subscribes again on the same path. Restarting its +// arrival estimate from the cold-start prior gave the first outage after the resume a tenth of the +// histogram, so one late keyframe group held the shared delay near a second and froze the picture. +it("a resumed video track keeps its estimate, so one late burst does not raise the delay", async () => { + const clock = manualAt(0); + const playback = await guardedPlayback("legacy", [], [], true); + const track = playback.track as Moq.Track.Producer; + // One frame every 33ms in one-second groups, each written as it is due unless `held`. + const FRAME = 33; + const GOP = 30; + let frame = 0; + let group: Moq.Group.Producer | undefined; + let held = false; + const queued: number[] = []; + const run = async (until: number) => { + while (frame * FRAME <= until) { + await clock.advance(FRAME); + queued.push(frame * FRAME); + frame++; + if (!held) { + for (const media of queued.splice(0)) { + if (media % (FRAME * GOP) === 0) { + group?.finish(); + group = track.createGroup(BigInt(media / (FRAME * GOP))); + } + group?.writeFrame(legacyFrame(Duration.millis(media))); + } + } + await new Promise((resolve) => setImmediate(resolve)); + await microtasks(); + } + }; + + try { + playback.delay.set("auto"); + await microtasks(); + + // Long enough for the estimate to settle on a path where every frame lands on time. + await run(20_000); + const warm = playback.sync.out.delay.peek(); + + playback.enabled.set(false); + await run(23_000); + playback.enabled.set(true); + // Past the join, so the outage below is measured as the path's. + await run(25_000); + + // One outage: the frames from a keyframe on held back for 800ms and flushed at once. + held = true; + await run(25_740 + 800); + held = false; + await run(28_000); + + expect(playback.decoder.out.timestamp.peek()?.asMillis()).toBeGreaterThan(27_000); + expect(playback.sync.out.delay.peek().asMillis()).toBeLessThan(warm.asMillis() + 100); + } finally { + playback.close(); + await microtasks(); + clock.installed.close(); + } +}); + // Another path's arrivals say nothing about this one's, so a rendition read from elsewhere, such as // through a `broadcast` override, learns its own estimate instead of inheriting the outgoing one. it("a video track read from another path does not inherit the outgoing delay", async () => { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index f9afaa1f824..6c4e2a25e53 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -165,6 +165,9 @@ export class Decoder { // The path of the media the last rendition read, and its instance. Every handle to one instance // shares `closed`, so it identifies the instance where the handle itself does not. #instance?: { path: Moq.Path.Valid; closed: unknown }; + // The consumer the last released rendition read, and its path, so a rendition resumed after a pause + // or a hidden tab continues its arrival estimate: the viewer's stop says nothing about the path. + #retired?: { path: Moq.Path.Valid; consumer: Container.Consumer }; readonly #identity: Computed; #signals = new Effect(); @@ -331,10 +334,12 @@ export class Decoder { this.#instance = { path: active.path, closed: active.closed }; if (republished) this.sync.reset(); - // Only a rendition read from the same path continues the outgoing arrival estimate: another - // path's arrivals have nothing to do with what this one learned. + // Only a rendition read from the same path continues the outgoing arrival estimate, or the one + // released last when none is active: another path's arrivals have nothing to do with what this + // one learned. const outgoing = this.#active.peek(); - const predecessor = outgoing?.broadcast.path === active.path ? outgoing.consumer : undefined; + const previous = outgoing ? { path: outgoing.broadcast.path, consumer: outgoing.consumer } : this.#retired; + const predecessor = previous?.path === active.path ? previous.consumer : undefined; const rendition = new Rendition({ sync: this.sync, @@ -405,7 +410,10 @@ export class Decoder { return; } - effect.cleanup(() => active.close()); + effect.cleanup(() => { + if (active.consumer) this.#retired = { path: active.broadcast.path, consumer: active.consumer }; + active.close(); + }); effect.run((inner) => { const error = inner.get(active.error); From 47047feb497338b8ca3338c23f1049b4938a048d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 18:23:31 -0800 Subject: [PATCH 3/4] test(interop): a declared silence before any audio is not playback A resume onto the fixture's on-demand audio first reads the epoch its encoder ended while unwatched, which the player reports as a declared silence rather than a stall. measurePlayback opened its window there and graded the silence until the retained audio arrived. Co-Authored-By: Claude Opus 5.5 --- test/interop/clients/js/driver.ts | 5 ++--- test/interop/clients/js/media.ts | 16 +++++++++------- test/interop/clients/js/src/contract.test.ts | 6 +++++- test/interop/clients/js/src/contract.ts | 16 ++++++++++++---- test/interop/clients/js/tone.test.ts | 1 + test/interop/clients/js/tone.ts | 9 ++++++--- 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/test/interop/clients/js/driver.ts b/test/interop/clients/js/driver.ts index e90b2896842..5dd70198f4c 100644 --- a/test/interop/clients/js/driver.ts +++ b/test/interop/clients/js/driver.ts @@ -38,7 +38,7 @@ import { waitForState, waitForWatch, } from "./harness"; -import { type Sample, TONE_CHECK } from "./src/contract"; +import { audioPlays, type Sample, TONE_CHECK } from "./src/contract"; import { ToneCheck } from "./tone"; /** Remove the page's players, which closes their sessions, and wait until every transport has. */ @@ -192,8 +192,7 @@ async function watch(codec: string): Promise { const state = failedAudio ?? (await readPlayerState(page)); const wrongTone = state.audioCodec?.split(".")[0] === codec && - state.audioContext === "running" && - !state.audioStalled && + audioPlays(state) && state.audioChannels === tones.length && state.tones?.length === tones.length && state.tones.every(({ rms }) => rms >= TONE_CHECK.minRms); diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index 18d0cc8ef09..6759d5e12db 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -62,6 +62,7 @@ import { type Capabilities, capturePath, missing, probe, type Role } from "./src import { ALT_AUDIO, AUDIBLE_RMS, + audioPlays, CHANGED_SAMPLE_RATE, FAULTS, KEYFRAME_INTERVAL_MS, @@ -360,10 +361,9 @@ async function waitForLive( } lag = fixture.frameId - state.frameId; first ??= lag; - audio = - state.audioContext === "running" && !state.audioStalled - ? "playing" - : `${state.audioContext ?? "not built"}${state.audioStalled ? " and buffering" : ""}`; + audio = audioPlays(state) + ? "playing" + : `${state.audioContext ?? "not built"}${state.audioStalled ? " and buffering" : ""}${state.audioTimestamp ? "" : " with nothing anchored"}`; moved = frame !== undefined && state.frameId !== frame; if (moved && !started) { started = true; @@ -394,15 +394,17 @@ async function waitForLive( * * A player shows its first frame as soon as it decodes, then holds that frame silent while it buffers * up to its sync delay, which rises past 400ms under load. A window opened on that held frame counts - * the buffering as missing tone. Only the ring filling opens the window, never a tone, so silence - * still fails {@link assertMedia} rather than this wait. + * the buffering as missing tone. A resume onto the fixture's on-demand audio first reads the epoch + * its encoder ended while unwatched, a declared silence rather than a stall, until the retained audio + * it encodes again arrives. Only the ring playing anchored audio opens the window, never a tone, so + * silence still fails {@link assertMedia} rather than this wait. */ async function measurePlayback(player: Page, playerErrors: BrowserErrors, label: string): Promise { await waitForState(player, playerErrors, { deadline: Date.now() + SETTLE_MS, assertion: "playback starts", description: `${label}: the audio buffer to fill to the player's delay and play`, - predicate: (state) => state.audioContext === "running" && !state.audioStalled, + predicate: audioPlays, }); await (await observe(player, playerErrors, label)).grade(); } diff --git a/test/interop/clients/js/src/contract.test.ts b/test/interop/clients/js/src/contract.test.ts index 4d0efa7c1c5..b5a7bbd7db4 100644 --- a/test/interop/clients/js/src/contract.test.ts +++ b/test/interop/clients/js/src/contract.test.ts @@ -22,9 +22,13 @@ const nearLive = { previous: 183, source: 196, bound: 45, - sample: { frameId: 186, audioContext: "running", audioStalled: false }, + sample: { frameId: 186, audioContext: "running", audioStalled: false, audioTimestamp: 6200 }, }; +test("a declared silence before any audio is not live playback", () => { + expect(reachesLive({ ...nearLive, sample: { ...nearLive.sample, audioTimestamp: 0 } })).toBe(false); +}); + test("an advancing preview while audio buffers is not live playback", () => { expect(reachesLive({ ...nearLive, sample: { ...nearLive.sample, audioStalled: true } })).toBe(false); }); diff --git a/test/interop/clients/js/src/contract.ts b/test/interop/clients/js/src/contract.ts index 3a389885576..ef39173afca 100644 --- a/test/interop/clients/js/src/contract.ts +++ b/test/interop/clients/js/src/contract.ts @@ -163,7 +163,7 @@ export type Sample = { audioBytes: number; /** `AudioContext.state`, absent until the graph exists. */ audioContext?: string; - /** Playback timestamp reported by the render worklet, in milliseconds. */ + /** Playback timestamp reported by the render worklet, in milliseconds, 0 until a frame anchors the ring. */ audioTimestamp?: number; /** Whether the audio buffer is waiting to refill. */ audioStalled: boolean; @@ -229,17 +229,25 @@ export type LongFrame = { /** The `window` property each page keeps its {@link LongFrame} log on. */ export const LONG_FRAMES = "__moqLongFrames"; +/** + * Whether audio plays: the graph runs, the ring is not refilling, and a frame has anchored it. A + * declared silence is not a stall, so a ring that took a publisher's end before any audio, as a + * resume onto an on-demand encoder's ended epoch does, reads unstalled while it holds nothing. + */ +export function audioPlays(sample: Pick): boolean { + return sample.audioContext === "running" && !sample.audioStalled && (sample.audioTimestamp ?? 0) > 0; +} + /** Whether a changed presented frame belongs to playback within the live bound. */ export function reachesLive(observation: { previous?: number; source: number; - sample: Pick; + sample: Pick; bound: number; }): boolean { const { previous, source, sample, bound } = observation; return ( - sample.audioContext === "running" && - !sample.audioStalled && + audioPlays(sample) && previous !== undefined && sample.frameId !== undefined && sample.frameId !== previous && diff --git a/test/interop/clients/js/tone.test.ts b/test/interop/clients/js/tone.test.ts index f33ecad9d88..98a181cc37a 100644 --- a/test/interop/clients/js/tone.test.ts +++ b/test/interop/clients/js/tone.test.ts @@ -5,6 +5,7 @@ const wrong = { at: 0, audioContext: "running", audioStalled: false, + audioTimestamp: 1000, audioCodec: "opus", audioChannels: 2, tones: [ diff --git a/test/interop/clients/js/tone.ts b/test/interop/clients/js/tone.ts index dd401f06463..b5426b32241 100644 --- a/test/interop/clients/js/tone.ts +++ b/test/interop/clients/js/tone.ts @@ -1,9 +1,12 @@ -import { type Sample, TONE_CHECK } from "./src/contract"; +import { audioPlays, type Sample, TONE_CHECK } from "./src/contract"; /** How long rendered audio may play before it has to carry the tone: several analyser windows. */ const GRACE_MS = 3000; -type Observation = Pick; +type Observation = Pick< + Sample, + "at" | "audioContext" | "audioStalled" | "audioTimestamp" | "audioCodec" | "audioChannels" | "tones" +>; /** Judge the expected tones after giving the running player a bounded grace to fill its analyser. */ export class ToneCheck { @@ -17,7 +20,7 @@ export class ToneCheck { /** Wait through buffering, accept the tone, or fail after enough audio has played to judge it. */ read(state: Observation): "waiting" | "heard" | "failed" { - const playing = state.audioContext === "running" && !state.audioStalled; + const playing = audioPlays(state); // Samples use the page's clock, so polling the same observation cannot spend the grace. // Both ends must be playing: a refill or suspension contributes no playback time. if (playing && this.#previous?.playing) this.#played += state.at - this.#previous.at; From 9582607b9e396f10cfc8b463de47a6366757f798 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 18:23:46 -0800 Subject: [PATCH 4/4] chore(quest): finish media-resume-delay Co-Authored-By: Claude Opus 5.5 --- quest/m0/interop-green/media-resume-delay.md | 25 -------------------- quest/m0/nightly/README.md | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 quest/m0/interop-green/media-resume-delay.md diff --git a/quest/m0/interop-green/media-resume-delay.md b/quest/m0/interop-green/media-resume-delay.md deleted file mode 100644 index e50bb8696e0..00000000000 --- a/quest/m0/interop-green/media-resume-delay.md +++ /dev/null @@ -1,25 +0,0 @@ -# [L] A viewer's resume doesn't freeze playback on a late video burst - -## Goal - -The interop `media` step passes under CPU load: after a pause and resume, or a codec rename, audio and video play without a freeze inside the measurement window, in the scheduled Interop run and Nightly's `Media (firefox player)` job. Both have failed every day since 2026-11-12. - -## Plan - -Both fail `FAIL audio tone: after resume: N/81 samples carried the fixture tone` in "media output + lifecycle". The `remote error: 1`/`54`, `stream aborted` and `fetch error … not found: group 10` lines are the expected cancels of pause and rename, also present in passing runs. - -The first bad commit was [moq-dev/moq#8507](https://github.com/moq-dev/moq/pull/8507) (`4262a0691e`; after resume, audio caught up by playing up to 1.6x). That looks fixed by [moq-dev/moq#8675](https://github.com/moq-dev/moq/pull/8675): the pause case alone under load now passes 4/4. - -What remains on `main` fails about 25% under load: after resume, the first frames of a new video group arrive together 400–900 ms late, the jitter estimator (`js/hang/src/container/jitter.ts`) counts that as path delay, and the auto delay jumps from 162 ms to 700–880 ms, so audio and video freeze about 0.5 s. The last good commit showed the same bursts, smaller (180–280 ms), so this may be old flakiness made visible. - -Steps: - -1. Find whether the burst is real (the Chromium publisher's keyframe encode stalling under load) or a re-subscription artifact the estimator should treat as join backlog. Its join rule excludes only frames produced before the first arrival. Log per-group arrival at the publisher, relay and player. -2. Decide what a large auto-delay rise does during playback: today it pads silence and freezes video for the whole step, while #8507 meant the ring to slow onto a rise. Coordinate with [audio playout](/quest/m2/audio-playout/README.md), which owns that rule. -3. Explain the 11-17 CI trace: audio reports timestamp 0 and is silent 0.6 s while video plays, then video freezes and jumps 600 ms. Why does `measurePlayback`'s "playback starts" wait pass while the ring has no audio yet? - -Reproduce from `test/`: `just media --media-cases rename,pause` under `taskset -c 0-3` with two busy loops on the same cores, and `HARNESS_RUNS` set to a short path (pulse.py's Unix socket path is too long from a worktree's `.tmp`). Without load it passes almost every time (12/13). - -Fix the root cause; no retries or looser thresholds. - -Public API: none. Wire: none. diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 97152e99d7c..fccb7229faa 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -86,7 +86,7 @@ Decided 2026-11-18 (maintainer, /quest-plan), from the triage of runs [44342367698](https://github.com/moq-dev/moq/actions/runs/44342367698): four more children (JS resume window, audio-quality p50 ceilings, publisher drop cost, resume mutants). Owned elsewhere: the firefox player by -[media resume delay](/quest/m0/interop-green/media-resume-delay.md), OBS by +[moq-dev/moq#9141](https://github.com/moq-dev/moq/pull/9141), OBS by [moq-dev/moq#9016](https://github.com/moq-dev/moq/pull/9016), and fixes in PRs from the same triage: benches ([moq-dev/moq#9089](https://github.com/moq-dev/moq/pull/9089),