From 16d6b8df082a832db8a80404d4c9c9d7dfdb1d3c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 01:45:44 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m0/interop-green/replacement-audio.md Quest-Claim: be1c1c2c83afe552 From 2363828bee89adc17dac6c0f4990c5c7f96009fb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 03:43:01 -0800 Subject: [PATCH 2/3] fix(watch): release the audio backlog a ring trailing the clock will skip After a route switch the audio ring re-anchors on the incumbent's retained backlog while the live picture moves the shared clock ahead. The ring skips onto the clock, but the decode gate metered the backlog by the ring's own playhead, one floor and window per worklet state report, so on a starved page the audio stayed stalled past the replacement case's switch bound. The gate now counts from the bound behind the clock when the playhead trails it. The replacement case stays quarantined: its quest is rewritten to the remaining cause, a decoder restart at each skipped group trimming 80 ms of warmup. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 50 ++++++++++++++++++ js/watch/src/audio/buffer.ts | 43 ++++++++++++---- js/watch/src/audio/replay.ts | 13 ++--- quest/m0/interop-green/README.md | 4 +- quest/m0/interop-green/replacement-audio.md | 56 ++++++++++++--------- test/interop/README.md | 6 +-- test/interop/clients/js/media.ts | 2 +- 7 files changed, 128 insertions(+), 46 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index faef9cc57bb..168d0023b61 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -322,6 +322,26 @@ describe("Backpressure", () => { }); } + // A ring re-anchored on a retained backlog trails the clock, which a live picture moved on, and + // skips onto it. Metered by its playhead, the backlog would come one floor and window per state + // report, each skipped as late on arrival. + it("releases at once what a ring trailing the clock past its bound skips", () => { + const gate = new Backpressure(1000, false, Duration.millis(200), Duration.millis(200)); + const signal = new AbortController().signal; + gate.observe(ms(0), false, ms(0)); + for (let t = 0; t < 240; t += 20) gate.inserted(ms(t), ms(t + 20)); + const wait = gate.wait(ms(240), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + // Within the bound the ring stretches onto the clock, playing everything from its playhead. + gate.observe(ms(0), false, ms(200)); + expect(Bun.peek.status(wait)).toBe("pending"); + gate.observe(ms(0), true, ms(2000)); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + // Released up to a floor and window past where the ring skips to. + for (let t = 260; t <= 2020; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(2040), signal))).toBe("pending"); + }); + // The next frame past a hole lies far ahead of the playhead, which never nears it while the ring // drains, so it is released by what the ring holds and the hole plays as silence. it("releases the frame past a hole once the ring runs short", () => { @@ -604,6 +624,36 @@ describe("AudioBuffer steering", () => { } }); + // The gate reads the clock where the ring rendered its reported playhead, on the ring's timeline. + it("releases the backlog of a ring trailing the clock past its bound", () => { + let position = 1000; + const clock = started(); + const { buffer, state, context } = fake(100, { + bound: 200, + clock: { position: () => position, out: { reference: new Signal(0) } }, + }); + const signal = new AbortController().signal; + try { + context.currentTime = 1; + context.sounding = 1; + buffer.insert(ms(1000), [new Float32Array(160)]); + state(ms(1000), false, 0, 1); + const wait = buffer.wait(ms(1160), signal); + expect(Bun.peek.status(wait)).toBe("pending"); + position = 1200; + state(ms(1000), false, 0, 1); + expect(Bun.peek.status(wait)).toBe("pending"); + position = 3000; + state(ms(1000), true, 0, 1); + expect(Bun.peek.status(wait)).toBe("fulfilled"); + expect(Bun.peek.status(buffer.wait(ms(2800), signal))).toBe("fulfilled"); + expect(Bun.peek.status(buffer.wait(ms(2950), signal))).toBe("pending"); + } finally { + buffer.close(); + clock.close(); + } + }); + // Before media zero's turn, as while an opening waits for its output to sound it, the clock pairs // at its negative position rather than the zero it reads as. it("pairs a clock short of media zero at its negative position", () => { diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index bc1ab3b5c05..3f7e77a29f1 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -31,7 +31,9 @@ const JUMP = 10_000; * full window to stretch. The frame's own timestamp does not matter: across a hole the next * frame lies far ahead, and the ring plays the hole as silence. A stall keeps releasing by the * playhead: the ring resumes on a cushion and slows down to rebuild, and media past the floor - * stays encoded, counted toward the level the ring decides on. + * stays encoded, counted toward the level the ring decides on. A playhead trailing the shared + * clock past the ring's bound counts from the bound behind the clock instead, where the ring + * skips to, so a backlog behind it goes at once rather than a window per state report. * * The playhead comes from the worklet's state messages. * @@ -54,6 +56,10 @@ export class Backpressure { // The ring as last observed. The playhead is undefined until one is known on the current timeline. #playhead?: number; + // Where the shared clock stood as the ring played the playhead, unset without a clock. + #clock?: number; + // How far the playhead may trail the clock before the ring skips onto it. + #bound = WINDOW; // Whether the ring has played on the current timeline. #started = false; @@ -75,10 +81,10 @@ export class Backpressure { #waiters: Array<{ timestamp: number; resolve: () => void }> = []; - constructor(rate: number, buffered: boolean, delay: Duration) { + constructor(rate: number, buffered: boolean, delay: Duration, bound = Duration.ZERO) { this.#rate = rate; this.#buffered = buffered; - this.setLatency(delay); + this.setLatency(delay, bound); } /** The ring's floor in samples: the part of the target held as decoded PCM. */ @@ -91,17 +97,25 @@ export class Backpressure { return this.#offset; } - /** Split `delay` between the floor and the hold. */ - setLatency(delay: Duration): void { + /** + * Split `delay` between the floor and the hold, and note how far audio may trail the clock before + * the ring skips onto it, at least a stretch window as the ring's own threshold is. + */ + setLatency(delay: Duration, bound = Duration.ZERO): void { + this.#bound = Math.max(bound.asMicros(), WINDOW); this.#floor = ringSamples(this.#rate, delay); this.#headroom = (this.#floor / this.#rate) * 1_000_000; this.#hold = ringHold(delay).asMicros(); this.#release(); } - /** Note the ring's playhead and whether it is stalled, and release every frame now due. */ - observe(playhead: number | undefined, stalled: boolean): void { + /** + * Note the ring's playhead, whether it is stalled, and where the shared clock stood as it played + * there, and release every frame now due. + */ + observe(playhead: number | undefined, stalled: boolean, clock?: number): void { this.#playhead = playhead; + this.#clock = clock; if (playhead !== undefined && !stalled) this.#started = true; this.#release(); } @@ -176,6 +190,7 @@ export class Backpressure { /** The ring was flushed for a new timeline: its playhead is unknown until the next frame anchors it. */ reset(): void { this.#playhead = undefined; + this.#clock = undefined; this.#started = false; this.#decoding = []; this.#inserted = undefined; @@ -214,7 +229,11 @@ export class Backpressure { #due(timestamp: number): boolean { const playhead = this.#playhead; - if (this.#started && playhead !== undefined) return this.#reach(playhead) - playhead < this.#headroom + WINDOW; + if (this.#started && playhead !== undefined) { + // The ring skips audio trailing the clock past its bound, so it plays on from there. + const from = this.#clock === undefined ? playhead : Math.max(playhead, this.#clock - this.#bound); + return this.#reach(from) - from < this.#headroom + WINDOW; + } if (!this.#covered(timestamp)) return false; // A live ring holds no more than the hold leaves uncovered, so it fills from everything due. A @@ -345,7 +364,7 @@ export class AudioBuffer { // The output latency the context already reports, so startup holds only what the ring must. this.#observeOutput(); const held = this.held(props.delay); - this.#backpressure = new Backpressure(rate, buffered, held); + this.#backpressure = new Backpressure(rate, buffered, held, this.#bound); const msg: Init = { type: "init", @@ -375,7 +394,6 @@ export class AudioBuffer { this.#accelerated.set(Duration.millis(data.accelerated)); this.#silent.set(data.silent); if (data.epoch === this.#epoch) { - this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled); if (clock) { const msg: Clock = { type: "clock", @@ -383,6 +401,9 @@ export class AudioBuffer { }; this.#worklet.port.postMessage(msg); } + // The clock as the ring played the reported playhead, on the ring's timeline. + const at = this.#offset === undefined ? undefined : this.#offset + data.time * 1_000_000; + this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled, at); } } }); @@ -505,7 +526,7 @@ export class AudioBuffer { */ setLatency(delay: Duration, bound: Duration = this.#bound): void { this.#bound = bound; - this.#backpressure.setLatency(this.held(delay)); + this.#backpressure.setLatency(this.held(delay), bound); // The clock moves with the delay, so its offset starts over, from a reading sent with the terms. this.#offset = undefined; const sync = this.#sync; diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index faf92d34c63..29a02c9dfcc 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -289,13 +289,14 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato await settle(); let delay = sync.out.delay.peek(); - const terms = () => ({ - bound: bound({ delay, maxDelay: sync.out.maxDelay.peek(), stretch }).asMillis(), - }); + const skip = () => bound({ delay, maxDelay: sync.out.maxDelay.peek(), stretch }); + const terms = () => ({ bound: skip().asMillis() }); // The decode gate, and the worklet's ring on the floor it sets, the way `AudioBuffer` drives both. - const gate = new Backpressure(rate, buffered, delay); + const gate = new Backpressure(rate, buffered, delay, skip()); const ring = new AudioRingBuffer({ rate, channels: 1, latency: millis(rate, gate.floor), ...terms() }); - const observe = () => gate.observe(ring.anchored ? ring.timestamp : undefined, ring.stalled); + // The ring's playhead is where the shared clock stands at the end of the quantum it read. + const observe = () => + gate.observe(ring.anchored ? ring.timestamp : undefined, ring.stalled, sync.now()?.asMicros()); // Arrived but not yet decoded media, read or still in the consumer, is what covers the part of the // target held encoded before playback starts. @@ -412,7 +413,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato if (!resolved.eq(delay)) { delay = resolved; observe(); - gate.setLatency(delay); + gate.setLatency(delay, skip()); ring.setLatency(millis(rate, gate.floor), terms()); } diff --git a/quest/m0/interop-green/README.md b/quest/m0/interop-green/README.md index 3f95fab0f09..daf75be7135 100644 --- a/quest/m0/interop-green/README.md +++ b/quest/m0/interop-green/README.md @@ -223,9 +223,11 @@ Scheduled runs `skipping missing groups` on the audio track before the stall, so the audio stalls past the 1039 ms bound after the route falls back. Reproduced at 1 of 8 and 2 of 12 loaded local runs (the audio ring still stalled 600 ms after the first - audio group of the returned incumbent); not root-caused. Decided 2026-11-15 (AFK, recommendation, for review): + audio group of the returned incumbent). Decided 2026-11-15 (AFK, recommendation, for review): `replacement` is quarantined by default on every engine again, and the child [replacement audio](/quest/m0/interop-green/replacement-audio.md) root-causes and restores it. + Its first PR found the stall: the decode gate released the backlog behind the clock one window + per worklet report. What keeps the quarantine is in the child. Public API: none. Wire: none. diff --git a/quest/m0/interop-green/replacement-audio.md b/quest/m0/interop-green/replacement-audio.md index d1055bf5be8..69410ba768e 100644 --- a/quest/m0/interop-green/replacement-audio.md +++ b/quest/m0/interop-green/replacement-audio.md @@ -1,4 +1,4 @@ -# [M] Restore the replacement media case: audio after a route switch +# [M] Restore the replacement media case: audio across recurring gaps ## Goal @@ -8,28 +8,36 @@ default media run runs it again. Remove its quarantine in `test/interop/clients/ ## Plan -Evidence from the quest's PR, with the case at `ce616b52c` and `cbb742cec` on a loaded host -(`quest gate just test media --media-cases replacement` in a loop, with and without 24 busy -loops): it failed 1 of 8, then 2 of 12 runs, and none of 31 runs after a temporary `console.warn` -per decoded audio frame, so it is a timing race. Each failure is `audioStalled:true` in -`live incumbent fallback plays` (lite-06, lite-07) or `live replacement plays`, with the picture -painting. - -One failing lite-07 fallback, timed from the log: - -- 238 ms after the newer publisher stopped the incumbent's announce returned; its audio - subscription was ok at 298 ms. The incumbent's `fetch error: ... group=62 error=group is past - the budget` and `skipping missing groups: track=audio 0 -> 69` follow, so the first audio group - the reader got was 69, about a second of retained audio behind the incumbent's live picture - (the stalled ring's `audioTimestamp` 9507 ms against a picture at 10548 ms). -- 600 ms after that first group, at the 1039 ms bound, the ring was still stalled. - -Open: where those 600 ms go. The decoder resets the ring on a changed instance -(`js/watch/src/audio/decoder.ts` `#runDecoder`), so the join snaps onto the clock; the question is -whether the ring waits for audio at the clock behind a backlog it must drop (then a join at the -live edge is the fix), or the starved page just decodes late (then the bound needs a load term -derived from the host, not a constant). Trace the ring's join and un-stall in the worklet on a -failing run; the largest-group replay at tune-in -([moq-dev/moq#8844](https://github.com/moq-dev/moq/pull/8844)) may overlap. +The switch itself is fixed: the quest's PR found that after a fallback or replacement the audio +waited behind the incumbent's retained backlog, which the live picture had moved the shared clock +past, and the decode gate released that backlog one floor and window per worklet state report. +The gate now counts from where the ring skips to. Measured on a loaded host, the two builds +interleaved: before, 15 of 161 switches took 600 ms or more and one missed the 1039 ms bound; after, +1 of 165 did and none missed it. + +What still fails under that load, at about one run in ten either way: + +- A mid-stream gap restarts the audio codec and trims the catalog `warmup` (80 ms of Opus) from the + first output after it (`js/watch/src/audio/terminal.ts` `update`, `rs/moq-audio/src/decode/consumer.rs` + `warmup_pending`). One lost 20 ms group costs 100 ms of audio, and when the starved viewer skips a + group more often than every 80 ms, as it does on a backlog read behind the clock, no output + survives at all: one failing lite-06 fallback stayed silent (`audioTimestamp` 0) past its bound + with every decoded output trimmed to zero frames. About 40% of the steady-state stalls in failing + `audio tone` windows followed such a trim. +- The rest are underruns of the case's fixed 50 ms delay while the host is near 0% idle. The default + run's other cases fail the same way at that load (`audio rendition switch back`, 1 of 4). + +Recommended for the gap (a review can undo it): re-prime the restarted decoder on the encoded frames +just before the gap, restamped to end where the first frame after it starts, and drop their output, +so the warmup trims pre-roll rather than wanted audio. Chrome stamps an `AudioDecoder`'s output from +its first input, so the pre-roll must carry those restamped timestamps. moq-audio does the same, so +the native player keeps parity. Rejected: dropping the warmup trim after a gap reverses +[moq-dev/moq#5569](https://github.com/moq-dev/moq/pull/5569)'s decision to play only clean +output; keeping the decoder across a gap and restamping its output by input order assumes one +output per input, which AAC priming breaks. + +Then rerun the replacement case on a loaded host and lift the quarantine if what fails is only the +starvation its sibling cases share. Measure with a loop like the PR's: `taskset` pins the run and +some busy loops to the same cores, so the load stays off other agents' cores. Public API: none. Wire: none. diff --git a/test/interop/README.md b/test/interop/README.md index 5927bc14502..84b1c7cf582 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -551,9 +551,9 @@ playback delay, and one page sample plus one driver poll. Publisher preparation is not part of it. The replacement case is quarantined from the default run on every engine: under CPU -load it intermittently leaves the audio stalled past the switch bound after a route -switch or fallback (the viewer logs `skipping missing groups` on the audio track first), -and Firefox also showed intermittent missing tone samples. Run it explicitly with +load its audio intermittently goes silent, as each group the viewer skips restarts the +decoder and trims its 80 ms warmup, and Firefox also showed intermittent missing tone +samples. Run it explicitly with `just test media --media-cases replacement`, or `--media-cases replacement --media-engines firefox` for Firefox; its content, sync, switch, and fallback assertions remain active. diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index 74eaa6210af..3fe05fe741a 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -163,7 +163,7 @@ const playerEngine = requestedPlayer as Engine; const wants = (name: Case) => selected.has(name); if (!native && values.cases === undefined) { console.error( - "QUARANTINE\treplacement\tintermittent audio stall after a route switch under CPU load; see quest/m0/interop-green/README.md", + "QUARANTINE\treplacement\tintermittent audio silence after skipped groups under CPU load; see quest/m0/interop-green/replacement-audio.md", ); } From ef3b7ab8c19c05774b681f4a4e6e138126f52ac4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 16 Nov 2026 12:46:09 -0800 Subject: [PATCH 3/3] fix(watch): release the backlog past the clock the ring skips onto The ring skips a trailing playhead all the way onto the clock, not to the bound behind it, so a floor short of the bound left the skip landing on an empty ring that stalled and drained one window per report again. Count from the clock once the playhead trails it past the bound. The two Opus relay-mic replays catch up faster at startup and settle a fraction of a millisecond apart; their lag budgets are re-measured. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 24 +++++++++++++++++++----- js/watch/src/audio/buffer.ts | 8 +++++--- test/audio-quality/budgets.json | 6 +++--- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 168d0023b61..f777dd1ecd8 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -337,9 +337,22 @@ describe("Backpressure", () => { expect(Bun.peek.status(wait)).toBe("pending"); gate.observe(ms(0), true, ms(2000)); expect(Bun.peek.status(wait)).toBe("fulfilled"); - // Released up to a floor and window past where the ring skips to. - for (let t = 260; t <= 2020; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); - expect(Bun.peek.status(gate.wait(ms(2040), signal))).toBe("pending"); + // Released up to a floor and window past the clock, where the ring skips to. + for (let t = 260; t <= 2220; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(2240), signal))).toBe("pending"); + }); + + // The ring skips all the way onto the clock, not to the bound behind it, so a floor short of the + // bound must still be released past the clock or the skip lands on an empty ring and stalls. + it("releases past the clock with a floor short of the bound", () => { + // At 48 kHz the floor is the 50 ms asked for, not a render quantum's 128 ms at 1 kHz. + const gate = new Backpressure(48_000, false, Duration.millis(50), Duration.millis(200)); + const signal = new AbortController().signal; + gate.observe(ms(0), false, ms(0)); + for (let t = 0; t < 80; t += 20) gate.inserted(ms(t), ms(t + 20)); + gate.observe(ms(0), true, ms(2000)); + for (let t = 80; t <= 2060; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled"); + expect(Bun.peek.status(gate.wait(ms(2080), signal))).toBe("pending"); }); // The next frame past a hole lies far ahead of the playhead, which never nears it while the ring @@ -646,8 +659,9 @@ describe("AudioBuffer steering", () => { position = 3000; state(ms(1000), true, 0, 1); expect(Bun.peek.status(wait)).toBe("fulfilled"); - expect(Bun.peek.status(buffer.wait(ms(2800), signal))).toBe("fulfilled"); - expect(Bun.peek.status(buffer.wait(ms(2950), signal))).toBe("pending"); + // Released up to a floor and window past the clock, where the ring skips to. + for (const t of [2800, 2950, 3100]) expect(Bun.peek.status(buffer.wait(ms(t), signal))).toBe("fulfilled"); + expect(Bun.peek.status(buffer.wait(ms(3250), signal))).toBe("pending"); } finally { buffer.close(); clock.close(); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 3f7e77a29f1..98bf4983e74 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -32,7 +32,7 @@ const JUMP = 10_000; * frame lies far ahead, and the ring plays the hole as silence. A stall keeps releasing by the * playhead: the ring resumes on a cushion and slows down to rebuild, and media past the floor * stays encoded, counted toward the level the ring decides on. A playhead trailing the shared - * clock past the ring's bound counts from the bound behind the clock instead, where the ring + * clock past the ring's bound, stalled or not, counts from the clock instead, where the ring * skips to, so a backlog behind it goes at once rather than a window per state report. * * The playhead comes from the worklet's state messages. @@ -230,8 +230,10 @@ export class Backpressure { #due(timestamp: number): boolean { const playhead = this.#playhead; if (this.#started && playhead !== undefined) { - // The ring skips audio trailing the clock past its bound, so it plays on from there. - const from = this.#clock === undefined ? playhead : Math.max(playhead, this.#clock - this.#bound); + // The ring skips onto the clock once it trails it past the bound, so it plays on from there. + // Within the bound it stretches onto the clock, playing everything from its playhead. + const clock = this.#clock; + const from = clock !== undefined && clock - playhead > this.#bound ? clock : playhead; return this.#reach(from) - from < this.#headroom + WINDOW; } diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 87bf5117bb4..445d4a410fc 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -482,8 +482,8 @@ "stalled_share": 0, "silence_share": 0, "target_ms_p95": 80, - "av_lag_ms_p50": -0.3, - "av_lag_ms_p95": 9 + "av_lag_ms_p50": 0.2, + "av_lag_ms_p95": 7.7 }, { "runtime": "replay", @@ -514,7 +514,7 @@ "stalled_share": 0.092, "silence_share": 0.092, "target_ms_p95": 80, - "av_lag_ms_p50": 1.7, + "av_lag_ms_p50": 2.3, "av_lag_ms_p95": 32.1 } ]