diff --git a/js/watch/src/audio/config.ts b/js/watch/src/audio/config.ts index 9e52ee077c2..9783582d18d 100644 --- a/js/watch/src/audio/config.ts +++ b/js/watch/src/audio/config.ts @@ -78,7 +78,7 @@ export function decoderDescription(codec: string, description: Uint8Array | unde * Undefined for Opus, whose packets state their own duration (see `Util.Opus.packetSamples`), and * for codecs with no constant. Never learned from observed timestamps. */ -export function frameDuration(config: Pick): Duration | undefined { +export function frameDuration(config: { readonly codec: string; readonly sampleRate: number }): Duration | undefined { const samples = frameSamples(config); if (samples === undefined) return undefined; return Duration.millis((samples * 1000) / config.sampleRate); @@ -96,7 +96,7 @@ export function packetDuration(codec: string, frame: Container.Frame): Duration return samples === undefined ? undefined : Duration.millis((samples * 1000) / OPUS_RATE); } -function frameSamples(config: Pick): number | undefined { +function frameSamples(config: { readonly codec: string; readonly sampleRate: number }): number | undefined { switch (Util.Aac.objectType(config.codec)) { case 2: return AAC_LC_FRAME_SAMPLES; diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 62904202848..4e40314484a 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -481,7 +481,7 @@ for (const description of [undefined, "1190"]) { } for (const cmaf of [false, true]) { - it(`a ${cmaf ? "CMAF" : "legacy"} gap releases accounting for codec outputs it discarded`, async () => { + it(`a ${cmaf ? "CMAF" : "legacy"} gap lands the codec outputs still pending before restarting it`, async () => { const clock = new Manual(); const installed = clock.install(); deferAudioOutput = true; @@ -496,15 +496,9 @@ for (const cmaf of [false, true]) { expect(playback.inserted()).toHaveLength(0); playback.push({ kind: "gap", timestamp: Duration.micros(140_000), duration: Duration.micros(20_000) }); await microtasks(); + // A starved decoder can hold everything since the last gap, which a reset alone would drop. expect(pendingAudio).toHaveLength(0); - deferAudioOutput = false; - frameTimestamp = 160_000; - const resumed = frame(); - playback.push({ ...resumed, group: 2n }); - await microtasks(); - expect(audioInputs).toHaveLength(7); - expect(playback.inserted()).toContain(resumed.frame.timestamp.asMicros()); - await playback.play(5); + expect(playback.inserted()).toHaveLength(6); expect(playback.sync.playing.peek()).toBe(true); } finally { playback.close(); @@ -530,6 +524,69 @@ it("decodes an audio packet inside a group after a gap", async () => { } }); +// The warmup the restarted decoder needs is the last frames before the gap, restamped to end where +// the frame after it starts, so it trims their output instead of the frame's. +it("re-primes the decoder on the warmup before a gap, so the frame after it plays", async () => { + const playback = await play(Duration.millis(100)); + try { + playback.restore({ warmup: 80 }); + await microtasks(); + await playback.play(); + const played = audioChunks.length; + const decoded = playback.feedback().audio?.decoded ?? 0; + playback.push({ + kind: "gap", + timestamp: Duration.micros(frameTimestamp + 20_000), + duration: Duration.millis(100), + }); + frameTimestamp += 100_000; + const next = frame(); + playback.push({ ...next, group: 1n }); + await microtasks(); + const at = next.frame.timestamp.asMicros(); + expect(audioChunks.slice(played).map((chunk) => chunk.timestamp)).toEqual([ + at - 80_000, + at - 60_000, + at - 40_000, + at - 20_000, + at, + ]); + expect(playback.inserted()).toContain(at); + // The pre-roll's output is not counted as decoded again; only the frame after the gap is. + expect(playback.feedback().audio?.decoded).toBe(decoded + 1); + } finally { + playback.close(); + await microtasks(); + } +}); + +// AAC frames state no duration, so the codec's frame length (1024 samples) is what the pre-roll covers. +it("re-primes an AAC decoder on frames that state no duration", async () => { + const playback = await play(Duration.millis(100)); + try { + playback.restore({ codec: "mp4a.40.2", sampleRate: 48_000, numberOfChannels: 2, warmup: 80 }); + await microtasks(); + await playback.play(); + const played = audioChunks.length; + playback.push({ + kind: "gap", + timestamp: Duration.micros(frameTimestamp + 20_000), + duration: Duration.millis(100), + }); + frameTimestamp += 100_000; + const next = frame(); + playback.push({ ...next, group: 1n }); + await microtasks(); + // Four 21.3 ms frames cover the 80 ms warmup, ahead of the frame after the gap. + const chunks = audioChunks.slice(played).map((chunk) => chunk.timestamp); + expect(chunks).toHaveLength(5); + expect(chunks.at(-1)).toBe(next.frame.timestamp.asMicros()); + } finally { + playback.close(); + await microtasks(); + } +}); + // A subscription's head arrives newest-first, one group per frame. Chrome's AudioDecoder stamps every // output from its first input, so decoding the older frames after it shifts the rest of the track. for (const cmaf of [false, true]) { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 00bc3ce885f..e4db557dbd4 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -686,8 +686,7 @@ export class Decoder { let written: number | undefined; const decoder = new AudioDecoder({ output: (data) => { - this.#count(key, { decoded: 1 }); - const landed = this.#emit(data); + const landed = this.#emit(data, key); if (landed !== undefined) written = Math.max(written ?? landed, landed); }, error: (error) => failure.set(error), @@ -725,9 +724,7 @@ export class Decoder { await this.#onFinish(effect, decoder, () => written); break; } - if (this.#onRead(next)) { - this.#reprime(decoder, decoderConfig, order); - } + if (!(await this.#apply(effect, decoder, decoderConfig, order, next))) return; continue; } const { frame } = next; @@ -783,8 +780,7 @@ export class Decoder { let written: number | undefined; const decoder = new AudioDecoder({ output: (data) => { - this.#count(key, { decoded: 1 }); - const landed = this.#emit(data); + const landed = this.#emit(data, key); if (landed !== undefined) written = Math.max(written ?? landed, landed); }, error: (error) => failure.set(error), @@ -819,9 +815,7 @@ export class Decoder { await this.#onFinish(effect, decoder, () => written); break; } - if (this.#onRead(next)) { - this.#reprime(decoder, decoderConfig, order); - } + if (!(await this.#apply(effect, decoder, decoderConfig, order, next))) return; continue; } const { frame } = next; @@ -854,12 +848,31 @@ export class Decoder { reads: FrameRead[], ): Promise { for (const read of reads) { - if (this.#onRead(read)) { - this.#reprime(decoder, config, order); - } + if (!(await this.#apply(effect, decoder, config, order, read))) return false; if (!order.admit(read)) continue; - if (!(await this.#decode(effect, decoder, read.frame))) return false; + // A frame stating no duration, such as AAC's, runs the codec's frame length, as queuedRanges + // counts it. + const duration = packetDuration(config.codec, read.frame) ?? frameDuration(config); + if (!(await this.#decode(effect, decoder, read.frame, duration))) return false; + } + return true; + } + + // Apply an ordered read, restarting the codec when it ends an epoch, and report whether to keep + // going. The decoder first hands over what it holds, mapped on the epoch it was decoded in: a reset + // would drop it, and a starved decoder skipping groups can hold everything since the last skip. + async #apply( + effect: Effect, + decoder: AudioDecoder, + config: AudioDecoderConfig, + order: Order, + read: Container.Read, + ): Promise { + if (this.#terminal.ends(read) && decoder.state === "configured") { + await decoder.flush(); + if (effect.abort.aborted) return false; } + if (this.#onRead(read)) this.#reprime(decoder, config, order); return true; } @@ -871,8 +884,14 @@ export class Decoder { order.reset(); } - // Decode one frame once backpressure admits it, reporting whether to keep going. - async #decode(effect: Effect, decoder: AudioDecoder, frame: Container.ReceivedFrame): Promise { + // Decode one frame once backpressure admits it, behind any pre-roll a gap left, reporting whether + // to keep going. + async #decode( + effect: Effect, + decoder: AudioDecoder, + frame: Container.ReceivedFrame, + duration: Duration | undefined, + ): Promise { const ring = await this.#ready(effect); if (!ring) return false; @@ -885,6 +904,9 @@ export class Decoder { // A fatal decode error closes the decoder, so decoding again throws InvalidStateError out // of the loop. Stop instead: the error callback already reported the real failure. if (decoder.state === "closed") return false; + for (const preroll of this.#terminal.preroll()) { + decoder.decode(new EncodedAudioChunk({ type: "key", data: preroll.payload, timestamp: preroll.timestamp })); + } decoder.decode( new EncodedAudioChunk({ // WebCodecs audio chunks are key even inside a MoQ group. @@ -893,6 +915,7 @@ export class Decoder { timestamp: frame.timestamp.asMicros(), }), ); + this.#terminal.decoded(frame, duration); return true; } @@ -1028,17 +1051,21 @@ export class Decoder { this.#handover.declared(end); } - // Place decoded audio, returning where the PCM written ends, in microseconds. - #emit(sample: AudioData): number | undefined { + // Count and place decoded audio, returning where the PCM written ends, in microseconds. + #emit(sample: AudioData, key: Key | undefined): number | undefined { // Whatever becomes of it, the output answers the frames it was decoded from, so the gate counts // only what still decodes. Reported after any insert, so the gate always counts the frame as // either decoding or in the ring. + // Pre-roll output answers no frame: it only primes the decoder, and the warmup trims it. It is + // not counted as decoded either, since the frames it came from already were. + const preroll = this.#terminal.primes(sample); + if (!preroll) this.#count(key, { decoded: 1 }); const decoded = sample.timestamp + (sample.numberOfFrames / sample.sampleRate) * 1_000_000; let landed: number | undefined; try { landed = this.#place(sample); } finally { - this.#ring.peek()?.decoded(decoded, landed); + if (!preroll) this.#ring.peek()?.decoded(decoded, landed); } return landed; } diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index 3eae3beb7d5..73423885d11 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -112,3 +112,106 @@ test("a hole preserves a declared endpoint until a later group starts a new time expect(terminal.update(frame(2n, 1_000_000))).toBe(true); expect(terminal.end).toBeUndefined(); }); + +const packet = (timestamp: number) => ({ + timestamp: Duration.micros(timestamp), + payload: new Uint8Array([timestamp / 20_000]), +}); +const opus = (timestamp: number) => ({ timestamp, sampleRate: 48_000, numberOfFrames: 960 }); + +test("a gap re-primes the decoder on the warmup before it, so the audio after it plays whole", () => { + const terminal = new Terminal(); + terminal.clear(312, 80_000); + for (let i = 0; i < 6; i++) { + terminal.update(frame(0n, i * 20_000)); + terminal.decoded(packet(i * 20_000), Duration.micros(20_000)); + } + expect(terminal.update(gap)).toBe(true); + terminal.update(frame(2n, 1_000_000)); + + // The newest 80 ms, back to back and ending where the frame after the gap starts. + const preroll = terminal.preroll(); + expect(preroll.map((p) => p.timestamp)).toEqual([920_000, 940_000, 960_000, 980_000]); + expect(preroll.map((p) => p.payload[0])).toEqual([2, 3, 4, 5]); + expect(terminal.preroll()).toEqual([]); + + // The pre-roll's output takes the pre-skip and the warmup, and answers no frame. + for (const { timestamp } of preroll) { + expect(terminal.primes(opus(timestamp))).toBe(true); + expect(terminal.span(opus(timestamp)).frames).toBe(0); + } + // The overlap the pre-skip carries over the restart starts before the frame, so only it is trimmed. + expect(terminal.primes(opus(1_000_000))).toBe(false); + expect(terminal.span(opus(1_000_000))).toEqual({ timestamp: 1_000_000, frameOffset: 312, frames: 648 }); + expect(terminal.span(opus(1_020_000))).toEqual({ timestamp: 1_013_500, frameOffset: 0, frames: 960 }); +}); + +// A decoder that coalesces outputs can hand back the pre-roll and the frame after the gap together. +test("output reaching past the pre-roll holds the audio after the gap", () => { + const terminal = new Terminal(); + terminal.clear(0, 80_000); + for (let i = 0; i < 4; i++) { + terminal.update(frame(0n, i * 20_000)); + terminal.decoded(packet(i * 20_000), Duration.micros(20_000)); + } + terminal.update(gap); + terminal.update(frame(1n, 1_000_000)); + expect(terminal.preroll()).toHaveLength(4); + + // 20 ms of pre-roll alone, then the other 60 ms of it with the frame after the gap. + expect(terminal.primes(opus(920_000))).toBe(true); + const coalesced = { timestamp: 940_000, sampleRate: 48_000, numberOfFrames: 4 * 960 }; + expect(terminal.primes(coalesced)).toBe(false); + expect(terminal.span(coalesced)).toEqual({ timestamp: 1_000_000, frameOffset: 3 * 960, frames: 960 }); +}); + +test("gaps closer than the warmup still play, the pre-roll carrying over the last one", () => { + const terminal = new Terminal(); + terminal.clear(0, 80_000); + for (let i = 0; i < 4; i++) { + terminal.update(frame(0n, i * 20_000)); + terminal.decoded(packet(i * 20_000), Duration.micros(20_000)); + } + for (const start of [1_000_000, 2_000_000]) { + terminal.update(gap); + terminal.update(frame(1n, start)); + expect(terminal.preroll().map((p) => p.timestamp)).toEqual([ + start - 80_000, + start - 60_000, + start - 40_000, + start - 20_000, + ]); + terminal.decoded(packet(start), Duration.micros(20_000)); + expect(terminal.span(opus(start)).frames).toBe(960); + } +}); + +test("a short history trims only the warmup it leaves uncovered", () => { + const terminal = new Terminal(); + terminal.clear(0, 80_000); + terminal.update(frame(0n, 0)); + terminal.decoded(packet(0), Duration.micros(20_000)); + terminal.update(frame(0n, 20_000)); + // A frame of unknown duration cannot be restamped, so the pre-roll starts after it. + terminal.decoded(packet(20_000), undefined); + terminal.update(frame(0n, 40_000)); + terminal.decoded(packet(40_000), Duration.micros(20_000)); + terminal.update(gap); + terminal.update(frame(1n, 1_000_000)); + expect(terminal.preroll().map((p) => p.timestamp)).toEqual([980_000]); + const span = terminal.span({ timestamp: 1_000_000, sampleRate: 48_000, numberOfFrames: 4_800 }); + expect(span).toEqual({ timestamp: 1_060_000, frameOffset: 2_880, frames: 1_920 }); +}); + +test("a fresh encoder after a declared end takes no pre-roll from the stopped one", () => { + const terminal = new Terminal(); + terminal.clear(0, 80_000); + terminal.update(frame(0n, 0)); + terminal.decoded(packet(0), Duration.micros(20_000)); + terminal.update(end(0n, 20_000)); + expect(terminal.update(frame(1n, 1_000_000))).toBe(true); + terminal.update(gap); + terminal.update(frame(2n, 2_000_000)); + expect(terminal.preroll()).toEqual([]); + expect(terminal.span(opus(2_000_000)).frames).toBe(0); +}); diff --git a/js/watch/src/audio/terminal.ts b/js/watch/src/audio/terminal.ts index cbc37e02298..9ed6dde0c8a 100644 --- a/js/watch/src/audio/terminal.ts +++ b/js/watch/src/audio/terminal.ts @@ -12,6 +12,19 @@ type Update = | { readonly kind: "gap" } | { readonly kind: "end"; readonly group: bigint; readonly end: Duration }; +// How far, in microseconds, pre-roll output may end past the pre-roll: it absorbs the restamped +// timestamps' rounding, and is shorter than any codec frame (Opus's shortest is 2.5 ms). +const PREROLL_SLACK = 1_000; + +/** An encoded frame handed to the decoder, on the timeline the decoder saw it. */ +export interface Encoded { + /** Microseconds. */ + readonly timestamp: number; + /** Microseconds. */ + readonly duration: number; + readonly payload: Uint8Array; +} + /** The portion of decoded audio that belongs to the source timeline. */ export interface DecodedSpan { /** Source timestamp of the first retained frame, in microseconds. */ @@ -35,6 +48,13 @@ export class Terminal { #warmup = 0; #prime = true; #before?: number; + // The newest frames decoded, back to back and covering the warmup, which re-prime the decoder a + // gap restarts. Their output is pre-roll for the warmup to trim instead of the audio after the gap. + #history: Encoded[] = []; + #covered = 0; + #preroll: Encoded[] = []; + // Where the pre-roll ends: output before it only primes the decoder. + #prerollEnd?: number; /** How far decoded output lands before its frame timestamp: the Opus pre-skip, or zero. */ get delay(): number { @@ -55,14 +75,23 @@ export class Terminal { this.#warmup = warmup; this.#prime = true; this.#before = undefined; + this.#forget(); this.#resetEpoch(); } + /** Whether applying a read would end a codec epoch, as `update` reports. */ + ends(read: Update): boolean { + if (read.kind === "gap") return this.#started; + return read.kind === "frame" && this.#endGroup !== undefined && read.group > this.#endGroup; + } + /** Apply one ordered consumer read and report whether a codec epoch ended before it. */ update(read: Update): boolean { if (read.kind === "gap") { this.#prime = true; this.#before = undefined; + this.#preroll = []; + this.#prerollEnd = undefined; this.#resetEpoch(); return this.#started; } @@ -73,24 +102,92 @@ export class Terminal { } // A later group after a declared end comes from a fresh encoder; the end's own group carries // the stopped encoder's final packets. - const reset = this.#endGroup !== undefined && read.group > this.#endGroup; + const reset = this.ends(read); if (reset) this.#restart(); this.#started = true; const timestamp = read.frame.timestamp.asMicros(); this.#epoch ??= timestamp; if (this.#prime) { - this.#before = timestamp + this.#warmup; + const primed = this.#reprime(timestamp); + this.#before = timestamp + Math.max(0, this.#warmup - primed); this.#prime = false; } return reset; } + /** + * Note a frame handed to the decoder, with its duration in microseconds if known. A frame of unknown + * duration cannot be restamped, so the pre-roll starts after it. + */ + decoded(frame: { readonly timestamp: Duration; readonly payload: Uint8Array }, duration?: Duration): void { + if (this.#warmup === 0) return; + const micros = duration?.asMicros() ?? 0; + if (micros <= 0) { + this.#history = []; + this.#covered = 0; + return; + } + this.#history.push({ timestamp: frame.timestamp.asMicros(), duration: micros, payload: frame.payload }); + this.#covered += micros; + while (this.#covered - this.#history[0].duration >= this.#warmup) { + this.#covered -= this.#history[0].duration; + this.#history.shift(); + } + } + + /** Take the frames to decode, output discarded, before the first frame after a gap. */ + preroll(): Encoded[] { + const preroll = this.#preroll; + this.#preroll = []; + return preroll; + } + + /** + * Whether a decoded sample is pre-roll output alone, which only primes the decoder. A sample + * reaching past the pre-roll holds the audio after the gap, so it is not. + */ + primes(sample: SampleSpan): boolean { + const end = this.#prerollEnd; + if (end === undefined) return false; + const sampleEnd = sample.timestamp + (sample.numberOfFrames * 1_000_000) / sample.sampleRate; + if (sampleEnd <= end + PREROLL_SLACK) return true; + this.#prerollEnd = undefined; + return false; + } + + // Restamp the history to end where the first frame after a gap starts, as the pre-roll that + // re-primes the restarted decoder, returning how much of the warmup it covers. Chrome stamps an + // AudioDecoder's output from its first input, so the pre-roll carries the timestamps it plays at. + #reprime(timestamp: number): number { + let start = timestamp; + const preroll: Encoded[] = []; + for (let i = this.#history.length - 1; i >= 0; i--) { + start -= this.#history[i].duration; + preroll[i] = { ...this.#history[i], timestamp: start }; + } + this.#history = preroll; + this.#preroll = preroll; + if (preroll.length === 0) return 0; + this.#epoch = start; + this.#prerollEnd = timestamp; + return timestamp - start; + } + + #forget(): void { + this.#history = []; + this.#covered = 0; + this.#preroll = []; + this.#prerollEnd = undefined; + } + // Start a fresh codec epoch after its declared end, on the same timeline. #restart(): void { this.#started = false; this.#end = undefined; this.#endGroup = undefined; this.#before = undefined; + // A fresh encoder's first packets need no frames of the stopped one ahead of them. + this.#forget(); this.#resetEpoch(); } diff --git a/quest/m0/interop-green/README.md b/quest/m0/interop-green/README.md index daf75be7135..ccc4f91dcdf 100644 --- a/quest/m0/interop-green/README.md +++ b/quest/m0/interop-green/README.md @@ -17,8 +17,8 @@ so it only needs to stay green. with `gh run list --repo moq-dev/moq --workflow interop.yml --event schedule`, then the run's STEPS summary. The JS route port that [moq-dev/moq#8290](https://github.com/moq-dev/moq/pull/8290) waited on landed in [moq-dev/moq#8443](https://github.com/moq-dev/moq/pull/8443), - so the pass now runs the datagram step; the `replacement` case is - quarantined by default until its child quest restores it. Decided 2026-11-09 (maintainer, AFK, accepting the + so the pass now runs the datagram step; the `replacement` case runs by + default on Chromium, and its child quest restores it on Firefox. Decided 2026-11-09 (maintainer, AFK, accepting the recommendation): close this quest on a pass, which the maintainer set with those cases skipped; the pass that closes it runs all but `replacement`. Release remeasure now updates stale planned-break pins after publication @@ -227,7 +227,7 @@ Scheduled runs `replacement` is quarantined by default on every engine again, and the child [replacement audio](/quest/m0/interop-green/replacement-audio.md) root-causes and restores it. Its first PR found the stall: the decode gate released the backlog behind the clock one window - per worklet report. What keeps the quarantine is in the child. + per worklet report. Its second restored the case on Chromium; Firefox's quarantine is in the child. Public API: none. Wire: none. diff --git a/quest/m0/interop-green/replacement-audio.md b/quest/m0/interop-green/replacement-audio.md index 69410ba768e..71e2a81ecbb 100644 --- a/quest/m0/interop-green/replacement-audio.md +++ b/quest/m0/interop-green/replacement-audio.md @@ -1,43 +1,28 @@ -# [M] Restore the replacement media case: audio across recurring gaps +# [M] Restore the replacement media case on Firefox ## Goal -`just test media --media-cases replacement` passes every run on Chromium under CPU load, and the -default media run runs it again. Remove its quarantine in `test/interop/clients/js/media.ts` and -`test/interop/README.md`. +`just test media --media-cases replacement --media-engines firefox` passes every run, and the +default media run on Firefox runs it again. Remove the quarantine off Chromium in +`test/interop/clients/js/media.ts` and `test/interop/README.md`. ## Plan -The switch itself is fixed: the quest's PR found that after a fallback or replacement the audio -waited behind the incumbent's retained backlog, which the live picture had moved the shared clock -past, and the decode gate released that backlog one floor and window per worklet state report. -The gate now counts from where the ring skips to. Measured on a loaded host, the two builds -interleaved: before, 15 of 161 switches took 600 ms or more and one missed the 1039 ms bound; after, -1 of 165 did and none missed it. - -What still fails under that load, at about one run in ten either way: - -- A mid-stream gap restarts the audio codec and trims the catalog `warmup` (80 ms of Opus) from the - first output after it (`js/watch/src/audio/terminal.ts` `update`, `rs/moq-audio/src/decode/consumer.rs` - `warmup_pending`). One lost 20 ms group costs 100 ms of audio, and when the starved viewer skips a - group more often than every 80 ms, as it does on a backlog read behind the clock, no output - survives at all: one failing lite-06 fallback stayed silent (`audioTimestamp` 0) past its bound - with every decoded output trimmed to zero frames. About 40% of the steady-state stalls in failing - `audio tone` windows followed such a trim. -- The rest are underruns of the case's fixed 50 ms delay while the host is near 0% idle. The default - run's other cases fail the same way at that load (`audio rendition switch back`, 1 of 4). - -Recommended for the gap (a review can undo it): re-prime the restarted decoder on the encoded frames -just before the gap, restamped to end where the first frame after it starts, and drop their output, -so the warmup trims pre-roll rather than wanted audio. Chrome stamps an `AudioDecoder`'s output from -its first input, so the pre-roll must carry those restamped timestamps. moq-audio does the same, so -the native player keeps parity. Rejected: dropping the warmup trim after a gap reverses -[moq-dev/moq#5569](https://github.com/moq-dev/moq/pull/5569)'s decision to play only clean -output; keeping the decoder across a gap and restamping its output by input order assumes one -output per input, which AAC priming breaks. - -Then rerun the replacement case on a loaded host and lift the quarantine if what fails is only the -starvation its sibling cases share. Measure with a loop like the PR's: `taskset` pins the run and -some busy loops to the same cores, so the load stays off other agents' cores. +Chromium runs the case again by default. Its audio no longer goes silent across the groups a +starved viewer skips: the restarted decoder is re-primed on the frames before each gap, and a gap +lands what the decoder still holds instead of resetting it away. + +Decided 2026-11-16 (maintainer, on [moq-dev/moq#8980](https://github.com/moq-dev/moq/pull/8980#issuecomment-6943194650)): + +- ✅ Lift the quarantine on Chromium even though loaded runs still fail about one in six (rejected: + keep it until the underruns are fixed). What still fails there is the steady-state starvation the + sibling cases share, not the gaps. +- ✅ Leave Firefox to this quest (rejected: fix it in that PR). Its fallback misses the bound as + often before the gap fixes as after, so it is a separate cause. + +On Firefox, the fallback (`live incumbent fallback plays`) misses its 1039 ms bound about one run in +five on an unloaded host, as often before those fixes as after: one failing run had neither a +picture nor started audio, another a picture with the audio stalled. Root-cause it on Firefox first, +reading the player's state across the fallback. WebKit was never run; check it once Firefox passes. Public API: none. Wire: none. diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 20b5d1f7f8f..baadb095ff2 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -133,6 +133,9 @@ pub struct Consumer { /// first packet, which the codec's own delay already primes. warmup: Duration, warmup_pending: bool, + /// Media decoded so far, up to the warmup. A gap keeps the codec's history, which primes + /// it as the browser's pre-roll does, so only what this falls short of is trimmed. + primed: Duration, /// Whether a packet was read or lost yet, so the track's first packet can be told apart. started: bool, warmup_until: Option, @@ -278,6 +281,7 @@ impl Consumer { max_age, warmup, warmup_pending: !warmup.is_zero(), + primed: Duration::ZERO, warmup_until: None, started: false, resolved_sample_rate: sample_rate, @@ -393,6 +397,7 @@ impl Consumer { // own group carries the stopped encoder's final packets, trimmed below. if self.end.is_some() && group != self.end_group { self.restart()?; + self.fresh_encoder(); } // A group with nothing below it, which its publisher declares by skipping // every start before it, is the track's first, and a group delivers from its @@ -405,7 +410,13 @@ impl Consumer { Some(Encoded::Gap { timestamp, duration }) => { self.boundary = Some(Duration::try_from(timestamp)?); self.started = true; + // Groups lost past a declared end were the fresh encoder's, so the packet + // after them still comes from it. + let fresh = self.end.is_some(); self.restart()?; + if fresh { + self.fresh_encoder(); + } self.warmup_pending = !self.warmup.is_zero(); self.ready.push_back(Read::Gap { timestamp, duration }); continue; @@ -429,7 +440,7 @@ impl Consumer { let timestamp = mux_frame.timestamp; // Undeclared holes are routine: a skipped stalled group, a packet the - // decoder refused, an ingest that resynced. Drop every stage's state at + // decoder refused, an ingest that resynced. Drop the timeline's state at // the edge, before the packet after it goes anywhere near the decoder. // // Skipped once an end marker arrives, because from there the terminal @@ -475,9 +486,12 @@ impl Consumer { // first packet has no history to warm up from: the codec's pre-skip, trimmed by // the decoder, is all the priming its encoder declared. if std::mem::take(&mut self.warmup_pending) && !track_start { - let warmup = moq_net::Timestamp::try_from(self.warmup)? - .convert(timestamp.scale().ok_or(moq_net::TimeOverflow)?)?; - self.warmup_until = Some(timestamp.checked_add(warmup)?); + let warmup = self.warmup.saturating_sub(self.primed); + if !warmup.is_zero() { + let warmup = moq_net::Timestamp::try_from(warmup)? + .convert(timestamp.scale().ok_or(moq_net::TimeOverflow)?)?; + self.warmup_until = Some(timestamp.checked_add(warmup)?); + } } // Codec delay trimmed off the front is media this packet covered even // though no samples came out, so it still moves the packet after it along. @@ -491,6 +505,7 @@ impl Consumer { if covered > 0 { self.target.set_frame(covered as f64 * 1000.0 / rate as f64); self.frame = Duration::from_secs_f64(covered as f64 / rate as f64); + self.primed = self.primed.saturating_add(self.frame).min(self.warmup); } if let Some(end) = self.end { let terminal_start = *self @@ -736,7 +751,9 @@ impl Consumer { } /// Start a new timeline after a gap or past a declared end. The decoder is not reset: - /// the next group already starts on a keyframe, and pre-skip is a play-path concern. + /// the next group already starts on a keyframe, and the history it keeps primes the + /// packets after the break, so a gap trims no more warmup than the decoder still lacks, + /// and no pre-skip, which only a fresh encoder starts with. /// /// Samples the resampler still holds arrived before the break, so they are drained /// as their own frame first rather than filtered together with what follows it. @@ -753,21 +770,26 @@ impl Consumer { self.epoch = None; self.delay_trimmed = 0; self.warmup_until = None; - self.decoder.reapply_delay(); Ok(()) } - /// Reset codec prediction and resampling state at a hole, returning whatever the - /// resampler was still holding from before it. + /// Expect a fresh encoder's packets: its pre-skip is trimmed again, and the stopped + /// encoder's history primes none of its warmup. + fn fresh_encoder(&mut self) { + self.decoder.reapply_delay(); + self.primed = Duration::ZERO; + } + + /// Reset resampling state at a hole, returning whatever the resampler was still + /// holding from before it. /// /// Those samples arrived before the hole and belong before it, so they come /// out as their own frame rather than being filtered together with the audio /// on the far side. The resampler starts over from there, which is what makes /// the next packet's output stamp from the packet itself: nothing is buffered - /// to reach back over. + /// to reach back over. The codec keeps its history, which primes the packets + /// after the hole as a pre-roll would. fn gap(&mut self) -> Result, Error> { - self.decoder.reset_prediction()?; - let frame = self.drain()?; self.next_start = None; @@ -1145,7 +1167,19 @@ mod tests { } } - async fn warmup_output(declared: bool) -> (Vec, usize) { + /// How the timeline breaks between `warmup_output`'s two runs of packets. + #[derive(Clone, Copy, Debug)] + enum Break { + /// A declared end, so a fresh encoder follows. + Declared, + /// A lost group: a seek opens the next without declaring the starts before it skipped. + Lost, + /// A timestamp jump across contiguous groups. + Undeclared, + } + + /// Decodes `first` packets, then six more a second later, past a `cut` in the timeline. + async fn warmup_output(cut: Break, first: u64) -> (Vec, usize) { let broadcast = moq_net::broadcast::Info::new().produce(); let track = broadcast .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) @@ -1159,10 +1193,14 @@ mod tests { // Joined mid-stream: the track's first packet needs no warmup. producer.seek(1).unwrap(); for epoch in 0..2 { - if epoch != 0 && declared { - producer.discontinuity().unwrap(); + if epoch != 0 { + match cut { + Break::Declared => producer.discontinuity().unwrap(), + Break::Lost => producer.seek(1_000_000).unwrap(), + Break::Undeclared => {} + } } - for packet in 0..6 { + for packet in 0..if epoch == 0 { first } else { 6 } { producer .write(moq_mux::container::Frame { timestamp: Timestamp::from_micros(epoch * 1_000_000 + packet * 20_000).unwrap(), @@ -1455,7 +1493,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn warmup_trims_the_join_without_trimming_groups_or_declared_returns() { - let (stamps, frames) = warmup_output(true).await; + let (stamps, frames) = warmup_output(Break::Declared, 6).await; assert_eq!( stamps, [ @@ -1465,11 +1503,21 @@ mod tests { assert_eq!(frames, 8 * 960); } + /// The codec's history primes it across a gap, so only the warmup it falls short of is trimmed. #[tokio::test(start_paused = true)] - async fn warmup_restarts_after_an_undeclared_gap() { - let (stamps, frames) = warmup_output(false).await; - assert_eq!(stamps, [80_000, 100_000, 1_080_000, 1_100_000]); - assert_eq!(frames, 4 * 960); + async fn a_gap_trims_the_warmup_the_history_leaves() { + for cut in [Break::Lost, Break::Undeclared] { + let (stamps, frames) = warmup_output(cut, 2).await; + assert_eq!(stamps, [1_040_000, 1_060_000, 1_080_000, 1_100_000], "{cut:?}"); + assert_eq!(frames, 4 * 960, "{cut:?}"); + // A primed decoder trims nothing after the gap. + let (stamps, _) = warmup_output(cut, 6).await; + assert_eq!( + stamps[2..], + [1_000_000, 1_020_000, 1_040_000, 1_060_000, 1_080_000, 1_100_000], + "{cut:?}" + ); + } } /// A latest join skips the cached backlog and primes the decoder on the live edge's own @@ -2597,8 +2645,8 @@ mod tests { let frames = first.data.len() / size_of::(); assert!(frames < 960, "the pre-skip should be trimmed, got {frames} frames"); - // The hole is real, so codec prediction starts over but stream-level pre-skip - // does not. The audio is stamped where the packet says rather than 2.5 ms early. + // The hole is real, but stream-level pre-skip does not start over. The audio is + // stamped where the packet says rather than 2.5 ms early. let second = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert_eq!(second.timestamp.as_micros().unwrap(), 22_500); assert_eq!(second.data.len() / size_of::(), 960, "pre-skip was reapplied"); @@ -2614,6 +2662,102 @@ mod tests { ); } + /// A lost group leaves the same encoder running, so its pre-skip, trimmed once at the + /// stream's start, is not trimmed again off the media after the loss. + #[tokio::test(start_paused = true)] + async fn a_lost_group_does_not_trim_the_pre_skip_again() { + assert_eq!(after_loss(false, None).await[0], (1_000_000, 960)); + } + + /// Groups lost past a declared end were the fresh encoder's, so the packet after them + /// still comes from it: its pre-skip is trimmed, leaving its media at the packet's stamp. + #[tokio::test(start_paused = true)] + async fn a_loss_after_a_declared_end_trims_the_fresh_encoders_pre_skip() { + assert_eq!(after_loss(true, None).await[0], (1_000_000, 960 - 312)); + } + + /// The stopped encoder's history primes none of the fresh encoder's warmup, so a loss of + /// its first groups trims the whole warmup after them. + #[tokio::test(start_paused = true)] + async fn a_loss_after_a_declared_end_trims_the_whole_warmup() { + let warmup = Some(Duration::from_millis(10)); + assert_eq!(after_loss(true, warmup).await[0].0, 1_010_000); + } + + /// Decodes two Opus packets, then, past lost groups and a declared end if `declared`, one + /// more a second later, returning the stamp and length of each frame after the loss. + async fn after_loss(declared: bool, warmup: Option) -> Vec<(u128, usize)> { + let input = Input { + format: Format::F32, + sample_rate: 48_000, + layout: Layout::Mono, + }; + let config = EncodeConfig::new(input.sample_rate, input.layout); + let mut encoder = Encoder::new(&config).unwrap(); + let mut catalog = encoder.catalog(); + catalog.warmup = warmup; + + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let pcm = vec![0.25f32; encoder.frame_size()]; + for epoch in 0..2u64 { + if epoch == 1 { + if declared { + producer.discontinuity().unwrap(); + encoder = Encoder::new(&config).unwrap(); + } + producer.seek(1_000_000).unwrap(); + } + for packet in 0..if epoch == 0 { 2 } else { 1 } { + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(epoch * 1_000_000 + packet * 20_000).unwrap(), + duration: None, + payload: crate::single(encoder.encode_all(&pcm).unwrap()).payload, + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + } + } + let _finished = producer.finish().unwrap(); + let mut consumer = Consumer::new( + &broadcast.consume(), + &catalog, + "audio", + Options { + delay: Delay { + max: Some(Duration::from_secs(10)), + ..Delay::default() + }, + ..Options::new() + }, + ) + .await + .unwrap(); + + let mut lost = false; + let mut after = Vec::new(); + while let Some(read) = consumer.read().await.unwrap() { + match read { + Read::Gap { .. } => lost = true, + Read::Frame(frame) if lost => after.push(( + frame.timestamp.as_micros().unwrap(), + frame.data.len() / size_of::(), + )), + Read::Frame(_) | Read::End { .. } => {} + } + } + assert!(!after.is_empty(), "no frame after the loss"); + after + } + thread_local! { /// A pinned receiver clock for [`now`], so a test replays an arrival trace /// instantly. Thread-local because a `tokio::test` runs on one thread. diff --git a/rs/moq-audio/src/decode/decoder.rs b/rs/moq-audio/src/decode/decoder.rs index 4be4798c97d..87811c81fda 100644 --- a/rs/moq-audio/src/decode/decoder.rs +++ b/rs/moq-audio/src/decode/decoder.rs @@ -81,7 +81,7 @@ impl Decoder { /// Reset codec history and reapply startup delay for a new discontinuous epoch. pub fn reset(&mut self) -> Result<(), Error> { - self.reset_prediction()?; + self.backend.reset()?; self.reapply_delay(); Ok(()) } @@ -91,11 +91,6 @@ impl Decoder { self.delay_remaining = self.delay; } - /// Reset codec prediction after packet loss without reapplying stream startup delay. - pub(super) fn reset_prediction(&mut self) -> Result<(), Error> { - self.backend.reset() - } - /// How much startup delay is still to be trimmed, in native-rate frames. /// /// Trimmed samples are media the packet covered even though nothing came out of diff --git a/test/interop/README.md b/test/interop/README.md index 5ae0b0d7e3a..7fdd7dfdd1d 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -551,12 +551,11 @@ at QUIC's 1 ms timer granularity, the 500 ms GOP, one frame to decode, the 50 ms playback delay, and one page sample plus one driver poll. Publisher preparation is not part of it. -The replacement case is quarantined from the default run on every engine: under CPU -load its audio intermittently goes silent, as each group the viewer skips restarts the -decoder and trims its 80 ms warmup, and Firefox also showed intermittent missing tone -samples. Run it explicitly with -`just test media --media-cases replacement`, or `--media-cases replacement --media-engines firefox` for Firefox; -its content, sync, switch, and fallback assertions remain active. +The replacement case is quarantined from the default run off Chromium: on Firefox its +fallback intermittently misses the switch bound, as often with the audio fixes that +restored it on Chromium as without them. Run it explicitly with +`--media-cases replacement --media-engines firefox`; its content, sync, switch, and +fallback assertions remain active. The CLI integration suite's `instances` cases exercise the real QUIC relay with lite-07, lite-06, and IETF draft 22. Held catalog snapshots and media diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index cb3aea75fae..9efef98118f 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -129,11 +129,13 @@ const native = values.publisher === "native"; const isEngine = (name: string): name is Engine => ENGINES.some((engine) => engine === name); const requestedPublisher = values["publisher-engine"] ?? "chromium"; const requestedPlayer = values["player-engine"] ?? "chromium"; +// Restored on Chromium only: on Firefox its fallback intermittently misses the switch bound. +const quarantined = !native && values.cases === undefined && requestedPlayer !== "chromium"; const selected = new Set( values.cases === undefined ? native ? NATIVE_CASES - : CASES.filter((name) => name !== "replacement") + : CASES.filter((name) => !quarantined || name !== "replacement") : values.cases === "none" ? [] : values.cases.split(","), @@ -161,9 +163,9 @@ if ( const publisherEngine = requestedPublisher as Engine; const playerEngine = requestedPlayer as Engine; const wants = (name: Case) => selected.has(name); -if (!native && values.cases === undefined) { +if (quarantined) { console.error( - "QUARANTINE\treplacement\tintermittent audio silence after skipped groups under CPU load; see quest/m0/interop-green/replacement-audio.md", + "QUARANTINE\treplacement\tintermittent fallback past the switch bound off Chromium; see quest/m0/interop-green/replacement-audio.md", ); }