diff --git a/js/hang/src/container/consumer.ordered.test.ts b/js/hang/src/container/consumer.ordered.test.ts index e5dabb08fa..09c721befb 100644 --- a/js/hang/src/container/consumer.ordered.test.ts +++ b/js/hang/src/container/consumer.ordered.test.ts @@ -4,21 +4,33 @@ import { expect, test } from "bun:test"; import { Duration } from "@moq/clock"; import { Manual } from "@moq/clock/test"; import { Time, Track } from "@moq/net"; +import { u53 } from "../catalog/integers.ts"; import { Consumer, type Read } from "./consumer.ts"; import { encodeFrame, Format } from "./legacy.ts"; const flush = () => new Promise((resolve) => setImmediate(resolve)); +// One 20 ms CELT packet: its TOC byte states the duration, which an Opus format reads. +const OPUS = new Uint8Array([0xf8]); + // An audio track whose missing groups an unserved fill owes, as a session's feed would, read by a // consumer whose ordered reader waits `wait` for a hole and whose arrival subscription reaches 2 s. -function setup(wait: number) { +// Frames state no duration unless `opus`. +function setup(wait: number, opus = false) { const clock = new Manual(); const installed = clock.install(); const track = new Track.Producer("audio"); const requests = track.requests(); const ordered = track.subscribe({ delay: { min: Duration.ZERO, max: Duration.millis(wait) } }).ordered(); const consumer = new Consumer(track.subscribe({ delay: { max: Duration.secs(2) } }), { - format: new Format("audio"), + format: opus + ? new Format({ + codec: "opus", + sampleRate: u53(48_000), + numberOfChannels: u53(2), + container: { kind: "legacy" }, + }) + : new Format("audio"), delay: { min: Duration.ZERO, max: Duration.secs(2) }, ordered, }); @@ -34,7 +46,7 @@ function setup(wait: number) { const write = (sequence: bigint, ms: number) => { const group = track.createGroup(sequence); group.writeFrame({ - payload: encodeFrame(new Uint8Array([1]), Duration.millis(ms)), + payload: encodeFrame(opus ? OPUS : new Uint8Array([1]), Duration.millis(ms)), timestamp: Time.Timestamp.fromMillis(ms), }); group.finish(); @@ -140,3 +152,68 @@ test("the estimate observes late groups the ordered reader gave up on", async () }; expect((await spread(300)).gt(await spread(0))).toBe(true); }); + +test("a gap over keys that held no media is no loss", async () => { + const t = setup(100, true); + try { + // Keyed by timestamp in microseconds, with the keys between never declared skipped. The group + // past them opens before its frame lands, and the reader gives the keys up while it waits. + t.write(0n, 0); + const group = t.track.createGroup(20_000n); + await t.clock.advance(Duration.millis(100)); + // A later group the reader hands out at once wakes the consumer before the frame says nothing + // was missing. + t.track.skipGroups(20_001n, 40_000n); + t.write(40_000n, 40); + await flush(); + group.writeFrame({ payload: encodeFrame(OPUS, Duration.millis(20)), timestamp: Time.Timestamp.fromMillis(20) }); + group.finish(); + await flush(); + expect(t.read()).toEqual(["frame 0", "frame 20", "frame 40"]); + } finally { + await t.close(); + } +}); + +test("a gap over keys that held media is still a loss", async () => { + const t = setup(100, true); + try { + t.write(0n, 0); + t.write(40_000n, 40); + await t.clock.advance(Duration.millis(100)); + await flush(); + expect(t.read()).toEqual(["frame 0", "gap 20+20", "frame 40"]); + } finally { + await t.close(); + } +}); + +test("an open group that never delivers a frame is passed once the reader's wait ages it", async () => { + const t = setup(100, true); + try { + t.write(0n, 0); + // Opened, never written or finished. + t.track.createGroup(1n); + // Group 2 lands at 20 ms. The reader expires the open group a wait after its successor landed. + for (let i = 2; i <= 6; i++) { + await t.clock.advance(Duration.millis(20)); + t.write(BigInt(i), i * 20); + } + await t.clock.advance(Duration.millis(19)); + await flush(); + expect(t.read()).toEqual(["frame 0"]); + await t.clock.advance(Duration.millis(1)); + await flush(); + expect(t.read()).toEqual([ + "frame 0", + "gap 20+20", + "frame 40", + "frame 60", + "frame 80", + "frame 100", + "frame 120", + ]); + } finally { + await t.close(); + } +}); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 3da434e5de..c1bcb402b8 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -77,8 +77,8 @@ export interface ConsumerProps { /** * Read the media in key order from this reader of the same track instead of `track`'s arrival * order, so it decides every hole. It must be a second subscription, never `track`'s own, which - * keeps reading in arrival order to feed the estimate: a missing group is a hole once the reader reports it as a gap, - * and never waited for here. Its `delay.max` is then how long a hole is waited for, so size it to + * keeps reading in arrival order to feed the estimate: a missing group is a hole once the reader reports it as a gap + * and the media after it does not continue the timeline, and never waited for here. Its `delay.max` is then how long a hole is waited for, so size it to * what the player can still play late. `track` only feeds the arrival estimate, read frame by frame * as it lands, so give it a `delay.max` reaching as far as lateness can move the estimate. */ @@ -246,6 +246,9 @@ export class Consumer { #mediaGroup?: bigint; // Media went missing since the last item read, so the next one is preceded by a gap. #lost = false; + // The ordered reader gave up on keys since the last item read: a loss only if the next item does + // not continue the timeline, since a publisher keying groups by timestamp never produced most keys. + #gapped?: { from: bigint; to: bigint }; // The furthest presentation point read so far: max(timestamp + duration), endpoints included. // Undefined until the first item, so a loss before it is no gap. #edge?: number; @@ -863,8 +866,9 @@ export class Consumer { const nextStart = next?.frames.at(0)?.timestamp.asMicros(); if (!next || nextStart === undefined || active.end < nextStart) return false; + // Routine for a group of one frame whose stream finishes after the next one starts, and nothing + // is lost, so it says nothing. this.#groups.shift(); - console.warn(`skipping covered group: ${active.consumer.start} -> ${next.consumer.start}`); this.#pass(active, next.consumer.start); this.#discard(active); @@ -1086,7 +1090,10 @@ export class Consumer { ) { // What an ordered reader gives up it reports as a gap; a key it skips without one was never a // group. A gap an empty group carries still marks the next frame, which follows the loss. - if (!contiguous && (decided ? head.gap : !empty)) { + if (decided && head.gap && !contiguous) { + // The head's first frame may not have landed yet, so the next item read decides. + this.#gapped = { from: this.#gapped?.from ?? this.#active, to: head.consumer.start }; + } else if (!decided && !contiguous && !empty) { this.#lost = true; if (!startup) { console.warn( @@ -1318,8 +1325,14 @@ export class Consumer { #emit(read: Exclude): Read { const resume = (read.kind === "frame" ? read.frame.timestamp : read.end).asMicros(); const edge = this.#edge; - const lost = this.#lost; + const gapped = this.#gapped; + this.#gapped = undefined; + let lost = this.#lost; this.#lost = false; + if (gapped && edge !== undefined && !ptsContiguous(edge, resume)) { + lost = true; + console.warn(`skipping missing groups: track=${this.#track.name} ${gapped.from} -> ${gapped.to}`); + } this.#advance(read.kind === "frame" ? resume + (read.frame.duration?.asMicros() ?? 0) : resume); if (!lost || edge === undefined) return read; diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index 6d9b6d3096..8639344348 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -104,6 +104,27 @@ describe("replay", () => { expect(warn).toHaveBeenCalled(); }); + // A publisher keying groups by timestamp never uses the keys between them, so only keys around + // missing media are owed. Media that overlaps the group below counts as back to back, as the consumer + // reads it. + it("waits out only the keys around missing media when groups are keyed by timestamp", async () => { + const options = { rate: RATE, config: CONFIG, delay: Duration.millis(100), duration: observed(10) }; + const keyed = (duration: number) => + paced(10, () => 30).map((arrival) => ({ ...arrival, group: arrival.timestamp * 1000, duration })); + // Audio heard, in ms: owed keys nobody sends would starve the whole replay. + const heard = async (trace: Arrival[]) => { + let samples = 0; + for await (const { output } of replay(trace, options)) samples += output.filter((v) => v !== 0).length; + return (samples * 1000) / RATE; + }; + const whole = await heard(keyed(20)); + expect(whole).toBeGreaterThan(9000); + expect(await heard(keyed(22))).toBe(whole); + expect(await underruns(keyed(20), options)).toBe(0); + const missing = keyed(20).filter((arrival) => arrival.timestamp !== 5000); + expect(await underruns(missing, options)).toBeGreaterThan(0); + }); + it("leaves a frame missing inside a group as missing audio", async () => { // Five frames a group, delivered on time, with one from the middle of a group never sent. const trace = paced(10, () => 30) diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 21b4d708bf..5170612361 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -1,7 +1,8 @@ /** * Replay a recorded arrival trace through the player's consumer and ring on a simulated clock. * - * Every frame is written into its group at the instant the trace says it arrived, and the player's + * Every frame is written into its group at the instant the trace says it arrived, the keys a + * timestamp-keyed publisher skipped declared as its group lands, and the player's * real `Container.Consumer` decides what to deliver and skip, at the max delay the decoder gives it, * reading the ordered reader that decides each hole at the budget the decoder gives that. Delivered frames go into the real ring, which carries on through a gap the way the decoder's * does: a gap restarts only the codec. One render quantum is read every quantum's worth of that same @@ -38,6 +39,9 @@ export const QUANTUM = 128; */ const WIRE_MAX_DELAY = null; +/** How far, in ms, a group may start past the end of the one below and still read as back to back, as the consumer allows. */ +const CONTIGUOUS = 1; + /** Any non-empty payload: an empty one is the legacy container's end marker. */ const PAYLOAD = new Uint8Array(1); @@ -208,6 +212,25 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato trace.forEach((arrival, i) => { ends.set(arrival.group, i); }); + // A publisher keying groups by timestamp never produces the keys between two groups whose media + // is back to back, and each GROUP's Previous Start tells the subscriber so as it lands. Keys + // between media that is not are owed, since a group there went missing. + const spans = new Map(); + for (const { group, timestamp, duration } of trace) { + const span = spans.get(group); + spans.set(group, { + start: Math.min(span?.start ?? timestamp, timestamp), + end: Math.max(span?.end ?? timestamp, timestamp + duration), + }); + } + const keys = [...spans.keys()].sort((a, b) => a - b); + const skipped = new Map(); + keys.forEach((key, i) => { + const below = keys[i - 1]; + if (below === undefined || below + 1 === key) return; + const end = spans.get(below)?.end ?? 0; + if ((spans.get(key)?.start ?? 0) <= end + CONTIGUOUS) skipped.set(key, below + 1); + }); const lookahead = Duration.millis(options.lookahead ?? 0); const buffered = !lookahead.isZero(); @@ -361,6 +384,8 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato } let group = groups.get(arrival.group); if (!group) { + const from = skipped.get(arrival.group); + if (from !== undefined) track.skipGroups(BigInt(from), BigInt(arrival.group)); group = track.createGroup(BigInt(arrival.group)); groups.set(arrival.group, group); } diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index 73423885d1..b3581701e7 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -140,12 +140,39 @@ test("a gap re-primes the decoder on the warmup before it, so the audio after it expect(terminal.primes(opus(timestamp))).toBe(true); expect(terminal.span(opus(timestamp)).frames).toBe(0); } - // The overlap the pre-skip carries over the restart starts before the frame, so only it is trimmed. + // An Opus packet decodes to the encoder's lookahead before its timestamp, so the frame after the gap + // plays whole from a pre-skip early, as it would have with no gap. expect(terminal.primes(opus(1_000_000))).toBe(false); - expect(terminal.span(opus(1_000_000))).toEqual({ timestamp: 1_000_000, frameOffset: 312, frames: 648 }); + expect(terminal.span(opus(1_000_000))).toEqual({ timestamp: 993_500, frameOffset: 0, frames: 960 }); expect(terminal.span(opus(1_020_000))).toEqual({ timestamp: 1_013_500, frameOffset: 0, frames: 960 }); }); +// A reader can report a gap where nothing went missing, such as over keys a publisher never used. +test("a gap with nothing missing leaves no hole in Opus output", () => { + const terminal = new Terminal(); + terminal.clear(312, 80_000); + const ends: number[] = []; + const starts: number[] = []; + const play = (timestamp: number) => { + if (terminal.primes(opus(timestamp))) return; + const span = terminal.span(opus(timestamp)); + if (span.frames === 0) return; + starts.push(span.timestamp); + ends.push(span.timestamp + (span.frames * 1_000_000) / 48_000); + }; + for (let i = 0; i < 10; i++) { + terminal.update(frame(BigInt(i), i * 20_000)); + terminal.decoded(packet(i * 20_000), Duration.micros(20_000)); + play(i * 20_000); + } + expect(terminal.update(gap)).toBe(true); + terminal.update(frame(10n, 200_000)); + for (const { timestamp } of terminal.preroll()) play(timestamp); + play(200_000); + // Output is back to back across the restart. + expect(starts.at(-1)).toBe(ends.at(-2)); +}); + // A decoder that coalesces outputs can hand back the pre-roll and the frame after the gap together. test("output reaching past the pre-roll holds the audio after the gap", () => { const terminal = new Terminal(); diff --git a/js/watch/src/audio/terminal.ts b/js/watch/src/audio/terminal.ts index 9ed6dde0c8..fccbe8716d 100644 --- a/js/watch/src/audio/terminal.ts +++ b/js/watch/src/audio/terminal.ts @@ -109,7 +109,9 @@ export class Terminal { this.#epoch ??= timestamp; if (this.#prime) { const primed = this.#reprime(timestamp); - this.#before = timestamp + Math.max(0, this.#warmup - primed); + // On the timeline `span` maps output onto, where this frame's output lands: a pre-skip early, + // since an Opus packet decodes to the encoder's lookahead before its timestamp. + this.#before = timestamp - this.delay + Math.max(0, this.#warmup - primed); this.#prime = false; } return reset; diff --git a/quest/m2/audio-playout/browser-audio-hole-skip.md b/quest/m2/audio-playout/browser-audio-hole-skip.md index 55d07dfecc..760d2d88de 100644 --- a/quest/m2/audio-playout/browser-audio-hole-skip.md +++ b/quest/m2/audio-playout/browser-audio-hole-skip.md @@ -1,12 +1,13 @@ -# [M] Browser audio's ordered reader grades no worse than before #8799 +# [S] Browser audio's ordered reader grades no worse than before #8799 on the runner ## Goal The Chromium `bursty` and `congestion` rows of `just test audio-quality`, with browser audio reading its holes from `@moq/net`'s ordered reader, grade no worse than before [moq-dev/moq#8799](https://github.com/moq-dev/moq/pull/8799) -gave the audio consumer the player's clock, measured on the nightly runner, -and the one full starvation seen locally is explained or fixed. +on the nightly runner, and no scheduled night shows a browser row starving +(`stalled_share` near 1, or a playhead holding still while the render clock +runs) that this quest has not explained. ## Plan @@ -16,32 +17,38 @@ audio hole, through an ordered reader whose budget is the ring's size the container consumer; keep the playhead rule and the separate wait). [moq-dev/moq#9068](https://github.com/moq-dev/moq/pull/9068) moved browser -audio onto the ordered reader: `Ordered.nextItem` returns a gap for each -half-open range of keys it gives up on, audio reads through a second -subscription whose budget is what the ring holds plus the stretch bound and -the warmup, and the audio container consumer plays each gap as its hole. Firefox's -`after resume` window stays audible. - -Its local Chromium measurement, three runs each of the PR, its merge base, -and `cf23b5ae04~1` (before #8799), could not grade the rows: the machine sat -at a load average of 30 to 70, and render underruns reached 580 a minute on -the merge base. The player-side numbers were level with the merge base on -`congestion`, which also trimmed about half the audio it did before #8799. -On `bursty` they were mixed, and one Opus `bursty` run of seven starved for -its last 7.5 s: the playhead held still while the render clock ran, which no -merge-base or pre-#8799 run did, and four reruns did not repeat it. +audio onto the ordered reader. Its first scheduled night (2026-11-18, +[44342367698](https://github.com/moq-dev/moq/actions/runs/44342367698)) put +112 tap underruns a minute on `chromium-opus-48000-bursty`, where every +earlier night had none: about one 4 to 6 ms hole a second, with no player +underrun and no media missing from the capture. The PR for this quest found +and fixed both causes. A reader gap over keys a timestamp-keyed publisher +never used was marked a loss whenever the consumer judged it before the +group's first frame landed. And every gap restarted the Opus codec with a cut +at the frame's timestamp rather than a pre-skip earlier, which dropped 6.5 ms +of real audio. A local live run reproduced the night's numbers without the +fix (136.8 underruns a minute) and measured zero with it. Replaying that +night's captures, now possible since the replay declares a timestamp-keyed +publisher's skipped keys, graded `bursty` level with the player before +#9068. `congestion` trimmed up to 350 ms a minute more audio there, with +stalled and silence shares level: the audio the old player never received +now arrives late and is skipped onto the clock. + +The 7.5 s starvation one local run showed during #9068 was not reproduced. +The mechanism suspected for it, an active group that opens but never delivers +a frame, is bounded: the ordered subscription expires such a group one hole +wait after its successor lands, which a consumer test now holds. So a trace +does not need to record when each group opens. Work: -- Grade the Chromium `bursty` and `congestion` rows from the nightly - runner's `audio-quality-run` artifacts after #9068 lands, against the - nights before it and before #8799. -- Find what starved that run. The audio container consumer - (`js/hang/src/container/consumer.ts`) in ordered mode has nothing that - passes an active group that opened but never delivers a frame until the - max buffer fills, which #8799's playhead covered. A trace recorded with - `--trace` holds only the frames that arrived, so replaying such a group - also needs the trace to record when each group opens. +- Grade the Chromium `bursty` and `congestion` rows from the + `audio-quality-run` artifacts of the first three scheduled nights after + the fix lands, against the nights before #8799 (2026-11-13 to 11-15) and + the PR's table. Opus `bursty` should file no tap underruns again. +- If a night shows a browser row starving, replay its capture + (`test/audio-quality/README.md`, "To replay a capture") and explain it + here. ## Related diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 83b97d5664..5764925624 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -128,7 +128,10 @@ untraced control and never use traced rows to calibrate quality budgets. To replay a capture, put its arrivals, final environment configuration, and `largest` (the observer's join, `environment.largest` in `.page.json`) into the version 4 `Trace` below, subtract the first arrival's `at` from every arrival and from the last sample's -`at` for the duration, and supply the observed minimum RTT as a diagnostic. Replay measures the +`at` for the duration, and supply the observed minimum RTT as a diagnostic. A publisher that keys +groups by timestamp never produces the keys between them, so the replay declares those skipped where +one group's media ends as the next begins, as each group's append does on the wire, and leaves a hole +in the media owed. Replay measures the target from the arrivals and holds the configuration constant; compare the raw samples when it changed during the live run. The observer only records frames delivered under the player's maximum age, so a replay cannot recover groups the