Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 80 additions & 3 deletions js/hang/src/container/consumer.ordered.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,33 @@ import { expect, test } from "bun:test";
import { Duration } from "@moq/clock";
import { Manual } from "@moq/clock/test";
import { Time, Track } from "@moq/net";
import { u53 } from "../catalog/integers.ts";
import { Consumer, type Read } from "./consumer.ts";
import { encodeFrame, Format } from "./legacy.ts";

const flush = () => new Promise<void>((resolve) => setImmediate(resolve));

// One 20 ms CELT packet: its TOC byte states the duration, which an Opus format reads.
const OPUS = new Uint8Array([0xf8]);

// An audio track whose missing groups an unserved fill owes, as a session's feed would, read by a
// consumer whose ordered reader waits `wait` for a hole and whose arrival subscription reaches 2 s.
function setup(wait: number) {
// Frames state no duration unless `opus`.
function setup(wait: number, opus = false) {
const clock = new Manual();
const installed = clock.install();
const track = new Track.Producer("audio");
const requests = track.requests();
const ordered = track.subscribe({ delay: { min: Duration.ZERO, max: Duration.millis(wait) } }).ordered();
const consumer = new Consumer(track.subscribe({ delay: { max: Duration.secs(2) } }), {
format: new Format("audio"),
format: opus
? new Format({
codec: "opus",
sampleRate: u53(48_000),
numberOfChannels: u53(2),
container: { kind: "legacy" },
})
: new Format("audio"),
delay: { min: Duration.ZERO, max: Duration.secs(2) },
ordered,
});
Expand All @@ -34,7 +46,7 @@ function setup(wait: number) {
const write = (sequence: bigint, ms: number) => {
const group = track.createGroup(sequence);
group.writeFrame({
payload: encodeFrame(new Uint8Array([1]), Duration.millis(ms)),
payload: encodeFrame(opus ? OPUS : new Uint8Array([1]), Duration.millis(ms)),
timestamp: Time.Timestamp.fromMillis(ms),
});
group.finish();
Expand Down Expand Up @@ -140,3 +152,68 @@ test("the estimate observes late groups the ordered reader gave up on", async ()
};
expect((await spread(300)).gt(await spread(0))).toBe(true);
});

test("a gap over keys that held no media is no loss", async () => {
const t = setup(100, true);
try {
// Keyed by timestamp in microseconds, with the keys between never declared skipped. The group
// past them opens before its frame lands, and the reader gives the keys up while it waits.
t.write(0n, 0);
const group = t.track.createGroup(20_000n);
await t.clock.advance(Duration.millis(100));
// A later group the reader hands out at once wakes the consumer before the frame says nothing
// was missing.
t.track.skipGroups(20_001n, 40_000n);
t.write(40_000n, 40);
await flush();
group.writeFrame({ payload: encodeFrame(OPUS, Duration.millis(20)), timestamp: Time.Timestamp.fromMillis(20) });
group.finish();
await flush();
expect(t.read()).toEqual(["frame 0", "frame 20", "frame 40"]);
} finally {
await t.close();
}
});

test("a gap over keys that held media is still a loss", async () => {
const t = setup(100, true);
try {
t.write(0n, 0);
t.write(40_000n, 40);
await t.clock.advance(Duration.millis(100));
await flush();
expect(t.read()).toEqual(["frame 0", "gap 20+20", "frame 40"]);
} finally {
await t.close();
}
});

test("an open group that never delivers a frame is passed once the reader's wait ages it", async () => {
const t = setup(100, true);
try {
t.write(0n, 0);
// Opened, never written or finished.
t.track.createGroup(1n);
// Group 2 lands at 20 ms. The reader expires the open group a wait after its successor landed.
for (let i = 2; i <= 6; i++) {
await t.clock.advance(Duration.millis(20));
t.write(BigInt(i), i * 20);
}
await t.clock.advance(Duration.millis(19));
await flush();
expect(t.read()).toEqual(["frame 0"]);
await t.clock.advance(Duration.millis(1));
await flush();
expect(t.read()).toEqual([
"frame 0",
"gap 20+20",
"frame 40",
"frame 60",
"frame 80",
"frame 100",
"frame 120",
]);
} finally {
await t.close();
}
});
23 changes: 18 additions & 5 deletions js/hang/src/container/consumer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ export interface ConsumerProps {
/**
* Read the media in key order from this reader of the same track instead of `track`'s arrival
* order, so it decides every hole. It must be a second subscription, never `track`'s own, which
* keeps reading in arrival order to feed the estimate: a missing group is a hole once the reader reports it as a gap,
* and never waited for here. Its `delay.max` is then how long a hole is waited for, so size it to
* keeps reading in arrival order to feed the estimate: a missing group is a hole once the reader reports it as a gap
* and the media after it does not continue the timeline, and never waited for here. Its `delay.max` is then how long a hole is waited for, so size it to
* what the player can still play late. `track` only feeds the arrival estimate, read frame by frame
* as it lands, so give it a `delay.max` reaching as far as lateness can move the estimate.
*/
Expand Down Expand Up @@ -246,6 +246,9 @@ export class Consumer {
#mediaGroup?: bigint;
// Media went missing since the last item read, so the next one is preceded by a gap.
#lost = false;
// The ordered reader gave up on keys since the last item read: a loss only if the next item does
// not continue the timeline, since a publisher keying groups by timestamp never produced most keys.
#gapped?: { from: bigint; to: bigint };
// The furthest presentation point read so far: max(timestamp + duration), endpoints included.
// Undefined until the first item, so a loss before it is no gap.
#edge?: number;
Expand Down Expand Up @@ -863,8 +866,9 @@ export class Consumer {
const nextStart = next?.frames.at(0)?.timestamp.asMicros();
if (!next || nextStart === undefined || active.end < nextStart) return false;

// Routine for a group of one frame whose stream finishes after the next one starts, and nothing
// is lost, so it says nothing.
this.#groups.shift();
console.warn(`skipping covered group: ${active.consumer.start} -> ${next.consumer.start}`);
this.#pass(active, next.consumer.start);

this.#discard(active);
Expand Down Expand Up @@ -1086,7 +1090,10 @@ export class Consumer {
) {
// What an ordered reader gives up it reports as a gap; a key it skips without one was never a
// group. A gap an empty group carries still marks the next frame, which follows the loss.
if (!contiguous && (decided ? head.gap : !empty)) {
if (decided && head.gap && !contiguous) {
// The head's first frame may not have landed yet, so the next item read decides.
this.#gapped = { from: this.#gapped?.from ?? this.#active, to: head.consumer.start };
} else if (!decided && !contiguous && !empty) {
this.#lost = true;
if (!startup) {
console.warn(
Expand Down Expand Up @@ -1318,8 +1325,14 @@ export class Consumer {
#emit(read: Exclude<Read, { kind: "gap" }>): Read {
const resume = (read.kind === "frame" ? read.frame.timestamp : read.end).asMicros();
const edge = this.#edge;
const lost = this.#lost;
const gapped = this.#gapped;
this.#gapped = undefined;
let lost = this.#lost;
this.#lost = false;
if (gapped && edge !== undefined && !ptsContiguous(edge, resume)) {
lost = true;
console.warn(`skipping missing groups: track=${this.#track.name} ${gapped.from} -> ${gapped.to}`);
}
this.#advance(read.kind === "frame" ? resume + (read.frame.duration?.asMicros() ?? 0) : resume);
if (!lost || edge === undefined) return read;

Expand Down
21 changes: 21 additions & 0 deletions js/watch/src/audio/replay.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,27 @@ describe("replay", () => {
expect(warn).toHaveBeenCalled();
});

// A publisher keying groups by timestamp never uses the keys between them, so only keys around
// missing media are owed. Media that overlaps the group below counts as back to back, as the consumer
// reads it.
it("waits out only the keys around missing media when groups are keyed by timestamp", async () => {
const options = { rate: RATE, config: CONFIG, delay: Duration.millis(100), duration: observed(10) };
const keyed = (duration: number) =>
paced(10, () => 30).map((arrival) => ({ ...arrival, group: arrival.timestamp * 1000, duration }));
// Audio heard, in ms: owed keys nobody sends would starve the whole replay.
const heard = async (trace: Arrival[]) => {
let samples = 0;
for await (const { output } of replay(trace, options)) samples += output.filter((v) => v !== 0).length;
return (samples * 1000) / RATE;
};
const whole = await heard(keyed(20));
expect(whole).toBeGreaterThan(9000);
expect(await heard(keyed(22))).toBe(whole);
expect(await underruns(keyed(20), options)).toBe(0);
const missing = keyed(20).filter((arrival) => arrival.timestamp !== 5000);
expect(await underruns(missing, options)).toBeGreaterThan(0);
});

it("leaves a frame missing inside a group as missing audio", async () => {
// Five frames a group, delivered on time, with one from the middle of a group never sent.
const trace = paced(10, () => 30)
Expand Down
27 changes: 26 additions & 1 deletion js/watch/src/audio/replay.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
/**
* Replay a recorded arrival trace through the player's consumer and ring on a simulated clock.
*
* Every frame is written into its group at the instant the trace says it arrived, and the player's
* Every frame is written into its group at the instant the trace says it arrived, the keys a
* timestamp-keyed publisher skipped declared as its group lands, and the player's
* real `Container.Consumer` decides what to deliver and skip, at the max delay the decoder gives it,
* reading the ordered reader that decides each hole at the budget the decoder gives that. Delivered frames go into the real ring, which carries on through a gap the way the decoder's
* does: a gap restarts only the codec. One render quantum is read every quantum's worth of that same
Expand Down Expand Up @@ -38,6 +39,9 @@ export const QUANTUM = 128;
*/
const WIRE_MAX_DELAY = null;

/** How far, in ms, a group may start past the end of the one below and still read as back to back, as the consumer allows. */
const CONTIGUOUS = 1;

/** Any non-empty payload: an empty one is the legacy container's end marker. */
const PAYLOAD = new Uint8Array(1);

Expand Down Expand Up @@ -208,6 +212,25 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato
trace.forEach((arrival, i) => {
ends.set(arrival.group, i);
});
// A publisher keying groups by timestamp never produces the keys between two groups whose media
// is back to back, and each GROUP's Previous Start tells the subscriber so as it lands. Keys
// between media that is not are owed, since a group there went missing.
const spans = new Map<number, { start: number; end: number }>();
for (const { group, timestamp, duration } of trace) {
const span = spans.get(group);
spans.set(group, {
start: Math.min(span?.start ?? timestamp, timestamp),
end: Math.max(span?.end ?? timestamp, timestamp + duration),
});
}
const keys = [...spans.keys()].sort((a, b) => a - b);
const skipped = new Map<number, number>();
keys.forEach((key, i) => {
const below = keys[i - 1];
if (below === undefined || below + 1 === key) return;
const end = spans.get(below)?.end ?? 0;
if ((spans.get(key)?.start ?? 0) <= end + CONTIGUOUS) skipped.set(key, below + 1);
});

const lookahead = Duration.millis(options.lookahead ?? 0);
const buffered = !lookahead.isZero();
Expand Down Expand Up @@ -361,6 +384,8 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato
}
let group = groups.get(arrival.group);
if (!group) {
const from = skipped.get(arrival.group);
if (from !== undefined) track.skipGroups(BigInt(from), BigInt(arrival.group));
group = track.createGroup(BigInt(arrival.group));
groups.set(arrival.group, group);
}
Expand Down
31 changes: 29 additions & 2 deletions js/watch/src/audio/terminal.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -140,12 +140,39 @@ test("a gap re-primes the decoder on the warmup before it, so the audio after it
expect(terminal.primes(opus(timestamp))).toBe(true);
expect(terminal.span(opus(timestamp)).frames).toBe(0);
}
// The overlap the pre-skip carries over the restart starts before the frame, so only it is trimmed.
// An Opus packet decodes to the encoder's lookahead before its timestamp, so the frame after the gap
// plays whole from a pre-skip early, as it would have with no gap.
expect(terminal.primes(opus(1_000_000))).toBe(false);
expect(terminal.span(opus(1_000_000))).toEqual({ timestamp: 1_000_000, frameOffset: 312, frames: 648 });
expect(terminal.span(opus(1_000_000))).toEqual({ timestamp: 993_500, frameOffset: 0, frames: 960 });
expect(terminal.span(opus(1_020_000))).toEqual({ timestamp: 1_013_500, frameOffset: 0, frames: 960 });
});

// A reader can report a gap where nothing went missing, such as over keys a publisher never used.
test("a gap with nothing missing leaves no hole in Opus output", () => {
const terminal = new Terminal();
terminal.clear(312, 80_000);
const ends: number[] = [];
const starts: number[] = [];
const play = (timestamp: number) => {
if (terminal.primes(opus(timestamp))) return;
const span = terminal.span(opus(timestamp));
if (span.frames === 0) return;
starts.push(span.timestamp);
ends.push(span.timestamp + (span.frames * 1_000_000) / 48_000);
};
for (let i = 0; i < 10; i++) {
terminal.update(frame(BigInt(i), i * 20_000));
terminal.decoded(packet(i * 20_000), Duration.micros(20_000));
play(i * 20_000);
}
expect(terminal.update(gap)).toBe(true);
terminal.update(frame(10n, 200_000));
for (const { timestamp } of terminal.preroll()) play(timestamp);
play(200_000);
// Output is back to back across the restart.
expect(starts.at(-1)).toBe(ends.at(-2));
});

// A decoder that coalesces outputs can hand back the pre-roll and the frame after the gap together.
test("output reaching past the pre-roll holds the audio after the gap", () => {
const terminal = new Terminal();
Expand Down
4 changes: 3 additions & 1 deletion js/watch/src/audio/terminal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ export class Terminal {
this.#epoch ??= timestamp;
if (this.#prime) {
const primed = this.#reprime(timestamp);
this.#before = timestamp + Math.max(0, this.#warmup - primed);
// On the timeline `span` maps output onto, where this frame's output lands: a pre-skip early,
// since an Opus packet decodes to the encoder's lookahead before its timestamp.
this.#before = timestamp - this.delay + Math.max(0, this.#warmup - primed);
this.#prime = false;
}
return reset;
Expand Down
59 changes: 33 additions & 26 deletions quest/m2/audio-playout/browser-audio-hole-skip.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# [M] Browser audio's ordered reader grades no worse than before #8799
# [S] Browser audio's ordered reader grades no worse than before #8799 on the runner

## Goal

The Chromium `bursty` and `congestion` rows of `just test audio-quality`, with
browser audio reading its holes from `@moq/net`'s ordered reader, grade no
worse than before [moq-dev/moq#8799](https://github.com/moq-dev/moq/pull/8799)
gave the audio consumer the player's clock, measured on the nightly runner,
and the one full starvation seen locally is explained or fixed.
on the nightly runner, and no scheduled night shows a browser row starving
(`stalled_share` near 1, or a playhead holding still while the render clock
runs) that this quest has not explained.

## Plan

Expand All @@ -16,32 +17,38 @@ audio hole, through an ordered reader whose budget is the ring's size
the container consumer; keep the playhead rule and the separate wait).

[moq-dev/moq#9068](https://github.com/moq-dev/moq/pull/9068) moved browser
audio onto the ordered reader: `Ordered.nextItem` returns a gap for each
half-open range of keys it gives up on, audio reads through a second
subscription whose budget is what the ring holds plus the stretch bound and
the warmup, and the audio container consumer plays each gap as its hole. Firefox's
`after resume` window stays audible.

Its local Chromium measurement, three runs each of the PR, its merge base,
and `cf23b5ae04~1` (before #8799), could not grade the rows: the machine sat
at a load average of 30 to 70, and render underruns reached 580 a minute on
the merge base. The player-side numbers were level with the merge base on
`congestion`, which also trimmed about half the audio it did before #8799.
On `bursty` they were mixed, and one Opus `bursty` run of seven starved for
its last 7.5 s: the playhead held still while the render clock ran, which no
merge-base or pre-#8799 run did, and four reruns did not repeat it.
audio onto the ordered reader. Its first scheduled night (2026-11-18,
[44342367698](https://github.com/moq-dev/moq/actions/runs/44342367698)) put
112 tap underruns a minute on `chromium-opus-48000-bursty`, where every
earlier night had none: about one 4 to 6 ms hole a second, with no player
underrun and no media missing from the capture. The PR for this quest found
and fixed both causes. A reader gap over keys a timestamp-keyed publisher
never used was marked a loss whenever the consumer judged it before the
group's first frame landed. And every gap restarted the Opus codec with a cut
at the frame's timestamp rather than a pre-skip earlier, which dropped 6.5 ms
of real audio. A local live run reproduced the night's numbers without the
fix (136.8 underruns a minute) and measured zero with it. Replaying that
night's captures, now possible since the replay declares a timestamp-keyed
publisher's skipped keys, graded `bursty` level with the player before
#9068. `congestion` trimmed up to 350 ms a minute more audio there, with
stalled and silence shares level: the audio the old player never received
now arrives late and is skipped onto the clock.

The 7.5 s starvation one local run showed during #9068 was not reproduced.
The mechanism suspected for it, an active group that opens but never delivers
a frame, is bounded: the ordered subscription expires such a group one hole
wait after its successor lands, which a consumer test now holds. So a trace
does not need to record when each group opens.

Work:

- Grade the Chromium `bursty` and `congestion` rows from the nightly
runner's `audio-quality-run` artifacts after #9068 lands, against the
nights before it and before #8799.
- Find what starved that run. The audio container consumer
(`js/hang/src/container/consumer.ts`) in ordered mode has nothing that
passes an active group that opened but never delivers a frame until the
max buffer fills, which #8799's playhead covered. A trace recorded with
`--trace` holds only the frames that arrived, so replaying such a group
also needs the trace to record when each group opens.
- Grade the Chromium `bursty` and `congestion` rows from the
`audio-quality-run` artifacts of the first three scheduled nights after
the fix lands, against the nights before #8799 (2026-11-13 to 11-15) and
the PR's table. Opus `bursty` should file no tap underruns again.
- If a night shows a browser row starving, replay its capture
(`test/audio-quality/README.md`, "To replay a capture") and explain it
here.

## Related

Expand Down
Loading