Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions js/watch/src/audio/buffer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,39 @@ describe("Backpressure", () => {
});
}

// A ring re-anchored on a retained backlog trails the clock, which a live picture moved on, and
// skips onto it. Metered by its playhead, the backlog would come one floor and window per state
// report, each skipped as late on arrival.
it("releases at once what a ring trailing the clock past its bound skips", () => {
const gate = new Backpressure(1000, false, Duration.millis(200), Duration.millis(200));
const signal = new AbortController().signal;
gate.observe(ms(0), false, ms(0));
for (let t = 0; t < 240; t += 20) gate.inserted(ms(t), ms(t + 20));
const wait = gate.wait(ms(240), signal);
expect(Bun.peek.status(wait)).toBe("pending");
// Within the bound the ring stretches onto the clock, playing everything from its playhead.
gate.observe(ms(0), false, ms(200));
expect(Bun.peek.status(wait)).toBe("pending");
gate.observe(ms(0), true, ms(2000));
expect(Bun.peek.status(wait)).toBe("fulfilled");
// Released up to a floor and window past the clock, where the ring skips to.
for (let t = 260; t <= 2220; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled");
expect(Bun.peek.status(gate.wait(ms(2240), signal))).toBe("pending");
});

// The ring skips all the way onto the clock, not to the bound behind it, so a floor short of the
// bound must still be released past the clock or the skip lands on an empty ring and stalls.
it("releases past the clock with a floor short of the bound", () => {
// At 48 kHz the floor is the 50 ms asked for, not a render quantum's 128 ms at 1 kHz.
const gate = new Backpressure(48_000, false, Duration.millis(50), Duration.millis(200));
const signal = new AbortController().signal;
gate.observe(ms(0), false, ms(0));
for (let t = 0; t < 80; t += 20) gate.inserted(ms(t), ms(t + 20));
gate.observe(ms(0), true, ms(2000));
for (let t = 80; t <= 2060; t += 20) expect(Bun.peek.status(gate.wait(ms(t), signal))).toBe("fulfilled");
expect(Bun.peek.status(gate.wait(ms(2080), signal))).toBe("pending");
});

// The next frame past a hole lies far ahead of the playhead, which never nears it while the ring
// drains, so it is released by what the ring holds and the hole plays as silence.
it("releases the frame past a hole once the ring runs short", () => {
Expand Down Expand Up @@ -604,6 +637,37 @@ describe("AudioBuffer steering", () => {
}
});

// The gate reads the clock where the ring rendered its reported playhead, on the ring's timeline.
it("releases the backlog of a ring trailing the clock past its bound", () => {
let position = 1000;
const clock = started();
const { buffer, state, context } = fake(100, {
bound: 200,
clock: { position: () => position, out: { reference: new Signal<number | undefined>(0) } },
});
const signal = new AbortController().signal;
try {
context.currentTime = 1;
context.sounding = 1;
buffer.insert(ms(1000), [new Float32Array(160)]);
state(ms(1000), false, 0, 1);
const wait = buffer.wait(ms(1160), signal);
expect(Bun.peek.status(wait)).toBe("pending");
position = 1200;
state(ms(1000), false, 0, 1);
expect(Bun.peek.status(wait)).toBe("pending");
position = 3000;
state(ms(1000), true, 0, 1);
expect(Bun.peek.status(wait)).toBe("fulfilled");
// Released up to a floor and window past the clock, where the ring skips to.
for (const t of [2800, 2950, 3100]) expect(Bun.peek.status(buffer.wait(ms(t), signal))).toBe("fulfilled");
expect(Bun.peek.status(buffer.wait(ms(3250), signal))).toBe("pending");
} finally {
buffer.close();
clock.close();
}
});

// Before media zero's turn, as while an opening waits for its output to sound it, the clock pairs
// at its negative position rather than the zero it reads as.
it("pairs a clock short of media zero at its negative position", () => {
Expand Down
45 changes: 34 additions & 11 deletions js/watch/src/audio/buffer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ const JUMP = 10_000;
* full window to stretch. The frame's own timestamp does not matter: across a hole the next
* frame lies far ahead, and the ring plays the hole as silence. A stall keeps releasing by the
* playhead: the ring resumes on a cushion and slows down to rebuild, and media past the floor
* stays encoded, counted toward the level the ring decides on.
* stays encoded, counted toward the level the ring decides on. A playhead trailing the shared
* clock past the ring's bound, stalled or not, counts from the clock instead, where the ring
* skips to, so a backlog behind it goes at once rather than a window per state report.
*
* The playhead comes from the worklet's state messages.
*
Expand All @@ -54,6 +56,10 @@ export class Backpressure {

// The ring as last observed. The playhead is undefined until one is known on the current timeline.
#playhead?: number;
// Where the shared clock stood as the ring played the playhead, unset without a clock.
#clock?: number;
// How far the playhead may trail the clock before the ring skips onto it.
#bound = WINDOW;
// Whether the ring has played on the current timeline.
#started = false;

Expand All @@ -75,10 +81,10 @@ export class Backpressure {

#waiters: Array<{ timestamp: number; resolve: () => void }> = [];

constructor(rate: number, buffered: boolean, delay: Duration) {
constructor(rate: number, buffered: boolean, delay: Duration, bound = Duration.ZERO) {
this.#rate = rate;
this.#buffered = buffered;
this.setLatency(delay);
this.setLatency(delay, bound);
}

/** The ring's floor in samples: the part of the target held as decoded PCM. */
Expand All @@ -91,17 +97,25 @@ export class Backpressure {
return this.#offset;
}

/** Split `delay` between the floor and the hold. */
setLatency(delay: Duration): void {
/**
* Split `delay` between the floor and the hold, and note how far audio may trail the clock before
* the ring skips onto it, at least a stretch window as the ring's own threshold is.
*/
setLatency(delay: Duration, bound = Duration.ZERO): void {
this.#bound = Math.max(bound.asMicros(), WINDOW);
this.#floor = ringSamples(this.#rate, delay);
this.#headroom = (this.#floor / this.#rate) * 1_000_000;
this.#hold = ringHold(delay).asMicros();
this.#release();
}

/** Note the ring's playhead and whether it is stalled, and release every frame now due. */
observe(playhead: number | undefined, stalled: boolean): void {
/**
* Note the ring's playhead, whether it is stalled, and where the shared clock stood as it played
* there, and release every frame now due.
*/
observe(playhead: number | undefined, stalled: boolean, clock?: number): void {
this.#playhead = playhead;
this.#clock = clock;
if (playhead !== undefined && !stalled) this.#started = true;
this.#release();
}
Expand Down Expand Up @@ -176,6 +190,7 @@ export class Backpressure {
/** The ring was flushed for a new timeline: its playhead is unknown until the next frame anchors it. */
reset(): void {
this.#playhead = undefined;
this.#clock = undefined;
this.#started = false;
this.#decoding = [];
this.#inserted = undefined;
Expand Down Expand Up @@ -214,7 +229,13 @@ export class Backpressure {

#due(timestamp: number): boolean {
const playhead = this.#playhead;
if (this.#started && playhead !== undefined) return this.#reach(playhead) - playhead < this.#headroom + WINDOW;
if (this.#started && playhead !== undefined) {
// The ring skips onto the clock once it trails it past the bound, so it plays on from there.
// Within the bound it stretches onto the clock, playing everything from its playhead.
const clock = this.#clock;
const from = clock !== undefined && clock - playhead > this.#bound ? clock : playhead;
return this.#reach(from) - from < this.#headroom + WINDOW;
}

if (!this.#covered(timestamp)) return false;
// A live ring holds no more than the hold leaves uncovered, so it fills from everything due. A
Expand Down Expand Up @@ -345,7 +366,7 @@ export class AudioBuffer {
// The output latency the context already reports, so startup holds only what the ring must.
this.#observeOutput();
const held = this.held(props.delay);
this.#backpressure = new Backpressure(rate, buffered, held);
this.#backpressure = new Backpressure(rate, buffered, held, this.#bound);

const msg: Init = {
type: "init",
Expand Down Expand Up @@ -375,14 +396,16 @@ export class AudioBuffer {
this.#accelerated.set(Duration.millis(data.accelerated));
this.#silent.set(data.silent);
if (data.epoch === this.#epoch) {
this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled);
if (clock) {
const msg: Clock = {
type: "clock",
...this.#read(clock.position(), clock.out.reference.peek(), heard),
};
this.#worklet.port.postMessage(msg);
}
// The clock as the ring played the reported playhead, on the ring's timeline.
const at = this.#offset === undefined ? undefined : this.#offset + data.time * 1_000_000;
this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled, at);
}
}
});
Expand Down Expand Up @@ -505,7 +528,7 @@ export class AudioBuffer {
*/
setLatency(delay: Duration, bound: Duration = this.#bound): void {
this.#bound = bound;
this.#backpressure.setLatency(this.held(delay));
this.#backpressure.setLatency(this.held(delay), bound);
// The clock moves with the delay, so its offset starts over, from a reading sent with the terms.
this.#offset = undefined;
const sync = this.#sync;
Expand Down
13 changes: 7 additions & 6 deletions js/watch/src/audio/replay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -289,13 +289,14 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato
await settle();

let delay = sync.out.delay.peek();
const terms = () => ({
bound: bound({ delay, maxDelay: sync.out.maxDelay.peek(), stretch }).asMillis(),
});
const skip = () => bound({ delay, maxDelay: sync.out.maxDelay.peek(), stretch });
const terms = () => ({ bound: skip().asMillis() });
// The decode gate, and the worklet's ring on the floor it sets, the way `AudioBuffer` drives both.
const gate = new Backpressure(rate, buffered, delay);
const gate = new Backpressure(rate, buffered, delay, skip());
const ring = new AudioRingBuffer({ rate, channels: 1, latency: millis(rate, gate.floor), ...terms() });
const observe = () => gate.observe(ring.anchored ? ring.timestamp : undefined, ring.stalled);
// The ring's playhead is where the shared clock stands at the end of the quantum it read.
const observe = () =>
gate.observe(ring.anchored ? ring.timestamp : undefined, ring.stalled, sync.now()?.asMicros());

// Arrived but not yet decoded media, read or still in the consumer, is what covers the part of the
// target held encoded before playback starts.
Expand Down Expand Up @@ -412,7 +413,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato
if (!resolved.eq(delay)) {
delay = resolved;
observe();
gate.setLatency(delay);
gate.setLatency(delay, skip());
ring.setLatency(millis(rate, gate.floor), terms());
}

Expand Down
4 changes: 3 additions & 1 deletion quest/m0/interop-green/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -223,9 +223,11 @@ Scheduled runs
`skipping missing groups` on the audio track before the stall, so the audio
stalls past the 1039 ms bound after the route falls back. Reproduced
at 1 of 8 and 2 of 12 loaded local runs (the audio ring still stalled 600 ms after the first
audio group of the returned incumbent); not root-caused. Decided 2026-11-15 (AFK, recommendation, for review):
audio group of the returned incumbent). Decided 2026-11-15 (AFK, recommendation, for review):
`replacement` is quarantined by default on every engine again, and the child
[replacement audio](/quest/m0/interop-green/replacement-audio.md) root-causes and restores it.
Its first PR found the stall: the decode gate released the backlog behind the clock one window
per worklet report. What keeps the quarantine is in the child.

Public API: none. Wire: none.

56 changes: 32 additions & 24 deletions quest/m0/interop-green/replacement-audio.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# [M] Restore the replacement media case: audio after a route switch
# [M] Restore the replacement media case: audio across recurring gaps

## Goal

Expand All @@ -8,28 +8,36 @@ default media run runs it again. Remove its quarantine in `test/interop/clients/

## Plan

Evidence from the quest's PR, with the case at `ce616b52c` and `cbb742cec` on a loaded host
(`quest gate just test media --media-cases replacement` in a loop, with and without 24 busy
loops): it failed 1 of 8, then 2 of 12 runs, and none of 31 runs after a temporary `console.warn`
per decoded audio frame, so it is a timing race. Each failure is `audioStalled:true` in
`live incumbent fallback plays` (lite-06, lite-07) or `live replacement plays`, with the picture
painting.

One failing lite-07 fallback, timed from the log:

- 238 ms after the newer publisher stopped the incumbent's announce returned; its audio
subscription was ok at 298 ms. The incumbent's `fetch error: ... group=62 error=group is past
the budget` and `skipping missing groups: track=audio 0 -> 69` follow, so the first audio group
the reader got was 69, about a second of retained audio behind the incumbent's live picture
(the stalled ring's `audioTimestamp` 9507 ms against a picture at 10548 ms).
- 600 ms after that first group, at the 1039 ms bound, the ring was still stalled.

Open: where those 600 ms go. The decoder resets the ring on a changed instance
(`js/watch/src/audio/decoder.ts` `#runDecoder`), so the join snaps onto the clock; the question is
whether the ring waits for audio at the clock behind a backlog it must drop (then a join at the
live edge is the fix), or the starved page just decodes late (then the bound needs a load term
derived from the host, not a constant). Trace the ring's join and un-stall in the worklet on a
failing run; the largest-group replay at tune-in
([moq-dev/moq#8844](https://github.com/moq-dev/moq/pull/8844)) may overlap.
The switch itself is fixed: the quest's PR found that after a fallback or replacement the audio
waited behind the incumbent's retained backlog, which the live picture had moved the shared clock
past, and the decode gate released that backlog one floor and window per worklet state report.
The gate now counts from where the ring skips to. Measured on a loaded host, the two builds
interleaved: before, 15 of 161 switches took 600 ms or more and one missed the 1039 ms bound; after,
1 of 165 did and none missed it.

What still fails under that load, at about one run in ten either way:

- A mid-stream gap restarts the audio codec and trims the catalog `warmup` (80 ms of Opus) from the
first output after it (`js/watch/src/audio/terminal.ts` `update`, `rs/moq-audio/src/decode/consumer.rs`
`warmup_pending`). One lost 20 ms group costs 100 ms of audio, and when the starved viewer skips a
group more often than every 80 ms, as it does on a backlog read behind the clock, no output
survives at all: one failing lite-06 fallback stayed silent (`audioTimestamp` 0) past its bound
with every decoded output trimmed to zero frames. About 40% of the steady-state stalls in failing
`audio tone` windows followed such a trim.
- The rest are underruns of the case's fixed 50 ms delay while the host is near 0% idle. The default
run's other cases fail the same way at that load (`audio rendition switch back`, 1 of 4).

Recommended for the gap (a review can undo it): re-prime the restarted decoder on the encoded frames
just before the gap, restamped to end where the first frame after it starts, and drop their output,
so the warmup trims pre-roll rather than wanted audio. Chrome stamps an `AudioDecoder`'s output from
its first input, so the pre-roll must carry those restamped timestamps. moq-audio does the same, so
the native player keeps parity. Rejected: dropping the warmup trim after a gap reverses
[moq-dev/moq#5569](https://github.com/moq-dev/moq/pull/5569)'s decision to play only clean
output; keeping the decoder across a gap and restamping its output by input order assumes one
output per input, which AAC priming breaks.

Then rerun the replacement case on a loaded host and lift the quarantine if what fails is only the
starvation its sibling cases share. Measure with a loop like the PR's: `taskset` pins the run and
some busy loops to the same cores, so the load stays off other agents' cores.

Public API: none. Wire: none.
6 changes: 3 additions & 3 deletions test/audio-quality/budgets.json
Original file line number Diff line number Diff line change
Expand Up @@ -482,8 +482,8 @@
"stalled_share": 0,
"silence_share": 0,
"target_ms_p95": 80,
"av_lag_ms_p50": -0.3,
"av_lag_ms_p95": 9
"av_lag_ms_p50": 0.2,
"av_lag_ms_p95": 7.7
},
{
"runtime": "replay",
Expand Down Expand Up @@ -514,7 +514,7 @@
"stalled_share": 0.092,
"silence_share": 0.092,
"target_ms_p95": 80,
"av_lag_ms_p50": 1.7,
"av_lag_ms_p50": 2.3,
"av_lag_ms_p95": 32.1
}
]
Expand Down
6 changes: 3 additions & 3 deletions test/interop/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -551,9 +551,9 @@ playback delay, and one page sample plus one driver poll. Publisher preparation
is not part of it.

The replacement case is quarantined from the default run on every engine: under CPU
load it intermittently leaves the audio stalled past the switch bound after a route
switch or fallback (the viewer logs `skipping missing groups` on the audio track first),
and Firefox also showed intermittent missing tone samples. Run it explicitly with
load its audio intermittently goes silent, as each group the viewer skips restarts the
decoder and trims its 80 ms warmup, and Firefox also showed intermittent missing tone
samples. Run it explicitly with
`just test media --media-cases replacement`, or `--media-cases replacement --media-engines firefox` for Firefox;
its content, sync, switch, and fallback assertions remain active.

Expand Down
2 changes: 1 addition & 1 deletion test/interop/clients/js/media.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ const playerEngine = requestedPlayer as Engine;
const wants = (name: Case) => selected.has(name);
if (!native && values.cases === undefined) {
console.error(
"QUARANTINE\treplacement\tintermittent audio stall after a route switch under CPU load; see quest/m0/interop-green/README.md",
"QUARANTINE\treplacement\tintermittent audio silence after skipped groups under CPU load; see quest/m0/interop-green/replacement-audio.md",
);
}

Expand Down