diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 1e74a898e4e..e572ab997e1 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -49,9 +49,12 @@ group is observed when the stall clears. A missing group, one that has delivered no frame, is waited for only the playout delay plus any lookahead, since past that it would play late: a reordered backlog's holes waited out to the whole budget would delay the frames behind them, which the estimator would measure as -jitter. The browser player waits less still, only until its clock reaches the -group past the hole: a viewer resuming behind a replayed backlog, which arrives -newest first, would otherwise hold the live audio for the backlog's oldest group. A caller that estimates keeps the budget past the target, which is why +jitter. The browser player waits less still, only until the group past the hole +trails its clock by the stretch threshold, past which the ring would skip the +hole's audio anyway: a viewer resuming behind a replayed backlog, which arrives +newest first, would otherwise hold the live audio for the backlog's oldest group, +while a late burst that fills a hole a moment after the group past it lands +still plays it. A caller that estimates keeps the budget past the target, which is why `moq play` waits on a stalled audio group for the delay plus the estimate's 2 s ceiling. The observation is also only as honest as the caller is prompt: `read()` is timed when it resolves, so a player drains it diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index e2654ef0c52..c91d8777512 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -82,16 +82,19 @@ export interface ConsumerProps { predecessor?: Consumer; /** * The playout clock this reader feeds, such as the player's `Sync` (default: none). A missing - * group whose successor starts at or before the media time it presents now could only arrive - * too late to play, so it is skipped then rather than waited for: a reader resuming behind the - * live edge moves on once the clock passes the hole, not a whole {@link ConsumerProps.wait} later. + * group whose successor starts at or before the oldest media the clock still plays could only + * arrive too late to play, so it is skipped then rather than waited for: a reader resuming behind + * the live edge moves on once the clock passes the hole, not a whole {@link ConsumerProps.wait} later. */ playhead?: Playhead; } /** A playout clock, which a {@link Consumer} reads to tell when a missing group could no longer play. */ export interface Playhead { - /** The media time presenting now, or undefined while nothing presents. */ + /** + * The oldest media time the player still plays, or undefined while nothing presents: the media + * presenting now, less however far a player that catches up late media lets it trail. + */ now(): Duration | undefined; /** Changes whenever {@link Playhead.now} jumps rather than advancing with the wall clock: a re-anchor, a new delay, or a pause. */ readonly anchor: Getter; diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index f777dd1ecd8..63ffcc5be86 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -725,6 +725,36 @@ describe("AudioBuffer steering", () => { } }); + // The ring renders ahead of what sounds by the context's current lead, which a fall in the output + // latency shortens at once, where `output` holds its peak for five seconds. + it("reports how far the worklet renders ahead of what sounds now", () => { + const clock = started(); + const { buffer, state, context } = fake(300); + try { + context.baseLatency = 0.01; + context.outputLatency = 0.25; + context.currentTime = 1; + context.sounding = 0.74; + state(ms(0), false, 0); + expect(buffer.lead.peek()).toEqual(Duration.millis(260)); + // The output latency fell to 20 ms: the peak stays, and the lead follows. + context.outputLatency = 0.02; + context.currentTime = 2; + context.sounding = 1.97; + state(ms(0), false, 0); + expect(buffer.output.peek()).toEqual(Duration.millis(260)); + expect(buffer.lead.peek()).toEqual(Duration.millis(30)); + // A move of a couple of milliseconds, a render callback's worth of noise, is left alone. + context.currentTime = 3; + context.sounding = 2.972; + state(ms(0), false, 0); + expect(buffer.lead.peek()).toEqual(Duration.millis(30)); + } finally { + buffer.close(); + clock.close(); + } + }); + // Startup holds only what the ring must from the first message, before any state report. it("holds the delay less the output latency the context reports at construction", () => { const sent: Message[] = []; diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 98bf4983e74..93215eb37ce 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -3,8 +3,11 @@ import { Effect, type Getter, Signal } from "@moq/signals"; import { Peak, ringHold, ringSamples } from "./latency"; import type { Clock, Data, End, Init, Latency, Playing, Reading, Received, Reset, State, Truncate } from "./render"; -// The stretch's 30 ms processing window, in microseconds. -const WINDOW = 30_000; +// The stretch's 30 ms processing window, in microseconds. @internal +export const WINDOW = 30_000; + +// The least move, in milliseconds, of how far the worklet renders ahead of what sounds that is reported. +const LEAD_STEP = 5; // How far the clock's offset from the context's time may fall per state report: far past any drift // between the two clocks, and well short of the render callback a reading can land within. @@ -284,6 +287,14 @@ export class AudioBuffer { */ readonly output: Getter = this.#output; + readonly #lead = new Signal(Duration.ZERO); + /** + * How far the worklet renders ahead of what sounds now, as the context's output position last + * reported it, or `output` until it reports one. Unlike `output`, it follows an output latency + * that fell at once rather than holding its peak, and moves only by a few milliseconds at a time. + */ + readonly lead: Getter = this.#lead; + readonly #stalled = new Signal(true); /** Whether the ring is stalled (waiting to fill). */ readonly stalled: Getter = this.#stalled; @@ -365,6 +376,7 @@ export class AudioBuffer { // The output latency the context already reports, so startup holds only what the ring must. this.#observeOutput(); + this.#lead.set(this.#output.peek()); const held = this.held(props.delay); this.#backpressure = new Backpressure(rate, buffered, held, this.#bound); @@ -388,6 +400,7 @@ export class AudioBuffer { if (data?.type === "state") { const heard = this.#sounding(); this.#observeOutput(); + this.#observeLead(heard); this.#timestamp.set(this.#heardAt(data, heard)); this.#stalled.set(data.stalled); this.#underruns.set(data.underruns); @@ -448,6 +461,15 @@ export class AudioBuffer { this.#output.set(Duration.secs(context.baseLatency || 0).add(this.#heard)); } + // Fold in how far the context renders ahead of the output position sounding at `heard`. The + // position is only as fine as a render callback, so a smaller move is left alone, since a signal + // that notifies on every state report would wake whatever reads it as often. + #observeLead(heard: number | undefined): void { + const lead = + heard === undefined ? this.#output.peek() : Duration.secs(Math.max(0, this.#context.currentTime - heard)); + if (Math.abs(lead.asMillis() - this.#lead.peek().asMillis()) >= LEAD_STEP) this.#lead.set(lead); + } + // Where the playhead sounding at context time `heard` stands, from what the worklet rendered: the // report rendered last at or before it, advanced at real time up to the next. Without a position, // the reported playhead. diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 0f9b299970d..84409af017c 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -31,6 +31,7 @@ import { import { Handover } from "./handover"; import { bound, maxDelay, ringHold, target } from "./latency"; import { Order } from "./order"; +import { playable } from "./playhead"; // A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; @@ -190,6 +191,8 @@ export class Decoder { // Audio ring bridging the main thread and the worklet. #ring = new Signal(undefined); + // How far ahead of the clock the ring renders, so what it renders sounds when due. + #output: Getter; // The context, worklet, and ring are keyed on this alone. It outlives a rendition's absence, so // the queued tail plays out and a return with the same shape reuses the graph. @@ -283,6 +286,10 @@ export class Decoder { complete, }; }); + this.#output = this.#signals.computed((effect) => { + const ring = effect.get(this.#ring); + return ring ? effect.get(ring.lead) : undefined; + }); this.#track = this.sync.track({ target: playout, startup }); this.#signals.cleanup(() => this.#track.close()); this.#identity = this.#signals.computed((effect) => { @@ -947,7 +954,7 @@ export class Decoder { wait: this.sync.out.maxDelay, warmup: Duration.millis(config.warmup ?? 0).min(DELAY_MAX), predecessor: continues ? outgoing.consumer : undefined, - playhead: this.sync, + playhead: playable(this.sync, this.#output), }); effect.cleanup(() => consumer.close()); this.#phase.set({ ...phase, consumer: { path, codec: config.codec, consumer } }); diff --git a/js/watch/src/audio/playhead.test.ts b/js/watch/src/audio/playhead.test.ts new file mode 100644 index 00000000000..4a3f1048e18 --- /dev/null +++ b/js/watch/src/audio/playhead.test.ts @@ -0,0 +1,66 @@ +import { expect, it } from "bun:test"; +import { Duration } from "@moq/clock"; +import { Manual } from "@moq/clock/test"; +import { Signal } from "@moq/signals"; +import { Sync } from "../sync"; +import { playable } from "./playhead"; + +it("reads the clock the ring renders at, less the stretch it plays late audio within", async () => { + const clock = new Manual(); + const installed = clock.install(); + const sync = new Sync({ delay: Duration.millis(300), stretch: Duration.millis(200) }); + const span = new Signal<{ start: Duration; end: Duration } | null>(null); + const audio = sync.track({ target: new Signal(Duration.millis(300)), startup: span }); + const output = new Signal(Duration.millis(40)); + const playhead = playable(sync, output); + try { + await clock.advance(Duration.ZERO); + audio.received(Duration.ZERO); + span.set({ start: Duration.ZERO, end: Duration.millis(300) }); + await clock.advance(Duration.millis(1000)); + const now = sync.now(); + if (now === undefined) throw new Error("the clock never started"); + expect(playhead.now()).toEqual(now.add(Duration.millis(40)).saturatingSub(Duration.millis(200))); + + // The trail moving is a jump, which wakes a consumer waiting on a hole. + let jumped = false; + const dispose = playhead.anchor.subscribe(() => { + jumped = true; + }); + await clock.advance(Duration.ZERO); + jumped = false; + output.set(Duration.millis(60)); + await clock.advance(Duration.ZERO); + expect(jumped).toBe(true); + expect(playhead.now()).toEqual(now.add(Duration.millis(60)).saturatingSub(Duration.millis(200))); + dispose(); + } finally { + audio.close(); + sync.close(); + installed.close(); + } +}); + +it("applies the output latency before media zero's turn, when the ring already renders it", async () => { + const clock = new Manual(); + const installed = clock.install(); + const sync = new Sync({ delay: Duration.millis(300), lookahead: Duration.millis(1000), stretch: Duration.ZERO }); + const span = new Signal<{ start: Duration; end: Duration; lead?: Duration } | null>(null); + const audio = sync.track({ target: new Signal(Duration.millis(300)), startup: span }); + const playhead = playable(sync, new Signal(Duration.millis(120))); + try { + await clock.advance(Duration.ZERO); + audio.received(Duration.ZERO); + span.set({ start: Duration.ZERO, end: Duration.millis(180), lead: Duration.millis(120) }); + await clock.advance(Duration.millis(50)); + // 70 ms short of media zero's turn, which the clock reads as zero, the ring renders 50 ms, and + // plays audio trailing it by its 30 ms window even with the stretch off. + expect(sync.now()).toEqual(Duration.ZERO); + expect(sync.position()).toBe(-70); + expect(playhead.now()).toEqual(Duration.millis(20)); + } finally { + audio.close(); + sync.close(); + installed.close(); + } +}); diff --git a/js/watch/src/audio/playhead.ts b/js/watch/src/audio/playhead.ts new file mode 100644 index 00000000000..900233d0fe8 --- /dev/null +++ b/js/watch/src/audio/playhead.ts @@ -0,0 +1,34 @@ +import { Duration } from "@moq/clock"; +import type * as Container from "@moq/hang/container"; +import { Derived, type Getter, Signal } from "@moq/signals"; +import type { Sync } from "../sync"; +import { WINDOW } from "./buffer"; +import { bound } from "./latency"; + +/** + * The oldest media the ring still plays, which the audio consumer reads as its clock. The ring renders + * `output` (none until known) ahead of the shared clock, so what it renders now sounds when the clock + * reaches it, and stretches audio trailing that within its {@link bound} back onto it, at least one stretch window. So a missing + * group can still play until the group past it trails the ring by more than the bound, as when a late + * burst fills the hole a moment after the group past it lands. + * + * @internal Exported for the replay harness. + */ +export function playable(sync: Sync, output: Getter = new Signal(undefined)): Container.Playhead { + // How far the oldest playable media trails the clock, in ms; negative when the output leads by more. + const trail = new Derived( + [sync.out.delay, sync.out.maxDelay, sync.in.stretch, output], + (delay, maxDelay, stretch, output) => + // The ring skips only past its bound, never less than one stretch window, even with the stretch off. + Math.max(bound({ delay, maxDelay, stretch }).asMillis(), WINDOW / 1000) - (output?.asMillis() ?? 0), + ); + return { + now: () => { + // The signed position, since before media zero's turn the ring may already render it. + const position = sync.position(); + if (position === undefined) return undefined; + return Duration.millis(Math.max(0, position - trail.peek())); + }, + anchor: new Derived([sync.anchor, trail], (anchor, trail) => ({ anchor, trail })), + }; +} diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index 0297cdb37da..b40a259bd26 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -197,6 +197,23 @@ describe("replay against the shared clock", () => { const flushed = paced(60, (i) => 30 + (4 - (i % 5)) * 20); const fixed = { rate: RATE, config: CONFIG, delay: Duration.millis(40), duration: observed(60) }; + // A late burst lands out of order: the group before lands a moment after the one past it, which the + // clock already passed. The ring still plays audio trailing the clock within the stretch, so the + // consumer waits for that group rather than skipping a hole the burst fills; with the stretch off, + // the ring would skip it anyway. + it("keeps a hole a late burst fills once the clock passed the group past it", async () => { + // Groups 250 to 259 are held and released together at 5230 ms, 250 a millisecond after the rest. + const held = (i: number) => (i >= 250 && i < 260 ? 5230 - i * 20 + (i === 250 ? 1 : 0) : 30); + const trace = paced(10, held).sort((a, b) => a.at - b.at); + const options = { rate: RATE, config: CONFIG, delay: Duration.millis(100), duration: observed(10) }; + + // The consumer warns on every group it skips. + await lag(trace, options, 0); + expect(warn).not.toHaveBeenCalled(); + await lag(trace, { ...options, stretch: 0 }, 0); + expect(warn).toHaveBeenCalled(); + }); + it("underruns a fixed target short of the spread with the stretch off, rather than trailing the clock", async () => { const { lags, median, underruns, accelerated } = await lag(flushed, { ...fixed, stretch: 0 }, 30_000); expect(underruns).toBeGreaterThanOrEqual(flushed.length / 5); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 29a02c9dfcc..fcc98ceac75 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -2,8 +2,8 @@ * Replay a recorded arrival trace through the player's consumer and ring on a simulated clock. * * Every frame is written into its group at the instant the trace says it arrived, and the player's - * real `Container.Consumer` decides what to deliver, wait for, and skip, at the hole wait and max delay - * the decoder sets. Delivered frames go into the real ring, which carries on through a gap the way the decoder's + * real `Container.Consumer` decides what to deliver, wait for, and skip, at the hole wait, max delay, and + * clock the decoder gives it. Delivered frames go into the real ring, which carries on through a gap the way the decoder's * does: a gap restarts only the codec. One render quantum is read every quantum's worth of that same * clock, which is what the AudioWorklet does. A real {@link Sync} resolves the delay from the playout target the decoder * registers, measured by that same consumer from the recorded catalog config, and the ring and the @@ -26,6 +26,7 @@ import { type Delay, STRETCH, Sync } from "../sync"; import { Backpressure } from "./buffer"; import { frameDuration } from "./config"; import { maxDelay as audioMaxDelay, bound, ringHold, target } from "./latency"; +import { playable } from "./playhead"; import { AudioRingBuffer } from "./ring-buffer"; /** An AudioWorklet render quantum, in frames. */ @@ -258,6 +259,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato format: new Container.Legacy.Format("audio"), delay: new Derived([sync.out.delay, maxDelay], (min, max) => ({ min, max })), wait: sync.out.maxDelay, + playhead: playable(sync), }, ); diff --git a/quest/m2/audio-playout/browser-audio-hole-skip.md b/quest/m2/audio-playout/browser-audio-hole-skip.md index 773c76df121..e1f8ac7c76c 100644 --- a/quest/m2/audio-playout/browser-audio-hole-skip.md +++ b/quest/m2/audio-playout/browser-audio-hole-skip.md @@ -19,6 +19,18 @@ audio hole, through an ordered reader whose budget is the ring's size the container consumer, as this quest planned before; keep the playhead rule and the separate wait). +Interim rule, decided 2026-11-17 (maintainer, /quest-run, +[moq-dev/moq#8982](https://github.com/moq-dev/moq/pull/8982)): ✅ browser +audio ends a hole once the group past it trails the clock by the ring's +stretch bound (`playable(sync, output)` in `js/watch/src/audio/playhead.ts`), +the rule #8982 ships (rejected: main's bare clock with only the replay harness +fixed and re-budgeted; the estimate wait, which brought back the Firefox resume +silence). It is the only option with no measured regression in the browser, +and native audio keeps its estimate wait until +[moq-dev/moq#8984](https://github.com/moq-dev/moq/pull/8984) settles one rule. +The work below replaces `playable` with the ordered reader's `Gap`, so +`playable` and the audio consumer's clock go with it. + Why the playhead rule goes: natively, ending an audio hole once the clock reaches the group past it took `bursty` from 3 to 155 underruns a minute (`just test audio-quality --runtimes native --codecs opus --duration 40 @@ -55,11 +67,6 @@ Work: #8799's repro, `taskset -c 4-5 nix develop --command just test media --media-engines firefox --media-cases rename,pause`. -## Waiting - -- moq-dev/moq: [#8910](https://github.com/moq-dev/moq/pull/8910) merges, - which defines `Item` and `Gap` for the JS mirror to follow. - ## Related - [A hole is waited for by the subscription's delay](/quest/m2/delay-hole-wait.md) - settles the wait rule this moves into the transport diff --git a/quest/m2/delay-hole-wait.md b/quest/m2/delay-hole-wait.md index 2cabe356b79..d57368f874d 100644 --- a/quest/m2/delay-hole-wait.md +++ b/quest/m2/delay-hole-wait.md @@ -51,6 +51,14 @@ Facts found by the audit: lag p95 of 808 to 946 ms against 288 to 631 ms. The estimate wait gives every hole the estimate from the arrival of the group past it, which a late burst needs. +- Browser audio ends a hole once the group past it trails the clock by the + ring's stretch bound, the furthest the ring plays late audio + ([moq-dev/moq#8982](https://github.com/moq-dev/moq/pull/8982)): the + variant that lost natively on `congestion`. In the browser it graded no + worse than the bare clock or the wait alone on the Chromium `bursty` and + `congestion` rows, and the replays show the bare clock skipping holes a + burst fills 1 ms later. The estimate wait alone (no clock) left the Firefox + resume window silent longer (9 to 41 samples against 6 to 12). For the PR, settle the home of native audio's live estimate first. Two ways: