diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 31b6a6f9643..60f974bca53 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -441,7 +441,8 @@ already inside every number on this page: a publisher writing 100 ms of media at once produces arrivals whose relative delay reaches 100 ms less one frame, on a perfect network, with no help from the catalog. -So the advertised value is a **floor**, not an addend: +So the advertised value is a **floor**, not an addend, in the network's share of +the target (audio adds its output latency [at the speaker](#at-the-speaker)): ``` target = max(measured, advertised) + frame @@ -517,6 +518,30 @@ interval in progress and any open stall, folds into the continued histogram as if that interval closed at the switch, judged by the outgoing track's join: a switch most often follows an outage, and would otherwise drop the outage's one observation. +## At the speaker + +Audio is placed on the clock where it is heard, not where it is rendered. The +output takes time to sound what the player hands it, so an audio track's target +also counts that **output latency**, and video waits for it too, which lip sync +on that device needs: + +``` +target = max(measured, advertised) + frame + output +``` + +The player reads the clock at the moment the output sounds a sample and renders +the media due then; its buffer holds the target less the delay between render +and speaker. The output latency is the most the device reported over the last +5 s, since a host's report moves with where each read lands. + +| | Placed at | `output` | +| --- | --- | --- | +| Browser | `getOutputTimestamp()`, the context time sounding at a page instant, against which the clock is paired at the greatest offset, falling 0.1 ms a report | `baseLatency` plus the peak `outputLatency` | +| Native | the callback's playback timestamp | the host's reported delay plus one device period | + +A fixed browser delay is still taken literally, and a fixed native delay is +raised to the output latency, as [Fixed targets](#fixed-targets) describes. + ## Rise, fall, startup, and the ceiling There is **no separate rise or fall limiter**. The histogram is the only @@ -631,6 +656,12 @@ An impossible fixed target is refused, not silently adjusted. A negative delay, or one beyond the track's max age, is an error at configuration time. Supported or refused, not warned about and continued. +The one exception is the [output latency](#at-the-speaker), which no delay can +undercut. Natively, `moq play` raises a fixed delay below its device's output +latency to that latency, since audio cannot sound sooner. The browser keeps the +delay as given, so one shorter than the jitter plus the output latency +underruns rather than letting audio trail video. + In the browser this is the numeric `delay` setting; natively it is `decode::Options::delay.min` with `decode::Options::estimate` set to `false`. `estimate` defaults to `true`, which runs the estimator instead and never goes diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index eab22d06b2b..7c38a2d6f67 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -1,11 +1,39 @@ import { describe, expect, it } from "bun:test"; -import { Duration } from "@moq/clock"; +import { Duration, Instant, now } from "@moq/clock"; +import { Manual } from "@moq/clock/test"; import { type Getter, Signal } from "@moq/signals"; import { AudioBuffer, Backpressure } from "./buffer"; import type { Init, Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; const HOUR = 60 * 60 * 1000; + +// A manual clock moved past the page's origin, where a performance time of zero means none. +function started(): { close(): void } { + const manual = new Manual(); + const installed = manual.install(); + manual.jump(Duration.secs(1)); + return installed; +} + +// A context that reports its output position as sounding `sounding` now, or nothing until it is set. +function silent(): { + currentTime: number; + baseLatency: number; + outputLatency: number; + sounding?: number; + getOutputTimestamp(): AudioTimestamp; +} { + return { + currentTime: 0, + baseLatency: 0, + outputLatency: 0, + getOutputTimestamp() { + if (this.sounding === undefined) return { contextTime: 0, performanceTime: 0 }; + return { contextTime: this.sounding, performanceTime: now().since(Instant.ZERO).asMillis() }; + }, + }; +} const ms = (value: number) => value * 1000; // A buffered buffer over a real port, with the worklet's end of it left to the test. @@ -16,7 +44,7 @@ function open(): { close: () => void; } { const channel = new MessageChannel(); - const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { + const buffer = new AudioBuffer({ port: channel.port1, context: silent() } as unknown as AudioWorkletNode, { channels: 1, rate: 48_000, delay: Duration.millis(100), @@ -30,6 +58,7 @@ function open(): { const msg: State = { type: "state", timestamp: timestamp ?? 0, + time: 0, stalled, anchored: timestamp !== undefined, underruns: 0, @@ -56,7 +85,7 @@ function open(): { describe("AudioBuffer", () => { it("transfers PCM ownership and orders control messages after the data they change", async () => { const channel = new MessageChannel(); - const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { + const buffer = new AudioBuffer({ port: channel.port1, context: silent() } as unknown as AudioWorkletNode, { channels: 2, rate: 48_000, delay: Duration.millis(100), @@ -407,7 +436,7 @@ function fake( delay: number, props: { bound?: number; - clock?: { now(): Duration | undefined; out: { reference: Getter } }; + clock?: { position(): number | undefined; out: { reference: Getter } }; } = {}, ) { const sent: Message[] = []; @@ -416,7 +445,7 @@ function fake( start() {}, }); // The context's time in seconds, which the test advances. - const context = { currentTime: 0 }; + const context = silent(); const buffer = new AudioBuffer({ port, context } as unknown as AudioWorkletNode, { channels: 1, rate: 1000, @@ -425,10 +454,11 @@ function fake( ...(props.bound !== undefined && { bound: Duration.millis(props.bound) }), clock: props.clock, }); - const state = (timestamp: number, stalled: boolean, epoch: number) => { + const state = (timestamp: number, stalled: boolean, epoch: number, time = 0) => { const data: State = { type: "state", timestamp, + time, stalled, anchored: true, underruns: 0, @@ -452,7 +482,7 @@ describe("AudioBuffer past its decoded floor", () => { postMessage: (msg: Message) => sent.push(msg), start() {}, }); - new AudioBuffer({ port } as unknown as AudioWorkletNode, { + new AudioBuffer({ port, context: silent() } as unknown as AudioWorkletNode, { channels: 2, rate: 48_000, delay: Duration.millis(delay), @@ -517,43 +547,149 @@ describe("AudioBuffer steering", () => { } }); - // The context's time advances a render quantum at a time, so a reading of the clock lands up to a - // quantum late against it. The offset keeps the earliest, rises only as slowly as drift needs, and - // jumps when the delay moves the clock for real. - it("sends the clock at its earliest offset from the context's time", () => { + // A browser that reads its output position once per render callback reports it up to a callback + // early, so the offset keeps the greatest, falls only as slowly as drift needs, and moves whole when + // the delay or the reference moves the clock for real. + it("sends the clock against the context time sounding at its greatest offset", () => { let now: Duration | undefined = Duration.millis(1000); const reference = new Signal(0); - const { buffer, sent, state, context } = fake(100, { clock: { now: () => now, out: { reference } } }); - const report = (time: number, clock: number | undefined) => { + const clock = started(); + const { buffer, sent, state, context } = fake(100, { + clock: { position: () => now?.asMillis(), out: { reference } }, + }); + const report = (time: number, sounding: number, clock: number | undefined) => { context.currentTime = time; + context.sounding = sounding; now = clock === undefined ? undefined : Duration.millis(clock); state(ms(0), false, 0); }; try { - report(1, 1000); - // Read 2 ms late: the offset rises by only 100 us. - report(1.25, 1252); - report(1.5, 1500); + // The output sounds 100 ms behind the context's time. + report(1, 0.9, 1000); + // Read 2 ms early: the offset falls by only 100 us. + report(1.25, 1.152, 1250); + report(1.5, 1.4, 1500); // A delay change moved the clock 50 ms. - report(1.75, 1800); + report(1.75, 1.65, 1800); // The reference moved 5 ms earlier: the offset takes it whole, under the filter's. reference.set(-5); - report(2, 2055); - // A later reference rises only as fast as the filter does. + report(2, 1.9, 2055); + // A later reference moves it back, though it is no jump. reference.set(0); - report(2.25, 2300); - report(2.5, undefined); + report(2.25, 2.15, 2300); + report(2.5, 2.4, undefined); expect(sent.filter((msg) => msg.type === "clock")).toEqual([ - { type: "clock", timestamp: 1_000_000, time: 1 }, - { type: "clock", timestamp: 1_250_100, time: 1.25 }, - { type: "clock", timestamp: 1_500_000, time: 1.5 }, - { type: "clock", timestamp: 1_800_000, time: 1.75 }, - { type: "clock", timestamp: 2_055_000, time: 2 }, - { type: "clock", timestamp: 2_300_000, time: 2.25 }, + { type: "clock", timestamp: 1_100_000, time: 1 }, + { type: "clock", timestamp: 1_349_900, time: 1.25 }, + { type: "clock", timestamp: 1_600_000, time: 1.5 }, + { type: "clock", timestamp: 1_900_000, time: 1.75 }, + { type: "clock", timestamp: 2_155_000, time: 2 }, + { type: "clock", timestamp: 2_400_000, time: 2.25 }, { type: "clock", time: 2.5 }, ]); } finally { buffer.close(); + clock.close(); + } + }); + + // Before media zero's turn, as while an opening waits for its output to sound it, the clock pairs + // at its negative position rather than the zero it reads as. + it("pairs a clock short of media zero at its negative position", () => { + const clock = started(); + const { buffer, sent, state, context } = fake(100, { + clock: { position: () => -120, out: { reference: new Signal(0) } }, + }); + try { + context.sounding = 1; + state(ms(0), false, 0); + expect(sent.filter((msg) => msg.type === "clock")).toEqual([ + { type: "clock", timestamp: -1_120_000, time: 0 }, + ]); + } finally { + buffer.close(); + clock.close(); + } + }); + + // Before the output has played anything there is no position to pair the clock with. + it("sends no clock until the output reports a position", () => { + const { buffer, sent, state } = fake(100, { + clock: { position: () => 1000, out: { reference: new Signal(0) } }, + }); + try { + state(ms(0), false, 0); + expect(sent.filter((msg) => msg.type === "clock")).toEqual([{ type: "clock", time: 0 }]); + } finally { + buffer.close(); + } + }); + + // The ring renders ahead of the speaker by the output latency, so it holds the delay less that, + // decoded floor included, and the delay the clock holds counts the base latency and the output + // latency's recent peak. + it("holds the delay less the output latency it renders ahead by", () => { + const { buffer, sent, state, context } = fake(300); + try { + context.baseLatency = 0.01; + context.outputLatency = 0.12; + state(ms(0), false, 0); + context.outputLatency = 0.104; + state(ms(0), false, 0); + expect(buffer.output.peek()).toEqual(Duration.millis(130)); + buffer.setLatency(Duration.millis(300)); + // A delay short of the output latency still holds one callback. + expect(buffer.held(Duration.millis(50))).toEqual(Duration.millis(10)); + buffer.setLatency(Duration.millis(50)); + expect(sent.filter((msg) => msg.type === "latency")).toEqual([ + { type: "latency", latency: 180, bound: 0 }, + { type: "latency", latency: 128, bound: 0 }, + ]); + } finally { + buffer.close(); + } + }); + + // Startup holds only what the ring must from the first message, before any state report. + it("holds the delay less the output latency the context reports at construction", () => { + const sent: Message[] = []; + const port = Object.assign(new EventTarget(), { postMessage: (msg: Message) => sent.push(msg), start() {} }); + const context = silent(); + context.outputLatency = 0.12; + const buffer = new AudioBuffer({ port, context } as unknown as AudioWorkletNode, { + channels: 1, + rate: 1000, + delay: Duration.millis(300), + buffered: false, + }); + try { + expect(buffer.held(Duration.millis(300))).toEqual(Duration.millis(180)); + expect(sent[0]).toMatchObject({ type: "init", latency: 180 }); + } finally { + buffer.close(); + } + }); + + // The speaker sounds what was rendered an output latency ago, so the playback timestamp is where + // the rendered playhead stood then, advancing at real time, and holds through a stall. + it("reports the playhead sounding now", () => { + const clock = started(); + const { buffer, state, context } = fake(100); + try { + context.sounding = 1; + state(ms(5000), false, 0, 1.1); + // Nothing rendered at or before what is sounding: the first report, back by the difference. + expect(buffer.timestamp.peek()).toBe(ms(4900)); + context.sounding = 1.12; + state(ms(5050), false, 0, 1.15); + expect(buffer.timestamp.peek()).toBe(ms(5020)); + // The ring stalled at 5050: what sounds holds there once it has caught up. + context.sounding = 1.17; + state(ms(5050), true, 0, 1.2); + expect(buffer.timestamp.peek()).toBe(ms(5050)); + } finally { + buffer.close(); + clock.close(); } }); }); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 3d5165249b8..8ccf426db9c 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,18 +1,17 @@ -import { Duration } from "@moq/clock"; +import { Duration, Instant, now } from "@moq/clock"; import { Effect, type Getter, Signal } from "@moq/signals"; -import { ringHold, ringSamples } from "./latency"; +import { Peak, ringHold, ringSamples } from "./latency"; import type { Clock, Data, End, Init, Latency, Playing, Reading, Received, Reset, State, Truncate } from "./render"; // The stretch's 30 ms processing window, in microseconds. const WINDOW = 30_000; -// How far the clock's offset from the context's time may rise per state report: far past any drift +// How far the clock's offset from the context's time may fall per state report: far past any drift // between the two clocks, and well short of the render callback a reading can land within. -const RISE = 100; +const FALL = 100; // An offset that moves further than this moved for real, as a delay or a reference change moves it, -// rather than by where a reading landed within a render callback. Readings after a longer callback -// all land just after it, so they agree without the filter. +// rather than by where a reading landed within a render callback. const JUMP = 10_000; /** @@ -241,11 +240,20 @@ export class AudioBuffer { readonly rate: number; readonly channels: number; #worklet: AudioWorkletNode; + // The realtime context the worklet renders in, which reports where its output stands. + #context: AudioContext; readonly #timestamp = new Signal(0); - /** Current playback timestamp, as the worklet last reported it. */ + /** The playback timestamp sounding now, as of the worklet's last report. */ readonly timestamp: Getter = this.#timestamp; + readonly #output = new Signal(Duration.ZERO); + /** + * How long the output takes to sound what the ring renders: the context's base latency plus the + * highest output latency it reported over the last five seconds. + */ + readonly output: Getter = this.#output; + readonly #stalled = new Signal(true); /** Whether the ring is stalled (waiting to fill). */ readonly stalled: Getter = this.#stalled; @@ -278,12 +286,18 @@ export class AudioBuffer { #playing?: Getter; // How far audio may trail the shared clock before the ring skips ahead. #bound: Duration; + // The peak output latency, between the render and the speaker, which the ring does not hold. + #latency = new Peak(); + #heard = Duration.ZERO; + // What the worklet reported rendering, oldest first, back to what is sounding now. + #rendered: { time: number; timestamp: number }[] = []; // The shared clock the ring plays onto, and the reference it plays from. - #sync?: { now(): Duration | undefined; out: { reference: Getter } }; - // The shared clock less the context's time, in microseconds, as last sent. The context's time - // advances a render quantum at a time, so a reading lands up to a quantum late: the least offset is - // the reading taken nearest a quantum's start. It rises only slowly, as drift needs, unless it moves - // past any quantum, as a delay or reference change moves it. + #sync?: { position(): number | undefined; out: { reference: Getter } }; + // The shared clock less the context time sounding at the same instant, in microseconds, as last + // sent. A browser that reads its output position only once per render callback reports it up to a + // callback early, so the greatest offset is the reading taken nearest a callback's start. It falls + // only slowly, as drift needs, unless it moves past any callback, as a delay or reference change + // moves it. #offset?: number; // The shared clock's reference as last sent, in milliseconds. One that moved earlier stepped the // clock ahead, which the offset takes whole rather than as where a reading landed. @@ -303,7 +317,7 @@ export class AudioBuffer { /** Anchor to the first frame and hold the lookahead upstream until playback starts. */ buffered: boolean; /** The shared clock the ring plays onto, and the reference it plays from. */ - clock?: { now(): Duration | undefined; out: { reference: Getter } }; + clock?: { position(): number | undefined; out: { reference: Getter } }; /** Hold consumption until the shared startup floor is ready. */ playing?: Getter; }, @@ -311,12 +325,16 @@ export class AudioBuffer { const { channels, rate, buffered, playing, clock } = props; this.#playing = props.playing; this.#worklet = worklet; + this.#context = worklet.context as AudioContext; this.channels = channels; this.rate = rate; this.#bound = props.bound ?? Duration.ZERO; this.#sync = clock; - this.#backpressure = new Backpressure(rate, buffered, props.delay); + // The output latency the context already reports, so startup holds only what the ring must. + this.#observeOutput(); + const held = this.held(props.delay); + this.#backpressure = new Backpressure(rate, buffered, held); const msg: Init = { type: "init", @@ -336,7 +354,9 @@ export class AudioBuffer { this.#signals.event(worklet.port, "message", (ev: Event) => { const data = (ev as MessageEvent).data; if (data?.type === "state") { - this.#timestamp.set(data.timestamp); + const heard = this.#sounding(); + this.#observeOutput(); + this.#timestamp.set(this.#heardAt(data, heard)); this.#stalled.set(data.stalled); this.#underruns.set(data.underruns); this.#skips.set(data.skips); @@ -346,7 +366,10 @@ export class AudioBuffer { if (data.epoch === this.#epoch) { this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled); if (clock) { - const msg: Clock = { type: "clock", ...this.#read(clock.now(), clock.out.reference.peek()) }; + const msg: Clock = { + type: "clock", + ...this.#read(clock.position(), clock.out.reference.peek(), heard), + }; this.#worklet.port.postMessage(msg); } } @@ -356,6 +379,17 @@ export class AudioBuffer { worklet.port.start(); } + /** + * What the ring holds of a delay: the delay less the output latency it renders ahead by, though + * never less than one callback, the least it can render from. Its floor, its startup, and what + * stays encoded upstream all follow this, so a start never queues media the clock then skips. + * @internal + */ + held(delay: Duration): Duration { + const base = Duration.secs(this.#context.baseLatency || 0); + return delay.saturatingSub(this.#heard).max(base.min(delay)); + } + // The floor and bound as the worklet takes them. #terms(): Omit { return { @@ -364,24 +398,55 @@ export class AudioBuffer { }; } - // Where the shared clock stands, in the ring's coordinates, read together with the context's time - // so a late message misplaces neither. Each reading follows a state report, which lands just after - // the context renders a callback, when its time runs up to that callback ahead of the wall clock: - // the ring then trails the clock by up to one callback, the audio a callback pulls at once. - #read(now: Duration | undefined, reference: number | undefined): Reading { - const time = this.#worklet.context.currentTime; + // The context time sounding now, in seconds, or undefined before the output has played anything. + // The browser reports its output position with the instant it was heard, on the page's clock. + #sounding(): number | undefined { + const position = this.#context.getOutputTimestamp(); + if (!position.performanceTime || position.contextTime === undefined) return undefined; + return position.contextTime + (now().since(Instant.ZERO).asMillis() - position.performanceTime) / 1000; + } + + // Fold in the output latency the context reports now. `baseLatency` is the callback the context + // renders at once; `outputLatency`, where reported, the rest of the way to the speaker. + #observeOutput(): void { + const context = this.#context; + this.#heard = this.#latency.observe(Duration.secs(context.outputLatency || 0), now()); + this.#output.set(Duration.secs(context.baseLatency || 0).add(this.#heard)); + } + + // Where the playhead sounding at context time `heard` stands, from what the worklet rendered: the + // report rendered last at or before it, advanced at real time up to the next. Without a position, + // the reported playhead. + #heardAt(state: State, heard: number | undefined): number { + if (heard === undefined) { + this.#rendered = []; + return state.timestamp; + } + const rendered = this.#rendered; + rendered.push({ time: state.time, timestamp: state.timestamp }); + while (rendered.length > 1 && rendered[1].time <= heard) rendered.shift(); + const [at, next] = rendered; + const projected = at.timestamp + (heard - at.time) * 1_000_000; + return Math.max(0, next ? Math.min(projected, next.timestamp) : Math.min(projected, at.timestamp)); + } + + // Where the shared clock stands, in the ring's coordinates, against the context time sounding at + // the same instant, so the ring renders each sample to sound when the clock says it is due. The + // reading carries the context's time it was taken at, so a late message misplaces neither. + #read(position: number | undefined, reference: number | undefined, heard: number | undefined): Reading { + const time = this.#context.currentTime; const last = this.#reference; this.#reference = reference; - if (now === undefined) { + if (position === undefined || heard === undefined) { this.#offset = undefined; return { time }; } - // A reference moved earlier steps the clock, not where a reading landed, so the offset takes it whole. - const stepped = last !== undefined && reference !== undefined ? Math.max(0, last - reference) * 1000 : 0; - const offset = now.asMicros() - this.#backpressure.offset - time * 1_000_000; - const previous = this.#offset === undefined ? undefined : this.#offset + stepped; + // A reference move is a step of the clock, not where a reading landed, so the offset takes it whole. + const moved = last !== undefined && reference !== undefined ? (last - reference) * 1000 : 0; + const offset = position * 1000 - this.#backpressure.offset - heard * 1_000_000; + const previous = this.#offset === undefined ? undefined : this.#offset + moved; this.#offset = - previous !== undefined && Math.abs(offset - previous) < JUMP ? Math.min(offset, previous + RISE) : offset; + previous !== undefined && Math.abs(offset - previous) < JUMP ? Math.max(offset, previous - FALL) : offset; return { timestamp: this.#offset + time * 1_000_000, time }; } @@ -429,14 +494,14 @@ export class AudioBuffer { */ setLatency(delay: Duration, bound: Duration = this.#bound): void { this.#bound = bound; - this.#backpressure.setLatency(delay); + this.#backpressure.setLatency(this.held(delay)); // The clock moves with the delay, so its offset starts over, from a reading sent with the terms. this.#offset = undefined; const sync = this.#sync; const msg: Latency = { type: "latency", ...this.#terms(), - ...(sync && { clock: this.#read(sync.now(), sync.out.reference.peek()) }), + ...(sync && { clock: this.#read(sync.position(), sync.out.reference.peek(), this.#sounding()) }), }; this.#worklet.port.postMessage(msg); } @@ -497,6 +562,7 @@ export class AudioBuffer { this.#worklet.port.postMessage(msg); this.#backpressure.reset(); this.#offset = undefined; + this.#rendered = []; } /** diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 5cd6d5e9cd4..68533171728 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -33,7 +33,13 @@ let moduleLoaded: Promise = Promise.resolve(); class FakeContext extends EventTarget { state: AudioContextState = "running"; readonly currentTime = 0; + readonly baseLatency = 0; + readonly outputLatency = 0; readonly sampleRate: number; + // No output position yet, so the ring plays without a clock. + getOutputTimestamp(): AudioTimestamp { + return { contextTime: 0, performanceTime: 0 }; + } readonly audioWorklet = { addModule: () => moduleLoaded }; readonly calls = { suspend: 0, resume: 0, close: 0 }; constructor(options: AudioContextOptions) { @@ -113,6 +119,7 @@ class FakeWorklet { const state: State = { type: "state", timestamp: ring.timestamp, + time: 0, stalled: ring.stalled, anchored: ring.anchored, underruns: ring.underruns, diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index ab219d5a7b6..482139e59e4 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -68,7 +68,7 @@ export type DecoderOutput = { sampleRate: Signal; stats: Signal; - // Current playback timestamp from worklet + // The playback timestamp sounding at the speaker, as of the worklet's last report. timestamp: Signal; // Whether the audio buffer is stalled (waiting to fill). A silence the publisher declared, such as @@ -225,11 +225,14 @@ export class Decoder { const measured = effect.get(this.#measured); if (measured === undefined) return undefined; const delay = effect.get(this.source.out.config)?.delay; + const ring = effect.get(this.#ring); return target({ measured, advertised: effect.get(this.source.out.jitter), frame: effect.get(this.#frame), delay: delay !== undefined ? Duration.millis(delay) : undefined, + // Audio is placed on the clock where it is heard, so video waits for the output too. + output: ring ? effect.get(ring.output) : undefined, }); }); const startup = this.#signals.computed((effect) => { @@ -244,8 +247,17 @@ export class Decoder { for (const range of effect.get(this.#queued) ?? []) { queued = queued.add(range.end.saturatingSub(range.start.max(span.end))); } - const hold = ringHold(effect.get(this.sync.out.delay)); - return { start: span.start, end: span.end.add(queued.min(hold)) }; + // The ring holds the delay less the output latency, which the speaker covers: the opening + // sounds that much after it renders, so it comes due then rather than being skipped. + const delay = effect.get(this.sync.out.delay); + const ring = effect.get(this.#ring); + if (ring) effect.get(ring.output); + const held = ring ? ring.held(delay) : delay; + return { + start: span.start, + end: span.end.add(queued.min(ringHold(held))), + lead: delay.saturatingSub(held), + }; }); this.#track = this.sync.track({ target: playout, startup }); this.#signals.cleanup(() => this.#track.close()); @@ -511,7 +523,9 @@ export class Decoder { if (!ring) return; // The ring plays onto the shared clock, slowing down onto a rise and speeding up onto a fall, - // and skips onto it only audio trailing it past the stretch threshold. + // and skips onto it only audio trailing it past the stretch threshold. It holds the delay less + // the output latency, which moves its floor even under a fixed delay. + effect.get(ring.output); const budget = this.#budget(effect); ring.setLatency(budget.delay, bound(budget)); } diff --git a/js/watch/src/audio/latency.test.ts b/js/watch/src/audio/latency.test.ts index 20e39dffdcb..4d8f0578d80 100644 --- a/js/watch/src/audio/latency.test.ts +++ b/js/watch/src/audio/latency.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "bun:test"; -import { Duration } from "@moq/clock"; -import { ringCapacity, ringHold, ringSamples, target } from "./latency"; +import { Duration, Instant } from "@moq/clock"; +import { Peak, ringCapacity, ringHold, ringSamples, target } from "./latency"; const ms = (value: number) => Duration.millis(value); @@ -9,6 +9,10 @@ describe("target", () => { expect(target({ measured: ms(40), frame: ms(20) })).toEqual(ms(60)); }); + it("adds the output latency, since audio is placed where it is heard", () => { + expect(target({ measured: ms(40), frame: ms(20), output: ms(42) })).toEqual(ms(102)); + }); + it("floors the measured term at the advertised span instead of adding it", () => { expect(target({ measured: ms(40), advertised: ms(200), frame: ms(20) })).toEqual(ms(220)); expect(target({ measured: ms(300), advertised: ms(200), frame: ms(20) })).toEqual(ms(320)); @@ -57,3 +61,16 @@ describe("ringHold", () => { expect(ringHold(ms(500))).toEqual(Duration.micros(0)); }); }); + +describe("Peak", () => { + // A lower reading holds until the higher one ages out of the window; a higher one takes over at once. + it("holds the highest output latency for five seconds", () => { + const at = (secs: number) => Instant.ZERO.saturatingAdd(Duration.secs(secs)); + const peak = new Peak(); + expect(peak.observe(ms(120), at(0))).toEqual(ms(120)); + expect(peak.observe(ms(104), at(1))).toEqual(ms(120)); + expect(peak.observe(ms(112), at(5))).toEqual(ms(120)); + expect(peak.observe(ms(108), at(6))).toEqual(ms(112)); + expect(peak.observe(ms(128), at(7))).toEqual(ms(128)); + }); +}); diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index 060b010ba6e..b1e2f4aa869 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -1,4 +1,4 @@ -import { Duration } from "@moq/clock"; +import { Duration, type Instant } from "@moq/clock"; /** The terms of the audio playout target. */ export interface Target { @@ -10,11 +10,13 @@ export interface Target { frame?: Duration; /** The catalog `delay`: how far this rendition trails the broadcast's others, if any. */ delay?: Duration; + /** How long the output takes to sound what the ring renders, once known. */ + output?: Duration; } /** * The "auto" playout target: the measured term floored by the advertised span, plus one frame, plus - * the rendition's catalog delay. + * the rendition's catalog delay, plus the output latency. * * The advertised span is a floor rather than an addend, because the receiver's measurement already * contains the publisher's batching. The catalog delay is an addend, because the measurement is @@ -23,7 +25,29 @@ export interface Target { */ export function target(props: Target): Duration { const floored = props.measured.max(props.advertised ?? Duration.ZERO); - return floored.add(props.frame ?? Duration.ZERO).add(props.delay ?? Duration.ZERO); + return floored + .add(props.frame ?? Duration.ZERO) + .add(props.delay ?? Duration.ZERO) + .add(props.output ?? Duration.ZERO); +} + +// How long the highest reported output latency holds. A browser's report moves with where each read +// lands, so the delay the clock holds for it follows the peak rather than every reading, and falls +// once a lower latency, as after a switch off a Bluetooth headset, has held this long. +const OUTPUT_WINDOW = Duration.secs(5); + +/** The highest output latency reported within the last five seconds. */ +export class Peak { + // Each reading that no later one is at least as high as, oldest first, so the first is the peak. + #readings: { at: Instant; latency: Duration }[] = []; + + /** Fold in the latency reported at `at`, returning the peak. */ + observe(latency: Duration, at: Instant): Duration { + while (this.#readings.length > 0 && (this.#readings.at(-1)?.latency.le(latency) ?? false)) this.#readings.pop(); + this.#readings.push({ at, latency }); + while (this.#readings.length > 1 && at.since(this.#readings[0].at).gt(OUTPUT_WINDOW)) this.#readings.shift(); + return this.#readings[0].latency; + } } // The arrival estimate's ceiling, `BUCKET * BUCKETS` in @moq/hang's jitter estimator: no lateness past diff --git a/js/watch/src/audio/render-worklet.test.ts b/js/watch/src/audio/render-worklet.test.ts index 7c924218a08..374f2e01007 100644 --- a/js/watch/src/audio/render-worklet.test.ts +++ b/js/watch/src/audio/render-worklet.test.ts @@ -46,7 +46,14 @@ function renderer() { sent: render.sent, read(samples: number) { const output = new Float32Array(samples); - expect(render.process([], [[output]], {})).toBe(true); + // The context's time, at zero unless a test runs it. + const time = Object.getOwnPropertyDescriptor(globalThis, "currentTime"); + if (!time) Object.defineProperty(globalThis, "currentTime", { configurable: true, value: 0 }); + try { + expect(render.process([], [[output]], {})).toBe(true); + } finally { + if (!time) Reflect.deleteProperty(globalThis, "currentTime"); + } return output; }, }; @@ -82,6 +89,8 @@ describe("render dispatch", () => { type: "state", epoch: 7, timestamp: 560_000, + // The end of the 20-sample quantum at 1 kHz, at a context time of zero. + time: 0.02, stalled: false, anchored: true, underruns: 0, diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 92e915deda2..e60a16c353e 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -108,6 +108,7 @@ class Render extends AudioWorkletProcessor { const state: State = { type: "state", timestamp: ring.timestamp, + time: currentTime + output[0].length / ring.rate, stalled: ring.stalled, anchored: ring.anchored, underruns: ring.underruns, diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index 9d649ddf16d..13a26be431c 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -61,13 +61,14 @@ export interface Latency extends Terms { } /** - * Where the shared clock stands, for the ring to time-stretch onto. The clock runs at real time, so one - * reading maps it onto the worklet's own time until the next. + * Where the shared clock stands, for the ring to time-stretch onto, at the context time the output + * sounds then. The clock runs at real time, so one reading maps it onto the worklet's own time until + * the next. */ export interface Reading { - // The clock's media time at context time `time`, in the ring's coordinates, unset while it has none. + // The media time due at context time `time`, in the ring's coordinates, unset while the clock has none. timestamp?: number; - // The AudioContext's `currentTime` when the clock was read, in seconds. + // A context time in seconds. time: number; } @@ -86,6 +87,8 @@ export interface Received { export interface State { type: "state"; timestamp: number; + // The context time `timestamp` was rendered at, in seconds: the end of the quantum just read. + time: number; stalled: boolean; // Whether `timestamp` is on the current timeline: it reads 0 until a frame anchors the ring. anchored: boolean; diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 9e5cf9f09ef..6fd55dd37eb 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -418,6 +418,33 @@ describe("buffered startup", () => { } }); } + // Audio's output sounds its opening a while after it renders, so that wait counts toward the + // delay, and the opening, even at media zero, comes due as it first sounds instead of being skipped. + it("holds the opening's turn back by the time its output takes to sound it", async () => { + const clock = new Manual(); + const installed = clock.install(); + const sync = new Sync({ delay: Duration.millis(300), lookahead: Duration.millis(1000) }); + const span = new Signal<{ start: Duration; end: Duration; lead?: Duration } | null>(null); + const audio = sync.track({ target: new Signal(Duration.millis(300)), startup: span }); + try { + await clock.advance(Duration.ZERO); + audio.received(Duration.ZERO); + span.set({ start: Duration.ZERO, end: Duration.millis(170), lead: Duration.millis(120) }); + await clock.advance(Duration.ZERO); + expect(sync.now()).toBeUndefined(); + span.set({ start: Duration.ZERO, end: Duration.millis(180), lead: Duration.millis(120) }); + await clock.advance(Duration.ZERO); + // Short of media zero, which the clock reads as zero until the opening's turn. + expect(sync.now()).toEqual(Duration.ZERO); + await clock.advance(Duration.millis(150)); + expect(sync.now()).toEqual(Duration.millis(30)); + } finally { + audio.close(); + sync.close(); + installed.close(); + } + }); + it("releases startup waits on abort, reset, and close without starting playback", async () => { const clock = new Manual(); const installed = clock.install(); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index a162d5a3304..43c636858e6 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -87,8 +87,12 @@ export type SyncOutput = { maxDelay: Signal; }; -/** A decoded span from the opening of a track's timeline. @internal */ -export type Span = { start: Duration; end: Duration }; +/** + * What a track holds from the opening of its timeline, and how long its output takes to sound the + * opening once released, which counts toward the delay and holds the opening's turn back as long. + * @internal + */ +export type Span = { start: Duration; end: Duration; lead?: Duration }; /** What a track holds in the shared clock. @internal */ export type TrackProps = { @@ -269,11 +273,17 @@ export class Clock { } now(): Duration | undefined { + const position = this.position(); + // Media time is nonnegative, so a playhead before the first frame's turn reads as zero. + return position === undefined ? undefined : Duration.millis(Math.max(0, position)); + } + + // The media time due now in milliseconds, negative before media zero's turn. + position(): number | undefined { if (!this.playing.peek()) return undefined; const reference = this.out.reference.peek(); if (reference === undefined) return undefined; - // Media time is nonnegative, so a playhead before the first frame's turn reads as zero. - return Duration.millis(Math.max(0, wall() - reference - this.out.delay.peek().asMillis())); + return wall() - reference - this.out.delay.peek().asMillis(); } async wait(part: Part, timestamp: Duration, signal?: AbortSignal): Promise { @@ -443,22 +453,25 @@ export class Sync { } if (clock.phase.peek() === "playing") return; const ready = spans.flatMap(({ part, span }) => (span ? [{ part, span }] : [])); + const lead = (span: Span) => span.lead ?? Duration.ZERO; if ( reference === undefined || ready.length !== spans.length || - ready.some(({ span }) => span.end.saturatingSub(span.start).lt(delay)) + ready.some(({ span }) => span.end.add(lead(span)).saturatingSub(span.start).lt(delay)) ) { clock.phase.set("starting"); return; } - // The track with the earliest opening holds the reference it anchors. - const earliest = ready.reduce((a, b) => (b.span.start.lt(a.span.start) ? b : a)); - const first = earliest.span.start; + // The track whose opening sounds earliest holds the reference it anchors. + const opens = (span: Span) => span.start.asMillis() - lead(span).asMillis(); + const earliest = ready.reduce((a, b) => (opens(b.span) < opens(a.span) ? b : a)); + const first = opens(earliest.span); clock.owner = earliest.part; // Anchor at the retained opening, not at its arrival: startup may have waited on another track. // Every track's own anchor moves by the same offset, keeping the gaps between tracks that - // playing would have held, so a later handoff moves the playhead only by that gap. - const anchored = wall() - first.asMillis() - delay.asMillis(); + // playing would have held, so a later handoff moves the playhead only by that gap. An opening + // whose output takes a while to sound comes due that much later. + const anchored = wall() - first - delay.asMillis(); for (const part of clock.parts.peek()) { if (part.anchor !== undefined) part.anchor += anchored - reference; } @@ -518,6 +531,15 @@ export class Sync { return this.#clock.now(); } + /** + * {@link Sync.now} in milliseconds, negative rather than zero before media time zero's turn, as + * when an opening waits for its output to sound it. + * @internal + */ + position(): number | undefined { + return this.#clock.position(); + } + /** Stop and release everything this holds. */ close() { this.#clock.closed.set(true); diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 95fe8762473..cee096e187c 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -56,5 +56,5 @@ audit ([moq-dev/moq#8648](https://github.com/moq-dev/moq/pull/8648)), the size report's output ([moq-dev/moq#8654](https://github.com/moq-dev/moq/pull/8654)), the firefox resume ([moq-dev/moq#8635](https://github.com/moq-dev/moq/pull/8635)), and the chromium `av_lag` rows -([output latency](/quest/m2/audio-playout/output-latency.md)). +([moq-dev/moq#8676](https://github.com/moq-dev/moq/pull/8676)). diff --git a/quest/m2/audio-playout/README.md b/quest/m2/audio-playout/README.md index 13321dddbd4..668de17641e 100644 --- a/quest/m2/audio-playout/README.md +++ b/quest/m2/audio-playout/README.md @@ -97,9 +97,11 @@ Order: the 200 ms snap both ways and the that replaced the NetEq controller and splices under it, are done, and the [clock step](/quest/m2/audio-playout/clock-step.md) builds on the lag rule. The estimator stops folding a join's intervals -([moq-dev/moq#8730](https://github.com/moq-dev/moq/pull/8730)); -[output latency](/quest/m2/audio-playout/output-latency.md) and the -[join's skipped backlog](/quest/m2/audio-playout/join-backlog.md) are independent. +([moq-dev/moq#8730](https://github.com/moq-dev/moq/pull/8730)). Output latency +landed in [moq-dev/moq#8676](https://github.com/moq-dev/moq/pull/8676): both +players place audio where it is heard and count the output latency in the +target. The [join's skipped backlog](/quest/m2/audio-playout/join-backlog.md) +is independent. The [calibration](/quest/m3/audio-calibrate.md) re-measures every ceiling after this epic. diff --git a/quest/m2/audio-playout/output-latency.md b/quest/m2/audio-playout/output-latency.md deleted file mode 100644 index c53bf9e2047..00000000000 --- a/quest/m2/audio-playout/output-latency.md +++ /dev/null @@ -1,46 +0,0 @@ -# [M] Audio is placed on the clock where it is heard - -## Goal - -Each player compares audio with the shared clock at the moment the audio -leaves the speaker, so audio and video stay in sync on any output device, and -the audio target counts that output latency. The browser rests on the clock -as native does, without the underruns a clock pairing alone caused, and -native's fixed 50 ms floor, which raises the shared delay for video too, is -replaced by its device's actual latency. - -## Plan - -Planned 2026-11-11 (maintainer, /quest-merge pick of -[moq-dev/moq#8507](https://github.com/moq-dev/moq/pull/8507)) for the -browser's render callback. Widened 2026-11-13 (maintainer, /quest-plan): -✅ lag is measured where audio is heard, in both players; the -[epic](/quest/m2/audio-playout/README.md) records the decision. - -Neither player reads its output latency today. The browser never reads -`baseLatency` or `outputLatency`. Natively `playback/driver.rs` places audio -by the host clock at the callback and hands the device's reported latency -only to the playback ledger, and `moq play` floors the audio target at 50 ms -(`AUDIO_BUFFER_MIN` in `play/media.rs`) instead. So audio is -heard later than video by the device latency: tens of milliseconds on a -desktop sink, 150 to 250 ms over Bluetooth. - -The browser evidence: headless Chromium renders 2048-frame callbacks -(`baseLatency` 42.7 ms) and pairs the clock with `currentTime` just after -each one, so it rests 28 to 35 ms behind where native rests within a few ms; -Nightly's near-zero rows show +12 to +20 ms since #8507. A max-filtered -wall-clock pairing put audio on the clock (p50 -1.3 to 4.5 ms) but underran -near-zero Opus 45 times a minute, since the target did not count the -callback. - -Recommendations, in one PR: - -1. Add the output latency to each player's audio target: in the browser, - `baseLatency` (or the measured callback size where it is missing) plus - `outputLatency` where reported, and natively the device period plus the - latency the host reports. The - shared delay then covers it, so video waits that much longer, which lip - sync on that device needs. -2. Read the clock at output time: in the browser, pair it on the wall clock - with the max filter; natively, use the callback's playback timestamp. -3. Measure the live rows before and after in both players. diff --git a/rs/moq-audio/src/playback/driver.rs b/rs/moq-audio/src/playback/driver.rs index fa9ef91be44..5a4981329cb 100644 --- a/rs/moq-audio/src/playback/driver.rs +++ b/rs/moq-audio/src/playback/driver.rs @@ -873,8 +873,11 @@ impl Driver { let played_at = timestamp.playback.saturating_duration_since(epoch) + Duration::from_secs_f64((data.len() / channels) as f64 / sample_rate); mixer.complete(now); - mixer.latency(timestamp.playback.checked_duration_since(timestamp.callback)); - mixer.fill(data, played_at, crate::ring::host()); + // Audio is placed on the clock where it is heard: the buffer's first + // frame sounds this long after the callback. + let delay = timestamp.playback.checked_duration_since(timestamp.callback); + mixer.latency(delay, data.len() / channels); + mixer.fill(data, played_at, crate::ring::host() + delay.unwrap_or_default()); }, move |error| { failures.report(&error); diff --git a/rs/moq-audio/src/playback/mixer.rs b/rs/moq-audio/src/playback/mixer.rs index af2a03cc668..e70fa653d0a 100644 --- a/rs/moq-audio/src/playback/mixer.rs +++ b/rs/moq-audio/src/playback/mixer.rs @@ -170,6 +170,8 @@ pub(super) struct Mixer { scratch: Vec, /// Counts what the mix played, for an [`Observer`](super::Observer). ledger: Ledger, + /// Frames the device delays a read by before it sounds, as last reported. + heard: usize, /// Where echo cancellation reads what was played. Fed the clipped mix, since /// that is the signal the speaker gets and therefore the one the microphone /// hears back. @@ -212,6 +214,7 @@ impl Mixer { bus: vec![0.0; CHUNK * channels], scratch: vec![0.0; CHUNK * channels], ledger: Ledger::new(observe, rate), + heard: 0, #[cfg(feature = "aec")] reference: None, #[cfg(feature = "aec")] @@ -245,9 +248,11 @@ impl Mixer { } } - /// Report the host's latency for the buffer about to be filled. - pub(super) fn latency(&self, latency: Option) { - self.ledger.latency(latency); + /// Report the host's latency for the buffer of `frames` about to be filled: + /// how long after its read its first frame sounds. + pub(super) fn latency(&mut self, latency: Option, frames: usize) { + self.ledger.latency(latency, frames); + self.heard = latency.map_or(0, |latency| (latency.as_secs_f64() * self.rate as f64).round() as usize); } /// Fill one device buffer, interleaved at the device's channel count and in @@ -323,6 +328,7 @@ impl Mixer { bus, scratch, ledger, + heard, #[cfg(feature = "aec")] reference, #[cfg(feature = "aec")] @@ -363,6 +369,7 @@ impl Mixer { // stays on the timeline instead of queueing up and jumping when // it unmutes. The status only reports under/overflow, which the // channel has already corrected for. + entry.cons.heard(*heard); let status = entry.cons.read_at(&mut scratch[..samples], now, total - done); if !matches!(status, Read::NotReady | Read::Underflow { frames: 0 }) { entry.played_at = played_at; diff --git a/rs/moq-audio/src/playback/observe.rs b/rs/moq-audio/src/playback/observe.rs index ffc3e1e832d..dda9bb9b258 100644 --- a/rs/moq-audio/src/playback/observe.rs +++ b/rs/moq-audio/src/playback/observe.rs @@ -101,6 +101,9 @@ pub(super) struct Shared { clock: AtomicU64, /// [`Observation::latency`] in nanoseconds, or [`UNKNOWN`]. latency: AtomicU64, + /// [`Shared::output`] in nanoseconds, or [`UNKNOWN`]: published whole, so + /// a reader never sums one callback's delay with another's period. + output: AtomicU64, quanta: AtomicU64, quiet: AtomicU64, /// [`Observation::comfort`] in nanoseconds. @@ -143,6 +146,7 @@ impl Default for Shared { Self { clock: AtomicU64::new(0), latency: AtomicU64::new(UNKNOWN), + output: AtomicU64::new(UNKNOWN), quanta: AtomicU64::new(0), quiet: AtomicU64::new(0), comfort: AtomicU64::new(0), @@ -170,6 +174,16 @@ impl Shared { let _ = self.driver.set(driver); } + /// The device's output latency: the host's delay plus one device period, + /// the most of a buffer that sounds after its read. `None` before a host + /// has reported a delay. + pub(super) fn output(&self) -> Option { + match self.output.load(Ordering::Relaxed) { + UNKNOWN => None, + nanos => Some(Duration::from_nanos(nanos)), + } + } + /// Wake a [`Observer::finish`] whose measurement a callback or the last stream /// going away closed out. Locks, so the driver thread calls it, never the /// audio thread. @@ -344,10 +358,16 @@ impl Ledger { } } - /// The host's latency for the buffer being filled, if it reports one. - pub(super) fn latency(&self, latency: Option) { - let nanos = latency.map_or(UNKNOWN, |l| l.as_nanos().min(UNKNOWN as u128 - 1) as u64); - self.shared.latency.store(nanos, Ordering::Relaxed); + /// The host's latency for the buffer being filled, if it reports one, and + /// that buffer's length in frames. + pub(super) fn latency(&self, latency: Option, frames: usize) { + let nanos = |latency: Duration| latency.as_nanos().min(UNKNOWN as u128 - 1) as u64; + self.shared + .latency + .store(latency.map_or(UNKNOWN, nanos), Ordering::Relaxed); + let period = Duration::from_secs_f64(frames as f64 / self.rate); + let output = latency.map_or(UNKNOWN, |latency| nanos(latency + period)); + self.shared.output.store(output, Ordering::Relaxed); } /// Fold one mixed buffer in. `supplied` is how many of its leading frames diff --git a/rs/moq-audio/src/playback/sink.rs b/rs/moq-audio/src/playback/sink.rs index db798300cc3..4698a5ff946 100644 --- a/rs/moq-audio/src/playback/sink.rs +++ b/rs/moq-audio/src/playback/sink.rs @@ -54,8 +54,10 @@ pub struct Input { /// absorb without a dropout, and it is also delay: the sample handed over now /// sounds this much later. A player that presents video against a playout /// delay sets this to match, so the sink is where that delay lives rather - /// than something the picture has to be held back for separately. Must be - /// non-zero: a ring with no depth can never be read from. + /// than something the picture has to be held back for separately. Once a + /// clock is [`due`](Sink::due), the device's own delay counts toward it and + /// the sink holds the rest. Must be non-zero: a ring with no depth can never + /// be read from. pub latency: Duration, } @@ -238,21 +240,30 @@ impl Sink { } /// How much audio is queued between the last [`write`](Self::write) and the - /// speaker. + /// speaker, the device's own delay included. /// - /// The pacing signal for A/V sync: the sample playing right now was written + /// The pacing signal for A/V sync: the sample sounding right now was written /// at roughly `last_timestamp - buffered()`, so a video clock can steer /// against it. It settles near [`Input::latency`] once playback is running, - /// climbs when the writer runs ahead, and falls toward zero when it falls - /// behind. + /// climbs when the writer runs ahead, and falls toward the device's delay + /// when it falls behind. pub fn buffered(&self) -> Duration { - self.channel.lock().unwrap().prod.buffered() + self.channel.lock().unwrap().prod.heard_in(crate::ring::host()) + } + + /// How long the device takes to sound what it reads: the delay its host + /// reports plus one device period. `None` until the device has called back. + /// + /// A player holding a playout delay counts this in it, since the sink places + /// audio on [`due`](Self::due)'s clock where it is heard. + pub fn output_latency(&self) -> Option { + self.shared.observe.output() } /// Set the playout target, converging by stretching received PCM. /// - /// This is the buffer the sink keeps against jitter: it slows down while - /// what it holds runs short of it, and speeds up past it only to catch up + /// This is the buffer the sink keeps against jitter, counting the device's + /// own delay: it slows down while what it holds runs short of it, and speeds up past it only to catch up /// with the clock [`due`](Self::due) anchors it to. /// /// Must be non-zero, at most [`moq_mux::container::DELAY_MAX`], and fit in the current @@ -274,7 +285,7 @@ impl Sink { } /// Anchor the audio written so far to the caller's clock: its end is due to - /// sound at `at`. + /// sound at `at`, as heard after the device's own delay. /// /// The sink then plays onto that clock rather than at its latency alone: /// behind it, it speeds up, and ahead of it, it slows down, at a rate diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index 33e39582824..b5d4bc2fe4e 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -111,6 +111,9 @@ pub(crate) enum Read { /// before it exists. #[derive(Default)] struct Shared { + /// Frames the device delays a read by before it sounds. Reads are placed + /// where they are heard, so the queue holds the target less this. + heard: AtomicUsize, pushed: AtomicBool, read: AtomicBool, /// Effective input frames played or discarded, including stretch's length @@ -120,6 +123,11 @@ struct Shared { /// Frames of the writer's stream the reader has played or skipped: /// `consumed` before the stretch's length correction. played: AtomicUsize, + /// `consumed` as of the last read in the low half, and the frame on the + /// clock [`Consumer::read_at_frame`] counts at which that read's last frame + /// is heard in the high half, each wrapping at 32 bits. One word, so a + /// writer never pairs one read's position with another's end. + drained: AtomicU64, /// The writer's end less `played`, at its latest push. Adding the reader's /// `played` less `consumed` gives the depth that push left, less what the /// stretch removed since: the level the stretch decides on. @@ -146,6 +154,16 @@ struct Shared { resume_frames: AtomicUsize, } +impl Shared { + /// What the queue holds of the target: the rest is the device's delay + /// after a read, which the target counts. + fn queue(&self) -> usize { + self.target + .load(Ordering::Relaxed) + .saturating_sub(self.heard.load(Ordering::Relaxed)) + } +} + /// Set in [`Shared::resume`] while a resume waits for the reader. const PENDING: u64 = 1 << 63; @@ -191,6 +209,7 @@ pub(crate) fn channel(channels: usize, input_rate: u32, output_rate: u32, config .then(|| Box::new(crate::stretch::Stretch::new(output_rate, channels))); let shared = Arc::new(Shared { target: AtomicUsize::new(latency), + heard: AtomicUsize::new(0), level: AtomicUsize::new(latency), ..Default::default() }); @@ -297,7 +316,7 @@ impl Producer { } self.declared = false; let queued = self.written.wrapping_sub(self.shared.consumed.load(Ordering::Relaxed)); - let frames = self.shared.target.load(Ordering::Relaxed).saturating_sub(queued); + let frames = self.shared.queue().saturating_sub(queued); let ends = self.shared.ends.load(Ordering::Relaxed); let mut state = self.shared.resume.load(Ordering::Acquire); while state & PENDING != 0 { @@ -367,7 +386,7 @@ impl Producer { if let Some(pad) = self.pad { let queued = self.ring.occupied_len(); - let target = self.shared.target.load(Ordering::Relaxed) * self.channels; + let target = self.shared.queue() * self.channels; if queued <= pad && queued < target { let samples = self.ring.push_iter(std::iter::repeat_n(0.0, target - queued)); self.written = self.written.wrapping_add(samples / self.channels); @@ -465,11 +484,23 @@ impl Producer { } /// How much audio is queued ahead of the reader. + #[cfg(test)] pub(crate) fn buffered(&self) -> Duration { Duration::from_secs_f64( self.written.wrapping_sub(self.shared.consumed.load(Ordering::Relaxed)) as f64 / self.rate as f64, ) } + + /// How long after `now` the last frame written is heard: what is queued, + /// plus what the last read took that has yet to sound, since a read is + /// placed at the instant its first frame is heard. + pub(crate) fn heard_in(&self, now: Instant) -> Duration { + let drained = self.shared.drained.load(Ordering::Relaxed); + let queued = (self.written as u32).wrapping_sub(drained as u32); + let now = (now.saturating_duration_since(self.base).as_secs_f64() * self.rate as f64).round() as usize; + let sounding = ((drained >> 32) as u32).wrapping_sub(now as u32) as i32; + Duration::from_secs_f64((queued as f64 + sounding.max(0) as f64) / self.rate as f64) + } } /// Converts whole chunks of input to the reader's rate. @@ -653,6 +684,12 @@ pub(crate) struct Consumer { } impl Consumer { + /// Note that the device sounds what is read `frames` after the read, which + /// the target counts and the queue therefore does not hold. + pub(crate) fn heard(&mut self, frames: usize) { + self.shared.heard.store(frames, Ordering::Relaxed); + } + /// Fill `output` with queued interleaved frames, and silence past them. pub(crate) fn read(&mut self, output: &mut [f32]) -> Read { let demand = output.len() / self.channels; @@ -669,6 +706,15 @@ impl Consumer { /// [`read_at`](Self::read_at) on the frame clock [`Producer::due_frame`] counts. pub(crate) fn read_at_frame(&mut self, output: &mut [f32], now: usize, demand: usize) -> Read { + let read = self.read_frames(output, now, demand); + // Published once the whole read is done: what it consumed sounds by its end. + let end = now.wrapping_add(output.len() / self.channels); + let drained = (end as u32 as u64) << 32 | self.playhead() as u32 as u64; + self.shared.drained.store(drained, Ordering::Relaxed); + read + } + + fn read_frames(&mut self, output: &mut [f32], now: usize, demand: usize) -> Read { self.start(); self.demand = demand; self.now = now; @@ -747,7 +793,12 @@ impl Consumer { let lag = clock.wrapping_add(self.now + elapsed).wrapping_sub(playhead) as isize; return (lag as i64, (anchor >> 32) as i64); } - let target = self.shared.target.load(Ordering::Relaxed); + // The target counts the device's delay, which the queue does not hold, + // though it still holds at least the callback being read. + let target = self + .shared + .queue() + .max(self.demand.min(self.shared.target.load(Ordering::Relaxed))); let level = self .shared .level @@ -1759,6 +1810,27 @@ mod tests { assert!(cons.lag(0).0 <= 0); } + /// Reads are placed where they sound, so the queue holds the target less + /// the device's delay, though never less than the callback being read. + #[test] + fn the_device_delay_comes_off_the_target_the_queue_holds() { + let rate = 48_000; + let (mut prod, mut cons, pcm) = tone(rate, Duration::from_millis(100)); + let base = prod.base; + assert_eq!(prod.push(&pcm), Push::Ok); + let mut out = vec![0.0; 480 * 2]; + // The unclocked bound is three quarters of the headroom above the target. + let capacity = cons.ring.capacity().get() / 2; + cons.heard(1_440); + pull(&mut cons, &mut out, base); + assert_eq!(cons.lag(0).1, ((capacity - (4_800 - 1_440)) * 3 / 4) as i64); + // A resume tops the queue up to as much, so the next utterance sounds one target out. + assert_eq!(prod.shared.queue(), 4_800 - 1_440); + cons.heard(48_000); + pull(&mut cons, &mut out, base); + assert_eq!(cons.lag(0).1, ((capacity - 480) * 3 / 4) as i64); + } + #[test] fn staged_audio_is_buffered_and_finish_drains_it() { let (mut prod, mut cons) = channel( @@ -1976,6 +2048,27 @@ mod tests { assert!(out.iter().all(|&sample| sample == 0.0)); } + /// The last frame written is heard after what is queued and the part of + /// the last read still sounding, which shrinks until the next read. + #[test] + fn the_last_write_is_heard_behind_the_read_still_sounding() { + let (mut prod, mut cons) = queue(48_000, 48_000, Duration::from_secs(1)); + let base = prod.base; + let millis = |at: Duration| (at.as_secs_f64() * 1000.0).round() as u64; + assert_eq!(prod.push(&[0.1; 4800 * STEREO]), Push::Ok); + // The 50 ms opening cushion and 100 ms written, before any read. + assert_eq!(millis(prod.heard_in(base)), 150); + + // A 10 ms read the device sounds 20 ms after `base`. + let mut out = vec![0.0; 480 * STEREO]; + pull(&mut cons, &mut out, base + Duration::from_millis(20)); + assert_eq!(millis(prod.heard_in(base)), 170); + assert_eq!(millis(prod.heard_in(base + Duration::from_millis(25))), 145); + assert_eq!(millis(prod.heard_in(base + Duration::from_millis(30))), 140); + // Past the read's end, only the queue is left. + assert_eq!(millis(prod.heard_in(base + Duration::from_millis(40))), 140); + } + /// A next utterance pushed while the end's tail still plays waits behind it, /// with comfort noise topping the wait up to one latency. #[test] diff --git a/rs/moq-audio/tests/integration/playback.rs b/rs/moq-audio/tests/integration/playback.rs index fd26a672ed3..e31aab1dbc3 100644 --- a/rs/moq-audio/tests/integration/playback.rs +++ b/rs/moq-audio/tests/integration/playback.rs @@ -121,9 +121,11 @@ async fn plays_a_tone_through_the_default_device() { kio::clock::sleep(Duration::from_millis(100)).await; peaked |= sink.control().peak() > 0.4; - // The device is consuming if the queue stays near the target latency - // rather than growing by a full chunk every write. - drained |= sink.buffered() < Duration::from_millis(250); + // The device is consuming if what is buffered stays near the target + // latency rather than growing by a full chunk every write. It runs to the + // speaker, so it counts the device's own latency too. + let output = sink.output_latency().unwrap_or_default(); + drained |= sink.buffered() < Duration::from_millis(250) + output; } assert!(peaked, "the mixer never saw the tone"); @@ -217,7 +219,7 @@ async fn switches_devices_without_dropping_sinks() { let _ = sink.write(&tone).expect("write"); kio::clock::sleep(Duration::from_millis(200)).await; assert!( - sink.buffered() < Duration::from_secs(1), + sink.buffered() < Duration::from_secs(1) + sink.output_latency().unwrap_or_default(), "{} did not drain after the switch", device.name ); diff --git a/rs/moq-cli/src/play/fake.rs b/rs/moq-cli/src/play/fake.rs index fb2221dd7d0..d72dd2d1751 100644 --- a/rs/moq-cli/src/play/fake.rs +++ b/rs/moq-cli/src/play/fake.rs @@ -250,6 +250,11 @@ impl Sink for FakeSink { self.next = true; } + /// A device that has not reported its latency, as before its first callback. + fn output_latency(&self) -> Option { + None + } + fn buffered(&self) -> Duration { if let Some(prime) = self.prime { prime diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index f564bea1911..b63eb1dbc35 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -27,11 +27,17 @@ enum Crossing { Loss, } -/// The floor on the speaker's buffer, whatever delay was asked for. The device -/// pulls on a fixed clock, so a ring shallower than this drops out on ordinary -/// network jitter, and one with no depth at all can never be read from. +/// The depth a sink opens on, and its floor until the device reports its own +/// latency, which replaces it. A ring with no depth at all can never be read +/// from. const AUDIO_BUFFER_MIN: Duration = Duration::from_millis(50); +/// How long the device's highest reported output latency holds. A host's report +/// moves with where each callback lands, so the delay the clock holds for it +/// follows the peak rather than every reading, and falls once a lower latency, +/// as after a switch off a Bluetooth headset, has held this long. +const OUTPUT_WINDOW: Duration = Duration::from_secs(5); + /// How far past the latency it opened on the sink may be asked to hold. Its ring /// reaches three seconds past that latency, less a 30 ms processing window /// (`Sink::set_latency`), so this leaves margin, and it covers audio's reach past the delay. @@ -610,6 +616,7 @@ async fn play_audio( let arrived = tokio::sync::Notify::new(); let popped = tokio::sync::Notify::new(); let max_delay = consumer.max_delay(); + let estimate = consumer.options().estimate; let sample_rate = consumer.sample_rate(); let layout = consumer.layout(); let channels = layout.channels(); @@ -645,11 +652,33 @@ async fn play_audio( let mut timeline = AudioTimeline::default(); + // The device's output latency the clock holds for. + let peak = Mutex::new(Peak::default()); + // Audio is placed on the clock where it is heard, so the device's output + // latency is part of the delay video waits for too. A fixed delay is floored + // at it instead, since audio cannot sound any sooner. Capped as the catalog + // delay is, so a hostile advertised jitter cannot make playback retain media + // without bound. + let audible = |delay: Duration| { + let device = peak.lock().unwrap().peak(); + if estimate { + delay + device.unwrap_or_default() + } else { + delay.max(device.unwrap_or(AUDIO_BUFFER_MIN)) + } + .min(moq_mux::container::DELAY_MAX) + }; + + // How far ahead of its due time a write may sound: what the sink holds, at + // most its cap, plus the device's latency after it. + let reach = || cap + peak.lock().unwrap().peak().unwrap_or_default(); + // A replacement's opening cushion and its timeline plan must agree. The // callback now owns target convergence, rather than write-time padding // subtracting the incoming frame before it reaches the sink. Like a hole, - // the cushion is silence the sink must hold. - let lead = |delay: Duration| latency.max(delay.min(cap).max(AUDIO_BUFFER_MIN)); + // the cushion is silence the sink must hold, and a write ahead of its due + // time leads it by the device's latency too. + let lead = |delay: Duration| latency.max(audible(delay).min(reach()).max(AUDIO_BUFFER_MIN)); let schedule = presentation.consume(); // The consumer decodes a packet once the sink has room for it. Its playhead is @@ -669,9 +698,15 @@ async fn play_audio( } }; steer(None); - // What the sink holds against jitter: the shared delay, floored at what the - // device needs and capped at what the sink was sized for. - let target = || schedule.delay().clamp(AUDIO_BUFFER_MIN, cap); + // What the sink holds: the shared delay, which counts the device's own + // latency, never below that latency, and capped last at what the sink was + // sized for, so a device reporting more than that cannot get it refused. + let target = |sink: &::Sink| { + schedule + .delay() + .max(sink.output_latency().unwrap_or(AUDIO_BUFFER_MIN)) + .min(cap) + }; // Anchor the sink to the shared clock: the media written so far ends at // `end`, and sounds when the clock says it is due. The callback stretches // onto that clock by its lag alone, and skips onto it what trails it past @@ -854,7 +889,7 @@ async fn play_audio( // No write may come to carry a delay change, so the speaker follows it now. _ = role.delay_changed(seen) => { steer(wrote); - sink.set_latency(target())?; + sink.set_latency(target(&sink))?; if let Some((_, end)) = wrote { anchor(&mut sink, end); } @@ -988,18 +1023,15 @@ async fn play_audio( // clock holds both while the sink keeps sizing for jitter alone. The // clock holds the whole measured target, which the subscription follows, // though the sink holds no more of it than its cap: the rest stays encoded. - // Capped as the catalog delay is, so a hostile advertised jitter cannot - // make playback retain media without bound. - let held = latency - .max(delay.max(AUDIO_BUFFER_MIN)) - .min(moq_mux::container::DELAY_MAX); + peak.lock().unwrap().observe(sink.output_latency(), Instant::now()); + let held = audible(delay); *jitter.lock().unwrap() = Some(held); moved |= role.set_target(capped(held, *stated.borrow())); // The consumer releases media due further out than the sink holds only // when its timeline ends before the media behind it covers the hold, so // that little waits here instead of overflowing the sink. let held_back = wait(&mut catalog, &|| { - presentation.due(mapped).and_then(|due| due.checked_sub(cap)) + presentation.due(mapped).and_then(|due| due.checked_sub(reach())) }) .await; // A fresh sink plays its first write behind an opening cushion sized before @@ -1018,7 +1050,7 @@ async fn play_audio( // Convergence is the callback's: it slows down when its buffer runs low and // catches up with the clock once it holds more than its target, and only // skips what trails the clock past what the subscription still delivers. - sink.set_latency(target())?; + sink.set_latency(target(&sink))?; // The packet pays for the channel count; the catalog cannot allocate // a duration's worth of silence before any samples arrive. @@ -1077,6 +1109,40 @@ async fn play_audio( Ok(sink) } +/// The highest output latency the device reported within [`OUTPUT_WINDOW`]. +#[derive(Default)] +struct Peak { + /// Each reading that no later one is at least as high as, oldest first, so + /// the front is the peak. + readings: VecDeque<(Instant, Duration)>, +} + +impl Peak { + /// Fold in the latency the device reports now, if any, returning the peak. + fn observe(&mut self, latency: Option, now: Instant) -> Option { + if let Some(latency) = latency { + while self.readings.back().is_some_and(|&(_, last)| last <= latency) { + self.readings.pop_back(); + } + self.readings.push_back((now, latency)); + } + while self + .readings + .front() + .is_some_and(|&(at, _)| now.saturating_duration_since(at) > OUTPUT_WINDOW) + && self.readings.len() > 1 + { + self.readings.pop_front(); + } + self.peak() + } + + /// The peak as of the last reading. + fn peak(&self) -> Option { + self.readings.front().map(|&(_, latency)| latency) + } +} + /// Play out what a retired sink still holds, instead of cutting the tail off /// by dropping it. async fn drain(sink: impl Sink) { @@ -2728,6 +2794,24 @@ mod tests { ); } + /// The latency the clock holds is the device's peak over the window: a lower + /// reading holds until the higher one ages out, and a higher one takes over at once. + #[test] + fn the_output_latency_holds_its_peak_for_the_window() { + let start = Instant::now(); + let at = |secs: u64| start.saturating_add(Duration::from_secs(secs)); + let ms = Duration::from_millis; + let mut peak = Peak::default(); + assert_eq!(peak.observe(None, at(0)), None); + assert_eq!(peak.observe(Some(ms(20)), at(0)), Some(ms(20))); + assert_eq!(peak.observe(Some(ms(14)), at(1)), Some(ms(20))); + assert_eq!(peak.observe(None, at(5)), Some(ms(20))); + assert_eq!(peak.observe(Some(ms(15)), at(6)), Some(ms(15))); + assert_eq!(peak.observe(Some(ms(40)), at(7)), Some(ms(40))); + // Unreported since, the last peak stands. + assert_eq!(peak.observe(None, at(60)), Some(ms(40))); + } + #[tokio::test] async fn a_finite_audio_track_plays_its_final_samples() { tokio::time::pause(); diff --git a/rs/moq-cli/src/play/output.rs b/rs/moq-cli/src/play/output.rs index b8a4a8bd008..054d346cfba 100644 --- a/rs/moq-cli/src/play/output.rs +++ b/rs/moq-cli/src/play/output.rs @@ -49,6 +49,9 @@ pub(super) trait Sink: Send + Sync + 'static { /// How much audio is queued ahead of the speaker. fn buffered(&self) -> Duration; + /// How long the device takes to sound what it reads, once it has reported it. + fn output_latency(&self) -> Option; + /// Play every queued sample before releasing the sink. fn finish(self) -> impl Future + Send; @@ -88,6 +91,10 @@ impl Sink for moq_audio::playback::Sink { moq_audio::playback::Sink::buffered(self) } + fn output_latency(&self) -> Option { + moq_audio::playback::Sink::output_latency(self) + } + fn end(&mut self) { moq_audio::playback::Sink::end(self); } diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index a6d936d9b81..d158f06fc89 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -97,7 +97,8 @@ in what the native player does: - Its ring stretches received PCM onto the shared clock by the page's rule. A ring that still runs dry emits silence until a cushion of received PCM arrives; `silence` counts that padding. Native `stalled` is only a sink with nothing written yet. -- The playhead is projected from the player's last write at the wall rate and held at its end. +- The playhead is projected from the player's last write at the wall rate and held at its end; the + write sounds after what the sink holds and the device's own delay. - `device` is the host's reported output latency, with no `baseLatency`, and `render_load` is unmeasured. @@ -318,8 +319,9 @@ player's own count, which its stats panel shows, is read alongside: that starts within 60 ms of a stall is that stall, and is not an underrun. - `silence`: the share of rendered quanta under -60 dBFS at the output, whatever the cause. - `target_ms`: the resolved playout delay (`sync.out.delay`). -- `av_lag_ms`: how far the audio playhead trails the shared clock video plays from (`sync.now()`), - sampled while playing. Audio and video keep separate clocks that align once the delay covers the +- `av_lag_ms`: how far the audio sounding at the speaker trails the shared clock video plays from + (`sync.now()`), sampled while playing. The browser's playhead (`audio.out.timestamp`) is the one + sounding as of its last report, projected at the wall rate to the sample. Audio and video keep separate clocks that align once the delay covers the arrivals, so this is the latency audio adds beyond the target: positive is audio late, negative early. Its `p50` is where audio rests, so latency spent to buy fewer underruns shows there; its `p95` includes audio trailing through a stall. Natively, the presentation clock `moq play` paces diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts index 6d95173576e..f5632dde895 100644 --- a/test/audio-quality/clients/js/src/probe.ts +++ b/test/audio-quality/clients/js/src/probe.ts @@ -77,6 +77,20 @@ export function probe(watch: MoqWatch, capture = false): Probe { // The tap follows the output node, which the player rebuilds with its graph. A tap left on the old // node would report a perfect run as one long silence. + // The playhead the player last reported, and when. It reports once per render callback, so a sample + // projects it at real time to the instant it is read, as the native player's is. + // A stall ending re-anchors it too: the playhead held through the stall, so it resumes from now. + let reported: { timestamp: number; at: number } | undefined; + signals.run((effect) => { + const timestamp = effect.get(audio.out.timestamp); + effect.get(audio.out.stalled); + reported = timestamp === undefined ? undefined : { timestamp: timestamp.asMillis(), at: performance.now() }; + }); + const playhead = (at: number): number | undefined => { + if (!reported) return undefined; + return audio.out.stalled.peek() ? reported.timestamp : reported.timestamp + (at - reported.at); + }; + let tapped: AudioNode | undefined; let current: Tap | undefined; let renderLoad: number | undefined; @@ -120,11 +134,12 @@ export function probe(watch: MoqWatch, capture = false): Probe { const gaps = current?.take() ?? []; const taken = stalls; stalls = []; + const at = performance.now(); return { // Read back to back, so the pair calibrates the render clock against the viewer's. - at: performance.now(), + at, render: context ? context.currentTime * 1000 : undefined, - timestamp: audio.out.timestamp.peek()?.asMillis(), + timestamp: playhead(at), stalled: audio.out.stalled.peek(), clock: sync.now()?.asMillis(), delay: sync.out.delay.peek().asMillis(), diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index 0028602aa98..a8cbc11b3d7 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -393,8 +393,10 @@ export type Sample = { render?: Ms; /** - * `audio.out.timestamp`: the playhead on the `media` clock. Undefined while there is none. - * Natively, projected from the last write into the speaker at the wall rate, held at its end. + * The playhead sounding at the speaker, on the `media` clock. Undefined while there is none. In + * the browser `audio.out.timestamp`, projected at the wall rate from when the player last reported + * it, and held while stalled. Natively, projected from the last write into the speaker at the wall + * rate, held at its end. */ timestamp?: Ms; /**