From 9163a90f3ddc0a82b23b158c19a141f8f8e4b058 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 13:47:54 -0800 Subject: [PATCH 01/18] chore(quest): claim quest/m2/audio-playout/output-latency.md Quest-Claim: d5c3ec1dc0d122d3 From bbea5f33e423a30925a0c5c4fc04c4d67f475163 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 14:09:53 -0800 Subject: [PATCH 02/18] feat(moq-audio,moq-cli): place native audio where it is heard and hold the device latency Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/playback/driver.rs | 7 ++- rs/moq-audio/src/playback/mixer.rs | 13 +++-- rs/moq-audio/src/playback/observe.rs | 28 ++++++++++- rs/moq-audio/src/playback/sink.rs | 32 +++++++++---- rs/moq-audio/src/ring.rs | 13 +++++ rs/moq-cli/src/play/fake.rs | 5 ++ rs/moq-cli/src/play/media.rs | 72 +++++++++++++++++++++++----- rs/moq-cli/src/play/output.rs | 7 +++ 8 files changed, 149 insertions(+), 28 deletions(-) diff --git a/rs/moq-audio/src/playback/driver.rs b/rs/moq-audio/src/playback/driver.rs index fa9ef91be44..5a4981329cb 100644 --- a/rs/moq-audio/src/playback/driver.rs +++ b/rs/moq-audio/src/playback/driver.rs @@ -873,8 +873,11 @@ impl Driver { let played_at = timestamp.playback.saturating_duration_since(epoch) + Duration::from_secs_f64((data.len() / channels) as f64 / sample_rate); mixer.complete(now); - mixer.latency(timestamp.playback.checked_duration_since(timestamp.callback)); - mixer.fill(data, played_at, crate::ring::host()); + // Audio is placed on the clock where it is heard: the buffer's first + // frame sounds this long after the callback. + let delay = timestamp.playback.checked_duration_since(timestamp.callback); + mixer.latency(delay, data.len() / channels); + mixer.fill(data, played_at, crate::ring::host() + delay.unwrap_or_default()); }, move |error| { failures.report(&error); diff --git a/rs/moq-audio/src/playback/mixer.rs b/rs/moq-audio/src/playback/mixer.rs index 42cd3a5f81c..ae65a6be331 100644 --- a/rs/moq-audio/src/playback/mixer.rs +++ b/rs/moq-audio/src/playback/mixer.rs @@ -170,6 +170,8 @@ pub(super) struct Mixer { scratch: Vec, /// Counts what the mix played, for an [`Observer`](super::Observer). ledger: Ledger, + /// Frames the device delays a read by before it sounds, as last reported. + heard: usize, /// Where echo cancellation reads what was played. Fed the clipped mix, since /// that is the signal the speaker gets and therefore the one the microphone /// hears back. @@ -212,6 +214,7 @@ impl Mixer { bus: vec![0.0; CHUNK * channels], scratch: vec![0.0; CHUNK * channels], ledger: Ledger::new(observe, rate), + heard: 0, #[cfg(feature = "aec")] reference: None, #[cfg(feature = "aec")] @@ -245,9 +248,11 @@ impl Mixer { } } - /// Report the host's latency for the buffer about to be filled. - pub(super) fn latency(&self, latency: Option) { - self.ledger.latency(latency); + /// Report the host's latency for the buffer of `frames` about to be filled: + /// how long after its read its first frame sounds. + pub(super) fn latency(&mut self, latency: Option, frames: usize) { + self.ledger.latency(latency, frames); + self.heard = latency.map_or(0, |latency| (latency.as_secs_f64() * self.rate as f64).round() as usize); } /// Fill one device buffer, interleaved at the device's channel count and in @@ -323,6 +328,7 @@ impl Mixer { bus, scratch, ledger, + heard, #[cfg(feature = "aec")] reference, #[cfg(feature = "aec")] @@ -363,6 +369,7 @@ impl Mixer { // stays on the timeline instead of queueing up and jumping when // it unmutes. The status only reports under/overflow, which the // channel has already corrected for. + entry.cons.heard(*heard); let status = entry.cons.read_at(&mut scratch[..samples], now, total - done); if !matches!(status, Read::NotReady | Read::Underflow { frames: 0 }) { entry.played_at = played_at; diff --git a/rs/moq-audio/src/playback/observe.rs b/rs/moq-audio/src/playback/observe.rs index ffc3e1e832d..f81a6ce69f6 100644 --- a/rs/moq-audio/src/playback/observe.rs +++ b/rs/moq-audio/src/playback/observe.rs @@ -101,6 +101,9 @@ pub(super) struct Shared { clock: AtomicU64, /// [`Observation::latency`] in nanoseconds, or [`UNKNOWN`]. latency: AtomicU64, + /// The last buffer's duration at the device rate in nanoseconds: one + /// device period. Zero before the first. + period: AtomicU64, quanta: AtomicU64, quiet: AtomicU64, /// [`Observation::comfort`] in nanoseconds. @@ -143,6 +146,7 @@ impl Default for Shared { Self { clock: AtomicU64::new(0), latency: AtomicU64::new(UNKNOWN), + period: AtomicU64::new(0), quanta: AtomicU64::new(0), quiet: AtomicU64::new(0), comfort: AtomicU64::new(0), @@ -170,6 +174,23 @@ impl Shared { let _ = self.driver.set(driver); } + /// How long after a read the device sounds it, as the host last reported, + /// or `None` before a host has. + pub(super) fn delay(&self) -> Option { + match self.latency.load(Ordering::Relaxed) { + UNKNOWN => None, + nanos => Some(Duration::from_nanos(nanos)), + } + } + + /// The device's output latency: the host's delay plus one device period, + /// the most of a buffer that sounds after its read. `None` before a host + /// has reported a delay. + pub(super) fn output(&self) -> Option { + let period = Duration::from_nanos(self.period.load(Ordering::Relaxed)); + self.delay().map(|delay| delay + period) + } + /// Wake a [`Observer::finish`] whose measurement a callback or the last stream /// going away closed out. Locks, so the driver thread calls it, never the /// audio thread. @@ -344,10 +365,13 @@ impl Ledger { } } - /// The host's latency for the buffer being filled, if it reports one. - pub(super) fn latency(&self, latency: Option) { + /// The host's latency for the buffer being filled, if it reports one, and + /// that buffer's length in frames. + pub(super) fn latency(&self, latency: Option, frames: usize) { let nanos = latency.map_or(UNKNOWN, |l| l.as_nanos().min(UNKNOWN as u128 - 1) as u64); self.shared.latency.store(nanos, Ordering::Relaxed); + let period = (frames as f64 / self.rate * 1e9) as u64; + self.shared.period.store(period, Ordering::Relaxed); } /// Fold one mixed buffer in. `supplied` is how many of its leading frames diff --git a/rs/moq-audio/src/playback/sink.rs b/rs/moq-audio/src/playback/sink.rs index 988b22c18c3..9afdc41b27b 100644 --- a/rs/moq-audio/src/playback/sink.rs +++ b/rs/moq-audio/src/playback/sink.rs @@ -54,8 +54,10 @@ pub struct Input { /// absorb without a dropout, and it is also delay: the sample handed over now /// sounds this much later. A player that presents video against a playout /// delay sets this to match, so the sink is where that delay lives rather - /// than something the picture has to be held back for separately. Must be - /// non-zero: a ring with no depth can never be read from. + /// than something the picture has to be held back for separately. Once a + /// clock is [`due`](Sink::due), the device's own delay counts toward it and + /// the sink holds the rest. Must be non-zero: a ring with no depth can never + /// be read from. pub latency: Duration, } @@ -238,21 +240,31 @@ impl Sink { } /// How much audio is queued between the last [`write`](Self::write) and the - /// speaker. + /// speaker, the device's own delay included. /// - /// The pacing signal for A/V sync: the sample playing right now was written + /// The pacing signal for A/V sync: the sample sounding right now was written /// at roughly `last_timestamp - buffered()`, so a video clock can steer /// against it. It settles near [`Input::latency`] once playback is running, - /// climbs when the writer runs ahead, and falls toward zero when it falls - /// behind. + /// climbs when the writer runs ahead, and falls toward the device's delay + /// when it falls behind. pub fn buffered(&self) -> Duration { - self.channel.lock().unwrap().prod.buffered() + let queued = self.channel.lock().unwrap().prod.buffered(); + queued + self.shared.observe.delay().unwrap_or_default() + } + + /// How long the device takes to sound what it reads: the delay its host + /// reports plus one device period. `None` until the device has called back. + /// + /// A player holding a playout delay counts this in it, since the sink places + /// audio on [`due`](Self::due)'s clock where it is heard. + pub fn output_latency(&self) -> Option { + self.shared.observe.output() } /// Set the playout target, converging by stretching received PCM. /// - /// This is the buffer the sink keeps against jitter: it slows down while - /// what it holds runs short of it, and speeds up past it only to catch up + /// This is the buffer the sink keeps against jitter, counting the device's + /// own delay: it slows down while what it holds runs short of it, and speeds up past it only to catch up /// with the clock [`due`](Self::due) anchors it to. /// /// Must be non-zero, at most [`moq_mux::container::DELAY_MAX`], and fit in the current @@ -274,7 +286,7 @@ impl Sink { } /// Anchor the audio written so far to the caller's clock: its end is due to - /// sound at `at`. + /// sound at `at`, as heard after the device's own delay. /// /// The sink then plays onto that clock rather than at its latency alone. /// Behind it, it speeds up once it holds more than its latency; ahead of diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index c6934a35ea6..df13d4fe241 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -204,6 +204,7 @@ pub(crate) fn channel(channels: usize, input_rate: u32, output_rate: u32, config refill: false, prime: latency, demand: 0, + heard: 0, consumed: 0, position: 0, cleared: 0, @@ -613,6 +614,9 @@ pub(crate) struct Consumer { prime: usize, /// Frames left in the device's callback as of the current read. demand: usize, + /// Frames the device delays a read by before it sounds. Reads are placed + /// where they are heard, so the queue holds the target less this. + heard: usize, consumed: usize, /// Frames taken off the ring plus the comfort noise of each resume played /// out: the writer's `written`, as far as the reader has reached. @@ -655,6 +659,12 @@ impl Consumer { self.shared.stretch.store(true, Ordering::Relaxed); } + /// Note that the device sounds what is read `frames` after the read, which + /// the target counts and the queue therefore does not hold. + pub(crate) fn heard(&mut self, frames: usize) { + self.heard = frames; + } + /// Fill `output` with queued interleaved frames, and silence past them. pub(crate) fn read(&mut self, output: &mut [f32]) -> Read { let demand = output.len() / self.channels; @@ -732,7 +742,10 @@ impl Consumer { .clocked .load(Ordering::Acquire) .then(|| self.shared.anchor.load(Ordering::Acquire)); + // The target counts the device's delay, which the queue does not hold, + // though it still holds at least the callback being read. let target = self.shared.target.load(Ordering::Relaxed); + let target = target.saturating_sub(self.heard).max(self.demand.min(target)); let level = self .shared .level diff --git a/rs/moq-cli/src/play/fake.rs b/rs/moq-cli/src/play/fake.rs index 86bec2998e2..fb6061a15f3 100644 --- a/rs/moq-cli/src/play/fake.rs +++ b/rs/moq-cli/src/play/fake.rs @@ -250,6 +250,11 @@ impl Sink for FakeSink { self.next = true; } + /// A device that has not reported its latency, as before its first callback. + fn output_latency(&self) -> Option { + None + } + fn buffered(&self) -> Duration { if let Some(prime) = self.prime { prime diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 288c2a7f465..175c8627071 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -27,11 +27,17 @@ enum Crossing { Loss, } -/// The floor on the speaker's buffer, whatever delay was asked for. The device -/// pulls on a fixed clock, so a ring shallower than this drops out on ordinary -/// network jitter, and one with no depth at all can never be read from. +/// The depth a sink opens on, and its floor until the device reports its own +/// latency, which replaces it. A ring with no depth at all can never be read +/// from. const AUDIO_BUFFER_MIN: Duration = Duration::from_millis(50); +/// How long the device's highest reported output latency holds. A host's report +/// moves with where each callback lands, so the delay the clock holds for it +/// follows the peak rather than every reading, and falls once a lower latency, +/// as after a switch off a Bluetooth headset, has held this long. +const OUTPUT_WINDOW: Duration = Duration::from_secs(5); + /// How far past the latency it opened on the sink may be asked to hold. Its ring /// reaches three seconds past that latency, less a 30 ms processing window /// (`Sink::set_latency`), so this leaves margin, and it covers the default stretch. @@ -610,6 +616,7 @@ async fn play_audio( let arrived = tokio::sync::Notify::new(); let popped = tokio::sync::Notify::new(); let max_delay = consumer.max_delay(); + let estimate = consumer.options().estimate; let sample_rate = consumer.sample_rate(); let layout = consumer.layout(); let channels = layout.channels(); @@ -670,9 +677,16 @@ async fn play_audio( } }; steer(None); - // What the sink holds against jitter: the shared delay, floored at what the - // device needs and capped at what the sink was sized for. - let target = || schedule.delay().clamp(AUDIO_BUFFER_MIN, cap); + // What the sink holds: the shared delay, which counts the device's own + // latency, capped at what the sink was sized for and never below that latency. + let target = |sink: &::Sink| { + schedule + .delay() + .min(cap) + .max(sink.output_latency().unwrap_or(AUDIO_BUFFER_MIN)) + }; + // The device's output latency the clock holds for. + let output = Mutex::new(Peak::default()); // Anchor the sink to the shared clock: the media written so far ends at // `end`, and sounds when the clock says it is due. The callback stretches // onto that clock, and skips only what trails it past the stretch, which @@ -873,7 +887,7 @@ async fn play_audio( // No write may come to carry a delay change, so the speaker follows it now. _ = role.delay_changed(seen) => { steer(wrote); - sink.set_latency(target())?; + sink.set_latency(target(&sink))?; if let Some((_, end)) = wrote { anchor(&mut sink, &mut anchored, end); } @@ -1009,9 +1023,16 @@ async fn play_audio( // though the sink holds no more of it than its cap: the rest stays encoded. // Capped as the catalog delay is, so a hostile advertised jitter cannot // make playback retain media without bound. - let held = latency - .max(delay.max(AUDIO_BUFFER_MIN)) - .min(moq_mux::container::DELAY_MAX); + // Audio is placed on the clock where it is heard, so the device's output + // latency is part of the delay video waits for too. A fixed delay is + // floored at it instead, since audio cannot sound any sooner. + let device = output.lock().unwrap().observe(sink.output_latency(), Instant::now()); + let held = if estimate { + delay + device.unwrap_or_default() + } else { + delay.max(device.unwrap_or(AUDIO_BUFFER_MIN)) + } + .min(moq_mux::container::DELAY_MAX); *jitter.lock().unwrap() = Some(held); moved |= role.set_target(capped(held, *stated.borrow())); // The consumer releases media due further out than the sink holds only @@ -1037,7 +1058,7 @@ async fn play_audio( // Convergence is the callback's: it slows down when its buffer runs low and // catches up with the clock once it holds more than its target, and only // skips what trails the clock past what the subscription still delivers. - sink.set_latency(target())?; + sink.set_latency(target(&sink))?; // The packet pays for the channel count; the catalog cannot allocate // a duration's worth of silence before any samples arrive. @@ -1096,6 +1117,35 @@ async fn play_audio( Ok(sink) } +/// The highest output latency the device reported within [`OUTPUT_WINDOW`]. +#[derive(Default)] +struct Peak { + /// Each reading that no later one is at least as high as, oldest first, so + /// the front is the peak. + readings: VecDeque<(Instant, Duration)>, +} + +impl Peak { + /// Fold in the latency the device reports now, if any, returning the peak. + fn observe(&mut self, latency: Option, now: Instant) -> Option { + if let Some(latency) = latency { + while self.readings.back().is_some_and(|&(_, last)| last <= latency) { + self.readings.pop_back(); + } + self.readings.push_back((now, latency)); + } + while self + .readings + .front() + .is_some_and(|&(at, _)| now.saturating_duration_since(at) > OUTPUT_WINDOW) + && self.readings.len() > 1 + { + self.readings.pop_front(); + } + self.readings.front().map(|&(_, latency)| latency) + } +} + /// Play out what a retired sink still holds, instead of cutting the tail off /// by dropping it. async fn drain(sink: impl Sink) { diff --git a/rs/moq-cli/src/play/output.rs b/rs/moq-cli/src/play/output.rs index 823e88abfe9..263c5dd665a 100644 --- a/rs/moq-cli/src/play/output.rs +++ b/rs/moq-cli/src/play/output.rs @@ -50,6 +50,9 @@ pub(super) trait Sink: Send + Sync + 'static { /// How much audio is queued ahead of the speaker. fn buffered(&self) -> Duration; + /// How long the device takes to sound what it reads, once it has reported it. + fn output_latency(&self) -> Option; + /// Play every queued sample before releasing the sink. fn finish(self) -> impl Future + Send; @@ -89,6 +92,10 @@ impl Sink for moq_audio::playback::Sink { moq_audio::playback::Sink::buffered(self) } + fn output_latency(&self) -> Option { + moq_audio::playback::Sink::output_latency(self) + } + fn end(&mut self) { moq_audio::playback::Sink::end(self); } From 2fb1f81f29c1ca50a9840ea73773608f7bc75209 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 14:14:49 -0800 Subject: [PATCH 03/18] feat(watch): place browser audio where it is heard and hold the output latency Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 144 ++++++++++++++++++---- js/watch/src/audio/buffer.ts | 106 ++++++++++++---- js/watch/src/audio/decoder.test.ts | 7 ++ js/watch/src/audio/decoder.ts | 7 +- js/watch/src/audio/latency.ts | 30 ++++- js/watch/src/audio/render-worklet.test.ts | 11 +- js/watch/src/audio/render-worklet.ts | 1 + js/watch/src/audio/render.ts | 11 +- 8 files changed, 255 insertions(+), 62 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 17ae04d469e..9477ee607d7 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -1,11 +1,38 @@ import { describe, expect, it } from "bun:test"; -import { Duration } from "@moq/clock"; +import { Duration, Instant, Manual, now } from "@moq/clock"; import { type Getter, Signal } from "@moq/signals"; import { AudioBuffer, Backpressure } from "./buffer"; import type { Init, Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; const HOUR = 60 * 60 * 1000; + +// A manual clock moved past the page's origin, where a performance time of zero means none. +function started(): { close(): void } { + const manual = new Manual(); + const installed = manual.install(); + manual.jump(Duration.secs(1)); + return installed; +} + +// A context that reports its output position as sounding `sounding` now, or nothing until it is set. +function silent(): { + currentTime: number; + baseLatency: number; + outputLatency: number; + sounding?: number; + getOutputTimestamp(): AudioTimestamp; +} { + return { + currentTime: 0, + baseLatency: 0, + outputLatency: 0, + getOutputTimestamp() { + if (this.sounding === undefined) return { contextTime: 0, performanceTime: 0 }; + return { contextTime: this.sounding, performanceTime: now().since(Instant.ZERO).asMillis() }; + }, + }; +} const ms = (value: number) => value * 1000; // A buffered buffer over a real port, with the worklet's end of it left to the test. @@ -16,7 +43,7 @@ function open(): { close: () => void; } { const channel = new MessageChannel(); - const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { + const buffer = new AudioBuffer({ port: channel.port1, context: silent() } as unknown as AudioWorkletNode, { channels: 1, rate: 48_000, delay: Duration.millis(100), @@ -30,6 +57,7 @@ function open(): { const msg: State = { type: "state", timestamp: timestamp ?? 0, + time: 0, stalled, anchored: timestamp !== undefined, underruns: 0, @@ -56,7 +84,7 @@ function open(): { describe("AudioBuffer", () => { it("transfers PCM ownership and orders control messages after the data they change", async () => { const channel = new MessageChannel(); - const buffer = new AudioBuffer({ port: channel.port1 } as unknown as AudioWorkletNode, { + const buffer = new AudioBuffer({ port: channel.port1, context: silent() } as unknown as AudioWorkletNode, { channels: 2, rate: 48_000, delay: Duration.millis(100), @@ -416,7 +444,7 @@ function fake( start() {}, }); // The context's time in seconds, which the test advances. - const context = { currentTime: 0 }; + const context = silent(); const buffer = new AudioBuffer({ port, context } as unknown as AudioWorkletNode, { channels: 1, rate: 1000, @@ -425,10 +453,11 @@ function fake( ...(props.bound !== undefined && { bound: Duration.millis(props.bound) }), clock: props.clock, }); - const state = (timestamp: number, stalled: boolean, epoch: number) => { + const state = (timestamp: number, stalled: boolean, epoch: number, time = 0) => { const data: State = { type: "state", timestamp, + time, stalled, anchored: true, underruns: 0, @@ -452,7 +481,7 @@ describe("AudioBuffer past its decoded floor", () => { postMessage: (msg: Message) => sent.push(msg), start() {}, }); - new AudioBuffer({ port } as unknown as AudioWorkletNode, { + new AudioBuffer({ port, context: silent() } as unknown as AudioWorkletNode, { channels: 2, rate: 48_000, delay: Duration.millis(delay), @@ -517,43 +546,106 @@ describe("AudioBuffer steering", () => { } }); - // The context's time advances a render quantum at a time, so a reading of the clock lands up to a - // quantum late against it. The offset keeps the earliest, rises only as slowly as drift needs, and - // jumps when the delay moves the clock for real. - it("sends the clock at its earliest offset from the context's time", () => { + // A browser that reads its output position once per render callback reports it up to a callback + // early, so the offset keeps the greatest, falls only as slowly as drift needs, and moves whole when + // the delay or the reference moves the clock for real. + it("sends the clock against the context time sounding at its greatest offset", () => { let now: Duration | undefined = Duration.millis(1000); const reference = new Signal(0); + const clock = started(); const { buffer, sent, state, context } = fake(100, { clock: { now: () => now, out: { reference } } }); - const report = (time: number, clock: number | undefined) => { + const report = (time: number, sounding: number, clock: number | undefined) => { context.currentTime = time; + context.sounding = sounding; now = clock === undefined ? undefined : Duration.millis(clock); state(ms(0), false, 0); }; try { - report(1, 1000); - // Read 2 ms late: the offset rises by only 100 us. - report(1.25, 1252); - report(1.5, 1500); + // The output sounds 100 ms behind the context's time. + report(1, 0.9, 1000); + // Read 2 ms early: the offset falls by only 100 us. + report(1.25, 1.152, 1250); + report(1.5, 1.4, 1500); // A delay change moved the clock 50 ms. - report(1.75, 1800); + report(1.75, 1.65, 1800); // The reference moved 5 ms earlier: a jump the offset takes whole, under the filter's. reference.set(-5); - report(2, 2055); - // A later reference is no jump. + report(2, 1.9, 2055); + // A later reference moves it back, though it is no jump. reference.set(0); - report(2.25, 2300); - report(2.5, undefined); + report(2.25, 2.15, 2300); + report(2.5, 2.4, undefined); expect(sent.filter((msg) => msg.type === "clock")).toEqual([ - { type: "clock", timestamp: 1_000_000, time: 1 }, - { type: "clock", timestamp: 1_250_100, time: 1.25 }, - { type: "clock", timestamp: 1_500_000, time: 1.5 }, - { type: "clock", timestamp: 1_800_000, time: 1.75 }, - { type: "clock", timestamp: 2_055_000, time: 2, jumped: 5000 }, - { type: "clock", timestamp: 2_300_000, time: 2.25 }, + { type: "clock", timestamp: 1_100_000, time: 1 }, + { type: "clock", timestamp: 1_349_900, time: 1.25 }, + { type: "clock", timestamp: 1_600_000, time: 1.5 }, + { type: "clock", timestamp: 1_900_000, time: 1.75 }, + { type: "clock", timestamp: 2_155_000, time: 2, jumped: 5000 }, + { type: "clock", timestamp: 2_400_000, time: 2.25 }, { type: "clock", time: 2.5 }, ]); } finally { buffer.close(); + clock.close(); + } + }); + + // Before the output has played anything there is no position to pair the clock with. + it("sends no clock until the output reports a position", () => { + const { buffer, sent, state } = fake(100, { + clock: { now: () => Duration.millis(1000), out: { reference: new Signal(0) } }, + }); + try { + state(ms(0), false, 0); + expect(sent.filter((msg) => msg.type === "clock")).toEqual([{ type: "clock", time: 0 }]); + } finally { + buffer.close(); + } + }); + + // The ring renders ahead of the speaker by the output latency, so it holds the delay less that, and + // the delay the clock holds counts the base latency and the output latency's recent peak. + it("holds the delay less the output latency it renders ahead by", () => { + const { buffer, sent, state, context } = fake(300); + try { + context.baseLatency = 0.01; + context.outputLatency = 0.12; + state(ms(0), false, 0); + context.outputLatency = 0.104; + state(ms(0), false, 0); + expect(buffer.output.peek()).toEqual(Duration.millis(130)); + buffer.setLatency(Duration.millis(300)); + // A delay short of the output latency still holds one callback. + buffer.setLatency(Duration.millis(50)); + expect(sent.filter((msg) => msg.type === "latency")).toEqual([ + { type: "latency", latency: 300, target: 180, bound: 0 }, + { type: "latency", latency: 128, target: 10, bound: 0 }, + ]); + } finally { + buffer.close(); + } + }); + + // The speaker sounds what was rendered an output latency ago, so the playback timestamp is where + // the rendered playhead stood then, advancing at real time, and holds through a stall. + it("reports the playhead sounding now", () => { + const clock = started(); + const { buffer, state, context } = fake(100); + try { + context.sounding = 1; + state(ms(5000), false, 0, 1.1); + // Nothing rendered at or before what is sounding: the first report, back by the difference. + expect(buffer.timestamp.peek()).toBe(ms(4900)); + context.sounding = 1.12; + state(ms(5050), false, 0, 1.15); + expect(buffer.timestamp.peek()).toBe(ms(5020)); + // The ring stalled at 5050: what sounds holds there once it has caught up. + context.sounding = 1.17; + state(ms(5050), true, 0, 1.2); + expect(buffer.timestamp.peek()).toBe(ms(5050)); + } finally { + buffer.close(); + clock.close(); } }); }); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index be3d2ae17fc..94807097a86 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -1,18 +1,17 @@ -import { Duration } from "@moq/clock"; +import { Duration, Instant, now } from "@moq/clock"; import { Effect, type Getter, Signal } from "@moq/signals"; -import { ringHold, ringSamples } from "./latency"; +import { Peak, ringHold, ringSamples } from "./latency"; import type { Clock, Data, End, Init, Latency, Playing, Received, Reset, State, Truncate } from "./render"; // The stretch's 30 ms processing window, in microseconds. const WINDOW = 30_000; -// How far the clock's offset from the context's time may rise per state report: far past any drift +// How far the clock's offset from the context's time may fall per state report: far past any drift // between the two clocks, and well short of the render callback a reading can land within. -const RISE = 100; +const FALL = 100; // An offset that moves further than this moved for real, as a delay or a reference change moves it, -// rather than by where a reading landed within a render callback. Readings after a longer callback -// all land just after it, so they agree without the filter. +// rather than by where a reading landed within a render callback. const JUMP = 10_000; /** @@ -241,11 +240,20 @@ export class AudioBuffer { readonly rate: number; readonly channels: number; #worklet: AudioWorkletNode; + // The realtime context the worklet renders in, which reports where its output stands. + #context: AudioContext; readonly #timestamp = new Signal(0); - /** Current playback timestamp, as the worklet last reported it. */ + /** The playback timestamp sounding now, as of the worklet's last report. */ readonly timestamp: Getter = this.#timestamp; + readonly #output = new Signal(Duration.ZERO); + /** + * How long the output takes to sound what the ring renders: the context's base latency plus the + * highest output latency it reported over the last five seconds. + */ + readonly output: Getter = this.#output; + readonly #stalled = new Signal(true); /** Whether the ring is stalled (waiting to fill). */ readonly stalled: Getter = this.#stalled; @@ -278,10 +286,16 @@ export class AudioBuffer { #playing?: Getter; // How far audio may trail the shared clock before the ring skips ahead. #bound: Duration; - // The shared clock less the context's time, in microseconds, as last sent. The context's time - // advances a render quantum at a time, so a reading lands up to a quantum late: the least offset is - // the reading taken nearest a quantum's start. It rises only slowly, as drift needs, unless it moves - // past any quantum, as a delay or reference change moves it. + // The peak output latency, between the render and the speaker, which the ring does not hold. + #latency = new Peak(); + #heard = Duration.ZERO; + // What the worklet reported rendering, oldest first, back to what is sounding now. + #rendered: { time: number; timestamp: number }[] = []; + // The shared clock less the context time sounding at the same instant, in microseconds, as last + // sent. A browser that reads its output position only once per render callback reports it up to a + // callback early, so the greatest offset is the reading taken nearest a callback's start. It falls + // only slowly, as drift needs, unless it moves past any callback, as a delay or reference change + // moves it. #offset?: number; // The shared clock's reference as last sent, in milliseconds. One that moved earlier jumped the // clock ahead, which the ring skips rather than speeds through, as video skips the frames it passed. @@ -309,6 +323,7 @@ export class AudioBuffer { const { channels, rate, buffered, playing, clock } = props; this.#playing = props.playing; this.#worklet = worklet; + this.#context = worklet.context as AudioContext; this.channels = channels; this.rate = rate; this.#bound = props.bound ?? Duration.ZERO; @@ -333,7 +348,9 @@ export class AudioBuffer { this.#signals.event(worklet.port, "message", (ev: Event) => { const data = (ev as MessageEvent).data; if (data?.type === "state") { - this.#timestamp.set(data.timestamp); + const heard = this.#sounding(); + this.#observeOutput(); + this.#timestamp.set(this.#heardAt(data, heard)); this.#stalled.set(data.stalled); this.#underruns.set(data.underruns); this.#skips.set(data.skips); @@ -342,7 +359,7 @@ export class AudioBuffer { this.#silent.set(data.silent); if (data.epoch === this.#epoch) { this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled); - if (clock) this.#clock(clock.now(), clock.out.reference.peek()); + if (clock) this.#clock(clock.now(), clock.out.reference.peek(), heard); } } }); @@ -350,35 +367,69 @@ export class AudioBuffer { worklet.port.start(); } - // The terms as the worklet takes them. + // The terms as the worklet takes them. The ring's target is the delay less the output latency it + // renders ahead by, though never less than one callback, the least it can render from. #terms(delay: Duration): Omit { + const base = Duration.secs(this.#context.baseLatency || 0); return { latency: (this.#backpressure.floor / this.rate) * 1000, - target: delay.asMillis(), + target: delay.saturatingSub(this.#heard).max(base.min(delay)).asMillis(), bound: this.#bound.asMillis(), }; } - // Send where the shared clock stands, in the ring's coordinates, read together with the context's - // time so a late message misplaces neither. Each reading follows a state report, which lands just - // after the context renders a callback, when its time runs up to that callback ahead of the wall - // clock: the ring then trails the clock by up to one callback, the audio a callback pulls at once. - #clock(now: Duration | undefined, reference: number | undefined): void { - const time = this.#worklet.context.currentTime; + // The context time sounding now, in seconds, or undefined before the output has played anything. + // The browser reports its output position with the instant it was heard, on the page's clock. + #sounding(): number | undefined { + const position = this.#context.getOutputTimestamp(); + if (!position.performanceTime || position.contextTime === undefined) return undefined; + return position.contextTime + (now().since(Instant.ZERO).asMillis() - position.performanceTime) / 1000; + } + + // Fold in the output latency the context reports now. `baseLatency` is the callback the context + // renders at once; `outputLatency`, where reported, the rest of the way to the speaker. + #observeOutput(): void { + const context = this.#context; + this.#heard = this.#latency.observe(Duration.secs(context.outputLatency || 0), now()); + this.#output.set(Duration.secs(context.baseLatency || 0).add(this.#heard)); + } + + // Where the playhead sounding at context time `heard` stands, from what the worklet rendered: the + // report rendered last at or before it, advanced at real time up to the next. Without a position, + // the reported playhead. + #heardAt(state: State, heard: number | undefined): number { + if (heard === undefined) { + this.#rendered = []; + return state.timestamp; + } + const rendered = this.#rendered; + rendered.push({ time: state.time, timestamp: state.timestamp }); + while (rendered.length > 1 && rendered[1].time <= heard) rendered.shift(); + const [at, next] = rendered; + const projected = at.timestamp + (heard - at.time) * 1_000_000; + return Math.max(0, next ? Math.min(projected, next.timestamp) : Math.min(projected, at.timestamp)); + } + + // Send where the shared clock stands, in the ring's coordinates, against the context time sounding + // at the same instant, so the ring renders each sample to sound when the clock says it is due. + #clock(now: Duration | undefined, reference: number | undefined, heard: number | undefined): void { + const time = this.#context.currentTime; const last = this.#reference; this.#reference = reference; - if (now === undefined) { + if (now === undefined || heard === undefined) { this.#offset = undefined; const msg: Clock = { type: "clock", time }; this.#worklet.port.postMessage(msg); return; } - // A jump is a step of the clock, not where a reading landed, so the offset takes it whole. - const jumped = last !== undefined && reference !== undefined ? Math.max(0, last - reference) * 1000 : 0; - const offset = now.asMicros() - this.#backpressure.offset - time * 1_000_000; - const previous = this.#offset === undefined ? undefined : this.#offset + jumped; + // A reference move is a step of the clock, not where a reading landed, so the offset takes it + // whole. One earlier jumped the clock ahead, which the ring skips rather than speeds through. + const moved = last !== undefined && reference !== undefined ? (last - reference) * 1000 : 0; + const jumped = Math.max(0, moved); + const offset = now.asMicros() - this.#backpressure.offset - heard * 1_000_000; + const previous = this.#offset === undefined ? undefined : this.#offset + moved; this.#offset = - previous !== undefined && Math.abs(offset - previous) < JUMP ? Math.min(offset, previous + RISE) : offset; + previous !== undefined && Math.abs(offset - previous) < JUMP ? Math.max(offset, previous - FALL) : offset; const msg: Clock = { type: "clock", timestamp: this.#offset + time * 1_000_000, @@ -496,6 +547,7 @@ export class AudioBuffer { this.#worklet.port.postMessage(msg); this.#backpressure.reset(); this.#offset = undefined; + this.#rendered = []; } /** diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index d2028a5eebe..71718ef29ab 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -32,7 +32,13 @@ let moduleLoaded: Promise = Promise.resolve(); class FakeContext extends EventTarget { state: AudioContextState = "running"; readonly currentTime = 0; + readonly baseLatency = 0; + readonly outputLatency = 0; readonly sampleRate: number; + // No output position yet, so the ring plays without a clock. + getOutputTimestamp(): AudioTimestamp { + return { contextTime: 0, performanceTime: 0 }; + } readonly audioWorklet = { addModule: () => moduleLoaded }; readonly calls = { suspend: 0, resume: 0, close: 0 }; constructor(options: AudioContextOptions) { @@ -113,6 +119,7 @@ class FakeWorklet { const state: State = { type: "state", timestamp: ring.timestamp, + time: 0, stalled: ring.stalled, anchored: ring.anchored, underruns: ring.underruns, diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 2a07d2f0383..16fa51c71df 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -225,11 +225,14 @@ export class Decoder { const measured = effect.get(this.#measured); if (measured === undefined) return undefined; const delay = effect.get(this.source.out.config)?.delay; + const ring = effect.get(this.#ring); return target({ measured, advertised: effect.get(this.source.out.jitter), frame: effect.get(this.#frame), delay: delay !== undefined ? Duration.millis(delay) : undefined, + // Audio is placed on the clock where it is heard, so video waits for the output too. + output: ring ? effect.get(ring.output) : undefined, }); }); const startup = this.#signals.computed((effect) => { @@ -511,7 +514,9 @@ export class Decoder { if (!ring) return; // The ring plays onto the shared clock, slowing down onto a rise and speeding up onto a fall, - // and skips only audio trailing it past what the subscription still delivers. + // and skips only audio trailing it past what the subscription still delivers. It holds the + // delay less the output latency, which moves its target even under a fixed delay. + effect.get(ring.output); const budget = this.#budget(effect); ring.setLatency(budget.delay, bound(budget)); } diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index 8ee53995323..569c445ccea 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -1,4 +1,4 @@ -import { Duration } from "@moq/clock"; +import { Duration, type Instant } from "@moq/clock"; /** The terms of the audio playout target. */ export interface Target { @@ -10,11 +10,13 @@ export interface Target { frame?: Duration; /** The catalog `delay`: how far this rendition trails the broadcast's others, if any. */ delay?: Duration; + /** How long the output takes to sound what the ring renders, once known. */ + output?: Duration; } /** * The "auto" playout target: the measured term floored by the advertised span, plus one frame, plus - * the rendition's catalog delay. + * the rendition's catalog delay, plus the output latency. * * The advertised span is a floor rather than an addend, because the receiver's measurement already * contains the publisher's batching. The catalog delay is an addend, because the measurement is @@ -23,7 +25,29 @@ export interface Target { */ export function target(props: Target): Duration { const floored = props.measured.max(props.advertised ?? Duration.ZERO); - return floored.add(props.frame ?? Duration.ZERO).add(props.delay ?? Duration.ZERO); + return floored + .add(props.frame ?? Duration.ZERO) + .add(props.delay ?? Duration.ZERO) + .add(props.output ?? Duration.ZERO); +} + +// How long the highest reported output latency holds. A browser's report moves with where each read +// lands, so the delay the clock holds for it follows the peak rather than every reading, and falls +// once a lower latency, as after a switch off a Bluetooth headset, has held this long. +const OUTPUT_WINDOW = Duration.secs(5); + +/** The highest output latency reported within the last five seconds. */ +export class Peak { + // Each reading that no later one is at least as high as, oldest first, so the first is the peak. + #readings: { at: Instant; latency: Duration }[] = []; + + /** Fold in the latency reported at `at`, returning the peak. */ + observe(latency: Duration, at: Instant): Duration { + while (this.#readings.length > 0 && (this.#readings.at(-1)?.latency.le(latency) ?? false)) this.#readings.pop(); + this.#readings.push({ at, latency }); + while (this.#readings.length > 1 && at.since(this.#readings[0].at).gt(OUTPUT_WINDOW)) this.#readings.shift(); + return this.#readings[0].latency; + } } /** diff --git a/js/watch/src/audio/render-worklet.test.ts b/js/watch/src/audio/render-worklet.test.ts index e9fbbab923b..1c5bc729f2c 100644 --- a/js/watch/src/audio/render-worklet.test.ts +++ b/js/watch/src/audio/render-worklet.test.ts @@ -46,7 +46,14 @@ function renderer(target = 100) { sent: render.sent, read(samples: number) { const output = new Float32Array(samples); - expect(render.process([], [[output]], {})).toBe(true); + // The context's time, at zero unless a test runs it. + const time = Object.getOwnPropertyDescriptor(globalThis, "currentTime"); + if (!time) Object.defineProperty(globalThis, "currentTime", { configurable: true, value: 0 }); + try { + expect(render.process([], [[output]], {})).toBe(true); + } finally { + if (!time) Reflect.deleteProperty(globalThis, "currentTime"); + } return output; }, }; @@ -85,6 +92,8 @@ describe("render dispatch", () => { type: "state", epoch: 7, timestamp: 560_000, + // The end of the 20-sample quantum at 1 kHz, at a context time of zero. + time: 0.02, stalled: false, anchored: true, underruns: 0, diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 9b9d25982e9..8a2cc46a3b4 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -109,6 +109,7 @@ class Render extends AudioWorkletProcessor { const state: State = { type: "state", timestamp: ring.timestamp, + time: currentTime + output[0].length / ring.rate, stalled: ring.stalled, anchored: ring.anchored, underruns: ring.underruns, diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index cc3861dfd66..6f51cf2a0eb 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -61,14 +61,15 @@ export interface Latency extends Terms { } /** - * Where the shared clock stands, for the ring to time-stretch onto. The clock runs at real time, so one - * reading maps it onto the worklet's own time until the next. + * Where the shared clock stands, for the ring to time-stretch onto, at the context time the output + * sounds then. The clock runs at real time, so one reading maps it onto the worklet's own time until + * the next. */ export interface Clock { type: "clock"; - // The clock's media time at context time `time`, in the ring's coordinates, unset while it has none. + // The media time due at context time `time`, in the ring's coordinates, unset while the clock has none. timestamp?: number; - // The AudioContext's `currentTime` when the clock was read, in seconds. + // A context time in seconds. time: number; // How far the clock's reference moved earlier since the last reading, in microseconds, which the // ring skips rather than speeds through. @@ -85,6 +86,8 @@ export interface Received { export interface State { type: "state"; timestamp: number; + // The context time `timestamp` was rendered at, in seconds: the end of the quantum just read. + time: number; stalled: boolean; // Whether `timestamp` is on the current timeline: it reads 0 until a frame anchors the ring. anchored: boolean; From c8207d65a27a92daf2c1e1b2de352988a0744714 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 14:16:10 -0800 Subject: [PATCH 04/18] fix(moq-cli): don't shadow the output with the latency peak Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 175c8627071..ae9406bb7f1 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -686,7 +686,7 @@ async fn play_audio( .max(sink.output_latency().unwrap_or(AUDIO_BUFFER_MIN)) }; // The device's output latency the clock holds for. - let output = Mutex::new(Peak::default()); + let peak = Mutex::new(Peak::default()); // Anchor the sink to the shared clock: the media written so far ends at // `end`, and sounds when the clock says it is due. The callback stretches // onto that clock, and skips only what trails it past the stretch, which @@ -1026,7 +1026,7 @@ async fn play_audio( // Audio is placed on the clock where it is heard, so the device's output // latency is part of the delay video waits for too. A fixed delay is // floored at it instead, since audio cannot sound any sooner. - let device = output.lock().unwrap().observe(sink.output_latency(), Instant::now()); + let device = peak.lock().unwrap().observe(sink.output_latency(), Instant::now()); let held = if estimate { delay + device.unwrap_or_default() } else { From 1f250a9f7161b758425f00d7fcfe96799f72d76e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 14:34:16 -0800 Subject: [PATCH 05/18] test(audio-quality): project the browser's playhead to the instant it is sampled Co-Authored-By: Claude Opus 5.5 --- test/audio-quality/clients/js/src/probe.ts | 17 +++++++++++++++-- test/audio-quality/clients/js/src/schema.ts | 6 ++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts index 60d416e3074..3dc4d3084dc 100644 --- a/test/audio-quality/clients/js/src/probe.ts +++ b/test/audio-quality/clients/js/src/probe.ts @@ -78,6 +78,18 @@ export function probe(watch: MoqWatch, capture = false): Probe { // The tap follows the output node, which the player rebuilds with its graph. A tap left on the old // node would report a perfect run as one long silence. + // The playhead the player last reported, and when. It reports once per render callback, so a sample + // projects it at real time to the instant it is read, as the native player's is. + let reported: { timestamp: number; at: number } | undefined; + signals.run((effect) => { + const timestamp = effect.get(audio.out.timestamp); + reported = timestamp === undefined ? undefined : { timestamp: timestamp.asMillis(), at: performance.now() }; + }); + const playhead = (at: number): number | undefined => { + if (!reported) return undefined; + return audio.out.stalled.peek() ? reported.timestamp : reported.timestamp + (at - reported.at); + }; + let tapped: AudioNode | undefined; let current: Tap | undefined; let renderLoad: number | undefined; @@ -121,11 +133,12 @@ export function probe(watch: MoqWatch, capture = false): Probe { const gaps = current?.take() ?? []; const taken = stalls; stalls = []; + const at = performance.now(); return { // Read back to back, so the pair calibrates the render clock against the viewer's. - at: performance.now(), + at, render: context ? context.currentTime * 1000 : undefined, - timestamp: audio.out.timestamp.peek()?.asMillis(), + timestamp: playhead(at), stalled: audio.out.stalled.peek(), clock: sync.now()?.asMillis(), delay: sync.out.delay.peek().asMillis(), diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index a28b3af79ef..e4b6b2c3131 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -393,8 +393,10 @@ export type Sample = { render?: Ms; /** - * `audio.out.timestamp`: the playhead on the `media` clock. Undefined while there is none. - * Natively, projected from the last write into the speaker at the wall rate, held at its end. + * The playhead sounding at the speaker, on the `media` clock. Undefined while there is none. In + * the browser `audio.out.timestamp`, projected at the wall rate from when the player last reported + * it, and held while stalled. Natively, projected from the last write into the speaker at the wall + * rate, held at its end. */ timestamp?: Ms; /** From f183be888f5db7dc2e269574a0c68cb421cfe809 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 14:43:31 -0800 Subject: [PATCH 06/18] test,doc: cover the output latency and document placing audio where it is heard Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 24 ++++++++++++++++++++++++ js/watch/src/audio/decoder.ts | 2 +- js/watch/src/audio/latency.test.ts | 21 +++++++++++++++++++-- rs/moq-audio/src/ring.rs | 17 +++++++++++++++++ rs/moq-cli/src/play/media.rs | 18 ++++++++++++++++++ test/audio-quality/README.md | 8 +++++--- 6 files changed, 84 insertions(+), 6 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index d52fff6e7f5..be1510a5fd5 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -415,6 +415,30 @@ reference window restarts, so the new track's frames are measured against its own fastest. Its opening backlog would read as seconds late against the old track's arrivals. +## At the speaker + +Audio is placed on the clock where it is heard, not where it is rendered. The +output takes time to sound what the player hands it, so an audio track's target +also counts that **output latency**, and video waits for it too, which lip sync +on that device needs: + +``` +target = max(measured, advertised) + frame + output +``` + +The player reads the clock at the moment the output sounds a sample and renders +the media due then; its buffer holds the target less the delay between render +and speaker. The output latency is the most the device reported over the last +5 s, since a host's report moves with where each read lands. + +| | Placed at | `output` | +| --- | --- | --- | +| Browser | `getOutputTimestamp()`, the context time sounding at a page instant, against which the clock is paired at the greatest offset, falling 0.1 ms a report | `baseLatency` plus the peak `outputLatency` | +| Native | the callback's playback timestamp | the host's reported delay plus one device period | + +A fixed browser delay is still taken literally. A fixed native delay is floored +at the output latency, since audio cannot sound sooner. + ## Rise, fall, startup, and the ceiling There is **no separate rise or fall limiter**. The histogram is the only diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 16fa51c71df..059f458f0e6 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -68,7 +68,7 @@ export type DecoderOutput = { sampleRate: Signal; stats: Signal; - // Current playback timestamp from worklet + // The playback timestamp sounding at the speaker, as of the worklet's last report. timestamp: Signal; // Whether the audio buffer is stalled (waiting to fill). A silence the publisher declared, such as diff --git a/js/watch/src/audio/latency.test.ts b/js/watch/src/audio/latency.test.ts index 20e39dffdcb..4d8f0578d80 100644 --- a/js/watch/src/audio/latency.test.ts +++ b/js/watch/src/audio/latency.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "bun:test"; -import { Duration } from "@moq/clock"; -import { ringCapacity, ringHold, ringSamples, target } from "./latency"; +import { Duration, Instant } from "@moq/clock"; +import { Peak, ringCapacity, ringHold, ringSamples, target } from "./latency"; const ms = (value: number) => Duration.millis(value); @@ -9,6 +9,10 @@ describe("target", () => { expect(target({ measured: ms(40), frame: ms(20) })).toEqual(ms(60)); }); + it("adds the output latency, since audio is placed where it is heard", () => { + expect(target({ measured: ms(40), frame: ms(20), output: ms(42) })).toEqual(ms(102)); + }); + it("floors the measured term at the advertised span instead of adding it", () => { expect(target({ measured: ms(40), advertised: ms(200), frame: ms(20) })).toEqual(ms(220)); expect(target({ measured: ms(300), advertised: ms(200), frame: ms(20) })).toEqual(ms(320)); @@ -57,3 +61,16 @@ describe("ringHold", () => { expect(ringHold(ms(500))).toEqual(Duration.micros(0)); }); }); + +describe("Peak", () => { + // A lower reading holds until the higher one ages out of the window; a higher one takes over at once. + it("holds the highest output latency for five seconds", () => { + const at = (secs: number) => Instant.ZERO.saturatingAdd(Duration.secs(secs)); + const peak = new Peak(); + expect(peak.observe(ms(120), at(0))).toEqual(ms(120)); + expect(peak.observe(ms(104), at(1))).toEqual(ms(120)); + expect(peak.observe(ms(112), at(5))).toEqual(ms(120)); + expect(peak.observe(ms(108), at(6))).toEqual(ms(112)); + expect(peak.observe(ms(128), at(7))).toEqual(ms(128)); + }); +}); diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index df13d4fe241..45fb66f40fe 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -1687,6 +1687,23 @@ mod tests { assert!(cons.inputs(0).lag <= 0); } + /// Reads are placed where they sound, so the queue holds the target less + /// the device's delay, though never less than the callback being read. + #[test] + fn the_device_delay_comes_off_the_target_the_queue_holds() { + let rate = 48_000; + let (mut prod, mut cons, pcm) = tone(rate, Duration::from_millis(100)); + let base = prod.base; + assert_eq!(prod.push(&pcm), Push::Ok); + let mut out = vec![0.0; 480 * 2]; + cons.heard(1_440); + pull(&mut cons, &mut out, base); + assert_eq!(cons.inputs(0).target, 4_800 - 1_440); + cons.heard(48_000); + pull(&mut cons, &mut out, base); + assert_eq!(cons.inputs(0).target, 480); + } + #[test] fn staged_audio_is_buffered_and_finish_drains_it() { let (mut prod, mut cons) = channel( diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index ae9406bb7f1..cb0e7e1f054 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -2797,6 +2797,24 @@ mod tests { ); } + /// The latency the clock holds is the device's peak over the window: a lower + /// reading holds until the higher one ages out, and a higher one takes over at once. + #[test] + fn the_output_latency_holds_its_peak_for_the_window() { + let start = Instant::now(); + let at = |secs: u64| start.saturating_add(Duration::from_secs(secs)); + let ms = Duration::from_millis; + let mut peak = Peak::default(); + assert_eq!(peak.observe(None, at(0)), None); + assert_eq!(peak.observe(Some(ms(20)), at(0)), Some(ms(20))); + assert_eq!(peak.observe(Some(ms(14)), at(1)), Some(ms(20))); + assert_eq!(peak.observe(None, at(5)), Some(ms(20))); + assert_eq!(peak.observe(Some(ms(15)), at(6)), Some(ms(15))); + assert_eq!(peak.observe(Some(ms(40)), at(7)), Some(ms(40))); + // Unreported since, the last peak stands. + assert_eq!(peak.observe(None, at(60)), Some(ms(40))); + } + #[tokio::test] async fn a_finite_audio_track_plays_its_final_samples() { tokio::time::pause(); diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 687f76d877b..b9f71e1e4f5 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -97,7 +97,8 @@ in what the native player does: - Its ring stretches received PCM onto the shared clock by the page's rule. A ring that still runs dry emits silence until a cushion of received PCM arrives; `silence` counts that padding. Native `stalled` is only a sink with nothing written yet. -- The playhead is projected from the player's last write at the wall rate and held at its end. +- The playhead is projected from the player's last write at the wall rate and held at its end; the + write sounds after what the sink holds and the device's own delay. - `device` is the host's reported output latency, with no `baseLatency`, and `render_load` is unmeasured. @@ -318,8 +319,9 @@ player's own count, which its stats panel shows, is read alongside: that starts within 60 ms of a stall is that stall, and is not an underrun. - `silence`: the share of rendered quanta under -60 dBFS at the output, whatever the cause. - `target_ms`: the resolved playout delay (`sync.out.delay`). -- `av_lag_ms`: how far the audio playhead trails the shared clock video plays from (`sync.now()`), - sampled while playing. Audio and video keep separate clocks that align once the delay covers the +- `av_lag_ms`: how far the audio sounding at the speaker trails the shared clock video plays from + (`sync.now()`), sampled while playing. The browser's playhead (`audio.out.timestamp`) is the one + sounding as of its last report, projected at the wall rate to the sample. Audio and video keep separate clocks that align once the delay covers the arrivals, so this is the latency audio adds beyond the target: positive is audio late, negative early. Its `p50` is where audio rests, so latency spent to buy fewer underruns shows there; its `p95` includes audio trailing through a stall. Natively, the presentation clock `moq play` paces From df19bf57d2aba55274e503acbb9fa7359537cce4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 14:43:56 -0800 Subject: [PATCH 07/18] quest: finish output latency Co-Authored-By: Claude Opus 5.5 --- quest/m0/nightly/README.md | 2 +- quest/m2/audio-playout/README.md | 6 ++-- quest/m2/audio-playout/output-latency.md | 46 ------------------------ 3 files changed, 5 insertions(+), 49 deletions(-) delete mode 100644 quest/m2/audio-playout/output-latency.md diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 42a03707b87..ab4ddf718d5 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -56,6 +56,6 @@ audit ([moq-dev/moq#8648](https://github.com/moq-dev/moq/pull/8648)), the size report's output ([moq-dev/moq#8654](https://github.com/moq-dev/moq/pull/8654)), the firefox resume ([resume sync](/quest/m0/interop-green/resume-sync.md)), and the chromium `av_lag` rows -([output latency](/quest/m2/audio-playout/output-latency.md) and +([moq-dev/moq#8676](https://github.com/moq-dev/moq/pull/8676) and [congestion catch-up](/quest/m2/audio-playout/congestion-catch-up.md)). diff --git a/quest/m2/audio-playout/README.md b/quest/m2/audio-playout/README.md index e944482a3e3..05978683a23 100644 --- a/quest/m2/audio-playout/README.md +++ b/quest/m2/audio-playout/README.md @@ -91,8 +91,10 @@ Order: [congestion catch-up](/quest/m2/audio-playout/congestion-catch-up.md) gives the stall its snap on today's splices, the [lag rule](/quest/m2/audio-playout/lag-rule.md) replaces the controller under it, and the [clock step](/quest/m2/audio-playout/clock-step.md) builds -on the lag rule. [Target fall](/quest/m2/audio-playout/target-fall.md) and -[output latency](/quest/m2/audio-playout/output-latency.md) are independent. +on the lag rule. [Target fall](/quest/m2/audio-playout/target-fall.md) is +independent. Output latency landed in +[moq-dev/moq#8676](https://github.com/moq-dev/moq/pull/8676): both players +place audio where it is heard and count the output latency in the target. The [calibration](/quest/m3/audio-calibrate.md) re-measures every ceiling after this epic. diff --git a/quest/m2/audio-playout/output-latency.md b/quest/m2/audio-playout/output-latency.md deleted file mode 100644 index c53bf9e2047..00000000000 --- a/quest/m2/audio-playout/output-latency.md +++ /dev/null @@ -1,46 +0,0 @@ -# [M] Audio is placed on the clock where it is heard - -## Goal - -Each player compares audio with the shared clock at the moment the audio -leaves the speaker, so audio and video stay in sync on any output device, and -the audio target counts that output latency. The browser rests on the clock -as native does, without the underruns a clock pairing alone caused, and -native's fixed 50 ms floor, which raises the shared delay for video too, is -replaced by its device's actual latency. - -## Plan - -Planned 2026-11-11 (maintainer, /quest-merge pick of -[moq-dev/moq#8507](https://github.com/moq-dev/moq/pull/8507)) for the -browser's render callback. Widened 2026-11-13 (maintainer, /quest-plan): -✅ lag is measured where audio is heard, in both players; the -[epic](/quest/m2/audio-playout/README.md) records the decision. - -Neither player reads its output latency today. The browser never reads -`baseLatency` or `outputLatency`. Natively `playback/driver.rs` places audio -by the host clock at the callback and hands the device's reported latency -only to the playback ledger, and `moq play` floors the audio target at 50 ms -(`AUDIO_BUFFER_MIN` in `play/media.rs`) instead. So audio is -heard later than video by the device latency: tens of milliseconds on a -desktop sink, 150 to 250 ms over Bluetooth. - -The browser evidence: headless Chromium renders 2048-frame callbacks -(`baseLatency` 42.7 ms) and pairs the clock with `currentTime` just after -each one, so it rests 28 to 35 ms behind where native rests within a few ms; -Nightly's near-zero rows show +12 to +20 ms since #8507. A max-filtered -wall-clock pairing put audio on the clock (p50 -1.3 to 4.5 ms) but underran -near-zero Opus 45 times a minute, since the target did not count the -callback. - -Recommendations, in one PR: - -1. Add the output latency to each player's audio target: in the browser, - `baseLatency` (or the measured callback size where it is missing) plus - `outputLatency` where reported, and natively the device period plus the - latency the host reports. The - shared delay then covers it, so video waits that much longer, which lip - sync on that device needs. -2. Read the clock at output time: in the browser, pair it on the wall clock - with the max filter; natively, use the callback's playback timestamp. -3. Measure the live rows before and after in both players. From 3fd16bacde54515fc8fda1aa48f37d26c44c64a5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:02:32 -0800 Subject: [PATCH 08/18] fix(watch,moq-audio,moq-cli): hold only the post-output depth at startup and resume, and publish the output latency whole Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 7 +++-- js/watch/src/audio/buffer.ts | 24 ++++++++++------ rs/moq-audio/src/playback/observe.rs | 25 +++++++++------- rs/moq-audio/src/ring.rs | 32 +++++++++++++++------ rs/moq-cli/src/play/media.rs | 43 +++++++++++++++++----------- 5 files changed, 85 insertions(+), 46 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 9477ee607d7..0c5d547814f 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -603,8 +603,9 @@ describe("AudioBuffer steering", () => { } }); - // The ring renders ahead of the speaker by the output latency, so it holds the delay less that, and - // the delay the clock holds counts the base latency and the output latency's recent peak. + // The ring renders ahead of the speaker by the output latency, so it holds the delay less that, + // decoded floor included, and the delay the clock holds counts the base latency and the output + // latency's recent peak. it("holds the delay less the output latency it renders ahead by", () => { const { buffer, sent, state, context } = fake(300); try { @@ -618,7 +619,7 @@ describe("AudioBuffer steering", () => { // A delay short of the output latency still holds one callback. buffer.setLatency(Duration.millis(50)); expect(sent.filter((msg) => msg.type === "latency")).toEqual([ - { type: "latency", latency: 300, target: 180, bound: 0 }, + { type: "latency", latency: 180, target: 180, bound: 0 }, { type: "latency", latency: 128, target: 10, bound: 0 }, ]); } finally { diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 94807097a86..26504cc67bf 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -328,13 +328,14 @@ export class AudioBuffer { this.rate = rate; this.#bound = props.bound ?? Duration.ZERO; - this.#backpressure = new Backpressure(rate, buffered, props.delay); + const held = this.#held(props.delay); + this.#backpressure = new Backpressure(rate, buffered, held); const msg: Init = { type: "init", channels, rate, - ...this.#terms(props.delay), + ...this.#terms(held), ...(playing && { playing: playing.peek() }), }; worklet.port.postMessage(msg); @@ -367,13 +368,19 @@ export class AudioBuffer { worklet.port.start(); } - // The terms as the worklet takes them. The ring's target is the delay less the output latency it - // renders ahead by, though never less than one callback, the least it can render from. - #terms(delay: Duration): Omit { + // What the ring holds of a delay: the delay less the output latency it renders ahead by, though + // never less than one callback, the least it can render from. Its floor, its startup, and what + // stays encoded upstream all follow this, so a start never queues media the clock then skips. + #held(delay: Duration): Duration { const base = Duration.secs(this.#context.baseLatency || 0); + return delay.saturatingSub(this.#heard).max(base.min(delay)); + } + + // The terms as the worklet takes them, for what the ring holds. + #terms(held: Duration): Omit { return { latency: (this.#backpressure.floor / this.rate) * 1000, - target: delay.saturatingSub(this.#heard).max(base.min(delay)).asMillis(), + target: held.asMillis(), bound: this.#bound.asMillis(), }; } @@ -484,10 +491,11 @@ export class AudioBuffer { */ setLatency(delay: Duration, bound: Duration = this.#bound): void { this.#bound = bound; - this.#backpressure.setLatency(delay); + const held = this.#held(delay); + this.#backpressure.setLatency(held); // The clock moves with the delay, so its offset starts over with the next reading. this.#offset = undefined; - const msg: Latency = { type: "latency", ...this.#terms(delay) }; + const msg: Latency = { type: "latency", ...this.#terms(held) }; this.#worklet.port.postMessage(msg); } diff --git a/rs/moq-audio/src/playback/observe.rs b/rs/moq-audio/src/playback/observe.rs index f81a6ce69f6..921a1691611 100644 --- a/rs/moq-audio/src/playback/observe.rs +++ b/rs/moq-audio/src/playback/observe.rs @@ -101,9 +101,9 @@ pub(super) struct Shared { clock: AtomicU64, /// [`Observation::latency`] in nanoseconds, or [`UNKNOWN`]. latency: AtomicU64, - /// The last buffer's duration at the device rate in nanoseconds: one - /// device period. Zero before the first. - period: AtomicU64, + /// [`Shared::output`] in nanoseconds, or [`UNKNOWN`]: published whole, so + /// a reader never sums one callback's delay with another's period. + output: AtomicU64, quanta: AtomicU64, quiet: AtomicU64, /// [`Observation::comfort`] in nanoseconds. @@ -146,7 +146,7 @@ impl Default for Shared { Self { clock: AtomicU64::new(0), latency: AtomicU64::new(UNKNOWN), - period: AtomicU64::new(0), + output: AtomicU64::new(UNKNOWN), quanta: AtomicU64::new(0), quiet: AtomicU64::new(0), comfort: AtomicU64::new(0), @@ -187,8 +187,10 @@ impl Shared { /// the most of a buffer that sounds after its read. `None` before a host /// has reported a delay. pub(super) fn output(&self) -> Option { - let period = Duration::from_nanos(self.period.load(Ordering::Relaxed)); - self.delay().map(|delay| delay + period) + match self.output.load(Ordering::Relaxed) { + UNKNOWN => None, + nanos => Some(Duration::from_nanos(nanos)), + } } /// Wake a [`Observer::finish`] whose measurement a callback or the last stream @@ -368,10 +370,13 @@ impl Ledger { /// The host's latency for the buffer being filled, if it reports one, and /// that buffer's length in frames. pub(super) fn latency(&self, latency: Option, frames: usize) { - let nanos = latency.map_or(UNKNOWN, |l| l.as_nanos().min(UNKNOWN as u128 - 1) as u64); - self.shared.latency.store(nanos, Ordering::Relaxed); - let period = (frames as f64 / self.rate * 1e9) as u64; - self.shared.period.store(period, Ordering::Relaxed); + let nanos = |latency: Duration| latency.as_nanos().min(UNKNOWN as u128 - 1) as u64; + self.shared + .latency + .store(latency.map_or(UNKNOWN, nanos), Ordering::Relaxed); + let period = Duration::from_secs_f64(frames as f64 / self.rate); + let output = latency.map_or(UNKNOWN, |latency| nanos(latency + period)); + self.shared.output.store(output, Ordering::Relaxed); } /// Fold one mixed buffer in. `supplied` is how many of its leading frames diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index 45fb66f40fe..1a92514e50c 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -97,6 +97,9 @@ pub(crate) enum Read { /// before it exists. #[derive(Default)] struct Shared { + /// Frames the device delays a read by before it sounds. Reads are placed + /// where they are heard, so the queue holds the target less this. + heard: AtomicUsize, pushed: AtomicBool, read: AtomicBool, /// Effective input frames played or discarded, including stretch's length @@ -136,6 +139,16 @@ struct Shared { resume_frames: AtomicUsize, } +impl Shared { + /// What the queue holds of the target: the rest is the device's delay + /// after a read, which the target counts. + fn queue(&self) -> usize { + self.target + .load(Ordering::Relaxed) + .saturating_sub(self.heard.load(Ordering::Relaxed)) + } +} + /// Set in [`Shared::resume`] while a resume waits for the reader. const PENDING: u64 = 1 << 63; @@ -175,6 +188,7 @@ pub(crate) fn channel(channels: usize, input_rate: u32, output_rate: u32, config let shared = Arc::new(Shared { target: AtomicUsize::new(latency), + heard: AtomicUsize::new(0), level: AtomicUsize::new(latency), ..Default::default() }); @@ -204,7 +218,6 @@ pub(crate) fn channel(channels: usize, input_rate: u32, output_rate: u32, config refill: false, prime: latency, demand: 0, - heard: 0, consumed: 0, position: 0, cleared: 0, @@ -284,7 +297,7 @@ impl Producer { } self.declared = false; let queued = self.written.wrapping_sub(self.shared.consumed.load(Ordering::Relaxed)); - let frames = self.shared.target.load(Ordering::Relaxed).saturating_sub(queued); + let frames = self.shared.queue().saturating_sub(queued); let ends = self.shared.ends.load(Ordering::Relaxed); let mut state = self.shared.resume.load(Ordering::Acquire); while state & PENDING != 0 { @@ -354,7 +367,7 @@ impl Producer { if let Some(pad) = self.pad.filter(|_| !self.shared.stretch.load(Ordering::Relaxed)) { let queued = self.ring.occupied_len(); - let target = self.shared.target.load(Ordering::Relaxed) * self.channels; + let target = self.shared.queue() * self.channels; if queued <= pad && queued < target { let samples = self.ring.push_iter(std::iter::repeat_n(0.0, target - queued)); self.written = self.written.wrapping_add(samples / self.channels); @@ -614,9 +627,6 @@ pub(crate) struct Consumer { prime: usize, /// Frames left in the device's callback as of the current read. demand: usize, - /// Frames the device delays a read by before it sounds. Reads are placed - /// where they are heard, so the queue holds the target less this. - heard: usize, consumed: usize, /// Frames taken off the ring plus the comfort noise of each resume played /// out: the writer's `written`, as far as the reader has reached. @@ -662,7 +672,7 @@ impl Consumer { /// Note that the device sounds what is read `frames` after the read, which /// the target counts and the queue therefore does not hold. pub(crate) fn heard(&mut self, frames: usize) { - self.heard = frames; + self.shared.heard.store(frames, Ordering::Relaxed); } /// Fill `output` with queued interleaved frames, and silence past them. @@ -744,8 +754,10 @@ impl Consumer { .then(|| self.shared.anchor.load(Ordering::Acquire)); // The target counts the device's delay, which the queue does not hold, // though it still holds at least the callback being read. - let target = self.shared.target.load(Ordering::Relaxed); - let target = target.saturating_sub(self.heard).max(self.demand.min(target)); + let target = self + .shared + .queue() + .max(self.demand.min(self.shared.target.load(Ordering::Relaxed))); let level = self .shared .level @@ -1699,6 +1711,8 @@ mod tests { cons.heard(1_440); pull(&mut cons, &mut out, base); assert_eq!(cons.inputs(0).target, 4_800 - 1_440); + // A resume tops the queue up to as much, so the next utterance sounds one target out. + assert_eq!(prod.shared.queue(), 4_800 - 1_440); cons.heard(48_000); pull(&mut cons, &mut out, base); assert_eq!(cons.inputs(0).target, 480); diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index cb0e7e1f054..80cecc29d4e 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -653,11 +653,29 @@ async fn play_audio( let mut timeline = AudioTimeline::default(); + // The device's output latency the clock holds for. + let peak = Mutex::new(Peak::default()); + // Audio is placed on the clock where it is heard, so the device's output + // latency is part of the delay video waits for too. A fixed delay is floored + // at it instead, since audio cannot sound any sooner. Capped as the catalog + // delay is, so a hostile advertised jitter cannot make playback retain media + // without bound. + let audible = |delay: Duration| { + let device = peak.lock().unwrap().peak(); + if estimate { + delay + device.unwrap_or_default() + } else { + delay.max(device.unwrap_or(AUDIO_BUFFER_MIN)) + } + .min(moq_mux::container::DELAY_MAX) + }; + // A replacement's opening cushion and its timeline plan must agree. The // callback now owns target convergence, rather than write-time padding // subtracting the incoming frame before it reaches the sink. Like a hole, - // the cushion is silence the sink must hold. - let lead = |delay: Duration| latency.max(delay.min(cap).max(AUDIO_BUFFER_MIN)); + // the cushion is silence the sink must hold, and a write ahead of its due + // time leads it by the device's latency too. + let lead = |delay: Duration| latency.max(audible(delay).min(cap).max(AUDIO_BUFFER_MIN)); let schedule = presentation.consume(); // The consumer decodes a packet once the sink has room for it. Its playhead is @@ -685,8 +703,6 @@ async fn play_audio( .min(cap) .max(sink.output_latency().unwrap_or(AUDIO_BUFFER_MIN)) }; - // The device's output latency the clock holds for. - let peak = Mutex::new(Peak::default()); // Anchor the sink to the shared clock: the media written so far ends at // `end`, and sounds when the clock says it is due. The callback stretches // onto that clock, and skips only what trails it past the stretch, which @@ -1021,18 +1037,8 @@ async fn play_audio( // clock holds both while the sink keeps sizing for jitter alone. The // clock holds the whole measured target, which the subscription follows, // though the sink holds no more of it than its cap: the rest stays encoded. - // Capped as the catalog delay is, so a hostile advertised jitter cannot - // make playback retain media without bound. - // Audio is placed on the clock where it is heard, so the device's output - // latency is part of the delay video waits for too. A fixed delay is - // floored at it instead, since audio cannot sound any sooner. - let device = peak.lock().unwrap().observe(sink.output_latency(), Instant::now()); - let held = if estimate { - delay + device.unwrap_or_default() - } else { - delay.max(device.unwrap_or(AUDIO_BUFFER_MIN)) - } - .min(moq_mux::container::DELAY_MAX); + peak.lock().unwrap().observe(sink.output_latency(), Instant::now()); + let held = audible(delay); *jitter.lock().unwrap() = Some(held); moved |= role.set_target(capped(held, *stated.borrow())); // The consumer releases media due further out than the sink holds only @@ -1142,6 +1148,11 @@ impl Peak { { self.readings.pop_front(); } + self.peak() + } + + /// The peak as of the last reading. + fn peak(&self) -> Option { self.readings.front().map(|&(_, latency)| latency) } } From c1d98771808ba7861748aa7ceb8b84eafda4f2d8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:19:27 -0800 Subject: [PATCH 09/18] fix(watch,moq-cli): count the output latency in buffered startup and in native write-ahead Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.ts | 15 +++++++++------ js/watch/src/audio/decoder.ts | 8 ++++++-- rs/moq-cli/src/play/media.rs | 8 ++++++-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 26504cc67bf..2320e0ec2d7 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -328,7 +328,7 @@ export class AudioBuffer { this.rate = rate; this.#bound = props.bound ?? Duration.ZERO; - const held = this.#held(props.delay); + const held = this.held(props.delay); this.#backpressure = new Backpressure(rate, buffered, held); const msg: Init = { @@ -368,10 +368,13 @@ export class AudioBuffer { worklet.port.start(); } - // What the ring holds of a delay: the delay less the output latency it renders ahead by, though - // never less than one callback, the least it can render from. Its floor, its startup, and what - // stays encoded upstream all follow this, so a start never queues media the clock then skips. - #held(delay: Duration): Duration { + /** + * What the ring holds of a delay: the delay less the output latency it renders ahead by, though + * never less than one callback, the least it can render from. Its floor, its startup, and what + * stays encoded upstream all follow this, so a start never queues media the clock then skips. + * @internal + */ + held(delay: Duration): Duration { const base = Duration.secs(this.#context.baseLatency || 0); return delay.saturatingSub(this.#heard).max(base.min(delay)); } @@ -491,7 +494,7 @@ export class AudioBuffer { */ setLatency(delay: Duration, bound: Duration = this.#bound): void { this.#bound = bound; - const held = this.#held(delay); + const held = this.held(delay); this.#backpressure.setLatency(held); // The clock moves with the delay, so its offset starts over with the next reading. this.#offset = undefined; diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 059f458f0e6..73e2a81809d 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -247,8 +247,12 @@ export class Decoder { for (const range of effect.get(this.#queued) ?? []) { queued = queued.add(range.end.saturatingSub(range.start.max(span.end))); } - const hold = ringHold(effect.get(this.sync.out.delay)); - return { start: span.start, end: span.end.add(queued.min(hold)) }; + // The ring holds the delay less the output latency, which the speaker covers. + const delay = effect.get(this.sync.out.delay); + const ring = effect.get(this.#ring); + if (ring) effect.get(ring.output); + const held = ring ? ring.held(delay) : delay; + return { start: span.start, end: span.end.add(queued.min(ringHold(held))).add(delay.saturatingSub(held)) }; }); this.#track = this.sync.track({ target: playout, startup }); this.#signals.cleanup(() => this.#track.close()); diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 80cecc29d4e..de33d249300 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -670,12 +670,16 @@ async fn play_audio( .min(moq_mux::container::DELAY_MAX) }; + // How far ahead of its due time a write may sound: what the sink holds, at + // most its cap, plus the device's latency after it. + let reach = || cap + peak.lock().unwrap().peak().unwrap_or_default(); + // A replacement's opening cushion and its timeline plan must agree. The // callback now owns target convergence, rather than write-time padding // subtracting the incoming frame before it reaches the sink. Like a hole, // the cushion is silence the sink must hold, and a write ahead of its due // time leads it by the device's latency too. - let lead = |delay: Duration| latency.max(audible(delay).min(cap).max(AUDIO_BUFFER_MIN)); + let lead = |delay: Duration| latency.max(audible(delay).min(reach()).max(AUDIO_BUFFER_MIN)); let schedule = presentation.consume(); // The consumer decodes a packet once the sink has room for it. Its playhead is @@ -1045,7 +1049,7 @@ async fn play_audio( // when its timeline ends before the media behind it covers the hold, so // that little waits here instead of overflowing the sink. let held_back = wait(&mut catalog, &|| { - presentation.due(mapped).and_then(|due| due.checked_sub(cap)) + presentation.due(mapped).and_then(|due| due.checked_sub(reach())) }) .await; // A fresh sink plays its first write behind an opening cushion sized before From e18ff56cdb6a60f7d0f6d4b93a05eb022d12a1a3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:26:21 -0800 Subject: [PATCH 10/18] fix(watch): start buffered playback with the opening due as it first sounds Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/decoder.ts | 9 +++++++-- js/watch/src/sync.ts | 6 +++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 73e2a81809d..53653cb8676 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -247,12 +247,17 @@ export class Decoder { for (const range of effect.get(this.#queued) ?? []) { queued = queued.add(range.end.saturatingSub(range.start.max(span.end))); } - // The ring holds the delay less the output latency, which the speaker covers. + // The ring holds the delay less the output latency, which the speaker covers: the opening + // sounds that much after it renders, so the span reaches back by it. The clock anchors on the + // earliest start, which puts the opening due as it first sounds rather than skipping it. const delay = effect.get(this.sync.out.delay); const ring = effect.get(this.#ring); if (ring) effect.get(ring.output); const held = ring ? ring.held(delay) : delay; - return { start: span.start, end: span.end.add(queued.min(ringHold(held))).add(delay.saturatingSub(held)) }; + return { + start: span.start.saturatingSub(delay.saturatingSub(held)), + end: span.end.add(queued.min(ringHold(held))), + }; }); this.#track = this.sync.track({ target: playout, startup }); this.#signals.cleanup(() => this.#track.close()); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 176a7fb6d66..75b3cbf18cf 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -86,7 +86,11 @@ export type SyncOutput = { maxDelay: Signal; }; -/** A decoded span from the opening of a track's timeline. @internal */ +/** + * What a track holds from the opening of its timeline. `start` is the media time the clock should + * read as playback starts: the opening itself, or earlier by however long the track's output takes to + * sound it. @internal + */ export type Span = { start: Duration; end: Duration }; /** What a track holds in the shared clock. @internal */ From f70c29f04d3fb1a4ad9dc527a69728bf2c2c0684 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:32:25 -0800 Subject: [PATCH 11/18] fix(watch): carry the output's lead in the startup span, so an opening at media zero keeps it Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 16 ++++++++++++++++ js/watch/src/audio/buffer.ts | 3 ++- js/watch/src/audio/decoder.ts | 6 +++--- js/watch/src/sync.test.ts | 27 +++++++++++++++++++++++++++ js/watch/src/sync.ts | 23 +++++++++++++---------- 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 0c5d547814f..9452d240610 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -590,6 +590,22 @@ describe("AudioBuffer steering", () => { } }); + // A clock short of media zero reads as zero, which would pair it late, so it waits until it moves. + it("sends no clock while it reads media zero", () => { + const clock = started(); + const { buffer, sent, state, context } = fake(100, { + clock: { now: () => Duration.ZERO, out: { reference: new Signal(0) } }, + }); + try { + context.sounding = 1; + state(ms(0), false, 0); + expect(sent.filter((msg) => msg.type === "clock")).toEqual([{ type: "clock", time: 0 }]); + } finally { + buffer.close(); + clock.close(); + } + }); + // Before the output has played anything there is no position to pair the clock with. it("sends no clock until the output reports a position", () => { const { buffer, sent, state } = fake(100, { diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 2320e0ec2d7..2d8023ebff5 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -426,7 +426,8 @@ export class AudioBuffer { const time = this.#context.currentTime; const last = this.#reference; this.#reference = reference; - if (now === undefined || heard === undefined) { + // A clock still short of media zero reads as zero, which would pair it late; wait it out. + if (now === undefined || now.isZero() || heard === undefined) { this.#offset = undefined; const msg: Clock = { type: "clock", time }; this.#worklet.port.postMessage(msg); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 53653cb8676..24095173734 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -248,15 +248,15 @@ export class Decoder { queued = queued.add(range.end.saturatingSub(range.start.max(span.end))); } // The ring holds the delay less the output latency, which the speaker covers: the opening - // sounds that much after it renders, so the span reaches back by it. The clock anchors on the - // earliest start, which puts the opening due as it first sounds rather than skipping it. + // sounds that much after it renders, so it comes due then rather than being skipped. const delay = effect.get(this.sync.out.delay); const ring = effect.get(this.#ring); if (ring) effect.get(ring.output); const held = ring ? ring.held(delay) : delay; return { - start: span.start.saturatingSub(delay.saturatingSub(held)), + start: span.start, end: span.end.add(queued.min(ringHold(held))), + lead: delay.saturatingSub(held), }; }); this.#track = this.sync.track({ target: playout, startup }); diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index 4a6056993cc..925ca2f5df5 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -417,6 +417,33 @@ describe("buffered startup", () => { } }); } + // Audio's output sounds its opening a while after it renders, so that wait counts toward the + // delay, and the opening, even at media zero, comes due as it first sounds instead of being skipped. + it("holds the opening's turn back by the time its output takes to sound it", async () => { + const clock = new Manual(); + const installed = clock.install(); + const sync = new Sync({ delay: Duration.millis(300), lookahead: Duration.millis(1000) }); + const span = new Signal<{ start: Duration; end: Duration; lead?: Duration } | null>(null); + const audio = sync.track({ target: new Signal(Duration.millis(300)), startup: span }); + try { + await clock.advance(Duration.ZERO); + audio.received(Duration.ZERO); + span.set({ start: Duration.ZERO, end: Duration.millis(170), lead: Duration.millis(120) }); + await clock.advance(Duration.ZERO); + expect(sync.now()).toBeUndefined(); + span.set({ start: Duration.ZERO, end: Duration.millis(180), lead: Duration.millis(120) }); + await clock.advance(Duration.ZERO); + // Short of media zero, which the clock reads as zero until the opening's turn. + expect(sync.now()).toEqual(Duration.ZERO); + await clock.advance(Duration.millis(150)); + expect(sync.now()).toEqual(Duration.millis(30)); + } finally { + audio.close(); + sync.close(); + installed.close(); + } + }); + it("releases startup waits on abort, reset, and close without starting playback", async () => { const clock = new Manual(); const installed = clock.install(); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 75b3cbf18cf..75d12287b30 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -87,11 +87,11 @@ export type SyncOutput = { }; /** - * What a track holds from the opening of its timeline. `start` is the media time the clock should - * read as playback starts: the opening itself, or earlier by however long the track's output takes to - * sound it. @internal + * What a track holds from the opening of its timeline, and how long its output takes to sound the + * opening once released, which counts toward the delay and holds the opening's turn back as long. + * @internal */ -export type Span = { start: Duration; end: Duration }; +export type Span = { start: Duration; end: Duration; lead?: Duration }; /** What a track holds in the shared clock. @internal */ export type TrackProps = { @@ -446,22 +446,25 @@ export class Sync { } if (clock.phase.peek() === "playing") return; const ready = spans.flatMap(({ part, span }) => (span ? [{ part, span }] : [])); + const lead = (span: Span) => span.lead ?? Duration.ZERO; if ( reference === undefined || ready.length !== spans.length || - ready.some(({ span }) => span.end.saturatingSub(span.start).lt(delay)) + ready.some(({ span }) => span.end.add(lead(span)).saturatingSub(span.start).lt(delay)) ) { clock.phase.set("starting"); return; } - // The track with the earliest opening holds the reference it anchors. - const earliest = ready.reduce((a, b) => (b.span.start.lt(a.span.start) ? b : a)); - const first = earliest.span.start; + // The track whose opening sounds earliest holds the reference it anchors. + const opens = (span: Span) => span.start.asMillis() - lead(span).asMillis(); + const earliest = ready.reduce((a, b) => (opens(b.span) < opens(a.span) ? b : a)); + const first = opens(earliest.span); clock.owner = earliest.part; // Anchor at the retained opening, not at its arrival: startup may have waited on another track. // Every track's own anchor moves by the same offset, keeping the gaps between tracks that - // playing would have held, so a later handoff moves the playhead only by that gap. - const anchored = wall() - first.asMillis() - delay.asMillis(); + // playing would have held, so a later handoff moves the playhead only by that gap. An opening + // whose output takes a while to sound comes due that much later. + const anchored = wall() - first - delay.asMillis(); for (const part of clock.parts.peek()) { if (part.anchor !== undefined) part.anchor += anchored - reference; } From 1567b48f73957c1297bb390c116f65c16813b88e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 15:41:09 -0800 Subject: [PATCH 12/18] fix(watch): pair the clock at its signed position before media zero's turn Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 19 ++++++++++++------- js/watch/src/audio/buffer.ts | 11 +++++------ js/watch/src/sync.ts | 19 +++++++++++++++++-- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 9452d240610..8cdd72d021e 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -435,7 +435,7 @@ function fake( delay: number, props: { bound?: number; - clock?: { now(): Duration | undefined; out: { reference: Getter } }; + clock?: { position(): number | undefined; out: { reference: Getter } }; } = {}, ) { const sent: Message[] = []; @@ -553,7 +553,9 @@ describe("AudioBuffer steering", () => { let now: Duration | undefined = Duration.millis(1000); const reference = new Signal(0); const clock = started(); - const { buffer, sent, state, context } = fake(100, { clock: { now: () => now, out: { reference } } }); + const { buffer, sent, state, context } = fake(100, { + clock: { position: () => now?.asMillis(), out: { reference } }, + }); const report = (time: number, sounding: number, clock: number | undefined) => { context.currentTime = time; context.sounding = sounding; @@ -590,16 +592,19 @@ describe("AudioBuffer steering", () => { } }); - // A clock short of media zero reads as zero, which would pair it late, so it waits until it moves. - it("sends no clock while it reads media zero", () => { + // Before media zero's turn, as while an opening waits for its output to sound it, the clock pairs + // at its negative position rather than the zero it reads as. + it("pairs a clock short of media zero at its negative position", () => { const clock = started(); const { buffer, sent, state, context } = fake(100, { - clock: { now: () => Duration.ZERO, out: { reference: new Signal(0) } }, + clock: { position: () => -120, out: { reference: new Signal(0) } }, }); try { context.sounding = 1; state(ms(0), false, 0); - expect(sent.filter((msg) => msg.type === "clock")).toEqual([{ type: "clock", time: 0 }]); + expect(sent.filter((msg) => msg.type === "clock")).toEqual([ + { type: "clock", timestamp: -1_120_000, time: 0 }, + ]); } finally { buffer.close(); clock.close(); @@ -609,7 +614,7 @@ describe("AudioBuffer steering", () => { // Before the output has played anything there is no position to pair the clock with. it("sends no clock until the output reports a position", () => { const { buffer, sent, state } = fake(100, { - clock: { now: () => Duration.millis(1000), out: { reference: new Signal(0) } }, + clock: { position: () => 1000, out: { reference: new Signal(0) } }, }); try { state(ms(0), false, 0); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 2d8023ebff5..d813624821e 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -315,7 +315,7 @@ export class AudioBuffer { /** Anchor to the first frame and hold the lookahead upstream until playback starts. */ buffered: boolean; /** The shared clock the ring plays onto, and the reference it plays from. */ - clock?: { now(): Duration | undefined; out: { reference: Getter } }; + clock?: { position(): number | undefined; out: { reference: Getter } }; /** Hold consumption until the shared startup floor is ready. */ playing?: Getter; }, @@ -360,7 +360,7 @@ export class AudioBuffer { this.#silent.set(data.silent); if (data.epoch === this.#epoch) { this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled); - if (clock) this.#clock(clock.now(), clock.out.reference.peek(), heard); + if (clock) this.#clock(clock.position(), clock.out.reference.peek(), heard); } } }); @@ -422,12 +422,11 @@ export class AudioBuffer { // Send where the shared clock stands, in the ring's coordinates, against the context time sounding // at the same instant, so the ring renders each sample to sound when the clock says it is due. - #clock(now: Duration | undefined, reference: number | undefined, heard: number | undefined): void { + #clock(position: number | undefined, reference: number | undefined, heard: number | undefined): void { const time = this.#context.currentTime; const last = this.#reference; this.#reference = reference; - // A clock still short of media zero reads as zero, which would pair it late; wait it out. - if (now === undefined || now.isZero() || heard === undefined) { + if (position === undefined || heard === undefined) { this.#offset = undefined; const msg: Clock = { type: "clock", time }; this.#worklet.port.postMessage(msg); @@ -437,7 +436,7 @@ export class AudioBuffer { // whole. One earlier jumped the clock ahead, which the ring skips rather than speeds through. const moved = last !== undefined && reference !== undefined ? (last - reference) * 1000 : 0; const jumped = Math.max(0, moved); - const offset = now.asMicros() - this.#backpressure.offset - heard * 1_000_000; + const offset = position * 1000 - this.#backpressure.offset - heard * 1_000_000; const previous = this.#offset === undefined ? undefined : this.#offset + moved; this.#offset = previous !== undefined && Math.abs(offset - previous) < JUMP ? Math.max(offset, previous - FALL) : offset; diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 75d12287b30..3f77976467e 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -272,11 +272,17 @@ export class Clock { } now(): Duration | undefined { + const position = this.position(); + // Media time is nonnegative, so a playhead before the first frame's turn reads as zero. + return position === undefined ? undefined : Duration.millis(Math.max(0, position)); + } + + // The media time due now in milliseconds, negative before media zero's turn. + position(): number | undefined { if (!this.playing.peek()) return undefined; const reference = this.out.reference.peek(); if (reference === undefined) return undefined; - // Media time is nonnegative, so a playhead before the first frame's turn reads as zero. - return Duration.millis(Math.max(0, wall() - reference - this.out.delay.peek().asMillis())); + return wall() - reference - this.out.delay.peek().asMillis(); } async wait(part: Part, timestamp: Duration, signal?: AbortSignal): Promise { @@ -524,6 +530,15 @@ export class Sync { return this.#clock.now(); } + /** + * {@link Sync.now} in milliseconds, negative rather than zero before media time zero's turn, as + * when an opening waits for its output to sound it. + * @internal + */ + position(): number | undefined { + return this.#clock.position(); + } + /** Stop and release everything this holds. */ close() { this.#clock.closed.set(true); From 275db581409ae7e5c67f4e2e1fa084d0205eb871 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 18:50:02 -0800 Subject: [PATCH 13/18] docs(audio-jitter): a short fixed browser delay underruns rather than desyncs Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index be1510a5fd5..735c38e824b 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -436,8 +436,9 @@ and speaker. The output latency is the most the device reported over the last | Browser | `getOutputTimestamp()`, the context time sounding at a page instant, against which the clock is paired at the greatest offset, falling 0.1 ms a report | `baseLatency` plus the peak `outputLatency` | | Native | the callback's playback timestamp | the host's reported delay plus one device period | -A fixed browser delay is still taken literally. A fixed native delay is floored -at the output latency, since audio cannot sound sooner. +A fixed browser delay is still taken literally: one shorter than the jitter plus +the output latency underruns rather than letting audio trail video. A fixed +native delay is floored at the output latency, since audio cannot sound sooner. ## Rise, fall, startup, and the ceiling From 5497e5c9721cdf56bf59c3b66d20d3f64c42fff2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 19:05:35 -0800 Subject: [PATCH 14/18] fix(moq-audio): count the part of the last read still sounding in Sink::buffered A callback consumes a whole period at once, so right after it the queue left out up to a period the device had yet to sound. The ring now publishes its consumed position with the instant its last read is heard, in one word. Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/playback/observe.rs | 9 ------ rs/moq-audio/src/playback/sink.rs | 3 +- rs/moq-audio/src/ring.rs | 47 ++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/rs/moq-audio/src/playback/observe.rs b/rs/moq-audio/src/playback/observe.rs index 921a1691611..dda9bb9b258 100644 --- a/rs/moq-audio/src/playback/observe.rs +++ b/rs/moq-audio/src/playback/observe.rs @@ -174,15 +174,6 @@ impl Shared { let _ = self.driver.set(driver); } - /// How long after a read the device sounds it, as the host last reported, - /// or `None` before a host has. - pub(super) fn delay(&self) -> Option { - match self.latency.load(Ordering::Relaxed) { - UNKNOWN => None, - nanos => Some(Duration::from_nanos(nanos)), - } - } - /// The device's output latency: the host's delay plus one device period, /// the most of a buffer that sounds after its read. `None` before a host /// has reported a delay. diff --git a/rs/moq-audio/src/playback/sink.rs b/rs/moq-audio/src/playback/sink.rs index 9afdc41b27b..8aa3b9a2b0d 100644 --- a/rs/moq-audio/src/playback/sink.rs +++ b/rs/moq-audio/src/playback/sink.rs @@ -248,8 +248,7 @@ impl Sink { /// climbs when the writer runs ahead, and falls toward the device's delay /// when it falls behind. pub fn buffered(&self) -> Duration { - let queued = self.channel.lock().unwrap().prod.buffered(); - queued + self.shared.observe.delay().unwrap_or_default() + self.channel.lock().unwrap().prod.heard_in(crate::ring::host()) } /// How long the device takes to sound what it reads: the delay its host diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index 1a92514e50c..0adeb116226 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -109,6 +109,11 @@ struct Shared { /// Frames of the writer's stream the reader has played or skipped: /// `consumed` before the stretch's length correction. played: AtomicUsize, + /// `consumed` as of the last read in the low half, and the frame on the + /// clock [`Consumer::read_at_frame`] counts at which that read's last frame + /// is heard in the high half, each wrapping at 32 bits. One word, so a + /// writer never pairs one read's position with another's end. + drained: AtomicU64, /// The writer's end less `played`, at its latest push. Adding the reader's /// `played` less `consumed` gives the depth that push left, less what the /// stretch removed since: the level the stretch decides on. @@ -469,11 +474,23 @@ impl Producer { } /// How much audio is queued ahead of the reader. + #[cfg(test)] pub(crate) fn buffered(&self) -> Duration { Duration::from_secs_f64( self.written.wrapping_sub(self.shared.consumed.load(Ordering::Relaxed)) as f64 / self.rate as f64, ) } + + /// How long after `now` the last frame written is heard: what is queued, + /// plus what the last read took that has yet to sound, since a read is + /// placed at the instant its first frame is heard. + pub(crate) fn heard_in(&self, now: Instant) -> Duration { + let drained = self.shared.drained.load(Ordering::Relaxed); + let queued = (self.written as u32).wrapping_sub(drained as u32); + let now = (now.saturating_duration_since(self.base).as_secs_f64() * self.rate as f64).round() as usize; + let sounding = ((drained >> 32) as u32).wrapping_sub(now as u32) as i32; + Duration::from_secs_f64((queued as f64 + sounding.max(0) as f64) / self.rate as f64) + } } /// Converts whole chunks of input to the reader's rate. @@ -691,6 +708,15 @@ impl Consumer { /// [`read_at`](Self::read_at) on the frame clock [`Producer::due_frame`] counts. pub(crate) fn read_at_frame(&mut self, output: &mut [f32], now: usize, demand: usize) -> Read { + let read = self.read_frames(output, now, demand); + // Published once the whole read is done: what it consumed sounds by its end. + let end = now.wrapping_add(output.len() / self.channels); + let drained = (end as u32 as u64) << 32 | self.consumed as u32 as u64; + self.shared.drained.store(drained, Ordering::Relaxed); + read + } + + fn read_frames(&mut self, output: &mut [f32], now: usize, demand: usize) -> Read { self.start(); self.demand = demand; self.now = now; @@ -1943,6 +1969,27 @@ mod tests { assert!(out.iter().all(|&sample| sample == 0.0)); } + /// The last frame written is heard after what is queued and the part of + /// the last read still sounding, which shrinks until the next read. + #[test] + fn the_last_write_is_heard_behind_the_read_still_sounding() { + let (mut prod, mut cons) = queue(48_000, 48_000, Duration::from_secs(1)); + let base = prod.base; + let millis = |at: Duration| (at.as_secs_f64() * 1000.0).round() as u64; + assert_eq!(prod.push(&[0.1; 4800 * STEREO]), Push::Ok); + // The 50 ms opening cushion and 100 ms written, before any read. + assert_eq!(millis(prod.heard_in(base)), 150); + + // A 10 ms read the device sounds 20 ms after `base`. + let mut out = vec![0.0; 480 * STEREO]; + pull(&mut cons, &mut out, base + Duration::from_millis(20)); + assert_eq!(millis(prod.heard_in(base)), 170); + assert_eq!(millis(prod.heard_in(base + Duration::from_millis(25))), 145); + assert_eq!(millis(prod.heard_in(base + Duration::from_millis(30))), 140); + // Past the read's end, only the queue is left. + assert_eq!(millis(prod.heard_in(base + Duration::from_millis(40))), 140); + } + /// A next utterance pushed while the end's tail still plays waits behind it, /// with comfort noise topping the wait up to one latency. #[test] From ded421b4b7d291e4add4ff4054b08e449a73cdc0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 19:42:15 -0800 Subject: [PATCH 15/18] test(moq-audio): count the device latency in the device tests' drain checks Sink::buffered runs to the speaker now, so a high-latency default device such as a Bluetooth headset would fail the 250 ms check while draining normally. Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/tests/integration/playback.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rs/moq-audio/tests/integration/playback.rs b/rs/moq-audio/tests/integration/playback.rs index fd26a672ed3..e31aab1dbc3 100644 --- a/rs/moq-audio/tests/integration/playback.rs +++ b/rs/moq-audio/tests/integration/playback.rs @@ -121,9 +121,11 @@ async fn plays_a_tone_through_the_default_device() { kio::clock::sleep(Duration::from_millis(100)).await; peaked |= sink.control().peak() > 0.4; - // The device is consuming if the queue stays near the target latency - // rather than growing by a full chunk every write. - drained |= sink.buffered() < Duration::from_millis(250); + // The device is consuming if what is buffered stays near the target + // latency rather than growing by a full chunk every write. It runs to the + // speaker, so it counts the device's own latency too. + let output = sink.output_latency().unwrap_or_default(); + drained |= sink.buffered() < Duration::from_millis(250) + output; } assert!(peaked, "the mixer never saw the tone"); @@ -217,7 +219,7 @@ async fn switches_devices_without_dropping_sinks() { let _ = sink.write(&tone).expect("write"); kio::clock::sleep(Duration::from_millis(200)).await; assert!( - sink.buffered() < Duration::from_secs(1), + sink.buffered() < Duration::from_secs(1) + sink.output_latency().unwrap_or_default(), "{} did not drain after the switch", device.name ); From 1ca3afe5a94faeac52e90010fb11b6bf3e066f56 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 20:02:54 -0800 Subject: [PATCH 16/18] fix(watch,moq-cli): cap the native target last, and read the output latency before the ring's first hold The device floor came after the native target's cap, so a device reporting more latency than the sink was sized for got its target refused. The browser ring read the output latency only on its first state report, so a startup released before it held the full delay. The probe re-anchors its projection when a stall ends, and the docs fold the output-latency exception into Fixed targets. Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 11 ++++++++--- js/watch/src/audio/buffer.test.ts | 20 ++++++++++++++++++++ js/watch/src/audio/buffer.ts | 2 ++ rs/moq-cli/src/play/media.rs | 5 +++-- test/audio-quality/clients/js/src/probe.ts | 2 ++ 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 735c38e824b..1b59900c733 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -436,9 +436,8 @@ and speaker. The output latency is the most the device reported over the last | Browser | `getOutputTimestamp()`, the context time sounding at a page instant, against which the clock is paired at the greatest offset, falling 0.1 ms a report | `baseLatency` plus the peak `outputLatency` | | Native | the callback's playback timestamp | the host's reported delay plus one device period | -A fixed browser delay is still taken literally: one shorter than the jitter plus -the output latency underruns rather than letting audio trail video. A fixed -native delay is floored at the output latency, since audio cannot sound sooner. +A fixed browser delay is still taken literally, and a fixed native delay is +raised to the output latency, as [Fixed targets](#fixed-targets) describes. ## Rise, fall, startup, and the ceiling @@ -547,6 +546,12 @@ An impossible fixed target is refused, not silently adjusted. A negative delay, or one beyond the track's max age, is an error at configuration time. Supported or refused, not warned about and continued. +The one exception is the [output latency](#at-the-speaker), which no delay can +undercut. Natively, `moq play` raises a fixed delay below its device's output +latency to that latency, since audio cannot sound sooner. The browser keeps the +delay as given, so one shorter than the jitter plus the output latency +underruns rather than letting audio trail video. + In the browser this is the numeric `delay` setting; natively it is `decode::Options::delay.min` with `decode::Options::estimate` set to `false`. `estimate` defaults to `true`, which runs the estimator instead and never goes diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index 8cdd72d021e..50b411f9908 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -648,6 +648,26 @@ describe("AudioBuffer steering", () => { } }); + // Startup holds only what the ring must from the first message, before any state report. + it("holds the delay less the output latency the context reports at construction", () => { + const sent: Message[] = []; + const port = Object.assign(new EventTarget(), { postMessage: (msg: Message) => sent.push(msg), start() {} }); + const context = silent(); + context.outputLatency = 0.12; + const buffer = new AudioBuffer({ port, context } as unknown as AudioWorkletNode, { + channels: 1, + rate: 1000, + delay: Duration.millis(300), + buffered: false, + }); + try { + expect(buffer.held(Duration.millis(300))).toEqual(Duration.millis(180)); + expect(sent[0]).toMatchObject({ type: "init", target: 180 }); + } finally { + buffer.close(); + } + }); + // The speaker sounds what was rendered an output latency ago, so the playback timestamp is where // the rendered playhead stood then, advancing at real time, and holds through a stall. it("reports the playhead sounding now", () => { diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index d813624821e..138a53c7a3a 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -328,6 +328,8 @@ export class AudioBuffer { this.rate = rate; this.#bound = props.bound ?? Duration.ZERO; + // The output latency the context already reports, so startup holds only what the ring must. + this.#observeOutput(); const held = this.held(props.delay); this.#backpressure = new Backpressure(rate, buffered, held); diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index de33d249300..0bfc7a431cb 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -700,12 +700,13 @@ async fn play_audio( }; steer(None); // What the sink holds: the shared delay, which counts the device's own - // latency, capped at what the sink was sized for and never below that latency. + // latency, never below that latency, and capped last at what the sink was + // sized for, so a device reporting more than that cannot get it refused. let target = |sink: &::Sink| { schedule .delay() - .min(cap) .max(sink.output_latency().unwrap_or(AUDIO_BUFFER_MIN)) + .min(cap) }; // Anchor the sink to the shared clock: the media written so far ends at // `end`, and sounds when the clock says it is due. The callback stretches diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts index 3dc4d3084dc..5deb2318f39 100644 --- a/test/audio-quality/clients/js/src/probe.ts +++ b/test/audio-quality/clients/js/src/probe.ts @@ -80,9 +80,11 @@ export function probe(watch: MoqWatch, capture = false): Probe { // node would report a perfect run as one long silence. // The playhead the player last reported, and when. It reports once per render callback, so a sample // projects it at real time to the instant it is read, as the native player's is. + // A stall ending re-anchors it too: the playhead held through the stall, so it resumes from now. let reported: { timestamp: number; at: number } | undefined; signals.run((effect) => { const timestamp = effect.get(audio.out.timestamp); + effect.get(audio.out.stalled); reported = timestamp === undefined ? undefined : { timestamp: timestamp.asMillis(), at: performance.now() }; }); const playhead = (at: number): number | undefined => { From 537e9296929182b7106c5f9fa38155dd028030b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 09:54:43 -0800 Subject: [PATCH 17/18] fix(watch): drop the worklet's target term and Manual import the merge left in the tests Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/buffer.test.ts | 10 ++++++---- js/watch/src/audio/buffer.ts | 5 ++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/js/watch/src/audio/buffer.test.ts b/js/watch/src/audio/buffer.test.ts index c25cdd52907..7c38a2d6f67 100644 --- a/js/watch/src/audio/buffer.test.ts +++ b/js/watch/src/audio/buffer.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "bun:test"; -import { Duration, Instant, Manual, now } from "@moq/clock"; +import { Duration, Instant, now } from "@moq/clock"; +import { Manual } from "@moq/clock/test"; import { type Getter, Signal } from "@moq/signals"; import { AudioBuffer, Backpressure } from "./buffer"; import type { Init, Message, State } from "./render"; @@ -638,10 +639,11 @@ describe("AudioBuffer steering", () => { expect(buffer.output.peek()).toEqual(Duration.millis(130)); buffer.setLatency(Duration.millis(300)); // A delay short of the output latency still holds one callback. + expect(buffer.held(Duration.millis(50))).toEqual(Duration.millis(10)); buffer.setLatency(Duration.millis(50)); expect(sent.filter((msg) => msg.type === "latency")).toEqual([ - { type: "latency", latency: 180, target: 180, bound: 0 }, - { type: "latency", latency: 128, target: 10, bound: 0 }, + { type: "latency", latency: 180, bound: 0 }, + { type: "latency", latency: 128, bound: 0 }, ]); } finally { buffer.close(); @@ -662,7 +664,7 @@ describe("AudioBuffer steering", () => { }); try { expect(buffer.held(Duration.millis(300))).toEqual(Duration.millis(180)); - expect(sent[0]).toMatchObject({ type: "init", target: 180 }); + expect(sent[0]).toMatchObject({ type: "init", latency: 180 }); } finally { buffer.close(); } diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 41709cb444a..8ccf426db9c 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -366,7 +366,10 @@ export class AudioBuffer { if (data.epoch === this.#epoch) { this.#backpressure.observe(data.anchored ? data.timestamp : undefined, data.stalled); if (clock) { - const msg: Clock = { type: "clock", ...this.#read(clock.position(), clock.out.reference.peek(), heard) }; + const msg: Clock = { + type: "clock", + ...this.#read(clock.position(), clock.out.reference.peek(), heard), + }; this.#worklet.port.postMessage(msg); } } From 8f08697c43978b55d782fa1c566b4b00a22180e0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 14 Nov 2026 11:50:01 -0800 Subject: [PATCH 18/18] docs(audio-jitter): the first target formula is the network's share Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index 273aece2966..60f974bca53 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -441,7 +441,8 @@ already inside every number on this page: a publisher writing 100 ms of media at once produces arrivals whose relative delay reaches 100 ms less one frame, on a perfect network, with no help from the catalog. -So the advertised value is a **floor**, not an addend: +So the advertised value is a **floor**, not an addend, in the network's share of +the target (audio adds its output latency [at the speaker](#at-the-speaker)): ``` target = max(measured, advertised) + frame