From 836ab77b870d445db26c3fd0ebe8a4c38790ca79 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 02:06:07 -0800 Subject: [PATCH 1/7] chore(quest): claim quest/m1/audio-jitter-target/native-av-lag.md Quest-Claim: 0962abb6ef0106f0 From e67ad6b2c2ebe26c256913789fec38f5ddd9f540 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 02:06:56 -0800 Subject: [PATCH 2/7] feat(play): moq play --metrics reports the presentation clock beside the playhead Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/metrics.rs | 7 ++++++- rs/moq-mux/src/sync.rs | 5 +++++ test/audio-quality/README.md | 3 ++- test/audio-quality/clients/js/src/schema.ts | 4 ++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/rs/moq-cli/src/play/metrics.rs b/rs/moq-cli/src/play/metrics.rs index e5325d74b5a..6cbb1e3e1cf 100644 --- a/rs/moq-cli/src/play/metrics.rs +++ b/rs/moq-cli/src/play/metrics.rs @@ -5,7 +5,8 @@ //! and meanings, so its analyzer reduces a native run exactly as it does a //! browser one. Durations are milliseconds as floats. `at` is this process's //! monotonic clock from the first sample, `render` the device's clock, and -//! `timestamp` the media clock; fields this player cannot read are left out +//! `timestamp` the media clock, with `clock` where the presentation clock video +//! plays from stands on it at the same instant; fields this player cannot read are left out //! rather than written as zero. `comfort` and `utterances` have no browser //! counterpart: they show how a declared silence played, for push-to-talk. @@ -39,6 +40,8 @@ struct Sample { #[serde(skip_serializing_if = "Option::is_none")] timestamp: Option, #[serde(skip_serializing_if = "Option::is_none")] + clock: Option, + #[serde(skip_serializing_if = "Option::is_none")] stalled: Option, #[serde(skip_serializing_if = "Option::is_none")] delay: Option, @@ -241,11 +244,13 @@ impl Metrics { // Read straight after the device clock, so the two pair up. let now = Instant::now(); let (playhead, delay) = (self.playhead.at(now), self.presentation.delay()); + let clock = self.presentation.position(now); let sample = Sample { at: ms(now.duration_since(self.start)), render: observation.as_ref().map(|_| ms(totals.clock)), timestamp: playhead.map(ms), + clock: clock.map(ms), stalled: observation.as_ref().map(|(o, _)| o.stalled), delay: Some(ms(delay)), output_latency: observation.as_ref().and_then(|(o, _)| o.latency).map(ms), diff --git a/rs/moq-mux/src/sync.rs b/rs/moq-mux/src/sync.rs index c56adbf5aba..d8f810a52a6 100644 --- a/rs/moq-mux/src/sync.rs +++ b/rs/moq-mux/src/sync.rs @@ -710,6 +710,11 @@ impl Consumer { self.current.lock().unwrap().state.lock().unwrap().due(timestamp) } + /// The media time the clock presents at `at`, or `None` before the first anchor. + pub fn position(&self, at: Instant) -> Option { + self.current.lock().unwrap().state.lock().unwrap().position(at) + } + /// The newest due picture's index in presentation order, or `None` to repeat the previous one. /// /// Pass queued timestamps in ascending order and discard entries through the diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 57a8072ad54..04039206453 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -318,7 +318,8 @@ player's own count, which its stats panel shows, is read alongside: - `av_lag_ms`: how far the audio playhead trails the shared clock video plays from (`sync.now()`), sampled while playing. Audio and video keep separate clocks that align once the delay covers the arrivals, so this is the latency audio adds beyond the target: positive is audio late, negative - early. Its replay ceilings hold audio from settling later to buy fewer underruns. Unset natively. + early. Its replay ceilings hold audio from settling later to buy fewer underruns. Natively, the + presentation clock `moq play` paces video from (`sync::Consumer::position`). - `converge_ms`: from first audio to the last time the target moved more than one 20 ms bucket from its final value. - `render_load`: Chromium's render capacity, the share of each quantum's budget the graph used. diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index fece2bea085..6bfad590044 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -157,7 +157,7 @@ export const METRICS = { clock: "media", aggregations: ["p50", "p95"], description: - "How far the audio playhead trails the shared clock video plays from (`sync.now()`), sampled while playing: latency audio adds beyond the resolved target. Negative is audio ahead. Unset natively.", + "How far the audio playhead trails the shared clock video plays from (`sync.now()`), sampled while playing: latency audio adds beyond the resolved target. Negative is audio ahead.", }, target_ms: { unit: "ms", @@ -410,7 +410,7 @@ export type Sample = { * is only a sink with nothing written yet, or none at all. */ stalled?: boolean; - /** `sync.now()`: where the shared clock puts video, on the `media` clock. Unset natively. */ + /** `sync.now()`: where the shared clock puts video, on the `media` clock. Natively, `sync::Consumer::position`. */ clock?: Ms; /** `sync.out.delay`: the resolved playout target. Natively, the target the speaker is held on. */ delay?: Ms; From 5f5e107519b5bc2d6d5b6c52fb808279ddaccc37 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 02:26:05 -0800 Subject: [PATCH 3/7] fix(audio): native playback decides its stretch where audio arrives Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter/stretch/corpus.json | 38 ++---- js/watch/src/audio/stretch.test.ts | 17 +-- rs/moq-audio/src/ring.rs | 125 ++++++++++++++++++- rs/moq-audio/src/stretch.rs | 53 ++++---- 4 files changed, 171 insertions(+), 62 deletions(-) diff --git a/doc/concept/audio-jitter/stretch/corpus.json b/doc/concept/audio-jitter/stretch/corpus.json index e864b8552e1..b761dbb07fe 100644 --- a/doc/concept/audio-jitter/stretch/corpus.json +++ b/doc/concept/audio-jitter/stretch/corpus.json @@ -53,8 +53,7 @@ -0.2377641350030899, -0.22275163233280182, -0.2022542506456375, -0.1767766922712326, -0.14694631099700928, -0.11349762231111526, -0.07725425064563751, -0.03910861536860466 ], - "buffered": 320, - "target": 640, + "excess": -320, "cooldown": 0, "decision": "Expand", "lag": 40, @@ -171,8 +170,7 @@ -0.2377641350030899, -0.22275163233280182, -0.2022542506456375, -0.1767766922712326, -0.14694631099700928, -0.11349762231111526, -0.07725425064563751, -0.03910861536860466 ], - "buffered": 800, - "target": 640, + "excess": 159, "cooldown": 0, "decision": "Accelerate", "lag": 40, @@ -273,8 +271,7 @@ -0.2377641350030899, -0.22275163233280182, -0.2022542506456375, -0.1767766922712326, -0.14694631099700928, -0.11349762231111526, -0.07725425064563751, -0.03910861536860466 ], - "buffered": 3200, - "target": 640, + "excess": 2560, "cooldown": 160, "decision": "Fast", "lag": 120, @@ -306,7 +303,7 @@ ] }, { - "name": "hold-in-band", + "name": "hold-on-target", "rate": 8000, "channels": 1, "input": [ @@ -359,8 +356,7 @@ -0.2377641350030899, -0.22275163233280182, -0.2022542506456375, -0.1767766922712326, -0.14694631099700928, -0.11349762231111526, -0.07725425064563751, -0.03910861536860466 ], - "buffered": 720, - "target": 640, + "excess": 0, "cooldown": 0, "decision": "Hold", "lag": 0, @@ -469,8 +465,7 @@ -0.2377641350030899, -0.22275163233280182, -0.2022542506456375, -0.1767766922712326, -0.14694631099700928, -0.11349762231111526, -0.07725425064563751, -0.03910861536860466 ], - "buffered": 320, - "target": 640, + "excess": -320, "cooldown": 1, "decision": "Hold", "lag": 0, @@ -580,8 +575,7 @@ -0.6991210579872131, -0.7436450123786926, 0.511226236820221, 0.30140137672424316, -0.3033992648124695, 0.7745504975318909, -0.3500271439552307 ], - "buffered": 320, - "target": 640, + "excess": -320, "cooldown": 0, "decision": "Expand", "lag": 0, @@ -703,8 +697,7 @@ -0.0009706376004032791, 0.0006069837254472077, 0.00024918612325564027, 0.0006460933363996446, -0.0007419105968438089, -0.00017247976211365312, 0.00025107196415774524, 0.0007953110034577549 ], - "buffered": 320, - "target": 640, + "excess": -320, "cooldown": 0, "decision": "Expand", "lag": 62, @@ -889,8 +882,7 @@ -0.1767766922712326, 0.1767766922712326, -0.14694631099700928, 0.14694631099700928, -0.11349762231111526, 0.11349762231111526, -0.07725425064563751, 0.07725425064563751, -0.03910861536860466, 0.03910861536860466 ], - "buffered": 320, - "target": 640, + "excess": -320, "cooldown": 0, "decision": "Expand", "lag": 40, @@ -1294,8 +1286,7 @@ -0.04953653737902641, -0.04253482446074486, -0.03549857810139656, -0.028433511033654213, -0.021345358341932297, -0.014239874668419361, -0.007122828625142574 ], - "buffered": 1764, - "target": 3528, + "excess": -1764, "cooldown": 0, "decision": "Expand", "lag": 441, @@ -1976,8 +1967,7 @@ -0.045558881014585495, -0.03910861536860466, -0.03263154625892639, -0.026132116094231606, -0.019614774733781815, -0.013083988800644875, -0.006544237025082111 ], - "buffered": 4800, - "target": 3840, + "excess": 959, "cooldown": 0, "decision": "Accelerate", "lag": 240, @@ -2286,8 +2276,7 @@ -0.603949248790741, -0.10583518445491791, -0.32648566365242004, -0.4554884731769562, -0.27422934770584106, -0.09860859811306 ], - "buffered": 800, - "target": 640, + "excess": 159, "cooldown": 0, "decision": "Accelerate", "lag": 0, @@ -2398,8 +2387,7 @@ -0.603949248790741, -0.10583518445491791, -0.32648566365242004, -0.4554884731769562, -0.27422934770584106, -0.09860859811306 ], - "buffered": 3200, - "target": 640, + "excess": 2560, "cooldown": 0, "decision": "Fast", "lag": 81, diff --git a/js/watch/src/audio/stretch.test.ts b/js/watch/src/audio/stretch.test.ts index 9d6de528ad0..74dec6e5c4b 100644 --- a/js/watch/src/audio/stretch.test.ts +++ b/js/watch/src/audio/stretch.test.ts @@ -9,8 +9,8 @@ interface Vector { rate: number; channels: number; input: number[]; - buffered: number; - target: number; + /** Frames the ring sits above its target right after an arrival; negative is short of it. */ + excess: number; cooldown: number; /** As native playback decides: it also expands, which this player never does. */ decision: Decision | "Expand"; @@ -29,16 +29,9 @@ const vectors = JSON.parse(readFileSync(corpus, "utf8")) as Vector[]; const planar = (samples: number[], channels: number) => Array.from({ length: channels }, (_, channel) => samples.filter((_, i) => i % channels === channel)); -// The corpus decides as NetEq does, on the buffered depth against a band around the target. This -// player decides on the excess above the target alone and never expands, so the corpus grades its -// seams: an excess past the 20 ms band fast-accelerates, one inside it accelerates, and a ring at or -// short of its target, where NetEq holds or expands, plays its window unchanged. -const excess = (vector: Vector) => { - const band = Math.floor((vector.rate * 20) / 1000); - if (vector.decision === "Fast") return vector.buffered - vector.target; - if (vector.decision === "Accelerate") return band - 1; - return Math.min(0, vector.buffered - vector.target); -}; +// This player never expands, so a ring short of its target, where native playback expands, plays +// its window unchanged. +const excess = (vector: Vector) => Math.max(0, vector.excess); for (const vector of vectors) { it(`stretches the corpus PCM: ${vector.name}`, () => { diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index ce324ba4733..b3c5259de17 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -83,6 +83,9 @@ struct Shared { /// correction. One publication avoids sampling the ring and staging buffer /// on opposite sides of a window transfer. consumed: AtomicUsize, + /// The writer's `written`, published after each push, so the reader sees + /// when audio arrives. + written: AtomicUsize, target: AtomicUsize, finished: AtomicBool, stretch: AtomicBool, @@ -130,6 +133,7 @@ pub(crate) fn channel(channels: usize, input_rate: u32, output_rate: u32, config prod.push_iter(std::iter::repeat_n(0.0, latency * channels)); let shared = Arc::new(Shared { + written: AtomicUsize::new(latency), target: AtomicUsize::new(latency), ..Default::default() }); @@ -153,6 +157,12 @@ pub(crate) fn channel(channels: usize, input_rate: u32, output_rate: u32, config refill: false, prime: latency, consumed: 0, + rendered: 0, + seen: latency, + newest: latency, + deepest: [latency; 2], + second: 0, + half: 0, position: 0, cleared: 0, cushioned: 0, @@ -258,6 +268,12 @@ impl Producer { /// Input frames are admitted only while their output has room, so a full /// queue drops the newest audio rather than overwriting what is queued. pub(crate) fn push(&mut self, input: &[f32]) -> Push { + let push = self.admit(input); + self.shared.written.store(self.written, Ordering::Release); + push + } + + fn admit(&mut self, input: &[f32]) -> Push { self.shared.pushed.store(true, Ordering::Relaxed); if !self.shared.read.load(Ordering::Relaxed) { return Push::NotReady; @@ -316,6 +332,7 @@ impl Producer { } _ => true, }; + self.shared.written.store(self.written, Ordering::Release); self.shared.finished.store(true, Ordering::Relaxed); flushed } @@ -487,6 +504,23 @@ pub(crate) struct Consumer { /// The startup cushion is silence, not received audio to stretch. prime: usize, consumed: usize, + /// Frames handed to the output, ever. `consumed` less this is what the + /// stretch removed (net of what it added) and what was discarded. + rendered: usize, + /// The writer's `written` when the reader last looked. + seen: usize, + /// The newest arrival, as `written - rendered` when the reader saw it, so + /// its depth is that plus `rendered - consumed`: the depth then, less what + /// the stretch removed or the reader discarded since. See + /// [`excess`](Self::excess). + newest: usize, + /// The deepest arrival of the previous second rendered and of this one, in + /// the same form. + deepest: [usize; 2], + /// A second at the reader's rate, set with the stretcher. + second: usize, + /// Frames rendered into the current second. + half: usize, /// Frames taken off the ring plus the comfort noise of each resume played /// out: the writer's `written`, as far as the reader has reached. position: usize, @@ -510,9 +544,53 @@ impl Consumer { /// Allocate the native playout stretcher before handing this ring to a device. pub(crate) fn stretch(&mut self, rate: u32) { self.stretch = Some(Box::new(crate::stretch::Stretch::new(rate, self.channels))); + self.second = rate as usize; self.shared.stretch.store(true, Ordering::Relaxed); } + /// Frames to stretch the next window by: removed while positive, added while + /// negative. + /// + /// The depth is read where audio has just arrived, as the browser's ring + /// does. Between arrivals the queue drains toward the next one, which says + /// nothing about how far the playhead trails the newest audio, so steering + /// on the current depth would expand into every gap and settle audio a + /// burst behind the shared clock. Any arrival deeper than the target + /// accelerates. Only when every arrival over the last one to two seconds + /// fell short does it expand: a publisher clock slower than the device's, + /// or a raised target, rather than one late arrival. + /// + /// The reader sees an arrival at its next callback, so the depth it reads + /// includes up to one device period the device has since played. + fn excess(&mut self) -> isize { + let written = self.shared.written.load(Ordering::Acquire); + let depth = |anchor: usize| anchor.wrapping_add(self.rendered).wrapping_sub(self.consumed) as isize; + if written != self.seen { + self.seen = written; + self.newest = written.wrapping_sub(self.rendered); + if depth(self.newest) > depth(self.deepest[1]) { + self.deepest[1] = self.newest; + } + } + let target = self.shared.target.load(Ordering::Relaxed) as isize; + let newest = depth(self.newest) - target; + if newest > 0 { + return newest; + } + (depth(self.deepest[0]).max(depth(self.deepest[1])) - target).min(0) + } + + /// Count `frames` handed to the output, starting the next second's deepest + /// arrival at the newest once a second has rendered. + fn render(&mut self, frames: usize) { + self.rendered = self.rendered.wrapping_add(frames); + self.half += frames; + if self.second > 0 && self.half >= self.second { + self.half -= self.second; + self.deepest = [self.deepest[1], self.newest]; + } + } + /// Fill `output` with queued interleaved frames, and silence past them. pub(crate) fn read(&mut self, output: &mut [f32]) -> Read { self.start(); @@ -617,6 +695,7 @@ impl Consumer { } done += frames * channels; self.consumed = self.consumed.wrapping_add(frames); + self.render(frames); continue; } @@ -625,7 +704,9 @@ impl Consumer { if let Some((ahead, _)) = self.resume() { if ahead == 0 { let room = (output.len() - done) / channels; - done += self.cushion(Some(&mut output[done..]), room) * channels; + let frames = self.cushion(Some(&mut output[done..]), room); + self.render(frames); + done += frames * channels; continue; } frames = frames.min(ahead); @@ -633,6 +714,7 @@ impl Consumer { // A declared end's tail drains as received: running dry after it is // silence, not an underrun, so nothing should expand toward the target. let declared = self.declared(); + let excess = self.excess(); let Some(stretch) = self.stretch.as_mut() else { let frames = frames.min((output.len() - done) / channels); if frames == 0 { @@ -645,6 +727,7 @@ impl Consumer { self.comfort.played(&output[done + prime * channels..done + read]); self.position = self.position.wrapping_add(frames); self.consumed = self.consumed.wrapping_add(frames); + self.render(frames); done += read; continue; }; @@ -660,7 +743,7 @@ impl Consumer { let read = self.ring.pop_slice(&mut stretch.input[..frames * channels]); debug_assert_eq!(read, frames * channels); self.position = self.position.wrapping_add(frames); - stretch.process(frames, buffered, self.shared.target.load(Ordering::Relaxed), enabled); + stretch.process(frames, excess, enabled); self.accelerated += frames.saturating_sub(stretch.available()) as u64; // The writer still counts the input while it is being analysed. // Publish only the resulting duration change, then the rendered @@ -1007,6 +1090,44 @@ mod tests { } } + /// Audio arriving in bursts rests on the target right after each burst + /// rather than expanding into the gaps between them, which would settle it a + /// burst behind the clock. + #[test] + fn bursts_settle_on_the_target_where_they_arrive() { + let rate = 48_000; + let (mut prod, mut cons) = channel( + 2, + rate, + rate, + Config { + latency: Duration::from_millis(120), + capacity: Duration::from_secs(3), + pad: false, + trim: false, + }, + ); + cons.stretch(rate); + cons.start(); + let mut pcm = vec![0.0; rate as usize / 10 * 2]; + for (i, frame) in pcm.as_chunks_mut::<2>().0.iter_mut().enumerate() { + frame.fill((i as f32 * std::f32::consts::TAU * 200.0 / rate as f32).sin() * 0.25); + } + let mut out = vec![0.0; rate as usize / 100 * 2]; + // 100 ms of audio every 100 ms, read in 10 ms device periods, against a + // target that covers the gap. + for tick in 0..2000 { + if tick % 10 == 0 { + assert_eq!(prod.push(&pcm), Push::Ok); + if tick >= 500 { + let depth = prod.buffered().as_secs_f64() * 1000.0; + assert!((120.0..=150.0).contains(&depth), "tick {tick}: {depth} ms"); + } + } + assert_eq!(cons.read(&mut out), Read::Ok, "tick {tick}"); + } + } + #[test] fn staged_audio_is_buffered_and_finish_drains_it() { let (mut prod, mut cons) = channel( diff --git a/rs/moq-audio/src/stretch.rs b/rs/moq-audio/src/stretch.rs index e3496279b4a..a6b452e89a7 100644 --- a/rs/moq-audio/src/stretch.rs +++ b/rs/moq-audio/src/stretch.rs @@ -60,18 +60,25 @@ impl Stretch { frames } - /// All input is new, received PCM. Short blocks and finished sinks cannot - /// stretch, so a finite input never becomes concealment of an absent frame. - pub fn process(&mut self, frames: usize, buffered: usize, target: usize, enabled: bool) { + /// Stretch the first `frames` of [`input`](Self::input) by at most `excess` + /// frames: removed while positive, added while negative. All input is new, + /// received PCM. Short blocks and finished sinks cannot stretch, so a finite + /// input never becomes concealment of an absent frame. + pub fn process(&mut self, frames: usize, excess: isize, enabled: bool) { let operation = if enabled && frames == self.window { - self.decide(buffered, target) + self.decide(excess) } else { Operation::Hold }; - let lag = if operation == Operation::Hold { - 0 - } else { - pitch(&self.input, self.channels, self.rate, operation == Operation::Fast) + let lag = match operation { + Operation::Hold => 0, + Operation::Fast => pitch(&self.input, self.channels, self.rate, true), + // An ordinary stretch takes one period, and only one that fits, so the + // ring never lands past its target. + Operation::Accelerate | Operation::Expand => { + let lag = pitch(&self.input, self.channels, self.rate, false); + if lag <= excess.unsigned_abs() { lag } else { 0 } + } }; self.position = 0; self.length = frames; @@ -109,18 +116,19 @@ impl Stretch { self.cooldown = self.rate as usize * 50 / 1000; } - fn decide(&self, buffered: usize, target: usize) -> Operation { - let high = target + self.rate as usize * 20 / 1000; - if buffered >= high * 4 { + /// What a full window decides given `excess` frames above the target, as + /// the browser's ring decides, plus the expand it never takes. An excess past + /// NetEq's 20 ms granularity fast-accelerates even during cooldown; any other + /// excess accelerates, and any shortfall expands. + fn decide(&self, excess: isize) -> Operation { + if excess >= (self.rate as usize * 20 / 1000) as isize { Operation::Fast - } else if self.cooldown > 0 { + } else if self.cooldown > 0 || excess == 0 { Operation::Hold - } else if buffered >= high { + } else if excess > 0 { Operation::Accelerate - } else if buffered < target { - Operation::Expand } else { - Operation::Hold + Operation::Expand } } } @@ -205,8 +213,7 @@ mod tests { rate: u32, channels: usize, input: Vec, - buffered: usize, - target: usize, + excess: isize, cooldown: usize, decision: String, lag: usize, @@ -222,12 +229,12 @@ mod tests { stretch.input.copy_from_slice(&vector.input); stretch.cooldown = vector.cooldown; assert_eq!( - format!("{:?}", stretch.decide(vector.buffered, vector.target)), + format!("{:?}", stretch.decide(vector.excess)), vector.decision, "{}", vector.name ); - stretch.process(stretch.window, vector.buffered, vector.target, true); + stretch.process(stretch.window, vector.excess, true); assert_eq!(stretch.length.abs_diff(stretch.window), vector.lag, "{}", vector.name); assert_eq!(stretch.length * vector.channels, vector.output.len(), "{}", vector.name); for (index, (&actual, &expected)) in stretch.output[..vector.output.len()] @@ -250,7 +257,7 @@ mod tests { let mut stretch = Stretch::new(rate, 2); stretch.input.fill(0.25); stretch.cooldown = 0; - stretch.process(stretch.window, stretch.window, rate as usize, true); + stretch.process(stretch.window, -(rate as isize), true); assert!(stretch.length > stretch.window); let mut output = vec![0.0; 2 * rate as usize]; let (_, allocations) = crate::allocation::measure(|| { @@ -258,9 +265,9 @@ mod tests { assert_eq!(stretch.read(&mut output), 0); }); assert_eq!(allocations.operations, 0); - stretch.process(stretch.window, stretch.window, rate as usize, false); + stretch.process(stretch.window, -(rate as isize), false); assert_eq!(stretch.length, stretch.window, "a finished sink drains unchanged"); - stretch.process(stretch.window / 2, stretch.window, rate as usize, true); + stretch.process(stretch.window / 2, -(rate as isize), true); assert_eq!(stretch.length, stretch.window / 2, "a short block drains unchanged"); } } From 99db817c890858902dcc4e5a7af0cb98e4d69f9a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 02:41:48 -0800 Subject: [PATCH 4/7] docs(audio): the native stretch decision, and what native A/V lag still waits on Co-Authored-By: Claude Opus 5.5 --- doc/concept/audio-jitter.md | 81 +++++++++-------- quest/m1/audio-jitter-target/README.md | 2 +- quest/m1/audio-jitter-target/native-av-lag.md | 90 +++++++++++-------- .../m1/audio-jitter-target/native-stretch.md | 14 +-- 4 files changed, 107 insertions(+), 80 deletions(-) diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index a7d55b60b45..468059a1a45 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -620,8 +620,8 @@ preemptive expand, and the browser in the AudioWorklet's ring in `js/watch` with its accelerate alone. Neither generates replacement audio for a missing packet, so a hole in the media plays as a hole. The estimator above is unchanged. Both share the pitch search, admission, seams, and -[PCM corpus](audio-jitter/stretch/corpus.json) below; they differ only in -[what they decide on](#browser-decision). The browser stretches only live +[PCM corpus](audio-jitter/stretch/corpus.json) and +[decision](#decision) below; native playback alone also expands. The browser stretches only live playback: buffered playback plays every sample it holds, as it always has. The source is WebRTC's @@ -646,45 +646,48 @@ does the tail before a [declared end](#declared-silence): running dry after it is silence rather than an underrun, so nothing expands toward a target that no arrival will refill. -`buffered` counts unread frames in the ring plus staged output. `target` is -the current playout target. The state is `cooldown`, initially 60 ms -(NetEq's six startup ticks), reduced -by each frame rendered, saturating at zero. At a window boundary, native -playback chooses in this order, with `high = target + 20 ms`: +`target` is the current playout target. The state is `cooldown`, initially +60 ms (NetEq's six startup ticks), reduced by each frame rendered, saturating +at zero. A successful stretch sets cooldown to 50 ms (NetEq's five 10 ms +ticks). A rejected seam leaves it unchanged. There is no concealment state. -1. `buffered >= 4 * high`: fast accelerate, even during cooldown. -2. `cooldown > 0`: hold. -3. `buffered >= high`: accelerate. -4. `buffered < target`: preemptive expand. -5. Otherwise: hold. +### Decision -A successful stretch sets cooldown to 50 ms (NetEq's five 10 ms ticks). -A rejected seam leaves it unchanged. This preserves NetEq's 20 ms adjustment -granularity, 4x fast threshold, and minimum interval. NetEq's extra -packet-history maximum-delay term is not added again: this controller sees the -already decoded PCM queue and its jitter target rather than an RTP playout -timestamp and packet-arrival history. There is no concealment state. - -### Browser decision - -The browser's ring starts on its target and plays at the shared clock video -plays from, so slowing it down only ever settles audio behind that clock. It -never expands. It decides on `peak`, the ring's length right after the latest -write less what the stretch has removed since: how far the playhead trails the -newest arrival, which is the playout delay NetEq's `DecisionLogic` compares. -`buffered` drains between arrivals because the next one has not landed, not +Both players decide on `peak`, the ring's depth (unread frames plus staged +output) right after the latest arrival, less what the stretch has removed +since and plus what it has added: how far the playhead trails the newest +arrival, which is the playout delay NetEq's `DecisionLogic` compares. The +depth between arrivals drains because the next one has not landed, not because audio is ahead of the clock, so deciding on it expands into every gap -and settles a burst behind. With `excess = peak - target`, choose in this order: +and settles a burst behind. Native playback sees an arrival at its next device +callback, so its `peak` also counts up to one device period already played. + +With `excess = peak - target`, choose in this order: 1. `excess >= 20 ms`: fast accelerate, even during cooldown. -2. `cooldown > 0` or `excess <= 0`: hold. -3. Otherwise: accelerate, admitting only a lag `L <= excess`. +2. `cooldown > 0`: hold. +3. `excess > 0`: accelerate, admitting only a lag `L <= excess`. +4. Native playback only, when every arrival over the last one to two seconds + rendered left `peak` short of the target: preemptive expand by the + shortfall of the deepest of them, admitting only `L <= shortfall`. +5. Otherwise: hold. So the ring rests within one pitch period of its target right after each -arrival, where dropping back to the target used to leave it, and a fast -accelerate after a refill catches up about as fast as that drop did. Arrivals that -outrun the target underrun instead: that is the cost of a target short of the -spread, and the estimator's to fix, not the stretch's. +arrival, and a fast accelerate after a refill catches up about as fast as +dropping the backlog would. Arrivals that outrun the target underrun instead: +that is the cost of a target short of the spread, and the estimator's to fix, +not the stretch's. One late arrival never expands, since slowing down for it +only adds latency; a publisher clock slower than the device's leaves every +arrival short, and native playback follows it. The browser's ring starts on +its target and plays at the shared clock, which cannot follow such a +publisher, so it never expands. The second-long halves are counted in rendered +frames: the newest arrival starts each, and the window is the current half and +the one before it. + +This keeps NetEq's 20 ms adjustment granularity and minimum interval. NetEq's +extra packet-history maximum-delay term is not added again: this controller +sees the already decoded PCM queue and its jitter target rather than an RTP +playout timestamp and packet-arrival history. ### Pitch search and admission @@ -749,7 +752,7 @@ search. Finishing bypasses that wait so a short tail can drain. Trimming is emergency recovery at the ring's capacity watermark. The CLI's incoming-media trim is reserved for a distance at the fast threshold, not ordinary 20 ms drift. The browser's ring receives each decoded frame as a -write, so it re-anchors on its target there, at the lower of `4 * high` and +write, so it re-anchors on its target there, at the lower of `4 * (target + 20 ms)` and three quarters of its headroom above the target, and counts each re-anchor as a skip in its stats. These bounds do not promise concealment of missing media or convergence of a signal whose seams continually fail admission. @@ -757,15 +760,15 @@ media or convergence of a signal whose seams continually fail admission. ### Stretch corpus `audio-jitter/stretch/corpus.json` holds literal interleaved input and expected -output PCM, device rate and channels, buffered and target frame counts, -incoming cooldown, requested decision, and admitted lag (zero means hold). +output PCM, device rate and channels, the `excess` in frames (negative is a +shortfall native playback expands by), incoming cooldown, requested decision, +and admitted lag (zero means hold). Compare output within 1e-6 absolute sample amplitude. It includes hold, cooldown, fast-mode bypass, linked stereo, passive noise, rejected active noise, and 8/44.1/48 kHz seams. Both implementations consume these exact vectors; generating expected output from the implementation under test is not conformance. The vectors decide as native playback does, so the browser -grades its accelerate and fast seams against them, at an excess inside and -past the 20 ms band, and holds wherever native expands. +holds wherever native expands. ## Comfort noise diff --git a/quest/m1/audio-jitter-target/README.md b/quest/m1/audio-jitter-target/README.md index b025813de04..f7911997e59 100644 --- a/quest/m1/audio-jitter-target/README.md +++ b/quest/m1/audio-jitter-target/README.md @@ -61,7 +61,7 @@ native playback against its budgets. - [Native stretch](/quest/m1/audio-jitter-target/native-stretch.md) - `moq play` copies NetEq's accelerate and preemptive expand, sharing the browser's seams and vectors, so its native rows play with no audible glitch once the target settles - [Buffered converge](/quest/m1/audio-jitter-target/buffered-converge.md) - a buffered `` time-stretches its audio back onto the shared clock after a stall or drift, so lip sync returns and no audio group is dropped - [Capped budgets](/quest/m1/audio-jitter-target/capped-budgets.md) - the AAC browser budgets are re-measured from nightlies after the TS importer groups each PES -- [Native A/V lag](/quest/m1/audio-jitter-target/native-av-lag.md) - `moq play` reports how far audio trails its presentation clock, and native stops expanding into the gap between arrivals if it settles audio late +- [Native A/V lag](/quest/m1/audio-jitter-target/native-av-lag.md) - `moq play` reports how far audio trails its presentation clock, and native decides its stretch where audio arrives rather than expanding into the gap between arrivals - [Native rendition delay](/quest/m1/audio-jitter-target/native-rendition-delay.md) - `moq play` holds each subscribed rendition's catalog `delay` plus its target, as the browser does - [Chromium Opus stalled share](/quest/m1/audio-jitter-target/opus-stalled-share.md) - the Chromium Opus browser rows stay under the 0.01 stalled-share budget every night diff --git a/quest/m1/audio-jitter-target/native-av-lag.md b/quest/m1/audio-jitter-target/native-av-lag.md index e7b0af6ea4d..932c8eea11a 100644 --- a/quest/m1/audio-jitter-target/native-av-lag.md +++ b/quest/m1/audio-jitter-target/native-av-lag.md @@ -4,11 +4,10 @@ `moq play --metrics` reports how far the native audio playhead trails the presentation clock video plays from, so every native row in -`test/audio-quality` grades `av_lag_ms` (p50, p95) as the replay rows do. If -native's time stretch settles audio behind that clock, as the browser's did -before [moq-dev/moq#8058](https://github.com/moq-dev/moq/pull/8058), native -stops spending latency on it: at steady state its p50 lag is at or below zero -on every native row, and its ceilings guard that. +`test/audio-quality` grades `av_lag_ms` (p50, p95) as the replay rows do, and +native's time stretch no longer settles audio behind that clock: at steady +state its p50 lag is at or below zero on every native row, and its ceilings +guard that. Boundaries: the jitter estimator and its target stay as `doc/concept/audio-jitter.md` specifies them. Underruns during congestion are @@ -20,37 +19,58 @@ drift between publisher and device, which the browser's ring cannot. Decided 2026-11-08 (maintainer, on [moq-dev/moq#8058](https://github.com/moq-dev/moq/pull/8058)): never add latency to chase zero underruns; audio and video keep separate clocks that -align at steady state once the delay is right. The browser's ring now decides -on its depth right after each arrival and never expands, because expanding on -the depth between arrivals settled audio a burst behind the shared clock -(`bursty-aac` lag p50 1 to 289 ms). Native's `Stretch::decide` -(`rs/moq-audio/src/stretch.rs`) still decides on `buffered` at each callback, -so it may do the same. - -1. Report the lag. The metrics writer (`rs/moq-cli/src/play/metrics.rs`) - already holds the presentation `moq_mux::sync::Consumer` and the - `Playhead`. Write each sample's `clock`, the presentation clock's media - time at the instant the playhead is read, with the field and meaning - `test/audio-quality/clients/js/src/schema.ts` gives the browser's, so - `analyze.ts` reduces native samples to `av_lag_ms` unchanged. Leave it out - while nothing is mapped, as other unreadable fields are. -2. Measure. Run the native rows locally against `main`'s decision, then read - the scheduled Nightlies. Lag p50 near or below zero means native already - aligns: budget `av_lag_ms` by the native rule and finish. -3. If p50 sits above zero on any row, decide on the depth right after each - arrival, less what the stretch removed since, as the browser's - `AudioRingBuffer` does, and expand only to follow a publisher clock - slower than the device's, not to fill the gap between arrivals. Hold the - listened drift case (media per render second about 0.998 against a 0.998 - publisher) and the shared stretch corpus, and update the spec's native - decision in the same PR. - -[Native stretch](/quest/m1/audio-jitter-target/native-stretch.md) is -calibrating the native ceilings from seven Nightlies of today's decision. -Step 3 moves those ceilings, so a decision change re-measures them by the -same rule rather than stacking on the old calibration. +align at steady state once the delay is right. + +[moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361) writes each +sample's presentation clock (`clock`, from `sync::Consumer::position`) and +moves native playback onto the browser's decision: the depth right after each +arrival, less what the stretch removed since, with an expand only once every +arrival over the last one to two seconds fell short, which still follows a +slower publisher clock. One local run of the 12 native rows each way (seed 7, +60 s, a loaded developer host) measured: + +- `main`'s decision on `buffered`: A/V lag p50 11 to 59 ms on Opus and 67 to + 88 ms on AAC, which accelerated about 2 s a minute even at `fixed-250` + while expanding into the gaps between PES bursts. +- The arrival decision: p50 1.3 to 6.6 ms on the adaptive rows, -13 and -16 ms + at `fixed-250`, accelerating 10 to 340 ms a minute. Underruns rose on + `step` (AAC 0 to 613.6 a minute, Opus 9.8 to 25) and AAC `wide` (0 to 17.3). + +The AAC `step` underruns start at 47 s, when the estimator relaxes its target +from 223 to 186 ms. `moq play` sets the sink's target per frame to +`due(end) - now` on the shared clock, whose reference is the earliest arrival +ever and never moves later, while the estimator measures against the fastest +frame of the last 2 s. After the shaper's delay steps from 5 to 60 ms, the +clock-aligned depth stays 55 ms short of the estimator's target, below AAC's +~160 ms PES burst spacing. `main`'s decision hid that by settling a burst +behind the clock. + +The remaining positive p50 is the accelerate's resting band: an ordinary +accelerate removes one pitch period only when it fits in the excess, so the +ring rests up to one period above its target, as the browser's does. + +Once the decisions below settle, the native `av_lag_ms` ceilings come from +[Native stretch](/quest/m1/audio-jitter-target/native-stretch.md)'s +seven-Nightly calibration of this decision. + +## Waiting + +- Maintainer: AAC `step` underruns after the shared clock stops matching a + persistent delay increase. Options: (1, recommended) accept them here and + plan a quest so the shared clock's reference (`moq_mux::sync` and + `js/watch` `Sync`) follows a delay increase every arrival in the + estimator's window shows, which also re-aligns video; (2) floor the native + sink target at the estimator's target, settling audio behind the clock by + the step; (3) keep `main`'s decision. Answer on + [moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361). +- Maintainer: the adaptive rows' p50 of 1.3 to 6.6 ms, the accelerate's + one-pitch-period resting band, against the Goal's "at or below zero". + Options: (1, recommended) accept a p50 within one pitch period as aligned + and budget it as measured; (2) let an ordinary accelerate overshoot the + target by up to one period, trading underruns for it in both players. + Answer on [moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361). ## Related -- [Native stretch](/quest/m1/audio-jitter-target/native-stretch.md) - the native ceilings this lag joins, calibrated from the current decision +- [Native stretch](/quest/m1/audio-jitter-target/native-stretch.md) - the native ceilings this lag joins, calibrated from this decision - [Browser ceilings](/quest/m1/audio-jitter-target/browser-ceilings.md) - budgets `av_lag_ms` on the chromium rows the same way diff --git a/quest/m1/audio-jitter-target/native-stretch.md b/quest/m1/audio-jitter-target/native-stretch.md index 4fdd9cff0e8..13d628bda61 100644 --- a/quest/m1/audio-jitter-target/native-stretch.md +++ b/quest/m1/audio-jitter-target/native-stretch.md @@ -38,9 +38,11 @@ cooldowns. end's tail unstretched, as finishing does. Its browser half decides on the ring's depth right after each arrival and never expands, since expanding on the depth between arrivals settled audio a burst behind the shared clock -(decided 2026-11-08). Native playback still decides on `buffered`, so -whether it settles behind its clock the same way is worth measuring once its -rows report A/V lag. +(decided 2026-11-08). +[moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361) measured native +settling behind its clock the same way (A/V lag p50 up to 88 ms) and moves it +onto that decision, with an expand only for a publisher clock slower than the +device's. Analysis samples NetEq's 48 kHz FIR on a virtual clock for other device rates and uses floating-point peak refinement and fades. Those arithmetic adaptations are documented, not claimed bit-identical to int16 @@ -151,8 +153,10 @@ bench waits below remain. ## Waiting - moq-dev/moq: seven scheduled Nightlies on a `main` that includes - [moq-dev/moq#8308](https://github.com/moq-dev/moq/pull/8308), whose - native rows report `player_skips` and `trimmed_ms`. Find them with `gh run list --workflow + [moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361), whose + native rows decide where audio arrives and report `av_lag_ms` beside + [moq-dev/moq#8308](https://github.com/moq-dev/moq/pull/8308)'s + `player_skips` and `trimmed_ms`; the calibration budgets `av_lag_ms` too. Find them with `gh run list --workflow nightly.yml --event schedule`, download each with `gh run download -n audio-quality-run` within its 7 days, keep its native rows in the calibration PR's body, and apply the calibration rule once seven exist. From 5a5f5536fb173f1c6a1e131ee8c5784f15d7b908 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 02:51:51 -0800 Subject: [PATCH 5/7] refactor(audio): keep the arrival state beside the boxed stretcher Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/ring.rs | 181 ++++++++++++++++++++++----------------- 1 file changed, 103 insertions(+), 78 deletions(-) diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index b3c5259de17..6ad96c35e37 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -157,12 +157,6 @@ pub(crate) fn channel(channels: usize, input_rate: u32, output_rate: u32, config refill: false, prime: latency, consumed: 0, - rendered: 0, - seen: latency, - newest: latency, - deepest: [latency; 2], - second: 0, - half: 0, position: 0, cleared: 0, cushioned: 0, @@ -495,7 +489,7 @@ impl Convert { pub(crate) struct Consumer { ring: HeapCons, channels: usize, - stretch: Option>, + stretch: Option>, /// Whether the window staged in the stretcher is startup cushion. staged_prime: bool, /// An actual underrun waits for received PCM, emitting emergency silence @@ -504,23 +498,6 @@ pub(crate) struct Consumer { /// The startup cushion is silence, not received audio to stretch. prime: usize, consumed: usize, - /// Frames handed to the output, ever. `consumed` less this is what the - /// stretch removed (net of what it added) and what was discarded. - rendered: usize, - /// The writer's `written` when the reader last looked. - seen: usize, - /// The newest arrival, as `written - rendered` when the reader saw it, so - /// its depth is that plus `rendered - consumed`: the depth then, less what - /// the stretch removed or the reader discarded since. See - /// [`excess`](Self::excess). - newest: usize, - /// The deepest arrival of the previous second rendered and of this one, in - /// the same form. - deepest: [usize; 2], - /// A second at the reader's rate, set with the stretcher. - second: usize, - /// Frames rendered into the current second. - half: usize, /// Frames taken off the ring plus the comfort noise of each resume played /// out: the writer's `written`, as far as the reader has reached. position: usize, @@ -543,52 +520,21 @@ pub(crate) struct Consumer { impl Consumer { /// Allocate the native playout stretcher before handing this ring to a device. pub(crate) fn stretch(&mut self, rate: u32) { - self.stretch = Some(Box::new(crate::stretch::Stretch::new(rate, self.channels))); - self.second = rate as usize; - self.shared.stretch.store(true, Ordering::Relaxed); - } - - /// Frames to stretch the next window by: removed while positive, added while - /// negative. - /// - /// The depth is read where audio has just arrived, as the browser's ring - /// does. Between arrivals the queue drains toward the next one, which says - /// nothing about how far the playhead trails the newest audio, so steering - /// on the current depth would expand into every gap and settle audio a - /// burst behind the shared clock. Any arrival deeper than the target - /// accelerates. Only when every arrival over the last one to two seconds - /// fell short does it expand: a publisher clock slower than the device's, - /// or a raised target, rather than one late arrival. - /// - /// The reader sees an arrival at its next callback, so the depth it reads - /// includes up to one device period the device has since played. - fn excess(&mut self) -> isize { + // Whatever the writer has pushed so far, the primed silence at least, + // arrived before the first window. let written = self.shared.written.load(Ordering::Acquire); - let depth = |anchor: usize| anchor.wrapping_add(self.rendered).wrapping_sub(self.consumed) as isize; - if written != self.seen { - self.seen = written; - self.newest = written.wrapping_sub(self.rendered); - if depth(self.newest) > depth(self.deepest[1]) { - self.deepest[1] = self.newest; - } - } - let target = self.shared.target.load(Ordering::Relaxed) as isize; - let newest = depth(self.newest) - target; - if newest > 0 { - return newest; - } - (depth(self.deepest[0]).max(depth(self.deepest[1])) - target).min(0) - } - - /// Count `frames` handed to the output, starting the next second's deepest - /// arrival at the newest once a second has rendered. - fn render(&mut self, frames: usize) { - self.rendered = self.rendered.wrapping_add(frames); - self.half += frames; - if self.second > 0 && self.half >= self.second { - self.half -= self.second; - self.deepest = [self.deepest[1], self.newest]; - } + self.stretch = Some(Box::new(Native { + stretch: crate::stretch::Stretch::new(rate, self.channels), + arrivals: Arrivals { + rendered: 0, + seen: written, + newest: written, + deepest: [written; 2], + second: rate as usize, + half: 0, + }, + })); + self.shared.stretch.store(true, Ordering::Relaxed); } /// Fill `output` with queued interleaved frames, and silence past them. @@ -688,14 +634,14 @@ impl Consumer { let channels = self.channels; let mut done = 0; while done < output.len() { - if let Some(stretch) = self.stretch.as_mut().filter(|stretch| stretch.available() > 0) { - let frames = stretch.read(&mut output[done..]); + if let Some(native) = self.stretch.as_mut().filter(|native| native.stretch.available() > 0) { + let frames = native.stretch.read(&mut output[done..]); + native.arrivals.render(frames); if !self.staged_prime { self.comfort.played(&output[done..done + frames * channels]); } done += frames * channels; self.consumed = self.consumed.wrapping_add(frames); - self.render(frames); continue; } @@ -705,7 +651,9 @@ impl Consumer { if ahead == 0 { let room = (output.len() - done) / channels; let frames = self.cushion(Some(&mut output[done..]), room); - self.render(frames); + if let Some(native) = &mut self.stretch { + native.arrivals.render(frames); + } done += frames * channels; continue; } @@ -714,8 +662,7 @@ impl Consumer { // A declared end's tail drains as received: running dry after it is // silence, not an underrun, so nothing should expand toward the target. let declared = self.declared(); - let excess = self.excess(); - let Some(stretch) = self.stretch.as_mut() else { + let Some(native) = self.stretch.as_mut() else { let frames = frames.min((output.len() - done) / channels); if frames == 0 { break; @@ -727,10 +674,15 @@ impl Consumer { self.comfort.played(&output[done + prime * channels..done + read]); self.position = self.position.wrapping_add(frames); self.consumed = self.consumed.wrapping_add(frames); - self.render(frames); done += read; continue; }; + let excess = native.arrivals.excess( + self.shared.written.load(Ordering::Acquire), + self.consumed, + self.shared.target.load(Ordering::Relaxed), + ); + let stretch = &mut native.stretch; let frames = frames.min(stretch.window); if frames == 0 { break; @@ -788,13 +740,13 @@ impl Consumer { } let buffered = self.ring.occupied_len() / self.channels; let cushion = self.resume().map_or(0, |(_, frames)| frames); - buffered + cushion + self.stretch.as_ref().map_or(0, |stretch| stretch.available()) + buffered + cushion + self.stretch.as_ref().map_or(0, |native| native.stretch.available()) } /// Drop up to `frames` queued frames unread, returning how many of them were /// pushed rather than the startup silence or a resume's comfort noise. pub(crate) fn discard(&mut self, frames: usize) -> usize { - let staged = self.stretch.as_mut().map_or(0, |stretch| stretch.discard(frames)); + let staged = self.stretch.as_mut().map_or(0, |native| native.stretch.discard(frames)); self.consumed = self.consumed.wrapping_add(staged); let mut pushed = if self.staged_prime { 0 } else { staged }; let mut left = frames - staged; @@ -823,6 +775,79 @@ impl Consumer { } } +/// Native playback's stretcher and the arrivals it decides on. +struct Native { + stretch: crate::stretch::Stretch, + arrivals: Arrivals, +} + +/// Where audio arrives in the queue, which native playback stretches toward +/// its target. +/// +/// The depth is read where audio has just arrived, as the browser's ring +/// does. Between arrivals the queue drains toward the next one, which says +/// nothing about how far the playhead trails the newest audio, so steering on +/// the current depth would expand into every gap and settle audio a burst +/// behind the shared clock. Any arrival deeper than the target accelerates. +/// Only when every arrival over the last one to two seconds fell short does it +/// expand: a publisher clock slower than the device's, or a raised target, +/// rather than one late arrival. +/// +/// The reader sees an arrival at its next callback, so the depth it reads +/// includes up to one device period the device has since played. +struct Arrivals { + /// Frames handed to the output, ever. The reader's `consumed` less this is + /// what the stretch removed (net of what it added) and what was discarded. + rendered: usize, + /// The writer's `written` when the reader last looked. + seen: usize, + /// The newest arrival, as `written - rendered` when the reader saw it, so + /// its depth is that plus `rendered - consumed`: the depth then, less what + /// the stretch removed or the reader discarded since. + newest: usize, + /// The deepest arrival of the previous second rendered and of this one, in + /// the same form. + deepest: [usize; 2], + /// A second at the reader's rate. + second: usize, + /// Frames rendered into the current second. + half: usize, +} + +impl Arrivals { + /// Frames to stretch the next window by toward `target`: removed while + /// positive, added while negative. `written` is the writer's latest and + /// `consumed` the reader's. + fn excess(&mut self, written: usize, consumed: usize, target: usize) -> isize { + let rendered = self.rendered; + let depth = |anchor: usize| anchor.wrapping_add(rendered).wrapping_sub(consumed) as isize; + if written != self.seen { + self.seen = written; + self.newest = written.wrapping_sub(rendered); + if depth(self.newest) > depth(self.deepest[1]) { + self.deepest[1] = self.newest; + } + } + let target = target as isize; + let newest = depth(self.newest) - target; + if newest > 0 { + return newest; + } + (depth(self.deepest[0]).max(depth(self.deepest[1])) - target).min(0) + } + + /// Count `frames` handed to the output, starting the next second's deepest + /// arrival at the newest once a second has rendered. + fn render(&mut self, frames: usize) { + self.rendered = self.rendered.wrapping_add(frames); + self.half += frames; + if self.second > 0 && self.half >= self.second { + self.half -= self.second; + self.deepest = [self.deepest[1], self.newest]; + } + } +} + #[cfg(test)] mod tests { use super::*; From ea9ac331177409ca2946cf764e049ba17af4daff Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:24:23 -0800 Subject: [PATCH 6/7] quest: settle native A/V lag decisions; the Nightly wait moves to Native stretch Co-Authored-By: Claude Opus 5.5 --- quest/m1/audio-jitter-target/README.md | 1 - quest/m1/audio-jitter-target/native-av-lag.md | 76 ------------------- .../m1/audio-jitter-target/native-stretch.md | 6 +- 3 files changed, 5 insertions(+), 78 deletions(-) delete mode 100644 quest/m1/audio-jitter-target/native-av-lag.md diff --git a/quest/m1/audio-jitter-target/README.md b/quest/m1/audio-jitter-target/README.md index f7911997e59..07e8f82e1cf 100644 --- a/quest/m1/audio-jitter-target/README.md +++ b/quest/m1/audio-jitter-target/README.md @@ -61,7 +61,6 @@ native playback against its budgets. - [Native stretch](/quest/m1/audio-jitter-target/native-stretch.md) - `moq play` copies NetEq's accelerate and preemptive expand, sharing the browser's seams and vectors, so its native rows play with no audible glitch once the target settles - [Buffered converge](/quest/m1/audio-jitter-target/buffered-converge.md) - a buffered `` time-stretches its audio back onto the shared clock after a stall or drift, so lip sync returns and no audio group is dropped - [Capped budgets](/quest/m1/audio-jitter-target/capped-budgets.md) - the AAC browser budgets are re-measured from nightlies after the TS importer groups each PES -- [Native A/V lag](/quest/m1/audio-jitter-target/native-av-lag.md) - `moq play` reports how far audio trails its presentation clock, and native decides its stretch where audio arrives rather than expanding into the gap between arrivals - [Native rendition delay](/quest/m1/audio-jitter-target/native-rendition-delay.md) - `moq play` holds each subscribed rendition's catalog `delay` plus its target, as the browser does - [Chromium Opus stalled share](/quest/m1/audio-jitter-target/opus-stalled-share.md) - the Chromium Opus browser rows stay under the 0.01 stalled-share budget every night diff --git a/quest/m1/audio-jitter-target/native-av-lag.md b/quest/m1/audio-jitter-target/native-av-lag.md deleted file mode 100644 index 932c8eea11a..00000000000 --- a/quest/m1/audio-jitter-target/native-av-lag.md +++ /dev/null @@ -1,76 +0,0 @@ -# [M] Native audio never settles behind its presentation clock - -## Goal - -`moq play --metrics` reports how far the native audio playhead trails the -presentation clock video plays from, so every native row in -`test/audio-quality` grades `av_lag_ms` (p50, p95) as the replay rows do, and -native's time stretch no longer settles audio behind that clock: at steady -state its p50 lag is at or below zero on every native row, and its ceilings -guard that. - -Boundaries: the jitter estimator and its target stay as -`doc/concept/audio-jitter.md` specifies them. Underruns during congestion are -accepted, never bought down with latency. Native keeps compensating clock -drift between publisher and device, which the browser's ring cannot. - -## Plan - -Decided 2026-11-08 (maintainer, on -[moq-dev/moq#8058](https://github.com/moq-dev/moq/pull/8058)): never add -latency to chase zero underruns; audio and video keep separate clocks that -align at steady state once the delay is right. - -[moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361) writes each -sample's presentation clock (`clock`, from `sync::Consumer::position`) and -moves native playback onto the browser's decision: the depth right after each -arrival, less what the stretch removed since, with an expand only once every -arrival over the last one to two seconds fell short, which still follows a -slower publisher clock. One local run of the 12 native rows each way (seed 7, -60 s, a loaded developer host) measured: - -- `main`'s decision on `buffered`: A/V lag p50 11 to 59 ms on Opus and 67 to - 88 ms on AAC, which accelerated about 2 s a minute even at `fixed-250` - while expanding into the gaps between PES bursts. -- The arrival decision: p50 1.3 to 6.6 ms on the adaptive rows, -13 and -16 ms - at `fixed-250`, accelerating 10 to 340 ms a minute. Underruns rose on - `step` (AAC 0 to 613.6 a minute, Opus 9.8 to 25) and AAC `wide` (0 to 17.3). - -The AAC `step` underruns start at 47 s, when the estimator relaxes its target -from 223 to 186 ms. `moq play` sets the sink's target per frame to -`due(end) - now` on the shared clock, whose reference is the earliest arrival -ever and never moves later, while the estimator measures against the fastest -frame of the last 2 s. After the shaper's delay steps from 5 to 60 ms, the -clock-aligned depth stays 55 ms short of the estimator's target, below AAC's -~160 ms PES burst spacing. `main`'s decision hid that by settling a burst -behind the clock. - -The remaining positive p50 is the accelerate's resting band: an ordinary -accelerate removes one pitch period only when it fits in the excess, so the -ring rests up to one period above its target, as the browser's does. - -Once the decisions below settle, the native `av_lag_ms` ceilings come from -[Native stretch](/quest/m1/audio-jitter-target/native-stretch.md)'s -seven-Nightly calibration of this decision. - -## Waiting - -- Maintainer: AAC `step` underruns after the shared clock stops matching a - persistent delay increase. Options: (1, recommended) accept them here and - plan a quest so the shared clock's reference (`moq_mux::sync` and - `js/watch` `Sync`) follows a delay increase every arrival in the - estimator's window shows, which also re-aligns video; (2) floor the native - sink target at the estimator's target, settling audio behind the clock by - the step; (3) keep `main`'s decision. Answer on - [moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361). -- Maintainer: the adaptive rows' p50 of 1.3 to 6.6 ms, the accelerate's - one-pitch-period resting band, against the Goal's "at or below zero". - Options: (1, recommended) accept a p50 within one pitch period as aligned - and budget it as measured; (2) let an ordinary accelerate overshoot the - target by up to one period, trading underruns for it in both players. - Answer on [moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361). - -## Related - -- [Native stretch](/quest/m1/audio-jitter-target/native-stretch.md) - the native ceilings this lag joins, calibrated from this decision -- [Browser ceilings](/quest/m1/audio-jitter-target/browser-ceilings.md) - budgets `av_lag_ms` on the chromium rows the same way diff --git a/quest/m1/audio-jitter-target/native-stretch.md b/quest/m1/audio-jitter-target/native-stretch.md index 13d628bda61..dcff147342a 100644 --- a/quest/m1/audio-jitter-target/native-stretch.md +++ b/quest/m1/audio-jitter-target/native-stretch.md @@ -156,7 +156,11 @@ bench waits below remain. [moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361), whose native rows decide where audio arrives and report `av_lag_ms` beside [moq-dev/moq#8308](https://github.com/moq-dev/moq/pull/8308)'s - `player_skips` and `trimmed_ms`; the calibration budgets `av_lag_ms` too. Find them with `gh run list --workflow + `player_skips` and `trimmed_ms`; the calibration budgets `av_lag_ms` too, + as measured: a p50 within one pitch period counts as aligned, and the AAC + `step` underruns that follow a persistent delay increase are accepted + (decided 2026-11-09 on + [moq-dev/moq#8361](https://github.com/moq-dev/moq/pull/8361)). Find them with `gh run list --workflow nightly.yml --event schedule`, download each with `gh run download -n audio-quality-run` within its 7 days, keep its native rows in the calibration PR's body, and apply the calibration rule once seven exist. From b93e1d12e8516e2c3f740d90d0f0f4e4e70cc851 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:51:06 -0800 Subject: [PATCH 7/7] test(audio): the arrival test sets main's new comfort field Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/ring.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rs/moq-audio/src/ring.rs b/rs/moq-audio/src/ring.rs index 05289723fb2..673f8007c55 100644 --- a/rs/moq-audio/src/ring.rs +++ b/rs/moq-audio/src/ring.rs @@ -1145,6 +1145,7 @@ mod tests { capacity: Duration::from_secs(3), pad: false, trim: false, + comfort: true, }, ); cons.stretch(rate);