From 3d72ddd81a2982906df00b7c76652d7015ea3d04 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 02:34:34 -0800 Subject: [PATCH 01/11] chore(quest): claim quest/m1/audio-jitter-target/native-rendition-delay.md Quest-Claim: 53f1c613bd5b99dc From b8582ad34c6be62efcd2471746222841e518d4cf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 02:41:24 -0800 Subject: [PATCH 02/11] fix(cli): moq play holds each rendition's catalog delay Under --delay auto, video holds its rendition's catalog delay as its sync target and audio adds its own to the jitter target, so a slow video encoder delays audio to match, as the browser does. The delay follows catalog updates and only rises; a fixed --delay stays fixed. Co-Authored-By: Claude Opus 5.5 --- quest/m1/audio-jitter-target/README.md | 1 - .../native-rendition-delay.md | 26 ------- rs/moq-cli/src/play/media.rs | 58 +++++++++++++-- rs/moq-cli/src/play/video.rs | 74 ++++++++++++++++++- 4 files changed, 126 insertions(+), 33 deletions(-) delete mode 100644 quest/m1/audio-jitter-target/native-rendition-delay.md diff --git a/quest/m1/audio-jitter-target/README.md b/quest/m1/audio-jitter-target/README.md index b025813de04..1799536cd95 100644 --- a/quest/m1/audio-jitter-target/README.md +++ b/quest/m1/audio-jitter-target/README.md @@ -62,7 +62,6 @@ native playback against its budgets. - [Buffered converge](/quest/m1/audio-jitter-target/buffered-converge.md) - a buffered `` time-stretches its audio back onto the shared clock after a stall or drift, so lip sync returns and no audio group is dropped - [Capped budgets](/quest/m1/audio-jitter-target/capped-budgets.md) - the AAC browser budgets are re-measured from nightlies after the TS importer groups each PES - [Native A/V lag](/quest/m1/audio-jitter-target/native-av-lag.md) - `moq play` reports how far audio trails its presentation clock, and native stops expanding into the gap between arrivals if it settles audio late -- [Native rendition delay](/quest/m1/audio-jitter-target/native-rendition-delay.md) - `moq play` holds each subscribed rendition's catalog `delay` plus its target, as the browser does - [Chromium Opus stalled share](/quest/m1/audio-jitter-target/opus-stalled-share.md) - the Chromium Opus browser rows stay under the 0.01 stalled-share budget every night ## Closes diff --git a/quest/m1/audio-jitter-target/native-rendition-delay.md b/quest/m1/audio-jitter-target/native-rendition-delay.md deleted file mode 100644 index da8e90055c5..00000000000 --- a/quest/m1/audio-jitter-target/native-rendition-delay.md +++ /dev/null @@ -1,26 +0,0 @@ -# [S] moq play holds each rendition's catalog delay - -## Goal - -`moq play` sets its playout from the catalog as the browser does: -`max over subscribed renditions of (delay + target)` -(`doc/concept/audio-jitter.md`), so a rendition whose publisher states a -`delay`, such as a slow video encoder, plays in sync with audio instead of -behind it. - -## Plan - -Planned 2026-11-08 (maintainer, approved follow-up of -[moq-dev/moq#8272](https://github.com/moq-dev/moq/pull/8272), which gives -moq-mux's playout sync per-track targets and the largest-target delay). Milestone: -m1, an additive fix to shipped playback. #8272 has merged. - -On `main`, nothing under `rs/moq-cli/src/play/` reads a rendition's -`delay`; the browser does (`js/watch/src/video/playhead.ts`, -`js/watch/src/audio/decoder.ts`). With #8272, each track's target is what it -holds through `Track::set_target`: add the rendition's catalog `delay` to it -(video holds `delay` alone, since it has no estimator), recompute when a -rendition is subscribed, dropped, or its catalog entry rises, and test that a -200 ms video `delay` delays audio to match. A fixed `--delay` stays fixed. - -Public API: none. Wire: none. diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 59235ddcb5a..76095492142 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -205,6 +205,12 @@ impl Media { { playback.retire(Kind::Audio); } + if let Some(demand) = &video_demand { + demand.hold(&self.args, snapshot.video.renditions.get(&demand.name).and_then(|config| config.delay)); + } + if let Some(demand) = &audio_demand { + demand.hold(&self.args, snapshot.audio.renditions.get(&demand.name).and_then(|config| config.delay)); + } playback.received(snapshot); }, None => { @@ -342,21 +348,26 @@ impl Media { refresh = refresh.is_some(), "playing video rendition" ); + let (delay, held) = tokio::sync::watch::channel(None); let video = Video { presentation: self.presentation.clone(), frames: self.video.clone(), changed: self.drained.clone(), output: self.output.clone(), max_delay, + delay: held, }; let (stop, stopped) = tokio::sync::oneshot::channel(); let (retire, retired) = tokio::sync::oneshot::channel(); - video_demand = Some(Demand { + let demand = Demand { name: name.clone(), track: reference.clone(), stop, retire: Some(retire), - }); + delay, + }; + demand.hold(&self.args, config.delay); + video_demand = Some(demand); tasks.spawn(async move { // Video holds no tail beyond the frames already queued for the window, // so a retirement stops it too. @@ -419,7 +430,9 @@ impl Media { if speaker.is_none() { speaker = Some(device.get_or_try_init(|| self.output.speaker()).await?.clone()); } + let (delay, held) = tokio::sync::watch::channel(None); let audio = AudioPlayback { + delay: held, speaker: speaker.clone().expect("opened above"), presentation: self.presentation.clone(), playhead: self.playhead.clone(), @@ -429,12 +442,15 @@ impl Media { }; let (stop, stopped) = tokio::sync::oneshot::channel(); let (retire, retired) = tokio::sync::oneshot::channel(); - audio_demand = Some(Demand { + let demand = Demand { name: name.clone(), track: reference.clone(), stop, retire: Some(retire), - }); + delay, + }; + demand.hold(&self.args, config.delay); + audio_demand = Some(demand); tasks.spawn(async move { // A dropped sender retires nothing. let retired = async { @@ -489,6 +505,8 @@ struct Demand { stop: tokio::sync::oneshot::Sender<()>, /// Ends the rendition as though its track finished, taken once sent. retire: Option>, + /// The catalog `delay` the rendition holds, `None` under a fixed `--delay`. + delay: tokio::sync::watch::Sender>, } impl Demand { @@ -502,6 +520,19 @@ impl Demand { let _ = retire.send(()); true } + + /// Hold the rendition's newest catalog `delay`, unless the delay is fixed. + /// It only grows, so a stale snapshot never lowers it. + fn hold(&self, args: &Args, delay: Option) { + let delay = delay.filter(|_| args.fixed_delay().is_none()); + self.delay.send_if_modified(|held| { + let raised = delay > *held; + if raised { + *held = delay; + } + raised + }); + } } // Cancel through the task's own result so it still reports which half ended. @@ -535,6 +566,8 @@ struct AudioPlayback { /// The depth the sink opens on, which also sizes its ring. latency: Duration, output: O, + /// The rendition's catalog `delay`, added to the jitter target this track holds. + delay: tokio::sync::watch::Receiver>, } /// Play a track until it ends, or until `retired` resolves, which ends it where @@ -551,6 +584,7 @@ async fn play_audio( playhead, latency, output, + delay: encoder, } = playback; // The playout delay is the consumer's to size, from how unevenly packets arrive, @@ -827,7 +861,9 @@ async fn play_audio( // steers, never what moves it. The target is this track's own jitter // target, which playback holds unless another track asks for more. let mut moved = role.received(mapped, arrived_at); - moved |= role.set_target(input.latency); + // The publisher's encoder delay is on top of the arrival jitter, so the + // clock holds both while the sink keeps sizing for jitter alone. + moved |= role.set_target(input.latency + encoder.borrow().unwrap_or_default()); // The depth that makes this frame's end sound when the shared clock says // it is due, so the callback stretches the speaker onto the clock: behind // it after a stall, it plays faster until it catches up. Floored at what @@ -938,6 +974,7 @@ mod tests { .unwrap(); let recorder = Recorder::default(); let playback = AudioPlayback { + delay: tokio::sync::watch::channel(None).1, changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(AUDIO_BUFFER_MIN), @@ -974,6 +1011,7 @@ mod tests { .unwrap(); let recorder = Recorder::default(); let playback = AudioPlayback { + delay: tokio::sync::watch::channel(None).1, changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(delay), @@ -1050,6 +1088,7 @@ mod tests { .unwrap(); let recorder = Recorder::default(); let playback = AudioPlayback { + delay: tokio::sync::watch::channel(None).1, changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(delay), @@ -1189,6 +1228,7 @@ mod tests { .await .unwrap(); let audio_play = AudioPlayback { + delay: tokio::sync::watch::channel(None).1, changed: media.drained.clone(), speaker: recorder.clone(), presentation: media.presentation.clone(), @@ -1202,6 +1242,7 @@ mod tests { changed: media.drained.clone(), output: recorder.clone(), max_delay: delay, + delay: tokio::sync::watch::channel(None).1, }; let audio_task = tokio::spawn(async move { play_audio(consumer, audio_play, std::future::pending()).await.unwrap(); @@ -1315,6 +1356,7 @@ mod tests { .await .unwrap(); let audio_play = AudioPlayback { + delay: tokio::sync::watch::channel(None).1, changed: media.drained.clone(), speaker: recorder.clone(), presentation: media.presentation.clone(), @@ -1328,6 +1370,7 @@ mod tests { changed: media.drained.clone(), output: recorder.clone(), max_delay: delay, + delay: tokio::sync::watch::channel(None).1, }; let audio_task = tokio::spawn(async move { play_audio(consumer, audio_play, std::future::pending()) @@ -1477,6 +1520,7 @@ mod tests { .unwrap(); let recorder = Recorder::default(); let playback = AudioPlayback { + delay: tokio::sync::watch::channel(None).1, changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(delay), @@ -1558,6 +1602,7 @@ mod tests { .unwrap(); let recorder = Recorder::default(); let playback = AudioPlayback { + delay: tokio::sync::watch::channel(None).1, changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(delay), @@ -1615,6 +1660,7 @@ mod tests { let presentation = Presentation::new(delay); let changed = Arc::new(tokio::sync::Notify::new()); let playback = AudioPlayback { + delay: tokio::sync::watch::channel(None).1, changed: changed.clone(), speaker: recorder.clone(), presentation: presentation.clone(), @@ -1693,6 +1739,7 @@ mod tests { changed: media.drained.clone(), output: recorder.clone(), max_delay: delay, + delay: tokio::sync::watch::channel(None).1, }; let video_task = tokio::spawn(video_play.run( moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), @@ -2432,6 +2479,7 @@ mod tests { changed: media.drained.clone(), output: recorder.clone(), max_delay: delay, + delay: tokio::sync::watch::channel(None).1, } .run(track, decoder, None), ); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 4d990ea3c40..859f486d556 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -54,6 +54,8 @@ pub(super) struct Video { pub changed: Arc, pub output: O, pub max_delay: Duration, + /// The rendition's catalog `delay`, held as this track's playout target. + pub delay: tokio::sync::watch::Receiver>, } /// The max-delay decoder of a rendition with a catalog `warmup`, and the track it @@ -193,7 +195,15 @@ impl Video { ..Default::default() }); } - if active && buffers.len() == 1 && role.received(role.map(frame.timestamp)?, Instant::now()) { + // Video has no jitter estimate, so its target is the encoder delay alone, + // which holds audio back with it. + let held = self.delay.borrow().is_some_and(|delay| role.set_target(delay)); + let anchored = + active && buffers.len() == 1 && role.received(role.map(frame.timestamp)?, Instant::now()); + if held { + self.changed.notify_one(); + } + if held || anchored { self.output.send(Event::Wake); } let back = buffers.back_mut().unwrap(); @@ -739,6 +749,7 @@ mod tests { changed: media.drained.clone(), output: media.output.clone(), max_delay, + delay: tokio::sync::watch::channel(None).1, } } @@ -808,6 +819,67 @@ mod tests { (min, max) } + /// A video rendition's catalog `delay` holds audio back with it, and a rise moves both. + #[tokio::test] + async fn a_video_encoder_delay_delays_audio_to_match() { + tokio::time::pause(); + let jitter = Duration::from_millis(50); + let media = media(jitter); + let start = Instant::now(); + let audio = media.presentation.track(moq_mux::container::Kind::Audio); + audio.received(moq_net::Timestamp::ZERO, start); + audio.set_target(jitter); + assert_eq!(media.presentation.due(moq_net::Timestamp::ZERO), Some(start + jitter)); + + let (encoder, delay) = tokio::sync::watch::channel(Some(Duration::from_millis(200))); + let budget = Duration::from_secs(1); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = track + .consume() + .subscribe( + moq_net::track::Subscription::default().with_delay(moq_net::track::Delay::default().with_max(budget)), + ) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Data); + let track_consumer = moq_mux::container::Consumer::new(subscriber, format()); + let mut producer = moq_mux::container::Producer::new(track, format()); + let settle = async || { + for _ in 0..10 { + tokio::task::yield_now().await; + } + }; + let video = Video { + delay, + ..playback(&media, budget) + }; + let task = tokio::spawn(video.run(track_consumer, Buffered::default(), None)); + producer.write(frame(0, true)).unwrap(); + settle().await; + assert_eq!( + media.presentation.due(moq_net::Timestamp::ZERO), + Some(start + Duration::from_millis(200)), + "audio plays in sync with the delayed video" + ); + + encoder.send_replace(Some(Duration::from_millis(300))); + producer.write(frame(33, false)).unwrap(); + settle().await; + let clock = media.presentation.consume(); + assert_eq!( + clock.delay(), + Duration::from_millis(300), + "a rising catalog delay moves the schedule" + ); + + producer.finish().unwrap(); + task.await.unwrap().unwrap(); + assert_eq!(clock.delay(), jitter, "released with its track"); + } + async fn burst(speaker: bool, paused_window: bool) { burst_with(speaker, paused_window, false).await; } From 745d24b4e34a15fa6ab1c873f7a6df45c9c4b5ff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 02:46:25 -0800 Subject: [PATCH 03/11] fix(cli): grow video budgets with its catalog delay, applied on change Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 6 +++ rs/moq-cli/src/play/video.rs | 73 +++++++++++++++++++++++++++++------- 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 76095492142..ef533c4a2d1 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -335,6 +335,7 @@ impl Media { track, decoder, interval, + warmup, }) }; Ok::<_, anyhow::Error>((track, decoder, refresh)) @@ -355,6 +356,7 @@ impl Media { changed: self.drained.clone(), output: self.output.clone(), max_delay, + min_delay: self.args.video_delay(), delay: held, }; let (stop, stopped) = tokio::sync::oneshot::channel(); @@ -1242,6 +1244,7 @@ mod tests { changed: media.drained.clone(), output: recorder.clone(), max_delay: delay, + min_delay: delay, delay: tokio::sync::watch::channel(None).1, }; let audio_task = tokio::spawn(async move { @@ -1370,6 +1373,7 @@ mod tests { changed: media.drained.clone(), output: recorder.clone(), max_delay: delay, + min_delay: delay, delay: tokio::sync::watch::channel(None).1, }; let audio_task = tokio::spawn(async move { @@ -1739,6 +1743,7 @@ mod tests { changed: media.drained.clone(), output: recorder.clone(), max_delay: delay, + min_delay: delay, delay: tokio::sync::watch::channel(None).1, }; let video_task = tokio::spawn(video_play.run( @@ -2479,6 +2484,7 @@ mod tests { changed: media.drained.clone(), output: recorder.clone(), max_delay: delay, + min_delay: delay, delay: tokio::sync::watch::channel(None).1, } .run(track, decoder, None), diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 859f486d556..7ee8f28d40b 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -54,7 +54,10 @@ pub(super) struct Video { pub changed: Arc, pub output: O, pub max_delay: Duration, - /// The rendition's catalog `delay`, held as this track's playout target. + /// The subscription's playout delay before any catalog `delay`. + pub min_delay: Duration, + /// The rendition's catalog `delay`, held as this track's playout target and + /// added to every budget above. pub delay: tokio::sync::watch::Receiver>, } @@ -68,6 +71,8 @@ pub(super) struct Refresh { pub decoder: D, /// The catalog's frame interval, when it declares a rate. pub interval: Option, + /// The extra budget its subscription holds over the min-delay decoder's. + pub warmup: Duration, } /// The production sink and the deterministic buffered decoder used by tests. @@ -123,6 +128,14 @@ fn soonest(a: Option, b: Option) -> Option { } impl Video { + /// A subscription's budget once it holds the catalog `delay`, plus `warmup`. + fn budget(&self, delay: Duration, warmup: Duration) -> moq_net::track::Delay { + moq_net::track::Delay { + min: Some(self.min_delay + delay + warmup), + max: Some(self.max_delay + delay + warmup), + } + } + /// Play `track`, and with `refresh` also run its max-delay decoder. pub async fn run( self, @@ -149,6 +162,20 @@ impl Video { let advanced = tokio::sync::Notify::new(); let role = self.presentation.track(moq_mux::container::Kind::Video); let receive = async { + // Video has no jitter estimate, so its target is the encoder delay alone, + // which holds audio back with it. Every budget grows with it, or media the + // later schedule still waits for would be evicted first. + let mut encoder = self.delay.clone(); + let hold = |track: &mut Track, delay: Option| { + let Some(delay) = delay else { return Duration::ZERO }; + track.set_delay(self.budget(delay, Duration::ZERO)); + if role.set_target(delay) { + self.changed.notify_one(); + self.output.send(Event::Wake); + } + delay + }; + let mut held = hold(&mut track, *encoder.borrow_and_update()); // The group of a declared end whose restart waits for a later group, so the // media buffered before the end still plays out, and the transition that // later group resumes. @@ -166,11 +193,17 @@ impl Video { .filter(|segment| segment.oldest().is_some()) .filter_map(|segment| segment.arrived) .min() - .is_some_and(|arrived| arrived.elapsed() > self.max_delay) + .is_some_and(|arrived| arrived.elapsed() > self.max_delay + held) { advanced.notified().await; } - let read = track.read().await?; + let read = tokio::select! { + read = track.read() => read?, + Ok(()) = encoder.changed() => { + held = hold(&mut track, *encoder.borrow_and_update()); + continue; + } + }; let mut buffers = buffers.lock().unwrap(); let active = buffers.len() == 1 && !advancing.load(std::sync::atomic::Ordering::Relaxed); let before = buffers.front().unwrap().generation; @@ -195,15 +228,7 @@ impl Video { ..Default::default() }); } - // Video has no jitter estimate, so its target is the encoder delay alone, - // which holds audio back with it. - let held = self.delay.borrow().is_some_and(|delay| role.set_target(delay)); - let anchored = - active && buffers.len() == 1 && role.received(role.map(frame.timestamp)?, Instant::now()); - if held { - self.changed.notify_one(); - } - if held || anchored { + if active && buffers.len() == 1 && role.received(role.map(frame.timestamp)?, Instant::now()) { self.output.send(Event::Wake); } let back = buffers.back_mut().unwrap(); @@ -212,7 +237,7 @@ impl Video { if back.resumes.is_none() && back.after.is_none() { origin.lock().unwrap().get_or_insert(group); } - back.push(frame, self.max_delay); + back.push(frame, self.max_delay + held); back.arrived.get_or_insert_with(Instant::now); false } @@ -523,6 +548,7 @@ impl Video { let Some(Refresh { track: mut refresh, decoder: mut codec, + warmup, .. }) = refresh else { @@ -544,8 +570,21 @@ impl Video { // The group of the last end this decoder read. Its own subscription may // deliver the replacement before the min-delay decoder reads that end. let mut fence = None; + let mut encoder = self.delay.clone(); + if let Some(delay) = *encoder.borrow_and_update() { + refresh.set_delay(self.budget(delay, warmup)); + } loop { - let (group, mut frame) = match refresh.read().await? { + let read = tokio::select! { + read = refresh.read() => read?, + Ok(()) = encoder.changed() => { + if let Some(delay) = *encoder.borrow_and_update() { + refresh.set_delay(self.budget(delay, warmup)); + } + continue; + } + }; + let (group, mut frame) = match read { Some(Read::Frame { group, frame }) => (group, frame), Some(Read::Gap { .. }) => { push(codec.flush().await?); @@ -749,6 +788,7 @@ mod tests { changed: media.drained.clone(), output: media.output.clone(), max_delay, + min_delay: max_delay, delay: tokio::sync::watch::channel(None).1, } } @@ -906,6 +946,7 @@ mod tests { track: max, decoder: refresh_decoder(Duration::ZERO), interval: Some(Duration::from_millis(33)), + warmup: Duration::ZERO, }; tokio::spawn(playback(&media, delay).run(track, decoder, Some(refresh))) } else { @@ -1617,6 +1658,7 @@ mod tests { // Slower than real time to start, so the earlier sweep takes a while. decoder: refresh_decoder(Duration::from_millis(10)), interval: Some(Duration::from_millis(33)), + warmup: Duration::ZERO, }; let task = tokio::spawn(playback(&media, delay).run(min, Buffered::new(0), Some(refresh))); let shown = live(&media, publish, task, 31..90, 0..0, Duration::ZERO).await; @@ -1649,6 +1691,7 @@ mod tests { track: max, decoder: refresh_decoder(Duration::from_millis(10)), interval: Some(Duration::from_millis(33)), + warmup: Duration::ZERO, }; let task = tokio::spawn(playback(&media, delay).run(min, Buffered::new(0), Some(refresh))); // The fourth sweep is held up past the min-delay budget, not the max-delay one. @@ -1694,6 +1737,7 @@ mod tests { track: max, decoder: refresh_decoder(Duration::from_millis(10)), interval: Some(Duration::from_millis(33)), + warmup: Duration::ZERO, }; let task = tokio::spawn(playback(&media, delay).run(min, Buffered::new(0), Some(refresh))); let start = Instant::now(); @@ -1774,6 +1818,7 @@ mod tests { track: max_track, decoder: refresh_decoder(Duration::ZERO), interval: Some(Duration::from_millis(33)), + warmup: Duration::ZERO, }; let task = tokio::spawn(playback(&media, delay).run(min_track, Buffered::new(0), Some(refresh))); let start = Instant::now(); From 9c0118edbd90d823f9bc772e329d07d429e0fa96 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 02:49:10 -0800 Subject: [PATCH 04/11] fix(cli): apply a rising catalog delay while playback waits Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 25 ++++++++++++++++++++++--- rs/moq-cli/src/play/video.rs | 5 ++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index ef533c4a2d1..f3dc48a4af3 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -586,7 +586,7 @@ async fn play_audio( playhead, latency, output, - delay: encoder, + delay: stated, } = playback; // The playout delay is the consumer's to size, from how unevenly packets arrive, @@ -643,7 +643,18 @@ async fn play_audio( // declared before the replacement media that resumes it is planned. A short // timeline can declare its end before its first PCM, so ends queue until a // frame crosses them. + // The jitter target the last write held, so a rise in the stated delay applies + // at once rather than waiting on the next write. + let jitter = Mutex::new(None); + let retarget = |delay: Option| { + let Some(jitter) = *jitter.lock().unwrap() else { return }; + if role.set_target(jitter + delay.unwrap_or_default()) { + changed.notify_one(); + output.send(Event::Wake); + } + }; let receive = async { + let mut stated = stated.clone(); let mut retired = std::pin::pin!(retired); // Media went missing since the last frame, so the next one does not continue it. let mut lost = false; @@ -658,11 +669,18 @@ async fn play_audio( .front() .is_some_and(|(.., queued): &(_, _, _, Instant)| queued.elapsed() > max_delay) { - popped.notified().await; + tokio::select! { + () = popped.notified() => {} + Ok(()) = stated.changed() => retarget(*stated.borrow_and_update()), + } } let event = tokio::select! { event = consumer.read() => event, () = &mut retired => Ok(None), + Ok(()) = stated.changed() => { + retarget(*stated.borrow_and_update()); + continue; + } }; let event = match event { Err(err) if !matches!(err, moq_audio::Error::Decode(_)) => return Err(err.into()), @@ -865,7 +883,8 @@ async fn play_audio( let mut moved = role.received(mapped, arrived_at); // The publisher's encoder delay is on top of the arrival jitter, so the // clock holds both while the sink keeps sizing for jitter alone. - moved |= role.set_target(input.latency + encoder.borrow().unwrap_or_default()); + *jitter.lock().unwrap() = Some(input.latency); + moved |= role.set_target(input.latency + stated.borrow().unwrap_or_default()); // The depth that makes this frame's end sound when the shared clock says // it is due, so the callback stretches the speaker onto the clock: behind // it after a stall, it plays faster until it catches up. Floored at what diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 7ee8f28d40b..e690231e2dc 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -195,7 +195,10 @@ impl Video { .min() .is_some_and(|arrived| arrived.elapsed() > self.max_delay + held) { - advanced.notified().await; + tokio::select! { + () = advanced.notified() => {} + Ok(()) = encoder.changed() => held = hold(&mut track, *encoder.borrow_and_update()), + } } let read = tokio::select! { read = track.read() => read?, From 9d7ba6bf6f0b7f7d158c71464ed932038aae986c Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 09:07:43 -0800 Subject: [PATCH 05/11] fix(cli): refresh decoder applies delay rises while parked Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/video.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index e690231e2dc..9194b6d2fa7 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -658,12 +658,22 @@ impl Video { } } }; - match at { - Some(at) => tokio::select! { - _ = woken => {}, - _ = kio::clock::sleep_until(at) => {}, - }, - None => woken.await, + // A delay rise while parked must grow this subscription's budget now, + // or it evicts frames the later schedule still waits for. + let sleep = async { + match at { + Some(at) => kio::clock::sleep_until(at).await, + None => std::future::pending().await, + } + }; + tokio::select! { + _ = woken => {}, + _ = sleep => {}, + Ok(()) = encoder.changed() => { + if let Some(delay) = *encoder.borrow_and_update() { + refresh.set_delay(self.budget(delay, warmup)); + } + } } }; let Some(mapped) = mapped else { From 97da08f618d4faee3fab99d9c12257642033ebc5 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:25:06 -0800 Subject: [PATCH 06/11] fix(cli): audio holds a shared delay past its budget until due Audio waits on decoded media due further out than its sink holds, and retargets the sink when the shared delay changes during a delivery gap. Catalog delays cap at the sink's LATENCY_MAX. Adds moq_mux::sync::Track::delay_changed. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 133 +++++++++++++++++++++++++++++++---- rs/moq-mux/src/sync.rs | 11 +++ 2 files changed, 131 insertions(+), 13 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index f3dc48a4af3..f7b68a4bfeb 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -526,7 +526,11 @@ impl Demand { /// Hold the rendition's newest catalog `delay`, unless the delay is fixed. /// It only grows, so a stale snapshot never lowers it. fn hold(&self, args: &Args, delay: Option) { - let delay = delay.filter(|_| args.fixed_delay().is_none()); + // Capped at the deepest delay a sink holds, so a hostile catalog cannot make + // playback retain media without bound. + let delay = delay + .filter(|_| args.fixed_delay().is_none()) + .map(|delay| delay.min(moq_audio::playback::Input::LATENCY_MAX)); self.delay.send_if_modified(|held| { let raised = delay > *held; if raised { @@ -635,6 +639,21 @@ async fn play_audio( // subtracting the incoming frame before it reaches the sink. let lead = |delay: Duration| latency.max(delay.min(max_delay).max(AUDIO_BUFFER_MIN)); + // The most the sink holds. A shared delay past it, such as a slow video + // encoder's, holds decoded media here until it is due instead. + let cap = max_delay.max(AUDIO_BUFFER_MIN); + let schedule = presentation.consume(); + // The depth that makes media ending at `end` sound when the shared clock says it is due. + let hold = |end: Duration| { + moq_net::Timestamp::try_from(end) + .ok() + .and_then(|end| presentation.due(end)) + .map(|due| { + due.saturating_duration_since(Instant::now()) + .clamp(AUDIO_BUFFER_MIN, cap) + }) + }; + // Tracks whether the last read failed, so a stream the decoder can't read at // all logs once rather than once per packet. let mut dropping = false; @@ -667,7 +686,7 @@ async fn play_audio( .lock() .unwrap() .front() - .is_some_and(|(.., queued): &(_, _, _, Instant)| queued.elapsed() > max_delay) + .is_some_and(|(.., queued): &(_, _, _, Instant)| queued.elapsed() > max_delay + schedule.delay()) { tokio::select! { () = popped.notified() => {} @@ -749,13 +768,24 @@ async fn play_audio( let mut utterance = false; // The last frame written was trimmed, so a trim of this one continues that skip. let mut trimming = false; + // Where the media written so far ends. + let mut wrote = None; loop { let (event, delay, crossing, arrived_at) = loop { if let Some(event) = received.lock().unwrap().pop_front() { popped.notify_one(); break event; } - arrived.notified().await; + let seen = schedule.delay(); + tokio::select! { + () = arrived.notified() => {} + // No write may come to carry a delay change, so the speaker follows it now. + _ = role.delay_changed(seen) => { + if let Some(held) = wrote.and_then(hold) { + sink.set_latency(held)?; + } + } + } }; // A replacement sink must open with the current cushion, not the // old floor and a target update after its initial silence was fixed. @@ -876,7 +906,6 @@ async fn play_audio( // which leaves it running ahead of media time until the clock below // re-anchors, taking the video with it. So the hole goes in as silence ahead // of the frame, and the pair is fitted to the target as one write. - let buffered = samples(sink.buffered()); // The arrival, not the write: how far the speaker trails is what the clock // steers, never what moves it. The target is this track's own jitter // target, which playback holds unless another track asks for more. @@ -885,18 +914,30 @@ async fn play_audio( // clock holds both while the sink keeps sizing for jitter alone. *jitter.lock().unwrap() = Some(input.latency); moved |= role.set_target(input.latency + stated.borrow().unwrap_or_default()); + // Media due further out than the sink holds waits here, following any + // delay change meanwhile. + loop { + let seen = schedule.delay(); + // The frame's start, so a delay the sink can hold never waits here. + let Some(at) = presentation + .due(mapped) + .and_then(|due| due.checked_sub(cap)) + .filter(|at| *at > Instant::now()) + else { + break; + }; + tokio::select! { + () = kio::clock::sleep_until(at) => {} + _ = role.delay_changed(seen) => {} + } + } // The depth that makes this frame's end sound when the shared clock says // it is due, so the callback stretches the speaker onto the clock: behind // it after a stall, it plays faster until it catches up. Floored at what - // the device needs, and capped at the age budget: audio older than it is - // skipped rather than held, so a deeper target could never fill. The - // budget is also what the sink's ring was sized to hold. - let now = Instant::now(); - let held = moq_net::Timestamp::try_from(timing.end) - .ok() - .and_then(|end| presentation.due(end)) - .map_or(input.latency, |due| due.saturating_duration_since(now)) - .clamp(AUDIO_BUFFER_MIN, max_delay.max(AUDIO_BUFFER_MIN)); + // the device needs, and capped at the age budget the sink was sized for; + // the wait above holds anything due later. + let buffered = samples(sink.buffered()); + let held = hold(timing.end).unwrap_or(input.latency.clamp(AUDIO_BUFFER_MIN, cap)); let target = samples(held); sink.set_latency(held)?; // Leave normal convergence to the callback. Only an excess at NetEq's @@ -940,6 +981,7 @@ async fn play_audio( silent = true; } + wrote = Some(timing.end); playhead.wrote(timing.end, sink.buffered(), Instant::now()); // A move has to wake the window: it is asleep on a deadline computed from // the old schedule, and every queued frame is due earlier now. @@ -1069,6 +1111,71 @@ mod tests { assert!(old.last().unwrap().to <= new.first().unwrap().from); } + /// A video encoder delay past audio's own budget still plays audio in sync: + /// the media waits here until it is due rather than sounding early. + #[tokio::test] + async fn audio_holds_a_shared_delay_past_its_budget() { + tokio::time::pause(); + let budget = Duration::from_millis(200); + let encoder = Duration::from_secs(1); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut options = moq_audio::decode::Options::new(); + options.delay = moq_audio::decode::Delay { + min: Some(budget), + max: Some(budget), + }; + options.estimate = false; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let presentation = Presentation::new(budget); + let video = presentation.track(moq_mux::container::Kind::Video); + video.set_target(encoder); + let recorder = Recorder::default(); + let playback = AudioPlayback { + delay: tokio::sync::watch::channel(None).1, + changed: Default::default(), + speaker: recorder.clone(), + presentation: presentation.clone(), + playhead: Default::default(), + // The fake's opening silence stands in for the padding a real ring adds + // once it runs dry, which tops it up to its target before the held write. + latency: budget, + output: recorder.clone(), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback, std::future::pending()) + .await + .unwrap() + .finish() + .await; + }); + let start = Instant::now(); + for index in 0..5 { + producer.write(packet(index, 0.25)).unwrap(); + } + producer.finish().unwrap(); + task.await.unwrap(); + let first = recorder.played().first().copied().expect("played audio"); + let due = start + encoder; + assert!( + first.from.max(due) - first.from.min(due) <= 2 * PACKET_DURATION, + "audio sounded at {:?}, due {:?}", + first.from - start, + encoder + ); + } + /// A push-to-talk mute declares its silence and sends nothing until the next /// press. The sink carries on through it, so the native ring fills it with /// comfort noise, and the next press plays one delay after it arrives. diff --git a/rs/moq-mux/src/sync.rs b/rs/moq-mux/src/sync.rs index c56adbf5aba..caa87de943f 100644 --- a/rs/moq-mux/src/sync.rs +++ b/rs/moq-mux/src/sync.rs @@ -364,6 +364,17 @@ impl Track { moved } + /// Wait until playback trails the reference by something other than `seen`, returning the new delay. + #[cfg_attr(feature = "async", kio::wrapper)] + pub fn poll_delay_changed(&self, waiter: &kio::Waiter, seen: Duration) -> Poll { + let mut delay = seen; + let _ = std::task::ready!(self.clock.changed.poll(waiter, |_| { + delay = self.clock.state.lock().unwrap().delay(); + if delay == seen { Poll::Pending } else { Poll::Ready(()) } + })); + Poll::Ready(delay) + } + /// Stop while a tail still plays until `drained`, holding a declared end until /// then. Dropping instead discards the tail with the end. pub fn retire(self, drained: Option) { From 41e46634f4d29c136da365e3eba158d429e7efc9 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:23:33 -0800 Subject: [PATCH 07/11] fix(cli): held audio writes its missing lead as silence; waits follow delay changes Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 86 ++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 33 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index f7b68a4bfeb..4d10417259b 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -753,6 +753,26 @@ async fn play_audio( } }; let play = async { + let mut catalog = stated.clone(); + // Sleep until `deadline`, recomputing it whenever the shared or the catalog delay + // changes, so a change while asleep never leaves audio on a stale deadline. + // Reports whether it slept at all. + let wait = async |catalog: &mut tokio::sync::watch::Receiver>, + deadline: &(dyn Fn() -> Option + Sync)| { + let mut slept = false; + loop { + let seen = schedule.delay(); + let Some(at) = deadline().filter(|at| *at > Instant::now()) else { + return slept; + }; + slept = true; + tokio::select! { + () = kio::clock::sleep_until(at) => {} + _ = role.delay_changed(seen) => {} + Ok(()) = catalog.changed() => retarget(*catalog.borrow_and_update()), + } + } + }; // Whether a timeline is playing and its end is already declared, and an end // declared past that one, for a later timeline that can only be mapped once it // resumes. @@ -813,12 +833,11 @@ async fn play_audio( let first = lead(delay); let mapping = role.resume(transition, frame.timestamp, Some(first)).ready().await?; role.apply(transition, mapping); - let at = presentation - .due(role.map(frame.timestamp)?) - .and_then(|at| at.checked_sub(first)); - if let Some(at) = at { - kio::clock::sleep_until(at).await; - } + let start = role.map(frame.timestamp)?; + wait(&mut catalog, &|| { + presentation.due(start).and_then(|at| at.checked_sub(first)) + }) + .await; ended = false; // An end this frame did not cross ends the timeline it resumes. if let Some(end) = deferred.take() @@ -895,10 +914,10 @@ async fn play_audio( // spacing instead of fitting that queued burst as if it arrived live. // The lead follows the current target so jitter can still deepen the sink. if sequential { - let at = presentation.due(mapped).and_then(|at| at.checked_sub(lead(delay))); - if let Some(at) = at { - kio::clock::sleep_until(at).await; - } + wait(&mut catalog, &|| { + presentation.due(mapped).and_then(|at| at.checked_sub(lead(delay))) + }) + .await; } // A hole in the media is a hole in the audio, not a splice. Handing the next @@ -914,23 +933,21 @@ async fn play_audio( // clock holds both while the sink keeps sizing for jitter alone. *jitter.lock().unwrap() = Some(input.latency); moved |= role.set_target(input.latency + stated.borrow().unwrap_or_default()); - // Media due further out than the sink holds waits here, following any - // delay change meanwhile. - loop { - let seen = schedule.delay(); - // The frame's start, so a delay the sink can hold never waits here. - let Some(at) = presentation - .due(mapped) - .and_then(|due| due.checked_sub(cap)) - .filter(|at| *at > Instant::now()) - else { - break; - }; - tokio::select! { - () = kio::clock::sleep_until(at) => {} - _ = role.delay_changed(seen) => {} - } - } + // Media due further out than the sink holds waits here. The sink then + // holds only what was already written, its opening silence included, so + // the rest of the lead goes in as silence ahead of the frame. + let held_back = wait(&mut catalog, &|| { + presentation.due(mapped).and_then(|due| due.checked_sub(cap)) + }) + .await; + let short = presentation + .due(mapped) + .filter(|_| held_back) + .map(|due| { + due.saturating_duration_since(Instant::now()) + .saturating_sub(sink.buffered()) + }) + .map_or(0, samples); // The depth that makes this frame's end sound when the shared clock says // it is due, so the callback stretches the speaker onto the clock: behind // it after a stall, it plays faster until it catches up. Floored at what @@ -943,7 +960,7 @@ async fn play_audio( // Leave normal convergence to the callback. Only an excess at NetEq's // fast threshold (four times target + granularity) is an emergency // re-anchor; a 20 ms trim would compete with every pitch correction. - let fit = fit(buffered, target, slack, timing.silence, length as u64); + let fit = fit(buffered, target, slack, timing.silence + short, length as u64); // The packet pays for the channel count; the catalog cannot allocate // a duration's worth of silence before any samples arrive. @@ -1148,9 +1165,8 @@ mod tests { speaker: recorder.clone(), presentation: presentation.clone(), playhead: Default::default(), - // The fake's opening silence stands in for the padding a real ring adds - // once it runs dry, which tops it up to its target before the held write. - latency: budget, + // What `--delay auto` opens on, well short of the lead the held media needs. + latency: AUDIO_BUFFER_MIN, output: recorder.clone(), }; let task = tokio::spawn(async move { @@ -1166,10 +1182,14 @@ mod tests { } producer.finish().unwrap(); task.await.unwrap(); - let first = recorder.played().first().copied().expect("played audio"); + let first = recorder + .played() + .into_iter() + .find(|part| part.sample == 0.25) + .expect("played audio"); let due = start + encoder; assert!( - first.from.max(due) - first.from.min(due) <= 2 * PACKET_DURATION, + first.from.max(due) - first.from.min(due) <= Duration::from_millis(1), "audio sounded at {:?}, due {:?}", first.from - start, encoder From aebf851e7ae5c72e75c8ad07249defd3d1dd6a54 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:30:57 -0800 Subject: [PATCH 08/11] fix(cli): held audio's lead silence covers a hole before it Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 4d10417259b..7129b3d18ff 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -935,7 +935,8 @@ async fn play_audio( moved |= role.set_target(input.latency + stated.borrow().unwrap_or_default()); // Media due further out than the sink holds waits here. The sink then // holds only what was already written, its opening silence included, so - // the rest of the lead goes in as silence ahead of the frame. + // the rest of the lead goes in as silence ahead of the frame, which already + // covers any hole before it. let held_back = wait(&mut catalog, &|| { presentation.due(mapped).and_then(|due| due.checked_sub(cap)) }) @@ -960,7 +961,7 @@ async fn play_audio( // Leave normal convergence to the callback. Only an excess at NetEq's // fast threshold (four times target + granularity) is an emergency // re-anchor; a 20 ms trim would compete with every pitch correction. - let fit = fit(buffered, target, slack, timing.silence + short, length as u64); + let fit = fit(buffered, target, slack, timing.silence.max(short), length as u64); // The packet pays for the channel count; the catalog cannot allocate // a duration's worth of silence before any samples arrive. From adb99190b0d2ef94b4985f4860b75b120ca3fedc Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:41:44 -0800 Subject: [PATCH 09/11] fix(cli): held audio starting an utterance takes the sink's cushion, not extra lead silence Also drop stretch.md's link to this quest, which main added. Co-Authored-By: Claude Opus 5.5 --- quest/m1/audio-jitter-target/stretch.md | 1 - rs/moq-cli/src/play/media.rs | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/quest/m1/audio-jitter-target/stretch.md b/quest/m1/audio-jitter-target/stretch.md index c8395c11e11..9e73984f2b1 100644 --- a/quest/m1/audio-jitter-target/stretch.md +++ b/quest/m1/audio-jitter-target/stretch.md @@ -135,5 +135,4 @@ Public API: none expected. Wire: none. ## Related - [The `stretch` setting](/quest/m1/audio-jitter-target/stretch-setting.md) - makes this quest's skip bound configurable, and zero turns the stretch off -- [Native rendition delay](/quest/m1/audio-jitter-target/native-rendition-delay.md) - the PCM queue native backpressure replaces - [Audio outlives video under bufferbloat](/quest/m4/audio-bufferbloat-av.md) - the audio and video congestion rows this behavior is asserted on diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 54b840a36d9..6cef7959a80 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -941,9 +941,11 @@ async fn play_audio( presentation.due(mapped).and_then(|due| due.checked_sub(cap)) }) .await; + // A frame starting an utterance is instead placed one target out by the + // sink itself, and `hold` below sizes that target to the due time. let short = presentation .due(mapped) - .filter(|_| held_back) + .filter(|_| held_back && !utterance) .map(|due| { due.saturating_duration_since(Instant::now()) .saturating_sub(sink.buffered()) From b46a8eb5857cb4bbc03fe2ad6565d7753ea42b72 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 20:42:37 -0800 Subject: [PATCH 10/11] fix(cli): port play tests to Instant::checked_add after merging main --- rs/moq-cli/src/play/media.rs | 2 +- rs/moq-cli/src/play/video.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 9c7b11c326a..c18b80f353a 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -1190,7 +1190,7 @@ mod tests { .into_iter() .find(|part| part.sample == 0.25) .expect("played audio"); - let due = start + encoder; + let due = start.checked_add(encoder).expect("test deadline overflows"); assert!( first.from.max(due) - first.from.min(due) <= Duration::from_millis(1), "audio sounded at {:?}, due {:?}", diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 72cfcbbd983..f646b9cf65f 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -901,7 +901,10 @@ mod tests { let audio = media.presentation.track(moq_mux::container::Kind::Audio); audio.received(moq_net::Timestamp::ZERO, start); audio.set_target(jitter); - assert_eq!(media.presentation.due(moq_net::Timestamp::ZERO), Some(start + jitter)); + assert_eq!( + media.presentation.due(moq_net::Timestamp::ZERO), + start.checked_add(jitter) + ); let (encoder, delay) = tokio::sync::watch::channel(Some(Duration::from_millis(200))); let budget = Duration::from_secs(1); @@ -933,7 +936,7 @@ mod tests { settle().await; assert_eq!( media.presentation.due(moq_net::Timestamp::ZERO), - Some(start + Duration::from_millis(200)), + start.checked_add(Duration::from_millis(200)), "audio plays in sync with the delayed video" ); From 734665485e6e82a8722afebef893a8a565bb4087 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:32:54 -0800 Subject: [PATCH 11/11] fix(cli): video budgets follow the shared playout delay Video's encoded retention and both subscriptions' budgets now grow by however far the shared playout delay trails past the subscription's own, whichever track set it. Audio declaring a 5 s catalog delay no longer leaves video at its 2 s budget, evicting pictures before they are due. Records the maintainer's 2026-11-09 decisions in native-backpressure (no early audio start under the ring's cap) and stretch-setting (delete AUTO_MAX_DELAY). Co-Authored-By: Claude Opus 5.5 --- .../native-backpressure.md | 10 ++ .../m1/audio-jitter-target/stretch-setting.md | 8 + rs/moq-cli/src/play/video.rs | 152 ++++++++++++++---- 3 files changed, 141 insertions(+), 29 deletions(-) diff --git a/quest/m1/audio-jitter-target/native-backpressure.md b/quest/m1/audio-jitter-target/native-backpressure.md index 8856b759e61..5554e3c36c1 100644 --- a/quest/m1/audio-jitter-target/native-backpressure.md +++ b/quest/m1/audio-jitter-target/native-backpressure.md @@ -52,6 +52,16 @@ them passed in: a gate on it would decode at a depth the sink has already outrun and could not replace #8362's hold. +Decided 2026-11-09 (maintainer, /quest-unblock): the gate must not start +audio early when a shared catalog delay is under the ring's cap, with a test. +✅ Fix it here (rejected: pad the missing lead in +[moq-dev/moq#8362](https://github.com/moq-dev/moq/pull/8362)). With a shared delay +that fits the sink's cap, that PR writes the frame without its hold-path lead +padding, so audio can start up to ~150 ms early +(`rs/moq-cli/src/play/media.rs`). The browser-mirroring refill starts with the +whole target ahead of the playhead and replaces that padding path, so the fix +lands with the gate. + Keep the gate inside moq-audio's consumer if that stays additive; if it needs a public API change, it moves to m2. diff --git a/quest/m1/audio-jitter-target/stretch-setting.md b/quest/m1/audio-jitter-target/stretch-setting.md index 9610c8eafb5..3fac8727b62 100644 --- a/quest/m1/audio-jitter-target/stretch-setting.md +++ b/quest/m1/audio-jitter-target/stretch-setting.md @@ -10,6 +10,9 @@ silence, and drift between the publisher's clock and the device's is corrected by a small skip or pad. The default is 2 s whether `delay` is `auto` or fixed, so a fixed-delay live player also keeps late audio and catches up. +`AUTO_MAX_DELAY` is deleted in both `js/watch/src/audio/latency.ts` and +`rs/moq-cli/src/play/args.rs`: every budget it fed (the subscription max +delay, the sink cap, and video retention) is derived from `delay + stretch`. ## Plan @@ -28,6 +31,11 @@ want stretching. - Default: ✅ 2 s for `auto` and fixed delays alike (rejected: a fixed delay keeping max delay equal to the delay). - Milestone: ✅ m1 (rejected: m2). The attribute and flag are additive. +- Decided 2026-11-09 (maintainer, /quest-unblock): ✅ delete + `AUTO_MAX_DELAY` in both players and derive every budget it fed + (subscription max delay, sink cap, video retention) from `delay + stretch` + (rejected: keep the 2 s constant beside the setting). The 2 s is an + arbitrary constant nobody can trace. The audio subscription's max delay is what makes late audio arrive at all: the relay and the container consumer drop media older than it. Today it is 2 s diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index f646b9cf65f..522c70d059f 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -54,10 +54,9 @@ pub(super) struct Video { pub changed: Arc, pub output: O, pub max_delay: Duration, - /// The subscription's playout delay before any catalog `delay`. + /// The playout delay the subscription was opened for; a shared delay past it grows every budget by the excess. pub min_delay: Duration, - /// The rendition's catalog `delay`, held as this track's playout target and - /// added to every budget above. + /// The rendition's catalog `delay`, held as this track's playout target. pub delay: tokio::sync::watch::Receiver>, } @@ -136,11 +135,17 @@ fn soonest(a: Option, b: Option) -> Option { } impl Video { - /// A subscription's budget once it holds the catalog `delay`, plus `warmup`. - fn budget(&self, delay: Duration, warmup: Duration) -> moq_net::track::Delay { + /// How much longer than its own budgets video retains media once playback + /// trails the reference by `shared`, whichever track's target set it. + fn held(&self, shared: Duration) -> Duration { + shared.saturating_sub(self.min_delay) + } + + /// A subscription's budget with `held` and `warmup` on top of its own. + fn budget(&self, held: Duration, warmup: Duration) -> moq_net::track::Delay { moq_net::track::Delay { - min: Some(self.min_delay + delay + warmup), - max: Some(self.max_delay + delay + warmup), + min: Some(self.min_delay + held + warmup), + max: Some(self.max_delay + held + warmup), } } @@ -171,19 +176,30 @@ impl Video { let role = self.presentation.track(moq_mux::container::Kind::Video); let receive = async { // Video has no jitter estimate, so its target is the encoder delay alone, - // which holds audio back with it. Every budget grows with it, or media the - // later schedule still waits for would be evicted first. + // which holds audio back with it. let mut encoder = self.delay.clone(); - let hold = |track: &mut Track, delay: Option| { - let Some(delay) = delay else { return Duration::ZERO }; - track.set_delay(self.budget(delay, Duration::ZERO)); - if role.set_target(delay) { + let target = |delay: Option| { + if let Some(delay) = delay + && role.set_target(delay) + { self.changed.notify_one(); self.output.send(Event::Wake); } - delay }; - let mut held = hold(&mut track, *encoder.borrow_and_update()); + target(*encoder.borrow_and_update()); + // Every budget follows the shared delay, whichever track set it, or media + // the later schedule still waits for would be evicted first. + let schedule = self.presentation.consume(); + let mut shared = schedule.delay(); + let mut held = Duration::ZERO; + let follow = |track: &mut Track, held: &mut Duration, shared: Duration| { + let next = self.held(shared); + if next != *held { + *held = next; + track.set_delay(self.budget(next, Duration::ZERO)); + } + }; + follow(&mut track, &mut held, shared); // The group of a declared end whose restart waits for a later group, so the // media buffered before the end still plays out, and the transition that // later group resumes. @@ -205,13 +221,22 @@ impl Video { { tokio::select! { () = advanced.notified() => {} - Ok(()) = encoder.changed() => held = hold(&mut track, *encoder.borrow_and_update()), + Ok(()) = encoder.changed() => target(*encoder.borrow_and_update()), + delay = role.delay_changed(shared) => { + shared = delay; + follow(&mut track, &mut held, shared); + } } } let read = tokio::select! { read = track.read() => read?, Ok(()) = encoder.changed() => { - held = hold(&mut track, *encoder.borrow_and_update()); + target(*encoder.borrow_and_update()); + continue; + } + delay = role.delay_changed(shared) => { + shared = delay; + follow(&mut track, &mut held, shared); continue; } }; @@ -580,17 +605,23 @@ impl Video { // The group of the last end this decoder read. Its own subscription may // deliver the replacement before the min-delay decoder reads that end. let mut fence = None; - let mut encoder = self.delay.clone(); - if let Some(delay) = *encoder.borrow_and_update() { - refresh.set_delay(self.budget(delay, warmup)); - } + let schedule = self.presentation.consume(); + let mut shared = schedule.delay(); + let mut held = Duration::ZERO; + let follow = |refresh: &mut Track, held: &mut Duration, shared: Duration| { + let next = self.held(shared); + if next != *held { + *held = next; + refresh.set_delay(self.budget(next, warmup)); + } + }; + follow(&mut refresh, &mut held, shared); loop { let read = tokio::select! { read = refresh.read() => read?, - Ok(()) = encoder.changed() => { - if let Some(delay) = *encoder.borrow_and_update() { - refresh.set_delay(self.budget(delay, warmup)); - } + delay = role.delay_changed(shared) => { + shared = delay; + follow(&mut refresh, &mut held, shared); continue; } }; @@ -679,10 +710,9 @@ impl Video { tokio::select! { _ = woken => {}, _ = sleep => {}, - Ok(()) = encoder.changed() => { - if let Some(delay) = *encoder.borrow_and_update() { - refresh.set_delay(self.budget(delay, warmup)); - } + delay = role.delay_changed(shared) => { + shared = delay; + follow(&mut refresh, &mut held, shared); } } }; @@ -955,6 +985,70 @@ mod tests { assert_eq!(clock.delay(), jitter, "released with its track"); } + /// Audio's 5 s catalog delay sets the shared schedule while video states none, + /// so video keeps every picture that schedule still waits for, well past its own + /// 2 s budget, rather than evicting them and freezing. + #[tokio::test] + async fn an_audio_encoder_delay_grows_video_retention() { + tokio::time::pause(); + let (min, max) = (Duration::from_millis(100), Duration::from_secs(2)); + let shared = Duration::from_secs(5); + let media = media(min); + let audio = media.presentation.track(moq_mux::container::Kind::Audio); + audio.set_target(shared); + + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = source + .consume() + .subscribe( + moq_net::track::Subscription::default().with_delay(moq_net::track::Delay { + min: Some(min), + max: Some(max), + }), + ) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new(source.clone(), format()); + let track = moq_mux::container::Consumer::new(subscriber, format()); + let video = Video { + min_delay: min, + ..playback(&media, max) + }; + let task = tokio::spawn(video.run(track, Buffered::new(0), None)); + + let mut first = None; + for at in 0..=5_500u64 { + if at % 100 == 0 { + producer.write(frame(at, true)).unwrap(); + } + tokio::task::yield_now().await; + if let Some(picture) = media.output.present(&media) + && first.is_none() + { + first = Some((picture.as_millis(), at)); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + let (picture, at) = first.expect("video froze: its pictures were evicted before they were due"); + assert_eq!(picture, 0, "the first picture was kept until it was due"); + assert!((5_000..5_100).contains(&at), "played in sync with audio: {at}"); + let held = shared - min; + assert_eq!( + source.subscription().map(|subscription| subscription.delay), + Some(moq_net::track::Delay { + min: Some(min + held), + max: Some(max + held), + }), + "the subscription's budgets grow with the shared delay" + ); + producer.finish().unwrap(); + task.await.unwrap().unwrap(); + } + async fn burst(speaker: bool, paused_window: bool) { burst_with(speaker, paused_window, false).await; }