From 0f4a307a87f5301ce67e37b694c89c2716bd8d09 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 23:31:56 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m2/mux-sync.md Quest-Claim: 979859ff485444d7 From 6e68ab0de046fd5b1076369c6d85c7f62d7e5921 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 7 Nov 2026 23:58:26 -0800 Subject: [PATCH 2/4] feat(mux)!: moq_mux::sync keeps tracks decoupled on one wall-clock reference (WIP) Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/headless.rs | 7 +- rs/moq-cli/src/play/media.rs | 220 ++- rs/moq-cli/src/play/metrics.rs | 14 +- rs/moq-cli/src/play/playback.rs | 7 +- rs/moq-cli/src/play/timeline.rs | 106 ++ rs/moq-cli/src/play/video.rs | 65 +- rs/moq-cli/src/play/window.rs | 5 +- rs/moq-mux/Cargo.toml | 2 +- rs/moq-mux/benches/{timeline.rs => sync.rs} | 35 +- rs/moq-mux/src/lib.rs | 4 +- rs/moq-mux/src/sync.rs | 1331 ++++++++++++++++ rs/moq-mux/src/timeline.rs | 1501 ------------------- 12 files changed, 1678 insertions(+), 1619 deletions(-) rename rs/moq-mux/benches/{timeline.rs => sync.rs} (63%) create mode 100644 rs/moq-mux/src/sync.rs delete mode 100644 rs/moq-mux/src/timeline.rs diff --git a/rs/moq-cli/src/play/headless.rs b/rs/moq-cli/src/play/headless.rs index b8bd1c2e9c5..5c9efdcc3d7 100644 --- a/rs/moq-cli/src/play/headless.rs +++ b/rs/moq-cli/src/play/headless.rs @@ -14,7 +14,8 @@ use super::media::Media; use super::metrics::Metrics; use super::output::Output; use super::window::Event; -use moq_mux::timeline::Producer as Presentation; +use super::timeline::Playhead; +use moq_mux::sync::Producer as Presentation; /// The speaker and the run loop's events, plus `--metrics`' observer feed. #[derive(Clone)] @@ -52,9 +53,10 @@ pub(crate) async fn run( ) -> anyhow::Result<()> { let (events, mut received) = mpsc::unbounded_channel(); let presentation = Presentation::new(args.video_delay()); + let playhead = Playhead::default(); let (mut metrics, observe) = match &args.metrics { Some(path) => { - let (metrics, observe) = Metrics::create(path, presentation.consume())?; + let (metrics, observe) = Metrics::create(path, presentation.consume(), playhead.clone())?; (Some(metrics), Some(observe)) } None => (None, None), @@ -65,6 +67,7 @@ pub(crate) async fn run( origin, broadcast, presentation, + playhead, args, video: Arc::new(Mutex::new(VecDeque::new())), drained: Arc::new(Notify::new()), diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 402a8bd2eca..28e765eafba 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -15,10 +15,10 @@ use kio::clock::Instant; use super::args::Args; use super::output::{Output, Sink, Speaker}; use super::playback::{Kind, Playback, Track, joined}; -use super::timeline::{AudioTimeline, fit}; +use super::timeline::{AudioTimeline, Playhead, fit}; use super::video::{Refresh, Video}; use super::window::Event; -use moq_mux::timeline::{Producer as Presentation, Transition}; +use moq_mux::sync::{Producer as Presentation, Transition}; /// What a frame that starts a new decoded timeline crossed. enum Crossing { @@ -54,6 +54,8 @@ pub(super) struct Media { pub(super) args: Args, pub(super) video: Arc>>, pub(super) presentation: Presentation, + /// What the speaker is sounding, for `--metrics`. + pub(super) playhead: Playhead, pub(super) drained: Arc, pub(super) output: O, } @@ -137,8 +139,8 @@ impl Media { let mut audio_demand: Option = None; // Shared by an audio rendition and the retired tails still playing beside // it, so their sinks mix on one stream: a second stream on an exclusive - // device would fail to open. Released once none of them is left to hand - // the presentation anchor back to video; the player keeps the device. + // device would fail to open. Released once none of them is left; the + // player keeps the device. let mut speaker = None; // Retired audio sinks still playing out what they hold. let mut tails = tokio::task::JoinSet::new(); @@ -159,10 +161,9 @@ impl Media { result = tasks.join_next(), if !tasks.is_empty() => { let ended = joined(result.expect("guarded by is_empty"))?.map(|(kind, (finished, sink))| { if kind == Kind::Audio { - // The tail still sounds, so the speaker keeps the anchor, but a - // replacement is a track boundary whose timestamps need not - // continue this one: its first frame re-pins. - self.presentation.audio_restarted(); + // A replacement is a track boundary whose timestamps need not + // continue this one. + self.playhead.restart(); } // The retired sink still holds a delay of audio, and a replacement // holds its own before its first sample sounds. Played one after @@ -213,10 +214,9 @@ impl Media { } } - // The last sink went quiet: nothing holds playback to the speaker's - // cadence any more, and video takes the anchor back. + // The last sink went quiet. if !playback.playing(Kind::Audio) && tails.is_empty() && speaker.take().is_some() { - self.presentation.audio_stopped(); + self.playhead.clear(); self.drained.notify_one(); } @@ -409,6 +409,7 @@ impl Media { let audio = AudioPlayback { speaker: speaker.clone().expect("opened above"), presentation: self.presentation.clone(), + playhead: self.playhead.clone(), latency: self.args.fixed_delay().unwrap_or_default().max(AUDIO_BUFFER_MIN), changed: self.drained.clone(), output: self.output.clone(), @@ -517,6 +518,7 @@ struct AudioPlayback { changed: Arc, speaker: O::Speaker, presentation: Presentation, + playhead: Playhead, /// The depth the sink opens on, which also sizes its ring. latency: Duration, output: O, @@ -533,6 +535,7 @@ async fn play_audio( changed, speaker, presentation, + playhead, latency, output, } = playback; @@ -542,10 +545,10 @@ async fn play_audio( // later. That estimate times each packet when `read` hands it over, so this // receiver never waits on the speaker between reads. Live playback lets // the callback stretch toward the target; a declared end plays comfort noise - // until the old media drains and its replacement is due. The window - // schedules video against where the speaker actually is, which keeps the two - // together whatever the target does. - let role = presentation.track(Kind::Audio); + // until the old media drains and its replacement is due. Each write steers + // the sink toward when the shared clock says its media is due, which keeps + // audio with video in the steady state without letting a stall hold video. + let role = presentation.track(moq_mux::container::Kind::Audio); let received = Mutex::new(VecDeque::new()); let arrived = tokio::sync::Notify::new(); let popped = tokio::sync::Notify::new(); @@ -678,7 +681,7 @@ async fn play_audio( // The frame being written starts the utterance after a declared silence. let mut utterance = false; loop { - let (event, delay, crossing, _) = loop { + let (event, delay, crossing, arrived_at) = loop { if let Some(event) = received.lock().unwrap().pop_front() { popped.notify_one(); break event; @@ -711,9 +714,7 @@ async fn play_audio( let first = lead(delay); let mapping = role.resume(transition, frame.timestamp, Some(first)).ready().await?; role.apply(transition, mapping); - let at = presentation - .due(role.map(frame.timestamp)?) - .and_then(|at| at.checked_sub(first)); + let at = role.due(role.map(frame.timestamp)?).and_then(|at| at.checked_sub(first)); if let Some(at) = at { kio::clock::sleep_until(at).await; } @@ -782,7 +783,10 @@ async fn play_audio( sequential = false; silent = false; drop(sink); - presentation.audio_restarted(); + // The media either side of the break is unrelated, so its arrivals + // stop holding the clock and the playhead starts over. + role.reset(); + playhead.restart(); sink = speaker.sink(input.clone())?; } @@ -790,7 +794,7 @@ async fn play_audio( // spacing instead of fitting that queued burst as if it arrived live. // The lead follows the current target so jitter can still deepen the sink. if sequential { - let at = presentation.due(mapped).and_then(|at| at.checked_sub(lead(delay))); + let at = role.due(mapped).and_then(|at| at.checked_sub(lead(delay))); if let Some(at) = at { kio::clock::sleep_until(at).await; } @@ -802,11 +806,23 @@ async fn play_audio( // re-anchors, taking the video with it. So the hole goes in as silence ahead // of the frame, and the pair is fitted to the target as one write. let buffered = samples(sink.buffered()); - // Capped at the age budget: audio older than it is skipped rather than held, - // so a deeper target could never fill. The advertised floor needs the cap, - // being a number the publisher declared about itself, unbounded. The budget - // is also what the sink's ring was sized to hold. - let held = input.latency; + // The arrival, not the write: how far the speaker trails is what the clock + // steers, never what moves it. The target is this track's own jitter + // target, which playback holds unless another track asks for more. + let mut moved = role.received(mapped, arrived_at); + moved |= role.set_target(input.latency); + // The depth that makes this frame's end sound when the shared clock says + // it is due, so the callback stretches the speaker onto the clock: behind + // it after a stall, it plays faster until it catches up. Floored at what + // the device needs, and capped at the age budget: audio older than it is + // skipped rather than held, so a deeper target could never fill. The + // budget is also what the sink's ring was sized to hold. + let now = Instant::now(); + let held = moq_net::Timestamp::try_from(timing.end) + .ok() + .and_then(|end| role.due(end)) + .map_or(input.latency, |due| due.saturating_duration_since(now)) + .clamp(AUDIO_BUFFER_MIN, max_delay.max(AUDIO_BUFFER_MIN)); let target = samples(held); sink.set_latency(held)?; // Leave normal convergence to the callback. Only an excess at NetEq's @@ -841,18 +857,9 @@ async fn play_audio( silent = true; } - // Anchor the playout clock on where the speaker has actually reached, which - // is the only half of the pipeline that cannot skip ahead. A move has to - // wake the window: it is asleep on a deadline computed from the old anchor, - // and every queued frame is due earlier now. - let moved = presentation.audio( - moq_mux::timeline::Audio { - end: timing.end, - buffered: sink.buffered(), - target: held, - }, - Instant::now(), - ); + playhead.wrote(timing.end, sink.buffered(), Instant::now()); + // A move has to wake the window: it is asleep on a deadline computed from + // the old schedule, and every queued frame is due earlier now. if moved { changed.notify_one(); output.send(Event::Wake); @@ -908,6 +915,7 @@ mod tests { changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(AUDIO_BUFFER_MIN), + playhead: Default::default(), latency: AUDIO_BUFFER_MIN, output: recorder, }; @@ -943,6 +951,7 @@ mod tests { changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(delay), + playhead: Default::default(), latency: AUDIO_BUFFER_MIN, output: recorder.clone(), }; @@ -1018,6 +1027,7 @@ mod tests { changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(delay), + playhead: Default::default(), latency: delay, output: recorder.clone(), }; @@ -1098,6 +1108,133 @@ mod tests { quick_resumes(true, true, Duration::from_millis(80)).await; } + /// Audio stalls by congestion, then its backlog lands late and too far ahead + /// to fill, so it starts a fresh sink. Video keeps its own schedule throughout: + /// a stalled speaker never holds the picture. + #[tokio::test] + async fn stalled_audio_no_longer_holds_video() { + struct Passthrough; + impl super::super::video::Decoder for Passthrough { + async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result> { + Ok(vec![moq_video::Frame::new( + moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?, + frame.timestamp, + )]) + } + async fn flush(&mut self) -> anyhow::Result> { + Ok(Vec::new()) + } + } + tokio::time::pause(); + let start = Instant::now(); + let delay = Duration::from_millis(200); + let recorder = Recorder::default(); + let origin = moq_tokio::origin::spawn(); + let media = media(&origin, delay, recorder.clone()); + let broadcast = moq_net::broadcast::Info::new().produce(); + let audio = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let video = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = video + .consume() + .subscribe( + moq_net::track::Subscription::default().with_delay(moq_net::track::Delay::default().with_max(delay)), + ) + .await + .unwrap(); + let mut audio = moq_mux::container::Producer::new(audio, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut video = moq_mux::container::Producer::new(video, Container::Legacy(moq_mux::container::Kind::Video)); + let mut options = moq_audio::decode::Options::new(); + options.delay = moq_audio::decode::Delay { + min: Some(delay), + max: Some(delay), + }; + options.estimate = false; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let audio_play = AudioPlayback { + changed: media.drained.clone(), + speaker: recorder.clone(), + presentation: media.presentation.clone(), + playhead: Default::default(), + latency: delay, + output: recorder.clone(), + }; + let video_play = Video { + presentation: media.presentation.clone(), + frames: media.video.clone(), + changed: media.drained.clone(), + output: recorder.clone(), + max_delay: delay, + }; + let audio_task = tokio::spawn(async move { + play_audio(consumer, audio_play, std::future::pending()).await.unwrap(); + }); + let video_task = tokio::spawn(video_play.run( + moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), + Passthrough, + None, + )); + let mut shown = Vec::new(); + for at in 0..1_400u64 { + if at % 20 == 0 && at < 1_200 { + let index = at / 20; + video + .write(moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_millis(at).unwrap(), + duration: Some(moq_net::Timestamp::from_millis(20).unwrap()), + keyframe: index == 0, + payload: Bytes::from_static(b"picture"), + }) + .unwrap(); + // Audio stops after 200 ms; its backlog lands in one late burst at 800 ms. + if index < 10 || index > 40 { + audio.write(packet(index, if index < 10 { 0.25 } else { 0.5 })).unwrap(); + } else if index == 40 { + for late in 10..=40 { + audio.write(packet(late, 0.5)).unwrap(); + } + } + } + tokio::task::yield_now().await; + if let Some(timestamp) = recorder.present(&media) { + shown.push((timestamp, Instant::now())); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + audio.finish().unwrap(); + video.finish().unwrap(); + video_task.await.unwrap().unwrap(); + audio_task.abort(); + assert!( + recorder.played().iter().any(|part| part.sample == 0.5), + "audio never resumed" + ); + assert!( + recorder.latencies().len() >= 2, + "the late burst never started a fresh sink" + ); + assert!(shown.len() > 50, "video stalled: {} pictures", shown.len()); + for (timestamp, at) in shown { + let due = start + delay + Duration::from(timestamp); + assert!( + at.saturating_duration_since(due) <= Duration::from_millis(1) + && due.saturating_duration_since(at) <= Duration::from_millis(1), + "picture {timestamp:?} shown at {at:?}, due {due:?}" + ); + } + } + async fn quick_resumes(audio_end: bool, video_end: bool, audio_latency: Duration) { struct Buffered(Vec); impl super::super::video::Decoder for Buffered { @@ -1155,6 +1292,7 @@ mod tests { changed: media.drained.clone(), speaker: recorder.clone(), presentation: media.presentation.clone(), + playhead: Default::default(), latency: audio_latency, output: recorder.clone(), }; @@ -1316,6 +1454,7 @@ mod tests { changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(delay), + playhead: Default::default(), latency: delay, output: recorder.clone(), }; @@ -1396,6 +1535,7 @@ mod tests { changed: Default::default(), speaker: recorder.clone(), presentation: Presentation::new(delay), + playhead: Default::default(), latency: delay, output: recorder.clone(), }; @@ -1452,6 +1592,7 @@ mod tests { changed: changed.clone(), speaker: recorder.clone(), presentation: presentation.clone(), + playhead: Default::default(), latency: delay, output: recorder.clone(), }; @@ -1467,7 +1608,7 @@ mod tests { kio::clock::sleep(PACKET_DURATION).await; } // Video declares an end two seconds out, holding the audio replacement back. - let video = presentation.track(Kind::Video); + let video = presentation.track(moq_mux::container::Kind::Video); video.declare(); video.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); producer.discontinuity().unwrap(); @@ -1609,6 +1750,7 @@ mod tests { }, video: Default::default(), presentation: Presentation::new(delay), + playhead: Default::default(), drained: Default::default(), output, } diff --git a/rs/moq-cli/src/play/metrics.rs b/rs/moq-cli/src/play/metrics.rs index d1b159c5a95..fbf439ba80c 100644 --- a/rs/moq-cli/src/play/metrics.rs +++ b/rs/moq-cli/src/play/metrics.rs @@ -20,7 +20,9 @@ use moq_audio::playback::{Observation, Observer}; use serde::Serialize; use tokio::sync::mpsc; -use moq_mux::timeline::Consumer; +use moq_mux::sync::Consumer; + +use super::timeline::Playhead; /// How often a sample is taken, the browser probe's interval. const INTERVAL: Duration = Duration::from_millis(250); @@ -107,6 +109,7 @@ impl Totals { pub(super) struct Metrics { file: BufWriter, presentation: Consumer, + playhead: Playhead, observers: mpsc::UnboundedReceiver, current: Option<(Observer, Totals)>, /// What closed engines counted, added to the current one's totals. @@ -122,6 +125,7 @@ impl Metrics { pub(super) fn create( path: &Path, presentation: Consumer, + playhead: Playhead, ) -> anyhow::Result<(Self, mpsc::UnboundedSender)> { let file = File::create(path).with_context(|| format!("failed to create {}", path.display()))?; let (observe, observers) = mpsc::unbounded_channel(); @@ -130,6 +134,7 @@ impl Metrics { let metrics = Self { file: BufWriter::new(file), presentation, + playhead, observers, current: None, base: Totals::default(), @@ -203,17 +208,14 @@ impl Metrics { }; // Read straight after the device clock, so the two pair up. let now = Instant::now(); - let (playhead, target) = { - let presentation = &self.presentation; - (presentation.playhead(now), presentation.target()) - }; + let (playhead, delay) = (self.playhead.at(now), self.presentation.delay()); let sample = Sample { at: ms(now.duration_since(self.start)), render: observation.as_ref().map(|(o, base)| ms(base.clock + o.clock)), timestamp: playhead.map(ms), stalled: observation.as_ref().map(|(o, _)| o.stalled), - delay: target.map(ms), + delay: Some(ms(delay)), output_latency: observation.as_ref().and_then(|(o, _)| o.latency).map(ms), quanta: observation.as_ref().map(|(o, base)| base.quanta + o.quanta), quiet: observation.as_ref().map(|(o, base)| base.quiet + o.quiet), diff --git a/rs/moq-cli/src/play/playback.rs b/rs/moq-cli/src/play/playback.rs index 217079a23fc..ef5bd1f215a 100644 --- a/rs/moq-cli/src/play/playback.rs +++ b/rs/moq-cli/src/play/playback.rs @@ -6,7 +6,12 @@ use std::collections::HashSet; use hang::catalog::Reference; use moq_mux::catalog; -pub(super) use moq_mux::timeline::Kind; +/// A half of the player: the picture or the speaker. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Kind { + Video, + Audio, +} /// The track a rendition plays from: its resolved broadcast path and its name there. #[derive(Clone, Debug, PartialEq, Eq, Hash)] diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 39aa050ae85..8df2dae7fe0 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -1,7 +1,68 @@ //! Sample accounting for the player's speaker. +use std::sync::{Arc, Mutex}; use std::time::Duration; +use kio::clock::Instant; + +/// The media time the speaker is sounding, which `--metrics` reports. +/// +/// The speaker converges on the shared clock rather than owning it, so where it +/// actually is stays its own: it runs dry in a stall and skips ahead to catch up. +#[derive(Clone, Default)] +pub(super) struct Playhead(Arc>); + +#[derive(Default)] +struct Written { + /// The last write: the media time it ends at, when that sounds, and the + /// playhead it may not fall back behind on the same timeline. + last: Option<(Duration, Instant, Option)>, + /// The next write starts a new timeline. + restart: bool, +} + +impl Playhead { + /// Record a write ending at media `end`, of which `buffered` is still queued at `now`. + pub(super) fn wrote(&self, end: Duration, buffered: Duration, now: Instant) { + let mut written = self.0.lock().unwrap(); + // Silence padded ahead of the frame to refill the speaker is not media: the + // playhead holds where it was until the padding has played, rather than + // projecting back over media already heard. A new timeline starts afresh. + let floor = if std::mem::take(&mut written.restart) { + None + } else { + written.playhead(now) + }; + written.last = now.checked_add(buffered).map(|sounds| (end, sounds, floor)); + } + + /// The speaker moved onto an unrelated timeline: a fresh sink or rendition. + pub(super) fn restart(&self) { + self.0.lock().unwrap().restart = true; + } + + /// The speaker went quiet. + pub(super) fn clear(&self) { + *self.0.lock().unwrap() = Written::default(); + } + + /// The media time sounding at `now`, or `None` with no speaker. + pub(super) fn at(&self, now: Instant) -> Option { + self.0.lock().unwrap().playhead(now) + } +} + +impl Written { + /// Projected from the last write at the wall rate, held at its end (a speaker + /// with nothing left plays nothing further), and never behind where it stood + /// at that write (refill padding plays no media). + fn playhead(&self, now: Instant) -> Option { + let (end, sounds, floor) = self.last?; + let projected = end.saturating_sub(sounds.saturating_duration_since(now)); + Some(floor.map_or(projected, |floor| projected.max(floor)).min(end)) + } +} + /// Where the audio track has reached, measured from its own origin so /// timestamp rounding can't accumulate into drift. #[derive(Default)] @@ -85,6 +146,51 @@ pub(super) struct Fit { mod tests { use super::*; + const DELAY: Duration = Duration::from_millis(100); + + /// The playhead runs at the wall rate from the last write and stops at its + /// end, which is what a speaker that ran dry is still sounding. + #[test] + fn the_playhead_projects_the_last_write_and_holds_at_its_end() { + let now = Instant::ZERO + Duration::from_secs(1); + let playhead = Playhead::default(); + assert_eq!(playhead.at(now), None); + + playhead.wrote(Duration::from_secs(10), Duration::from_millis(200), now); + assert_eq!(playhead.at(now), Some(Duration::from_millis(9_800))); + let later = now + Duration::from_millis(150); + assert_eq!(playhead.at(later), Some(Duration::from_millis(9_950))); + let dry = now + Duration::from_secs(1); + assert_eq!(playhead.at(dry), Some(Duration::from_secs(10))); + + playhead.clear(); + assert_eq!(playhead.at(dry), None); + } + + /// Silence padded ahead of a frame after the speaker ran dry plays no media, + /// so the playhead holds instead of replaying media already heard. + #[test] + fn the_playhead_holds_through_refill_padding() { + let now = Instant::ZERO + Duration::from_secs(1); + let playhead = Playhead::default(); + playhead.wrote(Duration::from_secs(10), Duration::from_millis(200), now); + let dry = now + Duration::from_secs(1); + assert_eq!(playhead.at(dry), Some(Duration::from_secs(10))); + + // 100 ms of padding, then a 20 ms frame continuing from 10 s. + playhead.wrote(Duration::from_millis(10_020), Duration::from_millis(120), dry); + assert_eq!(playhead.at(dry), Some(Duration::from_secs(10))); + let padded = dry + Duration::from_millis(100); + assert_eq!(playhead.at(padded), Some(Duration::from_secs(10))); + let played = dry + Duration::from_millis(110); + assert_eq!(playhead.at(played), Some(Duration::from_millis(10_010))); + + // A new timeline may start anywhere. + playhead.restart(); + playhead.wrote(Duration::from_secs(2), DELAY, played); + assert_eq!(playhead.at(played), Some(Duration::from_millis(1_900))); + } + #[test] fn audio_timeline_restarts_when_media_time_rewinds() { let mut timeline = AudioTimeline::default(); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 96d92b7564c..2b937f84c0e 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -12,7 +12,7 @@ use moq_video::decode::{Lane, Next, Selector}; use super::buffer::Buffer; use super::output::Output; use super::window::Event; -use moq_mux::timeline::{Producer as Presentation, Transition}; +use moq_mux::sync::{Producer as Presentation, Transition}; /// A few surfaces, independent of the requested playout delay. A codec batch /// that overflows it waits beside the queue rather than pushing out pictures @@ -145,7 +145,7 @@ impl Video { let buffers = Mutex::new(VecDeque::from([Segment::default()])); // Signalled when a segment held behind a resume becomes the front. let advanced = tokio::sync::Notify::new(); - let role = self.presentation.track(super::playback::Kind::Video); + let role = self.presentation.track(moq_mux::container::Kind::Video); let receive = async { // The group of a declared end whose restart waits for a later group, so the // media buffered before the end still plays out, and the transition that @@ -193,9 +193,7 @@ impl Video { ..Default::default() }); } - if active - && buffers.len() == 1 && self.presentation.video(role.map(frame.timestamp)?, Instant::now()) - { + if active && buffers.len() == 1 && role.received(role.map(frame.timestamp)?, Instant::now()) { self.output.send(Event::Wake); } let back = buffers.back_mut().unwrap(); @@ -242,7 +240,7 @@ impl Video { ..Default::default() }); } - self.presentation.video_restarted(); + role.reset(); false } Some(Read::End { group, end: endpoint }) => { @@ -479,9 +477,8 @@ impl Video { role.end(end)?; } advancing.store(false, std::sync::atomic::Ordering::Relaxed); - let presentation = &self.presentation; for frame in &buffers.front().unwrap().frames { - presentation.video(role.map(frame.timestamp)?, Instant::now()); + role.received(role.map(frame.timestamp)?, Instant::now()); } self.output.send(Event::Wake); continue; @@ -726,6 +723,7 @@ mod tests { }, video: Default::default(), presentation: Presentation::new(delay), + playhead: Default::default(), drained: Default::default(), output: Recorder::default(), } @@ -817,15 +815,11 @@ mod tests { let media = media(delay); let start = Instant::now(); let last = moq_net::Timestamp::from_millis(1_980).unwrap(); + // A speaker that anchored on the first frame's turn, and so trails the + // burst's live edge by all of it. + let audio = media.presentation.track(moq_mux::container::Kind::Audio); if speaker { - media.presentation.audio( - moq_mux::timeline::Audio { - end: Duration::ZERO, - buffered: delay, - target: delay, - }, - start, - ); + audio.received(moq_net::Timestamp::ZERO, start); } let decoder = Buffered::default(); let decoded = decoder.decoded.clone(); @@ -844,20 +838,14 @@ mod tests { tokio::spawn(playback(&media, delay).run(track, decoder, None)) }; tokio::task::yield_now().await; - let last_due = start - + delay + if speaker { - Duration::from_millis(1_980) - } else { - Duration::ZERO - }; + // The burst is the live edge, which no speaker holds video back from. + let last_due = start + delay; + assert_eq!(media.presentation.due(last), Some(last_due)); + // Audio's own tune-in burst lands on the same edge, so the tail keeps its + // schedule once video's track goes. + audio.received(last, start); assert_eq!(media.presentation.due(last), Some(last_due)); assert!(media.video.lock().unwrap().len() <= MAX_FRAMES); - if speaker { - assert!( - decoded.lock().unwrap().is_empty(), - "the delay was buffered as raw pictures" - ); - } let mut shown = Vec::new(); let mut sizes = Vec::new(); while Instant::now() <= last_due + Duration::from_millis(10) { @@ -900,7 +888,7 @@ mod tests { } #[tokio::test] - async fn a_dual_burst_follows_the_speakers_clock() { + async fn a_dual_burst_reaches_live_past_a_trailing_speaker() { burst_with(true, false, true).await; } @@ -915,7 +903,7 @@ mod tests { } #[tokio::test] - async fn a_video_burst_follows_the_speakers_clock() { + async fn a_video_burst_reaches_live_past_a_trailing_speaker() { burst(true, false).await; } @@ -1136,7 +1124,7 @@ mod tests { .due(moq_net::Timestamp::from_millis(66).unwrap()) .unwrap(); assert!(Instant::now() < last, "the track closed after its tail was due"); - let audio = media.presentation.track(super::super::playback::Kind::Audio); + let audio = media.presentation.track(moq_mux::container::Kind::Audio); let transition = audio.declare(); let _mapping = audio .resume(transition, moq_net::Timestamp::ZERO, None) @@ -1166,7 +1154,7 @@ mod tests { let mut producer = moq_mux::container::Producer::new(source, format()); let track = moq_mux::container::Consumer::new(subscriber, format()); let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0), None)); - let audio = media.presentation.track(super::super::playback::Kind::Audio); + let audio = media.presentation.track(moq_mux::container::Kind::Audio); // Two-picture segments, one per restart, published live for two seconds // while audio's end holds the first replacement back. const SEGMENTS: u64 = 50; @@ -1232,7 +1220,7 @@ mod tests { let track = moq_mux::container::Consumer::new(subscriber, format()); let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0), None)); // Audio's end two seconds out holds every replacement back. - let audio = media.presentation.track(super::super::playback::Kind::Audio); + let audio = media.presentation.track(moq_mux::container::Kind::Audio); audio.declare(); audio.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); producer.write(frame(0, true)).unwrap(); @@ -1264,14 +1252,7 @@ mod tests { audio.resume(transition, start, Some(delay)).ready().await.unwrap(), ); let first = audio.map(start).unwrap(); - media.presentation.audio( - moq_mux::timeline::Audio { - end: Duration::from(first), - buffered: delay, - target: delay, - }, - Instant::now(), - ); + audio.received(first, Instant::now()); let mut shown = None; for _ in 0..1_000 { tokio::task::yield_now().await; @@ -1323,7 +1304,7 @@ mod tests { }; step(3).await; // Audio waits on an end ten seconds out while the decoder idles. - let audio = media.presentation.track(super::super::playback::Kind::Audio); + let audio = media.presentation.track(moq_mux::container::Kind::Audio); let transition = audio.declare(); audio.end(moq_net::Timestamp::from_secs(10).unwrap()).unwrap(); let resume = tokio::spawn(async move { diff --git a/rs/moq-cli/src/play/window.rs b/rs/moq-cli/src/play/window.rs index 4e99d266ea8..017c2eaf8d8 100644 --- a/rs/moq-cli/src/play/window.rs +++ b/rs/moq-cli/src/play/window.rs @@ -21,7 +21,7 @@ use super::args::Args; use super::layout::fit; use super::media::Media; use super::output::Output; -use moq_mux::timeline::{Consumer, Producer as Presentation}; +use moq_mux::sync::{Consumer, Producer as Presentation}; /// How early a frame may be shown rather than waiting another wakeup for it. /// Under a display's frame interval, so it can't be seen, but enough that timer @@ -79,6 +79,7 @@ pub fn run( args, video: video.clone(), presentation: presentation.clone(), + playhead: Default::default(), drained: drained.clone(), output: Remote { proxy: proxy.clone(), @@ -195,7 +196,7 @@ impl App { return Ok(()); }; let mut video = self.video.lock().unwrap(); - // Re-read the shared anchor on every redraw: audio may have moved it. + // Re-read the shared schedule on every redraw: any track may have moved it. let cutoff = Instant::now() + VIDEO_EARLY_TOLERANCE; let mut due = None; if let Some(index) = self diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 7506d924c2d..f508b58553c 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -107,7 +107,7 @@ name = "containers" harness = false [[bench]] -name = "timeline" +name = "sync" harness = false [[bench]] diff --git a/rs/moq-mux/benches/timeline.rs b/rs/moq-mux/benches/sync.rs similarity index 63% rename from rs/moq-mux/benches/timeline.rs rename to rs/moq-mux/benches/sync.rs index e470551b42d..32104b1d27c 100644 --- a/rs/moq-mux/benches/timeline.rs +++ b/rs/moq-mux/benches/sync.rs @@ -3,23 +3,19 @@ use std::{hint::black_box, time::Duration}; use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; use kio::clock::Instant; -use moq_mux::timeline; +use moq_mux::{container::Kind, sync}; use moq_net::Timestamp; fn presentation(c: &mut Criterion) { - let mut group = c.benchmark_group("timeline_present"); + let mut group = c.benchmark_group("sync_present"); let now = Instant::ZERO + Duration::from_secs(1); for readers in [1, 8, 64] { for frames in [1, 3, 30] { - let clock = timeline::Producer::new(Duration::from_millis(100)); - clock.audio( - moq_mux::timeline::Audio { - end: Duration::from_secs(1), - buffered: Duration::from_millis(100), - target: Duration::from_millis(100), - }, - now, - ); + let clock = sync::Producer::new(Duration::from_millis(100)); + let audio = clock.track(Kind::Audio); + let video = clock.track(Kind::Video); + audio.received(Timestamp::from_millis(1_000).unwrap(), now); + video.received(Timestamp::from_millis(1_000).unwrap(), now); let consumers: Vec<_> = (0..readers).map(|_| clock.consume()).collect(); let timestamps: Vec<_> = (0..frames) .map(|index| Timestamp::from_millis(1_000 + index * 20).unwrap()) @@ -41,18 +37,11 @@ fn presentation(c: &mut Criterion) { fn update(c: &mut Criterion) { let now = Instant::ZERO + Duration::from_secs(1); - let clock = timeline::Producer::new(Duration::from_millis(100)); - c.bench_function("timeline_audio_update", |b| { - b.iter(|| { - black_box(clock.audio( - moq_mux::timeline::Audio { - end: black_box(Duration::from_secs(1)), - buffered: Duration::from_millis(100), - target: Duration::from_millis(100), - }, - now, - )) - }); + let clock = sync::Producer::new(Duration::from_millis(100)); + let audio = clock.track(Kind::Audio); + let _video = clock.track(Kind::Video); + c.bench_function("sync_received", |b| { + b.iter(|| black_box(audio.received(black_box(Timestamp::from_millis(1_000).unwrap()), now))); }); } diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 9247b609a88..8ceb7ddd1e9 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -28,7 +28,7 @@ //! - [`Pacer`] maps each exported frame's media timestamp to the wall-clock //! instant it should be delivered at, for byte streams whose spacing is //! part of the format (MPEG-TS). -//! - [`timeline`] shares one audio-led playout clock with video presenters. +//! - [`sync`] holds a broadcast's tracks on one playout clock without letting a slow one stall the rest. //! - [`rate`] turns bandwidth grants into stable media bitrate targets shared //! by audio, video, and transcode senders. //! - [`follow`] follows a broadcast across the publisher instances that @@ -61,7 +61,7 @@ pub mod rate; pub mod select; mod source; mod stats; -pub mod timeline; +pub mod sync; pub use clock::Clock; pub use error::*; diff --git a/rs/moq-mux/src/sync.rs b/rs/moq-mux/src/sync.rs new file mode 100644 index 00000000000..b43b19e8de7 --- /dev/null +++ b/rs/moq-mux/src/sync.rs @@ -0,0 +1,1331 @@ +//! One playout clock shared by a broadcast's tracks. +//! +//! Each [`Track`] folds its arrivals into the clock with [`Track::received`] +//! and reads when its media is due. The clock's reference is the earliest +//! arrival any live track has seen, measured against its timestamp, so it sits +//! on the wall clock and follows the live edge. Every track presents against +//! it, which holds them together in the steady state. One that falls behind, +//! such as audio stalled by congestion, only arrives late: it never holds or +//! moves the others. Audio converges on the clock by stretching toward its +//! [`due`](Track::due) time rather than owning it. +//! +//! Playback trails the reference by the largest target a live track holds +//! ([`Track::set_target`]), or by the delay passed to [`Producer::new`] while +//! none holds one. Give a [`Consumer`] to a presenter: [`Consumer::present`] +//! selects the newest due picture without waiting, so a canvas can repeat its +//! previous picture. +//! +//! A track also maps encoder segments onto the clock across declared ends. +//! Keep one active track per [`Kind`](crate::container::Kind); retire it before starting its successor. +//! Bound replacement media with the caller's retention budget and +//! [`Track::abandon`] transitions whose replacement is discarded. + +use kio::clock::Instant; +use std::time::Duration; + +use moq_net::Timestamp; +use std::{ + pin::Pin, + sync::{Arc, Mutex}, + task::Poll, +}; + +use crate::container::Kind; + +/// One slot per [`Kind`]. +const KINDS: usize = 3; + +struct Clock { + state: Mutex, + changed: kio::Shared<()>, +} + +impl Clock { + fn new(delay: Duration) -> Self { + Self { + state: Mutex::new(State::new(delay)), + changed: Default::default(), + } + } + + fn wake(&self) { + *self.changed.lock() = (); + } +} + +/// A schedule pinning media time to the wall clock: `media` arrived at `at`. +#[derive(Clone, Copy, Debug)] +struct Anchor { + at: Instant, + media: Duration, +} + +impl Anchor { + /// When `media` lands on this schedule, before any delay. A time before the + /// platform clock can express lands on the anchor, which is already past. + fn at(&self, media: Duration) -> Option { + match media.checked_sub(self.media) { + Some(ahead) => self.at.checked_add(ahead), + None => Some(self.at.checked_sub(self.media - media).unwrap_or(self.at)), + } + } + + /// Whether this schedule runs ahead of `other`, so its media lands earlier. + fn earlier(&self, other: &Self) -> bool { + match other.media.checked_sub(self.media) { + Some(ahead) => self.at.checked_add(ahead).is_some_and(|at| at < other.at), + None => other.at.checked_add(self.media - other.media).is_none_or(|at| self.at < at), + } + } +} + +/// Whether two optional schedules differ. +fn moved(before: Option, after: Option) -> bool { + match (before, after) { + (Some(before), Some(after)) => before.earlier(&after) || after.earlier(&before), + (before, after) => before.is_some() != after.is_some(), + } +} + +/// One track's hold on the clock. +#[derive(Clone, Copy, Default)] +struct Part { + /// The earliest arrival on this track's current timeline. + anchor: Option, + /// The delay this track asks playback to hold. + target: Option, +} + +/// The playout clock every track of one broadcast presents against. +/// +/// Each track keeps the earliest arrival on its own timeline, and the reference +/// is the earliest of those. That is what pulls playback toward live: a frame +/// arriving earlier than the reference predicted re-pins it, while a late +/// frame only reports that it is late. A track that resets or goes away gives +/// up its arrivals, so the reference falls back to the earliest track left. +struct State { + /// How far behind the reference playback runs while no track holds a target. + delay: Duration, + parts: [Part; KINDS], + /// The schedule the last track to go left behind, which what it already + /// queued still presents against until any track anchors again. + left: Option, + /// Endpoints declared by each live role, on the shared playback timeline. + ends: [Option; KINDS], + /// When each retired role's tail finishes playing, frozen on the wall clock: + /// a later rendition re-pins the anchor, which would carry a media endpoint + /// onto an unrelated timeline. Kept apart from `ends` so a successor role + /// never clears its predecessor's tail. + tails: Vec, + /// Encoder restarts whose replacements have not all started playing. + transitions: std::collections::VecDeque, + /// The latest transition each role declared. + declared: [Transition; KINDS], + next: Transition, +} + +/// One encoder restart, shared by the roles whose ends declared it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Transition(u64); + +struct Pending { + id: Transition, + /// Roles whose declared end opened or joined this transition. + members: [bool; KINDS], + /// Members that applied its mapping. + resumed: [bool; KINDS], + /// Each member's first replacement media. + plans: [Option; KINDS], + mapping: Option, +} + +struct Boundary { + starts: Option, + end: Duration, +} + +#[derive(Clone, Copy)] +struct Plan { + source: Duration, + delay: Duration, + /// When the replacement media arrived. + arrived: Instant, +} + +impl State { + fn new(delay: Duration) -> Self { + Self { + delay, + parts: Default::default(), + left: None, + ends: Default::default(), + tails: Vec::new(), + transitions: Default::default(), + declared: [Transition(0); KINDS], + next: Transition(1), + } + } + + /// The earliest schedule any track holds. + fn reference(&self) -> Option { + self.parts + .iter() + .filter_map(|part| part.anchor) + .reduce(|earliest, anchor| if anchor.earlier(&earliest) { anchor } else { earliest }) + .or(self.left) + } + + /// How far behind the reference playback runs. + fn delay(&self) -> Duration { + self.parts + .iter() + .filter_map(|part| part.target) + .max() + .unwrap_or(self.delay) + } + + /// Fold an arrival into `kind`'s timeline, reporting whether that moved the schedule. + fn received(&mut self, kind: Kind, timestamp: Timestamp, now: Instant) -> bool { + let arrival = Anchor { + at: now, + media: Duration::from(timestamp), + }; + let before = self.reference(); + let part = &mut self.parts[kind as usize]; + if part.anchor.is_none_or(|anchor| arrival.earlier(&anchor)) { + part.anchor = Some(arrival); + } + self.left = None; + moved(before, self.reference()) + } + + /// End `kind`'s timeline, reporting whether that moved the schedule. + fn reset(&mut self, kind: Kind) -> bool { + let before = self.reference(); + self.parts[kind as usize].anchor = None; + moved(before, self.reference()) + } + + /// Release `kind` as its track goes. + fn release(&mut self, kind: Kind) { + let part = std::mem::take(&mut self.parts[kind as usize]); + if self.parts.iter().all(|part| part.anchor.is_none()) { + self.left = part.anchor.or(self.left); + } + } + + /// Hold `target` for `kind`, reporting whether that moved the schedule. + fn set_target(&mut self, kind: Kind, target: Duration) -> bool { + let before = self.delay(); + self.parts[kind as usize].target = Some(target); + before != self.delay() + } + + /// How many transitions are still pending. + #[cfg(test)] + fn pending(&self) -> usize { + self.transitions.len() + } + + fn transition(&mut self, id: Transition) -> Option<&mut Pending> { + self.transitions.iter_mut().find(|pending| pending.id == id) + } + + /// Forget transitions nobody resumes, or that every member resumed once + /// their replacement started and nothing can join them. + fn prune(&mut self, now: Instant) { + self.transitions.retain(|pending| { + let members = pending.members.iter().any(|member| *member); + let resumed = (0..KINDS).all(|role| !pending.members[role] || pending.resumed[role]); + let started = pending.mapping.is_some_and(|mapping| mapping.started(now)); + members && !(resumed && started) + }); + } + + /// The media position the clock will reach at `at`. + fn position(&self, at: Instant) -> Option { + let reference = self.reference()?; + let at = at.checked_sub(self.delay()).unwrap_or(Instant::ZERO); + Some(if at >= reference.at { + reference.media.saturating_add(at.duration_since(reference.at)) + } else { + reference.media.saturating_sub(reference.at.duration_since(at)) + }) + } + + /// When `timestamp` should be presented, or `None` before any frame has + /// anchored the clock. `None` means as soon as possible: there is nothing + /// left to wait for. + fn due(&self, timestamp: Timestamp) -> Option { + let at = self.reference()?.at(Duration::from(timestamp))?; + Some(at.checked_add(self.delay()).unwrap_or(at)) + } +} + +/// One track's hold on the shared clock: its arrivals and target, and the +/// mapping of its encoder segments onto the shared timeline. +/// +/// Dropping it releases its arrivals and target. +pub struct Track { + clock: Arc, + role: Kind, + mapping: std::sync::Mutex, +} + +/// The current segment's mapping, and how long it held the segment's media +/// back behind the old segment's tail. +#[derive(Clone, Copy)] +struct Segment { + mapping: Mapping, + /// Added to each arrival: media held behind a tail arrived that much earlier + /// than its schedule, which says nothing about the live edge. + held: Duration, +} + +/// A segment's timestamp mapping, applied only after its first media can play. +#[derive(Clone, Copy)] +#[must_use] +pub struct Mapping { + source: Duration, + target: Duration, + starts: Option, +} + +impl Mapping { + fn map(self, source: Timestamp) -> Result { + let source = Duration::from(source); + let target = if source >= self.source { + self.target.checked_add(source - self.source).ok_or(Error::Overflow)? + } else { + self.target + .checked_sub(self.source - source) + .ok_or(Error::BeforeOrigin)? + }; + Timestamp::from_micros(target.as_micros().try_into().map_err(|_| Error::Overflow)?).map_err(|_| Error::Overflow) + } + + fn started(&self, now: Instant) -> bool { + self.starts.is_none_or(|starts| starts <= now) + } +} + +impl Track { + fn new(clock: Arc, role: Kind) -> Self { + Self { + clock, + role, + mapping: std::sync::Mutex::new(Segment { + mapping: Mapping { + source: Duration::ZERO, + target: Duration::ZERO, + starts: None, + }, + held: Duration::ZERO, + }), + } + } + + /// Fold a frame that arrived at `now` into the clock, reporting whether that + /// moved the schedule. + /// + /// `timestamp` is on the shared timeline, as [`map`](Self::map) returns it. + /// A frame arriving earlier than the schedule predicted pulls playback toward + /// live; a late one leaves it alone. A move makes every queued frame due + /// earlier, so a presenter has to recompute the deadline it is asleep on. + pub fn received(&self, timestamp: Timestamp, now: Instant) -> bool { + let held = self.mapping.lock().unwrap().held; + let now = now.checked_add(held).unwrap_or(now); + let moved = self.clock.state.lock().unwrap().received(self.role, timestamp, now); + if moved { + self.clock.wake(); + } + moved + } + + /// End this track's timeline at a discontinuity, such as lost media or a + /// rewind: its arrivals stop holding the clock until the next one. + pub fn reset(&self) { + if self.clock.state.lock().unwrap().reset(self.role) { + self.clock.wake(); + } + } + + /// Ask playback to trail the reference by at least `target`, reporting + /// whether that moved the schedule. Playback holds the largest target any + /// live track asks for. + pub fn set_target(&self, target: Duration) -> bool { + let moved = self.clock.state.lock().unwrap().set_target(self.role, target); + if moved { + self.clock.wake(); + } + moved + } + + /// When `timestamp`, on the shared timeline, should be presented, or `None` + /// before any frame has anchored the clock. + pub fn due(&self, timestamp: Timestamp) -> Option { + self.clock.state.lock().unwrap().due(timestamp) + } + + /// Stop while a tail still plays until `drained`, holding a declared end until + /// then. Dropping instead discards the tail with the end. + pub fn retire(self, drained: Option) { + let now = kio::clock::now(); + let mut state = self.clock.state.lock().unwrap(); + state.tails.retain(|tail| *tail > now); + if state.ends[self.role as usize].take().is_some() + && let Some(drained) = drained + { + state.tails.push(drained); + } + } + + /// Map a source timestamp onto the shared broadcast timeline. + pub fn map(&self, source: Timestamp) -> Result { + self.mapping.lock().unwrap().mapping.map(source) + } + + /// Declare that this role's encoder stopped, returning the transition its + /// replacement resumes. + /// + /// A peer's transition this role has not joined yet is the same restart, as + /// long as its replacement has not started playing; otherwise this one opens + /// a transition of its own. + pub fn declare(&self) -> Transition { + let now = kio::clock::now(); + let own = self.role as usize; + let mut state = self.clock.state.lock().unwrap(); + state.prune(now); + let after = state.declared[own]; + let joined = state + .transitions + .iter_mut() + .filter(|transition| transition.id > after) + .find(|transition| { + !transition.members[own] + && transition.members.iter().any(|member| *member) + && transition.mapping.is_none_or(|mapping| !mapping.started(now)) + }) + .map(|transition| { + transition.members[own] = true; + transition.id + }); + let id = joined.unwrap_or_else(|| { + let id = state.next; + state.next = Transition(id.0 + 1); + let mut members = [false; KINDS]; + members[own] = true; + state.transitions.push_back(Pending { + id, + members, + resumed: [false; KINDS], + plans: [None; KINDS], + mapping: None, + }); + id + }); + state.declared[own] = id; + drop(state); + self.clock.wake(); + id + } + + /// Mark where this role's current segment ends on the shared clock. + pub fn end(&self, source: Timestamp) -> Result<()> { + let mapping = self.mapping.lock().unwrap().mapping; + self.clock.state.lock().unwrap().ends[self.role as usize] = Some(Boundary { + starts: mapping.starts, + end: Duration::from(mapping.map(source)?), + }); + self.clock.wake(); + Ok(()) + } + + /// Record the first replacement media for `transition`, as it arrives. + pub fn plan(&self, transition: Transition, source: Timestamp, delay: Option) { + let arrived = kio::clock::now(); + let mut state = self.clock.state.lock().unwrap(); + let delay = delay.unwrap_or(state.delay()); + if let Some(pending) = state.transition(transition) { + pending.plans[self.role as usize] = Some(Plan { + source: Duration::from(source), + delay, + arrived, + }); + } + drop(state); + self.clock.wake(); + } + + /// Media was lost: nothing waits for the discarded tail. + pub fn gap(&self) { + self.clock.state.lock().unwrap().ends[self.role as usize] = None; + self.clock.wake(); + } + + /// This role will never resume `transition`, so a peer must not pair with its plan. + pub fn abandon(&self, transition: Transition) { + let own = self.role as usize; + let mut state = self.clock.state.lock().unwrap(); + if let Some(pending) = state.transition(transition) { + pending.members[own] = false; + pending.plans[own] = None; + } + state.prune(kio::clock::now()); + drop(state); + self.clock.wake(); + } + + /// Commit the mapping once its replacement media starts playing. + pub fn apply(&self, transition: Transition, mapping: Mapping) { + let mut segment = self.mapping.lock().unwrap(); + let mut state = self.clock.state.lock().unwrap(); + let reference = state.reference(); + let mut held = Duration::ZERO; + if let Some(pending) = state.transition(transition) { + pending.resumed[self.role as usize] = true; + // How much earlier the replacement arrived than the schedule it now + // plays on, having waited for the old tail to drain. + if let Some(plan) = pending.plans[self.role as usize] + && let Some(scheduled) = mapping + .map(media(plan.source)) + .ok() + .zip(reference) + .and_then(|(mapped, reference)| reference.at(Duration::from(mapped))) + { + held = scheduled.saturating_duration_since(plan.arrived); + } + } + *segment = Segment { mapping, held }; + drop(segment); + state.prune(kio::clock::now()); + drop(state); + self.clock.wake(); + } + + /// Prepare a mapping after the old media drains; dropping the wait leaves the track unchanged. + pub fn resume(&self, transition: Transition, source: Timestamp, delay: Option) -> Resume<'_> { + Resume { + track: self, + transition, + source, + delay, + timer: None, + } + } +} + +impl Drop for Track { + fn drop(&mut self) { + let own = self.role as usize; + { + let mut state = self.clock.state.lock().unwrap(); + state.release(self.role); + state.ends[own] = None; + for pending in state.transitions.iter_mut().filter(|pending| !pending.resumed[own]) { + pending.members[own] = false; + pending.plans[own] = None; + } + state.declared[own] = Transition(0); + state.prune(kio::clock::now()); + } + self.clock.wake(); + } +} + +/// A duration from the start of the track as a wire timestamp. +fn media(duration: Duration) -> Timestamp { + // A timestamp caps at 2^61 - 1, which is ~73,000 years of microseconds. + const MAX: u128 = (1 << 61) - 1; + Timestamp::from_micros(duration.as_micros().min(MAX) as u64).expect("clamped to the timestamp maximum") +} + +/// A pending segment mapping, driven by the caller's executor or waiter. +#[must_use = "drive the wait before applying its mapping"] +pub struct Resume<'a> { + track: &'a Track, + transition: Transition, + source: Timestamp, + delay: Option, + timer: Option, +} + +impl Resume<'_> { + /// The replacement's mapping, once all declared old media has drained. + #[cfg_attr(feature = "async", kio::wrapper)] + pub fn poll_ready(&mut self, waiter: &kio::Waiter) -> Poll> { + let _ = self.track.clock.changed.poll(waiter, |_| Poll::Pending); + let now = kio::clock::now(); + let mut state = self.track.clock.state.lock().unwrap(); + if let Some(mapping) = state.transition(self.transition).and_then(|pending| pending.mapping) { + return Poll::Ready(Ok(mapping)); + } + let end = state + .ends + .iter() + .flatten() + .filter(|boundary| boundary.starts.is_none_or(|start| start <= now)) + .filter_map(|boundary| state.due(media(boundary.end))) + .chain(state.tails.iter().copied()) + .max(); + if let Some(end) = end.filter(|end| *end > now) { + let timer = self.timer.get_or_insert_with(|| kio::clock::sleep_until(end)); + if timer.deadline() != end { + timer.reset(end); + } + std::task::ready!(waiter.poll_future(Pin::new(timer))); + } + + let own = self.track.role as usize; + // A transition abandoned under this resume, by a gap the caller rechecks, + // maps alone like one no peer joined. + let (mapping, paired) = state.transition(self.transition).map_or((None, None), |pending| { + let mut plans = pending.members.iter().zip(&pending.plans).filter(|(member, _)| **member); + let joined = pending.members[own] && plans.clone().count() > 1; + let origin = plans + .try_fold(None::, |earliest, (_, plan)| { + let plan = (*plan)?; + Some(Some(match earliest { + Some(earliest) if earliest.source <= plan.source => earliest, + _ => plan, + })) + }) + .flatten(); + (pending.mapping, origin.filter(|_| joined)) + }); + // The peer already resumed this transition. + if let Some(mapping) = mapping { + return Poll::Ready(Ok(mapping)); + } + // Every member has replacement media. The earliest defines their common + // origin, keeping the media offsets between them, even when its delay + // differs from the others'. Otherwise this role maps alone and a peer + // joining later follows it. + let (source, delay) = match paired { + Some(origin) => (origin.source, origin.delay), + None => (Duration::from(self.source), self.delay.unwrap_or(state.delay())), + }; + let starts = now.checked_add(delay).ok_or(Error::DelayOverflow)?; + let mapping = Mapping { + source, + target: state.position(starts).unwrap_or(source), + starts: Some(starts), + }; + if let Some(pending) = state.transition(self.transition) { + pending.mapping = Some(mapping); + } + Poll::Ready(Ok(mapping)) + } +} + +/// A timestamp or delay cannot be represented on the playout clock. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// Mapping the timestamp would overflow. + #[error("playback timeline overflow")] + Overflow, + /// A picture precedes the mapped segment's origin. + #[error("picture precedes the playback timeline")] + BeforeOrigin, + /// The requested delay exceeds the system clock. + #[error("playback delay exceeds the system clock")] + DelayOverflow, +} + +/// A result from mapping media onto the playout clock. +pub type Result = std::result::Result; + +/// Owns one broadcast's playout clock and hands out its tracks. +/// +/// Clones capture this broadcast's clock; resetting a handle never changes the +/// clock its previous clones or tracks update. +#[derive(Clone)] +pub struct Producer { + clock: Arc, + current: Arc>>, +} + +/// Reads the current playout schedule without moving it. +/// +/// Keeps the final schedule readable after its tracks drop, and follows the +/// replacement clock when a producer resets for another broadcast. +#[derive(Clone)] +pub struct Consumer { + current: Arc>>, +} + +impl Producer { + /// Create a broadcast clock that trails the reference by `delay` while no track holds a target. + pub fn new(delay: Duration) -> Self { + let clock = Arc::new(Clock::new(delay)); + Self { + clock: clock.clone(), + current: Arc::new(Mutex::new(clock)), + } + } + + /// Replace this producer with a new broadcast clock and switch readers to it. + /// + /// Previous clones and tracks retain their old broadcast's clock. + #[must_use = "use the returned producer for the replacement broadcast"] + pub fn reset(self, delay: Duration) -> Self { + let clock = Arc::new(Clock::new(delay)); + *self.current.lock().unwrap() = clock.clone(); + Self { + clock, + current: self.current, + } + } + + /// Return a reader that follows this handle's broadcast replacements. + pub fn consume(&self) -> Consumer { + Consumer { + current: self.current.clone(), + } + } + + /// Return a track bound to this broadcast's clock. + pub fn track(&self, kind: Kind) -> Track { + Track::new(self.clock.clone(), kind) + } + + /// When `timestamp` should be presented, or `None` before any frame has + /// anchored the clock. `None` means as soon as possible: there is nothing + /// left to wait for. + pub fn due(&self, timestamp: Timestamp) -> Option { + self.clock.state.lock().unwrap().due(timestamp) + } +} + +impl Consumer { + /// How far behind the reference playback runs. + pub fn delay(&self) -> Duration { + self.current.lock().unwrap().state.lock().unwrap().delay() + } + + /// When this timestamp is due, or `None` before the first anchor. + pub fn due(&self, timestamp: Timestamp) -> Option { + self.current.lock().unwrap().state.lock().unwrap().due(timestamp) + } + + /// The newest due picture's index in presentation order, or `None` to repeat the previous one. + /// + /// Pass queued timestamps in ascending order and discard entries through the + /// returned index after showing that picture. This stores no frames and leaves + /// their retention and backpressure to the caller. Query again after every + /// schedule move; a cached deadline can become stale. + pub fn present(&self, timestamps: impl IntoIterator, now: Instant) -> Option { + let clock = self.current.lock().unwrap(); + let state = clock.state.lock().unwrap(); + timestamps + .into_iter() + .take_while(|timestamp| state.due(*timestamp).is_none_or(|at| at <= now)) + .count() + .checked_sub(1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const DELAY: Duration = Duration::from_millis(100); + + fn ms(millis: u64) -> Timestamp { + Timestamp::from_millis(millis).unwrap() + } + + /// A clock with an audio and a video track, audio anchored at 10 s arriving `now`. + fn anchored(now: Instant) -> (Arc, Track, Track) { + let clock = Arc::new(Clock::new(DELAY)); + let audio = Track::new(clock.clone(), Kind::Audio); + let video = Track::new(clock.clone(), Kind::Video); + audio.received(ms(10_000), now); + (clock, audio, video) + } + + /// Earlier arrivals pull the reference forward so playback converges on the + /// live edge without giving up the delay; late ones leave it alone. + #[test] + fn the_earliest_arrival_sets_the_schedule() { + let start = Instant::ZERO + Duration::from_secs(1); + let clock = Producer::new(DELAY); + let video = clock.track(Kind::Video); + assert_eq!(clock.due(ms(0)), None, "nothing anchors before the first frame"); + + assert!(video.received(ms(0), start), "the first anchor"); + assert_eq!(clock.due(ms(0)), Some(start + DELAY)); + + // 40ms of media later, but only 20ms of wall clock: the first frame was + // 20ms late, so the schedule moves onto this one. + let now = start + Duration::from_millis(20); + assert!(video.received(ms(40), now), "an early frame left the schedule alone"); + assert_eq!(clock.due(ms(40)), Some(now + DELAY)); + + // A frame that merely arrives late keeps its media instant. The delay is + // the room it has to be late in. + let late = now + Duration::from_millis(70); + assert!(!video.received(ms(80), late), "a late arrival moved the schedule"); + assert_eq!(clock.due(ms(80)), Some(now + Duration::from_millis(140))); + } + + /// The required decoupling: audio stalled by congestion arrives late, which + /// neither holds video back nor moves it, while both share one schedule in + /// the steady state. + #[test] + fn stalled_audio_no_longer_holds_video() { + let start = Instant::ZERO + Duration::from_secs(1); + let clock = Producer::new(DELAY); + let audio = clock.track(Kind::Audio); + let video = clock.track(Kind::Video); + audio.received(ms(0), start); + video.received(ms(0), start); + let steady = clock.due(ms(1_000)); + assert_eq!(steady, Some(start + DELAY + Duration::from_secs(1))); + + // Video keeps arriving on time while audio stalls for 500ms, then audio + // lands in a late burst and resets its timeline on a fresh sink. + for frame in 1..=25 { + let at = start + Duration::from_millis(frame * 40); + assert!(!video.received(ms(frame * 40), at)); + } + let resumed = start + Duration::from_millis(1_000); + for frame in 0..25 { + assert!(!audio.received(ms(500 + frame * 20), resumed), "late audio moved video"); + } + audio.reset(); + assert!(!audio.received(ms(1_000), resumed + Duration::from_millis(100))); + assert_eq!(clock.due(ms(1_000)), steady, "stalled audio held video"); + assert_eq!(audio.due(ms(1_000)), steady, "audio converges on the shared schedule"); + } + + /// A video tune-in burst is the live edge: nothing owns the schedule, so it + /// moves every track onto it, audio included. + #[test] + fn a_video_burst_moves_audio_onto_the_live_edge() { + let start = Instant::ZERO + Duration::from_secs(1); + let clock = Producer::new(DELAY); + let audio = clock.track(Kind::Audio); + let video = clock.track(Kind::Video); + audio.received(ms(1_000), start); + assert!(video.received(ms(4_000), start)); + assert_eq!(audio.due(ms(4_000)), Some(start + DELAY)); + } + + /// A track that resets gives up its arrivals: the schedule falls back to the + /// earliest track left, and with none left the next frame anchors afresh. + #[test] + fn a_reset_hands_the_schedule_to_the_tracks_left() { + let start = Instant::ZERO + Duration::from_secs(1); + let clock = Producer::new(DELAY); + let audio = clock.track(Kind::Audio); + let video = clock.track(Kind::Video); + video.received(ms(0), start); + // Audio runs 30ms ahead of video, so it holds the schedule. + audio.received(ms(30), start); + assert_eq!(clock.due(ms(30)), Some(start + DELAY)); + + audio.reset(); + assert_eq!(clock.due(ms(30)), Some(start + DELAY + Duration::from_millis(30))); + + video.reset(); + assert_eq!(clock.due(ms(30)), None); + let now = start + Duration::from_secs(5); + audio.received(ms(500), now); + assert_eq!(clock.due(ms(500)), Some(now + DELAY)); + + // The last track to go leaves its schedule for what it queued, until + // another anchors afresh. + drop(audio); + assert_eq!(clock.due(ms(500)), Some(now + DELAY)); + let later = now + Duration::from_secs(5); + video.received(ms(0), later); + assert_eq!(clock.due(ms(0)), Some(later + DELAY)); + } + + /// Playback holds the deepest target a live track asks for, and the default + /// while none asks. + #[test] + fn the_delay_is_the_largest_target_held() { + let start = Instant::ZERO + Duration::from_secs(1); + let clock = Producer::new(DELAY); + let reader = clock.consume(); + let audio = clock.track(Kind::Audio); + let video = clock.track(Kind::Video); + video.received(ms(0), start); + assert_eq!(reader.delay(), DELAY); + + assert!(audio.set_target(Duration::from_millis(60))); + assert_eq!(reader.delay(), Duration::from_millis(60)); + assert_eq!(clock.due(ms(0)), Some(start + Duration::from_millis(60))); + assert!(video.set_target(Duration::from_millis(80))); + assert!(!audio.set_target(Duration::from_millis(70)), "a shallower target moved the delay"); + assert_eq!(reader.delay(), Duration::from_millis(80)); + + drop(video); + assert_eq!(reader.delay(), Duration::from_millis(70)); + drop(audio); + assert_eq!(reader.delay(), DELAY); + } + + #[test] + fn presentation_selects_due_frames_on_the_current_schedule() { + let clock = Producer::new(Duration::from_millis(40)); + let reader = clock.consume(); + let audio = clock.track(Kind::Audio); + let video = clock.track(Kind::Video); + let now = Instant::ZERO + Duration::from_secs(1); + video.received(ms(0), now); + let frames = [ms(0), ms(20), ms(40)]; + assert_eq!(reader.present(frames, now), None); + assert_eq!(reader.present(frames, now + Duration::from_millis(60)), Some(1)); + // Audio arrives 40ms ahead; stale video deadlines no longer apply. + audio.received(ms(40), now); + assert_eq!(reader.present(frames, now), Some(0)); + assert_eq!(reader.present(frames, now + Duration::from_millis(39)), Some(1)); + assert_eq!(reader.present(frames, now + Duration::from_millis(40)), Some(2)); + // Neither a stall nor a future-only queue invents a new picture. + assert_eq!(reader.present([], now + Duration::from_secs(1)), None); + assert_eq!(reader.present([ms(200)], now + Duration::from_millis(100)), None); + } + + #[tokio::test] + async fn replacing_a_broadcast_isolates_its_old_update_handles() { + tokio::time::pause(); + let clock = Producer::new(DELAY); + let reader = clock.consume(); + let previous = clock.clone(); + let old = clock.track(Kind::Audio); + old.declare(); + let clock = clock.reset(DELAY); + let now = kio::clock::now(); + let audio = clock.track(Kind::Audio); + audio.received(ms(0), now); + audio.declare(); + audio.end(ms(1_000)).unwrap(); + // A JoinSet abort can leave these updates and drops until after the new track started. + drop(old); + previous.track(Kind::Audio).received(ms(10_000), now); + assert_eq!(reader.due(ms(0)), Some(now + DELAY)); + let video = clock.track(Kind::Video); + let transition = video.declare(); + video.plan(transition, ms(0), None); + let resumed = kio::clock::timeout( + DELAY + Duration::from_millis(500), + video.resume(transition, ms(0), None).ready(), + ) + .await; + assert!(resumed.is_err(), "old handle cleared the new audio tail"); + } + + #[tokio::test] + async fn ordinary_arrivals_leave_a_parked_resume_asleep() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::task::{Wake, Waker}; + struct Wakes(AtomicUsize); + impl Wake for Wakes { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::Relaxed); + } + } + tokio::time::pause(); + let clock = Producer::new(DELAY); + let now = kio::clock::now(); + let audio = clock.track(Kind::Audio); + audio.received(ms(0), now); + audio.end(ms(1_000)).unwrap(); + let video = clock.track(Kind::Video); + let transition = video.declare(); + video.plan(transition, ms(0), None); + let wakes = Arc::new(Wakes(AtomicUsize::new(0))); + let waiter = kio::Waiter::new(Waker::from(wakes.clone())); + let mut resume = video.resume(transition, ms(0), None); + assert!(resume.poll_ready(&waiter).is_pending()); + let later = now + Duration::from_millis(20); + video.received(ms(20), later); + audio.received(ms(20), later); + assert_eq!(wakes.0.load(Ordering::Relaxed), 0); + audio.received(ms(40), later); + assert!( + wakes.0.load(Ordering::Relaxed) > 0, + "a moved schedule must wake the resume" + ); + } + + #[test] + fn discarded_replacements_release_their_transition_state() { + let clock = Arc::new(Clock::new(Duration::ZERO)); + let video = Track::new(clock.clone(), Kind::Video); + let audio = Track::new(clock.clone(), Kind::Audio); + let first = audio.declare(); + assert_eq!(first, video.declare()); + // Audio can hold its declared end while video discards any number of replacements. + for _ in 0..100 { + let transition = video.declare(); + video.plan(transition, Timestamp::ZERO, None); + video.abandon(transition); + assert_eq!(clock.state.lock().unwrap().pending(), 1); + } + video.abandon(first); + drop(audio); + assert_eq!(clock.state.lock().unwrap().pending(), 0); + } + + #[tokio::test] + async fn a_data_track_joins_a_paired_restart() { + tokio::time::pause(); + let now = kio::clock::now(); + let (clock, audio, video) = anchored(now); + let data = Track::new(clock.clone(), Kind::Data); + let transition = audio.declare(); + assert_eq!(video.declare(), transition); + assert_eq!(data.declare(), transition, "one restart"); + audio.plan(transition, ms(0), Some(DELAY)); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY).await; + video.apply( + transition, + video.resume(transition, ms(40), None).ready().await.unwrap(), + ); + // The data track has no replacement yet, so the others map alone. + assert_eq!(video.map(ms(40)).unwrap().as_millis(), 10_100); + data.plan(transition, ms(20), None); + data.apply(transition, data.resume(transition, ms(20), None).ready().await.unwrap()); + assert_eq!(data.map(ms(40)).unwrap(), video.map(ms(40)).unwrap()); + } + + #[tokio::test] + async fn one_ended_role_maps_to_the_continuous_peers_clock() { + tokio::time::pause(); + let start = kio::clock::now(); + let clock = Arc::new(Clock::new(DELAY)); + let audio = Track::new(clock.clone(), Kind::Audio); + let video = Track::new(clock.clone(), Kind::Video); + video.received(ms(10_000), start); + let transition = audio.declare(); + audio.end(ms(10_020)).unwrap(); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + let unchanged = clock.state.lock().unwrap().due(ms(10_500)); + audio.apply( + transition, + audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), + ); + assert_eq!(audio.map(ms(0)).unwrap().as_millis(), 10_120); + assert_eq!(video.map(ms(10_500)).unwrap().as_millis(), 10_500); + assert_eq!(clock.state.lock().unwrap().due(ms(10_500)), unchanged); + } + + /// A replacement that arrived while the old tail played waits for it, so it + /// plays later than it arrived. Its arrivals are on time for that schedule, + /// not early: they must not drag the other tracks ahead. + #[tokio::test] + async fn media_held_behind_a_tail_leaves_the_schedule_alone() { + tokio::time::pause(); + let start = kio::clock::now(); + let (clock, audio, video) = anchored(start); + video.received(ms(10_000), start); + let transition = audio.declare(); + audio.end(ms(10_100)).unwrap(); + // The replacement starts over at zero and lands at once. + audio.plan(transition, ms(0), Some(DELAY)); + let mapping = audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(); + audio.apply(transition, mapping); + assert!(kio::clock::now() >= start + DELAY * 2, "the replacement waited for the tail"); + let unchanged = clock.state.lock().unwrap().due(ms(10_500)); + for frame in 0..10 { + let at = start + Duration::from_millis(frame * 20); + assert!( + !audio.received(audio.map(ms(frame * 20)).unwrap(), at), + "held media moved the schedule" + ); + } + assert_eq!(clock.state.lock().unwrap().due(ms(10_500)), unchanged); + } + + #[tokio::test] + async fn a_late_paired_replacement_keeps_its_media_offset() { + tokio::time::pause(); + let now = kio::clock::now(); + let (_clock, audio, video) = anchored(now); + let transition = audio.declare(); + assert_eq!(video.declare(), transition, "one restart"); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + audio.plan(transition, ms(0), Some(DELAY)); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + audio.apply( + transition, + audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), + ); + // The video replacement lands after the resumed audio has started playing. + tokio::time::advance(DELAY * 2).await; + video.plan(transition, ms(40), None); + video.apply( + transition, + video.resume(transition, ms(40), None).ready().await.unwrap(), + ); + assert_eq!( + video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + } + + #[tokio::test] + async fn a_late_replacement_pairs_with_its_own_transition() { + tokio::time::pause(); + let now = kio::clock::now(); + let (_clock, audio, video) = anchored(now); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.apply( + transition, + video.resume(transition, ms(40), None).ready().await.unwrap(), + ); + // Video already queued its next replacement when audio's first one lands. + let next = video.declare(); + assert_ne!(next, transition); + video.end(ms(60)).unwrap(); + video.plan(next, ms(5_000), None); + audio.plan(transition, ms(0), Some(DELAY)); + audio.apply( + transition, + audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), + ); + assert_eq!( + video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + } + + #[tokio::test] + async fn a_peer_two_restarts_ahead_pairs_with_the_first() { + tokio::time::pause(); + let now = kio::clock::now(); + let (_clock, audio, video) = anchored(now); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.plan(transition, ms(40), None); + let first = video.resume(transition, ms(40), None).ready().await.unwrap(); + video.apply(transition, first); + // Video restarts again and resumes before audio's first replacement lands. + let next = video.declare(); + video.plan(next, ms(5_000), None); + tokio::time::advance(Duration::from_millis(10)).await; + video.apply(next, video.resume(next, ms(5_000), None).ready().await.unwrap()); + audio.plan(transition, ms(0), Some(DELAY)); + audio.apply( + transition, + audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), + ); + assert_eq!( + first.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + } + + #[tokio::test] + async fn an_abandoned_replacement_leaves_the_next_transition_alone() { + tokio::time::pause(); + let now = kio::clock::now(); + let (clock, audio, video) = anchored(now); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + // Video's first replacement is evicted before it plays, then it restarts again. + video.plan(transition, ms(40), None); + video.abandon(transition); + let next = video.declare(); + video.plan(next, ms(5_000), None); + audio.plan(transition, ms(0), Some(DELAY)); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + audio.apply( + transition, + audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), + ); + video.apply(next, video.resume(next, ms(5_000), None).ready().await.unwrap()); + // The later replacement maps onto the running clock, not the audio restart's. + let resumed = kio::clock::now() + DELAY; + assert_eq!( + clock.state.lock().unwrap().due(video.map(ms(5_000)).unwrap()), + Some(resumed) + ); + } + + #[tokio::test] + async fn an_abandoned_peer_plan_is_not_paired() { + tokio::time::pause(); + let now = kio::clock::now(); + let (clock, audio, video) = anchored(now); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + audio.plan(transition, ms(0), Some(Duration::from_millis(80))); + audio.abandon(transition); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.apply( + transition, + video.resume(transition, ms(40), None).ready().await.unwrap(), + ); + let resumed = kio::clock::now() + DELAY; + assert_eq!( + clock.state.lock().unwrap().due(video.map(ms(40)).unwrap()), + Some(resumed) + ); + } + + #[tokio::test] + async fn a_resumed_transition_ignores_the_peers_later_end() { + tokio::time::pause(); + let now = kio::clock::now(); + let (_clock, audio, video) = anchored(now); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + let mapping = video.resume(transition, ms(40), None).ready().await.unwrap(); + video.apply(transition, mapping); + // The resumed video plays, then its encoder stops again far ahead. + tokio::time::advance(DELAY * 2).await; + video.declare(); + video.end(ms(60_000)).unwrap(); + audio.plan(transition, ms(0), Some(DELAY)); + let resumed = kio::clock::timeout( + Duration::from_millis(1), + audio.resume(transition, ms(0), Some(DELAY)).ready(), + ) + .await; + assert!(resumed.is_ok(), "audio waited for video's next end"); + } + + #[tokio::test] + async fn a_cancelled_role_releases_its_declared_end() { + tokio::time::pause(); + let now = kio::clock::now(); + let (_clock, audio, video) = anchored(now); + // Video holds the schedule once audio goes. + video.received(ms(10_000), now); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + // Cancelled with its sink, so nothing plays the tail. + drop(audio); + let transition = video.declare(); + video.end(ms(10_000)).unwrap(); + video.plan(transition, ms(0), None); + let resume = kio::clock::timeout( + DELAY + Duration::from_millis(150), + video.resume(transition, ms(0), None).ready(), + ) + .await; + assert!(resume.is_ok(), "video waited for discarded audio"); + } + + #[tokio::test] + async fn a_closed_role_still_holds_its_declared_end() { + tokio::time::pause(); + let now = kio::clock::now(); + let (_clock, audio, video) = anchored(now); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + // The track closed, but its returned sink still plays the tail up to the end. + audio.retire(Some(now + Duration::from_millis(300))); + let transition = video.declare(); + video.end(ms(10_000)).unwrap(); + video.plan(transition, ms(0), None); + let resume = kio::clock::timeout( + DELAY + Duration::from_millis(150), + video.resume(transition, ms(0), None).ready(), + ) + .await; + assert!(resume.is_err(), "video resumed before the audio tail played"); + } + + #[tokio::test] + async fn a_retired_end_ignores_a_later_re_pin() { + tokio::time::pause(); + let now = kio::clock::now(); + let (clock, audio, video) = anchored(now); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + audio.retire(Some(now + Duration::from_millis(300))); + // A replacement rendition starts over at zero and anchors afresh. + let successor = Track::new(clock.clone(), Kind::Audio); + successor.received(ms(50), now); + let transition = video.declare(); + video.plan(transition, ms(0), None); + let resume = kio::clock::timeout( + Duration::from_millis(400), + video.resume(transition, ms(0), None).ready(), + ) + .await; + assert!(resume.is_ok(), "video waited past the drained tail"); + } + + #[tokio::test] + async fn a_cancelled_successor_keeps_a_retired_tail() { + tokio::time::pause(); + let now = kio::clock::now(); + let (clock, audio, video) = anchored(now); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + audio.retire(Some(now + Duration::from_millis(300))); + // The next rendition fails while the retired sink still plays. + let successor = Track::new(clock.clone(), Kind::Audio); + successor.declare(); + drop(successor); + let transition = video.declare(); + video.plan(transition, ms(0), None); + let resume = kio::clock::timeout( + Duration::from_millis(250), + video.resume(transition, ms(0), None).ready(), + ) + .await; + assert!(resume.is_err(), "video resumed over the retired audio tail"); + } + + #[tokio::test] + async fn paired_plans_keep_media_offsets_and_follow_the_earliest_plan() { + tokio::time::pause(); + let now = kio::clock::now(); + let (clock, audio, video) = anchored(now); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + audio.plan(transition, ms(0), Some(Duration::from_millis(80))); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.apply( + transition, + video.resume(transition, ms(40), None).ready().await.unwrap(), + ); + audio.apply( + transition, + audio + .resume(transition, ms(0), Some(Duration::from_millis(80))) + .ready() + .await + .unwrap(), + ); + assert_eq!( + video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + assert_eq!( + clock.state.lock().unwrap().due(audio.map(ms(0)).unwrap()), + Some(now + Duration::from_millis(200)) + ); + } +} diff --git a/rs/moq-mux/src/timeline.rs b/rs/moq-mux/src/timeline.rs deleted file mode 100644 index 0e4ff14a20d..00000000000 --- a/rs/moq-mux/src/timeline.rs +++ /dev/null @@ -1,1501 +0,0 @@ -//! One playout clock shared by a broadcast's audio and video. -//! -//! Feed video arrivals and the audio device's buffered position to a [`Producer`]. -//! Give its [`Consumer`] to a presenter: [`Consumer::present`] selects the newest -//! due picture without waiting, so a canvas can repeat its previous picture. -//! Audio owns the anchor while it plays; without audio, video uses the delay -//! passed to [`Producer::new`]. The audio jitter target comes from the caller, -//! keeping this module independent of decoders and output devices. -//! -//! A [`Track`] maps encoder segments onto the clock across declared ends. Keep -//! one active track per [`Kind`]; retire it before starting its successor. Bound -//! replacement media with the caller's retention budget and [`Track::abandon`] -//! transitions whose replacement is discarded. - -use kio::clock::Instant; -use std::time::Duration; - -use moq_net::Timestamp; -use std::{ - pin::Pin, - sync::{Arc, Mutex}, - task::Poll, -}; - -struct Clock { - state: Mutex, - changed: kio::Shared<()>, -} - -impl Clock { - fn new(delay: Duration) -> Self { - Self { - state: Mutex::new(State::new(delay)), - changed: Default::default(), - } - } - - fn wake(&self) { - *self.changed.lock() = (); - } -} - -/// The playout clock: one anchor the window and the speaker both present -/// against. -/// -/// Every arriving frame is folded into a single [`Pacer`](crate::Pacer), which -/// maps media time onto the wall clock and holds each result back by the -/// configured delay. That is what pulls playback toward live: a frame arriving -/// earlier than the anchor predicted re-pins it, and the delay is measured from -/// the edge it just set rather than from wherever the first frame happened to -/// land. -/// -/// While audio is playing it is the only half allowed to move the anchor. The -/// speaker drains on its own clock and cannot skip forward with a re-anchor, so -/// a video tune-in burst re-anchoring would leave the picture ahead of the -/// sound. Video reads the anchor and follows it. -struct State { - pacer: crate::Pacer, - /// How far behind the live edge playback runs. - delay: Duration, - /// Whether the speaker owns the anchor. - speaker: bool, - /// Set when the speaker restarts on a new timeline mid-track, so the next - /// audio frame pins the anchor outright instead of pacing against media it - /// will never play. - restart: bool, - /// Endpoints declared by each live role, on the shared playback timeline. - ends: [Option; 2], - /// When each retired role's tail finishes playing, frozen on the wall clock: - /// a later rendition re-pins the anchor, which would carry a media endpoint - /// onto an unrelated timeline. Kept apart from `ends` so a successor role - /// never clears its predecessor's tail. - tails: Vec, - /// Encoder restarts whose replacements have not all started playing. - transitions: std::collections::VecDeque, - /// The latest transition each role declared. - declared: [Transition; 2], - next: Transition, - reference: Option, - /// The last speaker write: the media time it ends at, when that sounds, and - /// the playhead it may not fall back behind on the same timeline. - written: Option<(Duration, Instant, Option)>, - /// The playout target the speaker holds, once audio is playing. - target: Option, -} - -/// One encoder restart, shared by the roles whose ends declared it. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct Transition(u64); - -struct Pending { - id: Transition, - /// Roles whose declared end opened or joined this transition. - members: [bool; 2], - /// Members that applied its mapping. - resumed: [bool; 2], - /// Each member's first replacement media. - plans: [Option; 2], - mapping: Option, -} - -struct Boundary { - starts: Option, - end: Duration, -} - -#[derive(Clone, Copy)] -struct Plan { - source: Duration, - delay: Duration, -} - -impl State { - fn new(delay: Duration) -> Self { - Self { - pacer: crate::Pacer::default().with_delay(delay), - delay, - speaker: false, - restart: false, - ends: Default::default(), - tails: Vec::new(), - transitions: Default::default(), - declared: [Transition(0); 2], - next: Transition(1), - reference: None, - written: None, - target: None, - } - } - - /// How many transitions are still pending. - #[cfg(test)] - fn pending(&self) -> usize { - self.transitions.len() - } - - fn transition(&mut self, id: Transition) -> Option<&mut Pending> { - self.transitions.iter_mut().find(|pending| pending.id == id) - } - - /// Forget transitions nobody resumes, or that every member resumed once - /// their replacement started and nothing can join them. - fn prune(&mut self, now: Instant) { - self.transitions.retain(|pending| { - let members = pending.members.iter().any(|member| *member); - let resumed = (0..2).all(|role| !pending.members[role] || pending.resumed[role]); - let started = pending.mapping.is_some_and(|mapping| mapping.started(now)); - members && !(resumed && started) - }); - } - - /// The media position the shared clock will reach at `at`. - fn position(&self, at: Instant) -> Option { - let reference = self.reference?; - let due = self.due(reference)?; - let reference = Duration::from(reference); - Some(if at >= due { - reference.saturating_add(at.duration_since(due)) - } else { - reference.saturating_sub(due.duration_since(at)) - }) - } - - /// Fold an arriving video frame into the anchor, unless the speaker owns it, - /// reporting whether that moved the schedule. - /// - /// A move makes every queued frame due earlier, so the window has to recompute - /// the deadline it is already asleep on. - fn video(&mut self, timestamp: Timestamp, now: Instant) -> bool { - if self.speaker { - return false; - } - - self.reference = Some(timestamp); - let before = self.pacer.due(timestamp); - before != Some(self.pacer.pace(timestamp, now)) - } - - /// Start a new video timeline without taking ownership from a speaker. - fn video_restarted(&mut self) { - if !self.speaker { - self.pacer = crate::Pacer::default().with_delay(self.delay); - } - } - - /// Fold the speaker's position into the anchor, reporting whether that moved - /// the schedule. - /// - /// `end` is the media time of the last sample written and `buffered` is how - /// much of the write is still queued, so `end` does not sound until - /// `buffered` from now. The anchor is pinned at the instant that leaves - /// exactly the delay before it does, which is where the live edge sits on a - /// clock trailing by that delay. Pinning the sample sounding *now* instead - /// would schedule video a whole delay behind the speaker. - /// - /// A move makes every queued video frame due earlier, so the window has to - /// recompute the deadline it is already asleep on. - fn audio(&mut self, end: Duration, buffered: Duration, now: Instant) -> bool { - // Silence padded ahead of the frame to refill the speaker is not media: the - // playhead holds where it was until the padding has played, rather than - // projecting back over media already heard. A new timeline starts afresh. - let floor = (self.speaker && !self.restart).then(|| self.playhead(now)).flatten(); - self.written = now.checked_add(buffered).map(|sounds| (end, sounds, floor)); - let timestamp = media(end); - self.reference = Some(timestamp); - let edge = now - .checked_add(buffered) - .and_then(|at| at.checked_sub(self.delay)) - .unwrap_or(now); - - // Taking ownership re-pins outright rather than pacing. Whatever set the - // anchor before is either video, which the speaker cannot follow, or a - // track that has since ended, and a timestamp that maps into the past under - // that anchor would never re-anchor on its own: the sink would play now - // while the picture stayed on a timeline nothing is feeding. - let before = self.pacer.due(timestamp); - let after = if !self.speaker || std::mem::take(&mut self.restart) { - self.pacer.hurry(timestamp, edge) - } else { - self.pacer.pace(timestamp, edge) - }; - self.speaker = true; - - before != Some(after) - } - - /// The speaker restarted on a new timeline, so the media either side of the - /// break is unrelated: re-pin the anchor on the next audio frame rather than - /// pacing across a jump the speaker never plays. - fn restarted(&mut self) { - self.restart = true; - } - - /// The speaker went quiet, so nothing holds playback to its cadence any - /// more and video takes the anchor back. - /// - /// A replacement rendition takes ownership again on its first frame, and - /// re-pins there: it is a track boundary, so its timestamps are not - /// necessarily continuous with what just ended. That covers a pending - /// restart, which would otherwise re-pin the frame after it too. - fn stopped(&mut self) { - self.speaker = false; - self.restart = false; - self.written = None; - self.target = None; - } - - /// Record the playout target the speaker is being held on. - fn hold(&mut self, target: Duration) { - self.target = Some(target); - } - - /// The media time sounding at `now`, or `None` with no speaker. - /// - /// Projected from the last write at the wall rate, held at its end (a speaker - /// with nothing left plays nothing further), and never behind where it stood - /// at that write (refill padding plays no media). - fn playhead(&self, now: Instant) -> Option { - let (end, sounds, floor) = self.written?; - let projected = end.saturating_sub(sounds.saturating_duration_since(now)); - Some(floor.map_or(projected, |floor| projected.max(floor)).min(end)) - } - - /// The playout target the speaker is held on, or `None` with no speaker. - fn target(&self) -> Option { - self.target - } - - /// When `timestamp` should be presented, or `None` before any frame has - /// anchored the clock. `None` means as soon as possible: there is nothing - /// left to wait for. - fn due(&self, timestamp: Timestamp) -> Option { - self.pacer.due(timestamp) - } -} - -/// Maps each encoder segment onto the shared playback clock. -pub struct Track { - presentation: Arc, - role: Kind, - mapping: std::sync::Mutex, -} - -/// A segment's timestamp mapping, applied only after its first media can play. -#[derive(Clone, Copy)] -#[must_use] -pub struct Mapping { - source: Duration, - target: Duration, - starts: Option, -} - -impl Mapping { - fn map(self, source: Timestamp) -> Result { - let source = Duration::from(source); - let target = if source >= self.source { - self.target.checked_add(source - self.source).ok_or(Error::Overflow)? - } else { - self.target - .checked_sub(self.source - source) - .ok_or(Error::BeforeOrigin)? - }; - Timestamp::from_micros(target.as_micros().try_into().map_err(|_| Error::Overflow)?).map_err(|_| Error::Overflow) - } - - fn started(&self, now: Instant) -> bool { - self.starts.is_none_or(|starts| starts <= now) - } -} - -impl Track { - fn new(presentation: Arc, role: Kind) -> Self { - Self { - presentation, - role, - mapping: std::sync::Mutex::new(Mapping { - source: Duration::ZERO, - target: Duration::ZERO, - starts: None, - }), - } - } - - /// Stop while a tail still plays until `drained`, holding a declared end until - /// then. Dropping instead discards the tail with the end. - pub fn retire(self, drained: Option) { - let now = kio::clock::now(); - let mut presentation = self.presentation.state.lock().unwrap(); - presentation.tails.retain(|tail| *tail > now); - if presentation.ends[self.role as usize].take().is_some() - && let Some(drained) = drained - { - presentation.tails.push(drained); - } - } - - /// Map a source timestamp onto the shared broadcast timeline. - pub fn map(&self, source: Timestamp) -> Result { - self.mapping.lock().unwrap().map(source) - } - - /// Declare that this role's encoder stopped, returning the transition its - /// replacement resumes. - /// - /// A peer's transition this role has not joined yet is the same restart, as - /// long as its replacement has not started playing; otherwise this one opens - /// a transition of its own. - pub fn declare(&self) -> Transition { - let now = kio::clock::now(); - let own = self.role as usize; - let mut presentation = self.presentation.state.lock().unwrap(); - presentation.prune(now); - let after = presentation.declared[own]; - let joined = presentation - .transitions - .iter_mut() - .filter(|transition| transition.id > after) - .find(|transition| { - transition.members[1 - own] - && !transition.members[own] - && transition.mapping.is_none_or(|mapping| !mapping.started(now)) - }) - .map(|transition| { - transition.members[own] = true; - transition.id - }); - let id = joined.unwrap_or_else(|| { - let id = presentation.next; - presentation.next = Transition(id.0 + 1); - let mut members = [false; 2]; - members[own] = true; - presentation.transitions.push_back(Pending { - id, - members, - resumed: [false; 2], - plans: [None, None], - mapping: None, - }); - id - }); - presentation.declared[own] = id; - drop(presentation); - self.presentation.wake(); - id - } - - /// Mark where this role's current segment ends on the shared clock. - pub fn end(&self, source: Timestamp) -> Result<()> { - let mapping = *self.mapping.lock().unwrap(); - self.presentation.state.lock().unwrap().ends[self.role as usize] = Some(Boundary { - starts: mapping.starts, - end: Duration::from(mapping.map(source)?), - }); - self.presentation.wake(); - Ok(()) - } - - /// Record the first replacement media for `transition`. - pub fn plan(&self, transition: Transition, source: Timestamp, delay: Option) { - let mut presentation = self.presentation.state.lock().unwrap(); - let delay = delay.unwrap_or(presentation.delay); - if let Some(pending) = presentation.transition(transition) { - pending.plans[self.role as usize] = Some(Plan { - source: Duration::from(source), - delay, - }); - } - drop(presentation); - self.presentation.wake(); - } - - /// Media was lost: nothing waits for the discarded tail. - pub fn gap(&self) { - self.presentation.state.lock().unwrap().ends[self.role as usize] = None; - self.presentation.wake(); - } - - /// This role will never resume `transition`, so a peer must not pair with its plan. - pub fn abandon(&self, transition: Transition) { - let own = self.role as usize; - let mut presentation = self.presentation.state.lock().unwrap(); - if let Some(pending) = presentation.transition(transition) { - pending.members[own] = false; - pending.plans[own] = None; - } - presentation.prune(kio::clock::now()); - drop(presentation); - self.presentation.wake(); - } - - /// Commit the mapping once its replacement media starts playing. - pub fn apply(&self, transition: Transition, mapping: Mapping) { - *self.mapping.lock().unwrap() = mapping; - let mut presentation = self.presentation.state.lock().unwrap(); - if let Some(pending) = presentation.transition(transition) { - pending.resumed[self.role as usize] = true; - } - presentation.prune(kio::clock::now()); - drop(presentation); - self.presentation.wake(); - } - - /// Prepare a mapping after the old media drains; dropping the wait leaves the track unchanged. - pub fn resume(&self, transition: Transition, source: Timestamp, delay: Option) -> Resume<'_> { - Resume { - track: self, - transition, - source, - delay, - timer: None, - } - } -} - -impl Drop for Track { - fn drop(&mut self) { - let own = self.role as usize; - { - let mut presentation = self.presentation.state.lock().unwrap(); - presentation.ends[own] = None; - for pending in presentation - .transitions - .iter_mut() - .filter(|pending| !pending.resumed[own]) - { - pending.members[own] = false; - pending.plans[own] = None; - } - presentation.declared[own] = Transition(0); - presentation.prune(kio::clock::now()); - } - self.presentation.wake(); - } -} - -/// A duration from the start of the track as a wire timestamp. -fn media(duration: Duration) -> Timestamp { - // A timestamp caps at 2^61 - 1, which is ~73,000 years of microseconds. - const MAX: u128 = (1 << 61) - 1; - Timestamp::from_micros(duration.as_micros().min(MAX) as u64).expect("clamped to the timestamp maximum") -} - -/// A pending segment mapping, driven by the caller's executor or waiter. -#[must_use = "drive the wait before applying its mapping"] -pub struct Resume<'a> { - track: &'a Track, - transition: Transition, - source: Timestamp, - delay: Option, - timer: Option, -} - -impl Resume<'_> { - /// The replacement's mapping, once all declared old media has drained. - #[cfg_attr(feature = "async", kio::wrapper)] - pub fn poll_ready(&mut self, waiter: &kio::Waiter) -> Poll> { - let _ = self.track.presentation.changed.poll(waiter, |_| Poll::Pending); - let now = kio::clock::now(); - let mut presentation = self.track.presentation.state.lock().unwrap(); - if let Some(mapping) = presentation - .transition(self.transition) - .and_then(|pending| pending.mapping) - { - return Poll::Ready(Ok(mapping)); - } - let end = presentation - .ends - .iter() - .flatten() - .filter(|boundary| boundary.starts.is_none_or(|start| start <= now)) - .filter_map(|boundary| presentation.due(media(boundary.end))) - .chain(presentation.tails.iter().copied()) - .max(); - if let Some(end) = end.filter(|end| *end > now) { - let timer = self.timer.get_or_insert_with(|| kio::clock::sleep_until(end)); - if timer.deadline() != end { - timer.reset(end); - } - std::task::ready!(waiter.poll_future(Pin::new(timer))); - } - - let peer = 1 - self.track.role as usize; - // A transition abandoned under this resume, by a gap the caller rechecks, - // maps alone like one the peer never joined. - let (mapping, paired) = presentation - .transition(self.transition) - .map_or((None, None), |pending| { - let both = pending.members[peer] && pending.plans.iter().all(Option::is_some); - (pending.mapping, pending.plans[Kind::Audio as usize].filter(|_| both)) - }); - // The peer already resumed this transition. - if let Some(mapping) = mapping { - return Poll::Ready(Ok(mapping)); - } - // Both roles have replacement media. The speaker's first sample defines - // their common origin even when its adaptive delay differs from video's. - // Otherwise this role maps alone and a peer joining later follows it. - let (source, delay) = match paired { - Some(audio) => (audio.source, audio.delay), - None => (Duration::from(self.source), self.delay.unwrap_or(presentation.delay)), - }; - let starts = now.checked_add(delay).ok_or(Error::DelayOverflow)?; - let mapping = Mapping { - source, - target: presentation.position(starts).unwrap_or(source), - starts: Some(starts), - }; - if let Some(pending) = presentation.transition(self.transition) { - pending.mapping = Some(mapping); - } - Poll::Ready(Ok(mapping)) - } -} - -/// Whether a track feeds the picture or the speaker. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Kind { - /// A picture track. - Video, - /// A sound track. - Audio, -} - -/// A timestamp or delay cannot be represented on the playout clock. -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -#[non_exhaustive] -pub enum Error { - /// Mapping the timestamp would overflow. - #[error("playback timeline overflow")] - Overflow, - /// A picture precedes the mapped segment's origin. - #[error("picture precedes the playback timeline")] - BeforeOrigin, - /// The requested delay exceeds the system clock. - #[error("playback delay exceeds the system clock")] - DelayOverflow, -} - -/// A result from mapping media onto the playout clock. -pub type Result = std::result::Result; - -/// The speaker's last write and caller-selected jitter target. -#[derive(Clone, Copy, Debug)] -pub struct Audio { - /// Media time at the end of the last write. - pub end: Duration, - /// How long until the end of that write sounds. - pub buffered: Duration, - /// The jitter target used to fill the speaker's buffer. - pub target: Duration, -} - -/// Updates one broadcast's playout clock, shared by audio and video tasks. -/// -/// Clones capture this broadcast's clock; resetting a handle never changes the -/// clock its previous clones or track handles update. -#[derive(Clone)] -pub struct Producer { - clock: Arc, - current: Arc>>, -} - -/// Reads the current playout schedule without moving its anchor. -/// -/// Keeps the final schedule readable after update handles drop, and follows -/// the replacement clock when a producer resets for another broadcast. -#[derive(Clone)] -pub struct Consumer { - current: Arc>>, -} - -impl Producer { - /// Create a broadcast clock with this video-only delay. - pub fn new(delay: Duration) -> Self { - let clock = Arc::new(Clock::new(delay)); - Self { - clock: clock.clone(), - current: Arc::new(Mutex::new(clock)), - } - } - - /// Replace this producer with a new broadcast clock and switch readers to it. - /// - /// Previous clones and track handles retain their old broadcast's clock. - #[must_use = "use the returned producer for the replacement broadcast"] - pub fn reset(self, delay: Duration) -> Self { - let clock = Arc::new(Clock::new(delay)); - *self.current.lock().unwrap() = clock.clone(); - Self { - clock, - current: self.current, - } - } - - /// Return a reader that follows this handle's broadcast replacements. - pub fn consume(&self) -> Consumer { - Consumer { - current: self.current.clone(), - } - } - - /// Return a track handle bound to this broadcast's clock. - pub fn track(&self, kind: Kind) -> Track { - Track::new(self.clock.clone(), kind) - } - - /// Fold an arriving video frame into the anchor, unless the speaker owns it, - /// reporting whether that moved the schedule. - /// - /// A move makes every queued frame due earlier, so the window has to recompute - /// the deadline it is already asleep on. - pub fn video(&self, timestamp: Timestamp, now: Instant) -> bool { - let clock = &self.clock; - let moved = clock.state.lock().unwrap().video(timestamp, now); - if moved { - clock.wake(); - } - moved - } - - /// Start a new video timeline without taking ownership from a speaker. - pub fn video_restarted(&self) { - let clock = &self.clock; - let moved = { - let mut state = clock.state.lock().unwrap(); - if state.speaker { - false - } else { - state.video_restarted(); - true - } - }; - if moved { - clock.wake(); - } - } - - /// Fold the speaker's position into the anchor, reporting whether that moved - /// the schedule. - /// - /// `audio.end` is the media time of the last sample written and `audio.buffered` is how - /// much of the write is still queued, so `end` does not sound until - /// `buffered` from now. The anchor is pinned at the instant that leaves - /// exactly the delay before it does, which is where the live edge sits on a - /// clock trailing by that delay. Pinning the sample sounding *now* instead - /// would schedule video a whole delay behind the speaker. - /// - /// A move makes every queued video frame due earlier, so the window has to - /// recompute the deadline it is already asleep on. - pub fn audio(&self, audio: Audio, now: Instant) -> bool { - let clock = &self.clock; - let moved = { - let mut state = clock.state.lock().unwrap(); - state.hold(audio.target); - state.audio(audio.end, audio.buffered, now) - }; - if moved { - clock.wake(); - } - moved - } - - /// The speaker restarted on a new timeline, so the media either side of the - /// break is unrelated: re-pin the anchor on the next audio frame rather than - /// pacing across a jump the speaker never plays. - pub fn audio_restarted(&self) { - self.clock.state.lock().unwrap().restarted() - } - - /// The speaker went quiet, so nothing holds playback to its cadence any - /// more and video takes the anchor back. - /// - /// A replacement rendition takes ownership again on its first frame, and - /// re-pins there: it is a track boundary, so its timestamps are not - /// necessarily continuous with what just ended. That covers a pending - /// restart, which would otherwise re-pin the frame after it too. - pub fn audio_stopped(&self) { - self.clock.state.lock().unwrap().stopped() - } - - /// When `timestamp` should be presented, or `None` before any frame has - /// anchored the clock. `None` means as soon as possible: there is nothing - /// left to wait for. - pub fn due(&self, timestamp: Timestamp) -> Option { - self.clock.state.lock().unwrap().due(timestamp) - } -} - -impl Consumer { - /// The media time sounding now, held at the last audio write's end. - pub fn playhead(&self, now: Instant) -> Option { - self.current.lock().unwrap().state.lock().unwrap().playhead(now) - } - - /// The caller-supplied audio playout target, or `None` without audio. - pub fn target(&self) -> Option { - self.current.lock().unwrap().state.lock().unwrap().target() - } - - /// When this timestamp is due, or `None` before the first anchor. - pub fn due(&self, timestamp: Timestamp) -> Option { - self.current.lock().unwrap().state.lock().unwrap().due(timestamp) - } - - /// The newest due picture's index in presentation order, or `None` to repeat the previous one. - /// - /// Pass queued timestamps in ascending order and discard entries through the - /// returned index after showing that picture. This stores no frames and leaves - /// their retention and backpressure to the caller. Query again after every - /// anchor update; a cached deadline can become stale. - pub fn present(&self, timestamps: impl IntoIterator, now: Instant) -> Option { - let clock = self.current.lock().unwrap(); - let state = clock.state.lock().unwrap(); - timestamps - .into_iter() - .take_while(|timestamp| state.due(*timestamp).is_none_or(|at| at <= now)) - .count() - .checked_sub(1) - } -} - -#[cfg(test)] -mod tests { - use super::*; - #[tokio::test] - async fn replacing_a_broadcast_isolates_its_old_update_handles() { - tokio::time::pause(); - let clock = Producer::new(DELAY); - let reader = clock.consume(); - let previous = clock.clone(); - let old = clock.track(Kind::Audio); - old.declare(); - let clock = clock.reset(DELAY); - let now = kio::clock::now(); - clock.audio( - Audio { - end: Duration::ZERO, - buffered: DELAY, - target: DELAY, - }, - now, - ); - let audio = clock.track(Kind::Audio); - audio.declare(); - audio.end(ms(1_000)).unwrap(); - // A JoinSet abort can leave these updates and drops until after the new track started. - drop(old); - previous.audio( - Audio { - end: Duration::from_secs(10), - buffered: Duration::ZERO, - target: Duration::ZERO, - }, - now, - ); - assert_eq!(reader.due(ms(0)), Some(now + DELAY)); - let video = clock.track(Kind::Video); - let transition = video.declare(); - video.plan(transition, ms(0), None); - let resumed = kio::clock::timeout( - DELAY + Duration::from_millis(500), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resumed.is_err(), "old handle cleared the new audio tail"); - } - - #[tokio::test] - async fn ordinary_arrivals_leave_a_parked_resume_asleep() { - use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }; - use std::task::{Wake, Waker}; - struct Wakes(AtomicUsize); - impl Wake for Wakes { - fn wake(self: Arc) { - self.0.fetch_add(1, Ordering::Relaxed); - } - } - tokio::time::pause(); - let clock = Producer::new(DELAY); - let now = kio::clock::now(); - clock.audio( - Audio { - end: Duration::ZERO, - buffered: DELAY, - target: DELAY, - }, - now, - ); - let audio = clock.track(Kind::Audio); - audio.end(ms(1_000)).unwrap(); - let video = clock.track(Kind::Video); - let transition = video.declare(); - video.plan(transition, ms(0), None); - let wakes = Arc::new(Wakes(AtomicUsize::new(0))); - let waiter = kio::Waiter::new(Waker::from(wakes.clone())); - let mut resume = video.resume(transition, ms(0), None); - assert!(resume.poll_ready(&waiter).is_pending()); - clock.video(ms(20), now); - clock.audio( - Audio { - end: Duration::from_millis(20), - buffered: DELAY, - target: DELAY, - }, - now + Duration::from_millis(20), - ); - assert_eq!(wakes.0.load(Ordering::Relaxed), 0); - clock.audio( - Audio { - end: Duration::from_millis(40), - buffered: DELAY, - target: DELAY, - }, - now + Duration::from_millis(20), - ); - assert!( - wakes.0.load(Ordering::Relaxed) > 0, - "a moved anchor must wake the resume" - ); - } - - #[test] - fn discarded_replacements_release_their_transition_state() { - let presentation = Arc::new(Clock::new(Duration::ZERO)); - let video = Track::new(presentation.clone(), Kind::Video); - let audio = Track::new(presentation.clone(), Kind::Audio); - let first = audio.declare(); - assert_eq!(first, video.declare()); - // Audio can hold its declared end while video discards any number of replacements. - for _ in 0..100 { - let transition = video.declare(); - video.plan(transition, Timestamp::ZERO, None); - video.abandon(transition); - assert_eq!(presentation.state.lock().unwrap().pending(), 1); - } - video.abandon(first); - drop(audio); - assert_eq!(presentation.state.lock().unwrap().pending(), 0); - } - - #[test] - fn presentation_selects_due_frames_on_the_current_audio_anchor() { - let clock = Producer::new(Duration::from_millis(40)); - let video = clock.consume(); - let now = Instant::now(); - clock.video(ms(0), now); - let frames = [ms(0), ms(20), ms(40)]; - assert_eq!(video.present(frames, now), None); - assert_eq!(video.present(frames, now + Duration::from_millis(60)), Some(1)); - // The speaker takes over with 100ms queued; stale video deadlines no longer apply. - clock.audio( - Audio { - end: Duration::from_millis(40), - buffered: Duration::from_millis(100), - target: Duration::from_millis(100), - }, - now, - ); - assert_eq!(video.present(frames, now + Duration::from_millis(60)), Some(0)); - assert_eq!(video.present(frames, now + Duration::from_millis(99)), Some(1)); - assert_eq!(video.present(frames, now + Duration::from_millis(100)), Some(2)); - // Neither a stall nor a future-only queue invents a new picture. - assert_eq!(video.present([], now + Duration::from_secs(1)), None); - assert_eq!(video.present([ms(80)], now + Duration::from_millis(100)), None); - clock.video(ms(1_000), now); - assert_eq!(video.due(ms(40)), Some(now + Duration::from_millis(100))); - } - - const DELAY: Duration = Duration::from_millis(100); - - fn ms(millis: u64) -> Timestamp { - Timestamp::from_millis(millis).unwrap() - } - - /// The playhead runs at the wall rate from the last write and stops at its - /// end, which is what a speaker that ran dry is still sounding. - #[test] - fn the_playhead_projects_the_last_write_and_holds_at_its_end() { - let now = Instant::now(); - let mut presentation = State::new(DELAY); - assert_eq!(presentation.playhead(now), None); - - presentation.audio(Duration::from_secs(10), Duration::from_millis(200), now); - presentation.hold(DELAY); - assert_eq!(presentation.playhead(now), Some(Duration::from_millis(9_800))); - let later = now + Duration::from_millis(150); - assert_eq!(presentation.playhead(later), Some(Duration::from_millis(9_950))); - let dry = now + Duration::from_secs(1); - assert_eq!(presentation.playhead(dry), Some(Duration::from_secs(10))); - assert_eq!(presentation.target(), Some(DELAY)); - - presentation.stopped(); - assert_eq!((presentation.playhead(dry), presentation.target()), (None, None)); - } - - /// Silence padded ahead of a frame after the speaker ran dry plays no media, - /// so the playhead holds instead of replaying media already heard. - #[test] - fn the_playhead_holds_through_refill_padding() { - let now = Instant::now(); - let mut presentation = State::new(DELAY); - presentation.audio(Duration::from_secs(10), Duration::from_millis(200), now); - let dry = now + Duration::from_secs(1); - assert_eq!(presentation.playhead(dry), Some(Duration::from_secs(10))); - - // 100 ms of padding, then a 20 ms frame continuing from 10 s. - presentation.audio(Duration::from_millis(10_020), Duration::from_millis(120), dry); - assert_eq!(presentation.playhead(dry), Some(Duration::from_secs(10))); - let padded = dry + Duration::from_millis(100); - assert_eq!(presentation.playhead(padded), Some(Duration::from_secs(10))); - let played = dry + Duration::from_millis(110); - assert_eq!(presentation.playhead(played), Some(Duration::from_millis(10_010))); - - // A new timeline may start anywhere. - presentation.restarted(); - presentation.audio(Duration::from_secs(2), Duration::from_millis(100), played); - assert_eq!(presentation.playhead(played), Some(Duration::from_millis(1_900))); - } - - #[tokio::test] - async fn one_ended_role_maps_to_the_continuous_peers_clock() { - tokio::time::pause(); - let start = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - presentation.state.lock().unwrap().video(ms(10_000), start); - let transition = audio.declare(); - audio.end(ms(10_020)).unwrap(); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - let unchanged = presentation.state.lock().unwrap().due(ms(10_500)); - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - assert_eq!(audio.map(ms(0)).unwrap().as_millis(), 10_120); - assert_eq!(video.map(ms(10_500)).unwrap().as_millis(), 10_500); - assert_eq!(presentation.state.lock().unwrap().due(ms(10_500)), unchanged); - } - - #[tokio::test] - async fn a_late_paired_replacement_keeps_its_media_offset() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - let transition = audio.declare(); - assert_eq!(video.declare(), transition, "one restart"); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - audio.plan(transition, ms(0), Some(DELAY)); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - // The video replacement lands after the resumed audio has started playing. - tokio::time::advance(DELAY * 2).await; - video.plan(transition, ms(40), None); - video.apply( - transition, - video.resume(transition, ms(40), None).ready().await.unwrap(), - ); - assert_eq!( - video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), - 40 - ); - } - - #[tokio::test] - async fn a_late_replacement_pairs_with_its_own_transition() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - video.plan(transition, ms(40), None); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.apply( - transition, - video.resume(transition, ms(40), None).ready().await.unwrap(), - ); - // Video already queued its next replacement when audio's first one lands. - let next = video.declare(); - assert_ne!(next, transition); - video.end(ms(60)).unwrap(); - video.plan(next, ms(5_000), None); - audio.plan(transition, ms(0), Some(DELAY)); - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - assert_eq!( - video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), - 40 - ); - } - - #[tokio::test] - async fn a_peer_two_restarts_ahead_pairs_with_the_first() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.plan(transition, ms(40), None); - let first = video.resume(transition, ms(40), None).ready().await.unwrap(); - video.apply(transition, first); - // Video restarts again and resumes before audio's first replacement lands. - let next = video.declare(); - video.plan(next, ms(5_000), None); - tokio::time::advance(Duration::from_millis(10)).await; - video.apply(next, video.resume(next, ms(5_000), None).ready().await.unwrap()); - audio.plan(transition, ms(0), Some(DELAY)); - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - assert_eq!( - first.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), - 40 - ); - } - - #[tokio::test] - async fn an_abandoned_replacement_leaves_the_next_transition_alone() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - // Video's first replacement is evicted before it plays, then it restarts again. - video.plan(transition, ms(40), None); - video.abandon(transition); - let next = video.declare(); - video.plan(next, ms(5_000), None); - audio.plan(transition, ms(0), Some(DELAY)); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - video.apply(next, video.resume(next, ms(5_000), None).ready().await.unwrap()); - // The later replacement maps onto the running clock, not the audio restart's. - let resumed = kio::clock::now() + DELAY; - assert_eq!( - presentation.state.lock().unwrap().due(video.map(ms(5_000)).unwrap()), - Some(resumed) - ); - } - - #[tokio::test] - async fn an_abandoned_peer_plan_is_not_paired() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - audio.plan(transition, ms(0), Some(Duration::from_millis(80))); - audio.abandon(transition); - video.plan(transition, ms(40), None); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.apply( - transition, - video.resume(transition, ms(40), None).ready().await.unwrap(), - ); - let resumed = kio::clock::now() + DELAY; - assert_eq!( - presentation.state.lock().unwrap().due(video.map(ms(40)).unwrap()), - Some(resumed) - ); - } - - #[tokio::test] - async fn a_resumed_transition_ignores_the_peers_later_end() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - video.plan(transition, ms(40), None); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - let mapping = video.resume(transition, ms(40), None).ready().await.unwrap(); - video.apply(transition, mapping); - // The resumed video plays, then its encoder stops again far ahead. - tokio::time::advance(DELAY * 2).await; - video.declare(); - video.end(ms(60_000)).unwrap(); - audio.plan(transition, ms(0), Some(DELAY)); - let resumed = kio::clock::timeout( - Duration::from_millis(1), - audio.resume(transition, ms(0), Some(DELAY)).ready(), - ) - .await; - assert!(resumed.is_ok(), "audio waited for video's next end"); - } - - #[tokio::test] - async fn a_cancelled_role_releases_its_declared_end() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - audio.declare(); - audio.end(ms(10_200)).unwrap(); - // Cancelled with its sink, so nothing plays the tail. - drop(audio); - let transition = video.declare(); - video.end(ms(10_000)).unwrap(); - video.plan(transition, ms(0), None); - let resume = kio::clock::timeout( - DELAY + Duration::from_millis(150), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resume.is_ok(), "video waited for discarded audio"); - } - - #[tokio::test] - async fn a_closed_role_still_holds_its_declared_end() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - audio.declare(); - audio.end(ms(10_200)).unwrap(); - // The track closed, but its returned sink still plays the tail up to the end. - audio.retire(Some(now + Duration::from_millis(300))); - let transition = video.declare(); - video.end(ms(10_000)).unwrap(); - video.plan(transition, ms(0), None); - let resume = kio::clock::timeout( - DELAY + Duration::from_millis(150), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resume.is_err(), "video resumed before the audio tail played"); - } - - #[tokio::test] - async fn a_retired_end_ignores_a_later_re_pin() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - audio.declare(); - audio.end(ms(10_200)).unwrap(); - audio.retire(Some(now + Duration::from_millis(300))); - // A replacement rendition starts over at zero and re-pins the anchor. - { - let mut presentation = presentation.state.lock().unwrap(); - presentation.restarted(); - presentation.audio(Duration::from_millis(50), DELAY, now); - } - let transition = video.declare(); - video.plan(transition, ms(0), None); - let resume = kio::clock::timeout( - Duration::from_millis(400), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resume.is_ok(), "video waited past the drained tail"); - } - - #[tokio::test] - async fn a_cancelled_successor_keeps_a_retired_tail() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - audio.declare(); - audio.end(ms(10_200)).unwrap(); - audio.retire(Some(now + Duration::from_millis(300))); - // The next rendition fails while the retired sink still plays. - let successor = Track::new(presentation.clone(), Kind::Audio); - successor.declare(); - drop(successor); - let transition = video.declare(); - video.plan(transition, ms(0), None); - let resume = kio::clock::timeout( - Duration::from_millis(250), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resume.is_err(), "video resumed over the retired audio tail"); - } - - #[tokio::test] - async fn paired_plans_keep_media_offsets_and_follow_the_speakers_delay() { - tokio::time::pause(); - let now = kio::clock::now(); - let presentation = Arc::new(Clock::new(DELAY)); - presentation - .state - .lock() - .unwrap() - .audio(Duration::from_secs(10), DELAY, now); - let audio = Track::new(presentation.clone(), Kind::Audio); - let video = Track::new(presentation.clone(), Kind::Video); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - audio.plan(transition, ms(0), Some(Duration::from_millis(80))); - video.plan(transition, ms(40), None); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.apply( - transition, - video.resume(transition, ms(40), None).ready().await.unwrap(), - ); - audio.apply( - transition, - audio - .resume(transition, ms(0), Some(Duration::from_millis(80))) - .ready() - .await - .unwrap(), - ); - assert_eq!( - video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), - 40 - ); - assert_eq!( - presentation.state.lock().unwrap().due(audio.map(ms(0)).unwrap()), - Some(now + Duration::from_millis(200)) - ); - } - - /// Earlier arrivals pull the anchor forward so playback converges on the live - /// edge without giving up the configured delay. - #[test] - fn a_late_first_frame_catches_up_to_live() { - let start = Instant::now(); - let mut presentation = State::new(DELAY); - - // Produced 500ms ago, though nothing here knows that yet. The window is - // asleep on the deadline the anchor last gave it, so every move is reported. - assert!(presentation.video(ms(0), start), "the first anchor"); - assert_eq!(presentation.due(ms(0)), Some(start + DELAY)); - - // 40ms of media later, but only 20ms of wall clock: the anchor was 480ms - // behind live, so it moves onto this frame. - let now = start + Duration::from_millis(20); - assert!(presentation.video(ms(40), now), "an early frame left the anchor alone"); - assert_eq!(presentation.due(ms(40)), Some(now + DELAY)); - - // A frame that merely arrives late keeps its media instant. The delay is - // the room it has to be late in. - let late = now + Duration::from_millis(70); - assert!(!presentation.video(ms(80), late), "a late arrival moved the anchor"); - assert_eq!(presentation.due(ms(80)), Some(now + Duration::from_millis(140))); - } - - /// The speaker cannot skip forward with a re-anchor, so while it is playing a - /// video burst must not move the anchor out from under it. - #[test] - fn the_speaker_owns_the_anchor_while_it_plays() { - let start = Instant::now(); - let mut presentation = State::new(DELAY); - - presentation.audio(Duration::from_secs(1), DELAY, start); - let anchored = presentation.due(ms(1_000)); - - // A tune-in burst: seconds of video arriving at once. - presentation.video(ms(1_040), start); - presentation.video(ms(4_000), start); - assert_eq!( - presentation.due(ms(1_000)), - anchored, - "video moved the speaker's anchor" - ); - - // Once audio stops, video anchors again. - presentation.stopped(); - presentation.video(ms(4_040), start); - assert_eq!(presentation.due(ms(4_040)), Some(start + DELAY)); - } - - /// The speaker reports the sample sounding now, which is a delay behind the - /// edge. Pinning that instant directly would schedule video a second delay - /// behind the sound. - #[test] - fn the_speaker_anchors_at_the_live_edge() { - let start = Instant::now(); - let mut presentation = State::new(DELAY); - - // The last sample written is a full delay from sounding, which is exactly - // where a settled sink sits: media time and the picture agree. - presentation.audio(Duration::from_secs(1), DELAY, start); - assert_eq!(presentation.due(ms(1_000)), Some(start + DELAY)); - - // Filling up, with only 40ms queued: that sample sounds in 40ms, so the - // picture that goes with it is due then too. - let now = start + Duration::from_millis(500); - presentation.audio(Duration::from_millis(1_500), Duration::from_millis(40), now); - assert_eq!(presentation.due(ms(1_500)), Some(now + Duration::from_millis(40))); - } - - /// Video usually anchors first, and audio's first frame can sit behind that - /// anchor. Pacing it would map it into the past, which never re-anchors, so - /// the sink would play now while the picture stayed on video's timeline. - #[test] - fn the_speaker_re_anchors_when_it_takes_over() { - let start = Instant::now(); - let mut presentation = State::new(DELAY); - presentation.video(ms(4_000), start); - - presentation.audio(Duration::from_secs(1), DELAY, start); - assert_eq!(presentation.due(ms(1_000)), Some(start + DELAY)); - - // A replacement rendition is a track boundary too: its timestamps need not - // continue the one that just ended. - presentation.stopped(); - let now = start + Duration::from_millis(20); - presentation.audio(Duration::from_millis(200), DELAY, now); - assert_eq!(presentation.due(ms(200)), Some(now + DELAY)); - } - - /// A hole too large to play through drops the buffered audio and starts a new - /// sink, and a rewind restarts the timeline outright. Pacing across either - /// would schedule video against samples the speaker never plays. - #[test] - fn a_restarted_speaker_re_anchors() { - let start = Instant::now(); - let mut presentation = State::new(DELAY); - presentation.audio(Duration::from_secs(10), DELAY, start); - - // The publisher rewound: without the restart this pins media 9 seconds - // behind the anchor, leaving the picture 9 seconds in the past. - let now = start + Duration::from_millis(20); - presentation.restarted(); - presentation.audio(Duration::from_secs(1), DELAY, now); - assert_eq!(presentation.due(ms(1_000)), Some(now + DELAY)); - } - - /// A retired rendition marks a restart for its replacement, which may never - /// come. The next track re-pins on its first frame for taking ownership, so a - /// restart left over would re-pin the frame after it onto that frame's jitter. - #[test] - fn a_stopped_speaker_drops_a_pending_restart() { - let start = Instant::now(); - let mut presentation = State::new(DELAY); - presentation.audio(Duration::from_secs(10), DELAY, start); - presentation.restarted(); - presentation.stopped(); - - presentation.audio(Duration::from_secs(1), DELAY, start); - // 20ms of media, 40ms of wall clock: a late frame paces and leaves the - // anchor where it was. - let now = start + Duration::from_millis(40); - presentation.audio(Duration::from_millis(1_020), DELAY, now); - assert_eq!( - presentation.due(ms(1_020)), - Some(start + Duration::from_millis(20) + DELAY) - ); - } - - /// The window sleeps on the deadline it last computed, so an anchor the - /// speaker moved has to say so or the queue presents late by that much. - #[test] - fn a_moved_anchor_is_reported() { - let start = Instant::now(); - let mut presentation = State::new(DELAY); - assert!( - presentation.audio(Duration::from_secs(1), DELAY, start), - "the first anchor" - ); - - // On schedule: 40ms of media, 40ms of wall clock, same sink depth. - let now = start + Duration::from_millis(40); - assert!( - !presentation.audio(Duration::from_millis(1_040), DELAY, now), - "a frame on the anchor moved it" - ); - - // Arriving early re-anchors, and the window has to hear about it. - assert!( - presentation.audio(Duration::from_millis(1_100), DELAY, now), - "an early frame left the anchor alone" - ); - } -} From 843e612e64b427f4ff51bec056c9020b8a8db9ad Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 00:18:14 -0800 Subject: [PATCH 3/4] feat(mux)!: finish the sync port, docs, and delete the mux-sync quest Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 3 +++ doc/lib/rs/moq-mux.md | 5 ++-- quest/m2/README.md | 1 - quest/m2/api-shape/media.md | 2 +- quest/m2/mux-sync.md | 38 ---------------------------- rs/justfile | 2 +- rs/moq-cli/src/play/args.rs | 6 ++--- rs/moq-cli/src/play/headless.rs | 2 +- rs/moq-cli/src/play/media.rs | 10 +++++--- rs/moq-mux/src/sync.rs | 44 +++++++++++++++++++-------------- sh/gh/nightly.sh | 2 +- 11 files changed, 44 insertions(+), 71 deletions(-) delete mode 100644 quest/m2/mux-sync.md diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 5a46af26322..480dac212d9 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -187,6 +187,9 @@ Playback starts at the newest group and trails the live edge by `--delay`. The default, `auto`, sizes the buffer to how unevenly audio arrives, with the same [algorithm](/concept/audio-jitter) as the browser player. A duration fixes the delay and is also how long a stalled group is waited on before it is skipped. +Audio and video play from one clock and stay in sync while both arrive on time; +a track that stalls plays late and catches up by playing faster, rather than +holding the other one back. Each role follows the catalog, switching rendition when the publisher retires the one playing. With `--video off`, `--metrics ` writes a JSON line every 250 ms with the playhead, the delay, and every gap and stall the speaker hit, in diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 88a2f0df13d..3c5ad76bced 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -10,8 +10,9 @@ description: Container import and export Turns existing container formats into [hang](/concept/hang) broadcasts and back. This is what `moq import` / `export` and the gateways are built on. -Native players also use its `timeline` to present a broadcast's audio and video -on one clock: audio drives the anchor while it plays, and video follows it. +Native players also use its `sync` to present a broadcast's tracks on one clock, +anchored on the earliest arrival any track has seen. Tracks stay together in the +steady state, and one that stalls plays late instead of holding the others. The caller supplies the jitter target, decoders, and output devices; a presenter can select the newest due picture without waiting and repeat it during a stall. diff --git a/quest/m2/README.md b/quest/m2/README.md index fa75bcb69f3..95df8fd2b87 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -201,7 +201,6 @@ quests that require them. - [No sentinel values in public API](/quest/m2/no-sentinels.md) - receive windows, cache durations, reconnect timeouts, and the FFI test clock use `Option` instead of a huge value or zero - [moq-video takes devices, not environment variables](/quest/m2/video-devices.md) - VA-API and V4L2 device paths are config fields, and `moq_video::ENV` never ships - [Gateways share one ingest config and one rate cap](/quest/m2/gateway-ingest.md) - `moq_mux::Ingest` replaces 18 per-gateway setters, and the live rate cap moves to `moq_net::bandwidth` -- [Native playout sync keeps tracks decoupled](/quest/m2/mux-sync.md) - `moq_mux::timeline` becomes `moq_mux::sync`, and one slow track no longer stalls or moves another - [JS time values are types, not numbers](/quest/m2/js-time-types.md) - every public JS duration and instant is a `Clock.Duration` or `Clock.Instant` object, so a unit mistake fails to compile - [Small API fixes before the cut](/quest/m2/api-shape/README.md) - one path and one spelling per item, names by role, and no dead or wrong-role public surface across net, transport, media, native, apps, JS, and bindings - [Public constructors spawn nothing, or say why](/quest/m2/spawn-constructors.md) - each public library entry point that spawns returns a driver its caller polls, or is listed as kept diff --git a/quest/m2/api-shape/media.md b/quest/m2/api-shape/media.md index fbd3ae4f7c3..06b54de64d9 100644 --- a/quest/m2/api-shape/media.md +++ b/quest/m2/api-shape/media.md @@ -41,7 +41,7 @@ Added 2026-11-07 (maintainer, /quest-plan from the 2026-11-07 API sweep of open Found by the 2026-11-07 API shape audit (media formats); see the [epic](/quest/m2/api-shape/README.md). The playout clock is -[mux-sync](/quest/m2/mux-sync.md), and rendition stats are +`moq_mux::sync`, already reshaped, and rendition stats are [rendition-stats-type](/quest/m2/rendition-stats-type.md). Check the `ts::Follower` and `ts::Release` collision too, and rename whichever name the PR shows is ambiguous. diff --git a/quest/m2/mux-sync.md b/quest/m2/mux-sync.md deleted file mode 100644 index 92f6a3296db..00000000000 --- a/quest/m2/mux-sync.md +++ /dev/null @@ -1,38 +0,0 @@ -# [M] Native playout sync keeps tracks decoupled - -## Goal - -`moq_mux::timeline` becomes `moq_mux::sync` and follows the browser -player's model: lip sync in the steady state, but one slow track never stalls -or moves another. A wall-clock reference holds the broadcast's delay, each -track converges on it (audio by time-stretching toward it), and audio no -longer owns the anchor. `moq play` (`rs/moq-cli/src/play/`) moves to it, and -the module's duplicate `Kind` gives way to `container::Kind`. - -## Plan - -Decided 2026-11-07 (maintainer, /quest-plan from the API shape audit): -✅ decouple and rename (rejected: rename only, keeping the audio anchor; -keeping it as is). The 2026-11-02 decision behind -[moq-dev/moq#7759](https://github.com/moq-dev/moq/pull/7759) ("while audio -plays, only audio moves the anchor") is superseded by the 2026-11-05 ruling -on [moq-dev/moq#7941](https://github.com/moq-dev/moq/pull/7941) for the JS -player: "I kind of don't want one slow track to impact everything. We want -lip sync in the steady state but desync during the congestion state." - -- ✅ Name: `moq_mux::sync`, matching `@moq/watch`'s `Sync` by role - (rejected: `presentation`, "presentation sucks"; `playhead`; `schedule`). - `container::Playout` keeps its name. -- Port the JS semantics from `js/watch/src/sync.ts` (per-track timelines on - one reference, rework in #7941) rather than inventing a third model. - `Producer::{audio_restarted, audio_stopped}` and `Consumer::target` likely - go. -- `moq play`'s timeline tests move with the module and change where audio - stalls used to hold video; add a case where stalled audio no longer holds - video. -- moq-dev/studio's moq-scene playout consumes this module through a path - dependency and adapts on its next pin; this PR names that migration in its - Impact section. - -Breaking: the module is new on main, but release had an unrelated -`moq_mux::timeline`, so the cut's MIGRATE entry says both are gone. diff --git a/rs/justfile b/rs/justfile index 23a8ab90add..c71cc38b049 100644 --- a/rs/justfile +++ b/rs/justfile @@ -570,7 +570,7 @@ codspeed: --features moq-audio/bus,moq-net/fuzz,moq-quic/fuzz \ --bench tasks --bench waiter --bench bus --bench protect --bench codec --bench table \ --bench catalog --bench containers --bench decode_clock --bench export --bench jitter \ - --bench parameter_sets --bench timeline --bench ts_programs --bench ts_schedule --bench ts_video \ + --bench parameter_sets --bench sync --bench ts_programs --bench ts_schedule --bench ts_video \ --bench announce --bench auth --bench bandwidth --bench canonical --bench frame --bench interests \ --bench priority --bench retirement --bench schedule --bench stats --bench synced --bench tail \ --bench tracks --bench varint --bench viewers --bench matching \ diff --git a/rs/moq-cli/src/play/args.rs b/rs/moq-cli/src/play/args.rs index c142cbb1cc8..e9ed4a74bfb 100644 --- a/rs/moq-cli/src/play/args.rs +++ b/rs/moq-cli/src/play/args.rs @@ -24,8 +24,8 @@ const DELAY_MAX: Duration = Duration::from_secs(10); /// lateness the estimate can represent. const AUTO_MAX_DELAY: Duration = Duration::from_secs(2); -/// How far video alone trails the live edge under `auto`. The estimate is -/// audio's; video follows the speaker whenever there is one. +/// How far playback trails the live edge under `auto` while audio holds no +/// estimate, as when there is none: video has no estimator of its own. const AUTO_VIDEO_DELAY: Duration = Duration::from_millis(100); /// How far playback trails the live edge. @@ -112,7 +112,7 @@ impl Args { self.fixed_delay().unwrap_or(AUTO_MAX_DELAY) } - /// How far video trails the live edge while no speaker is setting the pace. + /// How far playback trails the live edge while no audio estimate sets it. pub(super) fn video_delay(&self) -> Duration { self.fixed_delay().unwrap_or(AUTO_VIDEO_DELAY) } diff --git a/rs/moq-cli/src/play/headless.rs b/rs/moq-cli/src/play/headless.rs index 5c9efdcc3d7..c7f620720f8 100644 --- a/rs/moq-cli/src/play/headless.rs +++ b/rs/moq-cli/src/play/headless.rs @@ -13,8 +13,8 @@ use super::args::Args; use super::media::Media; use super::metrics::Metrics; use super::output::Output; -use super::window::Event; use super::timeline::Playhead; +use super::window::Event; use moq_mux::sync::Producer as Presentation; /// The speaker and the run loop's events, plus `--metrics`' observer feed. diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 28e765eafba..e1e96a77608 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -714,7 +714,9 @@ async fn play_audio( let first = lead(delay); let mapping = role.resume(transition, frame.timestamp, Some(first)).ready().await?; role.apply(transition, mapping); - let at = role.due(role.map(frame.timestamp)?).and_then(|at| at.checked_sub(first)); + let at = presentation + .due(role.map(frame.timestamp)?) + .and_then(|at| at.checked_sub(first)); if let Some(at) = at { kio::clock::sleep_until(at).await; } @@ -794,7 +796,7 @@ async fn play_audio( // spacing instead of fitting that queued burst as if it arrived live. // The lead follows the current target so jitter can still deepen the sink. if sequential { - let at = role.due(mapped).and_then(|at| at.checked_sub(lead(delay))); + let at = presentation.due(mapped).and_then(|at| at.checked_sub(lead(delay))); if let Some(at) = at { kio::clock::sleep_until(at).await; } @@ -820,7 +822,7 @@ async fn play_audio( let now = Instant::now(); let held = moq_net::Timestamp::try_from(timing.end) .ok() - .and_then(|end| role.due(end)) + .and_then(|end| presentation.due(end)) .map_or(input.latency, |due| due.saturating_duration_since(now)) .clamp(AUDIO_BUFFER_MIN, max_delay.max(AUDIO_BUFFER_MIN)); let target = samples(held); @@ -1198,7 +1200,7 @@ mod tests { }) .unwrap(); // Audio stops after 200 ms; its backlog lands in one late burst at 800 ms. - if index < 10 || index > 40 { + if !(10..=40).contains(&index) { audio.write(packet(index, if index < 10 { 0.25 } else { 0.5 })).unwrap(); } else if index == 40 { for late in 10..=40 { diff --git a/rs/moq-mux/src/sync.rs b/rs/moq-mux/src/sync.rs index b43b19e8de7..c56adbf5aba 100644 --- a/rs/moq-mux/src/sync.rs +++ b/rs/moq-mux/src/sync.rs @@ -1,13 +1,13 @@ //! One playout clock shared by a broadcast's tracks. //! -//! Each [`Track`] folds its arrivals into the clock with [`Track::received`] -//! and reads when its media is due. The clock's reference is the earliest +//! Each [`Track`] folds its arrivals into the clock with [`Track::received`], +//! and [`Producer::due`] says when media is due. The clock's reference is the earliest //! arrival any live track has seen, measured against its timestamp, so it sits //! on the wall clock and follows the live edge. Every track presents against //! it, which holds them together in the steady state. One that falls behind, //! such as audio stalled by congestion, only arrives late: it never holds or //! moves the others. Audio converges on the clock by stretching toward its -//! [`due`](Track::due) time rather than owning it. +//! [`due`](Producer::due) time rather than owning it. //! //! Playback trails the reference by the largest target a live track holds //! ([`Track::set_target`]), or by the delay passed to [`Producer::new`] while @@ -16,9 +16,9 @@ //! previous picture. //! //! A track also maps encoder segments onto the clock across declared ends. -//! Keep one active track per [`Kind`](crate::container::Kind); retire it before starting its successor. -//! Bound replacement media with the caller's retention budget and -//! [`Track::abandon`] transitions whose replacement is discarded. +//! Keep one active track per [`Kind`]; retire it before +//! starting its successor. Bound replacement media with the caller's retention +//! budget and [`Track::abandon`] transitions whose replacement is discarded. use kio::clock::Instant; use std::time::Duration; @@ -74,7 +74,10 @@ impl Anchor { fn earlier(&self, other: &Self) -> bool { match other.media.checked_sub(self.media) { Some(ahead) => self.at.checked_add(ahead).is_some_and(|at| at < other.at), - None => other.at.checked_add(self.media - other.media).is_none_or(|at| self.at < at), + None => other + .at + .checked_add(self.media - other.media) + .is_none_or(|at| self.at < at), } } } @@ -113,7 +116,7 @@ struct State { /// Endpoints declared by each live role, on the shared playback timeline. ends: [Option; KINDS], /// When each retired role's tail finishes playing, frozen on the wall clock: - /// a later rendition re-pins the anchor, which would carry a media endpoint + /// a later rendition anchors afresh, which would carry a media endpoint /// onto an unrelated timeline. Kept apart from `ends` so a successor role /// never clears its predecessor's tail. tails: Vec, @@ -361,12 +364,6 @@ impl Track { moved } - /// When `timestamp`, on the shared timeline, should be presented, or `None` - /// before any frame has anchored the clock. - pub fn due(&self, timestamp: Timestamp) -> Option { - self.clock.state.lock().unwrap().due(timestamp) - } - /// Stop while a tail still plays until `drained`, holding a declared end until /// then. Dropping instead discards the tail with the end. pub fn retire(self, drained: Option) { @@ -580,7 +577,11 @@ impl Resume<'_> { // A transition abandoned under this resume, by a gap the caller rechecks, // maps alone like one no peer joined. let (mapping, paired) = state.transition(self.transition).map_or((None, None), |pending| { - let mut plans = pending.members.iter().zip(&pending.plans).filter(|(member, _)| **member); + let mut plans = pending + .members + .iter() + .zip(&pending.plans) + .filter(|(member, _)| **member); let joined = pending.members[own] && plans.clone().count() > 1; let origin = plans .try_fold(None::, |earliest, (_, plan)| { @@ -797,7 +798,6 @@ mod tests { audio.reset(); assert!(!audio.received(ms(1_000), resumed + Duration::from_millis(100))); assert_eq!(clock.due(ms(1_000)), steady, "stalled audio held video"); - assert_eq!(audio.due(ms(1_000)), steady, "audio converges on the shared schedule"); } /// A video tune-in burst is the live edge: nothing owns the schedule, so it @@ -810,7 +810,7 @@ mod tests { let video = clock.track(Kind::Video); audio.received(ms(1_000), start); assert!(video.received(ms(4_000), start)); - assert_eq!(audio.due(ms(4_000)), Some(start + DELAY)); + assert_eq!(clock.due(ms(4_000)), Some(start + DELAY)); } /// A track that resets gives up its arrivals: the schedule falls back to the @@ -860,7 +860,10 @@ mod tests { assert_eq!(reader.delay(), Duration::from_millis(60)); assert_eq!(clock.due(ms(0)), Some(start + Duration::from_millis(60))); assert!(video.set_target(Duration::from_millis(80))); - assert!(!audio.set_target(Duration::from_millis(70)), "a shallower target moved the delay"); + assert!( + !audio.set_target(Duration::from_millis(70)), + "a shallower target moved the delay" + ); assert_eq!(reader.delay(), Duration::from_millis(80)); drop(video); @@ -1031,7 +1034,10 @@ mod tests { audio.plan(transition, ms(0), Some(DELAY)); let mapping = audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(); audio.apply(transition, mapping); - assert!(kio::clock::now() >= start + DELAY * 2, "the replacement waited for the tail"); + assert!( + kio::clock::now() >= start + DELAY * 2, + "the replacement waited for the tail" + ); let unchanged = clock.state.lock().unwrap().due(ms(10_500)); for frame in 0..10 { let at = start + Duration::from_millis(frame * 20); diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index e2cef86c86c..e3efb1510fc 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -143,7 +143,7 @@ case "$1" in criterion extra --no-fail-fast -p moq-crypto -p moq-json -p moq-archive -p moq-cli -p moq-mux -p moq-audio \ -p moq-quic -p moq-sock -p moq-tokio -p moq-transcode -p moq-hls --features moq-quic/fuzz \ --bench protect --bench sign --bench codec --bench table --bench allocations --bench window \ - --bench producer --bench play_buffer --bench jitter --bench timeline --bench decode_clock --bench ts_programs --bench ts_video \ + --bench producer --bench play_buffer --bench jitter --bench sync --bench decode_clock --bench ts_programs --bench ts_video \ --bench bus --bench batches --bench parameter_sets --bench sent_packets --bench demux --bench receive_bursts --bench ladder --bench fanout --bench include criterion audio-capture -p moq-audio --features capture --bench capture # moq-auth's published release is its own dev-dependency, so `-p moq-auth` is ambiguous. From a0c65b778a5e63e24cb67974bacf4074bc272412 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 8 Nov 2026 00:31:41 -0800 Subject: [PATCH 4/4] fix(cli): wake video when audio releases its target, and scope the playhead per broadcast Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 5 ++++ rs/moq-cli/src/play/timeline.rs | 52 +++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index e1e96a77608..a021962647d 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -101,6 +101,7 @@ impl Media { // Another instance's timeline is its own, so nothing of the old one carries over. drop(playing.take()); self.presentation = self.presentation.reset(self.args.video_delay()); + self.playhead = self.playhead.clone().reset(); self.video.lock().unwrap().clear(); self.drained.notify_one(); self.output.send(Event::Wake); @@ -164,6 +165,10 @@ impl Media { // A replacement is a track boundary whose timestamps need not // continue this one. self.playhead.restart(); + // The track released its target, which can lower the delay that + // video's decoder and window are asleep on. + self.drained.notify_one(); + self.output.send(Event::Wake); } // The retired sink still holds a delay of audio, and a replacement // holds its own before its first sample sounds. Played one after diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 8df2dae7fe0..83e0f835f81 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -9,8 +9,24 @@ use kio::clock::Instant; /// /// The speaker converges on the shared clock rather than owning it, so where it /// actually is stays its own: it runs dry in a stall and skips ahead to catch up. -#[derive(Clone, Default)] -pub(super) struct Playhead(Arc>); +/// +/// Clones write one broadcast's speaker; [`reset`](Self::reset) starts the next +/// broadcast's, which readers follow while the old one's late writes go nowhere. +#[derive(Clone)] +pub(super) struct Playhead { + own: Arc>, + current: Arc>>>, +} + +impl Default for Playhead { + fn default() -> Self { + let own = Arc::new(Mutex::new(Written::default())); + Self { + current: Arc::new(Mutex::new(own.clone())), + own, + } + } +} #[derive(Default)] struct Written { @@ -22,9 +38,20 @@ struct Written { } impl Playhead { + /// Start the next broadcast's playhead and switch readers to it. + #[must_use = "use the returned playhead for the replacement broadcast"] + pub(super) fn reset(self) -> Self { + let own = Arc::new(Mutex::new(Written::default())); + *self.current.lock().unwrap() = own.clone(); + Self { + own, + current: self.current, + } + } + /// Record a write ending at media `end`, of which `buffered` is still queued at `now`. pub(super) fn wrote(&self, end: Duration, buffered: Duration, now: Instant) { - let mut written = self.0.lock().unwrap(); + let mut written = self.own.lock().unwrap(); // Silence padded ahead of the frame to refill the speaker is not media: the // playhead holds where it was until the padding has played, rather than // projecting back over media already heard. A new timeline starts afresh. @@ -38,17 +65,17 @@ impl Playhead { /// The speaker moved onto an unrelated timeline: a fresh sink or rendition. pub(super) fn restart(&self) { - self.0.lock().unwrap().restart = true; + self.own.lock().unwrap().restart = true; } /// The speaker went quiet. pub(super) fn clear(&self) { - *self.0.lock().unwrap() = Written::default(); + *self.own.lock().unwrap() = Written::default(); } /// The media time sounding at `now`, or `None` with no speaker. pub(super) fn at(&self, now: Instant) -> Option { - self.0.lock().unwrap().playhead(now) + self.current.lock().unwrap().lock().unwrap().playhead(now) } } @@ -167,6 +194,19 @@ mod tests { assert_eq!(playhead.at(dry), None); } + /// A replaced broadcast's late writes never reach the reader. + #[test] + fn a_reset_playhead_isolates_the_old_broadcast() { + let now = Instant::ZERO + Duration::from_secs(1); + let reader = Playhead::default(); + let old = reader.clone(); + let new = reader.clone().reset(); + old.wrote(Duration::from_secs(10), DELAY, now); + assert_eq!(reader.at(now), None); + new.wrote(Duration::from_secs(2), DELAY, now); + assert_eq!(reader.at(now), Some(Duration::from_millis(1_900))); + } + /// Silence padded ahead of a frame after the speaker ran dry plays no media, /// so the playhead holds instead of replaying media already heard. #[test]