From 0913a5dd08bf2f4e61ec7466a69a208d0b5917ee Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:04:06 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m2/sync-segments.md Quest-Claim: b05d29a00866f339 From ec5c2dc94c4c26491a4b0b060a803a69bb63313d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:48:32 -0800 Subject: [PATCH 2/3] refactor(mux): sync resets at a segment transition, as JS does Timestamps keep moving forward across a declared end, so the shared clock needs no segment mapping. Delete moq_mux::sync's transitions (Transition, Mapping, Resume, declare/end/plan/apply/abandon/gap/retire/map) and the segment state behind a second lock. moq play's video track resets its part of the clock at the first later group after a declared end, and audio starts the next utterance on the same clock, as @moq/watch does. Co-Authored-By: Claude Opus 5.5 --- quest/m2/sync-segments.md | 29 -- rs/moq-cli/src/play/buffer.rs | 17 - rs/moq-cli/src/play/media.rs | 253 +++------ rs/moq-cli/src/play/video.rs | 458 ++++------------ rs/moq-mux/CHANGELOG.md | 2 +- rs/moq-mux/src/sync.rs | 953 +--------------------------------- 6 files changed, 183 insertions(+), 1529 deletions(-) delete mode 100644 quest/m2/sync-segments.md diff --git a/quest/m2/sync-segments.md b/quest/m2/sync-segments.md deleted file mode 100644 index bbb5ea4a80c..00000000000 --- a/quest/m2/sync-segments.md +++ /dev/null @@ -1,29 +0,0 @@ -# [M] moq_mux::sync resets at a segment transition, as JS does - -## Goal - -`moq_mux::sync` handles a declared end and its replacement media with one -rule that Rust and JS share, and its state follows REVIEW.md's State rules: -one enum per lifecycle, one lock per value. - -## Plan - -Found 2026-11-14 by the [feature review](/quest/m3/release/feature-review.md). -Segment transitions in `rs/moq-mux/src/sync.rs` keep parallel `members`, -`resumed`, and `plans` arrays in `Pending`, and one track's segment state sits -behind two locks. -[moq-dev/moq#8589](https://github.com/moq-dev/moq/pull/8589) and -[moq-dev/moq#8513](https://github.com/moq-dev/moq/pull/8513) patched it, and -#8589 says JS needs no mirror because `js/watch/src/sync.ts` has no segment -transitions, so Rust runs a third model the deleted mux-sync quest said not -to invent. - -Coordinate with -[moq-dev/moq#8749](https://github.com/moq-dev/moq/pull/8749), which changes -the same clock. - -Decided 2026-11-14 (maintainer, pick on [moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764)): ✅ (a) delete the transitions and reset the way JS does (rejected: (b) one enum per role, ported to JS). - -Public API: possibly `moq_mux::sync`'s transition surface, unpublished. -Wire: none. - diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index 218a525f6d4..e9e603a9e44 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -17,8 +17,6 @@ pub(super) struct Buffer { pub floor: Option, waiting_keyframe: bool, pub ended: bool, - /// Exclusive endpoint declared for this encoder segment. - pub end: Option, } impl Default for Buffer { @@ -33,21 +31,11 @@ impl Default for Buffer { // A join resumes at its first keyframe, like any other break. waiting_keyframe: true, ended: false, - end: None, } } } impl Buffer { - pub fn clear(&mut self) { - self.frames.clear(); - self.oldest.clear(); - self.bytes = 0; - self.edge = None; - self.generation += 1; - self.waiting_keyframe = true; - } - /// The earliest presentation time still buffered, wherever it sits in /// decode order. pub fn oldest(&self) -> Option { @@ -192,11 +180,6 @@ mod tests { [0, 99, 33, 66] ); assert_eq!(buffer.edge.unwrap().as_millis().unwrap(), 99); - buffer.clear(); - assert_eq!(buffer.bytes, 0); - assert!(buffer.frames.is_empty()); - buffer.push(frame(1, true), Duration::ZERO); - assert_eq!(buffer.edge.unwrap().as_millis().unwrap(), 1); } #[test] fn age_expiry_finds_a_reordered_picture_behind_the_front() { diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index e5e0f2c4b75..255ce87f188 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -17,12 +17,16 @@ use super::playback::{Kind, Playback, Track, joined}; use super::timeline::{AudioTimeline, Playhead}; use super::video::{Refresh, Video}; use super::window::Event; -use moq_mux::sync::{Producer as Presentation, Transition}; +use moq_mux::sync::Producer as Presentation; /// What a frame that starts a new decoded timeline crossed. enum Crossing { - /// A declared end, whose transition the frame resumes. - Resume(Transition), + /// A declared end: the frame starts the utterance after its silence. + End { + /// Another end is already declared past this frame, so the utterance it + /// starts is followed by a silence too. + ahead: bool, + }, /// Lost media: no end was declared. Loss, } @@ -620,7 +624,7 @@ async fn play_audio( // later. That estimate times each packet when `read` hands it over, so this // receiver never waits on the speaker between reads. Live playback lets // the callback stretch toward the target; a declared end plays comfort noise - // until the old media drains and its replacement is due. Each write anchors + // until the next utterance is due. Each write anchors // the sink to when the shared clock says its media is due, so the callback // plays onto that clock in the steady state without letting a stall hold video. let role = presentation.track(moq_mux::container::Kind::Audio); @@ -736,10 +740,6 @@ async fn play_audio( // all logs once rather than once per packet. let mut dropping = false; - // Resolved in delivery order, ahead of playback, so each end's transition is - // declared before the replacement media that resumes it is planned. A short - // timeline can declare its end before its first PCM, so ends queue until a - // frame crosses them. // The jitter target the last write held, so a rise in the stated delay applies // at once rather than waiting on the next write. let jitter = Mutex::new(None); @@ -760,6 +760,8 @@ async fn play_audio( let mut retired = std::pin::pin!(retired); // Media went missing since the last frame, so the next one does not continue it. let mut lost = false; + // Declared ends no frame has crossed yet. A short utterance can declare its + // end before its first PCM, so ends queue until a frame crosses them. let mut ends = VecDeque::new(); // The subscription reaches the estimator's ceiling past the shared delay wherever that // moves, never below what it opened with, so every frame late enough to move the target @@ -773,9 +775,9 @@ async fn play_audio( consumer.set_max_delay(max); asked = max; } - // Stop reading while play is held behind a resume with more than the age - // budget queued, so the subscription evicts the excess instead of this - // queue growing without bound. + // Stop reading while play is held with more than the age budget queued, + // so the subscription evicts the excess instead of this queue growing + // without bound. while received .lock() .unwrap() @@ -804,7 +806,7 @@ async fn play_audio( let delay = consumer.delay(); let crossing = match &event { Ok(Some(moq_audio::decode::Read::End { end })) => { - ends.push_back((Duration::try_from(*end)?, role.declare())); + ends.push_back(Duration::try_from(*end)?); None } // Playback restarts at the frame after it, so that is what carries it. @@ -815,25 +817,20 @@ async fn play_audio( Ok(Some(moq_audio::decode::Read::Frame(frame))) => { let lost = std::mem::take(&mut lost); // A frame at or past an end comes from the encoder that replaced it; one - // before it is the stopped encoder's drain. Only the last end crossed has - // replacement media; the ones before it ended empty timelines. + // before it is the stopped encoder's drain. Ends it crosses together + // closed utterances with no media. let at = Duration::try_from(frame.timestamp)?; - let mut crossed = None; - while let Some(&(end, transition)) = ends.front() - && at >= end - { + let mut crossed = false; + while ends.front().is_some_and(|end| at >= *end) { ends.pop_front(); - if let Some(skipped) = crossed.replace(transition) { - role.abandon(skipped); - } + crossed = true; } match crossed { - Some(transition) => { - role.plan(transition, frame.timestamp, Some(lead(delay))); - Some(Crossing::Resume(transition)) - } - None if lost => Some(Crossing::Loss), - None => None, + true => Some(Crossing::End { + ahead: !ends.is_empty(), + }), + false if lost => Some(Crossing::Loss), + false => None, } } _ => None, @@ -871,16 +868,9 @@ async fn play_audio( } } }; - // Whether a timeline is playing and its end is already declared, and an end - // declared past that one, for a later timeline that can only be mapped once it - // resumes. - let mut playing = false; - let mut ended = false; - let mut deferred = None; - let mut sequential = false; // The sink was told its audio ends in a declared silence and has not been told what follows. let mut silent = false; - // The frame being written resumes a timeline whose end is already declared. + // The frame being written starts an utterance whose end is already declared. let mut declare = false; // The frame being written starts the utterance after a declared silence. let mut utterance = false; @@ -914,13 +904,8 @@ async fn play_audio( input.latency = lead(delay); let frame = match event { Ok(Some(moq_audio::decode::Read::Frame(frame))) => { - playing = true; match crossing { Some(Crossing::Loss) => { - deferred = None; - ended = false; - role.gap(); - sequential = false; // What follows a declared silence starts the next utterance however many // groups were lost on the way, rather than filling the silence as a hole. if std::mem::take(&mut silent) { @@ -929,29 +914,12 @@ async fn play_audio( utterance = true; } } - Some(Crossing::Resume(transition)) => { + Some(Crossing::End { ahead }) => { // A declared boundary preserves the old sink, unlike an undeclared hole: - // its tail and then comfort noise play while the old media drains. - let first = lead(delay); - let mapping = role.resume(transition, frame.timestamp, Some(first)).ready().await?; - role.apply(transition, mapping); - let start = role.map(frame.timestamp)?; - wait(&mut catalog, &|| { - presentation.due(start).and_then(|at| at.checked_sub(first)) - }) - .await; - ended = false; - // An end this frame did not cross ends the timeline it resumes. - if let Some(end) = deferred.take() - && Duration::try_from(frame.timestamp)? < Duration::try_from(end)? - { - role.end(end)?; - ended = true; - declare = true; - } - sequential = true; + // its tail and then comfort noise play until this frame is due. The + // timestamps kept moving through the silence, so the clock carries on. + declare = ahead; timeline = AudioTimeline::default(); - // Comfort noise fills the wait, so this frame plays one target out. silent = false; sink.next(); utterance = true; @@ -960,18 +928,14 @@ async fn play_audio( } frame } - Ok(Some(moq_audio::decode::Read::End { end })) => { + // An end read while already silent belongs to an utterance still to start, + // which its first frame's crossing carries. + Ok(Some(moq_audio::decode::Read::End { .. })) => { // The stopped encoder's final frames may follow, and play before the silence. if !silent { sink.end(); silent = true; } - if playing && ended { - deferred = Some(end); - } else { - role.end(end)?; - ended = true; - } continue; } // Never queued: receive folds it into the next frame's crossing. @@ -994,15 +958,13 @@ async fn play_audio( dropping = false; let length = frame.data.len() / stride; - let mapped = role.map(frame.timestamp)?; - let start = Duration::try_from(mapped)?; + let start = Duration::try_from(frame.timestamp)?; let timing = timeline.push(start, length, sample_rate, fill_max); // A rewind or a hole too large to fill starts a new playback sink. The old // sink has no media clock, so its buffered audio cannot be carried across a // timeline region the player skipped. if timing.reset_sink { - sequential = false; silent = false; drop(sink); // The media either side of the break is unrelated, so its arrivals @@ -1013,16 +975,6 @@ async fn play_audio( fresh = true; } - // Resumed media was received while the old tail played. Keep its - // spacing instead of fitting that queued burst as if it arrived live. - // The lead follows the current target so jitter can still deepen the sink. - if sequential { - wait(&mut catalog, &|| { - presentation.due(mapped).and_then(|at| at.checked_sub(lead(delay))) - }) - .await; - } - // A hole in the media is a hole in the audio, not a splice. Handing the next // frame straight to the speaker shortens the track by the missing duration, // which leaves it running ahead of media time until the clock below @@ -1031,7 +983,7 @@ async fn play_audio( // The arrival, not the write: how far the speaker trails is what the clock // steers, never what moves it. The target is this track's own jitter // target, which playback holds unless another track asks for more. - let mut moved = role.received(mapped, arrived_at); + let mut moved = role.received(frame.timestamp, arrived_at); // The publisher's encoder delay is on top of the arrival jitter, so the // clock holds both while the sink keeps sizing for jitter alone. The // clock holds the whole measured target, which the subscription follows, @@ -1044,7 +996,9 @@ async fn play_audio( // when its timeline ends before the media behind it covers the hold, so // that little waits here instead of overflowing the sink. let held_back = wait(&mut catalog, &|| { - presentation.due(mapped).and_then(|due| due.checked_sub(reach())) + presentation + .due(frame.timestamp) + .and_then(|due| due.checked_sub(reach())) }) .await; // A fresh sink plays its first write behind an opening cushion sized before @@ -1053,7 +1007,7 @@ async fn play_audio( // which already covers any hole before it. A frame starting an utterance is // instead placed one target out by the sink itself. let short = presentation - .due(mapped) + .due(frame.timestamp) .filter(|_| (fresh || held_back) && !utterance) .map(|due| { due.saturating_duration_since(Instant::now()) @@ -1119,7 +1073,6 @@ async fn play_audio( Ok::<_, anyhow::Error>(sink) }; let (_, sink) = tokio::try_join!(receive, play)?; - role.retire(Instant::now().checked_add(sink.buffered())); Ok(sink) } @@ -1848,22 +1801,20 @@ mod tests { video.discontinuity().unwrap(); } for index in 0..3 { - if audio_end || index == 0 { - let mut pcm = packet(index + 3, 0.5); - pcm.keyframe = index == 0; - audio.write(pcm).unwrap(); - } - if video_end || index == 0 { - video.write(picture(index + 3, video_end && index == 0)).unwrap(); - } + let mut pcm = packet(index + 3, 0.5); + pcm.keyframe = index == 0; + audio.write(pcm).unwrap(); + video.write(picture(index + 3, video_end && index == 0)).unwrap(); + // Timestamps run on through the end, so the replacement arrives live. + kio::clock::sleep(PACKET_DURATION).await; } let (mut audio, mut video) = (Some(audio), Some(video)); let mut _audio_finished = audio.take_if(|_| audio_end).map(|audio| audio.finish().unwrap()); let mut _video_finished = video.take_if(|_| video_end).map(|video| video.finish().unwrap()); let mut shown = Vec::new(); for step in 0..800 { - if step >= 20 && step % 20 == 0 { - let index = 3 + step / 20; + if step % 20 == 0 { + let index = 6 + step / 20; if let Some(audio) = &mut audio { audio.write(packet(index, 0.5)).unwrap(); } @@ -2070,77 +2021,7 @@ mod tests { assert!(!resumed.is_empty(), "{played:?}"); assert!( resumed.iter().all(|part| part.utterance > 0), - "the replacement resumed its declared transition: {played:?}" - ); - } - - #[tokio::test] - async fn a_held_audio_resume_leaves_the_excess_to_the_subscription() { - tokio::time::pause(); - let delay = Duration::from_millis(200); - let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); - let mut options = moq_audio::decode::Options::new(); - options.delay = moq_audio::decode::Delay { - min: Some(delay), - max: Some(delay), - }; - options.estimate = false; - options.output.format = moq_audio::Format::F32; - let consumer = moq_audio::decode::Consumer::new( - &broadcast.consume(), - &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), - "audio", - options, - ) - .await - .unwrap(); - let recorder = Recorder::default(); - let presentation = Presentation::new(delay); - let changed = Arc::new(tokio::sync::Notify::new()); - let playback = AudioPlayback { - delay: tokio::sync::watch::channel(None).1, - changed: changed.clone(), - speaker: recorder.clone(), - presentation: presentation.clone(), - playhead: Default::default(), - latency: delay, - output: recorder.clone(), - stretch: Duration::ZERO, - }; - let task = tokio::spawn(async move { - play_audio(consumer, playback, std::future::pending()) - .await - .unwrap() - .finish() - .await; - }); - for index in 0..5 { - producer.write(packet(index, 0.25)).unwrap(); - kio::clock::sleep(PACKET_DURATION).await; - } - // Video declares an end six seconds out, holding the audio replacement - // back past its age budget, the delay plus the estimator's 2 s reach. - let video = presentation.track(moq_mux::container::Kind::Video); - video.declare(); - video.end(moq_net::Timestamp::from_millis(6_000).unwrap()).unwrap(); - producer.discontinuity().unwrap(); - const REPLACEMENT: u64 = 300; - for index in 0..REPLACEMENT { - producer.write(packet(5 + index, 0.5)).unwrap(); - kio::clock::sleep(PACKET_DURATION).await; - } - let _finished = producer.finish().unwrap(); - task.await.unwrap(); - drop(video); - let (resumed, _) = played(&recorder, 0.5); - assert!(!resumed.is_zero(), "the replacement played"); - assert!( - resumed < PACKET_DURATION * REPLACEMENT as u32 / 2, - "media held behind the resume past the age budget was queued, not evicted: {resumed:?}" + "the next utterance did not start after the declared end: {played:?}" ); } @@ -2180,7 +2061,7 @@ mod tests { ) .await .unwrap(); - let mut video = moq_mux::container::Producer::new(video, Container::Legacy(moq_mux::container::Kind::Video)); + let video = moq_mux::container::Producer::new(video, Container::Legacy(moq_mux::container::Kind::Video)); let video_play = Video { presentation: media.presentation.clone(), frames: media.video.clone(), @@ -2201,22 +2082,30 @@ mod tests { keyframe, payload: Bytes::from_static(b"picture"), }; - for index in 0..3 { - video.write(picture(index * 20, index == 0)).unwrap(); - kio::clock::sleep(PACKET_DURATION).await; - } - video.discontinuity().unwrap(); - // A burst longer than the age budget evicts this replacement's only keyframe. - for index in 0..15 { - video.write(picture(1_000 + index * 20, index == 0)).unwrap(); - } - video.discontinuity().unwrap(); - for index in 0..3 { - video.write(picture(5_000 + index * 20, index == 0)).unwrap(); - } - let _finished = video.finish().unwrap(); + let mut video = Some(video); + let mut _finished = None; let mut shown = Vec::new(); - for _ in 0..2_000 { + for at in 0..6_000 { + let producer = video.as_mut(); + match at { + 0 | 20 | 40 => producer.unwrap().write(picture(at, at == 0)).unwrap(), + // A burst longer than the age budget evicts this replacement's only keyframe. + 1_000 => { + let producer = producer.unwrap(); + producer.discontinuity().unwrap(); + for index in 0..15 { + producer.write(picture(at + index * 20, index == 0)).unwrap(); + } + } + 5_000 => { + let producer = producer.unwrap(); + producer.discontinuity().unwrap(); + producer.write(picture(at, true)).unwrap(); + } + 5_020 | 5_040 => producer.unwrap().write(picture(at, false)).unwrap(), + 5_060 => _finished = video.take().map(|video| video.finish().unwrap()), + _ => {} + } tokio::task::yield_now().await; if let Some(timestamp) = recorder.present(&media) { shown.push(timestamp.as_millis().unwrap()); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index b94dcfccbf5..e43f1473dcf 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -12,7 +12,7 @@ use moq_video::decode::{Lane, Next, Selector}; use super::buffer::Buffer; use super::output::Output; use super::window::Event; -use moq_mux::sync::{Producer as Presentation, Transition}; +use moq_mux::sync::Producer as Presentation; /// A few surfaces, independent of the requested playout delay. A codec batch /// that overflows it waits beside the queue rather than pushing out pictures @@ -28,7 +28,8 @@ enum Step { Decode, /// The track ended: drain what the codec still holds. Flush, - Resume, + /// The front segment drained: start the next one. + Next, } /// When the window's queue has room for another picture, or `None` if it does @@ -98,34 +99,6 @@ impl Decoder for moq_video::decode::Sink { } } -/// One encoder segment's media, and the transition it resumes when it -/// replaces the segment before it. -#[derive(Default)] -struct Segment { - media: Buffer, - resumes: Option, - /// When its first picture arrived. - arrived: Option, - /// The group of the end it replaced: its media is in later groups only. - after: Option, - /// The group of its own declared end, once read. - until: Option, -} - -impl std::ops::Deref for Segment { - type Target = Buffer; - - fn deref(&self) -> &Buffer { - &self.media - } -} - -impl std::ops::DerefMut for Segment { - fn deref_mut(&mut self) -> &mut Buffer { - &mut self.media - } -} - /// The earlier deadline, where `None` is no deadline. /// The later of two timestamps, `a` when they don't compare: [`moq_net::Timestamp`] has no `Ord`. fn later(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { @@ -137,20 +110,20 @@ fn earlier(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { if b < a { b } else { a } } -/// A timestamp on the broadcast timeline, ordered by time. +/// A presentation timestamp, ordered by time. /// /// [`moq_net::Timestamp`] has no `Ord`, since an unscaled one orders against no scaled one. -/// The presentation's `map` only returns scaled timestamps, and scaled ones order totally. +/// Video timestamps are scaled, and scaled ones order totally. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct Mapped(moq_net::Timestamp); +struct Ordered(moq_net::Timestamp); -impl Ord for Mapped { +impl Ord for Ordered { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.0.partial_cmp(&other.0).unwrap_or(std::cmp::Ordering::Equal) } } -impl PartialOrd for Mapped { +impl PartialOrd for Ordered { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } @@ -193,15 +166,11 @@ impl Video { // Wakes the max-delay decoder when the clock, the segments, or the selector change. let wake = tokio::sync::Notify::new(); let closed = std::sync::atomic::AtomicBool::new(false); - let advancing = std::sync::atomic::AtomicBool::new(false); - let revision = std::sync::atomic::AtomicU64::new(0); - // The first group the clock's mapping covers: the first the min-delay decoder - // read, then the one after each end it resumed past. A gap keeps the mapping, - // so this outlives the buffers it replaces. - let origin: Mutex> = Mutex::new(None); - let buffers = Mutex::new(VecDeque::from([Segment::default()])); - // Signalled when a segment held behind a resume becomes the front. - let advanced = tokio::sync::Notify::new(); + // One buffer per encoder between declared ends, oldest first. The codec drains + // each before decoding the next, which a fresh encoder opens with a keyframe. + // The front never holds the rest back for long: no picture is due later than + // one shared delay after it arrived, since the reference is the earliest arrival. + let buffers = Mutex::new(VecDeque::from([Buffer::default()])); let role = self.presentation.track(moq_mux::container::Kind::Video); let receive = async { // Video has no jitter estimate, so its target is the encoder delay alone, @@ -229,34 +198,11 @@ impl Video { } }; follow(&mut track, &mut held, shared); - // The group of a declared end whose restart waits for a later group, so the - // media buffered before the end still plays out, and the transition that - // later group resumes. + // The group of a declared end no later group has followed yet. The media + // buffered before the end still plays out. let mut end = None; loop { - // Stop reading while media queued behind a resume arrived more than the age - // budget ago, so the subscription evicts the excess instead of the queue - // growing a segment per restart without bound. - while buffers - .lock() - .unwrap() - .iter() - .skip(1) - .filter(|segment| segment.oldest().is_some()) - .filter_map(|segment| segment.arrived) - .min() - .is_some_and(|arrived| arrived.elapsed() > self.max_delay + held) - { - tokio::select! { - () = advanced.notified() => {} - Ok(()) = encoder.changed() => target(*encoder.borrow_and_update()), - delay = role.delay_changed(shared) => { - shared = delay; - follow(&mut track, &mut held, shared); - } - } - } let read = tokio::select! { read = track.read() => read?, Ok(()) = encoder.changed() => { @@ -270,90 +216,47 @@ impl Video { } }; let mut buffers = buffers.lock().unwrap(); - let active = buffers.len() == 1 && !advancing.load(std::sync::atomic::Ordering::Relaxed); let before = buffers.front().unwrap().generation; let ended = match read { Some(Read::Frame { group, frame }) => { - if let Some((ended, transition)) = end.take_if(|(end, _)| *end != group) { + // Only a group later than a declared end's starts the next encoder, as + // in `@moq/watch`: the stopped encoder's own group may still follow it. + if end.take_if(|end| group > *end).is_some() { // A superseded segment never receives another picture, so one that - // eviction emptied can never resume. + // eviction emptied has nothing left to decode. let mut index = 1; while index < buffers.len() { if buffers[index].oldest().is_some() { index += 1; - } else if let Some(skipped) = buffers.remove(index).and_then(|skipped| skipped.resumes) - { - role.abandon(skipped); + } else { + buffers.remove(index); } } - role.plan(transition, frame.timestamp, None); - buffers.push_back(Segment { - resumes: Some(transition), - after: Some(ended), - ..Default::default() - }); + buffers.push_back(Buffer::default()); + // Another timeline: its arrivals stop holding the clock until this one. + role.reset(); } - if active && buffers.len() == 1 && role.received(role.map(frame.timestamp)?, Instant::now()) { + if role.received(frame.timestamp, Instant::now()) { self.output.send(Event::Wake); } - let back = buffers.back_mut().unwrap(); - // The first segment's mapping starts where this decoder joined, so an - // end before the join needs no resume. - if back.resumes.is_none() && back.after.is_none() { - origin.lock().unwrap().get_or_insert(group); - } - back.push(frame, self.max_delay + held); - back.arrived.get_or_insert_with(Instant::now); + buffers.back_mut().unwrap().push(frame, self.max_delay + held); false } - // Applied in delivery order, so the restart lands before the frame after it. - // An end declared before the loss still holds, as on the audio side: what - // follows comes from the replacement encoder and resumes the latest - // transition, while the media lost with the gap is discarded. + // Lost media discards every picture still buffered. An end declared + // before the loss still holds, as on the audio side: a later group comes + // from the replacement encoder. Some(Read::Gap { .. }) => { - // The mapping is unchanged, so the front keeps its groups. - let (after, until) = buffers.front().map(|front| (front.after, front.until)).unwrap(); - let mut queued: Vec<_> = buffers - .iter() - .filter_map(|buffer| Some((buffer.resumes?, buffer.after))) - .collect(); - let resumes = queued.pop(); - for (transition, _) in queued { - role.abandon(transition); - } - revision.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - role.gap(); buffers.clear(); - buffers.push_back(Segment { - after, - until, - ..Default::default() - }); + buffers.push_back(Buffer::default()); let front = buffers.front_mut().unwrap(); - front.clear(); front.generation = before + 1; - front.ended = resumes.is_some() || end.is_some(); - if let Some((transition, after)) = resumes { - buffers.push_back(Segment { - resumes: Some(transition), - after, - ..Default::default() - }); - } + front.ended = end.is_some(); role.reset(); false } - Some(Read::End { group, end: endpoint }) => { - if let Some((_, transition)) = end.take() { - role.abandon(transition); - } - end = Some((group, role.declare())); - buffers.back_mut().unwrap().until = Some(group); + Some(Read::End { group, .. }) => { + end = Some(group); buffers.back_mut().unwrap().ended = true; - buffers.back_mut().unwrap().end = Some(endpoint); - if active { - role.end(endpoint)?; - } false } None => { @@ -387,7 +290,7 @@ impl Video { let mut held = BTreeSet::new(); // Decoded pictures waiting for room in the window's queue. let mut pending = VecDeque::new(); - let drained = loop { + loop { wake.notify_one(); let (current, oldest, ended, last, floor) = { let buffers = buffers.lock().unwrap(); @@ -397,7 +300,6 @@ impl Video { }; // A codec may return pictures it held across the keyframe that // resumed a skipped timeline. Those pictures no longer have a slot. - let floor = floor.map(|floor| role.map(floor)).transpose()?; if generation.is_some_and(|generation| generation != current) { // Whatever the codec still holds sits below the new floor, so it is // filtered on the way out and owes the window nothing. @@ -472,8 +374,7 @@ impl Video { // Every access unit up to the earliest picture still owed must be // decoded before that picture is due, however deep the stream // reorders or the codec holds pictures back. - let oldest = role.map(oldest)?; - let owed = held.first().map_or(oldest, |held: &Mapped| earlier(oldest, held.0)); + let owed = held.first().map_or(oldest, |held: &Ordered| earlier(oldest, held.0)); let at = presentation .due(owed) .and_then(|at| at.checked_sub(DECODE_AHEAD)) @@ -491,21 +392,10 @@ impl Video { } else if select.is_some() || (last && !done) { // The segment's last pictures are still owed to the window. Step::Wait(select) + } else if closed.load(std::sync::atomic::Ordering::Relaxed) && buffers.lock().unwrap().len() == 1 { + break; } else { - // A new encoder waits for every old picture to be presented. A closed - // track hands its last due time to whatever resumes after it. - if closed.load(std::sync::atomic::Ordering::Relaxed) && buffers.lock().unwrap().len() == 1 { - break queue.back().and_then(|frame| presentation.due(frame.timestamp)); - } else if !queue.is_empty() { - Step::Wait( - queue - .back() - .and_then(|frame| presentation.due(frame.timestamp)) - .filter(|at| *at > now), - ) - } else { - Step::Resume - } + Step::Next } }; match step { @@ -521,16 +411,15 @@ impl Video { continue; } Step::Decode => { - let mut frame = buffers + let frame = buffers .lock() .unwrap() .front_mut() .unwrap() .pop() .expect("no await since inspecting the buffer"); - frame.timestamp = role.map(frame.timestamp)?; generation = Some(current); - held.insert(Mapped(frame.timestamp)); + held.insert(Ordered(frame.timestamp)); { let mut edge = edge.lock().unwrap(); *edge = Some(edge.map_or(frame.timestamp, |edge| later(edge, frame.timestamp))); @@ -540,51 +429,26 @@ impl Video { // joined receiver continues observing arrivals during this await. decoder.decode(frame).await?; } - Step::Resume => { - // Read under the same lock as the pop: a gap replaces the buffers and - // bumps the revision while holding it. + Step::Next => { let next = { let mut buffers = buffers.lock().unwrap(); // Eviction can empty a replacement until its next keyframe; one a // later replacement already follows never gets another. while buffers.len() > 2 && buffers[1].oldest().is_none() { - if let Some(transition) = buffers.remove(1).and_then(|skipped| skipped.resumes) { - role.abandon(transition); - } + buffers.remove(1); } - match buffers.get(1).and_then(|next| Some((next.oldest()?, next.resumes?))) { - Some((start, transition)) => { - advancing.store(true, std::sync::atomic::Ordering::Relaxed); - buffers.pop_front(); - advanced.notify_one(); - Some((start, transition, revision.load(std::sync::atomic::Ordering::Relaxed))) - } - None => None, + let next = buffers.get(1).is_some_and(|next| next.oldest().is_some()); + if next { + buffers.pop_front(); } + next }; - let Some((start, transition, before)) = next else { + if !next { if closed.load(std::sync::atomic::Ordering::Relaxed) { - break None; + break; } self.changed.notified().await; - continue; - }; - let mapping = role.resume(transition, start, None).ready().await?; - let buffers = buffers.lock().unwrap(); - if revision.load(std::sync::atomic::Ordering::Relaxed) != before { - advancing.store(false, std::sync::atomic::Ordering::Relaxed); - continue; - } - role.apply(transition, mapping); - *origin.lock().unwrap() = buffers.front().unwrap().after.map(|after| after + 1); - if let Some(end) = buffers.front().unwrap().end { - role.end(end)?; } - advancing.store(false, std::sync::atomic::Ordering::Relaxed); - for frame in &buffers.front().unwrap().frames { - role.received(role.map(frame.timestamp)?, Instant::now()); - } - self.output.send(Event::Wake); continue; } Step::Flush => { @@ -606,11 +470,11 @@ impl Video { held.retain(|held| held.0 > frame.timestamp); selector.push(Lane::Min, Decoded::Frame(frame), now); } - }; - Ok::<_, anyhow::Error>(drained) + } + Ok::<_, anyhow::Error>(()) }; // Pushes the max-delay decoder's pictures to the selector. It never moves the - // clock, and it only decodes media the min-delay decoder's segment maps. + // clock. let refresh = async { let Some(Refresh { track: mut refresh, @@ -635,9 +499,6 @@ impl Video { // point reference media this decoder never had. let mut recover = true; let mut leading = None; - // The group of the last end this decoder read. Its own subscription may - // deliver the replacement before the min-delay decoder reads that end. - let mut fence = None; let schedule = self.presentation.consume(); let mut shared = schedule.delay(); let mut held = Duration::ZERO; @@ -658,8 +519,8 @@ impl Video { continue; } }; - let (group, mut frame) = match read { - Some(Read::Frame { group, frame }) => (group, frame), + let frame = match read { + Some(Read::Frame { frame, .. }) => frame, Some(Read::Gap { .. }) => { codec.flush().await?; push(&mut codec)?; @@ -667,10 +528,9 @@ impl Video { continue; } // A fresh encoder has no preceding GOP to reference, so no recovery. - Some(Read::End { group, .. }) => { + Some(Read::End { .. }) => { codec.flush().await?; push(&mut codec)?; - fence = Some(group); continue; } None => { @@ -688,48 +548,23 @@ impl Video { if leading.is_some_and(|floor| frame.timestamp < floor) { continue; } - // Wait until the frame is in the segment the clock maps and due to decode: - // not ahead of the min-delay decoder, nor of the decode lead. - let mapped = loop { + // Wait until the frame is due to decode: not ahead of the min-delay + // decoder, nor of the decode lead. + loop { let woken = wake.notified(); - let at = { - let buffers = buffers.lock().unwrap(); - let front = buffers.front().unwrap(); - // A segment the clock already moved past. - if front.after.is_some_and(|after| group <= after) { - break None; - } - // A segment the clock has not mapped yet. - let ahead = front.until.is_some_and(|until| group > until) - || fence.is_some_and(|fence| { - group > fence && origin.lock().unwrap().is_none_or(|origin| origin <= fence) - }) || advancing.load(std::sync::atomic::Ordering::Relaxed); - if ahead { - // Nor ever will: the track closed without resuming it. - if buffers.len() == 1 && closed.load(std::sync::atomic::Ordering::Relaxed) { - break None; - } - None - } else { - drop(buffers); - // Before the segment's mapped start, so the clock moved past it too. - let Ok(mapped) = role.map(frame.timestamp) else { - break None; - }; - if !selector.lock().unwrap().wants(Lane::Max) { - None - } else if edge.lock().unwrap().is_some_and(|edge| mapped <= edge) { - break Some(mapped); - } else { - let due = self.presentation.due(mapped); - match due - .and_then(|at| at.checked_sub(DECODE_AHEAD)) - .filter(|at| *at > Instant::now()) - { - None => break Some(mapped), - at => at, - } - } + let at = if !selector.lock().unwrap().wants(Lane::Max) { + None + } else if edge.lock().unwrap().is_some_and(|edge| frame.timestamp <= edge) { + break; + } else { + match self + .presentation + .due(frame.timestamp) + .and_then(|at| at.checked_sub(DECODE_AHEAD)) + .filter(|at| *at > Instant::now()) + { + None => break, + at => at, } }; // A delay rise while parked must grow this subscription's budget now, @@ -748,11 +583,7 @@ impl Video { follow(&mut refresh, &mut held, shared); } } - }; - let Some(mapped) = mapped else { - continue; - }; - frame.timestamp = mapped; + } // Never race a codec call: cancelling Sink::decode poisons it. codec.decode(frame).await?; push(&mut codec)?; @@ -764,8 +595,7 @@ impl Video { Err(err) = refresh => Err(err), } }; - let (_, drained) = tokio::try_join!(receive, decode)?; - role.retire(drained); + tokio::try_join!(receive, decode)?; Ok(()) } } @@ -1377,7 +1207,7 @@ mod tests { } #[tokio::test] - async fn a_closed_track_holds_its_end_until_its_last_picture_is_due() { + async fn a_closed_track_queues_its_tail_without_waiting_for_it() { tokio::time::pause(); let delay = Duration::from_millis(200); let media = media(delay); @@ -1414,82 +1244,6 @@ mod tests { .due(moq_net::Timestamp::from_millis(66).unwrap()) .unwrap(); assert!(Instant::now() < last, "the track closed after its tail was due"); - let audio = media.presentation.track(moq_mux::container::Kind::Audio); - let transition = audio.declare(); - let _mapping = audio - .resume(transition, moq_net::Timestamp::ZERO, None) - .ready() - .await - .unwrap(); - // It maps one delay early, so its first write plays as the tail ends. - assert!( - Instant::now() >= last.checked_sub(delay).unwrap(), - "audio resumed over the video tail" - ); - } - - #[tokio::test] - async fn a_held_video_resume_leaves_the_excess_to_the_subscription() { - tokio::time::pause(); - let delay = Duration::from_millis(200); - let media = media(delay); - let broadcast = moq_net::broadcast::Info::new().produce(); - let source = broadcast - .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let subscriber = source - .consume() - .subscribe( - moq_net::track::Subscription::default().with_delay(moq_net::track::Delay::default().with_max(delay)), - ) - .await - .unwrap(); - let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); - let mut producer = Some(moq_mux::container::Producer::new(source, format())); - let mut _finished = None; - let track = moq_mux::container::Consumer::new(subscriber, format()); - let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0), None)); - let audio = media.presentation.track(moq_mux::container::Kind::Audio); - // Two-picture segments, one per restart, published live for two seconds - // while audio's end holds the first replacement back. - const SEGMENTS: u64 = 50; - let mut shown = 0; - // Presents until the track drains, which replaying every queued segment would - // take far longer than this bound. - for at in 0..60_000u64 { - if task.is_finished() { - break; - } - if at < 60 && at % 20 == 0 { - producer.as_mut().unwrap().write(frame(at, at == 0)).unwrap(); - } else if at == 60 { - audio.declare(); - audio.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); - } - if (60..60 + SEGMENTS * 40).contains(&at) && at % 20 == 0 { - if at % 40 == 20 { - producer.as_mut().unwrap().discontinuity().unwrap(); - } - producer.as_mut().unwrap().write(frame(at, at % 40 == 20)).unwrap(); - } else if at == 60 + SEGMENTS * 40 { - _finished = producer.take().map(|producer| producer.finish().unwrap()); - } - tokio::task::yield_now().await; - if media - .output - .present(&media) - .is_some_and(|picture| picture.as_millis().unwrap() >= 60) - { - shown += 1; - } - tokio::time::advance(Duration::from_millis(1)).await; - } - task.await.unwrap().unwrap(); - assert!(shown > 0, "the replacements played"); - assert!( - shown < SEGMENTS, - "pictures held behind the resume past the age budget were queued, not evicted: {shown}" - ); } #[tokio::test] @@ -1514,10 +1268,6 @@ mod tests { let mut producer = moq_mux::container::Producer::new(source, format()); let track = moq_mux::container::Consumer::new(subscriber, format()); let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0), None)); - // Audio's end two seconds out holds every replacement back. - let audio = media.presentation.track(moq_mux::container::Kind::Audio); - audio.declare(); - audio.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); producer.write(frame(0, true)).unwrap(); // Each replacement is one GOP longer than the age budget, so eviction // takes its only keyframe and leaves it empty. @@ -1530,24 +1280,11 @@ mod tests { .unwrap(); } } - tokio::time::advance(Duration::from_millis(500)).await; - tokio::task::yield_now().await; - media.output.present(&media); - // Once the old tail drained, audio must pair with the surviving replacement, - // rather than a discarded segment that still has a transition or a plan. - tokio::time::advance(Duration::from_secs(3)).await; - tokio::task::yield_now().await; - let start = moq_net::Timestamp::from_millis(3_000).unwrap(); - let transition = audio.declare(); - audio.plan(transition, start, Some(delay)); + // The emptied segments must give way to the one after them, rather than + // leave the decoder waiting on media that never comes. + producer.discontinuity().unwrap(); producer.write(frame(3_040, true)).unwrap(); let _finished = producer.finish().unwrap(); - audio.apply( - transition, - audio.resume(transition, start, Some(delay)).ready().await.unwrap(), - ); - let first = audio.map(start).unwrap(); - audio.received(first, Instant::now()); let mut shown = None; for _ in 0..1_000 { tokio::task::yield_now().await; @@ -1555,14 +1292,11 @@ mod tests { tokio::time::advance(Duration::from_millis(1)).await; } task.await.unwrap().unwrap(); - assert_eq!( - shown.map(|shown| Duration::try_from(shown).unwrap()), - Some(Duration::try_from(first).unwrap() + Duration::from_millis(40)) - ); + assert_eq!(shown.map(|shown| shown.as_millis().unwrap()), Some(3_040)); } #[tokio::test] - async fn a_waiting_resume_leaves_the_decoders_wakeup_alone() { + async fn an_idle_decoder_wakes_for_the_next_picture() { tokio::time::pause(); let delay = Duration::from_millis(200); let media = media(delay); @@ -1598,17 +1332,7 @@ mod tests { } }; step(3).await; - // Audio waits on an end ten seconds out while the decoder idles. - let audio = media.presentation.track(moq_mux::container::Kind::Audio); - let transition = audio.declare(); - audio.end(moq_net::Timestamp::from_secs(10).unwrap()).unwrap(); - let resume = tokio::spawn(async move { - audio - .resume(transition, moq_net::Timestamp::ZERO, None) - .ready() - .await - .map(|_| ()) - }); + // The decoder idles. for _ in 0..200 { tokio::task::yield_now().await; media.output.present(&media); @@ -1624,7 +1348,6 @@ mod tests { sent.elapsed() ); let _finished = producer.finish().unwrap(); - resume.abort(); for _ in 0..500 { tokio::task::yield_now().await; media.output.present(&media); @@ -1634,7 +1357,7 @@ mod tests { } #[tokio::test] - async fn a_quick_resume_flushes_and_plays_every_old_picture_first() { + async fn a_declared_end_flushes_and_plays_every_old_picture_first() { tokio::time::pause(); let delay = Duration::from_millis(200); let media = media(delay); @@ -1681,7 +1404,7 @@ mod tests { task.await.unwrap().unwrap(); assert_eq!(&shown[..3], [0, 33_000, 66_000]); assert_eq!(shown.len(), 6); - assert!(shown[3] > shown[2], "the resumed segment follows every old picture"); + assert!(shown[3] > shown[2], "the next segment follows every old picture"); assert_eq!(shown[4] - shown[3], 33_000); assert_eq!(shown[5] - shown[4], 33_000); assert_eq!( @@ -1889,8 +1612,7 @@ mod tests { } /// A join right after an encoder restart starts the max-delay decoder on the end, - /// a group before the min-delay one. The clock already maps the replacement, so - /// that end holds nothing back. + /// a group before the min-delay one. That end holds nothing back. #[tokio::test] async fn a_join_right_after_a_restart_still_hands_over() { tokio::time::pause(); @@ -1957,10 +1679,11 @@ mod tests { } /// The max-delay decoder reads its own subscription, so it can see an encoder - /// restart before the min-delay decoder does. It holds the replacement until the - /// clock maps it rather than placing it on the old encoder's timeline. + /// restart before the min-delay decoder does. Timestamps run on through the end, + /// so it plays the replacement when the shared clock says it is due, without + /// waiting for the min-delay decoder to read the end. #[tokio::test] - async fn a_restart_the_max_delay_decoder_reads_first_waits_for_the_clock() { + async fn a_restart_the_max_delay_decoder_reads_first_plays_on_the_clock() { tokio::time::pause(); let delay = Duration::from_millis(200); let media = media(delay); @@ -2034,9 +1757,12 @@ mod tests { let timestamps: Vec<_> = shown.iter().map(|(at, _)| *at).collect(); assert_eq!(×tamps[..3], [0, 33, 66], "{shown:?}"); assert_eq!(timestamps.len(), 6, "{shown:?}"); - assert!( - shown[3].1 >= late, - "the replacement played before the clock mapped it: {shown:?}" + // The old pictures all arrived at once, so the clock holds the newest: 66 ms + // played at the delay. + assert_eq!( + shown[3].1, + delay + Duration::from_millis(1_000 - 66), + "the replacement missed its due time: {shown:?}" ); assert_eq!(timestamps[4] - timestamps[3], 33, "{shown:?}"); assert_eq!(timestamps[5] - timestamps[4], 33, "{shown:?}"); diff --git a/rs/moq-mux/CHANGELOG.md b/rs/moq-mux/CHANGELOG.md index 06322f35fa7..1a34c7dc83e 100644 --- a/rs/moq-mux/CHANGELOG.md +++ b/rs/moq-mux/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `sync::Consumer::poll_changed` (and `changed` with the `async` feature) waits until the playout schedule differs from a `sync::Schedule` read earlier with `Consumer::schedule`, and returns the new one. It resolves on a re-pinned reference, a moved delay, a lost anchor, or a producer reset onto a clock that schedules differently, and stays parked through transition bookkeeping that moves no due time. A presenter asleep on a picture's due time can wake and re-schedule instead of polling. `Schedule::due` and `Schedule::delay` read the snapshot it waits on. +- `sync::Consumer::poll_changed` (and `changed` with the `async` feature) waits until the playout schedule differs from a `sync::Schedule` read earlier with `Consumer::schedule`, and returns the new one. It resolves on a re-pinned reference, a moved delay, a lost anchor, or a producer reset onto a clock that schedules differently, and stays parked through anything that moves no due time. A presenter asleep on a picture's due time can wake and re-schedule instead of polling. `Schedule::due` and `Schedule::delay` read the snapshot it waits on. - MP4 and Matroska import fill a video rendition's `projection` and `stereo` from Spherical Video V2's `sv3d`/`st3d` boxes and Matroska's `Projection`/`StereoMode`, and Opus channel mapping family 2 (raw Opus, Matroska, and other OpusHead sources) imports with `layout: "ambisonic"`. What the catalog cannot describe is refused with `fmp4::Error::UnsupportedSpatial`, `mkv::Error::UnsupportedSpatial`, or `opus::Error::UnsupportedChannelCount`: a cubemap or mesh projection, a nonzero pose, uncentred or vertically cropped equirectangular bounds, a stereo packing other than mono, left-right, or top-bottom with the left eye first, and family 2 at zeroth order or with RFC 8486's non-diegetic stereo pair. - fMP4 carries surround Opus: a `dOps` channel mapping table (family 1, 2, or 255) imports into the OpusHead description, and an exported `dOps` writes the description's table back instead of refusing it with `opus::Error::UnsupportedMappingFamily`. Family 3, whose OpusHead carries a demixing matrix rather than a table, is still refused both ways. - `container::mp4::Export::stop` ends a recording before the broadcast does: `next` still returns the fragment each track was buffering, then `None`, so `finish` indexes it. diff --git a/rs/moq-mux/src/sync.rs b/rs/moq-mux/src/sync.rs index 2f28607c857..a391066e936 100644 --- a/rs/moq-mux/src/sync.rs +++ b/rs/moq-mux/src/sync.rs @@ -18,17 +18,17 @@ //! reads the [`Schedule`] it sleeps on and wakes when //! [`Consumer::poll_changed`] reports that it moved. //! -//! A track also maps encoder segments onto the clock across declared ends. -//! Keep one active track per [`Kind`]; retire it before -//! starting its successor. Bound replacement media with the caller's retention -//! budget and [`Track::abandon`] transitions whose replacement is discarded. +//! Timestamps keep moving forward across a declared end, so the clock has no +//! notion of one: media after it lands on the same schedule. A track whose +//! timeline breaks, such as at lost media or a later group after a declared end, +//! calls [`Track::reset`] so its arrivals stop holding the clock until the next +//! one, as `@moq/watch`'s `Sync` does. Keep one active track per [`Kind`]. use kio::clock::Instant; use std::time::Duration; use moq_net::Timestamp; use std::{ - pin::Pin, sync::{Arc, Mutex}, task::Poll, }; @@ -166,46 +166,6 @@ struct State { /// The schedule the last track to go left behind, which what it already /// queued still presents against until any track anchors again. left: Option, - /// Endpoints declared by each live role, on the shared playback timeline. - ends: [Option; KINDS], - /// When each retired role's tail finishes playing, frozen on the wall clock: - /// a later rendition anchors afresh, which would carry a media endpoint - /// onto an unrelated timeline. Kept apart from `ends` so a successor role - /// never clears its predecessor's tail. - tails: Vec, - /// Encoder restarts whose replacements have not all started playing. - transitions: std::collections::VecDeque, - /// The latest transition each role declared. - declared: [Transition; KINDS], - next: Transition, -} - -/// One encoder restart, shared by the roles whose ends declared it. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub struct Transition(u64); - -struct Pending { - id: Transition, - /// Roles whose declared end opened or joined this transition. - members: [bool; KINDS], - /// Members that applied its mapping. - resumed: [bool; KINDS], - /// Each member's first replacement media. - plans: [Option; KINDS], - mapping: Option, -} - -struct Boundary { - starts: Option, - end: Duration, -} - -#[derive(Clone, Copy)] -struct Plan { - source: Duration, - delay: Duration, - /// When the replacement media arrived. - arrived: Instant, } impl State { @@ -214,11 +174,6 @@ impl State { delay, parts: Default::default(), left: None, - ends: Default::default(), - tails: Vec::new(), - transitions: Default::default(), - declared: [Transition(0); KINDS], - next: Transition(1), } } @@ -282,27 +237,6 @@ impl State { before != self.delay() } - /// How many transitions are still pending. - #[cfg(test)] - fn pending(&self) -> usize { - self.transitions.len() - } - - fn transition(&mut self, id: Transition) -> Option<&mut Pending> { - self.transitions.iter_mut().find(|pending| pending.id == id) - } - - /// Forget transitions nobody resumes, or that every member resumed once - /// their replacement started and nothing can join them. - fn prune(&mut self, now: Instant) { - self.transitions.retain(|pending| { - let members = pending.members.iter().any(|member| *member); - let resumed = (0..KINDS).all(|role| !pending.members[role] || pending.resumed[role]); - let started = pending.mapping.is_some_and(|mapping| mapping.started(now)); - members && !(resumed && started) - }); - } - /// The media position the clock will reach at `at`. fn position(&self, at: Instant) -> Option { let reference = self.reference()?; @@ -361,81 +295,24 @@ impl PartialEq for Schedule { } } -/// One track's hold on the shared clock: its arrivals and target, and the -/// mapping of its encoder segments onto the shared timeline. +/// One track's hold on the shared clock: its arrivals and target. /// /// Dropping it releases its arrivals and target. pub struct Track { clock: Arc, role: Kind, - mapping: std::sync::Mutex, -} - -/// The current segment's mapping, and how long it held the segment's media -/// back behind the old segment's tail. -#[derive(Clone, Copy)] -struct Segment { - mapping: Mapping, - /// Added to each arrival: media held behind a tail arrived that much earlier - /// than its schedule, which says nothing about the live edge. - held: Duration, -} - -/// A segment's timestamp mapping, applied only after its first media can play. -#[derive(Clone, Copy)] -#[must_use] -pub struct Mapping { - source: Duration, - target: Duration, - starts: Option, -} - -impl Mapping { - fn map(self, source: Timestamp) -> Result { - let source = Duration::try_from(source).map_err(|_| Error::Unscaled)?; - let target = if source >= self.source { - self.target.checked_add(source - self.source).ok_or(Error::Overflow)? - } else { - self.target - .checked_sub(self.source - source) - .ok_or(Error::BeforeOrigin)? - }; - Timestamp::from_micros(target.as_micros().try_into().map_err(|_| Error::Overflow)?).map_err(|_| Error::Overflow) - } - - fn started(&self, now: Instant) -> bool { - self.starts.is_none_or(|starts| starts <= now) - } } impl Track { - fn new(clock: Arc, role: Kind) -> Self { - Self { - clock, - role, - mapping: std::sync::Mutex::new(Segment { - mapping: Mapping { - source: Duration::ZERO, - target: Duration::ZERO, - starts: None, - }, - held: Duration::ZERO, - }), - } - } - /// Fold a frame that arrived at `now` into the clock, reporting whether that /// moved the schedule. /// - /// `timestamp` is on the shared timeline, as [`map`](Self::map) returns it. /// A frame arriving earlier than the schedule predicted pulls playback toward /// live; a late one leaves it alone. A move makes every queued frame due /// earlier, so a presenter has to recompute the deadline it is asleep on. /// An [unscaled](Timestamp::unscaled) timestamp has no media time, so it /// moves nothing. pub fn received(&self, timestamp: Timestamp, now: Instant) -> bool { - let held = self.mapping.lock().unwrap().held; - let now = now.checked_add(held).unwrap_or(now); let moved = self.clock.state.lock().unwrap().received(self.role, timestamp, now); if moved { self.clock.wake(); @@ -443,14 +320,10 @@ impl Track { moved } - /// End this track's timeline at a discontinuity, such as lost media or a - /// rewind: its arrivals stop holding the clock until the next one. - /// - /// Media after the break was not held behind any tail, so its arrivals are - /// the live edge again. Keeping the hold would anchor the clock that much - /// late and make the media wait for it. + /// End this track's timeline at a discontinuity, such as lost media, a rewind, + /// or a later group after a declared end: its arrivals stop holding the clock + /// until the next one. pub fn reset(&self) { - self.mapping.lock().unwrap().held = Duration::ZERO; if self.clock.state.lock().unwrap().reset(self.role) { self.clock.wake(); } @@ -477,308 +350,15 @@ impl Track { })); Poll::Ready(delay) } - - /// Stop while a tail still plays until `drained`, holding a declared end until - /// then. Dropping instead discards the tail with the end. - pub fn retire(self, drained: Option) { - let now = kio::clock::now(); - let mut state = self.clock.state.lock().unwrap(); - state.tails.retain(|tail| *tail > now); - if state.ends[self.role as usize].take().is_some() - && let Some(drained) = drained - { - state.tails.push(drained); - } - } - - /// Map a source timestamp onto the shared broadcast timeline. Refuses an - /// [unscaled](Timestamp::unscaled) one with [`Error::Unscaled`]. - pub fn map(&self, source: Timestamp) -> Result { - self.mapping.lock().unwrap().mapping.map(source) - } - - /// Declare that this role's encoder stopped, returning the transition its - /// replacement resumes. - /// - /// A peer's transition this role has not joined yet is the same restart, as - /// long as its replacement has not started playing; otherwise this one opens - /// a transition of its own. - pub fn declare(&self) -> Transition { - let now = kio::clock::now(); - let own = self.role as usize; - let mut state = self.clock.state.lock().unwrap(); - state.prune(now); - let after = state.declared[own]; - let joined = state - .transitions - .iter_mut() - .filter(|transition| transition.id > after) - .find(|transition| { - !transition.members[own] - && transition.members.iter().any(|member| *member) - && transition.mapping.is_none_or(|mapping| !mapping.started(now)) - }) - .map(|transition| { - transition.members[own] = true; - transition.id - }); - let id = joined.unwrap_or_else(|| { - let id = state.next; - state.next = Transition(id.0 + 1); - let mut members = [false; KINDS]; - members[own] = true; - state.transitions.push_back(Pending { - id, - members, - resumed: [false; KINDS], - plans: [None; KINDS], - mapping: None, - }); - id - }); - state.declared[own] = id; - drop(state); - self.clock.wake(); - id - } - - /// Mark where this role's current segment ends on the shared clock. - pub fn end(&self, source: Timestamp) -> Result<()> { - let mapping = self.mapping.lock().unwrap().mapping; - self.clock.state.lock().unwrap().ends[self.role as usize] = Some(Boundary { - starts: mapping.starts, - end: Duration::try_from(mapping.map(source)?).expect("a mapped timestamp is in microseconds"), - }); - self.clock.wake(); - Ok(()) - } - - /// Record the first replacement media for `transition`, as it arrives. - /// - /// An [unscaled](Timestamp::unscaled) `source` has no media time to plan - /// from, so it records nothing and this role maps alone. - pub fn plan(&self, transition: Transition, source: Timestamp, delay: Option) { - let Ok(source) = Duration::try_from(source) else { - return; - }; - let arrived = kio::clock::now(); - let mut state = self.clock.state.lock().unwrap(); - let delay = delay.unwrap_or(state.delay()); - if let Some(pending) = state.transition(transition) { - pending.plans[self.role as usize] = Some(Plan { source, delay, arrived }); - } - drop(state); - self.clock.wake(); - } - - /// Media was lost: nothing waits for the discarded tail. - pub fn gap(&self) { - self.clock.state.lock().unwrap().ends[self.role as usize] = None; - self.clock.wake(); - } - - /// This role will never resume `transition`, so a peer must not pair with its plan. - pub fn abandon(&self, transition: Transition) { - let own = self.role as usize; - let mut state = self.clock.state.lock().unwrap(); - if let Some(pending) = state.transition(transition) { - pending.members[own] = false; - pending.plans[own] = None; - } - state.prune(kio::clock::now()); - drop(state); - self.clock.wake(); - } - - /// Commit the mapping once its replacement media starts playing. - pub fn apply(&self, transition: Transition, mapping: Mapping) { - let mut segment = self.mapping.lock().unwrap(); - let mut state = self.clock.state.lock().unwrap(); - let reference = state.reference(); - let mut held = Duration::ZERO; - if let Some(pending) = state.transition(transition) { - pending.resumed[self.role as usize] = true; - // How much earlier the replacement arrived than the schedule it now - // plays on, having waited for the old tail to drain. - if let Some(plan) = pending.plans[self.role as usize] - && let Some(scheduled) = mapping - .map(media(plan.source)) - .ok() - .zip(reference) - .and_then(|(mapped, reference)| reference.at(Duration::try_from(mapped).ok()?)) - { - held = scheduled.saturating_duration_since(plan.arrived); - } - } - *segment = Segment { mapping, held }; - drop(segment); - state.prune(kio::clock::now()); - drop(state); - self.clock.wake(); - } - - /// Prepare a mapping one delay before the old media drains; dropping the wait leaves the track unchanged. - pub fn resume(&self, transition: Transition, source: Timestamp, delay: Option) -> Resume<'_> { - Resume { - track: self, - transition, - source, - delay, - timer: None, - } - } } impl Drop for Track { fn drop(&mut self) { - let own = self.role as usize; - { - let mut state = self.clock.state.lock().unwrap(); - state.release(self.role); - state.ends[own] = None; - for pending in state.transitions.iter_mut().filter(|pending| !pending.resumed[own]) { - pending.members[own] = false; - pending.plans[own] = None; - } - state.declared[own] = Transition(0); - state.prune(kio::clock::now()); - } + self.clock.state.lock().unwrap().release(self.role); self.clock.wake(); } } -/// A duration from the start of the track as a wire timestamp. -fn media(duration: Duration) -> Timestamp { - // A timestamp caps at 2^61 - 1, which is ~73,000 years of microseconds. - const MAX: u128 = (1 << 61) - 1; - Timestamp::from_micros(duration.as_micros().min(MAX) as u64).expect("clamped to the timestamp maximum") -} - -/// A pending segment mapping, driven by the caller's executor or waiter. -#[must_use = "drive the wait before applying its mapping"] -pub struct Resume<'a> { - track: &'a Track, - transition: Transition, - source: Timestamp, - delay: Option, - timer: Option, -} - -impl Resume<'_> { - /// The replacement's mapping, once all declared old media is due to drain within one delay. - #[cfg_attr(feature = "async", kio::wrapper)] - pub fn poll_ready(&mut self, waiter: &kio::Waiter) -> Poll> { - let _ = self.track.clock.changed.poll(waiter, |_| Poll::Pending); - let now = kio::clock::now(); - let mut state = self.track.clock.state.lock().unwrap(); - if let Some(mapping) = state.transition(self.transition).and_then(|pending| pending.mapping) { - return Poll::Ready(Ok(mapping)); - } - let tail = state - .ends - .iter() - .flatten() - .filter(|boundary| boundary.starts.is_none_or(|start| start <= now)) - .filter_map(|boundary| state.due(media(boundary.end))) - .chain(state.tails.iter().copied()) - .max(); - // Wake one delay before the tail ends: media written then plays as it ends. - let lead = self.delay.unwrap_or(state.delay()); - if let Some(wake) = tail - .map(|end| end.checked_sub(lead).unwrap_or(now)) - .filter(|wake| *wake > now) - { - let timer = self.timer.get_or_insert_with(|| kio::clock::sleep_until(wake)); - if timer.deadline() != wake { - timer.reset(wake); - } - std::task::ready!(waiter.poll_future(Pin::new(timer))); - } - - let own = self.track.role as usize; - // A transition abandoned under this resume, by a gap the caller rechecks, - // maps alone like one no peer joined. - let (mapping, paired, own_plan) = state.transition(self.transition).map_or((None, None, None), |pending| { - let mut plans = pending - .members - .iter() - .zip(&pending.plans) - .filter(|(member, _)| **member); - let joined = pending.members[own] && plans.clone().count() > 1; - let origin = plans - .try_fold(None::, |earliest, (_, plan)| { - let plan = (*plan)?; - Some(Some(match earliest { - Some(earliest) if earliest.source <= plan.source => earliest, - _ => plan, - })) - }) - .flatten(); - (pending.mapping, origin.filter(|_| joined), pending.plans[own]) - }); - // The peer already resumed this transition. - if let Some(mapping) = mapping { - return Poll::Ready(Ok(mapping)); - } - // Every member has replacement media. The earliest defines their common - // origin, keeping the media offsets between them, even when its delay - // differs from the others'. Otherwise this role maps alone and a peer - // joining later follows it. - let (source, delay, arrived) = match paired { - Some(origin) => (origin.source, origin.delay, Some(origin.arrived)), - None => ( - Duration::try_from(self.source).map_err(|_| Error::Unscaled)?, - self.delay.unwrap_or(state.delay()), - own_plan.map(|plan| plan.arrived), - ), - }; - // Media that arrived before the old tail ended continues its schedule: - // it starts one delay after arriving, or as the tail ends if later, so - // back-to-back segments add no hold. Only after a real gap, where the - // sink underruns anyway, does it re-buffer one delay from now. - let starts = match (tail, arrived) { - (Some(end), Some(arrived)) if arrived <= end => { - // One tick past the end: a tail whose last frame declared no - // duration ends at that frame's timestamp, which the first new - // frame must not share. - let after = end.checked_add(Duration::from_micros(1)).ok_or(Error::DelayOverflow)?; - arrived.checked_add(delay).ok_or(Error::DelayOverflow)?.max(after) - } - _ => now.checked_add(delay).ok_or(Error::DelayOverflow)?, - }; - let mapping = Mapping { - source, - target: state.position(starts).unwrap_or(source), - starts: Some(starts), - }; - if let Some(pending) = state.transition(self.transition) { - pending.mapping = Some(mapping); - } - Poll::Ready(Ok(mapping)) - } -} - -/// A timestamp or delay cannot be represented on the playout clock. -#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] -#[non_exhaustive] -pub enum Error { - /// Mapping the timestamp would overflow. - #[error("playback timeline overflow")] - Overflow, - /// A picture precedes the mapped segment's origin. - #[error("picture precedes the playback timeline")] - BeforeOrigin, - /// The requested delay exceeds the system clock. - #[error("playback delay exceeds the system clock")] - DelayOverflow, - /// The timestamp is [unscaled](Timestamp::unscaled), a scale-less track's key with no media time. - #[error("timestamp has no timescale")] - Unscaled, -} - -/// A result from mapping media onto the playout clock. -pub type Result = std::result::Result; - /// Owns one broadcast's playout clock and hands out its tracks. /// /// Clones capture this broadcast's clock; resetting a handle never changes the @@ -833,7 +413,10 @@ impl Producer { /// Return a track bound to this broadcast's clock. pub fn track(&self, kind: Kind) -> Track { - Track::new(self.clock.clone(), kind) + Track { + clock: self.clock.clone(), + role: kind, + } } /// When `timestamp` should be presented, or `None` before any frame has @@ -920,22 +503,11 @@ mod tests { use super::*; const DELAY: Duration = Duration::from_millis(100); - /// How far past the tail's end a continuing segment starts. - const TICK: Duration = Duration::from_micros(1); fn ms(millis: u64) -> Timestamp { Timestamp::from_millis(millis).unwrap() } - /// A clock with an audio and a video track, audio anchored at 10 s arriving `now`. - fn anchored(now: Instant) -> (Arc, Track, Track) { - let clock = Arc::new(Clock::new(DELAY)); - let audio = Track::new(clock.clone(), Kind::Audio); - let video = Track::new(clock.clone(), Kind::Video); - audio.received(ms(10_000), now); - (clock, audio, video) - } - #[derive(serde::Deserialize)] struct Case { name: String, @@ -1157,501 +729,21 @@ mod tests { ); } - #[tokio::test] - async fn replacing_a_broadcast_isolates_its_old_update_handles() { - tokio::time::pause(); + #[test] + fn replacing_a_broadcast_isolates_its_old_update_handles() { + let now = Instant::ZERO.saturating_add(Duration::from_secs(1)); let clock = Producer::new(DELAY); let reader = clock.consume(); let previous = clock.clone(); let old = clock.track(Kind::Audio); - old.declare(); let clock = clock.reset(DELAY); - let now = kio::clock::now(); let audio = clock.track(Kind::Audio); audio.received(ms(0), now); - audio.declare(); - audio.end(ms(1_000)).unwrap(); // A JoinSet abort can leave these updates and drops until after the new track started. + old.received(ms(10_000), now); drop(old); previous.track(Kind::Audio).received(ms(10_000), now); assert_eq!(reader.due(ms(0)), Some(now.saturating_add(DELAY))); - let video = clock.track(Kind::Video); - let transition = video.declare(); - video.plan(transition, ms(0), None); - let resumed = kio::clock::timeout( - DELAY + Duration::from_millis(500), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resumed.is_err(), "old handle cleared the new audio tail"); - } - - #[tokio::test] - async fn ordinary_arrivals_leave_a_parked_resume_asleep() { - tokio::time::pause(); - let clock = Producer::new(DELAY); - let now = kio::clock::now(); - let audio = clock.track(Kind::Audio); - audio.received(ms(0), now); - audio.end(ms(1_000)).unwrap(); - let video = clock.track(Kind::Video); - let transition = video.declare(); - video.plan(transition, ms(0), None); - let (wakes, waiter) = Wakes::waiter(); - let mut resume = video.resume(transition, ms(0), None); - assert!(resume.poll_ready(&waiter).is_pending()); - let later = now.saturating_add(Duration::from_millis(20)); - video.received(ms(20), later); - audio.received(ms(20), later); - assert_eq!(wakes.count(), 0); - audio.received(ms(40), later); - assert!(wakes.count() > 0, "a moved schedule must wake the resume"); - } - - #[test] - fn discarded_replacements_release_their_transition_state() { - let clock = Arc::new(Clock::new(Duration::ZERO)); - let video = Track::new(clock.clone(), Kind::Video); - let audio = Track::new(clock.clone(), Kind::Audio); - let first = audio.declare(); - assert_eq!(first, video.declare()); - // Audio can hold its declared end while video discards any number of replacements. - for _ in 0..100 { - let transition = video.declare(); - video.plan(transition, Timestamp::ZERO, None); - video.abandon(transition); - assert_eq!(clock.state.lock().unwrap().pending(), 1); - } - video.abandon(first); - drop(audio); - assert_eq!(clock.state.lock().unwrap().pending(), 0); - } - - #[tokio::test] - async fn a_data_track_joins_a_paired_restart() { - tokio::time::pause(); - let now = kio::clock::now(); - let (clock, audio, video) = anchored(now); - let data = Track::new(clock.clone(), Kind::Data); - let transition = audio.declare(); - assert_eq!(video.declare(), transition); - assert_eq!(data.declare(), transition, "one restart"); - audio.plan(transition, ms(0), Some(DELAY)); - video.plan(transition, ms(40), None); - tokio::time::advance(DELAY).await; - video.apply( - transition, - video.resume(transition, ms(40), None).ready().await.unwrap(), - ); - // The data track has no replacement yet, so the others map alone. - assert_eq!(video.map(ms(40)).unwrap().as_millis().unwrap(), 10_100); - data.plan(transition, ms(20), None); - data.apply(transition, data.resume(transition, ms(20), None).ready().await.unwrap()); - assert_eq!(data.map(ms(40)).unwrap(), video.map(ms(40)).unwrap()); - } - - #[tokio::test] - async fn one_ended_role_maps_to_the_continuous_peers_clock() { - tokio::time::pause(); - let start = kio::clock::now(); - let clock = Arc::new(Clock::new(DELAY)); - let audio = Track::new(clock.clone(), Kind::Audio); - let video = Track::new(clock.clone(), Kind::Video); - video.received(ms(10_000), start); - let transition = audio.declare(); - audio.end(ms(10_020)).unwrap(); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - let unchanged = clock.state.lock().unwrap().due(ms(10_500)); - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - assert_eq!(audio.map(ms(0)).unwrap().as_millis().unwrap(), 10_120); - assert_eq!(video.map(ms(10_500)).unwrap().as_millis().unwrap(), 10_500); - assert_eq!(clock.state.lock().unwrap().due(ms(10_500)), unchanged); - } - - /// A replacement that arrived while the old tail played waits for it, so it - /// plays later than it arrived. Its arrivals are on time for that schedule, - /// not early: they must not drag the other tracks ahead. - #[tokio::test] - async fn media_held_behind_a_tail_leaves_the_schedule_alone() { - tokio::time::pause(); - let start = kio::clock::now(); - let (clock, audio, video) = anchored(start); - video.received(ms(10_000), start); - let transition = audio.declare(); - audio.end(ms(10_100)).unwrap(); - // The replacement starts over at zero and lands at once. - audio.plan(transition, ms(0), Some(DELAY)); - let mapping = audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(); - audio.apply(transition, mapping); - assert!( - kio::clock::now() >= start.saturating_add(DELAY), - "the replacement waited until one delay before the tail ends" - ); - let unchanged = clock.state.lock().unwrap().due(ms(10_500)); - for frame in 0..10 { - let at = start.saturating_add(Duration::from_millis(frame * 20)); - assert!( - !audio.received(audio.map(ms(frame * 20)).unwrap(), at), - "held media moved the schedule" - ); - } - assert_eq!(clock.state.lock().unwrap().due(ms(10_500)), unchanged); - } - - /// Media after a reset was not held behind the tail, so it plays one delay - /// after it arrives rather than waiting out the old hold too. - #[tokio::test] - async fn a_reset_drops_the_hold_behind_a_tail() { - tokio::time::pause(); - let start = kio::clock::now(); - let clock = Arc::new(Clock::new(DELAY)); - let audio = Track::new(clock.clone(), Kind::Audio); - audio.received(ms(10_000), start); - let transition = audio.declare(); - audio.end(ms(10_100)).unwrap(); - audio.plan(transition, ms(0), Some(DELAY)); - let mapping = audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(); - audio.apply(transition, mapping); - - audio.reset(); - let now = kio::clock::now(); - let next = audio.map(ms(1_000)).unwrap(); - audio.received(next, now); - assert_eq!(clock.state.lock().unwrap().due(next), now.checked_add(DELAY)); - } - - /// A press arriving while the previous one still plays continues its - /// schedule, so back-to-back presses add no delay; one after a gap - /// re-buffers one delay. - #[tokio::test] - async fn back_to_back_segments_keep_their_delay() { - tokio::time::pause(); - let start = kio::clock::now(); - let clock = Arc::new(Clock::new(DELAY)); - let audio = Track::new(clock.clone(), Kind::Audio); - audio.received(ms(0), start); - for press in 1..=5u64 { - // Each press lasts 1 s, and the next arrives as it ends. - let transition = audio.declare(); - audio.end(ms(1_000)).unwrap(); - tokio::time::advance(start.saturating_add(Duration::from_secs(press)) - kio::clock::now()).await; - let arrived = kio::clock::now(); - audio.plan(transition, ms(0), Some(DELAY)); - let mapping = audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(); - audio.apply(transition, mapping); - // Each press starts one tick past the tail it touches. - assert_eq!( - clock.state.lock().unwrap().due(audio.map(ms(0)).unwrap()), - arrived.checked_add(DELAY + TICK * press as u32), - "press {press} drifted" - ); - } - - // After a gap, the tail has drained: it re-buffers one delay from now. - let transition = audio.declare(); - audio.end(ms(1_000)).unwrap(); - tokio::time::advance(Duration::from_secs(5)).await; - let arrived = kio::clock::now(); - audio.plan(transition, ms(0), Some(DELAY)); - let mapping = audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(); - audio.apply(transition, mapping); - assert_eq!( - clock.state.lock().unwrap().due(audio.map(ms(0)).unwrap()), - arrived.checked_add(DELAY) - ); - } - - #[tokio::test] - async fn a_late_paired_replacement_keeps_its_media_offset() { - tokio::time::pause(); - let now = kio::clock::now(); - let (_clock, audio, video) = anchored(now); - let transition = audio.declare(); - assert_eq!(video.declare(), transition, "one restart"); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - audio.plan(transition, ms(0), Some(DELAY)); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - // The video replacement lands after the resumed audio has started playing. - tokio::time::advance(DELAY * 2).await; - video.plan(transition, ms(40), None); - video.apply( - transition, - video.resume(transition, ms(40), None).ready().await.unwrap(), - ); - assert_eq!( - video.map(ms(40)).unwrap().as_millis().unwrap() - audio.map(ms(0)).unwrap().as_millis().unwrap(), - 40 - ); - } - - #[tokio::test] - async fn a_late_replacement_pairs_with_its_own_transition() { - tokio::time::pause(); - let now = kio::clock::now(); - let (_clock, audio, video) = anchored(now); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - video.plan(transition, ms(40), None); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.apply( - transition, - video.resume(transition, ms(40), None).ready().await.unwrap(), - ); - // Video already queued its next replacement when audio's first one lands. - let next = video.declare(); - assert_ne!(next, transition); - video.end(ms(60)).unwrap(); - video.plan(next, ms(5_000), None); - audio.plan(transition, ms(0), Some(DELAY)); - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - assert_eq!( - video.map(ms(40)).unwrap().as_millis().unwrap() - audio.map(ms(0)).unwrap().as_millis().unwrap(), - 40 - ); - } - - #[tokio::test] - async fn a_peer_two_restarts_ahead_pairs_with_the_first() { - tokio::time::pause(); - let now = kio::clock::now(); - let (_clock, audio, video) = anchored(now); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.plan(transition, ms(40), None); - let first = video.resume(transition, ms(40), None).ready().await.unwrap(); - video.apply(transition, first); - // Video restarts again and resumes before audio's first replacement lands. - let next = video.declare(); - video.plan(next, ms(5_000), None); - tokio::time::advance(Duration::from_millis(10)).await; - video.apply(next, video.resume(next, ms(5_000), None).ready().await.unwrap()); - audio.plan(transition, ms(0), Some(DELAY)); - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - assert_eq!( - first.map(ms(40)).unwrap().as_millis().unwrap() - audio.map(ms(0)).unwrap().as_millis().unwrap(), - 40 - ); - } - - #[tokio::test] - async fn an_abandoned_replacement_leaves_the_next_transition_alone() { - tokio::time::pause(); - let now = kio::clock::now(); - let (clock, audio, video) = anchored(now); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - // Video's first replacement is evicted before it plays, then it restarts again. - video.plan(transition, ms(40), None); - video.abandon(transition); - let next = video.declare(); - video.plan(next, ms(5_000), None); - audio.plan(transition, ms(0), Some(DELAY)); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - audio.apply( - transition, - audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), - ); - video.apply(next, video.resume(next, ms(5_000), None).ready().await.unwrap()); - // The later replacement maps onto the running clock, not the audio - // restart's, continuing as the tail ends since it arrived before then. - let resumed = kio::clock::now().saturating_add(TICK); - assert_eq!( - clock.state.lock().unwrap().due(video.map(ms(5_000)).unwrap()), - Some(resumed) - ); - } - - #[tokio::test] - async fn an_abandoned_peer_plan_is_not_paired() { - tokio::time::pause(); - let now = kio::clock::now(); - let (clock, audio, video) = anchored(now); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - audio.plan(transition, ms(0), Some(Duration::from_millis(80))); - audio.abandon(transition); - video.plan(transition, ms(40), None); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.apply( - transition, - video.resume(transition, ms(40), None).ready().await.unwrap(), - ); - // Video arrived before the tail ended, so it continues as the tail ends. - let resumed = kio::clock::now().saturating_add(TICK); - assert_eq!( - clock.state.lock().unwrap().due(video.map(ms(40)).unwrap()), - Some(resumed) - ); - } - - #[tokio::test] - async fn a_resumed_transition_ignores_the_peers_later_end() { - tokio::time::pause(); - let now = kio::clock::now(); - let (_clock, audio, video) = anchored(now); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - video.plan(transition, ms(40), None); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - let mapping = video.resume(transition, ms(40), None).ready().await.unwrap(); - video.apply(transition, mapping); - // The resumed video plays, then its encoder stops again far ahead. - tokio::time::advance(DELAY * 2).await; - video.declare(); - video.end(ms(60_000)).unwrap(); - audio.plan(transition, ms(0), Some(DELAY)); - let resumed = kio::clock::timeout( - Duration::from_millis(1), - audio.resume(transition, ms(0), Some(DELAY)).ready(), - ) - .await; - assert!(resumed.is_ok(), "audio waited for video's next end"); - } - - #[tokio::test] - async fn a_cancelled_role_releases_its_declared_end() { - tokio::time::pause(); - let now = kio::clock::now(); - let (_clock, audio, video) = anchored(now); - // Video holds the schedule once audio goes. - video.received(ms(10_000), now); - audio.declare(); - audio.end(ms(10_200)).unwrap(); - // Cancelled with its sink, so nothing plays the tail. - drop(audio); - let transition = video.declare(); - video.end(ms(10_000)).unwrap(); - video.plan(transition, ms(0), None); - let resume = kio::clock::timeout( - DELAY + Duration::from_millis(150), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resume.is_ok(), "video waited for discarded audio"); - } - - #[tokio::test] - async fn a_closed_role_still_holds_its_declared_end() { - tokio::time::pause(); - let now = kio::clock::now(); - let (_clock, audio, video) = anchored(now); - audio.declare(); - audio.end(ms(10_200)).unwrap(); - // The track closed, but its returned sink still plays the tail up to the end. - audio.retire(Some(now.saturating_add(Duration::from_millis(300)))); - let transition = video.declare(); - video.end(ms(10_000)).unwrap(); - video.plan(transition, ms(0), None); - // It may map one delay before the tail ends, so its media plays as the tail ends. - let resume = kio::clock::timeout( - Duration::from_millis(150), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resume.is_err(), "video resumed before the audio tail played"); - } - - #[tokio::test] - async fn a_retired_end_ignores_a_later_re_pin() { - tokio::time::pause(); - let now = kio::clock::now(); - let (clock, audio, video) = anchored(now); - audio.declare(); - audio.end(ms(10_200)).unwrap(); - audio.retire(Some(now.saturating_add(Duration::from_millis(300)))); - // A replacement rendition starts over at zero and anchors afresh. - let successor = Track::new(clock.clone(), Kind::Audio); - successor.received(ms(50), now); - let transition = video.declare(); - video.plan(transition, ms(0), None); - let resume = kio::clock::timeout( - Duration::from_millis(400), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resume.is_ok(), "video waited past the drained tail"); - } - - #[tokio::test] - async fn a_cancelled_successor_keeps_a_retired_tail() { - tokio::time::pause(); - let now = kio::clock::now(); - let (clock, audio, video) = anchored(now); - audio.declare(); - audio.end(ms(10_200)).unwrap(); - audio.retire(Some(now.saturating_add(Duration::from_millis(300)))); - // The next rendition fails while the retired sink still plays. - let successor = Track::new(clock.clone(), Kind::Audio); - successor.declare(); - drop(successor); - let transition = video.declare(); - video.plan(transition, ms(0), None); - let resume = kio::clock::timeout( - Duration::from_millis(150), - video.resume(transition, ms(0), None).ready(), - ) - .await; - assert!(resume.is_err(), "video resumed over the retired audio tail"); - } - - #[tokio::test] - async fn paired_plans_keep_media_offsets_and_follow_the_earliest_plan() { - tokio::time::pause(); - let now = kio::clock::now(); - let (clock, audio, video) = anchored(now); - let transition = audio.declare(); - video.declare(); - audio.end(ms(10_020)).unwrap(); - video.end(ms(10_020)).unwrap(); - audio.plan(transition, ms(0), Some(Duration::from_millis(80))); - video.plan(transition, ms(40), None); - tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.apply( - transition, - video.resume(transition, ms(40), None).ready().await.unwrap(), - ); - audio.apply( - transition, - audio - .resume(transition, ms(0), Some(Duration::from_millis(80))) - .ready() - .await - .unwrap(), - ); - assert_eq!( - video.map(ms(40)).unwrap().as_millis().unwrap() - audio.map(ms(0)).unwrap().as_millis().unwrap(), - 40 - ); - assert_eq!( - clock.state.lock().unwrap().due(audio.map(ms(0)).unwrap()), - // The pair arrived before the tail ended, so it continues as the tail ends. - Some(now.saturating_add(Duration::from_millis(120) + TICK)) - ); } /// Counts the wakes a parked poll receives. @@ -1729,7 +821,7 @@ mod tests { } /// A wait resolves only when a due time moves, not on every signal the - /// clock raises for its transitions. + /// clock raises. #[test] fn the_change_waiter_ignores_what_leaves_the_schedule_alone() { let start = Instant::ZERO.saturating_add(Duration::from_secs(1)); @@ -1749,13 +841,6 @@ mod tests { assert!(!video.set_target(Duration::from_millis(150))); assert_eq!(wakes.count(), 0, "an unmoved schedule woke the waiter"); - // Declaring a restart and its end signals the transition's resume, but - // the schedule stays put, so the reader parks again. - let transition = video.declare(); - video.end(ms(1_000)).unwrap(); - video.plan(transition, ms(0), None); - assert!(reader.poll_changed(&waiter, seen).is_pending()); - // A target that moves and moves back before the reader looks is no change. assert!(audio.set_target(Duration::from_millis(400))); assert!(audio.set_target(Duration::from_millis(200))); From e265b7415ba269561779ec48f11f3e539b5e9938 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 08:12:13 -0800 Subject: [PATCH 3/3] fix(play): keep backpressure on segments queued behind a slow video decoder Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/video.rs | 120 ++++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 8 deletions(-) diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index e43f1473dcf..73e354eea31 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -99,6 +99,29 @@ impl Decoder for moq_video::decode::Sink { } } +/// One encoder's media between declared ends. The codec drains each one before +/// decoding the next, which a fresh encoder opens with a keyframe. +#[derive(Default)] +struct Segment { + media: Buffer, + /// When its first picture arrived. + arrived: Option, +} + +impl std::ops::Deref for Segment { + type Target = Buffer; + + fn deref(&self) -> &Buffer { + &self.media + } +} + +impl std::ops::DerefMut for Segment { + fn deref_mut(&mut self) -> &mut Buffer { + &mut self.media + } +} + /// The earlier deadline, where `None` is no deadline. /// The later of two timestamps, `a` when they don't compare: [`moq_net::Timestamp`] has no `Ord`. fn later(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { @@ -166,11 +189,10 @@ impl Video { // Wakes the max-delay decoder when the clock, the segments, or the selector change. let wake = tokio::sync::Notify::new(); let closed = std::sync::atomic::AtomicBool::new(false); - // One buffer per encoder between declared ends, oldest first. The codec drains - // each before decoding the next, which a fresh encoder opens with a keyframe. - // The front never holds the rest back for long: no picture is due later than - // one shared delay after it arrived, since the reference is the earliest arrival. - let buffers = Mutex::new(VecDeque::from([Buffer::default()])); + // One segment per encoder between declared ends, oldest first. + let buffers = Mutex::new(VecDeque::from([Segment::default()])); + // Signalled when a segment behind the front becomes the front. + let advanced = tokio::sync::Notify::new(); let role = self.presentation.track(moq_mux::container::Kind::Video); let receive = async { // Video has no jitter estimate, so its target is the encoder delay alone, @@ -203,6 +225,29 @@ impl Video { let mut end = None; loop { + // Stop reading while media queued behind the front segment arrived more + // than the age budget ago, as behind a decoder slower than real time, so + // the subscription evicts the excess. Each segment ages out only its own + // media, so without this the queue would grow a segment per restart. + while buffers + .lock() + .unwrap() + .iter() + .skip(1) + .filter(|segment| segment.oldest().is_some()) + .filter_map(|segment| segment.arrived) + .min() + .is_some_and(|arrived| arrived.elapsed() > self.max_delay + held) + { + tokio::select! { + () = advanced.notified() => {} + Ok(()) = encoder.changed() => target(*encoder.borrow_and_update()), + delay = role.delay_changed(shared) => { + shared = delay; + follow(&mut track, &mut held, shared); + } + } + } let read = tokio::select! { read = track.read() => read?, Ok(()) = encoder.changed() => { @@ -232,14 +277,16 @@ impl Video { buffers.remove(index); } } - buffers.push_back(Buffer::default()); + buffers.push_back(Segment::default()); // Another timeline: its arrivals stop holding the clock until this one. role.reset(); } if role.received(frame.timestamp, Instant::now()) { self.output.send(Event::Wake); } - buffers.back_mut().unwrap().push(frame, self.max_delay + held); + let back = buffers.back_mut().unwrap(); + back.push(frame, self.max_delay + held); + back.arrived.get_or_insert_with(Instant::now); false } // Lost media discards every picture still buffered. An end declared @@ -247,7 +294,7 @@ impl Video { // from the replacement encoder. Some(Read::Gap { .. }) => { buffers.clear(); - buffers.push_back(Buffer::default()); + buffers.push_back(Segment::default()); let front = buffers.front_mut().unwrap(); front.generation = before + 1; front.ended = end.is_some(); @@ -440,6 +487,7 @@ impl Video { let next = buffers.get(1).is_some_and(|next| next.oldest().is_some()); if next { buffers.pop_front(); + advanced.notify_one(); } next }; @@ -1414,6 +1462,62 @@ mod tests { ); } + /// A decoder slower than real time leaves short segments queued behind the + /// front. Each ages out only its own media, so receive stops reading once one + /// has waited past the age budget, and the subscription evicts the excess. + #[tokio::test] + async fn segments_behind_a_slow_decoder_leave_the_excess_to_the_subscription() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = source + .consume() + .subscribe( + moq_net::track::Subscription::default().with_delay(moq_net::track::Delay::default().with_max(delay)), + ) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = Some(moq_mux::container::Producer::new(source, format())); + let mut _finished = None; + let track = moq_mux::container::Consumer::new(subscriber, format()); + let decoder = Buffered { + cost: Duration::from_millis(100), + ..Buffered::new(0) + }; + let decoded = decoder.decoded.clone(); + let task = tokio::spawn(playback(&media, delay).run(track, decoder, None)); + // Two-picture segments, one per restart, published live for two seconds. + const SEGMENTS: u64 = 50; + for at in 0..60_000u64 { + if task.is_finished() { + break; + } + if at < SEGMENTS * 40 && at % 20 == 0 { + if at % 40 == 0 && at > 0 { + producer.as_mut().unwrap().discontinuity().unwrap(); + } + producer.as_mut().unwrap().write(frame(at, at % 40 == 0)).unwrap(); + } else if at == SEGMENTS * 40 { + _finished = producer.take().map(|producer| producer.finish().unwrap()); + } + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + let decoded = decoded.lock().unwrap().len() as u64; + assert!(decoded > 0, "nothing played"); + assert!( + decoded < SEGMENTS, + "media queued behind the front past the age budget was kept, not evicted: {decoded}" + ); + } + /// Subscribe to `track` as `moq play` does, starting `lookback` groups before /// the live edge. async fn join(track: &moq_net::track::Producer, budget: Duration, lookback: u64) -> Track {