From 18b24ba318d0222fd57bfb17cfb57d045b81826b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:04:24 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m2/warmup-one-decoder.md Quest-Claim: a25f3158d8f7ef78 From 6035f1a31cc6094d2fce6e31ab69ff7224c7c053 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:37:56 -0800 Subject: [PATCH 2/5] feat(moq-video,moq-cli): a warmup rendition plays through one decoder Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/benches/play_buffer.rs | 6 +- rs/moq-cli/src/play/buffer.rs | 104 ++- rs/moq-cli/src/play/fake.rs | 7 +- rs/moq-cli/src/play/media.rs | 119 ++- rs/moq-cli/src/play/video.rs | 887 ++++------------------- rs/moq-video/src/decode/backend/mod.rs | 5 - rs/moq-video/src/decode/backend/probe.rs | 47 +- rs/moq-video/src/decode/consumer.rs | 646 +++++------------ rs/moq-video/src/decode/mod.rs | 2 - rs/moq-video/src/decode/select.rs | 808 --------------------- 10 files changed, 437 insertions(+), 2194 deletions(-) delete mode 100644 rs/moq-video/src/decode/select.rs diff --git a/rs/moq-cli/benches/play_buffer.rs b/rs/moq-cli/benches/play_buffer.rs index bc1834faa4d..aabc17d4038 100644 --- a/rs/moq-cli/benches/play_buffer.rs +++ b/rs/moq-cli/benches/play_buffer.rs @@ -32,7 +32,11 @@ fn main() { for _ in 0..1_000 { let mut buffer = buffer::Buffer::default(); for frame in &frames { - buffer.push(black_box(frame.clone()), Duration::from_millis(delay_ms)); + buffer.push( + black_box(frame.clone()), + Duration::from_millis(delay_ms), + Duration::ZERO, + ); peak = peak.max(buffer.bytes); } black_box(buffer); diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index 218a525f6d4..d7ffe0c0f57 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -15,6 +15,13 @@ pub(super) struct Buffer { /// The keyframe the buffer last resumed at. A picture presenting before it references /// media the buffer never had, so it is neither decoded nor shown. pub floor: Option, + /// Pictures presenting before this are still warming up after a join or a skip: + /// decoded for their references, never shown, and never aged out, since they + /// trail the live edge by the warmup on purpose. + pub present: Option, + /// Whether the next keyframe resumes after a join or a loss, so a catalog `warmup` + /// applies. Cleared for an encoder's first picture, which has nothing before it. + pub joined: bool, waiting_keyframe: bool, pub ended: bool, /// Exclusive endpoint declared for this encoder segment. @@ -30,6 +37,8 @@ impl Default for Buffer { oldest: VecDeque::new(), generation: 0, floor: None, + present: None, + joined: true, // A join resumes at its first keyframe, like any other break. waiting_keyframe: true, ended: false, @@ -46,6 +55,7 @@ impl Buffer { self.edge = None; self.generation += 1; self.waiting_keyframe = true; + self.joined = true; } /// The earliest presentation time still buffered, wherever it sits in @@ -63,13 +73,19 @@ impl Buffer { Some(frame) } - pub fn push(&mut self, frame: Frame, max_delay: Duration) { + /// Append `frame`, aging out what trails the newest by more than `max_delay`. After a + /// join or a loss, the pictures before the resuming keyframe plus `warmup` are warming. + pub fn push(&mut self, frame: Frame, max_delay: Duration, warmup: Duration) { let edge = *self.edge.get_or_insert(frame.timestamp); // Timestamp has no `Ord`; one that doesn't compare keeps the edge. self.edge = Some(if frame.timestamp > edge { frame.timestamp } else { edge }); if frame.keyframe { if self.waiting_keyframe { self.floor = Some(frame.timestamp); + self.present = self + .floor + .filter(|_| self.joined) + .and_then(|floor| warmed(floor, warmup)); } self.waiting_keyframe = false; } @@ -78,8 +94,11 @@ impl Buffer { } self.index(frame); let edge = self.edge.unwrap(); - // An unscaled timestamp has no age, so it never goes stale. + let present = self.present; + // An unscaled timestamp has no age, so it never goes stale, and neither does a + // warming one. let stale = |oldest: &moq_net::Timestamp| match (edge.as_micros(), oldest.as_micros()) { + _ if present.is_some_and(|present| *oldest < present) => false, (Ok(edge), Ok(oldest)) => edge.saturating_sub(oldest) > max_delay.as_micros(), _ => false, }; @@ -93,6 +112,8 @@ impl Buffer { self.generation += 1; self.waiting_keyframe = self.frames.is_empty(); self.floor = self.frames.front().map(|frame| frame.timestamp); + // Resuming past a loss warms up again. + self.present = self.floor.and_then(|floor| warmed(floor, warmup)); // The retained keyframe's leading pictures referenced what was just dropped. let mut frames = std::mem::take(&mut self.frames); frames.retain(|frame| !self.below_floor(frame)); @@ -119,6 +140,18 @@ impl Buffer { } } +/// Where a `warmup` from `floor` ends, at its scale, or `None` when there is none. +fn warmed(floor: moq_net::Timestamp, warmup: Duration) -> Option { + if warmup.is_zero() { + return None; + } + let warmup = moq_net::Timestamp::try_from(warmup) + .ok()? + .convert(floor.scale()?) + .ok()?; + floor.checked_add(warmup).ok() +} + #[cfg(test)] mod tests { use super::*; @@ -136,20 +169,20 @@ mod tests { fn eviction_keeps_only_decodable_media_inside_the_age_window() { let mut buffer = Buffer::default(); let age = Duration::from_millis(100); - buffer.push(frame(0, true), age); - buffer.push(frame(50, false), age); - buffer.push(frame(100, true), age); - buffer.push(frame(150, false), age); + buffer.push(frame(0, true), age, Duration::ZERO); + buffer.push(frame(50, false), age, Duration::ZERO); + buffer.push(frame(100, true), age, Duration::ZERO); + buffer.push(frame(150, false), age, Duration::ZERO); assert_eq!(buffer.frames.front().unwrap().timestamp.as_millis().unwrap(), 100); assert_eq!(buffer.bytes, 14); buffer.pop(); assert_eq!(buffer.bytes, 7); - buffer.push(frame(260, false), age); + buffer.push(frame(260, false), age, Duration::ZERO); assert!(buffer.frames.is_empty(), "no retained keyframe"); assert_eq!(buffer.bytes, 0); - buffer.push(frame(280, false), age); + buffer.push(frame(280, false), age, Duration::ZERO); assert!(buffer.frames.is_empty(), "cannot resume on a dependent picture"); - buffer.push(frame(300, true), age); + buffer.push(frame(300, true), age, Duration::ZERO); assert_eq!(buffer.bytes, 7); } @@ -165,15 +198,15 @@ mod tests { (90, false), (110, false), ] { - buffer.push(frame(ms, keyframe), age); + buffer.push(frame(ms, keyframe), age, Duration::ZERO); } - buffer.push(frame(95, false), age); + buffer.push(frame(95, false), age, Duration::ZERO); let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis().unwrap()).collect(); assert_eq!(kept, [100, 110], "80, 90 and 95 reference the evicted group"); assert_eq!(buffer.bytes, 14); assert_eq!(buffer.oldest().unwrap().as_millis().unwrap(), 100); - buffer.push(frame(200, true), age); - buffer.push(frame(180, false), age); + buffer.push(frame(200, true), age, Duration::ZERO); + buffer.push(frame(180, false), age, Duration::ZERO); assert_eq!(buffer.frames.len(), 4, "a continuous group keeps its leading pictures"); } @@ -181,7 +214,7 @@ mod tests { fn reordered_timestamps_keep_decode_order_and_the_newest_edge() { let mut buffer = Buffer::default(); for (ms, keyframe) in [(0, true), (99, false), (33, false), (66, false)] { - buffer.push(frame(ms, keyframe), Duration::from_millis(100)); + buffer.push(frame(ms, keyframe), Duration::from_millis(100), Duration::ZERO); } assert_eq!( buffer @@ -195,7 +228,7 @@ mod tests { buffer.clear(); assert_eq!(buffer.bytes, 0); assert!(buffer.frames.is_empty()); - buffer.push(frame(1, true), Duration::ZERO); + buffer.push(frame(1, true), Duration::ZERO, Duration::ZERO); assert_eq!(buffer.edge.unwrap().as_millis().unwrap(), 1); } #[test] @@ -203,16 +236,51 @@ mod tests { let mut buffer = Buffer::default(); let age = Duration::from_millis(100); for (ms, keyframe) in [(0, true), (99, false), (33, false)] { - buffer.push(frame(ms, keyframe), age); + buffer.push(frame(ms, keyframe), age, Duration::ZERO); } buffer.pop(); - buffer.push(frame(150, false), age); + buffer.push(frame(150, false), age, Duration::ZERO); assert!( buffer.frames.is_empty(), "the 33ms picture expired behind the 99ms reference" ); assert_eq!(buffer.bytes, 0); - buffer.push(frame(160, true), age); + buffer.push(frame(160, true), age, Duration::ZERO); assert_eq!(buffer.frames.len(), 1); } + + /// After a join the warmup's pictures trail the live edge on purpose, so a zero age budget + /// keeps them, and an eviction past a loss warms up again from the keyframe it resumes at. + #[test] + fn a_join_warms_up_without_aging_out_the_warmup() { + let mut buffer = Buffer::default(); + let warmup = Duration::from_millis(100); + for ms in [0, 50, 100, 150] { + buffer.push(frame(ms, ms % 100 == 0), Duration::ZERO, warmup); + } + let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis().unwrap()).collect(); + assert_eq!(kept, [0, 50, 100, 150]); + assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 100); + + buffer.pop(); + buffer.pop(); + buffer.push(frame(200, true), Duration::ZERO, warmup); + let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis().unwrap()).collect(); + assert_eq!(kept, [200], "the aged group was evicted"); + assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 300); + } + + /// An encoder's first picture has nothing before it to warm up from. + #[test] + fn a_fresh_encoder_needs_no_warmup() { + let mut buffer = Buffer { + joined: false, + ..Buffer::default() + }; + buffer.push(frame(0, true), Duration::ZERO, Duration::from_millis(100)); + assert!(buffer.present.is_none()); + buffer.clear(); + buffer.push(frame(500, true), Duration::ZERO, Duration::from_millis(100)); + assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 600, "a loss warms up"); + } } diff --git a/rs/moq-cli/src/play/fake.rs b/rs/moq-cli/src/play/fake.rs index d72dd2d1751..b4c4a4ccf76 100644 --- a/rs/moq-cli/src/play/fake.rs +++ b/rs/moq-cli/src/play/fake.rs @@ -50,17 +50,12 @@ impl Recorder { /// Present the newest due frame, just as the window does on a redraw. pub fn present(&self, media: &super::media::Media) -> Option { - self.present_frame(media).map(|frame| frame.timestamp) - } - - /// [`present`](Self::present), returning the picture itself. - pub fn present_frame(&self, media: &super::media::Media) -> Option { let mut video = media.video.lock().unwrap(); let presentation = media.presentation.consume(); let mut shown = None; if let Some(index) = presentation.present(video.iter().map(|frame| frame.timestamp), kio::clock::now()) { for _ in 0..=index { - shown = video.pop_front(); + shown = video.pop_front().map(|frame| frame.timestamp); } } if shown.is_some() { diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index e5e0f2c4b75..cfe0fc9a53d 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -15,7 +15,7 @@ use super::args::{Args, REACH}; use super::output::{Output, Sink, Speaker}; use super::playback::{Kind, Playback, Track, joined}; use super::timeline::{AudioTimeline, Playhead}; -use super::video::{Refresh, Video}; +use super::video::Video; use super::window::Event; use moq_mux::sync::{Producer as Presentation, Transition}; @@ -274,9 +274,7 @@ impl Media { // Nothing older than the playhead is worth presenting, so the delay // doubles as the staleness budget on the wire. let max_delay = self.args.video_delay(); - // A decoder and its subscription from `start`, and the live edge when `edge` asks - // to wait for it. - let open = async |delay: moq_net::track::Delay, start: moq_net::track::Anchor, edge: bool| { + let opened = async { // The window's renderer refuses a GPU surface it cannot import, so // the decoder downloads those instead, all of them on a device // that imports none. @@ -291,80 +289,57 @@ impl Media { decode.v4l2 = self.args.v4l2_decoder.clone(); } let decoder = moq_video::decode::Sink::open(config, &decode).await?; + let delay = moq_net::track::Delay { + min: Some(self.args.video_delay()), + max: Some(max_delay), + }; + // An intra-refresh rendition decodes from a group earlier, presenting from + // its recovery point plus the warmup, as `moq_video::decode::Consumer` does. + let warmup = config.warmup.unwrap_or_default().min(moq_mux::container::DELAY_MAX); + let playout = moq_mux::container::Playout::new(delay, warmup); let track = rendition.track(hang::catalog::Reference::track(config, name))?; + // The whole delay from the first request, so a cold join never states a + // zero minimum before the container's update. The live edge, from the + // groups each hop holds from its live feed, never a warm copy's backlog, + // so it names no cached group for the shared publisher subscription to + // rewind to. let mut subscriber = track .subscribe( - moq_net::track::Subscription::default() - .with_priority(hang::catalog::PRIORITY.video) - // The whole delay from the first request, so a cold join never - // states a zero minimum before the container's update. - .with_delay(delay) - .with_ranges([moq_net::track::Range { start, end: None }]), + playout.subscription( + moq_net::track::Subscription::default() + .with_priority(hang::catalog::PRIORITY.video) + .with_ranges([moq_net::track::Range::live()]), + ), ) .await?; - // The live edge is known only once the feed is live: a warm copy still - // waiting on its source reports nothing yet. The wait ends with the track, - // and holds up the rest of playback, so only a warmup lane asks for it. - let edge = match edge { - true => subscriber.largest().await.map(|largest| largest.group), - false => None, - }; + // A warmup widens the subscription to the group before the live edge, the + // one containing the key just below it, as starts are sparse. The live + // edge is known only once the feed is live: a warm copy still waiting on + // its source reports nothing yet. The wait ends with the track. Every wire + // carries a time start, where a relative one past the edge is refused + // before lite-07; there it reaches the edge alone. + if !warmup.is_zero() + && let Some(edge) = subscriber.largest().await + { + let start = moq_net::track::Anchor::Time(edge.group.saturating_sub(1)); + let subscription = subscriber + .subscription() + .with_ranges([moq_net::track::Range { start, end: None }]); + subscriber.update(subscription)?; + } let format = catalog::hang::Container::try_from(config)?; - // The playout delay is the target latency, and the max how late a group may be. - let track = - moq_mux::container::Playout::new(delay, Duration::ZERO).consumer(subscriber, format); - Ok::<_, anyhow::Error>((track, decoder, edge)) - }; - let opened = async { - let delay = moq_net::track::Delay { - min: Some(self.args.video_delay()), - max: Some(max_delay), - }; - // An intra-refresh rendition also runs a decoder with one more sweep - // of budget, joining a group earlier so its pictures stay clean, as - // `moq_video::decode::Consumer` does. Both share one upstream request. - let warmup = config.warmup.unwrap_or_default().min(moq_mux::container::DELAY_MAX); - // The live edge, from the groups each hop holds from its live feed, never a warm - // copy's backlog, so it names no cached group for the shared publisher - // subscription to rewind to. - let (track, decoder, edge) = - open(delay, moq_net::track::Anchor::Relative(0), !warmup.is_zero()).await?; - let refresh = if warmup.is_zero() { - None - } else { - // The min-delay decoder's delay plus one sweep, as in - // `moq_video::decode::Consumer`. - let delay = moq_net::track::Delay { - min: delay.min.map(|min| min.saturating_add(warmup)), - max: Some(max_delay.saturating_add(warmup)), - }; - // The group before the live edge: the one containing the key just below it, as - // starts are sparse. Every wire carries a time start, where a relative one - // past the edge is refused before lite-07; there it reaches the edge alone. - let start = edge.map_or(moq_net::track::Anchor::Relative(0), |edge| { - moq_net::track::Anchor::Time(edge.saturating_sub(1)) - }); - let (track, decoder, _) = open(delay, start, false).await?; - let interval = config - .framerate - .filter(|rate| rate.is_finite() && *rate > 0.0) - .and_then(|rate| Duration::try_from_secs_f64(rate.recip()).ok()); - Some(Refresh { - track, - decoder, - interval, - warmup, - }) - }; - Ok::<_, anyhow::Error>((track, decoder, refresh)) + // The playout delay is the target latency, and the max how late a group may + // be; the start reaches the warmup further back. + let track = playout.consumer(subscriber, format); + Ok::<_, anyhow::Error>((track, decoder, warmup)) } .await; match opened { - Ok((track, decoder, refresh)) => { + Ok((track, decoder, warmup)) => { tracing::info!( track = name, decoder = decoder.name(), - refresh = refresh.is_some(), + ?warmup, "playing video rendition" ); let (delay, held) = tokio::sync::watch::channel(None); @@ -376,6 +351,7 @@ impl Media { max_delay, min_delay: self.args.video_delay(), delay: held, + warmup, }; let (stop, stopped) = tokio::sync::oneshot::channel(); let (retire, retired) = tokio::sync::oneshot::channel(); @@ -397,7 +373,7 @@ impl Media { Ok(()) = retired => {} } }; - let result = until_stopped(video.run(track, decoder, refresh), stopped).await; + let result = until_stopped(video.run(track, decoder), stopped).await; (Kind::Video, result.map(|(finished, _)| (finished, None))) }); playback.started(Kind::Video, reference); @@ -1679,6 +1655,7 @@ mod tests { max_delay: delay, min_delay: delay, delay: tokio::sync::watch::channel(None).1, + warmup: Duration::ZERO, }; let audio_task = tokio::spawn(async move { play_audio(consumer, audio_play, std::future::pending()).await.unwrap(); @@ -1686,7 +1663,6 @@ mod tests { let video_task = tokio::spawn(video_play.run( moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), Passthrough(None), - None, )); let mut shown = Vec::new(); for at in 0..1_400u64 { @@ -1815,6 +1791,7 @@ mod tests { max_delay: delay, min_delay: delay, delay: tokio::sync::watch::channel(None).1, + warmup: Duration::ZERO, }; let audio_task = tokio::spawn(async move { play_audio(consumer, audio_play, std::future::pending()) @@ -1826,7 +1803,6 @@ mod tests { let video_task = tokio::spawn(video_play.run( moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), Buffered(Vec::new(), Default::default()), - None, )); let picture = |index: u64, keyframe: bool| moq_mux::container::Frame { timestamp: moq_net::Timestamp::from_millis(index * 20).unwrap(), @@ -2189,11 +2165,11 @@ mod tests { max_delay: delay, min_delay: delay, delay: tokio::sync::watch::channel(None).1, + warmup: Duration::ZERO, }; let video_task = tokio::spawn(video_play.run( moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), Passthrough(None), - None, )); let picture = |millis: u64, keyframe: bool| moq_mux::container::Frame { timestamp: moq_net::Timestamp::from_millis(millis).unwrap(), @@ -2948,8 +2924,9 @@ mod tests { max_delay: delay, min_delay: delay, delay: tokio::sync::watch::channel(None).1, + warmup: Duration::ZERO, } - .run(track, decoder, None), + .run(track, decoder), ); loop { tokio::task::yield_now().await; diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index b94dcfccbf5..b8f0de64bd9 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -6,8 +6,6 @@ use std::time::Duration; use kio::clock::Instant; use moq_mux::container::{Frame, Read}; -use moq_video::decode::Read as Decoded; -use moq_video::decode::{Lane, Next, Selector}; use super::buffer::Buffer; use super::output::Output; @@ -58,19 +56,8 @@ pub(super) struct Video { pub min_delay: Duration, /// The rendition's catalog `delay`, held as this track's playout target. pub delay: tokio::sync::watch::Receiver>, -} - -/// The max-delay decoder of a rendition with a catalog `warmup`, and the track it -/// reads: subscribed with one more sweep of budget, starting a group earlier. -/// -/// Its pictures replace the min-delay decoder's once it catches up, through the -/// same [`Selector`] `moq_video::decode::Consumer` uses. It never moves the clock. -pub(super) struct Refresh { - pub track: Track, - pub decoder: D, - /// The catalog's frame interval, when it declares a rate. - pub interval: Option, - /// The extra budget its subscription holds over the min-delay decoder's. + /// The rendition's catalog `warmup`: after a join or a skip, pictures before the + /// recovery point plus this decode but never show. pub warmup: Duration, } @@ -106,10 +93,6 @@ struct Segment { resumes: Option, /// When its first picture arrived. arrived: Option, - /// The group of the end it replaced: its media is in later groups only. - after: Option, - /// The group of its own declared end, once read. - until: Option, } impl std::ops::Deref for Segment { @@ -126,13 +109,7 @@ impl std::ops::DerefMut for Segment { } } -/// The earlier deadline, where `None` is no deadline. -/// The later of two timestamps, `a` when they don't compare: [`moq_net::Timestamp`] has no `Ord`. -fn later(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { - if b > a { b } else { a } -} - -/// The earlier of two timestamps, `a` when they don't compare; see [`later`]. +/// The earlier of two timestamps, `a` when they don't compare: [`moq_net::Timestamp`] has no `Ord`. fn earlier(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { if b < a { b } else { a } } @@ -156,13 +133,6 @@ impl PartialOrd for Mapped { } } -fn soonest(a: Option, b: Option) -> Option { - match (a, b) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - } -} - impl Video { /// How much longer than its own budgets video retains media once playback /// trails the reference by `shared`, whichever track's target set it. @@ -170,35 +140,20 @@ impl Video { shared.saturating_sub(self.min_delay) } - /// A subscription's budget with `held` and `warmup` on top of its own. - fn budget(&self, held: Duration, warmup: Duration) -> moq_net::track::Delay { + /// A subscription's budget with `held` on top of its own. The container adds the + /// warmup to the subscription's max itself. + fn budget(&self, held: Duration) -> moq_net::track::Delay { moq_net::track::Delay { - min: Some(self.min_delay + held + warmup), - max: Some(self.max_delay + held + warmup), + min: Some(self.min_delay + held), + max: Some(self.max_delay + held), } } - /// Play `track`, and with `refresh` also run its max-delay decoder. - pub async fn run( - self, - mut track: Track, - mut decoder: D, - refresh: Option>, - ) -> anyhow::Result<()> { - let interval = refresh.as_ref().and_then(|refresh| refresh.interval); - let selector = Mutex::new(Selector::new(refresh.is_some(), interval)); - // The newest presentation time fed to the min-delay decoder, which the - // max-delay decoder may always decode up to. - let edge: Mutex> = Mutex::new(None); - // Wakes the max-delay decoder when the clock, the segments, or the selector change. - let wake = tokio::sync::Notify::new(); + /// Play `track` through `decoder`. + pub async fn run(self, mut track: Track, mut decoder: D) -> anyhow::Result<()> { let closed = std::sync::atomic::AtomicBool::new(false); let advancing = std::sync::atomic::AtomicBool::new(false); let revision = std::sync::atomic::AtomicU64::new(0); - // The first group the clock's mapping covers: the first the min-delay decoder - // read, then the one after each end it resumed past. A gap keeps the mapping, - // so this outlives the buffers it replaces. - let origin: Mutex> = Mutex::new(None); let buffers = Mutex::new(VecDeque::from([Segment::default()])); // Signalled when a segment held behind a resume becomes the front. let advanced = tokio::sync::Notify::new(); @@ -225,7 +180,7 @@ impl Video { let next = self.held(shared); if next != *held { *held = next; - track.set_delay(self.budget(next, Duration::ZERO)); + track.set_delay(self.budget(next)); } }; follow(&mut track, &mut held, shared); @@ -233,6 +188,8 @@ impl Video { // media buffered before the end still plays out, and the transition that // later group resumes. let mut end = None; + // Whether a frame was read yet, so only the first can be the track's start. + let mut started = false; loop { // Stop reading while media queued behind a resume arrived more than the age @@ -274,7 +231,7 @@ impl Video { let before = buffers.front().unwrap().generation; let ended = match read { Some(Read::Frame { group, frame }) => { - if let Some((ended, transition)) = end.take_if(|(end, _)| *end != group) { + if let Some((_, transition)) = end.take_if(|(end, _)| *end != group) { // A superseded segment never receives another picture, so one that // eviction emptied can never resume. let mut index = 1; @@ -287,22 +244,27 @@ impl Video { } } role.plan(transition, frame.timestamp, None); - buffers.push_back(Segment { + let mut resumed = Segment { resumes: Some(transition), - after: Some(ended), ..Default::default() - }); + }; + // A fresh encoder opens on a picture of its own, with nothing before it + // to warm up from. + resumed.joined = false; + buffers.push_back(resumed); } + // The clock anchors on the arrival furthest ahead of its media, so a warmup + // trailing the live edge never holds playback back. if active && buffers.len() == 1 && role.received(role.map(frame.timestamp)?, Instant::now()) { self.output.send(Event::Wake); } let back = buffers.back_mut().unwrap(); - // The first segment's mapping starts where this decoder joined, so an - // end before the join needs no resume. - if back.resumes.is_none() && back.after.is_none() { - origin.lock().unwrap().get_or_insert(group); + // A group with nothing below it is the track's first, decoded from the + // encoder's first picture. + if !std::mem::replace(&mut started, true) && track.is_skipped(0..group) { + back.joined = false; } - back.push(frame, self.max_delay + held); + back.push(frame, self.max_delay + held, self.warmup); back.arrived.get_or_insert_with(Instant::now); false } @@ -311,32 +273,22 @@ impl Video { // follows comes from the replacement encoder and resumes the latest // transition, while the media lost with the gap is discarded. Some(Read::Gap { .. }) => { - // The mapping is unchanged, so the front keeps its groups. - let (after, until) = buffers.front().map(|front| (front.after, front.until)).unwrap(); - let mut queued: Vec<_> = buffers - .iter() - .filter_map(|buffer| Some((buffer.resumes?, buffer.after))) - .collect(); + let mut queued: Vec<_> = buffers.iter().filter_map(|buffer| buffer.resumes).collect(); let resumes = queued.pop(); - for (transition, _) in queued { + for transition in queued { role.abandon(transition); } revision.fetch_add(1, std::sync::atomic::Ordering::Relaxed); role.gap(); buffers.clear(); - buffers.push_back(Segment { - after, - until, - ..Default::default() - }); + buffers.push_back(Segment::default()); let front = buffers.front_mut().unwrap(); front.clear(); front.generation = before + 1; front.ended = resumes.is_some() || end.is_some(); - if let Some((transition, after)) = resumes { + if resumes.is_some() { buffers.push_back(Segment { - resumes: Some(transition), - after, + resumes, ..Default::default() }); } @@ -348,7 +300,6 @@ impl Video { role.abandon(transition); } end = Some((group, role.declare())); - buffers.back_mut().unwrap().until = Some(group); buffers.back_mut().unwrap().ended = true; buffers.back_mut().unwrap().end = Some(endpoint); if active { @@ -373,7 +324,6 @@ impl Video { "buffered encoded video" ); self.changed.notify_one(); - wake.notify_one(); if ended { break; } @@ -388,80 +338,31 @@ impl Video { // Decoded pictures waiting for room in the window's queue. let mut pending = VecDeque::new(); let drained = loop { - wake.notify_one(); - let (current, oldest, ended, last, floor) = { + let (current, oldest, ended) = { let buffers = buffers.lock().unwrap(); let buffer = buffers.front().unwrap(); - let last = buffers.len() == 1 && closed.load(std::sync::atomic::Ordering::Relaxed); - (buffer.generation, buffer.oldest(), buffer.ended, last, buffer.floor) + (buffer.generation, buffer.oldest(), buffer.ended) }; - // A codec may return pictures it held across the keyframe that - // resumed a skipped timeline. Those pictures no longer have a slot. - let floor = floor.map(|floor| role.map(floor)).transpose()?; if generation.is_some_and(|generation| generation != current) { // Whatever the codec still holds sits below the new floor, so it is // filtered on the way out and owes the window nothing. generation = None; held.clear(); pending.clear(); - selector.lock().unwrap().clear(); } let step = { let mut queue = self.frames.lock().unwrap(); let presentation = &self.presentation; let now = Instant::now(); - let mut selector = selector.lock().unwrap(); - // The min-delay decoder drained its tail, so the selector waits only on - // the max-delay one, which ends with the same track. - if last && oldest.is_none() && ended && generation.is_none() { - selector.finish(Lane::Min); - } - // When the selector next has a picture, if it is holding one back. It - // hands over only once the window took everything it handed over before. - let mut select = None; - let mut done = false; let mut placed = false; - loop { - if pending.is_empty() { - loop { - match selector.next(now) { - Next::Read(Decoded::Frame(frame)) => { - // A picture that doesn't compare with the floor isn't known to precede it. - let below = floor.is_some_and( - |floor| matches!((frame.timestamp.as_micros(), floor.as_micros()), (Ok(at), Ok(floor)) if at < floor), - ); - if !below { - pending.push_back(frame); - } - } - // Only pictures are pushed. - Next::Read(_) => {} - Next::Wait(at) => { - select = at; - break; - } - Next::Done => { - done = true; - break; - } - } - } - if pending.is_empty() { - break; - } - } - while !pending.is_empty() && vacancy(&queue, presentation).is_none_or(|at| at <= now) { - if queue.len() >= MAX_FRAMES { - queue.pop_front(); - } - let frame: moq_video::Frame = pending.pop_front().expect("checked by the loop"); - let index = queue.partition_point(|queued| queued.timestamp <= frame.timestamp); - queue.insert(index, frame); - placed = true; - } - if !pending.is_empty() { - break; + while !pending.is_empty() && vacancy(&queue, presentation).is_none_or(|at| at <= now) { + if queue.len() >= MAX_FRAMES { + queue.pop_front(); } + let frame: moq_video::Frame = pending.pop_front().expect("checked by the loop"); + let index = queue.partition_point(|queued| queued.timestamp <= frame.timestamp); + queue.insert(index, frame); + placed = true; } if placed { self.output.send(Event::Wake); @@ -479,18 +380,13 @@ impl Video { .and_then(|at| at.checked_sub(DECODE_AHEAD)) .max(vacancy(&queue, presentation)); match at.filter(|at| *at > now) { - Some(at) => Step::Wait(soonest(Some(at), select)), - // Pictures the selector holds back wait for the max-delay decoder. - None if !selector.wants(Lane::Min) => Step::Wait(select), + Some(at) => Step::Wait(Some(at)), None => Step::Decode, } } else if !ended { - Step::Wait(select) + Step::Wait(None) } else if generation.is_some() { Step::Flush - } else if select.is_some() || (last && !done) { - // The segment's last pictures are still owed to the window. - Step::Wait(select) } else { // A new encoder waits for every old picture to be presented. A closed // track hands its last due time to whatever resumes after it. @@ -531,11 +427,6 @@ impl Video { frame.timestamp = role.map(frame.timestamp)?; generation = Some(current); held.insert(Mapped(frame.timestamp)); - { - let mut edge = edge.lock().unwrap(); - *edge = Some(edge.map_or(frame.timestamp, |edge| later(edge, frame.timestamp))); - } - wake.notify_one(); // Never race a codec call: cancelling Sink::decode poisons it. The // joined receiver continues observing arrivals during this await. decoder.decode(frame).await?; @@ -576,7 +467,6 @@ impl Video { continue; } role.apply(transition, mapping); - *origin.lock().unwrap() = buffers.front().unwrap().after.map(|after| after + 1); if let Some(end) = buffers.front().unwrap().end { role.end(end)?; } @@ -598,172 +488,33 @@ impl Video { } } } - let now = Instant::now(); - let mut selector = selector.lock().unwrap(); + // A codec may return pictures it held across the keyframe that resumed a + // skipped timeline, which no longer have a slot, and pictures still warming up + // after it, which would show refresh stripes. + let (floor, present) = { + let buffers = buffers.lock().unwrap(); + let front = buffers.front().unwrap(); + (front.floor, front.present) + }; + let floor = floor.map(|floor| role.map(floor)).transpose()?; + let present = present.map(|present| role.map(present)).transpose()?; + // A picture that doesn't compare with a bound isn't known to precede it. + let below = |at: moq_net::Timestamp, bound: Option| { + bound.is_some_and( + |bound| matches!((at.as_micros(), bound.as_micros()), (Ok(at), Ok(bound)) if at < bound), + ) + }; while let Some(frame) = decoder.receive()? { // A codec returns display order, so a picture held before one it // returned was dropped rather than delayed. held.retain(|held| held.0 > frame.timestamp); - selector.push(Lane::Min, Decoded::Frame(frame), now); + if !below(frame.timestamp, floor) && !below(frame.timestamp, present) { + pending.push_back(frame); + } } }; Ok::<_, anyhow::Error>(drained) }; - // Pushes the max-delay decoder's pictures to the selector. It never moves the - // clock, and it only decodes media the min-delay decoder's segment maps. - let refresh = async { - let Some(Refresh { - track: mut refresh, - decoder: mut codec, - warmup, - .. - }) = refresh - else { - return std::future::pending::>().await; - }; - let push = |codec: &mut D| -> anyhow::Result<()> { - let now = Instant::now(); - let mut selector = selector.lock().unwrap(); - while let Some(frame) = codec.receive()? { - selector.push(Lane::Max, Decoded::Frame(frame), now); - } - drop(selector); - self.changed.notify_one(); - Ok(()) - }; - // After a join or a loss, pictures presenting before the first recovery - // point reference media this decoder never had. - let mut recover = true; - let mut leading = None; - // The group of the last end this decoder read. Its own subscription may - // deliver the replacement before the min-delay decoder reads that end. - let mut fence = None; - let schedule = self.presentation.consume(); - let mut shared = schedule.delay(); - let mut held = Duration::ZERO; - let follow = |refresh: &mut Track, held: &mut Duration, shared: Duration| { - let next = self.held(shared); - if next != *held { - *held = next; - refresh.set_delay(self.budget(next, warmup)); - } - }; - follow(&mut refresh, &mut held, shared); - loop { - let read = tokio::select! { - read = refresh.read() => read?, - delay = role.delay_changed(shared) => { - shared = delay; - follow(&mut refresh, &mut held, shared); - continue; - } - }; - let (group, mut frame) = match read { - Some(Read::Frame { group, frame }) => (group, frame), - Some(Read::Gap { .. }) => { - codec.flush().await?; - push(&mut codec)?; - recover = true; - continue; - } - // A fresh encoder has no preceding GOP to reference, so no recovery. - Some(Read::End { group, .. }) => { - codec.flush().await?; - push(&mut codec)?; - fence = Some(group); - continue; - } - None => { - codec.flush().await?; - push(&mut codec)?; - selector.lock().unwrap().finish(Lane::Max); - self.changed.notify_one(); - return std::future::pending::>().await; - } - }; - if frame.keyframe { - leading = recover.then_some(frame.timestamp); - recover = false; - } - if leading.is_some_and(|floor| frame.timestamp < floor) { - continue; - } - // Wait until the frame is in the segment the clock maps and due to decode: - // not ahead of the min-delay decoder, nor of the decode lead. - let mapped = loop { - let woken = wake.notified(); - let at = { - let buffers = buffers.lock().unwrap(); - let front = buffers.front().unwrap(); - // A segment the clock already moved past. - if front.after.is_some_and(|after| group <= after) { - break None; - } - // A segment the clock has not mapped yet. - let ahead = front.until.is_some_and(|until| group > until) - || fence.is_some_and(|fence| { - group > fence && origin.lock().unwrap().is_none_or(|origin| origin <= fence) - }) || advancing.load(std::sync::atomic::Ordering::Relaxed); - if ahead { - // Nor ever will: the track closed without resuming it. - if buffers.len() == 1 && closed.load(std::sync::atomic::Ordering::Relaxed) { - break None; - } - None - } else { - drop(buffers); - // Before the segment's mapped start, so the clock moved past it too. - let Ok(mapped) = role.map(frame.timestamp) else { - break None; - }; - if !selector.lock().unwrap().wants(Lane::Max) { - None - } else if edge.lock().unwrap().is_some_and(|edge| mapped <= edge) { - break Some(mapped); - } else { - let due = self.presentation.due(mapped); - match due - .and_then(|at| at.checked_sub(DECODE_AHEAD)) - .filter(|at| *at > Instant::now()) - { - None => break Some(mapped), - at => at, - } - } - } - }; - // A delay rise while parked must grow this subscription's budget now, - // or it evicts frames the later schedule still waits for. - let sleep = async { - match at { - Some(at) => kio::clock::sleep_until(at).await, - None => std::future::pending().await, - } - }; - tokio::select! { - _ = woken => {}, - _ = sleep => {}, - delay = role.delay_changed(shared) => { - shared = delay; - follow(&mut refresh, &mut held, shared); - } - } - }; - let Some(mapped) = mapped else { - continue; - }; - frame.timestamp = mapped; - // Never race a codec call: cancelling Sink::decode poisons it. - codec.decode(frame).await?; - push(&mut codec)?; - } - }; - let decode = async { - tokio::select! { - drained = decode => drained, - Err(err) = refresh => Err(err), - } - }; let (_, drained) = tokio::try_join!(receive, decode)?; role.retire(drained); Ok(()) @@ -784,10 +535,6 @@ mod tests { /// earliest one out, the way a real decoder returns display order. struct Buffered { depth: usize, - /// The edge of every picture, which tells the two decoders apart. - size: u32, - /// How long each decode takes. - cost: Duration, held: Vec, ready: VecDeque, decoded: Arc>>, @@ -804,8 +551,6 @@ mod tests { fn new(depth: usize) -> Self { Self { depth, - size: MIN, - cost: Duration::ZERO, held: Vec::new(), ready: VecDeque::new(), decoded: Default::default(), @@ -814,28 +559,10 @@ mod tests { } } - /// The min-delay decoder's picture size in tests, and the max-delay one's. - const MIN: u32 = 16; - const MAX: u32 = 32; - - /// The max-delay decoder of a test: [`MAX`] pictures, each decode taking `cost`. - fn refresh_decoder(cost: Duration) -> Buffered { - Buffered { - size: MAX, - cost, - ..Buffered::new(0) - } - } - impl Decoder for Buffered { async fn decode(&mut self, frame: Frame) -> anyhow::Result<()> { - if !self.cost.is_zero() { - kio::clock::sleep(self.cost).await; - } self.decoded.lock().unwrap().push((frame.timestamp, Instant::now())); - let size = self.size as usize; - let surface = - moq_video::Surface::rgba(&vec![128; size * size * 4], moq_video::Size::new(self.size, self.size))?; + let surface = moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?; self.held.push(moq_video::Frame::new(surface, frame.timestamp)); self.held.sort_by(|a, b| a.timestamp.partial_cmp(&b.timestamp).unwrap()); let bumped = self.held.len().saturating_sub(self.depth); @@ -886,6 +613,7 @@ mod tests { max_delay, min_delay: max_delay, delay: tokio::sync::watch::channel(None).1, + warmup: Duration::ZERO, } } @@ -922,39 +650,6 @@ mod tests { moq_mux::container::Consumer::new(subscriber, format) } - /// Two subscriptions to one track: the min-delay decoder's, and the max-delay - /// one's with `warmup` more budget. - async fn tracks(frames: impl IntoIterator, delay: Duration, warmup: Duration) -> (Track, Track) { - let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast - .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let subscribe = async |budget| { - let subscriber = track - .consume() - .subscribe( - moq_net::track::Subscription::default() - .with_delay(moq_net::track::Delay::default().with_max(budget)), - ) - .await - .unwrap(); - moq_mux::container::Consumer::new( - subscriber, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Data), - ) - }; - let (min, max) = (subscribe(delay).await, subscribe(delay + warmup).await); - let mut producer = moq_mux::container::Producer::new( - track, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Data), - ); - for frame in frames { - producer.write(frame).unwrap(); - } - let _finished = producer.finish().unwrap(); - (min, max) - } - /// A video rendition's catalog `delay` holds audio back with it, and a rise moves both. #[tokio::test] async fn a_video_encoder_delay_delays_audio_to_match() { @@ -995,7 +690,7 @@ mod tests { delay, ..playback(&media, budget) }; - let task = tokio::spawn(video.run(track_consumer, Buffered::default(), None)); + let task = tokio::spawn(video.run(track_consumer, Buffered::default())); producer.write(frame(0, true)).unwrap(); settle().await; assert_eq!( @@ -1052,7 +747,7 @@ mod tests { min_delay: min, ..playback(&media, max) }; - let task = tokio::spawn(video.run(track, Buffered::new(0), None)); + let task = tokio::spawn(video.run(track, Buffered::new(0))); let mut first = None; for at in 0..=5_500u64 { @@ -1084,10 +779,6 @@ mod tests { } async fn burst(speaker: bool, paused_window: bool) { - burst_with(speaker, paused_window, false).await; - } - - async fn burst_with(speaker: bool, paused_window: bool, dual: bool) { tokio::time::pause(); let delay = Duration::from_secs(2); let media = media(delay); @@ -1103,19 +794,8 @@ mod tests { let decoded = decoder.decoded.clone(); let flushed = decoder.flushed.clone(); let frames = (0..=60).map(|index| frame(index * 33, index == 0)); - let task = if dual { - let (track, max) = tracks(frames, delay, Duration::from_secs(1)).await; - let refresh = Refresh { - track: max, - decoder: refresh_decoder(Duration::ZERO), - interval: Some(Duration::from_millis(33)), - warmup: Duration::ZERO, - }; - tokio::spawn(playback(&media, delay).run(track, decoder, Some(refresh))) - } else { - let track = track(frames, delay).await; - tokio::spawn(playback(&media, delay).run(track, decoder, None)) - }; + let track = track(frames, delay).await; + let task = tokio::spawn(playback(&media, delay).run(track, decoder)); tokio::task::yield_now().await; // The burst is the live edge, which no speaker holds video back from. let last_due = start.saturating_add(delay); @@ -1126,13 +806,11 @@ mod tests { assert_eq!(media.presentation.due(last), Some(last_due)); assert!(media.video.lock().unwrap().len() <= MAX_FRAMES); let mut shown = Vec::new(); - let mut sizes = Vec::new(); while Instant::now() <= last_due.saturating_add(Duration::from_millis(10)) { if (!paused_window || Instant::now() >= start.saturating_add(Duration::from_secs(1))) - && let Some(picture) = media.output.present_frame(&media) + && let Some(timestamp) = media.output.present(&media) { - shown.push((picture.timestamp, Instant::now())); - sizes.push(picture.size().width); + shown.push((timestamp, Instant::now())); } assert!(media.video.lock().unwrap().len() <= MAX_FRAMES, "unbounded raw queue"); tokio::time::advance(Duration::from_millis(1)).await; @@ -1152,27 +830,6 @@ mod tests { ); assert!(shown.windows(2).all(|pair| pair[0].0 < pair[1].0)); assert_eq!(decoded.lock().unwrap().len(), 61); - if dual { - // Both decoders start on the same keyframe, and the max-delay one takes over - // for good once it has decoded what the min-delay one has. - assert!( - sizes.iter().skip_while(|size| **size == MIN).all(|size| *size == MAX), - "{sizes:?}" - ); - assert_eq!(sizes.last(), Some(&MAX)); - } - } - - /// The max-delay decoder decodes past due pictures the min-delay one also - /// paced, without moving the clock or the schedule. - #[tokio::test] - async fn a_dual_burst_keeps_the_single_decoder_schedule() { - burst_with(false, false, true).await; - } - - #[tokio::test] - async fn a_dual_burst_reaches_live_past_a_trailing_speaker() { - burst_with(true, false, true).await; } #[tokio::test] @@ -1200,7 +857,7 @@ mod tests { delay, ) .await; - let task = tokio::spawn(playback(&media, delay).run(track, Buffered::default(), None)); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::default())); let mut shown = Vec::new(); for _ in 0..110 { tokio::task::yield_now().await; @@ -1239,7 +896,7 @@ mod tests { producer.write(frame(0, true)).unwrap(); let decoder = Buffered::default(); let decoded = decoder.decoded.clone(); - let task = tokio::spawn(playback(&media, delay).run(track, decoder, None)); + let task = tokio::spawn(playback(&media, delay).run(track, decoder)); tokio::task::yield_now().await; tokio::time::advance(delay).await; tokio::task::yield_now().await; @@ -1304,7 +961,7 @@ mod tests { ); let decoder = Buffered::new(0); let decoded = decoder.decoded.clone(); - let task = tokio::spawn(playback(&media, delay).run(track, decoder, None)); + let task = tokio::spawn(playback(&media, delay).run(track, decoder)); for _ in 0..1300 { tokio::task::yield_now().await; media.output.present(&media); @@ -1362,7 +1019,7 @@ mod tests { // The publisher pauses and sends nothing more for now. producer.discontinuity().unwrap(); // A codec that holds nothing back, so every picture is owed before the track ends. - let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0), None)); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); let mut shown = Vec::new(); for _ in 0..2_200 { tokio::task::yield_now().await; @@ -1402,7 +1059,7 @@ mod tests { let _finished = producer.finish().unwrap(); let track = moq_mux::container::Consumer::new(subscriber, format()); // The window never presents, so the decoded pictures stay queued. - let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0), None)); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); while !task.is_finished() { tokio::time::advance(Duration::from_millis(1)).await; tokio::task::yield_now().await; @@ -1448,7 +1105,7 @@ mod tests { let mut producer = Some(moq_mux::container::Producer::new(source, format())); let mut _finished = None; let track = moq_mux::container::Consumer::new(subscriber, format()); - let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0), None)); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); let audio = media.presentation.track(moq_mux::container::Kind::Audio); // Two-picture segments, one per restart, published live for two seconds // while audio's end holds the first replacement back. @@ -1513,7 +1170,7 @@ mod tests { let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); let mut producer = moq_mux::container::Producer::new(source, format()); let track = moq_mux::container::Consumer::new(subscriber, format()); - let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0), None)); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); // Audio's end two seconds out holds every replacement back. let audio = media.presentation.track(moq_mux::container::Kind::Audio); audio.declare(); @@ -1582,7 +1239,7 @@ mod tests { let track = moq_mux::container::Consumer::new(subscriber, format()); let decoder = Buffered::new(0); let decoded = decoder.decoded.clone(); - let task = tokio::spawn(playback(&media, delay).run(track, decoder, None)); + let task = tokio::spawn(playback(&media, delay).run(track, decoder)); for ms in [0, 20, 40] { producer.write(frame(ms, ms == 0)).unwrap(); } @@ -1665,11 +1322,8 @@ mod tests { let _finished = producer.finish().unwrap(); let decoder = Buffered::new(2); let flushed = decoder.flushed.clone(); - let task = tokio::spawn(playback(&media, delay).run( - moq_mux::container::Consumer::new(subscriber, format), - decoder, - None, - )); + let task = + tokio::spawn(playback(&media, delay).run(moq_mux::container::Consumer::new(subscriber, format), decoder)); let mut shown = Vec::new(); for _ in 0..1_000 { tokio::task::yield_now().await; @@ -1691,355 +1345,70 @@ mod tests { ); } - /// Subscribe to `track` as `moq play` does, starting `lookback` groups before - /// the live edge. - async fn join(track: &moq_net::track::Producer, budget: Duration, lookback: u64) -> Track { - let mut subscriber = track - .consume() - .subscribe( - moq_net::track::Subscription::default().with_delay(moq_net::track::Delay::default().with_max(budget)), - ) - .await - .unwrap(); - let largest = subscriber.largest().await.unwrap(); - subscriber = subscriber.with_range(largest.group.saturating_sub(lookback)..); - moq_mux::container::Consumer::new( - subscriber, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), - ) - } - - /// Publishes 33 ms pictures live into sweeps of `sweep` pictures, each its own - /// group, so a test can hold one back. - struct Live { - track: moq_net::track::Producer, - sweep: u64, - groups: std::collections::BTreeMap, - } - - impl Live { - fn new(track: moq_net::track::Producer, sweep: u64) -> Self { - Self { - track, - sweep, - groups: Default::default(), - } - } - - /// Publish picture `index`, the first of its sweep starting a group. - fn write(&mut self, index: u64) { - let sequence = index / self.sweep; - if !self.groups.contains_key(&sequence) { - // A publisher finishes a group when it starts the next one. - self.seal(sequence); - } - let group = self.groups.entry(sequence).or_insert_with(|| { - self.track - .create_group(moq_net::group::Info { start: sequence }) - .unwrap() - }); - let format = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); - format - .write(group, &[frame(index * 33, index.is_multiple_of(self.sweep))]) - .unwrap(); - } - - /// Finish every group before `sequence` still open. - fn seal(&mut self, sequence: u64) { - let open = self.groups.split_off(&sequence); - for (_, group) in std::mem::replace(&mut self.groups, open) { - group.close().unwrap(); - } - } - - fn finish(&mut self) { - self.seal(u64::MAX); - drop(self.track.clone().finish().unwrap()); - } - } - - /// The min-delay decoder's pictures, then the max-delay decoder's for good. - fn hands_over(sizes: &[u32]) -> bool { - let switch = sizes.iter().position(|size| *size == MAX); - switch.is_some_and(|switch| switch > 0 && sizes[switch..].iter().all(|size| *size == MAX)) - } - - /// Publish one picture every 33 ms from `from` until `until`, skipping - /// `held` until `late`, presenting every millisecond. Returns what was shown as - /// (timestamp, picture size). - async fn live( - media: &Media, - mut live: Live, - task: tokio::task::JoinHandle>, - pictures: std::ops::Range, - held: std::ops::Range, - late: Duration, - ) -> Vec<(u128, u32)> { - let start = Instant::now(); - let mut next = pictures.start; - let mut shown = Vec::new(); - let mut released = false; - let mut finished = false; - while !task.is_finished() { - let elapsed = Instant::now() - start; - while next < pictures.end && Duration::from_millis((next - pictures.start) * 33) <= elapsed { - if !held.contains(&next) { - live.write(next); - } - next += 1; - } - if !released && elapsed >= late { - held.clone().for_each(|index| live.write(index)); - // Finished on arrival, like the groups around it. - live.seal((next - 1) / live.sweep); - released = true; - } - if next == pictures.end && released && !finished { - live.finish(); - finished = true; - } - tokio::task::yield_now().await; - if let Some(picture) = media.output.present_frame(media) { - shown.push((picture.timestamp.as_millis().unwrap(), picture.size().width)); - } - tokio::time::advance(Duration::from_millis(1)).await; - assert!(elapsed < Duration::from_secs(30), "playback never finished"); - } - task.await.unwrap().unwrap(); - // The window still holds the last pictures the decoder queued. - for _ in 0..1_000 { - if let Some(picture) = media.output.present_frame(media) { - shown.push((picture.timestamp.as_millis().unwrap(), picture.size().width)); - } - tokio::time::advance(Duration::from_millis(1)).await; - } - shown - } - - /// A cold join shows the min-delay decoder's pictures right away, then the - /// max-delay decoder's once it has decoded the group before the live edge and - /// caught up. - #[tokio::test] - async fn a_cold_join_hands_over_to_the_max_delay_decoder() { - tokio::time::pause(); - let delay = Duration::from_millis(200); - let warmup = Duration::from_secs(1); - let media = media(delay); - let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast - .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - // Sweeps a second long, and the viewer joins at the start of the second. - let mut publish = Live::new(track.clone(), 30); - (0..=30).for_each(|index| publish.write(index)); - let min = join(&track, delay, 0).await; - let max = join(&track, delay + warmup, 1).await; - let refresh = Refresh { - track: max, - // Slower than real time to start, so the earlier sweep takes a while. - decoder: refresh_decoder(Duration::from_millis(10)), - interval: Some(Duration::from_millis(33)), - warmup: Duration::ZERO, - }; - let task = tokio::spawn(playback(&media, delay).run(min, Buffered::new(0), Some(refresh))); - let shown = live(&media, publish, task, 31..90, 0..0, Duration::ZERO).await; - let sizes: Vec<_> = shown.iter().map(|(_, size)| *size).collect(); - assert_eq!(shown.first().map(|(at, _)| *at), Some(30 * 33), "{shown:?}"); - assert!(hands_over(&sizes), "no clean handover: {shown:?}"); - assert!(shown.windows(2).all(|pair| pair[0].0 < pair[1].0), "{shown:?}"); - assert_eq!(shown.last().map(|(at, _)| *at), Some(89 * 33)); - } - - /// After a loss only the min-delay decoder skips, so it shows its pictures - /// right after the hole while the max-delay decoder catches up on the late - /// sweep, and then hands back over. Pictures behind the playhead never show. - #[tokio::test] - async fn a_skip_recovers_through_the_max_delay_decoder() { - tokio::time::pause(); - let delay = Duration::from_millis(100); + /// Play a group starting at `first` holding media from 1 s, and the live group from 2 s, + /// through a rendition with a one-second warmup. Returns each picture shown, in + /// milliseconds, and when after the groups were published. + async fn warmup_join(first: u64) -> Vec<(u128, Duration)> { + let delay = Duration::from_secs(2); let warmup = Duration::from_secs(1); let media = media(delay); let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast + let published = broadcast .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) .unwrap(); - // Sweeps a third of a second long. - let mut publish = Live::new(track.clone(), 10); - publish.write(0); - let min = join(&track, delay, 0).await; - let max = join(&track, delay + warmup, 1).await; - let refresh = Refresh { - track: max, - decoder: refresh_decoder(Duration::from_millis(10)), - interval: Some(Duration::from_millis(33)), - warmup: Duration::ZERO, - }; - let task = tokio::spawn(playback(&media, delay).run(min, Buffered::new(0), Some(refresh))); - // The fourth sweep is held up past the min-delay budget, not the max-delay one. - let shown = live(&media, publish, task, 1..90, 30..40, Duration::from_millis(1_500)).await; - assert!(shown.windows(2).all(|pair| pair[0].0 < pair[1].0), "{shown:?}"); - let (before, after): (Vec<_>, Vec<_>) = shown.iter().partition(|(at, _)| *at < 30 * 33); - assert_eq!(before.last().map(|(_, size)| *size), Some(MAX), "{shown:?}"); - assert!( - after.iter().all(|(at, _)| *at >= 40 * 33), - "the late sweep showed behind the playhead: {shown:?}" + let playout = moq_mux::container::Playout::new( + moq_net::track::Delay { + min: Some(delay), + max: Some(delay), + }, + warmup, ); - let sizes: Vec<_> = after.iter().map(|(_, size)| *size).collect(); - assert!(hands_over(&sizes), "no clean handover after the skip: {shown:?}"); - } - - /// A join right after an encoder restart starts the max-delay decoder on the end, - /// a group before the min-delay one. The clock already maps the replacement, so - /// that end holds nothing back. - #[tokio::test] - async fn a_join_right_after_a_restart_still_hands_over() { - tokio::time::pause(); - let delay = Duration::from_millis(200); - let warmup = Duration::from_secs(1); - let media = media(delay); - let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast - .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + let subscriber = published + .consume() + .subscribe(playout.subscription(moq_net::track::Subscription::default())) + .await .unwrap(); - let mut producer = Some(moq_mux::container::Producer::new( - track.clone(), - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), - )); - let mut _finished = None; - for ms in [0, 33, 66] { - producer.as_mut().unwrap().write(frame(ms, ms == 0)).unwrap(); + let wire = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + for (start, stamps) in [(first, 1_000..=1_900), (2_000, 2_000..=2_200)] { + let mut group = published.append_group(start).unwrap(); + let frames: Vec<_> = stamps + .step_by(100) + .enumerate() + .map(|(index, ms)| frame(ms, index == 0)) + .collect(); + wire().write(&mut group, &frames).unwrap(); + group.close().unwrap(); } - producer.as_mut().unwrap().discontinuity().unwrap(); - // The replacement is published live, one picture every 33 ms. - let picture = |index: u64| frame(1_000 + index * 33, index == 0); - producer.as_mut().unwrap().write(picture(0)).unwrap(); - let min = join(&track, delay, 0).await; - let max = join(&track, delay + warmup, 1).await; - let refresh = Refresh { - track: max, - decoder: refresh_decoder(Duration::from_millis(10)), - interval: Some(Duration::from_millis(33)), - warmup: Duration::ZERO, + let _published = published.finish().unwrap(); + let begin = Instant::now(); + let video = Video { + warmup, + ..playback(&media, delay) }; - let task = tokio::spawn(playback(&media, delay).run(min, Buffered::new(0), Some(refresh))); - let start = Instant::now(); - let mut next = 1; + let task = tokio::spawn(video.run(playout.consumer(subscriber, wire()), Buffered::new(0))); let mut shown = Vec::new(); - while !task.is_finished() { - let elapsed = Instant::now() - start; - if next < 60 && Duration::from_millis(next * 33) <= elapsed { - producer.as_mut().unwrap().write(picture(next)).unwrap(); - next += 1; - if next == 60 { - _finished = producer.take().map(|producer| producer.finish().unwrap()); - } - } + for _ in 0..10_000 { tokio::task::yield_now().await; - if let Some(picture) = media.output.present_frame(&media) { - shown.push((picture.timestamp.as_millis().unwrap(), picture.size().width)); + if let Some(timestamp) = media.output.present(&media) { + shown.push((timestamp.as_millis().unwrap(), Instant::now() - begin)); } tokio::time::advance(Duration::from_millis(1)).await; - assert!(elapsed < Duration::from_secs(30), "playback never finished: {shown:?}"); } task.await.unwrap().unwrap(); - // The window still holds the last pictures the decoder queued. - for _ in 0..1_000 { - if let Some(picture) = media.output.present_frame(&media) { - shown.push((picture.timestamp.as_millis().unwrap(), picture.size().width)); - } - tokio::time::advance(Duration::from_millis(1)).await; - } - assert_eq!(shown.first().map(|(at, _)| *at), Some(1_000), "{shown:?}"); - assert_eq!(shown.last().map(|(at, _)| *at), Some(1_000 + 59 * 33), "{shown:?}"); - let sizes: Vec<_> = shown.iter().map(|(_, size)| *size).collect(); - assert!(hands_over(&sizes), "no clean handover: {shown:?}"); - assert!(shown.windows(2).all(|pair| pair[0].0 < pair[1].0), "{shown:?}"); + shown } - /// The max-delay decoder reads its own subscription, so it can see an encoder - /// restart before the min-delay decoder does. It holds the replacement until the - /// clock maps it rather than placing it on the old encoder's timeline. - #[tokio::test] - async fn a_restart_the_max_delay_decoder_reads_first_waits_for_the_clock() { - tokio::time::pause(); - let delay = Duration::from_millis(200); - let media = media(delay); - let broadcast = moq_net::broadcast::Info::new().produce(); - let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); - // Separate tracks with the same groups, so each decoder's delivery is up to the test. - let publish = async |name: &str| { - let track = broadcast - .create_track(name, hang::container::track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let subscriber = track - .consume() - .subscribe( - moq_net::track::Subscription::default() - .with_delay(moq_net::track::Delay::default().with_max(delay)), - ) - .await - .unwrap(); - let mut producer = moq_mux::container::Producer::new(track, format()); - for ms in [0, 33, 66] { - producer.write(frame(ms, ms == 0)).unwrap(); - } - (producer, moq_mux::container::Consumer::new(subscriber, format())) - }; - let (min, min_track) = publish("min").await; - let (max, max_track) = publish("max").await; - // The replacement encoder's timestamps run ahead of the old one's. - let restart = |mut producer: moq_mux::container::Producer| { - producer.discontinuity().unwrap(); - for ms in [1_000, 1_033, 1_066] { - producer.write(frame(ms, ms == 1_000)).unwrap(); - } - producer.finish().unwrap() - }; - let _max = restart(max); - let mut min = Some(min); - let mut _min = None; - let refresh = Refresh { - track: max_track, - decoder: refresh_decoder(Duration::ZERO), - interval: Some(Duration::from_millis(33)), - warmup: Duration::ZERO, - }; - let task = tokio::spawn(playback(&media, delay).run(min_track, Buffered::new(0), Some(refresh))); - let start = Instant::now(); - let late = Duration::from_secs(2); - let mut shown = Vec::new(); - let mut restarted = false; - while !task.is_finished() { - if !restarted && Instant::now() >= start.saturating_add(late) { - _min = min.take().map(restart); - restarted = true; - } - tokio::task::yield_now().await; - if let Some(picture) = media.output.present_frame(&media) { - shown.push((picture.timestamp.as_millis().unwrap(), Instant::now() - start)); - } - tokio::time::advance(Duration::from_millis(1)).await; - assert!( - Instant::now() - start < Duration::from_secs(30), - "playback never finished" - ); - } - task.await.unwrap().unwrap(); - for _ in 0..1_000 { - if let Some(picture) = media.output.present_frame(&media) { - shown.push((picture.timestamp.as_millis().unwrap(), Instant::now() - start)); - } - tokio::time::advance(Duration::from_millis(1)).await; - } - let timestamps: Vec<_> = shown.iter().map(|(at, _)| *at).collect(); - assert_eq!(×tamps[..3], [0, 33, 66], "{shown:?}"); - assert_eq!(timestamps.len(), 6, "{shown:?}"); - assert!( - shown[3].1 >= late, - "the replacement played before the clock mapped it: {shown:?}" - ); - assert_eq!(timestamps[4] - timestamps[3], 33, "{shown:?}"); - assert_eq!(timestamps[5] - timestamps[4], 33, "{shown:?}"); + /// A join mid-track decodes the group before the live one as its warmup and never shows + /// it, so the live group's pictures play when they would have anyway. The track's first + /// group decodes from the encoder's first picture, so nothing warms up there. + #[tokio::test(start_paused = true)] + async fn a_warmup_join_shows_only_the_recovered_pictures() { + let joined = warmup_join(1_000).await; + let start = warmup_join(0).await; + let millis = |shown: &[(u128, Duration)]| shown.iter().map(|(ms, _)| *ms).collect::>(); + assert_eq!(millis(&joined), [2_000, 2_100, 2_200]); + assert_eq!(millis(&start), (1_000..=2_200).step_by(100).collect::>()); + assert_eq!(joined, start[10..], "the warmup moved the live group's schedule"); } /// Play one burst of 33 ms pictures, given by slot in decode order, through a @@ -2050,7 +1419,7 @@ mod tests { let delay = Duration::from_secs(2); let media = media(delay); let track = track(order.iter().map(|&slot| frame(slot * 33, slot == 0)), delay).await; - let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(depth), None)); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(depth))); tokio::task::yield_now().await; let last = moq_net::Timestamp::from_millis(order.iter().max().unwrap() * 33).unwrap(); let end = media.presentation.due(last).unwrap(); diff --git a/rs/moq-video/src/decode/backend/mod.rs b/rs/moq-video/src/decode/backend/mod.rs index c88a9247b1f..b0fd8c60912 100644 --- a/rs/moq-video/src/decode/backend/mod.rs +++ b/rs/moq-video/src/decode/backend/mod.rs @@ -299,11 +299,6 @@ const NAMED_ONLY: &[Candidate] = &[ supports: |c| matches!(c, Codec::H264), open: probe::Immediate::open, }, - Candidate { - name: probe::SINGLE_NAME, - supports: |c| matches!(c, Codec::H264), - open: probe::Single::open, - }, Candidate { name: probe::NATIVE_NAME, supports: |c| matches!(c, Codec::H264), diff --git a/rs/moq-video/src/decode/backend/probe.rs b/rs/moq-video/src/decode/backend/probe.rs index bf24165ca4d..3b6aa933ae4 100644 --- a/rs/moq-video/src/decode/backend/probe.rs +++ b/rs/moq-video/src/decode/backend/probe.rs @@ -10,6 +10,7 @@ use std::collections::VecDeque; use std::sync::Mutex; +#[cfg(not(apple))] use std::sync::atomic::{AtomicBool, Ordering}; use std::thread::ThreadId; @@ -32,19 +33,13 @@ pub(crate) const REORDER_NAME: &str = "probe-reorder"; /// A test decoder that holds every picture until a flush. pub(crate) const HOLDING_NAME: &str = "probe-holding"; -/// A test decoder that hands each picture straight back and records nothing, -/// for tests that run more than one decoder at a time. It refuses [`CORRUPT`]. +/// A test decoder that hands each picture straight back and records nothing. +/// It refuses [`CORRUPT`]. pub(crate) const IMMEDIATE_NAME: &str = "probe-immediate"; /// An access unit the immediate probe fails to decode. pub(crate) const CORRUPT: &[u8] = b"corrupt"; -/// A test decoder standing in for hardware with a single session: opening a -/// second while one is alive fails. -pub(crate) const SINGLE_NAME: &str = "probe-single"; - -static SINGLE_OPEN: AtomicBool = AtomicBool::new(false); - /// A test decoder whose flush waits until the test releases it. #[cfg(not(apple))] pub(crate) const BLOCKING_FLUSH_NAME: &str = "probe-blocking-flush"; @@ -168,8 +163,6 @@ pub(crate) struct Reorder { ready: VecDeque, } -pub(crate) struct Single(VecDeque); - pub(crate) struct Native(VecDeque); pub(crate) struct Refusing; @@ -266,40 +259,6 @@ impl Backend for Immediate { } } -impl Single { - pub(crate) fn open(_stream: &Stream, _config: &Config) -> Result, Error> { - if SINGLE_OPEN.swap(true, Ordering::SeqCst) { - return Err(anyhow::anyhow!("probe: out of decoder sessions").into()); - } - Ok(Box::new(Self(VecDeque::new()))) - } -} - -impl Backend for Single { - fn decode(&mut self, _access_unit: Bytes, timestamp: Timestamp, _keyframe: bool) -> Result<(), Error> { - self.0.push_back(frame(timestamp)?); - Ok(()) - } - - fn receive(&mut self) -> Result, Error> { - Ok(self.0.pop_front()) - } - - fn flush(&mut self) -> Result<(), Error> { - Ok(()) - } - - fn name(&self) -> &str { - SINGLE_NAME - } -} - -impl Drop for Single { - fn drop(&mut self) { - SINGLE_OPEN.store(false, Ordering::SeqCst); - } -} - impl Native { pub(crate) fn open(_stream: &Stream, config: &Config) -> Result, Error> { *NATIVE_OPENED.lock().unwrap() = Some(config.clone()); diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 0e19f3253e4..0b040830b42 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -2,14 +2,13 @@ use std::collections::VecDeque; use std::pin::Pin; -use std::task::{Context, Poll, ready}; +use std::task::{Context, Poll}; use hang::catalog::VideoConfig; use kio::clock::Instant; use moq_mux::container::Read as Encoded; use super::decoder::Config; -use super::select::{Lane, Next, Selector}; use super::sink::Sink; use crate::Error; use crate::Frame; @@ -68,9 +67,9 @@ pub enum Start { /// What a live player wants. Without it a rebuilt decoder walks the whole /// backlog at decode speed before reaching live media, which a viewer sees /// as playback jumping backwards and then sprinting to catch up. A - /// rendition with a catalog `warmup` also decodes the group before it, for - /// the max-delay decoder (see [`Consumer`]). Opening waits for the track's - /// live edge, or its end. + /// rendition with a catalog `warmup` also decodes the group before it, so + /// its first picture shown is clean (see [`Consumer`]). Opening waits for the + /// track's live edge, or its end. Latest, } @@ -96,7 +95,8 @@ pub struct Options { /// playout delay and `max` at or past it for a softer skip. The max is /// applied to the transport subscription, and both reach /// [`moq_mux::container::Consumer`]. A rendition with a catalog `warmup` - /// runs a second decoder at both plus the warmup (see [`Consumer`]). + /// starts that much further back, and its subscription's max holds it too + /// (see [`Consumer`]). pub delay: Delay, } @@ -114,16 +114,20 @@ impl Options { /// [`Config::output`](super::Config::output) asked for. The direct mirror of /// `moq_audio::decode::Consumer`. /// -/// A rendition with a catalog `warmup` (intra refresh, so no keyframes) runs -/// two decoders. One skips at [`Options::delay`]'s max and may show refresh -/// stripes after a join or a skip. The other starts and tolerates `warmup` more -/// and joins a group earlier, so its pictures stay clean, and they replace the first -/// decoder's once it catches up. Both subscriptions share one upstream request. +/// A rendition with a catalog `warmup` (intra refresh, so no keyframes) is +/// decoded from the group start and presented from that start plus the warmup, +/// as `moq_audio::decode::Consumer` does: after a join or a gap, the pictures +/// before then would show refresh stripes, so they are decoded but dropped. The +/// start reaches the warmup further back to make up for it, joining a group earlier. pub struct Consumer { - /// The min-delay and max-delay decoders, indexed by [`Lane`]. - lanes: [Slot; 2], + /// The decoder between reads, absent while `step` holds it. + reader: Option>, + /// A read in flight, kept across a cancelled [`read`](Self::read), since a + /// cancelled [`Sink`] call poisons the sink. + step: Option, name: String, - selector: Selector, + /// When the access unit behind the latest picture returned arrived. + arrival: Option, } /// Access units a [`Reader`] remembers the arrival of while its decoder holds them. @@ -136,17 +140,6 @@ pub struct Consumer { /// backwards within a group; the oldest go first. const ARRIVALS: usize = 32; -/// A decoder that is idle, decoding, or gone. -/// -/// A decode in flight is kept across [`Consumer::read`] calls rather than -/// cancelled when the other lane wins, because a cancelled [`Sink`] call -/// poisons the sink. -enum Slot { - Idle(Box), - Busy(Step), - Closed, -} - /// A [`Reader::read`] in flight, handing the reader back with its result. type Step = Pin, Result, Error>)> + Send>>; @@ -177,49 +170,31 @@ struct Reader { /// The first recovery group lacks the preceding GOP's references. recover: bool, leading_floor: Option, + /// The catalog's `warmup`, capped at [`moq_mux::container::DELAY_MAX`]. + warmup: std::time::Duration, + /// Pictures before this, in nanoseconds, are still warming up after a join or a + /// gap, so they are decoded but never returned. Nanoseconds, because a backend may + /// hand a picture back at another scale than its access unit's. + present: Option, + /// Whether a frame has been read, so a later keyframe is not the track's first. + started: bool, } impl Consumer { /// Subscribe to `name` in `broadcast`, decoding it per the catalog entry. - /// Errors if the rendition's codec is not supported by a native backend, or - /// if a `warmup` rendition's second decoder cannot open. + /// Errors if the rendition's codec is not supported by a native backend. pub async fn new( broadcast: &moq_net::broadcast::Consumer, catalog: &VideoConfig, name: impl Into, options: Options, ) -> Result { - let name = name.into(); - let (min, edge) = Reader::open(broadcast, catalog, &name, &options, options.delay, None).await?; - - let warmup = catalog.warmup.unwrap_or_default().min(moq_mux::container::DELAY_MAX); - let max = if warmup.is_zero() { - None - } else { - // One more sweep both to start and to fall behind, joining a group earlier, - // so this decoder starts at a recovery point a whole sweep before the - // min-delay one. - let delay = Delay { - min: Some(options.delay.min.unwrap_or_default().saturating_add(warmup)), - max: options.delay.max.map(|max| max.saturating_add(warmup)), - }; - let (max, _) = Reader::open(broadcast, catalog, &name, &options, delay, edge).await?; - Some(max) - }; - - // The catalog's rate when it has one, else whatever the pictures show. - let interval = catalog - .framerate - .filter(|rate| rate.is_finite() && *rate > 0.0) - .and_then(|rate| std::time::Duration::try_from_secs_f64(rate.recip()).ok()); - + let reader = Reader::open(broadcast, catalog, &name.into(), &options).await?; Ok(Self { - name: min.decoder.name().to_owned(), - selector: Selector::new(max.is_some(), interval), - lanes: [ - Slot::Idle(Box::new(min)), - max.map_or(Slot::Closed, |max| Slot::Idle(Box::new(max))), - ], + name: reader.decoder.name().to_owned(), + reader: Some(Box::new(reader)), + step: None, + arrival: None, }) } @@ -239,77 +214,44 @@ impl Consumer { /// it was never handed, or a stream that repeats a timestamp while the decoder /// holds both. pub fn arrival(&self) -> Option { - self.selector.arrival() + self.arrival } /// Read the next decoded picture or boundary, or `None` after the track ends and /// the decoder's buffered tail has been drained. /// - /// Join and delivery-gap recovery trim leading pictures whose previous GOP is absent. - /// Returned picture timestamps strictly increase. + /// Join and delivery-gap recovery trim leading pictures whose previous GOP is absent, + /// and, with a catalog `warmup`, the pictures before the recovery point plus the warmup. /// /// Cancel-safe: a decode in flight when the read is dropped is kept and - /// finished by the next read. A codec error from either decoder fails the - /// read. + /// finished by the next read. A codec error fails the read. pub async fn read(&mut self) -> Result, Error> { - let mut sleep = None; - std::future::poll_fn(|cx| self.poll_read(cx, &mut sleep)).await + std::future::poll_fn(|cx| self.poll_read(cx)).await } - fn poll_read( - &mut self, - cx: &mut Context<'_>, - sleep: &mut Option>>, - ) -> Poll, Error>> { - loop { - let deadline = match self.selector.next(Instant::now()) { - Next::Read(read) => return Poll::Ready(Ok(Some(read))), - Next::Done => return Poll::Ready(Ok(None)), - Next::Wait(deadline) => deadline, - }; - - let mut progress = false; - for lane in [Lane::Min, Lane::Max] { - let slot = &mut self.lanes[lane as usize]; - if matches!(slot, Slot::Idle(_)) && self.selector.wants(lane) { - let Slot::Idle(mut reader) = std::mem::replace(slot, Slot::Closed) else { - unreachable!("matched above"); - }; - *slot = Slot::Busy(Box::pin(async move { - let result = reader.read().await; - (reader, result) - })); - } - let Slot::Busy(step) = slot else { - continue; - }; - let Poll::Ready((reader, result)) = step.as_mut().poll(cx) else { - continue; - }; - progress = true; - *slot = Slot::Idle(reader); - match result { - Ok(Some((read, arrival))) => self.selector.push_arrived(lane, read, arrival, Instant::now()), - Ok(None) => { - *slot = Slot::Closed; - self.selector.finish(lane); - } - // Either decoder failing fails the read. Left idle, so the next - // read asks it again and gets the same answer, like a lone decoder. - Err(err) => return Poll::Ready(Err(err)), - } - } - if progress { - continue; - } - - let Some(deadline) = deadline else { - return Poll::Pending; - }; - let sleep = sleep.get_or_insert_with(|| Box::pin(kio::clock::sleep_until(deadline))); - sleep.as_mut().reset(deadline); - ready!(sleep.as_mut().poll(cx)); + fn poll_read(&mut self, cx: &mut Context<'_>) -> Poll, Error>> { + let reader = &mut self.reader; + let step = self.step.get_or_insert_with(|| { + let mut reader = reader.take().expect("the reader is idle while no read is in flight"); + Box::pin(async move { + let result = reader.read().await; + (reader, result) + }) + }); + let Poll::Ready((reader, result)) = step.as_mut().poll(cx) else { + return Poll::Pending; + }; + self.step = None; + // Left idle after an error, so the next read asks it again and gets the same answer. + self.reader = Some(reader); + let Some((read, arrival)) = result? else { + return Poll::Ready(Ok(None)); + }; + // A gap or an end leaves the arrival unchanged. + if let Read::Frame(_) = read { + self.arrival = arrival; } + Poll::Ready(Ok(Some(read))) } } @@ -326,27 +268,21 @@ impl Consumer { } } - /// The lane's decoder between reads; a lane decoding in the background is busy. - fn reader(&mut self, lane: Lane) -> &mut Reader { - let Slot::Idle(reader) = &mut self.lanes[lane as usize] else { - panic!("the {lane:?} decoder is not idle"); - }; - reader + /// The decoder between reads. + fn reader(&mut self) -> &mut Reader { + self.reader.as_deref_mut().expect("the decoder is not idle") } } impl Reader { - /// Subscribe with `delay`, starting at the live edge when the caller asked for - /// [`Start::Latest`], or at the group before `edge`, a live edge another reader found. - /// Returns the reader and the live edge it waited for. + /// Subscribe per `options`, starting at the live edge when the caller asked for + /// [`Start::Latest`], or at the group before it with a catalog `warmup`. async fn open( broadcast: &moq_net::broadcast::Consumer, catalog: &VideoConfig, name: &str, options: &Options, - delay: Delay, - edge: Option, - ) -> Result<(Self, Option), Error> { + ) -> Result { let decoder = Sink::open(catalog, &options.decoder).await.map_err(|err| match err { // Only a codec or description this build cannot decode is a refusal of the // rendition. Backend and worker faults keep their own variants. @@ -354,10 +290,9 @@ impl Reader { err => err, })?; - // The max-delay decoder tolerates its warmup as lateness, so its delay already holds the - // warmup on both bounds: neither lane passes the catalog warmup to the playout, which would - // count it twice, on the subscription and in the start's reach. - let playout = moq_mux::container::Playout::new(delay, std::time::Duration::ZERO); + let warmup = catalog.warmup.unwrap_or_default().min(moq_mux::container::DELAY_MAX); + // The subscription's max carries the warmup, and the start reaches it further back. + let playout = moq_mux::container::Playout::new(options.delay, warmup); let track = broadcast.track(hang::catalog::Reference::track(catalog, name))?; // The whole delay from the first request, as the container keeps it, so the publisher // never sees a stated zero min before the container's update. @@ -377,28 +312,31 @@ impl Reader { // key just below it, since starts are sparse. Every wire carries a time start, where a // relative one past the edge is refused before lite-07; there it reaches the edge alone. if options.start == Start::Latest { - let start = edge.map_or(moq_net::track::Anchor::Relative(0), |edge| { - moq_net::track::Anchor::Time(edge.saturating_sub(1)) - }); - subscription = subscription.with_ranges([moq_net::track::Range { start, end: None }]); + subscription = subscription.with_ranges([moq_net::track::Range::live()]); } let mut subscriber = track.subscribe(subscription).await?; // The live edge is known only once the feed is live: a warm copy still waiting on its // source reports nothing yet. Opening waits for it, or the track's end, as `Start::Latest` - // promises. - let edge = match options.start { - Start::Latest => subscriber.largest().await.map(|largest| largest.group), - _ => None, - }; + // promises. A warmup widens the same subscription to the group before it, before + // anything is read, so the start can reach the warmup back. + if options.start == Start::Latest + && let Some(edge) = subscriber.largest().await + && !warmup.is_zero() + { + let start = moq_net::track::Anchor::Time(edge.group.saturating_sub(1)); + let subscription = subscriber + .subscription() + .with_ranges([moq_net::track::Range { start, end: None }]); + subscriber.update(subscription)?; + } let track = subscriber; // The catalog says how the track is framed, and it is not always the legacy // wire: `moq import fmp4` publishes CMAF. Reading a moof+mdat fragment as a // varint timestamp plus a payload decodes to garbage rather than failing. let container = moq_mux::catalog::hang::Container::try_from(catalog)?; // On a cold network join the live edge's stream often lands first. The container starts - // at the group covering the live edge less the min delay, waiting for it, so the - // max-delay decoder, whose min delay includes the warmup, still gets the group its warmup - // needs. + // at the group covering the live edge less the min delay and the warmup, waiting for it, + // so the decoder still gets the group its warmup needs. let track = playout.consumer(track, container); let reader = Self { @@ -409,8 +347,11 @@ impl Reader { drained: false, recover: true, leading_floor: None, + warmup, + present: None, + started: false, }; - Ok((reader, edge)) + Ok(reader) } /// The next item this decoder alone produces, with its access unit's arrival @@ -425,7 +366,7 @@ impl Reader { } let mux_frame = match self.track.read().await? { - Some(Encoded::Frame { frame, .. }) => Some(frame), + Some(Encoded::Frame { group, frame }) => Some((group, frame)), // Drained so the pictures from before the boundary all come out ahead of it, // rather than whenever the media after it pushes them through. The decoder // stays usable and restarts on the next group's keyframe. @@ -435,16 +376,18 @@ impl Reader { self.recover = true; continue; } - // A fresh encoder has no preceding GOP to reference, so no recovery. + // A fresh encoder has no preceding GOP to reference, so no recovery, and its + // timeline may start over below the warmup's. Some(Encoded::End { end, .. }) => { self.drain().await?; self.pending.push_back((Read::End { end }, None)); + self.present = None; continue; } None => None, }; - let Some(mux_frame) = mux_frame else { + let Some((group, mux_frame)) = mux_frame else { // The flag goes up only once the tail is in hand, so a read // dropped before the drain ran retries it rather than reporting // an end the stream has not reached. Flushing twice is safe: the @@ -467,7 +410,17 @@ impl Reader { continue; }; + // A group with nothing below it is the track's first, decoded from the encoder's + // first picture, so it needs no warmup. + let track_start = !std::mem::replace(&mut self.started, true) && self.track.is_skipped(0..group); if mux_frame.keyframe { + if self.recover && !self.warmup.is_zero() { + // A frame with no time has no presentation to hold back. + let start = mux_frame.timestamp.as_nanos().ok(); + self.present = start + .filter(|_| !track_start) + .map(|start| start.saturating_add(self.warmup.as_nanos())); + } self.leading_floor = self.recover.then_some(mux_frame.timestamp); self.recover = false; } @@ -526,6 +479,11 @@ impl Reader { }); Some(arrived) }); + // Still warming up: decoded for its references, never shown. + let warming = self.present.zip(picture.timestamp.as_nanos().ok()); + if warming.is_some_and(|(present, at)| at < present) { + continue; + } self.pending.push_back((Read::Frame(picture), arrival)); } Ok(()) @@ -945,7 +903,7 @@ mod tests { [1000, 1100, 2000, 1800, 1900], "join trims, continuous delivery keeps every leading picture" ); - consumer.reader(Lane::Min).track.set_delay(Delay::default()); + consumer.reader().track.set_delay(Delay::default()); write(12, &[3000]); write(13, &[4000, 3800, 3900, 4100]); let _track = track.finish().unwrap(); @@ -1138,7 +1096,7 @@ mod tests { let mut sourced = Vec::new(); while consumer.next_frame().await.unwrap().is_some() { - assert!(consumer.reader(Lane::Min).arrivals.len() <= ARRIVALS); + assert!(consumer.reader().arrivals.len() <= ARRIVALS); sourced.push(consumer.arrival().is_some()); } let mut expected = vec![false; FRAMES as usize - ARRIVALS]; @@ -1365,8 +1323,7 @@ mod tests { assert!(consumer.next_frame().await.unwrap().is_none()); } - /// One frame per second, so the frame interval the selector holds a picture - /// for dwarfs a decode thread's round trip even on a loaded host. + /// One frame per second. const SECOND: u64 = 1_000_000; /// A sweep of four frames. @@ -1391,7 +1348,7 @@ mod tests { (broadcast, track) } - /// Append frames at `seconds` to `group`, the first frame of a group being its recovery point. + /// Append frames at `seconds` to `group`, the first frame of a sweep being its recovery point. fn write(group: &mut moq_net::group::Producer, seconds: &[u64]) { let wire = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); let frames: Vec<_> = seconds @@ -1406,30 +1363,27 @@ mod tests { wire.write(group, &frames).unwrap(); } - fn group(track: &mut moq_net::track::Producer, sequence: u64, seconds: &[u64]) -> moq_net::group::Producer { - let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); + /// Append a group at `start` holding frames at `seconds`, skipping the starts since the last. + fn group(track: &mut moq_net::track::Producer, start: u64, seconds: &[u64]) -> moq_net::group::Producer { + let mut group = track.append_group(start).unwrap(); write(&mut group, seconds); group } - async fn dual( - broadcast: &moq_net::broadcast::Producer, - start: Start, - max_delay: std::time::Duration, - kind: &str, - ) -> Consumer { + /// A consumer of a rendition with a four-second warmup and a zero max delay. + async fn warm(broadcast: &moq_net::broadcast::Producer, start: Start) -> Consumer { Consumer::new( &broadcast.consume(), &warmup_catalog(Some(WARMUP)), "video", Options { decoder: Config { - kind: Kind::Named(kind.into()), + kind: Kind::Named(probe::IMMEDIATE_NAME.into()), ..Config::new() }, start, delay: Delay { - max: Some(max_delay), + max: Some(std::time::Duration::ZERO), ..Delay::default() }, }, @@ -1438,8 +1392,8 @@ mod tests { .unwrap() } - /// Read one frame, as (seconds, the lane it was shown from). - async fn read(consumer: &mut Consumer) -> (u64, Lane) { + /// Read one picture, in seconds. + async fn read(consumer: &mut Consumer) -> u64 { let frame = kio::clock::timeout(std::time::Duration::from_secs(10), consumer.next_frame()) .await .expect("a frame within ten seconds") @@ -1447,71 +1401,40 @@ mod tests { .expect("the track is live"); let seconds = u64::try_from(frame.timestamp.as_micros().unwrap()).unwrap() / SECOND; assert!(consumer.arrival().is_some(), "picture {seconds} lost its arrival"); - (seconds, consumer.selector.source()) + seconds } - /// Let the decoders run with nothing new to show until the max-delay one - /// has caught up, or fail after ten seconds of trying. - async fn catch_up(consumer: &mut Consumer) { - let mut sleep = None; - let caught_up = std::future::poll_fn(|cx| { - loop { - match consumer.poll_read(cx, &mut sleep) { - // Breaks carry no picture, so `next_frame` would skip them too. - Poll::Ready(Ok(Some(Read::Gap { .. } | Read::End { .. }))) => {} - Poll::Ready(read) => panic!( - "nothing new was published, yet read returned {:?}", - read.map(|read| read.map(|read| match read { - Read::Frame(frame) => frame.timestamp, - _ => unreachable!("matched above"), - })) - ), - Poll::Pending => break, - } - } - match consumer.selector.source() { - Lane::Max => Poll::Ready(()), - Lane::Min => Poll::Pending, - } - }); - kio::clock::timeout(std::time::Duration::from_secs(10), caught_up) - .await - .expect("the max-delay decoder never caught up"); - } - - /// A cold join shows the min-delay decoder's pictures at the live edge first, - /// then the max-delay decoder's once it has decoded the sweep before them. + /// A join mid-track starts a group before the live edge, on one subscription whose max + /// holds the warmup, and shows nothing until the recovery point plus the warmup. Groups + /// are keyed by their start, so the one before the live edge lies a whole group's span + /// below it: the start counts a group back, not one below the live edge's start. #[tokio::test] - async fn a_cold_join_hands_over_to_the_max_delay_decoder() { + async fn a_join_starts_a_group_earlier_and_presents_after_the_warmup() { let (broadcast, mut track) = live_track(); let published = track.clone(); - group(&mut track, 0, &[0, 1, 2, 3]).close().unwrap(); - let mut live = group(&mut track, 1, &[4, 5]); + group(&mut track, 4 * SECOND, &[4, 5, 6, 7]).close().unwrap(); + let mut live = group(&mut track, 8 * SECOND, &[8, 9]); - let mut consumer = dual( - &broadcast, - Start::Latest, - std::time::Duration::ZERO, - probe::IMMEDIATE_NAME, - ) - .await; + let mut consumer = warm(&broadcast, Start::Latest).await; + let subscription = published.subscription().expect("the consumer subscribed"); + assert_eq!(subscription.delay.max, Some(WARMUP), "the max does not hold the warmup"); assert_eq!( - published.subscription().and_then(|sub| sub.delay.max), - Some(WARMUP), - "the two subscriptions did not share one request at the larger budget" + subscription.ranges, + vec![moq_net::track::Range { + start: moq_net::track::Anchor::Time(8 * SECOND - 1), + end: None, + }], + "the subscription does not reach the group before the live edge" ); - // The max-delay decoder starts at 0, so only the min-delay one can show 4 first. - assert_eq!(read(&mut consumer).await.0, 4, "the live edge shows first"); - assert_eq!(read(&mut consumer).await.0, 5); - catch_up(&mut consumer).await; - for second in 6..9 { - write(&mut live, &[second]); - assert_eq!(read(&mut consumer).await, (second, Lane::Max)); - } + // 4 through 7 decode as the warmup, never shown. + assert_eq!(read(&mut consumer).await, 8); + assert_eq!(read(&mut consumer).await, 9); + write(&mut live, &[10]); + assert_eq!(read(&mut consumer).await, 10); } - /// A catalog warmup past the cap widens the max-delay decoder's subscription by the cap alone. + /// A catalog warmup past the cap widens the subscription by the cap alone. #[tokio::test] async fn a_huge_warmup_reaches_back_by_the_cap() { let (broadcast, mut track) = live_track(); @@ -1541,203 +1464,30 @@ mod tests { ); } - /// On a cold network join the live edge's stream often lands first. The - /// max-delay decoder still starts at the group before it, and takes over. - #[tokio::test] - async fn a_cold_join_whose_live_edge_arrives_first_starts_a_group_earlier() { - let (broadcast, mut track) = live_track(); - let mut live = group(&mut track, 1, &[4, 5]); - - let mut consumer = dual( - &broadcast, - Start::Latest, - std::time::Duration::ZERO, - probe::IMMEDIATE_NAME, - ) - .await; - - // Only the live edge has landed: the max-delay decoder waits for the group below it. - // Mocked time, so the timeout and the warmup's deadline never race; no decode runs - // while it waits, and the probe's thread would let mocked time skip ahead if one did. - let max = consumer.reader(Lane::Max); - tokio::time::pause(); - assert!( - kio::clock::timeout(std::time::Duration::from_millis(50), max.read()) - .await - .is_err(), - "the max-delay decoder started at the live edge" - ); - tokio::time::resume(); - group(&mut track, 0, &[0, 1, 2, 3]).close().unwrap(); - let Ok(Some((Read::Frame(first), _))) = max.read().await else { - panic!("the max-delay decoder produced no picture"); - }; - assert_eq!( - first.timestamp, - Timestamp::from_micros(0).unwrap(), - "it starts a group earlier" - ); - - assert_eq!(read(&mut consumer).await, (4, Lane::Min), "the live edge shows first"); - assert_eq!(read(&mut consumer).await.0, 5); - catch_up(&mut consumer).await; - write(&mut live, &[6]); - assert_eq!(read(&mut consumer).await, (6, Lane::Max)); - } - - /// Groups are keyed by their start, so the one before the live edge lies a whole group's - /// span below it: the max-delay decoder reaches it by counting groups back, not by stepping - /// one below the live edge's start. - #[tokio::test] - async fn the_max_delay_decoder_counts_back_a_group_across_sparse_starts() { - let (broadcast, mut track) = live_track(); - group(&mut track, 0, &[0, 1, 2, 3]).close().unwrap(); - let _live = group(&mut track, 4 * SECOND, &[4, 5]); - - let mut consumer = dual( - &broadcast, - Start::Latest, - std::time::Duration::ZERO, - probe::IMMEDIATE_NAME, - ) - .await; - let Ok(Some((Read::Frame(first), _))) = consumer.reader(Lane::Max).read().await else { - panic!("the max-delay decoder produced no picture"); - }; - assert_eq!( - first.timestamp, - Timestamp::from_micros(0).unwrap(), - "it starts a group earlier" - ); - let Ok(Some((Read::Frame(first), _))) = consumer.reader(Lane::Min).read().await else { - panic!("the min-delay decoder produced no picture"); - }; - assert_eq!(first.timestamp, Timestamp::from_micros(4 * SECOND).unwrap()); - } - - /// A stall shorter than a sweep makes the min-delay decoder skip, while the - /// max-delay one waits it out, and takes the picture back once it catches up, - /// never alternating in between. + /// The track's first group decodes from the encoder's first picture, so it shows at + /// once. A skip past a stalled group warms up again: the pictures before its recovery + /// point plus the warmup decode but never show. #[tokio::test] - async fn a_short_stall_skips_only_the_min_delay_decoder() { + async fn a_gap_warms_up_again_but_the_track_start_does_not() { let (broadcast, mut track) = live_track(); - let mut consumer = dual( - &broadcast, - Start::Oldest, - std::time::Duration::ZERO, - probe::IMMEDIATE_NAME, - ) - .await; - - // One frame at a time once both decoders are caught up, so every picture - // below has exactly one lane that can show it. - let mut first = group(&mut track, 0, &[0]); - assert_eq!(read(&mut consumer).await.0, 0); - catch_up(&mut consumer).await; - let mut shown = Vec::new(); - for second in 1..4 { - write(&mut first, &[second]); - shown.push(read(&mut consumer).await); - } - first.close().unwrap(); - let mut stalled = group(&mut track, 1, &[4]); - shown.push(read(&mut consumer).await); - - // Group 1 stalls and group 2 starts: the min-delay decoder skips to it, - // leading by more than a frame, so it is shown at once. - let mut live = group(&mut track, 2, &[8]); - shown.push(read(&mut consumer).await); - // Mocked time, so the timeout fires only once every task has run, well inside the - // sweep; no decode runs while the max-delay decoder waits on the stalled group. - tokio::time::pause(); - assert!( - kio::clock::timeout(std::time::Duration::from_millis(100), consumer.next_frame()) - .await - .is_err() - ); - tokio::time::resume(); - assert_eq!( - consumer.selector.source(), - Lane::Min, - "the max-delay decoder skipped the stall too" - ); - - write(&mut stalled, &[5, 6, 7]); - stalled.close().unwrap(); - catch_up(&mut consumer).await; - for second in 9..12 { - write(&mut live, &[second]); - shown.push(read(&mut consumer).await); - } - - assert_eq!( - shown, - [ - (1, Lane::Max), - (2, Lane::Max), - (3, Lane::Max), - (4, Lane::Max), - (8, Lane::Min), - (9, Lane::Max), - (10, Lane::Max), - (11, Lane::Max), - ] - ); - } - - /// A stall longer than a sweep skips both decoders, so the stalled group's - /// late frames are never shown, and the max-delay decoder still takes over. - #[tokio::test] - async fn a_long_stall_skips_both_decoders() { - let (broadcast, mut track) = live_track(); - let mut consumer = dual( - &broadcast, - Start::Oldest, - std::time::Duration::ZERO, - probe::IMMEDIATE_NAME, - ) - .await; + let mut consumer = warm(&broadcast, Start::Oldest).await; // Written as it is read: a zero budget would skip a backlog. group(&mut track, 0, &[0, 1, 2, 3]).close().unwrap(); let mut shown = Vec::new(); for _ in 0..4 { - shown.push(read(&mut consumer).await.0); + shown.push(read(&mut consumer).await); } let mut stalled = group(&mut track, 1, &[4]); - shown.push(read(&mut consumer).await.0); + shown.push(read(&mut consumer).await); let mut live = group(&mut track, 2, &[8, 9, 10, 11, 12]); - for _ in 0..5 { - shown.push(read(&mut consumer).await.0); - } - catch_up(&mut consumer).await; + shown.push(read(&mut consumer).await); write(&mut stalled, &[5, 6, 7]); stalled.close().unwrap(); write(&mut live, &[13]); - shown.push(read(&mut consumer).await.0); - assert_eq!(shown, [0, 1, 2, 3, 4, 8, 9, 10, 11, 12, 13]); - } - - /// Both decoders hold their last picture until the track ends, and each - /// drains it exactly once. - #[tokio::test] - async fn the_tail_drains_from_both_decoders() { - let (broadcast, mut track) = live_track(); - group(&mut track, 0, &[0, 1, 2, 3]).close().unwrap(); - group(&mut track, 1, &[4, 5]).close().unwrap(); - let _track = track.finish().unwrap(); - - // A budget past the whole 5 s backlog, so the start keeps every cached group and both - // decoders read them all. - let mut consumer = dual(&broadcast, Start::Oldest, 2 * WARMUP, probe::BUFFERED_NAME).await; - let mut shown = Vec::new(); - while let Some(frame) = consumer.next_frame().await.unwrap() { - shown.push(u64::try_from(frame.timestamp.as_micros().unwrap()).unwrap() / SECOND); - } - assert_eq!(shown, [0, 1, 2, 3, 4, 5]); - assert!(consumer.next_frame().await.unwrap().is_none()); - assert!(consumer.lanes.iter().all(|slot| matches!(slot, Slot::Closed))); + shown.push(read(&mut consumer).await); + assert_eq!(shown, [0, 1, 2, 3, 4, 12, 13]); } /// The first request already states the min delay: a latest start waits for the live edge @@ -1773,13 +1523,17 @@ mod tests { assert_eq!(first.map(|sub| sub.delay), Some(delay)); } - /// A rendition without `warmup` opens one subscription and one decoder. + /// A rendition without `warmup` subscribes at the max delay alone and shows every picture + /// after a join. #[tokio::test] - async fn no_warmup_runs_one_decoder() { + async fn no_warmup_changes_nothing() { for warmup in [None, Some(std::time::Duration::ZERO)] { - let (broadcast, track) = live_track(); - let max_delay = std::time::Duration::from_millis(500); - let consumer = Consumer::new( + let (broadcast, mut track) = live_track(); + let published = track.clone(); + group(&mut track, 4 * SECOND, &[4, 5]).close().unwrap(); + let _track = track.finish().unwrap(); + let max_delay = std::time::Duration::from_secs(10); + let mut consumer = Consumer::new( &broadcast.consume(), &warmup_catalog(warmup), "video", @@ -1797,38 +1551,12 @@ mod tests { ) .await .unwrap(); - assert!(matches!(consumer.lanes[Lane::Max as usize], Slot::Closed)); - assert_eq!(track.subscription().and_then(|sub| sub.delay.max), Some(max_delay)); + assert_eq!(published.subscription().and_then(|sub| sub.delay.max), Some(max_delay)); + assert_eq!(read(&mut consumer).await, 4); + assert_eq!(read(&mut consumer).await, 5); } } - /// A warmup rendition needs both decoders: one that cannot open fails the - /// consumer instead of playing with stripes and no signal. - #[tokio::test] - async fn a_second_decoder_that_cannot_open_fails_the_consumer() { - let (broadcast, _track) = live_track(); - let result = Consumer::new( - &broadcast.consume(), - &warmup_catalog(Some(WARMUP)), - "video", - Options { - decoder: Config { - kind: Kind::Named(probe::SINGLE_NAME.into()), - ..Config::new() - }, - ..Options::new() - }, - ) - .await; - let Err(err) = result else { - panic!("the consumer opened without its max-delay decoder"); - }; - assert!( - err.to_string().contains("out of decoder sessions"), - "unexpected error: {err}" - ); - } - /// A device that cannot decode the stream refuses the rendition, so a caller /// can pick another one, the same as for a codec this build lacks. #[tokio::test] @@ -1854,48 +1582,6 @@ mod tests { } } - /// A max-delay decode error reaches the caller, like a min-delay one. - #[tokio::test] - async fn a_max_delay_decode_error_fails_the_read() { - let (broadcast, mut track) = live_track(); - // Only the max-delay decoder reads group 0, a group before the live edge. - let mut earlier = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video) - .write( - &mut earlier, - &[moq_mux::container::Frame { - timestamp: Timestamp::ZERO, - duration: None, - payload: Bytes::from_static(probe::CORRUPT), - keyframe: true, - }], - ) - .unwrap(); - write(&mut earlier, &[1, 2, 3]); - earlier.close().unwrap(); - let _live = group(&mut track, 1, &[4, 5]); - - let mut consumer = dual( - &broadcast, - Start::Latest, - std::time::Duration::ZERO, - probe::IMMEDIATE_NAME, - ) - .await; - for _ in 0..3 { - match kio::clock::timeout(std::time::Duration::from_secs(10), consumer.next_frame()).await { - Ok(Ok(Some(_))) => continue, - Ok(Err(err)) => { - assert!(err.to_string().contains("corrupt"), "unexpected error: {err}"); - return; - } - Ok(Ok(None)) => panic!("the live track ended"), - Err(_) => break, - } - } - panic!("the max-delay decoder's error was swallowed"); - } - /// A decoder that refuses an access unit never returns its picture, so a later /// access unit at the same timestamp reports its own arrival, not the refused one's. #[tokio::test(start_paused = true)] @@ -1955,7 +1641,7 @@ mod tests { .unwrap() .expect("the next access unit decodes"); assert_eq!(consumer.arrival(), Some(landed[1])); - assert_eq!(consumer.reader(Lane::Min).arrivals.len(), 0, "nothing stays queued"); + assert_eq!(consumer.reader().arrivals.len(), 0, "nothing stays queued"); } /// A timestamp two access units share, both still held by the decoder, matches either diff --git a/rs/moq-video/src/decode/mod.rs b/rs/moq-video/src/decode/mod.rs index 410c66f9ba2..313ba31a5d9 100644 --- a/rs/moq-video/src/decode/mod.rs +++ b/rs/moq-video/src/decode/mod.rs @@ -24,14 +24,12 @@ pub(crate) mod backend; mod conformance; mod consumer; mod decoder; -mod select; mod sink; mod vui; pub use backend::{Codec, NAMES}; pub use consumer::{Consumer, Delay, Options, Read, Start}; pub use decoder::{Config, Decoder, Kind}; -pub use select::{Lane, Next, Selector}; pub use sink::Sink; #[cfg(test)] diff --git a/rs/moq-video/src/decode/select.rs b/rs/moq-video/src/decode/select.rs deleted file mode 100644 index e262fd1a2db..00000000000 --- a/rs/moq-video/src/decode/select.rs +++ /dev/null @@ -1,808 +0,0 @@ -//! Picks which of a min-delay and a max-delay decoder's pictures to show. -//! -//! An intra-refresh rendition has no keyframes, so a decoder that joins or skips -//! mid-sweep shows refresh stripes until a whole sweep has passed. The min-delay -//! decoder keeps the caller's latency and may show them; the max-delay decoder -//! tolerates one more sweep of delay, so it keeps continuity. Both race on the -//! same frames in steady state, and taking whichever is newest would flicker -//! stripes between clean pictures, so the source is sticky instead. -//! -//! Gaps and ends ride in each lane's queue between its pictures, so they come out -//! in order with whichever lane is the source. Both lanes read the same track, so -//! the first copy of an end returned stands for both, and a loss the output has -//! already moved past is no loss in it. -//! -//! [`Consumer`](super::Consumer) runs both decoders and this selector for you. A -//! player that paces its own decoding, such as `moq play`, drives a [`Selector`] -//! directly: it pushes each lane's output and returns whatever [`Selector::next`] -//! hands back. - -use std::collections::VecDeque; -use std::time::Duration; - -use kio::clock::Instant; - -use super::Read; - -/// One of the two decoders. -/// -/// The discriminant indexes a two-element array, so a caller may keep its own -/// per-lane state as `[T; 2]` indexed by `lane as usize`. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Lane { - /// Skips at the caller's latency target, and may show stripes after a join or a skip. - Min = 0, - /// Tolerates one more sweep, joining a group earlier, so its pictures stay clean. - Max = 1, -} - -/// Decoded pictures a lane may hold before its decoder waits for the selector. -/// -/// Two, so a held min-delay picture still lets the decoder behind it move the -/// presentation clock the hold is measured on, without parking a backlog of -/// decoded surfaces a hardware pool may need back. Gaps and ends don't count: -/// they hold no surface, and a lane must reach its next picture past them to -/// lead the other. -const QUEUE: usize = 2; - -/// What [`Selector::next`] decided. -pub enum Next { - /// Show or apply this next. - Read(Read), - /// Nothing to show until a lane produces more, or until the deadline passes. - Wait(Option), - /// Both lanes finished and nothing is left to show. - Done, -} - -/// A queued read, with when the access unit behind a picture arrived and when the picture was -/// decoded. -struct Queued { - read: Read, - arrival: Option, - at: Instant, -} - -#[derive(Default)] -struct State { - queue: VecDeque, - /// Pictures in the queue. - pictures: usize, - /// The newest timestamp this lane has decoded, shown or not. - newest: Option, - /// Ended, or never opened. - done: bool, -} - -impl State { - /// Done, with nothing left to show either. - fn exhausted(&self) -> bool { - self.done && self.queue.is_empty() - } - - /// The first queued picture's timestamp, and when it was decoded; `None` when it has no time. - fn held(&self) -> Option<(Duration, Instant)> { - let (frame, at) = self.queue.iter().find_map(|queued| match &queued.read { - Read::Frame(frame) => Some((frame, queued.at)), - _ => None, - })?; - Some((time(frame.timestamp)?, at)) - } -} - -/// Chooses between a min-delay and a max-delay decoder's pictures. -/// -/// The max-delay picture is shown once that decoder has caught up to the -/// min-delay one, a newer min-delay picture waits at most one frame interval for -/// it, and with two lanes the returned picture timestamps strictly increase. -pub struct Selector { - lanes: [State; 2], - source: Lane, - /// Whether there are two lanes to choose between. - dual: bool, - /// The last picture timestamp returned with two lanes, which output strictly - /// increases past. A lone decoder's pictures pass through as they come. - last: Option, - /// When the access unit behind the last picture returned arrived. - arrival: Option, - /// The furthest a gap returned with two lanes resumed at. - resumed: Option, - /// The newest end returned with two lanes. - ended: Option, - /// The catalog's frame interval, or the smallest step the min-delay lane has - /// decoded. The smallest rather than the latest, so a skip never stretches it. - interval: Option, - measured: bool, -} - -impl Selector { - /// A selector for one lane, or for both when `dual`. - /// - /// `interval` is the catalog's frame interval, how long a newer min-delay - /// picture waits for the max-delay decoder. Without one, the selector uses - /// the smallest step the min-delay lane decodes. - pub fn new(dual: bool, interval: Option) -> Self { - let mut lanes: [State; 2] = Default::default(); - lanes[Lane::Max as usize].done = !dual; - Self { - lanes, - source: Lane::Min, - dual, - last: None, - arrival: None, - resumed: None, - ended: None, - measured: interval.is_none(), - interval, - } - } - - /// The lane the last picture came from, or will next. - #[cfg(test)] - pub(super) fn source(&self) -> Lane { - self.source - } - - /// When the access unit behind the last picture [`next`](Self::next) returned arrived, as - /// [`push_arrived`](Self::push_arrived) gave it. - pub(super) fn arrival(&self) -> Option { - self.arrival - } - - /// Whether `lane` can take another decoded picture. - pub fn wants(&self, lane: Lane) -> bool { - let state = &self.lanes[lane as usize]; - !state.done && state.pictures < QUEUE - } - - /// What `lane` read at `now`. - /// - /// Between two pictures a lane keeps one gap, spanning every loss there, and its - /// first end, where the pictures before it stop. Marker-only groups read while this - /// lane waits behind the source then cost a bounded queue, not an entry each. - pub fn push(&mut self, lane: Lane, read: Read, now: Instant) { - self.push_arrived(lane, read, None, now); - } - - /// [`push`](Self::push), with when the access unit behind a picture arrived, which - /// [`arrival`](Self::arrival) reports once the picture is returned. - pub(super) fn push_arrived(&mut self, lane: Lane, read: Read, arrival: Option, now: Instant) { - let state = &mut self.lanes[lane as usize]; - let mut run = state - .queue - .iter_mut() - .rev() - .map(|queued| &mut queued.read) - .take_while(|queued| !matches!(queued, Read::Frame(_))); - match &read { - Read::Frame(frame) => { - if let Some(timestamp) = time(frame.timestamp) { - if lane == Lane::Min - && self.measured && let Some(step) = - state.newest.and_then(|newest| timestamp.checked_sub(newest)) - && !step.is_zero() - { - self.interval = Some(self.interval.map_or(step, |interval| interval.min(step))); - } - state.newest = Some(state.newest.map_or(timestamp, |newest| newest.max(timestamp))); - } - state.pictures += 1; - } - Read::Gap { timestamp, duration } => { - if let Some(Read::Gap { - timestamp: start, - duration: span, - }) = run.find(|queued| matches!(queued, Read::Gap { .. })) - { - (*start, *span) = merge((*start, *span), (*timestamp, *duration)); - return; - } - } - Read::End { .. } => { - if run.any(|queued| matches!(queued, Read::End { .. })) { - return; - } - } - } - state.queue.push_back(Queued { read, arrival, at: now }); - } - - /// Forget every queued picture and what each lane has decoded, as after a - /// break that discards both decoders' output. What was already returned is - /// kept, so the output still never rewinds. - pub fn clear(&mut self) { - for state in &mut self.lanes { - state.queue.clear(); - state.pictures = 0; - state.newest = None; - } - } - - /// `lane` ended and will decode nothing more. - pub fn finish(&mut self, lane: Lane) { - self.lanes[lane as usize].done = true; - } - - /// The next item to return at `now`, or what to wait for. - pub fn next(&mut self, now: Instant) -> Next { - if self.dual { - self.prune(); - } - - let [min, max] = &self.lanes; - let interval = self.interval.unwrap_or_default(); - - if self.source == Lane::Min { - // Max-delay pictures take over once that decoder has caught up to a - // min-delay picture, not merely decoded first: on a join it starts a - // sweep behind. - let caught_up = !max.exhausted() && min.newest.is_some() && max.newest >= min.newest; - if caught_up || (min.exhausted() && !max.exhausted()) { - self.source = Lane::Max; - } - } else if let Some((held, at)) = min.held() { - // A newer min-delay picture waits for the max-delay decoder to reach - // it, but never for longer than a frame: not when it already leads by - // more than that, not once a frame's worth of wall or media time has - // passed, and not after the max-delay decoder has nothing left. - let leads = max.newest.is_none_or(|newest| held > newest + interval); - let wall = now.saturating_duration_since(at); - let media = min.newest.unwrap_or(held).saturating_sub(held); - if max.exhausted() || leads || wall.max(media) >= interval { - self.source = Lane::Min; - } - } else if max.exhausted() { - self.source = Lane::Min; - } - - let source = &mut self.lanes[self.source as usize]; - if let Some(Queued { read, arrival, .. }) = source.queue.pop_front() { - match &read { - Read::Frame(frame) => { - source.pictures -= 1; - self.arrival = arrival; - if self.dual - && let Some(timestamp) = time(frame.timestamp) - { - self.last = Some(timestamp); - } - } - Read::Gap { timestamp, duration } if self.dual => { - if let Some(resume) = resumes(*timestamp, *duration) { - self.resumed = Some(resume); - } - } - Read::End { end } if self.dual => { - if let Some(end) = time(*end) { - self.ended = Some(end); - } - } - _ => {} - } - return Next::Read(read); - } - - if self.lanes.iter().all(State::exhausted) { - return Next::Done; - } - - // Only a held min-delay picture has a deadline. - let deadline = match self.source { - Lane::Max => self.lanes[Lane::Min as usize] - .held() - .map(|(_, at)| at.saturating_add(interval)), - Lane::Min => None, - }; - Next::Wait(deadline) - } - - /// Drop what the output already moved past, so a cutover never repeats or - /// rewinds a picture, and never returns an end or a loss twice. What has no - /// time orders against nothing, so it stays. - fn prune(&mut self) { - let (last, resumed, ended) = (self.last, self.resumed, self.ended); - for state in &mut self.lanes { - state.queue.retain(|queued| match &queued.read { - Read::Frame(frame) => time(frame.timestamp).is_none_or(|at| last.is_none_or(|last| at > last)), - // Lost by this lane, but the output already reached where it resumes. - Read::Gap { timestamp, duration } => resumes(*timestamp, *duration) - .is_none_or(|resume| last.max(resumed).is_none_or(|reached| resume > reached)), - // Returned from the other lane, or stale behind pictures shown past it. - Read::End { end } => time(*end) - .is_none_or(|end| ended.is_none_or(|ended| end > ended) && last.is_none_or(|last| end >= last)), - }); - state.pictures = state - .queue - .iter() - .filter(|queued| matches!(queued.read, Read::Frame(_))) - .count(); - } - } -} - -/// A timestamp's time since its track's zero, or `None` when it is -/// [unscaled](moq_net::Timestamp::unscaled) and has no time to order by. -fn time(timestamp: moq_net::Timestamp) -> Option { - Duration::try_from(timestamp).ok() -} - -/// Where a gap resumes, or `None` when it has no time. -fn resumes(timestamp: moq_net::Timestamp, duration: moq_net::Timestamp) -> Option { - Some(time(timestamp)? + time(duration)?) -} - -/// One gap covering two with no picture between them, or the newer alone if their -/// scales differ. -fn merge( - first: (moq_net::Timestamp, moq_net::Timestamp), - second: (moq_net::Timestamp, moq_net::Timestamp), -) -> (moq_net::Timestamp, moq_net::Timestamp) { - let merged = || { - let start = if second.0 < first.0 { second.0 } else { first.0 }; - let earlier = first.0.checked_add(first.1).ok()?; - let later = second.0.checked_add(second.1).ok()?; - let end = if earlier > later { earlier } else { later }; - Some((start, end.checked_sub(start).ok()?)) - }; - merged().unwrap_or(second) -} - -#[cfg(test)] -mod tests { - use moq_net::Timestamp; - - use super::*; - - const FRAME: Duration = Duration::from_millis(33); - - fn picture(millis: u64) -> Read { - let i420 = crate::I420::new(crate::Size::new(2, 2), vec![0x80; 6]).unwrap(); - Read::Frame(crate::Frame::new( - crate::Surface::I420(i420), - Timestamp::from_millis(millis).unwrap(), - )) - } - - fn gap(millis: u64, duration: u64) -> Read { - Read::Gap { - timestamp: Timestamp::from_millis(millis).unwrap(), - duration: Timestamp::from_millis(duration).unwrap(), - } - } - - fn end(millis: u64) -> Read { - Read::End { - end: Timestamp::from_millis(millis).unwrap(), - } - } - - /// Everything `next` returns at `now`, as `frame 33`, `gap 66+100`, or `end 99`. - fn reads(selector: &mut Selector, now: Instant) -> Vec { - let mut out = Vec::new(); - while let Next::Read(read) = selector.next(now) { - out.push(match read { - Read::Frame(frame) => format!("frame {}", frame.timestamp.as_millis().unwrap()), - Read::Gap { timestamp, duration } => format!( - "gap {}+{}", - timestamp.as_millis().unwrap(), - duration.as_millis().unwrap() - ), - Read::End { end } => format!("end {}", end.as_millis().unwrap()), - }); - } - out - } - - /// The millisecond timestamps of every picture `next` returns at `now`. - fn drain(selector: &mut Selector, now: Instant) -> Vec { - let mut out = Vec::new(); - while let Next::Read(read) = selector.next(now) { - let Read::Frame(frame) = read else { - panic!("expected only pictures"); - }; - out.push(frame.timestamp.as_millis().unwrap()); - } - out - } - - fn selector() -> Selector { - Selector::new(true, Some(FRAME)) - } - - #[tokio::test] - async fn min_delay_pictures_show_until_the_max_delay_decoder_catches_up() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - - // A cold join: the min-delay decoder is at the live edge, the max-delay - // one a sweep behind and racing through it. - selector.push(Lane::Min, picture(1000), now); - selector.push(Lane::Max, picture(0), now); - assert_eq!(drain(&mut selector, now), [1000]); - assert_eq!(selector.source, Lane::Min); - - selector.push(Lane::Max, picture(500), now); - selector.push(Lane::Min, picture(1033), now); - assert_eq!(drain(&mut selector, now), [1033]); - - // Caught up: the max-delay decoder reached the newest min-delay picture. - selector.push(Lane::Max, picture(1033), now); - selector.push(Lane::Max, picture(1066), now); - assert_eq!(drain(&mut selector, now), [1066]); - assert_eq!(selector.source, Lane::Max); - } - - /// A read with no time, such as a frame a player built at an unscaled key, - /// orders against nothing: it passes through rather than panicking or being - /// pruned, and moves neither lane's clock. - #[tokio::test] - async fn an_unscaled_read_passes_through() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - let i420 = crate::I420::new(crate::Size::new(2, 2), vec![0x80; 6]).unwrap(); - let unscaled = || { - Read::Frame(crate::Frame::new( - crate::Surface::I420(i420.clone()), - Timestamp::unscaled(7), - )) - }; - - selector.push(Lane::Min, picture(1000), now); - assert_eq!(drain(&mut selector, now), [1000]); - selector.push(Lane::Min, unscaled(), now); - selector.push(Lane::Max, unscaled(), now); - selector.push( - Lane::Min, - Read::Gap { - timestamp: Timestamp::unscaled(7), - duration: Timestamp::unscaled(1), - }, - now, - ); - selector.push( - Lane::Min, - Read::End { - end: Timestamp::unscaled(8), - }, - now, - ); - - let mut out = Vec::new(); - while let Next::Read(read) = selector.next(now) { - out.push(match read { - Read::Frame(frame) => format!("frame {:?}", frame.timestamp), - Read::Gap { timestamp, duration } => format!("gap {timestamp:?}+{duration:?}"), - Read::End { end } => format!("end {end:?}"), - }); - } - assert_eq!(out, ["frame #7", "gap #7+#1", "end #8"]); - } - - /// The arrival handed out is the one pushed with the picture returned, from - /// whichever lane, and a picture pushed without one has none. - #[tokio::test] - async fn the_arrival_rides_with_its_picture() { - tokio::time::pause(); - let now = Instant::now(); - // Distinct for every picture and lane, so it says which one was shown. - let arrival = |lane: Lane, millis: u64| now.checked_add(Duration::from_millis(millis * 2 + lane as u64)); - let mut selector = selector(); - let push = |selector: &mut Selector, lane, millis| { - selector.push_arrived(lane, picture(millis), arrival(lane, millis), now); - }; - push(&mut selector, Lane::Min, 1000); - push(&mut selector, Lane::Max, 0); - assert_eq!(drain(&mut selector, now), [1000]); - assert_eq!(selector.arrival(), arrival(Lane::Min, 1000)); - - push(&mut selector, Lane::Max, 500); - push(&mut selector, Lane::Min, 1033); - push(&mut selector, Lane::Max, 1033); - push(&mut selector, Lane::Max, 1066); - assert_eq!(drain(&mut selector, now), [1033, 1066]); - assert_eq!(selector.source, Lane::Max); - assert_eq!(selector.arrival(), arrival(Lane::Max, 1066)); - - selector.push(Lane::Max, picture(1100), now); - assert_eq!(drain(&mut selector, now), [1100]); - assert_eq!(selector.arrival(), None); - } - - /// The max-delay decoder finishing its first picture before the min-delay - /// one has any is not caught up: that picture is a sweep behind the live edge. - #[tokio::test] - async fn a_join_waits_for_the_min_delay_picture() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), now); - assert!(matches!(selector.next(now), Next::Wait(None))); - selector.push(Lane::Min, picture(1000), now); - assert_eq!(drain(&mut selector, now), [1000]); - assert_eq!(selector.source, Lane::Min); - } - - #[tokio::test] - async fn the_max_delay_decoder_keeps_the_picture_without_alternating() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), now); - selector.push(Lane::Min, picture(0), now); - assert_eq!(drain(&mut selector, now), [0]); - assert_eq!(selector.source, Lane::Max); - - // The two race on every frame; whichever decodes first, the max-delay - // picture is the one shown. - for index in 1..10u64 { - selector.push(Lane::Min, picture(index * 33), now); - assert!(matches!(selector.next(now), Next::Wait(Some(_))), "held, not shown"); - selector.push(Lane::Max, picture(index * 33), now); - assert_eq!(drain(&mut selector, now), [u128::from(index * 33)]); - assert_eq!(selector.source, Lane::Max, "alternated at frame {index}"); - } - } - - #[tokio::test] - async fn a_min_delay_skip_hands_the_picture_back_until_max_delay_catches_up() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), now); - selector.push(Lane::Min, picture(0), now); - assert_eq!(drain(&mut selector, now), [0]); - - // The min-delay decoder skipped a stalled group; it leads by far more - // than a frame, so it is shown at once. - selector.push(Lane::Min, picture(1000), now); - assert_eq!(drain(&mut selector, now), [1000]); - assert_eq!(selector.source, Lane::Min); - - // The max-delay decoder waited out the stall and races through it, - // never showing the pictures already passed. - for millis in [33, 66, 500, 999] { - selector.push(Lane::Max, picture(millis), now); - assert_eq!(drain(&mut selector, now), Vec::::new()); - } - selector.push(Lane::Min, picture(1033), now); - assert_eq!(drain(&mut selector, now), [1033]); - selector.push(Lane::Max, picture(1000), now); - selector.push(Lane::Max, picture(1033), now); - assert_eq!(drain(&mut selector, now), Vec::::new()); - assert_eq!(selector.source, Lane::Max, "caught up"); - selector.push(Lane::Max, picture(1066), now); - assert_eq!(drain(&mut selector, now), [1066]); - } - - #[tokio::test] - async fn a_held_picture_shows_within_a_frame_of_wall_time() { - tokio::time::pause(); - let start = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), start); - selector.push(Lane::Min, picture(0), start); - assert_eq!(drain(&mut selector, start), [0]); - - // One newer min-delay picture, then the publisher pauses: the max-delay - // decoder never produces it. - selector.push(Lane::Min, picture(33), start); - let Next::Wait(Some(deadline)) = selector.next(start) else { - panic!("a held picture must carry a deadline"); - }; - assert_eq!(deadline, start.saturating_add(FRAME)); - assert!(matches!( - selector.next(deadline.saturating_sub(Duration::from_millis(1))), - Next::Wait(_) - )); - assert_eq!(drain(&mut selector, deadline), [33]); - } - - #[tokio::test] - async fn a_held_picture_shows_within_a_frame_of_media_time() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), now); - selector.push(Lane::Min, picture(0), now); - assert_eq!(drain(&mut selector, now), [0]); - - // No wall time passes (a caller draining faster than real time), but the - // min-delay decoder moved a frame past the held picture. - selector.push(Lane::Min, picture(20), now); - assert!(matches!(selector.next(now), Next::Wait(_))); - selector.push(Lane::Min, picture(53), now); - assert_eq!(drain(&mut selector, now), [20, 53]); - } - - #[tokio::test] - async fn a_held_picture_shows_when_the_max_delay_decoder_stops() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), now); - selector.push(Lane::Min, picture(0), now); - assert_eq!(drain(&mut selector, now), [0]); - - selector.push(Lane::Min, picture(33), now); - assert!(matches!(selector.next(now), Next::Wait(_))); - // The max-delay decoder ended, as at the end of the track. - selector.finish(Lane::Max); - assert_eq!(drain(&mut selector, now), [33]); - assert_eq!(selector.source, Lane::Min); - } - - #[tokio::test] - async fn the_tail_drains_from_both_lanes() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), now); - selector.push(Lane::Min, picture(0), now); - assert_eq!(drain(&mut selector, now), [0]); - - // The min-delay decoder ends first; the max-delay one still has a tail. - selector.push(Lane::Min, picture(33), now); - selector.finish(Lane::Min); - selector.push(Lane::Max, picture(33), now); - selector.push(Lane::Max, picture(66), now); - assert_eq!(drain(&mut selector, now), [33, 66]); - assert!(matches!(selector.next(now), Next::Wait(None))); - selector.finish(Lane::Max); - assert!(matches!(selector.next(now), Next::Done)); - - // And the other way around: max-delay ends first while min-delay drains. - let mut selector = Selector::new(true, Some(FRAME)); - selector.push(Lane::Min, picture(0), now); - selector.finish(Lane::Max); - selector.push(Lane::Min, picture(33), now); - selector.finish(Lane::Min); - assert_eq!(drain(&mut selector, now), [0, 33]); - assert!(matches!(selector.next(now), Next::Done)); - - // A lane that stopped still shows what it already decoded. - let mut selector = Selector::new(true, Some(FRAME)); - selector.push(Lane::Min, picture(0), now); - assert_eq!(drain(&mut selector, now), [0]); - selector.finish(Lane::Min); - selector.push(Lane::Max, picture(33), now); - selector.finish(Lane::Max); - assert_eq!(drain(&mut selector, now), [33]); - assert!(matches!(selector.next(now), Next::Done)); - } - - #[tokio::test] - async fn one_lane_passes_every_picture_through() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = Selector::new(false, None); - assert!(!selector.wants(Lane::Max)); - // Decode order, as a probe that does not reorder hands it back. - for millis in [0, 66, 33] { - selector.push(Lane::Min, picture(millis), now); - } - selector.finish(Lane::Min); - assert_eq!(drain(&mut selector, now), [0, 66, 33]); - assert!(matches!(selector.next(now), Next::Done)); - } - - #[tokio::test] - async fn one_lane_passes_gaps_and_ends_through_in_order() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = Selector::new(false, None); - for read in [picture(0), gap(33, 967), picture(1000), end(1033), picture(2000)] { - selector.push(Lane::Min, read, now); - } - selector.finish(Lane::Min); - assert_eq!( - reads(&mut selector, now), - ["frame 0", "gap 33+967", "frame 1000", "end 1033", "frame 2000"] - ); - assert!(matches!(selector.next(now), Next::Done)); - } - - /// Both decoders read the same end; it comes out once, between the stopped - /// encoder's pictures and the fresh one's. - #[tokio::test] - async fn an_end_both_lanes_read_is_returned_once() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), now); - selector.push(Lane::Min, picture(0), now); - assert_eq!(reads(&mut selector, now), ["frame 0"]); - - // The min-delay decoder reaches the end first, while the source is max-delay. - selector.push(Lane::Min, picture(33), now); - selector.push(Lane::Min, end(66), now); - for read in [picture(33), end(66), picture(1000)] { - selector.push(Lane::Max, read, now); - } - assert_eq!(reads(&mut selector, now), ["frame 33", "end 66", "frame 1000"]); - selector.push(Lane::Min, picture(1000), now); - selector.push(Lane::Min, picture(1033), now); - selector.push(Lane::Max, picture(1033), now); - assert_eq!(reads(&mut selector, now), ["frame 1033"]); - } - - /// A loss on one decoder that the other delivered past is no loss in what - /// comes out; one on the decoder that takes over is. - #[tokio::test] - async fn a_gap_counts_only_when_the_output_skips_it() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), now); - selector.push(Lane::Min, picture(0), now); - assert_eq!(reads(&mut selector, now), ["frame 0"]); - - // The min-delay decoder lost 33..66, which the max-delay one decoded. - selector.push(Lane::Max, picture(33), now); - selector.push(Lane::Max, picture(66), now); - selector.push(Lane::Min, gap(33, 33), now); - selector.push(Lane::Min, picture(66), now); - assert_eq!(reads(&mut selector, now), ["frame 33", "frame 66"]); - - // Then it skips a stalled group the max-delay decoder is still waiting on. - selector.push(Lane::Min, gap(99, 901), now); - selector.push(Lane::Min, picture(1000), now); - assert_eq!(reads(&mut selector, now), ["gap 99+901", "frame 1000"]); - assert_eq!(selector.source, Lane::Min); - } - - /// Boundaries the min-delay decoder reads while the max-delay source stalls fold - /// into one gap and the first end, however many marker groups arrive. - #[tokio::test] - async fn a_run_of_boundaries_holds_one_gap_and_one_end() { - tokio::time::pause(); - let now = Instant::now(); - let mut selector = selector(); - selector.push(Lane::Max, picture(0), now); - selector.push(Lane::Min, picture(0), now); - assert_eq!(reads(&mut selector, now), ["frame 0"]); - - selector.push(Lane::Min, end(33), now); - for index in 1..1000 { - selector.push(Lane::Min, gap(33 * index, 10), now); - selector.push(Lane::Min, end(33 * index + 10), now); - } - assert_eq!(selector.lanes[Lane::Min as usize].queue.len(), 2); - selector.push(Lane::Min, picture(40_000), now); - assert_eq!(reads(&mut selector, now), ["end 33", "gap 33+32944", "frame 40000"]); - } - - /// Whatever order the two lanes deliver in, and however often the source - /// changes, what comes out strictly increases. - #[tokio::test] - async fn output_strictly_increases_across_every_switch() { - tokio::time::pause(); - let start = Instant::now(); - let mut selector = Selector::new(true, None); - // A deterministic interleaving with skips, stalls, and catch-ups on both. - let min = [0, 33, 66, 1000, 1033, 1066, 1100, 2000, 2033, 2066, 2100, 2133]; - let max = [0, 33, 66, 100, 133, 1000, 1033, 1066, 1100, 1133, 2033, 2133, 2166]; - let mut out = Vec::new(); - let (mut a, mut b) = (0, 0); - let mut now = start; - while a < min.len() || b < max.len() { - // Min-delay runs two steps for every max-delay step, then the reverse. - let min_first = (a + b) % 5 < 3; - if (min_first || b >= max.len()) && a < min.len() { - selector.push(Lane::Min, picture(min[a]), now); - a += 1; - } else { - selector.push(Lane::Max, picture(max[b]), now); - b += 1; - } - now = now.saturating_add(Duration::from_millis(7)); - out.extend(drain(&mut selector, now)); - } - selector.finish(Lane::Min); - selector.finish(Lane::Max); - out.extend(drain(&mut selector, now)); - assert!(out.windows(2).all(|pair| pair[0] < pair[1]), "not increasing: {out:?}"); - assert_eq!(out.last(), Some(&2166), "the newest picture was lost: {out:?}"); - } -} From 9c1ffc28e5c07d95f4c78cbc600bfb271851376d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:44:11 -0800 Subject: [PATCH 3/5] feat(watch): a warmup rendition plays through one decoder Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/watch.md | 19 +- js/hang/src/container/consumer.ts | 2 +- js/watch/src/video/config.ts | 3 +- js/watch/src/video/decoder.test.ts | 438 +++----------------- js/watch/src/video/decoder.ts | 328 +++------------ js/watch/src/video/recovery.ts | 32 +- quest/m2/warmup-one-decoder.md | 51 --- quest/m4/intra-refresh/README.md | 18 +- quest/m4/intra-refresh/consumer-warmup.md | 4 +- quest/m4/intra-refresh/dual-decoder.md | 66 +-- quest/m4/intra-refresh/recovery-boundary.md | 9 +- rs/moq-ffi/src/video.rs | 2 +- rs/moq-mux/src/container/consumer.rs | 2 +- 13 files changed, 215 insertions(+), 759 deletions(-) delete mode 100644 quest/m2/warmup-one-decoder.md diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 5a0ea772564..a0b6f8ee103 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -65,19 +65,12 @@ broken media automatically. A video rendition encoded with intra refresh has no keyframes, and its catalog `warmup` says how long a decoder takes to clean up after joining. The player -decodes such a rendition twice: one decoder holds your delay and may show -refresh stripes after a join or a skip, while a second starts one group earlier -and tolerates `warmup` more lateness, so its picture is clean. The clean -picture shows whenever it has caught up. Each decoder costs its own CPU and -decoder memory, and both share one subscription. A rendition without `warmup` -uses one decoder. - -An H.264 group of such a rendition opens on a recovery point that is not an -IDR. Chromium's WebCodecs refuses to open a decoder on that picture: it wants -an IDR or a recovery point that counts zero frames. So when a decoder opens on -one, the player inserts a second recovery point with a zero count before its -first slice. The slices are untouched; the stripes clear as the sweep -completes. +joins such a rendition that much earlier and decodes from the group start, but +shows nothing until the warmup has passed, after a join and after each skip. +An H.264 group opens on a recovery point that is not an IDR, which Chromium's +WebCodecs refuses unless it counts zero frames, so the player inserts one that +does before the first slice. Firefox and Safari are untested, and Chromium +refuses an H.265 group that opens without an IDR. `el.broadcast.out.status` is `connecting`, `offline`, `loading`, `live`, `error`, or `unsupported`. It reads `connecting` until a session could have announced the broadcast: while the diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 870d7e4c781..c6a071e9d30 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -58,7 +58,7 @@ export interface ConsumerProps { /** * How much further than the min delay the start reaches, in milliseconds (default: 0), so a * decoder that needs that much media before its output is clean starts early enough, such as an - * Opus decoder or the max-delay decoder of an intra-refresh rendition. Pass the catalog + * Opus decoder or an intra-refresh video decoder. Pass the catalog * rendition's `warmup`. Mirrors Rust's `container::Playout`. * * Put the max delay plus the warmup on the subscription, since the publisher skips what falls diff --git a/js/watch/src/video/config.ts b/js/watch/src/video/config.ts index e7b58ee04f9..68cf7afa9be 100644 --- a/js/watch/src/video/config.ts +++ b/js/watch/src/video/config.ts @@ -3,7 +3,7 @@ import type * as Catalog from "@moq/hang/catalog"; /** The catalog fields that determine the demuxer and WebCodecs decoder instance. */ export type DecoderConfig = Pick< Catalog.VideoConfig, - "codec" | "container" | "description" | "displayAspectWidth" | "displayAspectHeight" + "codec" | "container" | "description" | "displayAspectWidth" | "displayAspectHeight" | "warmup" > & { optimizeForLatency: boolean; interlaced: boolean; @@ -31,6 +31,7 @@ export function playbackIdentity(config: Catalog.VideoConfig): PlaybackIdentity displayAspectHeight: config.displayAspectHeight, optimizeForLatency: config.optimizeForLatency ?? true, interlaced: config.interlaced ?? false, + warmup: config.warmup, }, }; } diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 526267a9840..77e40117d02 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -167,18 +167,15 @@ class Picture { displayWidth = 16; displayHeight = 16; readonly timestamp: number; - // The index of the codec that decoded it, in creation order, or -1 for one a test made. - readonly codec: number; readonly #pictures: Picture[]; - constructor(timestamp: number, pictures: Picture[], codec = -1) { + constructor(timestamp: number, pictures: Picture[]) { this.#pictures = pictures; pictures.push(this); this.timestamp = timestamp; - this.codec = codec; } clone() { if (this.closed) throw new Error("clone of a closed frame"); - return new Picture(this.timestamp, this.#pictures, this.codec); + return new Picture(this.timestamp, this.#pictures); } close() { this.closed = true; @@ -313,7 +310,7 @@ async function guardedPlayback( payloads[this.index].push((chunk as unknown as { data: Uint8Array }).data); const emit = () => { outputs.push(chunk.timestamp); - const picture = new Picture(chunk.timestamp, pictures, this.index); + const picture = new Picture(chunk.timestamp, pictures); void this.callbacks.output(picture as unknown as VideoFrame); }; if (owed.delete(chunk.timestamp)) { @@ -400,9 +397,7 @@ async function guardedPlayback( delay, fail: (error: DOMException, index = -1) => errors.at(index)?.(error), emit: (timestamp: number, index = -1) => - frames.at(index)?.( - new Picture(timestamp, pictures, (index + frames.length) % frames.length) as unknown as VideoFrame, - ), + frames.at(index)?.(new Picture(timestamp, pictures) as unknown as VideoFrame), get codecs() { return errors.length; }, @@ -1685,38 +1680,28 @@ for (const kind of ["legacy", "cmaf"] as const) { describe("Decoder with a warmup rendition", () => { // One refresh sweep, which is one group. const WARMUP = Catalog.u53(300); - // The codecs in creation order: the min-delay decoder opens first. - const MIN = 0; - const MAX = 1; // A frame of `group` at `ms`, a recovery point when `key`. const at = (group: bigint, ms: number, key = false) => sample(group, ms * 1000, key); - // A warmup rendition whose two container consumers each read their own feed, presenting at once. - // `hold` keeps those timestamps in the codec until `release`. - async function refresh(fields: Partial = { warmup: WARMUP }, hold: number[] = []) { + // A warmup rendition reading the container consumer's reads from a feed, presenting at once. + async function refresh(fields: Partial = { warmup: WARMUP }) { const feeds = new Feeds(); - const playback = await guardedPlayback("legacy", [], hold, false, 0, { fields, feeds }); - // Every picture shown, as [ms, codec]. - const shown: [number, number][] = []; + const playback = await guardedPlayback("legacy", [], [], false, 0, { fields, feeds }); + // Every picture shown, in ms. + const shown: number[] = []; const watch = new Effect(); watch.subscribe(playback.decoder.out.frame, (frame) => { const picture = frame as unknown as Picture | undefined; - if (picture) shown.push([picture.timestamp / 1000, picture.codec]); + if (picture) shown.push(picture.timestamp / 1000); }); - const current = () => { - const picture = playback.decoder.out.frame.peek() as unknown as Picture | undefined; - return picture && [picture.timestamp / 1000, picture.codec]; - }; - const push = async (lane: number, ...reads: Read[]) => { - feeds.push(lane, ...reads); - await microtasks(); - }; return { playback, shown, - current, - push, + push: async (...reads: Read[]) => { + feeds.push(0, ...reads); + await microtasks(); + }, async close() { watch.close(); playback.close(); @@ -1726,28 +1711,15 @@ describe("Decoder with a warmup rendition", () => { }; } - function ascending(shown: [number, number][]) { - for (let i = 1; i < shown.length; i++) { - expect(shown[i][0]).toBeGreaterThanOrEqual(shown[i - 1][0]); - } - } - - it("opens one subscription and one decoder without a warmup", async () => { - for (const [fields, decoders] of [ - [{}, 1], - [{ warmup: Catalog.u53(0) }, 1], - [{ warmup: WARMUP }, 2], - // An hour of warmup reaches back only as far as the delay cap. - [{ warmup: Catalog.u53(3_600_000) }, 2], - ] as const) { + it("opens one decoder whose subscription tolerates the warmup", async () => { + for (const fields of [{}, { warmup: Catalog.u53(0) }, { warmup: WARMUP }, { warmup: Catalog.u53(3_600_000) }]) { const subscribe = spyOn(Moq.Track.Consumer.prototype, "subscribe"); const run = await refresh(fields); try { - expect(run.playback.codecs).toBe(decoders); - const budgets = subscribe.mock.calls.map(([options]) => options?.delay?.max); - // The max-delay decoder tolerates one sweep more; js/net merges the two upstream. + expect(run.playback.codecs).toBe(1); + // An hour of warmup reaches back only as far as the delay cap. const warmup = Duration.millis(fields.warmup ?? 0).min(DELAY_MAX); - expect(budgets).toEqual(decoders === 1 ? [Duration.millis(0)] : [Duration.millis(0), warmup]); + expect(subscribe.mock.calls.map(([options]) => options?.delay?.max)).toEqual([warmup]); } finally { subscribe.mockRestore(); await run.close(); @@ -1755,231 +1727,41 @@ describe("Decoder with a warmup rendition", () => { } }); - it("shows the min-delay picture on a cold join, then the max-delay one once it catches up", async () => { - const run = await refresh(); - try { - expect(run.playback.codecs).toBe(2); - - // The max-delay decoder joined a group earlier. Before the min-delay decoder anchors the - // clock, its pictures have no place on it. - await run.push(MAX, at(9n, 700, true), at(9n, 733)); - expect(run.current()).toBeUndefined(); - - // The min-delay decoder's picture shows at once, stripes and all. - await run.push(MIN, at(10n, 1000, true)); - expect(run.current()).toEqual([1000, MIN]); - - // Pictures behind it never win, even though they are clean. - await run.push(MAX, at(9n, 966)); - expect(run.current()).toEqual([1000, MIN]); - - // Caught up: the max-delay picture wins the tie, and keeps the lead. - await run.push(MAX, at(10n, 1000, true)); - expect(run.current()).toEqual([1000, MAX]); - await run.push(MIN, at(10n, 1033)); - await run.push(MAX, at(10n, 1033)); - expect(run.current()).toEqual([1033, MAX]); - ascending(run.shown); - } finally { - await run.close(); - } - }); - - it("keeps the max-delay picture through a stall shorter than one sweep", async () => { - const run = await refresh(); - try { - for (const lane of [MIN, MAX]) await run.push(lane, at(10n, 1000, true), at(10n, 1033)); - expect(run.current()).toEqual([1033, MAX]); - - // Group 10 stalls. The min-delay decoder skips to group 11 and shows its stripes. - await run.push(MIN, gap, at(11n, 1300, true)); - expect(run.current()).toEqual([1300, MIN]); - - // The max-delay decoder tolerates the stall: it gets the rest of group 10, decodes it - // behind the playhead, and takes over again once it reaches the min-delay picture. - await run.push(MAX, at(10n, 1066), at(10n, 1100), at(10n, 1266)); - expect(run.current()).toEqual([1300, MIN]); - await run.push(MAX, at(11n, 1300, true)); - expect(run.current()).toEqual([1300, MAX]); - await run.push(MIN, at(11n, 1333)); - await run.push(MAX, at(11n, 1333)); - expect(run.current()).toEqual([1333, MAX]); - - // Neither codec restarted: the keyframe the min-delay decoder skipped to re-primes it. - expect(run.playback.resets).toBe(0); - expect(run.playback.decoded[MAX]).toEqual( - [1000, 1033, 1066, 1100, 1266, 1300, 1333].map((ms) => ms * 1000), - ); - ascending(run.shown); - } finally { - await run.close(); - } - }); - - it("skips both decoders on a stall longer than one sweep", async () => { + it("hides the warmup after a join and after a gap, but not across groups or a fresh encoder", async () => { const run = await refresh(); try { - for (const lane of [MIN, MAX]) await run.push(lane, at(10n, 1000, true), at(10n, 1033)); - - // Past the max-delay decoder's budget too: both skip, keeping their codecs. - await run.push(MIN, gap, at(12n, 1600, true)); - expect(run.current()).toEqual([1600, MIN]); - await run.push(MAX, gap, at(12n, 1600, true)); - expect(run.current()).toEqual([1600, MAX]); - expect(run.playback.resets).toBe(0); - ascending(run.shown); + // The join's recovery point at 1000: the next 300ms are decoded but not shown. + await run.push(at(10n, 1000, true), at(10n, 1100), at(10n, 1200), at(10n, 1300)); + // The next group continues the codec, so it shows from its first picture. + await run.push(at(11n, 1400, true), at(11n, 1500)); + expect(run.playback.decoded[0]).toEqual([1000, 1100, 1200, 1300, 1400, 1500].map((ms) => ms * 1000)); + expect(run.shown).toEqual([1300, 1400, 1500]); + + // A skip resumes at group 13's recovery point, warming up again. + await run.push(gap, at(13n, 2000, true), at(13n, 2200), at(13n, 2300), at(14n, 2400, true)); + expect(run.shown).toEqual([1300, 1400, 1500, 2300, 2400]); + + // A declared end, then a fresh encoder on another timeline: its recovery point is clean. + await run.push(marker(14n, 2_500_000), at(15n, 0, true), at(15n, 100)); + expect(run.shown).toEqual([1300, 1400, 1500, 2300, 2400, 0, 100]); + expect(run.playback.resets).toBe(1); } finally { await run.close(); } }); - it("moves the shared clock only from the min-delay decoder", async () => { - const run = await refresh(); - const received = spyOn(Track.prototype, "received"); - const reset = spyOn(Track.prototype, "reset"); - try { - await run.push(MIN, at(10n, 1000, true), at(10n, 1033)); - await run.push(MAX, at(9n, 700, true), at(9n, 733), at(10n, 1000, true), at(10n, 1033)); - expect(received.mock.calls.map(([timestamp]) => timestamp)).toEqual([ - Duration.millis(1000), - Duration.millis(1033), - ]); - expect(run.current()).toEqual([1033, MAX]); - - // The publisher restarts its timeline. The min-delay decoder restarts video's timeline on the - // clock and shows the new one; the max-delay picture still on the old one never wins. - await run.push(MIN, marker(10n, 1_066_000), at(11n, 0, true)); - expect(reset).toHaveBeenCalledTimes(1); - expect(run.current()).toEqual([0, MIN]); - - // Once the max-delay decoder reads the restart too, it takes over again, without its own reset. - await run.push(MAX, marker(10n, 1_066_000), at(11n, 0, true)); - expect(reset).toHaveBeenCalledTimes(1); - expect(run.current()).toEqual([0, MAX]); - expect(received.mock.calls.map(([timestamp]) => timestamp)).toEqual([ - Duration.millis(1000), - Duration.millis(1033), - Duration.millis(0), - ]); - } finally { - received.mockRestore(); - reset.mockRestore(); - await run.close(); - } - }); - - it("keeps a picture held from before a restart out of the new timeline", async () => { - const run = await refresh(undefined, [0]); - try { - for (const lane of [MIN, MAX]) await run.push(lane, at(10n, 1000, true)); - expect(run.current()).toEqual([1000, MAX]); - - // The min-delay decoder reads a restart, but its first new picture is still in the codec. - // The picture it holds from the old timeline must not show, nor block the new one. - await run.push(MIN, marker(10n, 1_066_000), at(11n, 0, true)); - expect(run.current()).toEqual([1000, MAX]); - run.playback.release(); - await microtasks(); - expect(run.current()).toEqual([0, MIN]); - } finally { - await run.close(); - } - }); - - it("promotes a pending rendition once the outgoing one restarts its timeline", async () => { - const run = await refresh({}, [0]); - try { - await run.push(MIN, at(10n, 1000, true)); - expect(run.current()).toEqual([1000, MIN]); - - // The outgoing rendition reads a restart whose first picture is still in the codec. It has - // presented nothing on the new timeline, so the replacement need not catch up to 1000. - await run.push(MIN, marker(10n, 1_066_000), at(11n, 0, true)); - expect(run.current()).toEqual([1000, MIN]); - run.playback.switchRendition(); - await microtasks(); - expect(run.playback.codecs).toBe(2); - // Emitted straight from the replacement codec, since decoding there would flush the held picture. - run.playback.emit(0, 1); - await microtasks(); - expect(run.current()).toEqual([0, 1]); - } finally { - await run.close(); - } - }); - - it("takes the max-delay picture when that decoder opens after a restart", async () => { + it("shows every picture of a rendition without a warmup", async () => { const run = await refresh({}); try { - expect(run.playback.codecs).toBe(1); - await run.push(MIN, at(10n, 1000, true), marker(10n, 1_066_000), at(11n, 0, true), at(11n, 33)); - expect(run.current()).toEqual([33, MIN]); - - // The catalog gains a warmup. The new decoder never saw the restart, yet reads the same timeline. - run.playback.switchRendition("video", { warmup: WARMUP }); - await microtasks(); - expect(run.playback.codecs).toBe(2); - await run.push(MAX, at(11n, 0, true), at(11n, 33), at(11n, 66)); - await run.push(MIN, at(11n, 66)); - expect(run.current()).toEqual([66, MAX]); - } finally { - await run.close(); - } - }); - - it("never shows a lower timestamp across a 100ms trailing gap and a recovery boundary", async () => { - const run = await refresh(); - try { - await run.push(MIN, at(10n, 1000, true)); - let min = 1000; - let max = 900; - for (let cycle = 0; cycle < 6; cycle++) { - // Trailing by 100ms, the max-delay decoder catches up at twice the frame rate. - while (max < min) { - min += 33; - run.playback.emit(min * 1000, MIN); - run.playback.emit(max * 1000, MAX); - max = Math.min(max + 66, min); - run.playback.emit(max * 1000, MAX); - await microtasks(); - } - // Level for a few frames, the max-delay picture winning each tie. - for (let i = 0; i < 3; i++) { - min += 33; - max = min; - run.playback.emit(min * 1000, MIN); - await microtasks(); - run.playback.emit(max * 1000, MAX); - await microtasks(); - expect(run.current()).toEqual([min, MAX]); - } - // A skip leaves it 100ms behind again. - min += 100; - } - expect(run.shown.some(([, codec]) => codec === MIN)).toBe(true); - ascending(run.shown); - } finally { - await run.close(); - } - }); - - it("hands the picture back when the max-delay decoder stops producing", async () => { - const run = await refresh(); - try { - for (const lane of [MIN, MAX]) await run.push(lane, at(10n, 1000, true)); - expect(run.current()).toEqual([1000, MAX]); - - await run.push(MIN, at(10n, 1033)); - expect(run.current()).toEqual([1033, MIN]); - await run.push(MIN, at(10n, 1066)); - expect(run.current()).toEqual([1066, MIN]); + await run.push(at(10n, 1000, true), at(10n, 1100), gap, at(13n, 2000, true), at(13n, 2100)); + expect(run.shown).toEqual([1000, 1100, 2000, 2100]); } finally { await run.close(); } }); - // Through real container consumers: the max-delay one reaches back one more sweep for its start. - it("joins the max-delay decoder one group earlier", async () => { + // Through a real container consumer, which reaches back one more sweep for its start. + it("joins one group earlier and plays the live edge at the delay", async () => { const clock = manualAt(5000); const playback = await guardedPlayback("legacy", [], [], true, 0, { fields: { warmup: WARMUP }, @@ -1997,115 +1779,29 @@ describe("Decoder with a warmup rendition", () => { // A relay serves a cold join newest first: the live group, then the sweep before it. write(track.createGroup(5n), 1500, 1620); await flush(); - const live = track.createGroup(4n); - write(live, 1200, 1470); - live.finish(); + const earlier = track.createGroup(4n); + write(earlier, 1200, 1470); + earlier.finish(); await flush(); - // The min-delay decoder starts at the live group, which covers its 100ms reach. The - // max-delay one reaches 400ms back, to the group before. - expect(playback.decoded[MIN][0]).toBe(1_500_000); - expect(playback.decoded[MAX][0]).toBe(1_200_000); - expect(playback.decoded[MAX]).toContain(1_620_000); - - // The poster is the min-delay picture. The burst moved the playhead to 1520, so by the - // time the rest comes due, the max-delay decoder has decoded it too, and its clean - // pictures win. - const shown = playback.decoder.out.frame.peek() as unknown as Picture; - expect([shown.timestamp, shown.codec]).toEqual([1_500_000, MIN]); + // The start reaches 400ms back, to the group before the live one, and decodes it all. + expect(playback.decoded[0][0]).toBe(1_200_000); + expect(playback.decoded[0]).toContain(1_620_000); + + // The poster is the first picture past the warmup. The live group's arrivals hold the clock + // 100ms behind the live edge, not another 300ms behind for the warmup's. + const poster = playback.decoder.out.frame.peek() as unknown as Picture; + expect(poster.timestamp).toBe(1_500_000); await clock.advance(10); await microtasks(); const due = playback.decoder.out.frame.peek() as unknown as Picture; - expect([due.timestamp, due.codec]).toEqual([1_530_000, MAX]); + expect(due.timestamp).toBe(1_530_000); } finally { playback.close(); await microtasks(); clock.installed.close(); } }); - - // Through real container consumers, on a live track of 300ms groups at 30ms per frame, 100ms - // behind. Group 2 stalls after 660ms while group 3 keeps arriving, until `release`. - async function stall(release: number) { - const clock = manualAt(0); - const playback = await guardedPlayback("legacy", [], [], true, 0, { - fields: { warmup: WARMUP }, - initial: Duration.millis(100), - }); - const track = playback.track as Moq.Track.Producer; - const shown: [number, number][] = []; - const watch = new Effect(); - watch.subscribe(playback.decoder.out.frame, (frame) => { - const picture = frame as unknown as Picture | undefined; - if (picture) shown.push([picture.timestamp / 1000, picture.codec]); - }); - const groups: Moq.Group.Producer[] = []; - const write = (ms: number) => { - const sequence = Math.floor(ms / 300); - groups[sequence] ??= track.createGroup(BigInt(sequence)); - groups[sequence].writeFrame(legacyFrame(Duration.millis(ms))); - if (ms % 300 === 270) groups[sequence].finish(); - }; - const stalled = (ms: number) => ms > 660 && ms < 900; - try { - for (let ms = 0; ms <= 1500; ms += 10) { - if (ms % 30 === 0 && !stalled(ms)) write(ms); - if (ms === release) for (let late = 690; late < 900; late += 30) write(late); - await clock.advance(10); - await microtasks(); - } - return { decoded: playback.decoded, resets: playback.resets, shown }; - } finally { - watch.close(); - playback.close(); - await microtasks(); - clock.installed.close(); - } - } - - it("keeps the max-delay decoder continuous through a live stall shorter than one sweep", async () => { - const { decoded, resets, shown } = await stall(1000); - const stalled = [690, 720, 750, 780, 810, 840, 870].map((ms) => ms * 1000); - // The min-delay decoder skipped the rest of group 2 and resumed at group 3. - for (const ms of stalled) expect(decoded[MIN]).not.toContain(ms); - expect(resets).toBe(0); - // The max-delay decoder waited for it. - for (const ms of stalled) expect(decoded[MAX]).toContain(ms); - // Stripes showed until the max-delay picture caught up, then it took over again. - expect(shown.some(([ms, codec]) => ms >= 900 && codec === MIN)).toBe(true); - expect(shown.at(-1)?.[1]).toBe(MAX); - ascending(shown); - }); - - it("skips both decoders on a live stall longer than one sweep", async () => { - const { decoded, resets, shown } = await stall(1400); - for (const lane of [MIN, MAX]) expect(decoded[lane]).not.toContain(690_000); - expect(resets).toBe(0); - expect(shown.at(-1)?.[1]).toBe(MAX); - ascending(shown); - }); - - it("holds a leading max-delay picture when that decoder closes", async () => { - const run = await refresh(); - try { - for (const lane of [MIN, MAX]) await run.push(lane, at(10n, 1000, true)); - // The min-delay decoder is slower to deliver, so the max-delay one leads. - await run.push(MAX, at(10n, 1033), at(10n, 1066)); - expect(run.current()).toEqual([1066, MAX]); - - // The catalog drops its warmup. The min-delay picture behind does not replace the lead. - run.playback.switchRendition("video", { warmup: Catalog.u53(0) }); - await microtasks(); - expect(run.current()).toEqual([1066, MAX]); - await run.push(MIN, at(10n, 1033)); - expect(run.current()).toEqual([1066, MAX]); - await run.push(MIN, at(10n, 1066), at(10n, 1100)); - expect(run.current()).toEqual([1100, MIN]); - ascending(run.shown); - } finally { - await run.close(); - } - }); }); // H.264 NAL units, without their start code or length prefix. @@ -2174,8 +1870,6 @@ describe("insertRecoveryPoint", () => { for (const kind of ["legacy", "cmaf"] as const) { describe(`Decoder opening a ${kind} warmup rendition`, () => { - const MIN = 0; - const MAX = 1; const recovery = avcc([NAL.sweep, NAL.slice]); const patched = avcc([NAL.sweep, NAL.recovered, NAL.slice]); const delta = avcc([NAL.slice]); @@ -2188,8 +1882,8 @@ for (const kind of ["legacy", "cmaf"] as const) { const playback = await guardedPlayback(kind, [], [], false, 0, { fields, feeds }); return { playback, - push: async (lane: number, ...reads: Read[]) => { - feeds.push(lane, ...reads); + push: async (...reads: Read[]) => { + feeds.push(0, ...reads); await microtasks(); }, }; @@ -2198,18 +1892,16 @@ for (const kind of ["legacy", "cmaf"] as const) { it("rewrites only the recovery point that opens each codec", async () => { const { playback, push } = await open({ warmup: Catalog.u53(300) }); try { - expect(playback.codecs).toBe(2); - for (const lane of [MIN, MAX]) { - await push(lane, at(10n, 1000, true, recovery), at(10n, 1033, false, delta)); - // The next group continues the open codec, so Chromium does not check it. - await push(lane, at(11n, 1300, true, recovery)); - } - for (const lane of [MIN, MAX]) expect(playback.payloads[lane]).toEqual([patched, delta, recovery]); + expect(playback.codecs).toBe(1); + await push(at(10n, 1000, true, recovery), at(10n, 1033, false, delta)); + // The next group continues the open codec, so Chromium does not check it. + await push(at(11n, 1300, true, recovery)); + expect(playback.payloads[0]).toEqual([patched, delta, recovery]); // A skip keeps the open codec, so the group it resumes on is left alone. - await push(MIN, gap, at(13n, 1900, true, recovery)); + await push(gap, at(13n, 1900, true, recovery)); expect(playback.resets).toBe(0); - expect(playback.payloads[MIN].at(-1)).toEqual(recovery); + expect(playback.payloads[0].at(-1)).toEqual(recovery); } finally { playback.close(); await microtasks(); @@ -2219,8 +1911,8 @@ for (const kind of ["legacy", "cmaf"] as const) { it("leaves an opening IDR unchanged", async () => { const { playback, push } = await open({ warmup: Catalog.u53(300) }); try { - for (const lane of [MIN, MAX]) await push(lane, at(10n, 1000, true, idr)); - for (const lane of [MIN, MAX]) expect(playback.payloads[lane]).toEqual([idr]); + await push(at(10n, 1000, true, idr)); + expect(playback.payloads[0]).toEqual([idr]); } finally { playback.close(); await microtasks(); @@ -2231,9 +1923,9 @@ for (const kind of ["legacy", "cmaf"] as const) { const { playback, push } = await open({}); try { expect(playback.codecs).toBe(1); - await push(MIN, at(10n, 1000, true, recovery), gap, at(13n, 1900, true, recovery)); + await push(at(10n, 1000, true, recovery), gap, at(13n, 1900, true, recovery)); expect(playback.resets).toBe(0); - expect(playback.payloads[MIN]).toEqual([recovery, recovery]); + expect(playback.payloads[0]).toEqual([recovery, recovery]); } finally { playback.close(); await microtasks(); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index f4d7d485c28..a40fb04ab11 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -334,7 +334,7 @@ export class Decoder { // Only a rendition read from the same path continues the outgoing arrival estimate: another // path's arrivals have nothing to do with what this one learned. const outgoing = this.#active.peek(); - const predecessor = outgoing?.broadcast.path === active.path ? outgoing : undefined; + const predecessor = outgoing?.broadcast.path === active.path ? outgoing.consumer : undefined; const rendition = new Rendition({ sync: this.sync, @@ -417,7 +417,7 @@ export class Decoder { this.#clearCurrentFrame(); }); - // Clone the frame so we own it independently of the DecoderTrack. + // Clone the frame so we own it independently of the Rendition. // proxy() would share the same reference, allowing the source to close our frame. effect.run((inner) => { const frame = inner.get(active.frame); @@ -552,16 +552,10 @@ export class Decoder { static supported = supported; } -interface DecoderTrackProps { +interface RenditionProps { sync: Sync; // Video's hold on the clock, shared by every rendition: their frames are one timeline. clock: Track; - // Set for the max-delay decoder of a rendition with a catalog `warmup`: its start and its - // lateness budget reach this much further, and it leaves the shared clock and stats alone. - warmup?: Getter; - // Whether the rendition has a catalog `warmup`, so its groups may open on a recovery point that - // is not an IDR. Set for both decoders. - refresh?: Getter; broadcast: Moq.Broadcast.Consumer; // The rendition ID; `name` is its track's name in `broadcast`. track: string; @@ -575,15 +569,21 @@ interface DecoderTrackProps { // arrival estimate it continues. predecessor?: Container.Consumer; - // Absent for the max-delay decoder: the bytes arrive once and the min-delay decoder counts them. - stats?: Signal; + stats: Signal; // The rendition's feedback counters, under `key`: its ID in the catalog it was selected from. - // Absent for the max-delay decoder, like `stats`. - tally?: Tally; + tally: Tally; key: Key; } -class DecoderTrack { +/** + * One rendition's playback through one WebCodecs decoder. + * + * A rendition with a catalog `warmup` (intra refresh, so no keyframes) joins that much earlier, + * through the container consumer's reach, and decodes from the group start without presenting the + * warmup that follows each join or gap, so its first picture is clean. Rust players do the same in + * `moq_video::decode::Consumer`. + */ +class Rendition { error = new Signal(undefined); sync: Sync; clock: Track; @@ -593,39 +593,21 @@ class DecoderTrack { config: DecoderConfig; identity: string; catalog: Getter; - stats?: Signal; - tally?: Tally; + stats: Signal; + tally: Tally; key: Key; timestamp = new Signal(undefined); frame = new Signal(undefined); - // The group the current `frame` was read from. Group sequences are shared by every subscription to - // the track, so they place a picture on a timeline for either decoder. - group?: bigint; - // The first group of the timeline this decoder reads, undefined until a publisher restart. - readonly start = new Signal(undefined); - - // Counts the timeline restarts this decoder read. A picture decoded before one is from the old - // timeline and never presents after it, even when its clock wait already resolved. - #generation = 0; - // The group each frame in the codec was read from, by timestamp. Each output found here retires its - // entry and every one before it. A publisher that never outputs a picture grows this by one entry - // per chunk; the maintainer accepted that as an exception to the bounded-state rule - // (moq-dev/moq#7555, 2026-11-01), since WebCodecs bounds no codec's pending output, so any cap is - // a guess that can evict a picture the codec still owes. - #groups = new Map(); - - // How far past the shared clock's budgets this decoder reaches. - #warmup: Getter; - // The playout delay is where a cold join starts, and the shared max delay how late it may fall. - // The max-delay decoder adds the warmup to both, so it starts at the group before and keeps - // it, as Rust's does, rather than passing the consumer a `warmup` that would reach further. - // Its subscription and its consumer take the same delay. + // The catalog `warmup`, capped like the delay so a publisher cannot hold the viewer without bound. + #warmup: Duration; + // The playout delay is where a cold join starts, and the shared max delay how late it may fall. The + // consumer's warmup reaches the start further back; the subscription widens its max by the same + // warmup, since the publisher skips what falls outside it. #delay: Getter; - // Whether this decoder drives the shared clock: only the min-delay one reports arrivals and - // restarts the timeline. - #drives: boolean; - #refresh: Getter; + #subscription: Getter; + // The running codec's join and gap state: which leading pictures to refuse, and which still warm up. + #recovery = new Recovery(); // Network jitter + decode buffer. buffered = new Signal([]); @@ -669,7 +651,7 @@ class DecoderTrack { #signals = new Effect(); - constructor(props: DecoderTrackProps) { + constructor(props: RenditionProps) { this.sync = props.sync; this.clock = props.clock; this.broadcast = props.broadcast; @@ -681,13 +663,12 @@ class DecoderTrack { this.stats = props.stats; this.tally = props.tally; this.key = props.key; - this.#warmup = props.warmup ?? getter(Duration.ZERO); - this.#delay = new Derived([this.sync.out.delay, this.sync.out.maxDelay, this.#warmup], (min, max, warmup) => ({ - min: min.add(warmup), - max: max.add(warmup), + this.#warmup = Duration.millis(this.config.warmup ?? 0).min(DELAY_MAX); + this.#delay = new Derived([this.sync.out.delay, this.sync.out.maxDelay], (min, max) => ({ min, max })); + this.#subscription = new Derived([this.sync.out.delay, this.sync.out.maxDelay], (min, max) => ({ + min, + max: max.add(this.#warmup), })); - this.#drives = props.warmup === undefined; - this.#refresh = props.refresh ?? getter(false); this.#predecessor = props.predecessor; this.#signals.run(this.#run.bind(this)); @@ -695,21 +676,20 @@ class DecoderTrack { // Count into this rendition's feedback and the decoder's running totals. #count(counts: Partial>, totals?: (stats: Stats) => Partial): void { - this.tally?.count(this.key, counts); - if (totals) this.stats?.update((current) => ({ ...(current ?? EMPTY), ...totals(current ?? EMPTY) })); + this.tally.count(this.key, counts); + if (totals) this.stats.update((current) => ({ ...(current ?? EMPTY), ...totals(current ?? EMPTY) })); } // Count a frame read from the track: its bytes even when rejected, and whether an accepted one // arrived too late for playout. Classifies ingress before decoder backpressure changes the - // delivery clock. Only the min-delay decoder moves the clock and counts: the other reads the same - // frames behind it. + // delivery clock. A warming frame anchors no later playhead: the live frames behind it skip the + // clock to within the max delay, and buffered startup anchors on the first picture shown. #receive(frame: Container.ReceivedFrame, accepted: boolean): void { - if (!this.#drives) return; const timestamp = frame.timestamp as Duration; const late = accepted && this.clock.received(timestamp, frame.arrival); const bytes = frame.payload.byteLength; const arrived = arrivedAt(frame.arrival); - this.tally?.arrive(this.key, timestamp, arrived); + this.tally.arrive(this.key, timestamp, arrived); this.#count({ frames: 1, bytes, late: late ? 1 : 0 }, (stats) => ({ frameCount: stats.frameCount + (accepted ? 1 : 0), bytesReceived: stats.bytesReceived + bytes, @@ -724,6 +704,7 @@ class DecoderTrack { const consumer = new Container.Consumer(sub, { format, delay: this.#delay, + warmup: this.#warmup, predecessor: this.consumer ?? this.#predecessor, }); effect.cleanup(() => consumer.close()); @@ -741,12 +722,14 @@ class DecoderTrack { broadcast: this.broadcast, track: this.name, priority: Catalog.PRIORITY.video, - // js/net merges both decoders' subscriptions into one request at the smaller stated min delay - // and the larger max delay. - delay: this.#delay, + delay: this.#subscription, }); if (!sub) return; + // Each codec joins afresh. + const recovery = new Recovery(this.#warmup); + this.#recovery = recovery; + // Keep output tied to this codec lifetime even if the effect opens another run. const signal = effect.abort; const decoder = new VideoDecoder({ @@ -756,18 +739,9 @@ class DecoderTrack { // still deliver output after close(). if (signal.aborted) return; this.#count({ decoded: 1 }); - const generation = this.#generation; - const group = this.#groups.get(frame.timestamp); - // Pictures come out in presentation order, so a chunk at or before this one that - // produced none, such as a hidden VP9 frame, never will. Only a picture with an entry - // proves that: one without was decoded before the last reset cleared the map, so its - // timestamp says nothing about the chunks fed since. - if (group !== undefined) { - for (const timestamp of this.#groups.keys()) { - if (timestamp <= frame.timestamp) this.#groups.delete(timestamp); - } - } const timestamp = Duration.micros(frame.timestamp); + // Still warming up after a join or gap: decoded for its references alone. + if (recovery.warming(timestamp)) return; if (this.#late(timestamp)) { // Late frame, don't render it. return; @@ -777,18 +751,13 @@ class DecoderTrack { // Render something while we wait for the sync to catch up, unless the frame is // already off the timeline: WebCodecs can hold a picture past a reset or a skip. - // Only the min-delay decoder posts one: the max-delay picture shows on the clock. - if (this.#drives && this.frame.peek() === undefined && this.clock.presentable(timestamp)) { - this.group = group; + if (this.frame.peek() === undefined && this.clock.presentable(timestamp)) { this.frame.set(frame.clone()); } // Resolves at once when the delay is "instant". Closing the track releases it. // Check the captured signal again in case cancellation followed the resolved wait. if (!(await this.clock.wait(timestamp, signal)) || signal.aborted) return; - // The max-delay decoder restarts its timeline after the clock did, so the clock's own - // epoch cannot tell its leftover pictures apart. - if (this.#generation !== generation) return; if (this.#late(timestamp)) { // Late frame, don't render it. @@ -801,7 +770,6 @@ class DecoderTrack { // Trim the decode buffer as frames are rendered this.#trimBuffered(timestamp); - this.group = group; this.frame.update((prev) => { prev?.close(); return frame.clone(); // avoid closing the frame here @@ -863,7 +831,7 @@ class DecoderTrack { let previous: Duration | undefined; let latest: bigint | undefined; - const recovery = new Recovery(); + const recovery = this.#recovery; this.#spawn(effect, async () => { for (;;) { @@ -904,7 +872,6 @@ class DecoderTrack { latest = next.group; this.#resetAtKeyframe(decoder, frame); - this.#groups.set(frame.timestamp.asMicros(), next.group); this.#submit(decoder, frame); if (!this.sync.playing.peek()) await this.#waitStartup(effect, frame.timestamp); } @@ -942,7 +909,7 @@ class DecoderTrack { let previous: Duration | undefined; let latest: bigint | undefined; - const recovery = new Recovery(); + const recovery = this.#recovery; this.#spawn(effect, async () => { for (;;) { @@ -981,7 +948,6 @@ class DecoderTrack { if (decoder.state === "closed") break; latest = next.group; this.#resetAtKeyframe(decoder, frame); - this.#groups.set(frame.timestamp.asMicros(), next.group); this.#submit(decoder, frame); if (!this.sync.playing.peek()) await this.#waitStartup(effect, frame.timestamp); } @@ -1034,7 +1000,7 @@ class DecoderTrack { const ended = this.ended.peek(); if (ended === undefined || read.group <= ended) return; this.ended.set(undefined); - this.#onBreak(read.group); + this.#onBreak(); } } @@ -1052,15 +1018,13 @@ class DecoderTrack { // A fresh timeline invalidates parked pictures and its previous decoded span. The held picture // stays until the new keyframe renders; the codec resets before that keyframe is decoded. - #onBreak(group: bigint): void { + #onBreak(): void { this.first.set(undefined); this.timestamp.set(undefined); this.#buffered.set([]); - this.#generation++; - this.start.set(group); - // The min-delay decoder restarts video's timeline, leaving audio's. The max-delay one reads the - // same end later. - if (this.#drives) this.clock.reset(); + this.#recovery.restart(); + // Restarts video's timeline, leaving audio's. + this.clock.reset(); this.#resetPending = true; } @@ -1077,7 +1041,6 @@ class DecoderTrack { // A closed decoder throws on reset. if (!frame.keyframe || decoder.state === "closed") return; decoder.reset(); - this.#groups.clear(); this.#configure(decoder); } @@ -1092,7 +1055,7 @@ class DecoderTrack { let data = frame.payload; if (this.#opening) { this.#opening = false; - if (frame.keyframe && this.#refresh.peek() && isH264(this.config.codec)) { + if (frame.keyframe && !this.#warmup.isZero() && isH264(this.config.codec)) { data = insertRecoveryPoint(data, this.#description); } } @@ -1179,188 +1142,6 @@ class DecoderTrack { } } -/** Constructor properties for {@link Rendition}. */ -interface RenditionProps extends Omit { - stats: Signal; - tally: Tally; - // The rendition on screen when this one was selected from the same path, whose decoders' arrival - // estimates these continue. - predecessor?: Rendition; -} - -/** - * One rendition's playback: a min-delay decoder, plus a max-delay decoder while the catalog gives the - * rendition a nonzero `warmup` (intra refresh, so no keyframes). - * - * The min-delay decoder holds the app's latency and drives the shared clock, so it may show refresh - * stripes after a cold join or a skip. The max-delay decoder starts one group earlier and tolerates - * the warmup more lateness, so its pictures are clean, and it decodes what is already due as fast as - * it can without moving the clock. Its picture is shown whenever its timestamp is at or past the - * min-delay one's, and nothing shown ever steps back in time. Rust players do the same in - * `moq_video::decode::Consumer`. - */ -class Rendition { - readonly error = new Signal(undefined); - readonly broadcast: Moq.Broadcast.Consumer; - readonly track: string; - readonly config: DecoderConfig; - readonly identity: string; - readonly catalog: Getter; - // The rendition's feedback counters' key: its ID in the catalog it was selected from. - readonly key: Key; - - // The picture chosen between the two decoders, and its timestamp once it presented on the clock: - // undefined while it is only a poster. - readonly frame = new Signal(undefined); - readonly timestamp = new Signal(undefined); - - readonly min: DecoderTrack; - readonly max = new Signal(undefined); - - // The lane picture last shown, its timestamp, and the first group of the min-delay timeline it was - // shown on. Nothing below that timestamp shows again until the timeline restarts. - #shown?: VideoFrame; - #last?: Duration; - #start?: bigint; - - #signals = new Effect(); - - constructor(props: RenditionProps) { - this.broadcast = props.broadcast; - this.track = props.track; - this.config = props.config; - this.identity = props.identity; - this.catalog = props.catalog; - this.key = props.key; - - // A missing or zero `warmup` keeps one decoder. The budgets follow a changed value without - // reopening the max-delay decoder. - const warmup = new Signal(Duration.ZERO); - const dual = new Signal(false); - - const predecessor = props.predecessor; - this.min = new DecoderTrack({ ...props, refresh: dual, predecessor: predecessor?.consumer }); - - this.#signals.run((effect) => { - const config = effect.get(props.catalog)?.video?.renditions?.[props.track]; - // Capped like the delay, so a catalog `warmup` cannot hold the viewer without bound. - warmup.set(Duration.millis(config?.warmup ?? 0).min(DELAY_MAX)); - dual.set(!warmup.peek().isZero()); - }); - this.#signals.run((effect) => { - if (!effect.get(dual)) return; - const max = new DecoderTrack({ - ...props, - stats: undefined, - tally: undefined, - warmup, - refresh: dual, - predecessor: predecessor?.max.peek()?.consumer ?? predecessor?.consumer, - }); - effect.cleanup(() => max.close()); - effect.set(this.max, max); - }); - - // Either decoder failing fails the rendition, as an HEVC refresh frame sent as `key` does. - this.#signals.run((effect) => { - const max = effect.get(this.max); - const failed = effect.get(this.min.error) ?? (max && effect.get(max.error)); - if (failed) this.error.set(failed); - }); - - this.#signals.run((effect) => { - const max = effect.get(this.max); - this.#select( - { frame: effect.get(this.min.frame), lane: this.min }, - max && { frame: effect.get(max.frame), lane: max }, - effect.get(this.min.start), - ); - }); - } - - // The min-delay decoder's arrival estimate, which a replacement rendition continues. - get consumer(): Container.Consumer | undefined { - return this.min.consumer; - } - - // Startup reads the clock-driving decoder's first picture and its buffered media. - get first(): Getter { - return this.min.first; - } - - get buffered(): Getter { - return this.min.buffered; - } - - get spread(): Getter { - return this.min.spread; - } - - // The group the publisher declared the clock-driving decoder's media ends with. - get ended(): Getter { - return this.min.ended; - } - - // The publisher finishing the clock-driving decoder's track. - get finish(): Getter { - return this.min.finish; - } - - // Whether the clock-driving decoder's media all arrived. - get complete(): Getter { - return this.min.complete; - } - - // Show the max-delay picture when it is at or past the min-delay one, so it wins ties, else the - // min-delay picture. Only pictures read from `start` on, the min-delay decoder's timeline, are - // candidates: one held from before a restart neither shows nor sets the floor. Not `caughtUp`: its - // slack, applied per picture, would alternate backwards and forwards. - #select(min: Lane, max: Lane | undefined, start: bigint | undefined): void { - // A restarted timeline begins below the previous one's pictures. Like a decoder's own timestamp - // on a break, the selection's clears until the new timeline presents, so a pending rendition is - // not held to the old timeline's playhead. - if (start !== this.#start) { - this.#start = start; - this.#last = undefined; - this.timestamp.set(undefined); - } - const current = (lane: Lane | undefined): lane is Lane & { frame: VideoFrame } => - lane?.frame !== undefined && (start === undefined || (lane.lane.group ?? -1n) >= start); - // Even a clean max-delay picture is a sweep behind until the min-delay decoder has one. - if (!current(min)) return; - - let pick = min; - if (current(max) && max.frame.timestamp >= min.frame.timestamp) pick = max; - if (pick.frame === this.#shown) return; - - const timestamp = Duration.micros(pick.frame.timestamp); - if (this.#last?.gt(timestamp)) return; - this.#shown = pick.frame; - this.#last = timestamp; - - this.timestamp.set(pick.lane.timestamp.peek()); - this.frame.update((prev) => { - prev?.close(); - return pick.frame.clone(); - }); - } - - close(): void { - this.#signals.close(); - this.min.close(); - this.frame.update((prev) => { - prev?.close(); - return undefined; - }); - } -} - -// One decoder's current picture, for the selector. -interface Lane { - frame: VideoFrame | undefined; - lane: DecoderTrack; -} - function isH264(codec: string): boolean { return codec.startsWith("avc1.") || codec.startsWith("avc3."); } @@ -1381,9 +1162,8 @@ const RECOVERY_SEI = new Uint8Array([0x06, 0x06, 0x01, 0x84, 0x80]); * 0` (`AnalyzeAnnexB` in `media/formats/mp4/avc.cc`). Otherwise `decode()` throws "A key frame is * required". An intra-refresh group opens on a recovery point that counts a whole sweep, so every * join and skip of a `warmup` rendition would fail. The extra message declares that one picture - * recovered, which is what a `warmup` rendition already accepts: the min-delay decoder shows the - * refresh stripes until the sweep completes, and the max-delay decoder covers them. The slices - * are unchanged; the message goes right before the first one, after any SEI the stream carries. + * recovered, which is what a `warmup` rendition already accepts: the decoder hides the pictures + * after a join or skip until the warmup passes. The slices are unchanged; the message goes right before the first one, after any SEI the stream carries. * @internal */ export function insertRecoveryPoint(payload: Uint8Array, description?: Uint8Array): Uint8Array { diff --git a/js/watch/src/video/recovery.ts b/js/watch/src/video/recovery.ts index 63134fa3b37..5e8061b4313 100644 --- a/js/watch/src/video/recovery.ts +++ b/js/watch/src/video/recovery.ts @@ -1,22 +1,50 @@ -import type { Duration } from "@moq/clock"; +import { Duration } from "@moq/clock"; import type { Container } from "@moq/hang"; -/** Refuse leading pictures whose previous GOP was not delivered to this decoder. @internal */ +/** + * Refuse leading pictures whose previous GOP was not delivered to this decoder, and hold back the + * catalog `warmup` of pictures after each join, which a recovery point is not clean before. @internal + */ export class Recovery { #needed = true; #floor?: Duration; + readonly #warmup: Duration; + // The pictures still warming up since the last join or gap: from its recovery point, for the warmup. + #warming?: { start: Duration; end: Duration }; + + constructor(warmup: Duration = Duration.ZERO) { + this.#warmup = warmup; + } /** Arm recovery after media was lost, preserving continuous groups and declared ends. */ gap(): void { this.#needed = true; } + /** Forget the warmup once a fresh encoder starts another timeline, whose pictures need none. */ + restart(): void { + this.#warming = undefined; + } + /** Whether this access unit has the references available at the current delivery boundary. */ accept(frame: Container.ReceivedFrame): boolean { if (frame.keyframe) { + if (this.#needed && !this.#warmup.isZero()) { + this.#warming = { start: frame.timestamp, end: frame.timestamp.add(this.#warmup) }; + } this.#floor = this.#needed ? frame.timestamp : undefined; this.#needed = false; } return this.#floor === undefined || frame.timestamp.ge(this.#floor); } + + /** + * Whether the picture at `timestamp` is still warming up after the last join or gap, so it is + * decoded but neither shown nor placed on the clock. Pictures from before that recovery point + * that the codec still holds are not. + */ + warming(timestamp: Duration): boolean { + const warming = this.#warming; + return warming !== undefined && timestamp.ge(warming.start) && timestamp.lt(warming.end); + } } diff --git a/quest/m2/warmup-one-decoder.md b/quest/m2/warmup-one-decoder.md deleted file mode 100644 index 8d1aad14389..00000000000 --- a/quest/m2/warmup-one-decoder.md +++ /dev/null @@ -1,51 +0,0 @@ -# [M] Warmup video plays through one decoder - -## Goal - -A video rendition with a `warmup` plays through one decoder that joins one -group earlier through the container consumer's existing warmup reach, as -audio does, in `@moq/watch`, `moq_video::decode`, and `moq play`; the -max-delay lane and its selector are gone, and the docs describe what ships. - -## Plan - -Found 2026-11-14 by the [feature review](/quest/m3/release/feature-review.md). -[moq-dev/moq#7555](https://github.com/moq-dev/moq/pull/7555), -[moq-dev/moq#5842](https://github.com/moq-dev/moq/pull/5842), and -[moq-dev/moq#7376](https://github.com/moq-dev/moq/pull/7376) added about -2,800 lines, none on `release`. - -- No producer in the stack sets a video `warmup`: the only writers are Opus - importers and encoders. The [intra-refresh](/quest/m4/intra-refresh/README.md) - import that would set one moved to m4 on 2026-11-12, for want of a consumer. -- Once `warmup` is nonzero, the max-delay lane never stops - (`js/watch/src/video/decoder.ts`; `rs/moq-video/src/decode/select.rs` says - both lanes race in steady state), so a viewer decodes twice for the whole - session to improve the moments after a join or skip. -- [Dual decoder](/quest/m4/intra-refresh/dual-decoder.md) says Firefox and - Safari are unverified and HEVC refresh fails in Chromium, while - `doc/lib/js/watch.md` presents the feature as working. -- The Rust and JS selection policies differ (Rust holds a sticky source, - JS picks per picture), though the JS docs say Rust does the same. -- `js/hang`'s container consumer and `moq_mux::container::Playout` already - decode from the group start and present at start plus `warmup`; video opts - out of passing it. - -The two-decoder design was the maintainer's decision on 2026-10-14; the new -evidence is the 2026-11-12 move to m4 and the absence of any producer. - -Pass `warmup` to the container consumer for video, delete -`Rendition`'s selector and the max-delay lanes in JS, `decode/select.rs` and -`moq play`'s dual path in Rust, keep `insertRecoveryPoint` only if Chromium -needs it for one decoder, trim `doc/lib/js/watch.md`, and rescope the m4 -dual-decoder quest to deciding whether a second decoder pays for itself, with -catch-up measurements. - -Decided 2026-11-14 (maintainer, pick on [moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764)): ✅ (a) one decoder through the container consumer's warmup reach, revisiting two in m4 with measurements (rejected: (b) keep it as Chromium-only and experimental; (c) keep it as is). - -Public API: (a) removes the dual-decoder selection from `@moq/watch` and -`moq_video::decode`, unpublished. Wire: none. - -## Related - -- [Transcoder warmup](/quest/m4/intra-refresh/consumer-warmup.md) - its Goal assumes the players' two decoders; reword it if (a) is picked diff --git a/quest/m4/intra-refresh/README.md b/quest/m4/intra-refresh/README.md index ca268987a97..a54e8da6a1d 100644 --- a/quest/m4/intra-refresh/README.md +++ b/quest/m4/intra-refresh/README.md @@ -7,8 +7,8 @@ a stripe of the picture, so a decoder that starts at the beginning of a sweep is clean once the sweep completes, and the bitrate never spikes. This questline makes such video a first-class hang broadcast on import and playback: streams contributed that way import cleanly, and viewers stay live while recovering -to a clean picture. A cold join or skip may show refresh stripes until the -second decoder catches up, as the playback decision below permits. +to a clean picture. A player hides the refresh stripes after a cold join or +skip, as the playback decision below says. Measured on 2026-10-06: at fixed quality, a 60s keyframe interval saves 10-16% against 2s on the 240p-720p rungs and about 10% at 1080p (x264 @@ -35,12 +35,14 @@ make these presentation guarantees true even when sweep lengths change. exactly one cycle and relay shedding keeps its meaning. - The catalog carries a `warmup` duration per rendition. The transcoder decodes enough preceding groups to cover it as history before each group it - transcodes. A player uses it only to gate and size its second decoder. That decoder is clean because it joins a whole group - earlier, not because of the value, so an imprecise `warmup` only changes how - long stripes last. The field is generic: audio gets the same one for Opus convergence. -- A player stays live, in the browser and in Rust: a min-delay decoder may - show stripes after a cold join or skip, and a second decoder with one more - sweep of delay takes over once its timestamp catches up. The transcoder + transcodes. A player joins that much earlier and hides that much after each + join or skip, so an imprecise `warmup` shows stripes or hides clean pictures. + The field is generic: audio gets the same one for Opus convergence. +- A player plays through one decoder, in the browser and in Rust: it joins one + group earlier through the container consumer's warmup reach and presents + from the group start plus `warmup`, so a cold join stays at the playout delay + and a skip freezes for one sweep. Whether a second decoder that shows stripes + meanwhile pays for itself is left to measure. The transcoder never encodes a partially refreshed picture: every fetched group and every group after a skip is preceded by decode-only history. A group that opens on a true IDR needs none. diff --git a/quest/m4/intra-refresh/consumer-warmup.md b/quest/m4/intra-refresh/consumer-warmup.md index 37797e8ba06..7d2e3a5c255 100644 --- a/quest/m4/intra-refresh/consumer-warmup.md +++ b/quest/m4/intra-refresh/consumer-warmup.md @@ -6,8 +6,8 @@ The transcoder, given a source rendition with `warmup` set, never encodes a picture its decoder has not fully recovered. Every group it transcodes, whether fetched on demand for a rung or read live after a skip, is preceded by enough earlier groups decoded as history that its first picture is clean. -Players stay live with stripes instead, via the min-delay and max-delay -decoder pairs in `@moq/watch` and the native `moq_video::decode::Consumer`. +Players instead join a group earlier and hide the warmup after each join or +skip, in `@moq/watch` and the native `moq_video::decode::Consumer`. Renditions without `warmup` behave exactly as today. ## Plan diff --git a/quest/m4/intra-refresh/dual-decoder.md b/quest/m4/intra-refresh/dual-decoder.md index 84c29d1153a..49be709516c 100644 --- a/quest/m4/intra-refresh/dual-decoder.md +++ b/quest/m4/intra-refresh/dual-decoder.md @@ -1,39 +1,49 @@ -# [S] Browser decoders join intra-refresh H.264 in every engine +# [M] Decide whether a second decoder pays for itself ## Goal -A js/watch viewer of an H.264 intra-refresh rendition (catalog `warmup` set) -joins and recovers from skips in Chromium, Firefox, and Safari. Each of its two -decoders (the min-delay and max-delay pair in `js/watch/src/video/decoder.ts`) -can open on a group's first frame, a recovery point that is not an IDR, and -decodes on from there, showing stripes until the sweep completes. +Measure what a second decoder would buy a viewer of an intra-refresh rendition +(catalog `warmup` set), and either add it to `@moq/watch`, `moq_video::decode`, +and `moq play` or delete this quest. Today each player decodes through one +decoder that joins one group earlier and hides the warmup after each join or +skip, so a skip freezes the picture for one sweep. ## Plan -The two decoders, their selector, and the Chromium join landed with this -quest's first PR ([moq-dev/moq#7555](https://github.com/moq-dev/moq/pull/7555)). -What remains is checking the other engines. - -- Chromium (done, verified on 156): WebCodecs accepts an H.264 chunk that opens - a codec only when it is an IDR or carries an SEI recovery point with - `recovery_frame_cnt == 0` (`media/formats/mp4/avc.cc`, `AnalyzeAnnexB`). An - intra-refresh recovery point counts a whole sweep. - ✅ Decided 2026-11-01 (option A): on the chunk that opens a codec (after - `configure` or `reset`), when the rendition has `warmup` and the chunk is - H.264 with no IDR, `insertRecoveryPoint` in `decoder.ts` inserts an SEI - recovery point with a zero count before the first slice. Rejected: treating - H.264 `warmup` renditions as unsupported in Chromium (B), and asking Chromium - to accept a nonzero count (C), an outside party. -- Firefox and Safari: unverified. Playwright's Firefox build reports H.264 - unsupported and its WebKit build would not launch on the Linux host. Check - both with a real browser, joining at a non-IDR recovery point sent as `key`, - with and without the inserted recovery point. If one rejects the join, find - what its key check accepts. -- HEVC: Chromium rejects a non-IRAP H.265 frame sent as `key`, so an HEVC - refresh rendition fails through the decoder's existing `error` signal. +Decided 2026-11-14 (maintainer, pick on [moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764)): +one decoder ships, and two are revisited here with measurements. +[moq-dev/moq#8857](https://github.com/moq-dev/moq/pull/8857) removed the earlier +pair: a min-delay decoder that showed stripes and drove the clock, and a +max-delay decoder a sweep behind whose clean pictures replaced them once it +caught up. That pair decoded twice for the whole session, since the max-delay +lane never stops in steady state, to improve only the moments after a join or +skip. Its Rust and JS selection policies also differed (Rust held a sticky +source, JS picked per picture). + +Measure, on a real intra-refresh import with the +[H.264 import](/quest/m4/intra-refresh/h264-import.md): + +- How long a skip freezes with one decoder, against how long stripes show and + how often the picture would switch sources with two. +- The steady-state CPU, power, and decoder memory a second decoder costs, on + desktop and mobile hardware decoders. +- Whether Firefox and Safari open a codec on a non-IDR H.264 recovery point + sent as `key`, with and without the zero-count recovery point + `insertRecoveryPoint` inserts for Chromium. Playwright's Firefox build + reports H.264 unsupported and its WebKit build would not launch on the Linux + host, so check both with a real browser. Chromium refuses a non-IRAP H.265 + frame sent as `key`, so an HEVC refresh rendition fails through the + decoder's `error` signal in either design. + +A second decoder that only runs for one sweep after each join or skip, rather +than for the whole session, is the variant to measure first. + +## Hosts + +- linux - Chromium and Firefox +- macos - Safari and VideoToolbox ## Related -- [Warmup with one decoder](/quest/m2/warmup-one-decoder.md) - may replace the players' two decoders with one before the cut, leaving this to decide whether a second pays for itself - [H.264 import](/quest/m4/intra-refresh/h264-import.md) - publishes the real `warmup` for an end-to-end x264 check - [Consumer warmup](/quest/m4/intra-refresh/consumer-warmup.md) - the transcoder decodes history so it never encodes stripes diff --git a/quest/m4/intra-refresh/recovery-boundary.md b/quest/m4/intra-refresh/recovery-boundary.md index 764fa7dd239..d8082505796 100644 --- a/quest/m4/intra-refresh/recovery-boundary.md +++ b/quest/m4/intra-refresh/recovery-boundary.md @@ -32,10 +32,11 @@ Decided 2026-10-22 (maintainer): precision is settled; the hang draft rounds a fractional-millisecond `warmup` up ([#5212](https://github.com/moq-dev/moq/pull/5212)). The draft's `warmup` rule is amended so a consumer running a parallel max-delay decoder MAY present -min-delay pictures before the group start plus `warmup`, which keeps the -merged dual decoder in `decode::Consumer` -([#5842](https://github.com/moq-dev/moq/pull/5842)), and the draft now says -so. Publication ordering for a later, longer sweep remains to settle. +min-delay pictures before the group start plus `warmup`, and the draft now says +so. The players have since gone back to one decoder that hides those pictures +([moq-dev/moq#8857](https://github.com/moq-dev/moq/pull/8857)); the +[dual decoder](/quest/m4/intra-refresh/dual-decoder.md) quest decides whether a +second one returns. Publication ordering for a later, longer sweep remains to settle. Ask the maintainer how the catalog duration stays safe for a longer sweep, such as a bound published before the affected content or a fixed bound from diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 31e19910f59..6fecfc14cc6 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -1409,7 +1409,7 @@ mod decode_tests { } } - /// The max-delay decoder opens only for a catalog warmup, so it must survive the trip through + /// The decoder joins earlier and hides the warmup it reads, so it must survive the trip through /// the FFI record. #[test] fn video_config_keeps_the_warmup() { diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index ffd6cb8822d..3b6d01fae2d 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -318,7 +318,7 @@ fn gap(edge: std::time::Duration, resume: Timestamp) -> Result Date: Sun, 15 Nov 2026 08:22:04 -0800 Subject: [PATCH 4/5] fix(watch,moq-video,moq-cli): warmup review round 1 Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.test.ts | 31 +++++++++++- js/watch/src/video/recovery.ts | 31 +++++++++--- rs/moq-cli/src/play/buffer.rs | 77 +++++++++++++++++++++-------- rs/moq-video/src/decode/consumer.rs | 50 ++++++++++++------- 4 files changed, 140 insertions(+), 49 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 77e40117d02..608e8027d7b 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1685,9 +1685,10 @@ describe("Decoder with a warmup rendition", () => { const at = (group: bigint, ms: number, key = false) => sample(group, ms * 1000, key); // A warmup rendition reading the container consumer's reads from a feed, presenting at once. - async function refresh(fields: Partial = { warmup: WARMUP }) { + // `hold` keeps those timestamps in the codec until the next chunk. + async function refresh(fields: Partial = { warmup: WARMUP }, hold: number[] = []) { const feeds = new Feeds(); - const playback = await guardedPlayback("legacy", [], [], false, 0, { fields, feeds }); + const playback = await guardedPlayback("legacy", [], hold, false, 0, { fields, feeds }); // Every picture shown, in ms. const shown: number[] = []; const watch = new Effect(); @@ -1750,6 +1751,32 @@ describe("Decoder with a warmup rendition", () => { } }); + it("keeps a warming picture the codec still holds hidden past a later gap", async () => { + const run = await refresh({ warmup: WARMUP }, [1_100_000]); + try { + // 1100 is still inside the codec when the skip moves the warmup to group 13. + await run.push(at(10n, 1000, true), at(10n, 1100), gap, at(13n, 2000, true), at(13n, 2300)); + expect(run.playback.outputs).toContain(1_100_000); + expect(run.shown).toEqual([2300]); + } finally { + await run.close(); + } + }); + + it("needs no warmup for a fresh encoder, even after a join or a gap", async () => { + const run = await refresh(); + try { + // Joined on a declared end: the next group is a fresh encoder's first. + await run.push(marker(9n, 900_000), at(10n, 1000, true), at(10n, 1100)); + expect(run.shown).toEqual([1000, 1100]); + // A skip, then a declared end before any recovery point: the fresh encoder is clean too. + await run.push(gap, marker(12n, 1_300_000), at(13n, 0, true), at(13n, 100)); + expect(run.shown).toEqual([1000, 1100, 0, 100]); + } finally { + await run.close(); + } + }); + it("shows every picture of a rendition without a warmup", async () => { const run = await refresh({}); try { diff --git a/js/watch/src/video/recovery.ts b/js/watch/src/video/recovery.ts index 5e8061b4313..483d41ca50f 100644 --- a/js/watch/src/video/recovery.ts +++ b/js/watch/src/video/recovery.ts @@ -9,8 +9,9 @@ export class Recovery { #needed = true; #floor?: Duration; readonly #warmup: Duration; - // The pictures still warming up since the last join or gap: from its recovery point, for the warmup. - #warming?: { start: Duration; end: Duration }; + // The pictures still warming up since the last join or gap: from its recovery point, for the + // warmup. `passed` once the codec output a picture past it, so none of it can still come out. + #warming?: { start: Duration; end: Duration; passed: boolean }; constructor(warmup: Duration = Duration.ZERO) { this.#warmup = warmup; @@ -21,8 +22,13 @@ export class Recovery { this.#needed = true; } - /** Forget the warmup once a fresh encoder starts another timeline, whose pictures need none. */ + /** + * A fresh encoder starts another timeline after a declared end. Its first picture references + * nothing earlier and needs no warmup, even after a join or a gap. + */ restart(): void { + this.#needed = false; + this.#floor = undefined; this.#warming = undefined; } @@ -30,7 +36,11 @@ export class Recovery { accept(frame: Container.ReceivedFrame): boolean { if (frame.keyframe) { if (this.#needed && !this.#warmup.isZero()) { - this.#warming = { start: frame.timestamp, end: frame.timestamp.add(this.#warmup) }; + // The codec may still hold pictures of a warmup it has not output past, which stay + // hidden: the hidden span reaches back over them rather than growing a list. + const previous = this.#warming; + const start = previous && !previous.passed ? previous.start.min(frame.timestamp) : frame.timestamp; + this.#warming = { start, end: frame.timestamp.add(this.#warmup), passed: false }; } this.#floor = this.#needed ? frame.timestamp : undefined; this.#needed = false; @@ -39,12 +49,17 @@ export class Recovery { } /** - * Whether the picture at `timestamp` is still warming up after the last join or gap, so it is - * decoded but neither shown nor placed on the clock. Pictures from before that recovery point - * that the codec still holds are not. + * Whether the decoded picture at `timestamp` is still warming up after the last join or gap, + * so it is decoded but not shown. Called on each picture the codec outputs, in presentation + * order. */ warming(timestamp: Duration): boolean { const warming = this.#warming; - return warming !== undefined && timestamp.ge(warming.start) && timestamp.lt(warming.end); + if (warming === undefined) return false; + if (timestamp.ge(warming.end)) { + warming.passed = true; + return false; + } + return timestamp.ge(warming.start); } } diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index d7ffe0c0f57..6c5d9260cb4 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -16,8 +16,8 @@ pub(super) struct Buffer { /// media the buffer never had, so it is neither decoded nor shown. pub floor: Option, /// Pictures presenting before this are still warming up after a join or a skip: - /// decoded for their references, never shown, and never aged out, since they - /// trail the live edge by the warmup on purpose. + /// decoded for their references and never shown. They age out the warmup later + /// than other media, since they trail the live edge by it on purpose. pub present: Option, /// Whether the next keyframe resumes after a join or a loss, so a catalog `warmup` /// applies. Cleared for an encoder's first picture, which has nothing before it. @@ -95,12 +95,18 @@ impl Buffer { self.index(frame); let edge = self.edge.unwrap(); let present = self.present; - // An unscaled timestamp has no age, so it never goes stale, and neither does a - // warming one. - let stale = |oldest: &moq_net::Timestamp| match (edge.as_micros(), oldest.as_micros()) { - _ if present.is_some_and(|present| *oldest < present) => false, - (Ok(edge), Ok(oldest)) => edge.saturating_sub(oldest) > max_delay.as_micros(), - _ => false, + // An unscaled timestamp has no age, so it never goes stale. A warming one may + // trail by the warmup too. + let stale = |oldest: &moq_net::Timestamp| { + let warming = present.is_some_and(|present| *oldest < present); + let budget = match warming { + true => max_delay.saturating_add(warmup), + false => max_delay, + }; + match (edge.as_micros(), oldest.as_micros()) { + (Ok(edge), Ok(oldest)) => edge.saturating_sub(oldest) > budget.as_micros(), + _ => false, + } }; if self.oldest.front().is_some_and(stale) { // Dropping a reference picture invalidates the rest of its GOP. Resume @@ -112,7 +118,8 @@ impl Buffer { self.generation += 1; self.waiting_keyframe = self.frames.is_empty(); self.floor = self.frames.front().map(|frame| frame.timestamp); - // Resuming past a loss warms up again. + // Resuming past a loss warms up again, even at a keyframe that has yet to arrive. + self.joined = true; self.present = self.floor.and_then(|floor| warmed(floor, warmup)); // The retained keyframe's leading pictures referenced what was just dropped. let mut frames = std::mem::take(&mut self.frames); @@ -249,25 +256,53 @@ mod tests { assert_eq!(buffer.frames.len(), 1); } - /// After a join the warmup's pictures trail the live edge on purpose, so a zero age budget - /// keeps them, and an eviction past a loss warms up again from the keyframe it resumes at. + /// After a join the warmup's pictures trail the live edge on purpose, so they age out the + /// warmup later than other media, and an eviction past a loss warms up again from the + /// keyframe it resumes at. #[test] - fn a_join_warms_up_without_aging_out_the_warmup() { + fn a_join_ages_out_the_warmup_a_warmup_later() { let mut buffer = Buffer::default(); - let warmup = Duration::from_millis(100); - for ms in [0, 50, 100, 150] { - buffer.push(frame(ms, ms % 100 == 0), Duration::ZERO, warmup); + let (age, warmup) = (Duration::from_millis(100), Duration::from_millis(100)); + for ms in [0, 50, 100, 150, 200] { + buffer.push(frame(ms, ms % 100 == 0), age, warmup); } let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis().unwrap()).collect(); - assert_eq!(kept, [0, 50, 100, 150]); + assert_eq!( + kept, + [0, 50, 100, 150, 200], + "the warmup trails by the age plus the warmup" + ); assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 100); - buffer.pop(); - buffer.pop(); - buffer.push(frame(200, true), Duration::ZERO, warmup); + buffer.push(frame(250, false), age, warmup); let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis().unwrap()).collect(); - assert_eq!(kept, [200], "the aged group was evicted"); - assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 300); + assert_eq!( + kept, + [200, 250], + "the warmup past its budget was evicted, then the stale group" + ); + assert_eq!( + buffer.present.unwrap().as_millis().unwrap(), + 300, + "resuming past a loss warms up" + ); + } + + /// An eviction that empties a fresh encoder's segment is a loss: the keyframe it resumes at + /// warms up. + #[test] + fn an_emptied_fresh_segment_warms_up_at_its_next_keyframe() { + let mut buffer = Buffer { + joined: false, + ..Buffer::default() + }; + let warmup = Duration::from_millis(100); + buffer.push(frame(0, true), Duration::ZERO, warmup); + assert!(buffer.present.is_none()); + buffer.push(frame(50, false), Duration::ZERO, warmup); + assert!(buffer.frames.is_empty(), "aged out with no keyframe to resume at"); + buffer.push(frame(300, true), Duration::ZERO, warmup); + assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 400); } /// An encoder's first picture has nothing before it to warm up from. diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 0b040830b42..f5dcd8f44ce 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -120,11 +120,7 @@ impl Options { /// before then would show refresh stripes, so they are decoded but dropped. The /// start reaches the warmup further back to make up for it, joining a group earlier. pub struct Consumer { - /// The decoder between reads, absent while `step` holds it. - reader: Option>, - /// A read in flight, kept across a cancelled [`read`](Self::read), since a - /// cancelled [`Sink`] call poisons the sink. - step: Option, + slot: Slot, name: String, /// When the access unit behind the latest picture returned arrived. arrival: Option, @@ -140,6 +136,15 @@ pub struct Consumer { /// backwards within a group; the oldest go first. const ARRIVALS: usize = 32; +/// The decoder, idle between reads or decoding. +/// +/// A read in flight is kept across a cancelled [`Consumer::read`] and finished by +/// the next one, since a cancelled [`Sink`] call poisons the sink. +enum Slot { + Idle(Box), + Busy(Step), +} + /// A [`Reader::read`] in flight, handing the reader back with its result. type Step = Pin, Result, Error>)> + Send>>; @@ -192,8 +197,7 @@ impl Consumer { let reader = Reader::open(broadcast, catalog, &name.into(), &options).await?; Ok(Self { name: reader.decoder.name().to_owned(), - reader: Some(Box::new(reader)), - step: None, + slot: Slot::Idle(Box::new(reader)), arrival: None, }) } @@ -230,20 +234,26 @@ impl Consumer { } fn poll_read(&mut self, cx: &mut Context<'_>) -> Poll, Error>> { - let reader = &mut self.reader; - let step = self.step.get_or_insert_with(|| { - let mut reader = reader.take().expect("the reader is idle while no read is in flight"); - Box::pin(async move { + if matches!(self.slot, Slot::Idle(_)) { + // A pending placeholder holds the slot only while the reader moves into its step. + let Slot::Idle(mut reader) = + std::mem::replace(&mut self.slot, Slot::Busy(Box::pin(std::future::pending()))) + else { + unreachable!("matched above"); + }; + self.slot = Slot::Busy(Box::pin(async move { let result = reader.read().await; (reader, result) - }) - }); + })); + } + let Slot::Busy(step) = &mut self.slot else { + unreachable!("started above"); + }; let Poll::Ready((reader, result)) = step.as_mut().poll(cx) else { return Poll::Pending; }; - self.step = None; // Left idle after an error, so the next read asks it again and gets the same answer. - self.reader = Some(reader); + self.slot = Slot::Idle(reader); let Some((read, arrival)) = result? else { return Poll::Ready(Ok(None)); }; @@ -270,7 +280,10 @@ impl Consumer { /// The decoder between reads. fn reader(&mut self) -> &mut Reader { - self.reader.as_deref_mut().expect("the decoder is not idle") + let Slot::Idle(reader) = &mut self.slot else { + panic!("the decoder is not idle"); + }; + reader } } @@ -376,11 +389,12 @@ impl Reader { self.recover = true; continue; } - // A fresh encoder has no preceding GOP to reference, so no recovery, and its - // timeline may start over below the warmup's. + // A fresh encoder has no preceding GOP to reference and nothing to warm up from, + // even after a join or a gap, and its timeline may start over below the warmup's. Some(Encoded::End { end, .. }) => { self.drain().await?; self.pending.push_back((Read::End { end }, None)); + self.recover = false; self.present = None; continue; } From 6baf89329f589443a20276ab207d28453b577848 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 12:57:26 -0800 Subject: [PATCH 5/5] fix(watch,moq-cli): drop a picture resuming past a timeline break, round the warmup up Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.test.ts | 29 +++++++++++++++++++++++++++++ js/watch/src/video/decoder.ts | 8 ++++++++ rs/moq-cli/src/play/buffer.rs | 24 +++++++++++++++++++----- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 608e8027d7b..2c7e70b7183 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -1763,6 +1763,35 @@ describe("Decoder with a warmup rendition", () => { } }); + it("drops a picture whose clock wait resolved before a timeline break it resumes after", async () => { + // Hold 1100's wait open: resolving it once the break is read stands in for a wait that returned + // true but whose output callback resumed only after the break. + let release: ((present: boolean) => void) | undefined; + const original = Track.prototype.wait; + const wait = spyOn(Track.prototype, "wait").mockImplementation(function (this: Track, timestamp, signal) { + if (timestamp.asMicros() !== 1_100_000) return original.call(this, timestamp, signal); + return new Promise((resolve) => { + release = resolve; + }); + }); + const run = await refresh({}); + try { + await run.push(at(10n, 1000, true), at(10n, 1100)); + expect(run.shown).toEqual([1000]); + // A declared end, then a fresh encoder starting over at 0. + await run.push(marker(10n, 1_200_000), at(11n, 0, true)); + expect(run.shown).toEqual([1000, 0]); + release?.(true); + await microtasks(); + // The old timeline's picture neither shows nor late-rejects the new timeline's next one. + await run.push(at(11n, 100)); + expect(run.shown).toEqual([1000, 0, 100]); + } finally { + wait.mockRestore(); + await run.close(); + } + }); + it("needs no warmup for a fresh encoder, even after a join or a gap", async () => { const run = await refresh(); try { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index a40fb04ab11..0114c69d7c2 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -629,6 +629,9 @@ class Rendition { // A timeline restart makes the next decoded frame decide whether the codec resets. #resetPending = false; + // Counts timeline restarts. A picture whose clock wait resolved before one, but whose callback + // resumes after it, belongs to the old timeline and must neither show nor stamp `timestamp`. + #generation = 0; // A hole was read, so the next decoded frame resets the codec only if it rewinds. #holed = false; // The newest timestamp fed to the codec since it last opened. @@ -740,6 +743,7 @@ class Rendition { if (signal.aborted) return; this.#count({ decoded: 1 }); const timestamp = Duration.micros(frame.timestamp); + const generation = this.#generation; // Still warming up after a join or gap: decoded for its references alone. if (recovery.warming(timestamp)) return; if (this.#late(timestamp)) { @@ -758,6 +762,9 @@ class Rendition { // Resolves at once when the delay is "instant". Closing the track releases it. // Check the captured signal again in case cancellation followed the resolved wait. if (!(await this.clock.wait(timestamp, signal)) || signal.aborted) return; + // The wait checks the clock's epoch, but a break can still land between its resolution + // and this continuation, clearing the state #late reads below. + if (this.#generation !== generation) return; if (this.#late(timestamp)) { // Late frame, don't render it. @@ -1019,6 +1026,7 @@ class Rendition { // A fresh timeline invalidates parked pictures and its previous decoded span. The held picture // stays until the new keyframe renders; the codec resets before that keyframe is decoded. #onBreak(): void { + this.#generation++; this.first.set(undefined); this.timestamp.set(undefined); this.#buffered.set([]); diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index 6c5d9260cb4..40e85bca6bb 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -147,15 +147,18 @@ impl Buffer { } } -/// Where a `warmup` from `floor` ends, at its scale, or `None` when there is none. +/// Where a `warmup` from `floor` ends, at its scale, or `None` when there is none. Rounds +/// up to the next tick, so a coarse scale never hides less than the whole warmup. fn warmed(floor: moq_net::Timestamp, warmup: Duration) -> Option { if warmup.is_zero() { return None; } - let warmup = moq_net::Timestamp::try_from(warmup) - .ok()? - .convert(floor.scale()?) - .ok()?; + let scale = floor.scale()?; + let ticks = warmup + .as_nanos() + .checked_mul(u128::from(scale.as_u64()))? + .div_ceil(1_000_000_000); + let warmup = moq_net::Timestamp::new(u64::try_from(ticks).ok()?, scale).ok()?; floor.checked_add(warmup).ok() } @@ -305,6 +308,17 @@ mod tests { assert_eq!(buffer.present.unwrap().as_millis().unwrap(), 400); } + /// A coarse scale rounds the warmup up a tick rather than hiding less than all of it. + #[test] + fn a_coarse_scale_rounds_the_warmup_up() { + let floor = moq_net::Timestamp::from_scale(10, 10).unwrap(); + let present = warmed(floor, Duration::from_millis(150)).unwrap(); + assert_eq!(present.value(), 12, "150 ms at 10 Hz ends at the second tick"); + let present = warmed(floor, Duration::from_millis(200)).unwrap(); + assert_eq!(present.value(), 12, "an exact warmup does not round"); + assert!(warmed(moq_net::Timestamp::unscaled(3), Duration::from_millis(150)).is_none()); + } + /// An encoder's first picture has nothing before it to warm up from. #[test] fn a_fresh_encoder_needs_no_warmup() {