diff --git a/rs/moq-mux/src/container/declared.rs b/rs/moq-mux/src/container/declared.rs new file mode 100644 index 00000000000..c5aea9039d5 --- /dev/null +++ b/rs/moq-mux/src/container/declared.rs @@ -0,0 +1,254 @@ +use std::collections::VecDeque; + +use hang::catalog::VideoConfig; + +/// The reorder depth `config` declares, or `None` when it declares none. +/// +/// Refuses a depth its codec and level cannot reach, so a declared clock never holds more frames +/// than a conforming decoder's picture buffer. +pub(crate) fn depth(config: &VideoConfig) -> crate::Result> { + let Some(reorder) = config.reorder else { + return Ok(None); + }; + let max = crate::codec::video::max_reorder(config); + if u32::from(reorder) > max { + return Err(ClockError::Depth { + reorder, + max, + codec: config.codec.to_string(), + } + .into()); + } + Ok(Some(usize::from(reorder))) +} + +/// One group's decode clock, at its rendition's declared reorder depth. +/// +/// The n-th frame of a group in decode order decodes at the group's (n - depth)-th presentation +/// slot. Once the group's first depth + 1 frames are read, no frame still to come can be presented +/// below the earliest of them, so from then on each frame is released as it is read: it takes the +/// least slot not yet taken, which a later frame could undercut only by following more than depth +/// frames in presentation order. The first depth frames decode one tick apart up to that earliest +/// slot, which needs no frame spacing, so a DTS can fall before the timeline's zero. A tick is the +/// caller's timescale, the finest its container writes: 90 kHz for MPEG-TS, 1 ms for FLV. +/// +/// A clock lives for one group, so its DTS depends on nothing read before the group, and a frame +/// reordered deeper than declared fails rather than move the timeline. +pub(crate) struct DeclaredClock { + depth: usize, + /// The group's earliest PTS, once enough frames are read that none to come can precede it. + earliest: Option, + /// Frames still to be released that decode before [`Self::earliest`]. + leading: usize, + /// Frames read but not yet released, in decode order, with their PTS. + held: VecDeque<(T, u64)>, + /// Presentation slots not yet taken, ascending. + slots: Vec, + /// The largest depth + 1 PTS read, ascending: all a deeper reorder is checked against. + highest: Vec, + /// The last DTS released. + last: Option, +} + +impl DeclaredClock { + /// A clock for one group of a rendition reordering at most `depth` frames, from [`depth`]. + pub(crate) fn new(depth: usize) -> Self { + Self { + depth, + earliest: None, + leading: depth, + held: VecDeque::new(), + slots: Vec::new(), + highest: Vec::new(), + last: None, + } + } + + /// How many frames are held. + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.held.len() + } + + /// Hold the group's next frame in decode order, presented at `pts`. + pub(crate) fn push(&mut self, item: T, pts: u64) -> crate::Result<()> { + if self.held.len() > self.depth { + return Err(ClockError::Undrained.into()); + } + let at = self.highest.partition_point(|&prior| prior <= pts); + if self.highest.len() - at > self.depth { + return Err(ClockError::Reordered(self.depth).into()); + } + self.highest.insert(at, pts); + if self.highest.len() > self.depth + 1 { + self.highest.remove(0); + } + let at = self.slots.partition_point(|&slot| slot <= pts); + self.slots.insert(at, pts); + self.held.push_back((item, pts)); + Ok(()) + } + + /// The next frame and its DTS in signed ticks, once the group's earliest slot is known, + /// or at once when `flush` because the group ended. + pub(crate) fn pop(&mut self, flush: bool) -> crate::Result> { + if self.held.is_empty() || (self.earliest.is_none() && !flush && self.held.len() <= self.depth) { + return Ok(None); + } + let earliest = *self.earliest.get_or_insert(self.slots[0]); + let dts = match self.leading { + 0 => i128::from(self.slots.remove(0)), + leading => { + self.leading -= 1; + i128::from(earliest) - leading as i128 + } + }; + let (item, pts) = self.held.pop_front().expect("the held queue is nonempty"); + if dts > i128::from(pts) || self.last.is_some_and(|last| dts <= last) { + return Err(ClockError::Presentation.into()); + } + self.last = Some(dts); + Ok(Some((item, dts))) + } +} + +#[derive(Debug, thiserror::Error)] +enum ClockError { + #[error("video reorder depth {reorder} exceeds the {max} that {codec} can reach")] + Depth { reorder: u8, max: u32, codec: String }, + #[error("video reorders past its declared depth {0}")] + Reordered(usize), + #[error("video presentation slots cannot author increasing DTS at or before PTS")] + Presentation, + #[error("the decode clock must be drained before reading another frame")] + Undrained, +} + +impl From for crate::Error { + fn from(err: ClockError) -> Self { + anyhow::Error::new(err).into() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A B-pyramid (`max_num_reorder_frames` 2) in decode order, in 40 ms frames. + const PYRAMID: [u64; 9] = [0, 4, 2, 1, 3, 8, 6, 5, 7]; + const FRAME: u64 = 3_600; + + /// The DTS of one group read through a declared clock of `depth`, released as each frame is + /// read, checking it never holds more than `depth` frames between reads. + fn declared_dts(depth: usize, read: &[u64]) -> Vec { + let mut clock = DeclaredClock::new(depth); + let mut out = Vec::new(); + for &pts in read { + clock.push((), pts).unwrap(); + while let Some(((), dts)) = clock.pop(false).unwrap() { + out.push(dts); + } + assert!(clock.len() <= depth); + } + while let Some(((), dts)) = clock.pop(true).unwrap() { + out.push(dts); + } + out + } + + #[test] + fn declared_slots_shift_by_count_and_start_before_zero() { + let ticks = |frames: &[i128]| frames.iter().map(|i| i * FRAME as i128).collect::>(); + let leading = |dts: Vec| [vec![-2, -1], dts].concat(); + assert_eq!( + declared_dts(2, &PYRAMID.map(|i| i * FRAME)), + leading(ticks(&[0, 1, 2, 3, 4, 5, 6])) + ); + // References spaced wider than any frame period still take their slots by count. + let far = [0, 6, 10, 1, 2, 3, 4, 5, 7, 8, 9].map(|i| i * FRAME); + assert_eq!(declared_dts(2, &far), leading(ticks(&[0, 1, 2, 3, 4, 5, 6, 7, 8]))); + } + + /// Open-GOP leading pictures present before the keyframe that starts their group, so the + /// group's first frames decode below its earliest slot, not the keyframe's. + #[test] + fn leading_pictures_decode_before_the_keyframe() { + let read = [10, 8, 9, 14, 12, 11, 13]; + assert_eq!(declared_dts(2, &read), [6, 7, 8, 9, 10, 11, 12]); + // A group that ends before the lookahead fills still takes its earliest slot. + assert_eq!(declared_dts(2, &read[..2]), [6, 7]); + } + + #[test] + fn a_zero_depth_releases_each_frame_at_its_pts() { + let mut clock = DeclaredClock::new(0); + for pts in [0, 1, 90_000, u64::MAX] { + clock.push(pts, pts).unwrap(); + assert_eq!(clock.pop(false).unwrap(), Some((pts, i128::from(pts)))); + assert_eq!(clock.len(), 0); + } + clock.push(0, 0).unwrap_err(); + } + + /// Flat or backward PTS read past the declared depth fail within depth + 1 frames, however + /// many follow and however large they are, rather than accumulate. + #[test] + fn malformed_presentation_is_refused_without_retaining_its_payload() { + for backwards in [false, true] { + for count in [16, 128, 1024] { + for size in [256, 16_384] { + let mut clock = DeclaredClock::new(2); + let refused = (0..count).position(|index| { + let pts = 90_000 - if backwards { index } else { 0 }; + let refused = clock.push(vec![0u8; size], pts).is_err() + || std::iter::from_fn(|| clock.pop(false).transpose()).any(|popped| popped.is_err()); + assert!(clock.len() <= 2); + refused + }); + assert!( + refused.is_some_and(|index| index <= 3), + "backwards={backwards} count={count}" + ); + } + } + } + } + + #[test] + fn a_declared_depth_fits_the_codec_level() { + use hang::catalog::{H264, VideoCodec}; + + let config = |codec: VideoCodec, size: Option<(u32, u32)>, reorder: u8| { + let mut config = VideoConfig::new(codec); + config.coded_width = size.map(|(width, _)| width); + config.coded_height = size.map(|(_, height)| height); + config.reorder = Some(reorder); + config + }; + let h264 = |level| H264 { + profile: 100, + constraints: 0, + level, + inline: false, + }; + let h265: VideoCodec = "hvc1.1.6.L123.B0".parse().unwrap(); + let vp9: VideoCodec = "vp09.00.10.08".parse().unwrap(); + let cases = [ + // Level 4.0's 32768 macroblocks hold four 1920x1088 frames. + (config(h264(40).into(), Some((1920, 1080)), 4), true), + (config(h264(40).into(), Some((1920, 1080)), 5), false), + // Without a size, the deepest buffer any picture of the level gets. + (config(h264(40).into(), None, 16), true), + (config(h264(40).into(), None, 17), false), + // Level 4.1's MaxLumaPs holds six 1920x1080 pictures, one of them the current. + (config(h265.clone(), Some((1920, 1080)), 5), true), + (config(h265, Some((1920, 1080)), 6), false), + (config(vp9.clone(), None, 0), true), + (config(vp9, None, 1), false), + ]; + for (config, valid) in cases { + assert_eq!(depth(&config).is_ok(), valid, "{config:?}"); + } + assert_eq!(depth(&VideoConfig::new(VideoCodec::VP8)).unwrap(), None); + } +} diff --git a/rs/moq-mux/src/container/decode.rs b/rs/moq-mux/src/container/decode.rs index 76c8e785584..b065ff28fa3 100644 --- a/rs/moq-mux/src/container/decode.rs +++ b/rs/moq-mux/src/container/decode.rs @@ -1,10 +1,10 @@ +//! The bench `decode_clock` compiles this file on its own, so it names nothing else in the crate. + use std::collections::VecDeque; use std::time::Duration; use moq_clock::Instant; -use hang::catalog::VideoConfig; - /// The largest supported reorder span in 90 kHz ticks. pub(crate) const MAX_REORDER: u64 = 2 * 90_000; @@ -160,254 +160,3 @@ impl DecodeClock { lookahead.max(self.reach).min(MAX_REORDER) } } - -/// The reorder depth `config` declares, or `None` when it declares none. -/// -/// Refuses a depth its codec and level cannot reach, so a declared clock never holds more frames -/// than a conforming decoder's picture buffer. -pub(crate) fn declared(config: &VideoConfig) -> crate::Result> { - let Some(reorder) = config.reorder else { - return Ok(None); - }; - let max = crate::codec::video::max_reorder(config); - if u32::from(reorder) > max { - return Err(ClockError::Depth { - reorder, - max, - codec: config.codec.to_string(), - } - .into()); - } - Ok(Some(usize::from(reorder))) -} - -/// One group's decode clock, at its rendition's declared reorder depth. -/// -/// The n-th frame of a group in decode order decodes at the group's (n - depth)-th presentation -/// slot. Once the group's first depth + 1 frames are read, no frame still to come can be presented -/// below the earliest of them, so from then on each frame is released as it is read: it takes the -/// least slot not yet taken, which a later frame could undercut only by following more than depth -/// frames in presentation order. The first depth frames decode one tick apart up to that earliest -/// slot, which needs no frame spacing, so a DTS can fall before the timeline's zero. A tick is the -/// caller's timescale, the finest its container writes: 90 kHz for MPEG-TS, 1 ms for FLV. -/// -/// A clock lives for one group, so its DTS depends on nothing read before the group, and a frame -/// reordered deeper than declared fails rather than move the timeline. -pub(crate) struct DeclaredClock { - depth: usize, - /// The group's earliest PTS, once enough frames are read that none to come can precede it. - earliest: Option, - /// Frames still to be released that decode before [`Self::earliest`]. - leading: usize, - /// Frames read but not yet released, in decode order, with their PTS. - held: VecDeque<(T, u64)>, - /// Presentation slots not yet taken, ascending. - slots: Vec, - /// The largest depth + 1 PTS read, ascending: all a deeper reorder is checked against. - highest: Vec, - /// The last DTS released. - last: Option, -} - -impl DeclaredClock { - /// A clock for one group of a rendition reordering at most `depth` frames, from [`declared`]. - pub(crate) fn new(depth: usize) -> Self { - Self { - depth, - earliest: None, - leading: depth, - held: VecDeque::new(), - slots: Vec::new(), - highest: Vec::new(), - last: None, - } - } - - /// How many frames are held. - #[cfg(test)] - pub(crate) fn len(&self) -> usize { - self.held.len() - } - - /// Hold the group's next frame in decode order, presented at `pts`. - pub(crate) fn push(&mut self, item: T, pts: u64) -> crate::Result<()> { - if self.held.len() > self.depth { - return Err(ClockError::Undrained.into()); - } - let at = self.highest.partition_point(|&prior| prior <= pts); - if self.highest.len() - at > self.depth { - return Err(ClockError::Reordered(self.depth).into()); - } - self.highest.insert(at, pts); - if self.highest.len() > self.depth + 1 { - self.highest.remove(0); - } - let at = self.slots.partition_point(|&slot| slot <= pts); - self.slots.insert(at, pts); - self.held.push_back((item, pts)); - Ok(()) - } - - /// The next frame and its DTS in signed ticks, once the group's earliest slot is known, - /// or at once when `flush` because the group ended. - pub(crate) fn pop(&mut self, flush: bool) -> crate::Result> { - if self.held.is_empty() || (self.earliest.is_none() && !flush && self.held.len() <= self.depth) { - return Ok(None); - } - let earliest = *self.earliest.get_or_insert(self.slots[0]); - let dts = match self.leading { - 0 => i128::from(self.slots.remove(0)), - leading => { - self.leading -= 1; - i128::from(earliest) - leading as i128 - } - }; - let (item, pts) = self.held.pop_front().expect("the held queue is nonempty"); - if dts > i128::from(pts) || self.last.is_some_and(|last| dts <= last) { - return Err(ClockError::Presentation.into()); - } - self.last = Some(dts); - Ok(Some((item, dts))) - } -} - -#[derive(Debug, thiserror::Error)] -enum ClockError { - #[error("video reorder depth {reorder} exceeds the {max} that {codec} can reach")] - Depth { reorder: u8, max: u32, codec: String }, - #[error("video reorders past its declared depth {0}")] - Reordered(usize), - #[error("video presentation slots cannot author increasing DTS at or before PTS")] - Presentation, - #[error("the decode clock must be drained before reading another frame")] - Undrained, -} - -impl From for crate::Error { - fn from(err: ClockError) -> Self { - anyhow::Error::new(err).into() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A B-pyramid (`max_num_reorder_frames` 2) in decode order, in 40 ms frames. - const PYRAMID: [u64; 9] = [0, 4, 2, 1, 3, 8, 6, 5, 7]; - const FRAME: u64 = 3_600; - - /// The DTS of one group read through a declared clock of `depth`, released as each frame is - /// read, checking it never holds more than `depth` frames between reads. - fn declared_dts(depth: usize, read: &[u64]) -> Vec { - let mut clock = DeclaredClock::new(depth); - let mut out = Vec::new(); - for &pts in read { - clock.push((), pts).unwrap(); - while let Some(((), dts)) = clock.pop(false).unwrap() { - out.push(dts); - } - assert!(clock.len() <= depth); - } - while let Some(((), dts)) = clock.pop(true).unwrap() { - out.push(dts); - } - out - } - - #[test] - fn declared_slots_shift_by_count_and_start_before_zero() { - let ticks = |frames: &[i128]| frames.iter().map(|i| i * FRAME as i128).collect::>(); - let leading = |dts: Vec| [vec![-2, -1], dts].concat(); - assert_eq!( - declared_dts(2, &PYRAMID.map(|i| i * FRAME)), - leading(ticks(&[0, 1, 2, 3, 4, 5, 6])) - ); - // References spaced wider than any frame period still take their slots by count. - let far = [0, 6, 10, 1, 2, 3, 4, 5, 7, 8, 9].map(|i| i * FRAME); - assert_eq!(declared_dts(2, &far), leading(ticks(&[0, 1, 2, 3, 4, 5, 6, 7, 8]))); - } - - /// Open-GOP leading pictures present before the keyframe that starts their group, so the - /// group's first frames decode below its earliest slot, not the keyframe's. - #[test] - fn leading_pictures_decode_before_the_keyframe() { - let read = [10, 8, 9, 14, 12, 11, 13]; - assert_eq!(declared_dts(2, &read), [6, 7, 8, 9, 10, 11, 12]); - // A group that ends before the lookahead fills still takes its earliest slot. - assert_eq!(declared_dts(2, &read[..2]), [6, 7]); - } - - #[test] - fn a_zero_depth_releases_each_frame_at_its_pts() { - let mut clock = DeclaredClock::new(0); - for pts in [0, 1, 90_000, u64::MAX] { - clock.push(pts, pts).unwrap(); - assert_eq!(clock.pop(false).unwrap(), Some((pts, i128::from(pts)))); - assert_eq!(clock.len(), 0); - } - clock.push(0, 0).unwrap_err(); - } - - /// Flat or backward PTS read past the declared depth fail within depth + 1 frames, however - /// many follow and however large they are, rather than accumulate. - #[test] - fn malformed_presentation_is_refused_without_retaining_its_payload() { - for backwards in [false, true] { - for count in [16, 128, 1024] { - for size in [256, 16_384] { - let mut clock = DeclaredClock::new(2); - let refused = (0..count).position(|index| { - let pts = 90_000 - if backwards { index } else { 0 }; - let refused = clock.push(vec![0u8; size], pts).is_err() - || std::iter::from_fn(|| clock.pop(false).transpose()).any(|popped| popped.is_err()); - assert!(clock.len() <= 2); - refused - }); - assert!( - refused.is_some_and(|index| index <= 3), - "backwards={backwards} count={count}" - ); - } - } - } - } - - #[test] - fn a_declared_depth_fits_the_codec_level() { - use hang::catalog::{H264, VideoCodec}; - - let config = |codec: VideoCodec, size: Option<(u32, u32)>, reorder: u8| { - let mut config = VideoConfig::new(codec); - config.coded_width = size.map(|(width, _)| width); - config.coded_height = size.map(|(_, height)| height); - config.reorder = Some(reorder); - config - }; - let h264 = |level| H264 { - profile: 100, - constraints: 0, - level, - inline: false, - }; - let h265: VideoCodec = "hvc1.1.6.L123.B0".parse().unwrap(); - let vp9: VideoCodec = "vp09.00.10.08".parse().unwrap(); - let cases = [ - // Level 4.0's 32768 macroblocks hold four 1920x1088 frames. - (config(h264(40).into(), Some((1920, 1080)), 4), true), - (config(h264(40).into(), Some((1920, 1080)), 5), false), - // Without a size, the deepest buffer any picture of the level gets. - (config(h264(40).into(), None, 16), true), - (config(h264(40).into(), None, 17), false), - // Level 4.1's MaxLumaPs holds six 1920x1080 pictures, one of them the current. - (config(h265.clone(), Some((1920, 1080)), 5), true), - (config(h265, Some((1920, 1080)), 6), false), - (config(vp9.clone(), None, 0), true), - (config(vp9, None, 1), false), - ]; - for (config, valid) in cases { - assert_eq!(declared(&config).is_ok(), valid, "{config:?}"); - } - assert_eq!(declared(&VideoConfig::new(VideoCodec::VP8)).unwrap(), None); - } -} diff --git a/rs/moq-mux/src/container/flv/export.rs b/rs/moq-mux/src/container/flv/export.rs index 703cb9e2081..f7334dea2c4 100644 --- a/rs/moq-mux/src/container/flv/export.rs +++ b/rs/moq-mux/src/container/flv/export.rs @@ -35,7 +35,7 @@ use super::{ }; use crate::catalog::Stream; use crate::container::consumer::Event; -use crate::container::decode::DeclaredClock; +use crate::container::declared::DeclaredClock; use crate::container::{ExportSource, Frame}; /// Before the header, unbind every track the catalog no longer lists (removed, or disabled) or @@ -232,7 +232,7 @@ impl FlvTrack { ) -> anyhow::Result<()> { self.release(key, Some(arrived), shift, true, jitter)?; let depth = match &self.config { - Some(config) if self.flavor.has_composition_time() => crate::container::decode::declared(config)?, + Some(config) if self.flavor.has_composition_time() => crate::container::declared::depth(config)?, _ => None, }; self.declared = depth.map(DeclaredClock::new); diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 14be1a9a40d..3df1957f5ab 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -76,7 +76,7 @@ struct MkvTrack { /// The inferred decode clock of a rendition that declares no `reorder`. clock: DecodeClock>, /// The current group's clock, when its rendition declared `reorder` at the group's start. - declared: Option>>, + declared: Option>>, config: Option, restart: u64, reorder: Option<(u64, u64)>, @@ -163,14 +163,14 @@ impl MkvTrack { /// rendition declares now, or on the inferred clock when it declares none. fn start_group(&mut self, name: &str, jitter: &mut crate::jitter::Buffer) -> Result<()> { let depth = match &self.config { - Some(config) => crate::container::decode::declared(config)?, + Some(config) => crate::container::declared::depth(config)?, None => None, }; // The inferred clock spans groups, so it is only flushed when a declaration replaces it. if self.declared.is_some() || depth.is_some() { self.release(name, jitter, true)?; } - self.declared = depth.map(crate::container::decode::DeclaredClock::new); + self.declared = depth.map(crate::container::declared::DeclaredClock::new); Ok(()) } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index b51bb2b71e5..562f5c15948 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -13,6 +13,7 @@ use bytes::Bytes; mod consumer; +mod declared; mod decode; mod group; mod producer; diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index a02d6053c39..6f557c3dca7 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -47,7 +47,8 @@ use crate::codec::annexb::InBandCodec; use crate::codec::video::{Hrd, Reorder}; use crate::codec::{aac, annexb, opus}; use crate::container::consumer::Event; -use crate::container::decode::{DeclaredClock, DecodeClock, MAX_REORDER, Settle}; +use crate::container::declared::DeclaredClock; +use crate::container::decode::{DecodeClock, MAX_REORDER, Settle}; use crate::container::{ExportSource, Frame}; use crate::jitter::{self, Arrival, Push}; @@ -771,7 +772,7 @@ impl Timing { self.jitter = config.jitter.map_or(0, |t| (t.as_micros() * 90_000 / 1_000_000) as u64); self.framerate = config.framerate.filter(|fps| fps.is_finite() && *fps > 0.0); self.level = Some(video_level(config, name)); - self.reorder = crate::container::decode::declared(config)?; + self.reorder = crate::container::declared::depth(config)?; Ok(()) }