diff --git a/rs/moq-mux/src/container/declared.rs b/rs/moq-mux/src/container/declared.rs index c48134b61e0..d95a81be746 100644 --- a/rs/moq-mux/src/container/declared.rs +++ b/rs/moq-mux/src/container/declared.rs @@ -32,6 +32,10 @@ pub(crate) fn depth(config: &VideoConfig) -> crate::Result> { /// slot, which needs no frame spacing, so a DTS can fall before the timeline's zero. A tick is the /// caller's timescale, the finest its container writes: 90 kHz for MPEG-TS, 1 ms for FLV. /// +/// Distinct source timestamps can round onto one tick. A frame then moves up to the tick after +/// every frame presented before it, so each keeps its own presentation slot and order; identical +/// source timestamps fail, since no order between them is known. +/// /// A clock lives for one group, so its DTS depends on nothing read before the group, and a frame /// reordered deeper than declared fails rather than move the timeline. pub(crate) struct DeclaredClock { @@ -44,8 +48,9 @@ pub(crate) struct DeclaredClock { held: VecDeque<(T, u64)>, /// Presentation slots not yet taken, ascending. slots: Vec, - /// The largest depth + 1 PTS read, ascending: all a deeper reorder is checked against. - highest: Vec, + /// The largest depth + 1 PTS read, ascending, with their source timestamps in nanoseconds: + /// all a deeper reorder or a tie is checked against. + highest: Vec<(u64, u128)>, /// The last DTS released. last: Option, } @@ -78,19 +83,30 @@ impl DeclaredClock { /// Whether a frame presented at `pts` would follow more than the depth frames read before it /// in presentation order, which [`Self::push`] refuses. pub(crate) fn reorders_past(&self, pts: u64) -> bool { - self.highest.len() - self.highest.partition_point(|&prior| prior <= pts) > self.depth + self.highest.len() - self.highest.partition_point(|&(prior, _)| prior <= pts) > self.depth } - /// Hold the group's next frame in decode order, presented at `pts`. - pub(crate) fn push(&mut self, item: T, pts: u64) -> crate::Result<()> { + /// Hold the group's next frame in decode order, presented at `pts` ticks, read from `source`. + pub(crate) fn push(&mut self, item: T, pts: u64, source: moq_net::Timestamp) -> crate::Result<()> { if self.held.len() > self.depth { return Err(ClockError::Undrained.into()); } + let source = source.as_nanos(); + // Only `highest` matters: a frame below all of it reorders past the depth. + let mut pts = pts; + for &(tick, prior) in &self.highest { + if prior < source && tick >= pts { + pts = tick.checked_add(1).ok_or(ClockError::Tie)?; + } + } + if (self.highest.iter()).any(|&(tick, prior)| prior >= source && tick <= pts) { + return Err(ClockError::Tie.into()); + } if self.reorders_past(pts) { return Err(ClockError::Reordered(self.depth).into()); } - let at = self.highest.partition_point(|&prior| prior <= pts); - self.highest.insert(at, pts); + let at = self.highest.partition_point(|&(prior, _)| prior <= pts); + self.highest.insert(at, (pts, source)); if self.highest.len() > self.depth + 1 { self.highest.remove(0); } @@ -100,9 +116,10 @@ impl DeclaredClock { Ok(()) } - /// The next frame and its DTS in signed ticks, once the group's earliest slot is known, - /// or at once when `flush` because the group ended. - pub(crate) fn pop(&mut self, flush: bool) -> crate::Result> { + /// The next frame with its DTS in signed ticks and its PTS, past the tick it was pushed at if a + /// tie moved it, once the group's earliest slot is known, or at once when `flush` because the + /// group ended. + pub(crate) fn pop(&mut self, flush: bool) -> crate::Result> { if self.held.is_empty() || (self.earliest.is_none() && !flush && self.held.len() <= self.depth) { return Ok(None); } @@ -119,7 +136,7 @@ impl DeclaredClock { return Err(ClockError::Presentation.into()); } self.last = Some(dts); - Ok(Some((item, dts))) + Ok(Some((item, dts, pts))) } } @@ -131,6 +148,8 @@ enum ClockError { Reordered(usize), #[error("video presentation slots cannot author increasing DTS at or before PTS")] Presentation, + #[error("video presents two frames at one timestamp")] + Tie, #[error("the decode clock must be drained before reading another frame")] Undrained, } @@ -149,19 +168,24 @@ mod tests { const PYRAMID: [u64; 9] = [0, 4, 2, 1, 3, 8, 6, 5, 7]; const FRAME: u64 = 3_600; + /// A source timestamp exactly on 90 kHz tick `pts`. + fn at(pts: u64) -> moq_net::Timestamp { + moq_net::Timestamp::from_scale(pts, 90_000).unwrap() + } + /// The DTS of one group read through a declared clock of `depth`, released as each frame is /// read, checking it never holds more than `depth` frames between reads. fn declared_dts(depth: usize, read: &[u64]) -> Vec { let mut clock = DeclaredClock::new(depth); let mut out = Vec::new(); for &pts in read { - clock.push((), pts).unwrap(); - while let Some(((), dts)) = clock.pop(false).unwrap() { + clock.push((), pts, at(pts)).unwrap(); + while let Some(((), dts, _)) = clock.pop(false).unwrap() { out.push(dts); } assert!(clock.len() <= depth); } - while let Some(((), dts)) = clock.pop(true).unwrap() { + while let Some(((), dts, _)) = clock.pop(true).unwrap() { out.push(dts); } out @@ -193,12 +217,63 @@ mod tests { #[test] fn a_zero_depth_releases_each_frame_at_its_pts() { let mut clock = DeclaredClock::new(0); - for pts in [0, 1, 90_000, u64::MAX] { - clock.push(pts, pts).unwrap(); - assert_eq!(clock.pop(false).unwrap(), Some((pts, i128::from(pts)))); + for (index, pts) in [0, 1, 90_000, u64::MAX].into_iter().enumerate() { + let source = moq_net::Timestamp::from_nanos(index as u64).unwrap(); + clock.push(pts, pts, source).unwrap(); + assert_eq!(clock.pop(false).unwrap(), Some((pts, i128::from(pts), pts))); assert_eq!(clock.len(), 0); } - clock.push(0, 0).unwrap_err(); + clock.push(0, 0, at(0)).unwrap_err(); + } + + /// Distinct source timestamps that round onto one millisecond tick, as FLV clocks them, move + /// the later frame up to the next free tick; identical ones still fail. + #[test] + fn sub_tick_ties_move_to_the_next_free_tick() { + let micros = |micros: u64| moq_net::Timestamp::from_micros(micros).unwrap(); + let released = |depth: usize, read: &[u64]| { + let mut clock = DeclaredClock::new(depth); + let mut out = Vec::new(); + for &source in read { + let source = micros(source); + clock.push((), source.as_millis() as u64, source)?; + while let Some(((), dts, pts)) = clock.pop(false)? { + out.push((dts, pts)); + } + } + while let Some(((), dts, pts)) = clock.pop(true)? { + out.push((dts, pts)); + } + crate::Result::Ok(out) + }; + // MPEG-TS can carry a repeated timestamp one 90 kHz tick (11 µs) later. + assert_eq!( + released(0, &[182_815_000, 182_856_000, 182_856_011, 182_897_000]).unwrap(), + [ + (182_815, 182_815), + (182_856, 182_856), + (182_857, 182_857), + (182_897, 182_897) + ] + ); + // A tie moved onto the next frame's tick moves that frame up in turn, however many share + // the tick. + assert_eq!( + released(0, &[10_000, 10_400, 11_000, 13_000]).unwrap(), + [(10, 10), (11, 11), (12, 12), (13, 13)] + ); + assert_eq!( + released(0, &[40_000, 40_011, 40_022, 40_033, 80_000]).unwrap(), + [(40, 40), (41, 41), (42, 42), (43, 43), (80, 80)] + ); + // Reordered frames tie the same way: the later-presented one moves. + assert_eq!( + released(1, &[0, 3_000, 1_000, 1_400, 4_000]).unwrap(), + [(-1, 0), (0, 3), (1, 1), (2, 2), (3, 4)] + ); + released(0, &[10_000, 10_000]).unwrap_err(); + // A frame presented below one already on its tick has nowhere to go. + released(1, &[0, 10_400, 10_000]).unwrap_err(); } /// Flat or backward PTS read past the declared depth fail within depth + 1 frames, however @@ -211,7 +286,7 @@ mod tests { let mut clock = DeclaredClock::new(2); let refused = (0..count).position(|index| { let pts = 90_000 - if backwards { index } else { 0 }; - let refused = clock.push(vec![0u8; size], pts).is_err() + let refused = clock.push(vec![0u8; size], pts, at(pts)).is_err() || std::iter::from_fn(|| clock.pop(false).transpose()).any(|popped| popped.is_err()); assert!(clock.len() <= 2); refused diff --git a/rs/moq-mux/src/container/flv/export.rs b/rs/moq-mux/src/container/flv/export.rs index 2e86444b449..ece9dd839da 100644 --- a/rs/moq-mux/src/container/flv/export.rs +++ b/rs/moq-mux/src/container/flv/export.rs @@ -253,7 +253,11 @@ impl FlvTrack { return Ok(()); }; let released = (|| { - while let Some(((mut arrival, record), dts)) = clock.pop(flush)? { + while let Some(((mut arrival, record), dts, pts)) = clock.pop(flush)? { + // A tie the clock moved presents on the tick it moved to. + if arrival.item.timestamp.as_millis() != u128::from(pts) { + arrival.item.timestamp = moq_net::Timestamp::from_millis(pts)?; + } if let Some(settled) = settled { arrival.arrived = arrival.arrived.max(settled); } @@ -527,9 +531,9 @@ impl Export { match &mut track.declared { Some(clock) => { // Clocked in FLV's milliseconds, so the leading frames step a tag tick apart. - let pts = arrival.item.timestamp.as_millis() as u64; + let source = arrival.item.timestamp; let arrived = arrival.arrived; - clock.push((arrival, record), pts)?; + clock.push((arrival, record), source.as_millis() as u64, source)?; track.release(key, Some(arrived), self.shift, false, &mut self.jitter)?; } None => { diff --git a/rs/moq-mux/src/container/flv/export_test.rs b/rs/moq-mux/src/container/flv/export_test.rs index 3c1e8bcca87..657fd73c8ce 100644 --- a/rs/moq-mux/src/container/flv/export_test.rs +++ b/rs/moq-mux/src/container/flv/export_test.rs @@ -1350,7 +1350,7 @@ async fn export_authors_dts_and_composition_time_for_reordered_avc() { } /// The `(DTS, PTS)` of every AVC frame tag, and the audio tag timestamps, an export renders of -/// a broadcast that declares `reorder` and carries a group of `stamps` (ms, decode order) for +/// a broadcast that declares `reorder` and carries a group of `stamps` (µs, decode order) for /// each entry, plus one audio frame at 20 ms. With `open`, the last group is cut but the video /// track stays open. async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow::Result<(Vec<(u32, i64)>, Vec)> { @@ -1385,8 +1385,8 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow:: .renditions .insert(audio_track.name().to_string(), audio_config); } - let frame = |ms: u64, payload: &'static [u8], keyframe| crate::container::Frame { - timestamp: Timestamp::from_millis(ms).unwrap(), + let frame = |micros: u64, payload: &'static [u8], keyframe| crate::container::Frame { + timestamp: Timestamp::from_micros(micros).unwrap(), duration: None, payload: Bytes::from_static(payload), keyframe, @@ -1396,13 +1396,13 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow:: crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); for stamps in groups { - for (index, &ms) in stamps.iter().enumerate() { + for (index, µs) in stamps.iter().enumerate() { let payload: &'static [u8] = if index == 0 { &[0, 0, 0, 1, 0x65] } else { &[0, 0, 0, 1, 0x41] }; - video.write(frame(ms, payload, index == 0)).unwrap(); + video.write(frame(micros, payload, index == 0)).unwrap(); } } match open { @@ -1413,7 +1413,7 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow:: audio_track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - audio.write(frame(20, &[0xde, 0xad], true)).unwrap(); + audio.write(frame(20_000, &[0xde, 0xad], true)).unwrap(); let _finished = audio.finish().unwrap(); catalog.finish().unwrap(); @@ -1452,7 +1452,10 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow:: /// presentation slots. The whole stream shifts by the depth, so those fit FLV's unsigned clock. #[tokio::test(start_paused = true)] async fn export_declared_reorder_decodes_each_group_from_its_frames() { - let groups: [&[u64]; 2] = [&[0, 160, 80, 40, 120], &[400, 560, 480, 440, 520]]; + let groups: [&[u64]; 2] = [ + &[0, 160_000, 80_000, 40_000, 120_000], + &[400_000, 560_000, 480_000, 440_000, 520_000], + ]; let (video, audio) = export_declared(2, &groups, false).await.unwrap(); assert_eq!( video, @@ -1472,19 +1475,27 @@ async fn export_declared_reorder_decodes_each_group_from_its_frames() { assert_eq!(audio, [22]); // A first group cut short leaves the next decoding where it would have. - let cut: [&[u64]; 2] = [&[0, 160, 80], &[400, 560, 480, 440, 520]]; + let cut: [&[u64]; 2] = [&[0, 160_000, 80_000], groups[1]]; let (cut, _) = export_declared(2, &cut, false).await.unwrap(); assert_eq!(cut[3..], video[5..]); // A depth of zero decodes at presentation, and refuses a frame that reorders. - let (video, _) = export_declared(0, &[&[0, 40, 80]], false).await.unwrap(); + let (video, _) = export_declared(0, &[&[0, 40_000, 80_000]], false).await.unwrap(); assert_eq!(video, [(0, 0), (40, 40), (80, 80)]); - let err = export_declared(0, &[&[0, 80, 40]], false).await.unwrap_err(); + let err = export_declared(0, &[&[0, 80_000, 40_000]], false).await.unwrap_err(); assert!(err.to_string().contains("declared depth"), "{err}"); + // Distinct timestamps within one millisecond present a tick apart; identical ones fail. + let (video, _) = export_declared(0, &[&[0, 40_000, 40_011, 80_000]], false) + .await + .unwrap(); + assert_eq!(video, [(0, 0), (40, 40), (41, 41), (80, 80)]); + let err = export_declared(0, &[&[0, 40_000, 40_000]], false).await.unwrap_err(); + assert!(err.to_string().contains("one timestamp"), "{err}"); + // A group that ends goes out whole while the track stays open, though fewer frames than the // depth settle nothing. - let (video, _) = export_declared(2, &[&[0, 80]], true).await.unwrap(); + let (video, _) = export_declared(2, &[&[0, 80_000]], true).await.unwrap(); assert_eq!(video, [(0, 2), (1, 82)]); } diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 3a6bb7fc5dd..2b40636146c 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -183,7 +183,10 @@ impl MkvTrack { ) -> Result<()> { let pts = (arrival.item.timestamp.as_nanos() * 90_000 / 1_000_000_000) as u64; match &mut self.declared { - Some(clock) => clock.push(arrival, pts)?, + Some(clock) => { + let source = arrival.item.timestamp; + clock.push(arrival, pts, source)? + } None => { let (arrived, bytes) = (arrival.arrived, arrival.bytes); self.clock.push(arrival, pts, arrived, bytes) @@ -194,7 +197,11 @@ impl MkvTrack { fn release(&mut self, name: &str, jitter: &mut crate::jitter::Buffer, flush: bool) -> Result<()> { if let Some(clock) = &mut self.declared { - while let Some((mut arrival, dts)) = clock.pop(flush)? { + while let Some((mut arrival, dts, pts)) = clock.pop(flush)? { + // A tie the clock moved presents on the tick it moved to. + if (arrival.item.timestamp.as_nanos() * 90_000 / 1_000_000_000) as u64 != pts { + arrival.item.timestamp = moq_net::Timestamp::from_scale(pts, 90_000)?; + } if !self.finished && !flush { arrival.arrived = arrival.arrived.max(self.latest); } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 7344d2722eb..50bc46e43e4 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -513,7 +513,7 @@ impl Track { // exporter that kept it. return match &mut self.declared { Some(clock) => { - clock.push(None, to_ticks(pending.frame.timestamp))?; + clock.push(None, to_ticks(pending.frame.timestamp), pending.frame.timestamp)?; self.release(name, jitter, false) } None => Ok(()), @@ -533,11 +533,12 @@ impl Track { } self.declared = depth.map(DeclaredClock::new); } - let pts = to_ticks(pending.frame.timestamp); + let source = pending.frame.timestamp; + let pts = to_ticks(source); let (arrived, bytes) = (pending.arrived, pending.frame.payload.len()); let item = (pending, self.source.description().cloned()); match &mut self.declared { - Some(clock) => clock.push(Some(item), pts)?, + Some(clock) => clock.push(Some(item), pts, source)?, None => self.clock.push(item, pts, arrived, bytes), } self.release(name, jitter, false) @@ -808,10 +809,16 @@ impl Track { ) -> anyhow::Result<()> { if let Some(mut clock) = self.declared.take() { let released = (|| { - while let Some((item, dts)) = clock.pop(flush)? { + while let Some((item, dts, pts)) = clock.pop(flush)? { let Some((mut pending, description)) = item else { continue; }; + // A tie the clock moved presents on the tick it moved to: the least microsecond + // that `to_ticks` reads back as that tick. + if to_ticks(pending.frame.timestamp) != pts { + let micros = u64::try_from((u128::from(pts) * 100).div_ceil(9))?; + pending.frame.timestamp = Timestamp::from_micros(micros)?; + } if self.read == Read::Live { pending.arrived = pending.arrived.max(self.latest); } @@ -5233,7 +5240,7 @@ mod tests { let pts = super::to_ticks(ms(at)); match &mut track.declared { Some(clock) => { - clock.push(Some((pending, None)), pts).unwrap(); + clock.push(Some((pending, None)), pts, ms(at)).unwrap(); while clock.pop(false).unwrap().is_some() {} } None => track.clock.push((pending, None), pts, arrived, 0),