From 93d66a3c699a91703cccc8a2ba2bbd38b500c0b7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 19:48:55 -0800 Subject: [PATCH 1/2] fix(moq-mux): move a sub-tick presentation tie up one tick on the declared clock Two frames with distinct source timestamps can round onto one container tick (one FLV millisecond, one 90 kHz tick). The declared clock then authored two equal DTS and failed the export. The later-presented frame now moves up to the next free tick, and the FLV, MKV, and MPEG-TS exports present it there. Identical source timestamps still fail. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/declared.rs | 107 ++++++++++++++++---- rs/moq-mux/src/container/flv/export.rs | 10 +- rs/moq-mux/src/container/flv/export_test.rs | 33 ++++-- rs/moq-mux/src/container/mkv/export.rs | 11 +- rs/moq-mux/src/container/ts/export.rs | 17 +++- 5 files changed, 138 insertions(+), 40 deletions(-) diff --git a/rs/moq-mux/src/container/declared.rs b/rs/moq-mux/src/container/declared.rs index c48134b61e0..ee7364282bc 100644 --- a/rs/moq-mux/src/container/declared.rs +++ b/rs/moq-mux/src/container/declared.rs @@ -32,6 +32,10 @@ pub(crate) fn depth(config: &VideoConfig) -> crate::Result> { /// slot, which needs no frame spacing, so a DTS can fall before the timeline's zero. A tick is the /// caller's timescale, the finest its container writes: 90 kHz for MPEG-TS, 1 ms for FLV. /// +/// Distinct source timestamps can round onto one tick. The frame presented later then moves up a +/// tick at a time until its tick is free, so each frame keeps its own presentation slot and order; +/// identical source timestamps fail, since no order between them is known. +/// /// A clock lives for one group, so its DTS depends on nothing read before the group, and a frame /// reordered deeper than declared fails rather than move the timeline. pub(crate) struct DeclaredClock { @@ -44,8 +48,9 @@ pub(crate) struct DeclaredClock { held: VecDeque<(T, u64)>, /// Presentation slots not yet taken, ascending. slots: Vec, - /// The largest depth + 1 PTS read, ascending: all a deeper reorder is checked against. - highest: Vec, + /// The largest depth + 1 PTS read, ascending, with their source timestamps in nanoseconds: + /// all a deeper reorder or a tie is checked against. + highest: Vec<(u64, u128)>, /// The last DTS released. last: Option, } @@ -78,19 +83,29 @@ impl DeclaredClock { /// Whether a frame presented at `pts` would follow more than the depth frames read before it /// in presentation order, which [`Self::push`] refuses. pub(crate) fn reorders_past(&self, pts: u64) -> bool { - self.highest.len() - self.highest.partition_point(|&prior| prior <= pts) > self.depth + self.highest.len() - self.highest.partition_point(|&(prior, _)| prior <= pts) > self.depth } - /// Hold the group's next frame in decode order, presented at `pts`. - pub(crate) fn push(&mut self, item: T, pts: u64) -> crate::Result<()> { + /// Hold the group's next frame in decode order, presented at `pts` ticks, read from `source`. + pub(crate) fn push(&mut self, item: T, pts: u64, source: moq_net::Timestamp) -> crate::Result<()> { if self.held.len() > self.depth { return Err(ClockError::Undrained.into()); } + let source = source.as_nanos(); + let mut pts = pts; + // Only `highest` can hold the tick: the group's ticks are distinct, and one below all of + // `highest` reorders past the depth. + while let Some(&(_, prior)) = self.highest.iter().find(|&&(tick, _)| tick == pts) { + if prior >= source { + return Err(ClockError::Tie.into()); + } + pts = pts.checked_add(1).ok_or(ClockError::Tie)?; + } if self.reorders_past(pts) { return Err(ClockError::Reordered(self.depth).into()); } - let at = self.highest.partition_point(|&prior| prior <= pts); - self.highest.insert(at, pts); + let at = self.highest.partition_point(|&(prior, _)| prior <= pts); + self.highest.insert(at, (pts, source)); if self.highest.len() > self.depth + 1 { self.highest.remove(0); } @@ -100,9 +115,10 @@ impl DeclaredClock { Ok(()) } - /// The next frame and its DTS in signed ticks, once the group's earliest slot is known, - /// or at once when `flush` because the group ended. - pub(crate) fn pop(&mut self, flush: bool) -> crate::Result> { + /// The next frame with its DTS in signed ticks and its PTS, past the tick it was pushed at if a + /// tie moved it, once the group's earliest slot is known, or at once when `flush` because the + /// group ended. + pub(crate) fn pop(&mut self, flush: bool) -> crate::Result> { if self.held.is_empty() || (self.earliest.is_none() && !flush && self.held.len() <= self.depth) { return Ok(None); } @@ -119,7 +135,7 @@ impl DeclaredClock { return Err(ClockError::Presentation.into()); } self.last = Some(dts); - Ok(Some((item, dts))) + Ok(Some((item, dts, pts))) } } @@ -131,6 +147,8 @@ enum ClockError { Reordered(usize), #[error("video presentation slots cannot author increasing DTS at or before PTS")] Presentation, + #[error("video presents two frames at one timestamp")] + Tie, #[error("the decode clock must be drained before reading another frame")] Undrained, } @@ -149,19 +167,24 @@ mod tests { const PYRAMID: [u64; 9] = [0, 4, 2, 1, 3, 8, 6, 5, 7]; const FRAME: u64 = 3_600; + /// A source timestamp exactly on 90 kHz tick `pts`. + fn at(pts: u64) -> moq_net::Timestamp { + moq_net::Timestamp::from_scale(pts, 90_000).unwrap() + } + /// The DTS of one group read through a declared clock of `depth`, released as each frame is /// read, checking it never holds more than `depth` frames between reads. fn declared_dts(depth: usize, read: &[u64]) -> Vec { let mut clock = DeclaredClock::new(depth); let mut out = Vec::new(); for &pts in read { - clock.push((), pts).unwrap(); - while let Some(((), dts)) = clock.pop(false).unwrap() { + clock.push((), pts, at(pts)).unwrap(); + while let Some(((), dts, _)) = clock.pop(false).unwrap() { out.push(dts); } assert!(clock.len() <= depth); } - while let Some(((), dts)) = clock.pop(true).unwrap() { + while let Some(((), dts, _)) = clock.pop(true).unwrap() { out.push(dts); } out @@ -193,12 +216,58 @@ mod tests { #[test] fn a_zero_depth_releases_each_frame_at_its_pts() { let mut clock = DeclaredClock::new(0); - for pts in [0, 1, 90_000, u64::MAX] { - clock.push(pts, pts).unwrap(); - assert_eq!(clock.pop(false).unwrap(), Some((pts, i128::from(pts)))); + for (index, pts) in [0, 1, 90_000, u64::MAX].into_iter().enumerate() { + let source = moq_net::Timestamp::from_nanos(index as u64).unwrap(); + clock.push(pts, pts, source).unwrap(); + assert_eq!(clock.pop(false).unwrap(), Some((pts, i128::from(pts), pts))); assert_eq!(clock.len(), 0); } - clock.push(0, 0).unwrap_err(); + clock.push(0, 0, at(0)).unwrap_err(); + } + + /// Distinct source timestamps that round onto one millisecond tick, as FLV clocks them, move + /// the later frame up to the next free tick; identical ones still fail. + #[test] + fn sub_tick_ties_move_to_the_next_free_tick() { + let micros = |micros: u64| moq_net::Timestamp::from_micros(micros).unwrap(); + let released = |depth: usize, read: &[u64]| { + let mut clock = DeclaredClock::new(depth); + let mut out = Vec::new(); + for &source in read { + let source = micros(source); + clock.push((), source.as_millis() as u64, source)?; + while let Some(((), dts, pts)) = clock.pop(false)? { + out.push((dts, pts)); + } + } + while let Some(((), dts, pts)) = clock.pop(true)? { + out.push((dts, pts)); + } + crate::Result::Ok(out) + }; + // MPEG-TS can carry a repeated timestamp one 90 kHz tick (11 µs) later. + assert_eq!( + released(0, &[182_815_000, 182_856_000, 182_856_011, 182_897_000]).unwrap(), + [ + (182_815, 182_815), + (182_856, 182_856), + (182_857, 182_857), + (182_897, 182_897) + ] + ); + // A tie moved onto the next frame's tick moves that frame up in turn. + assert_eq!( + released(0, &[10_000, 10_400, 11_000, 13_000]).unwrap(), + [(10, 10), (11, 11), (12, 12), (13, 13)] + ); + // Reordered frames tie the same way: the later-presented one moves. + assert_eq!( + released(1, &[0, 3_000, 1_000, 1_400, 4_000]).unwrap(), + [(-1, 0), (0, 3), (1, 1), (2, 2), (3, 4)] + ); + released(0, &[10_000, 10_000]).unwrap_err(); + // A frame presented below one already on its tick has nowhere to go. + released(1, &[0, 10_400, 10_000]).unwrap_err(); } /// Flat or backward PTS read past the declared depth fail within depth + 1 frames, however @@ -211,7 +280,7 @@ mod tests { let mut clock = DeclaredClock::new(2); let refused = (0..count).position(|index| { let pts = 90_000 - if backwards { index } else { 0 }; - let refused = clock.push(vec![0u8; size], pts).is_err() + let refused = clock.push(vec![0u8; size], pts, at(pts)).is_err() || std::iter::from_fn(|| clock.pop(false).transpose()).any(|popped| popped.is_err()); assert!(clock.len() <= 2); refused diff --git a/rs/moq-mux/src/container/flv/export.rs b/rs/moq-mux/src/container/flv/export.rs index 2e86444b449..ece9dd839da 100644 --- a/rs/moq-mux/src/container/flv/export.rs +++ b/rs/moq-mux/src/container/flv/export.rs @@ -253,7 +253,11 @@ impl FlvTrack { return Ok(()); }; let released = (|| { - while let Some(((mut arrival, record), dts)) = clock.pop(flush)? { + while let Some(((mut arrival, record), dts, pts)) = clock.pop(flush)? { + // A tie the clock moved presents on the tick it moved to. + if arrival.item.timestamp.as_millis() != u128::from(pts) { + arrival.item.timestamp = moq_net::Timestamp::from_millis(pts)?; + } if let Some(settled) = settled { arrival.arrived = arrival.arrived.max(settled); } @@ -527,9 +531,9 @@ impl Export { match &mut track.declared { Some(clock) => { // Clocked in FLV's milliseconds, so the leading frames step a tag tick apart. - let pts = arrival.item.timestamp.as_millis() as u64; + let source = arrival.item.timestamp; let arrived = arrival.arrived; - clock.push((arrival, record), pts)?; + clock.push((arrival, record), source.as_millis() as u64, source)?; track.release(key, Some(arrived), self.shift, false, &mut self.jitter)?; } None => { diff --git a/rs/moq-mux/src/container/flv/export_test.rs b/rs/moq-mux/src/container/flv/export_test.rs index 3c1e8bcca87..657fd73c8ce 100644 --- a/rs/moq-mux/src/container/flv/export_test.rs +++ b/rs/moq-mux/src/container/flv/export_test.rs @@ -1350,7 +1350,7 @@ async fn export_authors_dts_and_composition_time_for_reordered_avc() { } /// The `(DTS, PTS)` of every AVC frame tag, and the audio tag timestamps, an export renders of -/// a broadcast that declares `reorder` and carries a group of `stamps` (ms, decode order) for +/// a broadcast that declares `reorder` and carries a group of `stamps` (µs, decode order) for /// each entry, plus one audio frame at 20 ms. With `open`, the last group is cut but the video /// track stays open. async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow::Result<(Vec<(u32, i64)>, Vec)> { @@ -1385,8 +1385,8 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow:: .renditions .insert(audio_track.name().to_string(), audio_config); } - let frame = |ms: u64, payload: &'static [u8], keyframe| crate::container::Frame { - timestamp: Timestamp::from_millis(ms).unwrap(), + let frame = |micros: u64, payload: &'static [u8], keyframe| crate::container::Frame { + timestamp: Timestamp::from_micros(micros).unwrap(), duration: None, payload: Bytes::from_static(payload), keyframe, @@ -1396,13 +1396,13 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow:: crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); for stamps in groups { - for (index, &ms) in stamps.iter().enumerate() { + for (index, µs) in stamps.iter().enumerate() { let payload: &'static [u8] = if index == 0 { &[0, 0, 0, 1, 0x65] } else { &[0, 0, 0, 1, 0x41] }; - video.write(frame(ms, payload, index == 0)).unwrap(); + video.write(frame(micros, payload, index == 0)).unwrap(); } } match open { @@ -1413,7 +1413,7 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow:: audio_track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - audio.write(frame(20, &[0xde, 0xad], true)).unwrap(); + audio.write(frame(20_000, &[0xde, 0xad], true)).unwrap(); let _finished = audio.finish().unwrap(); catalog.finish().unwrap(); @@ -1452,7 +1452,10 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow:: /// presentation slots. The whole stream shifts by the depth, so those fit FLV's unsigned clock. #[tokio::test(start_paused = true)] async fn export_declared_reorder_decodes_each_group_from_its_frames() { - let groups: [&[u64]; 2] = [&[0, 160, 80, 40, 120], &[400, 560, 480, 440, 520]]; + let groups: [&[u64]; 2] = [ + &[0, 160_000, 80_000, 40_000, 120_000], + &[400_000, 560_000, 480_000, 440_000, 520_000], + ]; let (video, audio) = export_declared(2, &groups, false).await.unwrap(); assert_eq!( video, @@ -1472,19 +1475,27 @@ async fn export_declared_reorder_decodes_each_group_from_its_frames() { assert_eq!(audio, [22]); // A first group cut short leaves the next decoding where it would have. - let cut: [&[u64]; 2] = [&[0, 160, 80], &[400, 560, 480, 440, 520]]; + let cut: [&[u64]; 2] = [&[0, 160_000, 80_000], groups[1]]; let (cut, _) = export_declared(2, &cut, false).await.unwrap(); assert_eq!(cut[3..], video[5..]); // A depth of zero decodes at presentation, and refuses a frame that reorders. - let (video, _) = export_declared(0, &[&[0, 40, 80]], false).await.unwrap(); + let (video, _) = export_declared(0, &[&[0, 40_000, 80_000]], false).await.unwrap(); assert_eq!(video, [(0, 0), (40, 40), (80, 80)]); - let err = export_declared(0, &[&[0, 80, 40]], false).await.unwrap_err(); + let err = export_declared(0, &[&[0, 80_000, 40_000]], false).await.unwrap_err(); assert!(err.to_string().contains("declared depth"), "{err}"); + // Distinct timestamps within one millisecond present a tick apart; identical ones fail. + let (video, _) = export_declared(0, &[&[0, 40_000, 40_011, 80_000]], false) + .await + .unwrap(); + assert_eq!(video, [(0, 0), (40, 40), (41, 41), (80, 80)]); + let err = export_declared(0, &[&[0, 40_000, 40_000]], false).await.unwrap_err(); + assert!(err.to_string().contains("one timestamp"), "{err}"); + // A group that ends goes out whole while the track stays open, though fewer frames than the // depth settle nothing. - let (video, _) = export_declared(2, &[&[0, 80]], true).await.unwrap(); + let (video, _) = export_declared(2, &[&[0, 80_000]], true).await.unwrap(); assert_eq!(video, [(0, 2), (1, 82)]); } diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 3a6bb7fc5dd..2b40636146c 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -183,7 +183,10 @@ impl MkvTrack { ) -> Result<()> { let pts = (arrival.item.timestamp.as_nanos() * 90_000 / 1_000_000_000) as u64; match &mut self.declared { - Some(clock) => clock.push(arrival, pts)?, + Some(clock) => { + let source = arrival.item.timestamp; + clock.push(arrival, pts, source)? + } None => { let (arrived, bytes) = (arrival.arrived, arrival.bytes); self.clock.push(arrival, pts, arrived, bytes) @@ -194,7 +197,11 @@ impl MkvTrack { fn release(&mut self, name: &str, jitter: &mut crate::jitter::Buffer, flush: bool) -> Result<()> { if let Some(clock) = &mut self.declared { - while let Some((mut arrival, dts)) = clock.pop(flush)? { + while let Some((mut arrival, dts, pts)) = clock.pop(flush)? { + // A tie the clock moved presents on the tick it moved to. + if (arrival.item.timestamp.as_nanos() * 90_000 / 1_000_000_000) as u64 != pts { + arrival.item.timestamp = moq_net::Timestamp::from_scale(pts, 90_000)?; + } if !self.finished && !flush { arrival.arrived = arrival.arrived.max(self.latest); } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 7344d2722eb..50bc46e43e4 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -513,7 +513,7 @@ impl Track { // exporter that kept it. return match &mut self.declared { Some(clock) => { - clock.push(None, to_ticks(pending.frame.timestamp))?; + clock.push(None, to_ticks(pending.frame.timestamp), pending.frame.timestamp)?; self.release(name, jitter, false) } None => Ok(()), @@ -533,11 +533,12 @@ impl Track { } self.declared = depth.map(DeclaredClock::new); } - let pts = to_ticks(pending.frame.timestamp); + let source = pending.frame.timestamp; + let pts = to_ticks(source); let (arrived, bytes) = (pending.arrived, pending.frame.payload.len()); let item = (pending, self.source.description().cloned()); match &mut self.declared { - Some(clock) => clock.push(Some(item), pts)?, + Some(clock) => clock.push(Some(item), pts, source)?, None => self.clock.push(item, pts, arrived, bytes), } self.release(name, jitter, false) @@ -808,10 +809,16 @@ impl Track { ) -> anyhow::Result<()> { if let Some(mut clock) = self.declared.take() { let released = (|| { - while let Some((item, dts)) = clock.pop(flush)? { + while let Some((item, dts, pts)) = clock.pop(flush)? { let Some((mut pending, description)) = item else { continue; }; + // A tie the clock moved presents on the tick it moved to: the least microsecond + // that `to_ticks` reads back as that tick. + if to_ticks(pending.frame.timestamp) != pts { + let micros = u64::try_from((u128::from(pts) * 100).div_ceil(9))?; + pending.frame.timestamp = Timestamp::from_micros(micros)?; + } if self.read == Read::Live { pending.arrived = pending.arrived.max(self.latest); } @@ -5233,7 +5240,7 @@ mod tests { let pts = super::to_ticks(ms(at)); match &mut track.declared { Some(clock) => { - clock.push(Some((pending, None)), pts).unwrap(); + clock.push(Some((pending, None)), pts, ms(at)).unwrap(); while clock.pop(false).unwrap().is_some() {} } None => track.clock.push((pending, None), pts, arrived, 0), From ddbcc8b14c8ca058ed2a21a8073454e1312d7269 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 19:52:45 -0800 Subject: [PATCH 2/2] fix(moq-mux): lift a tie past every earlier-presented frame still in view Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/declared.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/rs/moq-mux/src/container/declared.rs b/rs/moq-mux/src/container/declared.rs index ee7364282bc..d95a81be746 100644 --- a/rs/moq-mux/src/container/declared.rs +++ b/rs/moq-mux/src/container/declared.rs @@ -32,9 +32,9 @@ pub(crate) fn depth(config: &VideoConfig) -> crate::Result> { /// slot, which needs no frame spacing, so a DTS can fall before the timeline's zero. A tick is the /// caller's timescale, the finest its container writes: 90 kHz for MPEG-TS, 1 ms for FLV. /// -/// Distinct source timestamps can round onto one tick. The frame presented later then moves up a -/// tick at a time until its tick is free, so each frame keeps its own presentation slot and order; -/// identical source timestamps fail, since no order between them is known. +/// Distinct source timestamps can round onto one tick. A frame then moves up to the tick after +/// every frame presented before it, so each keeps its own presentation slot and order; identical +/// source timestamps fail, since no order between them is known. /// /// A clock lives for one group, so its DTS depends on nothing read before the group, and a frame /// reordered deeper than declared fails rather than move the timeline. @@ -92,14 +92,15 @@ impl DeclaredClock { return Err(ClockError::Undrained.into()); } let source = source.as_nanos(); + // Only `highest` matters: a frame below all of it reorders past the depth. let mut pts = pts; - // Only `highest` can hold the tick: the group's ticks are distinct, and one below all of - // `highest` reorders past the depth. - while let Some(&(_, prior)) = self.highest.iter().find(|&&(tick, _)| tick == pts) { - if prior >= source { - return Err(ClockError::Tie.into()); + for &(tick, prior) in &self.highest { + if prior < source && tick >= pts { + pts = tick.checked_add(1).ok_or(ClockError::Tie)?; } - pts = pts.checked_add(1).ok_or(ClockError::Tie)?; + } + if (self.highest.iter()).any(|&(tick, prior)| prior >= source && tick <= pts) { + return Err(ClockError::Tie.into()); } if self.reorders_past(pts) { return Err(ClockError::Reordered(self.depth).into()); @@ -255,11 +256,16 @@ mod tests { (182_897, 182_897) ] ); - // A tie moved onto the next frame's tick moves that frame up in turn. + // A tie moved onto the next frame's tick moves that frame up in turn, however many share + // the tick. assert_eq!( released(0, &[10_000, 10_400, 11_000, 13_000]).unwrap(), [(10, 10), (11, 11), (12, 12), (13, 13)] ); + assert_eq!( + released(0, &[40_000, 40_011, 40_022, 40_033, 80_000]).unwrap(), + [(40, 40), (41, 41), (42, 42), (43, 43), (80, 80)] + ); // Reordered frames tie the same way: the later-presented one moves. assert_eq!( released(1, &[0, 3_000, 1_000, 1_400, 4_000]).unwrap(),