Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 94 additions & 19 deletions rs/moq-mux/src/container/declared.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ pub(crate) fn depth(config: &VideoConfig) -> crate::Result<Option<usize>> {
/// slot, which needs no frame spacing, so a DTS can fall before the timeline's zero. A tick is the
/// caller's timescale, the finest its container writes: 90 kHz for MPEG-TS, 1 ms for FLV.
///
/// Distinct source timestamps can round onto one tick. A frame then moves up to the tick after
/// every frame presented before it, so each keeps its own presentation slot and order; identical
/// source timestamps fail, since no order between them is known.
///
/// A clock lives for one group, so its DTS depends on nothing read before the group, and a frame
/// reordered deeper than declared fails rather than move the timeline.
pub(crate) struct DeclaredClock<T> {
Expand All @@ -44,8 +48,9 @@ pub(crate) struct DeclaredClock<T> {
held: VecDeque<(T, u64)>,
/// Presentation slots not yet taken, ascending.
slots: Vec<u64>,
/// The largest depth + 1 PTS read, ascending: all a deeper reorder is checked against.
highest: Vec<u64>,
/// The largest depth + 1 PTS read, ascending, with their source timestamps in nanoseconds:
/// all a deeper reorder or a tie is checked against.
highest: Vec<(u64, u128)>,
/// The last DTS released.
last: Option<i128>,
}
Expand Down Expand Up @@ -78,19 +83,30 @@ impl<T> DeclaredClock<T> {
/// Whether a frame presented at `pts` would follow more than the depth frames read before it
/// in presentation order, which [`Self::push`] refuses.
pub(crate) fn reorders_past(&self, pts: u64) -> bool {
self.highest.len() - self.highest.partition_point(|&prior| prior <= pts) > self.depth
self.highest.len() - self.highest.partition_point(|&(prior, _)| prior <= pts) > self.depth
}

/// Hold the group's next frame in decode order, presented at `pts`.
pub(crate) fn push(&mut self, item: T, pts: u64) -> crate::Result<()> {
/// Hold the group's next frame in decode order, presented at `pts` ticks, read from `source`.
pub(crate) fn push(&mut self, item: T, pts: u64, source: moq_net::Timestamp) -> crate::Result<()> {
if self.held.len() > self.depth {
return Err(ClockError::Undrained.into());
}
let source = source.as_nanos();
// Only `highest` matters: a frame below all of it reorders past the depth.
let mut pts = pts;
for &(tick, prior) in &self.highest {
if prior < source && tick >= pts {
pts = tick.checked_add(1).ok_or(ClockError::Tie)?;
}
}
if (self.highest.iter()).any(|&(tick, prior)| prior >= source && tick <= pts) {
return Err(ClockError::Tie.into());
}
if self.reorders_past(pts) {
return Err(ClockError::Reordered(self.depth).into());
}
let at = self.highest.partition_point(|&prior| prior <= pts);
self.highest.insert(at, pts);
let at = self.highest.partition_point(|&(prior, _)| prior <= pts);
self.highest.insert(at, (pts, source));
if self.highest.len() > self.depth + 1 {
self.highest.remove(0);
}
Expand All @@ -100,9 +116,10 @@ impl<T> DeclaredClock<T> {
Ok(())
}

/// The next frame and its DTS in signed ticks, once the group's earliest slot is known,
/// or at once when `flush` because the group ended.
pub(crate) fn pop(&mut self, flush: bool) -> crate::Result<Option<(T, i128)>> {
/// The next frame with its DTS in signed ticks and its PTS, past the tick it was pushed at if a
/// tie moved it, once the group's earliest slot is known, or at once when `flush` because the
/// group ended.
pub(crate) fn pop(&mut self, flush: bool) -> crate::Result<Option<(T, i128, u64)>> {
if self.held.is_empty() || (self.earliest.is_none() && !flush && self.held.len() <= self.depth) {
return Ok(None);
}
Expand All @@ -119,7 +136,7 @@ impl<T> DeclaredClock<T> {
return Err(ClockError::Presentation.into());
}
self.last = Some(dts);
Ok(Some((item, dts)))
Ok(Some((item, dts, pts)))
}
}

Expand All @@ -131,6 +148,8 @@ enum ClockError {
Reordered(usize),
#[error("video presentation slots cannot author increasing DTS at or before PTS")]
Presentation,
#[error("video presents two frames at one timestamp")]
Tie,
#[error("the decode clock must be drained before reading another frame")]
Undrained,
}
Expand All @@ -149,19 +168,24 @@ mod tests {
const PYRAMID: [u64; 9] = [0, 4, 2, 1, 3, 8, 6, 5, 7];
const FRAME: u64 = 3_600;

/// A source timestamp exactly on 90 kHz tick `pts`.
fn at(pts: u64) -> moq_net::Timestamp {
moq_net::Timestamp::from_scale(pts, 90_000).unwrap()
}

/// The DTS of one group read through a declared clock of `depth`, released as each frame is
/// read, checking it never holds more than `depth` frames between reads.
fn declared_dts(depth: usize, read: &[u64]) -> Vec<i128> {
let mut clock = DeclaredClock::new(depth);
let mut out = Vec::new();
for &pts in read {
clock.push((), pts).unwrap();
while let Some(((), dts)) = clock.pop(false).unwrap() {
clock.push((), pts, at(pts)).unwrap();
while let Some(((), dts, _)) = clock.pop(false).unwrap() {
out.push(dts);
}
assert!(clock.len() <= depth);
}
while let Some(((), dts)) = clock.pop(true).unwrap() {
while let Some(((), dts, _)) = clock.pop(true).unwrap() {
out.push(dts);
}
out
Expand Down Expand Up @@ -193,12 +217,63 @@ mod tests {
#[test]
fn a_zero_depth_releases_each_frame_at_its_pts() {
let mut clock = DeclaredClock::new(0);
for pts in [0, 1, 90_000, u64::MAX] {
clock.push(pts, pts).unwrap();
assert_eq!(clock.pop(false).unwrap(), Some((pts, i128::from(pts))));
for (index, pts) in [0, 1, 90_000, u64::MAX].into_iter().enumerate() {
let source = moq_net::Timestamp::from_nanos(index as u64).unwrap();
clock.push(pts, pts, source).unwrap();
assert_eq!(clock.pop(false).unwrap(), Some((pts, i128::from(pts), pts)));
assert_eq!(clock.len(), 0);
}
clock.push(0, 0).unwrap_err();
clock.push(0, 0, at(0)).unwrap_err();
}

/// Distinct source timestamps that round onto one millisecond tick, as FLV clocks them, move
/// the later frame up to the next free tick; identical ones still fail.
#[test]
fn sub_tick_ties_move_to_the_next_free_tick() {
let micros = |micros: u64| moq_net::Timestamp::from_micros(micros).unwrap();
let released = |depth: usize, read: &[u64]| {
let mut clock = DeclaredClock::new(depth);
let mut out = Vec::new();
for &source in read {
let source = micros(source);
clock.push((), source.as_millis() as u64, source)?;
while let Some(((), dts, pts)) = clock.pop(false)? {
out.push((dts, pts));
}
}
while let Some(((), dts, pts)) = clock.pop(true)? {
out.push((dts, pts));
}
crate::Result::Ok(out)
};
// MPEG-TS can carry a repeated timestamp one 90 kHz tick (11 µs) later.
assert_eq!(
released(0, &[182_815_000, 182_856_000, 182_856_011, 182_897_000]).unwrap(),
[
(182_815, 182_815),
(182_856, 182_856),
(182_857, 182_857),
(182_897, 182_897)
]
);
// A tie moved onto the next frame's tick moves that frame up in turn, however many share
// the tick.
assert_eq!(
released(0, &[10_000, 10_400, 11_000, 13_000]).unwrap(),
[(10, 10), (11, 11), (12, 12), (13, 13)]
);
assert_eq!(
released(0, &[40_000, 40_011, 40_022, 40_033, 80_000]).unwrap(),
[(40, 40), (41, 41), (42, 42), (43, 43), (80, 80)]
);
// Reordered frames tie the same way: the later-presented one moves.
assert_eq!(
released(1, &[0, 3_000, 1_000, 1_400, 4_000]).unwrap(),
[(-1, 0), (0, 3), (1, 1), (2, 2), (3, 4)]
);
released(0, &[10_000, 10_000]).unwrap_err();
// A frame presented below one already on its tick has nowhere to go.
released(1, &[0, 10_400, 10_000]).unwrap_err();
}

/// Flat or backward PTS read past the declared depth fail within depth + 1 frames, however
Expand All @@ -211,7 +286,7 @@ mod tests {
let mut clock = DeclaredClock::new(2);
let refused = (0..count).position(|index| {
let pts = 90_000 - if backwards { index } else { 0 };
let refused = clock.push(vec![0u8; size], pts).is_err()
let refused = clock.push(vec![0u8; size], pts, at(pts)).is_err()
|| std::iter::from_fn(|| clock.pop(false).transpose()).any(|popped| popped.is_err());
assert!(clock.len() <= 2);
refused
Expand Down
10 changes: 7 additions & 3 deletions rs/moq-mux/src/container/flv/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,11 @@ impl FlvTrack {
return Ok(());
};
let released = (|| {
while let Some(((mut arrival, record), dts)) = clock.pop(flush)? {
while let Some(((mut arrival, record), dts, pts)) = clock.pop(flush)? {
// A tie the clock moved presents on the tick it moved to.
if arrival.item.timestamp.as_millis() != u128::from(pts) {
arrival.item.timestamp = moq_net::Timestamp::from_millis(pts)?;
}
if let Some(settled) = settled {
arrival.arrived = arrival.arrived.max(settled);
}
Expand Down Expand Up @@ -527,9 +531,9 @@ impl<S: Stream> Export<S> {
match &mut track.declared {
Some(clock) => {
// Clocked in FLV's milliseconds, so the leading frames step a tag tick apart.
let pts = arrival.item.timestamp.as_millis() as u64;
let source = arrival.item.timestamp;
let arrived = arrival.arrived;
clock.push((arrival, record), pts)?;
clock.push((arrival, record), source.as_millis() as u64, source)?;
track.release(key, Some(arrived), self.shift, false, &mut self.jitter)?;
}
None => {
Expand Down
33 changes: 22 additions & 11 deletions rs/moq-mux/src/container/flv/export_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1350,7 +1350,7 @@ async fn export_authors_dts_and_composition_time_for_reordered_avc() {
}

/// The `(DTS, PTS)` of every AVC frame tag, and the audio tag timestamps, an export renders of
/// a broadcast that declares `reorder` and carries a group of `stamps` (ms, decode order) for
/// a broadcast that declares `reorder` and carries a group of `stamps` (µs, decode order) for
/// each entry, plus one audio frame at 20 ms. With `open`, the last group is cut but the video
/// track stays open.
async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow::Result<(Vec<(u32, i64)>, Vec<u32>)> {
Expand Down Expand Up @@ -1385,8 +1385,8 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow::
.renditions
.insert(audio_track.name().to_string(), audio_config);
}
let frame = |ms: u64, payload: &'static [u8], keyframe| crate::container::Frame {
timestamp: Timestamp::from_millis(ms).unwrap(),
let frame = |micros: u64, payload: &'static [u8], keyframe| crate::container::Frame {
timestamp: Timestamp::from_micros(micros).unwrap(),
duration: None,
payload: Bytes::from_static(payload),
keyframe,
Expand All @@ -1396,13 +1396,13 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow::
crate::catalog::hang::Container::Legacy(crate::container::Kind::Data),
);
for stamps in groups {
for (index, &ms) in stamps.iter().enumerate() {
for (index, &micros) in stamps.iter().enumerate() {
let payload: &'static [u8] = if index == 0 {
&[0, 0, 0, 1, 0x65]
} else {
&[0, 0, 0, 1, 0x41]
};
video.write(frame(ms, payload, index == 0)).unwrap();
video.write(frame(micros, payload, index == 0)).unwrap();
}
}
match open {
Expand All @@ -1413,7 +1413,7 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow::
audio_track,
crate::catalog::hang::Container::Legacy(crate::container::Kind::Data),
);
audio.write(frame(20, &[0xde, 0xad], true)).unwrap();
audio.write(frame(20_000, &[0xde, 0xad], true)).unwrap();
let _finished = audio.finish().unwrap();
catalog.finish().unwrap();

Expand Down Expand Up @@ -1452,7 +1452,10 @@ async fn export_declared(reorder: u8, groups: &[&[u64]], open: bool) -> anyhow::
/// presentation slots. The whole stream shifts by the depth, so those fit FLV's unsigned clock.
#[tokio::test(start_paused = true)]
async fn export_declared_reorder_decodes_each_group_from_its_frames() {
let groups: [&[u64]; 2] = [&[0, 160, 80, 40, 120], &[400, 560, 480, 440, 520]];
let groups: [&[u64]; 2] = [
&[0, 160_000, 80_000, 40_000, 120_000],
&[400_000, 560_000, 480_000, 440_000, 520_000],
];
let (video, audio) = export_declared(2, &groups, false).await.unwrap();
assert_eq!(
video,
Expand All @@ -1472,19 +1475,27 @@ async fn export_declared_reorder_decodes_each_group_from_its_frames() {
assert_eq!(audio, [22]);

// A first group cut short leaves the next decoding where it would have.
let cut: [&[u64]; 2] = [&[0, 160, 80], &[400, 560, 480, 440, 520]];
let cut: [&[u64]; 2] = [&[0, 160_000, 80_000], groups[1]];
let (cut, _) = export_declared(2, &cut, false).await.unwrap();
assert_eq!(cut[3..], video[5..]);

// A depth of zero decodes at presentation, and refuses a frame that reorders.
let (video, _) = export_declared(0, &[&[0, 40, 80]], false).await.unwrap();
let (video, _) = export_declared(0, &[&[0, 40_000, 80_000]], false).await.unwrap();
assert_eq!(video, [(0, 0), (40, 40), (80, 80)]);
let err = export_declared(0, &[&[0, 80, 40]], false).await.unwrap_err();
let err = export_declared(0, &[&[0, 80_000, 40_000]], false).await.unwrap_err();
assert!(err.to_string().contains("declared depth"), "{err}");

// Distinct timestamps within one millisecond present a tick apart; identical ones fail.
let (video, _) = export_declared(0, &[&[0, 40_000, 40_011, 80_000]], false)
.await
.unwrap();
assert_eq!(video, [(0, 0), (40, 40), (41, 41), (80, 80)]);
let err = export_declared(0, &[&[0, 40_000, 40_000]], false).await.unwrap_err();
assert!(err.to_string().contains("one timestamp"), "{err}");

// A group that ends goes out whole while the track stays open, though fewer frames than the
// depth settle nothing.
let (video, _) = export_declared(2, &[&[0, 80]], true).await.unwrap();
let (video, _) = export_declared(2, &[&[0, 80_000]], true).await.unwrap();
assert_eq!(video, [(0, 2), (1, 82)]);
}

Expand Down
11 changes: 9 additions & 2 deletions rs/moq-mux/src/container/mkv/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,10 @@ impl MkvTrack {
) -> Result<()> {
let pts = (arrival.item.timestamp.as_nanos() * 90_000 / 1_000_000_000) as u64;
match &mut self.declared {
Some(clock) => clock.push(arrival, pts)?,
Some(clock) => {
let source = arrival.item.timestamp;
clock.push(arrival, pts, source)?
}
None => {
let (arrived, bytes) = (arrival.arrived, arrival.bytes);
self.clock.push(arrival, pts, arrived, bytes)
Expand All @@ -194,7 +197,11 @@ impl MkvTrack {

fn release(&mut self, name: &str, jitter: &mut crate::jitter::Buffer<String, Frame>, flush: bool) -> Result<()> {
if let Some(clock) = &mut self.declared {
while let Some((mut arrival, dts)) = clock.pop(flush)? {
while let Some((mut arrival, dts, pts)) = clock.pop(flush)? {
// A tie the clock moved presents on the tick it moved to.
if (arrival.item.timestamp.as_nanos() * 90_000 / 1_000_000_000) as u64 != pts {
arrival.item.timestamp = moq_net::Timestamp::from_scale(pts, 90_000)?;
}
if !self.finished && !flush {
arrival.arrived = arrival.arrived.max(self.latest);
}
Expand Down
17 changes: 12 additions & 5 deletions rs/moq-mux/src/container/ts/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@ impl Track {
// exporter that kept it.
return match &mut self.declared {
Some(clock) => {
clock.push(None, to_ticks(pending.frame.timestamp))?;
clock.push(None, to_ticks(pending.frame.timestamp), pending.frame.timestamp)?;
self.release(name, jitter, false)
}
None => Ok(()),
Expand All @@ -533,11 +533,12 @@ impl Track {
}
self.declared = depth.map(DeclaredClock::new);
}
let pts = to_ticks(pending.frame.timestamp);
let source = pending.frame.timestamp;
let pts = to_ticks(source);
let (arrived, bytes) = (pending.arrived, pending.frame.payload.len());
let item = (pending, self.source.description().cloned());
match &mut self.declared {
Some(clock) => clock.push(Some(item), pts)?,
Some(clock) => clock.push(Some(item), pts, source)?,
None => self.clock.push(item, pts, arrived, bytes),
}
self.release(name, jitter, false)
Expand Down Expand Up @@ -808,10 +809,16 @@ impl Track {
) -> anyhow::Result<()> {
if let Some(mut clock) = self.declared.take() {
let released = (|| {
while let Some((item, dts)) = clock.pop(flush)? {
while let Some((item, dts, pts)) = clock.pop(flush)? {
let Some((mut pending, description)) = item else {
continue;
};
// A tie the clock moved presents on the tick it moved to: the least microsecond
// that `to_ticks` reads back as that tick.
if to_ticks(pending.frame.timestamp) != pts {
let micros = u64::try_from((u128::from(pts) * 100).div_ceil(9))?;
pending.frame.timestamp = Timestamp::from_micros(micros)?;
}
if self.read == Read::Live {
pending.arrived = pending.arrived.max(self.latest);
}
Expand Down Expand Up @@ -5233,7 +5240,7 @@ mod tests {
let pts = super::to_ticks(ms(at));
match &mut track.declared {
Some(clock) => {
clock.push(Some((pending, None)), pts).unwrap();
clock.push(Some((pending, None)), pts, ms(at)).unwrap();
while clock.pop(false).unwrap().is_some() {}
}
None => track.clock.push((pending, None), pts, arrived, 0),
Expand Down