From e22c299353c2d9a87ca52b8d13138bce4c2b2c04 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:44:27 -0700 Subject: [PATCH 1/2] chore(quest): claim quest/m1/release-audit/export-reorder.md Quest-Claim: fcb4e1c52a746f51 From e74d92e33918fcf6caaa7d33e86184e0020be4d7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:26:09 -0700 Subject: [PATCH 2/2] fix(mux): bound what the decode clock holds in MKV and TS export A video track whose PTS stopped advancing past the reorder lookahead in an open group held every frame in the decode clock, outside the jitter buffer's backpressure, and a flush then drained it in quadratic time. The frame held longest now settles once it has waited, by the latest arrival read, longer than its lookahead plus the export's delay, so what the clock holds stays within what the source sends in that time. The lookahead is clamped to MAX_REORDER in the clock, which TS's catalog jitter skipped. Slots are a VecDeque and each slot/frame pair bounds the delay once, so a drain is linear. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EeyiEy5FWz3z44DYZVo7vU --- quest/m1/release-audit/README.md | 1 - quest/m1/release-audit/export-reorder.md | 36 ------- rs/moq-mux/Cargo.toml | 4 + rs/moq-mux/benches/decode_clock.rs | 98 +++++++++++++++++++ rs/moq-mux/src/container/decode.rs | 101 ++++++++++++++++---- rs/moq-mux/src/container/mkv/export.rs | 17 +++- rs/moq-mux/src/container/mkv/export_test.rs | 79 +++++++++++++++ rs/moq-mux/src/container/ts/export.rs | 31 +++--- rs/moq-mux/src/container/ts/export_test.rs | 33 +++++++ sh/gh/nightly.sh | 2 +- 10 files changed, 328 insertions(+), 74 deletions(-) delete mode 100644 quest/m1/release-audit/export-reorder.md create mode 100644 rs/moq-mux/benches/decode_clock.rs diff --git a/quest/m1/release-audit/README.md b/quest/m1/release-audit/README.md index 7a5bccd119b..e1b14d4e1bc 100644 --- a/quest/m1/release-audit/README.md +++ b/quest/m1/release-audit/README.md @@ -94,7 +94,6 @@ child records whether its behavior is new on main or inherited from release. - [Review HLS admission](/quest/m1/release-audit/hls-admission.md) - resolve the security-sensitive resource-boundary review before the cut - [Bound decoded datagram retention](/quest/m1/release-audit/datagram-memory.md) - shared ownership accounts for retained plain and compressed payloads across tracks -- [Bound export reorder storage](/quest/m1/release-audit/export-reorder.md) - flat timestamps cannot bypass MKV/TS backpressure or cause quadratic draining - [Validate CMAF header fields](/quest/m1/release-audit/cmaf-counts.md) - tiny fragments cannot force allocations proportional to an untrusted sample count, and a zero timescale is refused - [Preserve fMP4 sample runs](/quest/m1/release-audit/fmp4-passthrough.md) - passthrough does not substitute gap or interleaved bytes for media samples - [JS honors subscription bounds](/quest/m1/release-audit/js-subscription-bounds.md) - each reader and each datagram stays inside its own subscription's group bounds, not the aggregate's diff --git a/quest/m1/release-audit/export-reorder.md b/quest/m1/release-audit/export-reorder.md deleted file mode 100644 index 5be921c2d62..00000000000 --- a/quest/m1/release-audit/export-reorder.md +++ /dev/null @@ -1,36 +0,0 @@ -# [M] Include decode reordering in exporter backpressure - -## Goal - -MKV and TS export cannot retain unlimited frames or spend quadratic time -draining them when timestamps stop advancing. Preserve valid B-frame timing -while bounding all exporter-owned payloads, including lookahead storage. - -## Plan - -P1, introduced since release by -[moq-dev/moq#5309](https://github.com/moq-dev/moq/pull/5309), with later TS reuse. -At main 6369eca, `rs/moq-mux/src/container/decode.rs:62-93` inserts every frame -in `held` and `slots`, but timestamp-driven `pop` waits for the highest PTS to -advance past lookahead. `MAX_REORDER` bounds media time, not retained bytes. -Compiling this actual source and pushing 100,000 1KiB items at PTS 90,000, -calling `pop(9_000, 0, false)` after each, released zero frames. An explicit -flush recovered all 102,400,000 payload bytes. Flushing also repeatedly scans -remaining slots and removes vector element zero. - -MKV export gates reads on the downstream jitter queue (`mkv/export.rs:351`) -but inserts video into DecodeClock first (`:398-406`); a stuck clock leaves -that queue empty. Catalog jitter or SPS metadata supplies lookahead, and the -normal CLI buffered export defaults to 500ms. MKV ignores group ends. TS has -the same open-group exposure for timestamp-driven clocks (`ts/export.rs:417`, -`:431`, `:798`), although it flushes at later group boundaries and some SPS -paths use a bounded reorder depth. Consumer integration is source-traced; -the reproduction exercised the clock component. - -Tie reorder storage to exporter capacity/ownership, with explicit behavior -for unsupported timestamp progress and without a timer or arbitrary frame -limit. Add exporter-level open-group tests with flat/backward PTS and valid -reordering. Sweep buffered items and payload sizes to measure retained memory -and drain cost. Source-cache accounting cannot solve this downstream queue. -No public API or wire change is expected; report any changed malformed-input -behavior. Current consumers are CLI MKV and TS export. diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 16f90fc6ab0..031ba8793c4 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -65,6 +65,10 @@ harness = false name = "jitter" harness = false +[[bench]] +name = "decode_clock" +harness = false + [[bench]] name = "containers" harness = false diff --git a/rs/moq-mux/benches/decode_clock.rs b/rs/moq-mux/benches/decode_clock.rs new file mode 100644 index 00000000000..edfbab3820f --- /dev/null +++ b/rs/moq-mux/benches/decode_clock.rs @@ -0,0 +1,98 @@ +//! The decode clock swept over how many frames it holds and how their PTS run: draining what +//! it holds, and settling frames as they are read until the overdue rule bounds them. +use std::time::Duration; + +use bytes::Bytes; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_clock::Instant; + +#[path = "../src/container/decode.rs"] +#[allow(dead_code)] +mod decode; + +/// A 40 ms frame in 90 kHz ticks. +const FRAME: u64 = 3_600; + +/// The `index`-th frame's PTS in decode order: stuck, running backward, or a valid B-pyramid. +fn pts(shape: &str, index: u64) -> u64 { + let base = 1_000 * 90_000; + match shape { + "flat" => base, + "backward" => base - index, + _ => { + const PYRAMID: [u64; 9] = [0, 4, 2, 1, 3, 8, 6, 5, 7]; + base + (index / 9 * 9 + PYRAMID[(index % 9) as usize]) * FRAME + } + } +} + +/// Flush `held` frames read with nothing settled, as at a track's end. +fn drain(c: &mut Criterion) { + let mut group = c.benchmark_group("decode_clock_drain"); + group.measurement_time(Duration::from_secs(1)); + group.warm_up_time(Duration::from_millis(200)); + group.sample_size(10); + for shape in ["flat", "backward", "pyramid"] { + for held in [1_000u64, 10_000, 100_000] { + for size in [64usize, 4_096] { + let payload = Bytes::from(vec![0; size]); + group.throughput(Throughput::Elements(held)); + group.bench_function(BenchmarkId::new(format!("{shape}/{size}B"), held), |b| { + b.iter(|| { + let mut clock = decode::DecodeClock::default(); + for index in 0..held { + clock.push(payload.clone(), pts(shape, index), Instant::ZERO); + } + let mut drained = 0; + while clock.pop(decode::MAX_REORDER, 0, decode::Settle::All).is_some() { + drained += 1; + } + assert_eq!(drained, held); + }) + }); + } + } + } + group.finish(); +} + +/// Read `frames` frames 1 ms apart, a 120 ms lookahead and 500 ms delay, settling each as an +/// exporter does: flat or backward PTS holds what the overdue rule lets through. +fn read(c: &mut Criterion) { + let mut group = c.benchmark_group("decode_clock_read"); + group.measurement_time(Duration::from_secs(1)); + group.warm_up_time(Duration::from_millis(200)); + group.sample_size(10); + let lookahead = 120 * 90; + let delay = Duration::from_millis(500); + for shape in ["flat", "backward", "pyramid"] { + for frames in [1_000u64, 10_000, 100_000] { + group.throughput(Throughput::Elements(frames)); + group.bench_function(BenchmarkId::new(shape, frames), |b| { + b.iter(|| { + let mut clock = decode::DecodeClock::default(); + let (mut held, mut most) = (0, 0); + for index in 0..frames { + let latest = Instant::ZERO + Duration::from_millis(index); + clock.push(index, pts(shape, index), latest); + held += 1; + loop { + let settle = clock.settle(lookahead, latest, delay); + if clock.pop(lookahead, 0, settle).is_none() { + break; + } + held -= 1; + } + most = most.max(held); + } + // What arrives in the deepest lookahead and the delay, a frame a millisecond. + assert!(most <= 2_501, "{shape}: {most} held"); + }) + }); + } + } + group.finish(); +} + +criterion_group!(benches, drain, read); +criterion_main!(benches); diff --git a/rs/moq-mux/src/container/decode.rs b/rs/moq-mux/src/container/decode.rs index 68ad5d67e0d..52ed4e79176 100644 --- a/rs/moq-mux/src/container/decode.rs +++ b/rs/moq-mux/src/container/decode.rs @@ -1,8 +1,22 @@ use std::collections::VecDeque; +use std::time::Duration; + +use moq_clock::Instant; /// The largest supported reorder span in 90 kHz ticks. pub(crate) const MAX_REORDER: u64 = 2 * 90_000; +/// Which frames [`DecodeClock::pop`] gives out. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Settle { + /// Those whose slot the highest PTS read settled. + Due, + /// Those, and the frame held longest, which is overdue ([`DecodeClock::settle`]). + Front, + /// Every frame held, as nothing still to come is presented before them. + All, +} + /// A video rendition's decode clock, authored from presentation timestamps. /// /// [`crate::container::Frame`] carries only a presentation timestamp and frames reach the muxer in decode order, @@ -23,6 +37,11 @@ pub(crate) const MAX_REORDER: u64 = 2 * 90_000; /// (reordering deeper than all of those) is nudged one tick past the last DTS, so the clock /// stays strictly increasing. /// +/// A source whose PTS stops advancing would hold every frame after it here, so a frame that +/// has waited longer than its lookahead plus the export's delay is overdue, and settled at +/// once ([`Self::settle`]): a live source's PTS keeps pace with its arrival, so the frames +/// held stay within what the source sends in that time, like the jitter buffer's. +/// /// That settling, and a delay that only grows, depend on everything read before, so two /// exporters that joined or lost frames at different points can clock the same frame apart. /// [`Self::pop_counted`] depends on the frames held alone, for exporters that must agree. @@ -30,10 +49,13 @@ pub(crate) const MAX_REORDER: u64 = 2 * 90_000; /// Ticks are continuous (unwrapped) 90 kHz, so the clock never wraps mid-stream; the 33-bit /// wire wrap happens once at emission. pub(crate) struct DecodeClock { - /// Frames read but not yet given a DTS, in decode order, with their PTS. - held: VecDeque<(T, u64)>, + /// Frames read but not yet given a DTS, in decode order, with their PTS and arrival. + held: VecDeque<(T, u64, Instant)>, /// The held frames' PTS, ascending: the slots they decode in, in turn. - slots: Vec, + slots: VecDeque, + /// How many leading slots, paired with the held frame of the same rank, already bound the + /// delay, so draining a backlog scans each pair once. + bounded: usize, /// The highest PTS read. high: Option, /// The furthest a frame was presented below the highest PTS read before it, plus a tick. @@ -48,7 +70,8 @@ impl Default for DecodeClock { fn default() -> Self { Self { held: VecDeque::new(), - slots: Vec::new(), + slots: VecDeque::new(), + bounded: 0, high: None, reach: 0, delay: 0, @@ -58,8 +81,8 @@ impl Default for DecodeClock { } impl DecodeClock { - /// Hold the next frame in decode order, presented at `pts`. - pub(crate) fn push(&mut self, item: T, pts: u64) { + /// Hold the next frame in decode order, presented at `pts`, which `arrived` when read. + pub(crate) fn push(&mut self, item: T, pts: u64, arrived: Instant) { if let Some(high) = self.high && pts < high { @@ -68,34 +91,57 @@ impl DecodeClock { self.high = Some(self.high.map_or(pts, |high| high.max(pts))); let at = self.slots.partition_point(|&slot| slot <= pts); self.slots.insert(at, pts); - self.held.push_back((item, pts)); + self.bounded = self.bounded.min(at); + self.held.push_back((item, pts, arrived)); } /// The next frame and its DTS, once its slot is settled `lookahead` past (the catalog - /// `jitter`), or at once when `flush`. `floor` is the least reorder delay, in ticks. - pub(crate) fn pop(&mut self, lookahead: u64, floor: u64, flush: bool) -> Option<(T, u64)> { + /// `jitter`), or at once as `settle` says. `floor` is the least reorder delay, in ticks. + pub(crate) fn pop(&mut self, lookahead: u64, floor: u64, settle: Settle) -> Option<(T, u64)> { let high = self.high?; - let lookahead = lookahead.max(self.reach); - let settled = |slot: u64| flush || high >= slot.saturating_add(lookahead); - if !settled(*self.slots.first()?) { + let lookahead = self.lookahead(lookahead); + let front = *self.slots.front()?; + let settled = |slot: u64| match settle { + Settle::All => true, + Settle::Front if slot <= front => true, + _ => high >= slot.saturating_add(lookahead), + }; + if !settled(front) { return None; } // Every settled frame bounds the delay, so one decoding later never steps it back. let mut delay = self.delay.max(floor); - for (&slot, (_, pts)) in self.slots.iter().zip(&self.held) { + let mut bounded = self.bounded; + for (&slot, (_, pts, _)) in self.slots.range(bounded..).zip(self.held.range(bounded..)) { if !settled(slot) { break; } delay = delay.max(slot.saturating_sub(*pts)); + bounded += 1; } self.delay = delay.min(MAX_REORDER); + self.bounded = bounded - 1; - let slot = self.slots.remove(0); - let (item, _) = self.held.pop_front()?; + let slot = self.slots.pop_front()?; + let (item, ..) = self.held.pop_front()?; let dts = slot.saturating_sub(self.delay); Some((item, self.after_last(dts))) } + /// What [`Self::pop`] settles short of a flush: [`Settle::Front`] once the frame held + /// longest is overdue, having waited, by the `latest` arrival read, longer than `lookahead` + /// plus the export's `delay`, else [`Settle::Due`]. + pub(crate) fn settle(&self, lookahead: u64, latest: Instant, delay: Duration) -> Settle { + let Some((.., arrived)) = self.held.front() else { + return Settle::Due; + }; + let lookahead = Duration::from_nanos(self.lookahead(lookahead) * 1_000_000_000 / 90_000); + match latest.saturating_duration_since(*arrived) > lookahead + delay { + true => Settle::Front, + false => Settle::Due, + } + } + /// The next frame and its DTS, once more than `depth` frames are held, or at once when /// `flush`. `depth` is the most frames the sequence header lets precede one in decode order /// and follow it in output order, so no frame still to come is presented before all of @@ -104,16 +150,29 @@ impl DecodeClock { if !flush && self.held.len() <= depth { return None; } - let (item, pts) = self.held.pop_front()?; - let slot = self.slots.remove(0); + let (item, pts, _) = self.held.pop_front()?; + let slot = self.slots.pop_front()?; + self.bounded = self.bounded.saturating_sub(1); let dts = slot.saturating_sub(delay).min(pts); Some((item, self.after_last(dts))) } + /// How many frames are held. + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.held.len() + } + /// Forget every held frame, as if it had never been read. pub(crate) fn discard(&mut self) { self.held.clear(); self.slots.clear(); + self.bounded = 0; + } + + /// How far past a slot the highest PTS must be to settle it, under `lookahead`. + fn lookahead(&self, lookahead: u64) -> u64 { + lookahead.max(self.reach).min(MAX_REORDER) } fn after_last(&mut self, dts: u64) -> u64 { @@ -141,10 +200,10 @@ mod tests { let mut out = Vec::new(); let pop = |clock: &mut DecodeClock, flush| match counted { true => clock.pop_counted(2, 2 * FRAME, flush), - false => clock.pop(4 * FRAME, 2 * FRAME, flush), + false => clock.pop(4 * FRAME, 2 * FRAME, if flush { Settle::All } else { Settle::Due }), }; for &pts in read { - clock.push(pts, pts); + clock.push(pts, pts, Instant::ZERO); out.extend(std::iter::from_fn(|| pop(&mut clock, false))); } out.extend(std::iter::from_fn(|| pop(&mut clock, true))); @@ -196,12 +255,12 @@ mod tests { let mut clock = DecodeClock::default(); let mut out = Vec::new(); for &pts in &first[..8] { - clock.push(pts, pts); + clock.push(pts, pts, Instant::ZERO); out.extend(std::iter::from_fn(|| clock.pop_counted(2, 2 * FRAME, false))); } clock.discard(); for pts in next { - clock.push(pts, pts); + clock.push(pts, pts, Instant::ZERO); out.extend(std::iter::from_fn(|| clock.pop_counted(2, 2 * FRAME, false))); } out.extend(std::iter::from_fn(|| clock.pop_counted(2, 2 * FRAME, true))); diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 2b129d434d1..680ecb63ba2 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -13,6 +13,7 @@ use crate::catalog::Stream; use crate::container::ExportSource; use crate::container::Frame; use crate::container::consumer::Event; +use crate::container::decode::{DecodeClock, Settle}; use crate::container::mkv::Error; /// Matroska TimestampScale: 1 ms (in nanoseconds). @@ -67,7 +68,7 @@ pub struct Export { } struct MkvTrack { - clock: crate::container::decode::DecodeClock>, + clock: DecodeClock>, config: Option, restart: u64, reorder: Option<(u64, u64)>, @@ -106,7 +107,14 @@ impl MkvTrack { .min(limit); (floor.min(limit), lookahead) }); - while let Some((mut arrival, dts)) = self.clock.pop(lookahead, floor, flush) { + loop { + let settle = match flush { + true => Settle::All, + false => self.clock.settle(lookahead, self.latest, jitter.delay()), + }; + let Some((mut arrival, dts)) = self.clock.pop(lookahead, floor, settle) else { + break; + }; if !self.finished && !flush { arrival.arrived = arrival.arrived.max(self.latest); } @@ -335,7 +343,7 @@ impl Export { /// How many frames the export holds between its sources and the output. #[cfg(test)] pub(super) fn buffered(&self) -> usize { - self.jitter.len() + self.jitter.len() + self.tracks.values().map(|track| track.clock.len()).sum::() } #[cfg(test)] @@ -402,7 +410,8 @@ impl Export { track.restart = arrival.restart; } let pts = (arrival.item.timestamp.as_nanos() * 90_000 / 1_000_000_000) as u64; - track.clock.push(arrival, pts); + let arrived = arrival.arrived; + track.clock.push(arrival, pts, arrived); track.release(name, &mut self.jitter, false)?; } else { self.jitter.push(name.clone(), arrival)?; diff --git a/rs/moq-mux/src/container/mkv/export_test.rs b/rs/moq-mux/src/container/mkv/export_test.rs index e1d418735ac..2823f14a3d4 100644 --- a/rs/moq-mux/src/container/mkv/export_test.rs +++ b/rs/moq-mux/src/container/mkv/export_test.rs @@ -1132,3 +1132,82 @@ async fn disabled_renditions_stay_out_of_the_header_and_idle_after_it() { live.track.write(raw_frame(20_000, b"main-1", true)).unwrap(); assert!(chunk_now(&mut export).await.windows(6).any(|part| part == b"main-1")); } + +/// Video whose PTS stops advancing past the reorder lookahead in an open group: the export +/// holds no more than what arrives within the lookahead and the delay. Flat PTS waits the +/// catalog `jitter`; PTS running backward reads as reordering up to the deepest supported. +#[tokio::test(start_paused = true)] +async fn stalled_presentation_time_keeps_the_export_bounded() { + use std::time::Duration; + let step = Duration::from_millis(20); + let delay = Duration::from_millis(100); + let jitter = Duration::from_millis(120); + let deepest = Duration::from_secs(2); + let flat: fn(u64) -> u64 = |_| 10_000_000; + let backward: fn(u64) -> u64 = |index| 10_000_000 - index * 10_000; + for (name, pts, lookahead) in [("flat", flat, jitter), ("backward", backward, deepest)] { + let mut live = Live::avc3(); + for config in live.catalog.modify().unwrap().video.renditions.values_mut() { + config.jitter = Some(jitter); + } + let mut export = super::Export::new(live.source().await, live.catalog_stream().await) + .with_delay(delay) + .with_fragment_duration(Duration::ZERO); + live.track.write(video_frame(pts(0), true)).unwrap(); + export.next().await.unwrap().unwrap(); + let mut most = 0; + for index in 1..500 { + tokio::time::advance(step).await; + live.track.write(video_frame(pts(index), false)).unwrap(); + while let std::task::Poll::Ready(chunk) = export.poll_next(&kio::Waiter::noop()) { + chunk.unwrap().unwrap(); + } + most = most.max(export.buffered()); + } + // What arrived within the wait, the frame just read, and one due to go out. + let bound = ((lookahead + delay).as_millis() / step.as_millis()) as usize + 2; + assert!(most <= bound, "{name}: {most} frames buffered, over {bound}"); + } +} + +/// Live B-frames, read as they are sent, keep every frame and its presentation order: the +/// bound on what the decode clock holds never cuts a valid reorder short. +#[tokio::test(start_paused = true)] +async fn live_reordered_video_keeps_every_frame() { + use std::time::Duration; + let frame = 40_000; + let mut live = Live::avc3(); + for config in live.catalog.modify().unwrap().video.renditions.values_mut() { + config.jitter = Some(Duration::from_millis(120)); + } + let mut export = super::Export::new(live.source().await, live.catalog_stream().await) + .with_delay(Duration::from_millis(500)) + .with_fragment_duration(Duration::ZERO); + // A B-pyramid in decode order, a group of nine 40 ms frames at a time. + let pyramid = [0, 4, 2, 1, 3, 8, 6, 5, 7]; + let order: Vec = (0..10u64) + .flat_map(|group| pyramid.map(|i| 1_000_000 + (group * 9 + i) * frame)) + .collect(); + live.track.write(video_frame(order[0], true)).unwrap(); + export.next().await.unwrap().unwrap(); + let mut blocks = 0; + let mut count = |chunk: Bytes| { + let mut cursor = Cursor::new(chunk); + blocks += WebmIterator::new(&mut cursor, &[]) + .filter(|tag| matches!(tag, Ok(MatroskaSpec::SimpleBlock(_)))) + .count(); + }; + for (index, &pts) in order.iter().enumerate().skip(1) { + tokio::time::advance(Duration::from_micros(frame)).await; + live.track.write(video_frame(pts, index % 9 == 0)).unwrap(); + while let std::task::Poll::Ready(chunk) = export.poll_next(&kio::Waiter::noop()) { + count(chunk.unwrap().unwrap()); + } + } + live.track.finish().unwrap(); + live.catalog.finish().unwrap(); + while let Some(chunk) = export.next().await.unwrap() { + count(chunk); + } + assert_eq!(blocks, order.len()); +} diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 06163cb48a9..1d76f947e34 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -48,7 +48,7 @@ use crate::codec::annexb::InBandCodec; use crate::codec::video::{Hrd, Reorder}; use crate::codec::{aac, annexb, opus}; use crate::container::consumer::Event; -use crate::container::decode::{DecodeClock, MAX_REORDER}; +use crate::container::decode::{DecodeClock, MAX_REORDER, Settle}; use crate::container::{ExportSource, Frame}; use crate::jitter::{self, Arrival, Push}; @@ -415,7 +415,9 @@ impl Track { .describe(stream_type, self.source.description(), name, self.sync.is_some()); } let pts = to_ticks(pending.frame.timestamp); - self.clock.push((pending, self.source.description().cloned()), pts); + let arrived = pending.arrived; + self.clock + .push((pending, self.source.description().cloned()), pts, arrived); self.release(name, jitter, false) } @@ -783,7 +785,8 @@ impl Track { Ok((key, start)) } - /// Queue every held video frame whose DTS is settled, or all of them when `flush`. + /// Queue every held video frame whose DTS is settled or that is overdue + /// ([`DecodeClock::settle`]), or all of them when `flush`. /// /// A held frame is ready no earlier than the latest frame the source delivered, which /// settled it, so it is judged as arriving then. Frames held through a silence, @@ -800,7 +803,13 @@ impl Track { let counted = self.sync.as_ref().and(self.timing.depth()); while let Some(((mut pending, description), dts)) = match counted { Some(depth) => self.clock.pop_counted(depth, delay, flush), - None => self.clock.pop(lookahead, delay, flush), + None => { + let settle = match flush { + true => Settle::All, + false => self.clock.settle(lookahead, self.latest, jitter.delay()), + }; + self.clock.pop(lookahead, delay, settle) + } } { if !self.finished { pending.arrived = pending.arrived.max(self.latest); @@ -2071,10 +2080,10 @@ where stats } - /// How many access units are waiting in the jitter buffer. + /// How many access units are waiting in the jitter buffer or a decode clock. #[cfg(test)] pub(super) fn buffered(&self) -> usize { - self.jitter.len() + self.jitter.len() + self.tracks.values().map(|track| track.clock.len()).sum::() } /// When the next queued frame is due, or the next grid slot while media is queued or @@ -4055,8 +4064,8 @@ mod tests { use moq_net::Timestamp; use super::{ - Buffer, DecodeClock, Dolby, Kind, PSI_INTERVAL, PesPacket, dolby_removals, due, is_complete_section, si_due, - slot, unwrap_pts, verbatim_buffer, + Buffer, DecodeClock, Dolby, Kind, PSI_INTERVAL, PesPacket, Settle, dolby_removals, due, is_complete_section, + si_due, slot, unwrap_pts, verbatim_buffer, }; /// The same available video/audio backlog must choose the same start in either fill order. @@ -4261,12 +4270,12 @@ mod tests { let mut clock = DecodeClock::default(); let mut dts = vec![0; pts.len()]; for (i, &p) in pts.iter().enumerate() { - clock.push(i, p); - while let Some((i, d)) = clock.pop(jitter, floor, false) { + clock.push(i, p, moq_clock::Instant::ZERO); + while let Some((i, d)) = clock.pop(jitter, floor, Settle::Due) { dts[i] = d; } } - while let Some((i, d)) = clock.pop(jitter, floor, true) { + while let Some((i, d)) = clock.pop(jitter, floor, Settle::All) { dts[i] = d; } dts diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 895d4b725ce..4b979995b20 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -9856,6 +9856,39 @@ async fn a_backlog_stays_in_the_source_until_due() { } } +/// Video whose PTS stops advancing past the reorder lookahead in an open group: the export +/// holds no more than what arrives within the lookahead and the delay. Flat PTS waits the +/// catalog `jitter`; PTS running backward reads as reordering up to the deepest supported. +#[tokio::test(start_paused = true)] +async fn stalled_presentation_time_keeps_the_export_bounded() { + use crate::container::test_util::{Live, video_frame}; + + let step = Duration::from_millis(20); + let delay = Duration::from_millis(100); + let jitter = Duration::from_millis(120); + let deepest = Duration::from_secs(2); + let flat: fn(u64) -> u64 = |_| 10_000_000; + let backward: fn(u64) -> u64 = |index| 10_000_000 - index * 10_000; + for (name, pts, lookahead) in [("flat", flat, jitter), ("backward", backward, deepest)] { + let mut live = Live::avc3(); + for config in live.catalog.modify().unwrap().video.renditions.values_mut() { + config.jitter = Some(jitter); + } + let mut export = open_export(live.source().await).await.unwrap().with_delay(delay); + live.track.write(video_frame(pts(0), true)).unwrap(); + let mut most = 0; + for index in 1..500 { + tokio::time::advance(step).await; + live.track.write(video_frame(pts(index), false)).unwrap(); + poll_frames(&mut export); + most = most.max(export.buffered()); + } + // What arrived within the wait, the frame just read, and one due to go out. + let bound = ((lookahead + delay).as_millis() / step.as_millis()) as usize + 2; + assert!(most <= bound, "{name}: {most} frames buffered, over {bound}"); + } +} + /// The PID of the whole-PES private data track in the tests below. const PLACED_PID: u16 = 0x104; diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index fb8af010de2..288d2ef5a11 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -93,7 +93,7 @@ case "$1" in # only Criterion estimates enter the trend report. criterion extra --no-fail-fast -p moq-e2ee -p moq-json -p moq-archive -p moq-cli -p moq-mux \ --bench protect --bench codec --bench table --bench allocations --bench window \ - --bench producer --bench play_buffer --bench jitter + --bench producer --bench play_buffer --bench jitter --bench decode_clock ;; rust-uring) # One run per target makes an unsupported/skipped ring fail report