From 9af40f95ace08f7e6a262682769064707a038a72 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 17:56:05 -0700 Subject: [PATCH 1/9] quest: claim m1/ts-reliability/recovery From f7cf7a5a8e75e62d063cf2d7ec94b2e20a49a84a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 18:57:50 -0700 Subject: [PATCH 2/9] test(ts): grade recovery from missing MoQ content against a delivery ledger Add recovery_test.rs: each arm imports an FFmpeg-generated H.264/AAC clip, republishes its groups on a paused clock under a fault schedule (a group skipped or cut, the video or a track stalled, a join, A/V skew, the PTS wrap, open GOP), and grades the export against the ledger of what was delivered and when: nothing undelivered or late goes out, every surviving dependency chain does, every loss is counted, and FFmpeg decodes the output from each acquisition point as the continuous source decode's pictures. Negative controls exercise the graders' failure paths. The arms found two export bugs, fixed here: - A group's end now settles the DTS of the pictures the decode clock holds, so a hole after it no longer makes its tail late. - A group cut short upstream now says so (consumer Event::Cut) as soon as it is evicted, so what it delivered goes out instead of waiting on the next group and being dropped. A whole-path stall still lets held frames out late, because an export nothing wakes judges arrival on when it last ran; that arm is ignored pending a decision. The raw lane asserts its source's skip count for the skipped, cut and late arms. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/consumer.rs | 24 +- rs/moq-mux/src/container/fmp4/export.rs | 2 + rs/moq-mux/src/container/source.rs | 8 +- rs/moq-mux/src/container/ts/export.rs | 20 +- rs/moq-mux/src/container/ts/export_test.rs | 4 +- rs/moq-mux/src/container/ts/mod.rs | 2 + rs/moq-mux/src/container/ts/recovery_test.rs | 1126 ++++++++++++++++++ rs/moq-mux/src/container/ts/release.rs | 8 + rs/moq-mux/src/container/ts/release_test.rs | 3 + test/ts/README.md | 12 +- 10 files changed, 1195 insertions(+), 14 deletions(-) create mode 100644 rs/moq-mux/src/container/ts/recovery_test.rs diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 150766f8345..382ea46fc12 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -5,7 +5,7 @@ use moq_net::Timestamp; use super::{Container, Frame, Read, TimestampRewind}; -/// Media and clean group boundaries in delivery order. +/// Media and group boundaries in delivery order. pub(crate) enum Event { Frame(Frame), /// An endpoint marker. `opens` when no media precedes it in its group: a declared stop @@ -14,7 +14,11 @@ pub(crate) enum Event { end: Timestamp, opens: bool, }, + /// The group finished cleanly. GroupEnd, + /// The group was lost upstream partway, so nothing more of it arrives. The gap it leaves is + /// read once the next group resumes the track. + Cut, } /// Read items and exporter-only boundaries in delivery order. @@ -22,6 +26,7 @@ pub(crate) enum Item { Read(Read), FrameEnd(Timestamp), GroupEnd, + Cut, } /// Decode a moq-lite track into a stream of media [`Frame`]s in age-bounded @@ -253,7 +258,7 @@ impl Consumer { loop { match ready!(self.poll_item(waiter))? { Some(Item::Read(read)) => return Poll::Ready(Ok(Some(read))), - Some(Item::FrameEnd(_) | Item::GroupEnd) => continue, + Some(Item::FrameEnd(_) | Item::GroupEnd | Item::Cut) => continue, None => return Poll::Ready(Ok(None)), } } @@ -274,6 +279,7 @@ impl Consumer { Some(Event::FrameEnd { end, opens: true }) => Read::End { group: self.group, end }, Some(Event::FrameEnd { end, opens: false }) => return Poll::Ready(Ok(Some(Item::FrameEnd(end)))), Some(Event::GroupEnd) => return Poll::Ready(Ok(Some(Item::GroupEnd))), + Some(Event::Cut) => return Poll::Ready(Ok(Some(Item::Cut))), None => return Poll::Ready(Ok(None)), }; let resume = match &read { @@ -369,7 +375,7 @@ impl Consumer { group.sequence > current && group.buffered.is_empty() && group.fault.is_none() && group.poll_aborted(waiter) })?; - 'read: loop { + loop { if self.pending.front().is_none_or(|group| group.sequence <= self.current) { self.hole = None; self.hole_timer = None; @@ -434,7 +440,9 @@ impl Consumer { let current = self.current; self.discard(|group| group.sequence <= current)?; self.current = self.pending.front().map_or(self.current + 1, |g| g.sequence); - continue 'read; + // Said now rather than with the gap, which waits for the next group: what + // the caller holds of this one need not wait for a successor. + return Poll::Ready(Ok(Some(Event::Cut))); } // Cleanly finished group: advance to the next sequence. Poll::Ready(Ok(None)) => { @@ -2410,8 +2418,12 @@ mod tests { let waiter = kio::Waiter::noop(); - // The aborted group 0 skips straight to its successor; a short clean group emits - // GroupEnd. + // The aborted group 0 says it was cut, then skips straight to its successor; a short + // clean group emits GroupEnd. + assert!( + matches!(consumer.poll_event(&waiter), Poll::Ready(Ok(Some(Event::Cut)))), + "the cut is said before the successor" + ); let event = consumer.poll_event(&waiter); let Poll::Ready(Ok(Some(Event::Frame(frame)))) = event else { panic!( diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index f7356f20438..df03588ebcf 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -580,6 +580,8 @@ impl Export { } } } + // The fragment ends at the gap the next group reads. + Poll::Ready(Some(Event::Cut)) => {} Poll::Ready(Some(Event::GroupEnd)) => { if track.buffer.is_empty() { continue; diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 0f57a1dd450..06b93be5717 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -351,7 +351,7 @@ impl ExportSource { loop { match ready!(self.poll_event(waiter))? { Some(Event::Frame(frame)) => return Poll::Ready(Ok(Some(frame))), - Some(Event::GroupEnd | Event::FrameEnd { .. }) => continue, + Some(Event::GroupEnd | Event::FrameEnd { .. } | Event::Cut) => continue, None => return Poll::Ready(Ok(None)), } } @@ -436,6 +436,12 @@ impl ExportSource { self.open = false; return Poll::Ready(Ok(Some(Event::GroupEnd))); } + // The skip is counted with the gap the next group reads. + Some(Item::Cut) => { + self.finish_ends(); + self.cuts += u64::from(std::mem::take(&mut self.open)); + return Poll::Ready(Ok(Some(Event::Cut))); + } None => { self.finish_ends(); return Poll::Ready(Ok(None)); diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 7ad13c6f47a..a3059655bce 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -46,6 +46,7 @@ use crate::catalog::hang::Catalog; use crate::codec::annexb::InBandCodec; use crate::codec::video::{Hrd, Reorder}; use crate::codec::{aac, annexb, opus}; +use crate::container::consumer::Event; use crate::container::decode::{DecodeClock, MAX_REORDER}; use crate::container::{ExportSource, Frame}; use crate::jitter::{self, Arrival, Push}; @@ -274,6 +275,12 @@ struct Track { /// The source's cut count at the last frame queued. A cut since clears what a track that /// has not started holds. held: Option, + /// The source ended a group, cleanly or cut. The next group presents after every picture + /// of this one, its leading pictures included, so the end settles each DTS the clock holds: + /// waiting on the next group instead would make what this one delivered late behind a hole + /// upstream. Settled once the source has nothing more right now, so a track's end, read + /// with its last group's, still takes the end of the stream's settlement. + settle: bool, /// The PTS of the last keyframe queued, which leading pictures present before. keyframe: Option, /// What the catalog and the SPS declare about the video's reordering. @@ -313,6 +320,7 @@ impl Track { recovery_gap: None, leading_floor: None, held: None, + settle: false, keyframe: None, timing: Timing::default(), buffer: None, @@ -370,8 +378,11 @@ impl Track { if !input.waiting_for_header && input.jitter.full(&self.pid) { break; } - match self.source.poll_read(waiter)? { - Poll::Ready(Some(frame)) => { + match self.source.poll_event(waiter)? { + Poll::Ready(Some(Event::Frame(frame))) => { + if std::mem::take(&mut self.settle) && !input.waiting_for_header { + self.release(name, input.jitter, true)?; + } if input.waiting_for_header && !self.source.header_ready() { *input.trimmed += 1; continue; @@ -398,11 +409,16 @@ impl Track { } self.queue(name, pending, input.jitter)?; } + Poll::Ready(Some(Event::GroupEnd | Event::Cut)) => self.settle = true, + Poll::Ready(Some(Event::FrameEnd { .. })) => {} Poll::Ready(None) => { self.finished = true; break; } Poll::Pending => { + if std::mem::take(&mut self.settle) && !input.waiting_for_header { + self.release(name, input.jitter, true)?; + } self.empty = moq_clock::now(); break; } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 83f5a3659a7..a1810eb0fcf 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -3238,7 +3238,7 @@ fn count_pid(frames: &[Frame], pid: u16) -> usize { } /// Count the TS packets whose adaptation field sets `discontinuity_indicator`. -fn count_discontinuity(frames: &[Frame]) -> usize { +pub(super) fn count_discontinuity(frames: &[Frame]) -> usize { frames .iter() .flat_map(|f| f.payload.as_chunks::<188>().0.iter()) @@ -5265,7 +5265,7 @@ fn in_decode_order(decode: &[u64]) -> bool { /// Every PES unit, each PID in its own decode order, finishes arriving before it decodes, on /// the clock a receiver recovers from the PCRs. Returns how many units were judged. -fn assert_on_time(frames: &[Frame]) -> usize { +pub(super) fn assert_on_time(frames: &[Frame]) -> usize { const WRAP: i128 = 1 << 33; let ts: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); // PCRs unwrapped from the first, which a start a window before the timeline may wrap. diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index e7e1fe9d533..78e1df6ae0c 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -67,4 +67,6 @@ mod passthrough_test; #[cfg(test)] mod pes_packet_test; #[cfg(test)] +mod recovery_test; +#[cfg(test)] mod release_test; diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs new file mode 100644 index 00000000000..8ab2be5fb38 --- /dev/null +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -0,0 +1,1126 @@ +//! Recovery from MoQ content that never arrives, or arrives too late, after the output began. +//! +//! Each arm imports a clip FFmpeg generates in a declared software profile, records every group +//! and frame the import publishes, and republishes them on a paused clock under a fault schedule +//! at the MoQ content boundary: a group omitted or cut short, frames held past their deadline, a +//! track stalled. Loss here is content the path never delivers, not IP packets QUIC would +//! retransmit. The schedule is the ledger of what was delivered and when, and the export of the +//! republished broadcast is graded against it rather than against what the export reports: +//! +//! - Nothing goes out that was not delivered, or that the schedule made late. Every picture that +//! does keeps its dependency chain: a group's pictures are a decode-order prefix from its +//! keyframe, and its leading pictures go out only after the group before went out whole. +//! - What the schedule left alone goes out. A track the fault missed keeps running, and each +//! track resumes at the first group whose chain survives. +//! - Every delivered frame missing from the output is counted: `trimmed` before its track's +//! first frame went out, `dropped` after, and each skip upstream is one of `gaps`. +//! - Every unit arrives before it decodes, and the continuity counters run unbroken. +//! - FFmpeg decodes the output without an error from each point a receiver acquires at, and +//! presents every picture that went out, once, as the continuous source decode's picture at +//! the same presentation time. +//! +//! The raw lane's equivalents are in [`release_test`](super::release_test). + +use std::time::Duration; + +use mpeg2ts::pes::{PesPacketReader, ReadPesPacket}; +use mpeg2ts::ts::TsPacketReader; +use tokio::time::Instant; + +use super::export_test::{assert_on_time, count_discontinuity, open_export}; +use super::stats::{self, Class}; +use crate::catalog::hang::Catalog; +use crate::container::Frame; + +/// 25 fps. +const FRAME: Duration = Duration::from_millis(40); +/// AAC-LC at 48 kHz: 1024 samples, exactly 1920 ticks of the 90 kHz clock. +const AUDIO_TICKS: u64 = 1920; +const WRAP: u64 = 1 << 33; + +/// How long the export holds each frame past its decode time: `export ts --delay`'s default. +const DELAY: Duration = Duration::from_secs(1); +/// How long after its nominal time a frame arrives on a clean path, about a frame's transit. +const LAG: Duration = Duration::from_millis(40); +/// How far a frame's lateness may sit from a bound before the ledger calls it either way: the +/// export deadlines each picture on the decode time it derives, a reorder depth before it +/// presents, and anchors its clock on the track sent latest. +const MARGIN: Duration = Duration::from_millis(160); +/// How long a drain waits for the next frame before taking the export to have stalled. +const STALL: Duration = Duration::from_secs(10); + +/// A software profile FFmpeg generates: 6 s of 320x240 H.264 High at 25 fps, a group every +/// twelve pictures, beside AAC-LC. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Profile { + /// Closed GOP with B pictures, the profile `test/ts/decode.py` grades. + Closed, + /// The same, starting 2.7 s before the 33-bit PTS and PCR wrap. + Wrap, + /// Open GOP: after the first IDR every group opens on a recovery point, behind three + /// leading pictures that reference the group before. + /// + /// Resuming at a recovery point after a hole leaves a `frame_num` gap, which H.264 takes + /// for unintentional loss (8.2.5.2), so a decoder that ran through the hole conceals around + /// it. A receiver acquiring at the recovery point decodes it cleanly. + Open, +} + +impl Profile { + /// The clip, generated once per process. + fn clip(self) -> &'static [u8] { + static CLIPS: [std::sync::OnceLock>; 3] = [const { std::sync::OnceLock::new() }; 3]; + CLIPS[self as usize].get_or_init(|| { + let x264 = match self { + Self::Closed | Self::Wrap => "keyint=12:min-keyint=12:scenecut=0", + Self::Open => "keyint=12:min-keyint=12:scenecut=0:open-gop=1:bframes=3:b-adapt=0", + }; + let mut args = vec![ + "-nostdin", + "-hide_banner", + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc=size=320x240:rate=25", + "-f", + "lavfi", + "-i", + "sine=frequency=1000:sample_rate=48000", + "-t", + "6", + "-c:v", + "libx264", + "-profile:v", + "high", + "-preset", + "veryfast", + "-threads", + "1", + "-pix_fmt", + "yuv420p", + "-x264-params", + x264, + "-b:v", + "400k", + "-c:a", + "aac", + "-b:a", + "96k", + ]; + if self == Self::Wrap { + // 2^33 ticks is 95,443.7 s. + args.extend(["-output_ts_offset", "95441"]); + } + args.extend(["-pcr_period", "20", "-f", "mpegts", "pipe:1"]); + let output = std::process::Command::new("ffmpeg") + .args(&args) + .output() + .expect("ffmpeg is in the dev shell"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success() && stderr.is_empty(), "ffmpeg: {stderr}"); + output.stdout + }) + } +} + +/// One track as the import published it: each group's sequence and frames, in decode order. +struct Recorded { + name: String, + video: bool, + groups: Vec<(u64, Vec)>, +} + +/// What the import of a clip published. +struct Recording { + catalog: Catalog, + tracks: Vec, +} + +/// Import `ts` whole and read back every frame of every track. +async fn record(ts: &[u8]) -> Recording { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let consumer = broadcast.consume(); + let mut import = super::Import::new(broadcast.clone(), catalog.reserve()); + import.decode(ts).unwrap(); + import.finish().unwrap(); + let snapshot = catalog.snapshot(); + + let names = (snapshot.video.renditions.keys().map(|name| (name.clone(), true))) + .chain(snapshot.audio.renditions.keys().map(|name| (name.clone(), false))); + let mut tracks = Vec::new(); + for (name, video) in names { + let mut track = consumer + .track(&name) + .unwrap() + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(30))) + .await + .unwrap(); + let mut groups = Vec::new(); + while let Some(mut group) = tokio::time::timeout(Duration::from_secs(1), track.recv_group()) + .await + .unwrap() + .unwrap() + { + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await.unwrap() { + frames.push(frame); + } + groups.push((group.sequence, frames)); + } + tracks.push(Recorded { name, video, groups }); + } + assert_eq!( + tracks.iter().map(|track| track.video).collect::>(), + [true, false], + "the clip publishes a video and an audio track" + ); + Recording { + catalog: snapshot, + tracks, + } +} + +/// The presentation time of a published frame, in microseconds on the catalog clock. +fn micros(frame: &moq_net::frame::Frame) -> u64 { + hang::container::Frame::decode(frame.payload.clone()) + .unwrap() + .timestamp + .as_micros() as u64 +} + +/// The 90 kHz presentation time a microsecond timestamp goes out at, wrapped. +fn ticks(micros: u64) -> u64 { + (micros * 9).div_ceil(100) % WRAP +} + +/// A frame's place in a recording, and when a clean path delivers it after the first frame. +#[derive(Clone, Copy, Debug)] +struct Slot { + video: bool, + /// Index into the track's recorded groups. + group: usize, + /// Index into the group's frames. + frame: usize, + nominal: Duration, +} + +/// Where a schedule puts a frame. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Delivery { + At(Duration), + Omitted, +} + +/// A fault schedule: where each frame goes, from its slot. A group delivers a prefix of its +/// frames; one that loses the rest is evicted once its next frame would have arrived. +type Schedule = fn(Slot) -> Delivery; + +fn clean(slot: Slot) -> Delivery { + Delivery::At(slot.nominal) +} + +/// What the schedule decided for one recorded frame, and what the export made of it. +#[derive(Clone, Debug)] +struct Unit { + group: usize, + /// 90 kHz presentation time, wrapped as the output carries it. + pts: u64, + /// Presented before its group's keyframe, so it references the group before. + leading: bool, + delivered: bool, + /// Whether it arrived inside both the source's age budget and its deadline, `None` when + /// too close to call. + on_time: Option, + output: bool, +} + +/// One track's ledger, in decode order. +#[derive(Clone, Debug)] +struct Ledger { + video: bool, + units: Vec, + /// Skips the schedule made upstream: each run of omitted groups, and each cut with the run + /// after it. + gaps: u64, +} + +/// What an export of a republished recording wrote. +struct Run { + ledgers: Vec, + frames: Vec, + stats: stats::Export, + /// The stats after each output frame, and when, from the republisher's start. + samples: Vec<(Duration, stats::Export)>, +} + +impl Run { + fn ts(&self) -> Vec { + self.frames + .iter() + .flat_map(|frame| frame.payload.iter().copied()) + .collect() + } + + fn ledger(&self, video: bool) -> &Ledger { + self.ledgers.iter().find(|ledger| ledger.video == video).unwrap() + } + + /// The PID of the output's audio or video stream. + fn pid(&self, class: Class) -> u16 { + let [pid] = self + .stats + .streams + .iter() + .filter(|(_, row)| row.class == class) + .map(|(pid, _)| *pid) + .collect::>()[..] + else { + panic!("one {class:?} row: {:?}", self.stats.streams); + }; + pid + } +} + +/// Republish `recording` under `schedule`, start an export `join` in, and match its output to +/// the ledger. +async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { + let origin = recording + .tracks + .iter() + .filter_map(|track| track.groups.first()) + .map(|(_, frames)| micros(&frames[0])) + .min() + .unwrap(); + + // Every frame's delivery, and each group's end: (at, track, group, frame or the end). + let mut events = Vec::new(); + let mut ledgers = Vec::new(); + for (t, track) in recording.tracks.iter().enumerate() { + let mut units = Vec::new(); + let mut gaps = 0; + let mut skipping = false; + // The group a subscription made at the join starts at: the newest begun by then. + let mut joined = 0; + for (g, (_, frames)) in track.groups.iter().enumerate() { + let keyframe = micros(&frames[0]); + let mut kept = 0; + for (f, frame) in frames.iter().enumerate() { + let at = micros(frame); + let nominal = match track.video { + // A group's pictures go out in decode order, a frame apart. + true => Duration::from_micros(keyframe - origin) + FRAME * f as u32, + false => Duration::from_micros(at - origin), + }; + let delivery = schedule(Slot { + video: track.video, + group: g, + frame: f, + nominal, + }); + let on_time = match delivery { + Delivery::At(arrival) => { + let late = arrival.saturating_sub(nominal) + LAG; + // The source skips a group older than half the delay behind the newest. The + // clock anchors on the video, which the export sends up to a delay ahead of + // its decode time, so audio, sent barely ahead, has nearly a delay more. + let slack = if track.video { DELAY } else { 2 * DELAY }; + match (late + MARGIN < DELAY / 2, late > slack + MARGIN) { + (true, _) => Some(true), + (_, true) => Some(false), + _ => None, + } + } + Delivery::Omitted => None, + }; + if let Delivery::At(arrival) = delivery { + assert_eq!(kept, f, "{} group {g} resumes after a hole", track.name); + if f == 0 && arrival + LAG <= join { + joined = g; + } + kept += 1; + events.push((arrival + LAG, t, g, Some(f))); + } + units.push(Unit { + group: g, + pts: ticks(at), + leading: track.video && at < keyframe, + delivered: delivery != Delivery::Omitted, + on_time, + output: false, + }); + } + let whole = kept == frames.len(); + if whole { + let last = events.last().map(|(at, ..)| *at).unwrap(); + events.push((last, t, g, None)); + } else if kept > 0 { + let next = Duration::from_micros(keyframe - origin) + FRAME * kept as u32; + events.push((next + LAG, t, g, None)); + } + if kept > 0 && skipping { + gaps += 1; + } + skipping = match kept { + 0 => skipping || units.iter().any(|unit| unit.delivered), + _ => !whole, + }; + } + assert!(!skipping, "{}: a schedule ends a track on a hole", track.name); + // What came before the join was never offered to the export. + for unit in units.iter_mut().filter(|unit| unit.group < joined) { + unit.delivered = false; + } + ledgers.push(Ledger { + video: track.video, + units, + gaps, + }); + } + // Stable, so a group's end follows its last frame. + events.sort_by_key(|&(at, ..)| at); + + // The republisher: the same catalog and tracks on a new broadcast, each frame at its time. + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let snapshot = recording.catalog.clone(); + catalog + .mutate(|catalog| { + catalog.video = snapshot.video; + catalog.audio = snapshot.audio; + }) + .unwrap(); + let producers: Vec<_> = recording + .tracks + .iter() + .map(|track| { + let priority = match track.video { + true => hang::catalog::PRIORITY.video, + false => hang::catalog::PRIORITY.audio, + }; + broadcast + .create_track(track.name.clone(), catalog.track_info(priority)) + .unwrap() + }) + .collect(); + let consumer = broadcast.consume(); + let groups: Vec)>> = + recording.tracks.iter().map(|track| track.groups.clone()).collect(); + let start = Instant::now(); + let publisher = tokio::spawn(async move { + let _keep = (&broadcast, &catalog); + let mut open: Vec> = producers.iter().map(|_| None).collect(); + let mut kept = vec![0; producers.len()]; + for (at, t, g, step) in events { + tokio::time::sleep_until(start + at).await; + let (sequence, frames) = &groups[t][g]; + let Some(f) = step else { + let mut group = open[t].take().unwrap(); + match kept[t] == frames.len() { + true => group.finish().unwrap(), + false => group.abort(moq_net::Error::Old).unwrap(), + } + continue; + }; + let group = open[t].get_or_insert_with(|| { + kept[t] = 0; + let info = moq_net::group::Info { sequence: *sequence }; + producers[t].create_group(info).unwrap() + }); + group + .write_frame(frames[f].timestamp, frames[f].payload.clone()) + .unwrap(); + kept[t] += 1; + } + assert!(open.iter().all(Option::is_none), "every group ends"); + for producer in &producers { + producer.finish().unwrap(); + } + std::future::pending::<()>().await; + }); + + tokio::time::sleep_until(start + join).await; + let source = crate::source::announced(&consumer).await; + let mut export = open_export(source).await.unwrap().with_delay(DELAY); + let mut frames = Vec::new(); + let mut samples = Vec::new(); + while let Ok(next) = tokio::time::timeout(STALL, export.next()).await { + let Some(frame) = next.expect("export error") else { + break; + }; + frames.push(frame); + samples.push((start.elapsed(), export.stats())); + } + publisher.abort(); + let mut run = Run { + ledgers, + frames, + stats: export.stats(), + samples, + }; + + // Match each PES, and each ADTS frame of an audio PES, to the unit it carries. + let mut reader = PesPacketReader::new(TsPacketReader::new(std::io::Cursor::new(run.ts()))); + while let Some(pes) = reader.read_pes_packet().unwrap() { + let video = pes.header.stream_id.is_video(); + assert!(video || pes.header.stream_id.is_audio(), "only audio and video go out"); + let pts = pes.header.pts.expect("every PES carries a PTS").as_u64(); + let count = match video { + true => 1, + false => adts_frames(&pes.data), + }; + let ledger = run.ledgers.iter_mut().find(|ledger| ledger.video == video).unwrap(); + for k in 0..count as u64 { + let pts = (pts + k * AUDIO_TICKS) % WRAP; + let matched: Vec = (0..ledger.units.len()) + .filter(|&i| ledger.units[i].pts.abs_diff(pts) <= 1) + .collect(); + let [i] = matched[..] else { + panic!("output PTS {pts} matches units {matched:?} of the video={video} ledger"); + }; + assert!(!ledger.units[i].output, "unit {i} went out twice"); + ledger.units[i].output = true; + } + } + run +} + +/// How many ADTS frames an AAC PES carries. +fn adts_frames(mut data: &[u8]) -> usize { + let mut count = 0; + while data.len() > 7 { + assert!(data[0] == 0xff && data[1] & 0xf0 == 0xf0, "an ADTS sync word"); + let len = usize::from(data[3] & 3) << 11 | usize::from(data[4]) << 3 | usize::from(data[5] >> 5); + data = &data[len.clamp(7, data.len())..]; + count += 1; + } + count +} + +/// Whether each unit's chain survived the schedule, so it must go out, failing on a picture that +/// went out without its own. +fn chain(ledger: &Ledger) -> Result, String> { + let units = &ledger.units; + let mut chained = vec![false; units.len()]; + if !ledger.video { + for (i, unit) in units.iter().enumerate() { + chained[i] = unit.delivered && unit.on_time == Some(true); + } + return Ok(chained); + } + // Whether every picture of the group before went out, but the orphans it left out. + let mut whole = false; + let mut i = 0; + while i < units.len() { + let group = units[i].group; + let (mut alive, mut broken) = (true, false); + let mut complete = true; + while i < units.len() && units[i].group == group { + let unit = &units[i]; + if unit.leading && !whole { + if unit.output { + return Err(format!("video unit {i} went out without the group before it: {unit:?}")); + } + i += 1; + continue; + } + alive &= unit.delivered && unit.on_time == Some(true); + chained[i] = alive; + match unit.output { + true if broken => return Err(format!("video unit {i} went out after its chain broke: {unit:?}")), + true => {} + false => (broken, complete) = (true, false), + } + i += 1; + } + whole = complete; + } + Ok(chained) +} + +/// Grade an export against its ledger and the counters it reported. +fn grade(ledgers: &[Ledger], stats: &stats::Export) -> Result<(), String> { + let video = ledgers.iter().find(|ledger| ledger.video).unwrap(); + let Some(start) = video.units.iter().find(|unit| unit.output) else { + return Err("no video went out".into()); + }; + let keyframe = video.units.iter().find(|unit| unit.group == start.group).unwrap(); + if keyframe.pts != start.pts { + return Err(format!("the video starts mid-group, at {start:?}")); + } + // Missing frames before and after their track's first went out, those the schedule made late, + // and the skips it made. + let (mut before, mut after, mut late, mut gaps) = (0u64, 0u64, 0u64, 0u64); + for ledger in ledgers { + let name = if ledger.video { "video" } else { "audio" }; + let units = &ledger.units; + for (i, unit) in units.iter().enumerate() { + if unit.output && (!unit.delivered || unit.on_time == Some(false)) { + return Err(format!("{name} unit {i} went out, though {unit:?}")); + } + } + let chained = chain(ledger)?; + + // The export starts on the video's first group, with the audio presented from its + // keyframe on; audio straddling the keyframe may be trimmed with what precedes it. + for (i, unit) in units.iter().enumerate() { + let started = match ledger.video { + true => unit.group >= start.group, + false => (AUDIO_TICKS..WRAP / 2).contains(&((unit.pts + WRAP - keyframe.pts) % WRAP)), + }; + if started && chained[i] && !unit.output { + return Err(format!( + "{name} unit {i} survived the schedule but did not go out: {unit:?}" + )); + } + } + + let first = units.iter().position(|unit| unit.output).unwrap_or(units.len()); + for (i, unit) in units.iter().enumerate() { + if unit.delivered && !unit.output { + *if i < first { &mut before } else { &mut after } += 1; + late += u64::from(unit.on_time != Some(true)); + } + } + gaps += ledger.gaps; + } + + // The source skips frames that came older than its age budget, each run one more gap, and + // they are neither trimmed nor dropped. + let skips = stats + .gaps + .checked_sub(gaps) + .ok_or(format!("{} gaps reported for the {gaps} the schedule made", stats.gaps))?; + let counted = stats.dropped + stats.trimmed; + let skipped = (before + after).checked_sub(counted).ok_or(format!( + "{counted} frames counted dropped or trimmed, but only {} are missing", + before + after + ))?; + let exact = (stats.dropped, stats.trimmed) == (after, before); + if !(exact || (skipped > 0 && stats.dropped <= after && stats.trimmed <= before)) + || (skips > 0) != (skipped > 0) + || skips > skipped + || skipped > late + { + return Err(format!( + "(dropped, trimmed, gaps) reported {:?}; {before} missing before each track started, \ + {after} after, {late} of them late, and {gaps} gaps scheduled", + (stats.dropped, stats.trimmed, stats.gaps) + )); + } + Ok(()) +} + +/// Every PID's continuity counter steps by one on each packet with a payload. +fn assert_continuous(ts: &[u8]) { + let mut last = std::collections::HashMap::new(); + for (at, packet) in ts.as_chunks::<188>().0.iter().enumerate() { + let pid = u16::from_be_bytes([packet[1] & 0x1f, packet[2]]); + if pid == 0x1fff || packet[3] & 0x10 == 0 { + continue; + } + let counter = packet[3] & 0x0f; + if let Some(previous) = last.insert(pid, counter) { + assert_eq!( + counter, + (previous + 1) & 0x0f, + "PID {pid:#x} skips its counter at packet {at}" + ); + } + } +} + +/// What FFmpeg decoded: each picture's 90 kHz presentation time and hash, and the audio's +/// sample count. +struct Decoded { + pictures: Vec<(u64, String)>, + samples: u64, +} + +/// Decode `ts` with software FFmpeg, its audio too unless `video_only`, failing on any error it +/// logs. +fn decode(ts: &[u8], video_only: bool) -> Result { + use std::io::Write as _; + let mut args = vec![ + "-nostdin", + "-hide_banner", + "-v", + "error", + "-xerror", + "-err_detect", + "explode", + "-copyts", + "-threads", + "1", + "-i", + "pipe:0", + "-map", + "0:v:0", + "-c:v", + "rawvideo", + "-fps_mode", + "passthrough", + "-enc_time_base:v", + "1:90000", + ]; + if !video_only { + args.extend(["-map", "0:a:0", "-c:a", "pcm_s16le"]); + } + args.extend(["-f", "framemd5", "pipe:1"]); + let mut child = std::process::Command::new("ffmpeg") + .args(&args) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("ffmpeg is in the dev shell"); + let mut stdin = child.stdin.take().unwrap(); + let input = ts.to_vec(); + let feed = std::thread::spawn(move || stdin.write_all(&input)); + let output = child.wait_with_output().unwrap(); + // FFmpeg may stop reading at its first error, which the status reports. + let _ = feed.join().unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + if !output.status.success() || !stderr.is_empty() { + return Err(format!("FFmpeg failed ({}): {stderr}", output.status)); + } + let mut decoded = Decoded { + pictures: Vec::new(), + samples: 0, + }; + for line in String::from_utf8(output.stdout).unwrap().lines() { + if line.starts_with("#tb 0:") { + assert_eq!(line, "#tb 0: 1/90000", "pictures are stamped at 90 kHz"); + } + if line.starts_with('#') || line.is_empty() { + continue; + } + let fields: Vec<&str> = line.split(',').map(str::trim).collect(); + let [index, _dts, pts, duration, _size, hash] = fields[..] else { + panic!("framemd5 line {line}"); + }; + match index { + "0" => { + let pts: i64 = pts.parse().unwrap(); + decoded + .pictures + .push((pts.rem_euclid(WRAP as i64) as u64, hash.to_string())); + } + _ => decoded.samples += duration.parse::().unwrap(), + } + } + if decoded.pictures.is_empty() || (!video_only && decoded.samples == 0) { + return Err("FFmpeg decoded no pictures or no audio".into()); + } + Ok(decoded) +} + +/// The continuous source decode's pictures, by presentation time on the catalog clock. +struct Reference(std::collections::HashMap); + +impl Reference { + fn new(profile: Profile, video: &Ledger) -> Self { + let source = decode(profile.clip(), false).unwrap(); + // The import moved the source onto the catalog clock; the first pictures give the shift. + let earliest = |pts: &mut dyn Iterator| pts.min_by_key(|pts| (pts + WRAP / 2) % WRAP).unwrap(); + let shift = earliest(&mut source.pictures.iter().map(|(pts, _)| *pts)) + WRAP + - earliest(&mut video.units.iter().map(|unit| unit.pts)); + let pictures = source.pictures.into_iter(); + Self( + pictures + .map(|(pts, hash)| ((pts + WRAP - shift % WRAP) % WRAP, hash)) + .collect(), + ) + } + + /// FFmpeg presented each `emitted` picture exactly once, as the source's, and nothing else. + fn compare(&self, decoded: &Decoded, emitted: &[&Unit]) -> Result<(), String> { + let mut presented = vec![false; emitted.len()]; + for (pts, hash) in &decoded.pictures { + let Some(i) = emitted.iter().position(|unit| unit.pts.abs_diff(*pts) <= 1) else { + return Err(format!("FFmpeg presented a picture at {pts} that never went out")); + }; + if std::mem::replace(&mut presented[i], true) { + return Err(format!("FFmpeg presented the picture at {pts} twice")); + } + let source = (pts.saturating_sub(1)..=pts + 1).find_map(|pts| self.0.get(&pts)); + if source != Some(hash) { + return Err(format!("the picture at {pts} differs from the source's")); + } + } + match presented.iter().filter(|presented| !**presented).count() { + 0 => Ok(()), + withheld => Err(format!( + "FFmpeg withheld {withheld} of the {} pictures that went out", + emitted.len() + )), + } + } +} + +/// The video from each point a receiver acquires at, as a fresh decoder's input: its first +/// keyframe out, and each one it resumes at after a group did not go out whole. Each segment is +/// the output's leading tables and its packets up to the next, without the audio, beside the +/// pictures it carries. +fn segments(run: &Run) -> Vec<(Vec, Vec<&Unit>)> { + let ts = run.ts(); + let packets = ts.as_chunks::<188>().0; + let (video, audio) = (run.pid(Class::Video), run.pid(Class::Audio)); + let pid = |packet: &[u8; 188]| u16::from_be_bytes([packet[1] & 0x1f, packet[2]]); + // The packet each picture's PES starts at, by its PTS. + let starts: Vec<(usize, u64)> = (packets.iter().enumerate()) + .filter(|(_, packet)| pid(packet) == video && packet[1] & 0x40 != 0) + .map(|(at, packet)| { + let adaptation = match packet[3] & 0x20 { + 0 => 0, + _ => usize::from(packet[4]) + 1, + }; + let pes = &packet[4 + adaptation..]; + let pts = u64::from((pes[9] >> 1) & 7) << 30 + | u64::from(pes[10]) << 22 + | u64::from(pes[11] >> 1) << 15 + | u64::from(pes[12]) << 7 + | u64::from(pes[13] >> 1); + (at, pts) + }) + .collect(); + + // Where each segment starts, and its pictures. + let mut cuts: Vec<(usize, Vec<&Unit>)> = Vec::new(); + let mut whole = false; + for group in run.ledger(true).units.chunk_by(|a, b| a.group == b.group) { + let emitted: Vec<&Unit> = group.iter().filter(|unit| unit.output).collect(); + match emitted.first() { + Some(first) if !whole => { + let (at, _) = starts.iter().find(|(_, pts)| pts.abs_diff(first.pts) <= 1).unwrap(); + cuts.push((*at, emitted)); + } + _ => cuts + .last_mut() + .into_iter() + .for_each(|(_, units)| units.extend(&emitted)), + } + // The group before must have gone out whole for its leading pictures to. + whole = group.iter().all(|unit| unit.output || (unit.leading && !whole)); + } + + let tables = packets[..cuts[0].0].iter().filter(|packet| pid(packet) != audio); + let tables: Vec = tables.flatten().copied().collect(); + let ends: Vec = cuts.iter().skip(1).map(|(at, _)| *at).chain([packets.len()]).collect(); + (cuts.into_iter().zip(ends)) + .map(|((at, units), end)| { + let mut bytes = tables.clone(); + bytes.extend(packets[at..end].iter().filter(|packet| pid(packet) != audio).flatten()); + (bytes, units) + }) + .collect() +} + +/// Republish `profile`'s clip under `schedule`, export it from `join` on, and grade the export +/// and its decode: from each point a receiver acquires at, and as a whole when the profile's +/// resumptions are decodable straight through. +async fn recover(profile: Profile, schedule: Schedule, join: Duration) -> Run { + let recording = record(profile.clip()).await; + let run = run(&recording, schedule, join).await; + grade(&run.ledgers, &run.stats).unwrap(); + assert!(assert_on_time(&run.frames) > 200, "too little output to judge"); + assert_continuous(&run.ts()); + assert_eq!(count_discontinuity(&run.frames), 0, "a loss is not a new program clock"); + + let reference = Reference::new(profile, run.ledger(true)); + for (i, (ts, emitted)) in segments(&run).iter().enumerate() { + let output = decode(ts, true).unwrap_or_else(|err| panic!("segment {i}: {err}")); + (reference.compare(&output, emitted)).unwrap_or_else(|err| panic!("segment {i}: {err}")); + } + if profile != Profile::Open { + let output = decode(&run.ts(), false).unwrap(); + let emitted: Vec<&Unit> = run.ledger(true).units.iter().filter(|unit| unit.output).collect(); + reference.compare(&output, &emitted).unwrap(); + let frames = out(run.ledger(false)) as u64; + assert_eq!(output.samples, frames * 1024, "every audio frame that went out decodes"); + } + run +} + +/// How many of a ledger's units went out. +fn out(ledger: &Ledger) -> usize { + ledger.units.iter().filter(|unit| unit.output).count() +} + +#[tokio::test(start_paused = true)] +async fn a_clean_path_loses_nothing() { + let run = recover(Profile::Closed, clean, Duration::ZERO).await; + let video = run.ledger(true); + assert_eq!(out(video), video.units.len(), "every picture goes out"); + assert_eq!((run.stats.dropped, run.stats.gaps), (0, 0)); +} + +/// A group the path never delivers is skipped whole: the group before keeps its tail, though +/// nothing after it settles their decode times until the hole is past, and the export resumes +/// on the next keyframe. +#[tokio::test(start_paused = true)] +async fn a_skipped_group_resumes_at_the_next() { + let run = recover( + Profile::Closed, + |slot| match slot.video && slot.group == 5 { + true => Delivery::Omitted, + false => clean(slot), + }, + Duration::ZERO, + ) + .await; + assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); +} + +/// A group cut short keeps the pictures it delivered, though they are fewer than it takes to +/// settle their decode times, and loses only its tail. +#[tokio::test(start_paused = true)] +async fn a_cut_group_keeps_what_it_delivered() { + let run = recover( + Profile::Closed, + |slot| match slot.video && slot.group == 5 && slot.frame >= 4 { + true => Delivery::Omitted, + false => clean(slot), + }, + Duration::ZERO, + ) + .await; + let video = run.ledger(true); + let kept = video.units.iter().filter(|unit| unit.group == 5 && unit.output).count(); + assert_eq!(kept, 4, "the cut group's delivered pictures go out"); + assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); +} + +/// The video stalls for 2.5 s while the audio runs on, then delivers what it held at once: the +/// source skips the groups it held past the age budget as one gap, nothing late goes out, and the +/// video resumes on the first keyframe delivered in time, on the same clock. +#[tokio::test(start_paused = true)] +async fn a_stall_drops_what_it_made_late() { + let run = recover(Profile::Closed, |slot| hold(slot.video, slot), Duration::ZERO).await; + let (video, audio) = (run.ledger(true), run.ledger(false)); + assert!( + video.units.iter().any(|unit| unit.on_time == Some(false)), + "the schedule made pictures late" + ); + assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); + let first = audio.units.iter().position(|unit| unit.output).unwrap(); + assert!( + audio.units[first..].iter().all(|unit| unit.output), + "the audio runs on through the stall" + ); +} + +/// The whole path stalls, so nothing wakes the export until it delivers what it held. The export +/// judges a frame as arriving when its source was last found empty, which is when the export last +/// ran, so what the stall held behind the deadline still goes out, late. +#[tokio::test(start_paused = true)] +#[ignore = "an export nothing wakes judges a frame on when it last ran (quest/m1/ts-reliability/recovery.md)"] +async fn a_path_stall_drops_what_it_made_late() { + recover(Profile::Closed, |slot| hold(true, slot), Duration::ZERO).await; +} + +/// Hold what is `held` and due in [1.92 s, 4.4 s) until the end, delivering the rest cleanly. +fn hold(held: bool, slot: Slot) -> Delivery { + let (from, until) = (Duration::from_millis(1920), Duration::from_millis(4400)); + match held && (from..until).contains(&slot.nominal) { + true => Delivery::At(until), + false => clean(slot), + } +} + +/// The audio stalls for 1.5 s while the video runs on: every picture still goes out, the +/// audio's row stops counting and its quiet grows on the output's clock, and the audio resumes. +#[tokio::test(start_paused = true)] +async fn a_stalled_track_leaves_the_others_running() { + const FROM: Duration = Duration::from_millis(2000); + const UNTIL: Duration = Duration::from_millis(3500); + let run = recover( + Profile::Closed, + |slot| match !slot.video && (FROM..UNTIL).contains(&slot.nominal) { + true => Delivery::Omitted, + false => clean(slot), + }, + Duration::ZERO, + ) + .await; + let video = run.ledger(true); + assert_eq!(out(video), video.units.len(), "every picture goes out"); + assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); + + let (audio, pictures) = (run.pid(Class::Audio), run.pid(Class::Video)); + // The longest stretch of output the audio row sat still through. + let during = (run + .samples + .chunk_by(|(_, a), (_, b)| a.streams[&audio].units == b.streams[&audio].units)) + .max_by_key(|run| run.last().unwrap().0 - run[0].0) + .unwrap(); + let ((from, first), (until, last)) = (&during[0], during.last().unwrap()); + assert!( + *until - *from > Duration::from_millis(1200), + "the audio row stops counting, for only {:?}", + *until - *from + ); + assert!( + last.streams[&pictures].units > first.streams[&pictures].units, + "the video row keeps counting" + ); + let quiet = last.streams[&audio].quiet.expect("the output carries a PCR"); + assert!(quiet > Duration::from_millis(1200), "audio quiet for only {quiet:?}"); + assert!( + run.stats.streams[&audio].units > last.streams[&audio].units, + "the audio resumes" + ); +} + +/// Audio sent 200 ms behind the video, within the delay, survives a skipped video group whole. +#[tokio::test(start_paused = true)] +async fn skewed_audio_survives_a_video_skip() { + let run = recover( + Profile::Closed, + |slot| match (slot.video, slot.group) { + (true, 5) => Delivery::Omitted, + (true, _) => clean(slot), + (false, _) => Delivery::At(slot.nominal + Duration::from_millis(200)), + }, + Duration::ZERO, + ) + .await; + assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); +} + +/// A skipped group around the 33-bit wrap keeps the clock, and the pictures either side decode. +#[tokio::test(start_paused = true)] +async fn a_skip_across_the_wrap_keeps_the_clock() { + let run = recover( + Profile::Wrap, + |slot| match slot.video && slot.group == 5 { + true => Delivery::Omitted, + false => clean(slot), + }, + Duration::ZERO, + ) + .await; + let video = run.ledger(true); + let pts: Vec = video + .units + .iter() + .filter(|unit| unit.output) + .map(|unit| unit.pts) + .collect(); + assert!( + pts.windows(2).any(|pair| pair[1] < pair[0] / 2), + "the output crosses the wrap" + ); + assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); +} + +/// Open GOP: a skipped group and a cut one each orphan the next group's leading pictures, which +/// are trimmed, while every group after a whole one keeps its own. +#[tokio::test(start_paused = true)] +async fn open_gop_trims_only_the_orphans() { + let run = recover( + Profile::Open, + |slot| match slot.video && (slot.group == 5 || (slot.group == 8 && slot.frame >= 6)) { + true => Delivery::Omitted, + false => clean(slot), + }, + Duration::ZERO, + ) + .await; + let video = run.ledger(true); + let leading = |group: usize| { + let units: Vec<&Unit> = video + .units + .iter() + .filter(|unit| unit.group == group && unit.leading) + .collect(); + assert!(!units.is_empty(), "group {group} has leading pictures"); + units.iter().filter(|unit| unit.output).count() + }; + assert_eq!((leading(6), leading(9)), (0, 0), "the orphans are trimmed"); + assert!(leading(7) > 0 && leading(10) > 0, "a continuous group keeps them"); + assert_eq!(run.stats.gaps, 2); +} + +/// A subscriber joining 2.3 s in starts on a whole group and loses nothing after it. +#[tokio::test(start_paused = true)] +async fn a_join_starts_on_a_whole_group() { + let run = recover(Profile::Closed, clean, Duration::from_millis(2300)).await; + let video = run.ledger(true); + let first = video.units.iter().find(|unit| unit.output).unwrap().group; + assert!(first >= 4, "the export starts at the live edge, group {first}"); + assert_eq!((run.stats.dropped, run.stats.gaps), (0, 0)); +} + +/// The graders fail what they should: a picture that went out after its chain broke, a group +/// that survived but did not go out, a loss the counters do not show, and a picture FFmpeg +/// cannot decode as the source's. +#[tokio::test(start_paused = true)] +async fn the_graders_fail_what_they_should() { + let recording = record(Profile::Closed.clip()).await; + let run = run( + &recording, + |slot| match slot.video && slot.group == 5 { + true => Delivery::Omitted, + false => clean(slot), + }, + Duration::ZERO, + ) + .await; + grade(&run.ledgers, &run.stats).unwrap(); + let video = run.ledgers.iter().position(|ledger| ledger.video).unwrap(); + let unit = |group: usize, frame: usize| { + run.ledgers[video] + .units + .iter() + .position(|unit| unit.group == group) + .unwrap() + frame + }; + let edited = |edit: &dyn Fn(&mut Vec, &mut stats::Export)| { + let (mut ledgers, mut stats) = (run.ledgers.clone(), run.stats.clone()); + edit(&mut ledgers, &mut stats); + grade(&ledgers, &stats).unwrap_err() + }; + let err = edited(&|ledgers, _| ledgers[video].units[unit(5, 0)].output = true); + assert!(err.contains("went out, though"), "{err}"); + let err = edited(&|ledgers, stats| { + ledgers[video].units[unit(6, 3)].output = false; + stats.dropped += 1; + }); + assert!(err.contains("chain broke"), "{err}"); + let err = edited(&|ledgers, stats| { + for frame in 0..12 { + ledgers[video].units[unit(7, frame)].output = false; + } + stats.dropped += 12; + }); + assert!(err.contains("did not go out"), "{err}"); + let err = edited(&|_, stats| stats.dropped += 1); + assert!(err.contains("dropped"), "{err}"); + let err = edited(&|_, stats| stats.gaps = 0); + assert!(err.contains("gaps reported"), "{err}"); + + // The output less one P picture, with the ledger agreeing it never went out: FFmpeg must + // refuse what follows it, or decode it as other than the source's. + let reference = Reference::new(Profile::Closed, run.ledger(true)); + let pid = run.pid(Class::Video); + let target = unit(3, 1); + let ts = run.ts(); + let mut starts = ts + .as_chunks::<188>() + .0 + .iter() + .enumerate() + .filter(|(_, packet)| u16::from_be_bytes([packet[1] & 0x1f, packet[2]]) == pid && packet[1] & 0x40 != 0) + .map(|(at, _)| at); + let from = starts.nth(target).unwrap(); + let to = starts.next().unwrap(); + let mut damaged = ts[..from * 188].to_vec(); + damaged.extend_from_slice(&ts[to * 188..]); + let mut emitted: Vec<&Unit> = run.ledgers[video].units.iter().filter(|unit| unit.output).collect(); + emitted.retain(|unit| unit.pts != run.ledgers[video].units[target].pts); + let verdict = decode(&damaged, true).and_then(|output| reference.compare(&output, &emitted)); + assert!(verdict.is_err(), "a lost reference decoded as the source"); +} diff --git a/rs/moq-mux/src/container/ts/release.rs b/rs/moq-mux/src/container/ts/release.rs index 27457ce4e5f..b7bc7ff02a5 100644 --- a/rs/moq-mux/src/container/ts/release.rs +++ b/rs/moq-mux/src/container/ts/release.rs @@ -182,6 +182,14 @@ impl Release { self.jitter.out_of_tolerance() } + /// Times the source skipped objects it will never deliver, as the TS export's `gaps`. + #[cfg(test)] + pub(super) fn gaps(&self) -> u64 { + self.track + .as_ref() + .map_or(0, |(_, source)| source.skips() - source.markers()) + } + /// How many objects are waiting in the jitter buffer. #[cfg(test)] pub(super) fn buffered(&self) -> usize { diff --git a/rs/moq-mux/src/container/ts/release_test.rs b/rs/moq-mux/src/container/ts/release_test.rs index 6b71b858b1a..1943be144a2 100644 --- a/rs/moq-mux/src/container/ts/release_test.rs +++ b/rs/moq-mux/src/container/ts/release_test.rs @@ -334,6 +334,7 @@ async fn an_object_past_its_deadline_is_a_gap_in_that_release_only() { stalled_release.dropped(), "every late object is counted" ); + assert_eq!(stalled_release.gaps(), 0, "the late objects were delivered"); assert!( loss.missing.windows(2).all(|pair| pair[1] == pair[0] + 1), "one run is missing: {:?}", @@ -377,6 +378,7 @@ async fn a_group_never_delivered_leaves_its_time_out() { } ); assert_eq!(release.dropped(), 0, "nothing arrived late"); + assert_eq!(release.gaps(), 1, "the source skipped once"); } /// A group cut short loses the objects after the cut alone. @@ -396,6 +398,7 @@ async fn a_group_cut_short_loses_only_its_tail() { } ); assert_eq!(release.dropped(), 0, "nothing arrived late"); + assert_eq!(release.gaps(), 1, "the source skipped once"); } /// What the captures don't carry goes out at its exact position too: null packets, tables, a PID diff --git a/test/ts/README.md b/test/ts/README.md index a51183e0251..257b1e2f838 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -666,7 +666,12 @@ generated clip keeps one frame step, so it reads the capture's timestamps agains the source's, independently of what the exporter reports. Either way a stream may start up to 3 s late: the subscriber joins at the relay's latest group once the catalog is out, which skips the first 1-2 s on a clean path. Omitting MoQ content -on purpose belongs to the [recovery tests](/quest/m1/ts-reliability/recovery.md). +on purpose is graded on a paused clock in +`rs/moq-mux/src/container/ts/recovery_test.rs`: each arm republishes a generated +clip under a fault schedule (a group skipped or cut, a track or the video held +past its deadline, a join, A/V skew, the PTS wrap, open GOP), grades the output +against the ledger of what was delivered and when, and decodes it with FFmpeg +from each point a receiver acquires at, against the continuous source decode. ### Lossy path (`--loss`) @@ -850,8 +855,9 @@ PCR against a wrong rate or an empty measurement. Process controls also require nonzero/timeout exit codes to fail despite nonempty output. A missing tool or an unexpected failure category is a failing test, not a successful negative. -Other codecs, open-GOP acquisition, loss/recovery, raw carriage, and arbitrary -source captures do not yet have decoded-content comparisons. Their existing +Other codecs, raw carriage, and arbitrary source captures do not yet have +decoded-content comparisons; H.264 loss, recovery and open-GOP acquisition have +them in `recovery_test.rs`. Their existing specific checks remain active and the harness explicitly reports the narrower coverage. VBR and independent-clock MPTS need applicable timing models before claiming the CBR PCR-accuracy profile. From ee1ddeb5f583298efb4fbf59ae7ffb355e01665e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 21:06:28 -0700 Subject: [PATCH 3/9] test(ts): finish the recovery arms and fix what they found Adds the remaining arms to recovery_test.rs: a late catalog, a resolution change, a declared PCR discontinuity with loss on either side or its marker lost, HEVC RASL and RADL, and a bottleneck below the media rate behind a deep tail-dropping FIFO. Every acquisition point must now carry its own PAT and PMT. A two-minute fault run is wired into ts.yml's nightly steps through `just rs ts-recovery`, keeping a failing arm's evidence under $MOQ_TEST_RUNS. Fixes the arms found: - The import dropped the last picture before a declared PCR discontinuity: the packet carrying the flag starts a PES that proves its predecessor whole. - After a loss next to a declared discontinuity, the export laid the previous generation's PCR slots on the next generation's clock, running the PCR unflagged ahead and sending the old audio late. - The export trimmed HEVC RADL pictures as orphans after a join or a hole. Deletes quest/m1/ts-reliability/recovery.md. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts.yml | 8 + quest/m1/ts-reliability/README.md | 1 - quest/m1/ts-reliability/recovery.md | 68 - quest/m2/intra-refresh/plan-boundary.md | 6 +- rs/justfile | 4 + rs/moq-mux/src/codec/h265/mod.rs | 14 + rs/moq-mux/src/container/ts/export.rs | 20 +- rs/moq-mux/src/container/ts/import.rs | 7 + rs/moq-mux/src/container/ts/recovery_test.rs | 1167 ++++++++++++++---- rs/moq-mux/src/jitter.rs | 17 +- test/ts/README.md | 19 +- 11 files changed, 999 insertions(+), 332 deletions(-) delete mode 100644 quest/m1/ts-reliability/recovery.md diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index b562c579d02..b728f845b39 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -258,6 +258,14 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs MOQ_QUIC_CONGESTION_CONTROL: loss + # Two minutes of MoQ content faults and a congested bottleneck, on a paused clock: each + # recovery graded against what was delivered and decoded, and the latency must not drift. + - name: TS long recovery + if: ${{ !cancelled() && github.event_name != 'pull_request' }} + run: nix develop --command just rs ts-recovery + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + - name: Upload failure artifacts if: ${{ failure() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index f543017f979..0d9b2c728d2 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -100,7 +100,6 @@ remain with their current quests. ## Required - [Plan: semantic fidelity](/quest/m1/ts-reliability/fidelity.md) - explicit profiles and PMT lifecycle contracts produce implementation quests -- [Loss and recovery tests](/quest/m1/ts-reliability/recovery.md) - actual MoQ omissions exercise reacquisition and unaffected tracks ## Related diff --git a/quest/m1/ts-reliability/recovery.md b/quest/m1/ts-reliability/recovery.md deleted file mode 100644 index 5e67266b10e..00000000000 --- a/quest/m1/ts-reliability/recovery.md +++ /dev/null @@ -1,68 +0,0 @@ -# [L] TS recovery is tested against actual missing MoQ content - -## Goal - -Automated tests show how TS output and software decoding recover when MoQ -objects, groups, or tracks are absent or too late, including after playback -has started. Unaffected tracks continue according to the supported profile; -missing dependency chains are not presented as independently decodable. - -## Plan - -[#5713](https://github.com/moq-dev/moq/pull/5713) settled the cold-join -start: an export begins on a whole group, and `ts::stats::Export` counts -`trimmed`, `gaps` and `dropped`. Its paused-clock regressions -(`a_start_group_cut_short_starts_at_the_next`, `a_join_drops_its_group_whole`) -already cover a start group cut before output, so this quest does not repeat -them. Extend the evidence to loss after output began and to decoded output, -and check each fault schedule against what the export reported: a hole the -oracle accepts must show in those counters, or for a track that delivers -nothing, in its stream row's `units` and `quiet`. - -Test content loss separately from IP packet loss. QUIC may retransmit IP -loss; a netem percentage does not establish that any MoQ object was omitted. -Keep the real network impairment arm for transport and pacing evidence, -using a rate below the media bitrate behind a deep but finite queue with -verified overflow drops. Random loss only proves retransmission correctness, -never output quality or latency. Reuse the overflowing lane in -`test/ts/README.md`, and -add deterministic fault schedules at the MoQ content boundary. - -Cover a cold join, a skipped group, a partially delivered group, late objects -that miss release, one stalled/missing track, and resumed delivery. Put loss -around PAT/PMT or catalog acquisition, codec configuration and recovery -points, audio/video skew, PCR/PTS wrap and declared clock discontinuity. -Choose fixtures from declared supported profiles, not an unbounded matrix. - -The raw passthrough lane already has the packet/position and release-schedule -oracle (`rs/moq-mux/src/container/ts/oracle.rs`, `test/ts/passthrough.py`) -with skipped-group, cut-group and late-object regressions in `release_test.rs`; -extend it rather than build a second harness. Assert there that each omission -shows in the release's counters: `dropped` counts late objects, and the -skipped and cut groups need the `gaps` count -[#5849](https://github.com/moq-dev/moq/pull/5849) adds to `Release::stats`. - -Use mocked time for the Rust scheduling regressions. Integration captures -record exactly which content was delivered, omitted or late, so the oracle -does not explain every output gap as acceptable loss. Decode surviving -content with FFmpeg, compare recovered frames/samples against a continuous -source decode where applicable, and assert recovery at the profile's valid -acquisition point. No arbitrary recovery timeout becomes a substitute for -the encoded dependency rule. Exercise the graders' failure paths in CI. - -For semantic TS, validate continuity/discontinuity signalling, surviving -access-unit deadlines, decoder buffers and actual decoded output. Recovery -trims apply after a latency skip or discontinuity as well as the first join. -Coordinate the existing open-GOP and gradual-refresh quests: an H.264 -recovery-point SEI does not always promise immediate recovery, and HEVC CRA, -RADL and RASL do not have identical reference rules. Continuous subscribers -must retain valid pictures. - -Short deterministic arms run on relevant PRs. Long drift and impairment -arms run nightly with failure artifacts. Keep byte-position PCR accuracy -and measured release timing as separate checks. - -## Related - -- [Automated validation](/test/ts/README.md) - shared graders, profiles and known-bad controls -- [Gradual recovery](/quest/m2/intra-refresh/README.md) - recovery counts and per-group warmup diff --git a/quest/m2/intra-refresh/plan-boundary.md b/quest/m2/intra-refresh/plan-boundary.md index adf8467aa08..4f3c755c7ad 100644 --- a/quest/m2/intra-refresh/plan-boundary.md +++ b/quest/m2/intra-refresh/plan-boundary.md @@ -24,7 +24,10 @@ build reference state; withholding presentation is not dropping their input. Review the H.264 recovery-point flags and count semantics and HEVC CRA/RADL/RASL separately. A timestamp-before-keyframe rule is a conservative -leading-picture policy, not a complete codec dependency model. State which +leading-picture policy, not a complete codec dependency model; the TS export +already keeps HEVC RADL pictures, by NAL type, and +`rs/moq-mux/src/container/ts/recovery_test.rs` grades actual group skips and +recovered output for H.264 open GOP and HEVC RASL and RADL. State which profiles it supports without changing an existing wire contract by inference. Produce implementation and automated software-decode tests in the existing @@ -35,4 +38,3 @@ reference-decode comparisons for continuous playback and each recovery case. - [H.264 import](/quest/m2/intra-refresh/h264-import.md) - publishes recovery information - [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - uses it to size the transcoder's decode history -- [TS loss tests](/quest/m1/ts-reliability/recovery.md) - exercise actual group skips and recovered output diff --git a/rs/justfile b/rs/justfile index d487a863ebc..60d7ef308e2 100644 --- a/rs/justfile +++ b/rs/justfile @@ -348,6 +348,10 @@ multicast: just rs test -p moq-tokio --features mdns --lib --run-ignored only -E 'test(=mdns::tests::apps_partition_discovery) | test(/^mdns::tests::daemon_/)' just rs test -p moq-relay --features cluster-lan --run-ignored only -E 'test(/^lan_mesh::/)' +# Run the long TS recovery arms; a failing arm keeps its export and ledgers under $MOQ_TEST_RUNS. +ts-recovery: + just rs test -p moq-mux --lib --run-ignored only -E 'test(/^container::ts::recovery_test::long_/)' + # Run moq-audio's ignored output-device tests against a private PulseAudio null sink. audio-devices: #!/usr/bin/env bash diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index dd56cbb0995..0bff5b7de3f 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -50,6 +50,20 @@ fn hvc1_is_keyframe(data: &[u8], length_size: usize) -> bool { .any(|nal| nal.first().is_some_and(|header| is_irap(split::nal_unit_type(*header)))) } +/// Whether an hvc1-shaped (length-prefixed) access unit is a RADL picture: one that leads its +/// IRAP but references only it, so it decodes wherever that IRAP does (H.265 8.1.2). +pub(crate) fn hvc1_is_radl(data: &[u8], length_size: usize) -> bool { + let Ok(nals) = crate::codec::annexb::length_prefixed_nals(data, length_size) else { + return false; + }; + // RADL_N and RADL_R; types below 32 are slices. + let mut slices = (nals.map_while(std::result::Result::ok)) + .filter_map(|nal| nal.first().map(|header| (header >> 1) & 0x3f)) + .filter(|kind| *kind < 32) + .peekable(); + slices.peek().is_some() && slices.all(|kind| matches!(kind, 6 | 7)) +} + /// True for the IRAP (intra random access point) NAL types, the ones that open a group. /// HEVC random access is broader than H.264's single IDR type. pub(crate) fn is_irap(nal_type: NALUnitType) -> bool { diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index a3059655bce..0c78b345c36 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -353,6 +353,7 @@ impl Track { } if let Some((floor, ahead)) = self.leading_floor && pending.frame.timestamp < floor + && !radl(stream_type, self.source.description(), &pending.frame) { jitter.lose(&self.pid, ahead); return Ok(()); @@ -595,7 +596,10 @@ impl Track { .unwrap_or(frame.timestamp); let (key, pes) = self.release_key(decode, jitter.delay(), &frame)?; let video = matches!(self.kind, Kind::Video(_)); - let leading = video && !frame.keyframe && self.keyframe.is_some_and(|keyframe| frame.timestamp < keyframe); + let leading = video + && !frame.keyframe + && self.keyframe.is_some_and(|keyframe| frame.timestamp < keyframe) + && !matches!(self.kind, Kind::Video(stream_type) if radl(stream_type, description.as_ref(), &frame)); if video && frame.keyframe { self.keyframe = Some(frame.timestamp); } @@ -2448,7 +2452,8 @@ where fn slot_due(&self) -> Option<(u128, moq_clock::Instant)> { let (known, settled) = self.slot_settled()?; let settled = Timestamp::from_micros(u64::try_from(settled / 1_000).ok()?).ok()?; - Some((known, self.jitter.at(settled)?)) + // The tail of the generation still going out keeps its own clock after the next is acquired. + Some((known, self.jitter.at(self.generation, settled)?)) } /// The `known` that settles the next grid slot, and the jitter buffer key, in nanoseconds, @@ -3255,6 +3260,15 @@ fn ac3_sync_prefix(bytes: &[u8]) -> bool { bytes.iter().zip([0x0b, 0x77]).all(|(&byte, sync)| byte == sync) } +/// Whether a video frame is an HEVC RADL picture, which leads its keyframe but references only +/// it, so it goes out wherever that keyframe does rather than being trimmed as an orphan. +fn radl(stream_type: StreamType, description: Option<&Bytes>, frame: &Frame) -> bool { + stream_type == StreamType::H265 + && description + .and_then(|description| crate::codec::h265::hvcc_params(description).ok()) + .is_some_and(|(length_size, _)| crate::codec::h265::hvc1_is_radl(&frame.payload, length_size)) +} + /// Build the Annex-B elementary-stream payload for one video frame: rewrite the /// length-prefixed NALs to start-code-delimited NALs, injecting the parameter /// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC after a leading AUD on @@ -3595,7 +3609,7 @@ mod tests { // The last available audio frame, including its scheduling lead, anchors both orders. let (decode, key, _) = tracks[1].released.unwrap(); assert_eq!(decode.as_micros(), 800_000); - assert_eq!(jitter.at(key), Some(start + Duration::from_millis(500)), "{order:?}"); + assert_eq!(jitter.at(0, key), Some(start + Duration::from_millis(500)), "{order:?}"); tokio::time::advance(Duration::from_millis(500)).await; let video = std::iter::from_fn(|| match jitter.poll_next(&waiter) { std::task::Poll::Ready(ready) => Some(ready), diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 700b188ba6e..2ed6088e33b 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -343,6 +343,13 @@ impl Import { // out of it would break every track in the program on line noise. if self.pcr_pid.is_some_and(|p| p.as_u16() == pid) && pkt[1] & 0x80 == 0 { if discontinuity_indicator(&pkt) { + // A PES this packet starts ends the one before it on its PID whole, on the old + // clock; only the other PIDs' are cut short by the break. + if pkt[1] & 0x40 != 0 + && let Ok(pid) = Pid::new(pid) + { + self.flush(pid)?; + } self.timebase_break()?; // The new timebase starts with the PCR the flag rides, if it rides one. if let Some(pcr) = pcr(&pkt) { diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs index 8ab2be5fb38..3412b96c5d0 100644 --- a/rs/moq-mux/src/container/ts/recovery_test.rs +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -3,24 +3,28 @@ //! Each arm imports a clip FFmpeg generates in a declared software profile, records every group //! and frame the import publishes, and republishes them on a paused clock under a fault schedule //! at the MoQ content boundary: a group omitted or cut short, frames held past their deadline, a -//! track stalled. Loss here is content the path never delivers, not IP packets QUIC would -//! retransmit. The schedule is the ledger of what was delivered and when, and the export of the -//! republished broadcast is graded against it rather than against what the export reports: +//! track stalled, the catalog late, a bottleneck whose queue overflows. Loss here is content the +//! path never delivers, not IP packets QUIC would retransmit. The schedule is the ledger of what +//! was delivered and when, and the export of the republished broadcast is graded against it +//! rather than against what the export reports: //! //! - Nothing goes out that was not delivered, or that the schedule made late. Every picture that //! does keeps its dependency chain: a group's pictures are a decode-order prefix from its -//! keyframe, and its leading pictures go out only after the group before went out whole. +//! keyframe, and its leading pictures that reference the group before go out only after that +//! group went out whole. //! - What the schedule left alone goes out. A track the fault missed keeps running, and each //! track resumes at the first group whose chain survives. //! - Every delivered frame missing from the output is counted: `trimmed` before its track's //! first frame went out, `dropped` after, and each skip upstream is one of `gaps`. -//! - Every unit arrives before it decodes, and the continuity counters run unbroken. -//! - FFmpeg decodes the output without an error from each point a receiver acquires at, and -//! presents every picture that went out, once, as the continuous source decode's picture at -//! the same presentation time. +//! - Every unit arrives before it decodes, the continuity counters run unbroken, and the PCR +//! signals a discontinuity exactly where the source declared one. +//! - Each point a receiver acquires at carries its own PAT and PMT, and FFmpeg decodes the output +//! without an error from there, presenting every picture that went out, once, as the +//! continuous source decode's picture at the same presentation time. //! //! The raw lane's equivalents are in [`release_test`](super::release_test). +use std::ops::Range; use std::time::Duration; use mpeg2ts::pes::{PesPacketReader, ReadPesPacket}; @@ -49,80 +53,320 @@ const MARGIN: Duration = Duration::from_millis(160); /// How long a drain waits for the next frame before taking the export to have stalled. const STALL: Duration = Duration::from_secs(10); -/// A software profile FFmpeg generates: 6 s of 320x240 H.264 High at 25 fps, a group every -/// twelve pictures, beside AAC-LC. +/// Where a spliced profile changes over from its first encode to its second: the keyframe that +/// opens the seventh group. +const SPLICE: Duration = Duration::from_millis(2880); +/// How far the clock jumps at a declared discontinuity. +const JUMP: Duration = Duration::from_secs(20); + +/// A software profile FFmpeg generates: 320x240 at 25 fps, a group every twelve pictures, beside +/// AAC-LC. Six seconds long unless it says otherwise. #[derive(Clone, Copy, PartialEq, Eq, Debug)] enum Profile { - /// Closed GOP with B pictures, the profile `test/ts/decode.py` grades. + /// H.264 High, closed GOP with B pictures, the profile `test/ts/decode.py` grades. Closed, /// The same, starting 2.7 s before the 33-bit PTS and PCR wrap. Wrap, - /// Open GOP: after the first IDR every group opens on a recovery point, behind three + /// H.264 open GOP: after the first IDR every group opens on a recovery point, behind three /// leading pictures that reference the group before. /// /// Resuming at a recovery point after a hole leaves a `frame_num` gap, which H.264 takes /// for unintentional loss (8.2.5.2), so a decoder that ran through the hole conceals around /// it. A receiver acquiring at the recovery point decodes it cleanly. Open, + /// HEVC open GOP: after the first IDR every group opens on a CRA, behind three RASL + /// pictures that reference the group before. + Rasl, + /// HEVC closed GOP: after the first IDR every group opens on an IDR_W_RADL, behind two RADL + /// pictures that lead it but reference only it. + Radl, + /// H.264 closed GOP that drops from 320x240 to 176x144 at [`SPLICE`], the new SPS and PPS in + /// band. + Resized, + /// H.264 closed GOP whose PCR declares a discontinuity at [`SPLICE`], where the clock jumps + /// [`JUMP`] ahead. + Spliced, + /// [`Closed`](Self::Closed) for two minutes. + Long, +} + +/// One FFmpeg encode. +struct Encode { + size: &'static str, + seconds: &'static str, + /// The x264 or x265 parameters. + x26x: &'static str, + hevc: bool, + /// Seconds the timestamps start late by. + offset: u64, +} + +impl Encode { + fn run(&self) -> Vec { + let source = format!("testsrc=size={}:rate=25", self.size); + let offset = self.offset.to_string(); + let x26x = match self.hevc { + true => format!("{}:log-level=error", self.x26x), + false => self.x26x.to_string(), + }; + let mut args = vec![ + "-nostdin", + "-hide_banner", + "-v", + "error", + "-f", + "lavfi", + "-i", + &source, + "-f", + "lavfi", + "-i", + "sine=frequency=1000:sample_rate=48000", + "-t", + self.seconds, + ]; + args.extend(match self.hevc { + true => ["-c:v", "libx265", "-x265-params"], + false => ["-c:v", "libx264", "-x264-params"], + }); + args.push(&x26x); + if !self.hevc { + args.extend(["-profile:v", "high"]); + } + args.extend([ + "-preset", + "veryfast", + "-threads", + "1", + "-pix_fmt", + "yuv420p", + "-b:v", + "400k", + "-c:a", + "aac", + "-b:a", + "96k", + "-output_ts_offset", + &offset, + // One audio frame a PES, so a splice cuts no PES but the one it starts at. + "-pes_payload_size", + "0", + "-pcr_period", + "20", + "-f", + "mpegts", + "pipe:1", + ]); + let output = std::process::Command::new("ffmpeg") + .args(&args) + .output() + .expect("ffmpeg is in the dev shell"); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success() && stderr.is_empty(), "ffmpeg: {stderr}"); + output.stdout + } +} + +/// A generated clip, and the encodes its pictures come from. +struct Clip { + ts: Vec, + /// Each encode, and the 90 kHz presentation times the clip takes from it. + parts: Vec<(Vec, Range)>, } impl Profile { /// The clip, generated once per process. - fn clip(self) -> &'static [u8] { - static CLIPS: [std::sync::OnceLock>; 3] = [const { std::sync::OnceLock::new() }; 3]; + fn clip(self) -> &'static Clip { + static CLIPS: [std::sync::OnceLock; 8] = [const { std::sync::OnceLock::new() }; 8]; CLIPS[self as usize].get_or_init(|| { - let x264 = match self { - Self::Closed | Self::Wrap => "keyint=12:min-keyint=12:scenecut=0", - Self::Open => "keyint=12:min-keyint=12:scenecut=0:open-gop=1:bframes=3:b-adapt=0", + const CLOSED: &str = "keyint=12:min-keyint=12:scenecut=0"; + let encode = |size, x26x, hevc, offset| Encode { + size, + seconds: "6", + x26x, + hevc, + offset, }; - let mut args = vec![ - "-nostdin", - "-hide_banner", - "-v", - "error", - "-f", - "lavfi", - "-i", - "testsrc=size=320x240:rate=25", - "-f", - "lavfi", - "-i", - "sine=frequency=1000:sample_rate=48000", - "-t", - "6", - "-c:v", - "libx264", - "-profile:v", - "high", - "-preset", - "veryfast", - "-threads", - "1", - "-pix_fmt", - "yuv420p", - "-x264-params", - x264, - "-b:v", - "400k", - "-c:a", - "aac", - "-b:a", - "96k", - ]; - if self == Self::Wrap { + let whole = |encode: Encode| { + let ts = encode.run(); + let parts = vec![(ts.clone(), 0..u64::MAX)]; + Clip { ts, parts } + }; + match self { + Self::Closed => whole(encode("320x240", CLOSED, false, 0)), // 2^33 ticks is 95,443.7 s. - args.extend(["-output_ts_offset", "95441"]); + Self::Wrap => whole(encode("320x240", CLOSED, false, 95441)), + Self::Open => whole(encode( + "320x240", + "keyint=12:min-keyint=12:scenecut=0:open-gop=1:bframes=3:b-adapt=0", + false, + 0, + )), + Self::Rasl => whole(encode( + "320x240", + "keyint=12:min-keyint=12:scenecut=0:open-gop=1:bframes=3:b-adapt=0", + true, + 0, + )), + Self::Radl => whole(encode( + "320x240", + "keyint=12:min-keyint=12:scenecut=0:open-gop=0:radl=2:bframes=3:b-adapt=0", + true, + 0, + )), + Self::Resized => splice( + encode("320x240", CLOSED, false, 0).run(), + encode("176x144", CLOSED, false, 0).run(), + false, + ), + Self::Spliced => splice( + encode("320x240", CLOSED, false, 0).run(), + encode("320x240", CLOSED, false, JUMP.as_secs()).run(), + true, + ), + Self::Long => whole(Encode { + seconds: "120", + ..encode("320x240", CLOSED, false, 0) + }), } - args.extend(["-pcr_period", "20", "-f", "mpegts", "pipe:1"]); - let output = std::process::Command::new("ffmpeg") - .args(&args) - .output() - .expect("ffmpeg is in the dev shell"); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!(output.status.success() && stderr.is_empty(), "ffmpeg: {stderr}"); - output.stdout }) } + + /// How long the clip runs. + fn seconds(self) -> usize { + match self { + Self::Long => 120, + _ => 6, + } + } + + /// Whether its leading pictures reference the group before. + fn orphans(self) -> bool { + matches!(self, Self::Open | Self::Rasl) + } + + /// The PCR discontinuities the source declares. + fn breaks(self) -> usize { + usize::from(self == Self::Spliced) + } +} + +/// The PID, payload and payload unit start of a packet. +fn payload(packet: &[u8; 188]) -> (u16, &[u8], bool) { + let pid = u16::from_be_bytes([packet[1] & 0x1f, packet[2]]); + let start = match packet[3] & 0x30 { + 0x10 => 4, + 0x30 => (5 + usize::from(packet[4])).min(188), + _ => 188, + }; + (pid, &packet[start..], packet[1] & 0x40 != 0) +} + +/// The PTS a packet's PES starts with, if it starts one. +fn pes_pts(packet: &[u8; 188]) -> Option { + let (_, pes, start) = payload(packet); + if !start || pes.len() < 14 || pes[..3] != [0, 0, 1] || pes[7] & 0x80 == 0 { + return None; + } + Some( + u64::from((pes[9] >> 1) & 7) << 30 + | u64::from(pes[10]) << 22 + | u64::from(pes[11] >> 1) << 15 + | u64::from(pes[12]) << 7 + | u64::from(pes[13] >> 1), + ) +} + +/// Whether a packet starts a video PES. +fn starts_video(packet: &[u8; 188]) -> bool { + let (_, pes, start) = payload(packet); + start && pes.len() > 3 && pes[..3] == [0, 0, 1] && (0xe0..=0xef).contains(&pes[3]) +} + +/// `a` up to its keyframe at [`SPLICE`], then `b` from its own, declaring a PCR discontinuity +/// there if `declare`. Every other PID resumes in `b` at its first PES presented after `a`'s last +/// whole one, and every continuity counter carries on across the join. +fn splice(a: Vec, b: Vec, declare: bool) -> Clip { + let (a_packets, b_packets) = (a.as_chunks::<188>().0, b.as_chunks::<188>().0); + // The keyframe `SPLICE` after each encode's first picture. + let at = |packets: &[[u8; 188]]| { + let first = packets.iter().filter(|p| starts_video(p)).find_map(pes_pts).unwrap(); + let pts = first + SPLICE.as_micros() as u64 * 9 / 100; + let index = packets.iter().position(|p| starts_video(p) && pes_pts(p) == Some(pts)); + (index.expect("a keyframe at the splice"), pts) + }; + let ((from, from_pts), (to, to_pts)) = (at(a_packets), at(b_packets)); + + // Each PID's last whole PES in `a`: its PTS, and where an unfinished one started. + let mut last = std::collections::HashMap::, usize, usize, usize)>::new(); + for (index, packet) in a_packets[..from].iter().enumerate() { + let (pid, data, start) = payload(packet); + let entry = last.entry(pid).or_insert((None, index, 0, 0)); + if start && data.len() >= 6 && data[..3] == [0, 0, 1] { + if entry.3 == 0 || entry.2 >= entry.3 { + entry.0 = entry.0.max(pes_pts(packet)); + } + *entry = ( + entry.0, + index, + 0, + match u16::from_be_bytes([data[4], data[5]]) { + 0 => 0, + len => usize::from(len) + 6, + }, + ); + } + entry.2 += data.len(); + } + let mut out: Vec<[u8; 188]> = Vec::new(); + for (index, packet) in a_packets[..from].iter().enumerate() { + let (pid, ..) = payload(packet); + // A PES `a` leaves unfinished goes too. + match last.get(&pid) { + Some(&(_, started, have, want)) if want > have && index >= started => {} + _ => out.push(*packet), + } + } + let mut counters = std::collections::HashMap::new(); + for packet in &out { + if packet[3] & 0x10 != 0 { + counters.insert(payload(packet).0, packet[3] & 0x0f); + } + } + let mut resumed = std::collections::HashSet::new(); + for (index, packet) in b_packets[to..].iter().enumerate() { + let (pid, data, start) = payload(packet); + if !resumed.contains(&pid) { + let pes = data.len() >= 3 && data[..3] == [0, 0, 1]; + let after = match (pes_pts(packet), last.get(&pid)) { + (Some(pts), Some(&(Some(kept), ..))) => pts > kept, + _ => true, + }; + if !(start && (!pes || after) || index == 0) { + continue; + } + resumed.insert(pid); + } + let mut packet = *packet; + if packet[3] & 0x10 != 0 + && let Some(counter) = counters.get_mut(&pid) + { + *counter = (*counter + 1) & 0x0f; + packet[3] = (packet[3] & 0xf0) | *counter; + } + if declare && index == 0 { + assert!( + packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x10 != 0, + "the keyframe carries the PCR" + ); + packet[5] |= 0x80; + } + out.push(packet); + } + Clip { + ts: out.into_flattened(), + parts: vec![(a, 0..from_pts), (b, to_pts..u64::MAX)], + } } /// One track as the import published it: each group's sequence and frames, in decode order. @@ -134,17 +378,20 @@ struct Recorded { /// What the import of a clip published. struct Recording { + profile: Profile, catalog: Catalog, tracks: Vec, } -/// Import `ts` whole and read back every frame of every track. -async fn record(ts: &[u8]) -> Recording { +/// Import `profile`'s clip whole and read back every frame of every track. +async fn record(profile: Profile) -> Recording { let mut broadcast = moq_net::broadcast::Info::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + // Kept whole, however long the clip. + let config = crate::catalog::Config::default().with_max_age(Duration::from_secs(300)); + let catalog = crate::catalog::Producer::new(&mut broadcast, config).unwrap(); let consumer = broadcast.consume(); let mut import = super::Import::new(broadcast.clone(), catalog.reserve()); - import.decode(ts).unwrap(); + import.decode(&profile.clip().ts).unwrap(); import.finish().unwrap(); let snapshot = catalog.snapshot(); @@ -155,7 +402,7 @@ async fn record(ts: &[u8]) -> Recording { let mut track = consumer .track(&name) .unwrap() - .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(30))) + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(300))) .await .unwrap(); let mut groups = Vec::new(); @@ -177,18 +424,27 @@ async fn record(ts: &[u8]) -> Recording { [true, false], "the clip publishes a video and an audio track" ); + let pictures = (tracks[0].groups.iter().flat_map(|(_, frames)| frames)) + .filter(|frame| !decoded(frame).1) + .count(); + assert_eq!(pictures, profile.seconds() * 25, "the import publishes every picture"); Recording { + profile, catalog: snapshot, tracks, } } +/// The presentation time of a published frame, in microseconds on the catalog clock, and +/// whether it is the marker of a declared discontinuity. +fn decoded(frame: &moq_net::frame::Frame) -> (u64, bool) { + let frame = hang::container::Frame::decode(frame.payload.clone()).unwrap(); + (frame.timestamp.as_micros() as u64, frame.payload.is_empty()) +} + /// The presentation time of a published frame, in microseconds on the catalog clock. fn micros(frame: &moq_net::frame::Frame) -> u64 { - hang::container::Frame::decode(frame.payload.clone()) - .unwrap() - .timestamp - .as_micros() as u64 + decoded(frame).0 } /// The 90 kHz presentation time a microsecond timestamp goes out at, wrapped. @@ -205,6 +461,10 @@ struct Slot { /// Index into the group's frames. frame: usize, nominal: Duration, + /// The marker of a declared discontinuity, not media. + marker: bool, + /// Its payload's size in bytes. + size: usize, } /// Where a schedule puts a frame. @@ -222,14 +482,90 @@ fn clean(slot: Slot) -> Delivery { Delivery::At(slot.nominal) } +/// A bottleneck every track shares after the schedule's faults: while it is congested, frames +/// leave at `rate` in the order they arrive, behind a FIFO `queue` deep that tail-drops what +/// overflows it. A group that loses a frame there loses the rest of it too. +#[derive(Clone, Debug)] +struct Bottleneck { + /// Bits per second. + rate: u64, + queue: Duration, + /// When the path is congested; outside it, the link carries whatever reaches it at once. + congested: fn(Duration) -> bool, +} + +impl Bottleneck { + /// Pass every delivery through the link, in arrival order. + fn carry(&self, slots: &[(usize, Slot)], deliveries: &mut [Delivery]) { + let mut order: Vec = (0..slots.len()) + .filter(|&i| deliveries[i] != Delivery::Omitted) + .collect(); + order.sort_by_key(|&i| match deliveries[i] { + Delivery::At(at) => at, + Delivery::Omitted => unreachable!(), + }); + let mut free = Duration::ZERO; + let mut cut = std::collections::HashSet::new(); + for i in order { + let (track, slot) = slots[i]; + let Delivery::At(at) = deliveries[i] else { + unreachable!() + }; + if cut.contains(&(track, slot.group)) { + deliveries[i] = Delivery::Omitted; + continue; + } + if !(self.congested)(at) { + free = free.max(at); + deliveries[i] = Delivery::At(free); + continue; + } + if free.saturating_sub(at) > self.queue { + cut.insert((track, slot.group)); + deliveries[i] = Delivery::Omitted; + continue; + } + free = free.max(at) + Duration::from_nanos(slot.size as u64 * 8 * 1_000_000_000 / self.rate); + deliveries[i] = Delivery::At(free); + } + } +} + +/// What one arm republishes, and how. +#[derive(Clone)] +struct Arm { + profile: Profile, + schedule: Schedule, + /// A bottleneck after the faults. + path: Option, + /// When the export starts. + join: Duration, + /// When the catalog names the tracks. + catalog: Duration, +} + +/// `profile` under `schedule`, exported from the start. +fn arm(profile: Profile, schedule: Schedule) -> Arm { + Arm { + profile, + schedule, + path: None, + join: Duration::ZERO, + catalog: Duration::ZERO, + } +} + /// What the schedule decided for one recorded frame, and what the export made of it. #[derive(Clone, Debug)] struct Unit { group: usize, /// 90 kHz presentation time, wrapped as the output carries it. pts: u64, - /// Presented before its group's keyframe, so it references the group before. + nominal: Duration, + /// Presented before its group's keyframe. leading: bool, + /// A declared discontinuity's marker, which carries no media. + marker: bool, delivered: bool, /// Whether it arrived inside both the source's age budget and its deadline, `None` when /// too close to call. @@ -241,12 +577,21 @@ struct Unit { #[derive(Clone, Debug)] struct Ledger { video: bool, + /// Whether its leading pictures reference the group before. + orphans: bool, units: Vec, /// Skips the schedule made upstream: each run of omitted groups, and each cut with the run /// after it. gaps: u64, } +impl Ledger { + /// The units that carry media. + fn media(&self) -> impl Iterator { + self.units.iter().enumerate().filter(|(_, unit)| !unit.marker) + } +} + /// What an export of a republished recording wrote. struct Run { ledgers: Vec, @@ -284,9 +629,8 @@ impl Run { } } -/// Republish `recording` under `schedule`, start an export `join` in, and match its output to -/// the ledger. -async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { +/// Republish `recording` as `arm` says, start an export, and match its output to the ledger. +async fn run(recording: &Recording, arm: &Arm) -> Run { let origin = recording .tracks .iter() @@ -294,35 +638,79 @@ async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { .map(|(_, frames)| micros(&frames[0])) .min() .unwrap(); + // How far a declared break moved the clock, from the video's last picture before it to its + // first after, a frame on; and where the frames after it start. + let video = &recording.tracks[0].groups; + let (jumped, jump) = match video.iter().position(|(_, frames)| decoded(&frames[0]).1) { + Some(marker) => { + let before = micros(&video[marker].1[0]) + FRAME.as_micros() as u64; + let jump = micros(&video[marker + 1].1[0]) - before; + (before + jump / 2, Duration::from_micros(jump)) + } + None => (u64::MAX, Duration::ZERO), + }; + + // Every frame's slot, in recorded order, and where the schedule puts it. + let mut slots: Vec<(usize, Slot)> = Vec::new(); + for (t, track) in recording.tracks.iter().enumerate() { + let mut last = Duration::ZERO; + for (g, (_, frames)) in track.groups.iter().enumerate() { + let keyframe = micros(&frames[0]); + for (f, frame) in frames.iter().enumerate() { + let (at, marker) = decoded(frame); + let since = |micros: u64| match micros >= jumped { + true => Duration::from_micros(micros - origin) - jump, + false => Duration::from_micros(micros - origin), + }; + let nominal = match (marker, track.video) { + // Written at the break, right behind what came before it. + (true, _) => last, + // A group's pictures go out in decode order, a frame apart. + (false, true) => since(keyframe) + FRAME * f as u32, + (false, false) => since(at), + }; + last = nominal; + let size = frame.payload.len(); + let video = track.video; + let slot = Slot { + video, + group: g, + frame: f, + nominal, + marker, + size, + }; + slots.push((t, slot)); + } + } + } + let mut deliveries: Vec = slots.iter().map(|(_, slot)| (arm.schedule)(*slot)).collect(); + if let Some(path) = &arm.path { + path.carry(&slots, &mut deliveries); + } // Every frame's delivery, and each group's end: (at, track, group, frame or the end). let mut events = Vec::new(); let mut ledgers = Vec::new(); + let opened = arm.join.max(arm.catalog); + let mut next = 0; for (t, track) in recording.tracks.iter().enumerate() { let mut units = Vec::new(); let mut gaps = 0; let mut skipping = false; - // The group a subscription made at the join starts at: the newest begun by then. + // The group a subscription made when the export opens starts at: the newest begun by then. let mut joined = 0; for (g, (_, frames)) in track.groups.iter().enumerate() { let keyframe = micros(&frames[0]); let mut kept = 0; for (f, frame) in frames.iter().enumerate() { + let (_, slot) = slots[next]; + let delivery = deliveries[next]; + next += 1; let at = micros(frame); - let nominal = match track.video { - // A group's pictures go out in decode order, a frame apart. - true => Duration::from_micros(keyframe - origin) + FRAME * f as u32, - false => Duration::from_micros(at - origin), - }; - let delivery = schedule(Slot { - video: track.video, - group: g, - frame: f, - nominal, - }); let on_time = match delivery { Delivery::At(arrival) => { - let late = arrival.saturating_sub(nominal) + LAG; + let late = arrival.saturating_sub(slot.nominal) + LAG; // The source skips a group older than half the delay behind the newest. The // clock anchors on the video, which the export sends up to a delay ahead of // its decode time, so audio, sent barely ahead, has nearly a delay more. @@ -337,16 +725,18 @@ async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { }; if let Delivery::At(arrival) = delivery { assert_eq!(kept, f, "{} group {g} resumes after a hole", track.name); - if f == 0 && arrival + LAG <= join { + if f == 0 && arrival + LAG <= opened { joined = g; } kept += 1; - events.push((arrival + LAG, t, g, Some(f))); + events.push((arrival + LAG, Some((t, g, Some(f))))); } units.push(Unit { group: g, pts: ticks(at), + nominal: slot.nominal, leading: track.video && at < keyframe, + marker: slot.marker, delivered: delivery != Delivery::Omitted, on_time, output: false, @@ -355,10 +745,12 @@ async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { let whole = kept == frames.len(); if whole { let last = events.last().map(|(at, ..)| *at).unwrap(); - events.push((last, t, g, None)); + events.push((last, Some((t, g, None)))); } else if kept > 0 { - let next = Duration::from_micros(keyframe - origin) + FRAME * kept as u32; - events.push((next + LAG, t, g, None)); + // Evicted once its next frame would have arrived, after what it delivered. + let next = units[units.len() - frames.len() + kept].nominal + LAG; + let last = events.last().map(|(at, ..)| *at).unwrap(); + events.push((next.max(last), Some((t, g, None)))); } if kept > 0 && skipping { gaps += 1; @@ -369,29 +761,25 @@ async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { }; } assert!(!skipping, "{}: a schedule ends a track on a hole", track.name); - // What came before the join was never offered to the export. + // What came before the export opened was never offered to it. for unit in units.iter_mut().filter(|unit| unit.group < joined) { unit.delivered = false; } ledgers.push(Ledger { video: track.video, + orphans: recording.profile.orphans(), units, gaps, }); } - // Stable, so a group's end follows its last frame. - events.sort_by_key(|&(at, ..)| at); + // The catalog first among its peers; stable, so a group's end follows its last frame. + events.push((arm.catalog, None)); + events.sort_by_key(|&(at, event)| (at, event.is_some())); // The republisher: the same catalog and tracks on a new broadcast, each frame at its time. let mut broadcast = moq_net::broadcast::Info::new().produce(); let mut catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); let snapshot = recording.catalog.clone(); - catalog - .mutate(|catalog| { - catalog.video = snapshot.video; - catalog.audio = snapshot.audio; - }) - .unwrap(); let producers: Vec<_> = recording .tracks .iter() @@ -410,11 +798,21 @@ async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { recording.tracks.iter().map(|track| track.groups.clone()).collect(); let start = Instant::now(); let publisher = tokio::spawn(async move { - let _keep = (&broadcast, &catalog); + let _keep = &broadcast; let mut open: Vec> = producers.iter().map(|_| None).collect(); let mut kept = vec![0; producers.len()]; - for (at, t, g, step) in events { + for (at, event) in events { tokio::time::sleep_until(start + at).await; + let Some((t, g, step)) = event else { + let snapshot = snapshot.clone(); + catalog + .mutate(|catalog| { + catalog.video = snapshot.video; + catalog.audio = snapshot.audio; + }) + .unwrap(); + continue; + }; let (sequence, frames) = &groups[t][g]; let Some(f) = step else { let mut group = open[t].take().unwrap(); @@ -429,6 +827,7 @@ async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { let info = moq_net::group::Info { sequence: *sequence }; producers[t].create_group(info).unwrap() }); + assert_eq!(group.sequence, *sequence, "a schedule interleaves groups of track {t}"); group .write_frame(frames[f].timestamp, frames[f].payload.clone()) .unwrap(); @@ -438,10 +837,11 @@ async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { for producer in &producers { producer.finish().unwrap(); } + let _keep = catalog; std::future::pending::<()>().await; }); - tokio::time::sleep_until(start + join).await; + tokio::time::sleep_until(start + arm.join).await; let source = crate::source::announced(&consumer).await; let mut export = open_export(source).await.unwrap().with_delay(DELAY); let mut frames = Vec::new(); @@ -474,8 +874,9 @@ async fn run(recording: &Recording, schedule: Schedule, join: Duration) -> Run { let ledger = run.ledgers.iter_mut().find(|ledger| ledger.video == video).unwrap(); for k in 0..count as u64 { let pts = (pts + k * AUDIO_TICKS) % WRAP; - let matched: Vec = (0..ledger.units.len()) - .filter(|&i| ledger.units[i].pts.abs_diff(pts) <= 1) + let matched: Vec = (ledger.media()) + .filter(|(_, unit)| unit.pts.abs_diff(pts) <= 1) + .map(|(i, _)| i) .collect(); let [i] = matched[..] else { panic!("output PTS {pts} matches units {matched:?} of the video={video} ledger"); @@ -505,7 +906,7 @@ fn chain(ledger: &Ledger) -> Result, String> { let units = &ledger.units; let mut chained = vec![false; units.len()]; if !ledger.video { - for (i, unit) in units.iter().enumerate() { + for (i, unit) in ledger.media() { chained[i] = unit.delivered && unit.on_time == Some(true); } return Ok(chained); @@ -517,15 +918,28 @@ fn chain(ledger: &Ledger) -> Result, String> { let group = units[i].group; let (mut alive, mut broken) = (true, false); let mut complete = true; + if units[i].marker { + i += 1; + continue; + } while i < units.len() && units[i].group == group { let unit = &units[i]; - if unit.leading && !whole { + if unit.leading && ledger.orphans && !whole { if unit.output { return Err(format!("video unit {i} went out without the group before it: {unit:?}")); } i += 1; continue; } + // Nothing after a leading picture that needs only its own keyframe references it. + if unit.leading && !ledger.orphans { + chained[i] = alive && unit.delivered && unit.on_time == Some(true); + if unit.output && broken { + return Err(format!("video unit {i} went out after its chain broke: {unit:?}")); + } + i += 1; + continue; + } alive &= unit.delivered && unit.on_time == Some(true); chained[i] = alive; match unit.output { @@ -556,7 +970,7 @@ fn grade(ledgers: &[Ledger], stats: &stats::Export) -> Result<(), String> { for ledger in ledgers { let name = if ledger.video { "video" } else { "audio" }; let units = &ledger.units; - for (i, unit) in units.iter().enumerate() { + for (i, unit) in ledger.media() { if unit.output && (!unit.delivered || unit.on_time == Some(false)) { return Err(format!("{name} unit {i} went out, though {unit:?}")); } @@ -565,7 +979,7 @@ fn grade(ledgers: &[Ledger], stats: &stats::Export) -> Result<(), String> { // The export starts on the video's first group, with the audio presented from its // keyframe on; audio straddling the keyframe may be trimmed with what precedes it. - for (i, unit) in units.iter().enumerate() { + for (i, unit) in ledger.media() { let started = match ledger.video { true => unit.group >= start.group, false => (AUDIO_TICKS..WRAP / 2).contains(&((unit.pts + WRAP - keyframe.pts) % WRAP)), @@ -578,7 +992,7 @@ fn grade(ledgers: &[Ledger], stats: &stats::Export) -> Result<(), String> { } let first = units.iter().position(|unit| unit.output).unwrap_or(units.len()); - for (i, unit) in units.iter().enumerate() { + for (i, unit) in ledger.media() { if unit.delivered && !unit.output { *if i < first { &mut before } else { &mut after } += 1; late += u64::from(unit.on_time != Some(true)); @@ -640,7 +1054,7 @@ struct Decoded { } /// Decode `ts` with software FFmpeg, its audio too unless `video_only`, failing on any error it -/// logs. +/// logs. Pictures keep their own size, so a resolution change hashes each at its own. fn decode(ts: &[u8], video_only: bool) -> Result { use std::io::Write as _; let mut args = vec![ @@ -658,6 +1072,7 @@ fn decode(ts: &[u8], video_only: bool) -> Result { "pipe:0", "-map", "0:v:0", + "-noautoscale", "-c:v", "rawvideo", "-fps_mode", @@ -722,14 +1137,20 @@ struct Reference(std::collections::HashMap); impl Reference { fn new(profile: Profile, video: &Ledger) -> Self { - let source = decode(profile.clip(), false).unwrap(); + // Each encode decoded on its own, from its start, keeping the pictures the clip took. + let source: Vec<(u64, String)> = (profile.clip().parts.iter()) + .flat_map(|(ts, range)| { + let pictures = decode(ts, false).unwrap().pictures; + pictures.into_iter().filter(|(pts, _)| range.contains(pts)) + }) + .collect(); // The import moved the source onto the catalog clock; the first pictures give the shift. let earliest = |pts: &mut dyn Iterator| pts.min_by_key(|pts| (pts + WRAP / 2) % WRAP).unwrap(); - let shift = earliest(&mut source.pictures.iter().map(|(pts, _)| *pts)) + WRAP - - earliest(&mut video.units.iter().map(|unit| unit.pts)); - let pictures = source.pictures.into_iter(); + let shift = earliest(&mut source.iter().map(|(pts, _)| *pts)) + WRAP + - earliest(&mut video.media().map(|(_, unit)| unit.pts)); Self( - pictures + source + .into_iter() .map(|(pts, hash)| ((pts + WRAP - shift % WRAP) % WRAP, hash)) .collect(), ) @@ -760,37 +1181,31 @@ impl Reference { } } +/// A fresh decoder's input, and the pictures it carries. +type Segment<'a> = (Vec, Vec<&'a Unit>); + /// The video from each point a receiver acquires at, as a fresh decoder's input: its first -/// keyframe out, and each one it resumes at after a group did not go out whole. Each segment is -/// the output's leading tables and its packets up to the next, without the audio, beside the -/// pictures it carries. -fn segments(run: &Run) -> Vec<(Vec, Vec<&Unit>)> { - let ts = run.ts(); +/// keyframe out, and each one it resumes at after a group did not go out whole. Each segment +/// starts at the last PAT before its keyframe, which must follow the picture before it so a +/// receiver acquiring there finds the tables the export sent for it, and runs to the next +/// segment, without the audio. Each comes beside the pictures it carries. +fn segments<'a>(ts: &[u8], video: &'a Ledger, pids: (u16, u16)) -> Result>, String> { let packets = ts.as_chunks::<188>().0; - let (video, audio) = (run.pid(Class::Video), run.pid(Class::Audio)); - let pid = |packet: &[u8; 188]| u16::from_be_bytes([packet[1] & 0x1f, packet[2]]); + let pid = |packet: &[u8; 188]| payload(packet).0; + let (video_pid, audio_pid) = pids; // The packet each picture's PES starts at, by its PTS. let starts: Vec<(usize, u64)> = (packets.iter().enumerate()) - .filter(|(_, packet)| pid(packet) == video && packet[1] & 0x40 != 0) - .map(|(at, packet)| { - let adaptation = match packet[3] & 0x20 { - 0 => 0, - _ => usize::from(packet[4]) + 1, - }; - let pes = &packet[4 + adaptation..]; - let pts = u64::from((pes[9] >> 1) & 7) << 30 - | u64::from(pes[10]) << 22 - | u64::from(pes[11] >> 1) << 15 - | u64::from(pes[12]) << 7 - | u64::from(pes[13] >> 1); - (at, pts) - }) + .filter(|(_, packet)| pid(packet) == video_pid && packet[1] & 0x40 != 0) + .map(|(at, packet)| (at, pes_pts(packet).unwrap())) .collect(); - // Where each segment starts, and its pictures. + // Where each segment's keyframe starts, and its pictures. let mut cuts: Vec<(usize, Vec<&Unit>)> = Vec::new(); let mut whole = false; - for group in run.ledger(true).units.chunk_by(|a, b| a.group == b.group) { + for group in video.units.chunk_by(|a, b| a.group == b.group) { + if group[0].marker { + continue; + } let emitted: Vec<&Unit> = group.iter().filter(|unit| unit.output).collect(); match emitted.first() { Some(first) if !whole => { @@ -803,40 +1218,96 @@ fn segments(run: &Run) -> Vec<(Vec, Vec<&Unit>)> { .for_each(|(_, units)| units.extend(&emitted)), } // The group before must have gone out whole for its leading pictures to. - whole = group.iter().all(|unit| unit.output || (unit.leading && !whole)); + whole = group + .iter() + .all(|unit| unit.output || (unit.leading && video.orphans && !whole)); } - let tables = packets[..cuts[0].0].iter().filter(|packet| pid(packet) != audio); - let tables: Vec = tables.flatten().copied().collect(); - let ends: Vec = cuts.iter().skip(1).map(|(at, _)| *at).chain([packets.len()]).collect(); - (cuts.into_iter().zip(ends)) - .map(|((at, units), end)| { - let mut bytes = tables.clone(); - bytes.extend(packets[at..end].iter().filter(|packet| pid(packet) != audio).flatten()); - (bytes, units) + // Each segment from the PAT ahead of its keyframe, after the picture before it. + let mut froms = Vec::new(); + for (at, _) in &cuts { + let before = starts + .iter() + .rev() + .find(|(start, _)| start < at) + .map_or(0, |(start, _)| *start); + let Some(pat) = (before..*at).rev().find(|&i| pid(&packets[i]) == 0) else { + return Err(format!( + "no PAT between the picture at packet {before} and the keyframe at {at}" + )); + }; + froms.push(pat); + } + let ends: Vec = froms.iter().skip(1).copied().chain([packets.len()]).collect(); + Ok((froms.into_iter().zip(ends).zip(cuts)) + .map(|((from, end), (_, units))| { + let bytes = packets[from..end].iter().filter(|packet| pid(packet) != audio_pid); + (bytes.flatten().copied().collect(), units) }) - .collect() + .collect()) +} + +/// Where an arm leaves its evidence, kept when it fails or `MOQ_TEST_KEEP` is set: under +/// `$MOQ_TEST_RUNS/ts-recovery/`, by test name. +struct Evidence(Option); + +impl Evidence { + fn save(run: &Run) -> Self { + let Some(root) = std::env::var_os("MOQ_TEST_RUNS") else { + return Self(None); + }; + // libtest names each test's thread after it; a test run on the main thread has its process. + let name = match std::thread::current().name() { + Some(name) if name != "main" => name.replace("::", "-"), + _ => format!("arm-{}", std::process::id()), + }; + let dir = std::path::Path::new(&root).join("ts-recovery").join(name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("export.ts"), run.ts()).unwrap(); + std::fs::write(dir.join("ledgers.txt"), format!("{:#?}", run.ledgers)).unwrap(); + std::fs::write(dir.join("stats.txt"), format!("{:#?}", run.stats)).unwrap(); + Self(Some(dir)) + } } -/// Republish `profile`'s clip under `schedule`, export it from `join` on, and grade the export -/// and its decode: from each point a receiver acquires at, and as a whole when the profile's -/// resumptions are decodable straight through. -async fn recover(profile: Profile, schedule: Schedule, join: Duration) -> Run { - let recording = record(profile.clip()).await; - let run = run(&recording, schedule, join).await; +impl Drop for Evidence { + fn drop(&mut self) { + if let Some(dir) = &self.0 + && !std::thread::panicking() + && std::env::var_os("MOQ_TEST_KEEP").is_none() + { + let _ = std::fs::remove_dir_all(dir); + } + } +} + +/// Republish `arm`'s clip under its faults, export it, and grade the export and its decode: +/// from each point a receiver acquires at, and as a whole when the profile's resumptions are +/// decodable straight through. +async fn recover(arm: Arm) -> Run { + let profile = arm.profile; + let recording = record(profile).await; + let run = run(&recording, &arm).await; + let _evidence = Evidence::save(&run); grade(&run.ledgers, &run.stats).unwrap(); assert!(assert_on_time(&run.frames) > 200, "too little output to judge"); assert_continuous(&run.ts()); - assert_eq!(count_discontinuity(&run.frames), 0, "a loss is not a new program clock"); + assert_eq!( + count_discontinuity(&run.frames), + profile.breaks(), + "a PCR discontinuity where the source declared one, and no other" + ); let reference = Reference::new(profile, run.ledger(true)); - for (i, (ts, emitted)) in segments(&run).iter().enumerate() { + let pids = (run.pid(Class::Video), run.pid(Class::Audio)); + for (i, (ts, emitted)) in segments(&run.ts(), run.ledger(true), pids).unwrap().iter().enumerate() { let output = decode(ts, true).unwrap_or_else(|err| panic!("segment {i}: {err}")); (reference.compare(&output, emitted)).unwrap_or_else(|err| panic!("segment {i}: {err}")); } if profile != Profile::Open { let output = decode(&run.ts(), false).unwrap(); - let emitted: Vec<&Unit> = run.ledger(true).units.iter().filter(|unit| unit.output).collect(); + let video = run.ledger(true); + let emitted: Vec<&Unit> = video.units.iter().filter(|unit| unit.output).collect(); reference.compare(&output, &emitted).unwrap(); let frames = out(run.ledger(false)) as u64; assert_eq!(output.samples, frames * 1024, "every audio frame that went out decodes"); @@ -849,9 +1320,27 @@ fn out(ledger: &Ledger) -> usize { ledger.units.iter().filter(|unit| unit.output).count() } +/// How long after its nominal time each picture that went out did, in output order. +fn lags(run: &Run) -> Vec { + let pid = run.pid(Class::Video); + let emitted: Vec<&Unit> = run.ledger(true).units.iter().filter(|unit| unit.output).collect(); + assert_eq!( + run.stats.streams[&pid].units, + emitted.len() as u64, + "a PES for each picture" + ); + let mut lags = Vec::new(); + for (at, stats) in &run.samples { + while (lags.len() as u64) < stats.streams.get(&pid).map_or(0, |row| row.units) { + lags.push(at.saturating_sub(emitted[lags.len()].nominal)); + } + } + lags +} + #[tokio::test(start_paused = true)] async fn a_clean_path_loses_nothing() { - let run = recover(Profile::Closed, clean, Duration::ZERO).await; + let run = recover(arm(Profile::Closed, clean)).await; let video = run.ledger(true); assert_eq!(out(video), video.units.len(), "every picture goes out"); assert_eq!((run.stats.dropped, run.stats.gaps), (0, 0)); @@ -862,14 +1351,10 @@ async fn a_clean_path_loses_nothing() { /// on the next keyframe. #[tokio::test(start_paused = true)] async fn a_skipped_group_resumes_at_the_next() { - let run = recover( - Profile::Closed, - |slot| match slot.video && slot.group == 5 { - true => Delivery::Omitted, - false => clean(slot), - }, - Duration::ZERO, - ) + let run = recover(arm(Profile::Closed, |slot| match slot.video && slot.group == 5 { + true => Delivery::Omitted, + false => clean(slot), + })) .await; assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); } @@ -878,14 +1363,12 @@ async fn a_skipped_group_resumes_at_the_next() { /// settle their decode times, and loses only its tail. #[tokio::test(start_paused = true)] async fn a_cut_group_keeps_what_it_delivered() { - let run = recover( - Profile::Closed, - |slot| match slot.video && slot.group == 5 && slot.frame >= 4 { + let run = recover(arm(Profile::Closed, |slot| { + match slot.video && slot.group == 5 && slot.frame >= 4 { true => Delivery::Omitted, false => clean(slot), - }, - Duration::ZERO, - ) + } + })) .await; let video = run.ledger(true); let kept = video.units.iter().filter(|unit| unit.group == 5 && unit.output).count(); @@ -898,7 +1381,7 @@ async fn a_cut_group_keeps_what_it_delivered() { /// video resumes on the first keyframe delivered in time, on the same clock. #[tokio::test(start_paused = true)] async fn a_stall_drops_what_it_made_late() { - let run = recover(Profile::Closed, |slot| hold(slot.video, slot), Duration::ZERO).await; + let run = recover(arm(Profile::Closed, |slot| hold(slot.video, slot))).await; let (video, audio) = (run.ledger(true), run.ledger(false)); assert!( video.units.iter().any(|unit| unit.on_time == Some(false)), @@ -916,9 +1399,10 @@ async fn a_stall_drops_what_it_made_late() { /// judges a frame as arriving when its source was last found empty, which is when the export last /// ran, so what the stall held behind the deadline still goes out, late. #[tokio::test(start_paused = true)] -#[ignore = "an export nothing wakes judges a frame on when it last ran (quest/m1/ts-reliability/recovery.md)"] +#[ignore = "an export nothing wakes judges a frame on when it last ran; the fix, moq-net stamping each \ + frame's arrival, is its own pending quest"] async fn a_path_stall_drops_what_it_made_late() { - recover(Profile::Closed, |slot| hold(true, slot), Duration::ZERO).await; + recover(arm(Profile::Closed, |slot| hold(true, slot))).await; } /// Hold what is `held` and due in [1.92 s, 4.4 s) until the end, delivering the rest cleanly. @@ -934,17 +1418,7 @@ fn hold(held: bool, slot: Slot) -> Delivery { /// audio's row stops counting and its quiet grows on the output's clock, and the audio resumes. #[tokio::test(start_paused = true)] async fn a_stalled_track_leaves_the_others_running() { - const FROM: Duration = Duration::from_millis(2000); - const UNTIL: Duration = Duration::from_millis(3500); - let run = recover( - Profile::Closed, - |slot| match !slot.video && (FROM..UNTIL).contains(&slot.nominal) { - true => Delivery::Omitted, - false => clean(slot), - }, - Duration::ZERO, - ) - .await; + let run = recover(arm(Profile::Closed, stall_audio)).await; let video = run.ledger(true); assert_eq!(out(video), video.units.len(), "every picture goes out"); assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); @@ -974,18 +1448,23 @@ async fn a_stalled_track_leaves_the_others_running() { ); } +/// Omit the audio due in [2 s, 3.5 s). +fn stall_audio(slot: Slot) -> Delivery { + let (from, until) = (Duration::from_millis(2000), Duration::from_millis(3500)); + match !slot.video && (from..until).contains(&slot.nominal) { + true => Delivery::Omitted, + false => clean(slot), + } +} + /// Audio sent 200 ms behind the video, within the delay, survives a skipped video group whole. #[tokio::test(start_paused = true)] async fn skewed_audio_survives_a_video_skip() { - let run = recover( - Profile::Closed, - |slot| match (slot.video, slot.group) { - (true, 5) => Delivery::Omitted, - (true, _) => clean(slot), - (false, _) => Delivery::At(slot.nominal + Duration::from_millis(200)), - }, - Duration::ZERO, - ) + let run = recover(arm(Profile::Closed, |slot| match (slot.video, slot.group) { + (true, 5) => Delivery::Omitted, + (true, _) => clean(slot), + (false, _) => Delivery::At(slot.nominal + Duration::from_millis(200)), + })) .await; assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); } @@ -993,15 +1472,7 @@ async fn skewed_audio_survives_a_video_skip() { /// A skipped group around the 33-bit wrap keeps the clock, and the pictures either side decode. #[tokio::test(start_paused = true)] async fn a_skip_across_the_wrap_keeps_the_clock() { - let run = recover( - Profile::Wrap, - |slot| match slot.video && slot.group == 5 { - true => Delivery::Omitted, - false => clean(slot), - }, - Duration::ZERO, - ) - .await; + let run = recover(arm(Profile::Wrap, skip_group_5)).await; let video = run.ledger(true); let pts: Vec = video .units @@ -1016,59 +1487,250 @@ async fn a_skip_across_the_wrap_keeps_the_clock() { assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); } +fn skip_group_5(slot: Slot) -> Delivery { + match slot.video && slot.group == 5 { + true => Delivery::Omitted, + false => clean(slot), + } +} + +/// Skip video group 5 and cut group 8 after its sixth picture. +fn skip_and_cut(slot: Slot) -> Delivery { + match slot.video && (slot.group == 5 || (slot.group == 8 && slot.frame >= 6)) { + true => Delivery::Omitted, + false => clean(slot), + } +} + +/// How many of `group`'s leading pictures went out, failing if it has none. +fn leading_out(video: &Ledger, group: usize) -> usize { + let units: Vec<&Unit> = video + .units + .iter() + .filter(|unit| unit.group == group && unit.leading) + .collect(); + assert!(!units.is_empty(), "group {group} has leading pictures"); + units.iter().filter(|unit| unit.output).count() +} + /// Open GOP: a skipped group and a cut one each orphan the next group's leading pictures, which /// are trimmed, while every group after a whole one keeps its own. #[tokio::test(start_paused = true)] async fn open_gop_trims_only_the_orphans() { - let run = recover( - Profile::Open, - |slot| match slot.video && (slot.group == 5 || (slot.group == 8 && slot.frame >= 6)) { - true => Delivery::Omitted, - false => clean(slot), - }, - Duration::ZERO, - ) - .await; + let run = recover(arm(Profile::Open, skip_and_cut)).await; let video = run.ledger(true); - let leading = |group: usize| { - let units: Vec<&Unit> = video - .units - .iter() - .filter(|unit| unit.group == group && unit.leading) - .collect(); - assert!(!units.is_empty(), "group {group} has leading pictures"); - units.iter().filter(|unit| unit.output).count() - }; - assert_eq!((leading(6), leading(9)), (0, 0), "the orphans are trimmed"); - assert!(leading(7) > 0 && leading(10) > 0, "a continuous group keeps them"); + assert_eq!( + (leading_out(video, 6), leading_out(video, 9)), + (0, 0), + "the orphans are trimmed" + ); + assert!( + leading_out(video, 7) > 0 && leading_out(video, 10) > 0, + "a continuous group keeps them" + ); + assert_eq!(run.stats.gaps, 2); +} + +/// HEVC open GOP: the RASL pictures a skip or a cut orphans are trimmed, a continuous CRA keeps +/// its own, and the output decodes straight through the CRA it resumes at. +#[tokio::test(start_paused = true)] +async fn hevc_trims_only_orphaned_rasl() { + let run = recover(arm(Profile::Rasl, skip_and_cut)).await; + let video = run.ledger(true); + assert_eq!( + (leading_out(video, 6), leading_out(video, 9)), + (0, 0), + "the orphans are trimmed" + ); + assert!( + leading_out(video, 7) > 0 && leading_out(video, 10) > 0, + "a continuous group keeps them" + ); + assert_eq!(run.stats.gaps, 2); +} + +/// HEVC RADL pictures reference only their own IDR, so after a skip or a cut they go out with it. +#[tokio::test(start_paused = true)] +async fn hevc_keeps_radl_after_a_hole() { + let run = recover(arm(Profile::Radl, skip_and_cut)).await; + let video = run.ledger(true); + assert_eq!( + (leading_out(video, 6), leading_out(video, 9)), + (2, 2), + "RADL survives a hole" + ); assert_eq!(run.stats.gaps, 2); } /// A subscriber joining 2.3 s in starts on a whole group and loses nothing after it. #[tokio::test(start_paused = true)] async fn a_join_starts_on_a_whole_group() { - let run = recover(Profile::Closed, clean, Duration::from_millis(2300)).await; + let run = recover(Arm { + join: Duration::from_millis(2300), + ..arm(Profile::Closed, clean) + }) + .await; let video = run.ledger(true); let first = video.units.iter().find(|unit| unit.output).unwrap().group; assert!(first >= 4, "the export starts at the live edge, group {first}"); assert_eq!((run.stats.dropped, run.stats.gaps), (0, 0)); } -/// The graders fail what they should: a picture that went out after its chain broke, a group -/// that survived but did not go out, a loss the counters do not show, and a picture FFmpeg -/// cannot decode as the source's. +/// The catalog names the tracks only 2.3 s in, while the group in flight is cut short: the export +/// waits for it, starts on the next whole group with the tables ahead of it, and loses nothing +/// after. #[tokio::test(start_paused = true)] -async fn the_graders_fail_what_they_should() { - let recording = record(Profile::Closed.clip()).await; - let run = run( - &recording, - |slot| match slot.video && slot.group == 5 { +async fn a_late_catalog_starts_on_a_whole_group() { + let run = recover(Arm { + catalog: Duration::from_millis(2300), + ..arm(Profile::Closed, |slot| { + match slot.video && slot.group == 4 && slot.frame >= 10 { + true => Delivery::Omitted, + false => clean(slot), + } + }) + }) + .await; + let video = run.ledger(true); + let first = video.units.iter().find(|unit| unit.output).unwrap().group; + assert_eq!(first, 5, "the export starts on the first whole group after the catalog"); + assert_eq!(run.stats.dropped, 0); +} + +/// The picture size changes at a keyframe the path never delivers: the export resumes at the next +/// one with its new parameter sets, which a receiver acquiring there and one that ran through the +/// hole both decode at the new size. +#[tokio::test(start_paused = true)] +async fn a_skip_across_a_resolution_change_takes_the_new_parameters() { + let run = recover(arm(Profile::Resized, |slot| { + match slot.video && (slot.group == 6 || (slot.group == 5 && slot.frame >= 4)) { true => Delivery::Omitted, false => clean(slot), - }, - Duration::ZERO, - ) + } + })) + .await; + assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); +} + +/// The clock jumps 20 s at a declared discontinuity, and the path loses the group before the +/// break and the one after it: the output signals the break once, on the PCR, and resumes on the +/// keyframe after. +#[tokio::test(start_paused = true)] +async fn a_declared_discontinuity_survives_loss_either_side() { + let run = recover(arm(Profile::Spliced, |slot| { + match slot.video && (slot.group == 7 || (slot.group == 5 && slot.frame >= 4)) { + true => Delivery::Omitted, + false => clean(slot), + } + })) .await; + assert_eq!(run.stats.dropped, 0); +} + +/// The path loses the marker that declares the break, and every frame on either side arrives: +/// the export still finds the break where the clock jumps and signals it once. +#[tokio::test(start_paused = true)] +async fn a_lost_discontinuity_marker_still_breaks_the_clock() { + let run = recover(arm(Profile::Spliced, |slot| match slot.marker { + true => Delivery::Omitted, + false => clean(slot), + })) + .await; + assert_eq!(run.stats.dropped, 0); +} + +/// A path narrower than the media, behind a queue twice the delay deep, for two seconds: frames +/// queue past their deadline and the queue overflows. Nothing late goes out, every group the +/// export resumes at is whole from its keyframe, and its latency recovers once the path does. +#[tokio::test(start_paused = true)] +async fn a_congested_path_sheds_what_it_made_late() { + let run = recover(Arm { + path: Some(Bottleneck { + rate: 150_000, + queue: 2 * DELAY, + congested: |at| (Duration::from_millis(1500)..Duration::from_millis(3500)).contains(&at), + }), + ..arm(Profile::Closed, clean) + }) + .await; + let video = run.ledger(true); + assert!(video.units.iter().any(|unit| !unit.delivered), "the queue overflowed"); + assert!( + video.units.iter().any(|unit| unit.on_time == Some(false)), + "the queue made pictures late" + ); + let lags = lags(&run); + assert!( + lags.last().unwrap().abs_diff(lags[0]) <= FRAME, + "the latency recovers: {:?} at first, {:?} at last", + lags[0], + lags.last().unwrap() + ); +} + +/// One of each fault every twenty seconds, for the first five cycles: a skipped video group at 2 s, one cut +/// short at 6 s, the video held from 9 s to 11.5 s, and the audio lost from 13 s to 14.5 s. +fn faults(slot: Slot) -> Delivery { + const CYCLE: u128 = 20_000_000; + // Where the frame's group starts: a video group's keyframe, or the audio frame itself. + let start = match slot.video { + true => slot.nominal - FRAME * slot.frame as u32, + false => slot.nominal, + }; + if start >= Duration::from_secs(100) { + return clean(slot); + } + let cycled = |at: Duration| Duration::from_micros((at.as_micros() % CYCLE) as u64); + let within = + |from: u64, until: u64, at: Duration| (Duration::from_millis(from)..Duration::from_millis(until)).contains(&at); + let (since, group) = (cycled(slot.nominal), cycled(start)); + match slot.video { + true if within(2000, 2480, group) => Delivery::Omitted, + true if within(6000, 6480, group) && slot.frame >= 4 => Delivery::Omitted, + true if within(9000, 11500, group) => Delivery::At(slot.nominal - since + Duration::from_millis(11500)), + false if within(13000, 14500, since) => Delivery::Omitted, + _ => clean(slot), + } +} + +/// Two minutes of faults, and a congested path between them: every recovery is graded as in the +/// short arms, and the latency never grows past where it started. +#[tokio::test(start_paused = true)] +#[ignore = "long: run nightly by `just rs ts-recovery`"] +async fn long_faults_hold_their_latency() { + let run = recover(Arm { + path: Some(Bottleneck { + rate: 150_000, + queue: 2 * DELAY, + // Two seconds of every twenty. + congested: |at| (17..19).contains(&(at.as_secs() % 20)) && at < Duration::from_secs(100), + }), + ..arm(Profile::Long, faults) + }) + .await; + assert!(run.stats.gaps >= 20, "every fault skipped: {:?}", run.stats); + let lags = lags(&run); + let (least, most) = (lags.iter().min().unwrap(), lags.iter().max().unwrap()); + assert!( + *most - *least <= MARGIN, + "the latency ranged from {least:?} to {most:?}" + ); + // The first and last ten seconds of pictures, each on average. + let mean = |lags: &[Duration]| lags.iter().sum::() / lags.len() as u32; + let (first, last) = (mean(&lags[..250]), mean(&lags[lags.len() - 250..])); + assert!( + first.abs_diff(last) <= FRAME, + "the latency drifted from {first:?} to {last:?}" + ); +} + +/// The graders fail what they should: a picture that went out after its chain broke, a group +/// that survived but did not go out, a loss the counters do not show, a resumption without its +/// own tables, and a picture FFmpeg cannot decode as the source's. +#[tokio::test(start_paused = true)] +async fn the_graders_fail_what_they_should() { + let recording = record(Profile::Closed).await; + let run = run(&recording, &arm(Profile::Closed, skip_group_5)).await; grade(&run.ledgers, &run.stats).unwrap(); let video = run.ledgers.iter().position(|ledger| ledger.video).unwrap(); let unit = |group: usize, frame: usize| { @@ -1102,18 +1764,31 @@ async fn the_graders_fail_what_they_should() { let err = edited(&|_, stats| stats.gaps = 0); assert!(err.contains("gaps reported"), "{err}"); + // The output less the tables ahead of the keyframe it resumes at. + let ts = run.ts(); + let pid = run.pid(Class::Video); + let pids = (pid, run.pid(Class::Audio)); + let packets = ts.as_chunks::<188>().0; + let resume = (packets.iter()) + .position(|packet| { + payload(packet).0 == pid && pes_pts(packet) == Some(run.ledgers[video].units[unit(6, 0)].pts) + }) + .unwrap(); + let tables = (packets[..resume].iter()).rposition(starts_video).unwrap(); + let stripped: Vec = (packets.iter().enumerate()) + .filter(|&(at, packet)| !((tables..resume).contains(&at) && payload(packet).0 == 0)) + .flat_map(|(_, packet)| packet.iter().copied()) + .collect(); + segments(&ts, &run.ledgers[video], pids).unwrap(); + let err = segments(&stripped, &run.ledgers[video], pids).unwrap_err(); + assert!(err.contains("no PAT"), "{err}"); + // The output less one P picture, with the ledger agreeing it never went out: FFmpeg must // refuse what follows it, or decode it as other than the source's. let reference = Reference::new(Profile::Closed, run.ledger(true)); - let pid = run.pid(Class::Video); let target = unit(3, 1); - let ts = run.ts(); - let mut starts = ts - .as_chunks::<188>() - .0 - .iter() - .enumerate() - .filter(|(_, packet)| u16::from_be_bytes([packet[1] & 0x1f, packet[2]]) == pid && packet[1] & 0x40 != 0) + let mut starts = (packets.iter().enumerate()) + .filter(|(_, packet)| payload(packet).0 == pid && packet[1] & 0x40 != 0) .map(|(at, _)| at); let from = starts.nth(target).unwrap(); let to = starts.next().unwrap(); diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 7d6d245b5ad..d2bbbf2f0dd 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -754,9 +754,14 @@ impl Buffer { Ok(()) } - /// When a frame decoding at `decode` goes out on the current clock, once there is one. - pub fn at(&self, decode: Timestamp) -> Option { - self.deadline(self.clock.as_ref()?, decode) + /// When a frame of `generation` decoding at `decode` goes out: on the current clock, or on the + /// previous one while that generation's tail is still going out. + pub fn at(&self, generation: u64, decode: Timestamp) -> Option { + let clock = [self.clock, self.previous] + .into_iter() + .flatten() + .find(|clock| clock.generation == generation)?; + self.deadline(&clock, decode) } /// When a frame decoding at `decode` goes out on `clock`, if any instant holds it. @@ -1090,7 +1095,7 @@ mod tests { } } assert!(due(&mut buffer).is_empty()); - assert_eq!(buffer.at(ms(800)), Some(start + Duration::from_millis(500))); + assert_eq!(buffer.at(0, ms(800)), Some(start + Duration::from_millis(500))); tokio::time::advance(Duration::from_millis(500)).await; let first = buffer.poll_next(&kio::Waiter::noop()); assert!( @@ -1559,7 +1564,7 @@ mod tests { item: "v", }; let held = buffer - .at(frame.decode) + .at(0, frame.decode) .map(|deadline| since(arrived, deadline) as f64 / 1e9); match buffer.push(1, frame) { Ok(push) => assert_eq!(push, Push::Queued, "{ppm} ppm: frame {k} was late"), @@ -1571,7 +1576,7 @@ mod tests { } if hour >= settled && queued.is_zero() - && let Some(deadline) = buffer.at(Timestamp::from_micros(decode.as_micros() as u64).unwrap()) + && let Some(deadline) = buffer.at(0, Timestamp::from_micros(decode.as_micros() as u64).unwrap()) { let slack = since(arrived, deadline) as f64 / 1e9; run.worst = run.worst.max((slack - delay.as_secs_f64()).abs()); diff --git a/test/ts/README.md b/test/ts/README.md index 257b1e2f838..ed73d848998 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -669,9 +669,16 @@ catalog is out, which skips the first 1-2 s on a clean path. Omitting MoQ conten on purpose is graded on a paused clock in `rs/moq-mux/src/container/ts/recovery_test.rs`: each arm republishes a generated clip under a fault schedule (a group skipped or cut, a track or the video held -past its deadline, a join, A/V skew, the PTS wrap, open GOP), grades the output -against the ledger of what was delivered and when, and decodes it with FFmpeg -from each point a receiver acquires at, against the continuous source decode. +past its deadline, a join, a late catalog, A/V skew, the PTS wrap, a resolution +change, a declared PCR discontinuity or its lost marker, H.264 open GOP, HEVC +RASL and RADL, and a bottleneck below the media rate behind a FIFO twice the +delay deep that tail-drops), grades the output against the ledger of what was +delivered and when, and decodes it with FFmpeg from each point a receiver +acquires at, against the continuous source decode. Each of those points must +carry its own PAT and PMT, sent after the picture before it. Its graders' failure +paths run beside it, and `just rs ts-recovery` runs two minutes of the same faults +nightly, failing if the latency drifts and keeping a failing arm's export and +ledgers under `$MOQ_TEST_RUNS/ts-recovery/`. ### Lossy path (`--loss`) @@ -856,8 +863,8 @@ nonzero/timeout exit codes to fail despite nonempty output. A missing tool or an unexpected failure category is a failing test, not a successful negative. Other codecs, raw carriage, and arbitrary source captures do not yet have -decoded-content comparisons; H.264 loss, recovery and open-GOP acquisition have -them in `recovery_test.rs`. Their existing +decoded-content comparisons; H.264 and HEVC loss, recovery and open-GOP +acquisition have them in `recovery_test.rs`. Their existing specific checks remain active and the harness explicitly reports the narrower coverage. VBR and independent-clock MPTS need applicable timing models before claiming the CBR PCR-accuracy profile. @@ -883,7 +890,7 @@ Nightly and manual runs also invoke `just test ts --pair`, `just test ts --live --duration 120`, `just test ts --headroom --rate 7000000 --duration 120`, the same with `--queue 250ms` and `MOQ_QUIC_CONGESTION_CONTROL=loss`, and `just test ts --passthrough --rate 7000000 --queue 250ms --duration 120` -with `MOQ_QUIC_CONGESTION_CONTROL=loss`. +with `MOQ_QUIC_CONGESTION_CONTROL=loss`, and `just rs ts-recovery`. The pair exercises late-joining subscribers; the live window distinguishes ongoing clock drift from initial buffer filling; the congested paths judge what the export does when the path cannot carry the media. From 3356ef57534384aef5dd7c9f2d8570b41555cf45 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 21:11:07 -0700 Subject: [PATCH 4/9] test(ts): the spliced clip's jump is approximate Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/recovery_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs index 3412b96c5d0..6c688512412 100644 --- a/rs/moq-mux/src/container/ts/recovery_test.rs +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -84,7 +84,7 @@ enum Profile { /// band. Resized, /// H.264 closed GOP whose PCR declares a discontinuity at [`SPLICE`], where the clock jumps - /// [`JUMP`] ahead. + /// about [`JUMP`] ahead. Spliced, /// [`Closed`](Self::Closed) for two minutes. Long, From 0cc8790e4e21e2e541039f60868dff016e53accf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 21:13:37 -0700 Subject: [PATCH 5/9] test(ts): name the arrival-stamps quest in the ignored stall arm Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/recovery_test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs index 6c688512412..144ad7c5d2b 100644 --- a/rs/moq-mux/src/container/ts/recovery_test.rs +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -1399,8 +1399,8 @@ async fn a_stall_drops_what_it_made_late() { /// judges a frame as arriving when its source was last found empty, which is when the export last /// ran, so what the stall held behind the deadline still goes out, late. #[tokio::test(start_paused = true)] -#[ignore = "an export nothing wakes judges a frame on when it last ran; the fix, moq-net stamping each \ - frame's arrival, is its own pending quest"] +#[ignore = "an export nothing wakes judges a frame on when it last ran; moq-net stamping each frame's \ + arrival fixes it: /quest/m1/ts-reliability/arrival-stamps.md"] async fn a_path_stall_drops_what_it_made_late() { recover(arm(Profile::Closed, |slot| hold(true, slot))).await; } From 5333c864519891784b86e0996750365677a37a20 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 09:43:16 -0700 Subject: [PATCH 6/9] test(ts): read the recovery arms' counters from the export's release stats Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/recovery_test.rs | 56 ++++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs index 144ad7c5d2b..89dcbded5fb 100644 --- a/rs/moq-mux/src/container/ts/recovery_test.rs +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -1003,17 +1003,17 @@ fn grade(ledgers: &[Ledger], stats: &stats::Export) -> Result<(), String> { // The source skips frames that came older than its age budget, each run one more gap, and // they are neither trimmed nor dropped. - let skips = stats - .gaps - .checked_sub(gaps) - .ok_or(format!("{} gaps reported for the {gaps} the schedule made", stats.gaps))?; - let counted = stats.dropped + stats.trimmed; + let skips = stats.release.gaps.checked_sub(gaps).ok_or(format!( + "{} gaps reported for the {gaps} the schedule made", + stats.release.gaps + ))?; + let counted = stats.release.dropped + stats.release.trimmed; let skipped = (before + after).checked_sub(counted).ok_or(format!( "{counted} frames counted dropped or trimmed, but only {} are missing", before + after ))?; - let exact = (stats.dropped, stats.trimmed) == (after, before); - if !(exact || (skipped > 0 && stats.dropped <= after && stats.trimmed <= before)) + let exact = (stats.release.dropped, stats.release.trimmed) == (after, before); + if !(exact || (skipped > 0 && stats.release.dropped <= after && stats.release.trimmed <= before)) || (skips > 0) != (skipped > 0) || skips > skipped || skipped > late @@ -1021,7 +1021,7 @@ fn grade(ledgers: &[Ledger], stats: &stats::Export) -> Result<(), String> { return Err(format!( "(dropped, trimmed, gaps) reported {:?}; {before} missing before each track started, \ {after} after, {late} of them late, and {gaps} gaps scheduled", - (stats.dropped, stats.trimmed, stats.gaps) + (stats.release.dropped, stats.release.trimmed, stats.release.gaps) )); } Ok(()) @@ -1343,7 +1343,7 @@ async fn a_clean_path_loses_nothing() { let run = recover(arm(Profile::Closed, clean)).await; let video = run.ledger(true); assert_eq!(out(video), video.units.len(), "every picture goes out"); - assert_eq!((run.stats.dropped, run.stats.gaps), (0, 0)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 0)); } /// A group the path never delivers is skipped whole: the group before keeps its tail, though @@ -1356,7 +1356,7 @@ async fn a_skipped_group_resumes_at_the_next() { false => clean(slot), })) .await; - assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 1)); } /// A group cut short keeps the pictures it delivered, though they are fewer than it takes to @@ -1373,7 +1373,7 @@ async fn a_cut_group_keeps_what_it_delivered() { let video = run.ledger(true); let kept = video.units.iter().filter(|unit| unit.group == 5 && unit.output).count(); assert_eq!(kept, 4, "the cut group's delivered pictures go out"); - assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 1)); } /// The video stalls for 2.5 s while the audio runs on, then delivers what it held at once: the @@ -1387,7 +1387,7 @@ async fn a_stall_drops_what_it_made_late() { video.units.iter().any(|unit| unit.on_time == Some(false)), "the schedule made pictures late" ); - assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 1)); let first = audio.units.iter().position(|unit| unit.output).unwrap(); assert!( audio.units[first..].iter().all(|unit| unit.output), @@ -1421,7 +1421,7 @@ async fn a_stalled_track_leaves_the_others_running() { let run = recover(arm(Profile::Closed, stall_audio)).await; let video = run.ledger(true); assert_eq!(out(video), video.units.len(), "every picture goes out"); - assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 1)); let (audio, pictures) = (run.pid(Class::Audio), run.pid(Class::Video)); // The longest stretch of output the audio row sat still through. @@ -1466,7 +1466,7 @@ async fn skewed_audio_survives_a_video_skip() { (false, _) => Delivery::At(slot.nominal + Duration::from_millis(200)), })) .await; - assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 1)); } /// A skipped group around the 33-bit wrap keeps the clock, and the pictures either side decode. @@ -1484,7 +1484,7 @@ async fn a_skip_across_the_wrap_keeps_the_clock() { pts.windows(2).any(|pair| pair[1] < pair[0] / 2), "the output crosses the wrap" ); - assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 1)); } fn skip_group_5(slot: Slot) -> Delivery { @@ -1528,7 +1528,7 @@ async fn open_gop_trims_only_the_orphans() { leading_out(video, 7) > 0 && leading_out(video, 10) > 0, "a continuous group keeps them" ); - assert_eq!(run.stats.gaps, 2); + assert_eq!(run.stats.release.gaps, 2); } /// HEVC open GOP: the RASL pictures a skip or a cut orphans are trimmed, a continuous CRA keeps @@ -1546,7 +1546,7 @@ async fn hevc_trims_only_orphaned_rasl() { leading_out(video, 7) > 0 && leading_out(video, 10) > 0, "a continuous group keeps them" ); - assert_eq!(run.stats.gaps, 2); + assert_eq!(run.stats.release.gaps, 2); } /// HEVC RADL pictures reference only their own IDR, so after a skip or a cut they go out with it. @@ -1559,7 +1559,7 @@ async fn hevc_keeps_radl_after_a_hole() { (2, 2), "RADL survives a hole" ); - assert_eq!(run.stats.gaps, 2); + assert_eq!(run.stats.release.gaps, 2); } /// A subscriber joining 2.3 s in starts on a whole group and loses nothing after it. @@ -1573,7 +1573,7 @@ async fn a_join_starts_on_a_whole_group() { let video = run.ledger(true); let first = video.units.iter().find(|unit| unit.output).unwrap().group; assert!(first >= 4, "the export starts at the live edge, group {first}"); - assert_eq!((run.stats.dropped, run.stats.gaps), (0, 0)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 0)); } /// The catalog names the tracks only 2.3 s in, while the group in flight is cut short: the export @@ -1594,7 +1594,7 @@ async fn a_late_catalog_starts_on_a_whole_group() { let video = run.ledger(true); let first = video.units.iter().find(|unit| unit.output).unwrap().group; assert_eq!(first, 5, "the export starts on the first whole group after the catalog"); - assert_eq!(run.stats.dropped, 0); + assert_eq!(run.stats.release.dropped, 0); } /// The picture size changes at a keyframe the path never delivers: the export resumes at the next @@ -1609,7 +1609,7 @@ async fn a_skip_across_a_resolution_change_takes_the_new_parameters() { } })) .await; - assert_eq!((run.stats.dropped, run.stats.gaps), (0, 1)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 1)); } /// The clock jumps 20 s at a declared discontinuity, and the path loses the group before the @@ -1624,7 +1624,7 @@ async fn a_declared_discontinuity_survives_loss_either_side() { } })) .await; - assert_eq!(run.stats.dropped, 0); + assert_eq!(run.stats.release.dropped, 0); } /// The path loses the marker that declares the break, and every frame on either side arrives: @@ -1636,7 +1636,7 @@ async fn a_lost_discontinuity_marker_still_breaks_the_clock() { false => clean(slot), })) .await; - assert_eq!(run.stats.dropped, 0); + assert_eq!(run.stats.release.dropped, 0); } /// A path narrower than the media, behind a queue twice the delay deep, for two seconds: frames @@ -1708,7 +1708,7 @@ async fn long_faults_hold_their_latency() { ..arm(Profile::Long, faults) }) .await; - assert!(run.stats.gaps >= 20, "every fault skipped: {:?}", run.stats); + assert!(run.stats.release.gaps >= 20, "every fault skipped: {:?}", run.stats); let lags = lags(&run); let (least, most) = (lags.iter().min().unwrap(), lags.iter().max().unwrap()); assert!( @@ -1749,19 +1749,19 @@ async fn the_graders_fail_what_they_should() { assert!(err.contains("went out, though"), "{err}"); let err = edited(&|ledgers, stats| { ledgers[video].units[unit(6, 3)].output = false; - stats.dropped += 1; + stats.release.dropped += 1; }); assert!(err.contains("chain broke"), "{err}"); let err = edited(&|ledgers, stats| { for frame in 0..12 { ledgers[video].units[unit(7, frame)].output = false; } - stats.dropped += 12; + stats.release.dropped += 12; }); assert!(err.contains("did not go out"), "{err}"); - let err = edited(&|_, stats| stats.dropped += 1); + let err = edited(&|_, stats| stats.release.dropped += 1); assert!(err.contains("dropped"), "{err}"); - let err = edited(&|_, stats| stats.gaps = 0); + let err = edited(&|_, stats| stats.release.gaps = 0); assert!(err.contains("gaps reported"), "{err}"); // The output less the tables ahead of the keyframe it resumes at. From 558ee7956761a02ab6aa0f28bf524856d88b8ad6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 12:22:40 -0700 Subject: [PATCH 7/9] fix(ts): an adaptation-only PUSI at a clock break proves no PES whole A flagged discontinuity flushed the PCR PID's pending PES whenever PUSI was set, even on a packet with no payload, so an unbounded picture open across the break went out as complete instead of being cut short. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 33 ++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index d512bc55097..2475d7ef16c 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -345,8 +345,10 @@ impl Import { if self.pcr_pid.is_some_and(|p| p.as_u16() == pid) && pkt[1] & 0x80 == 0 { if discontinuity_indicator(&pkt) { // A PES this packet starts ends the one before it on its PID whole, on the old - // clock; only the other PIDs' are cut short by the break. + // clock; only the other PIDs' are cut short by the break. An adaptation-only + // packet starts none whatever its PUSI says, as `pes_packet` treats it. if pkt[1] & 0x40 != 0 + && matches!(payload(&pkt), Payload::Bytes(_)) && let Ok(pid) = Pid::new(pid) { self.flush(pid)?; @@ -8412,6 +8414,35 @@ pub(super) mod test { ); } + /// A flagged clock break that sets PUSI on an adaptation-only packet starts no PES, so it + /// proves nothing whole: the unbounded picture open across it is cut short and dropped. + #[test] + fn flagged_break_without_payload_drops_the_open_picture() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let unbounded = |cc, pts, keyframe| { + let mut packet = video_pes(VIDEO, cc, pts, None, &annexb_au(keyframe)); + let start = 5 + usize::from(packet[4]); + packet[start + 4..start + 6].fill(0); + packet + }; + let mut data = synth_pmt(&[(StreamType::H264, VIDEO)], false); + data.extend(unbounded(0, 90_000, true)); + data.extend(unbounded(1, 90_000 + FRAME, false)); + let mut flagged = clock_break_packet(VIDEO); + flagged[1] |= 0x40; + flagged[3] |= 1; + data.extend(flagged); + import.decode(&data).unwrap(); + import.finish().unwrap(); + assert_eq!( + import.stats().streams[&VIDEO].units, + 1, + "only the keyframe the second start proved whole" + ); + } + /// A malformed unbounded PES drained at EOF is still refused and counted. #[test] fn damaged_final_access_unit_is_refused() { From c1c140b0e66593cf6157601d9d8c494baacb014f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 12:24:44 -0700 Subject: [PATCH 8/9] quest: the release clock alarms no longer wait on the recovery arms, which land here Co-Authored-By: Claude Opus 5.5 --- quest/m1/ts-reliability/release-clock-false-alarms.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/quest/m1/ts-reliability/release-clock-false-alarms.md b/quest/m1/ts-reliability/release-clock-false-alarms.md index 9a0f34efa28..82dfc88cda3 100644 --- a/quest/m1/ts-reliability/release-clock-false-alarms.md +++ b/quest/m1/ts-reliability/release-clock-false-alarms.md @@ -25,10 +25,6 @@ Reproduce each first, then make the arms assert both counters stay zero. Public API: none. Wire: none. -## Required - -- [Loss and recovery tests](/quest/m1/ts-reliability/recovery.md) - lands the arms these alarms fire in - ## Related - [Arrival stamps](/quest/m1/ts-reliability/arrival-stamps.md) - also changes when the export releases a frame From 6784665d78003af79e0d351ba1a5cf1f4ccd38e4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 12:43:38 -0700 Subject: [PATCH 9/9] test(ts): the recovery bottleneck admits a frame only if all of it fits Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/recovery_test.rs | 33 ++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs index 89dcbded5fb..c8e5bdfe0e3 100644 --- a/rs/moq-mux/src/container/ts/recovery_test.rs +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -520,12 +520,14 @@ impl Bottleneck { deliveries[i] = Delivery::At(free); continue; } - if free.saturating_sub(at) > self.queue { + // The frame is admitted only if all of it fits: the backlog ahead plus its own bytes. + let leaves = free.max(at) + Duration::from_nanos(slot.size as u64 * 8 * 1_000_000_000 / self.rate); + if leaves - at > self.queue { cut.insert((track, slot.group)); deliveries[i] = Delivery::Omitted; continue; } - free = free.max(at) + Duration::from_nanos(slot.size as u64 * 8 * 1_000_000_000 / self.rate); + free = leaves; deliveries[i] = Delivery::At(free); } } @@ -1668,6 +1670,33 @@ async fn a_congested_path_sheds_what_it_made_late() { ); } +/// The bottleneck tail-drops a frame whose own bytes don't fit behind the backlog. +#[test] +fn the_bottleneck_admits_only_what_fits() { + // At 8 kbit/s, 1000 bytes take the whole one-second queue. + let link = Bottleneck { + rate: 8_000, + queue: Duration::from_secs(1), + congested: |_| true, + }; + let slot = |group, size| Slot { + video: true, + group, + frame: 0, + nominal: Duration::ZERO, + marker: false, + size, + }; + let slots = [(0, slot(0, 1000)), (0, slot(1, 1))]; + let mut deliveries = [Delivery::At(Duration::ZERO); 2]; + link.carry(&slots, &mut deliveries); + assert_eq!( + deliveries, + [Delivery::At(Duration::from_secs(1)), Delivery::Omitted], + "a full queue has no room for one more byte" + ); +} + /// One of each fault every twenty seconds, for the first five cycles: a skipped video group at 2 s, one cut /// short at 6 s, the video held from 9 s to 11.5 s, and the audio lost from 13 s to 14.5 s. fn faults(slot: Slot) -> Delivery {