Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion doc/concept/stats.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@ A TS import or export adds sections of its own, which
cumulative and omitted at zero, and a gauge is omitted until it is measured.

`mpegts` holds `streams`, one row per elementary stream keyed by its PID in
decimal, and the stream-wide counters. A row has `track` (the track suffix),
decimal (plus a `.si` row for an SI PID that dropped sections), and the
stream-wide counters. A row has `track` (the track suffix),
`class` (`audio`, `video`, or `data`; an unknown class reads as `data`),
`units` (access units delivered), and `quiet`, a gauge: milliseconds of
program clock since its last access unit. Its losses are `resyncs`, bytes
Expand Down
1 change: 0 additions & 1 deletion quest/m1/ts-reliability/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ catalog linger (#5305).

## Required

- [State bound](/quest/m1/ts-reliability/state-bound.md) - SI and revision state retire with what the source stops repeating
- [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a revised PAT never leaves a broadcast meaning another program
- [Stream routes](/quest/m1/ts-reliability/stream-routes.md) - every PID gets a declared route, and what is not carried is counted

Expand Down
54 changes: 0 additions & 54 deletions quest/m1/ts-reliability/state-bound.md

This file was deleted.

4 changes: 4 additions & 0 deletions rs/moq-cli/tests/integration/report.rs
Original file line number Diff line number Diff line change
Expand Up @@ -196,10 +196,14 @@ fn suppress(clip: &[u8], pid: u16, from: f64, to: f64) -> Vec<u8> {
}

/// Each PID's access units in `first` fell short of `last`'s: the rows advance while it runs.
/// An SI row counts only the sections the program clock could not place, so it has none.
fn assert_advancing(first: &ts::stats::Snapshot, last: &ts::stats::Snapshot) {
assert!(!last.streams.is_empty(), "no elementary stream reported");
let first = units(first);
for (pid, units) in units(last) {
if last.streams[&pid].track == ".si" {
continue;
}
let before = first.get(&pid).copied().unwrap_or(0);
assert!(before < units, "PID {pid}: {before} access units, then {units}");
}
Expand Down
44 changes: 44 additions & 0 deletions rs/moq-mux/src/catalog/producer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -956,6 +956,21 @@ impl<E: CatalogExt> Producer<E> {
Ok(recorder)
}

/// Retire `track`'s timeline once its producer dropped, for an owner that never enrolls the
/// name again: the timeline track finishes and leaves the root `archive` entry.
pub(crate) fn retire(&mut self, track: &str) -> crate::Result<()> {
self.timeline.retire(track)?;
let advertised = take(&self.current)
.catalog
.archive
.as_ref()
.is_some_and(|archive| archive.timelines.contains_key(track));
if advertised && let Some(archive) = self.modify()?.archive.as_mut() {
archive.timelines.remove(track);
}
Ok(())
}

/// The broadcast's [`Timelines`](crate::timeline::Timelines): each enrolled track's index.
///
/// A track's timeline is created (and advertised in the catalog's root `archive` entry) when
Expand Down Expand Up @@ -1938,6 +1953,35 @@ mod test {
assert!(catalog.enroll("video0", None).is_err());
}

/// A retired track's timeline ends for the readers already on it and leaves the archive, while
/// one still recording is refused.
#[test]
fn retiring_a_track_ends_its_timeline() {
let mut broadcast = moq_net::broadcast::Info::new().produce();
let mut catalog = Producer::new(&mut broadcast, Config::default()).unwrap();
let recorder = catalog
.enroll("video0", Some(std::time::Duration::from_secs(4)))
.unwrap();
let waiter = kio::Waiter::noop();
let mut subscribing = broadcast
.consume()
.track("video0.timeline.z")
.unwrap()
.subscribe(None)
.into_inner();
let std::task::Poll::Ready(Ok(mut reader)) = kio::Task::poll(&mut subscribing, &waiter) else {
panic!("the timeline track is live");
};
assert!(catalog.retire("video0").is_err(), "its recorder is still alive");

drop(recorder);
catalog.retire("video0").unwrap();
let archive = catalog.snapshot().archive.unwrap();
assert_eq!(archive.timelines.keys().collect::<Vec<_>>(), vec!["catalog.json"]);
assert_eq!(catalog.timeline().len(), 1);
assert!(matches!(reader.poll_finished(&waiter), std::task::Poll::Ready(Ok(_))));
}

#[test]
fn enrolling_advertises_the_catalog_section() {
let mut broadcast = moq_net::broadcast::Info::new().produce();
Expand Down
128 changes: 81 additions & 47 deletions rs/moq-mux/src/container/ts/export_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3237,14 +3237,18 @@ async fn service_layer_survives_roundtrip() {
let data = include_bytes!("test_data/bbb.ts");
let nit = make_long_section(0x40, 0x1234, 1, 0, 0, &[0xff, 0x01]);

// Prepend a synthetic NIT Actual packet (0x0010); prepend keeps bbb's alignment.
// Twice, because real SI repeats every few seconds: the repetition must collapse
// into the same single committed section rather than cutting a second group. The
// repeat varies the continuity counter so it reaches the store (an identical
// packet would be dropped as a TS duplicate before reassembly).
let mut input = si_packet(0x0010, &nit);
// Append a synthetic NIT Actual packet (0x0010), past bbb's PCRs: SI before the
// program clock can place it is dropped, which is where bbb's own SDT sits, so it is
// repeated there too. The NIT goes twice, because real SI repeats every few seconds:
// the repetition must collapse into the same single committed section rather than
// cutting a second group. Each repeat advances the continuity counter so it reaches
// the store (an identical packet would be dropped as a TS duplicate before reassembly).
let mut input = data.to_vec();
input.extend_from_slice(&si_packet(0x0010, &nit));
input.extend_from_slice(&si_packet_cc(0x0010, &nit, 1));
input.extend_from_slice(&data[..]);
let mut sdt = data[..188].to_vec();
sdt[3] = (sdt[3] & 0xf0) | ((sdt[3] + 1) & 0x0f);
input.extend_from_slice(&sdt);

let mut broadcast = moq_net::broadcast::Info::new().produce();
let consumer = broadcast.consume();
Expand Down Expand Up @@ -3330,9 +3334,16 @@ async fn service_layer_survives_roundtrip() {
}
assert!(checked_pat, "missing PAT");

// Re-import: the SDT and NIT must come back byte-for-byte.
// The SDT and NIT go out byte-for-byte.
assert_eq!(ts_sections(&ts, 0x0011), vec![sdt], "the SDT survived byte-for-byte");
assert_eq!(
ts_sections(&ts, 0x0010),
vec![Bytes::from(nit)],
"the NIT survived byte-for-byte"
);

// Re-import: the identity comes back.
let mut broadcast2 = moq_net::broadcast::Info::new().produce();
let consumer2 = broadcast2.consume();
let catalog2 = crate::catalog::Producer::new(
&mut broadcast2,
crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::<tscat::Ext>::default()),
Expand All @@ -3358,17 +3369,6 @@ async fn service_layer_survives_roundtrip() {
"program number survived"
);
assert_eq!(program2.pmt_pid, program.pmt_pid, "PMT PID survived");
assert_eq!(snapshot2.ext.mpegts.si, si, "every SI entry survived the round-trip");
assert_eq!(
read_si_sections(&consumer2, &snapshot2.ext.mpegts.si[&0x0011][&0x42].track).await,
vec![sdt],
"the SDT survived byte-for-byte"
);
assert_eq!(
read_si_sections(&consumer2, &snapshot2.ext.mpegts.si[&0x0010][&0x40].track).await,
vec![Bytes::from(nit)],
"the NIT survived byte-for-byte"
);
}

/// Each SI PID must be re-emitted on its own interval, independently of the PSI cadence
Expand Down Expand Up @@ -4134,24 +4134,17 @@ async fn multi_packet_si_section_is_captured() {
let input = si_packets_multi(0x0011, &sdt);
assert!(input.len() > 188, "the SDT must span more than one TS packet");

let mut broadcast = moq_net::broadcast::Info::new().produce();
let consumer = broadcast.consume();
let catalog = crate::catalog::Producer::new(
&mut broadcast,
crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::<tscat::Ext>::default()),
)
.unwrap();
let mut import = crate::container::ts::Import::new(broadcast, catalog.reserve());
import.decode(&BytesMut::from(&input[..])).unwrap();
import.finish().unwrap();
let mut rig = si_rig();
rig.import.decode(&BytesMut::from(&input[..])).unwrap();
rig.import.finish().unwrap();

let si = catalog.snapshot().ext.mpegts.si.clone();
let si = rig.catalog.snapshot().ext.mpegts.si.clone();
let entry = si
.get(&0x0011)
.and_then(|tables| tables.get(&0x42))
.expect("an SDT entry");
assert_eq!(
read_si_sections(&consumer, &entry.track).await,
read_si_sections(&rig.consumer, &entry.track).await,
vec![Bytes::from(sdt)],
"the multi-packet SDT was reassembled and captured byte-for-byte"
);
Expand All @@ -4166,6 +4159,13 @@ struct SiRig {
}

fn si_rig() -> SiRig {
let mut rig = si_import();
rig.import.decode(&BytesMut::from(&si_clock(1)[..])).unwrap();
rig
}

/// A rig with no input decoded yet.
fn si_import() -> SiRig {
let mut broadcast = moq_net::broadcast::Info::new().produce();
let consumer = broadcast.consume();
let catalog = crate::catalog::Producer::new(
Expand All @@ -4181,6 +4181,16 @@ fn si_rig() -> SiRig {
}
}

/// A program clock for SI-only input to be placed by: a PAT, a PMT for `program` with no
/// streams, and one PCR interval.
fn si_clock(program: u16) -> Vec<u8> {
use super::fixture::{Flags, MS, Mux};
let mut mux = Mux::default();
mux.pat(&[(program, 0x20)]).pmt(0x20, program, 0x1ff, &[], 0);
mux.packet(0x1ff, Flags::pcr(0)).packet(0x1ff, Flags::pcr(40 * MS));
mux.out
}

/// #2881: a multi-section table revised one section at a time must never publish a
/// torn set. The store buffers the incoming generation and commits it atomically,
/// so every snapshot group holds a single version.
Expand Down Expand Up @@ -4271,9 +4281,11 @@ fn si_rig_selecting(program: u16) -> SiRig {
import,
catalog,
consumer,
} = si_rig();
} = si_import();
let mut import = import.with_program(program);
import.decode(&BytesMut::from(&si_clock(program)[..])).unwrap();
SiRig {
import: import.with_program(program),
import,
catalog,
consumer,
}
Expand Down Expand Up @@ -4432,10 +4444,11 @@ async fn an_sdt_revision_dropping_the_service_retires_it() {
.unwrap();
let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone());
capture.select(1);
// One PCR interval, so the program clock places SI.
capture.pcr(0, 0);
capture.pcr(40 * 27_000, 10);
let mut revise = |version: u8, services: &[(u16, &[u8])]| {
capture
.section(0x0011, sdt_actual(version, 0, 0, services).into())
.unwrap();
assert!(capture.section(0x0011, sdt_actual(version, 0, 0, services).into(), 10));
capture.flush(Timestamp::ZERO, true).unwrap();
catalog
.snapshot()
Expand Down Expand Up @@ -4477,8 +4490,11 @@ async fn a_corrupt_sdt_keeps_the_last_good_snapshot() {
.unwrap();
let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone());
capture.select(1);
// One PCR interval, so the program clock places SI.
capture.pcr(0, 0);
capture.pcr(40 * 27_000, 10);
let mut feed = |section: Vec<u8>| {
capture.section(0x0011, section.into()).unwrap();
assert!(capture.section(0x0011, section.into(), 10));
capture.flush(Timestamp::ZERO, true).unwrap();
catalog
.snapshot()
Expand Down Expand Up @@ -4518,8 +4534,11 @@ async fn a_partial_sdt_revision_keeps_the_last_good_snapshot() {
.unwrap();
let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone());
capture.select(1);
// One PCR interval, so the program clock places SI.
capture.pcr(0, 0);
capture.pcr(40 * 27_000, 10);
let mut feed = |section: Vec<u8>| {
capture.section(0x0011, section.into()).unwrap();
assert!(capture.section(0x0011, section.into(), 10));
capture.flush(Timestamp::ZERO, true).unwrap();
catalog
.snapshot()
Expand Down Expand Up @@ -4564,12 +4583,15 @@ async fn a_selection_filters_eit_schedule() {
.unwrap();
let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone());
capture.select(1);
// One PCR interval, so the program clock places SI.
capture.pcr(0, 0);
capture.pcr(40 * 27_000, 10);
let schedule = |service: u16| {
let body = [0x00, 0x01, 0x00, 0x02, 0x00, 0x50, service as u8];
make_long_section(0x50, service, 0, 0, 0, &body)
};
for service in [1, 2] {
capture.section(0x0012, schedule(service).into()).unwrap();
assert!(capture.section(0x0012, schedule(service).into(), 10));
}
capture.finish(Timestamp::ZERO).unwrap();

Expand Down Expand Up @@ -4604,20 +4626,31 @@ async fn a_selected_programs_si_survives_export() {
let mut reader = TsPacketReader::new(Cursor::new(ts.as_ref()));
while reader.read_ts_packet().unwrap().is_some() {}

let mut again = si_rig();
again.import.decode(&BytesMut::from(ts.as_ref())).unwrap();
again.import.finish().unwrap();
let si = again.catalog.snapshot().ext.mpegts.si.clone();
let sdt = read_si_sections(&again.consumer, &si[&0x0011][&0x42].track).await;
let sdt = ts_sections(&ts, 0x0011);
assert_eq!(sdt.len(), 1, "one SDT actual section");
assert_sdt_lists_only(&sdt[0], 0, 2, b"Two");
assert_eq!(
read_si_sections(&again.consumer, &si[&0x0012][&0x4E].track).await,
ts_sections(&ts, 0x0012),
vec![eit_pf(2, 0), eit_pf(2, 1)],
"only program 2's EIT"
);
}

/// The sections `ts` carries on `pid`, each whole in one packet. Read off the wire rather
/// than by a re-import: SI an export leads with precedes its PCRs, so an import drops it
/// until it repeats.
fn ts_sections(ts: &[u8], pid: u16) -> Vec<Bytes> {
ts.chunks(188)
.filter(|p| (u16::from(p[1] & 0x1f) << 8 | u16::from(p[2])) == pid && p[1] & 0x40 != 0)
.map(|p| {
let pointer = if p[3] & 0x20 != 0 { 5 + usize::from(p[4]) } else { 4 };
let start = pointer + 1 + usize::from(p[pointer]);
let len = 3 + (usize::from(p[start + 1] & 0x0f) << 8 | usize::from(p[start + 2]));
Bytes::copy_from_slice(&p[start..start + len])
})
.collect()
}

/// #2842: SDT other sections from two networks that reuse a transport_stream_id
/// must not collide. The identity reads original_network_id (bytes 8..10) for
/// table_id 0x46, so both survive as separate sub-tables; a revision within one
Expand Down Expand Up @@ -6596,7 +6629,8 @@ async fn si_rapid_revisions_are_rate_bounded() {
#[tokio::test(start_paused = true)]
async fn overlapping_capture_teardown_keeps_the_survivors_mapping() {
let sdt = make_long_section(0x42, 1, 0, 0, 0, &[0xaa; 4]);
let mut input = si_packet(0x0011, &sdt);
let mut input = si_clock(1);
input.extend_from_slice(&si_packet(0x0011, &sdt));
input.extend_from_slice(&si_packet_cc(0x0011, &sdt, 1));

let mut broadcast = moq_net::broadcast::Info::new().produce();
Expand Down
Loading
Loading