diff --git a/doc/bin/ts.md b/doc/bin/ts.md index daa351a2626..1ba31d7be73 100644 --- a/doc/bin/ts.md +++ b/doc/bin/ts.md @@ -85,7 +85,9 @@ A broadcast holds one program. A multi-program stream is refused until broadcast (`event.hang` becomes `event/1.hang`, `event/2.hang`), following the PAT: a program it adds publishes, one it drops ends, and one that returns publishes as a new [instance](/concept/moq-lite#publisher-instances). A listed -program whose PMT does not arrive within 0.5 s of program clock ends the import. +program whose PMT does not arrive within 0.5 s of program clock counts as +`PMT_error_2` in the feed health while the others publish, and starts once its +PMT arrives. A PMT that adds, removes, or retypes a stream, or reuses a PID, ends the old track and continues under a fresh name; the export announces each layout at its diff --git a/quest/m1/ts-late-pmt.md b/quest/m1/ts-late-pmt.md deleted file mode 100644 index c273c76650d..00000000000 --- a/quest/m1/ts-late-pmt.md +++ /dev/null @@ -1,25 +0,0 @@ -# [S] A late PMT ends only its own program - -## Goal - -Under `--program all`, a program the PAT lists whose PMT does not arrive is -reported as TR 101 290 `PMT_error_2` in the import's health, and every other -program keeps publishing; the late program publishes once its PMT arrives. - -## Plan - -Found 2026-11-14 by the [feature review](/quest/m3/release/feature-review.md). -`Programs` (`rs/moq-mux/src/container/ts/programs.rs`, from -[moq-dev/moq#6252](https://github.com/moq-dev/moq/pull/6252)) returns -`Err(self.late())` once any listed program waits more than 0.5 s of program -clock, so an MPTS whose PAT keeps listing a service after its encoder dies, or -announces one before its PMT starts, ends every program's broadcast. The -guide documents that ("A listed program whose PMT does not arrive within -0.5 s of program clock ends the import"), but its Loss section says feed -health "is logged and reported without changing what publishes". Treat the -late program's PIDs as unclaimed and update `doc/bin/ts.md`. - -Decided 2026-11-14 (maintainer, pick on [moq-dev/moq#8764](https://github.com/moq-dev/moq/pull/8764)): ✅ (a) report and continue (rejected: (b) keep ending the import and reword the Loss section). - -Public API: none. Wire: none. - diff --git a/rs/moq-mux/src/container/ts/health.rs b/rs/moq-mux/src/container/ts/health.rs index d08c87c31e5..175b9db2d07 100644 --- a/rs/moq-mux/src/container/ts/health.rs +++ b/rs/moq-mux/src/container/ts/health.rs @@ -144,6 +144,11 @@ impl Errors { } } + /// Count `count` more `PMT_error_2` intervals on `pid`. + pub(super) fn pmt_error(&mut self, pid: u16, count: u64) { + self.pid(pid).pmt_error += count; + } + fn pid(&mut self, pid: u16) -> &mut PidErrors { self.pids.entry(pid).or_default() } @@ -346,10 +351,10 @@ impl Health { } } } else if let Some(timer) = self.pmt.get_mut(&pid) { - if scrambled { - self.errors.pid(pid).pmt_error += 1; - } else if section_start(pkt) == Some(PMT_TABLE_ID) { - timer.reset(now); + match PmtPacket::of(pkt, continuation) { + PmtPacket::Scrambled => self.errors.pid(pid).pmt_error += 1, + PmtPacket::Section => timer.reset(now), + PmtPacket::Other => {} } } if self.pcr_pid == Some(pid) && self.pcr(pid, pkt, at.count) { @@ -665,6 +670,32 @@ impl Delta { } } +/// What a packet on a PMT PID means for its `PMT_error_2` timer. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum PmtPacket { + /// It starts a PMT section, whatever its CRC, so the timer restarts. + Section, + /// It is scrambled, itself a `PMT_error_2`. + Scrambled, + /// Neither, as one continuing a section, or one the [`Multiplex`] graded corrupt or a + /// duplicate. + Other, +} + +impl PmtPacket { + pub(super) fn of(pkt: &[u8; TsPacket::SIZE], continuation: Continuation) -> Self { + if matches!(continuation, Continuation::Corrupt | Continuation::Duplicate) { + Self::Other + } else if pkt[3] & 0xc0 != 0 { + Self::Scrambled + } else if section_start(pkt) == Some(PMT_TABLE_ID) { + Self::Section + } else { + Self::Other + } + } +} + /// The table_id of the first section a PSI packet starts, if it starts one. fn section_start(pkt: &[u8; TsPacket::SIZE]) -> Option { if pkt[1] & 0x40 == 0 { diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index a3df17d2172..340daed4d03 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -6,7 +6,7 @@ use anyhow::Context; use mpeg2ts::ts::Pid; use super::demux::{Demux, Packet}; -use super::health::{Checks, Health}; +use super::health::{Checks, Health, PmtPacket}; use super::import::SectionReassembler; use super::unclaimed::Unclaimed; use super::{Ext, Import, catalog::SI_PIDS, health, psi, stats}; @@ -19,10 +19,12 @@ use crate::catalog; /// format detection still sees it; a name without one gains a plain `/n`. A program starts once /// its PMT lists an elementary stream; until then it publishes nothing. One whose PMT has not /// arrived 0.5 s of program clock after the PAT listed it, the bound TR 101 290 sets for -/// `PMT_error_2`, refuses the multiplex. The clock is a running program's, so nothing is late -/// before one runs, and one whose PMT arrived listing no stream yet waits. Input ahead of that -/// PMT is dropped, as a single [`Import`] drops media ahead of its PSI; its SI arrives with the -/// next repetition, as for an import joining mid-stream. +/// `PMT_error_2`, is late: each 0.5 s its PMT PID carries no PMT section counts in +/// [`stats::Snapshot::pmt_error`], its PIDs count as unclaimed, and the other programs carry on. +/// It starts whenever its PMT arrives. The clock is a running program's, so nothing is late +/// before one runs, and one whose PMT arrived listing no stream yet is not late. Input ahead of +/// that PMT is dropped, as a single [`Import`] drops media ahead of its PSI; its SI arrives with +/// the next repetition, as for an import joining mid-stream. /// /// The PAT is followed as it changes. A program a later PAT adds waits for its PMT as the first /// PAT's do, and each program follows its PMT to the PID the latest PAT names. One a later PAT @@ -33,7 +35,7 @@ use crate::catalog; /// and each program reads only the packets on the PIDs its PAT entry and PMT claim. A packet on a /// PID no program reads, the PAT's PMT PIDs, the SI PIDs and the null PID aside, is counted once /// for the multiplex (see [`stats::Snapshot::streams`]). Counting waits until every program the -/// PAT lists has its PMT, since until then a PID may be a program's still to come. +/// PAT lists has its PMT or is late, since until then a PID may be a program's still to come. /// /// Each PID is counted once, so a PID several programs read has one continuous set of /// counters while they start and retire. The PMT sections are reassembled here, once per PMT @@ -51,10 +53,12 @@ pub struct Programs { /// The latest whole PAT, from the first that lists a program, which seeds every program. pat: Option, /// The programs the latest PAT lists and no PMT has started yet, by the PMT PID it names - /// for them, with how long each has waited. + /// for them. waiting: HashMap>, - /// The PMT sections on each PMT PID the latest PAT lists, reassembled once for every program. - pmts: HashMap, + /// `PMT_error_2` on the PMT PIDs only waiting programs read. + pmt_errors: health::Errors, + /// Each PMT PID the latest PAT lists. + pmts: HashMap, /// Whether each PID, by index, is one of [`Self::pmts`], read without hashing every packet. pmt_pids: Vec, /// PMT sections dropped for a bad CRC, by PID. @@ -72,8 +76,11 @@ pub struct Programs { announced: bool, /// Times the programs waiting for their PMT. clock: Clock, - /// The [`Clock`] reading past which the longest-waiting program is late, while one waits. + /// The [`Clock`] reading past which a waiting program is late or a PMT PID it waits on is + /// overdue, while one waits. deadline: Option, + /// Whether a waiting program is not late yet, so a PID no program claims may be its. + joining: bool, /// Where the input can next expire a program's health check, so a program that reads no /// packet is advanced without a scan of every program per packet. Kept early when a routed /// packet moves its check later. @@ -85,11 +92,52 @@ pub struct Programs { /// `PMT_error_2`: a PMT recurs at least every 0.5 s of program clock. const PMT_LIMIT: u64 = super::passthrough::PCR_HZ / 2; +/// Whether a running program reads its PMT on `pid`, so its health times `PMT_error_2` there. +fn timed(tallies: &BTreeMap<(u16, Checks), Tally>, pid: u16) -> bool { + tallies + .get(&(pid, Checks::Table)) + .is_some_and(|tally| tally.owner.is_some()) +} + +/// A PMT PID the latest PAT lists. +struct PmtPid { + /// Its PMT sections, reassembled once for every program. + sections: SectionReassembler, + /// The [`Clock`] reading from which its next `PMT_error_2` is timed: that of the packet + /// starting its last PMT section, or of the PAT that named it, moved on 0.5 s for each + /// interval already past. Kept whichever programs read the PID, so a waiting program times + /// it on from where a running one left it. + section: u64, +} + +impl PmtPid { + fn new(now: u64) -> Self { + Self { + sections: SectionReassembler::default(), + section: now, + } + } + + /// Move [`Self::section`] past each 0.5 s interval without a PMT section through `now`, + /// returning how many it passed. + fn overdue(&mut self, now: u64) -> u64 { + let silent = now.saturating_sub(self.section); + if silent <= PMT_LIMIT { + return 0; + } + let count = (silent - 1) / PMT_LIMIT; + self.section += count * PMT_LIMIT; + count + } +} + /// How long a listed program has waited for its PMT, on the [`Clock`]. #[derive(Clone, Copy)] enum Wait { /// Since this many ticks. Since(u64), + /// Past the bound: its PIDs count as unclaimed, and it starts whenever its PMT arrives. + Late, /// Its PMT arrived listing no stream yet, so it is not late. Described, } @@ -312,6 +360,7 @@ impl Programs { demux: Demux::new(true), pat: None, waiting: HashMap::new(), + pmt_errors: health::Errors::default(), pmts: HashMap::new(), pmt_pids: vec![false; 1 << 13], crc_errors: BTreeMap::new(), @@ -322,6 +371,7 @@ impl Programs { announced: false, clock: Clock::default(), deadline: None, + joining: false, due: health::Due::default(), unclaimed: Unclaimed::default(), } @@ -397,7 +447,7 @@ impl Programs { // The waiting programs have no clock of their own until their PMT names one. self.clock.advance(&self.programs); if self.deadline.is_some_and(|deadline| self.clock.ticks > deadline) { - return Err(self.late()); + self.expire(); } } demux.drain(); @@ -451,6 +501,13 @@ impl Programs { if self.pat.is_none() && listed.is_empty() { return Ok(()); } + // A running program's health counted the intervals its PMT PID has already gone + // without a PMT, so a program left waiting on the PID once it retires counts on from + // there. + let ticks = self.clock.ticks; + for pmt in self.pmts.values_mut() { + pmt.overdue(ticks); + } self.retire(&listed)?; let mut moved = Vec::new(); self.waiting.retain(|pid, waiting| { @@ -465,7 +522,7 @@ impl Programs { !waiting.is_empty() }); // A PMT on a new PID is due 0.5 s from the PAT that names it. - let now = Wait::Since(self.clock.ticks); + let now = Wait::Since(ticks); for (number, pid) in moved { self.waiting.entry(pid).or_default().insert(number, now); } @@ -491,7 +548,7 @@ impl Programs { } self.pmts.retain(|pid, _| named.contains(pid)); for pid in named { - self.pmts.entry(pid).or_default(); + self.pmts.entry(pid).or_insert_with(|| PmtPid::new(ticks)); self.pmt_pids[usize::from(pid)] = true; } self.pat = Some(pat.clone()); @@ -499,35 +556,46 @@ impl Programs { Ok(()) } - /// Time the longest wait for a PMT, once the programs waiting have changed. + /// Time the next wait for a PMT to run out, once the programs waiting have changed. fn time_waits(&mut self) { - self.deadline = self - .waiting - .values() - .flat_map(HashMap::values) - .filter_map(|wait| match wait { - Wait::Since(since) => Some(since + PMT_LIMIT), - Wait::Described => None, - }) - .min(); - } - - /// The refusal once a listed program has waited more than 0.5 s of program clock for its - /// PMT, naming the programs that have. - fn late(&self) -> anyhow::Error { - let mut late: Vec = self - .waiting - .values() - .flat_map(|waiting| waiting.iter()) - .filter(|(_, wait)| matches!(wait, Wait::Since(since) if self.clock.ticks > since + PMT_LIMIT)) - .map(|(&number, _)| number) - .collect(); - late.sort_unstable(); - anyhow::anyhow!( - "{} TS programs the PAT lists sent no PMT within 0.5 s of program clock, the TR 101 290 PMT_error_2 bound: {:?}", - late.len(), - &late[..late.len().min(10)] - ) + self.joining = false; + self.deadline = None; + for (pid, waiting) in &self.waiting { + let overdue = self.pmts.get(pid).map(|pmt| pmt.section + PMT_LIMIT); + let late = waiting + .values() + .filter_map(|wait| match wait { + Wait::Since(since) => Some(since + PMT_LIMIT), + Wait::Late | Wait::Described => None, + }) + .min(); + self.joining |= late.is_some(); + self.deadline = self.deadline.into_iter().chain(overdue).chain(late).min(); + } + } + + /// Mark each program waiting more than 0.5 s of program clock for its PMT late, and count + /// `PMT_error_2` for each 0.5 s a PMT PID only waiting programs read carries no PMT section. + /// A running program reading its PMT on the PID counts it instead. + fn expire(&mut self) { + let now = self.clock.ticks; + for (&pid, waiting) in &mut self.waiting { + for (&number, wait) in waiting.iter_mut() { + if matches!(*wait, Wait::Since(since) if now > since + PMT_LIMIT) { + tracing::warn!( + program = number, + pmt_pid = pid, + "TS program the PAT lists sent no PMT within 0.5 s of program clock (PMT_error_2); the others carry on" + ); + *wait = Wait::Late; + } + } + let count = self.pmts.get_mut(&pid).map_or(0, |pmt| pmt.overdue(now)); + if count > 0 && !timed(&self.tallies, pid) { + self.pmt_errors.pmt_error(pid, count); + } + } + self.time_waits(); } /// End every started program `listed` no longer names: finish its tracks, announcing it @@ -648,8 +716,8 @@ impl Programs { if packet.pkt[1] & 0x80 != 0 || Unclaimed::fixed(packet.pid, true) || self.pmt_pids[usize::from(packet.pid)] { return; } - // A program still waiting for its PMT may claim it, and a late one refuses the multiplex. - if self.pat.is_some() && self.deadline.is_none() { + // A program still waiting for its PMT may claim it, until it is late. + if self.pat.is_some() && !self.joining { self.unclaimed.packet(packet.pid); } } @@ -661,10 +729,19 @@ impl Programs { /// so its own PMT path applies the PMT and starts its clock from the PCR already seen. Of /// several revisions in the packet, the last decides. fn describe(&mut self, packet: &Packet) -> anyhow::Result<()> { - let Some(sections) = self.pmts.get_mut(&packet.pid) else { + let Some(pmt) = self.pmts.get_mut(&packet.pid) else { return Ok(()); }; let mut waiting = self.waiting.get_mut(&packet.pid); + // Timed as a running program's health times its PMT PID, before any CRC check. + match PmtPacket::of(packet.pkt, packet.continuation) { + PmtPacket::Section => pmt.section = self.clock.ticks, + PmtPacket::Scrambled if waiting.is_some() && !timed(&self.tallies, packet.pid) => { + self.pmt_errors.pmt_error(packet.pid, 1); + } + PmtPacket::Scrambled | PmtPacket::Other => {} + } + let sections = &mut pmt.sections; let before = waiting.is_some().then(|| sections.clone()); let mut completed = Vec::new(); sections.push(packet.pkt, &mut completed); @@ -868,6 +945,7 @@ impl Programs { self.unclaimed.report(&mut stats.streams, |pid| { Unclaimed::fixed(pid, true) || self.pmt_pids[usize::from(pid)] || !self.readers[usize::from(pid)].is_empty() }); + errors.combine(&self.pmt_errors, u64::saturating_add); stats.crc_error = self.crc_errors.values().sum::() + self.demux.pat_crc_error(); errors.report(&mut stats); stats @@ -1098,11 +1176,12 @@ mod test { assert_eq!(programs.stats().streams[&0x71].units, 4, "program 2 read on"); } - /// A program the PAT lists whose PMT never arrives refuses the multiplex at the first packet - /// past 0.5 s of program clock, timed on the program that runs, without waiting for another - /// PAT; one whose PMT arrives listing no stream waits. + /// A program the PAT lists whose PMT does not arrive is late at the first packet past 0.5 s + /// of program clock, timed on the program that runs: its PMT PID counts `PMT_error_2` each + /// 0.5 s, its PIDs count as unclaimed, and the running program carries on. It starts once its + /// PMT arrives. One whose PMT arrives listing no stream is not late. #[test] - fn a_program_without_a_pmt_is_refused_after_half_a_second() { + fn a_late_pmt_ends_only_its_own_program() { let origin = crate::source::produce_origin(); let mut programs = Programs::new( origin, @@ -1111,18 +1190,129 @@ mod test { ); let mut mux = Mux::default(); mux.pat(&[(1, 0x100), (3, 0x300), (4, 0x400)]); - let mut step = |programs: &mut Programs, ms: u64| { + let step = |programs: &mut Programs, mux: &mut Mux, ms: u64| { mux.pmt(0x100, 1, 0x1ff, &[(MP2, 0x61)], 0) .pmt(0x400, 4, 0x1ff, &[], 0) + .packet(0x73, Flags::default()) .packet(0x1ff, Flags::pcr(27_000_000 + ms * MS)); - programs.decode(&std::mem::take(&mut mux.out)) + programs.decode(&std::mem::take(&mut mux.out)).unwrap(); + let stats = programs.stats(); + let refused = stats.streams.get(&0x73).map_or(0, |row| row.refused); + (stats.pmt_error, refused) }; // Program 1's clock starts at its first PCR, and the wait with it. for ms in (0..=500).step_by(100) { - step(&mut programs, ms).unwrap(); + assert_eq!(step(&mut programs, &mut mux, ms), (0, 0), "{ms} ms"); + } + assert_eq!( + step(&mut programs, &mut mux, 501), + (1, 0), + "late at the packet past 0.5 s" + ); + assert_eq!( + step(&mut programs, &mut mux, 1000), + (1, 1), + "its PIDs count as unclaimed" + ); + assert_eq!(step(&mut programs, &mut mux, 1001), (2, 2), "each 0.5 s counts"); + let running: Vec<_> = instances(&programs).into_iter().map(|(number, _)| number).collect(); + assert_eq!(running, [1], "the running program carries on"); + + mux.pmt(0x300, 3, 0x1ff, &[(MP2, 0x73)], 0); + assert_eq!( + step(&mut programs, &mut mux, 1100), + (2, 2), + "the late program claims its PID" + ); + let running: Vec<_> = instances(&programs).into_iter().map(|(number, _)| number).collect(); + assert_eq!(running, [1, 3], "the late program starts once its PMT arrives"); + } + + /// A PMT PID a running program reads counts its `PMT_error_2` once, however many programs + /// still wait on it. + #[test] + fn a_late_pmt_on_a_running_programs_pid_counts_once() { + let pmt_error = |listed: &[(u16, u16)]| { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new( + origin, + moq_net::Path::from_static("event.hang"), + catalog::Config::default(), + ); + let mut mux = Mux::default(); + mux.pat(listed); + for ms in (0..=2000).step_by(100) { + if ms <= 400 { + mux.pmt(0x100, 1, 0x1ff, &[(MP2, 0x61)], 0); + } + mux.packet(0x1ff, Flags::pcr(27_000_000 + ms * MS)); + programs.decode(&std::mem::take(&mut mux.out)).unwrap(); + } + programs.stats().pmt_error + }; + let alone = pmt_error(&[(1, 0x100)]); + assert!(alone > 0); + assert_eq!(pmt_error(&[(1, 0x100), (3, 0x100)]), alone); + } + + /// A waiting program's PMT PID is timed as a running program's health times its own: a PMT + /// section with a bad CRC still restarts the interval, so it counts as a CRC error alone. + #[test] + fn a_late_pmt_with_a_bad_crc_counts_only_its_crc() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new( + origin, + moq_net::Path::from_static("event.hang"), + catalog::Config::default(), + ); + let mut mux = Mux::default(); + mux.pat(&[(1, 0x100), (3, 0x300)]); + for ms in (0..=2000).step_by(100) { + mux.pmt(0x100, 1, 0x1ff, &[(MP2, 0x61)], 0) + .section(0x300, &corrupt(pmt_section(3, 0, &[(MP2, 0x73, &[])]))) + .packet(0x1ff, Flags::pcr(27_000_000 + ms * MS)); + programs.decode(&std::mem::take(&mut mux.out)).unwrap(); + } + let stats = programs.stats(); + assert_eq!(stats.crc_error, 21, "each PMT section counts its bad CRC"); + assert_eq!(stats.pmt_error, 0, "and no PMT_error_2"); + } + + /// A PMT PID a running program leaves to a waiting one keeps its interval: the waiting + /// program counts `PMT_error_2` 0.5 s after the PID's last PMT section, not after the PAT + /// that added it. + #[test] + fn a_late_pmt_times_on_from_a_retired_programs_pid() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new( + origin, + moq_net::Path::from_static("event.hang"), + catalog::Config::default(), + ); + let mut mux = Mux::default(); + let mut pmt_error = Vec::new(); + for ms in (0..=1000).step_by(50) { + match ms { + 0..400 => mux.pat(&[(1, 0x100), (5, 0x500)]), + 400..450 => mux.pat(&[(1, 0x100), (2, 0x100), (5, 0x500)]), + _ => mux.pat(&[(2, 0x100), (5, 0x500)]), + }; + match ms { + 0 => mux.pmt(0x100, 1, 0x1ff, &[(MP2, 0x61)], 0), + 800 => mux.pmt(0x100, 2, 0x1ff, &[(MP2, 0x62)], 0), + _ => &mut mux, + }; + mux.pmt(0x500, 5, 0x1ff, &[(MP2, 0x65)], 0) + .packet(0x1ff, Flags::pcr(27_000_000 + ms * MS)); + programs.decode(&std::mem::take(&mut mux.out)).unwrap(); + pmt_error.push((ms, programs.stats().pmt_error)); } - let err = step(&mut programs, 501).unwrap_err(); - assert!(format!("{err:#}").ends_with("bound: [3]"), "{err:#}"); + let at = |ms: u64| pmt_error.iter().find(|&&(at, _)| at == ms).unwrap().1; + assert_eq!(at(500), 0, "program 1 retired before its PMT was due"); + assert_eq!(at(550), 1, "due 0.5 s after program 1's last PMT"); + assert_eq!(at(1000), 1, "program 2's PMT ends the wait"); + let running: Vec<_> = instances(&programs).into_iter().map(|(number, _)| number).collect(); + assert_eq!(running, [5, 2], "program 2 starts once its PMT arrives"); } /// A program leaving the PAT flushes the picture it shares with a program still running,