From 20f348e124c54aefe9bd8029e90ae2f995e2e0d6 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:57:45 -0700 Subject: [PATCH 1/5] chore(quest): claim quest/m1/ts-reliability/shared-parser.md Quest-Claim: 84557ef1d36a7199 From dc441e8e8fad31a4a82e1edfad817104851612e1 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:40:31 -0700 Subject: [PATCH 2/5] perf(mux): parse an MPTS once for every program `ts::Programs` fed the whole multiplex to an independent `Import` per program, so framing, continuity, the PAT, SI reassembly and health grading ran once per program: ingest cost was program count times input rate. A shared `Demux` now frames the input and reads what every program shares (sync, per-PID continuity, the latest PCR per PID, the PAT, SI sections). `Programs` routes each packet only to the programs whose PAT entry or PMT claims its PID, through a PID-indexed reader table, and flushes only programs a call touched. A single `Import` runs the same parser. `health::Health` splits into the multiplex checks and the per-program timers; the mux-rate meter takes the parser's packet count. New `ts_programs` bench: at 16 programs, 4 Mb/s each, throughput goes from 87 to ~770-850 MiB/s, flat with program count. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EeyiEy5FWz3z44DYZVo7vU --- quest/m1/ts-reliability/README.md | 4 +- quest/m1/ts-reliability/pat-lifecycle.md | 13 +- quest/m1/ts-reliability/shared-parser.md | 27 -- quest/m1/ts-reliability/stream-routes.md | 20 +- rs/moq-mux/Cargo.toml | 4 + rs/moq-mux/benches/ts_programs.rs | 209 ++++++++++ rs/moq-mux/src/container/ts/demux.rs | 154 +++++++ rs/moq-mux/src/container/ts/export_test.rs | 10 +- rs/moq-mux/src/container/ts/health.rs | 90 +++-- rs/moq-mux/src/container/ts/import.rs | 449 +++++++++++---------- rs/moq-mux/src/container/ts/mod.rs | 1 + rs/moq-mux/src/container/ts/mux_rate.rs | 37 +- rs/moq-mux/src/container/ts/passthrough.rs | 7 +- rs/moq-mux/src/container/ts/programs.rs | 186 +++++---- rs/moq-mux/src/container/ts/si.rs | 3 +- 15 files changed, 810 insertions(+), 404 deletions(-) delete mode 100644 quest/m1/ts-reliability/shared-parser.md create mode 100644 rs/moq-mux/benches/ts_programs.rs create mode 100644 rs/moq-mux/src/container/ts/demux.rs diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index 4d6c81d7df5..5296db25a7a 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -84,8 +84,7 @@ refused means an error or a counted drop. Decided in the 2026-10-22 root-cause audit: the rows above kept getting patched one at a time because two foundations were missing. Decode time is reconstructed from history instead of decode order, and import state has no -owner that retires it. Those quests, and the shared parser that the routes -and PAT plans build on, rank ahead of the remaining rows. Rejected: carrying +owner that retires it. Those quests rank ahead of the remaining rows. Rejected: carrying decode time in every frame container, one shared ingest byte budget, and replacing the demux with GStreamer or another parser (it fixes neither cause). Out of scope for now: decoupling catalog publication from mux-rate @@ -99,7 +98,6 @@ remain with their current quests. - [State bound](/quest/m1/ts-reliability/state-bound.md) - SI and revision state retire with what the source stops repeating - [Decode order](/quest/m1/ts-reliability/decode-order.md) - exporters author DTS from decode order and fixed per-rendition constants -- [Shared parser](/quest/m1/ts-reliability/shared-parser.md) - `--program all` scans the multiplex once - [PMT export](/quest/m1/ts-reliability/pmt-export.md) - a TS export announces each PMT revision, mux rate included, at the next `version_number` - [Access units](/quest/m1/ts-reliability/pes-access-units.md) - a PES carrying several access units publishes each at its own time - [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a revised PAT never leaves a broadcast meaning another program diff --git a/quest/m1/ts-reliability/pat-lifecycle.md b/quest/m1/ts-reliability/pat-lifecycle.md index 2f0de125d9d..54a005098b8 100644 --- a/quest/m1/ts-reliability/pat-lifecycle.md +++ b/quest/m1/ts-reliability/pat-lifecycle.md @@ -14,10 +14,11 @@ What the code does today: - A PAT naming another PMT PID for the program adds a reassembler but never drops the old one, and `program.pmtPid` keeps the first. - `Programs` ignores a program a later PAT adds, without a warning. -- Under `--program all`, a later PAT that removes one program reaches every - per-program `Import`; the removed program's `handle_pat` errors, - `Programs::feed` propagates it, and the whole multiplex ingest stops, - including unrelated programs. +- Under `--program all`, the shared parser hands a later PAT that removes one + program to every program's `Import`; the removed program's `handle_pat` + errors, `Programs::decode` propagates it, and the whole multiplex ingest + stops, including unrelated programs. `Programs` sees each whole PAT first + (`Packet::pat`), so it can retire or start programs there instead. - A PAT naming another `transport_stream_id`, or a single-program import's PAT naming another program, already refuses the import. @@ -43,10 +44,6 @@ Tests use the generated `fixture::Mux` builder: a PMT PID move, and a it: the other program runs uninterrupted, and the returning one publishes a new epoch. -## Required - -- [Shared parser](/quest/m1/ts-reliability/shared-parser.md) - one parser owns the PID map this plan's accounting needs - ## Related - [PMT export](/quest/m1/ts-reliability/pmt-export.md) - the export follows a moved PMT PID with a new PAT diff --git a/quest/m1/ts-reliability/shared-parser.md b/quest/m1/ts-reliability/shared-parser.md deleted file mode 100644 index 9dddeb81fa2..00000000000 --- a/quest/m1/ts-reliability/shared-parser.md +++ /dev/null @@ -1,27 +0,0 @@ -# [L] One TS packet parser serves every program - -## Goal - -`--program all` scans the multiplex once: one packet parser owns the PAT, -the PID map and continuity, and dispatches each PID's packets to the -program that claims it. The cost of an MPTS ingest grows with input rate, -not with program count times input rate. - -## Plan - -Today `Programs` (`rs/moq-mux/src/container/ts/programs.rs`) feeds the same -bytes to an independent `Import` per program, so packet scanning, table -handling and health run once per program. - -Decided 2026-10-22 by the maintainer: this lands before -[stream routes](/quest/m1/ts-reliability/stream-routes.md) and -[PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md), whose plans work -around per-program reparsing (multiplex-wide PID rows moving between owners, -a removed program's `handle_pat` error stopping the whole ingest). Revisit -both plans once it lands. - -Add a benchmark in `rs/moq-mux/benches` that feeds `Programs` a generated -MPTS and sweeps program count and input rate independently; land it first -so the slope this removes shows. - -Public API: none expected; report any change to `Programs`. Wire: none. diff --git a/quest/m1/ts-reliability/stream-routes.md b/quest/m1/ts-reliability/stream-routes.md index 0587cdc4992..b9979ab89c8 100644 --- a/quest/m1/ts-reliability/stream-routes.md +++ b/quest/m1/ts-reliability/stream-routes.md @@ -59,9 +59,10 @@ program's PMT arrives. A listed PMT that does not arrive within the 0.5 s of program clock TR 101 290 allows (`PMT_error_2`, already counted) stops holding the gate, so a stale or corrupt program never disables the count. -Under `--program all`, every per-program `Import` reads the whole multiplex -and `Programs::stats` merges their rows by PID, so a per-program drop row -would shadow the program that carries the PID. `Programs` owns the +Under `--program all`, one parser reads the multiplex and `Programs` routes +each PID only to the programs whose PAT entry or PMT claims it +(`Programs::readers`), while `Programs::stats` merges their rows by PID, so a +per-program drop row would shadow the program that carries the PID. `Programs` owns the unclaimed-PID rows instead: a PID is dropped only when no program's PMT and no captured SI claims it, counted once at the multiplex rather than per program. A `--program N` import of an MPTS reassembles only its own PMT, so @@ -70,11 +71,10 @@ own program's PIDs and leaves multiplex-wide accounting to `--program all`. When the PAT lists only the selected program, it counts as above. Chosen over passively acquiring every PMT, which adds a reassembler per program to an import that serves one. -Classifying a PID looks it up in one map of every claimed PID, kept as -PMTs arrive, never by searching each program's PMT. A benchmark in -`rs/moq-mux/benches` feeds `Programs` a generated MPTS and sweeps program -count and PID count independently, against a baseline without the -accounting, so any slope it adds beyond the existing per-program feed shows. +Classifying a PID reads `Programs::readers`, the PID-indexed table of every +claimed PID kept as PMTs arrive: a PID no program reads is unclaimed. Extend +`rs/moq-mux/benches/ts_programs.rs` to sweep PID count beside program count +and bitrate, so any slope the accounting adds shows. A dropped PID reuses `stats::Stream`, with no new field: a row keyed by its PID in `Snapshot::streams`, an empty `track`, `Class::Data`, and `refused` @@ -106,7 +106,3 @@ program 2's PIDs, asserting the catalog, the stats rows (the unlisted PID's row asserted field by field), and an export round trip of the carried streams. The MPEG-2 video arm also runs the TSDuck round trip and an FFmpeg decode of the export in `test/ts`. - -## Required - -- [Shared parser](/quest/m1/ts-reliability/shared-parser.md) - one parser owns the PID map this plan's accounting needs diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 16f90fc6ab0..e3638e3a93a 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -57,6 +57,10 @@ harness = false name = "ts_schedule" harness = false +[[bench]] +name = "ts_programs" +harness = false + [[bench]] name = "export" harness = false diff --git a/rs/moq-mux/benches/ts_programs.rs b/rs/moq-mux/benches/ts_programs.rs new file mode 100644 index 00000000000..52be36723e5 --- /dev/null +++ b/rs/moq-mux/benches/ts_programs.rs @@ -0,0 +1,209 @@ +//! `--program all` TS ingest across program count and per-program bitrate, over one second of +//! a generated multiplex. Throughput is the whole input, so a parse that repeats per program +//! shows as a falling rate with program count. + +use std::time::Duration; + +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; + +const FPS: u64 = 25; +const GOP: u64 = 25; +/// PSI repeats every this many frames: 160 ms. +const PSI_EVERY: u64 = 4; +const SPS: &[u8] = &[ + 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, 0x01, + 0xd4, 0xc0, 0x80, +]; +const PPS: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; + +/// ISO/IEC 13818-1 CRC-32: polynomial 0x04C11DB7, no reflection, no final XOR. +fn crc32(data: &[u8]) -> u32 { + let mut crc = 0xffff_ffffu32; + for &byte in data { + crc ^= u32::from(byte) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { + (crc << 1) ^ 0x04c1_1db7 + } else { + crc << 1 + }; + } + } + crc +} + +/// A long-form PSI section: `table_id`, the 16-bit extension, version 0, one section. +fn section(table_id: u8, extension: u16, body: &[u8]) -> Vec { + let length = 5 + body.len() + 4; + let mut section = vec![table_id, 0xb0 | (length >> 8) as u8, length as u8]; + section.extend_from_slice(&extension.to_be_bytes()); + section.extend_from_slice(&[0xc1, 0, 0]); + section.extend_from_slice(body); + section.extend_from_slice(&crc32(§ion).to_be_bytes()); + section +} + +fn pmt_pid(program: u16) -> u16 { + 0x0100 + program +} + +fn video_pid(program: u16) -> u16 { + 0x0200 + program +} + +#[derive(Default)] +struct Mux { + out: Vec, + cc: std::collections::HashMap, +} + +impl Mux { + fn header(&mut self, pid: u16, start: bool, adaptation: bool) { + let cc = self.cc.entry(pid).or_default(); + self.out.extend_from_slice(&[ + 0x47, + u8::from(start) << 6 | (pid >> 8) as u8, + pid as u8, + if adaptation { 0x30 } else { 0x10 } | *cc, + ]); + *cc = (*cc + 1) & 0x0f; + } + + fn psi(&mut self, pid: u16, section: &[u8]) { + assert!(section.len() < 184, "one packet per section"); + self.header(pid, true, false); + let end = self.out.len() + 184; + self.out.push(0); + self.out.extend_from_slice(section); + self.out.resize(end, 0xff); + } + + /// One access unit on `pid` in `packets` packets, its first carrying `pcr` (27 MHz) and a + /// PES header with PTS `pts` (90 kHz). + fn frame(&mut self, pid: u16, packets: usize, pcr: u64, pts: u64, keyframe: bool) { + let mut pes = vec![0, 0, 1, 0xe0, 0, 0, 0x80, 0x80, 5]; + pes.extend_from_slice(&[ + 0x21 | ((pts >> 29) & 0x0e) as u8, + (pts >> 22) as u8, + 0x01 | ((pts >> 14) & 0xfe) as u8, + (pts >> 7) as u8, + 0x01 | ((pts << 1) & 0xfe) as u8, + ]); + let nals: &[&[u8]] = match keyframe { + true => &[SPS, PPS, &[0x65, 0x88, 0x84, 0x21]], + false => &[&[0x41, 0x9a, 0x02, 0x04]], + }; + for nal in nals { + pes.extend_from_slice(&[0, 0, 0, 1]); + pes.extend_from_slice(nal); + } + // The first packet gives 8 bytes to its PCR's adaptation field. Nonzero padding cannot + // add a start code. + pes.resize(packets * 184 - 8, 0xff); + + self.header(pid, true, true); + let (base, ext) = (pcr / 300, pcr % 300); + self.out.extend_from_slice(&[ + 7, + 0x10 | u8::from(keyframe) << 6, + (base >> 25) as u8, + (base >> 17) as u8, + (base >> 9) as u8, + (base >> 1) as u8, + ((base & 1) << 7) as u8 | 0x7e | (ext >> 8) as u8, + ext as u8, + ]); + let (first, rest) = pes.split_at(176); + self.out.extend_from_slice(first); + for chunk in rest.chunks(184) { + self.header(pid, false, false); + self.out.extend_from_slice(chunk); + } + } +} + +/// One second of `programs` H.264 programs at `bitrate` bits per second each, interleaved +/// frame by frame, each program carrying its PCR on its video PID. +fn multiplex(programs: u16, bitrate: u64) -> Vec { + let packets = (bitrate / FPS / 8).div_ceil(188) as usize; + let mut mux = Mux::default(); + let mut pat = Vec::new(); + for program in 1..=programs { + pat.extend_from_slice(&program.to_be_bytes()); + pat.extend_from_slice(&(0xe000 | pmt_pid(program)).to_be_bytes()); + } + let pat = section(0x00, 1, &pat); + let pmts: Vec<_> = (1..=programs) + .map(|program| { + let video = video_pid(program); + let mut body = (0xe000 | video).to_be_bytes().to_vec(); + body.extend_from_slice(&[0xf0, 0x00, 0x1b]); + body.extend_from_slice(&(0xe000 | video).to_be_bytes()); + body.extend_from_slice(&[0xf0, 0x00]); + section(0x02, program, &body) + }) + .collect(); + for frame in 0..FPS { + if frame % PSI_EVERY == 0 { + mux.psi(0, &pat); + for (program, pmt) in (1..=programs).zip(&pmts) { + mux.psi(pmt_pid(program), pmt); + } + } + let pts = 90_000 + frame * 90_000 / FPS; + for program in 1..=programs { + mux.frame(video_pid(program), packets, pts * 300, pts, frame % GOP == 0); + } + } + mux.out +} + +/// A fresh importer, with the origin driver it needs kept alive but unpolled. +fn setup() -> (moq_mux::container::ts::Programs, impl Sized) { + let (origin, driver) = moq_net::origin::Producer::new(Default::default()); + let programs = moq_mux::container::ts::Programs::new(origin, "bench.hang", moq_mux::catalog::Config::default()); + (programs, driver) +} + +/// Import all of `input` and finish, for the counters. +fn ingest(programs: u16, input: &[u8]) -> (moq_mux::container::ts::stats::Snapshot, impl Sized) { + let (mut importer, driver) = setup(); + importer.decode(input).unwrap(); + let finished = importer.finish().unwrap(); + assert_eq!(finished.len(), usize::from(programs)); + (importer.stats(), (driver, finished)) +} + +fn programs(c: &mut Criterion) { + let mut group = c.benchmark_group("ts_programs"); + group.measurement_time(Duration::from_secs(2)); + group.warm_up_time(Duration::from_millis(300)); + group.sample_size(10); + for programs in [1u16, 4, 16] { + for mbps in [1u64, 4] { + let input = multiplex(programs, mbps * 1_000_000); + let (stats, _driver) = ingest(programs, &input); + assert!( + stats.streams.len() == usize::from(programs) && stats.streams.values().all(|s| s.units == FPS), + "every program delivers every frame" + ); + group.throughput(Throughput::Bytes(input.len() as u64)); + group.bench_function(BenchmarkId::new(format!("{programs}"), format!("{mbps}mbps")), |b| { + b.iter_batched( + setup, + |(mut programs, driver)| { + for chunk in input.chunks(188 * 7) { + programs.decode(chunk).unwrap(); + } + (programs, driver) + }, + BatchSize::PerIteration, + ) + }); + } + } + group.finish(); +} + +criterion_group!(benches, programs); +criterion_main!(benches); diff --git a/rs/moq-mux/src/container/ts/demux.rs b/rs/moq-mux/src/container/ts/demux.rs new file mode 100644 index 00000000000..63cc28374f7 --- /dev/null +++ b/rs/moq-mux/src/container/ts/demux.rs @@ -0,0 +1,154 @@ +//! The packet layer of a TS import, run once however many programs read the input. + +use std::collections::HashMap; + +use bytes::Bytes; +use mpeg2ts::ts::{Pid, TsPacket}; + +use super::catalog::SI_PIDS; +use super::health::{Continuation, Errors, Multiplex}; +use super::import::{Framer, PatReader, SectionReassembler, adaptation_valid, pcr}; +use super::passthrough::{PCR_OFFSET, Sample}; +use super::psi; + +const NULL_PID: u16 = 0x1fff; + +/// Frames the input into packets and reads what every program shares: sync, each PID's +/// continuity, the latest PCR on each PID, the PAT, and the standalone SI sections. +#[derive(Default)] +pub(super) struct Demux { + /// Whole-packet accumulator; a trailing partial packet waits for the next call. + scratch: Vec, + /// The first byte of `scratch` not yet framed. + off: usize, + framer: Framer, + /// The input offset of the first byte in `scratch`. + position: u64, + /// Packets framed so far, every PID and null stuffing included. + packets: u64, + health: Multiplex, + pat: PatReader, + /// PAT sections dropped for a bad CRC. + pat_crc_error: u64, + /// The latest PCR on each PID, so a program whose PMT names its clock starts from the PCR + /// already seen. The 13-bit PID namespace bounds it. + pcrs: HashMap, + /// Reassemblers for the standalone SI PIDs, when anything reads them. + si: Option>, + /// The SI sections the current packet completed. + sections: Vec, +} + +/// One packet, and what the shared layer read out of it. +pub(super) struct Packet<'a> { + pub pkt: &'a [u8; TsPacket::SIZE], + pub pid: u16, + /// The packet's offset in the input. + pub offset: u64, + /// Packets framed through this one, every PID and null stuffing included. + pub count: u64, + /// What it means for the bytes already accumulated on its PID. + pub continuation: Continuation, + /// The whole PAT this packet completed. + pub pat: Option, + /// The SI sections this packet completed. + pub sections: &'a [Bytes], + /// The latest PCR on each PID, through this packet. + pub pcrs: &'a HashMap, +} + +impl Demux { + /// A parser that reassembles the standalone SI PIDs when `si` is set. + pub fn new(si: bool) -> Self { + Self { + si: si.then(HashMap::new), + ..Default::default() + } + } + + /// Append input; [`next`](Self::next) frames it. + pub fn push(&mut self, data: &[u8]) { + self.scratch.extend_from_slice(data); + } + + /// The next whole packet pushed, or `None` once only a partial packet remains, at which + /// point the caller [`drain`](Self::drain)s. + pub fn next(&mut self) -> Option> { + let at = self.framer.next(&self.scratch, &mut self.off)?; + self.health.routed(&self.scratch, at); + self.packets += 1; + let offset = self.position + at as u64; + let pkt: &[u8; TsPacket::SIZE] = self.scratch[at..at + TsPacket::SIZE].try_into().unwrap(); + let pid = (u16::from(pkt[1] & 0x1f) << 8) | u16::from(pkt[2]); + let continuation = self.health.packet(pkt); + + // A retransmission repeats a clock sample rather than taking a new one, and a packet the + // demodulator flagged corrupt has an adaptation field as untrustworthy as its payload. + if continuation != Continuation::Duplicate + && pid != NULL_PID + && pkt[1] & 0x80 == 0 + && adaptation_valid(pkt) + && let Some(ticks) = pcr(pkt) + { + self.pcrs.insert( + pid, + Sample { + at: offset + PCR_OFFSET, + ticks, + }, + ); + } + + let mut pat = None; + self.sections.clear(); + if pid == Pid::PAT { + pat = self.pat.push(pkt, &mut self.pat_crc_error); + } else if let Some(si) = &mut self.si + && SI_PIDS.contains(&pid) + { + let mut sections = Vec::new(); + si.entry(pid).or_default().push(pkt, &mut sections); + self.sections.extend(sections.into_iter().map(Bytes::from)); + } + + Some(Packet { + pkt, + pid, + offset, + count: self.packets, + continuation, + pat, + sections: &self.sections, + pcrs: &self.pcrs, + }) + } + + /// Drop the framed input, keeping a trailing partial packet for the next push. + pub fn drain(&mut self) { + self.health.drained(&self.scratch, self.off); + self.scratch.drain(..self.off); + self.position += self.off as u64; + self.off = 0; + } + + /// `pid` changed route: its next packet starts a fresh chain. + pub fn restart(&mut self, pid: u16) { + self.health.restart(pid); + } + + /// The TR 101 290 counts read off packets alone. + pub fn errors(&self) -> &Errors { + self.health.errors() + } + + /// PAT sections dropped for a bad CRC. + pub fn pat_crc_error(&self) -> u64 { + self.pat_crc_error + } + + /// Input held for the next push: at most a partial packet once drained. + #[cfg(test)] + pub fn held(&self) -> usize { + self.scratch.len() + } +} diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 895d4b725ce..a3d9aab4268 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -4198,7 +4198,9 @@ async fn an_sdt_revision_dropping_the_service_retires_it() { let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone()); capture.select(1); let mut revise = |version: u8, services: &[(u16, &[u8])]| { - capture.section(0x0011, sdt_actual(version, 0, 0, services)).unwrap(); + capture + .section(0x0011, sdt_actual(version, 0, 0, services).into()) + .unwrap(); capture.flush(Timestamp::ZERO, true).unwrap(); catalog .snapshot() @@ -4241,7 +4243,7 @@ async fn a_corrupt_sdt_keeps_the_last_good_snapshot() { let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone()); capture.select(1); let mut feed = |section: Vec| { - capture.section(0x0011, section).unwrap(); + capture.section(0x0011, section.into()).unwrap(); capture.flush(Timestamp::ZERO, true).unwrap(); catalog .snapshot() @@ -4282,7 +4284,7 @@ async fn a_partial_sdt_revision_keeps_the_last_good_snapshot() { let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone()); capture.select(1); let mut feed = |section: Vec| { - capture.section(0x0011, section).unwrap(); + capture.section(0x0011, section.into()).unwrap(); capture.flush(Timestamp::ZERO, true).unwrap(); catalog .snapshot() @@ -4332,7 +4334,7 @@ async fn a_selection_filters_eit_schedule() { make_long_section(0x50, service, 0, 0, 0, &body) }; for service in [1, 2] { - capture.section(0x0012, schedule(service)).unwrap(); + capture.section(0x0012, schedule(service).into()).unwrap(); } capture.finish(Timestamp::ZERO).unwrap(); diff --git a/rs/moq-mux/src/container/ts/health.rs b/rs/moq-mux/src/container/ts/health.rs index eb6e0fbfad2..230ba18bd34 100644 --- a/rs/moq-mux/src/container/ts/health.rs +++ b/rs/moq-mux/src/container/ts/health.rs @@ -1,9 +1,10 @@ //! The ETSI TR 101 290 checks an [`Import`](super::Import) grades its input against. //! //! Counters only, at the fixed limits of TR 101 290 V1.4.1: nothing here decides what is -//! published, and nothing is configured. The importer feeds every packet through -//! [`Health::packet`], and the media PIDs and the clock route on its continuity verdict, so -//! the counter and the PES reassembly cannot disagree about which packets broke a chain. +//! published, and nothing is configured. The parser grades every packet through +//! [`Multiplex::packet`] once, however many programs read it, and the media PIDs and the clock +//! route on its continuity verdict, so the counter and the PES reassembly cannot disagree about +//! which packets broke a chain. Each program times its own tables, PCR and PTS in [`Health`]. //! //! Intervals run on the program clock the importer already keeps //! ([`Liveness`](super::import::Liveness)): the PCR summed step by step, which the 33-bit wrap, @@ -36,8 +37,8 @@ const SYNC_ACQUIRED: u8 = 5; const NULL_PID: u16 = 0x1fff; const SYNC_BYTE: u8 = 0x47; -/// What one importer counted, kept per PID so importers reading the same multiplex can be -/// merged without counting a packet twice. +/// What one checker counted, kept per PID so programs reading the same PID can be merged +/// without counting a packet twice. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(super) struct Errors { ts_sync_loss: u64, @@ -57,8 +58,8 @@ struct PidErrors { } impl Errors { - /// Fold in another importer's count of the same input. Every importer of a multiplex sees - /// every packet, so each check takes the larger count rather than the sum. + /// Fold in another checker's count of the same input. Programs sharing a PAT, PMT or PCR PID + /// time the same packets, so each check takes the larger count rather than the sum. pub(super) fn max(&mut self, other: &Self) { self.ts_sync_loss = self.ts_sync_loss.max(other.ts_sync_loss); self.sync_byte_error = self.sync_byte_error.max(other.sync_byte_error); @@ -102,24 +103,17 @@ impl Errors { } } -/// The TR 101 290 checks on one importer's input. See the [module](self). +/// The TR 101 290 checks that read packets alone: sync and per-PID continuity. One per input, +/// however many programs read it. #[derive(Default)] -pub(super) struct Health { +pub(super) struct Multiplex { errors: Errors, sync: Sync, /// Per PID, every PID but the null PID. continuity: HashMap, - pcr_pid: Option, - /// The last PCR value on the PCR PID, forgotten across a signalled discontinuity. - pcr: Option, - pat: Timer, - /// The PMT PIDs the PAT names for the imported program. - pmt: HashMap, - /// The elementary streams with a PTS cadence, from their first PTS. - pts: HashMap, } -impl Health { +impl Multiplex { pub(super) fn errors(&self) -> &Errors { &self.errors } @@ -129,18 +123,18 @@ impl Health { self.sync.routed(buf, at, &mut self.errors); } - /// The importer is about to drain `buf` up to `off`: check the sync byte positions in the + /// The parser is about to drain `buf` up to `off`: check the sync byte positions in the /// rest of it, which no packet will be routed from, and carry the grid over. pub(super) fn drained(&mut self, buf: &[u8], off: usize) { self.sync.drained(buf, off, &mut self.errors); } - /// Grade one packet, and say what it means for the bytes already accumulated for its PID. + /// Grade one packet's continuity, and say what it means for the bytes already accumulated + /// for its PID. /// /// A packet with `transport_error_indicator` set counts `Transport_error` and nothing else: - /// TR 101 290 derives no further error from it, so it feeds no continuity, table or PCR - /// check. A duplicate feeds none of them either, after its own continuity check. - pub(super) fn packet(&mut self, pkt: &[u8; TsPacket::SIZE], now: Option) -> Continuation { + /// TR 101 290 derives no further error from it. + pub(super) fn packet(&mut self, pkt: &[u8; TsPacket::SIZE]) -> Continuation { let pid = (u16::from(pkt[1] & 0x1f) << 8) | u16::from(pkt[2]); if pkt[1] & 0x80 != 0 { self.errors.pid(pid).transport_error += 1; @@ -153,20 +147,51 @@ impl Health { let continuity = self.continuity.entry(pid).or_default(); let continuation = continuity.observe(pkt); match continuation { - Continuation::Corrupt => return continuation, // ISO/IEC 13818-1 2.4.3.3 permits one duplicate; a third copy is an error. - Continuation::Duplicate => { - if continuity.repeats > 1 { - self.errors.pid(pid).continuity_count_error += 1; - } - return continuation; + Continuation::Duplicate if continuity.repeats > 1 => { + self.errors.pid(pid).continuity_count_error += 1; } Continuation::Broken if !discontinuity_indicator(pkt) => { self.errors.pid(pid).continuity_count_error += 1; } - Continuation::Broken | Continuation::Contiguous => {} + _ => {} } + continuation + } + + /// `pid` changed route: forget the chain behind it, so the new route's first packet is not + /// judged against bytes it never accumulated. + pub(super) fn restart(&mut self, pid: u16) { + self.continuity.remove(&pid); + } +} + +/// The TR 101 290 checks timed on one program's clock: table, PCR and PTS intervals. +#[derive(Default)] +pub(super) struct Health { + errors: Errors, + pcr_pid: Option, + /// The last PCR value on the PCR PID, forgotten across a signalled discontinuity. + pcr: Option, + pat: Timer, + /// The PMT PIDs the PAT names for the imported program. + pmt: HashMap, + /// The elementary streams with a PTS cadence, from their first PTS. + pts: HashMap, +} +impl Health { + pub(super) fn errors(&self) -> &Errors { + &self.errors + } + + /// Time one packet the [`Multiplex`] graded as `continuation`. A corrupt packet or a + /// duplicate feeds no table or PCR check. + pub(super) fn packet(&mut self, pkt: &[u8; TsPacket::SIZE], continuation: Continuation, now: Option) { + if matches!(continuation, Continuation::Corrupt | Continuation::Duplicate) { + return; + } + let pid = (u16::from(pkt[1] & 0x1f) << 8) | u16::from(pkt[2]); let scrambled = pkt[3] & 0xc0 != 0; if pid == Pid::PAT { if scrambled { @@ -188,7 +213,6 @@ impl Health { if self.pcr_pid == Some(pid) { self.pcr(pid, pkt); } - continuation } /// `PCR_repetition_error` and `PCR_discontinuity_indicator_error`, graded on consecutive @@ -267,10 +291,8 @@ impl Health { self.pts.retain(|pid, _| pids.contains(pid)); } - /// `pid` changed route: forget the chain behind it, so the new route's first packet is not - /// judged against bytes it never accumulated, and stop grading its PTS until it has one. + /// `pid` changed route: stop grading its PTS until it has one. pub(super) fn restart(&mut self, pid: u16) { - self.continuity.remove(&pid); self.pts.remove(&pid); } } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 1308354e8ac..a5d8a58ca18 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -25,6 +25,7 @@ use mpeg2ts::ts::{Pid, TsPacket}; use super::adts; use super::catalog; +use super::demux::{Demux, Packet}; use super::health::{Continuation, Continuity, Health}; use super::mux_rate::PCR_WRAP; use super::passthrough::{PCR_HZ, PCR_OFFSET, Sample, locate}; @@ -111,12 +112,18 @@ pub struct Import { /// its offset onto the catalog clock. timebase: crate::catalog::Timebase, - /// The PAT, reassembled off PID 0. - pat: PatReader, + /// Frames the input [`decode`](Self::decode) reads. [`Programs`](super::Programs) runs one + /// parser for every program instead and hands each its packets. + demux: Demux, + /// PIDs whose route changed while handling the current packet, for the parser to restart. + restarts: Vec, + /// Whether a PAT or PMT was handled since [`claims_changed`](Self::claims_changed) last + /// said so. + claims_changed: bool, /// The PMT PIDs PATs have named for the imported program, each with its reassembler. /// Several programs' PMTs may share one. pmt_sections: HashMap, - /// PAT and PMT sections dropped for a CRC mismatch, keyed by PID. + /// PMT sections dropped for a CRC mismatch, keyed by PID. crc_errors: BTreeMap, /// Per elementary-stream-PID codec routing. streams: HashMap>, @@ -134,8 +141,6 @@ pub struct Import { /// Partial clear PES headers, at most 9 + 255 bytes per elementary PID, each with where its /// start code sits on the program clock ([`Placement::span`]). headers: HashMap, Vec)>, - /// The input offset of the first byte in `scratch`, which places every packet in the input. - position: u64, /// The program clock at each input byte, which places whole PES packets and sections. placement: Placement, /// Whole PES and sections dropped for want of a clock to place them, keyed by PID. @@ -147,16 +152,16 @@ pub struct Import { unclocked: HashMap, /// One program-wide budget for PES payloads and sections awaiting their clock. staging: Arc, - /// The TR 101 290 checks, and the per-PID continuity state routing reads. + /// The TR 101 290 checks timed on the program clock. health: Health, /// PID the PMT designates as carrying the program clock reference. An /// adaptation-field `discontinuity_indicator` means a *system time-base* /// discontinuity only here; on any other PID it says nothing but that the /// continuity counter jumped ([`Continuation::Broken`]). pcr_pid: Option, - /// Latest valid PCR on each PID before a PMT names the clock. Exporters may lead - /// their first PAT/PMT with PCR; the 13-bit PID namespace bounds this table. - pcr_candidates: HashMap, + /// The input offset from which a PCR may start the clock a PMT names, while none does. + /// Exporters may lead their first PAT/PMT with PCR, which the parser keeps per PID. + unclocked_since: Option, /// The multiplex rate measured off the PCR PID, recorded in the `mpegts` section /// while the source holds one. Only fed with `mpegts` catalog support. mux_rate: super::mux_rate::Meter, @@ -167,10 +172,6 @@ pub struct Import { /// True once a PMT with at least one supported stream has been parsed. initialized: bool, - /// Whole-packet accumulator. Bytes are routed one TS packet at a time; a trailing - /// partial packet is kept here for the next call. - scratch: Vec, - framer: Framer, /// Section-framed verbatim PIDs. Private sections (SCTE-35 table_id 0xFC and others) /// are not PES, so they are reassembled as sections rather than PES-parsed. Keyed by /// PID. SCTE-35 is detected via the PMT 'CUEI' registration descriptor. @@ -190,10 +191,6 @@ pub struct Import { /// What the PMT in force says of each PID it lists, so a revision that says otherwise ends /// the route rather than carrying the new stream under the old name. listed: HashMap, - /// Reassemblers for the standalone SI PIDs ([`catalog::SI_PIDS`]), keyed by PID, so - /// the service layer survives the round-trip. Only populated with `mpegts` catalog - /// support. - si_sections: HashMap, /// The SI store: buffers each sub-table to a complete generation, publishes the /// per-`(PID, table_id)` snapshot tracks, and advertises them in the catalog. si: super::si::Capture, @@ -241,7 +238,9 @@ impl Import { mux_rate_reservation, held: std::time::Duration::ZERO, held_edge: None, - pat: PatReader::default(), + demux: Demux::new(supports_mpegts), + restarts: Vec::new(), + claims_changed: false, pmt_sections: HashMap::new(), crc_errors: BTreeMap::new(), streams: HashMap::new(), @@ -252,7 +251,6 @@ impl Import { liveness: Liveness::default(), pending: HashMap::new(), headers: HashMap::new(), - position: 0, placement: Placement::default(), unplaced: BTreeMap::new(), refused: BTreeMap::new(), @@ -260,19 +258,16 @@ impl Import { staging: Arc::default(), health: Health::default(), pcr_pid: None, - pcr_candidates: HashMap::new(), + unclocked_since: Some(0), mux_rate: Default::default(), published: false, initialized: false, - scratch: Vec::new(), - framer: Framer::default(), sections: HashMap::new(), supports_mpegts, es_descriptors: HashMap::new(), recorded_media: HashSet::new(), program_info, listed: HashMap::new(), - si_sections: HashMap::new(), si, identity: None, last_pts: None, @@ -319,185 +314,211 @@ impl Import { /// now completes. The buffer is fully consumed; a trailing partial packet /// (< 188 bytes) is retained for the next call. pub fn decode(&mut self, data: &[u8]) -> anyhow::Result<()> { - self.damage_log.flush(moq_clock::now()); - self.scratch.extend_from_slice(data); + let mut demux = std::mem::take(&mut self.demux); + let result = self.decode_with(&mut demux, data); + self.demux = demux; + result + } + fn decode_with(&mut self, demux: &mut Demux, data: &[u8]) -> anyhow::Result<()> { + demux.push(data); // Route one whole packet at a time, so a PMT is parsed (and any PID it declares // registered) before the packets that follow it in the same chunk route. - let mut off = 0; - while let Some(at) = self.framer.next(&self.scratch, &mut off) { - self.health.routed(&self.scratch, at); - let offset = self.position + at as u64; - let pkt: [u8; TsPacket::SIZE] = self.scratch[at..at + TsPacket::SIZE].try_into().unwrap(); - let pid = (((pkt[1] & 0x1f) as u16) << 8) | pkt[2] as u16; - // Every packet paces the multiplex, null stuffing and retransmissions included. - if self.supports_mpegts { - self.mux_rate.packet(); - } - // Every packet is graded, but only the media PIDs and the clock route on the - // verdict: PSI and section PIDs keep their reassemblers' own. - let continuation = self.health.packet(&pkt, self.liveness.now()); - let continuation = Pid::new(pid) - .ok() - .filter(|pid| self.streams.contains_key(pid) || self.pcr_pid == Some(*pid)) - .map(|_| continuation); - // A retransmitted payload must not repeat the clock reset it carried. - if matches!(continuation, Some(Continuation::Duplicate)) { - continue; + loop { + for pid in self.restarts.drain(..) { + demux.restart(pid); } - // A media or PCR packet whose adaptation field overruns itself is refused before - // its clock bits can reach the program clock. Its counter already joined the - // chain, so a gap in front of it still salvages what came before. - if let Ok(pid) = Pid::new(pid) - && (self.streams.contains_key(&pid) || self.pcr_pid == Some(pid)) - && pkt[1] & 0x80 == 0 - && !adaptation_valid(&pkt) - { - if matches!(continuation, Some(Continuation::Broken)) { - self.salvage(pid)?; + let Some(packet) = demux.next() else { break }; + self.packet(&packet)?; + } + demux.drain(); + self.flush_batch() + } + + /// Cut the snapshot groups for whatever SI committed since the last call, and flush the + /// damage log. Batching per decode call (plus the store's own host-clock debounce) + /// coalesces a junction's burst of sub-table commits into few groups instead of one per + /// commit. + pub(super) fn flush_batch(&mut self) -> anyhow::Result<()> { + self.damage_log.flush(moq_clock::now()); + self.si.flush(self.last_pts.unwrap_or(Timestamp::ZERO), false) + } + + /// The PIDs this program reads besides the PAT and the standalone SI PIDs: its PMTs, its + /// elementary streams and its clock. + pub(super) fn claims(&self) -> Vec { + let mut pids: Vec = self + .pmt_sections + .keys() + .chain(self.streams.keys()) + .chain(self.pcr_pid.iter()) + .map(Pid::as_u16) + .chain(self.sections.keys().copied()) + .collect(); + pids.sort_unstable(); + pids.dedup(); + pids + } + + /// Whether [`claims`](Self::claims) may have changed since the last call. + pub(super) fn claims_changed(&mut self) -> bool { + std::mem::take(&mut self.claims_changed) + } + + /// Drain the PIDs whose route changed since the last call, for the parser to restart. + pub(super) fn restarts(&mut self) -> std::vec::Drain<'_, u16> { + self.restarts.drain(..) + } + + /// Route one packet the parser framed. Packets on PIDs this program does not + /// [`claim`](Self::claims) may be skipped, except the PAT's and the SI PIDs' whole sections. + pub(super) fn packet(&mut self, packet: &Packet) -> anyhow::Result<()> { + let Packet { + pkt, + pid, + offset, + continuation, + .. + } = *packet; + // Every packet is graded, but only the media PIDs and the clock route on the + // verdict: PSI and section PIDs keep their reassemblers' own. + self.health.packet(pkt, continuation, self.liveness.now()); + let continuation = Pid::new(pid) + .ok() + .filter(|pid| self.streams.contains_key(pid) || self.pcr_pid == Some(*pid)) + .map(|_| continuation); + // A retransmitted payload must not repeat the clock reset it carried. + if matches!(continuation, Some(Continuation::Duplicate)) { + return Ok(()); + } + // A media or PCR packet whose adaptation field overruns itself is refused before + // its clock bits can reach the program clock. Its counter already joined the + // chain, so a gap in front of it still salvages what came before. + if let Ok(pid) = Pid::new(pid) + && (self.streams.contains_key(&pid) || self.pcr_pid == Some(pid)) + && pkt[1] & 0x80 == 0 + && !adaptation_valid(pkt) + { + if matches!(continuation, Some(Continuation::Broken)) { + self.salvage(pid)?; + } + return self.damage(pid, &anyhow::anyhow!("malformed TS adaptation field")); + } + // Read the clock's own flag before routing, so the break lands between the media + // either side of it. The PCR PID can be a dedicated one nothing below routes, and + // the flag rides an adaptation-only packet as readily as a payload one. + // A packet the demodulator flagged corrupt (`transport_error_indicator`) is not + // read: its adaptation field is as untrustworthy as its payload, and taking a bit + // out of it would break every track in the program on line noise. + if self.pcr_pid.is_some_and(|p| p.as_u16() == pid) && pkt[1] & 0x80 == 0 { + if discontinuity_indicator(pkt) { + // A PES this packet starts ends the one before it on its PID whole, on the old + // clock; only the other PIDs' are cut short by the break. An adaptation-only + // packet starts none whatever its PUSI says, as `pes_packet` treats it. + if pkt[1] & 0x40 != 0 + && matches!(payload(pkt), Payload::Bytes(_)) + && let Ok(pid) = Pid::new(pid) + { + self.flush(pid)?; } - self.damage(pid, &anyhow::anyhow!("malformed TS adaptation field"))?; - continue; - } - if self.pcr_pid.is_none() - && pid != 0x1fff - && pkt[1] & 0x80 == 0 - && adaptation_valid(&pkt) - && let Some(ticks) = pcr(&pkt) - { - self.pcr_candidates.insert( - Pid::new(pid)?, - Sample { - at: offset + PCR_OFFSET, - ticks, - }, - ); - } - // Read the clock's own flag before routing, so the break lands between the media - // either side of it. The PCR PID can be a dedicated one nothing below routes, and - // the flag rides an adaptation-only packet as readily as a payload one. - // A packet the demodulator flagged corrupt (`transport_error_indicator`) is not - // read: its adaptation field is as untrustworthy as its payload, and taking a bit - // out of it would break every track in the program on line noise. - if self.pcr_pid.is_some_and(|p| p.as_u16() == pid) && pkt[1] & 0x80 == 0 { - if discontinuity_indicator(&pkt) { - // A PES this packet starts ends the one before it on its PID whole, on the old - // clock; only the other PIDs' are cut short by the break. An adaptation-only - // packet starts none whatever its PUSI says, as `pes_packet` treats it. - if pkt[1] & 0x40 != 0 - && matches!(payload(&pkt), Payload::Bytes(_)) - && let Ok(pid) = Pid::new(pid) - { - self.flush(pid)?; - } - self.timebase_break()?; - // The new timebase starts with the PCR the flag rides, if it rides one. - if let Some(pcr) = pcr(&pkt) { - self.place_pcr(pcr, offset)?; - } - } else if let Some(pcr) = pcr(&pkt) { + self.timebase_break()?; + // The new timebase starts with the PCR the flag rides, if it rides one. + if let Some(pcr) = pcr(pkt) { self.place_pcr(pcr, offset)?; - self.liveness.pcr(pcr); - if let Some(now) = self.liveness.now() { - self.health.tick(now); + } + } else if let Some(pcr) = pcr(pkt) { + self.place_pcr(pcr, offset)?; + self.liveness.pcr(pcr); + if let Some(now) = self.liveness.now() { + self.health.tick(now); + } + if self.supports_mpegts { + let ready = self.mux_rate.ready(); + if self.mux_rate.pcr(pcr, packet.count) { + self.record_mux_rate()?; } - if self.supports_mpegts { - let ready = self.mux_rate.ready(); - if self.mux_rate.pcr(pcr) { - self.record_mux_rate()?; - } - if self.mux_rate.ready() { - self.mux_rate_reservation = None; - } else if ready { - // A backwards clock voided the measurement. - self.rearm_mux_rate(); - } + if self.mux_rate.ready() { + self.mux_rate_reservation = None; + } else if ready { + // A backwards clock voided the measurement. + self.rearm_mux_rate(); } } } - if let Some(section) = self.sections.get_mut(&pid) { - let unplaced = section.packet(&pkt, offset, &self.placement)?; - let (units, dropped) = section.release(&self.timebase, &self.pts, self.placement.generation)?; - let unplaced = unplaced + dropped; - if unplaced > 0 { - *self.unplaced.entry(pid).or_default() += unplaced; - } - self.published |= units > 0; - self.liveness.delivered(pid, units); - continue; + } + if let Some(section) = self.sections.get_mut(&pid) { + let unplaced = section.packet(pkt, offset, &self.placement)?; + let (units, dropped) = section.release(&self.timebase, &self.pts, self.placement.generation)?; + let unplaced = unplaced + dropped; + if unplaced > 0 { + *self.unplaced.entry(pid).or_default() += unplaced; } - // The standalone SI PIDs carry the service layer, not media, and no PAT or PMT - // names them. - if self.supports_mpegts && catalog::SI_PIDS.contains(&pid) { - self.si_section(pid, &pkt)?; - continue; + self.published |= units > 0; + self.liveness.delivered(pid, units); + return Ok(()); + } + // The standalone SI PIDs carry the service layer, not media, and no PAT or PMT + // names them. The parser reassembled their sections, each captured whatever its + // `table_id`. + if self.supports_mpegts && catalog::SI_PIDS.contains(&pid) { + for section in packet.sections { + self.si.section(pid, section.clone())?; } - // An elementary stream's PES is as vulnerable to a break as a section is. A - // looping publisher wraps with its last PES still open and short of its declared - // length, so the next loop's leading packets would otherwise complete it and hand - // the codec one buffer straddling the cut. - if let Ok(pid) = Pid::new(pid) - && self.streams.contains_key(&pid) - { - match continuation.expect("media PID continuity was classified") { - Continuation::Duplicate => continue, - // Flagged corrupt, so the packet joins the partial rather than opening a - // new PES out of bytes the demodulator already disowned. - Continuation::Corrupt => { - self.damage(pid, &anyhow::anyhow!("transport error indicator"))?; - continue; - } - Continuation::Broken => { - self.broken(pid)?; - // This packet still routes normally: a PUSI opens a fresh PES, while a - // continuation finds no pending entry and is dropped, so the stream - // resumes at the next PES start rather than mid-frame. - } - Continuation::Contiguous => {} + return Ok(()); + } + // An elementary stream's PES is as vulnerable to a break as a section is. A + // looping publisher wraps with its last PES still open and short of its declared + // length, so the next loop's leading packets would otherwise complete it and hand + // the codec one buffer straddling the cut. + if let Ok(pid) = Pid::new(pid) + && self.streams.contains_key(&pid) + { + match continuation.expect("media PID continuity was classified") { + Continuation::Duplicate => return Ok(()), + // Flagged corrupt, so the packet joins the partial rather than opening a + // new PES out of bytes the demodulator already disowned. + Continuation::Corrupt => { + self.damage(pid, &anyhow::anyhow!("transport error indicator"))?; + return Ok(()); } - } - let Ok(pid) = Pid::new(pid) else { - continue; - }; - if pid.as_u16() == Pid::PAT { - self.pat_packet(&pkt)?; - } else if self.pmt_sections.contains_key(&pid) { - self.pmt_packet(pid, &pkt)?; - } else { - // PIDs we don't decode and don't carry (`Stream::Ignored`: a base catalog's - // undecoded streams, or an ambiguous 0x86 PID without CUEI) are dropped, as is - // every PID before the PSI that declares it: a live capture joins mid-stream, - // so PES arrive before their PMT. - match self.streams.get(&pid) { - None | Some(Stream::Ignored) => {} - Some(_) => self.pes_packet(pid, &pkt, offset)?, + Continuation::Broken => { + self.broken(pid)?; + // This packet still routes normally: a PUSI opens a fresh PES, while a + // continuation finds no pending entry and is dropped, so the stream + // resumes at the next PES start rather than mid-frame. } + Continuation::Contiguous => {} } } - - self.health.drained(&self.scratch, off); - self.scratch.drain(..off); - self.position += off as u64; - // Cut the snapshot groups for whatever SI committed in this batch. Batching per - // decode call (plus the store's own host-clock debounce) coalesces a junction's - // burst of sub-table commits into few groups instead of one per commit. - self.si.flush(self.last_pts.unwrap_or(Timestamp::ZERO), false)?; - Ok(()) - } - - /// Feed one packet on PID 0 to the PAT's reassembler, applying every whole table. - fn pat_packet(&mut self, pkt: &[u8; TsPacket::SIZE]) -> anyhow::Result<()> { - let crc_error = self.crc_errors.entry(Pid::PAT).or_default(); - if let Some(pat) = self.pat.push(pkt, crc_error) { - self.handle_pat(&pat)?; + let Ok(pid) = Pid::new(pid) else { + return Ok(()); + }; + if pid.as_u16() == Pid::PAT { + if let Some(pat) = &packet.pat { + self.handle_pat(pat)?; + } + } else if self.pmt_sections.contains_key(&pid) { + self.pmt_packet(pid, packet)?; + } else { + // PIDs we don't decode and don't carry (`Stream::Ignored`: a base catalog's + // undecoded streams, or an ambiguous 0x86 PID without CUEI) are dropped, as is + // every PID before the PSI that declares it: a live capture joins mid-stream, + // so PES arrive before their PMT. + match self.streams.get(&pid) { + None | Some(Stream::Ignored) => {} + Some(_) => self.pes_packet(pid, pkt, offset)?, + } } Ok(()) } /// Feed one packet on a PMT PID to its reassembler, applying every good PMT section. - fn pmt_packet(&mut self, pid: Pid, pkt: &[u8; TsPacket::SIZE]) -> anyhow::Result<()> { + fn pmt_packet(&mut self, pid: Pid, packet: &Packet) -> anyhow::Result<()> { let mut sections = Vec::new(); - self.pmt_sections.entry(pid).or_default().push(pkt, &mut sections); + self.pmt_sections + .entry(pid) + .or_default() + .push(packet.pkt, &mut sections); for section in sections { // Other tables may share the PID; only a PMT is this path's to check. if section.first() != Some(&psi::PMT_TABLE_ID) { @@ -509,12 +530,33 @@ impl Import { } if let Some(pmt) = Pmt::parse(§ion) { self.handle_pmt(pmt)?; + self.seed_clock(packet)?; } } Ok(()) } - /// A PAT or PMT section failed its CRC: it is dropped whole, and the table already in + /// Start a clock a PMT just named from its latest PCR, if one arrived since the program + /// last had none, or note from when one may start it. + fn seed_clock(&mut self, packet: &Packet) -> anyhow::Result<()> { + match self.pcr_pid { + Some(pid) => { + if let Some(since) = self.unclocked_since.take() + && let Some(&sample) = packet.pcrs.get(&pid.as_u16()) + && sample.at - PCR_OFFSET >= since + { + self.place_pcr(sample.ticks, sample.at - PCR_OFFSET)?; + } + } + None => { + self.unclocked_since + .get_or_insert(packet.offset + TsPacket::SIZE as u64); + } + } + Ok(()) + } + + /// A PMT section failed its CRC: it is dropped whole, and the table already in /// force stays. fn crc_error(&mut self, pid: Pid) { *self.crc_errors.entry(pid.as_u16()).or_default() += 1; @@ -526,6 +568,7 @@ impl Import { if self.program.is_some_and(|program| program != pmt.program_number) { return Ok(()); } + self.claims_changed = true; let mut seen = HashSet::new(); if let Some(es) = pmt.streams.iter().find(|es| !seen.insert(es.pid)) { anyhow::bail!("the PMT lists PID {} twice", es.pid.as_u16()); @@ -630,13 +673,6 @@ impl Import { ) ) }); - if let Some(pid) = self.pcr_pid { - let initial = self.pcr_candidates.remove(&pid); - self.pcr_candidates.clear(); - if let Some(sample) = initial { - self.place_pcr(sample.ticks, sample.at - PCR_OFFSET)?; - } - } let pids: Vec = pmt.streams.iter().map(|es| es.pid.as_u16()).collect(); self.health.pmt_streams(&pids); Ok(()) @@ -860,6 +896,7 @@ impl Import { } self.streams.insert(pid, stream); self.health.restart(pid.as_u16()); + self.restarts.push(pid.as_u16()); Ok(()) } @@ -896,6 +933,7 @@ impl Import { self.pending.remove(&pid); self.headers.remove(&pid); self.health.restart(pid.as_u16()); + self.restarts.push(pid.as_u16()); if !self.supports_mpegts { // Always route to Ignored, replacing any prior codec on this PID (a later PMT // can reassign it), so a private section is never PES-parsed. Warn once. @@ -1516,6 +1554,7 @@ impl Import { ); } self.pmt_sections.entry(entry.pmt_pid).or_default(); + self.claims_changed = true; self.health.pmt_pids(&[entry.pmt_pid.as_u16()], self.liveness.now()); if self.identity.is_none() { self.identity = Some(identity); @@ -1539,25 +1578,6 @@ impl Import { Ok(()) } - /// Feed one TS packet on a standalone SI PID to its reassembler, folding each - /// completed section into the SI store. - /// - /// Every section is captured, whatever its `table_id`: the SDT PID also carries - /// the BAT, the EIT PID carries now/next and the schedule, and a table we don't - /// recognize is exactly as worth preserving as one we do. A selected program is the - /// exception: its SI describes that service alone ([`with_program`](Self::with_program)). - /// The store buffers each sub-table to a complete generation and commits it atomically, - /// so a plain repetition (SI repeats every couple of seconds) publishes nothing and a - /// torn multi-section transition is never visible. - fn si_section(&mut self, pid: u16, pkt: &[u8]) -> anyhow::Result<()> { - let mut sections = Vec::new(); - self.si_sections.entry(pid).or_default().push(pkt, &mut sections); - for section in sections { - self.si.section(pid, section)?; - } - Ok(()) - } - /// Close the current group on every track and reopen at `sequence`. pub fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { for stream in self.streams.values_mut() { @@ -1648,20 +1668,22 @@ impl Import { } let mut stats = stats::Snapshot { streams, - crc_error: self.crc_errors.values().sum(), + crc_error: self.crc_errors.values().sum::() + self.demux.pat_crc_error(), ..Default::default() }; - self.health.errors().report(&mut stats); + let mut errors = self.demux.errors().clone(); + errors.max(self.health.errors()); + errors.report(&mut stats); stats } - /// The TR 101 290 counts behind [`Self::stats`], for merging importers that read the same + /// The TR 101 290 counts this program timed, for merging programs that read the same /// multiplex. pub(super) fn errors(&self) -> &super::health::Errors { self.health.errors() } - /// [`stats::Snapshot::crc_error`] per PID, for merging importers that read the same PSI. + /// PMT sections dropped for a bad CRC per PID, for merging programs that read the same PMT. pub(super) fn crc_errors(&self) -> &BTreeMap { &self.crc_errors } @@ -7356,7 +7378,8 @@ pub(super) mod test { import.stats().streams[&0x0061].units > units, "the stream kept delivering" ); - assert_eq!(import.crc_errors(), &BTreeMap::from([(0x0000, 1), (0x0100, 1)])); + assert_eq!(import.crc_errors(), &BTreeMap::from([(0x0100, 1)])); + assert_eq!(import.demux.pat_crc_error(), 1); assert_eq!(import.stats().crc_error, 2); let mut data = psi_packets(0x0100, &mut pmt_cc, &[], &pmt_section(1, 2, &two)); @@ -8432,7 +8455,7 @@ pub(super) mod test { import .decode(&mux.out[..resume_at]) .expect("damage must stay local to the unit"); - assert!(import.scratch.is_empty(), "every damaged packet must be consumed"); + assert_eq!(import.demux.held(), 0, "every damaged packet must be consumed"); assert!( !import.pending.contains_key(&Pid::new(VIDEO).unwrap()), "no damaged PES remains" diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index 94a424abac5..61a4138ee1a 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -25,6 +25,7 @@ mod adts; mod damage; +mod demux; mod export; mod follower; mod health; diff --git a/rs/moq-mux/src/container/ts/mux_rate.rs b/rs/moq-mux/src/container/ts/mux_rate.rs index 84d37401451..c04035e56ef 100644 --- a/rs/moq-mux/src/container/ts/mux_rate.rs +++ b/rs/moq-mux/src/container/ts/mux_rate.rs @@ -2,8 +2,8 @@ //! //! A constant-rate multiplexer paces its packets on the PCR clock, so the packets //! between two PCRs over the ticks between them is the rate of the whole -//! multiplex, stuffing included. [`Meter`] takes every packet and every PCR on -//! the clock PID and decides when that rate is worth recording: a full window +//! multiplex, stuffing included. [`Meter`] takes every PCR on the clock PID, with +//! the input's packet count at it, and decides when that rate is worth recording: a full window //! whose samples agree on one value publishes it, and a window that no longer //! agrees clears it, so a VBR source (whose samples never agree) records nothing //! and a rate change lands once rather than as churn. @@ -66,10 +66,10 @@ fn rate(packets: u64, ticks: u64) -> u64 { /// Tracks the multiplex rate and the value currently recorded for it. #[derive(Default)] pub(super) struct Meter { - /// Packets since the last PCR, every PID and null stuffing included. - packets: u64, /// The last PCR seen, in 27 MHz ticks. last: Option, + /// The input's packet count at `last`. + mark: u64, /// The sample being pooled from PCR intervals. sample: Interval, /// The completed samples of the window being collected. @@ -91,15 +91,11 @@ impl Meter { self.ready } - /// One transport packet went by. - pub fn packet(&mut self) { - self.packets += 1; - } - - /// A PCR arrived on the clock PID, carried by the packet just counted. Returns - /// whether [`published`](Self::published) changed. - pub fn pcr(&mut self, pcr: u64) -> bool { - let packets = std::mem::take(&mut self.packets); + /// A PCR arrived on the clock PID, carried by packet number `packets` of the input, + /// counting every PID and null stuffing. Returns whether [`published`](Self::published) + /// changed. + pub fn pcr(&mut self, pcr: u64, packets: u64) -> bool { + let packets = packets - std::mem::replace(&mut self.mark, packets); let Some(last) = self.last.replace(pcr) else { return false; }; @@ -126,7 +122,6 @@ impl Meter { /// Returns whether [`published`](Self::published) changed. pub fn discontinuity(&mut self) -> bool { self.ready = false; - self.packets = 0; self.last = None; self.sample = Interval::default(); self.window.clear(); @@ -178,16 +173,15 @@ mod test { fn feed(meter: &mut Meter, intervals: impl IntoIterator) -> Vec> { let mut changes = Vec::new(); let mut pcr = meter.last.unwrap_or(0); + let mut count = meter.mark; if meter.last.is_none() { - meter.packet(); - assert!(!meter.pcr(pcr)); + count += 1; + assert!(!meter.pcr(pcr, count)); } for (packets, ticks) in intervals { - for _ in 0..packets { - meter.packet(); - } + count += packets; pcr = (pcr + ticks) % PCR_WRAP; - if meter.pcr(pcr) { + if meter.pcr(pcr, count) { changes.push(meter.published()); } } @@ -272,8 +266,7 @@ mod test { feed(&mut meter, std::iter::repeat_n(CBR, PER_WINDOW)), vec![Some(2_499_999)] ); - meter.packet(); - assert!(meter.pcr(1), "the clock jumped backwards"); + assert!(meter.pcr(1, meter.mark + 1), "the clock jumped backwards"); assert_eq!(meter.published(), None); } diff --git a/rs/moq-mux/src/container/ts/passthrough.rs b/rs/moq-mux/src/container/ts/passthrough.rs index 07542da0c9c..94645ff3e00 100644 --- a/rs/moq-mux/src/container/ts/passthrough.rs +++ b/rs/moq-mux/src/container/ts/passthrough.rs @@ -452,7 +452,6 @@ impl Passthrough { { self.pmt_packet(pmt_pid, pkt)?; } - self.mux_rate.packet(); // A packet the demodulator flagged corrupt, or whose adaptation field overruns it, is // carried like any other, but nothing is read from it. @@ -516,7 +515,7 @@ impl Passthrough { samples: VecDeque::from([Sample { at, ticks: raw }]), }); self.waiting_since = self.mapped.then_some(raw); - if self.mux_rate.pcr(raw) { + if self.mux_rate.pcr(raw, self.offset / TsPacket::SIZE as u64) { self.record()?; } return Ok(None); @@ -546,7 +545,7 @@ impl Passthrough { if self.mux_rate.discontinuity() { self.record()?; } - if self.mux_rate.pcr(raw) { + if self.mux_rate.pcr(raw, self.offset / TsPacket::SIZE as u64) { self.record()?; } // The jump is not elapsed time, so the wait for the first group starts over. @@ -563,7 +562,7 @@ impl Passthrough { timebase.samples.pop_front(); } self.drain_queue()?; - if self.mux_rate.pcr(raw) { + if self.mux_rate.pcr(raw, self.offset / TsPacket::SIZE as u64) { self.record()?; } diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index e48dd5780ce..0277618c3b0 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -3,10 +3,10 @@ use std::collections::BTreeMap; use anyhow::Context; -use mpeg2ts::ts::{Pid, TsPacket}; +use mpeg2ts::ts::Pid; -use super::import::{Framer, PatReader}; -use super::{Ext, Import, psi, stats}; +use super::demux::Demux; +use super::{Ext, Import, catalog::SI_PIDS, psi, stats}; use crate::catalog; /// Imports every program the first PAT lists, each as its own broadcast on `origin`, with its @@ -16,16 +16,21 @@ use crate::catalog; /// format detection still sees it; a name without one gains a plain `/n`. Input ahead of that /// PAT is dropped, as a single [`Import`] drops media ahead of its PSI. A program a later PAT /// adds is not published; one it removes ends the import. +/// +/// The multiplex is parsed once: framing, continuity, the PAT and the SI sections are shared, +/// and each program reads only the packets on the PIDs its PAT entry and PMT claim. pub struct Programs { origin: moq_net::origin::Producer, name: moq_net::PathOwned, config: catalog::Config, - /// Input not yet read for the PAT: a trailing partial packet, or what follows the PAT. - pending: Vec, - /// Reads the PAT out of the input. - scan: PatScan, + demux: Demux, /// Empty until the PAT arrives. programs: Vec, + /// The programs reading each PID they claim, indexed by PID, in program order. The PAT and + /// the SI PIDs go to every program instead. + readers: Vec>, + /// The programs the current [`decode`](Self::decode) call routed a packet to. + touched: Vec, } /// One program's broadcast. @@ -39,6 +44,10 @@ struct Program { /// The route and catalog track until the catalog first names a servable track: announced any /// sooner, a relay would ask for tracks it cannot serve yet. pending: Option<(moq_net::origin::Route, moq_net::track::Consumer)>, + /// The PIDs this program is listed under in [`Programs::readers`], sorted. + claims: Vec, + /// Whether the current [`Programs::decode`] call routed it a packet. + touched: bool, } impl Program { @@ -68,21 +77,75 @@ impl Programs { origin, name: name.as_path().to_owned(), config, - pending: Vec::new(), - scan: PatScan::default(), + demux: Demux::new(true), programs: Vec::new(), + readers: vec![Vec::new(); 1 << 13], + touched: Vec::new(), } } /// Demux a chunk of the multiplex. A trailing partial packet is retained for the next call. pub fn decode(&mut self, data: &[u8]) -> anyhow::Result<()> { - if !self.programs.is_empty() { - return self.feed(data); + let mut demux = std::mem::take(&mut self.demux); + let result = self.decode_with(&mut demux, data); + self.demux = demux; + result + } + + fn decode_with(&mut self, demux: &mut Demux, data: &[u8]) -> anyhow::Result<()> { + demux.push(data); + while let Some(packet) = demux.next() { + if self.programs.is_empty() { + match &packet.pat { + Some(pat) if !pat.program_numbers().is_empty() => self.start(pat)?, + _ => continue, + } + } + // Every program reads the PAT, and the SI sections the parser reassembled. + let everyone = packet.pid == Pid::PAT || (SI_PIDS.contains(&packet.pid) && !packet.sections.is_empty()); + let readers = match everyone { + true => None, + false => match self.readers[usize::from(packet.pid)].as_slice() { + [] => continue, + readers => Some(readers), + }, + }; + let mut changed = Vec::new(); + for at in 0..readers.map_or(self.programs.len(), <[usize]>::len) { + let index = readers.map_or(at, |readers| readers[at]); + let program = &mut self.programs[index]; + if !std::mem::replace(&mut program.touched, true) { + self.touched.push(index); + } + let import = &mut program.import; + import.packet(&packet)?; + if import.claims_changed() { + changed.push(index); + } + } + drop(packet); + // Only a PAT or PMT moves claims or restarts a route. + for index in changed { + for pid in self.programs[index].import.restarts() { + demux.restart(pid); + } + self.reclaim(index); + } } - self.pending.extend_from_slice(data); - let Some(pat) = self.scan.scan(&mut self.pending) else { - return Ok(()); - }; + demux.drain(); + // Only a program that read something has anything to flush or announce, so a call + // costs what it carried rather than one pass over every program. + for index in self.touched.drain(..) { + let program = &mut self.programs[index]; + program.touched = false; + program.import.flush_batch()?; + program.announce_if_listed()?; + } + Ok(()) + } + + /// Create a broadcast for every program `pat` lists. + fn start(&mut self, pat: &psi::Pat) -> anyhow::Result<()> { for program in pat.program_numbers() { let name = program_broadcast(self.name.as_str(), program); let mut broadcast = self @@ -96,10 +159,7 @@ impl Programs { .clone() .with_catalog(catalog::hang::Catalog::::default()); let catalog = catalog::Producer::new(&mut broadcast, config)?; - let mut import = Import::new(broadcast.clone(), catalog.reserve()).with_program(program); - // Seeded with the scanner's PAT rather than replaying its packets, so every section - // reaches the importer and nothing the scanner counted is counted again. - import.handle_pat(&pat)?; + let import = Import::new(broadcast.clone(), catalog.reserve()).with_program(program); // Each program is its own publisher instance for this run, so a restart // replaces it rather than resuming into it. let route = moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); @@ -110,29 +170,37 @@ impl Programs { import, catalog, pending: Some((route, listed)), + claims: Vec::new(), + touched: false, }); } - let pending = std::mem::take(&mut self.pending); - self.feed(&pending) + Ok(()) } - fn feed(&mut self, data: &[u8]) -> anyhow::Result<()> { - for program in &mut self.programs { - program.import.decode(data)?; - program.announce_if_listed()?; + /// Move program `index` in [`Self::readers`] to the PIDs it claims now. + fn reclaim(&mut self, index: usize) { + let program = &mut self.programs[index]; + let claims = program.import.claims(); + for &pid in program.claims.iter().filter(|pid| claims.binary_search(pid).is_err()) { + self.readers[usize::from(pid)].retain(|&reader| reader != index); } - Ok(()) + for &pid in claims.iter().filter(|pid| program.claims.binary_search(pid).is_err()) { + let readers = &mut self.readers[usize::from(pid)]; + let at = readers.partition_point(|&reader| reader < index); + readers.insert(at, index); + } + program.claims = claims; } /// Every program's counters in one map: PIDs are unique across a multiplex. /// - /// Every importer reads the same PAT, and programs sharing a PMT PID read the same PMT - /// sections, so a dropped section counts once however many read it. Every importer also - /// grades every packet of the multiplex, so each TR 101 290 error counts once too. + /// The multiplex is graded once. Programs sharing a PMT PID read the same PMT sections, so + /// a dropped section counts once however many read it, and programs sharing a PCR PID + /// count its faults once too. pub fn stats(&self) -> stats::Snapshot { let mut stats = stats::Snapshot::default(); let mut crc_errors = BTreeMap::::new(); - let mut errors = super::health::Errors::default(); + let mut errors = self.demux.errors().clone(); for program in &self.programs { stats.streams.extend(program.import.stats().streams); for (&pid, &count) in program.import.crc_errors() { @@ -141,9 +209,7 @@ impl Programs { } errors.max(program.import.errors()); } - // The importers only see the PATs after the one that started them. - *crc_errors.entry(Pid::PAT).or_default() += self.scan.crc_error; - stats.crc_error = crc_errors.values().sum(); + stats.crc_error = crc_errors.values().sum::() + self.demux.pat_crc_error(); errors.report(&mut stats); stats } @@ -187,39 +253,6 @@ fn program_broadcast(name: &str, program: u16) -> String { } } -/// Finds the first whole PAT listing a program, before any importer exists to read one. -#[derive(Default)] -struct PatScan { - framer: Framer, - pat: PatReader, - /// PAT sections dropped for a bad CRC before the first whole PAT. - crc_error: u64, -} - -impl PatScan { - /// Read `pending` up to the first whole PAT listing a program, returning it with `pending` - /// trimmed to start just after the packet that completed it. Otherwise only a trailing - /// partial packet is kept: the reader holds a partial PAT itself. - fn scan(&mut self, pending: &mut Vec) -> Option { - let mut off = 0; - let mut found = None; - while let Some(at) = self.framer.next(pending, &mut off) { - let pkt: &[u8; TsPacket::SIZE] = pending[at..at + TsPacket::SIZE].try_into().unwrap(); - if (u16::from(pkt[1] & 0x1f) << 8 | u16::from(pkt[2])) != Pid::PAT { - continue; - } - if let Some(pat) = self.pat.push(pkt, &mut self.crc_error) - && !pat.program_numbers().is_empty() - { - found = Some(pat); - break; - } - } - pending.drain(..off); - found - } -} - #[cfg(test)] mod test { use std::time::Duration; @@ -258,9 +291,16 @@ mod test { assert_eq!(program_broadcast("event", 1), "event/1"); } - /// The PAT in a whole buffer, as one [`PatScan`] reads it. + /// The first whole PAT in a buffer, as the parser reads it. fn pat_programs(data: &[u8]) -> Option> { - Some(PatScan::default().scan(&mut data.to_vec())?.program_numbers()) + let mut demux = Demux::new(false); + demux.push(data); + while let Some(packet) = demux.next() { + if let Some(pat) = packet.pat { + return Some(pat.program_numbers()); + } + } + None } #[test] @@ -274,7 +314,7 @@ mod test { assert_eq!(pat_programs(&data[..100]), None); } - /// A PAT too big for one packet is read whole, and each importer is handed all of it, so + /// A PAT too big for one packet is read whole, and each program is handed all of it, so /// the program listed in its second packet publishes. #[tokio::test] async fn a_pat_spanning_two_packets_starts_every_program() { @@ -334,7 +374,7 @@ mod test { catalog.audio.renditions.len() } - /// A PAT in two sections starts both programs, each importer seeded with the whole table + /// A PAT in two sections starts both programs, each handed the whole table /// rather than only the section that completed it. #[tokio::test] async fn a_pat_in_two_sections_starts_every_program() { @@ -444,11 +484,7 @@ mod test { programs.decode(&null).unwrap(); } assert!(programs.programs.is_empty()); - assert!( - programs.pending.len() < 2 * 188, - "held {} bytes", - programs.pending.len() - ); + assert!(programs.demux.held() < 188, "held {} bytes", programs.demux.held()); } /// A corrupt PAT section sharing a packet with the good one that starts the programs is diff --git a/rs/moq-mux/src/container/ts/si.rs b/rs/moq-mux/src/container/ts/si.rs index faf7e3535d0..82b7c92f606 100644 --- a/rs/moq-mux/src/container/ts/si.rs +++ b/rs/moq-mux/src/container/ts/si.rs @@ -418,7 +418,7 @@ impl Capture { /// Fold one completed section from `pid` into its `(pid, table_id)` entry, /// creating the entry on first sight of the pair. - pub fn section(&mut self, pid: u16, section: Vec) -> anyhow::Result<()> { + pub fn section(&mut self, pid: u16, section: Bytes) -> anyhow::Result<()> { let Some(&table_id) = section.first() else { return Ok(()); }; @@ -426,7 +426,6 @@ impl Capture { // state, not the current one; carrying it would need a second generation slot // per sub-table for a table nothing re-emits, so it is dropped until the // version becomes current. - let section = Bytes::from(section); if long_form(§ion) && section[5] & 0x01 == 0 { return Ok(()); } From 3bcceff048c66b8ae9ef6e52c3bb27e6c9d3ac5c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 16:27:26 -0700 Subject: [PATCH 3/5] fix(mux): keep a shared PID's continuity chain when another program claims it The shared demux tracks one continuity chain per PID. A program registering a PID another program already reads reset that chain, so a duplicate packet right after the PMT read as contiguous and a counter gap went uncounted for the existing reader. Only the PID's sole reader restarts it now. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 2 +- rs/moq-mux/src/container/ts/programs.rs | 37 +++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index ada4d983988..34efcdfee50 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -7226,7 +7226,7 @@ pub(super) mod test { } /// A PMT section listing `(stream_type, pid, es_descriptors)`, the clock on the first. - fn pmt_section(program: u16, version: u8, streams: &[(u8, u16, &[u8])]) -> Vec { + pub(in crate::container::ts) fn pmt_section(program: u16, version: u8, streams: &[(u8, u16, &[u8])]) -> Vec { let pcr = streams.first().map_or(0x1fff, |&(_, pid, _)| pid); let mut body = vec![0xe0 | (pcr >> 8) as u8, pcr as u8, 0xf0, 0x00]; for &(stream_type, pid, descriptors) in streams { diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index 0277618c3b0..1c51b65dd2a 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -127,7 +127,11 @@ impl Programs { // Only a PAT or PMT moves claims or restarts a route. for index in changed { for pid in self.programs[index].import.restarts() { - demux.restart(pid); + // Another program still reading the PID judges its partial bytes against + // this chain, so only the PID's sole reader restarts it. + if self.readers[usize::from(pid)].iter().all(|&reader| reader == index) { + demux.restart(pid); + } } self.reclaim(index); } @@ -261,7 +265,7 @@ mod test { use crate::catalog::hang::Container; use crate::container::Consumer; use crate::container::ts::import::test::{ - corrupt, fifty_programs, pat_section, psi_packets, two_programs, two_section_pat, + corrupt, fifty_programs, pat_section, pmt_section, psi_packets, two_programs, two_section_pat, }; #[tokio::test(start_paused = true)] @@ -504,6 +508,35 @@ mod test { assert_eq!(programs.stats().crc_error, 1); } + /// A second program claiming an elementary PID another already reads keeps the PID's + /// continuity chain, so a counter gap right after its PMT still counts. + #[test] + fn a_shared_pid_keeps_its_chain_when_another_program_claims_it() { + const MP2: u8 = 0x03; + let es = |cc: u8| { + let mut pkt = vec![0x47, 0x00, 0x61, 0x10 | cc]; + pkt.resize(188, 0xff); + pkt + }; + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", catalog::Config::default()); + let pmt = |program: u16| { + psi_packets( + program << 8, + &mut 0, + &[], + &pmt_section(program, 0, &[(MP2, 0x0061, &[])]), + ) + }; + let mut data = psi_packets(0, &mut 0, &[], &pat_section(&[(1, 0x0100), (2, 0x0200)])); + data.extend(pmt(1)); + data.extend(es(0)); + data.extend(pmt(2)); + data.extend(es(2)); + programs.decode(&data).unwrap(); + assert_eq!(programs.stats().continuity_count_error, 1); + } + /// The input is held until the PAT is whole, then each program publishes as its own /// broadcast carrying only that program, its catalog clock anchored on its own first frame /// apart on one shared clock. From b85c41db83c7a98bdd47236388d5631153260c18 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 19:27:38 -0700 Subject: [PATCH 4/5] ci(nightly): run the ts_programs bench nightly Co-Authored-By: Claude Opus 5.5 --- sh/gh/nightly.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 76d4251ffb6..8246e479a26 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -120,7 +120,7 @@ case "$1" in # only Criterion estimates enter the trend report. criterion extra --no-fail-fast -p moq-e2ee -p moq-json -p moq-archive -p moq-cli -p moq-mux \ --bench protect --bench codec --bench table --bench allocations --bench window \ - --bench producer --bench play_buffer --bench jitter + --bench producer --bench play_buffer --bench jitter --bench ts_programs ;; rust-uring) # One run per target makes an unsupported/skipped ring fail report From 74503453115850d6fda38acd503475f73a636ab9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 02:10:18 -0700 Subject: [PATCH 5/5] test(mux): hand main's SI expiry tests Bytes sections Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/si.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rs/moq-mux/src/container/ts/si.rs b/rs/moq-mux/src/container/ts/si.rs index 0e121274cfb..ac60dc02f96 100644 --- a/rs/moq-mux/src/container/ts/si.rs +++ b/rs/moq-mux/src/container/ts/si.rs @@ -709,8 +709,8 @@ mod tests { (Capture::new(broadcast, catalog.clone()), catalog) } - fn section(table: u8, id: u16, version: u8, last: u8) -> Vec { - psi::section(table, id, version, 0, last, &[0, 0, 0, 0, 0, 0]) + fn section(table: u8, id: u16, version: u8, last: u8) -> Bytes { + psi::section(table, id, version, 0, last, &[0, 0, 0, 0, 0, 0]).into() } #[moq_clock::test] @@ -871,12 +871,12 @@ mod tests { async fn short_form_repetitions_replace_their_deadline_and_latest_value() { let (mut capture, _) = capture(); let mut time = vec![0x70, 0x70, 5, 0, 0, 0, 0, 0]; - capture.section(0x14, time.clone()).unwrap(); + capture.section(0x14, Bytes::from(time.clone())).unwrap(); capture.advance(Duration::from_secs(29)); - capture.section(0x14, time.clone()).unwrap(); + capture.section(0x14, Bytes::from(time.clone())).unwrap(); capture.advance(Duration::from_secs(31)); time[7] = 1; - capture.section(0x14, time.clone()).unwrap(); + capture.section(0x14, Bytes::from(time.clone())).unwrap(); capture.advance(Duration::from_secs(60)); capture.flush(Timestamp::ZERO, false).unwrap(); let entry = &capture.entries[&(0x14, 0x70)];