From e77eeaf2ad6d0531ce856bbfe2ae847d94b5c095 Mon Sep 17 00:00:00 2001 From: t0ms Date: Wed, 21 Oct 2026 22:46:44 +0100 Subject: [PATCH 1/7] fix(mux): retire TS tracks a PMT revision drops or changes Co-authored-by: Claude Opus 5.5 --- drafts/draft-lcurley-moq-mpegts.md | 7 +- quest/m1/catalog-identity-rs.md | 3 +- quest/m1/ts-reliability/README.md | 8 +- quest/m1/ts-reliability/pat-lifecycle.md | 48 +++ quest/m1/ts-reliability/pmt-export.md | 49 +++ quest/m1/ts-reliability/pmt-lifecycle.md | 96 ----- quest/m1/ts-reliability/stream-routes.md | 4 - rs/moq-mux/examples/scte35_inject.rs | 14 +- rs/moq-mux/src/container/ts/fixture.rs | 15 +- rs/moq-mux/src/container/ts/import.rs | 329 ++++++++++++++++-- .../src/container/ts/pes_packet_test.rs | 176 +++++++++- .../container/ts/test_data/scte35/bbb5s.ts | Bin 145512 -> 145512 bytes .../container/ts/test_data/scte35/ffmpeg.ts | Bin 82532 -> 82532 bytes .../container/ts/test_data/scte35/gst480i.ts | Bin 58468 -> 58468 bytes 14 files changed, 613 insertions(+), 136 deletions(-) create mode 100644 quest/m1/ts-reliability/pat-lifecycle.md create mode 100644 quest/m1/ts-reliability/pmt-export.md delete mode 100644 quest/m1/ts-reliability/pmt-lifecycle.md diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index 0ad3fbb16a8..ecf9fff23dc 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -135,13 +135,16 @@ A publisher MUST NOT record 0x0000 (PAT) or 0x1FFF (null packets), which carry n `descriptors` are the track's ES-level descriptors from the PMT ({{descriptor}}), in PMT order. +A publisher MUST NOT change an entry's `pid`, `descriptors`, or carriage record in place, so a track name means one elementary stream for its life. +A stream the PMT revises ends its track and continues as a new track under a new name; one the PMT drops leaves `tracks` as its track ends, and a PID listed again later gets a new name too. + `verbatim` and `pesPacket` mark a track carrying an undecoded elementary stream ({{verbatim}}). An entry MUST NOT carry both, and a consumer MUST refuse a catalog with one that does. A decoded track MAY appear here too, to preserve its `pid` and `descriptors`. A consumer MUST ignore an entry with no `verbatim` record, no `pesPacket` record, and no track the catalog describes, so one that predates `pesPacket` drops such a track from the stream it rebuilds. ## programDescriptors -The PMT's program-level descriptors (`program_info`), in PMT order ({{descriptor}}). +The PMT's program-level descriptors (`program_info`), in PMT order ({{descriptor}}), updated in place as the PMT revises them. ## program {#field-program} The program identity from the PAT: @@ -158,6 +161,7 @@ type Program = { They are the only service-layer fields this document parses; everything else about the program stays inside the opaque `si` tables. A publisher MUST include `program` when the broadcast came from a transport stream, and omit it otherwise. +A publisher MUST NOT change `transportStreamId` or `programNumber` in place: the carried `si` describes that service, so a source whose PAT names another ends the broadcast. ## si {#field-si} The standalone service information tables, keyed by the PID they ride on and then by `table_id`: @@ -449,6 +453,7 @@ A broadcast demultiplexed from a DVB transport stream: video and audio described - `verbatim` carries sections only, breaking -00: `framing` is required and `"section"` its only value, and `streamId` is gone. A consumer refuses a -00 catalog with `"pes"` framing or a `verbatim` record without `framing`. A publisher carries an undecoded PES stream as `pesPacket`, and refuses one it cannot carry unaltered. - A flagged PCR jump on an `m2ts` track follows hang's discontinuity marker rather than a skipped group sequence. - An `m2ts` track unwraps PCRs from its history, clamps extrapolated object times at a forward discontinuity, and cuts a fallback group after one second. `randomAccess` covers every published group; `muxRate` follows the measured source rate. +- A `tracks` entry is never changed in place: a stream the PMT revises continues under a new name, and one it drops leaves the section. `programDescriptors` follows the PMT; `program`'s `transportStreamId` and `programNumber` do not change. ## draft-lcurley-moq-mpegts-00 {:numbered="false"} diff --git a/quest/m1/catalog-identity-rs.md b/quest/m1/catalog-identity-rs.md index 9f14d21ce33..610a4a9fba1 100644 --- a/quest/m1/catalog-identity-rs.md +++ b/quest/m1/catalog-identity-rs.md @@ -54,8 +54,7 @@ minting a name where the identity changes and updating in place otherwise: explicit pad name after caps renegotiation. - TS AAC/MP2/AC-3 and FLV legacy audio fix the config from the first header and silently ignore a later change; refuse or mint instead. A PMT that - changes a PID's stream type belongs to - [Live PAT/PMT changes](/quest/m1/ts-reliability/pmt-lifecycle.md). + changes a PID's stream type or descriptors already opens a new track. - `moq-audio`/`moq-ffi` `set_bitrate` should update the catalog's bitrate. FLV, fMP4, MKV, HLS import, and `moq-transcode` rungs already comply; keep diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index 44a9c9c8742..0328b3473b0 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -108,8 +108,9 @@ refused means an error or a counted drop. | DSM-CC types (0x0A to 0x0D), 0x86 without `CUEI` | Today PES-parsed into damage, or dropped uncounted | [routes](/quest/m1/ts-reliability/stream-routes.md) | | CAT, EMM/ECM, ATSC PSIP, other SI PIDs, unlisted PIDs, TS scrambling | Outside the profile, raw lane; counted when the import reads every program. Today dropped uncounted | [routes](/quest/m1/ts-reliability/stream-routes.md) | | DVB SI on 0x0010, 0x0011, 0x0012, 0x0014 | Preserved: snapshot track per PID and `table_id` | landed | -| Program identity, program and ES descriptors | Preserved, but captured once | [PMT changes](/quest/m1/ts-reliability/pmt-lifecycle.md) | -| PMT add, remove, retype; PID reuse; PAT changes | Normalized: in place under fresh track names, PMT rebuilt at the next `version_number`; program identity changes refused; a removed program retires, and returns as a new epoch. Today stale tracks on import, refusal or stale PMT on export | [PMT changes](/quest/m1/ts-reliability/pmt-lifecycle.md) | +| Program identity, program and ES descriptors | Preserved: descriptors follow each PMT revision; an identity change refuses | landed | +| PMT add, remove, retype; PID reuse | Normalized: in place under fresh track names, PMT rebuilt at the next `version_number`. Import landed; today refusal or stale PMT on export | [PMT export](/quest/m1/ts-reliability/pmt-export.md) | +| PAT changes: moved PMT PID, programs added or removed | Normalized: followed; a removed program retires, and returns as a new epoch. Today ignored, or the whole multiplex stops | [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) | | Programs without video | Preserved when the PMT names a PCR PID; refused without one. Today export refuses sections without video and any program without media; import refuses a program without media | [clockless export](/quest/m1/ts-reliability/clockless-export.md) | | PCR | Normalized: rebased, carried on the first video PID, else audio, else a dedicated PCR-only PID; today video or audio only | [clockless export](/quest/m1/ts-reliability/clockless-export.md) | | Mux rate, null stuffing | Normalized: re-padded to `muxRate` while the source is CBR | landed | @@ -123,7 +124,8 @@ remain with their current quests. ## Required -- [Live PAT/PMT changes](/quest/m1/ts-reliability/pmt-lifecycle.md) - a revised PMT or PAT never leaves a track name meaning other content +- [PMT export](/quest/m1/ts-reliability/pmt-export.md) - a TS export announces each PMT revision at the next `version_number` +- [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a revised PAT never leaves a broadcast meaning another program - [Loss and recovery tests](/quest/m1/ts-reliability/recovery.md) - actual MoQ omissions exercise reacquisition and unaffected tracks - [Stream routes](/quest/m1/ts-reliability/stream-routes.md) - every PID gets a declared route, and what is not carried is counted - [Clockless export](/quest/m1/ts-reliability/clockless-export.md) - TS import and export carry audio-only and data-only programs diff --git a/quest/m1/ts-reliability/pat-lifecycle.md b/quest/m1/ts-reliability/pat-lifecycle.md new file mode 100644 index 00000000000..3998e01bae2 --- /dev/null +++ b/quest/m1/ts-reliability/pat-lifecycle.md @@ -0,0 +1,48 @@ +# [M] Live PAT changes keep programs honest + +## Goal + +A live TS source whose PAT changes mid-stream keeps every broadcast meaning +one program. A moved PMT PID, a program added, and a program removed or +returning each have one observable outcome: followed, published, retired, +or published as a new epoch. None is silently ignored. + +## Plan + +What the code does today: + +- A PAT naming another PMT PID for the program adds a reassembler but never + drops the old one, and `program.pmtPid` keeps the first. +- `Programs` ignores a program a later PAT adds, without a warning. +- Under `--program all`, a later PAT that removes one program reaches every + per-program `Import`; the removed program's `handle_pat` errors, + `Programs::feed` propagates it, and the whole multiplex ingest stops, + including unrelated programs. +- A PAT naming another `transport_stream_id`, or a single-program import's + PAT naming another program, already refuses the import. + +Decided 2026-10-20 by the maintainer: + +- A new PMT PID for the same program moves the reassembler and updates + `program.pmtPid`, which the export follows with a new PAT. `Programs` + publishes a program a later PAT adds as its own broadcast, as it does for + the first PAT, rather than ignoring it. +- A later PAT that removes a program retires that program's broadcast: its + tracks finish and it is unannounced, while the other programs carry on. + If the program returns, it publishes a new epoch of its broadcast, never + the retired instance's names for new content. A single-program import + whose selected program leaves the PAT ends, as today. +- Open: a PAT that names a program whose PMT never arrives. Decide whether + the import refuses it or waits, and which clock bounds the wait (found in + [moq-dev/moq#6122](https://github.com/moq-dev/moq/pull/6122)). + +Rust: `rs/moq-mux` TS `Import::handle_pat` and `Programs`. + +Tests use the generated `fixture::Mux` builder: a PMT PID move, and a +`--program all` MPTS whose PAT adds a program, drops one and later restores +it: the other program runs uninterrupted, and the returning one publishes a +new epoch. + +## Related + +- [PMT export](/quest/m1/ts-reliability/pmt-export.md) - the export follows a moved PMT PID with a new PAT diff --git a/quest/m1/ts-reliability/pmt-export.md b/quest/m1/ts-reliability/pmt-export.md new file mode 100644 index 00000000000..07036239381 --- /dev/null +++ b/quest/m1/ts-reliability/pmt-export.md @@ -0,0 +1,49 @@ +# [M] TS export follows a PMT revision + +## Goal + +A TS export of a broadcast whose layout changes mid-stream announces the +revised layout as an ordinary PMT revision. A track added, retired, or +replaced under a fresh name changes the PMT at the next `version_number`; +nothing else blips. + +## Plan + +The import already keeps track names honest: a changed `stream_type` or ES +descriptor loop finishes the old track, drops its catalog entry, and opens +a new uniquely named one; a removed stream finishes its track and leaves +the catalog; `programDescriptors` updates in place; a program identity +change refuses. + +What the export does today: + +- `Export` builds the PAT/PMT once. A track added afterward fails the export + ("TS track layout changed after PAT/PMT was emitted"), so every revision + the import publishes ends the export. +- A track that leaves the catalog keeps its PID in the PMT forever, and + `programDescriptors` is read once. +- A switch to another instance already rebuilds the tables at the next + `version_number`. + +Decided 2026-10-20 by the maintainer: + +- Export rebuilds the PMT at the next `version_number` when the track set + or an entry changes, drops a retired track's PID once its track ends, and + keeps the PCR PID unless its track retires. A TS receiver sees an + ordinary PMT revision and a Hang player sees a rendition switch. + +Rust: `rs/moq-mux` TS export. Draft +([mpegts](/drafts/draft-lcurley-moq-mpegts.md)): the rebuild section says +how a consumer revises the PMT. + +Tests use the generated `fixture::Mux` builder through import and export: +PMT add, remove, retype, descriptor-only change, version rollover (31 to +0), PID reuse after removal, and a program descriptor change, each +asserting the exported PMT's streams and version sequence. Extend the +TSDuck round trip with a generated mid-stream PMT revision so +`compliance.py` checks the rebuilt version sequence in CI. + +## Related + +- [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a moved PMT PID reaches the export as a new PAT +- [Hitless TS](/quest/m2/ts-hitless.md) - identical legs must revise their PMT identically diff --git a/quest/m1/ts-reliability/pmt-lifecycle.md b/quest/m1/ts-reliability/pmt-lifecycle.md deleted file mode 100644 index 56eb20cf8e8..00000000000 --- a/quest/m1/ts-reliability/pmt-lifecycle.md +++ /dev/null @@ -1,96 +0,0 @@ -# [L] Live PAT and PMT changes keep track names honest - -## Goal - -A live TS source that revises its PMT or PAT mid-stream produces a broadcast -whose every track name still means one elementary stream, and a TS export -that announces the revised layout. Added, removed and retyped streams, changed -descriptors, PID reuse and program changes each have one observable outcome: -carried under a fresh name, retired, or refused with an error. None is -silently kept stale. - -## Plan - -Reproduced 2026-10-20 in the fidelity audit. A PMT that retypes the private PES -on a PID from 0x06 (`KLVA` registration) to 0x15 (metadata descriptor) keeps -the `pesPacket` track, its catalog entry still says 0x06 with the old -descriptor, and the 0x15 payloads publish under the same name. A later PMT -that drops the PID leaves the entry and track in the catalog. - -What the code does today: - -- `Import::handle_pmt` only walks the streams the new PMT lists. Nothing - retires a PID it no longer lists, and `ensure_stream` returns early for any - PID that already has a route, so a new `stream_type` or ES descriptor loop - never reaches it. Only a section/PES flip and the PCR transitions re-route. -- Catalog fields are captured once: `program_descriptors` - (`program_recorded`), program identity (`identity_recorded`), and a decoded - track's ES descriptors (`recorded_media`). The decoded codec importers fix - their config from the first header; the identity guard for that lands in - [Rust identity enforcement](/quest/m1/catalog-identity-rs.md), and this - quest owns the PMT-level remap. -- A PAT naming another PMT PID adds a reassembler but never drops the old - one. `Programs` ignores a program a later PAT adds, without a warning. -- Under `--program all`, a later PAT that removes one program reaches every - per-program `Import`; the removed program's `handle_pat` errors, - `Programs::feed` propagates it, and the whole multiplex ingest stops, - including unrelated programs. -- `Export` builds the PAT/PMT once. A track added afterward fails the export - ("TS track layout changed after PAT/PMT was emitted"), and a track that - leaves the catalog keeps its PID in the PMT forever. A switch to another - instance already rebuilds the tables at the next `version_number`. - -Decided 2026-10-20 by the maintainer: change in place with fresh names. - -- A changed `stream_type` or ES descriptor loop finishes the old track, - drops its catalog entry, and opens a new uniquely named track, as for a - new PID. A removed stream finishes its track and leaves the catalog. - `programDescriptors` updates in place, and a CUEI change re-routes the - affected PIDs. -- Export rebuilds the PMT at the next `version_number` when the track set - or an entry changes, drops a retired track's PID once its track ends, and - keeps the PCR PID unless its track retires. A TS receiver sees an - ordinary PMT revision and a Hang player sees a rendition switch; nothing - else blips. -- A change of program identity (`transport_stream_id`, `program_number`) - refuses with an error: the carried SI describes the old service. -- A new PMT PID for the same program moves the reassembler and updates - `program.pmtPid`, which the export follows with a new PAT. `Programs` - publishes a program a later PAT adds as its own broadcast, as it does for - the first PAT, rather than ignoring it. -- A later PAT that removes a program retires that program's broadcast: its - tracks finish and it is unannounced, while the other programs carry on. - If the program returns, it publishes a new epoch of its broadcast, never - the retired instance's names for new content. A single-program import - whose selected program leaves the PAT ends, as today. -- Open: a PAT that names a program whose PMT never arrives. Decide whether - the import refuses it or waits, and which clock bounds the wait (found in - [moq-dev/moq#6122](https://github.com/moq-dev/moq/pull/6122)). - -Why: it matches the 2026-10-15 identity decision (changed content gets a new -name; a new epoch stays reserved for a whole-broadcast restart). Rejected: -replacing the broadcast with a new epoch on any ES layout change, which makes -every subscriber reacquire every track so an audio-language add interrupts -video too; and refusing every layout change, which stops the ingest on common -live transitions such as a regional opt-out adding audio or a subtitle PID -coming and going. - -Rust: `rs/moq-mux` TS import and export. JS: none; `js/msf` passes the -`mpegts` section through untyped and nothing in `js/` muxes TS. Draft -([mpegts](/drafts/draft-lcurley-moq-mpegts.md)): a publisher MUST NOT change -an entry's `pid`, `descriptors`, or carriage record in place, and the rebuild -section says how a consumer revises the PMT. Hang's core stays TS-free. - -Tests use the generated `fixture::Mux` builder: PMT add, remove, retype, -descriptor-only change, version rollover (31 to 0), PID reuse after removal, -program descriptor and CUEI changes, PMT PID move, and an identity change -that must refuse, plus a `--program all` MPTS whose PAT drops one program -and later restores it: the other program runs uninterrupted, and the -returning one publishes a new epoch. Each asserts the catalog, the track names, and the exported -PMT. Extend the TSDuck round trip with a generated mid-stream PMT revision so -`compliance.py` checks the rebuilt version sequence in CI. - -## Related - -- [Rust identity enforcement](/quest/m1/catalog-identity-rs.md) - codec config changes behind one PID mint names the same way -- [Stream routes](/quest/m1/ts-reliability/stream-routes.md) - which route a revised stream type gets diff --git a/quest/m1/ts-reliability/stream-routes.md b/quest/m1/ts-reliability/stream-routes.md index 17b861eb302..81235e9582a 100644 --- a/quest/m1/ts-reliability/stream-routes.md +++ b/quest/m1/ts-reliability/stream-routes.md @@ -106,7 +106,3 @@ program 2's PIDs, asserting the catalog, the stats rows (the unlisted PID's row asserted field by field), and an export round trip of the carried streams. The MPEG-2 video arm also runs the TSDuck round trip and an FFmpeg decode of the export in `test/ts`. - -## Related - -- [Live PAT/PMT changes](/quest/m1/ts-reliability/pmt-lifecycle.md) - a revised `stream_type` takes the route this quest declares diff --git a/rs/moq-mux/examples/scte35_inject.rs b/rs/moq-mux/examples/scte35_inject.rs index 337b9cbedab..9bde414d41b 100644 --- a/rs/moq-mux/examples/scte35_inject.rs +++ b/rs/moq-mux/examples/scte35_inject.rs @@ -164,8 +164,9 @@ fn main() -> anyhow::Result<()> { }) .collect::>()?; - // Pass the input packets through; insert the augmented PMT right after the first PMT, - // and splice the cues in at evenly spread positions so each lands on a later PTS. + // Pass the input packets through; insert the augmented PMT right after the first PMT and + // rewrite every later one to match, since a PMT that stops listing the SCTE-35 PID retires + // its track. Splice the cues in at evenly spread positions so each lands on a later PTS. anyhow::ensure!( input.len() % 188 == 0, "input TS length {} is not a multiple of 188", @@ -195,7 +196,14 @@ fn main() -> anyhow::Result<()> { let mut out = Vec::new(); for (i, p) in packets.iter().enumerate() { - out.extend_from_slice(p); + if i > pmt_idx && pid_of(p) == pmt_pid.as_u16() { + anyhow::ensure!(p[1] & 0x40 != 0, "a PMT spanning packets cannot be rewritten in place"); + // The input's own continuity counter, so the rewrite adds no counter error. + out.extend_from_slice(&[p[0], p[1], p[2], (aug_pmt[3] & 0xf0) | (p[3] & 0x0f)]); + out.extend_from_slice(&aug_pmt[4..]); + } else { + out.extend_from_slice(p); + } if i == pmt_idx { out.extend_from_slice(&aug_pmt); } diff --git a/rs/moq-mux/src/container/ts/fixture.rs b/rs/moq-mux/src/container/ts/fixture.rs index cc0605d69ba..126d9eaa25e 100644 --- a/rs/moq-mux/src/container/ts/fixture.rs +++ b/rs/moq-mux/src/container/ts/fixture.rs @@ -214,6 +214,19 @@ impl Mux { pcr_pid: u16, streams: &[(u8, u16, &[u8])], info: usize, + ) -> &mut Self { + self.pmt_at(0, pid, program, pcr_pid, streams, info) + } + + /// A [`Self::pmt_with`] at `version_number` `version`. + pub(super) fn pmt_at( + &mut self, + version: u8, + pid: u16, + program: u16, + pcr_pid: u16, + streams: &[(u8, u16, &[u8])], + info: usize, ) -> &mut Self { let mut descriptors = Vec::new(); while descriptors.len() < info { @@ -239,7 +252,7 @@ impl Mux { ]); body.extend_from_slice(info); } - self.section(pid, &psi::section(psi::PMT_TABLE_ID, program, 0, 0, 0, &body)) + self.section(pid, &psi::section(psi::PMT_TABLE_ID, program, version, 0, 0, &body)) } pub(super) fn bytes(&self) -> Vec { diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index e5d4b7295fd..2dc9641f3d0 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -174,9 +174,11 @@ pub struct Import { /// Decoded media PIDs already recorded into `mpegts.tracks`, so the reconcile in /// [`Self::flush`] runs once per track rather than on every frame. recorded_media: HashSet, - /// Whether the PMT program-level descriptors have been recorded yet (set once; - /// PMT `program_info` is stable for the program's life). - program_recorded: bool, + /// The PMT program-level descriptors last recorded, rewritten in place as the PMT revises them. + program_info: Vec, + /// What the PMT in force says of each PID it lists, so a revision that says otherwise ends + /// the route rather than carrying the new stream under the old name. + listed: HashMap, /// Reassemblers for the standalone SI PIDs ([`catalog::SI_PIDS`]), keyed by PID, so /// the service layer survives the round-trip. Only populated with `mpegts` catalog /// support. @@ -184,9 +186,10 @@ pub struct Import { /// The SI store: buffers each sub-table to a complete generation, publishes the /// per-`(PID, table_id)` snapshot tracks, and advertises them in the catalog. si: super::si::Capture, - /// Whether the service identity (TSID/service_id/PMT PID) has been captured from - /// the PAT into the catalog service record yet (set once; the PAT is stable). - identity_recorded: bool, + /// The `(transport_stream_id, program_number)` the first PAT named, also captured into the + /// catalog service record with the PMT PID. A PAT naming another refuses the import: the + /// carried SI describes this one. + identity: Option<(u16, u16)>, /// Latest video PTS, used to timestamp service-information snapshots. last_pts: Option, /// The program's PTS history, which picks each value's 33-bit wrap. @@ -248,10 +251,11 @@ impl Import { supports_mpegts, es_descriptors: HashMap::new(), recorded_media: HashSet::new(), - program_recorded: false, + program_info: Vec::new(), + listed: HashMap::new(), si_sections: HashMap::new(), si, - identity_recorded: false, + identity: None, last_pts: None, pts: PtsUnwrap::default(), program: None, @@ -501,15 +505,44 @@ impl Import { .iter() .any(|d| d.tag == 0x05 && d.data.len() >= 4 && &d.data[0..4] == b"CUEI"); - // Record the program-level descriptors once (PMT program_info is stable); - // export re-emits them verbatim, including the original CUEI. - if self.supports_mpegts && !self.program_recorded && !pmt.program_info.is_empty() { + // Record the program-level descriptors as the PMT revises them; export re-emits them + // verbatim, including the original CUEI. + if self.supports_mpegts && self.program_info != pmt.program_info { if let Some(mpegts) = self.catalog.modify()?.ext.mpegts_mut() { mpegts.program_descriptors = pmt.program_info.clone(); } - self.program_recorded = true; + self.program_info = pmt.program_info.clone(); } + // A stream the PMT no longer lists, or lists as something else, ends here, so every + // track name keeps meaning one elementary stream; a revised one opens under a new name. + let listed: HashMap = pmt + .streams + .iter() + .map(|es| { + let listing = Listing { + stream_type: es.stream_type, + descriptors: es.descriptors.clone(), + sections: section_framed(es.stream_type, cuei), + }; + (es.pid, listing) + }) + .collect(); + let mut ended: Vec = self + .listed + .iter() + .filter(|(pid, listing)| listed.get(pid) != Some(listing)) + .map(|(&pid, _)| pid) + .collect(); + ended.sort_by_key(Pid::as_u16); + for pid in ended { + self.end_route(pid)?; + if !listed.contains_key(&pid) { + tracing::info!(pid = pid.as_u16(), "TS elementary stream left the PMT"); + } + } + self.listed = listed; + for es in &pmt.streams { let before = self.route(es.pid); // Stash ES descriptors so a decoded media track can record them once its @@ -517,11 +550,10 @@ impl Import { if self.supports_mpegts { self.es_descriptors.insert(es.pid.as_u16(), es.descriptors.clone()); } - // Section-framed private data is reassembled as sections, never PES-parsed: - // private sections (0x05) and CUEI-marked SCTE-35 (0x86). Everything else - // routes through ensure_stream (a decoded codec, PES-framed verbatim, or - // dropped). - if es.stream_type == 0x05 || (cuei && es.stream_type == StreamType::Dts8ChannelLosslessAudio as u8) { + // Section-framed private data is reassembled as sections, never PES-parsed. + // Everything else routes through ensure_stream (a decoded codec, PES-framed + // verbatim, or dropped). + if section_framed(es.stream_type, cuei) { self.ensure_section(es.pid, es.stream_type, &es.descriptors)?; } else { self.ensure_stream(es.pid, es.stream_type, &es.descriptors)?; @@ -846,6 +878,48 @@ impl Import { Some(stream) } + /// End the route on `pid` for a PMT that dropped or revised its stream, as the end of input + /// ends every route: the PES it began goes out, what waits on the clock is placed from the + /// PCRs so far, and its track finishes and leaves the catalog. + fn end_route(&mut self, pid: Pid) -> anyhow::Result<()> { + if self.headers.remove(&pid).is_some() { + self.damage(pid, &anyhow::anyhow!("PES header truncated by a PMT revision"))?; + } + self.flush(pid)?; + let generation = self.placement.generation; + let (mut units, mut unplaced) = (0, 0); + if let Some(mut section) = self.sections.remove(&pid.as_u16()) { + section.settle(&self.placement); + (units, unplaced) = section.release(&self.timebase, &self.pts, generation)?; + unplaced += section.held.len() as u64; + section.finish()?; + } + if let Some(mut stream) = self.retire_stream(pid) { + if let Stream::PesPacket(stream) = &mut stream { + stream.settle(&self.placement); + let (published, dropped) = stream.release(&self.timebase, &self.pts, generation)?; + units += published; + unplaced += dropped + stream.held.len() as u64; + } + // A verbatim entry leaves with its stream; a decoded track's is recorded here. + if self.recorded_media.remove(&pid) + && let Some(name) = stream.media_track_name() + && let Some(mpegts) = self.catalog.modify()?.ext.mpegts_mut() + { + mpegts.tracks.remove(&name); + } + units += stream.finish()?; + } + if unplaced > 0 { + *self.unplaced.entry(pid.as_u16()).or_default() += unplaced; + } + self.published |= units > 0; + self.liveness.delivered(pid.as_u16(), units); + self.liveness.retire(pid.as_u16()); + self.es_descriptors.remove(&pid.as_u16()); + Ok(()) + } + /// Open the PES whose header `pes` begins where `span` says, if any PCR came before it. fn handle_pes_start(&mut self, pid: Pid, pes: PesStart, span: Option) -> anyhow::Result<()> { // Finish the preceding PES before accepting another header. @@ -1285,8 +1359,9 @@ impl Import { /// Pick the imported program from a whole PAT and learn its PMT PID, so `decode` /// reassembles that PMT and no other. /// - /// Every PAT is checked, so a program added mid-stream ends an unselected import, and a - /// selected program that disappears ends a selected one; media already published stays. + /// Every PAT is checked, so a program added mid-stream ends an unselected import, a + /// selected program that disappears ends a selected one, and a PAT naming the program under + /// another identity ends either; media already published stays. pub(super) fn handle_pat(&mut self, pat: &psi::Pat) -> anyhow::Result<()> { // program_number 0 is the network PID association, not a program. let programs: Vec<_> = pat.programs.iter().filter(|entry| entry.program_number != 0).collect(); @@ -1305,15 +1380,29 @@ impl Import { _ => return Err(MultipleProgramsError { programs: numbers() }.into()), }, }; + let identity = (pat.transport_stream_id, entry.program_number); + if let Some((transport_stream_id, program_number)) = self.identity + && (transport_stream_id, program_number) != identity + { + anyhow::bail!( + "the PAT changed the program's identity from transport_stream_id {transport_stream_id} program {program_number} to transport_stream_id {} program {}, which the carried service information does not describe", + identity.0, + identity.1 + ); + } self.pmt_sections.entry(entry.pmt_pid).or_default(); self.health.pmt_pids(&[entry.pmt_pid.as_u16()], self.liveness.now()); - self.record_program_identity(pat.transport_stream_id, entry) + if self.identity.is_none() { + self.identity = Some(identity); + self.record_program_identity(pat.transport_stream_id, entry)?; + } + Ok(()) } /// Capture the transport/service identity (TSID, service number, PMT PID) from the - /// PAT into the catalog service record, once. No-op without `mpegts` support. + /// PAT into the catalog service record. No-op without `mpegts` support. fn record_program_identity(&mut self, transport_stream_id: u16, entry: &psi::Association) -> anyhow::Result<()> { - if !self.supports_mpegts || self.identity_recorded { + if !self.supports_mpegts { return Ok(()); } if let Some(mpegts) = self.catalog.modify()?.ext.mpegts_mut() { @@ -1322,7 +1411,6 @@ impl Import { program.program_number = entry.program_number; program.pmt_pid = entry.pmt_pid.as_u16(); } - self.identity_recorded = true; Ok(()) } @@ -1488,6 +1576,22 @@ fn list_programs(programs: &[u16]) -> String { programs.iter().map(u16::to_string).collect::>().join(", ") } +/// What a PMT says of one elementary stream. A route lasts as long as its PID's listing does. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Listing { + stream_type: u8, + descriptors: Vec, + /// Reassembled as sections: [`section_framed`] under the program descriptors it came with. + sections: bool, +} + +/// Whether a stream of `stream_type` carries sections rather than PES: private sections (0x05), +/// and SCTE-35 (0x86) in a program whose descriptors carry the 'CUEI' registration. 0x86 is also +/// a DTS audio type, so without it the stream is not read as cues. +fn section_framed(stream_type: u8, cuei: bool) -> bool { + stream_type == 0x05 || (cuei && stream_type == StreamType::Dts8ChannelLosslessAudio as u8) +} + // Match the raw TS accumulator: half a MoQ group bounds one program's staged input. const STAGING_BYTES_MAX: usize = (moq_net::group::MAX_CACHE_BYTES / 2) as usize; const STAGING_UNITS_MAX: usize = moq_net::group::MAX_GROUP_FRAMES / 2; @@ -2154,6 +2258,8 @@ pub(super) struct Liveness { /// Per PID: access units delivered, and `elapsed` at the last one, or at registration. /// Kept per PID rather than per route, so a PMT remap carries it like `retired_stats`. streams: HashMap, + /// PIDs the PMT stopped listing: their counts stay, but they are not quiet. + retired: HashSet, } impl Liveness { @@ -2196,7 +2302,17 @@ impl Liveness { /// The PMT declared `pid`. Its silence counts from here until it delivers. pub(super) fn register(&mut self, pid: u16) { let now = self.elapsed.unwrap_or(0); - self.streams.entry(pid).or_insert((0, now)); + let entry = self.streams.entry(pid).or_insert((0, now)); + if self.retired.remove(&pid) { + entry.1 = now; + } + } + + /// The route on `pid` ended. Until a PMT declares it again it is not silent, only gone. + pub(super) fn retire(&mut self, pid: u16) { + if self.streams.contains_key(&pid) { + self.retired.insert(pid); + } } /// `pid` delivered `units` access units just now. @@ -2216,6 +2332,9 @@ impl Liveness { let Some(&(units, last)) = self.streams.get(&pid) else { return (0, None); }; + if self.retired.contains(&pid) { + return (units, None); + } let quiet = self .elapsed .map(|now| std::time::Duration::from_nanos((now - last) * 1_000 / 27)); @@ -9657,4 +9776,166 @@ pub(super) mod test { import.handle_pmt(pmt(StreamType::H264 as u8)).unwrap(); assert!(matches!(import.streams[&pid], super::Stream::H264 { .. })); } + + /// A PMT that changes a decoded stream's ES descriptors, here an audio language retag, + /// finishes its track: its rendition and `mpegts` entry leave the catalog, and the next PES + /// opens a rendition under a new name whose entry records the revised descriptors. + #[test] + fn a_relabelled_decoded_stream_reopens_under_a_new_name() { + use crate::container::ts::psi::{Es, Pmt}; + const AUDIO_PID: u16 = 0x0061; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let pmt = |language: &'static [u8]| Pmt { + program_number: 1, + pcr_pid: Some(Pid::new(AUDIO_PID).unwrap()), + program_info: Vec::new(), + streams: vec![Es { + stream_type: MP2, + pid: Pid::new(AUDIO_PID).unwrap(), + descriptors: vec![super::catalog::Descriptor { + tag: 0x0a, + data: bytes::Bytes::from_static(language), + }], + }], + }; + let language = |catalog: &crate::catalog::hang::Catalog, name: &str| { + catalog.ext.mpegts.tracks[name].descriptors[0].data.clone() + }; + + import.handle_pmt(pmt(b"eng\0")).unwrap(); + // The legacy path publishes once a later PES confirms the frames before it. + import.decode(&mp2_pes(AUDIO_PID, 0, 90_000, [0xAA, 0xBB])).unwrap(); + import.decode(&mp2_pes(AUDIO_PID, 1, 180_000, [0xAA, 0xBB])).unwrap(); + let before = catalog.snapshot(); + let old = before + .audio + .renditions + .keys() + .next() + .expect("the MP2 rendition") + .clone(); + assert_eq!(language(&before, &old).as_ref(), b"eng\0"); + + import.handle_pmt(pmt(b"fra\0")).unwrap(); + let between = catalog.snapshot(); + assert!( + between.audio.renditions.is_empty(), + "the old rendition left the catalog" + ); + assert!(between.ext.mpegts.tracks.is_empty(), "the old entry left the catalog"); + + import.decode(&mp2_pes(AUDIO_PID, 2, 270_000, [0xCC, 0xDD])).unwrap(); + import.decode(&mp2_pes(AUDIO_PID, 3, 360_000, [0xCC, 0xDD])).unwrap(); + let after = catalog.snapshot(); + let new = after + .audio + .renditions + .keys() + .next() + .expect("the relabelled rendition") + .clone(); + assert_ne!(new, old); + assert_eq!(language(&after, &new).as_ref(), b"fra\0"); + import.finish().unwrap(); + } + + /// The program descriptors follow each PMT revision in place, and adding or removing the + /// 'CUEI' registration reroutes the 0x86 PID: carried as cues under it, dropped without it, + /// and carried under a new name when it returns. + #[test] + fn program_descriptors_follow_the_pmt_and_reroute_the_cue_pid() { + use crate::container::ts::psi::{Es, Pmt}; + const CUE_PID: u16 = 0x0021; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let descriptor = |tag, data: &'static [u8]| super::catalog::Descriptor { + tag, + data: bytes::Bytes::from_static(data), + }; + let pmt = |program_info: Vec| Pmt { + program_number: 1, + pcr_pid: Some(Pid::new(PCR_PID).unwrap()), + program_info, + streams: vec![ + Es { + stream_type: StreamType::Mpeg2Video as u8, + pid: Pid::new(PCR_PID).unwrap(), + descriptors: Vec::new(), + }, + Es { + stream_type: StreamType::Dts8ChannelLosslessAudio as u8, + pid: Pid::new(CUE_PID).unwrap(), + descriptors: Vec::new(), + }, + ], + }; + let cues = || { + let tracks = catalog.snapshot().ext.mpegts.tracks; + tracks + .into_iter() + .find(|(_, track)| track.pid == CUE_PID) + .map(|(name, _)| name) + }; + + let cuei = vec![descriptor(0x05, b"CUEI")]; + import.handle_pmt(pmt(cuei.clone())).unwrap(); + assert_eq!(catalog.snapshot().ext.mpegts.program_descriptors, cuei); + let first = cues().expect("cues under CUEI"); + + let private = vec![descriptor(0x80, &[0xaa, 0xbb])]; + import.handle_pmt(pmt(private.clone())).unwrap(); + assert_eq!(catalog.snapshot().ext.mpegts.program_descriptors, private); + assert_eq!(cues(), None, "without CUEI the PID is not cues"); + assert!(matches!( + import.streams.get(&Pid::new(CUE_PID).unwrap()), + Some(super::Stream::Ignored) + )); + + import.handle_pmt(pmt(cuei.clone())).unwrap(); + assert_eq!(catalog.snapshot().ext.mpegts.program_descriptors, cuei); + let second = cues().expect("cues under CUEI again"); + assert_ne!(second, first, "the returning cue stream gets a new name"); + + import.handle_pmt(pmt(Vec::new())).unwrap(); + assert!(catalog.snapshot().ext.mpegts.program_descriptors.is_empty()); + } + + /// A PAT that names the program under another `transport_stream_id`, or a single-program + /// import's PAT naming another program, refuses the import: the carried SI describes the + /// first. A repeated PAT does not. + #[test] + fn a_pat_naming_another_identity_refuses_the_import() { + use crate::container::ts::psi::{Association, Pat}; + let pat = |transport_stream_id, program_number| Pat { + transport_stream_id, + programs: vec![Association { + program_number, + pmt_pid: Pid::new(0x20).unwrap(), + }], + }; + for (selected, transport_stream_id, program_number) in [(None, 2, 1), (None, 1, 2), (Some(1), 2, 1)] { + let (_catalog, import) = psi_import(); + let mut import = match selected { + Some(program) => import.with_program(program), + None => import, + }; + import.handle_pat(&pat(1, 1)).unwrap(); + import.handle_pat(&pat(1, 1)).unwrap(); + let err = import + .handle_pat(&pat(transport_stream_id, program_number)) + .expect_err("another identity is refused"); + assert!(err.to_string().contains("identity"), "{err:#}"); + } + } } diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index 668565f45a5..2fbfb37ffd3 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -228,6 +228,16 @@ impl Imported { /// Every frame on track `name`, and the error that ended it if one did. async fn read(consumer: &moq_net::broadcast::Consumer, name: &str) -> (Vec, Option) { + let (frames, end) = read_until(consumer, name).await; + (frames, end.err()) +} + +/// Every frame on track `name`, and how it ended: `Ok(true)` finished, `Ok(false)` gone quiet +/// still open, or the error that ended it. +async fn read_until( + consumer: &moq_net::broadcast::Consumer, + name: &str, +) -> (Vec, Result) { let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(3600)); let track = consumer.track(name).unwrap().subscribe(subscription).await.unwrap(); let mut reader = crate::container::Consumer::new( @@ -238,8 +248,9 @@ async fn read(consumer: &moq_net::broadcast::Consumer, name: &str) -> (Vec frames.push(frame), - Ok(Ok(None)) | Err(_) => return (frames, None), - Ok(Err(err)) => return (frames, Some(err.to_string())), + Ok(Ok(None)) => return (frames, Ok(true)), + Err(_) => return (frames, Ok(false)), + Ok(Err(err)) => return (frames, Err(err.to_string())), } } } @@ -788,3 +799,164 @@ async fn a_pmt_naming_or_dropping_the_pcr_reroutes_private_pes() { assert!(imported.entry().is_none(), "the track left the catalog"); assert_eq!(imported.stats().refused, 2); } + +/// The `KLVA` registration with an `ISO_639_language_descriptor` for English after it. +const KLVA_ENG: &[u8] = &[0x05, 0x04, b'K', b'L', b'V', b'A', 0x0a, 0x04, b'e', b'n', b'g', 0x00]; + +/// The descriptors a catalog entry records, as `(tag, data)`. +fn recorded(track: &super::catalog::Track) -> Vec<(u8, Vec)> { + track.descriptors.iter().map(|d| (d.tag, d.data.to_vec())).collect() +} + +/// A raw descriptor loop as `(tag, data)`. +fn descriptor_loop(mut raw: &[u8]) -> Vec<(u8, Vec)> { + let mut out = Vec::new(); + while let [tag, len, rest @ ..] = raw { + let (data, tail) = rest.split_at(usize::from(*len)); + out.push((*tag, data.to_vec())); + raw = tail; + } + out +} + +/// Start reading track `name` before the input that ends it is imported. +async fn reader( + imported: &Imported, + name: &str, +) -> tokio::task::JoinHandle<(Vec, Result)> { + let consumer = imported.consumer.clone(); + let name = name.to_string(); + let reader = tokio::spawn(async move { read_until(&consumer, &name).await }); + for _ in 0..10 { + tokio::task::yield_now().await; + } + reader +} + +/// A PMT revision that retypes the private stream, from 0x06 under `KLVA` to 0x15 under the +/// metadata descriptor, or changes its descriptors alone, finishes the old track and carries +/// what follows under a new name whose entry records the revision. The PES still waiting for a +/// PCR when the revision lands goes out on the old track, placed from the PCRs before it. +#[tokio::test(start_paused = true)] +async fn a_revised_private_stream_publishes_under_a_fresh_name() { + for (stream_type, revised) in [(0x15, METADATA), (0x06, KLVA_ENG)] { + let mut feed = Feed::new(0x06, KLVA); + let start = 1_000 * MS; + feed.pcr(start) + .picture(start) + .data(klv(1, 8)) + .pcr(start + 40 * MS) + .data(klv(2, 8)); + let held = placed(feed.clock(feed.pes[1].0).unwrap(), 0); + let head = feed.bytes(); + let streams: &[(u8, u16, &[u8])] = &[(0x02, VIDEO_PID, &[]), (stream_type, DATA_PID, revised)]; + feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, streams, 0); + let at = start + 80 * MS; + feed.pcr(at) + .picture(at) + .data(sync_klv(3, at / 300 + LEAD)) + .pcr(at + 40 * MS); + + let mut imported = start_import(&head); + let (old, _) = imported.entry().unwrap(); + let old_reader = reader(&imported, &old).await; + imported.import.decode(&feed.bytes()[head.len()..]).unwrap(); + + let (frames, end) = old_reader.await.unwrap(); + assert_eq!(end, Ok(true), "the old track finishes"); + assert_eq!(frames.len(), 2); + assert_eq!(frames[1].timestamp.as_micros() as u64, held); + let tracks = imported.catalog.snapshot().ext.mpegts.tracks; + let listed: Vec<_> = tracks.iter().filter(|(_, track)| track.pid == DATA_PID).collect(); + assert_eq!(listed.len(), 1, "one entry for the PID: {tracks:?}"); + let (name, entry) = listed[0]; + assert_ne!(*name, old, "the revised stream is carried under a new name"); + assert_eq!(entry.pes_packet.as_ref().unwrap().stream_type, stream_type); + assert_eq!(recorded(entry), descriptor_loop(revised)); + + let name = name.clone(); + imported.import.finish().unwrap(); + let (frames, err) = read(&imported.consumer, &name).await; + assert!(err.is_none(), "{err:?}"); + assert_eq!(frames.len(), 1); + assert_eq!(&frames[0].payload[..], &feed.pes[2].1[..]); + } +} + +/// A PMT that stops listing the private stream finishes its track and takes it out of the +/// catalog, its PID no longer counted as quiet; a later PMT listing the PID again opens a new +/// track rather than reviving the retired name. +#[tokio::test(start_paused = true)] +async fn a_dropped_private_stream_finishes_and_its_reused_pid_opens_a_new_track() { + let mut feed = Feed::new(0x06, KLVA); + let start = 1_000 * MS; + feed.pcr(start).picture(start).data(klv(1, 8)).pcr(start + 40 * MS); + let head = feed.bytes(); + feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, &[(0x02, VIDEO_PID, &[])], 0); + feed.picture(start + 40 * MS).data(klv(2, 8)).pcr(start + 80 * MS); + let dropped = feed.bytes(); + let streams: &[(u8, u16, &[u8])] = &[(0x02, VIDEO_PID, &[]), (0x06, DATA_PID, KLVA)]; + feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, streams, 0); + feed.picture(start + 80 * MS).data(klv(3, 8)).pcr(start + 120 * MS); + + let mut imported = start_import(&head); + let (old, _) = imported.entry().unwrap(); + let old_reader = reader(&imported, &old).await; + imported.import.decode(&dropped[head.len()..]).unwrap(); + let (frames, end) = old_reader.await.unwrap(); + assert_eq!(end, Ok(true), "the dropped stream's track finishes"); + assert_eq!(frames.len(), 1, "nothing after the drop is carried"); + assert!(imported.entry().is_none(), "the track left the catalog"); + assert_eq!(imported.stats().units, 1); + assert_eq!(imported.stats().quiet, None, "a PID the PMT dropped is not silent"); + + imported.import.decode(&feed.bytes()[dropped.len()..]).unwrap(); + imported.import.finish().unwrap(); + let (name, _) = imported.entry().expect("the PID is carried again"); + assert_ne!(name, old, "a reused PID gets a new name"); + let (frames, err) = read(&imported.consumer, &name).await; + assert!(err.is_none(), "{err:?}"); + assert_eq!(frames.len(), 1); + assert_eq!(&frames[0].payload[..], &feed.pes[2].1[..]); +} + +/// Revisions apply by what they say, across the 5-bit `version_number` wrap: versions 30, 31, +/// then 0 each relabel the stream and open their own track, while a repetition of any of them +/// keeps the track it opened. +#[test] +fn pmt_revisions_apply_across_the_version_wrap() { + let mut feed = Feed::new(0x06, KLVA); + let mut imported = start(false); + let mut names = Vec::new(); + let mut decoded = 0; + let revisions: [(u8, u8, &[u8]); 4] = [ + (0, 0x06, KLVA), + (30, 0x06, KLVA_ENG), + (31, 0x15, METADATA), + (0, 0x06, KLVA), + ]; + for (i, (version, stream_type, descriptors)) in revisions.into_iter().enumerate() { + let streams: &[(u8, u16, &[u8])] = &[(0x02, VIDEO_PID, &[]), (stream_type, DATA_PID, descriptors)]; + if i > 0 { + feed.mux.pmt_at(version, PMT_PID, 1, CLOCK_PID, streams, 0); + } + let bytes = feed.bytes(); + imported.import.decode(&bytes[decoded..]).unwrap(); + let (name, entry) = imported.entry().unwrap(); + assert_eq!( + entry.pes_packet.as_ref().unwrap().stream_type, + stream_type, + "version {version}" + ); + assert_eq!(recorded(&entry), descriptor_loop(descriptors), "version {version}"); + + feed.mux.pmt_at(version, PMT_PID, 1, CLOCK_PID, streams, 0); + let repeated = feed.bytes(); + imported.import.decode(&repeated[bytes.len()..]).unwrap(); + decoded = repeated.len(); + assert_eq!(imported.entry().unwrap().0, name, "a repetition of version {version}"); + names.push(name); + } + let distinct: std::collections::HashSet<_> = names.iter().collect(); + assert_eq!(distinct.len(), revisions.len(), "{names:?}"); +} diff --git a/rs/moq-mux/src/container/ts/test_data/scte35/bbb5s.ts b/rs/moq-mux/src/container/ts/test_data/scte35/bbb5s.ts index 01a8ed69b2a1f15372797bc5dd4e524d50be3ef6..99c0ad9f2597637262deff9abe3a09fd3e561f8e 100644 GIT binary patch delta 755 zcmY+?Pbh<79KdnUkMY_&g?HMnLs66%+x*+Ra*>;sa!?d0rxi6TW{15O&1vsp2Z}65 zqF4Q-73JVSQMjp*#6cMcJB;#-ywA_={r2`g@AG_~-*ZlS&Pls*2edEfdmRhKHPDOe5ihk96RnRB;A!(POU$n70S^<91r)mCPH5)pj{Akfs^$n8^*Y=K0_d>=< z9NJtIA$ZF_A_xtSd=mlzJR!bFNIxEsjr_}bCH08q8!gmh{EMH@^a$W*Da%?22)H%A zmiDtDY?fI;B08A0vj`(Rs5qE5Onfjn!<-5M0)LFJ7yDh$h_Z;4Xi47L>JqrxT+-jP zd7NHm!_*Gvl8j%jz{A8A+o&OW%&X_L2u>WZYAP$fiU|9 Dzjr0g delta 576 zcmX|;O(;ZR6o%*AVceNJG(W~L{olAYZwY!s1FRuc<^YZh};v$1fKg(xFS zqPsXq`6&yrD|YZPwhq`7C}aE_Tylb+v3~qgfpW&}$Xg*)FotHfY!F~v(+DdW zKlScnsA7CwEt8>|@o;0P2x=IwcIdS>hXs9>pk^EfwRP76D5ko%?=%+R#EhSmsQKN) zPxM*ruD;U@-koxt5yP!UjPriQ%zWI~^CDWNp)fBp{6l-P2xhH7%qEm<*J+~86R}Z& zZ&`@pb1BS&FhjQ=mcv2~!;=v33O7pkL@6B=X;XtahfsC&S-PfO7U@2*WTf?3oQOj> zuZ~lzKuxnd#&;p%7V=h#7%6=YM~~pemh;B!89aK0IE@O$XjBZTnv5bUz>C-1E`J?# y{C9xfFF19I5-@I+sB1naZYmPpUewU!ll~5hV|5bs$`{49W{GajC&hH9Gz-6&wasAw diff --git a/rs/moq-mux/src/container/ts/test_data/scte35/ffmpeg.ts b/rs/moq-mux/src/container/ts/test_data/scte35/ffmpeg.ts index e4626389a53547864dcaa71fd36beb8c3456149c..3661f5a70e15e656fd4d8edded81196157bca34d 100644 GIT binary patch delta 778 zcmY+BKS)AR6o+~D^@1o+p+69jqz}U-skCQWDMU3WG&o2!I7m=aLo|egL_$;310f0= zq^XAJ4mU7bY-%Y88gdB|8gpxipx=Xd=Qw`nch0?h=d>$EyJB2_dc}0r_DxjnCt=FE z?@AXl#Qnn^A!c0`@eEoswr6ladPj4*h`N#Vk8k&g81Zt@G{hWnFpS(HUYkN5C!UPH z8)BZgnLwT(uBX9C;-gjY0&y{W-YXV~8#&}l#Ah4e6mh9w8nz|wBJo?2M4VT8}sdT7ZhgoC0k?d6FnN Lw^v_jZD0EX?b{GA delta 581 zcmX|-JxD@P7>4`OTM7P_nwdY8Bsf${Li7VV)M5o84h|6x4HXRy5n&F3(9{rT2$w>l zNs#IcCmRAGh?)u_u%<$43Zf+%g1+;eo7=;C-*fJB-bO)d6tt6fJ2==jRC&o!F+eAw z$^F#|T@sD)meB;8cO{t3Y36os(g04X_ya3Oa1q8sVGZ1beGyE%38#iI^$-q>ylT)x zSR2FCOIV5__7di&5q*T2nS(a)OWeOPgOy(=JsJe)Q!$OLAmP?xSYuhZ;OQllR{pi+ zH@ohi$zqfIc|4D(l>2rNNtszH?Yv|6S=VN!BP%a6{E<@66|tz)7Y>{1KgXCV^?Dgo zrM_{7sZvj!Ba%AZKXDn<&FTKo)x*F2>8)BX{990UjZFQME~?;YG?VJESk7KADrhxbK%t4-jh+*`XJR* Date: Wed, 21 Oct 2026 23:22:25 +0100 Subject: [PATCH 2/7] fix(mux): route a PID listed twice by its first entry; keep a retired section's stats row Co-authored-by: Claude Opus 5.5 --- quest/m1/ts-reliability/README.md | 2 +- quest/m1/ts-reliability/pmt-export.md | 3 ++ rs/moq-mux/examples/scte35_inject.rs | 8 ++++- rs/moq-mux/src/container/ts/import.rs | 36 ++++++++++++++++++- .../src/container/ts/pes_packet_test.rs | 28 +++++++++++++++ 5 files changed, 74 insertions(+), 3 deletions(-) diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index 0328b3473b0..08eaa061916 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -109,7 +109,7 @@ refused means an error or a counted drop. | CAT, EMM/ECM, ATSC PSIP, other SI PIDs, unlisted PIDs, TS scrambling | Outside the profile, raw lane; counted when the import reads every program. Today dropped uncounted | [routes](/quest/m1/ts-reliability/stream-routes.md) | | DVB SI on 0x0010, 0x0011, 0x0012, 0x0014 | Preserved: snapshot track per PID and `table_id` | landed | | Program identity, program and ES descriptors | Preserved: descriptors follow each PMT revision; an identity change refuses | landed | -| PMT add, remove, retype; PID reuse | Normalized: in place under fresh track names, PMT rebuilt at the next `version_number`. Import landed; today refusal or stale PMT on export | [PMT export](/quest/m1/ts-reliability/pmt-export.md) | +| PMT add, remove, retype; PID reuse | Normalized without a new epoch: a changed or reused PID ends its track and continues under a fresh name, and the PMT is rebuilt at the next `version_number`. Import landed; today refusal or stale PMT on export | [PMT export](/quest/m1/ts-reliability/pmt-export.md) | | PAT changes: moved PMT PID, programs added or removed | Normalized: followed; a removed program retires, and returns as a new epoch. Today ignored, or the whole multiplex stops | [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) | | Programs without video | Preserved when the PMT names a PCR PID; refused without one. Today export refuses sections without video and any program without media; import refuses a program without media | [clockless export](/quest/m1/ts-reliability/clockless-export.md) | | PCR | Normalized: rebased, carried on the first video PID, else audio, else a dedicated PCR-only PID; today video or audio only | [clockless export](/quest/m1/ts-reliability/clockless-export.md) | diff --git a/quest/m1/ts-reliability/pmt-export.md b/quest/m1/ts-reliability/pmt-export.md index 07036239381..8ddcfacf05a 100644 --- a/quest/m1/ts-reliability/pmt-export.md +++ b/quest/m1/ts-reliability/pmt-export.md @@ -32,6 +32,9 @@ Decided 2026-10-20 by the maintainer: keeps the PCR PID unless its track retires. A TS receiver sees an ordinary PMT revision and a Hang player sees a rendition switch. +A `programDescriptors` change is an entry change too: the PMT carries the +revised `program_info` at the next `version_number`. + Rust: `rs/moq-mux` TS export. Draft ([mpegts](/drafts/draft-lcurley-moq-mpegts.md)): the rebuild section says how a consumer revises the PMT. diff --git a/rs/moq-mux/examples/scte35_inject.rs b/rs/moq-mux/examples/scte35_inject.rs index 9bde414d41b..6e85b7d92a1 100644 --- a/rs/moq-mux/examples/scte35_inject.rs +++ b/rs/moq-mux/examples/scte35_inject.rs @@ -194,10 +194,16 @@ fn main() -> anyhow::Result<()> { cue_at.len() ); + let first = &packets[pmt_idx]; let mut out = Vec::new(); for (i, p) in packets.iter().enumerate() { if i > pmt_idx && pid_of(p) == pmt_pid.as_u16() { - anyhow::ensure!(p[1] & 0x40 != 0, "a PMT spanning packets cannot be rewritten in place"); + // Only a repeat of the first PMT has its augmented form: a revised PMT, another + // program's section, or a PMT spanning packets would each need its own. + anyhow::ensure!( + p[..3] == first[..3] && p[3] & 0xf0 == first[3] & 0xf0 && p[4..] == first[4..], + "packet {i} on the PMT PID is not a repeat of the first PMT, which this generator cannot rewrite" + ); // The input's own continuity counter, so the rewrite adds no counter error. out.extend_from_slice(&[p[0], p[1], p[2], (aug_pmt[3] & 0xf0) | (p[3] & 0x0f)]); out.extend_from_slice(&aug_pmt[4..]); diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 2dc9641f3d0..1a522e4f7dd 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -479,11 +479,14 @@ impl Import { tracing::warn!(pid = pid.as_u16(), "dropped a PSI section with a bad CRC"); } - fn handle_pmt(&mut self, pmt: Pmt) -> anyhow::Result<()> { + fn handle_pmt(&mut self, mut pmt: Pmt) -> anyhow::Result<()> { // PMTs of several programs may share one PID; only the chosen one maps streams. if self.program.is_some_and(|program| program != pmt.program_number) { return Ok(()); } + // A PID listed twice is routed by its first entry, so the listing kept below says the same. + let mut seen = HashSet::new(); + pmt.streams.retain(|es| seen.insert(es.pid)); // Which PID speaks for the program clock, so a `discontinuity_indicator` there // can be read as a timebase reset rather than a counter jump. if self.pcr_pid != pmt.pcr_pid { @@ -889,6 +892,10 @@ impl Import { let generation = self.placement.generation; let (mut units, mut unplaced) = (0, 0); if let Some(mut section) = self.sections.remove(&pid.as_u16()) { + // Its row outlives the route, as a retired stream's does. + self.retired_stats + .entry(pid.as_u16()) + .or_insert_with(|| stats::Stream::new(".ts", stats::Class::Data)); section.settle(&self.placement); (units, unplaced) = section.release(&self.timebase, &self.pts, generation)?; unplaced += section.held.len() as u64; @@ -9938,4 +9945,31 @@ pub(super) mod test { assert!(err.to_string().contains("identity"), "{err:#}"); } } + + /// A section-framed PID the PMT drops keeps its stats row, so what it delivered stays counted. + #[test] + fn a_dropped_section_route_keeps_its_stats_row() { + const VIDEO_PID: u16 = 0x0050; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let streams = [ + (StreamType::Mpeg2Video, VIDEO_PID), + (StreamType::Dts8ChannelLosslessAudio, 0x21), + ]; + let mut bytes = synth_pmt(&streams, true); + bytes.extend_from_slice(&pcr_packet(VIDEO_PID, 27_000_000)); + bytes.extend_from_slice(&pes_packet(VIDEO_PID, 99_000)); + bytes.extend_from_slice(&packet(true, 0, 0, &CUE)); + bytes.extend_from_slice(&pcr_packet(VIDEO_PID, 29_700_000)); + bytes.extend_from_slice(&synth_pmt(&streams[..1], true)); + import.decode(&bytes).unwrap(); + + let row = &import.stats().streams[&0x21]; + assert_eq!((row.units, row.quiet), (1, None)); + } } diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index 2fbfb37ffd3..019753f55e7 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -920,6 +920,34 @@ async fn a_dropped_private_stream_finishes_and_its_reused_pid_opens_a_new_track( assert_eq!(&frames[0].payload[..], &feed.pes[2].1[..]); } +/// A PMT listing the private PID twice is routed by its first entry, so a revision keeping only +/// the second is a change: the first's track gives way to one for the second. +#[tokio::test(start_paused = true)] +async fn a_pid_listed_twice_is_routed_by_its_first_entry() { + let mut feed = Feed::new(0x06, KLVA); + let twice: &[(u8, u16, &[u8])] = &[ + (0x02, VIDEO_PID, &[]), + (0x06, DATA_PID, KLVA), + (0x15, DATA_PID, METADATA), + ]; + feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, twice, 0); + let start = 1_000 * MS; + feed.pcr(start).picture(start).data(klv(1, 8)).pcr(start + 40 * MS); + let head = feed.bytes(); + let second: &[(u8, u16, &[u8])] = &[(0x02, VIDEO_PID, &[]), (0x15, DATA_PID, METADATA)]; + feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, second, 0); + feed.picture(start + 40 * MS).data(klv(2, 8)).pcr(start + 80 * MS); + + let mut imported = start_import(&head); + let (old, entry) = imported.entry().unwrap(); + assert_eq!(entry.pes_packet.as_ref().unwrap().stream_type, 0x06); + imported.import.decode(&feed.bytes()[head.len()..]).unwrap(); + imported.import.finish().unwrap(); + let (name, entry) = imported.entry().expect("the second entry's stream"); + assert_ne!(name, old); + assert_eq!(entry.pes_packet.as_ref().unwrap().stream_type, 0x15); +} + /// Revisions apply by what they say, across the 5-bit `version_number` wrap: versions 30, 31, /// then 0 each relabel the stream and open their own track, while a repetition of any of them /// keeps the track it opened. From 605059c4ad0cde16599605e9cd054aeba3c3fc2e Mon Sep 17 00:00:00 2001 From: t0ms Date: Thu, 22 Oct 2026 00:19:15 +0100 Subject: [PATCH 3/7] fix(mux): bound TS route churn and refuse a PMT listing a PID twice Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 205 +++++++++++++++++- .../src/container/ts/pes_packet_test.rs | 29 --- 2 files changed, 195 insertions(+), 39 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 1a522e4f7dd..398f8bb1861 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -117,6 +117,8 @@ pub struct Import { streams: HashMap>, /// Counters from routes a later PMT replaced, keyed by PID. retired_stats: BTreeMap, + /// Routes ended or refused so far; see [`Self::MAX_ENDED_ROUTES`]. + routes_ended: usize, /// Damaged packets, PES, or access units refused, cumulative across PID remaps. damaged: BTreeMap, damage_log: super::damage::Log, @@ -229,6 +231,7 @@ impl Import { crc_errors: BTreeMap::new(), streams: HashMap::new(), retired_stats: BTreeMap::new(), + routes_ended: 0, damaged: BTreeMap::new(), damage_log: Default::default(), liveness: Liveness::default(), @@ -479,14 +482,15 @@ impl Import { tracing::warn!(pid = pid.as_u16(), "dropped a PSI section with a bad CRC"); } - fn handle_pmt(&mut self, mut pmt: Pmt) -> anyhow::Result<()> { + fn handle_pmt(&mut self, pmt: Pmt) -> anyhow::Result<()> { // PMTs of several programs may share one PID; only the chosen one maps streams. if self.program.is_some_and(|program| program != pmt.program_number) { return Ok(()); } - // A PID listed twice is routed by its first entry, so the listing kept below says the same. let mut seen = HashSet::new(); - pmt.streams.retain(|es| seen.insert(es.pid)); + if let Some(es) = pmt.streams.iter().find(|es| !seen.insert(es.pid)) { + anyhow::bail!("the PMT lists PID {} twice", es.pid.as_u16()); + } // Which PID speaks for the program clock, so a `discontinuity_indicator` there // can be read as a timebase reset rather than a counter jump. if self.pcr_pid != pmt.pcr_pid { @@ -872,19 +876,43 @@ impl Import { fn retire_stream(&mut self, pid: Pid) -> Option> { self.unclocked.remove(&pid); let stream = self.streams.remove(&pid)?; - if let Some(current) = stream.stats() { + self.keep_stats(pid, stream.stats()); + Some(stream) + } + + /// Keep a retired route's counters for this importer, naming it the PID's latest route. + fn keep_stats(&mut self, pid: Pid, current: Option) { + if let Some(current) = current { self.retired_stats .entry(pid.as_u16()) .and_modify(|retired| retired.merge(¤t)) .or_insert(current); } - Some(stream) + } + + /// Routes one import may end or refuse. A route that reopens takes a fresh track name, and the + /// broadcast reserves every name written for its whole life, so a source that keeps revising + /// its PMT would otherwise grow the broadcast without bound. Under 10 MB of reserved names; + /// a feed revising its PMT 50 times a day would take years to reach it. + const MAX_ENDED_ROUTES: usize = 65_536; + + /// Count a route ending, refusing the import once its source has churned through + /// [`Self::MAX_ENDED_ROUTES`]. + fn count_ended_route(&mut self) -> anyhow::Result<()> { + self.routes_ended += 1; + anyhow::ensure!( + self.routes_ended <= Self::MAX_ENDED_ROUTES, + "the TS source ended {} elementary stream routes, more than an import keeps names for", + Self::MAX_ENDED_ROUTES + ); + Ok(()) } /// End the route on `pid` for a PMT that dropped or revised its stream, as the end of input /// ends every route: the PES it began goes out, what waits on the clock is placed from the /// PCRs so far, and its track finishes and leaves the catalog. fn end_route(&mut self, pid: Pid) -> anyhow::Result<()> { + self.count_ended_route()?; if self.headers.remove(&pid).is_some() { self.damage(pid, &anyhow::anyhow!("PES header truncated by a PMT revision"))?; } @@ -892,16 +920,14 @@ impl Import { let generation = self.placement.generation; let (mut units, mut unplaced) = (0, 0); if let Some(mut section) = self.sections.remove(&pid.as_u16()) { - // Its row outlives the route, as a retired stream's does. - self.retired_stats - .entry(pid.as_u16()) - .or_insert_with(|| stats::Stream::new(".ts", stats::Class::Data)); + self.keep_stats(pid, Some(stats::Stream::new(".ts", stats::Class::Data))); section.settle(&self.placement); (units, unplaced) = section.release(&self.timebase, &self.pts, generation)?; unplaced += section.held.len() as u64; section.finish()?; } - if let Some(mut stream) = self.retire_stream(pid) { + self.unclocked.remove(&pid); + if let Some(mut stream) = self.streams.remove(&pid) { if let Stream::PesPacket(stream) = &mut stream { stream.settle(&self.placement); let (published, dropped) = stream.release(&self.timebase, &self.pts, generation)?; @@ -916,6 +942,8 @@ impl Import { mpegts.tracks.remove(&name); } units += stream.finish()?; + // Finishing can publish a held frame, which its counters then include. + self.keep_stats(pid, stream.stats()); } if unplaced > 0 { *self.unplaced.entry(pid.as_u16()).or_default() += unplaced; @@ -1197,6 +1225,8 @@ impl Import { /// Refuse the undecoded stream on `pid` rather than carry it altered: its track ends with an /// error and leaves the catalog, and the PID is dropped from here on. fn refuse(&mut self, pid: Pid, why: &str) -> anyhow::Result<()> { + // A PMT that keeps losing and regaining its PCR reopens the stream each time. + self.count_ended_route()?; self.pending.remove(&pid); self.headers.remove(&pid); if let Some(section) = self.sections.remove(&pid.as_u16()) { @@ -9972,4 +10002,159 @@ pub(super) mod test { let row = &import.stats().streams[&0x21]; assert_eq!((row.units, row.quiet), (1, None)); } + + /// A PID decoded as audio, then revised to private sections, then dropped, reports the + /// section route it ended on. + #[test] + fn a_dropped_route_reports_the_route_it_ended_on() { + use crate::container::ts::psi::{Es, Pmt}; + const AUDIO_PID: u16 = 0x0062; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let pmt = |streams: &[u8]| Pmt { + program_number: 1, + pcr_pid: Some(Pid::new(PCR_PID).unwrap()), + program_info: Vec::new(), + streams: streams + .iter() + .map(|&stream_type| Es { + stream_type, + pid: Pid::new(AUDIO_PID).unwrap(), + descriptors: Vec::new(), + }) + .collect(), + }; + + import.handle_pmt(pmt(&[MP2])).unwrap(); + import.decode(&mp2_pes(AUDIO_PID, 0, 90_000, [0xAA, 0xBB])).unwrap(); + import.decode(&mp2_pes(AUDIO_PID, 1, 180_000, [0xAA, 0xBB])).unwrap(); + import.handle_pmt(pmt(&[0x05])).unwrap(); + import.handle_pmt(pmt(&[])).unwrap(); + + let row = &import.stats().streams[&AUDIO_PID]; + assert_eq!((row.track.as_str(), row.class), (".ts", super::stats::Class::Data)); + } + + /// Ending a self-describing audio route drains the frame it held for a successor, and its + /// counters include that frame. + #[test] + fn a_dropped_audio_route_counts_the_frame_its_end_drains() { + use crate::container::ts::psi::{Es, Pmt}; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let pmt = |streams: Vec| Pmt { + program_number: 1, + pcr_pid: Some(Pid::new(PCR_PID).unwrap()), + program_info: Vec::new(), + streams, + }; + let audio = Es { + stream_type: MP2, + pid: Pid::new(PCR_PID).unwrap(), + descriptors: Vec::new(), + }; + + // The second frame spans two PES, so it begins in a carried tail and is held for a + // successor header that never comes. + let (mut head, second) = (mp2_frame(0xAA), mp2_frame(0xBB)); + head.extend_from_slice(&second[..36]); + import.handle_pmt(pmt(vec![audio])).unwrap(); + import.decode(&audio_pes_packet(PCR_PID, 0, 90_000, &head)).unwrap(); + import + .decode(&audio_pes_packet(PCR_PID, 1, 92_160, &second[36..])) + .unwrap(); + assert_eq!(import.stats().streams[&PCR_PID].unconfirmed, 0); + import.handle_pmt(pmt(Vec::new())).unwrap(); + + assert_eq!(import.stats().streams[&PCR_PID].unconfirmed, 1); + } + + /// A PMT listing one PID twice is malformed, so it refuses the import rather than pick an + /// entry to route by. + #[test] + fn a_pmt_listing_a_pid_twice_refuses_the_import() { + use crate::container::ts::psi::{Es, Pmt}; + let (_catalog, mut import) = psi_import(); + let es = |stream_type| Es { + stream_type, + pid: Pid::new(0x0062).unwrap(), + descriptors: Vec::new(), + }; + let pmt = Pmt { + program_number: 1, + pcr_pid: Some(Pid::new(PCR_PID).unwrap()), + program_info: Vec::new(), + streams: vec![es(0x06), es(0x15)], + }; + let err = import.handle_pmt(pmt).expect_err("a PID listed twice is refused"); + assert!(err.to_string().contains("twice"), "{err:#}"); + } + + /// Every ended route leaves its track name reserved, so a source that keeps revising its PMT + /// is refused once it has ended as many routes as an import keeps names for. + #[test] + fn pmt_churn_past_the_route_bound_refuses_the_import() { + use crate::container::ts::psi::{Es, Pmt}; + let (_catalog, mut import) = psi_import(); + let pmt = |streams: &[u8]| Pmt { + program_number: 1, + pcr_pid: Some(Pid::new(PCR_PID).unwrap()), + program_info: Vec::new(), + streams: streams + .iter() + .map(|&stream_type| Es { + stream_type, + pid: Pid::new(0x0062).unwrap(), + descriptors: Vec::new(), + }) + .collect(), + }; + import.handle_pmt(pmt(&[0x06])).unwrap(); + import.routes_ended = super::Import::<()>::MAX_ENDED_ROUTES - 1; + + import + .handle_pmt(pmt(&[0x15])) + .expect("the last route within the bound"); + let err = import.handle_pmt(pmt(&[])).expect_err("one route past the bound"); + assert!(err.to_string().contains("routes"), "{err:#}"); + } + + /// A PMT that keeps dropping and restoring its PCR reopens the stream each time, so its + /// refusals count toward the same bound. + #[test] + fn pcr_churn_past_the_route_bound_refuses_the_import() { + use crate::container::ts::psi::{Es, Pmt}; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let pmt = |pcr_pid: Option| Pmt { + program_number: 1, + pcr_pid: pcr_pid.map(|pid| Pid::new(pid).unwrap()), + program_info: Vec::new(), + streams: vec![Es { + stream_type: 0x06, + pid: Pid::new(0x0062).unwrap(), + descriptors: Vec::new(), + }], + }; + import.handle_pmt(pmt(Some(PCR_PID))).unwrap(); + import.routes_ended = super::Import::::MAX_ENDED_ROUTES; + + let err = import.handle_pmt(pmt(None)).expect_err("a refusal past the bound"); + assert!(err.to_string().contains("routes"), "{err:#}"); + } } diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index 019753f55e7..48d137a386e 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -919,35 +919,6 @@ async fn a_dropped_private_stream_finishes_and_its_reused_pid_opens_a_new_track( assert_eq!(frames.len(), 1); assert_eq!(&frames[0].payload[..], &feed.pes[2].1[..]); } - -/// A PMT listing the private PID twice is routed by its first entry, so a revision keeping only -/// the second is a change: the first's track gives way to one for the second. -#[tokio::test(start_paused = true)] -async fn a_pid_listed_twice_is_routed_by_its_first_entry() { - let mut feed = Feed::new(0x06, KLVA); - let twice: &[(u8, u16, &[u8])] = &[ - (0x02, VIDEO_PID, &[]), - (0x06, DATA_PID, KLVA), - (0x15, DATA_PID, METADATA), - ]; - feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, twice, 0); - let start = 1_000 * MS; - feed.pcr(start).picture(start).data(klv(1, 8)).pcr(start + 40 * MS); - let head = feed.bytes(); - let second: &[(u8, u16, &[u8])] = &[(0x02, VIDEO_PID, &[]), (0x15, DATA_PID, METADATA)]; - feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, second, 0); - feed.picture(start + 40 * MS).data(klv(2, 8)).pcr(start + 80 * MS); - - let mut imported = start_import(&head); - let (old, entry) = imported.entry().unwrap(); - assert_eq!(entry.pes_packet.as_ref().unwrap().stream_type, 0x06); - imported.import.decode(&feed.bytes()[head.len()..]).unwrap(); - imported.import.finish().unwrap(); - let (name, entry) = imported.entry().expect("the second entry's stream"); - assert_ne!(name, old); - assert_eq!(entry.pes_packet.as_ref().unwrap().stream_type, 0x15); -} - /// Revisions apply by what they say, across the 5-bit `version_number` wrap: versions 30, 31, /// then 0 each relabel the stream and open their own track, while a repetition of any of them /// keeps the track it opened. From 701ef6009217e838975b575b0e94f3cf561bf0c2 Mon Sep 17 00:00:00 2001 From: t0ms Date: Thu, 22 Oct 2026 01:00:00 +0100 Subject: [PATCH 4/7] fix(mux): refuse a PMT revision that would drop unanchored TS data Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 40 +++++++++++++++++-- .../src/container/ts/pes_packet_test.rs | 19 +++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 398f8bb1861..20dd926c0f3 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -910,7 +910,8 @@ impl Import { /// End the route on `pid` for a PMT that dropped or revised its stream, as the end of input /// ends every route: the PES it began goes out, what waits on the clock is placed from the - /// PCRs so far, and its track finishes and leaves the catalog. + /// PCRs so far (refusing the import, as end of input does, if no media timestamp anchored + /// it), and its track finishes and leaves the catalog. fn end_route(&mut self, pid: Pid) -> anyhow::Result<()> { self.count_ended_route()?; if self.headers.remove(&pid).is_some() { @@ -923,7 +924,10 @@ impl Import { self.keep_stats(pid, Some(stats::Stream::new(".ts", stats::Class::Data))); section.settle(&self.placement); (units, unplaced) = section.release(&self.timebase, &self.pts, generation)?; - unplaced += section.held.len() as u64; + anyhow::ensure!( + section.held.is_empty(), + "TS sections reached a PMT revision without a media timestamp" + ); section.finish()?; } self.unclocked.remove(&pid); @@ -932,7 +936,11 @@ impl Import { stream.settle(&self.placement); let (published, dropped) = stream.release(&self.timebase, &self.pts, generation)?; units += published; - unplaced += dropped + stream.held.len() as u64; + unplaced += dropped; + anyhow::ensure!( + stream.held.is_empty(), + "TS private PES reached a PMT revision without a media timestamp" + ); } // A verbatim entry leaves with its stream; a decoded track's is recorded here. if self.recorded_media.remove(&pid) @@ -10003,6 +10011,32 @@ pub(super) mod test { assert_eq!((row.units, row.quiet), (1, None)); } + /// A PMT dropping a section PID before any media timestamp anchors the clock refuses the + /// import, as end of input does, rather than drop the section it holds. + #[test] + fn a_section_route_dropped_before_the_media_clock_refuses_the_import() { + const VIDEO_PID: u16 = 0x0050; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let streams = [ + (StreamType::Mpeg2Video, VIDEO_PID), + (StreamType::Dts8ChannelLosslessAudio, 0x21), + ]; + let mut bytes = synth_pmt(&streams, true); + bytes.extend_from_slice(&pcr_packet(VIDEO_PID, 27_000_000)); + bytes.extend_from_slice(&packet(true, 0, 0, &CUE)); + bytes.extend_from_slice(&pcr_packet(VIDEO_PID, 29_700_000)); + import.decode(&bytes).unwrap(); + + let err = import.decode(&synth_pmt(&streams[..1], true)).unwrap_err(); + assert!(err.to_string().contains("without a media timestamp"), "{err:#}"); + } + /// A PID decoded as audio, then revised to private sections, then dropped, reports the /// section route it ended on. #[test] diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index 48d137a386e..8a2a42501b0 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -919,6 +919,25 @@ async fn a_dropped_private_stream_finishes_and_its_reused_pid_opens_a_new_track( assert_eq!(frames.len(), 1); assert_eq!(&frames[0].payload[..], &feed.pes[2].1[..]); } + +/// A PMT dropping a private stream before any media timestamp anchors the clock refuses the +/// import, as end of input does, rather than drop the PES it holds. +#[test] +fn a_private_stream_dropped_before_the_media_clock_refuses_the_import() { + let mut feed = Feed::new(0x06, KLVA); + let start = 1_000 * MS; + feed.pcr(start).data(klv(1, 8)).pcr(start + 40 * MS); + let head = feed.bytes(); + feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, &[(0x02, VIDEO_PID, &[])], 0); + + let mut imported = start_import(&head); + let err = imported + .import + .decode(&feed.bytes()[head.len()..]) + .expect_err("the held PES has no media timestamp to be placed by"); + assert!(err.to_string().contains("without a media timestamp"), "{err:#}"); +} + /// Revisions apply by what they say, across the 5-bit `version_number` wrap: versions 30, 31, /// then 0 each relabel the stream and open their own track, while a repetition of any of them /// keeps the track it opened. From 3cfc92bcb42839435a9a60ef53ac8baeeec0463a Mon Sep 17 00:00:00 2001 From: t0ms Date: Thu, 22 Oct 2026 02:26:10 +0100 Subject: [PATCH 5/7] fix(mux): bound TS program descriptor churn and start from the catalog's descriptors Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 136 ++++++++++++++++++++------ 1 file changed, 107 insertions(+), 29 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 20dd926c0f3..8ba786b6b8c 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -117,8 +117,8 @@ pub struct Import { streams: HashMap>, /// Counters from routes a later PMT replaced, keyed by PID. retired_stats: BTreeMap, - /// Routes ended or refused so far; see [`Self::MAX_ENDED_ROUTES`]. - routes_ended: usize, + /// Stream revisions applied so far; see [`Self::MAX_REVISIONS`]. + revisions: usize, /// Damaged packets, PES, or access units refused, cumulative across PID remaps. damaged: BTreeMap, damage_log: super::damage::Log, @@ -176,7 +176,8 @@ pub struct Import { /// Decoded media PIDs already recorded into `mpegts.tracks`, so the reconcile in /// [`Self::flush`] runs once per track rather than on every frame. recorded_media: HashSet, - /// The PMT program-level descriptors last recorded, rewritten in place as the PMT revises them. + /// The program-level descriptors the catalog advertises, rewritten in place as the PMT + /// revises them. program_info: Vec, /// What the PMT in force says of each PID it lists, so a revision that says otherwise ends /// the route rather than carrying the new stream under the old name. @@ -211,7 +212,12 @@ impl Import { // Sample the real catalog once at construction, not E::default(): an extension // may carry the section by value, and a snapshot clones under the mutex (no publish). let mut snapshot = catalog.snapshot(); - let supports_mpegts = snapshot.ext.mpegts_mut().is_some(); + let mpegts = snapshot.ext.mpegts_mut(); + let supports_mpegts = mpegts.is_some(); + // What the catalog already advertises, so the first PMT republishes only a difference. + let program_info = mpegts + .map(|mpegts| std::mem::take(&mut mpegts.program_descriptors)) + .unwrap_or_default(); if supports_mpegts { catalog.count_ext_tracks(|ext: &mut E| ext.mpegts_mut().is_some_and(|mpegts| mpegts.has_track())); } @@ -231,7 +237,7 @@ impl Import { crc_errors: BTreeMap::new(), streams: HashMap::new(), retired_stats: BTreeMap::new(), - routes_ended: 0, + revisions: 0, damaged: BTreeMap::new(), damage_log: Default::default(), liveness: Liveness::default(), @@ -254,7 +260,7 @@ impl Import { supports_mpegts, es_descriptors: HashMap::new(), recorded_media: HashSet::new(), - program_info: Vec::new(), + program_info, listed: HashMap::new(), si_sections: HashMap::new(), si, @@ -515,6 +521,7 @@ impl Import { // Record the program-level descriptors as the PMT revises them; export re-emits them // verbatim, including the original CUEI. if self.supports_mpegts && self.program_info != pmt.program_info { + self.count_revision()?; if let Some(mpegts) = self.catalog.modify()?.ext.mpegts_mut() { mpegts.program_descriptors = pmt.program_info.clone(); } @@ -890,20 +897,22 @@ impl Import { } } - /// Routes one import may end or refuse. A route that reopens takes a fresh track name, and the - /// broadcast reserves every name written for its whole life, so a source that keeps revising - /// its PMT would otherwise grow the broadcast without bound. Under 10 MB of reserved names; - /// a feed revising its PMT 50 times a day would take years to reach it. - const MAX_ENDED_ROUTES: usize = 65_536; + /// Stream revisions one import may apply: routes ended or refused, and program descriptor + /// changes. A route that reopens takes a fresh track name, which the broadcast reserves for + /// its whole life, and each revision republishes the catalog to every viewer, so a source that + /// keeps revising its PMT would otherwise grow the broadcast and churn its viewers without + /// bound. Under 10 MB of reserved names; a feed revising its PMT 50 times a day would take + /// years to reach it. + const MAX_REVISIONS: usize = 65_536; - /// Count a route ending, refusing the import once its source has churned through - /// [`Self::MAX_ENDED_ROUTES`]. - fn count_ended_route(&mut self) -> anyhow::Result<()> { - self.routes_ended += 1; + /// Count a revision, refusing the import once its source has churned through + /// [`Self::MAX_REVISIONS`]. + fn count_revision(&mut self) -> anyhow::Result<()> { + self.revisions += 1; anyhow::ensure!( - self.routes_ended <= Self::MAX_ENDED_ROUTES, - "the TS source ended {} elementary stream routes, more than an import keeps names for", - Self::MAX_ENDED_ROUTES + self.revisions <= Self::MAX_REVISIONS, + "the TS source revised its streams {} times, more than an import accepts", + Self::MAX_REVISIONS ); Ok(()) } @@ -913,7 +922,7 @@ impl Import { /// PCRs so far (refusing the import, as end of input does, if no media timestamp anchored /// it), and its track finishes and leaves the catalog. fn end_route(&mut self, pid: Pid) -> anyhow::Result<()> { - self.count_ended_route()?; + self.count_revision()?; if self.headers.remove(&pid).is_some() { self.damage(pid, &anyhow::anyhow!("PES header truncated by a PMT revision"))?; } @@ -1234,7 +1243,7 @@ impl Import { /// error and leaves the catalog, and the PID is dropped from here on. fn refuse(&mut self, pid: Pid, why: &str) -> anyhow::Result<()> { // A PMT that keeps losing and regaining its PCR reopens the stream each time. - self.count_ended_route()?; + self.count_revision()?; self.pending.remove(&pid); self.headers.remove(&pid); if let Some(section) = self.sections.remove(&pid.as_u16()) { @@ -10135,9 +10144,9 @@ pub(super) mod test { } /// Every ended route leaves its track name reserved, so a source that keeps revising its PMT - /// is refused once it has ended as many routes as an import keeps names for. + /// is refused once it has applied as many revisions as an import accepts. #[test] - fn pmt_churn_past_the_route_bound_refuses_the_import() { + fn pmt_churn_past_the_revision_bound_refuses_the_import() { use crate::container::ts::psi::{Es, Pmt}; let (_catalog, mut import) = psi_import(); let pmt = |streams: &[u8]| Pmt { @@ -10154,19 +10163,19 @@ pub(super) mod test { .collect(), }; import.handle_pmt(pmt(&[0x06])).unwrap(); - import.routes_ended = super::Import::<()>::MAX_ENDED_ROUTES - 1; + import.revisions = super::Import::<()>::MAX_REVISIONS - 1; import .handle_pmt(pmt(&[0x15])) - .expect("the last route within the bound"); - let err = import.handle_pmt(pmt(&[])).expect_err("one route past the bound"); - assert!(err.to_string().contains("routes"), "{err:#}"); + .expect("the last revision within the bound"); + let err = import.handle_pmt(pmt(&[])).expect_err("one revision past the bound"); + assert!(err.to_string().contains("revised"), "{err:#}"); } /// A PMT that keeps dropping and restoring its PCR reopens the stream each time, so its /// refusals count toward the same bound. #[test] - fn pcr_churn_past_the_route_bound_refuses_the_import() { + fn pcr_churn_past_the_revision_bound_refuses_the_import() { use crate::container::ts::psi::{Es, Pmt}; let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new( @@ -10186,9 +10195,78 @@ pub(super) mod test { }], }; import.handle_pmt(pmt(Some(PCR_PID))).unwrap(); - import.routes_ended = super::Import::::MAX_ENDED_ROUTES; + import.revisions = super::Import::::MAX_REVISIONS; let err = import.handle_pmt(pmt(None)).expect_err("a refusal past the bound"); - assert!(err.to_string().contains("routes"), "{err:#}"); + assert!(err.to_string().contains("revised"), "{err:#}"); + } + + /// Each program descriptor change republishes the catalog without ending a route, so it + /// counts toward the same bound. + #[test] + fn descriptor_churn_past_the_revision_bound_refuses_the_import() { + use crate::container::ts::psi::{Es, Pmt}; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let pmt = |data: &'static [u8]| Pmt { + program_number: 1, + pcr_pid: Some(Pid::new(PCR_PID).unwrap()), + program_info: vec![super::catalog::Descriptor { + tag: 0x80, + data: bytes::Bytes::from_static(data), + }], + streams: vec![Es { + stream_type: StreamType::Mpeg2Video as u8, + pid: Pid::new(PCR_PID).unwrap(), + descriptors: Vec::new(), + }], + }; + import.handle_pmt(pmt(&[0xaa])).unwrap(); + import.revisions = super::Import::::MAX_REVISIONS - 1; + + import + .handle_pmt(pmt(&[0xbb])) + .expect("the last revision within the bound"); + let err = import + .handle_pmt(pmt(&[0xaa])) + .expect_err("one revision past the bound"); + assert!(err.to_string().contains("revised"), "{err:#}"); + } + + /// The first PMT records its program descriptors over whatever the catalog held, even when it + /// carries none. + #[test] + fn the_first_pmt_records_its_program_descriptors_even_when_empty() { + use crate::container::ts::psi::{Es, Pmt}; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![super::catalog::Descriptor { + tag: 0x80, + data: bytes::Bytes::from_static(&[0xaa]), + }]; + let mut import = super::Import::new(broadcast, catalog.reserve()); + import + .handle_pmt(Pmt { + program_number: 1, + pcr_pid: Some(Pid::new(PCR_PID).unwrap()), + program_info: Vec::new(), + streams: vec![Es { + stream_type: StreamType::Mpeg2Video as u8, + pid: Pid::new(PCR_PID).unwrap(), + descriptors: Vec::new(), + }], + }) + .unwrap(); + + assert!(catalog.snapshot().ext.mpegts.program_descriptors.is_empty()); } } From ef89cf02c535224a0073eae1e4009fe5f16504bc Mon Sep 17 00:00:00 2001 From: t0ms Date: Thu, 22 Oct 2026 03:32:37 +0100 Subject: [PATCH 6/7] fix(mux): keep a revised TS route until it ends, so a refusal aborts it Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 15 +++++++++++---- rs/moq-mux/src/container/ts/pes_packet_test.rs | 12 +++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 8ba786b6b8c..449eb7fae77 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -929,8 +929,9 @@ impl Import { self.flush(pid)?; let generation = self.placement.generation; let (mut units, mut unplaced) = (0, 0); - if let Some(mut section) = self.sections.remove(&pid.as_u16()) { - self.keep_stats(pid, Some(stats::Stream::new(".ts", stats::Class::Data))); + // Each route stays in its map until it has finished, so a failure on the way leaves it + // for `abort` to end with the cause. + if let Some(section) = self.sections.get_mut(&pid.as_u16()) { section.settle(&self.placement); (units, unplaced) = section.release(&self.timebase, &self.pts, generation)?; anyhow::ensure!( @@ -939,9 +940,12 @@ impl Import { ); section.finish()?; } + if self.sections.remove(&pid.as_u16()).is_some() { + self.keep_stats(pid, Some(stats::Stream::new(".ts", stats::Class::Data))); + } self.unclocked.remove(&pid); - if let Some(mut stream) = self.streams.remove(&pid) { - if let Stream::PesPacket(stream) = &mut stream { + if let Some(stream) = self.streams.get_mut(&pid) { + if let Stream::PesPacket(stream) = &mut *stream { stream.settle(&self.placement); let (published, dropped) = stream.release(&self.timebase, &self.pts, generation)?; units += published; @@ -959,6 +963,8 @@ impl Import { mpegts.tracks.remove(&name); } units += stream.finish()?; + } + if let Some(stream) = self.streams.remove(&pid) { // Finishing can publish a held frame, which its counters then include. self.keep_stats(pid, stream.stats()); } @@ -10044,6 +10050,7 @@ pub(super) mod test { let err = import.decode(&synth_pmt(&streams[..1], true)).unwrap_err(); assert!(err.to_string().contains("without a media timestamp"), "{err:#}"); + assert!(import.sections.contains_key(&0x21), "the route is left for `abort`"); } /// A PID decoded as audio, then revised to private sections, then dropped, reports the diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index 8a2a42501b0..dddc3225e94 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -921,9 +921,10 @@ async fn a_dropped_private_stream_finishes_and_its_reused_pid_opens_a_new_track( } /// A PMT dropping a private stream before any media timestamp anchors the clock refuses the -/// import, as end of input does, rather than drop the PES it holds. -#[test] -fn a_private_stream_dropped_before_the_media_clock_refuses_the_import() { +/// import, as end of input does, rather than drop the PES it holds; aborting the import then +/// ends the stream's track with its cause. +#[tokio::test(start_paused = true)] +async fn a_private_stream_dropped_before_the_media_clock_refuses_the_import() { let mut feed = Feed::new(0x06, KLVA); let start = 1_000 * MS; feed.pcr(start).data(klv(1, 8)).pcr(start + 40 * MS); @@ -931,11 +932,16 @@ fn a_private_stream_dropped_before_the_media_clock_refuses_the_import() { feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, &[(0x02, VIDEO_PID, &[])], 0); let mut imported = start_import(&head); + let (name, _) = imported.entry().unwrap(); + let held = reader(&imported, &name).await; let err = imported .import .decode(&feed.bytes()[head.len()..]) .expect_err("the held PES has no media timestamp to be placed by"); assert!(err.to_string().contains("without a media timestamp"), "{err:#}"); + imported.import.abort(moq_net::Error::Unsupported); + let (_, end) = held.await.unwrap(); + assert!(end.as_ref().is_err_and(|err| err.contains("unsupported")), "{end:?}"); } /// Revisions apply by what they say, across the 5-bit `version_number` wrap: versions 30, 31, From 3c8b541c900b491a8621ab276622fd01e36fe2d8 Mon Sep 17 00:00:00 2001 From: t0ms Date: Thu, 22 Oct 2026 04:19:55 +0100 Subject: [PATCH 7/7] fix(mux): count a revised TS route's unit unplaced past a lone PCR, as end of input does Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/import.rs | 36 +++++++++++++++++-- .../src/container/ts/pes_packet_test.rs | 19 ++++++++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 449eb7fae77..ad8fa278f19 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -919,15 +919,17 @@ impl Import { /// End the route on `pid` for a PMT that dropped or revised its stream, as the end of input /// ends every route: the PES it began goes out, what waits on the clock is placed from the - /// PCRs so far (refusing the import, as end of input does, if no media timestamp anchored - /// it), and its track finishes and leaves the catalog. + /// PCRs so far or counted unplaced (refusing the import, as end of input does, if no media + /// timestamp anchored it), and its track finishes and leaves the catalog. fn end_route(&mut self, pid: Pid) -> anyhow::Result<()> { self.count_revision()?; if self.headers.remove(&pid).is_some() { self.damage(pid, &anyhow::anyhow!("PES header truncated by a PMT revision"))?; } self.flush(pid)?; - let generation = self.placement.generation; + // Released as if its timebase ended here, so a unit no later PCR will bracket for it is + // unplaced rather than held. + let generation = self.placement.generation + 1; let (mut units, mut unplaced) = (0, 0); // Each route stays in its map until it has finished, so a failure on the way leaves it // for `abort` to end with the cause. @@ -10053,6 +10055,34 @@ pub(super) mod test { assert!(import.sections.contains_key(&0x21), "the route is left for `abort`"); } + /// A PMT dropping a section PID past a lone PCR ends it as end of input would: the section + /// no PCR interval places is counted unplaced. + #[test] + fn a_section_route_dropped_past_a_lone_pcr_counts_its_section_unplaced() { + const VIDEO_PID: u16 = 0x0050; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let streams = [ + (StreamType::Mpeg2Video, VIDEO_PID), + (StreamType::Dts8ChannelLosslessAudio, 0x21), + ]; + let mut bytes = synth_pmt(&streams, true); + bytes.extend_from_slice(&pcr_packet(VIDEO_PID, 27_000_000)); + bytes.extend_from_slice(&pes_packet(VIDEO_PID, 99_000)); + bytes.extend_from_slice(&packet(true, 0, 0, &CUE)); + bytes.extend_from_slice(&synth_pmt(&streams[..1], true)); + import.decode(&bytes).unwrap(); + + let row = &import.stats().streams[&0x21]; + assert_eq!((row.units, row.unplaced), (0, 1)); + assert!(!import.sections.contains_key(&0x21)); + } + /// A PID decoded as audio, then revised to private sections, then dropped, reports the /// section route it ended on. #[test] diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index dddc3225e94..bbfc0978fa5 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -944,6 +944,25 @@ async fn a_private_stream_dropped_before_the_media_clock_refuses_the_import() { assert!(end.as_ref().is_err_and(|err| err.contains("unsupported")), "{end:?}"); } +/// A PMT dropping a private stream past a lone PCR ends it as end of input would: the PES no +/// PCR interval places is counted unplaced, and its track finishes. +#[tokio::test(start_paused = true)] +async fn a_private_stream_dropped_past_a_lone_pcr_counts_its_pes_unplaced() { + let mut feed = Feed::new(0x06, KLVA); + feed.pcr(1_000 * MS).picture(1_000 * MS).data(klv(0, 8)); + let head = feed.bytes(); + feed.mux.pmt_with(PMT_PID, 1, CLOCK_PID, &[(0x02, VIDEO_PID, &[])], 0); + + let mut imported = start_import(&head); + let (name, _) = imported.entry().unwrap(); + let held = reader(&imported, &name).await; + imported.import.decode(&feed.bytes()[head.len()..]).unwrap(); + let (frames, end) = held.await.unwrap(); + assert!(frames.is_empty()); + assert!(end.is_ok(), "{end:?}"); + assert_eq!(imported.stats().unplaced, 1); +} + /// Revisions apply by what they say, across the 5-bit `version_number` wrap: versions 30, 31, /// then 0 each relabel the stream and open their own track, while a repetition of any of them /// keeps the track it opened.