diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 5ae868e04f0..5eeb015d1ce 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -72,6 +72,15 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs shell: bash -leo pipefail {0} + # A KLV stream joins the program mid-stream and leaves it: the export announces each + # layout at the next PMT version, and carries the stream only while a PMT lists it. + - name: TS PMT revision + if: ${{ !cancelled() }} + run: nix develop --command just test ts --revision --strict + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + - name: TS audio-only SCTE-35 if: ${{ !cancelled() }} run: nix develop --command just test ts --clockless audio-scte35 diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index c91e3683676..450f7b1c76a 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -372,6 +372,7 @@ A consumer rebuilding a transport stream: - MUST place each track on its recorded `pid`, and MUST assign an unused PID to a track with none. - MUST put the PMT on `program.pmtPid` and build the PAT and PMT from `program`. - MUST choose a PCR PID and emit PCRs on it at the program clock's repetition interval. With no media stream it MUST use an unused PID with adaptation-only PCR packets and no PMT elementary-stream entry. It MUST NOT refuse a program for lacking video. +- MUST follow the catalog as it changes: when a track joins or leaves `tracks` or `programDescriptors` changes, it MUST emit a PMT at the next `version_number` (modulo 32) before any packet of a stream it adds, and MUST NOT emit a stream's packets after a PMT that no longer lists it. Each version it emits MUST differ from the one before, and no older version may follow a newer one. The PCR PID MUST NOT move until the stream it rides ends. - MUST re-emit each track's `descriptors` as its ES-level descriptors, and `programDescriptors` as the PMT's `program_info`. - MUST re-emit each `si` entry's sections byte-for-byte on that entry's PID, reading them from its track ({{si-track}}), at least as often as its `interval` when declared. - MUST packetize each `verbatim` frame as one section on its track's PID. @@ -457,6 +458,7 @@ A broadcast demultiplexed from a DVB transport stream: video and audio described - A flagged PCR jump on an `m2ts` track follows hang's discontinuity marker rather than a skipped group sequence. - An `m2ts` track unwraps PCRs from its history, clamps extrapolated object times at a forward discontinuity, and cuts a fallback group after one second. `randomAccess` covers every published group; `muxRate` follows the measured source rate. - A `tracks` entry is never changed in place: a stream the PMT revises continues under a new name, and one it drops leaves the section. `programDescriptors` follows the PMT; `program`'s `transportStreamId` and `programNumber` do not change. +- A consumer rebuilding a stream follows the catalog's revisions at the next PMT `version_number`, and carries a stream only while a PMT lists it. ## draft-lcurley-moq-mpegts-00 {:numbered="false"} diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index afe0a17fe83..ccbb46d0361 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -72,7 +72,7 @@ refused means an error or a counted drop. | CAT, EMM/ECM, ATSC PSIP, other SI PIDs, unlisted PIDs, TS scrambling | Outside the profile, raw lane; counted when the import reads every program. Today dropped uncounted | [routes](/quest/m1/ts-reliability/stream-routes.md) | | DVB SI on 0x0010, 0x0011, 0x0012, 0x0014 | Preserved: snapshot track per PID and `table_id` | landed | | Program identity, program and ES descriptors | Preserved: descriptors follow each PMT revision; an identity change refuses | landed | -| PMT add, remove, retype; PID reuse | Normalized without a new epoch: a changed or reused PID ends its track and continues under a fresh name, and the PMT is rebuilt at the next `version_number`. Import landed; today refusal or stale PMT on export | [PMT export](/quest/m1/ts-reliability/pmt-export.md) | +| PMT add, remove, retype; PID reuse | Normalized without a new epoch: a changed or reused PID ends its track and continues under a fresh name, and the export rebuilds the PMT at the next `version_number`. Today `export ts --sync` ends at an added track and announces no other revision | [PMT export](/quest/m1/ts-reliability/pmt-export.md) | | PAT changes: moved PMT PID, programs added or removed | Normalized: followed; a removed program retires, and returns as a new epoch. Today ignored, or the whole multiplex stops | [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) | | Programs without video | Preserved when the PMT names a PCR PID; refused without one | landed | | PCR | Normalized: rebased, carried on the first video PID, else audio, else a dedicated PCR-only PID | landed | @@ -102,7 +102,7 @@ remain with their current quests. - [State bound](/quest/m1/ts-reliability/state-bound.md) - SI and revision state retire with what the source stops repeating - [Decode order](/quest/m1/ts-reliability/decode-order.md) - TS, MKV and FLV author DTS from decode order and a declared `reorder` depth - [Shared parser](/quest/m1/ts-reliability/shared-parser.md) - `--program all` scans the multiplex once -- [PMT export](/quest/m1/ts-reliability/pmt-export.md) - a TS export announces each PMT revision, mux rate included, at the next `version_number` +- [PMT export](/quest/m1/ts-reliability/pmt-export.md) - synced TS legs revise their PMT and grid identically - [Access units](/quest/m1/ts-reliability/pes-access-units.md) - a PES carrying several access units publishes each at its own time - [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a revised PAT never leaves a broadcast meaning another program - [Stream routes](/quest/m1/ts-reliability/stream-routes.md) - every PID gets a declared route, and what is not carried is counted diff --git a/quest/m1/ts-reliability/pmt-export.md b/quest/m1/ts-reliability/pmt-export.md index 9eb77bb4d26..026d97b6ccc 100644 --- a/quest/m1/ts-reliability/pmt-export.md +++ b/quest/m1/ts-reliability/pmt-export.md @@ -1,61 +1,52 @@ -# [M] TS export follows a PMT revision +# [M] Synced TS legs revise their PMT identically ## Goal -A TS export of a broadcast whose layout changes mid-stream announces the -revised layout as an ordinary PMT revision. A track added, retired, or -replaced under a fresh name changes the PMT at the next `version_number`; -nothing else blips. +Two `moq export ts --sync` legs of a broadcast whose layout changes +mid-stream revise their PMT identically: the same `version_number`, in the +same slot, listing the same streams, wherever both legs hold the content +around it. A track added no longer ends a synced export. ## Plan -The import already keeps track names honest: a changed `stream_type` or ES -descriptor loop finishes the old track, drops its catalog entry, and opens -a new uniquely named one; a removed stream finishes its track and leaves -the catalog; `programDescriptors` updates in place; a program identity -change refuses. - -What the export does today: - -- `Export` builds the PAT/PMT once. A track added afterward fails the export - ("TS track layout changed after PAT/PMT was emitted"), so every revision - the import publishes ends the export. -- A track that leaves the catalog keeps its PID in the PMT forever, and - `programDescriptors` is read once. -- A switch to another instance already rebuilds the tables at the next - `version_number`. - -Decided 2026-10-20 by the maintainer: - -- Export rebuilds the PMT at the next `version_number` when the track set - or an entry changes, drops a retired track's PID once its track ends, and - keeps the PCR PID unless its track retires. A TS receiver sees an - ordinary PMT revision and a Hang player sees a rendition switch. - -A `programDescriptors` change is an entry change too: the PMT carries the -revised `program_info` at the next `version_number`. - -Under `moq export ts --sync`, the revision is a function of the catalog -alone, so identical legs revise their PMT identically. - -Decided 2026-10-22 by the maintainer: a revision is one transaction over -everything the catalog configures. A `muxRate` change applies with the -tables at the same boundary and, under `--sync`, rebuilds the grid there: -today `Schedule::set_rate` updates the rate while the `--sync` `Grid` keeps -the layout it was built at, so a running leg and a new joiner schedule the -same broadcast differently (reproduced in the 2026-10-22 audit). - -Rust: `rs/moq-mux` TS export. Draft -([mpegts](/drafts/draft-lcurley-moq-mpegts.md)): the rebuild section says -how a consumer revises the PMT. - -Tests use the generated `fixture::Mux` builder through import and export: -PMT add, remove, retype, descriptor-only change, version rollover (31 to -0), PID reuse after removal, a program descriptor change, and a `muxRate` -change under `--sync` with a leg joining after it, each asserting the -exported PMT's streams and version sequence, and the two legs' packets. Extend the -TSDuck round trip with a generated mid-stream PMT revision so -`compliance.py` checks the rebuilt version sequence in CI. +Outside `--sync`, the export follows a PMT revision at the next +`version_number`: a stream joins the PMT with its first frame, and leaves +once the catalog has dropped it and nothing of it is left to go out. A track +recorded on the PID of a dropped stream waits for that stream to drain, and +cuts it if it never ends. Each revision is one boundary: versions go out in +order, no packet of a stream a revision adds goes ahead of it, and its PCR +PID and `muxRate` take over after its last packet. The PCR keeps its PID +unless its track retires. + +What `--sync` does today: + +- The tables are laid once, from the catalog the leg opened on. A track + added afterward fails the export ("… under --sync"). +- A track that leaves the catalog keeps its PID and its lane, and an entry + or `programDescriptors` change is not announced. +- A `muxRate` change is warned and ignored: the leg keeps the rate its grid + was laid at. +- So legs that open on either side of a catalog change lay different PMTs + and grids, and a pair does not survive an added stream. + +Decided 2026-10-22 by the maintainer: under `moq export ts --sync`, the +revision is a function of the catalog alone, so identical legs revise +their PMT identically. A revision is one transaction over everything the +catalog configures: a `muxRate` change applies with the tables at the same +boundary and, under `--sync`, rebuilds the grid there. + +Open: where a revision lands. Each leg reads the catalog update at its own +moment, so the slot that carries the revised PMT, and its `version_number`, +must follow from something both legs read alike, such as an added track's +first unit on its lane or the end of a retired track's last group. A +revision that adds a track also needs a lane the rate can carry. + +Rust: `rs/moq-mux` TS export (`with_sync`). + +Tests: two synced legs, joined apart, across a PMT add, remove, retype, +descriptor change and `muxRate` change, write the same packets from the +revision on, PMTs included; a leg that joins after a revision lays the +revised PMT at the same `version_number`, and the revised grid. ## Related diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index bb20a3944f0..b61bc65f93c 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -125,25 +125,29 @@ where /// Tracks [`Self::follow`] left on the ended broadcast, resubscribed as the returned /// catalog lists them. stale: HashSet, - /// The PAT and PMT `version_number`, advanced by each switch to another instance so a - /// demux that caches tables by version reads the new ones. + /// The PAT and PMT `version_number` the program's first tables go out at, advanced by each + /// switch to another instance so a demux that caches tables by version reads the new ones. version: VersionNumber, /// After a switch to another instance, the PIDs whose first packet is queued flagging the /// break. `None` for an export that never switched. flagged: Option>, /// How long after its decode time each frame goes out. delay: Duration, - /// Holds every track's frames until `delay` past their decode time, keyed by PID. - jitter: jitter::Buffer, + /// Holds every track's frames until `delay` past their decode time, keyed by name. + jitter: jitter::Buffer, /// A frame the jitter buffer let go, waiting for the tail of the generation before it to go out. - held: Option>, + held: Option>, /// Jitter-buffer generation of the program being muxed; a newer one rewinds it. generation: u64, tracks: HashMap, - /// Tables muxed with frames still packed, which ride ahead of the next unit queued, so - /// they go out in the order they were written. + /// SI muxed with frames still packed, which rides ahead of the next unit queued, so it + /// goes out in the order it was written. tables: Vec, + /// The program tables are due ahead of the next unit queued. + psi_due: bool, + /// The program tables changed, so the next frame muxed is a tune-in point that leads with them. + revised: bool, /// The next continuity counter per PID, numbered as packets go out. counters: HashMap, /// PMT program-level descriptors captured on import, re-emitted in the PMT. @@ -161,7 +165,7 @@ where /// The trailing SI flush ran (once, just before end of stream). si_flushed: bool, - /// Program tables, built once the track layout is known. + /// Program tables, built once the track layout is known and revised as it changes. psi: Option, /// Media timestamp of the last PAT/PMT emission ([`due`]). last_psi: Option, @@ -220,6 +224,8 @@ struct Tally { pcr: (u64, bool), /// The PID of each access unit whose packets begin in the frame. units: Vec, + /// The last program tables to finish in the frame, as [`Psi::queued`] holds them. + tables: Option<(VersionNumber, psi::Pmt)>, } /// A frame read from its source. @@ -292,6 +298,17 @@ struct Track { base: (u64, u64, u64), finished: bool, pid: u16, + /// The PMT lists it: from its first frame out until its last has gone and the catalog has + /// dropped it. Its PID, kind and descriptors stay as listed. + listed: bool, + /// The catalog lists it. + cataloged: bool, + /// Read no further because the catalog withdrew it before the PMT listed it. A return + /// under the same name reads it on. + withdrawn: bool, + /// Frames released while its PID still carries the stream before it, which go out once + /// that stream's last has. + parked: VecDeque, kind: Kind, /// PMT ES-level descriptors to re-announce, captured verbatim on import (language, /// registration, ...). Empty for non-TS sources; AC-3/E-AC-3 then synthesize one. @@ -333,7 +350,7 @@ struct Track { struct Input<'a> { waiting_for_header: bool, video_start: Option, - jitter: &'a mut jitter::Buffer, + jitter: &'a mut jitter::Buffer, /// Counts the frames dropped ahead of the tune-in point or their codec config. trimmed: &'a mut u64, } @@ -350,6 +367,10 @@ impl Track { base: (0, 0, 0), finished: false, pid: 0, + listed: false, + cataloged: true, + withdrawn: false, + parked: VecDeque::new(), kind, descriptors: Vec::new(), clock: DecodeClock::default(), @@ -372,7 +393,12 @@ impl Track { /// The decode clock runs here, as frames are read, because the deadline is on decode time: /// reordered video arrives in decode order with PTS 0, 120, 40, 80, so a PTS deadline would /// strand a B-frame behind its reference or send it first. - fn queue(&mut self, name: &str, pending: Pending, jitter: &mut jitter::Buffer) -> anyhow::Result<()> { + fn queue( + &mut self, + name: &str, + pending: Pending, + jitter: &mut jitter::Buffer, + ) -> anyhow::Result<()> { if pending.restart != self.restart { // A marker declared a break in the timeline, so the decode clock restarts too. self.release(name, jitter, true)?; @@ -401,7 +427,7 @@ impl Track { && self.sync.is_none() && !radl(stream_type, self.source.description(), &pending.frame) { - jitter.lose(&self.pid, ahead); + jitter.lose(name, ahead); return Ok(()); } // Frames still held from a group cut short wait on frames that will not come, so under @@ -428,7 +454,7 @@ impl Track { // A placed frame is judged at release, against the keyframe's departure ([`Export::leads_video`]). let aligned = !is_video && !self.kind.placed(); loop { - if !input.waiting_for_header && input.jitter.full(&self.pid) { + if !input.waiting_for_header && input.jitter.full(name) { break; } match self.source.poll_event(waiter)? { @@ -442,7 +468,8 @@ impl Track { // A group arrives from its first frame, so the first read from it opens it. let sequence = self.source.group().unwrap_or_default(); let opens = self.group.replace(sequence) != Some(sequence); - if input.waiting_for_header && !self.source.header_ready() { + // A track waits for its codec config before the PMT can list it. + if (input.waiting_for_header || !self.listed) && !self.source.header_ready() { *input.trimmed += 1; continue; } @@ -481,7 +508,7 @@ impl Track { } Some(_) => self.clock.discard(), } - input.jitter.cut(&self.pid, self.base.2 + self.source.cuts()); + input.jitter.cut(&name.to_owned(), self.base.2 + self.source.cuts()); } Poll::Ready(Some(Event::GroupEnd | Event::Cut)) => self.settle = self.sync.is_none(), Poll::Ready(Some(Event::FrameEnd { .. })) => {} @@ -806,7 +833,7 @@ impl Track { /// /// Under sync, a stream whose SPS declares its depth is settled by counting frames at the /// SPS's delay, so a frame's DTS never depends on what the leg read before it. - fn release(&mut self, name: &str, jitter: &mut jitter::Buffer, flush: bool) -> anyhow::Result<()> { + fn release(&mut self, name: &str, jitter: &mut jitter::Buffer, flush: bool) -> anyhow::Result<()> { let (delay, lookahead) = self.timing.reorder(); let lookahead = lookahead.max(self.timing.jitter); let counted = self.sync.as_ref().and(self.timing.depth()); @@ -829,7 +856,7 @@ impl Track { pending: Pending, dts: Option, description: Option, - jitter: &mut jitter::Buffer, + jitter: &mut jitter::Buffer, ) -> anyhow::Result<()> { let Pending { frame, @@ -891,7 +918,7 @@ impl Track { planned, }, }; - if jitter.push(self.pid, arrival)? == Push::Late { + if jitter.push(name.to_string(), arrival)? == Push::Late { tracing::warn!( track = %name, dropped = jitter.dropped(), @@ -1066,14 +1093,35 @@ impl Kind { } } -/// The program tables plus the resolved PID layout. +/// The program tables: the PAT, fixed for the program's life, and the PMT, which lists the +/// streams on the air. struct Psi { pat: Bytes, - pmt: Bytes, - pcr_pid: u16, + /// What the PMT says now. + pmt: psi::Pmt, /// PID the PMT rides on: the source's original (preserved from the service /// record) or the synthesized [`PMT_PID`] for a media-only source. pmt_pid: u16, + /// The version and PMT last handed to the schedule, which the next copy follows. + pushed: Option<(VersionNumber, psi::Pmt)>, + /// The versions handed to the schedule and not yet returned, oldest first: they go out in + /// the order they were pushed ([`Schedule::lead`]). + queued: VecDeque<(VersionNumber, psi::Pmt)>, + /// The version and PMT of the last tables returned. + out: Option<(VersionNumber, psi::Pmt)>, +} + +impl Psi { + /// The PID the PMT names for the clock. + fn pcr_pid(&self) -> Option { + self.pmt.pcr_pid.map(|pid| pid.as_u16()) + } + + /// The PID the clock rides alone, which no stream may take. + fn own_clock(&self) -> Option { + self.pcr_pid() + .filter(|&pcr| self.pmt.streams.iter().all(|es| es.pid.as_u16() != pcr)) + } } /// Per-frame PES descriptor (everything but the payload bytes). @@ -1363,6 +1411,8 @@ where version: VersionNumber::default(), flagged: None, tables: Vec::new(), + psi_due: false, + revised: false, delay: Duration::ZERO, jitter: jitter::Buffer::new(Duration::ZERO), held: None, @@ -1596,6 +1646,7 @@ where if let Some(out) = self.pop() { return Poll::Ready(Ok(Some(out))); } + self.settle()?; let ready = match self.held.take() { Some(ready) => ready, None => match self.jitter.poll_next(waiter) { @@ -1618,19 +1669,23 @@ where ready.generation >= self.generation, "the jitter buffer released an older generation after a newer one" ); - let (name, kind) = self - .tracks - .iter() - .find(|(_, t)| t.pid == ready.track) - .map(|(name, t)| (name.clone(), t.kind.clone())) - .context("frame for an unknown PID")?; + let name = ready.track.clone(); + let kind = self.tracks.get(&name).context("missing track")?.kind.clone(); if self.leads_video(&kind, &ready.item) { self.jitter.discard(&ready); } else { self.last_timestamp = Some(ready.item.frame.timestamp); let decode = ready.item.decode.as_nanos(); self.lay_before(ready.item.key.as_nanos())?; - self.mux(&name, ready.item)?; + match self.list(&name)? { + true => self.mux(&name, ready.item)?, + false => self + .tracks + .get_mut(&name) + .expect("a listed track") + .parked + .push_back(ready.item), + } if self.delay.is_zero() { // Without a clock to lay slots on, a frame settles every slot before its own: a // later frame decodes no earlier, so it can be due no earlier than this, less the @@ -1667,6 +1722,7 @@ where self.flush_packs(|_, _| true)?; } self.lay_due(waiter)?; + self.settle()?; if let Some(out) = self.pop() { return Poll::Ready(Ok(Some(out))); } @@ -1676,7 +1732,7 @@ where // independent of the catalog: a retained track finishes while the broadcast stays // live, and holding its tail until the catalog closed would strand it indefinitely. let drained = !self.tracks.is_empty() - && self.tracks.values().all(|t| t.finished) + && self.tracks.values().all(|t| t.finished && t.parked.is_empty()) && self.jitter.is_empty() && self.held.is_none(); if drained { @@ -1710,6 +1766,11 @@ where /// Return the next queued frame, committing what it carries to the output boundary. fn pop(&mut self) -> Option { let (out, tally) = self.queue.pop_front()?; + if let Some(tables) = tally.tables + && let Some(psi) = &mut self.psi + { + psi.out = Some(tables); + } self.emitted_epoch = self.epoch; let (pcr, discontinuity) = tally.pcr; if discontinuity { @@ -1782,7 +1843,11 @@ where reject_ambiguous_tracks(&catalog, &mpegts)?; self.source.retain_selectable(&mut catalog); - self.program_descriptors = mpegts.program_descriptors.clone(); + // Under sync the tables and the grid are laid once, from the catalog the leg opened on. + let locked = self.sync && self.psi.is_some(); + if !locked { + self.program_descriptors = mpegts.program_descriptors.clone(); + } self.program = mpegts.program.clone(); // An explicit override wins even when it is refused (leaving the output // unpadded), so a bad flag cannot silently fall back to the catalog rate. @@ -1795,8 +1860,23 @@ where }) }), }; - self.mux_rate = mux_rate; - self.schedule.set_rate(mux_rate); + if mux_rate != self.mux_rate { + match locked { + true => tracing::warn!( + mux_rate, + kept = self.mux_rate, + "ignoring a multiplex rate change under --sync, which keeps the grid it was laid at" + ), + // A revision is one boundary: the rate rides the next tables out. + false => { + self.mux_rate = mux_rate; + self.revised = true; + } + } + } + if self.psi.is_none() { + self.schedule.set_rate(self.mux_rate); + } // Reconcile the SI subscriptions with the catalog's map. Entries may appear // after the PAT/PMT is built (a table acquired late): they ride standalone @@ -1859,69 +1939,110 @@ where } } - // The program tables are written once; reject a track added afterwards. - // - // A track that leaves the catalog is not a layout change: it keeps its PID and is - // read to its own end. A publisher retires a rendition as its track finishes or - // drops, and the catalog update races that end on another stream, so the leaving - // cannot say which it was. The track's end does: a finish ends it cleanly, and a - // drop is the error the export reports. if self.psi.is_some() { - for name in active.keys() { - anyhow::ensure!( - self.tracks.contains_key(name), - "TS track layout changed after PAT/PMT was emitted: '{name}' added" - ); + if locked { + for name in active.keys() { + anyhow::ensure!( + self.tracks.contains_key(name), + "TS track layout changed after PAT/PMT was emitted under --sync: '{name}' added" + ); + } } if !self.stale.is_empty() { self.resubscribe(&catalog, &mpegts)?; } - // The PCR may ride a PID of its own, which SI must not take either. - let mut es_pids: Vec = self.tracks.values().map(|t| t.pid).collect(); - es_pids.push(self.psi.as_ref().unwrap().pcr_pid); - reject_colliding_si_pids(&mpegts, self.pmt_pid(), &es_pids)?; - // The layout is locked but the timing is not: a `jitter` or `framerate` published - // after the tables still sizes the decode clock. + // A listed track keeps its PID, kind and descriptors, which its packets went out + // under: a stream the source revises comes under a new name. The timing is not + // fixed: a `jitter` or `framerate` published after the tables still sizes the + // decode clock. for (name, config) in catalog.video.renditions.iter() { - if let Some(track) = self.tracks.get_mut(name) { + if let Some(track) = self.tracks.get_mut(name).filter(|track| track.listed) { track.timing.configure(config, name); } } - return Ok(()); + // A track that leaves the catalog is read to its own end. A publisher retires a + // rendition as its track finishes or drops, and the catalog update races that end + // on another stream, so the leaving cannot say which it was. The track's end does: + // a finish ends it cleanly, and a drop is the error the export reports. One the PMT + // does not list yet is read no further, so a publisher that never ends what it + // withdraws leaves nothing behind but what it already sent, which still goes out. A + // name that returns before that is gone is the same stream, so it is read on. + for (name, track) in self.tracks.iter_mut() { + track.cataloged = locked || active.contains_key(name); + if track.cataloged && std::mem::take(&mut track.withdrawn) { + track.finished = false; + } + if !track.cataloged && !track.listed && !track.finished { + track.release(name, &mut self.jitter, true)?; + track.finished = true; + track.withdrawn = true; + } + } + self.revise()?; } - // Assign a PID to every desired track: prefer the original recorded in the - // `mpegts` section, then fill the rest from FIRST_ES_PID. The importer fills - // PIDs, descriptors, and stream_ids across several catalog publishes, so this - // runs every snapshot until the PMT is built and the tracks below are - // *refreshed*, not latched from the first (partial) snapshot. - let mut used: Vec = vec![0x0000, self.pmt_pid(), 0x1FFF]; + // Assign a PID to every desired track the PMT does not list yet: prefer the original + // recorded in the `mpegts` section, then fill the rest from FIRST_ES_PID. The importer + // fills PIDs, descriptors, and stream_ids across several catalog publishes, so this runs + // every snapshot until the PMT lists the track, which is *refreshed*, not latched from + // the first (partial) snapshot. A recorded PID may take over one a stream the catalog + // dropped still drains on ([`Self::list`]); any other PID the PMT lists is taken. + let pmt_pid = self.pmt_pid(); + let mut taken: Vec = vec![0x0000, pmt_pid, 0x1FFF]; + taken.extend(self.psi.as_ref().and_then(|psi| psi.own_clock())); + taken.extend(self.tracks.values().filter(|t| t.listed && t.cataloged).map(|t| t.pid)); + let draining: Vec = self + .tracks + .values() + .filter(|t| t.listed && !t.cataloged) + .map(|t| t.pid) + .collect(); + let joining: Vec<&String> = active + .keys() + .filter(|name| self.tracks.get(*name).is_none_or(|track| !track.listed)) + .collect(); let mut pids: BTreeMap = BTreeMap::new(); - for name in active.keys() { + for &name in &joining { if let Some(pid) = mpegts.tracks.get(name).map(|t| t.pid) - && !used.contains(&pid) + && !taken.contains(&pid) { - used.push(pid); + taken.push(pid); pids.insert(name.clone(), pid); } } - for name in active.keys() { + for &name in &joining { if !pids.contains_key(name) { let mut pid = FIRST_ES_PID; - while used.contains(&pid) { + while taken.contains(&pid) || draining.contains(&pid) { pid += 1; } - used.push(pid); + taken.push(pid); pids.insert(name.clone(), pid); } } - let es_pids: Vec = pids.values().copied().collect(); - reject_colliding_si_pids(&mpegts, self.pmt_pid(), &es_pids)?; + let es_pids: Vec = taken + .iter() + .chain(&draining) + .copied() + .filter(|&pid| pid >= 0x10 && pid != 0x1FFF) + .collect(); + reject_colliding_si_pids(&mpegts, pmt_pid, &es_pids)?; - // Reuse each track's existing source (and any pending frame) by name; refresh - // its PID, kind, and descriptors from this snapshot. Drop tracks no longer present. - let mut old = std::mem::take(&mut self.tracks); + // Reuse each joining track's source (and any pending frame) by name; refresh its PID, + // kind, and descriptors from this snapshot. Before the tables, drop tracks no longer + // present; after them, a track the catalog dropped stays until what it holds is gone + // ([`Self::settle`]). + let mut old: HashMap = self + .tracks + .extract_if(|name, track| !track.listed && active.contains_key(name)) + .collect(); + if self.psi.is_none() { + self.tracks.clear(); + } for (name, config) in catalog.video.renditions.iter() { + let Some(&pid) = pids.get(name) else { + continue; + }; let kind = video_kind(config, name)?; let mut track = match old.remove(name) { Some(track) => track, @@ -1934,9 +2055,12 @@ where }; track.kind = kind; track.timing.configure(config, name); - self.refresh(name, track, pids[name], track_descriptors(&mpegts, name)); + self.refresh(name, track, pid, track_descriptors(&mpegts, name)); } for (name, config) in catalog.audio.renditions.iter() { + let Some(&pid) = pids.get(name) else { + continue; + }; let kind = audio_kind(config, name)?; let mut track = match old.remove(name) { Some(track) => track, @@ -1949,9 +2073,12 @@ where }; track.buffer = Some(audio_buffer(config, &kind)); track.kind = kind; - self.refresh(name, track, pids[name], track_descriptors(&mpegts, name)); + self.refresh(name, track, pid, track_descriptors(&mpegts, name)); } for (name, entry) in mpegts.tracks.iter() { + let Some(&pid) = pids.get(name) else { + continue; + }; let kind = match (&entry.verbatim, &entry.pes_packet) { (Some(verbatim), _) => Kind::Verbatim { stream_type: verbatim.stream_type, @@ -1970,7 +2097,7 @@ where }; track.buffer = verbatim_buffer(&kind, &entry.descriptors); track.kind = kind; - self.refresh(name, track, pids[name], entry.descriptors.clone()); + self.refresh(name, track, pid, entry.descriptors.clone()); } // The clock anchors on whichever track a source sends latest against its decode time // (audio just in time while video runs most of a second ahead), so it waits to hear @@ -1979,9 +2106,10 @@ where // which would hold every join for the full bound. let continuous = self .tracks - .values() - .filter(|track| track.kind.class() != super::stats::Class::Data || track.buffer.is_some()); - self.jitter.expect(continuous.map(|track| track.pid)); + .iter() + .filter(|(_, track)| !track.finished) + .filter(|(_, track)| track.kind.class() != super::stats::Class::Data || track.buffer.is_some()); + self.jitter.expect(continuous.map(|(name, _)| name.clone())); Ok(()) } @@ -2069,7 +2197,7 @@ where if self.psi.is_none() { return stats; } - for track in self.tracks.values() { + for track in self.tracks.values().filter(|track| track.listed) { let (units, quiet) = self.liveness.stream(track.pid); let row = super::stats::Stream { track: track.kind.suffix().to_string(), @@ -2089,6 +2217,39 @@ where self.jitter.len() } + /// How many tracks the export carries, listed or not. + #[cfg(test)] + pub(super) fn tracks(&self) -> usize { + self.tracks.len() + } + + /// `pid`'s access units and how long it has been quiet, as [`Self::stats`] reads them. + #[cfg(test)] + pub(super) fn liveness(&self, pid: u16) -> (u64, Option) { + self.liveness.stream(pid) + } + + /// The track the PMT lists on `pid`. + #[cfg(test)] + pub(super) fn listed_on(&self, pid: u16) -> Option<&str> { + self.tracks + .iter() + .find(|(_, track)| track.listed && track.pid == pid) + .map(|(name, _)| name.as_str()) + } + + /// Take the next frame due out of the jitter buffer and hold it, as a generation boundary + /// holds the new generation's first frame behind the old one's tail. Returns its PID. + #[cfg(test)] + pub(super) fn hold_next(&mut self) -> Option { + let Poll::Ready(ready) = self.jitter.poll_next(&kio::Waiter::noop()) else { + return None; + }; + let pid = self.tracks.get(&ready.track)?.pid; + self.held = Some(ready); + Some(pid) + } + /// When the next queued frame is due, or the next grid slot while media is queued or /// every source has finished and the tail runs out. #[cfg(test)] @@ -2120,8 +2281,9 @@ where /// Carry on into `broadcast`, using the caller's replacement `catalog` stream. /// /// The same publisher instance (an equal epoch) continues the stream: the program and - /// its PSI stay as announced, and its catalog resubscribes each track as it lists it, so a - /// gap is a skip like any other. One that adds a track fails as a layout change. + /// its PSI carry on, and its catalog resubscribes each track as it lists it, so a gap is a + /// skip like any other. A track it adds or drops revises the PMT, as any catalog update + /// does, except under [`Self::with_sync`], where an added one fails as a layout change. /// /// Another instance (another epoch, or an epochless route) is a full program switch: a /// fresh export of `broadcast` with a new PMT from its catalog, and nothing carried from @@ -2161,12 +2323,19 @@ where next = next.with_mux_rate(rate); } // A switch away from a program that went out, or one still pending because this - // program never built its tables, carries to the next: its tables must read as new, - // every PID flags the break, and a pacing caller re-anchors on the new clock. Only a - // program that went out advances them again. - let emitted = self.psi.is_some(); + // program sent no tables, carries to the next: its tables must read as new, every PID + // flags the break, and a pacing caller re-anchors on the new clock. Only a program + // that went out advances them again. + let out = self + .psi + .as_ref() + .and_then(|psi| psi.out.as_ref()) + .map(|(version, _)| *version); + let emitted = out.is_some(); if (!same && emitted) || self.flagged.is_some() { - next.version = self.version; + // The last PMT version out, which a revision may have advanced. Versions still + // queued never reach a receiver. + next.version = out.unwrap_or(self.version); next.flagged = Some(HashSet::new()); next.pcr_discontinuity = true; next.epoch = self.epoch; @@ -2193,6 +2362,7 @@ where for (name, track) in self.tracks.iter_mut() { track.release(name, &mut self.jitter, true)?; track.finished = true; + track.withdrawn = false; self.stale.insert(name.clone()); } Ok(()) @@ -2230,9 +2400,9 @@ where } let next = self .tracks - .values() - .filter(|track| matches!(track.kind, Kind::Video(_))) - .filter_map(|track| self.jitter.peek(&track.pid)) + .iter() + .filter(|(_, track)| matches!(track.kind, Kind::Video(_))) + .filter_map(|(name, _)| self.jitter.peek(name.as_str())) .map(|next| match placed { true => next.key, false => next.frame.timestamp.max(start), @@ -2266,8 +2436,13 @@ where self.pcr_discontinuity = !self.sync; self.si_slot = None; self.tables.clear(); + if let Some(psi) = &mut self.psi { + psi.queued.clear(); + psi.pushed = psi.out.clone(); + } for track in self.tracks.values_mut() { track.pack = None; + track.parked.clear(); } } @@ -2303,7 +2478,7 @@ where /// to flag the break. The PCR PID's first packet always opens a slot with its PCR, flagged /// on its own ([`Self::lay`]). fn breaks(&mut self, pid: u16) -> bool { - let pcr = self.psi.as_ref().map(|psi| psi.pcr_pid); + let pcr = self.psi.as_ref().and_then(Psi::pcr_pid); match &mut self.flagged { Some(flagged) => Some(pid) != pcr && !self.counters.contains_key(&pid) && flagged.insert(pid), None => false, @@ -2312,7 +2487,11 @@ where /// PID the PMT rides on: the source's original (preserved in the service record), /// or the synthesized [`PMT_PID`] for a media-only source or an invalid (zero) value. + /// Once the tables are built they keep the PID they went out on. fn pmt_pid(&self) -> u16 { + if let Some(psi) = &self.psi { + return psi.pmt_pid; + } self.program .as_ref() .map(|s| s.pmt_pid) @@ -2340,36 +2519,82 @@ where } } - // Order tracks by PID for a stable layout; first video track carries the PCR. - let mut tracks: Vec<&Track> = self.tracks.values().collect(); - tracks.sort_by_key(|t| t.pid); - + for track in self.tracks.values_mut() { + track.listed = true; + } let pcr_pid = match sync { // A media PID flags the break at every group, which on the PCR's PID would declare a // break in the time base, so under sync the PCR rides a PID of its own. Some(_) => self.clock_pid()?, - None => tracks - .iter() - .find(|t| matches!(t.kind, Kind::Video(_))) - .or_else(|| { - tracks.iter().find(|t| { - matches!( - t.kind, - Kind::Aac { .. } | Kind::Opus { .. } | Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 - ) - }) + None => self.pick_clock()?, + }; + let pmt = self.program_map(pcr_pid)?; + + // Preserve the source's program identity so the rebuilt PAT stays consistent with the + // carried SI; synthesize a minimal identity otherwise. + let pmt_pid = self.pmt_pid(); + let transport_stream_id = self.program.as_ref().map(|s| s.transport_stream_id).unwrap_or(1); + let pat = Pat { + transport_stream_id, + version_number: self.version, + table: vec![ProgramAssociation { + program_num: pmt.program_number, + program_map_pid: Pid::new(pmt_pid)?, + }], + }; + + self.schedule.set_tables(pmt_pid); + self.schedule.set_clock(pcr_pid); + for track in self.tracks.values() { + self.liveness.register(track.pid); + } + let psi = Psi { + pat: super::psi::encode_pat(&pat)?, + pmt, + pmt_pid, + pushed: None, + queued: VecDeque::new(), + out: None, + }; + if let Some(rate) = sync { + self.lay_lanes(rate, &psi)?; + } + self.psi = Some(psi); + self.psi_due = !self.sync; + Ok(()) + } + + /// The PID to carry the clock: the first video track's, else the first audio track's, else + /// one used by nothing else, since sparse data cannot carry the clock's cadence. + fn pick_clock(&self) -> anyhow::Result { + let mut tracks: Vec<&Track> = self.tracks.values().filter(|t| t.listed).collect(); + tracks.sort_by_key(|t| t.pid); + tracks + .iter() + .find(|t| matches!(t.kind, Kind::Video(_))) + .or_else(|| { + tracks.iter().find(|t| { + matches!( + t.kind, + Kind::Aac { .. } | Kind::Opus { .. } | Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 + ) }) - .map(|t| t.pid) - // Sparse data cannot carry the clock's cadence: reserve a PID used only by PCR. - .or_else(|| { - (FIRST_ES_PID..0x1fff).find(|pid| { - *pid != self.pmt_pid() - && tracks.iter().all(|track| track.pid != *pid) - && self.si.keys().all(|(si, _)| si != pid) - }) + }) + .map(|t| t.pid) + .or_else(|| { + (FIRST_ES_PID..0x1fff).find(|pid| { + *pid != self.pmt_pid() + && self.tracks.values().all(|track| track.pid != *pid) + && self.si.keys().all(|(si, _)| si != pid) }) - .context("TS export has no unused PID for the PCR")?, - }; + }) + .context("TS export has no unused PID for the PCR") + } + + /// The PMT listing every listed track in PID order, with the clock on `pcr_pid`. + fn program_map(&self, pcr_pid: u16) -> anyhow::Result { + let mut tracks: Vec<&Track> = self.tracks.values().filter(|t| t.listed).collect(); + tracks.sort_by_key(|t| t.pid); let streams = tracks .iter() @@ -2424,47 +2649,182 @@ where Vec::new() }; - // Preserve the source's program identity so the rebuilt PAT/PMT stay consistent - // with the carried SI; synthesize a minimal identity otherwise. - let pmt_pid = self.pmt_pid(); - let transport_stream_id = self.program.as_ref().map(|s| s.transport_stream_id).unwrap_or(1); - let program_number = self - .program - .as_ref() - .map(|s| s.program_number) - .filter(|&id| id != 0) - .unwrap_or(1); - - let pat = Pat { - transport_stream_id, - version_number: self.version, - table: vec![ProgramAssociation { - program_num: program_number, - program_map_pid: Pid::new(pmt_pid)?, - }], + // A revision stays on the program the PAT went out naming. + let program_number = match &self.psi { + Some(psi) => psi.pmt.program_number, + None => self + .program + .as_ref() + .map(|s| s.program_number) + .filter(|&id| id != 0) + .unwrap_or(1), }; - let pmt = super::psi::Pmt { + Ok(super::psi::Pmt { program_number, pcr_pid: Some(Pid::new(pcr_pid)?), program_info, streams, + }) + } + + /// Rebuild the PMT from the listed tracks, leading the next frame muxed when it changed. The + /// clock stays on its PID unless the stream carrying it retired. Under sync the tables + /// were laid once and never change. + fn revise(&mut self) -> anyhow::Result<()> { + let Some(psi) = self.psi.as_ref().filter(|_| !self.sync) else { + return Ok(()); }; + let current = psi.pcr_pid().context("PMT without a clock")?; + let kept = + psi.own_clock() == Some(current) || self.tracks.values().any(|track| track.listed && track.pid == current); + let pcr_pid = match kept { + true => current, + false => self.pick_clock()?, + }; + let pmt = self.program_map(pcr_pid)?; + let psi = self.psi.as_mut().expect("checked above"); + if pmt != psi.pmt { + psi.pmt = pmt; + self.revised = true; + } + Ok(()) + } - self.schedule.set_tables(pmt_pid); - self.schedule.set_clock(pcr_pid); - for track in self.tracks.values() { + /// List `name` in the PMT ahead of its first frame: `false` while it waits for its PID, + /// which a stream the catalog dropped still holds. That stream is no longer read, so only + /// what it already holds drains ahead of this one ([`Self::settle`]). + fn list(&mut self, name: &str) -> anyhow::Result { + let track = self.tracks.get(name).context("missing track")?; + if track.listed { + return Ok(true); + } + if !track.parked.is_empty() { + return Ok(false); + } + let pid = track.pid; + if let Some((holder, track)) = self + .tracks + .iter_mut() + .find(|(other, track)| track.listed && track.pid == pid && other.as_str() != name) + { + if !track.finished { + track.release(holder, &mut self.jitter, true)?; + track.finished = true; + } + // Replaced, so a return of its name no longer reads it on. + track.withdrawn = false; + // A same-instance return that put another track on its PID replaced it. + self.stale.remove(holder.as_str()); + return Ok(false); + } + self.tracks.get_mut(name).expect("checked above").listed = true; + self.liveness.register(pid); + self.revise()?; + Ok(true) + } + + /// Retire each track the catalog dropped once nothing of it is left to go out, and list + /// the tracks that waited for its PID ([`Self::list`]). + fn settle(&mut self) -> anyhow::Result<()> { + if self.psi.is_none() || self.sync { + return Ok(()); + } + let gone: Vec = + self.tracks + .iter() + .filter(|(name, track)| { + // A track the same instance's return has yet to resubscribe stays, silent on its PID. + !track.cataloged + && !self.stale.contains(name.as_str()) + && track.finished && track.pack.is_none() + && track.parked.is_empty() + && !self.jitter.holds(name.as_str()) + && self.held.as_ref().is_none_or(|held| held.track != **name) + && !(track.listed && self.schedule.holds(track.pid)) + }) + .map(|(name, _)| name.clone()) + .collect(); + let mut changed = false; + for name in gone { + let track = self.tracks.remove(&name).expect("collected above"); + self.jitter.forget(name.as_str()); + self.stale.remove(&name); + if track.listed { + self.liveness.retire(track.pid); + self.schedule.forget(track.pid); + changed = true; + } + } + // A freed PID goes to the track that has waited for it longest; the rest wait their turn. + let mut waiting: Vec<(Timestamp, &String, u16)> = self + .tracks + .iter() + .filter_map(|(name, track)| Some((track.parked.front()?.key, name, track.pid))) + .collect(); + waiting.sort(); + let mut claimed: Vec = self.tracks.values().filter(|t| t.listed).map(|t| t.pid).collect(); + let mut free = Vec::new(); + for (_, name, pid) in waiting { + if !claimed.contains(&pid) { + claimed.push(pid); + free.push(name.clone()); + } + } + for name in &free { + let track = self.tracks.get_mut(name).expect("collected above"); + track.listed = true; self.liveness.register(track.pid); + changed = true; } - let psi = Psi { - pat: super::psi::encode_pat(&pat)?, - pmt: super::psi::encode_pmt(&pmt, self.version)?, - pcr_pid, - pmt_pid, - }; - if let Some(rate) = sync { - self.lay_lanes(rate, &psi)?; + if changed { + self.revise()?; } - self.psi = Some(psi); + for name in free { + let parked = std::mem::take(&mut self.tracks.get_mut(&name).expect("collected above").parked); + for queued in parked { + self.mux(&name, queued)?; + } + } + Ok(()) + } + + /// Hand the program tables to the schedule ahead of the next unit, if they are due. A PMT + /// that differs from the last handed over goes out under the next version. + fn lead_tables(&mut self) -> anyhow::Result<()> { + if !std::mem::take(&mut self.psi_due) { + return Ok(()); + } + let psi = self.psi.as_mut().context("PSI not built")?; + let version = match &psi.pushed { + Some((version, pmt)) if *pmt == psi.pmt => *version, + Some((version, _)) => { + let mut version = *version; + version.increment(); + version + } + None => self.version, + }; + let added: Vec = psi + .pmt + .streams + .iter() + .filter(|es| psi.pushed.as_ref().is_none_or(|(_, pmt)| !pmt.streams.contains(es))) + .map(|es| es.pid.as_u16()) + .collect(); + let section = super::psi::encode_pmt(&psi.pmt, version)?; + let clock = psi.pcr_pid().context("PMT without a clock")?; + psi.pushed = Some((version, psi.pmt.clone())); + psi.queued.push_back((version, psi.pmt.clone())); + let (pat, pmt_pid) = (psi.pat.clone(), psi.pmt_pid); + let mut packets = Vec::new(); + self.write_section(&mut packets, Pid::PAT, &pat)?; + self.write_section(&mut packets, pmt_pid, §ion)?; + self.schedule.lead(super::schedule::Tables { + packets, + clock, + rate: self.mux_rate, + added, + }); Ok(()) } @@ -2497,7 +2857,12 @@ where // Flagged, so each emission has the adaptation field a jump sets the flag in ([`Grid::lay`]). let mut tables = Vec::new(); section_packets(&mut tables, Pid::PAT, &psi.pat, true)?; - section_packets(&mut tables, psi.pmt_pid, &psi.pmt, true)?; + section_packets( + &mut tables, + psi.pmt_pid, + &super::psi::encode_pmt(&psi.pmt, self.version)?, + true, + )?; let tables = tables.as_chunks::<{ TsPacket::SIZE }>().0.to_vec(); let mut needs: Vec<(u16, Need)> = self .tracks @@ -2533,8 +2898,9 @@ where .. } = queued; let is_video = matches!(self.tracks.get(name).context("missing track")?.kind, Kind::Video(_)); - // Refresh PSI at keyframes or after the interval lapses. - let psi = (is_video && frame.keyframe) || due(frame.timestamp, self.last_psi, PSI_INTERVAL); + // Refresh PSI at keyframes or after the interval lapses, and lead with a revision. + let cadence = (is_video && frame.keyframe) || due(frame.timestamp, self.last_psi, PSI_INTERVAL); + let psi = std::mem::take(&mut self.revised) || cadence; if psi { // A receiver tunes in at the tables, so each AAC PID's next frame repeats its program // config element. Its older frames go on ahead of the tables, so that frame starts a @@ -2560,13 +2926,7 @@ where let mut out = Vec::with_capacity(TsPacket::SIZE); - if psi { - let psi = self.psi.as_ref().context("PSI not built")?; - let pmt_pid = psi.pmt_pid; - let pat = psi.pat.clone(); - let pmt = psi.pmt.clone(); - self.write_section(&mut out, Pid::PAT, &pat)?; - self.write_section(&mut out, pmt_pid, &pmt)?; + if cadence { self.last_psi = Some(frame.timestamp); } @@ -2614,13 +2974,18 @@ where } } + // The tables lead the unit that carries this frame, behind whatever went before it. if kind.placed() { + self.psi_due |= psi; return self.place(name, &frame, out); } match es_payload { // Section-framed verbatim (SCTE-35, ...) rides in private sections, not PES; // carry the bytes verbatim. - None => self.write_section(&mut out, pid, &frame.payload)?, + None => { + self.psi_due |= psi; + self.write_section(&mut out, pid, &frame.payload)?; + } Some(es_payload) => { let unit = PesUnit { pid, @@ -2633,9 +2998,10 @@ where // Only a delay leaves the frames time to wait for the ones after them. let track = self.tracks.get(name).context("missing track")?; if let Some((samples, rate)) = track.samples(&es_payload).filter(|_| !self.delay.is_zero()) { - return self.pack(name, unit, decode, out, es_payload, (samples, rate, pes)); + return self.pack(name, unit, decode, (psi, out), es_payload, (samples, rate, pes)); } self.flush(name)?; + self.psi_due |= psi; self.write_pes(&mut out, &unit, &es_payload)?; } } @@ -2688,6 +3054,7 @@ where if let Some(buffer) = buffer { self.schedule.set_buffer(pid, buffer); } + self.lead_tables()?; self.schedule.push_placed( pid, (place, decode.unwrap_or(place)), @@ -2741,7 +3108,8 @@ where } /// Mux an audio frame into `name`'s PES, sending the PES on first if the frame cannot join - /// it, and leave the tables muxed ahead of it to ride the next unit queued. A PES carries + /// it, and leave the program tables due (`psi`) and the SI muxed ahead of it to ride the + /// next unit queued. A PES carries /// frames while they run on from its PTS without a gap and fit the PID's decoder buffer /// whole with its header, so it arrives by its first frame's decode time as one frame /// would, and they span at most half the delay: its frames leave the jitter buffer with @@ -2757,7 +3125,7 @@ where name: &str, unit: PesUnit, decode: Timestamp, - tables: Vec, + (psi, tables): (bool, Vec), payload: Vec, (samples, rate, start): (u64, u32, Option), ) -> anyhow::Result<()> { @@ -2777,6 +3145,7 @@ where if !joins { self.flush(name)?; } + self.psi_due |= psi; self.tables.extend_from_slice(&tables); let track = self.tracks.get_mut(name).context("missing track")?; let pack = track.pack.get_or_insert(Pack { @@ -2872,6 +3241,7 @@ where ); } let by = decode.as_nanos().saturating_sub(track.drain().as_nanos()); + self.lead_tables()?; self.schedule.push_frames( pid, super::schedule::Timing { @@ -2978,10 +3348,9 @@ where /// grid instead: slots on the media timeline, shared by every exporter of the /// broadcast, like [`due`]. fn lay(&mut self, known: Option) -> anyhow::Result<()> { - let Some(psi) = self.psi.as_ref() else { + let Some(pmt_pid) = self.psi.as_ref().map(|psi| psi.pmt_pid) else { return Ok(()); }; - let (pcr_pid, pmt_pid) = (psi.pcr_pid, psi.pmt_pid); if known.is_none() { self.flush_packs(|_, _| true)?; } @@ -3005,7 +3374,7 @@ where let Some(slot) = self.schedule.next(known)? else { break; }; - let clock = pcr_packet(pcr_pid, slot.pcr)?; + let clock = pcr_packet(slot.clock.context("a slot without a clock PID")?, slot.pcr)?; let mut payload = slot.layout(&clock, pmt_pid, &NULL_PACKET); // The slot opens with its PCR, in the clock packet or the clock PID's packet that // carries it, whose adaptation field flags the break. @@ -3023,9 +3392,16 @@ where payload: Bytes::from(payload), keyframe: slot.keyframe, }; + let psi = self.psi.as_mut().expect("checked above"); + // Under sync the grid lays the one version the tables were built at. + let tables = match self.sync { + true => psi.out.is_none().then(|| (self.version, psi.pmt.clone())), + false => (0..slot.tables).filter_map(|_| psi.queued.pop_front()).last(), + }; let tally = Tally { pcr: (slot.pcr, discontinuity), units: slot.units, + tables, }; self.queue.push_back((frame, tally)); } @@ -4103,17 +4479,17 @@ mod tests { }; let start = moq_clock::now(); let mut jitter = super::jitter::Buffer::new(Duration::from_millis(500)); - jitter.expect([1, 2]); + jitter.expect(["video".to_string(), "audio".to_string()]); let mut trimmed = 0; for publisher in &mut publishers { publisher.write(frame(0)).unwrap(); } // As in Export::poll_next, the first held frame of each track arrives first. kio::wait(|waiter| { - for track in &mut tracks { + for (track, name) in tracks.iter_mut().zip(["video", "audio"]) { track .fill( - "track", + name, waiter, &mut super::Input { waiting_for_header: false, @@ -4142,7 +4518,7 @@ mod tests { for index in order { tracks[index] .fill( - "track", + ["video", "audio"][index], &waiter, &mut super::Input { waiting_for_header: false, @@ -4163,7 +4539,7 @@ mod tests { std::task::Poll::Ready(ready) => Some(ready), std::task::Poll::Pending => None, }) - .find(|ready| ready.track == 1) + .find(|ready| ready.track == "video") .expect("a video start"); assert_eq!(video.item.frame.timestamp.as_micros(), 600_000, "{order:?}"); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 895d4b725ce..3243a9db0dc 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -2450,6 +2450,93 @@ async fn aac_program_config_leads_a_packed_pes() { } } +/// A stream added mid-stream revises the tables off their cadence, and a receiver tunes in +/// there too: each AAC PID's next PES after any PAT leads with its program config element, the +/// one already open when the revision came included. +#[tokio::test(start_paused = true)] +async fn aac_program_config_follows_a_revision() { + let mut quad = vec![0x11, 0x80, 0x04, 0xC4, 0x04, 0x00, 0x21, 0x10, 0x0C]; + quad.extend_from_slice(b"Lavc63.1.101"); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut add = |name: &str| { + let track = broadcast + .create_track(name, hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 4); + cfg.container = Container::Legacy; + cfg.description = Some(Bytes::from(quad.clone())); + catalog.modify().unwrap().audio.renditions.insert(name.to_string(), cfg); + Producer::new(track, HangContainer::Legacy(Kind::Audio)) + }; + // 1,024 samples at 48 kHz, so the frames run on and share PESs, a few to each so one starts + // between the revision and the next repeat. + let write = |producer: &mut Producer, offset: u64, frames: std::ops::Range| { + for i in frames { + producer + .write(Frame { + timestamp: Timestamp::from_micros(offset + i * 1_024 * 1_000_000 / 48_000).unwrap(), + duration: None, + payload: Bytes::from(vec![0x20; 400]), + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + } + }; + let mut a = add("a.aac"); + let mut export = open_export(crate::source::announced(&consumer).await) + .await + .unwrap() + .with_delay(RECORDING_MAX_AGE) + .with_replay(); + write(&mut a, 0, 0..58); + let mut frames = poll_frames(&mut export); + // The second stream joins 1.25 s in, halfway between table repeats. + let mut b = add("b.aac"); + frames.extend(poll_frames(&mut export)); + write(&mut a, 0, 58..96); + write(&mut b, 1_250_000, 0..40); + a.finish().unwrap(); + b.finish().unwrap(); + catalog.finish().unwrap(); + frames.extend(drain_frames(&mut export).await); + let ts: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); + + // Whether each PES start, per PID, is the first on its PID since a PAT, and leads with the element. + let mut tuned: std::collections::HashSet = Default::default(); + let mut starts: Vec<(u16, bool, bool)> = Vec::new(); + let mut reader = TsPacketReader::new(Cursor::new(&ts[..])); + while let Some(packet) = reader.read_ts_packet().unwrap() { + let pid = packet.header.pid.as_u16(); + match packet.payload { + Some(TsPayload::Pat(_)) => tuned.clear(), + Some(TsPayload::PesStart(pes)) => { + let header = super::adts::Header::parse(&pes.data).unwrap(); + // ID_PCE in the first three bits of the raw data block. + starts.push((pid, tuned.insert(pid), pes.data[header.header_len] >> 5 == 5)); + } + _ => {} + } + } + assert_eq!( + starts + .iter() + .map(|(pid, ..)| pid) + .collect::>() + .len(), + 2 + ); + for (i, (pid, first, pce)) in starts.iter().enumerate() { + assert!( + !first || *pce, + "PES {i} on PID {pid}: the first since a PAT lacks the element" + ); + } +} + /// GStreamer 1.28 `fdkaacenc` output, 48 kHz stereo, remuxed to FLV by ffmpeg 9.0.1. Its /// AudioSpecificConfig signals SBR (and PS for v2) explicitly: object type 5 or 29 over an LC core /// at 24 kHz, stereo for v1 and mono for v2. ADTS has two bits for the object type, so export @@ -10520,3 +10607,228 @@ async fn placed_sections_keep_their_source_positions() { ); } } + +/// Returns of one instance whose catalogs rename the track on its PID retire each one left +/// out once the next holds the PID: the export keeps one track however many returns come. +#[tokio::test(start_paused = true)] +async fn a_track_renamed_on_its_pid_across_returns_retires_the_one_before() { + let origin = crate::source::produce_origin(); + let epoch = moq_net::Epoch::mint(); + let route = || moq_net::origin::Route::default().with_epoch(epoch.clone()); + let rendition = |broadcast: &mut moq_net::broadcast::Producer, + catalog: &mut crate::catalog::Producer, + name: &str| { + let track = aac_rendition(broadcast, catalog, name); + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(name.to_string(), tscat::Track::new(0x101)); + track + }; + + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + let mut track = rendition(&mut broadcast, &mut catalog, "0.aac"); + let source = crate::source::resolved(&origin, "live").await; + let mut export = open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap() + .with_delay(RECORDING_MAX_AGE) + .with_replay(); + let start = tokio::time::Instant::now(); + for ms in (0..200).step_by(20) { + write_aac(&mut track, ms); + } + drain_frames(&mut export).await; + drop((broadcast, catalog, track)); + let _ = drain_to_end(&mut export).await; + + for round in 1..=4 { + tokio::time::sleep(Duration::from_secs(1)).await; + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + let back = origin.consume().routed_broadcast("live").await.unwrap(); + let stream = crate::catalog::Consumer::new(&back, Default::default()).await.unwrap(); + export = export.follow(back, stream).unwrap(); + drain_frames(&mut export).await; + let mut track = rendition(&mut broadcast, &mut catalog, &format!("{round}.aac")); + let resumed = start.elapsed().as_millis() as u64; + for ms in (resumed..resumed + 200).step_by(20) { + write_aac(&mut track, ms); + } + drain_frames(&mut export).await; + drop((broadcast, catalog, track)); + let _ = drain_to_end(&mut export).await; + } + assert_eq!(export.tracks(), 1); +} + +/// A track a same-instance return renames on its PID counts its silence from its own listing, +/// not from the last unit of the stale one before it, which retires only once it is listed. +#[tokio::test(start_paused = true)] +async fn a_track_renamed_on_its_pid_across_a_return_is_quiet_from_its_listing() { + let origin = crate::source::produce_origin(); + let epoch = moq_net::Epoch::mint(); + let route = || moq_net::origin::Route::default().with_epoch(epoch.clone()); + let rendition = |broadcast: &mut moq_net::broadcast::Producer, + catalog: &mut crate::catalog::Producer, + name: &str| { + let track = aac_rendition(broadcast, catalog, name); + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(name.to_string(), tscat::Track::new(0x101)); + track + }; + + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + let mut track = rendition(&mut broadcast, &mut catalog, "0.aac"); + let source = crate::source::resolved(&origin, "live").await; + let mut export = open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap() + .with_delay(Duration::from_secs(1)) + .with_replay(); + let start = tokio::time::Instant::now(); + for ms in (0..200).step_by(20) { + write_aac(&mut track, ms); + } + drain_frames(&mut export).await; + drop((broadcast, catalog, track)); + let _ = drain_to_end(&mut export).await; + let (before, _) = export.liveness(0x101); + + tokio::time::sleep(Duration::from_secs(10)).await; + let (mut broadcast, mut catalog) = publish_live::(&origin, route()); + let back = origin.consume().routed_broadcast("live").await.unwrap(); + let stream = crate::catalog::Consumer::new(&back, Default::default()).await.unwrap(); + let mut export = export.follow(back, stream).unwrap(); + drain_frames(&mut export).await; + let mut track = rendition(&mut broadcast, &mut catalog, "1.aac"); + let resumed = start.elapsed().as_millis() as u64; + let mut quiet = Vec::new(); + for ms in (resumed..resumed + 4000).step_by(20) { + write_aac(&mut track, ms); + poll_frames(&mut export); + let (units, since) = export.liveness(0x101); + if export.listed_on(0x101) == Some("1.aac") && units == before { + quiet.push(since); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert!(!quiet.is_empty(), "the track was listed with none of its units out"); + assert!( + quiet + .iter() + .all(|since| since.is_some_and(|since| since < Duration::from_secs(1))), + "{quiet:?}" + ); + drop((broadcast, catalog, track)); +} + +/// A PMT revision no slot laid before a switch reached no receiver: the replacement's PMT +/// follows the last one out by one version. +#[tokio::test(start_paused = true)] +async fn a_switch_follows_the_last_pmt_out() { + let origin = crate::source::produce_origin(); + let route = || moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); + + let (mut old, mut old_catalog) = publish_live::(&origin, route()); + let mut audio = aac_rendition(&mut old, &mut old_catalog, "a.aac"); + let source = crate::source::resolved(&origin, "live").await; + let mut export = open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap() + .with_delay(RECORDING_MAX_AGE) + .with_replay(); + for ms in (0..200).step_by(20) { + write_aac(&mut audio, ms); + } + let mut frames = drain_frames(&mut export).await; + old_catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![tscat::Descriptor { + tag: 0x80, + data: Bytes::from_static(&[0xaa; 4]), + }]; + frames.extend(drain_frames(&mut export).await); + + let (mut new, mut new_catalog) = publish_live::(&origin, route()); + let mut audio = aac_rendition(&mut new, &mut new_catalog, "b.aac"); + let replacement = origin.consume().request_broadcast("live", None).await.unwrap(); + let stream = crate::catalog::Consumer::new(&replacement, Default::default()) + .await + .unwrap(); + let mut export = export.follow(replacement, stream).unwrap(); + for ms in (0..200).step_by(20) { + write_aac(&mut audio, ms); + } + frames.extend(drain_frames(&mut export).await); + audio.finish().unwrap(); + new_catalog.finish().unwrap(); + let (rest, end) = drain_to_end(&mut export).await; + frames.extend(rest); + end.unwrap(); + + let mut versions: Vec = pmts(&frames).iter().map(|(version, _)| *version).collect(); + versions.dedup(); + assert_eq!(versions, [0, 1]); + drop((old, old_catalog)); +} + +/// A switch onto an instance replaced after it built its tables but before a slot laid them +/// numbers the next program's from the last PMT out, not from the one that never went. +#[tokio::test(start_paused = true)] +async fn a_switch_past_an_instance_that_laid_no_tables_follows_the_last_pmt_out() { + let origin = crate::source::produce_origin(); + let route = || moq_net::origin::Route::default().with_epoch(moq_net::Epoch::mint()); + let follow = async |export: Export<_>| { + let replacement = origin.consume().request_broadcast("live", None).await.unwrap(); + let stream = crate::catalog::Consumer::new(&replacement, Default::default()) + .await + .unwrap(); + export.follow(replacement, stream).unwrap() + }; + + let (mut first, mut first_catalog) = publish_live::(&origin, route()); + let mut audio = aac_rendition(&mut first, &mut first_catalog, "a.aac"); + let source = crate::source::resolved(&origin, "live").await; + let mut export = open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap() + .with_delay(RECORDING_MAX_AGE) + .with_replay(); + for ms in (0..200).step_by(20) { + write_aac(&mut audio, ms); + } + let mut frames = drain_frames(&mut export).await; + + // Built from what it holds, and replaced before its first slot. + let (mut second, mut second_catalog) = publish_live::(&origin, route()); + let mut audio = aac_rendition(&mut second, &mut second_catalog, "b.aac"); + for ms in (0..200).step_by(20) { + write_aac(&mut audio, ms); + } + let mut export = follow(export).await; + assert!(poll_frames(&mut export).is_empty()); + + let (mut third, mut third_catalog) = publish_live::(&origin, route()); + let mut audio = aac_rendition(&mut third, &mut third_catalog, "c.aac"); + let mut export = follow(export).await; + for ms in (0..200).step_by(20) { + write_aac(&mut audio, ms); + } + frames.extend(drain_frames(&mut export).await); + audio.finish().unwrap(); + third_catalog.finish().unwrap(); + let (rest, end) = drain_to_end(&mut export).await; + frames.extend(rest); + end.unwrap(); + + let mut versions: Vec = pmts(&frames).iter().map(|(version, _)| *version).collect(); + versions.dedup(); + assert_eq!(versions, [0, 1]); + drop((first, first_catalog, second, second_catalog)); +} diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index 94a424abac5..7d9786ec916 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -71,3 +71,5 @@ mod pes_packet_test; mod recovery_test; #[cfg(test)] mod release_test; +#[cfg(test)] +mod revision_test; diff --git a/rs/moq-mux/src/container/ts/revision_test.rs b/rs/moq-mux/src/container/ts/revision_test.rs new file mode 100644 index 00000000000..39fc48f0e71 --- /dev/null +++ b/rs/moq-mux/src/container/ts/revision_test.rs @@ -0,0 +1,1677 @@ +//! A program whose PMT revises mid-stream, imported and exported again: the export announces +//! each revision with a PMT at the next `version_number`, its PAT unchanged. + +use std::time::Duration; + +use super::catalog::Ext; +use super::fixture::{Flags, MS, Mux}; +use crate::container::Frame; + +const PMT_PID: u16 = 0x20; +/// The PMT PID the export gives a program that recorded none. +const OWN_PMT_PID: u16 = 0x1000; +const CLOCK_PID: u16 = 0x1ff; +const AUDIO_PID: u16 = 0x101; +const DATA_PID: u16 = 0x102; +const OTHER_AUDIO_PID: u16 = 0x103; + +const AUDIO: u8 = 0x0f; +const PRIVATE: u8 = 0x06; +const METADATA: u8 = 0x15; + +/// ISO 639 language descriptors for English and French. +const ENG: &[u8] = &[0x0a, 0x04, b'e', b'n', b'g', 0x00]; +const FRA: &[u8] = &[0x0a, 0x04, b'f', b'r', b'a', 0x00]; +/// `metadata_descriptor` for asynchronous KLV (ISO 13818-1 2.6.60). +const KLV: &[u8] = &[0x26, 0x09, 0x01, 0x00, 0xff, b'K', b'L', b'V', b'A', 0x0f, 0x00]; + +/// How long the export holds each frame. +const DELAY: Duration = Duration::from_secs(2); + +/// A PMT the export wrote: its program number, `version_number`, PCR PID, program descriptors, +/// and each stream as `(stream_type, PID, descriptor bytes)`. +#[derive(Debug, PartialEq)] +struct Table { + program: u16, + version: u8, + pcr_pid: u16, + info: Vec, + streams: Vec<(u8, u16, Vec)>, +} + +/// One span of the source: the streams its PMT lists. +struct Span<'a> { + streams: &'a [(u8, u16, &'a [u8])], +} + +/// Import `spans` one by one, each opened by a PAT and a PMT at the next version and running +/// for `intervals` PCRs 40 ms apart (the first for three seconds), with an AAC frame on each +/// audio stream and a PES on each private one every interval. The export polls after each, +/// as a live one keeps up with the catalog, and runs to its end after the last. Returns +/// every PMT it wrote, each run of repeats once. +async fn reexport(spans: &[Span<'_>], intervals: u64) -> Vec { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + + let mut ticks = 1_000 * MS; + let mut export = None; + let mut frames = Vec::new(); + for (version, span) in spans.iter().enumerate() { + let mut mux = Mux::default(); + mux.pat(&[(1, PMT_PID)]) + .pmt_at(version as u8 % 32, PMT_PID, 1, CLOCK_PID, span.streams, 0); + // The import publishes its catalog once it has measured the multiplex rate. + let intervals = if version == 0 { 75 } else { intervals }; + for _ in 0..intervals { + mux.packet(CLOCK_PID, Flags::pcr(ticks)); + for &(stream_type, pid, _) in span.streams { + let payload = match stream_type { + AUDIO => aac(ticks / 300 + 9_000, pid as u8), + PRIVATE => pes(0xbd, pid as u8), + _ => pes(0xfc, pid as u8), + }; + mux.whole(pid, &payload); + } + ticks += 40 * MS; + } + import.decode(&mux.bytes()).unwrap(); + let export = match &mut export { + Some(export) => export, + None => { + let source = crate::source::announced(&consumer).await; + let opened = super::export_test::open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap(); + let export = export.insert(opened.with_delay(DELAY).with_replay()); + // The first frame out carries the tables built from this span alone. + let first = tokio::time::timeout(DELAY * 2, export.next()).await; + frames.push(first.expect("a first frame").unwrap().unwrap()); + export + } + }; + settle(export, &mut frames).await; + } + let mut mux = Mux::default(); + mux.packet(CLOCK_PID, Flags::pcr(ticks)); + import.decode(&mux.bytes()).unwrap(); + import.finish().unwrap(); + catalog.finish().unwrap(); + + drain(&mut export.unwrap(), &mut frames).await; + tables(&frames, PMT_PID) +} + +/// Run `export` until it waits on its source, as a live one keeps up with each catalog +/// snapshot as it comes. +async fn settle(export: &mut super::Export, frames: &mut Vec) +where + S::Ext: super::catalog::Catalog, +{ + while let Ok(frame) = tokio::time::timeout(Duration::from_millis(1), export.next()).await { + frames.push(frame.unwrap().expect("the export runs on")); + } +} + +/// Run `export` to its end. +async fn drain(export: &mut super::Export, frames: &mut Vec) +where + S::Ext: super::catalog::Catalog, +{ + loop { + match tokio::time::timeout(DELAY * 10, export.next()) + .await + .expect("the export ends") + { + Ok(Some(frame)) => frames.push(frame), + Ok(None) => break, + Err(err) => panic!("the export failed: {err}"), + } + } +} + +/// An ADTS AAC frame presenting at `pts`, its payload filled with `fill`. +fn aac(pts: u64, fill: u8) -> Vec { + let mut frame = super::adts::write_header(2, 48_000, 2, 64).unwrap().to_vec(); + frame.resize(7 + 64, fill); + let field = [ + 0x21 | ((pts >> 29) & 0x0e) as u8, + (pts >> 22) as u8, + 1 | ((pts >> 14) & 0xfe) as u8, + (pts >> 7) as u8, + 1 | ((pts << 1) & 0xfe) as u8, + ]; + let mut pes = vec![0, 0, 1, 0xc0]; + pes.extend_from_slice(&((3 + field.len() + frame.len()) as u16).to_be_bytes()); + pes.extend_from_slice(&[0x80, 0x80, field.len() as u8]); + pes.extend_from_slice(&field); + pes.extend_from_slice(&frame); + pes +} + +/// A PES of `stream_id` without a PTS, its payload filled with `fill`. +fn pes(stream_id: u8, fill: u8) -> Vec { + let payload = [fill; 24]; + let mut pes = vec![0, 0, 1, stream_id]; + pes.extend_from_slice(&((3 + payload.len()) as u16).to_be_bytes()); + pes.extend_from_slice(&[0x84, 0x00, 0x00]); + pes.extend_from_slice(&payload); + pes +} + +/// Every packet in `frames`. +fn packets(frames: &[Frame]) -> impl Iterator { + frames + .iter() + .flat_map(|frame| frame.payload.as_chunks::<188>().0.iter()) +} + +/// The PID of `packet`. +fn pid(packet: &[u8; 188]) -> u16 { + u16::from(packet[1] & 0x1f) << 8 | u16::from(packet[2]) +} + +/// Every PMT on `pmt_pid` in `frames`, each run of repeats once. +fn tables(frames: &[Frame], pmt_pid: u16) -> Vec
{ + let mut out: Vec
= Vec::new(); + for (_, table) in tables_at(frames, pmt_pid) { + if out.last() != Some(&table) { + out.push(table); + } + } + out +} + +/// Every PMT copy on `pmt_pid` in `frames`, with the index of the packet it ends in. +fn tables_at(frames: &[Frame], pmt_pid: u16) -> Vec<(usize, Table)> { + let mut out = Vec::new(); + let mut section = Vec::new(); + for (index, packet) in packets(frames).enumerate() { + if pid(packet) != pmt_pid { + continue; + } + let mut at = 4; + if packet[3] & 0x20 != 0 { + at += 1 + usize::from(packet[4]); + } + if packet[1] & 0x40 != 0 { + section = packet[at + 1 + usize::from(packet[at])..].to_vec(); + } else if !section.is_empty() { + section.extend_from_slice(&packet[at..]); + } else { + continue; + } + let len = 3 + (usize::from(section[1] & 0x0f) << 8 | usize::from(section[2])); + if section.len() < len { + continue; + } + let whole = std::mem::take(&mut section); + let section = &whole[..len]; + let pmt = super::psi::Pmt::parse(section).expect("a whole PMT"); + let bytes = |descriptors: &[super::catalog::Descriptor]| { + descriptors + .iter() + .flat_map(|d| [vec![d.tag, d.data.len() as u8], d.data.to_vec()].concat()) + .collect::>() + }; + let table = Table { + program: pmt.program_number, + version: (section[5] >> 1) & 0x1f, + pcr_pid: pmt.pcr_pid.map_or(0x1fff, |pid| pid.as_u16()), + info: bytes(&pmt.program_info), + streams: pmt + .streams + .iter() + .map(|es| (es.stream_type, es.pid.as_u16(), bytes(&es.descriptors))) + .collect(), + }; + out.push((index, table)); + } + out +} + +/// Each table's version and streams as `(stream_type, PID)`. +fn layouts(tables: &[Table]) -> Vec<(u8, Vec<(u8, u16)>)> { + tables + .iter() + .map(|table| { + let streams = table + .streams + .iter() + .map(|&(stream_type, pid, _)| (stream_type, pid)) + .collect(); + (table.version, streams) + }) + .collect() +} + +#[tokio::test(start_paused = true)] +async fn a_stream_added_mid_stream_revises_the_pmt() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])], + }, + ], + 25, + ) + .await; + assert_eq!( + layouts(&tables), + vec![ + (0, vec![(AUDIO, AUDIO_PID)]), + (1, vec![(AUDIO, AUDIO_PID), (PRIVATE, DATA_PID)]), + ] + ); + assert!( + tables.iter().all(|table| table.pcr_pid == AUDIO_PID), + "the clock stays put" + ); +} + +#[tokio::test(start_paused = true)] +async fn a_stream_removed_mid_stream_leaves_the_pmt_once_it_ends() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[])], + }, + ], + 25, + ) + .await; + assert_eq!( + layouts(&tables), + vec![ + (0, vec![(AUDIO, AUDIO_PID), (PRIVATE, DATA_PID)]), + (1, vec![(AUDIO, AUDIO_PID)]), + ] + ); +} + +/// A changed `stream_type` keeps the PID, and the PMT names the new type at the next version. +#[tokio::test(start_paused = true)] +async fn a_retyped_stream_revises_its_entry_in_place() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (METADATA, DATA_PID, KLV)], + }, + ], + 25, + ) + .await; + assert_eq!( + layouts(&tables), + vec![ + (0, vec![(AUDIO, AUDIO_PID), (PRIVATE, DATA_PID)]), + (1, vec![(AUDIO, AUDIO_PID), (METADATA, DATA_PID)]), + ] + ); + assert_eq!(tables[1].streams[1].2, KLV); +} + +/// An audio language retag reopens the rendition, which takes the PID and the clock over. +#[tokio::test(start_paused = true)] +async fn a_descriptor_change_revises_the_entry_and_keeps_the_clock() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, ENG)], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, FRA)], + }, + ], + 25, + ) + .await; + assert_eq!( + tables, + vec![ + Table { + program: 1, + version: 0, + pcr_pid: AUDIO_PID, + info: Vec::new(), + streams: vec![(AUDIO, AUDIO_PID, ENG.to_vec())], + }, + Table { + program: 1, + version: 1, + pcr_pid: AUDIO_PID, + info: Vec::new(), + streams: vec![(AUDIO, AUDIO_PID, FRA.to_vec())], + }, + ] + ); +} + +/// A listed track keeps the descriptors its packets went out under: a catalog revising them +/// under the same name revises nothing, as a revised stream comes under a new name. +#[tokio::test(start_paused = true)] +async fn a_listed_track_keeps_its_descriptors() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut audio) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let language = |language: &[u8]| super::catalog::Descriptor { + tag: language[0], + data: bytes::Bytes::copy_from_slice(&language[2..]), + }; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .get_mut(&name) + .unwrap() + .descriptors = vec![language(ENG)]; + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 0, 3_000).await; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .get_mut(&name) + .unwrap() + .descriptors = vec![language(FRA)]; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 3_000, 6_000).await; + audio.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let tables = tables(&frames, OWN_PMT_PID); + let streams: Vec<_> = tables.iter().map(|table| table.streams.clone()).collect(); + assert_eq!(streams, [vec![(AUDIO, AUDIO_PID, ENG.to_vec())]]); +} + +/// The PCR leaves a PID only with the track it rode on, for the next audio track. +#[tokio::test(start_paused = true)] +async fn the_clock_moves_only_when_its_track_retires() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, ENG), (AUDIO, OTHER_AUDIO_PID, FRA)], + }, + Span { + streams: &[(AUDIO, OTHER_AUDIO_PID, FRA)], + }, + ], + 25, + ) + .await; + let clocks: Vec<(u8, u16)> = tables.iter().map(|table| (table.version, table.pcr_pid)).collect(); + assert_eq!(clocks, vec![(0, AUDIO_PID), (1, OTHER_AUDIO_PID)]); + assert_eq!(tables[1].streams, vec![(AUDIO, OTHER_AUDIO_PID, FRA.to_vec())]); +} + +/// The PCR leaves the PID of a track that retires only behind a PMT naming the PID it moves +/// to: a receiver holding the PMT before finds the clock on the PID that one names. +#[tokio::test(start_paused = true)] +async fn the_clock_moves_behind_the_pmt_naming_its_pid() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut clock) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (_, mut other) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut clock, 0x21), (&mut other, 0x21)], + 0, + 3_000, + ) + .await; + remove(&mut catalog, &name); + clock.finish().unwrap(); + live(&mut export, &mut frames, start, &mut [(&mut other, 0x21)], 3_000, 6_000).await; + other.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let mut named = None; + let mut clocks = Vec::new(); + for frame in &frames { + for packet in packets(std::slice::from_ref(frame)) { + if packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x10 != 0 { + if let Some(named) = named { + assert_eq!(pid(packet), named, "a PCR on a PID the PMT out does not name"); + } + clocks.push(pid(packet)); + } + } + if let Some(table) = tables(std::slice::from_ref(frame), OWN_PMT_PID).last() { + named = Some(table.pcr_pid); + } + } + clocks.dedup(); + assert_eq!(clocks, [AUDIO_PID, OTHER_AUDIO_PID]); +} + +/// A PID the PMT dropped can come back as another stream. +#[tokio::test(start_paused = true)] +async fn a_pid_removed_can_return_as_another_stream() { + let tables = reexport( + &[ + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[])], + }, + Span { + streams: &[(AUDIO, AUDIO_PID, &[]), (METADATA, DATA_PID, KLV)], + }, + ], + 25, + ) + .await; + assert_eq!( + layouts(&tables), + vec![ + (0, vec![(AUDIO, AUDIO_PID), (PRIVATE, DATA_PID)]), + (1, vec![(AUDIO, AUDIO_PID)]), + (2, vec![(AUDIO, AUDIO_PID), (METADATA, DATA_PID)]), + ] + ); +} + +/// A changed `program_info` is a revision of its own. +#[tokio::test(start_paused = true)] +async fn a_program_descriptor_change_revises_the_pmt() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut audio) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 0, 3_000).await; + catalog.modify().unwrap().ext.mpegts.program_descriptors = vec![super::catalog::Descriptor { + tag: 0x80, + data: bytes::Bytes::from_static(&[0xaa; 6]), + }]; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 3_000, 6_000).await; + audio.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let versions: Vec<(u8, Vec)> = tables(&frames, OWN_PMT_PID) + .into_iter() + .map(|table| (table.version, table.info)) + .collect(); + assert_eq!( + versions, + vec![(0, Vec::new()), (1, vec![0x80, 6, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa])] + ); +} + +/// Layouts that change faster than the export lags behind its source still reach the wire +/// as a run of versions, each one up from the last and differing from it: a table copied into +/// a frame queued behind a revision never goes out after it, and a revision overtaken before +/// any frame carried it takes no version. +#[tokio::test(start_paused = true)] +async fn each_version_out_follows_the_last_by_one_change() { + let with: &[(u8, u16, &[u8])] = &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])]; + let without: &[(u8, u16, &[u8])] = &[(AUDIO, AUDIO_PID, &[])]; + let spans: Vec = (0..12) + .map(|i| Span { + streams: if i % 2 == 0 { with } else { without }, + }) + .collect(); + let tables = reexport(&spans, 10).await; + assert!(tables.len() > 2, "{tables:?}"); + for pair in tables.windows(2) { + assert_eq!(pair[1].version, (pair[0].version + 1) % 32, "{tables:?}"); + assert_ne!(pair[1].streams, pair[0].streams, "{tables:?}"); + } +} + +/// The `version_number` runs on from 31 to 0. +#[tokio::test(start_paused = true)] +async fn the_version_wraps_after_31() { + let with: &[(u8, u16, &[u8])] = &[(AUDIO, AUDIO_PID, &[]), (PRIVATE, DATA_PID, &[])]; + let without: &[(u8, u16, &[u8])] = &[(AUDIO, AUDIO_PID, &[])]; + let spans: Vec = (0..34) + .map(|i| Span { + streams: if i % 2 == 0 { with } else { without }, + }) + .collect(); + // Each span outlasts the export's lag behind its source: a PID dropped for less is + // listed again before the table dropping it goes out. + let tables = reexport(&spans, 40).await; + let versions: Vec = tables.iter().map(|table| table.version).collect(); + let expected: Vec = (0..34).map(|i| i % 32).collect(); + assert_eq!(versions, expected); + for (i, table) in tables.iter().enumerate() { + assert_eq!(table.streams.len(), if i % 2 == 0 { 2 } else { 1 }, "revision {i}"); + } +} + +/// A track's writer, its frames in the legacy container. +type Writer = crate::container::Producer; + +/// Open an AAC track on `broadcast` and list it in `catalog`, with `pid` recorded if given. +fn aac_track( + broadcast: &mut moq_net::broadcast::Producer, + catalog: &mut crate::catalog::Producer, + pid: Option, +) -> (String, Writer) { + let track = broadcast + .create_track( + broadcast.unique_name(".aac"), + hang::container::track_info(hang::catalog::PRIORITY.audio), + ) + .unwrap(); + let name = track.name().to_string(); + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AAC { profile: 2 }, 48_000, 2); + config.container = hang::catalog::Container::Legacy; + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions.insert(name.clone(), config); + if let Some(pid) = pid { + guard + .ext + .mpegts + .tracks + .insert(name.clone(), super::catalog::Track::new(pid)); + } + drop(guard); + let container = crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio); + (name, Writer::new(track, container)) +} + +/// Write an AAC frame at `millis` to each of `tracks`. +fn write_aac(tracks: &mut [&mut Writer], millis: u64) { + for track in tracks { + write_marked(track, millis, 0x21); + } +} + +/// Write an AAC frame at `millis` to `track`, its payload filled with `mark`. +fn write_marked(track: &mut Writer, millis: u64, mark: u8) { + let mut payload = vec![mark; 16]; + payload[0] = 0x21; + track + .write(Frame { + timestamp: moq_net::Timestamp::from_micros(millis * 1_000).unwrap(), + duration: None, + payload: bytes::Bytes::from(payload), + keyframe: true, + }) + .unwrap(); + track.cut(None).unwrap(); +} + +/// The marks of the frames ([`write_marked`]) that went out on `pid`, in order. +fn marks(frames: &[Frame], pid: u16, marks: &[u8]) -> Vec { + let mut payload = Vec::new(); + for packet in packets(frames).filter(|packet| self::pid(packet) == pid && packet[3] & 0x10 != 0) { + let at = if packet[3] & 0x20 != 0 { + 5 + usize::from(packet[4]) + } else { + 4 + }; + payload.extend_from_slice(&packet[at..]); + } + let mut out = Vec::new(); + for (at, run) in payload.windows(15).enumerate() { + let mark = run[0]; + if marks.contains(&mark) && run.iter().all(|&byte| byte == mark) && (at == 0 || payload[at - 1] != mark) { + out.push(mark); + } + } + out +} + +/// The export of `consumer`. +async fn export_of(consumer: &moq_net::broadcast::Consumer) -> super::Export> { + let source = crate::source::announced(consumer).await; + let opened = super::export_test::open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap(); + opened.with_delay(DELAY).with_replay() +} + +/// Open the export of `consumer` once its first frame is out. +async fn open( + consumer: &moq_net::broadcast::Consumer, + frames: &mut Vec, +) -> super::Export> { + let mut export = export_of(consumer).await; + let out = tokio::time::timeout(DELAY * 2, export.next()).await; + frames.push(out.expect("the tables went out").unwrap().unwrap()); + settle(&mut export, frames).await; + export +} + +/// Take `name` out of the catalog. +fn remove(catalog: &mut crate::catalog::Producer, name: &str) { + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions.remove(name); + guard.ext.mpegts.tracks.remove(name); +} + +/// Under sync the tables stay as laid, so a track added after them fails the export. +#[tokio::test(start_paused = true)] +async fn a_track_added_under_sync_fails_the_export() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + for millis in (0..3_000).step_by(20) { + write_aac(&mut [&mut first], millis); + } + let source = crate::source::announced(&consumer).await; + let opened = super::export_test::open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap(); + let mut export = opened.with_delay(DELAY).with_mux_rate(2_000_000).with_sync(); + let out = tokio::time::timeout(DELAY * 2, export.next()).await; + out.expect("the tables went out").unwrap().unwrap(); + + let _second = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let err = tokio::time::timeout(DELAY * 2, async { + loop { + if let Err(err) = export.next().await { + break err; + } + } + }) + .await + .expect("the export reads the catalog update"); + assert!(err.to_string().contains("under --sync"), "{err}"); +} + +/// A track the catalog lists after the tables went out, but before its PID is recorded, +/// takes the frames it holds along to the recorded PID when that arrives. +#[tokio::test(start_paused = true)] +async fn a_pid_recorded_after_its_track_takes_the_held_frames_along() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + for millis in (0..3_000).step_by(20) { + write_aac(&mut [&mut first], millis); + } + let source = crate::source::announced(&consumer).await; + let opened = super::export_test::open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap(); + let mut export = opened.with_delay(DELAY).with_replay(); + let out = tokio::time::timeout(DELAY * 2, export.next()).await; + let mut frames = vec![out.expect("the tables went out").unwrap().unwrap()]; + settle(&mut export, &mut frames).await; + + let (name, mut second) = aac_track(&mut broadcast, &mut catalog, None); + settle(&mut export, &mut frames).await; + for millis in (3_000..3_500).step_by(20) { + write_aac(&mut [&mut first, &mut second], millis); + } + settle(&mut export, &mut frames).await; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(name, super::catalog::Track::new(OTHER_AUDIO_PID)); + settle(&mut export, &mut frames).await; + for millis in (3_500..6_000).step_by(20) { + write_aac(&mut [&mut first, &mut second], millis); + } + first.finish().unwrap(); + second.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let tables = tables(&frames, OWN_PMT_PID); + assert_eq!( + tables.last().expect("a PMT").streams, + vec![(AUDIO, AUDIO_PID, Vec::new()), (AUDIO, OTHER_AUDIO_PID, Vec::new())] + ); + let carried: std::collections::BTreeSet = packets(&frames) + .filter(|packet| packet[1] & 0x40 != 0 && packet[3] & 0x10 != 0) + .filter(|packet| { + let at = if packet[3] & 0x20 != 0 { + 5 + usize::from(packet[4]) + } else { + 4 + }; + packet[at..].starts_with(&[0, 0, 1]) + }) + .map(pid) + .collect(); + assert_eq!( + carried, + [AUDIO_PID, OTHER_AUDIO_PID].into(), + "no PES left on another PID" + ); +} + +/// The next frame `export` writes. +async fn next(export: &mut super::Export) -> Frame +where + S::Ext: super::catalog::Catalog, +{ + let out = tokio::time::timeout(DELAY * 4, export.next()).await; + out.expect("a frame").unwrap().expect("the export runs on") +} + +/// A track not listed yet, whose first frame is held behind the old generation's tail, takes +/// that frame along to the PID recorded meanwhile. +#[tokio::test(start_paused = true)] +async fn a_pid_recorded_while_its_frame_is_held_takes_the_frame_along() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + for millis in (0..3_000).step_by(20) { + write_aac(&mut [&mut first], millis); + } + let mut frames = Vec::new(); + let mut export = open(&consumer, &mut frames).await; + while export.buffered() > 0 { + frames.push(next(&mut export).await); + } + let (name, mut second) = aac_track(&mut broadcast, &mut catalog, None); + settle(&mut export, &mut frames).await; + write_aac(&mut [&mut second], 3_000); + for millis in (3_020..3_500).step_by(20) { + write_aac(&mut [&mut first, &mut second], millis); + } + settle(&mut export, &mut frames).await; + tokio::time::advance(DELAY * 2).await; + let held = export.hold_next().expect("a frame due"); + assert_ne!(held, AUDIO_PID, "the second track's frame is held"); + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(name, super::catalog::Track::new(OTHER_AUDIO_PID)); + first.finish().unwrap(); + second.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + assert_eq!( + tables(&frames, OWN_PMT_PID).last().expect("a PMT").streams, + vec![(AUDIO, AUDIO_PID, Vec::new()), (AUDIO, OTHER_AUDIO_PID, Vec::new())] + ); +} + +/// A track whose frame is held behind the old generation's tail stays until that frame goes, +/// even once it ends and leaves the catalog. +#[tokio::test(start_paused = true)] +async fn a_track_ending_while_its_frame_is_held_stays_for_the_frame() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + for millis in (0..3_000).step_by(20) { + write_aac(&mut [&mut first], millis); + } + let mut frames = Vec::new(); + let mut export = open(&consumer, &mut frames).await; + while export.buffered() > 0 { + frames.push(next(&mut export).await); + } + let (name, mut second) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + settle(&mut export, &mut frames).await; + write_marked(&mut second, 3_000, 0x31); + for millis in (3_020..3_500).step_by(20) { + write_aac(&mut [&mut first], millis); + } + settle(&mut export, &mut frames).await; + tokio::time::advance(DELAY * 2).await; + assert_eq!( + export.hold_next(), + Some(OTHER_AUDIO_PID), + "the second track's frame is held" + ); + second.finish().unwrap(); + remove(&mut catalog, &name); + first.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + assert_eq!( + marks(&frames, OTHER_AUDIO_PID, &[0x31]), + [0x31], + "the held frame went out" + ); +} + +/// A track taking a PID over goes out behind everything the track before it there sent, even a +/// tail sent further ahead of its decode time than the new track's first frame: none of the +/// old stream goes out under the PMT revised for the new one. +#[tokio::test(start_paused = true)] +async fn a_track_taking_a_pid_over_goes_out_behind_the_tail_before_it() { + const OLD: u8 = 0xa5; + const NEW: u8 = 0x5a; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut old) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (_, mut other) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut old, OLD), (&mut other, 0x21)], + 0, + 3_000, + ) + .await; + + // The old stream's tail goes out ahead of its time, running on past where the new one starts. + for millis in (3_000..3_800).step_by(20) { + write_marked(&mut old, millis, OLD); + } + old.finish().unwrap(); + remove(&mut catalog, &name); + settle(&mut export, &mut frames).await; + let (_, mut new) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + settle(&mut export, &mut frames).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut new, NEW), (&mut other, 0x21)], + 3_000, + 5_000, + ) + .await; + new.finish().unwrap(); + other.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let marks = marks(&frames, AUDIO_PID, &[OLD, NEW]); + let first = marks + .iter() + .position(|&mark| mark == NEW) + .expect("the new stream went out"); + assert!(marks[..first].contains(&OLD), "the old stream went out first"); + assert!( + marks[first..].iter().all(|&mark| mark == NEW), + "the old stream's tail went out after the new one began: {marks:x?}" + ); +} + +/// A PID the PMT stops listing is gone rather than quiet, so a stream returning on it counts +/// its silence from its own listing, not from the last unit of the stream before it. +#[tokio::test(start_paused = true)] +async fn a_pid_that_leaves_the_pmt_is_gone_until_it_returns() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut old) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (_, mut other) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut old, 0x21), (&mut other, 0x21)], + 0, + 2_000, + ) + .await; + old.finish().unwrap(); + remove(&mut catalog, &name); + live(&mut export, &mut frames, start, &mut [(&mut other, 0x21)], 2_000, 5_000).await; + assert!(!export.stats().streams.contains_key(&AUDIO_PID), "the PID left the PMT"); + assert_eq!( + export.liveness(AUDIO_PID).1, + None, + "a PID the PMT left is gone, not quiet" + ); + + let (_, mut new) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + settle(&mut export, &mut frames).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut new, 0x21), (&mut other, 0x21)], + 5_000, + 7_500, + ) + .await; + let (_, quiet) = export.liveness(AUDIO_PID); + assert!( + quiet.is_some_and(|quiet| quiet < Duration::from_secs(1)), + "the returned stream is quiet for {quiet:?}" + ); +} + +/// A track whose first frame is held behind a generation boundary, then moved onto a PID taken +/// over, goes out behind the tail there all the same: the held frame drops. +#[tokio::test(start_paused = true)] +async fn a_held_frame_moved_onto_a_pid_taken_over_waits_for_the_tail() { + const OLD: u8 = 0xa5; + const NEW: u8 = 0x5a; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (old_name, mut old) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + for millis in (0..3_000).step_by(20) { + write_marked(&mut old, millis, OLD); + } + let mut frames = Vec::new(); + let mut export = open(&consumer, &mut frames).await; + while export.buffered() > 0 { + frames.push(next(&mut export).await); + } + let (name, mut new) = aac_track(&mut broadcast, &mut catalog, None); + settle(&mut export, &mut frames).await; + write_marked(&mut new, 3_000, NEW); + for millis in (3_020..3_500).step_by(20) { + write_marked(&mut old, millis, OLD); + write_marked(&mut new, millis, NEW); + } + settle(&mut export, &mut frames).await; + tokio::time::advance(DELAY * 2).await; + let held = export.hold_next().expect("a frame due"); + assert_ne!(held, AUDIO_PID, "the new track's frame is held"); + { + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions.remove(&old_name); + guard.ext.mpegts.tracks.remove(&old_name); + guard + .ext + .mpegts + .tracks + .insert(name, super::catalog::Track::new(AUDIO_PID)); + } + for millis in (3_500..4_000).step_by(20) { + write_marked(&mut old, millis, OLD); + write_marked(&mut new, millis, NEW); + } + old.finish().unwrap(); + new.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let marks = marks(&frames, AUDIO_PID, &[OLD, NEW]); + let first = marks + .iter() + .position(|&mark| mark == NEW) + .expect("the new stream went out"); + assert!( + marks[first..].iter().all(|&mark| mark == NEW), + "the old stream's tail went out after the new one began: {marks:x?}" + ); +} +/// A track recorded on the PID of a stream the catalog dropped that never ends cuts that stream: +/// what it already sent drains, then each track recorded there follows in turn. +#[tokio::test(start_paused = true)] +async fn a_pid_taken_from_a_stream_that_never_ends_cuts_it() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (name, mut old) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut old, 0x21)], 0, 2_000).await; + remove(&mut catalog, &name); + for round in 0..4 { + let (name, mut successor) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let from = 2_000 + u64::from(round) * 500; + live( + &mut export, + &mut frames, + start, + &mut [(&mut old, 0x21), (&mut successor, 0x31 + round)], + from, + from + 500, + ) + .await; + remove(&mut catalog, &name); + successor.finish().unwrap(); + } + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + let mut marks = marks(&frames, AUDIO_PID, &[0x21, 0x31, 0x32, 0x33, 0x34]); + marks.dedup(); + assert_eq!(marks, [0x21, 0x31, 0x32, 0x33, 0x34]); + drop(old); +} + +/// The PMT stays on the PID and program its tables went out with, whatever a later catalog +/// records, so a track a later catalog records on that PID takes another, and each revision is +/// still the program the PAT names. +#[tokio::test(start_paused = true)] +async fn a_track_recorded_on_the_pmt_pid_after_the_catalog_moves_it_takes_another() { + let program = |pmt_pid: u16, program_number: u16| super::catalog::Program { + transport_stream_id: 1, + program_number, + pmt_pid, + }; + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + catalog.modify().unwrap().ext.mpegts.program = Some(program(PMT_PID, 1)); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 0, 3_000).await; + catalog.modify().unwrap().ext.mpegts.program = Some(program(OWN_PMT_PID, 7)); + let (_, mut added) = aac_track(&mut broadcast, &mut catalog, Some(PMT_PID)); + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21), (&mut added, 0x21)], + 3_000, + 6_000, + ) + .await; + first.finish().unwrap(); + added.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + assert!(tables(&frames, OWN_PMT_PID).is_empty(), "the PMT stays on its PID"); + let tables = tables(&frames, PMT_PID); + let pids: Vec = tables.last().unwrap().streams.iter().map(|es| es.1).collect(); + assert_eq!(pids.len(), 2, "{tables:?}"); + assert!(!pids.contains(&PMT_PID), "{tables:?}"); + assert!(tables.iter().all(|table| table.program == 1), "{tables:?}"); +} + +/// An SCTE-35 `splice_null` section. +const CUE: &[u8] = &[ + 0xfc, 0x30, 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xf0, 0x0a, 0x05, 0x00, 0x00, 0x2b, 0xb4, 0x7f, + 0xdf, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xad, 0x25, 0xe8, 0x39, +]; + +/// Tables go out with the frame they were muxed with, and a placed frame waits for its +/// placement, so a revision muxed after another can go out first. The copies still queued +/// then carry the later table, under the version the earlier one took: the wire still +/// steps by one. +#[tokio::test(start_paused = true)] +async fn a_revision_out_before_the_one_it_follows_takes_its_version() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 0, 3_000).await; + + // A cue and an AAC track start together, the cue muxed first, on the lower PID: its + // revision waits with it for its placement, the AAC track's goes out now. + let track = broadcast + .unique_track(".scte35", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let entry = super::catalog::Track { + pid: DATA_PID, + descriptors: Vec::new(), + verbatim: Some(super::catalog::Verbatim::new(0x86, super::catalog::Framing::Section)), + pes_packet: None, + }; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(track.name().to_string(), entry); + let mut cue = Writer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + let (_, mut added) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + settle(&mut export, &mut frames).await; + tokio::time::sleep_until(start + Duration::from_millis(3_000)).await; + cue.write(Frame { + timestamp: moq_net::Timestamp::from_millis(3_000).unwrap(), + duration: None, + payload: bytes::Bytes::from_static(CUE), + keyframe: true, + }) + .unwrap(); + cue.cut(None).unwrap(); + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21), (&mut added, 0x21)], + 3_000, + 6_000, + ) + .await; + for track in [&mut first, &mut added, &mut cue] { + track.finish().unwrap(); + } + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let tables = tables(&frames, OWN_PMT_PID); + for pair in tables.windows(2) { + assert_eq!(pair[1].version, (pair[0].version + 1) % 32, "{tables:?}"); + } + let pids: Vec = tables.last().unwrap().streams.iter().map(|es| es.1).collect(); + assert_eq!(pids, [AUDIO_PID, DATA_PID, OTHER_AUDIO_PID], "{tables:?}"); +} + +/// A stream taking a PID over from a buffered one goes out on its own model, not the one +/// before it: a private section larger than the AAC buffer before it still goes out. +#[tokio::test(start_paused = true)] +async fn a_stream_taking_a_pid_over_leaves_the_buffer_before_it() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (name, mut before) = aac_track(&mut broadcast, &mut catalog, Some(DATA_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21), (&mut before, 0x21)], + 0, + 3_000, + ) + .await; + remove(&mut catalog, &name); + before.finish().unwrap(); + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 3_000, 6_000).await; + + let track = broadcast + .unique_track(".private", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let entry = super::catalog::Track { + pid: DATA_PID, + descriptors: Vec::new(), + verbatim: Some(super::catalog::Verbatim::new(0x05, super::catalog::Framing::Section)), + pes_packet: None, + }; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .insert(track.name().to_string(), entry); + let mut private = Writer::new( + track, + crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), + ); + settle(&mut export, &mut frames).await; + tokio::time::sleep_until(start + Duration::from_millis(6_000)).await; + let mut section = vec![0x80, 0x7f, 0xfd]; + section.resize(4_096, 0xab); + private + .write(Frame { + timestamp: moq_net::Timestamp::from_millis(6_000).unwrap(), + duration: None, + payload: bytes::Bytes::from(section), + keyframe: true, + }) + .unwrap(); + private.cut(None).unwrap(); + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 6_000, 9_000).await; + for track in [&mut first, &mut private] { + track.finish().unwrap(); + } + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let layout = layouts(&tables(&frames, OWN_PMT_PID)); + assert_eq!(layout.last().unwrap().1, [(AUDIO, AUDIO_PID), (0x05, DATA_PID)]); + let sent = packets(&frames).filter(|packet| pid(packet) == DATA_PID && packet[4..].iter().all(|&b| b == 0xab)); + assert_eq!(sent.count(), 4_096 / 184 - 1, "the section went out whole"); +} +/// A stream added while one that left drains, the two too many for a section beside the rest, +/// fails the export: a PMT that cannot list its program is refused, not cut short. +#[tokio::test(start_paused = true)] +async fn a_stream_too_many_for_the_pmt_section_fails_the_export() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + // A stream with 255 bytes of descriptors: three fit a section, four do not. + fn track( + broadcast: &mut moq_net::broadcast::Producer, + catalog: &mut crate::catalog::Producer, + pid: u16, + ) -> (String, Writer) { + let (name, writer) = aac_track(broadcast, catalog, Some(pid)); + let descriptor = super::catalog::Descriptor { + tag: 0x80, + data: bytes::Bytes::from(vec![pid as u8; 250]), + }; + catalog + .modify() + .unwrap() + .ext + .mpegts + .tracks + .get_mut(&name) + .unwrap() + .descriptors = vec![descriptor]; + (name, writer) + } + let (_, mut first) = track(&mut broadcast, &mut catalog, AUDIO_PID); + let (_, mut second) = track(&mut broadcast, &mut catalog, OTHER_AUDIO_PID); + let (name, mut leaving) = track(&mut broadcast, &mut catalog, 0x105); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + let mut tracks = [(&mut first, 0x21), (&mut second, 0x21), (&mut leaving, 0x21)]; + live(&mut export, &mut frames, start, &mut tracks, 0, 3_000).await; + remove(&mut catalog, &name); + let (_, mut added) = track(&mut broadcast, &mut catalog, 0x106); + let mut tracks = [ + (&mut first, 0x21), + (&mut second, 0x21), + (&mut leaving, 0x21), + (&mut added, 0x21), + ]; + let err = tokio::time::timeout(DELAY * 2, async { + loop { + for (track, mark) in tracks.iter_mut() { + write_marked(track, start.elapsed().as_millis() as u64, *mark); + } + tokio::time::sleep(Duration::from_millis(20)).await; + while let Ok(next) = tokio::time::timeout(Duration::from_millis(1), export.next()).await { + if let Err(err) = next { + return err; + } + } + } + }) + .await + .expect("the export fails"); + assert!(err.to_string().contains("exceeds 1021 bytes"), "{err}"); +} + +/// Tracks the catalog takes back before sending anything, their sources left open, go with +/// it, each on a PID of its own as they are. +#[tokio::test(start_paused = true)] +async fn tracks_withdrawn_before_sending_anything_go() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 0, 3_000).await; + let mut open = Vec::new(); + for at in (3_000..8_000).step_by(1_000) { + let (name, withdrawn) = aac_track(&mut broadcast, &mut catalog, None); + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], at, at + 500).await; + remove(&mut catalog, &name); + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21)], + at + 500, + at + 1_000, + ) + .await; + open.push(withdrawn); + } + assert_eq!(export.tracks(), 1); + first.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + drop(open); +} + +/// Write a frame stamped `millis`, filled with its mark, to each of `tracks` for each 20 ms +/// from `from` to `to`, `millis` after `start`, as a live source does, the export keeping up. +async fn live( + export: &mut super::Export, + frames: &mut Vec, + start: tokio::time::Instant, + tracks: &mut [(&mut Writer, u8)], + from: u64, + to: u64, +) where + S::Ext: super::catalog::Catalog, +{ + for millis in (from..to).step_by(20) { + tokio::time::sleep_until(start + Duration::from_millis(millis)).await; + for (track, mark) in tracks.iter_mut() { + write_marked(track, millis, *mark); + } + settle(export, frames).await; + } +} + +/// A broadcast and its `mpegts` catalog. +fn broadcast() -> ( + moq_net::broadcast::Producer, + moq_net::broadcast::Consumer, + crate::catalog::Producer, +) { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + (broadcast, consumer, catalog) +} + +/// The rate of `packets` a slot. +fn rate(packets: u64) -> u64 { + packets * 188 * 8 * 40 +} + +/// The PCR moves only once the whole PMT naming its new PID is out: at a rate low enough +/// that the PMT spans slots, no PCR goes out there before the PMT's last packet. +#[tokio::test(start_paused = true)] +async fn the_clock_moves_behind_the_last_packet_of_the_pmt_naming_its_pid() { + let (mut broadcast, consumer, mut catalog) = broadcast(); + let (name, mut clock) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let (_, mut other) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await.with_mux_rate(rate(4)); + // Frames 50 ms apart, for room beside the tables; the second track's from 500 ms, for room + // beside the first tables. + for millis in (0..6_000).step_by(50) { + tokio::time::sleep_until(start + Duration::from_millis(millis)).await; + match millis { + ..3_000 => write_marked(&mut clock, millis, 0x21), + // The revision moving the clock lays three packets of PMT, in slots of four. + 3_000 => { + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions.remove(&name); + guard.ext.mpegts.tracks.remove(&name); + guard.ext.mpegts.program_descriptors = (0x80..0x82) + .map(|tag| super::catalog::Descriptor { + tag, + data: bytes::Bytes::from(vec![tag; 250]), + }) + .collect(); + drop(guard); + clock.finish().unwrap(); + } + _ => {} + } + if millis >= 500 { + write_marked(&mut other, millis, 0x21); + } + settle(&mut export, &mut frames).await; + } + other.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let ends: Vec<(usize, u16)> = tables_at(&frames, OWN_PMT_PID) + .into_iter() + .map(|(at, table)| (at, table.pcr_pid)) + .collect(); + let mut clocks = Vec::new(); + for (index, packet) in packets(&frames).enumerate() { + if packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x10 != 0 { + if let Some(&(_, named)) = ends.iter().take_while(|(at, _)| *at < index).last() { + assert_eq!(pid(packet), named, "a PCR on a PID no whole PMT out names"); + } + clocks.push(pid(packet)); + } + } + clocks.dedup(); + assert_eq!(clocks, [AUDIO_PID, OTHER_AUDIO_PID]); +} + +/// A multiplex rate that changes with a revision applies behind it: the slots up to the one +/// the revised PMT starts out in are padded to the old rate, those after it to the new. +#[tokio::test(start_paused = true)] +async fn a_rate_change_applies_behind_the_revision_it_comes_with() { + const OLD: u64 = 20; + const NEW: u64 = 30; + let (mut broadcast, consumer, mut catalog) = broadcast(); + catalog.modify().unwrap().ext.mpegts.mux_rate = Some(rate(OLD)); + let (_, mut audio) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 0, 3_000).await; + { + let mut guard = catalog.modify().unwrap(); + guard.ext.mpegts.mux_rate = Some(rate(NEW)); + guard.ext.mpegts.program_descriptors = vec![super::catalog::Descriptor { + tag: 0x80, + data: bytes::Bytes::from_static(&[1, 2, 3, 4]), + }]; + } + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 3_000, 6_000).await; + audio.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + let revised = frames + .iter() + .position(|frame| { + tables(std::slice::from_ref(frame), OWN_PMT_PID) + .iter() + .any(|table| table.version == 1) + }) + .expect("a revised PMT"); + let counts: Vec = frames.iter().map(|frame| frame.payload.len() as u64 / 188).collect(); + assert!( + counts[revised - 20..=revised].iter().all(|&n| n == OLD), + "{revised}: {counts:?}" + ); + assert!( + counts[revised + 1..revised + 20].iter().all(|&n| n == NEW), + "{revised}: {counts:?}" + ); +} + +/// Under sync a catalog's rate change leaves the grid as laid, and says so. +#[tokio::test(start_paused = true)] +#[tracing_test::traced_test] +async fn a_rate_change_under_sync_keeps_the_grid() { + let (mut broadcast, consumer, mut catalog) = broadcast(); + catalog.modify().unwrap().ext.mpegts.mux_rate = Some(rate(20)); + let (_, mut audio) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let source = crate::source::announced(&consumer).await; + let opened = super::export_test::open_ts(source, crate::catalog::CatalogFormat::Hang) + .await + .unwrap(); + let mut export = opened.with_delay(DELAY).with_sync(); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 0, 3_000).await; + catalog.modify().unwrap().ext.mpegts.mux_rate = Some(rate(30)); + let before = frames.len(); + live(&mut export, &mut frames, start, &mut [(&mut audio, 0x21)], 3_000, 6_000).await; + assert!(frames.len() > before + 40); + assert!( + frames[before..].iter().all(|frame| frame.payload.len() == 20 * 188), + "the grid's rate changed" + ); + assert!(logs_contain("ignoring a multiplex rate change under --sync")); + drop((broadcast, audio, catalog)); +} + +/// A track the catalog withdraws before the PMT lists it, while its first frames wait in the +/// jitter buffer, and lists again under the same name is the same stream: it is read on. +#[tokio::test(start_paused = true)] +async fn a_track_withdrawn_before_its_listing_is_read_on_when_it_returns() { + withdraw_and_return(false).await; +} + +/// As above, with the first frames it sent before leaving going out, which lists it, and the +/// rest still waiting when it returns. +#[tokio::test(start_paused = true)] +async fn a_track_withdrawn_before_its_listing_is_read_on_when_it_returns_listed() { + withdraw_and_return(true).await; +} + +/// Withdraw a track whose first frames wait in the jitter buffer, let some of them go out when +/// `listed`, and list it again: the frames it sends after returning go out. +async fn withdraw_and_return(listed: bool) { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let (_, mut first) = aac_track(&mut broadcast, &mut catalog, Some(AUDIO_PID)); + let mut frames = Vec::new(); + let start = tokio::time::Instant::now(); + let mut export = export_of(&consumer).await; + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 0, 2_000).await; + + let (name, mut second) = aac_track(&mut broadcast, &mut catalog, Some(OTHER_AUDIO_PID)); + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21), (&mut second, 0x31)], + 2_000, + 2_100, + ) + .await; + let config = catalog.modify().unwrap().audio.renditions[&name].clone(); + remove(&mut catalog, &name); + settle(&mut export, &mut frames).await; + let back = match listed { + true => 2_050 + DELAY.as_millis() as u64, + false => 2_100, + }; + live(&mut export, &mut frames, start, &mut [(&mut first, 0x21)], 2_100, back).await; + assert_eq!( + marks(&frames, OTHER_AUDIO_PID, &[0x31]).is_empty(), + !listed, + "its frames went out before it returned" + ); + let mut guard = catalog.modify().unwrap(); + guard.audio.renditions.insert(name.clone(), config); + guard + .ext + .mpegts + .tracks + .insert(name, super::catalog::Track::new(OTHER_AUDIO_PID)); + drop(guard); + live( + &mut export, + &mut frames, + start, + &mut [(&mut first, 0x21), (&mut second, 0x32)], + back, + back + 1_000, + ) + .await; + first.finish().unwrap(); + second.finish().unwrap(); + catalog.finish().unwrap(); + drain(&mut export, &mut frames).await; + + assert_eq!( + marks(&frames, OTHER_AUDIO_PID, &[0x31, 0x32]).last(), + Some(&0x32), + "the returned track went out" + ); +} diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index d9f28b5d0fa..e0226eed4f4 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -4,6 +4,7 @@ use std::cmp::Reverse; use std::collections::{BinaryHeap, HashMap, VecDeque}; use std::time::Duration; +use anyhow::Context; use mpeg2ts::ts::TsPacket; use super::export::PCR_INTERVAL; @@ -150,8 +151,23 @@ pub(super) struct Timing { pub decode: u128, } +/// Program tables to go out ahead of the next unit pushed ([`Schedule::lead`]). +pub(super) struct Tables { + /// The PAT and PMT packets. + pub packets: Vec, + /// The PID every slot's clock rides once they have gone out. + pub clock: u16, + /// The multiplex rate from the slot after the one they finish in. + pub rate: Option, + /// The PIDs they list that the tables before them did not. Their units wait for them, and + /// so does the unit they lead. + pub added: Vec, +} + /// One access unit's packets, with the tables muxed ahead of it, still to go out. struct Unit { + /// Counts up with each unit pushed. + seq: u64, /// The elementary stream it carries, whose buffers bound it. pid: u16, /// The coarse deadline used by release windows and unpadded admission. @@ -176,6 +192,17 @@ struct Unit { frames: Vec<(u128, usize)>, /// The packets it carries into TBsys, one run per PID. runs: Vec, + /// For a unit of program tables, what they put in force and the PIDs they hold back. + tables: Option, +} + +/// What a unit of program tables puts in force once it has gone out ([`Tables`]). +struct Program { + clock: u16, + rate: Option, + holds: Vec, + /// The PID of the unit they lead, which they go out as far ahead of as it may. + leads: u16, } impl Unit { @@ -395,6 +422,11 @@ pub(super) struct Slot { pub clocked: bool, /// The slot does not follow the one laid before it, so its PCR flags the break. pub jump: bool, + /// The PID its clock rides. + pub clock: Option, + /// How many units of program tables finish in the slot ([`Schedule::lead`]), which go + /// out in the order they were pushed. + pub tables: usize, } impl Slot { @@ -557,6 +589,10 @@ pub(super) struct Schedule { ended: bool, /// The fixed lanes every slot is laid on instead, under [`Export::with_sync`](super::Export::with_sync). sync: Option, + /// The sequence number of the next unit pushed. + seq: u64, + /// Tables to go out ahead of the next unit pushed. + lead: Option, } impl Schedule { @@ -577,6 +613,8 @@ impl Schedule { system: None, mb: HashMap::new(), ended: false, + seq: 0, + lead: None, } } @@ -689,6 +727,27 @@ impl Schedule { self.clock = Some(pid); } + /// Forget the buffers of `pid`, which carries no stream any more, so one that takes it + /// over goes out under its own. + pub fn forget(&mut self, pid: u16) { + if self.buffers.remove(&pid).is_some() { + self.remap(); + } + } + + /// Whether a unit on `pid` is still to go out. + pub fn holds(&self, pid: u16) -> bool { + self.units.iter().any(|unit| unit.pid == pid && unit.tables.is_none()) + } + + /// Send `tables` ahead of the next unit pushed, under its deadline. Program tables ride + /// one lane, so each version goes out after the one before it, and a revision is one + /// boundary: no unit of a stream they add goes out before them, and their clock PID and + /// multiplex rate take over only once they have gone out. + pub fn lead(&mut self, tables: Tables) { + self.lead = Some(tables); + } + /// Queue an access unit of one frame. #[cfg(test)] pub fn push(&mut self, pid: u16, timing: Timing, packets: Vec, keyframe: bool) -> anyhow::Result<()> { @@ -737,13 +796,28 @@ impl Schedule { keyframe: bool, mut frames: Vec<(u128, usize)>, ) -> anyhow::Result<()> { + if let Some(tables) = self.lead.take() { + let pmt = self.pmt.context("program tables without a PMT PID")?; + // Later tables wait too, so each version follows the one before it. + let mut holds = tables.added; + holds.extend([pid, pmt]); + let program = Program { + clock: tables.clock, + rate: tables.rate, + holds, + leads: pid, + }; + // The unit they lead keeps its own start. + let last = self.last; + self.enqueue(pmt, timing, tables.packets, Some(program)); + self.last = last; + } if packets.is_empty() { return Ok(()); } if frames.len() < 2 { frames.clear(); } - let due = slot(timing.by); // An ended source relaxes deadlines, never buffer sizes: such a unit never fits. if let Some(size) = self.buffers.get(&pid).and_then(|buffer| buffer.size) { let on_pid = || { @@ -785,24 +859,64 @@ impl Schedule { .last() .map_or(0, |&(decode, _)| decode.saturating_sub(timing.decode)); self.horizon = self.horizon.max(Some(last.div_ceil(PCR_INTERVAL.as_nanos()))); + let unit = self.enqueue(pid, timing, packets, None); + unit.keyframe = keyframe; + unit.place = place; + unit.frames = frames; + Ok(()) + } + + /// Push a unit of `packets` on `pid` due by `timing`, after every unit pushed before it. + fn enqueue(&mut self, pid: u16, timing: Timing, packets: Vec, tables: Option) -> &mut Unit { + let due = slot(timing.by); let start = self.last.map_or(due, |last| (last + 1).min(due)); self.last = Some(due); let runs = self.runs(&packets); + let seq = self.seq; + self.seq += 1; self.units.push_back(Unit { + seq, pid, due, timing, start, packets, sent: 0, - keyframe, + keyframe: false, late: self.next.is_some_and(|next| due < next), - place, + place: None, held: 0, - frames, + frames: Vec::new(), runs, + tables, }); - Ok(()) + self.units.back_mut().expect("the unit just pushed") + } + + /// The PIDs the program tables still to go out hold back, each with the first of them that + /// does: a unit on one, pushed after those tables, waits for them. `left` is how many + /// packets each unit has still to send. + fn holding(&self, left: impl Fn(usize) -> usize) -> HashMap { + let mut held = HashMap::new(); + for (i, unit) in self.units.iter().enumerate() { + let Some(program) = unit.tables.as_ref().filter(|_| left(i) > 0) else { + continue; + }; + for &pid in &program.holds { + held.entry(pid).or_insert(unit.seq); + } + } + held + } + + /// Put in force what the tables of unit `i` carry, as their last packet goes out, if it is one. + fn enforce(&mut self, i: usize) -> bool { + let Some(program) = &self.units[i].tables else { + return false; + }; + self.clock = Some(program.clock); + self.rate = program.rate; + true } /// Whether a unit is still to go out. @@ -847,6 +961,7 @@ impl Schedule { self.horizon = None; self.last = None; self.ended = false; + self.lead = None; } /// The first slot `unit` may go out in: [`ahead`] of its due slot, and never before the @@ -867,7 +982,8 @@ impl Schedule { if unit.late { return 0; } - let ahead = ahead(self.buffers.get(&unit.pid), self.window); + let pid = unit.tables.as_ref().map_or(unit.pid, |program| program.leads); + let ahead = ahead(self.buffers.get(&pid), self.window); unit.due.saturating_sub(ahead).min(unit.start) } @@ -919,6 +1035,8 @@ impl Schedule { positions: Some(laid.positions), clocked: false, jump: laid.jump, + clock: self.clock, + tables: 0, })); } let Some(index) = self.index(known) else { @@ -932,11 +1050,28 @@ impl Schedule { return self.padded(index, rate).map(Some); } + let clock = self.clock; let mut take = self.admit(index, usize::MAX); - // A placed PES is due only once released, which may come after its due slot. - let released: Vec = self.units.iter().map(|unit| self.release(unit) <= index).collect(); - for ((unit, take), released) in self.units.iter_mut().zip(take.iter_mut()).zip(released) { - if unit.due <= index && released && *take < unit.remaining() { + // A placed PES is due only once released, which may come after its due slot. A unit + // forced out takes the tables ahead of it that hold it back along. + let mut forced: Vec = self + .units + .iter() + .map(|unit| unit.due <= index && self.release(unit) <= index) + .collect(); + for j in (0..self.units.len()).rev() { + if !forced[j] { + continue; + } + let pid = self.units[j].pid; + for (unit, forced) in self.units.iter().zip(forced.iter_mut()).take(j) { + if unit.tables.as_ref().is_some_and(|program| program.holds.contains(&pid)) { + *forced = true; + } + } + } + for ((unit, take), forced) in self.units.iter_mut().zip(take.iter_mut()).zip(forced) { + if forced && *take < unit.remaining() { if self.buffers.get(&unit.pid).is_some_and(|buffer| buffer.size.is_some()) { unit.held += unit.packets_on(unit.sent + *take..unit.count(), unit.pid) * PAYLOAD; } @@ -954,7 +1089,7 @@ impl Schedule { if unit.sent == 0 { keyframe |= unit.keyframe; // Only the tables muxed ahead of a frame that wrote nothing are no access unit. - if unit.packets_on(0..unit.count(), unit.pid) > 0 { + if unit.tables.is_none() && unit.packets_on(0..unit.count(), unit.pid) > 0 { units.extend(std::iter::repeat_n(unit.pid, unit.frames.len().max(1))); } } @@ -962,6 +1097,9 @@ impl Schedule { packets.extend_from_slice(&unit.packets[from..from + take * TsPacket::SIZE]); unit.sent += take; } + let tables = (0..self.units.len()) + .filter(|&i| self.units[i].remaining() == 0 && self.enforce(i)) + .count(); for unit in self.units.iter().filter(|unit| unit.remaining() == 0) { self.decoding.extend(unit.leaving()); } @@ -976,6 +1114,8 @@ impl Schedule { positions: None, clocked: false, jump: false, + clock, + tables, })) } @@ -998,8 +1138,11 @@ impl Schedule { positions: Some(Vec::new()), clocked: false, jump: false, + clock: self.clock, + tables: 0, }; let clock = self.clock.filter(|pid| self.buffers.contains_key(pid)); + let mut holding = self.holding(|i| self.units[i].remaining()); // A new multiplex rate changes Rsys from this slot on, and Bsys keeps the bytes it holds. let rsys = system_rate(rate); if let Some((empty, before)) = self.system @@ -1085,6 +1228,10 @@ impl Schedule { while let Some(Reverse((by, lane))) = ready.pop() { let i = *lanes[lane].front().unwrap(); let unit = &self.units[i]; + if holding.get(&unit.pid).is_some_and(|&seq| seq < unit.seq) { + passed.push(Reverse((by, lane))); + continue; + } let rest = &unit.packets[unit.sent * TsPacket::SIZE..]; let opening = match position { 0 => match clock.filter(|&clock| self::pid(rest) == clock) { @@ -1207,13 +1354,13 @@ impl Schedule { waiting.push(Reverse((last + TICK, lane))); continue; } - let held = if pid == unit.pid && buffer.is_some_and(|buffer| buffer.size.is_some()) { + let bytes = if pid == unit.pid && buffer.is_some_and(|buffer| buffer.size.is_some()) { payload(packet).len() } else { 0 }; if let Some(size) = buffer.and_then(|buffer| buffer.size) - && occupancy.get(&pid).copied().unwrap_or(0) + held > size + && occupancy.get(&pid).copied().unwrap_or(0) + bytes > size { if let Some(event) = removals.peek() { let ready = event.0 as i128; @@ -1254,18 +1401,22 @@ impl Schedule { let unit = &mut self.units[i]; if unit.sent == 0 { out.keyframe |= unit.keyframe; - if unit.packets_on(0..unit.count(), unit.pid) > 0 { + if unit.tables.is_none() && unit.packets_on(0..unit.count(), unit.pid) > 0 { out.units .extend(std::iter::repeat_n(unit.pid, unit.frames.len().max(1))); } } unit.sent += 1; if unit.last_decode() as i128 > now { - unit.held += held; - *occupancy.entry(unit.pid).or_default() += held; + unit.held += bytes; + *occupancy.entry(unit.pid).or_default() += bytes; } if unit.remaining() == 0 { lanes[lane].pop_front(); + if self.enforce(i) { + out.tables += 1; + holding = self.holding(|i| self.units[i].remaining()); + } } if let Some(&i) = lanes[lane].front() { let unit = &self.units[i]; @@ -1364,6 +1515,7 @@ impl Schedule { let mut take = vec![0; self.units.len()]; let mut carried: HashMap = HashMap::new(); let mut stopped: Vec = Vec::new(); + let mut holding = self.holding(|i| self.units[i].remaining()); for i in order { let unit = &self.units[i]; if budget == 0 { @@ -1374,7 +1526,8 @@ impl Schedule { } let size = self.buffers.get(&unit.pid).and_then(|buffer| buffer.size); let mut n = 0; - if self.release(unit) <= index { + let held = holding.get(&unit.pid).is_some_and(|&seq| seq < unit.seq); + if !held && self.release(unit) <= index { while n < unit.remaining() && budget > 0 { let pid = unit.pid_at(unit.sent + n); let cap = self @@ -1401,6 +1554,9 @@ impl Schedule { } if n < unit.remaining() { stopped.push(unit.pid); + } else if unit.tables.is_some() { + take[i] = n; + holding = self.holding(|j| self.units[j].remaining() - take[j]); } take[i] = n; } @@ -2665,6 +2821,8 @@ mod tests { positions: None, clocked: false, jump: false, + clock: None, + tables: 0, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -2704,6 +2862,8 @@ mod tests { positions: None, clocked: false, jump: false, + clock: None, + tables: 0, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 7495047a5ef..e141691ae1e 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -1,6 +1,7 @@ //! A jitter buffer that releases several tracks' frames in one decode order, each a fixed //! delay after its decode time. +use std::borrow::Borrow; use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::pin::Pin; use std::task::Poll; @@ -973,7 +974,10 @@ impl Buffer { /// Count a frame of `key` the caller dropped before queuing it: a loss once the track /// started or when the caller queued frames `ahead` of it, else trimmed. - pub fn lose(&mut self, key: &K, ahead: bool) { + pub fn lose(&mut self, key: &Q, ahead: bool) + where + K: Borrow, + { match ahead || self.tracks.get(key).is_some_and(|track| track.started) { true => self.dropped += 1, false => self.trimmed += 1, @@ -981,11 +985,39 @@ impl Buffer { } /// The frame `key` releases next, if it has one queued. - pub fn peek(&self, key: &K) -> Option<&T> { + pub fn peek(&self, key: &Q) -> Option<&T> + where + K: Borrow, + { let (_, _, _, item) = self.tracks.get(key)?.queue.front()?; Some(item) } + /// Whether `key` has a frame waiting, queued or held for a clock still being acquired. + pub fn holds(&self, key: &Q) -> bool + where + K: Borrow, + { + self.peek(key).is_some() + || self + .acquire + .as_ref() + .is_some_and(|acquire| acquire.tracks.contains_key(key)) + } + + /// Forget `key`, which holds nothing and will push nothing more, so what it kept to stay in + /// step goes with it. + pub fn forget(&mut self, key: &Q) + where + K: Borrow, + { + debug_assert!(!self.holds(key), "forgot a track that holds frames"); + self.tracks.remove(key); + self.expect.remove(key); + self.leads.remove(key); + self.steer.tracks.remove(key); + } + /// When the next queued frame is due, or the acquisition under way ends. #[cfg(test)] pub fn next_deadline(&self) -> Option { @@ -998,7 +1030,10 @@ impl Buffer { /// Whether `key` has a frame waiting that is already due (any frame, without a delay), so /// reading more of it now would only queue what the output cannot take yet. - pub fn full(&self, key: &K) -> bool { + pub fn full(&self, key: &Q) -> bool + where + K: Borrow, + { self.tracks .get(key) .and_then(|track| track.queue.front()) diff --git a/test/justfile b/test/justfile index 4e95f5cfa71..fc079055c94 100644 --- a/test/justfile +++ b/test/justfile @@ -185,6 +185,7 @@ ts *args: python3 ./ts/holes.test.py python3 ./ts/passthrough.test.py python3 ./ts/klv.test.py + python3 ./ts/revision.test.py python3 ./ts/clockless.test.py ./ts/run.sh "$@" diff --git a/test/ts/README.md b/test/ts/README.md index 8e7f71e464e..a8dd2f5fee8 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -44,6 +44,7 @@ just test ts --leak # 1080p video with no HRD, filling its level's 2 just test ts --full # the same with two audio PIDs, filling its multiplex with no nulls just test ts --fragment-pes # legal headers split across packets, with FFmpeg/TSDuck grading just test ts --klv # KLV and DVB AC-3 carried as whole PES, graded against the source +just test ts --revision # the PMT revised mid-stream, a KLV stream joining and leaving just test ts --delay 2s # pass the exporter's --delay just test ts --hrd --rate 7000000 # congest the subscriber's leg below the media rate just test ts --loss 1 # drop 1% of the relay's datagrams to the subscriber @@ -68,6 +69,15 @@ source placement on the program clock, and none after its PTS; and FFmpeg readin each as KLV with the same PES and PTS. `holes.py` and the T-STD model grade the AC-3, whose buffer drains each sync frame at its own decode time. +`--revision` has `revision.py` add asynchronous KLV to the default clip's null +slots from a third of its program clock to two thirds, and revise the PMT at each +end: the next `version_number` lists the stream before its first packet, and the +one after drops it once its last packet is out. `revision.py grade` holds the +capture's PMTs to the source's layouts in order, each at the version after the one +before and the KLV entry byte-equal, every packet of the stream between the PMT +listing it and the one dropping it, and every PES of the stream in order and +byte-equal. + `--live` swaps the analyzer, not the rig: the same round-trip runs, but the subscriber's stdout goes straight into `pcr-timing.py` instead of a capture file. That is the only arm that can see release timing at all, and it is what nightly @@ -116,6 +126,7 @@ Severities: **hard** checks fail the run by default; **shape** checks report as | `packet-size` | hard | 188 (or 204) bytes per packet | | `sync` | hard | no invalid sync bytes / transport-error packets | | `pat` / `pmt` | hard | valid PAT mapping programs to a PMT that lists the elementary streams | +| `pmt-versions` | hard | each program's PMT changes only at a new `version_number`, and never returns to an older one | | `psi-crc` | hard | no section dropped for a bad CRC | | `continuity` | hard | no continuity-counter discontinuities | | `pcr-presence` | hard | every program's declared PCR PID carries PCR (0x1FFF declares none) | @@ -894,7 +905,7 @@ browser interop. It runs nightly on `main`, and release PRs touching a workspace crate `run.sh` builds (kept in sync by `sh/gh/ts-paths.py`), the harness, or the toolchain run it too: `just test ts-controls`, `just test ts-pair-controls`, the default and 2 Mb/s round trips, HRD, leak, full, headroom, -open-GOP, KLV (strict), passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Each step runs even after another test fails. +open-GOP, KLV (strict), PMT revision, passthrough (generated, Kyrion and MPTS), EIT, and T-STD controls. Each step runs even after another test fails. All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. Nightly and manual runs also invoke `just test ts --pair`, diff --git a/test/ts/compliance.py b/test/ts/compliance.py index 829100951ef..083a7c465ee 100755 --- a/test/ts/compliance.py +++ b/test/ts/compliance.py @@ -14,9 +14,9 @@ in `pcr-timing.py`. Checks split into two severities: - - HARD (structural): fail the run by default. PAT/PMT, packet size, sync, - continuity counters, PSI CRC, PCR presence, PCR monotonicity, T-STD buffers, - system buffers. + - HARD (structural): fail the run by default. PAT/PMT, PMT versions, packet + size, sync, continuity counters, PSI CRC, PCR presence, PCR monotonicity, + T-STD buffers, system buffers. - SHAPE (broadcast profile): reported as WARN and only fail the run under `--strict`. Service descriptors (SDT). @@ -1527,6 +1527,64 @@ def take(self, start: float, end: float, first: bool, grade: Grade) -> None: grade.flag("Bsys overflow") +def pmt_tables(data: bytes, packet_size: int) -> dict[int, list[tuple[int, int, bytes]]]: + """Each program's PMTs in force (current_next_indicator set), by program_number: (index + of the packet completing the section, version_number, the section less its version and + CRC), on whichever PMT PID a PAT named, each run of repeats once.""" + pmt_pids = set() + for _, section in sections(packets_on(data, packet_size, {0}).get(0, [])): + if section[0] == 0x00 and section[5] & 0x01: + for at in range(8, len(section) - 4, 4): + if (section[at] << 8) | section[at + 1]: + pmt_pids.add(((section[at + 2] & 0x1F) << 8) | section[at + 3]) + found: dict[int, list[tuple[int, int, bytes]]] = {} + for packets in packets_on(data, packet_size, pmt_pids).values(): + for index, section in sections(packets): + if section[0] == 0x02 and section[5] & 0x01: + program = (section[3] << 8) | section[4] + body = section[:5] + section[6:-4] + found.setdefault(program, []).append((index, (section[5] >> 1) & 0x1F, body)) + tables: dict[int, list[tuple[int, int, bytes]]] = {} + for program, seen in found.items(): + runs = tables.setdefault(program, []) + for table in sorted(seen): + if not runs or runs[-1][1:] != table[1:]: + runs.append(table) + return tables + + +def check_pmt_versions(ts_path: str, packet_size: int) -> Check: + """Each program's PMT changes only at a new version_number (ISO 13818-1 2.4.4.9), and an + older version never follows a newer one. + + A demux keeps the table it has until the version changes, so a change made under the same + version never reaches it, and a copy of the table a revision replaced, arriving after it, + reverts the revision. A version one up to 15 behind the newest is older, modulo 32. So is + a table coming back under the version it went out at before the versions since have come + round to it, which modulo 32 alone reads as a skip forward. A revision that skips versions + is reported, not failed: what a receiver sees still changes at every new version.""" + with open(ts_path, "rb") as handle: + data = handle.read() + revisions = skipped = 0 + for program, runs in sorted(pmt_tables(data, packet_size).items()): + for at in range(1, len(runs)): + newest, (index, version, body) = runs[at - 1][1], runs[at] + behind = (newest - version) % 32 + last = max((n for n in range(at) if runs[n][1:] == (version, body)), default=None) + returned = last is not None and len({v for _, v, _ in runs[last + 1 : at]}) < 31 + if behind == 0: + detail = f"program {program}: the PMT changed at packet {index} under version {version}" + elif behind < 16 or returned: + detail = f"program {program}: version {version} at packet {index}, after version {newest}" + else: + revisions += 1 + skipped += (version - newest) % 32 - 1 + continue + return Check("pmt-versions", Severity.HARD, Status.FAIL, detail, {"revisions": revisions}) + detail = f"{revisions} revision(s), each at a new version" + (f", {skipped} version(s) skipped" if skipped else "") + return Check("pmt-versions", Severity.HARD, Status.PASS, detail, {"revisions": revisions, "skipped": skipped}) + + def check_system_buffers(ts_path: str, packet_size: int, scan: Scan) -> Check: """TBsys and Bsys (ISO 13818-1 2.4.2.3) for each program, on its PCR. @@ -1695,6 +1753,7 @@ def analyze(ts_path: str, reference_seconds: float | None = None) -> list[Check] pat, pmt = check_pat_pmt(analysis) checks.append(pat) checks.append(pmt) + checks.append(check_pmt_versions(ts_path, packet_size)) checks.append(check_psi_crc(analysis)) checks.append(check_continuity(analysis)) checks.append(check_pcr_presence(analysis, clock_by_pid)) diff --git a/test/ts/compliance.test.py b/test/ts/compliance.test.py index ee93a1f9a29..b2f2ade3115 100644 --- a/test/ts/compliance.test.py +++ b/test/ts/compliance.test.py @@ -1,10 +1,13 @@ import pathlib import sys +import tempfile import unittest sys.path.insert(0, str(pathlib.Path(__file__).parent)) import compliance # noqa: E402 +PMT_PID = 0x1000 + def eac3(stream: int) -> bytes: """A six-block 48 kHz E-AC-3 sync frame of 512 bytes; `stream` is byte 2's top bits.""" @@ -37,5 +40,66 @@ def test_reserved_reduced_rate_is_refused(self): compliance.ac3_frame(bytes(frame), 0) +def section(table_id: int, extension: int, version: int, body: bytes) -> bytes: + out = bytearray([table_id, 0xB0, 0, extension >> 8, extension & 0xFF, 0xC1 | version << 1, 0, 0]) + body + length = len(out) + 4 - 3 + out[1] |= length >> 8 + out[2] = length & 0xFF + return bytes(out) + compliance.crc32_mpeg(bytes(out)).to_bytes(4, "big") + + +def psi(pid: int, counter: int, table: bytes) -> bytes: + return bytes([0x47, 0x40 | pid >> 8, pid & 0xFF, 0x10 | counter & 15, 0]) + table + b"\xff" * (183 - len(table)) + + +def tables(*pmts: tuple[int, list[int]]) -> bytes: + """A PAT, then each PMT as (version, the PIDs of its AAC streams), on PMT_PID.""" + pat = section(0x00, 1, 0, bytes([0, 1, 0xE0 | PMT_PID >> 8, PMT_PID & 0xFF])) + out = [psi(0, 0, pat)] + for counter, (version, pids) in enumerate(pmts): + streams = b"".join(bytes([0x0F, 0xE0 | pid >> 8, pid & 0xFF, 0xF0, 0]) for pid in pids) + out.append(psi(PMT_PID, counter, section(0x02, 1, version, bytes([0xE1, 0x01, 0xF0, 0]) + streams))) + return b"".join(out) + + +class PmtVersions(unittest.TestCase): + def check(self, *pmts: tuple[int, list[int]]) -> compliance.Check: + with tempfile.NamedTemporaryFile(suffix=".ts") as handle: + handle.write(tables(*pmts)) + handle.flush() + return compliance.check_pmt_versions(handle.name, 188) + + def test_repeats_and_revisions_pass(self): + check = self.check((0, [0x101]), (0, [0x101]), (1, [0x101, 0x102]), (1, [0x101, 0x102]), (2, [0x101])) + self.assertEqual((check.status, check.metrics), (compliance.Status.PASS, {"revisions": 2, "skipped": 0})) + + def test_the_version_wraps(self): + check = self.check((31, [0x101]), (0, [0x101, 0x102])) + self.assertEqual((check.status, check.metrics["revisions"]), (compliance.Status.PASS, 1)) + + def test_a_skipped_version_is_reported(self): + check = self.check((0, [0x101]), (2, [0x101, 0x102])) + self.assertEqual((check.status, check.metrics["skipped"]), (compliance.Status.PASS, 1)) + + def test_a_change_under_the_same_version_fails(self): + self.assertEqual(self.check((0, [0x101]), (0, [0x101, 0x102])).status, compliance.Status.FAIL) + + def test_an_older_version_after_a_newer_fails(self): + check = self.check((0, [0x101]), (1, [0x101, 0x102]), (0, [0x101])) + self.assertEqual(check.status, compliance.Status.FAIL) + self.assertIn("after version 1", check.detail) + + def test_a_copy_past_half_the_versions_back_fails(self): + revisions = [(version, [0x101, 0x200 + version]) for version in range(18)] + check = self.check(*revisions, (0, [0x101, 0x200])) + self.assertEqual(check.status, compliance.Status.FAIL) + self.assertIn("after version 17", check.detail) + + def test_a_table_back_after_every_version_passes(self): + cycle = [(version % 32, [0x101, 0x200 + version % 32]) for version in range(33)] + check = self.check(*cycle) + self.assertEqual((check.status, check.metrics["revisions"]), (compliance.Status.PASS, 32)) + + if __name__ == "__main__": unittest.main() diff --git a/test/ts/revision.py b/test/ts/revision.py new file mode 100644 index 00000000000..15b7915628f --- /dev/null +++ b/test/ts/revision.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Revise a generated clip's PMT mid-stream, and grade a round-trip's capture of the revisions. + + ./revision.py inject source.ts out.ts [--report revision.json] + ./revision.py grade source.ts capture.ts [--report-json revision.json] + +`inject` adds asynchronous KLV (stream_type 0x06 with a `KLVA` registration, no PTS) in the +clip's null slots from a third of its program clock to two thirds, every 100 ms, and revises +the PMT at each end: it lists the stream at the next version_number before the stream's first +packet, and drops it at the version after that once its last packet is out. Every other packet +stays in place. + +`grade` holds the capture to what a TS export promises of a revision. Its PMT runs through the +source's layouts in order (the source's PIDs and stream types, the KLV entry byte-equal), each +at the version after the one before. Every packet of the stream falls between the PMT that lists +it and the one that drops it, and every PES of the stream arrives, in order and byte-equal. +compliance.py grades the rest of the capture, the version sequence included. + +Exit status is 0 when every check passes, 1 otherwise. +""" + +import argparse +import json +import sys +from pathlib import Path + +import compliance +import klv + +PID = 0x112 +STREAM = (0x06, PID, klv.KLVA) + + +class Failure(Exception): + """A named grading failure.""" + + def __init__(self, check, detail): + self.check = check + super().__init__(f"{check}: {detail}") + + +def revise(packet, version, streams): + """A PMT packet whose section is at `version` with `streams` appended, its CRC recomputed.""" + body = klv.payload(packet) + start = len(packet) - len(body) + 1 + body[0] + section = bytearray(klv.single_section(packet)) + section[5] = (section[5] & 0xC1) | version << 1 + entries = b"".join(bytes([kind, 0xE0 | route >> 8, route & 0xFF, 0xF0 | len(info) >> 8, len(info) & 0xFF]) + info for kind, route, info in streams) + section = section[:-4] + entries + length = len(section) + 4 - 3 + section[1] = (section[1] & 0xF0) | length >> 8 + section[2] = length & 0xFF + section += compliance.crc32_mpeg(bytes(section)).to_bytes(4, "big") + if start + len(section) > klv.SIZE: + raise ValueError("the PMT outgrows its packet") + out = bytearray(packet[:start]) + section + out += b"\xff" * (klv.SIZE - len(out)) + return bytes(out) + + +def clock_span(source, pcr_pid): + """The first and last PCR on `pcr_pid`, unwrapped.""" + values = [] + for packet in source: + if klv.pid(packet) == pcr_pid and (now := klv.pcr(packet)) is not None: + values.append(now if not values else values[-1] + (now - values[-1] % klv.PCR_WRAP) % klv.PCR_WRAP) + if len(values) < 2: + raise ValueError("fewer than two PCRs") + return values[0], values[-1] + + +def inject(data): + """`data` with the KLV stream and its two PMT revisions, and a report of what went in.""" + source = klv.packets(data) + prog = klv.program(data) + if PID in prog.streams: + raise ValueError(f"the clip already carries PID {PID:#x}") + first, last = clock_span(source, prog.pcr_pid) + start, end = first + (last - first) // 3, first + 2 * (last - first) // 3 + version = None + # 0 before the stream, 1 while it runs, 2 once it ended and its last packet is out. + phase = 0 + listed = False + queue = [] + counter = count = 0 + deadline = now = None + revised = [] + out = [] + for index, packet in enumerate(source): + route = klv.pid(packet) + if route == prog.pcr_pid and (value := klv.pcr(packet)) is not None: + now = value if now is None else now + (value - now % klv.PCR_WRAP) % klv.PCR_WRAP + if phase == 0 and now >= start: + phase, deadline = 1, now + while phase == 1 and deadline < end and now >= deadline: + pes = klv.async_klv(count) + queue.extend(klv.split(PID, pes, counter)) + counter += -(-len(pes) // 184) + count += 1 + deadline += klv.ASYNC_PERIOD + if phase == 1 and now >= end and not queue: + phase = 2 + if route == prog.pmt_pid and (klv.single_section(packet) or b"\0")[0] == 0x02: + version = (klv.single_section(packet)[5] >> 1) & 0x1F if version is None else version + if phase: + packet = revise(packet, (version + phase) % 32, [STREAM] if phase == 1 else []) + if len(revised) < phase: + revised.append(index) + listed = phase == 1 + if route == klv.NULL and queue and listed: + packet = queue.pop(0) + out.append(packet) + if phase != 2 or len(revised) != 2: + raise ValueError("the clip ended before the stream did and the PMT dropped it") + return b"".join(out), {"pes": count, "revised_at": revised} + + +def layouts(data): + """(index, version, {PID: (stream_type, descriptors)}) of each PMT of the first program, each + run of repeats once.""" + prog = klv.program(data) + found = [] + for index, packet in enumerate(klv.packets(data)): + section = klv.single_section(packet) + if klv.pid(packet) != prog.pmt_pid or section is None or section[0] != 0x02: + continue + table = ((section[5] >> 1) & 0x1F, klv.read_pmt(section)[1]) + if not found or found[-1][1:] != table: + found.append((index, *table)) + return found + + +def grade(source, capture): + """Grade `capture` against `source`, returning a report; raises Failure on the first miss.""" + types = lambda streams: {route: kind for route, (kind, _) in streams.items()} + src, cap = layouts(source), layouts(capture) + if [types(streams) for _, _, streams in cap] != [types(streams) for _, _, streams in src]: + raise Failure("layouts", f"the capture's PMTs list {[sorted(types(s).items()) for _, _, s in cap]}, the source's {[sorted(types(s).items()) for _, _, s in src]}") + for (_, newest, _), (index, version, _) in zip(cap, cap[1:]): + if version != (newest + 1) % 32: + raise Failure("version", f"version {version} at packet {index} follows version {newest}") + with_stream = [k for k, (_, _, streams) in enumerate(cap) if PID in streams] + if len(with_stream) != 1: + raise Failure("layouts", f"{len(with_stream)} of the capture's PMTs list PID {PID:#x}, not one") + listed = with_stream[0] + if cap[listed][2][PID] != (STREAM[0], STREAM[2]): + raise Failure("pmt", f"PID {PID:#x}: {cap[listed][2][PID]} in the capture, not {(STREAM[0], STREAM[2])}") + opened = cap[listed][0] + closed = cap[listed + 1][0] if listed + 1 < len(cap) else None + for index, packet in enumerate(klv.packets(capture)): + if klv.pid(packet) == PID and not (opened < index and (closed is None or index < closed)): + raise Failure("window", f"a packet on PID {PID:#x} at {index}, outside the PMTs listing it ({opened} to {closed})") + src_pes = [pes.data for pes in klv.private_pes(source, [PID])[PID]] + cap_pes = [pes.data for pes in klv.private_pes(capture, [PID])[PID]] + if cap_pes != src_pes: + raise Failure("pes", f"PID {PID:#x}: {len(cap_pes)} PES in the capture, not the source's {len(src_pes)} in order") + return {"versions": [version for _, version, _ in cap], "pes": len(cap_pes), "opened": opened, "closed": closed} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + add = sub.add_parser("inject") + add.add_argument("source", type=Path) + add.add_argument("output", type=Path) + add.add_argument("--report", type=Path) + check = sub.add_parser("grade") + check.add_argument("source", type=Path) + check.add_argument("capture", type=Path) + check.add_argument("--report-json", type=Path) + args = parser.parse_args() + + if args.command == "inject": + data, report = inject(args.source.read_bytes()) + args.output.write_bytes(data) + if args.report: + args.report.write_text(json.dumps(report, indent=2) + "\n") + print("revised the PMT:", report) + return 0 + + print("### PMT revisions (capture against the source)") + try: + report = grade(args.source.read_bytes(), args.capture.read_bytes()) + except Failure as err: + print(f" FAIL {err}") + return 1 + if args.report_json: + args.report_json.write_text(json.dumps(report, indent=2) + "\n") + print(f" PASS versions {report['versions']}; {report['pes']} PES on PID {PID:#x}, " + f"between the PMTs at packets {report['opened']} and {report['closed']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/ts/revision.test.py b/test/ts/revision.test.py new file mode 100644 index 00000000000..7df4b037c48 --- /dev/null +++ b/test/ts/revision.test.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Controls for the PMT revision fixture and its grader: each miss it exists to catch fails it.""" + +import unittest + +import compliance +import klv +import revision + +VIDEO = 0x100 +PMT = 0x1000 + + +def section(table_id, extension, body): + out = bytearray([table_id, 0xB0, 0, extension >> 8, extension & 0xFF, 0xC1, 0, 0]) + body + length = len(out) + 4 - 3 + out[1] |= length >> 8 + out[2] = length & 0xFF + return bytes(out) + compliance.crc32_mpeg(bytes(out)).to_bytes(4, "big") + + +def psi(route, table): + return bytes([0x47, 0x40 | route >> 8, route & 0xFF, 0x10, 0]) + table + b"\xff" * (183 - len(table)) + + +def picture(counter, pcr): + """A video packet carrying `pcr` and the start of an unbounded PES.""" + base, ext = pcr // 300, pcr % 300 + field = bytes([7, 0x10]) + (base >> 1).to_bytes(4, "big") + bytes([(base & 1) << 7 | 0x7E | ext >> 8, ext & 0xFF]) + pes = bytes([0, 0, 1, 0xE0, 0, 0, 0x80, 0x80, 5]) + klv.clock_field(0x2, base % klv.PTS_WRAP) + body = pes + b"\x00" * (184 - len(field) - len(pes)) + return bytes([0x47, 0x40 | VIDEO >> 8, VIDEO & 0xFF, 0x30 | counter & 15]) + field + body + + +NULL = bytes([0x47, 0x1F, 0xFF, 0x10]) + b"\xff" * 184 +PAT = psi(0, section(0x00, 1, bytes([0, 1, 0xE0 | PMT >> 8, PMT & 0xFF]))) +TABLE = psi(PMT, section(0x02, 1, bytes([0xE0 | VIDEO >> 8, VIDEO & 0xFF, 0xF0, 0, 0x1B, 0xE0 | VIDEO >> 8, VIDEO & 0xFF, 0xF0, 0]))) + + +def media(pictures=150): + """A clip of `pictures` 40 ms apart, its tables every ten, each picture followed by null slots.""" + out = [] + for k in range(pictures): + if k % 10 == 0: + out += [PAT, TABLE] + out.append(picture(k, 27_000_000 + k * 40 * 27_000)) + out.extend([NULL] * 6) + return b"".join(out) + + +def replace(data, index, packet): + return data[: index * klv.SIZE] + packet + data[(index + 1) * klv.SIZE :] + + +class Revision(unittest.TestCase): + def setUp(self): + self.source, self.report = revision.inject(media()) + + def assertFails(self, check, capture): + with self.assertRaises(revision.Failure) as caught: + revision.grade(self.source, capture) + self.assertEqual(caught.exception.check, check, str(caught.exception)) + + def test_the_source_revises_twice_and_grades_itself(self): + tables = revision.layouts(self.source) + self.assertEqual([version for _, version, _ in tables], [0, 1, 2]) + self.assertEqual([revision.PID in streams for _, _, streams in tables], [False, True, False]) + self.assertEqual(revision.grade(self.source, self.source)["versions"], [0, 1, 2]) + self.assertGreater(self.report["pes"], 10) + + def test_only_null_slots_and_tables_change(self): + for old, new in zip(klv.packets(media()), klv.packets(self.source)): + if klv.pid(old) not in (klv.NULL, PMT): + self.assertEqual(old, new) + + def test_the_stream_runs_between_its_revisions(self): + opened, closed = self.report["revised_at"] + on_pid = [k for k, packet in enumerate(klv.packets(self.source)) if klv.pid(packet) == revision.PID] + self.assertLess(opened, on_pid[0]) + self.assertLess(on_pid[-1], closed) + + def test_a_skipped_version_fails(self): + _, closed = self.report["revised_at"] + capture = self.source + for k, packet in enumerate(klv.packets(self.source)): + if k >= closed and klv.pid(packet) == PMT: + capture = replace(capture, k, revision.revise(TABLE, 3, [])) + self.assertFails("version", capture) + + def test_a_stale_table_after_the_revision_fails(self): + packets = klv.packets(self.source) + _, closed = self.report["revised_at"] + listing = packets[self.report["revised_at"][0]] + after = next(k for k in range(closed + 1, len(packets)) if klv.pid(packets[k]) == klv.NULL) + self.assertFails("layouts", replace(self.source, after, listing)) + + def test_a_packet_after_the_stream_was_dropped_fails(self): + packets = klv.packets(self.source) + _, closed = self.report["revised_at"] + stream = next(packet for packet in packets if klv.pid(packet) == revision.PID) + after = next(k for k in range(closed + 1, len(packets)) if klv.pid(packets[k]) == klv.NULL) + self.assertFails("window", replace(self.source, after, stream)) + + def test_a_dropped_pes_fails(self): + packets = klv.packets(self.source) + index = [k for k, packet in enumerate(packets) if klv.pid(packet) == revision.PID][3] + self.assertFails("pes", replace(self.source, index, NULL)) + + def test_a_capture_that_never_revised_fails(self): + self.assertFails("layouts", media()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/ts/run.sh b/test/ts/run.sh index cbca53d1473..58ded8b8552 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -26,6 +26,7 @@ # ./run.sh --fragment-pes # split generated clear PES headers across packets # ./run.sh --clockless sections # sections, pes, or audio-scte35 without video # ./run.sh --klv # add KLV and DVB AC-3, carried as whole PES, and grade them +# ./run.sh --revision # revise the PMT mid-stream, adding then dropping a KLV stream # ./run.sh --delay 2s # pass the exporter's --delay # ./run.sh --rate 7000000 # bufferbloat: 7 Mb/s to the subscriber behind a deep queue (moq-shaper) # ./run.sh --loss 10 # drop 10% of the relay's datagrams to the subscriber (moq-shaper) @@ -87,6 +88,7 @@ FULL="" # the same with audio, in a multiplex the source fills with no n FRAGMENT_PES="" # use following null slots for fragmented PES headers CLOCKLESS="" # generated programs clocked by audio or a dedicated PCR-only PID KLV="" # inject asynchronous and synchronous KLV beside DVB AC-3, and grade every PES +REVISION="" # revise the PMT twice mid-stream, and grade the export's revisions LOSS="" # percent of relay -> subscriber datagrams dropped at random by moq-shaper RATE="" # bits/s moq-shaper carries from the relay to the subscriber, behind QUEUE QUEUE="" # how long a datagram may wait for RATE before the path tail-drops it @@ -184,6 +186,10 @@ while [[ $# -gt 0 ]]; do KLV=1 shift ;; + --revision) + REVISION=1 + shift + ;; --delay) EXPORT=(--delay "$2") shift 2 @@ -286,6 +292,10 @@ if [[ -n "$KLV" && -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$FRAG echo "error: --klv generates its own clip and grades the demultiplexed export's PES" >&2 exit 1 fi +if [[ -n "$REVISION" && -n "$SOURCE$ANALYZE_ONLY$HRD$HEADROOM$LEAK$FULL$OPEN_GOP$FRAGMENT_PES$PAIR$LIVE$PASSTHROUGH$MPTS$KLV$CLOCKLESS" ]]; then + echo "error: --revision revises the generated closed-GOP clip and grades the demultiplexed export's PMT" >&2 + exit 1 +fi if [[ -n "$CLOCKLESS" ]]; then case "$CLOCKLESS" in audio-scte35 | sections | pes) ;; *) @@ -567,6 +577,12 @@ else -c:a aac -b:a 128k \ -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" fi +# A KLV stream joins the program a third of the way in and leaves it two thirds in, each +# change announced by a PMT revision, in the clip's null slots. +if [[ -n "$REVISION" ]]; then + mv "$SRC_TS" "$HARNESS_RUN/media.ts" + python3 "$DIR/revision.py" inject "$HARNESS_RUN/media.ts" "$SRC_TS" --report "$HARNESS_RUN/revision-injected.json" +fi if [[ -n "$FRAGMENT_PES" ]]; then python3 "$DIR/fragment_pes.py" "$SRC_TS" "$HARNESS_RUN/source-fragmented.ts" \ @@ -970,6 +986,17 @@ if [[ -n "$KLV" ]]; then fi fi +# The export announces each layout the source's PMT revised to, at the next version. +if [[ -n "$REVISION" ]]; then + echo + if ! python3 "$DIR/revision.py" grade "$SRC_TS" "$SUB_TS" --report-json "$HARNESS_RUN/revision.json"; then + echo >&2 + echo "error: PMT revision analysis failed (see round-trip logs below)" >&2 + dump_logs + exit 1 + fi +fi + # The delay's lead over the source's hides packets the export spends beyond the source's # until it runs out, so they are counted outright. if [[ -n "$FULL" ]]; then