From 79772dcb20a9d7ae6947e980cb30e241bd06a51c Mon Sep 17 00:00:00 2001 From: t0ms Date: Fri, 16 Oct 2026 20:24:43 +0100 Subject: [PATCH 01/11] feat(mux): cut passthrough groups and sections on the epoch's grid Fallback cuts fall on a one-second grid of the unwrapped PCR timeline the epoch places, and the M2ts section's random_access and mux_rate are judged per two-second window on the same timeline, so importers sharing an epoch publish the same groups and the same section wherever each joined. Co-authored-by: Claude Opus 5.5 --- quest/m2/README.md | 1 - quest/m2/epoch-anchor.md | 6 +- quest/m2/hop-aligned-import.md | 1 - quest/m2/ts-passthrough-epoch-cuts.md | 67 ----- rs/moq-cli/src/publish.rs | 50 ++-- rs/moq-mux/src/container/ts/mux_rate.rs | 197 +++++++++++++- rs/moq-mux/src/container/ts/passthrough.rs | 180 ++++++++----- .../src/container/ts/passthrough_test.rs | 247 +++++++++++++++--- 8 files changed, 550 insertions(+), 199 deletions(-) delete mode 100644 quest/m2/ts-passthrough-epoch-cuts.md diff --git a/quest/m2/README.md b/quest/m2/README.md index 0b4c6a04840..6b39f3e6ec9 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -26,7 +26,6 @@ ranked just below deterministic TS legs, which it requires. - [Release execution](/quest/m2/release/README.md) - publication, installed-artifact proof, cleanup, and downstream adoption after m1 - [P2P](/quest/m2/p2p/README.md) - application-selected peer sessions use shared priorities, routing, and same-epoch handover - [Epoch anchor](/quest/m2/epoch-anchor.md) - same-epoch importers each derive the catalog clock and timestamp era from the epoch and stay live, so they agree across a 33-bit wrap, a forward clock break, and a restart -- [Raw TS cuts on the epoch timeline](/quest/m2/ts-passthrough-epoch-cuts.md) - passthrough fallback cuts fall on the epoch's grid and the M2ts section derives from the epoch and content, so instances started apart publish identical groups and catalogs - [Same-epoch importers](/quest/m2/hop-aligned-import.md) - importers sharing one `--epoch` and fed one stream publish identical groups and timestamps, so failover between a redundant pair survives - [Hard-fork the binding generator](/quest/m2/uniffi-fork.md) - one moq-dev repo holds the chosen generator's core and every language backend, released under one tag and one pin - [Benchmark trend site](/quest/m2/bench-site.md) - a GitHub Pages site charts the binding, Rust, and JS nightly benchmarks over time diff --git a/quest/m2/epoch-anchor.md b/quest/m2/epoch-anchor.md index d6db4404c9e..efc119070ee 100644 --- a/quest/m2/epoch-anchor.md +++ b/quest/m2/epoch-anchor.md @@ -80,10 +80,8 @@ precondition. `Clock::wall_clock` never enters it. - Every catalog field an importer derives from its history rather than the content at hand derives from the epoch and the content instead (maintainer, - superseding #5412's adoption from the live catalog). In TS passthrough that - is the M2ts section's `random_access` and recorded `mux_rate`, owned by - [raw TS cuts](/quest/m2/ts-passthrough-epoch-cuts.md); audit the other - importers for more. + superseding #5412's adoption from the live catalog). TS passthrough's M2ts + section already does; audit the other importers for more. - A restart under the same epoch re-derives the same coordinate, including after a total outage, so the epoch keeps naming one timeline and resuming across the outage is a gap, not a stitch (maintainer, reversing #5412's diff --git a/quest/m2/hop-aligned-import.md b/quest/m2/hop-aligned-import.md index 81e686c5795..7b68bb78af2 100644 --- a/quest/m2/hop-aligned-import.md +++ b/quest/m2/hop-aligned-import.md @@ -76,7 +76,6 @@ stopping and restarting; repeat it with `--passthrough`. ## Required - [Epoch anchor](/quest/m2/epoch-anchor.md) - every importer derives one catalog clock and timestamp era from the epoch -- [Raw TS cuts on the epoch timeline](/quest/m2/ts-passthrough-epoch-cuts.md) - passthrough groups start where every instance cuts ## Related diff --git a/quest/m2/ts-passthrough-epoch-cuts.md b/quest/m2/ts-passthrough-epoch-cuts.md deleted file mode 100644 index 3f109700691..00000000000 --- a/quest/m2/ts-passthrough-epoch-cuts.md +++ /dev/null @@ -1,67 +0,0 @@ -# [S] Raw TS groups and their catalog section follow the epoch timeline - -## Goal - -Two `moq import ts --passthrough` instances sharing one `--epoch` and fed -one stream cut identical groups wherever each started, including on a -source that never marks random access and on a multi-program stream. -Objects already run from PCR to PCR, so only group starts differ: the -fallback cut counts `FALLBACK` from the current group's start, and before -the first group from when the importer began waiting, so on a source with no -random access points each importer keeps its own group phase for good. -Their catalogs' M2ts sections match too, though both fields depend on -history today: one non-random-access cut clears `random_access` for good, -and the recorded `mux_rate` is judged over windows that start at the -importer's first PCR. - -## Plan - -Decided 2026-10-13 by the maintainer: every group start is a function of -the content and the [epoch's timeline](/quest/m2/epoch-anchor.md), never of -when the importer started. - -- Fallback cuts fall on a grid of the epoch timeline, so an importer started - later first cuts where its partner did. A grid on the raw 27 MHz count will - not do: a 33-bit wrap is not a whole number of seconds, so such a grid - shifts by about 0.72 s at each wrap. -- An importer's first group starts only where a partner running since - before it would also cut: a random access point, a flagged forward break, - or the grid. -- Settle the grid rule so that a source marking random access more often - than `FALLBACK` is never cut away from its random access points, since any - other start publishes `random_access` false. -- Size cuts count from the group's start, so they follow once group starts - do. -- A cut is a position in the content, so under a shared epoch it follows - media time alone. Decided 2026-10-13 by the maintainer as an explicit - exception to timers advancing on max(wall, pts), because a wall-clock cut - would split a stalled instance from its partner. Rejected: keeping a wall - path. -- The M2ts section derives from the epoch and the content, never from the - live catalog or what came before the importer started (maintainer, - superseding #5412's adoption by a joiner). Today both fields are sticky - history. Settle the rule; recommended: each is a function of a window on - the epoch's grid (the rate measured over it, `random_access` true while its - cuts all fell on random access points), and an importer publishes its - catalog only once it has a whole window, so it never publishes a section - its partner did not. Rejected in #5412 and still out: conservative - constants under a shared epoch, which lose `random_access` and the rate - `--sync` uses. - -Tests: two instances started at different offsets into one capture publish -identical groups and objects from the later one's first group, and a -restarted instance rejoins identically. Cover a -source with no random access indicator, a multi-program stream, groups that -hit the size cap, the later instance started after a 33-bit wrap the -earlier crossed, and one started after the earlier made a fallback cut and -recorded a rate; their catalogs match. - -Public API: none. - -## Required - -- [Epoch anchor](/quest/m2/epoch-anchor.md) - the timeline the grid sits on - -## Related - -- [Same-epoch importers](/quest/m2/hop-aligned-import.md) - group sequences and the end-to-end pair diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 44c18fa97af..b26cc9b653a 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1187,24 +1187,32 @@ pub(crate) mod tests { #[tokio::test] async fn ts_passthrough_crosses_a_relay_through_a_flagged_jump() { const KYRION: &[u8] = include_bytes!("../../moq-mux/src/container/ts/test_data/kyrion_mpeg2av_ac3.ts"); - // The capture twice, the second copy's clock twenty seconds ahead of the first, its first - // PCR flagged as the discontinuity. Inside the track's max age, so the recording keeps - // every group. - const STEP: u64 = 20 * 90_000; - let mut second = KYRION.to_vec(); - let mut pcrs = second - .as_chunks_mut::<188>() - .0 - .iter_mut() - .filter(|pkt| pkt[3] & 0x20 != 0 && pkt[4] > 0 && pkt[5] & 0x10 != 0) - .peekable(); - pcrs.peek_mut().expect("a PCR")[5] |= 0x80; - for pkt in pcrs { - let base = pkt[6..11].iter().fold(0u64, |base, byte| base << 8 | u64::from(*byte)) >> 7; - let base = ((base + STEP) & ((1 << 33) - 1)) << 7 | u64::from(pkt[10] & 0x7f); - pkt[6..11].copy_from_slice(&base.to_be_bytes()[3..]); - } - let input = [KYRION, &second].concat(); + // The capture with its clock `seconds` ahead, its first PCR flagged as a discontinuity + // when `flagged`. + let shifted = |seconds: u64, flagged: bool| { + let mut copy = KYRION.to_vec(); + let mut pcrs = copy + .as_chunks_mut::<188>() + .0 + .iter_mut() + .filter(|pkt| pkt[3] & 0x20 != 0 && pkt[4] > 0 && pkt[5] & 0x10 != 0) + .peekable(); + if flagged { + pcrs.peek_mut().expect("a PCR")[5] |= 0x80; + } + for pkt in pcrs { + let base = pkt[6..11].iter().fold(0u64, |base, byte| base << 8 | u64::from(*byte)) >> 7; + let base = ((base + seconds * 90_000) & ((1 << 33) - 1)) << 7 | u64::from(pkt[10] & 0x7f); + pkt[6..11].copy_from_slice(&base.to_be_bytes()[3..]); + } + copy + }; + // The capture, carried straight on by a copy four seconds ahead so a whole catalog window + // follows the first group, then a copy twenty seconds past that one behind a flagged + // discontinuity. Inside the track's max age, so the recording keeps every group. + let lead = [KYRION, &shifted(4, false)].concat(); + let second = shifted(24, true); + let input = [&lead[..], &second].concat(); let _env = crate::test_env::EnvGuard::clear(&["MOQ_CONNECT", "MOQ_HOP"]); let _ = moq_tokio::crypto::install_default(); @@ -1263,10 +1271,10 @@ pub(crate) mod tests { .await .unwrap(); - // The catalog names the track once the first group is out, which the first copy holds; - // the jump is in the second. + // The catalog names the track once a whole window has followed the first group, which + // the lead holds; the jump is in the copy after it. const WAIT: Duration = Duration::from_secs(10); - decoder.decode_chunk(KYRION).unwrap(); + decoder.decode_chunk(&lead).unwrap(); let consumer = tokio::time::timeout(WAIT, remote.consume().routed_broadcast("pt.hang")) .await .unwrap() diff --git a/rs/moq-mux/src/container/ts/mux_rate.rs b/rs/moq-mux/src/container/ts/mux_rate.rs index 84d37401451..f55aad778ae 100644 --- a/rs/moq-mux/src/container/ts/mux_rate.rs +++ b/rs/moq-mux/src/container/ts/mux_rate.rs @@ -7,6 +7,10 @@ //! whose samples agree on one value publishes it, and a window that no longer //! agrees clears it, so a VBR source (whose samples never agree) records nothing //! and a rate change lands once rather than as churn. +//! +//! [`Windows`] measures the same rate over each window of a grid on the unwrapped +//! PCR timeline instead, carrying nothing from one window to the next, so importers +//! whose PCRs share a timeline measure the same rate for every window each saw whole. /// The PCR clock, in Hz. const PCR_HZ: u64 = 27_000_000; @@ -15,7 +19,7 @@ const PACKET_BITS: u64 = 188 * 8; /// The 33-bit base times 300 plus the 9-bit extension: the PCR wraps here. pub(super) const PCR_WRAP: u64 = (1 << 33) * 300; /// How much PCR time one window spans before it is judged. -const WINDOW: u64 = 2 * PCR_HZ; +pub(super) const WINDOW: u64 = 2 * PCR_HZ; /// Media time after which a source that has not completed a window publishes without a rate: /// twice the window, so a usable PCR clock always completes one first. pub(super) const UNMEASURED: std::time::Duration = std::time::Duration::from_secs(2 * WINDOW / PCR_HZ); @@ -35,6 +39,10 @@ const AGREEMENT: u64 = 20; /// How far a stable window must move before the published rate follows it, in /// thousandths, so measurement noise never touches the catalog. const DRIFT: u64 = 10; +/// The bits per second a [`Windows`] rate is rounded to: far coarser than a PCR +/// clock's jitter over a window, so a constant-rate source measures one value +/// window after window. +const STEP: u64 = 1_000; /// Packets over the PCR ticks they were paced across. #[derive(Clone, Copy, Default)] @@ -169,6 +177,107 @@ impl Meter { } } +/// The multiplex rate over each [`WINDOW`] of a grid on the unwrapped PCR timeline. +/// +/// Window `k` holds the PCR intervals that start in `[k * WINDOW, (k + 1) * WINDOW)`, +/// pooled into samples on a grid of [`SAMPLE`]. It is measured only if the intervals +/// since its start all arrived unbroken: no flagged jump, and none empty or longer +/// than [`MAX_INTERVAL`]. Its rate, rounded to [`STEP`], stands when every sample +/// agrees with it. +#[derive(Default)] +pub(super) struct Windows { + /// Packets since the last PCR, every PID and null stuffing included. + packets: u64, + /// The last PCR, unwrapped. + last: Option, + /// The PCR the unbroken run of intervals started at. + since: u64, + /// The window the last PCR falls in. + window: u64, + /// The window's samples, each with its index on the sample grid. + samples: Vec<(u64, Interval)>, +} + +/// A window the PCR clock has passed. +pub(super) struct Ended { + /// The window's index on the grid: it starts at `index * WINDOW` ticks. + pub index: u64, + /// The window's rate, if it was measured whole and its samples agree. + pub rate: Option, +} + +impl Windows { + /// One transport packet went by. + pub fn packet(&mut self) { + self.packets += 1; + } + + /// A PCR at `ticks` on the unwrapped timeline, carried by the packet just counted, + /// `broken` when a flagged jump brought it. Returns the window it ended, if it falls + /// in a later one; the windows a jump skips held nothing and end unreported. + pub fn pcr(&mut self, ticks: u64, broken: bool) -> Option { + let packets = std::mem::take(&mut self.packets); + let Some(last) = self.last.replace(ticks) else { + self.since = ticks; + self.window = ticks / WINDOW; + return None; + }; + let interval = Interval { + packets, + ticks: ticks.saturating_sub(last), + }; + if broken || interval.ticks == 0 || interval.ticks > MAX_INTERVAL { + self.since = ticks; + } else { + let index = last / SAMPLE; + match self.samples.last_mut() { + Some((at, sample)) if *at == index => sample.add(interval), + _ => self.samples.push((index, interval)), + } + } + if ticks / WINDOW == self.window { + return None; + } + let ended = Ended { + index: self.window, + rate: self.rate(&self.samples), + }; + self.window = ticks / WINDOW; + self.samples.clear(); + Some(ended) + } + + /// The rate of the window in progress, over its completed samples, if there are at + /// least two: for an input that ends before its first window does. + pub fn partial(&self) -> Option { + let in_progress = self.last.map(|last| last / SAMPLE); + let completed = match self.samples.last() { + Some((at, _)) if Some(*at) == in_progress => &self.samples[..self.samples.len() - 1], + _ => &self.samples[..], + }; + if completed.len() < 2 { + return None; + } + self.rate(completed) + } + + fn rate(&self, samples: &[(u64, Interval)]) -> Option { + if self.since > self.window * WINDOW { + return None; + } + let mut whole = Interval::default(); + samples.iter().for_each(|(_, s)| whole.add(*s)); + if whole.ticks == 0 { + return None; + } + let rate = whole.rate(); + samples + .iter() + .all(|(_, s)| s.agrees(rate)) + .then(|| (rate + STEP / 2) / STEP * STEP) + } +} + #[cfg(test)] mod test { use super::*; @@ -277,6 +386,92 @@ mod test { assert_eq!(meter.published(), None); } + /// Feed `windows` the PCRs `intervals` of (packets, ticks) lead to from `pcrs`'s last, + /// returning each window ended as (index, rate). + fn feed_windows( + windows: &mut Windows, + pcrs: &mut Vec, + intervals: impl IntoIterator, + ) -> Vec<(u64, Option)> { + let mut ended = Vec::new(); + for (packets, ticks) in intervals { + for _ in 0..packets { + windows.packet(); + } + let pcr = pcrs.last().map_or(0, |last| last + ticks); + pcrs.push(pcr); + ended.extend(windows.pcr(pcr, false).map(|window| (window.index, window.rate))); + } + ended + } + + #[test] + fn a_window_is_measured_alone() { + let mut first = Windows::default(); + let mut pcrs = Vec::new(); + let ended = feed_windows(&mut first, &mut pcrs, std::iter::repeat_n(CBR, PER_WINDOW * 5)); + assert_eq!(ended.len(), 5); + assert!(ended.iter().all(|&(_, rate)| rate == Some(2_500_000)), "{ended:?}"); + + // A meter started part-way through the second window measures the windows after it the same. + let mut second = Windows::default(); + let join = PER_WINDOW * 3 / 2; + let mut replay = vec![pcrs[join]]; + second.packet(); + assert!(second.pcr(pcrs[join], false).is_none()); + let joined = feed_windows( + &mut second, + &mut replay, + std::iter::repeat_n(CBR, PER_WINDOW * 5 - join - 1), + ); + assert_eq!(joined[0], (1, None), "the window it joined part-way through"); + assert_eq!(joined[1..], ended[2..]); + + // A file-paced source's samples never agree. + let mut vbr = Windows::default(); + let intervals = (0..PER_WINDOW as u64 * 3).map(|i| (if (i / 7) % 2 == 0 { 10 } else { 60 }, 714_701)); + let ended = feed_windows(&mut vbr, &mut Vec::new(), intervals); + assert!(ended.iter().all(|&(_, rate)| rate.is_none()), "{ended:?}"); + } + + #[test] + fn a_break_spoils_only_its_own_window() { + let mut windows = Windows::default(); + let mut pcrs = Vec::new(); + // Half a window, then a clock stalled past the longest interval, still inside window 0. + let stall = (100, MAX_INTERVAL + 1); + let intervals = std::iter::repeat_n(CBR, PER_WINDOW / 2).chain(std::iter::once(stall)); + let mut ended = feed_windows(&mut windows, &mut pcrs, intervals); + ended.extend(feed_windows( + &mut windows, + &mut pcrs, + std::iter::repeat_n(CBR, PER_WINDOW * 2), + )); + // A flagged jump ten windows forward, landing part-way into window 13. + windows.packet(); + let jump = pcrs.last().unwrap() + 10 * WINDOW; + ended.extend(windows.pcr(jump, true).map(|window| (window.index, window.rate))); + pcrs.push(jump); + ended.extend(feed_windows( + &mut windows, + &mut pcrs, + std::iter::repeat_n(CBR, PER_WINDOW * 2), + )); + let measured = Some(2_500_000); + assert_eq!( + ended, + vec![ + (0, None), + (1, measured), + (2, measured), + (3, None), + (13, None), + (14, measured) + ], + "only the windows the run broke in, or began part-way through, go unmeasured" + ); + } + #[test] fn survives_the_pcr_wrap() { let mut meter = Meter { diff --git a/rs/moq-mux/src/container/ts/passthrough.rs b/rs/moq-mux/src/container/ts/passthrough.rs index 0743e08e1cc..a12e0557bc4 100644 --- a/rs/moq-mux/src/container/ts/passthrough.rs +++ b/rs/moq-mux/src/container/ts/passthrough.rs @@ -14,21 +14,35 @@ //! program has no video; //! - a PCR with `discontinuity_indicator` that jumps forward, since no group may hold a PCR //! discontinuity; -//! - the first PCR at least [`FALLBACK`] after the group's start, for a source that never marks -//! random access; +//! - the first PCR at or past a line of a grid every [`FALLBACK`] on the PCR timeline, when no +//! random access point or flagged jump fell in the [`FALLBACK`] before the line, for a source +//! that marks random access rarely or never; //! - the first object that would take the group past [`GROUP_BYTES_MAX`] or //! [`GROUP_OBJECTS_MAX`], so a long GOP cannot outgrow what a relay caches. Objects are never //! split to meet the bound. //! -//! Only the first of these makes a group a random access point, so any other start publishes -//! [`random_access`](hang::catalog::M2ts::random_access) false from then on, as does any group -//! carried while the layout allows no random access (more than one program or video stream). A +//! A random access point or a jump counts against the grid at the PCR before it (its own PCR, if +//! its packet carries one). Only the first kind of start makes a group a random access point. A //! group the PCR discontinuity starts is behind a break even when it is also a random access //! point. //! //! Bytes before the first group are dropped: until the PMT, a PCR, and a group start have all -//! been seen there is no group to put them in. The track is byte-identical to the input from its -//! first group on, and only from there. Bytes off the 188-byte packet grid are dropped too. +//! been seen there is no group to put them in. A jump before the first group starts none, and a +//! line of the grid starts it only once the importer has watched for random access through the +//! whole [`FALLBACK`] before the line. The track is byte-identical to the input from its first +//! group on, and only from there. Bytes off the 188-byte packet grid are dropped too. +//! +//! # The catalog section +//! +//! The section is written for each window of a grid every two seconds on the PCR timeline, once +//! the clock passes it, and only for windows that start at or after the importer's first group: +//! the catalog lists the track from the first such window on, or at the end of the input if that +//! comes first. [`random_access`](hang::catalog::M2ts::random_access) is true when every group +//! the window started is a random access point and the layout allowed random access throughout it +//! (one program, one video stream); [`mux_rate`](hang::catalog::M2ts::mux_rate) is the rate the +//! window was paced at, when the PCRs arrived unbroken across it and agree on one. An input that +//! ends before its first window publishes the random access of everything it carried and the rate +//! of the window in progress. //! //! # Timestamps //! @@ -48,10 +62,11 @@ //! one before ends the import: [`PcrRewindError`] without the flag, [`PcrRestartError`] with it, //! since a signalled restart is new content and needs a new broadcast. So does a jump that carries //! the PCR more than half a wrap (about 13.25 hours) from the clock's reading, since it lands a -//! wrap back. The timeline therefore depends only on the bytes and the clock, and two importers of -//! one feed whose timebases one shared epoch placed (see -//! [`Timebase::place`](crate::catalog::Timebase::place)) publish identical objects wherever each -//! started. Objects timed past the last PCR before a flagged jump smaller than their lead can land +//! wrap back. The timeline therefore depends only on the bytes and the clock, and the grids above +//! lie on it, so two importers of one feed whose timebases one shared epoch placed (see +//! [`Timebase::place`](crate::catalog::Timebase::place)) publish identical objects, groups and +//! sections wherever each started, from the later one's first group on (group sequence numbers +//! aside). Objects timed past the last PCR before a flagged jump smaller than their lead can land //! after the first ones behind it; if one of them starts a group, the group behind the break would //! start before it, and the import ends there. @@ -61,7 +76,7 @@ use bytes::BytesMut; use mpeg2ts::ts::{Pid, TsPacket}; use super::import::{Framer, PatReader, SectionReassembler, adaptation_valid, discontinuity_indicator, pcr}; -use super::mux_rate::{Meter, PCR_WRAP}; +use super::mux_rate::{PCR_WRAP, WINDOW, Windows}; use super::psi::{self, Pmt}; /// The PCR clock, in Hz. @@ -73,7 +88,8 @@ const PCR: crate::catalog::Wrap = crate::catalog::Wrap { period: PCR_WRAP, }; -/// The longest a group runs without a random access point before a PCR starts the next. +/// The spacing of the grid that cuts a source without frequent random access points, so no group +/// runs much past two spacings. pub const FALLBACK: std::time::Duration = std::time::Duration::from_secs(1); const FALLBACK_TICKS: u64 = PCR_HZ; @@ -179,8 +195,8 @@ pub struct PcrGapError { /// timestamped. pub struct Passthrough { catalog: crate::catalog::Producer, - /// Held until the first group is written, so the first catalog already names the track with - /// its final `randomAccess`. + /// Held until the section is first written, so the first catalog already describes a whole + /// window. reservation: Option>, /// Puts every PCR in its era and shifts every object's PCR time onto the catalog clock, /// anchored by the first PCR. @@ -219,19 +235,26 @@ pub struct Passthrough { /// A `discontinuity_indicator` on the clock PID, waiting for the PCR that it flags (normally /// in the same packet). pending_break: bool, - /// Before the first group: the PCR time the importer started waiting for a group start at. - waiting_since: Option, + /// The first PCR at which the clock's program was mapped: every random access point since + /// has been seen. + watching_since: Option, + /// The PCR the latest random access point or flagged jump counts against the grid at. + last_mark: Option, /// The object being filled, from its first packet. open: Option, /// Closed objects waiting on the next PCR for their timestamps. queue: VecDeque, group: Option, + /// The PCR the first group started at. + first_mark: Option, /// Whether every group so far started at a random access point, and the layout allowed random - /// access throughout. + /// access throughout: the section of an input that ends before its first window. all_random_access: bool, + /// The same, for the window in progress. + window_random_access: bool, - mux_rate: Meter, + windows: Windows, } /// Owns the `m2ts` catalog section once written, removing it however the track ends, as the @@ -273,6 +296,8 @@ struct Sample { struct Object { /// Where the object's first byte sits in the input. at: u64, + /// The PCR at or before the object's first packet, which places a group it starts on the grids. + mark: u64, payload: BytesMut, /// Why this object starts a group, if it does. start: Option, @@ -290,9 +315,8 @@ enum Start { Full, } +#[derive(Default)] struct Group { - /// The PCR time of the group's first byte. - ticks: u64, bytes: u64, objects: usize, } @@ -329,12 +353,15 @@ impl Passthrough { mapped: false, timebase: None, pending_break: false, - waiting_since: None, + watching_since: None, + last_mark: None, open: None, queue: VecDeque::new(), group: None, + first_mark: None, all_random_access: true, - mux_rate: Meter::default(), + window_random_access: false, + windows: Windows::default(), }) } @@ -375,6 +402,9 @@ impl Passthrough { } self.close_object(); self.drain_queue()?; + if self.section.is_none() && self.group.is_some() { + self.record(self.single && self.all_random_access, self.windows.partial())?; + } self.reservation = None; self.media.finish()?; Ok(()) @@ -461,7 +491,7 @@ impl Passthrough { { self.pmt_packet(pmt_pid, pkt)?; } - self.mux_rate.packet(); + self.windows.packet(); // A packet the demodulator flagged corrupt, or whose adaptation field overruns it, is // carried like any other, but nothing is read from it. @@ -481,6 +511,7 @@ impl Passthrough { } if readable && self.access_pid == Some(pid) && random_access_indicator(pkt) && self.can_start() { start = Some(Start::RandomAccess); + self.last_mark = self.last_pcr(); } if start.is_some() || (clock && self.started()) { @@ -491,6 +522,7 @@ impl Passthrough { payload.extend_from_slice(pkt); self.open = Some(Object { at, + mark: self.last_pcr().expect("an object follows a PCR"), payload, start, discontinuity, @@ -525,10 +557,9 @@ impl Passthrough { raw, samples: VecDeque::from([Sample { at, ticks }]), }); - self.waiting_since = self.mapped.then_some(ticks); - if self.mux_rate.pcr(raw) { - self.record()?; - } + self.watching_since = self.mapped.then_some(ticks); + self.windows.pcr(ticks, false); + self.window_random_access = self.single; return Ok(None); }; @@ -551,16 +582,10 @@ impl Passthrough { raw, samples: VecDeque::from([Sample { at, ticks }]), }); - if self.mux_rate.discontinuity() { - self.record()?; - } - if self.mux_rate.pcr(raw) { - self.record()?; - } - // The jump is not elapsed time, so the wait for the first group starts over. - if self.waiting_since.is_some() { - self.waiting_since = Some(ticks); - } + // An importer already running cuts here, so the jump counts against the grid even + // before this one's first group. + self.last_mark = Some(ticks); + self.end_window(ticks, true)?; return Ok(self.started().then_some(Start::Discontinuity)); } @@ -570,18 +595,39 @@ impl Passthrough { timebase.samples.pop_front(); } self.drain_queue()?; - if self.mux_rate.pcr(raw) { - self.record()?; + self.end_window(ticks, false)?; + if self.mapped { + self.watching_since.get_or_insert(ticks); } - let since = match &self.group { - Some(group) => Some(group.ticks), - None if self.mapped => Some(*self.waiting_since.get_or_insert(ticks)), - None => None, + // The first PCR at or past a line of the grid cuts there, unless a random access point or + // a jump fell in the spacing before the line. Before the first group, only a line whose + // whole spacing was watched may: an importer running since before would cut there too. + let line = ticks / FALLBACK_TICKS * FALLBACK_TICKS; + let crossed = last < line; + let marked = self.last_mark.is_some_and(|mark| mark + FALLBACK_TICKS >= line); + let watched = self.started() || self.watching_since.is_some_and(|since| since + FALLBACK_TICKS <= line); + Ok((crossed && !marked && watched && self.can_start()).then_some(Start::Fallback)) + } + + /// The section for the window a PCR at `ticks` ended, if it ended one: written once the window + /// starts at or after the first group, since every group start and layout change in it is + /// then one an importer running since before saw the same way. + fn end_window(&mut self, ticks: u64, broken: bool) -> anyhow::Result<()> { + let Some(ended) = self.windows.pcr(ticks, broken) else { + return Ok(()); }; - Ok(since - .filter(|since| ticks.saturating_sub(*since) >= FALLBACK_TICKS && self.can_start()) - .map(|_| Start::Fallback)) + let random_access = std::mem::replace(&mut self.window_random_access, self.single); + if self.first_mark.is_some_and(|mark| mark <= ended.index * WINDOW) { + self.record(random_access, ended.rate)?; + } + Ok(()) + } + + /// The PCR the last packet followed, on the unwrapped timeline. + fn last_pcr(&self) -> Option { + let timebase = self.timebase.as_ref()?; + timebase.samples.back().map(|sample| sample.ticks) } /// Whether a group may start here: once the first one has, always; before it, only once the @@ -628,12 +674,10 @@ impl Passthrough { } if start != Start::RandomAccess || !self.single { self.all_random_access = false; + self.window_random_access = false; } - self.group = Some(Group { - ticks, - bytes: 0, - objects: 0, - }); + self.first_mark.get_or_insert(object.mark); + self.group = Some(Group::default()); } // Only an object that starts a group is ever opened before the first one. @@ -646,25 +690,19 @@ impl Passthrough { payload: object.payload.freeze(), keyframe: start.is_some(), })?; - if start.is_some() { - self.record()?; - self.reservation = None; - } Ok(()) } - /// Write the section to the catalog if it changed. Nothing is written before the first group. - fn record(&mut self) -> anyhow::Result<()> { - if self.group.is_none() && self.section.is_none() { - return Ok(()); - } + /// Write the section to the catalog if it changed, and let the catalog publish. + fn record(&mut self, random_access: bool, mux_rate: Option) -> anyhow::Result<()> { let mut section = hang::catalog::M2ts::new(self.media.name()); - section.random_access = self.single && self.all_random_access; - section.mux_rate = self.mux_rate.published(); + section.random_access = random_access; + section.mux_rate = mux_rate; if self.section.as_ref().is_some_and(|written| written.written == section) { return Ok(()); } self.catalog.mutate(|catalog| catalog.m2ts = Some(section.clone()))?; + self.reservation = None; match &mut self.section { Some(written) => written.written = section, None => { @@ -777,11 +815,19 @@ impl Passthrough { self.access_pid = access_pid; self.mapped = mapped; self.single = single; - // The group in progress now carries what no random access point covers, and it stays cached. - if !single && self.started() { - self.all_random_access = false; + if !single { + self.window_random_access = false; + // The group in progress now carries what no random access point covers. + if self.started() { + self.all_random_access = false; + } + } + // Unmapped, an importer that has not started misses random access points one that has + // sees, so its watch starts over. + if !mapped && !self.started() { + self.watching_since = None; } - self.record() + Ok(()) } } diff --git a/rs/moq-mux/src/container/ts/passthrough_test.rs b/rs/moq-mux/src/container/ts/passthrough_test.rs index 88d7afa5034..b39689c61c2 100644 --- a/rs/moq-mux/src/container/ts/passthrough_test.rs +++ b/rs/moq-mux/src/container/ts/passthrough_test.rs @@ -81,21 +81,6 @@ async fn import_onto( import_on(broadcast, catalog, timebase, data, chunk, pcr_pid).await } -/// [`import`] on a host whose wall clock reads `host`, its clock placed by a shared epoch at -/// `epoch`: PCR zero one wrap before it. -async fn import_placed( - data: &[u8], - epoch: std::time::SystemTime, - host: std::time::SystemTime, -) -> anyhow::Result { - crate::clock::fake_host_wall(host); - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default())?; - let timebase = catalog.timebase(); - timebase.place(moq_net::Timestamp::ZERO, epoch - Duration::from_micros(WRAP_MICROS))?; - import_on(broadcast, catalog, timebase, data, data.len(), None).await -} - /// [`import_onto`] on `timebase`, one of `catalog`'s. async fn import_on( broadcast: moq_net::broadcast::Producer, @@ -115,9 +100,17 @@ async fn import_on( passthrough.decode(chunk)?; } passthrough.finish()?; + read_back(&consumer, &name, &catalog).await +} +/// Every object of the finished track `name`, and the catalog as it stands. +async fn read_back( + consumer: &moq_net::broadcast::Consumer, + name: &str, + catalog: &crate::catalog::Producer, +) -> anyhow::Result { let mut track = consumer - .track(&name)? + .track(name)? .subscribe(moq_net::track::Subscription::default().with_max_delay(RECORDING_MAX_DELAY)) .await?; let mut objects = Vec::new(); @@ -361,21 +354,46 @@ async fn two_programs_on_one_clock_publish_no_random_access() { } } -/// Groups started while a second program was on air stay cached, so dropping back to one program -/// does not win random access back. +/// The window a second program was on air in publishes no random access, and the first window +/// carried whole on one program wins it back. #[tokio::test(start_paused = true)] -async fn random_access_lost_to_a_second_program_stays_lost() { +async fn random_access_returns_once_a_window_carries_one_program() { let mut mux = Mux::default(); mux.pat(&[(1, 0x20), (2, 0x30)]) .pmt(0x20, 1, 0x100, &[(0x1b, 0x100)], 0) .pmt(0x30, 2, 0x100, &[(0x1b, 0x200)], 0); paced(&mut mux, 0x100, 0, 30, 10, 4); program(&mut mux, 0x100); - paced(&mut mux, 0x100, 1_200 * MS, 30, 10, 4); + paced(&mut mux, 0x100, 1_200 * MS, 100, 10, 4); let data = mux.bytes(); + let cut = pcr_at(&data, 3_000 * MS); + let early = import(&data[..cut], cut, None).await.unwrap(); + assert!(!early.section().random_access, "the window with two programs"); let imported = import(&data, data.len(), None).await.unwrap(); - assert!(imported.groups().len() >= 6); - assert!(!imported.section().random_access); + assert!(imported.groups().len() >= 12); + assert!(imported.section().random_access, "the window after it"); +} + +/// The section waits for the first window that starts at or after the first group, so the catalog +/// first lists the track with a whole window behind it. +#[tokio::test(start_paused = true)] +async fn the_catalog_waits_for_a_whole_window() { + let mut mux = Mux::default(); + program(&mut mux, 0x100); + // The first group starts at 0.52 s, so the window from 2 s is the first after it. + paced(&mut mux, 0x100, 520 * MS, 120, 10, 4); + let data = mux.bytes(); + let end = pcr_at(&data, 4_000 * MS); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut passthrough = Passthrough::new(broadcast.clone(), catalog.reserve()).unwrap(); + passthrough.decode(&data[..end]).unwrap(); + assert!(catalog.snapshot().m2ts.is_none(), "listed before a whole window"); + passthrough.decode(&data[end..end + 188]).unwrap(); + let section = catalog.snapshot().m2ts.expect("listed once the window passed"); + assert!(section.random_access); + // 250 packets a window, and 5 random access points: 255 * 1504 bits over two seconds. + assert_eq!(section.mux_rate, Some(192_000)); } #[tokio::test(start_paused = true)] @@ -528,11 +546,15 @@ async fn objects_after_the_last_pcr_extrapolate_its_rate() { /// The section describes the track for as long as the importer lives, however the track ends. #[tokio::test(start_paused = true)] async fn the_section_goes_with_the_importer() { + let mut mux = Mux::default(); + program(&mut mux, 0x100); + paced(&mut mux, 0x100, 0, 75, 10, 4); + let data = mux.bytes(); for abort in [false, true] { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); let mut passthrough = Passthrough::new(broadcast.clone(), catalog.reserve()).unwrap(); - passthrough.decode(KYRION_H264).unwrap(); + passthrough.decode(&data).unwrap(); assert!(catalog.snapshot().m2ts.is_some(), "abort {abort}"); if abort { passthrough.abort(moq_net::Error::Cancel); @@ -613,30 +635,87 @@ fn pcr_at(data: &[u8], ticks: u64) -> usize { packet.expect("a packet carrying that PCR") * 188 } -/// Import `data` placed by `epoch` from its first byte, and again from the program and then the -/// packet carrying PCR `join`, three seconds later on a host clock 7 ms ahead. The second must -/// publish exactly the first's objects from its first group on. +/// A passthrough importer whose clock a shared epoch at `epoch` placed: PCR zero one wrap before it. +struct Placed { + broadcast: moq_net::broadcast::Consumer, + catalog: crate::catalog::Producer, + passthrough: Passthrough, + name: String, +} + +impl Placed { + fn new(epoch: std::time::SystemTime) -> Self { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let timebase = catalog.timebase(); + timebase + .place(moq_net::Timestamp::ZERO, epoch - Duration::from_micros(WRAP_MICROS)) + .unwrap(); + let passthrough = Passthrough::new(broadcast.clone(), timebase.reserve()).unwrap(); + Self { + broadcast: broadcast.consume(), + catalog, + name: passthrough.track().to_string(), + passthrough, + } + } + + async fn finish(mut self) -> Imported { + self.passthrough.finish().unwrap(); + read_back(&self.broadcast, &self.name, &self.catalog).await.unwrap() + } +} + +/// Whether each object starts a group. +fn starts(imported: &Imported) -> Vec { + let objects = &imported.objects; + (0..objects.len()) + .map(|i| i == 0 || objects[i].group != objects[i - 1].group) + .collect() +} + +/// Import `data` placed by one epoch from its first byte, and again, on a host clock 7 ms ahead, +/// from the program and then the packet carrying PCR `join`, the two fed in step. From the second's +/// first group on, both must publish the same objects in the same groups, and from the second's +/// first section on, the same section after every PCR. async fn import_twice_placed(data: &[u8], prefix: usize, join: u64) -> (Imported, Imported) { let epoch = epoch_time(); - let start = epoch + Duration::from_secs(600); - let first = import_placed(data, epoch, start).await.unwrap(); - let mut joined = data[..prefix].to_vec(); - joined.extend_from_slice(&data[pcr_at(data, join)..]); - tokio::time::advance(Duration::from_secs(3)).await; - let host = start + Duration::from_millis(3_007); - let second = import_placed(&joined, epoch, host).await.unwrap(); + let hosts = [epoch + Duration::from_secs(600), epoch + Duration::from_millis(600_007)]; + let split = pcr_at(data, join); + crate::clock::fake_host_wall(hosts[0]); + let mut first = Placed::new(epoch); + first.passthrough.decode(&data[..split]).unwrap(); + crate::clock::fake_host_wall(hosts[1]); + let mut second = Placed::new(epoch); + second.passthrough.decode(&data[..prefix]).unwrap(); + let mut described = false; + for pkt in data[split..].chunks(188) { + crate::clock::fake_host_wall(hosts[0]); + first.passthrough.decode(pkt).unwrap(); + crate::clock::fake_host_wall(hosts[1]); + second.passthrough.decode(pkt).unwrap(); + let Some(pcr) = header(pkt).pcr else { continue }; + let section = second.catalog.snapshot().m2ts; + described |= section.is_some(); + if described { + assert_eq!(section, first.catalog.snapshot().m2ts, "the sections after PCR {pcr}"); + } + } + assert!(described, "the joiner described no window"); + let (first, second) = (first.finish().await, second.finish().await); assert_eq!(first.catalog.clock, Some(epoch_clock(epoch))); assert_eq!(second.catalog.clock, first.catalog.clock); assert!(!second.objects.is_empty(), "the joiner published nothing"); - let tail = &first.objects[first.objects.len() - second.objects.len()..]; - for (index, (first, second)) in tail.iter().zip(&second.objects).enumerate() { + let skip = first.objects.len() - second.objects.len(); + for (index, (first, second)) in first.objects[skip..].iter().zip(&second.objects).enumerate() { assert_eq!( (first.micros, &first.payload), (second.micros, &second.payload), "object {index} of the importer that joined" ); } + assert_eq!(starts(&first)[skip..], starts(&second), "group starts"); (first, second) } @@ -649,7 +728,7 @@ async fn importers_sharing_an_epoch_publish_identical_objects_across_a_wrap() { program(&mut mux, 0x100); let prefix = mux.out.len(); let start = WRAP - 2_000 * MS; - for i in 0..150 { + for i in 0..250 { mux.packet(0x100, Flags::pcr((start + i * 40 * MS) % WRAP)); if i % 10 == 0 { mux.packet(0x100, ACCESS); @@ -674,13 +753,107 @@ async fn importers_sharing_an_epoch_agree_across_a_flagged_jump() { // Flagged three seconds on from the PCR due a second short of the wrap, so the jump crosses it. mux.packet(0x100, Flags::pcr(2_000 * MS).discontinuity()); mux.fill(0x101, 4); - paced(&mut mux, 0x100, 2_040 * MS, 50, 10, 4); + paced(&mut mux, 0x100, 2_040 * MS, 150, 10, 4); let data = mux.bytes(); let (first, _) = import_twice_placed(&data, prefix, 3_000 * MS).await; let micros: Vec = first.objects.iter().map(|object| object.micros).collect(); assert!(micros.windows(2).all(|pair| pair[0] <= pair[1]), "{micros:?}"); } +/// Importers placed by one epoch cut the same groups and describe them alike wherever the second +/// joined, as a restarted one does: on random access points rarer than the grid, on none, and on a +/// clock two programs share. +/// Every join falls after a cut on the grid, and the later ones after the first importer recorded a +/// rate. +#[tokio::test(start_paused = true)] +async fn importers_sharing_an_epoch_cut_the_same_groups() { + // Random access every `gop` PCRs on program 1's video, and how many programs share the clock. + for (gop, programs) in [(36, 1), (0, 1), (36, 2)] { + let mut mux = Mux::default(); + match programs { + 1 => program(&mut mux, 0x100), + _ => { + mux.pat(&[(1, 0x20), (2, 0x30)]) + .pmt(0x20, 1, 0x100, &[(0x1b, 0x100)], 0) + .pmt(0x30, 2, 0x100, &[(0x1b, 0x200)], 0); + } + } + let prefix = mux.out.len(); + for i in 0..400 { + mux.packet(0x100, Flags::pcr(i * 40 * MS)); + if gop != 0 && i % gop == 0 { + mux.packet(0x100, ACCESS); + } + if programs == 2 && i % 36 == 18 { + mux.packet(0x200, ACCESS); + } + mux.fill(0x101, 4); + } + let data = mux.bytes(); + for join in [2_520, 4_720, 7_360] { + let (first, _) = import_twice_placed(&data, prefix, join * MS).await; + let fallback = first + .groups() + .iter() + .any(|group| !header(&group[0].payload).random_access); + assert!(fallback, "gop {gop}: the grid cut"); + if gop == 0 || programs == 2 { + assert!(!first.section().random_access, "gop {gop}, {programs} programs"); + } + assert!(first.section().mux_rate.is_some(), "gop {gop}: a rate was recorded"); + } + } +} + +/// Importers placed by one epoch split groups at the object ceiling alike, the ceiling counted +/// from a group start both made. +#[tokio::test(start_paused = true)] +async fn importers_sharing_an_epoch_split_full_groups_alike() { + let mut mux = Mux::default(); + program(&mut mux, 0x100); + let prefix = mux.out.len(); + mux.packet(0x100, Flags::pcr(0).random_access()); + // A PCR in every packet, 200 µs apart: the ceiling fills before each line of the grid. + for i in 1..60_000 { + mux.packet(0x100, Flags::pcr(i * 5_400)); + } + let data = mux.bytes(); + let (first, second) = import_twice_placed(&data, prefix, 3_300 * MS).await; + let full = second + .groups() + .iter() + .filter(|group| group.len() == GROUP_OBJECTS_MAX) + .count(); + assert!(full >= 3, "{full} groups at the ceiling"); + assert!(!first.section().random_access); +} + +/// The grid lies on the timeline the epoch places, so it runs on regularly across a PCR wrap that +/// would shift a grid of raw ticks by about 0.72 s, and an importer joining after the wrap cuts on +/// it too. +#[tokio::test(start_paused = true)] +async fn the_grid_runs_on_across_a_wrap() { + const WRAP: u64 = (1 << 33) * 300; + let mut mux = Mux::default(); + program(&mut mux, 0x100); + let prefix = mux.out.len(); + let start = WRAP - 3_000 * MS; + for i in 0..300 { + mux.packet(0x100, Flags::pcr((start + i * 40 * MS) % WRAP)); + mux.fill(0x101, 4); + } + let data = mux.bytes(); + let (first, _) = import_twice_placed(&data, prefix, 2_000 * MS).await; + let groups = first.groups(); + assert!(groups.len() >= 10, "{} groups", groups.len()); + for group in &groups { + let pcr = header(&group[0].payload).pcr.expect("a cut on the grid falls on a PCR"); + let unwrapped = if pcr >= start { pcr } else { pcr + WRAP }; + let past = unwrapped % (1_000 * MS); + assert!(past < 40 * MS, "a cut {} ms past the line", past / MS); + } +} + /// With every PCR in the era nearest the clock and nothing clamping it, a flagged jump forward by /// less than the object a random access point started already reached would start its group /// before that one: the import ends rather than shift the timeline off the source's clock. From 8376d69e7f80d5546e56485d1ec249b41ae7111f Mon Sep 17 00:00:00 2001 From: t0ms Date: Sun, 18 Oct 2026 11:02:59 +0100 Subject: [PATCH 02/11] docs(mux): describe m2ts grid cuts and per-window sections in the draft The draft and the catalog docs still described the fallback cut as one second from the group's start and randomAccess as latching false for the rest of the track. State the grid on the PCR count, whose wraps follow the catalog clock, the first-group watch, and that randomAccess and muxRate describe the latest two-second window. Pin where a window's edge falls: a layout change before the first PCR past a line counts in the window before. Co-authored-by: Claude Opus 5.5 --- doc/concept/hang.md | 7 +++--- drafts/draft-lcurley-moq-mpegts.md | 19 ++++++++------ rs/hang/src/catalog/m2ts.rs | 9 ++++--- rs/moq-mux/src/container/ts/passthrough.rs | 4 ++- .../src/container/ts/passthrough_test.rs | 25 +++++++++++++++++++ 5 files changed, 48 insertions(+), 16 deletions(-) diff --git a/doc/concept/hang.md b/doc/concept/hang.md index a676fcaba06..01dd4a27aad 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -170,9 +170,10 @@ service. The root `m2ts` section names the one track holding it: Each frame is a run of whole 188-byte packets in input order, timestamped on the multiplex's own PCR clock, which never steps back: a source whose clock -steps back is a new broadcast. `randomAccess` says whether every group begins -where a receiver can start decoding, and `muxRate` is the constant multiplex -rate in bits per second, absent for a variable-rate source. A player has +steps back is a new broadcast. `randomAccess` says whether every group of the +latest two-second window began where a receiver can start decoding, and +`muxRate` is that window's constant multiplex rate in bits per second, absent +for a variable-rate source. A player has nothing to decode here. `moq import ts --passthrough` writes this section (see [moq-cli](/bin/cli)); the encoding is in [draft-lcurley-moq-mpegts](/draft/moq-mpegts). diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index d8429b75c25..356d20cf843 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -278,10 +278,10 @@ type M2ts = { `track` names the one track carrying the multiplex, in the same broadcast as the catalog. A second multiplex is a second broadcast. -`randomAccess` says that every group starts at a random access point ({{m2ts-groups}}). +`randomAccess` says that every group of the latest window the section describes started at a random access point ({{m2ts-groups}}). A publisher MUST write it, and a consumer MUST read an absent `randomAccess` as false. -`muxRate` follows {{field-mux-rate}}, measured over every packet of the track. +`muxRate` follows {{field-mux-rate}}, measured over every packet of that window. The section records no program, PID, descriptor, or table: they all reach the consumer inside the packets, as the publisher received them. A track named here MUST NOT also be described as a rendition or in an `mpegts` section. @@ -302,11 +302,13 @@ A group starts at: - a packet with `random_access_indicator` set on the video PID of the clock's program (the first PID its PMT declares with a video `stream_type`), or on the clock PID when that program has no video; - a PCR on the clock PID with `discontinuity_indicator` set that steps forward ({{m2ts-timestamps}}), since a group MUST NOT contain a PCR discontinuity; -- otherwise, the first PCR at least one second of PCR time after the group's start; +- the first PCR at or past a whole second of the PCR count ({{m2ts-timestamps}}), unless a random access point or a forward jump fell in the second before it, each counted at the PCR before it (its own, if its packet carries one); - the frame that would take a group past a size bound the publisher chooses below what a relay caches of one group; frames are never split to meet it. -`randomAccess` is true only while the PAT lists one program, its PMT declares at most one video stream, and the clock PID is that PMT's `PCR_PID`, and only if every group so far has started at a `random_access_indicator`. -The first group that starts any other way, or that carries packets while that layout does not hold, republishes the section with `randomAccess` false for the rest of the track. +Before its first group, a publisher starts one at a second only once it has watched the whole second before it, the PMT of the clock's program read and a PCR seen, so it never cuts where a publisher running longer saw a random access point. + +The section describes two-second windows of the PCR count, each holding the PCR intervals that start in it, and is written as the count passes each one, from the first window that starts at or after the publisher's first group; an input that ends before that window is described by everything it carried. +For a window, `randomAccess` is true only if the PAT listed one program, its PMT declared at most one video stream, and the clock PID was that PMT's `PCR_PID` throughout it, and every group it started began at a `random_access_indicator`. With more programs or video streams the random access points can stagger, so a group start is not one for all of them. Both indicators sit in the adaptation field, which TS-level scrambling leaves in the clear, so a scrambled multiplex groups like its clear twin. @@ -318,11 +320,11 @@ A PCR gives the time of the byte holding the last bit of its `program_clock_refe A byte between two PCRs is placed at the rate between them; a byte past the last PCR of a timebase takes the rate of the interval before it, and one before the first PCR of a timebase takes the rate of the interval after it. PCRs are unwrapped into a 64-bit count of 27 MHz ticks, and timestamps never decrease. -A step forward of at most half the 2^33 × 300 tick period adds to the count, across the wrap included, so a wrap is not a discontinuity. -A PCR with `discontinuity_indicator` set whose step is forward by that measure adds its step too, after the bytes before it are timed at the rate before it, and the group it starts MUST follow hang's discontinuity marker ({{hang}}). +Each PCR takes the number of 2^33 × 300 tick wraps that lands it nearest the current time on the catalog clock ({{hang}}), so the count runs on across the wrap, which is not a discontinuity, and a PCR more than half a wrap ahead of that time lands a wrap back. +A PCR with `discontinuity_indicator` set that lands at or past the one before it adds its step too, after the bytes before it are timed at the rate before it, and the group it starts MUST follow hang's discontinuity marker ({{hang}}). A PCR that steps backwards MUST end the track with an error, flagged or not: a flagged one is a restart, which is new content, so a publisher that carries on publishes it as a new broadcast. -The timeline depends only on the bytes since the publisher's first PCR, so two publishers that start at the same packet of one input publish identical frames. +The count therefore depends only on the bytes and the catalog clock, so publishers of one input whose catalog clocks agree publish identical frames, group boundaries, and sections wherever each started, from the later one's first group on. ## Mapping onto MSFTS {#m2ts-msfts} Stripping each frame's timestamp leaves an {{msfts}} object of whole source packets, and the groups carry over unchanged. @@ -443,6 +445,7 @@ A broadcast demultiplexed from a DVB transport stream: video and audio described - Added the `m2ts` section, a transport stream carried whole, with its mapping onto MSFTS; the comparison uses MSFTS's `mpeg2ts` names. - Added the `pesPacket` record: each frame one whole PES packet, timestamped with its placement. A consumer that predates it drops the track rather than refusing the catalog. - A flagged PCR jump on an `m2ts` track follows hang's discontinuity marker rather than a skipped group sequence. +- An `m2ts` track's fallback cut falls on whole seconds of a PCR count whose wraps follow the catalog clock, and `randomAccess` and `muxRate` describe the latest two-second window rather than the whole track, so publishers sharing a clock cut and describe the multiplex alike. ## draft-lcurley-moq-mpegts-00 {:numbered="false"} diff --git a/rs/hang/src/catalog/m2ts.rs b/rs/hang/src/catalog/m2ts.rs index 6857f9ed319..5557ddcdf1b 100644 --- a/rs/hang/src/catalog/m2ts.rs +++ b/rs/hang/src/catalog/m2ts.rs @@ -18,8 +18,9 @@ pub struct M2ts { /// The track carrying the packets, published in the same broadcast as the catalog. pub track: String, - /// Whether every group starts at a random access point of the multiplex's one video stream, - /// so a subscriber can decode from the first object of any group. + /// Whether every group the publisher's latest two-second window started began at a random + /// access point of the multiplex's one video stream, so a subscriber can decode from the first + /// object of any of them. /// /// Only ever true for a single program with at most one video stream: programs or video /// streams sharing a clock can stagger their GOPs, leaving no packet that starts all of them. @@ -27,8 +28,8 @@ pub struct M2ts { #[serde(default)] pub random_access: bool, - /// The rate the source's PCR paced the whole multiplex at, in bits per second, null packets - /// included. Present only while the source holds a constant rate. + /// The rate the source's PCR paced the whole multiplex at over that window, in bits per + /// second, null packets included. Present only while the source holds a constant rate. #[serde(default)] pub mux_rate: Option, } diff --git a/rs/moq-mux/src/container/ts/passthrough.rs b/rs/moq-mux/src/container/ts/passthrough.rs index a12e0557bc4..a0b20477ed8 100644 --- a/rs/moq-mux/src/container/ts/passthrough.rs +++ b/rs/moq-mux/src/container/ts/passthrough.rs @@ -35,7 +35,9 @@ //! # The catalog section //! //! The section is written for each window of a grid every two seconds on the PCR timeline, once -//! the clock passes it, and only for windows that start at or after the importer's first group: +//! the clock passes it; a window holds the PCR intervals that start in it, so the packets up to +//! the first PCR past a line, and any layout change among them, count in the window before the +//! line. It is written only for windows that start at or after the importer's first group: //! the catalog lists the track from the first such window on, or at the end of the input if that //! comes first. [`random_access`](hang::catalog::M2ts::random_access) is true when every group //! the window started is a random access point and the layout allowed random access throughout it diff --git a/rs/moq-mux/src/container/ts/passthrough_test.rs b/rs/moq-mux/src/container/ts/passthrough_test.rs index b39689c61c2..7492f5e975e 100644 --- a/rs/moq-mux/src/container/ts/passthrough_test.rs +++ b/rs/moq-mux/src/container/ts/passthrough_test.rs @@ -374,6 +374,31 @@ async fn random_access_returns_once_a_window_carries_one_program() { assert!(imported.section().random_access, "the window after it"); } +/// A window holds the PCR intervals that start in it: a second program carried past a line, up to +/// the first PCR after it, counts in the window before the line, and the window from the line on +/// carries one program throughout. +#[tokio::test(start_paused = true)] +async fn a_layout_change_before_the_first_pcr_past_a_line_counts_before_it() { + let mut mux = Mux::default(); + mux.pat(&[(1, 0x20), (2, 0x30)]) + .pmt(0x20, 1, 0x100, &[(0x1b, 0x100)], 0) + .pmt(0x30, 2, 0x100, &[(0x1b, 0x200)], 0); + // The last PCR before the 2 s line is at 1.98 s; most of the interval after it, program 2's + // packets included, lies past the line, and the PAT drops program 2 before the 2.02 s PCR. + paced(&mut mux, 0x100, 20 * MS, 50, 10, 4); + mux.fill(0x200, 20); + program(&mut mux, 0x100); + paced(&mut mux, 0x100, 2_020 * MS, 60, 10, 4); + let data = mux.bytes(); + let end = pcr_at(&data, 4_020 * MS); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut passthrough = Passthrough::new(broadcast.clone(), catalog.reserve()).unwrap(); + passthrough.decode(&data[..end + 188]).unwrap(); + let section = catalog.snapshot().m2ts.expect("the window from 2 s has passed"); + assert!(section.random_access, "the window from the line carried one program"); +} + /// The section waits for the first window that starts at or after the first group, so the catalog /// first lists the track with a whole window behind it. #[tokio::test(start_paused = true)] From 4345aa4a45693f1b28b22cd7526d948fbc749391 Mon Sep 17 00:00:00 2001 From: t0ms Date: Sun, 18 Oct 2026 11:20:22 +0100 Subject: [PATCH 03/11] docs(mux): say when the m2ts section changes and which PID an audio-only program cuts on Co-authored-by: Claude Opus 5.5 --- drafts/draft-lcurley-moq-mpegts.md | 4 +++- rs/hang/src/catalog/m2ts.rs | 4 ++-- rs/moq-mux/src/container/ts/passthrough.rs | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index 356d20cf843..ca88c3088ed 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -307,7 +307,9 @@ A group starts at: Before its first group, a publisher starts one at a second only once it has watched the whole second before it, the PMT of the clock's program read and a PCR seen, so it never cuts where a publisher running longer saw a random access point. -The section describes two-second windows of the PCR count, each holding the PCR intervals that start in it, and is written as the count passes each one, from the first window that starts at or after the publisher's first group; an input that ends before that window is described by everything it carried. +The section describes two-second windows of the PCR count, each holding the PCR intervals that start in it. +As the count passes each window, from the first that starts at or after the publisher's first group, the section takes that window's values; a window whose values match the last one's leaves the catalog unchanged. +An input that ends before its first such window is described by everything it carried. For a window, `randomAccess` is true only if the PAT listed one program, its PMT declared at most one video stream, and the clock PID was that PMT's `PCR_PID` throughout it, and every group it started began at a `random_access_indicator`. With more programs or video streams the random access points can stagger, so a group start is not one for all of them. diff --git a/rs/hang/src/catalog/m2ts.rs b/rs/hang/src/catalog/m2ts.rs index 5557ddcdf1b..de558c02888 100644 --- a/rs/hang/src/catalog/m2ts.rs +++ b/rs/hang/src/catalog/m2ts.rs @@ -19,8 +19,8 @@ pub struct M2ts { pub track: String, /// Whether every group the publisher's latest two-second window started began at a random - /// access point of the multiplex's one video stream, so a subscriber can decode from the first - /// object of any of them. + /// access point of the multiplex's video stream, or of its clock PID when it has none, so a + /// subscriber can decode from the first object of any of them. /// /// Only ever true for a single program with at most one video stream: programs or video /// streams sharing a clock can stagger their GOPs, leaving no packet that starts all of them. diff --git a/rs/moq-mux/src/container/ts/passthrough.rs b/rs/moq-mux/src/container/ts/passthrough.rs index a0b20477ed8..b67f6387160 100644 --- a/rs/moq-mux/src/container/ts/passthrough.rs +++ b/rs/moq-mux/src/container/ts/passthrough.rs @@ -41,7 +41,7 @@ //! the catalog lists the track from the first such window on, or at the end of the input if that //! comes first. [`random_access`](hang::catalog::M2ts::random_access) is true when every group //! the window started is a random access point and the layout allowed random access throughout it -//! (one program, one video stream); [`mux_rate`](hang::catalog::M2ts::mux_rate) is the rate the +//! (one program, at most one video stream); [`mux_rate`](hang::catalog::M2ts::mux_rate) is the rate the //! window was paced at, when the PCRs arrived unbroken across it and agree on one. An input that //! ends before its first window publishes the random access of everything it carried and the rate //! of the window in progress. From 71024db4efd8bda878de55d2addfa4de82993b76 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 23:26:38 -0700 Subject: [PATCH 04/11] fix(mux): describe only windows that start after the first group's PCR A joiner whose first PCR lands exactly on a window line can start its first group at a random access point in the PCR interval where its partner already cut on the grid, and so described that window as random access the partner had lost. Describe a window only when it starts strictly after the PCR the first group follows, so every group start in it is one both importers made. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-mpegts.md | 2 +- rs/moq-mux/src/container/ts/passthrough.rs | 13 ++++---- .../src/container/ts/passthrough_test.rs | 30 ++++++++++++++++++- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index ca88c3088ed..a665f3feab6 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -308,7 +308,7 @@ A group starts at: Before its first group, a publisher starts one at a second only once it has watched the whole second before it, the PMT of the clock's program read and a PCR seen, so it never cuts where a publisher running longer saw a random access point. The section describes two-second windows of the PCR count, each holding the PCR intervals that start in it. -As the count passes each window, from the first that starts at or after the publisher's first group, the section takes that window's values; a window whose values match the last one's leaves the catalog unchanged. +As the count passes each window, from the first that starts after the PCR the publisher's first group follows, the section takes that window's values; a window whose values match the last one's leaves the catalog unchanged. An input that ends before its first such window is described by everything it carried. For a window, `randomAccess` is true only if the PAT listed one program, its PMT declared at most one video stream, and the clock PID was that PMT's `PCR_PID` throughout it, and every group it started began at a `random_access_indicator`. With more programs or video streams the random access points can stagger, so a group start is not one for all of them. diff --git a/rs/moq-mux/src/container/ts/passthrough.rs b/rs/moq-mux/src/container/ts/passthrough.rs index b67f6387160..567ad76557f 100644 --- a/rs/moq-mux/src/container/ts/passthrough.rs +++ b/rs/moq-mux/src/container/ts/passthrough.rs @@ -37,9 +37,9 @@ //! The section is written for each window of a grid every two seconds on the PCR timeline, once //! the clock passes it; a window holds the PCR intervals that start in it, so the packets up to //! the first PCR past a line, and any layout change among them, count in the window before the -//! line. It is written only for windows that start at or after the importer's first group: -//! the catalog lists the track from the first such window on, or at the end of the input if that -//! comes first. [`random_access`](hang::catalog::M2ts::random_access) is true when every group +//! line. It is written only for windows that start after the PCR the importer's first group +//! follows: the catalog lists the track from the first such window on, or at the end of the input +//! if that comes first. [`random_access`](hang::catalog::M2ts::random_access) is true when every group //! the window started is a random access point and the layout allowed random access throughout it //! (one program, at most one video stream); [`mux_rate`](hang::catalog::M2ts::mux_rate) is the rate the //! window was paced at, when the PCRs arrived unbroken across it and agree on one. An input that @@ -613,14 +613,15 @@ impl Passthrough { } /// The section for the window a PCR at `ticks` ended, if it ended one: written once the window - /// starts at or after the first group, since every group start and layout change in it is - /// then one an importer running since before saw the same way. + /// starts after the PCR the first group follows, since every group start and layout change in + /// it is then one an importer running since before saw the same way. A window starting on that + /// PCR is not: the partner may have cut earlier in the same PCR interval. fn end_window(&mut self, ticks: u64, broken: bool) -> anyhow::Result<()> { let Some(ended) = self.windows.pcr(ticks, broken) else { return Ok(()); }; let random_access = std::mem::replace(&mut self.window_random_access, self.single); - if self.first_mark.is_some_and(|mark| mark <= ended.index * WINDOW) { + if self.first_mark.is_some_and(|mark| mark < ended.index * WINDOW) { self.record(random_access, ended.rate)?; } Ok(()) diff --git a/rs/moq-mux/src/container/ts/passthrough_test.rs b/rs/moq-mux/src/container/ts/passthrough_test.rs index 7492f5e975e..8fddd8110ef 100644 --- a/rs/moq-mux/src/container/ts/passthrough_test.rs +++ b/rs/moq-mux/src/container/ts/passthrough_test.rs @@ -573,7 +573,7 @@ async fn objects_after_the_last_pcr_extrapolate_its_rate() { async fn the_section_goes_with_the_importer() { let mut mux = Mux::default(); program(&mut mux, 0x100); - paced(&mut mux, 0x100, 0, 75, 10, 4); + paced(&mut mux, 0x100, 0, 125, 10, 4); let data = mux.bytes(); for abort in [false, true] { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -853,6 +853,34 @@ async fn importers_sharing_an_epoch_split_full_groups_alike() { assert!(!first.section().random_access); } +/// A joiner whose first PCR lands exactly on a window's line misses the grid cut its partner made +/// there, so the first window it describes is the one after: describing that window would publish +/// random access the partner lost to the cut. +#[tokio::test(start_paused = true)] +async fn a_joiner_on_a_window_line_describes_the_window_after() { + const WINDOW: u64 = 2_000 * MS; + let mut mux = Mux::default(); + program(&mut mux, 0x100); + let prefix = mux.out.len(); + // The placed epoch puts these PCRs one wrap on, so `offset` lands every fiftieth on a line. + let offset = (WINDOW - WRAP % WINDOW) % WINDOW; + for i in 0..400 { + mux.packet(0x100, Flags::pcr(offset + i * 40 * MS)); + // No random access before 4 s, then one just after the PCR on each whole second. + if i >= 100 && i % 25 == 0 { + mux.packet(0x100, ACCESS); + } + mux.fill(0x101, 4); + } + let data = mux.bytes(); + let (first, second) = import_twice_placed(&data, prefix, offset + 4_000 * MS).await; + assert!( + first.section().random_access, + "every group from 6 s on is a random access point" + ); + assert_eq!(first.section(), second.section()); +} + /// The grid lies on the timeline the epoch places, so it runs on regularly across a PCR wrap that /// would shift a grid of raw ticks by about 0.72 s, and an importer joining after the wrap cuts on /// it too. From dea48930f63ad373eea191762f3cf1230a265a72 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 23:40:58 -0700 Subject: [PATCH 05/11] docs(mux): say what an m2ts section shorter than a window takes Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-mpegts.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index a665f3feab6..69af11e63ad 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -309,7 +309,7 @@ Before its first group, a publisher starts one at a second only once it has watc The section describes two-second windows of the PCR count, each holding the PCR intervals that start in it. As the count passes each window, from the first that starts after the PCR the publisher's first group follows, the section takes that window's values; a window whose values match the last one's leaves the catalog unchanged. -An input that ends before its first such window is described by everything it carried. +An input that ends before its first such window takes `randomAccess` from every group it carried and `muxRate` from the window in progress. For a window, `randomAccess` is true only if the PAT listed one program, its PMT declared at most one video stream, and the clock PID was that PMT's `PCR_PID` throughout it, and every group it started began at a `random_access_indicator`. With more programs or video streams the random access points can stagger, so a group start is not one for all of them. From 533ef900cfaaa7f04ea670453cd013886c4be63c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 00:01:44 -0700 Subject: [PATCH 06/11] fix(mux): spoil a window whose broken interval starts on its line An unflagged repeated PCR exactly on a two-second line left the run's start on the line, so the window holding that empty interval was still measured without its packets. Track whether the window in progress missed an interval instead of where the run began. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/mux_rate.rs | 30 ++++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/rs/moq-mux/src/container/ts/mux_rate.rs b/rs/moq-mux/src/container/ts/mux_rate.rs index f55aad778ae..2760052e69e 100644 --- a/rs/moq-mux/src/container/ts/mux_rate.rs +++ b/rs/moq-mux/src/container/ts/mux_rate.rs @@ -190,8 +190,9 @@ pub(super) struct Windows { packets: u64, /// The last PCR, unwrapped. last: Option, - /// The PCR the unbroken run of intervals started at. - since: u64, + /// Whether the window in progress missed an interval: one started in it broken, or + /// the run of intervals began after its start. + spoiled: bool, /// The window the last PCR falls in. window: u64, /// The window's samples, each with its index on the sample grid. @@ -218,7 +219,7 @@ impl Windows { pub fn pcr(&mut self, ticks: u64, broken: bool) -> Option { let packets = std::mem::take(&mut self.packets); let Some(last) = self.last.replace(ticks) else { - self.since = ticks; + self.spoiled = !ticks.is_multiple_of(WINDOW); self.window = ticks / WINDOW; return None; }; @@ -226,8 +227,9 @@ impl Windows { packets, ticks: ticks.saturating_sub(last), }; - if broken || interval.ticks == 0 || interval.ticks > MAX_INTERVAL { - self.since = ticks; + let bad = broken || interval.ticks == 0 || interval.ticks > MAX_INTERVAL; + if bad { + self.spoiled = true; } else { let index = last / SAMPLE; match self.samples.last_mut() { @@ -243,6 +245,8 @@ impl Windows { rate: self.rate(&self.samples), }; self.window = ticks / WINDOW; + // A broken interval reaching into this window covered its start. + self.spoiled = bad && !ticks.is_multiple_of(WINDOW); self.samples.clear(); Some(ended) } @@ -262,7 +266,7 @@ impl Windows { } fn rate(&self, samples: &[(u64, Interval)]) -> Option { - if self.since > self.window * WINDOW { + if self.spoiled { return None; } let mut whole = Interval::default(); @@ -472,6 +476,20 @@ mod test { ); } + #[test] + fn an_empty_interval_on_a_line_spoils_its_window() { + let mut windows = Windows::default(); + let mut pcrs = Vec::new(); + // Fifty intervals a window, so a PCR lands on every line, then that PCR repeated. + let paced = (44, WINDOW / 50); + let mut ended = feed_windows(&mut windows, &mut pcrs, std::iter::repeat_n(paced, 51)); + ended.extend(feed_windows(&mut windows, &mut pcrs, [(44, 0)])); + ended.extend(feed_windows(&mut windows, &mut pcrs, std::iter::repeat_n(paced, 100))); + let measured = ended[0].1; + assert!(measured.is_some()); + assert_eq!(ended, vec![(0, measured), (1, None), (2, measured)]); + } + #[test] fn survives_the_pcr_wrap() { let mut meter = Meter { From f621c6c857bbb1223d3047fb2e4f5afc305e166d Mon Sep 17 00:00:00 2001 From: t0ms Date: Mon, 19 Oct 2026 09:59:15 +0100 Subject: [PATCH 07/11] fix(mux): count a flagged first PCR as a jump against the grid A same-epoch importer joining on the retransmitted copy of a flagged PCR on a grid line left its jump mark empty, so it fallback-cut a second later where its partner, which saw the original, did not. The first PCR's flag now marks the grid like any later jump. Co-authored-by: Claude Opus 5.5 --- drafts/draft-lcurley-moq-mpegts.md | 1 + rs/moq-mux/src/container/ts/passthrough.rs | 12 ++++++--- .../src/container/ts/passthrough_test.rs | 26 ++++++++++++++++++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index 69af11e63ad..20c8298ba27 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -306,6 +306,7 @@ A group starts at: - the frame that would take a group past a size bound the publisher chooses below what a relay caches of one group; frames are never split to meet it. Before its first group, a publisher starts one at a second only once it has watched the whole second before it, the PMT of the clock's program read and a PCR seen, so it never cuts where a publisher running longer saw a random access point. +A flagged first PCR counts as a forward jump, since a publisher running longer counted that jump, or the one it retransmits, at the same PCR. The section describes two-second windows of the PCR count, each holding the PCR intervals that start in it. As the count passes each window, from the first that starts after the PCR the publisher's first group follows, the section takes that window's values; a window whose values match the last one's leaves the catalog unchanged. diff --git a/rs/moq-mux/src/container/ts/passthrough.rs b/rs/moq-mux/src/container/ts/passthrough.rs index 567ad76557f..32ad35b4ea9 100644 --- a/rs/moq-mux/src/container/ts/passthrough.rs +++ b/rs/moq-mux/src/container/ts/passthrough.rs @@ -22,9 +22,10 @@ //! split to meet the bound. //! //! A random access point or a jump counts against the grid at the PCR before it (its own PCR, if -//! its packet carries one). Only the first kind of start makes a group a random access point. A -//! group the PCR discontinuity starts is behind a break even when it is also a random access -//! point. +//! its packet carries one), and a flagged first PCR counts as a jump, since an importer running +//! longer counted that one, or the one it retransmits. Only the first kind of start makes a group +//! a random access point. A group the PCR discontinuity starts is behind a break even when it is +//! also a random access point. //! //! Bytes before the first group are dropped: until the PMT, a PCR, and a group start have all //! been seen there is no group to put them in. A jump before the first group starts none, and a @@ -560,6 +561,11 @@ impl Passthrough { samples: VecDeque::from([Sample { at, ticks }]), }); self.watching_since = self.mapped.then_some(ticks); + // A flagged first PCR is a jump an importer running since before counted against the + // grid, the original if this one is a retransmitted copy, so it counts here too. + if flagged { + self.last_mark = Some(ticks); + } self.windows.pcr(ticks, false); self.window_random_access = self.single; return Ok(None); diff --git a/rs/moq-mux/src/container/ts/passthrough_test.rs b/rs/moq-mux/src/container/ts/passthrough_test.rs index 8fddd8110ef..96c5b34942a 100644 --- a/rs/moq-mux/src/container/ts/passthrough_test.rs +++ b/rs/moq-mux/src/container/ts/passthrough_test.rs @@ -704,9 +704,13 @@ fn starts(imported: &Imported) -> Vec { /// first group on, both must publish the same objects in the same groups, and from the second's /// first section on, the same section after every PCR. async fn import_twice_placed(data: &[u8], prefix: usize, join: u64) -> (Imported, Imported) { + import_twice_placed_from(data, prefix, pcr_at(data, join)).await +} + +/// [`import_twice_placed`], the second importer joining at byte `split`. +async fn import_twice_placed_from(data: &[u8], prefix: usize, split: usize) -> (Imported, Imported) { let epoch = epoch_time(); let hosts = [epoch + Duration::from_secs(600), epoch + Duration::from_millis(600_007)]; - let split = pcr_at(data, join); crate::clock::fake_host_wall(hosts[0]); let mut first = Placed::new(epoch); first.passthrough.decode(&data[..split]).unwrap(); @@ -881,6 +885,26 @@ async fn a_joiner_on_a_window_line_describes_the_window_after() { assert_eq!(first.section(), second.section()); } +/// A joiner whose first PCR is the retransmitted copy of a flagged jump on a line of the grid counts +/// the jump there as its partner did, so neither cuts on the line after. +#[tokio::test(start_paused = true)] +async fn a_joiner_on_a_retransmitted_jump_counts_it_against_the_grid() { + const LINE: u64 = 1_000 * MS; + let mut mux = Mux::default(); + program(&mut mux, 0x100); + let prefix = mux.out.len(); + // The placed epoch puts these PCRs one wrap on, so `offset` lands every 25th on a line. + let offset = (LINE - WRAP % LINE) % LINE; + paced(&mut mux, 0x100, offset, 100, 0, 4); + let jump = offset + 11 * LINE; + mux.packet(0x100, Flags::pcr(jump).discontinuity()); + mux.out.extend_from_within(mux.out.len() - 188..); + mux.fill(0x101, 4); + paced(&mut mux, 0x100, jump + 40 * MS, 250, 0, 4); + let data = mux.bytes(); + import_twice_placed_from(&data, prefix, pcr_at(&data, jump) + 188).await; +} + /// The grid lies on the timeline the epoch places, so it runs on regularly across a PCR wrap that /// would shift a grid of raw ticks by about 0.72 s, and an importer joining after the wrap cuts on /// it too. From b289ac99dac4fbcece248887f5feba29efcc6068 Mon Sep 17 00:00:00 2001 From: t0ms Date: Sat, 17 Oct 2026 15:57:09 +0100 Subject: [PATCH 08/11] feat(mux): number same-epoch groups on the catalog clock's grid Publishers sharing an `--epoch` number their groups from their own start, so a moq-lite 07 relay that resumes a subscription on the other at the same group and frame lands on other media, and the exporter reads a rewind (#4354). `Timebase::align_groups`, which the CLI calls with a shared `--epoch`, puts every media track the timebase reserves on a one-second grid of the catalog clock. A group opens at the first start in a second (a video keyframe, any audio frame, a data unit, a passthrough group start) and takes that second's number; later starts in the second join it, and a second with none gets an empty group. A process publishes nothing from the second it started in, so two fed one stream publish the same groups under the same sequences wherever each started. Part of #4354. Co-authored-by: Claude Opus 5.5 --- doc/bin/cli.md | 10 +- rs/moq-cli/src/args.rs | 3 +- rs/moq-cli/src/publish.rs | 1 + rs/moq-mux/src/catalog/timebase.rs | 21 +- rs/moq-mux/src/catalog/tracks.rs | 9 +- rs/moq-mux/src/container/fmp4/import.rs | 34 +- rs/moq-mux/src/container/fmp4/import_test.rs | 101 +++++ rs/moq-mux/src/container/grid.rs | 224 ++++++++++ rs/moq-mux/src/container/mod.rs | 1 + rs/moq-mux/src/container/producer.rs | 411 ++++++++++++++++-- rs/moq-mux/src/container/ts/import.rs | 129 +++++- rs/moq-mux/src/container/ts/passthrough.rs | 154 ++++++- .../src/container/ts/passthrough_test.rs | 115 ++++- 13 files changed, 1100 insertions(+), 113 deletions(-) create mode 100644 rs/moq-mux/src/container/grid.rs diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 703e4e07b0b..b21773d84b4 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -335,8 +335,14 @@ On moq-lite 07, two processes that pass the same `--epoch` (a UUIDv7, such as `uuidgen -7` prints) are one publisher: relays fail over between them mid-group, so they must produce identical tracks with aligned groups. Each derives its catalog clock from the epoch's wall time, so fed one stream they -publish the same clock and timestamps wherever each started, without reading -anything from the other; group numbering is not aligned yet. PTS zero is the +publish the same clock, timestamps, and media groups wherever each started, +without reading anything from the other; the catalog and timeline tracks are not +aligned yet. Every media track groups on that clock's one-second +grid: a group opens at the first place one may start in a second (a keyframe, +or any audio frame or data unit) and takes that second's number, later ones in +the second join it, and a second with none gets an empty group. A process +publishes nothing from the second it started in, and a source's timestamp break +publishes no break marker, only the jump. PTS zero is the epoch's time, and for `ts` one 33-bit wrap before it. The wall mapping is therefore off by the source clock's reading at the epoch's time: for a TS program clock, anywhere within 13.25 hours. Each process picks a TS timestamp's diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index aa03a9669bf..df9f2f16efe 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -446,7 +446,8 @@ pub struct MoqSide { /// then off by the source clock's reading at the epoch's time, which for a TS program /// clock is anywhere within half a wrap. Near either end of that range, processes whose /// clocks or delivery differ can place one timestamp a wrap apart, and redundant `flv` - /// imports stop matching at FLV's 49.7-day millisecond wrap. + /// imports stop matching at FLV's 49.7-day millisecond wrap. Groups are numbered by the + /// second of that clock they open in, so the publishers agree on those too. #[usage(long, env = "MOQ_EPOCH", help_heading = "MoQ")] pub epoch: Option, diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index b26cc9b653a..341d28818d0 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -278,6 +278,7 @@ pub(crate) fn timebase( timebase .place(moq_net::Timestamp::ZERO, zero) .with_context(|| format!("failed to place the clock on epoch {epoch}"))?; + timebase.align_groups(); } Ok(timebase) } diff --git a/rs/moq-mux/src/catalog/timebase.rs b/rs/moq-mux/src/catalog/timebase.rs index bb9786b0604..c4b820a0a42 100644 --- a/rs/moq-mux/src/catalog/timebase.rs +++ b/rs/moq-mux/src/catalog/timebase.rs @@ -1,3 +1,4 @@ +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, OnceLock}; use super::hang::CatalogExt; @@ -19,6 +20,7 @@ use super::{Producer, Reserved}; pub struct Timebase { pub(super) catalog: Producer, offset: Arc>, + aligned: Arc, } impl Timebase { @@ -26,6 +28,7 @@ impl Timebase { Self { catalog, offset: Default::default(), + aligned: Default::default(), } } @@ -50,12 +53,27 @@ impl Timebase { /// frame, this places the catalog's clock if it is not yet fixed and offsets onto it otherwise. /// A placement alone publishes nothing: the clock goes out with the catalog's first track. /// Refused once this timebase has an offset, and when the host's wall clock was unset, leaving no - /// wall to place it on. + /// wall to place it on. Importers that must also agree on their groups call + /// [`align_groups`](Self::align_groups) too. pub fn place(&self, pts: moq_net::Timestamp, wall: std::time::SystemTime) -> crate::Result<()> { self.catalog.clone().anchor(&self.offset, pts, Some(wall))?; Ok(()) } + /// Group every track reserved from now on by the catalog clock's one-second grid (see + /// [`container::Producer`](crate::container::Producer#grid)), for the publishers of one shared + /// epoch: [placed](Self::place) on it and fed one stream, they then publish the same groups under + /// the same sequences wherever each started, which a relay that resumes one's subscriptions on + /// another needs. + pub fn align_groups(&self) { + self.aligned.store(true, Ordering::Relaxed); + } + + /// Whether [`align_groups`](Self::align_groups) was called on this timebase or a clone. + pub(crate) fn aligned(&self) -> bool { + self.aligned.load(Ordering::Relaxed) + } + /// Anchor on the first frame's timestamp `pts`, returning the timebase's offset. /// /// The first call from any clone fixes the offset; every later one returns it unchanged. @@ -113,6 +131,7 @@ impl Clone for Timebase { Self { catalog: self.catalog.clone(), offset: self.offset.clone(), + aligned: self.aligned.clone(), } } } diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index 70d6a478e18..4a92d98bb49 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -465,9 +465,14 @@ impl Reserved { crate::Error: From, { let rendition = self.init(track.name())?; - self.timebase + let producer = self + .timebase .catalog - .media(track, container, rendition, config.into(), self.timeline_hint) + .media(track, container, rendition, config.into(), self.timeline_hint)?; + Ok(match self.timebase.aligned() { + true => producer.with_grid(), + false => producer, + }) } /// Resolve a timestamp on the broadcast's shared clock (see [`Producer::timestamp`]). diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 1b61dce3ee0..4a9f1e8f0e7 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -7,6 +7,7 @@ use std::collections::{HashMap, HashSet}; use super::{Error, Kind, sample}; use crate::Result; use crate::catalog::Estimator; +use crate::container::grid::Place; /// Converts fMP4/CMAF files into MoQ broadcast streams using CMAF passthrough. /// @@ -127,6 +128,10 @@ struct Fmp4Track { // group, which is what keeps audio on the same boundaries as video. segment: Option, + // Set when the timebase aligns groups, where a segment's first fragment opens one only on the grid + // (see `container::Producer`'s grid) and `seek` does nothing. + grid: Option, + // Measures the track's catalog jitter and bitrate from the fragments; the descriptor's own // bitrate, when it declares one, still wins over what this measures. estimator: Estimator, @@ -349,6 +354,7 @@ impl Import { track, group: None, segment: None, + grid: self.timebase.aligned().then(Default::default), recorder: Some(recorder), last_decode_time: None, sample_duration: None, @@ -993,7 +999,21 @@ impl Import { // The keyframe bit no longer decides that: audio flags every sample a sync sample, so // keying off it opened a group (a QUIC stream) per audio fragment. let new_segment = track.segment != Some(self.segment); - let start_group = track.group.is_none() || new_segment; + let mut start_group = track.group.is_none() || new_segment; + // A dropped fragment still moves the track into its segment, or the segment's next + // fragment would read as a start that an importer which kept this one does not see. + track.segment = Some(self.segment); + let mut fill = 0..0; + if let Some(grid) = track.grid.as_mut() { + match grid.place(timestamp, start_group, track.group.is_some()) { + Place::Drop => continue, + Place::Join => start_group = false, + Place::Open { fill: empty, sequence } => { + fill = empty; + track.pending_sequence = Some(sequence); + } + } + } let mut g = if start_group { if let Some(prev) = track.group.take() { prev.finish()?; @@ -1001,7 +1021,9 @@ impl Import { recorder.finish_group(prev.sequence); } } - track.segment = Some(self.segment); + for sequence in fill { + track.track.create_group(moq_net::group::Info { sequence })?.finish()?; + } match track.pending_sequence.take() { Some(sequence) => track.track.create_group(moq_net::group::Info { sequence })?, None => track.track.append_group()?, @@ -1102,13 +1124,14 @@ impl Import { /// control is intentionally not exposed. Skipping sequences is not a new timeline, so the /// next fragment must still advance past the last decode time; see /// [`discontinuity`](Self::discontinuity). Text spends a group per cue, so a text track that - /// is already past `sequence` continues from its own next group instead. + /// is already past `sequence` continues from its own next group instead. On a timebase a shared + /// epoch placed, the grid numbers the groups and this does nothing. pub fn seek(&mut self, sequence: u64) -> Result<()> { for track in self.text.values_mut() { let next = track.producer.next_sequence(); track.producer.seek(sequence.max(next))?; } - for track in self.tracks.values_mut() { + for track in self.tracks.values_mut().filter(|track| track.grid.is_none()) { track.estimator.cut(None); track.publish_estimate()?; if let Some(g) = track.group.take() { @@ -1132,6 +1155,9 @@ impl Import { } for track in self.tracks.values_mut() { track.last_decode_time = None; + if let Some(grid) = track.grid.as_mut() { + grid.restart(); + } } } } diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 53185aa13a5..687ab1ea683 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2535,3 +2535,104 @@ fn colr_nclc_keeps_its_code_points() { ); assert!(super::colr_color(&mp4_atom::Colr::new(300, 1, 1, false).unwrap()).is_err()); } + +// ---- Shared epoch ---- + +/// Where the shared-epoch source starts each segment, in milliseconds past ten minutes on from the +/// epoch: two pairs of segments share a second, and four seconds start none. +const PLACED_SEGMENTS: [u64; 11] = [ + 0, 1_500, 1_900, 3_000, 4_500, 4_800, 6_000, 7_500, 9_000, 10_500, 12_000, +]; + +/// Every group a finished track holds: its sequence and how many frames it carries. +fn drain_groups(track: &mut moq_net::track::Subscriber) -> Vec<(u64, usize)> { + let mut groups = Vec::new(); + while let Some(mut group) = track.recv_group().now_or_never().and_then(|r| r.ok().flatten()) { + let mut frames = 0; + while group + .next_frame() + .now_or_never() + .and_then(|r| r.ok().flatten()) + .is_some() + { + frames += 1; + } + groups.push((group.sequence, frames)); + } + groups +} + +/// The video and audio groups of an fMP4 import on a shared epoch's timebase, fed bbb's +/// init and then [`PLACED_SEGMENTS`] from segment `from` on, each segment one fragment per track. +async fn placed_groups(from: usize) -> [Vec<(u64, usize)>; 2] { + let epoch = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_millis(1_760_000_000_000); + crate::clock::fake_host_wall(epoch + std::time::Duration::from_secs(600)); + let (init, video, audio) = bbb_init(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let timebase = catalog.timebase(); + timebase.place(moq_net::Timestamp::ZERO, epoch).unwrap(); + timebase.align_groups(); + let mut fmp4 = super::Import::new(broadcast, timebase.reserve()); + fmp4.decode(&init).unwrap(); + + let snapshot = catalog.snapshot(); + let names = [ + snapshot.video.renditions.keys().next().unwrap(), + snapshot.audio.renditions.keys().next().unwrap(), + ]; + let mut tracks = Vec::new(); + for name in names { + tracks.push(consumer.track(name).unwrap().subscribe(replay()).await.unwrap()); + } + for (i, segment) in PLACED_SEGMENTS.iter().enumerate().skip(from) { + let next = PLACED_SEGMENTS.get(i + 1).map_or(segment + 1_000, |next| *next); + let (start, end) = (600_000_000 + segment * 1_000, 600_000_000 + next * 1_000); + fmp4.decode(&styp()).unwrap(); + for (id, scale) in [video, audio] { + let frames = [sample(start, true, Some(end - start))]; + fmp4.decode(&super::encode_fragment(info(id, scale, i as u32), &frames).unwrap()) + .unwrap(); + } + } + fmp4.finish().unwrap(); + let [video, audio] = &mut tracks[..] else { + unreachable!("two tracks") + }; + [drain_groups(video), drain_groups(audio)] +} + +/// fMP4 importers placed by one epoch publish the same groups wherever each started: one per +/// second of the catalog clock that starts a segment, numbered by that second, with segments that +/// share a second sharing its group and an empty group for each second that starts none. Neither +/// publishes from the second it started in. +#[tokio::test] +async fn importers_sharing_an_epoch_number_segments_alike() { + let expected = vec![ + (601, 2), + (602, 0), + (603, 1), + (604, 2), + (605, 0), + (606, 1), + (607, 1), + (608, 0), + (609, 1), + (610, 1), + (611, 0), + (612, 1), + ]; + let first = placed_groups(0).await; + assert_eq!(first, [expected.clone(), expected], "video, then audio"); + for from in 1..PLACED_SEGMENTS.len() - 1 { + let joined = placed_groups(from).await; + for (track, (first, joined)) in ["video", "audio"].into_iter().zip(first.iter().zip(&joined)) { + assert!(!joined.is_empty(), "{track} from segment {from} published nothing"); + assert!( + first.ends_with(joined), + "{track} from segment {from}: {joined:?} vs {first:?}" + ); + } + } +} diff --git a/rs/moq-mux/src/container/grid.rs b/rs/moq-mux/src/container/grid.rs new file mode 100644 index 00000000000..64f7f67d284 --- /dev/null +++ b/rs/moq-mux/src/container/grid.rs @@ -0,0 +1,224 @@ +//! Group numbering on the catalog clock's one-second grid, for a track whose timestamps every +//! publisher of its epoch shares. +//! +//! Each second of the catalog clock is a cell, numbered by its index. A group opens at the first +//! start in a cell no group has taken yet and takes that cell's number; later starts in the cell +//! join it, and every cell passed with no start gets an empty group, so sequences stay +//! consecutive. What counts as a start is the track's business: a video keyframe, any audio frame, +//! a passthrough group start. +//! +//! Two producers fed one stream therefore publish the same groups wherever each started, provided +//! neither publishes from a cell it may have seen only part of. That is the one it started in, and +//! the one it resumed in after a break: an earlier producer may have opened that cell's group at a +//! start the later one missed. + +use std::ops::Range; + +/// The spacing of the grid, in microseconds of the catalog clock. +pub(crate) const GRID_MICROS: u64 = 1_000_000; + +/// The most empty cells one step of the timeline fills. A longer step between two frames (or two +/// [`advance`](Grid::advance)s) starts a new watch instead, as a break does, so a source that leaps +/// hours ahead cannot open a group for every second it skipped. +const FILL_MAX: u64 = 60; + +/// The cell of the grid `timestamp` falls in. +fn cell(timestamp: moq_net::Timestamp) -> u64 { + u64::try_from(timestamp.as_micros() / u128::from(GRID_MICROS)).unwrap_or(u64::MAX) +} + +/// Where a frame lands on the grid. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum Place { + /// Give each sequence in `fill` an empty group, then open the group `sequence`. + Open { fill: Range, sequence: u64 }, + /// Join the open group. + Join, + /// No group is open, and the frame's cell may hold a start this producer missed. + Drop, +} + +/// Where a track stands on the grid. +#[derive(Debug, Default)] +pub(crate) struct Grid { + /// The newest cell a frame or [`advance`](Self::advance) reached. + now: Option, + /// The cell the watch began in, the newest one possibly seen only in part: set by the first + /// frame, advance or [`watch`](Self::watch), and again by the first after a [break](Self::restart). + origin: Option, + /// The highest cell given a group, empty or not. + last: Option, + /// Whether any group has opened. + opened: bool, +} + +impl Grid { + /// Place a frame at `timestamp`. `start` says the frame may open a group, `open` that one is + /// open for it to join. + pub fn place(&mut self, timestamp: moq_net::Timestamp, start: bool, open: bool) -> Place { + let at = self.reach(timestamp); + let origin = self.origin.expect("a reached grid has an origin"); + if start && self.last.is_none_or(|last| at > last) && (self.opened || at > origin) { + let fill = self.empty(at); + self.last = Some(at); + self.opened = true; + return Place::Open { fill, sequence: at }; + } + match open { + true => Place::Join, + false => Place::Drop, + } + } + + /// The timeline reached `now`, though no frame of the track did: the empty groups for the cells + /// it passed whole. The open group stays open, since only a start in a later cell closes it. + pub fn advance(&mut self, now: moq_net::Timestamp) -> Range { + let at = self.reach(now); + self.empty(at) + } + + /// The track sees every start only from `since` on, for an importer that can miss starts after + /// its first frames. Until the first group opens, a later call can only move that point later. + pub fn watch(&mut self, since: moq_net::Timestamp) { + if !self.opened { + let at = cell(since); + self.origin = Some(self.origin.map_or(at, |origin| origin.max(at))); + } + } + + /// A break: the cells it skipped get no empty group, and the watch begins again at the next + /// frame or advance. + pub fn restart(&mut self) { + self.origin = None; + } + + /// The cell the timeline reached with `timestamp`: its own, or the newest already reached, + /// since a passed cell never takes a group again. + fn reach(&mut self, timestamp: moq_net::Timestamp) -> u64 { + let own = cell(timestamp); + let at = self.now.map_or(own, |now| now.max(own)); + if self.now.is_some_and(|now| at - now > FILL_MAX) { + self.origin = None; + } + self.now = Some(at); + self.origin.get_or_insert(at); + at + } + + /// Claim every cell before `at` that was watched whole and holds no group yet. + fn empty(&mut self, at: u64) -> Range { + let origin = self.origin.expect("a reached grid has an origin"); + let from = self.last.map_or(origin, |last| last.max(origin)) + 1; + if from >= at { + return at..at; + } + self.last = Some(at - 1); + from..at + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(millis: u64) -> moq_net::Timestamp { + moq_net::Timestamp::from_millis(millis).unwrap() + } + + fn opens(sequence: u64, fill: Range) -> Place { + Place::Open { fill, sequence } + } + + #[test] + fn the_cell_a_producer_starts_in_publishes_nothing() { + let mut grid = Grid::default(); + assert_eq!(grid.place(at(10_000), true, false), Place::Drop); + assert_eq!(grid.place(at(10_500), true, false), Place::Drop); + assert_eq!(grid.place(at(11_200), true, false), opens(11, 11..11)); + } + + #[test] + fn later_starts_in_a_cell_join_its_group() { + let mut grid = Grid::default(); + grid.place(at(10_000), true, false); + assert_eq!(grid.place(at(11_000), true, false), opens(11, 11..11)); + assert_eq!(grid.place(at(11_500), true, true), Place::Join); + assert_eq!(grid.place(at(11_999), false, true), Place::Join); + } + + #[test] + fn cells_passed_without_a_start_get_empty_groups() { + let mut grid = Grid::default(); + grid.place(at(10_000), true, false); + grid.place(at(11_000), true, false); + assert_eq!(grid.place(at(12_500), false, true), Place::Join); + assert_eq!(grid.place(at(14_100), true, true), opens(14, 12..14)); + } + + #[test] + fn a_frame_behind_the_newest_cell_counts_in_it() { + let mut grid = Grid::default(); + grid.place(at(10_000), true, false); + grid.place(at(12_000), true, false); + assert_eq!(grid.place(at(11_900), true, true), Place::Join); + } + + #[test] + fn a_break_fills_nothing_it_skipped() { + let mut grid = Grid::default(); + grid.place(at(10_000), true, false); + grid.place(at(11_000), true, false); + grid.restart(); + assert_eq!(grid.place(at(15_000), true, true), opens(15, 15..15)); + assert_eq!(grid.place(at(17_000), true, true), opens(17, 16..17)); + } + + #[test] + fn a_leap_past_the_fill_bound_acts_as_a_break() { + let mut grid = Grid::default(); + grid.place(at(10_000), true, false); + grid.place(at(11_000), true, false); + assert_eq!(grid.place(at(3_600_000), true, true), opens(3_600, 3_600..3_600)); + } + + #[test] + fn advance_fills_cells_as_they_pass() { + let mut grid = Grid::default(); + assert_eq!(grid.advance(at(10_300)), 10..10); + assert_eq!(grid.advance(at(11_000)), 11..11); + assert_eq!(grid.advance(at(13_000)), 11..13); + assert_eq!(grid.place(at(13_400), true, false), opens(13, 13..13)); + } + + #[test] + fn watch_moves_the_origin_later_until_a_group_opens() { + let mut grid = Grid::default(); + grid.place(at(10_000), true, false); + grid.watch(at(12_000)); + assert_eq!(grid.place(at(12_500), true, false), Place::Drop); + assert_eq!(grid.place(at(13_000), true, false), opens(13, 13..13)); + grid.watch(at(20_000)); + assert_eq!(grid.place(at(14_000), true, true), opens(14, 14..14)); + } + + #[test] + fn producers_started_apart_agree_from_the_later_ones_first_group() { + let starts = [100, 900, 1_400, 2_050, 2_700, 2_800, 5_300, 5_900, 6_000, 6_400, 9_100]; + let run = |from: usize| { + let mut grid = Grid::default(); + let mut groups = Vec::new(); + for &start in &starts[from..] { + if let Place::Open { fill, sequence } = grid.place(at(10_000 + start), true, !groups.is_empty()) { + groups.extend(fill); + groups.push(sequence); + } + } + groups + }; + let early = run(0); + for from in 1..starts.len() { + let late = run(from); + assert!(early.ends_with(&late), "from {from}: {early:?} vs {late:?}"); + } + } +} diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index bea813647c5..85db47e58a5 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -16,6 +16,7 @@ use bytes::Bytes; mod consumer; mod decode; +mod grid; mod group; mod producer; mod source; diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index fc63b19569e..312137a385b 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -46,6 +46,16 @@ fn timestamp_lt(left: moq_net::Timestamp, right: moq_net::Timestamp) -> bool { /// - `finish()` is called. /// /// This is useful for CMAF where multiple samples should be packed into one moof+mdat. +/// +/// ## Grid +/// +/// On a timebase whose groups are [aligned](crate::catalog::Timebase::align_groups), the timeline +/// decides the groups, so two producers fed one stream publish the same groups under the same +/// sequences wherever each started. A group opens at the first start in each one-second cell of the +/// catalog clock and takes that cell's index as its sequence: a video keyframe, any audio frame, or +/// a keyframe on a data track. Later starts in the cell join it, and a cell with no start gets an +/// empty group, so sequences stay consecutive (see the `grid` module). [`cut`](Self::cut) and +/// [`seek`](Self::seek) do nothing. pub struct Producer { inner: moq_net::track::Producer, container: C, @@ -97,6 +107,13 @@ pub struct Producer { /// The catalog rendition this media producer owns, when it was created through a catalog. rendition: Option>>, + + /// Set when the timeline decides the groups; see [`with_grid`](Self::with_grid). + grid: Option, + + /// On the grid, a break cut the open group and nothing has continued it since: it closes + /// without a bound, and a video track waits for a keyframe. + broken: bool, } trait Rendition: Send { @@ -164,6 +181,8 @@ impl Producer { estimator: crate::catalog::Estimator::new(), bandwidth: None, rendition: None, + grid: None, + broken: false, } } } @@ -200,6 +219,8 @@ where estimator: rendition.estimator(), bandwidth: None, rendition: Some(Box::new(rendition)), + grid: None, + broken: false, } } @@ -379,6 +400,59 @@ where self } + /// Group on the one-second grid of the catalog clock (see [Grid](Self#grid)), for a track + /// whose timestamps every publisher of its epoch shares. + /// + /// A producer started late publishes nothing from the cell it started in: an earlier one may + /// have opened that cell's group at a start this one missed. A break publishes no marker + /// group, since the grid leaves it no sequence: a consumer sees the timestamps jump, or the + /// sequences skip when the break spans whole cells. + pub(crate) fn with_grid(mut self) -> Self { + self.grid = Some(super::grid::Grid::default()); + self + } + + /// Tell a grid-mode track it sees every start only from `since` on, for an importer that can + /// miss starts after its first frames. Until its first group opens, a later call can only move + /// that point later. Does nothing without [`with_grid`](Self::with_grid). + pub(crate) fn watch(&mut self, since: moq_net::Timestamp) { + if let Some(grid) = self.grid.as_mut() { + grid.watch(since); + } + } + + /// Tell a grid-mode track the timeline reached `now`, though no frame of its own did. + /// + /// A sparse track ticked this way watches from its first tick rather than its first frame, and + /// its empty cells get their empty groups as they pass rather than when its next frame comes. + /// The open group stays open: only a start in a later cell closes it. Does nothing without + /// [`with_grid`](Self::with_grid). + pub(crate) fn advance(&mut self, now: moq_net::Timestamp) -> crate::Result<()> { + let Some(grid) = self.grid.as_mut() else { + return Ok(()); + }; + let fill = grid.advance(now); + self.fill(fill) + } + + /// Whether the track groups on the [grid](Self#grid). + pub(crate) fn on_grid(&self) -> bool { + self.grid.is_some() + } + + /// The sequence of the open group, if one is open. + pub(crate) fn open_group(&self) -> Option { + self.group.as_ref().map(|group| group.sequence) + } + + /// Publish an empty group under each of `sequences`. + fn fill(&mut self, sequences: std::ops::Range) -> crate::Result<()> { + for sequence in sequences { + self.inner.create_group(moq_net::group::Info { sequence })?.finish()?; + } + Ok(()) + } + /// Raise the standing claim when the catalog estimate has a new peak. fn claim(&mut self) { let Some(bandwidth) = self.bandwidth.as_mut() else { @@ -415,49 +489,17 @@ where /// without writing, the way an oversized frame is refused. That is a restart, which is a new /// broadcast. Frames may still dip below the previous group's content: B-frames, open-GOP /// leading pictures, and a keyframe that overlaps the previous group's last frame. + /// + /// On the [grid](Self#grid) a start opens a group only in a cell no group has taken yet, and + /// a start the producer drops (in the cell it started in) returns `Ok` unwritten. pub fn write(&mut self, frame: Frame) -> crate::Result<()> { - let floor = match frame.keyframe { - true => self.start, - false => self.floor, + let frame = match self.grid.is_some() { + true => match self.group_on_grid(frame)? { + Some(frame) => frame, + None => return Ok(()), + }, + false => self.group_by_keyframe(frame)?, }; - if let Some(floor) = floor - && timestamp_lt(frame.timestamp, floor) - { - return Err(super::TimestampRewind { - timestamp: frame.timestamp, - floor, - } - .into()); - } - - // A keyframe cuts the previous group, using its timestamp as the boundary where the - // previous group's content ends, unless it overlaps that content. - if frame.keyframe { - let overlaps = self - .previous_timestamp - .is_some_and(|previous| timestamp_lt(frame.timestamp, previous)); - self.cut((!overlaps).then_some(frame.timestamp))?; - if overlaps { - self.cadence = None; - } - } - - // Start a new group if needed; the first frame of a group must be a keyframe. - if self.group.is_none() { - if !frame.keyframe { - // No group yet and this delta can't anchor one. The caller (e.g. a - // mid-stream join) decides whether to skip until the first keyframe. - return Err(super::MissingKeyframe.into()); - } - let group = match self.pending_sequence.take() { - Some(sequence) => self.inner.create_group(moq_net::group::Info { sequence })?, - None => self.inner.append_group()?, - }; - - self.floor = self.start; - self.start = Some(frame.timestamp); - self.group = Some(group); - } // Buffer or write the frame. if self.buffer_duration.is_zero() { @@ -501,6 +543,125 @@ where Ok(()) } + /// Refuse a frame below `floor`. + fn check_floor(frame: &Frame, floor: Option) -> crate::Result<()> { + if let Some(floor) = floor + && timestamp_lt(frame.timestamp, floor) + { + return Err(super::TimestampRewind { + timestamp: frame.timestamp, + floor, + } + .into()); + } + Ok(()) + } + + /// Open the group `frame` lands in, a keyframe closing the one before. + fn group_by_keyframe(&mut self, frame: Frame) -> crate::Result { + Self::check_floor( + &frame, + match frame.keyframe { + true => self.start, + false => self.floor, + }, + )?; + + // A keyframe cuts the previous group, using its timestamp as the boundary where the + // previous group's content ends, unless it overlaps that content. + if frame.keyframe { + self.close_before(&frame)?; + } + + // Start a new group if needed; the first frame of a group must be a keyframe. + if self.group.is_none() { + if !frame.keyframe { + // No group yet and this delta can't anchor one. The caller (e.g. a + // mid-stream join) decides whether to skip until the first keyframe. + return Err(super::MissingKeyframe.into()); + } + let group = match self.pending_sequence.take() { + Some(sequence) => self.inner.create_group(moq_net::group::Info { sequence })?, + None => self.inner.append_group()?, + }; + self.open(group, &frame); + } + Ok(frame) + } + + /// Place `frame` on the grid: open its cell's group if it is the cell's first start, else join + /// the open group. `None` when the frame is dropped. + fn group_on_grid(&mut self, mut frame: Frame) -> crate::Result> { + let kind = self.container.kind(); + let start = kind == Kind::Audio || frame.keyframe; + if self.broken && kind == Kind::Video && !frame.keyframe { + return Err(super::MissingKeyframe.into()); + } + // Every start bounds the frames after it, as a keyframe does off the grid, whether its group + // opens, joins, or is dropped: a rewind then ends producers alike wherever each started. + Self::check_floor( + &frame, + match start { + true => self.start, + false => self.floor, + }, + )?; + if start { + self.floor = self.start; + self.start = Some(frame.timestamp); + } + let grid = self.grid.as_mut().expect("grid mode"); + let place = grid.place(frame.timestamp, start, self.group.is_some()); + // Every frame raises the edge, a dropped one too, so a producer started late stamps an + // untimed payload where an earlier one does. + if let Some(end) = add_micros(frame.timestamp, frame.duration.unwrap_or(moq_net::Timestamp::ZERO)) { + self.raise_live_edge(end); + } + + match place { + super::grid::Place::Drop => { + return match start { + true => Ok(None), + false => Err(super::MissingKeyframe.into()), + }; + } + super::grid::Place::Join => frame.keyframe &= kind == Kind::Video, + super::grid::Place::Open { fill, sequence } => { + if self.group.is_some() { + match self.broken { + true => self.close(None, None)?, + false => self.close_before(&frame)?, + } + } + self.fill(fill)?; + self.group = Some(self.inner.create_group(moq_net::group::Info { sequence })?); + frame.keyframe = true; + } + } + self.broken = false; + Ok(Some(frame)) + } + + /// Close the open group ahead of a keyframe at `frame`'s timestamp, bounding it there unless the + /// keyframe overlaps its content. + fn close_before(&mut self, frame: &Frame) -> crate::Result<()> { + let overlaps = self + .previous_timestamp + .is_some_and(|previous| timestamp_lt(frame.timestamp, previous)); + self.bound((!overlaps).then_some(frame.timestamp))?; + if overlaps { + self.cadence = None; + } + Ok(()) + } + + /// Make `group`, opened at `frame`, the current group. + fn open(&mut self, group: moq_net::group::Producer, frame: &Frame) { + self.floor = self.start; + self.start = Some(frame.timestamp); + self.group = Some(group); + } + /// Cut the current group, flushing buffered frames and closing it. /// /// `end` bounds the final buffered frame when the publisher knows where the @@ -510,7 +671,16 @@ where /// does not bound the last frame in decode order. The next [`write`](Self::write) /// must be a keyframe. An explicit bound before the last ordered video frame /// returns [`InvalidEnd`](super::InvalidEnd) without flushing or closing the group. + /// On the [grid](Self#grid) this does nothing. pub fn cut(&mut self, end: Option) -> crate::Result<()> { + if self.grid.is_some() { + return Ok(()); + } + self.bound(end) + } + + /// Close the current group, bounded at `end` or its estimated end. + fn bound(&mut self, end: Option) -> crate::Result<()> { let marker_at = end.or_else(|| self.estimated_end()); self.close(end, marker_at) } @@ -605,8 +775,11 @@ where /// Close the current group (if any) and open the next group at the given sequence. /// /// The next [`write`](Self::write) must be a keyframe and will land in a group with - /// `sequence`. Useful for joining mid-stream. + /// `sequence`. Useful for joining mid-stream. On the [grid](Self#grid) this does nothing. pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { + if self.grid.is_some() { + return Ok(()); + } self.cut(None)?; self.pending_sequence = Some(sequence); Ok(()) @@ -639,7 +812,21 @@ where /// Otherwise the open group closes without a duration marker: what resumes may land sooner /// than one estimated frame later (a capture that reopens at once), and a guessed end past /// it would read as a rewind to every consumer. + /// + /// On the [grid](Self#grid) no marker is published and the open group stays open. The next + /// start in a later cell closes it without a bound, the cells the break skipped get no empty + /// group, and a video track takes no delta until its next keyframe. pub fn discontinuity(&mut self) -> crate::Result<()> { + if let Some(grid) = self.grid.as_mut() { + grid.restart(); + self.broken = true; + self.estimator.discontinuity(); + self.cadence = None; + if let Some(recorder) = self.recorder.as_mut() { + recorder.flush(); + } + return Ok(()); + } self.close(None, None)?; // Nothing is measured across the break: the frames still open on this side have no end, and // the gap to the far side is not a frame duration. @@ -725,7 +912,7 @@ where /// Finish the track, flushing any buffered frames and closing any open group. pub fn finish(&mut self) -> crate::Result<()> { - self.cut(None)?; + self.bound(None)?; self.inner.finish()?; self.publish_estimate()?; // Dropping the recorder flushes the open record, so a track that ends before its catalog @@ -1730,4 +1917,142 @@ mod tests { } assert_eq!(timestamps, [1_000_000, 1_040_000, 1_050_000]); } + + /// Every group a finished track holds: its sequence, and each frame's timestamp and size. + async fn grid_groups(mut consumer: moq_net::track::Subscriber) -> Vec<(u64, Vec<(u128, usize)>)> { + let mut groups = Vec::new(); + while let Some(mut group) = consumer.recv_group().await.unwrap() { + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await.unwrap() { + let decoded = hang::container::Frame::decode(frame.payload).unwrap(); + frames.push((decoded.timestamp.as_micros(), decoded.payload.len())); + } + groups.push((group.sequence, frames)); + } + groups + } + + /// A grid-mode producer of `kind` fed `frames`, skipping what a mid-stream join skips. + async fn on_grid(kind: crate::container::Kind, frames: &[Frame]) -> Vec<(u64, Vec<(u128, usize)>)> { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(kind)).with_grid(); + for frame in frames { + match producer.write(frame.clone()) { + Err(crate::Error::MissingKeyframe(_)) => {} + result => result.unwrap(), + } + producer.cut(None).unwrap(); + } + producer.finish().unwrap(); + grid_groups(consumer).await + } + + /// On the grid, video producers started anywhere in one stream publish the same groups, frames + /// and bounds under the same sequences, from the later one's first group on: keyframes 600 ms + /// apart share their second's group, and an importer's per-frame `cut` changes nothing. + #[tokio::test] + async fn on_the_grid_video_producers_started_apart_publish_the_same_groups() { + let frames: Vec = (0..200u64) + .map(|i| frame(10_000_000 + i * 40_000, i % 15 == 0)) + .collect(); + let first = on_grid(crate::container::Kind::Video, &frames).await; + let sequences: Vec = first.iter().map(|(sequence, _)| *sequence).collect(); + assert_eq!( + sequences, + (11..=17).collect::>(), + "the first second publishes nothing" + ); + assert_eq!(first[0].1.len(), 30 + 1, "cell 11 holds two GOPs and its bound"); + for from in [1, 14, 15, 16, 40, 80, 120] { + let joined = on_grid(crate::container::Kind::Video, &frames[from..]).await; + assert!( + !joined.is_empty() && first.ends_with(&joined), + "from frame {from}: {joined:?}" + ); + } + } + + /// On the grid every audio frame is a start, so one group holds each second's frames, the + /// first second's dropped. + #[tokio::test] + async fn on_the_grid_audio_groups_by_the_second() { + let frames: Vec = (0..150u64).map(|i| frame(5_000_000 + i * 20_000, false)).collect(); + let groups = on_grid(crate::container::Kind::Audio, &frames).await; + let shape: Vec<(u64, usize)> = groups + .iter() + .map(|(sequence, frames)| (*sequence, frames.len())) + .collect(); + assert_eq!(shape, [(6, 50), (7, 50)]); + } + + /// On the grid a rewind ends a producer that has published nothing yet, as it ends one that has. + #[tokio::test] + async fn on_the_grid_a_rewind_ends_a_producer_before_its_first_group() { + for (name, kind) in [ + ("audio", crate::container::Kind::Audio), + ("video", crate::container::Kind::Video), + ] { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut producer = Producer::new(track, Container::Legacy(kind)).with_grid(); + for i in 0..10u64 { + producer.write(frame(10_000_000 + i * 40_000, i == 0)).unwrap_or(()); + } + assert!( + matches!( + producer.write(frame(5_000_000, true)), + Err(crate::Error::TimestampRewind(_)) + ), + "{name}" + ); + } + } + + /// On the grid a break publishes no marker and leaves the cells it skipped without a group, and + /// video takes no delta until the next keyframe. + #[tokio::test] + async fn on_the_grid_a_break_skips_its_cells_and_waits_for_a_keyframe() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)).with_grid(); + for i in 0..60u64 { + producer + .write(frame(10_000_000 + i * 40_000, i % 25 == 0)) + .unwrap_or(()); + } + producer.discontinuity().unwrap(); + assert!(matches!( + producer.write(frame(20_000_000, false)), + Err(crate::Error::MissingKeyframe(_)) + )); + producer.write(frame(20_040_000, true)).unwrap(); + producer.write(frame(21_000_000, true)).unwrap(); + producer.finish().unwrap(); + let sequences: Vec = grid_groups(consumer) + .await + .iter() + .map(|(sequence, _)| *sequence) + .collect(); + assert_eq!(sequences, [11, 12, 20, 21]); + } + + /// On the grid a sparse track ticked by the media clock gets each empty second's group as the + /// second passes, from its first tick on. + #[tokio::test] + async fn on_the_grid_a_ticked_track_fills_empty_seconds() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Data)).with_grid(); + for tick in [3_200_000, 4_100_000, 6_500_000] { + producer.advance(Timestamp::from_micros(tick).unwrap()).unwrap(); + } + producer.write(frame(6_600_000, true)).unwrap(); + producer.finish().unwrap(); + let shape: Vec<(u64, usize)> = grid_groups(consumer) + .await + .iter() + .map(|(sequence, frames)| (*sequence, frames.len())) + .collect(); + assert_eq!(shape, [(4, 0), (5, 0), (6, 1)]); + } } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 2aaf418abe9..2fa06807e97 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -636,6 +636,7 @@ impl Import { pid.as_u16(), stream_type, descriptors, + self.timebase.aligned(), ) { Ok(stream) => Stream::Verbatim(Box::new(stream)), Err(err) => { @@ -708,6 +709,7 @@ impl Import { pid.as_u16(), stream_type, descriptors, + self.timebase.aligned(), )?; stream.retained = Retained::new(self.staging.clone()); self.sections.insert(pid.as_u16(), stream); @@ -779,6 +781,7 @@ impl Import { // previous one's. if pts.is_some() { self.last_pts = on_clock(pts, offset)?; + self.advance_data()?; } } if pts.is_some() { @@ -818,6 +821,23 @@ impl Import { Ok(()) } + /// Carry the media clock to the data tracks, which with aligned groups group on its grid + /// whether or not they carry anything. + fn advance_data(&mut self) -> anyhow::Result<()> { + let Some(now) = self.last_pts else { + return Ok(()); + }; + for section in self.sections.values_mut() { + section.track.advance(now)?; + } + for stream in self.streams.values_mut() { + if let Stream::Verbatim(stream) = stream { + stream.track.advance(now)?; + } + } + Ok(()) + } + /// Publish the sections held for the timebase's offset, now that it is known. They are stamped /// with the section clock at release: no timeline existed when they arrived. fn release_sections(&mut self, offset: Offset) -> anyhow::Result<()> { @@ -1350,6 +1370,7 @@ fn register_verbatim( stream_type: u8, framing: catalog::Framing, descriptors: Vec, + grid: bool, ) -> anyhow::Result> { // Verbatim payloads ride the legacy container, which normalizes the per-frame // timestamp to microseconds on the wire (see `hang::container::Frame::encode`), @@ -1366,10 +1387,13 @@ fn register_verbatim( // timeline track can collide), and the `VerbatimEntry` that removes this catalog // entry on drop only exists once this function returns successfully, so an entry // published first would be stranded. - let media = catalog.media_raw( + let mut media = catalog.media_raw( track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), )?; + if grid { + media = media.with_grid(); + } let mut guard = catalog.modify()?; let Some(mpegts) = guard.ext.mpegts_mut() else { @@ -1438,6 +1462,7 @@ impl SectionStream { pid: u16, stream_type: u8, descriptors: Vec, + grid: bool, ) -> anyhow::Result { let track = register_verbatim( &mut broadcast, @@ -1446,6 +1471,7 @@ impl SectionStream { stream_type, catalog::Framing::Section, descriptors, + grid, )?; let entry = VerbatimEntry { name: track.name().to_string(), @@ -1510,7 +1536,8 @@ impl SectionStream { fn discontinuity(&mut self, published: bool) -> anyhow::Result<()> { // The half-reassembled section is stamped with the clock that just ended. self.reassembler = SectionReassembler::default(); - if published { + // The grid's watch runs on the media clock's ticks, before anything publishes. + if published || self.track.on_grid() { self.track.discontinuity()?; } Ok(()) @@ -1582,6 +1609,7 @@ impl VerbatimStream { pid: u16, stream_type: u8, descriptors: Vec, + grid: bool, ) -> anyhow::Result { let track = register_verbatim( &mut broadcast, @@ -1590,6 +1618,7 @@ impl VerbatimStream { stream_type, catalog::Framing::Pes, descriptors, + grid, )?; let entry = VerbatimEntry { name: track.name().to_string(), @@ -1634,7 +1663,8 @@ impl VerbatimStream { } fn discontinuity(&mut self, published: bool) -> anyhow::Result<()> { - if published { + // The grid's watch runs on the media clock's ticks, before anything publishes. + if published || self.track.on_grid() { self.track.discontinuity()?; } Ok(()) @@ -6996,6 +7026,7 @@ pub(super) mod test { timebase .place(Timestamp::ZERO, epoch - Duration::from_micros(WRAP_MICROS)) .unwrap(); + timebase.align_groups(); let mut import = super::Import::new(broadcast, timebase.reserve()); let pmt = synth_pmt( &[(StreamType::Mpeg1Audio, PCR_PID), (StreamType::Mpeg1Audio, PEER_PID)], @@ -7033,9 +7064,34 @@ pub(super) mod test { frames } - /// Importers placed by one epoch publish one timeline wherever each joined the source: one - /// from before a wrap, one at it with its streams' first values either side of it, and one - /// after it on a host clock 7 ms ahead, taking the source 40 ms late. + /// Each rendition `catalog` lists, by name: every group's sequence and the fills of its frames. + async fn groups( + consumer: &moq_net::broadcast::Consumer, + catalog: &crate::catalog::Producer, + ) -> BTreeMap)>> { + let mut tracks = BTreeMap::new(); + for name in catalog.snapshot().audio.renditions.into_keys() { + let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(3600)); + let mut track = consumer.track(&name).unwrap().subscribe(subscription).await.unwrap(); + let mut groups = Vec::new(); + while let Ok(Ok(Some(mut group))) = + tokio::time::timeout(Duration::from_millis(50), track.recv_group()).await + { + let mut fills = Vec::new(); + while let Some(frame) = group.read_frame().await.unwrap() { + let frame = hang::container::Frame::decode(frame.payload).unwrap(); + fills.extend(frame.payload.get(4)); + } + groups.push((group.sequence, fills)); + } + tracks.insert(name, groups); + } + tracks + } + + /// Importers placed by one epoch publish one timeline and the same groups wherever each joined + /// the source: one from before a wrap, one at it with its streams' first values either side of + /// it, and one after it on a host clock 7 ms ahead, taking the source 40 ms late. #[tokio::test(start_paused = true)] async fn importers_sharing_an_epoch_publish_one_timeline_wherever_each_started() { let epoch = epoch_time(); @@ -7043,7 +7099,7 @@ pub(super) mod test { let first = WRAP - 10 * STEP; let (consumer, catalog, mut import) = epoch_import(epoch, start); let (mut at_wrap, mut late) = (None, None); - for i in 0..30 { + for i in 0..40 { let now = start + Duration::from_micros(i * 144_000); if i == 10 { at_wrap = Some(epoch_import(epoch, now)); @@ -7064,13 +7120,30 @@ pub(super) mod test { } import.finish().unwrap(); let all = timeline(&consumer, &catalog).await; - assert_eq!(all.len(), 120, "every frame of both streams"); - let pcr: Vec = (1..=60).map(|fill| all[&fill]).collect(); + // Nothing from the second it started in, at most fourteen 72 ms frames, then every frame. + let pcr: Vec = all.keys().copied().filter(|fill| *fill <= 80).collect(); + let peer: Vec = all.keys().copied().filter(|fill| *fill > 80).collect(); + assert!( + pcr.len() >= 66 && pcr == (81 - pcr.len() as u8..=80).collect::>(), + "{pcr:?}" + ); + assert!( + peer.len() >= 66 && peer == (0xd0 - peer.len() as u8..=0xcf).collect::>(), + "{peer:?}" + ); + let pcr: Vec = pcr.iter().map(|fill| all[fill]).collect(); assert!( pcr.windows(2).all(|w| (71_999..=72_001).contains(&(w[1] - w[0]))), "the first importer runs on through the wrap: {pcr:?}" ); assert_eq!(catalog.snapshot().clock, Some(epoch_clock(epoch))); + let first_groups = groups(&consumer, &catalog).await; + for (name, groups) in &first_groups { + assert!( + groups.windows(2).all(|pair| pair[1].0 == pair[0].0 + 1), + "{name}: consecutive sequences through the wrap" + ); + } let names = |catalog: &crate::catalog::Producer| catalog.snapshot().audio.renditions.into_keys().collect::>(); @@ -7088,10 +7161,12 @@ pub(super) mod test { "frame {fill:#x} of the importer joined {when}" ); } - if when == "at the wrap" { + for (name, groups) in groups(&consumer, &joined_catalog).await { + assert!(!groups.is_empty(), "{when}: {name} published no group"); assert!( - frames.contains_key(&21) && frames.contains_key(&0x94), - "its streams began either side of the wrap" + first_groups[&name].ends_with(&groups), + "{when}: {name}'s groups {groups:?} are not the first importer's {:?}", + first_groups[&name] ); } } @@ -7396,7 +7471,7 @@ pub(super) mod test { let jump = 600 * 90_000; let (consumer, catalog, mut import) = epoch_import(epoch, start); let mut joined = None; - for i in 0..20 { + for i in 0..30 { if i == 15 { joined = Some(epoch_import(epoch, start + Duration::from_micros(i * 144_000))); } @@ -7420,11 +7495,12 @@ pub(super) mod test { "the jump adds its step: {step} µs" ); + let first_groups = groups(&consumer, &catalog).await; let (consumer, joined_catalog, mut joined) = joined.unwrap(); joined.finish().unwrap(); assert_eq!(joined_catalog.snapshot().clock, catalog.snapshot().clock); let frames = timeline(&consumer, &joined_catalog).await; - assert!(frames.contains_key(&31), "{frames:?}"); + assert!(frames.len() >= 20, "{frames:?}"); for (fill, micros) in &frames { assert_eq!( all.get(fill), @@ -7432,6 +7508,13 @@ pub(super) mod test { "frame {fill:#x} of the importer that joined after the jump" ); } + for (name, groups) in groups(&consumer, &joined_catalog).await { + assert!( + first_groups[&name].ends_with(&groups), + "{name}'s groups {groups:?} are not the first importer's {:?}", + first_groups[&name] + ); + } } /// A flagged jump carrying the source more than half a wrap from the epoch's clock reads as a @@ -7467,20 +7550,21 @@ pub(super) mod test { // An hour short of the wrap. let first = WRAP - 3600 * 90_000; let (consumer, catalog, mut import) = epoch_import(epoch, start); - for i in 0..5 { + for i in 0..20 { import.decode(&source_pes(i, first + i * STEP)).unwrap(); tokio::time::advance(Duration::from_millis(144)).await; } import.finish().unwrap(); let before = timeline(&consumer, &catalog).await; + let stopped = groups(&consumer, &catalog).await; let clock = catalog.snapshot().clock; drop((consumer, catalog, import)); let outage = Duration::from_secs(2 * 3600); tokio::time::advance(outage).await; - let resumed = start + Duration::from_micros(5 * 144_000) + outage; + let resumed = start + Duration::from_micros(20 * 144_000) + outage; let (consumer, catalog, mut import) = epoch_import(epoch, resumed); - for i in 5..10 { + for i in 20..40 { let pts = (first + i * STEP + outage.as_secs() * 90_000) % WRAP; import.decode(&source_pes(i, pts)).unwrap(); tokio::time::advance(Duration::from_millis(144)).await; @@ -7488,11 +7572,18 @@ pub(super) mod test { import.finish().unwrap(); let after = timeline(&consumer, &catalog).await; assert_eq!(catalog.snapshot().clock, clock); - let gap = after[&11] - before[&1]; + // The last step of each run, since a run publishes nothing from the second it started in. + let gap = after[&79] - before[&39]; assert!( - (7_200_719_999..=7_200_720_001).contains(&gap), + (7_202_879_999..=7_202_880_001).contains(&gap), "the outage is a gap of {gap} µs" ); + for (name, groups) in groups(&consumer, &catalog).await { + // 7,200.072 s from the last frame before the outage to the first after it, plus the + // second the resumed run drops. + let (last, first) = (stopped[&name].last().unwrap().0, groups[0].0); + assert!((7_201..=7_202).contains(&(first - last)), "{name}: {last} then {first}"); + } } /// A 5-byte PES timestamp field with the 4-bit `prefix` (0b0010 PTS only, 0b0011 PTS of a diff --git a/rs/moq-mux/src/container/ts/passthrough.rs b/rs/moq-mux/src/container/ts/passthrough.rs index 32ad35b4ea9..f71cc16199a 100644 --- a/rs/moq-mux/src/container/ts/passthrough.rs +++ b/rs/moq-mux/src/container/ts/passthrough.rs @@ -27,6 +27,14 @@ //! a random access point. A group the PCR discontinuity starts is behind a break even when it is //! also a random access point. //! +//! On a timebase whose groups are [aligned](crate::catalog::Timebase::align_groups), the track +//! groups on the catalog clock's one-second grid (see +//! [`container::Producer`](crate::container::Producer#grid)), and the fallback grid's lines are +//! that clock's. A group then opens only at the first start in a second, numbered by that +//! second, and later starts in it join it, a flagged jump's included. A second with no start gets +//! an empty group, and no break marker is published. A group that reaches either bound ends the +//! import with [`GroupFullError`], since it cannot be split. +//! //! Bytes before the first group are dropped: until the PMT, a PCR, and a group start have all //! been seen there is no group to put them in. A jump before the first group starts none, and a //! line of the grid starts it only once the importer has watched for random access through the @@ -68,8 +76,8 @@ //! wrap back. The timeline therefore depends only on the bytes and the clock, and the grids above //! lie on it, so two importers of one feed whose timebases one shared epoch placed (see //! [`Timebase::place`](crate::catalog::Timebase::place)) publish identical objects, groups and -//! sections wherever each started, from the later one's first group on (group sequence numbers -//! aside). Objects timed past the last PCR before a flagged jump smaller than their lead can land +//! sections wherever each started, from the later one's first group on, under the same group +//! sequences. Objects timed past the last PCR before a flagged jump smaller than their lead can land //! after the first ones behind it; if one of them starts a group, the group behind the break would //! start before it, and the import ends there. @@ -81,6 +89,7 @@ use mpeg2ts::ts::{Pid, TsPacket}; use super::import::{Framer, PatReader, SectionReassembler, adaptation_valid, discontinuity_indicator, pcr}; use super::mux_rate::{PCR_WRAP, WINDOW, Windows}; use super::psi::{self, Pmt}; +use crate::container::grid::GRID_MICROS; /// The PCR clock, in Hz. const PCR_HZ: u64 = 27_000_000; @@ -179,6 +188,23 @@ pub struct PcrRestartError { pub to: u64, } +/// On a timebase whose groups are [aligned](crate::catalog::Timebase::align_groups), a group +/// outgrew [`GROUP_BYTES_MAX`] or [`GROUP_OBJECTS_MAX`]. +/// +/// Groups are then numbered by the second they start in, so one cannot be split to stay under +/// the bound as it is otherwise. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +#[error("group {sequence} reached {bytes} bytes in {objects} objects; an aligned group cannot split")] +pub struct GroupFullError { + /// The group's sequence. + pub sequence: u64, + /// The bytes it holds. + pub bytes: u64, + /// The objects it holds. + pub objects: usize, +} + /// The clock PID carried no PCR across more bytes than a group may hold. /// /// Those bytes cannot be timed, and holding them for a PCR that may never come is unbounded. @@ -204,6 +230,11 @@ pub struct Passthrough { /// Puts every PCR in its era and shifts every object's PCR time onto the catalog clock, /// anchored by the first PCR. shift: crate::catalog::Timebase, + /// Whether the timebase was placed, so the track groups on the catalog clock's grid and the + /// fallback cuts on its lines (see [`container::Producer`](crate::container::Producer#grid)). + grid: bool, + /// On the grid, the group the producer holds open and what it holds. + held: Option<(u64, Group)>, media: crate::container::Producer, /// The section as last written to the catalog, `None` before the first group. section: Option>, @@ -330,13 +361,20 @@ impl Passthrough { pub fn new(broadcast: moq_net::broadcast::Producer, reserved: crate::catalog::Reserved) -> crate::Result { let catalog = reserved.producer(); let track = broadcast.unique_track(".m2ts", catalog.track_info(hang::catalog::PRIORITY.video))?; - let media = catalog.media_raw( + let shift = reserved.timebase(); + let grid = shift.aligned(); + let mut media = catalog.media_raw( track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), )?; + if grid { + media = media.with_grid(); + } Ok(Self { catalog, - shift: reserved.timebase(), + shift, + grid, + held: None, reservation: Some(reserved), media, section: None, @@ -560,12 +598,15 @@ impl Passthrough { raw, samples: VecDeque::from([Sample { at, ticks }]), }); - self.watching_since = self.mapped.then_some(ticks); + if self.mapped { + self.watch(ticks)?; + } // A flagged first PCR is a jump an importer running since before counted against the // grid, the original if this one is a retransmitted copy, so it counts here too. if flagged { self.last_mark = Some(ticks); } + self.advance(ticks)?; self.windows.pcr(ticks, false); self.window_random_access = self.single; return Ok(None); @@ -594,6 +635,12 @@ impl Passthrough { // before this one's first group. self.last_mark = Some(ticks); self.end_window(ticks, true)?; + // On the catalog clock's grid the break lands here, between the bytes timed on + // either side of it, whether or not this importer has started. + if self.grid { + self.media.discontinuity()?; + } + self.advance(ticks)?; return Ok(self.started().then_some(Start::Discontinuity)); } @@ -603,21 +650,68 @@ impl Passthrough { timebase.samples.pop_front(); } self.drain_queue()?; + self.advance(ticks)?; self.end_window(ticks, false)?; - if self.mapped { - self.watching_since.get_or_insert(ticks); + if self.mapped && self.watching_since.is_none() { + self.watch(ticks)?; } // The first PCR at or past a line of the grid cuts there, unless a random access point or // a jump fell in the spacing before the line. Before the first group, only a line whose // whole spacing was watched may: an importer running since before would cut there too. - let line = ticks / FALLBACK_TICKS * FALLBACK_TICKS; - let crossed = last < line; - let marked = self.last_mark.is_some_and(|mark| mark + FALLBACK_TICKS >= line); - let watched = self.started() || self.watching_since.is_some_and(|since| since + FALLBACK_TICKS <= line); + // With aligned groups the lines are the catalog clock's, which number the groups. + let spacing = match self.grid { + true => i128::from(GRID_MICROS), + false => i128::from(FALLBACK_TICKS), + }; + let point = |at: u64| -> anyhow::Result { + Ok(match self.grid { + true => self.catalog_time(at)?.as_micros() as i128, + false => i128::from(at), + }) + }; + let line = point(ticks)?.div_euclid(spacing) * spacing; + let crossed = point(last)? < line; + let marked = match self.last_mark { + Some(mark) => point(mark)? + spacing >= line, + None => false, + }; + let watched = self.started() + || match self.watching_since { + Some(since) => point(since)? + spacing <= line, + None => false, + }; Ok((crossed && !marked && watched && self.can_start()).then_some(Start::Fallback)) } + /// The catalog clock's time at `ticks` on the PCR timeline, as an object there is stamped. + fn catalog_time(&self, ticks: u64) -> anyhow::Result { + let timestamp = moq_net::Timestamp::from_micros(ticks / (PCR_HZ / 1_000_000))?; + Ok(self.shift.shift(timestamp)?) + } + + /// Every random access point from the PCR at `ticks` on will be seen. On the grid the track + /// learns which cells it watched whole: those whose line comes a spacing after it, as a + /// fallback cut there needs. + fn watch(&mut self, ticks: u64) -> anyhow::Result<()> { + self.watching_since = Some(ticks); + if self.grid { + let since = self.catalog_time(ticks + FALLBACK_TICKS)?; + self.media.watch(since); + } + Ok(()) + } + + /// On the grid, carry the PCR at `ticks` to the track, so the cells it passes with no group + /// start get their empty groups. + fn advance(&mut self, ticks: u64) -> anyhow::Result<()> { + if self.grid && self.watching_since.is_some() { + let now = self.catalog_time(ticks)?; + self.media.advance(now)?; + } + Ok(()) + } + /// The section for the window a PCR at `ticks` ended, if it ended one: written once the window /// starts after the PCR the first group follows, since every group start and layout change in /// it is then one an importer running since before saw the same way. A window starting on that @@ -667,18 +761,19 @@ impl Passthrough { } fn write(&mut self, object: Object, ticks: u64) -> anyhow::Result<()> { - let timestamp = moq_net::Timestamp::from_micros(ticks / (PCR_HZ / 1_000_000))?; - let timestamp = self.shift.shift(timestamp)?; + let timestamp = self.catalog_time(ticks)?; let size = object.payload.len() as u64; let start = object.start.or_else(|| { self.group .as_ref() - .filter(|group| group.bytes + size > GROUP_BYTES_MAX || group.objects >= GROUP_OBJECTS_MAX) + .filter(|group| { + !self.grid && (group.bytes + size > GROUP_BYTES_MAX || group.objects >= GROUP_OBJECTS_MAX) + }) .map(|_| Start::Full) }); if let Some(start) = start { - if object.discontinuity && self.group.is_some() { + if object.discontinuity && self.group.is_some() && !self.grid { self.media.discontinuity()?; } if start != Start::RandomAccess || !self.single { @@ -693,12 +788,37 @@ impl Passthrough { let group = self.group.as_mut().expect("the first object starts a group"); group.bytes += size; group.objects += 1; - self.media.write(crate::container::Frame { + // On the grid an object in a second this importer publishes nothing from has no group. + match self.media.write(crate::container::Frame { timestamp, duration: None, payload: object.payload.freeze(), keyframe: start.is_some(), - })?; + }) { + Err(crate::Error::MissingKeyframe(_)) if self.grid => return Ok(()), + written => written?, + } + + // A start in a cell already open joins its group, so the bound is kept on the group as the + // grid holds it, and reaching it ends the import rather than splitting the cell. + if self.grid + && let Some(sequence) = self.media.open_group() + { + if self.held.as_ref().is_none_or(|(held, _)| *held != sequence) { + self.held = Some((sequence, Group::default())); + } + let (_, held) = self.held.as_mut().expect("held above"); + held.bytes += size; + held.objects += 1; + if held.bytes > GROUP_BYTES_MAX || held.objects > GROUP_OBJECTS_MAX { + return Err(GroupFullError { + sequence, + bytes: held.bytes, + objects: held.objects, + } + .into()); + } + } Ok(()) } diff --git a/rs/moq-mux/src/container/ts/passthrough_test.rs b/rs/moq-mux/src/container/ts/passthrough_test.rs index 96c5b34942a..1ee9a9d6f62 100644 --- a/rs/moq-mux/src/container/ts/passthrough_test.rs +++ b/rs/moq-mux/src/container/ts/passthrough_test.rs @@ -12,7 +12,8 @@ use bytes::Bytes; use super::fixture::{ACCESS, Flags, MS, Mux, WRAP, paced, program}; use super::import::test::{WRAP_MICROS, epoch_clock, epoch_time}; use super::passthrough::{ - GROUP_BYTES_MAX, GROUP_OBJECTS_MAX, PacketSizeError, Passthrough, PcrGapError, PcrRestartError, PcrRewindError, + GROUP_BYTES_MAX, GROUP_OBJECTS_MAX, GroupFullError, PacketSizeError, Passthrough, PcrGapError, PcrRestartError, + PcrRewindError, }; /// A delay budget no test timeline comes close to, so the reader sees every group. @@ -34,6 +35,8 @@ struct Imported { objects: Vec, /// The sequence of each break marker: a group of one object with no bytes. markers: Vec, + /// Every group's sequence, empty ones included, in the order received. + sequences: Vec, } impl Imported { @@ -115,7 +118,9 @@ async fn read_back( .await?; let mut objects = Vec::new(); let mut markers = Vec::new(); + let mut sequences = Vec::new(); while let Some(mut group) = tokio::time::timeout(Duration::from_secs(1), track.recv_group()).await?? { + sequences.push(group.sequence); while let Some(frame) = group.read_frame().await? { if frame.payload.is_empty() { markers.push(group.sequence); @@ -133,6 +138,7 @@ async fn read_back( catalog: catalog.snapshot(), objects, markers, + sequences, }) } @@ -670,12 +676,21 @@ struct Placed { impl Placed { fn new(epoch: std::time::SystemTime) -> Self { + Self::on(epoch, true) + } + + /// An importer on a timebase `epoch` places, which also aligns its groups when `aligned`, as the + /// CLI's shared epoch does. + fn on(epoch: std::time::SystemTime, aligned: bool) -> Self { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); let timebase = catalog.timebase(); timebase .place(moq_net::Timestamp::ZERO, epoch - Duration::from_micros(WRAP_MICROS)) .unwrap(); + if aligned { + timebase.align_groups(); + } let passthrough = Passthrough::new(broadcast.clone(), timebase.reserve()).unwrap(); Self { broadcast: broadcast.consume(), @@ -699,23 +714,25 @@ fn starts(imported: &Imported) -> Vec { .collect() } -/// Import `data` placed by one epoch from its first byte, and again, on a host clock 7 ms ahead, -/// from the program and then the packet carrying PCR `join`, the two fed in step. From the second's -/// first group on, both must publish the same objects in the same groups, and from the second's -/// first section on, the same section after every PCR. +/// [`import_twice`] with aligned groups, the second importer joining at the packet carrying PCR +/// `join`. async fn import_twice_placed(data: &[u8], prefix: usize, join: u64) -> (Imported, Imported) { - import_twice_placed_from(data, prefix, pcr_at(data, join)).await + import_twice(data, prefix, pcr_at(data, join), true).await } -/// [`import_twice_placed`], the second importer joining at byte `split`. -async fn import_twice_placed_from(data: &[u8], prefix: usize, split: usize) -> (Imported, Imported) { +/// Import `data` placed by one epoch from its first byte, and again, on a host clock 7 ms ahead, +/// from the program and then byte `split`, the two fed in step. From the second's first group on, +/// both must publish the same objects in the same groups, and when `aligned` under the same +/// sequences, empty groups included; from the second's first section on, the same section after +/// every PCR. +async fn import_twice(data: &[u8], prefix: usize, split: usize, aligned: bool) -> (Imported, Imported) { let epoch = epoch_time(); let hosts = [epoch + Duration::from_secs(600), epoch + Duration::from_millis(600_007)]; crate::clock::fake_host_wall(hosts[0]); - let mut first = Placed::new(epoch); + let mut first = Placed::on(epoch, aligned); first.passthrough.decode(&data[..split]).unwrap(); crate::clock::fake_host_wall(hosts[1]); - let mut second = Placed::new(epoch); + let mut second = Placed::on(epoch, aligned); second.passthrough.decode(&data[..prefix]).unwrap(); let mut described = false; for pkt in data[split..].chunks(188) { @@ -745,9 +762,24 @@ async fn import_twice_placed_from(data: &[u8], prefix: usize, split: usize) -> ( ); } assert_eq!(starts(&first)[skip..], starts(&second), "group starts"); + if aligned { + let groups = |imported: &Imported| imported.objects.iter().map(|object| object.group).collect::>(); + assert_eq!(groups(&first)[skip..], groups(&second), "each object's group sequence"); + assert!( + first.sequences.ends_with(&second.sequences), + "the joiner's groups: {:?} vs {:?}", + second.sequences, + first.sequences + ); + } (first, second) } +/// Whether `imported` skipped no sequence, as a timeline with no break never does on the grid. +fn consecutive(imported: &Imported) -> bool { + imported.sequences.windows(2).all(|pair| pair[1] == pair[0] + 1) +} + /// Importers placed by one epoch publish identical objects across a PCR wrap, the second joining /// after it. #[tokio::test(start_paused = true)] @@ -820,7 +852,8 @@ async fn importers_sharing_an_epoch_cut_the_same_groups() { } let data = mux.bytes(); for join in [2_520, 4_720, 7_360] { - let (first, _) = import_twice_placed(&data, prefix, join * MS).await; + let (first, second) = import_twice_placed(&data, prefix, join * MS).await; + assert!(consecutive(&first) && consecutive(&second), "gop {gop}, join {join}"); let fallback = first .groups() .iter() @@ -834,8 +867,8 @@ async fn importers_sharing_an_epoch_cut_the_same_groups() { } } -/// Importers placed by one epoch split groups at the object ceiling alike, the ceiling counted -/// from a group start both made. +/// Importers placed by one epoch without aligned groups split groups at the object ceiling alike, +/// the ceiling counted from a group start both made. #[tokio::test(start_paused = true)] async fn importers_sharing_an_epoch_split_full_groups_alike() { let mut mux = Mux::default(); @@ -847,7 +880,7 @@ async fn importers_sharing_an_epoch_split_full_groups_alike() { mux.packet(0x100, Flags::pcr(i * 5_400)); } let data = mux.bytes(); - let (first, second) = import_twice_placed(&data, prefix, 3_300 * MS).await; + let (first, second) = import_twice(&data, prefix, pcr_at(&data, 3_300 * MS), false).await; let full = second .groups() .iter() @@ -857,6 +890,28 @@ async fn importers_sharing_an_epoch_split_full_groups_alike() { assert!(!first.section().random_access); } +/// With aligned groups, a group is its second of the grid and cannot split, so one that reaches the +/// object ceiling ends the import. +#[tokio::test(start_paused = true)] +async fn an_aligned_group_at_the_ceiling_ends_the_import() { + let mut mux = Mux::default(); + program(&mut mux, 0x100); + mux.packet(0x100, Flags::pcr(0).random_access()); + // A PCR in every packet, 200 µs apart: 5,000 objects a second. + for i in 1..15_000 { + mux.packet(0x100, Flags::pcr(i * 5_400)); + } + let data = mux.bytes(); + crate::clock::fake_host_wall(epoch_time() + Duration::from_secs(600)); + let mut placed = Placed::new(epoch_time()); + let err = data + .chunks(188) + .try_for_each(|pkt| placed.passthrough.decode(pkt)) + .expect_err("a group past the ceiling"); + let full = err.downcast_ref::().expect("a full group"); + assert_eq!(full.objects, GROUP_OBJECTS_MAX + 1); +} + /// A joiner whose first PCR lands exactly on a window's line misses the grid cut its partner made /// there, so the first window it describes is the one after: describing that window would publish /// random access the partner lost to the cut. @@ -886,7 +941,7 @@ async fn a_joiner_on_a_window_line_describes_the_window_after() { } /// A joiner whose first PCR is the retransmitted copy of a flagged jump on a line of the grid counts -/// the jump there as its partner did, so neither cuts on the line after. +/// the jump there as its partner did, so neither cuts on the line after, with groups aligned or not. #[tokio::test(start_paused = true)] async fn a_joiner_on_a_retransmitted_jump_counts_it_against_the_grid() { const LINE: u64 = 1_000 * MS; @@ -902,12 +957,15 @@ async fn a_joiner_on_a_retransmitted_jump_counts_it_against_the_grid() { mux.fill(0x101, 4); paced(&mut mux, 0x100, jump + 40 * MS, 250, 0, 4); let data = mux.bytes(); - import_twice_placed_from(&data, prefix, pcr_at(&data, jump) + 188).await; + let split = pcr_at(&data, jump) + 188; + for aligned in [false, true] { + import_twice(&data, prefix, split, aligned).await; + } } -/// The grid lies on the timeline the epoch places, so it runs on regularly across a PCR wrap that -/// would shift a grid of raw ticks by about 0.72 s, and an importer joining after the wrap cuts on -/// it too. +/// The grid lies on the catalog clock the epoch places, so it runs on regularly across a PCR wrap +/// that would shift a grid of raw ticks by about 0.72 s, and an importer joining after the wrap +/// cuts on it too, each second's group numbered by that second. #[tokio::test(start_paused = true)] async fn the_grid_runs_on_across_a_wrap() { const WRAP: u64 = (1 << 33) * 300; @@ -920,15 +978,24 @@ async fn the_grid_runs_on_across_a_wrap() { mux.fill(0x101, 4); } let data = mux.bytes(); - let (first, _) = import_twice_placed(&data, prefix, 2_000 * MS).await; + let (first, second) = import_twice_placed(&data, prefix, 2_000 * MS).await; let groups = first.groups(); assert!(groups.len() >= 10, "{} groups", groups.len()); for group in &groups { - let pcr = header(&group[0].payload).pcr.expect("a cut on the grid falls on a PCR"); - let unwrapped = if pcr >= start { pcr } else { pcr + WRAP }; - let past = unwrapped % (1_000 * MS); - assert!(past < 40 * MS, "a cut {} ms past the line", past / MS); + assert!( + header(&group[0].payload).pcr.is_some(), + "a cut on the grid falls on a PCR" + ); + // An object starts ten bytes ahead of its PCR, so the one at the line can sit just short of it. + let micros = group[0].micros + 1_000; + assert!( + micros % 1_000_000 < 41_000, + "a cut {} µs past the line", + micros % 1_000_000 + ); + assert_eq!(group[0].group, micros / 1_000_000, "numbered by its second"); } + assert!(consecutive(&first) && consecutive(&second)); } /// With every PCR in the era nearest the clock and nothing clamping it, a flagged jump forward by From 1af2fefbb8b788e9b329afb0eb2fc470dbf1b244 Mon Sep 17 00:00:00 2001 From: t0ms Date: Sun, 18 Oct 2026 11:20:28 +0100 Subject: [PATCH 09/11] fix(mux): keep grid tracks safe across cuts, rewinds and early breaks - A video cut on the grid now waits for a keyframe, as a break does: the pictures after it may reference what the loss left out. - An fMP4 fragment below its track's latest segment start ends the import as a rewind instead of joining the open group. - A TS clock break restarts a video track's grid even before it has published, since its deltas before the first keyframe move the watch. - The draft states that publishers numbering m2ts groups by the second publish no discontinuity marker for a flagged jump. Co-authored-by: Claude Opus 5.5 --- drafts/draft-lcurley-moq-mpegts.md | 9 ++- rs/moq-mux/src/codec/h264/import.rs | 5 ++ rs/moq-mux/src/codec/h265/import.rs | 5 ++ rs/moq-mux/src/container/fmp4/import.rs | 23 ++++++ rs/moq-mux/src/container/fmp4/import_test.rs | 30 ++++++++ rs/moq-mux/src/container/producer.rs | 59 +++++++++++++-- rs/moq-mux/src/container/ts/import.rs | 79 +++++++++++++++++++- 7 files changed, 199 insertions(+), 11 deletions(-) diff --git a/drafts/draft-lcurley-moq-mpegts.md b/drafts/draft-lcurley-moq-mpegts.md index 20c8298ba27..9129f3bc98d 100644 --- a/drafts/draft-lcurley-moq-mpegts.md +++ b/drafts/draft-lcurley-moq-mpegts.md @@ -317,6 +317,12 @@ With more programs or video streams the random access points can stagger, so a g Both indicators sit in the adaptation field, which TS-level scrambling leaves in the clear, so a scrambled multiplex groups like its clear twin. A publisher parses the PAT and the PMT and nothing else past the adaptation field. +Publishers that share an epoch and number each group by the second of the catalog clock it starts in, so that each publishes the same groups under the same sequences, differ from the rules above in four ways. +The grid lines are the catalog clock's whole seconds. +A group opens only at the first start in a second no group has taken, and later starts in that second join it, a flagged jump's included: such a group can contain a PCR discontinuity, and no discontinuity marker precedes the jump, which a consumer sees as the timestamps' step, or as a sequence hole ({{hang}}) when the jump skips whole seconds. +A second with no start gets an empty group. +A group that reaches the size bound ends the track with an error, since it cannot be split. + ## Timestamps {#m2ts-timestamps} A frame's timestamp is the PCR time of its first byte, in microseconds rounded down. A PCR gives the time of the byte holding the last bit of its `program_clock_reference_base` ({{mpeg2}}), ten bytes into its packet. @@ -324,7 +330,7 @@ A byte between two PCRs is placed at the rate between them; a byte past the last PCRs are unwrapped into a 64-bit count of 27 MHz ticks, and timestamps never decrease. Each PCR takes the number of 2^33 × 300 tick wraps that lands it nearest the current time on the catalog clock ({{hang}}), so the count runs on across the wrap, which is not a discontinuity, and a PCR more than half a wrap ahead of that time lands a wrap back. -A PCR with `discontinuity_indicator` set that lands at or past the one before it adds its step too, after the bytes before it are timed at the rate before it, and the group it starts MUST follow hang's discontinuity marker ({{hang}}). +A PCR with `discontinuity_indicator` set that lands at or past the one before it adds its step too, after the bytes before it are timed at the rate before it, and the group it starts MUST follow hang's discontinuity marker ({{hang}}), unless the publisher numbers groups by the second ({{m2ts-groups}}). A PCR that steps backwards MUST end the track with an error, flagged or not: a flagged one is a restart, which is new content, so a publisher that carries on publishes it as a new broadcast. The count therefore depends only on the bytes and the catalog clock, so publishers of one input whose catalog clocks agree publish identical frames, group boundaries, and sections wherever each started, from the later one's first group on. @@ -449,6 +455,7 @@ A broadcast demultiplexed from a DVB transport stream: video and audio described - Added the `pesPacket` record: each frame one whole PES packet, timestamped with its placement. A consumer that predates it drops the track rather than refusing the catalog. - A flagged PCR jump on an `m2ts` track follows hang's discontinuity marker rather than a skipped group sequence. - An `m2ts` track's fallback cut falls on whole seconds of a PCR count whose wraps follow the catalog clock, and `randomAccess` and `muxRate` describe the latest two-second window rather than the whole track, so publishers sharing a clock cut and describe the multiplex alike. +- Publishers sharing an epoch may number `m2ts` groups by the catalog clock's second, which lets a flagged PCR jump join a group without a discontinuity marker. ## draft-lcurley-moq-mpegts-00 {:numbered="false"} diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 1d0a70db968..48894de1244 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -158,6 +158,11 @@ impl Import { Ok(()) } + /// Whether the track groups on the catalog clock's grid. + pub(crate) fn on_grid(&self) -> bool { + self.track.on_grid() + } + /// Close the current group and open the next one at `sequence`. pub fn seek(&mut self, sequence: u64) -> Result<()> { self.track.seek(sequence)?; diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 60f5256b18c..13812e9ee5b 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -163,6 +163,11 @@ impl Import { Ok(()) } + /// Whether the track groups on the catalog clock's grid. + pub(crate) fn on_grid(&self) -> bool { + self.track.on_grid() + } + /// Close the current group and open the next one at `sequence`. pub fn seek(&mut self, sequence: u64) -> Result<()> { self.track.seek(sequence)?; diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 4a9f1e8f0e7..49f0850a163 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -132,6 +132,11 @@ struct Fmp4Track { // (see `container::Producer`'s grid) and `seek` does nothing. grid: Option, + // On the grid, the time of the latest fragment that started a segment and of the one before it: + // no start may go below the first, and no other fragment below the second. + start: Option, + floor: Option, + // Measures the track's catalog jitter and bitrate from the fragments; the descriptor's own // bitrate, when it declares one, still wins over what this measures. estimator: Estimator, @@ -355,6 +360,8 @@ impl Import { group: None, segment: None, grid: self.timebase.aligned().then(Default::default), + start: None, + floor: None, recorder: Some(recorder), last_decode_time: None, sample_duration: None, @@ -1005,6 +1012,22 @@ impl Import { track.segment = Some(self.segment); let mut fill = 0..0; if let Some(grid) = track.grid.as_mut() { + // A fragment below the start before it ends the import, as it would off the grid by + // opening a group below the previous one, rather than joining the open group. Every + // start raises the floor, opened, joined or dropped, so importers end alike. + let floor = match start_group { + true => track.start, + false => track.floor, + }; + if let Some(floor) = floor + && timestamp.as_micros() < floor.as_micros() + { + return Err(crate::container::TimestampRewind { timestamp, floor }.into()); + } + if start_group { + track.floor = track.start; + track.start = Some(timestamp); + } match grid.place(timestamp, start_group, track.group.is_some()) { Place::Drop => continue, Place::Join => start_group = false, diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 687ab1ea683..817291551f7 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2636,3 +2636,33 @@ async fn importers_sharing_an_epoch_number_segments_alike() { } } } + +/// On a shared epoch's timebase a segment that starts below the one before ends the import, as it +/// does off the grid, rather than joining the open group of the second it falls in, across a +/// discontinuity too. +#[tokio::test] +async fn a_rewound_segment_on_a_shared_epoch_ends_the_import() { + let epoch = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_millis(1_760_000_000_000); + crate::clock::fake_host_wall(epoch + std::time::Duration::from_secs(600)); + let (init, (id, scale), _) = bbb_init(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let timebase = catalog.timebase(); + timebase.place(moq_net::Timestamp::ZERO, epoch).unwrap(); + timebase.align_groups(); + let mut fmp4 = super::Import::new(broadcast, timebase.reserve()); + fmp4.decode(&init).unwrap(); + let segment = |fmp4: &mut super::Import, i: u32, at: u64| { + fmp4.decode(&styp())?; + let frames = [sample(600_000_000 + at * 1_000, true, Some(200_000))]; + fmp4.decode(&super::encode_fragment(info(id, scale, i), &frames).unwrap()) + }; + for (i, at) in [0, 1_000, 1_500].into_iter().enumerate() { + segment(&mut fmp4, i as u32, at).unwrap(); + } + fmp4.discontinuity(); + assert!(matches!( + segment(&mut fmp4, 3, 1_200), + Err(crate::Error::TimestampRewind(_)) + )); +} diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 312137a385b..494f96333b6 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -54,8 +54,8 @@ fn timestamp_lt(left: moq_net::Timestamp, right: moq_net::Timestamp) -> bool { /// sequences wherever each started. A group opens at the first start in each one-second cell of the /// catalog clock and takes that cell's index as its sequence: a video keyframe, any audio frame, or /// a keyframe on a data track. Later starts in the cell join it, and a cell with no start gets an -/// empty group, so sequences stay consecutive (see the `grid` module). [`cut`](Self::cut) and -/// [`seek`](Self::seek) do nothing. +/// empty group, so sequences stay consecutive (see the `grid` module). [`seek`](Self::seek) does +/// nothing, and [`cut`](Self::cut) closes no group. pub struct Producer { inner: moq_net::track::Producer, container: C, @@ -111,8 +111,9 @@ pub struct Producer { /// Set when the timeline decides the groups; see [`with_grid`](Self::with_grid). grid: Option, - /// On the grid, a break cut the open group and nothing has continued it since: it closes - /// without a bound, and a video track waits for a keyframe. + /// On the grid, a break or a video [`cut`](Self::cut) ended the open group's content and + /// nothing has continued it since: it closes without a bound, and a video track waits for a + /// keyframe. broken: bool, } @@ -671,9 +672,15 @@ where /// does not bound the last frame in decode order. The next [`write`](Self::write) /// must be a keyframe. An explicit bound before the last ordered video frame /// returns [`InvalidEnd`](super::InvalidEnd) without flushing or closing the group. - /// On the [grid](Self#grid) this does nothing. + /// On the [grid](Self#grid) the open group stays open, since only a start in a later cell + /// closes it, and an audio or data track's cut does nothing. On a video track the next frame + /// must still be a keyframe, as after a break: the pictures before it may reference what the + /// cut left out, and the group closes at the next start without a bound. pub fn cut(&mut self, end: Option) -> crate::Result<()> { if self.grid.is_some() { + if self.container.kind() == Kind::Video { + self.broken = true; + } return Ok(()); } self.bound(end) @@ -1932,7 +1939,8 @@ mod tests { groups } - /// A grid-mode producer of `kind` fed `frames`, skipping what a mid-stream join skips. + /// A grid-mode producer of `kind` fed `frames`, skipping what a mid-stream join skips. Audio and + /// data are cut after every frame, as the importers that send each frame at once cut them. async fn on_grid(kind: crate::container::Kind, frames: &[Frame]) -> Vec<(u64, Vec<(u128, usize)>)> { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let consumer = track.subscribe(replay()); @@ -1942,7 +1950,9 @@ mod tests { Err(crate::Error::MissingKeyframe(_)) => {} result => result.unwrap(), } - producer.cut(None).unwrap(); + if kind != crate::container::Kind::Video { + producer.cut(None).unwrap(); + } } producer.finish().unwrap(); grid_groups(consumer).await @@ -1950,7 +1960,7 @@ mod tests { /// On the grid, video producers started anywhere in one stream publish the same groups, frames /// and bounds under the same sequences, from the later one's first group on: keyframes 600 ms - /// apart share their second's group, and an importer's per-frame `cut` changes nothing. + /// apart share their second's group. #[tokio::test] async fn on_the_grid_video_producers_started_apart_publish_the_same_groups() { let frames: Vec = (0..200u64) @@ -2036,6 +2046,39 @@ mod tests { assert_eq!(sequences, [11, 12, 20, 21]); } + /// On the grid a video cut, such as an importer's after lost packets, keeps the cell's group open + /// but takes no delta until the next keyframe; one in the same cell joins the group, and one in a + /// later cell closes it without a bound. + #[tokio::test] + async fn on_the_grid_a_video_cut_waits_for_a_keyframe() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Video)).with_grid(); + producer.write(frame(10_000_000, true)).unwrap(); + producer.write(frame(11_000_000, true)).unwrap(); + producer.write(frame(11_040_000, false)).unwrap(); + for at in [11_500_000, 12_000_000] { + producer.cut(None).unwrap(); + assert!(matches!( + producer.write(frame(at - 420_000, false)), + Err(crate::Error::MissingKeyframe(_)) + )); + producer.write(frame(at, true)).unwrap(); + producer.write(frame(at + 40_000, false)).unwrap(); + } + producer.finish().unwrap(); + let groups = grid_groups(consumer).await; + assert_eq!( + groups[0], + ( + 11, + vec![(11_000_000, 2), (11_040_000, 2), (11_500_000, 2), (11_540_000, 2)] + ) + ); + assert_eq!(groups[1].0, 12); + assert_eq!(groups[1].1[..2], [(12_000_000, 2), (12_040_000, 2)]); + } + /// On the grid a sparse track ticked by the media clock gets each empty second's group as the /// second passes, from its first tick on. #[tokio::test] diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 2fa06807e97..04fbc5983ef 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -2213,14 +2213,16 @@ impl Stream { /// The program clock restarted: mark the break on this track. fn discontinuity(&mut self, published: bool) -> anyhow::Result<()> { match self { + // On the grid a delta before the first keyframe moves the watch without counting as + // published, so the break restarts it either way. Stream::H264 { import, .. } => { - if published { + if published || import.on_grid() { import.discontinuity()?; } Ok(()) } Stream::H265 { import, .. } => { - if published { + if published || import.on_grid() { import.discontinuity()?; } Ok(()) @@ -7517,6 +7519,79 @@ pub(super) mod test { } } + /// A video importer that joined mid-GOP has published nothing when a flagged jump comes, though + /// its deltas moved its grid's watch; the jump restarts that watch as it does one that has + /// published, so neither gives the seconds the jump skipped an empty group. + #[tokio::test(start_paused = true)] + async fn a_video_importer_joined_mid_gop_agrees_across_a_jump() { + const VIDEO: u16 = 0x0061; + const PICTURE: u64 = 3_600; + let epoch = epoch_time(); + let start = epoch + Duration::from_secs(600); + let placed = |host: std::time::SystemTime| { + crate::clock::fake_host_wall(host); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let config = crate::catalog::Config::default().with_max_age(Duration::from_secs(86_400)); + let catalog = crate::catalog::Producer::new(&mut broadcast, config).unwrap(); + let timebase = catalog.timebase(); + timebase + .place(Timestamp::ZERO, epoch - Duration::from_micros(WRAP_MICROS)) + .unwrap(); + timebase.align_groups(); + let mut import = super::Import::new(broadcast, timebase.reserve()); + let pmt = synth_pmt(&[(StreamType::H264, VIDEO)], false); + import.decode(&bytes::BytesMut::from(&pmt[..])).unwrap(); + (consumer, catalog, import) + }; + async fn sequences(consumer: &moq_net::broadcast::Consumer, catalog: &crate::catalog::Producer) -> Vec { + let name = catalog.snapshot().video.renditions.into_keys().next().unwrap(); + let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(3600)); + let mut track = consumer.track(&name).unwrap().subscribe(subscription).await.unwrap(); + let mut sequences = Vec::new(); + while let Ok(Ok(Some(group))) = tokio::time::timeout(Duration::from_millis(50), track.recv_group()).await { + sequences.push(group.sequence); + } + sequences + } + + // A keyframe every second, the jump five seconds on at picture 40, before the joiner's first + // keyframe, and a keyframe right behind it. + let first = WRAP - 300 * 90_000; + let jump = 5 * 90_000; + let (consumer, catalog, mut import) = placed(start); + let mut joined = None; + let mut mux = Mux::default(); + for i in 0..140u64 { + if i == 30 { + joined = Some(placed(start + Duration::from_millis(i * 40))); + } + let mut data = Vec::new(); + if i == 40 { + data.extend(clock_break_packet(VIDEO)); + } + let keyframe = if i < 40 { i % 25 == 0 } else { (i - 40) % 25 == 0 }; + let pts = (first + i * PICTURE + if i >= 40 { jump } else { 0 }) % WRAP; + let cc = mux.cc(VIDEO); + data.extend(video_pes(VIDEO, cc, pts, None, &annexb_au(keyframe))); + import.decode(&data).unwrap(); + if let Some((_, _, joined)) = joined.as_mut() { + joined.decode(&data).unwrap(); + } + tokio::time::advance(Duration::from_millis(40)).await; + } + import.finish().unwrap(); + let first_groups = sequences(&consumer, &catalog).await; + let (consumer, joined_catalog, mut joined) = joined.unwrap(); + joined.finish().unwrap(); + let joined_groups = sequences(&consumer, &joined_catalog).await; + assert!(!joined_groups.is_empty()); + assert!( + first_groups.ends_with(&joined_groups), + "{joined_groups:?} are not the first importer's {first_groups:?}" + ); + } + /// A flagged jump carrying the source more than half a wrap from the epoch's clock reads as a /// rewind, so every importer of a pair ends alike rather than publishing it a wrap away. #[tokio::test(start_paused = true)] From a3e7afa041a15648f9dcd6f6efee801d67ba39d6 Mon Sep 17 00:00:00 2001 From: t0ms Date: Mon, 19 Oct 2026 11:14:04 +0100 Subject: [PATCH 10/11] fix(mux): keep dropped fMP4 segments and audio-only cues on the grid - On the grid only a new segment starts an fMP4 group, so a segment whose first fragment an importer dropped stays dropped, rather than its next fragment opening the following second with media a partner keeps in the segment's group. - In a TS program with no video, audio advances the section clock and the data tracks' grids. Every section of an aligned audio-only program otherwise landed in the second the import started in, so its cue track published nothing. Off the grid, such cues are now stamped at the current audio PTS rather than the first PES's. Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/fmp4/import.rs | 5 +- rs/moq-mux/src/container/fmp4/import_test.rs | 52 +++++++-- rs/moq-mux/src/container/ts/import.rs | 111 ++++++++++++++++--- 3 files changed, 142 insertions(+), 26 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 49f0850a163..21de99f70d1 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -1006,7 +1006,10 @@ impl Import { // The keyframe bit no longer decides that: audio flags every sample a sync sample, so // keying off it opened a group (a QUIC stream) per audio fragment. let new_segment = track.segment != Some(self.segment); - let mut start_group = track.group.is_none() || new_segment; + // On the grid a track holds no group only before its first or after a dropped segment + // start, whose later fragments an importer running longer keeps in that segment's + // group: none of them is a start, even in the next second. + let mut start_group = new_segment || (track.group.is_none() && track.grid.is_none()); // A dropped fragment still moves the track into its segment, or the segment's next // fragment would read as a start that an importer which kept this one does not see. track.segment = Some(self.segment); diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 817291551f7..d8db057a675 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2563,8 +2563,9 @@ fn drain_groups(track: &mut moq_net::track::Subscriber) -> Vec<(u64, usize)> { } /// The video and audio groups of an fMP4 import on a shared epoch's timebase, fed bbb's -/// init and then [`PLACED_SEGMENTS`] from segment `from` on, each segment one fragment per track. -async fn placed_groups(from: usize) -> [Vec<(u64, usize)>; 2] { +/// init and then [`PLACED_SEGMENTS`] from segment `from` on, each segment `fragments` equal +/// fragments per track, only the first opening on a video keyframe. +async fn placed_groups(from: usize, fragments: u64) -> [Vec<(u64, usize)>; 2] { let epoch = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_millis(1_760_000_000_000); crate::clock::fake_host_wall(epoch + std::time::Duration::from_secs(600)); let (init, video, audio) = bbb_init(); @@ -2588,12 +2589,16 @@ async fn placed_groups(from: usize) -> [Vec<(u64, usize)>; 2] { } for (i, segment) in PLACED_SEGMENTS.iter().enumerate().skip(from) { let next = PLACED_SEGMENTS.get(i + 1).map_or(segment + 1_000, |next| *next); - let (start, end) = (600_000_000 + segment * 1_000, 600_000_000 + next * 1_000); + let at = |part: u64| 600_000_000 + (segment + (next - segment) * part / fragments) * 1_000; fmp4.decode(&styp()).unwrap(); - for (id, scale) in [video, audio] { - let frames = [sample(start, true, Some(end - start))]; - fmp4.decode(&super::encode_fragment(info(id, scale, i as u32), &frames).unwrap()) - .unwrap(); + for part in 0..fragments { + let (start, end) = (at(part), at(part + 1)); + for (id, scale) in [video, audio] { + let frames = [sample(start, part == 0 || id != video.0, Some(end - start))]; + let sequence = (i as u64 * fragments + part) as u32; + fmp4.decode(&super::encode_fragment(info(id, scale, sequence), &frames).unwrap()) + .unwrap(); + } } } fmp4.finish().unwrap(); @@ -2623,10 +2628,39 @@ async fn importers_sharing_an_epoch_number_segments_alike() { (611, 0), (612, 1), ]; - let first = placed_groups(0).await; + assert_joiners_agree(1, expected).await; +} + +/// A late importer that drops a segment's first fragment, joining in that segment's second, drops +/// the rest of it too, though a later fragment falls in the next second: an importer running +/// longer keeps that fragment in the segment's group and gives the second an empty group. +#[tokio::test] +async fn a_dropped_segment_stays_dropped_across_a_second() { + let expected = vec![ + (601, 4), + (602, 0), + (603, 2), + (604, 4), + (605, 0), + (606, 2), + (607, 2), + (608, 0), + (609, 2), + (610, 2), + (611, 0), + (612, 2), + ]; + assert_joiners_agree(2, expected).await; +} + +/// Import [`PLACED_SEGMENTS`] in `fragments` fragments a segment from the first segment, whose +/// groups must be `expected` on both tracks, and from every later one, whose groups must end +/// those. +async fn assert_joiners_agree(fragments: u64, expected: Vec<(u64, usize)>) { + let first = placed_groups(0, fragments).await; assert_eq!(first, [expected.clone(), expected], "video, then audio"); for from in 1..PLACED_SEGMENTS.len() - 1 { - let joined = placed_groups(from).await; + let joined = placed_groups(from, fragments).await; for (track, (first, joined)) in ["video", "audio"].into_iter().zip(first.iter().zip(&joined)) { assert!(!joined.is_empty(), "{track} from segment {from} published nothing"); assert!( diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 04fbc5983ef..e1d2c57f1cf 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -162,14 +162,14 @@ pub struct Import { /// Whether the service identity (TSID/service_id/PMT PID) has been captured from /// the PAT into the catalog service record yet (set once; the PAT is stable). identity_recorded: bool, - /// Latest video PTS: the media clock used to timestamp private sections, which - /// carry no PES PTS of their own. + /// Latest video PTS, or audio's in a program without video: the media clock used to + /// timestamp private sections, which carry no PES PTS of their own. /// SPTS scope: one clock for the whole input. Under MPTS every program's video /// advances it, so a cue could be stamped with another program's PTS. last_pts: Option, /// The first PES's PTS on the catalog clock since the last timebase break: the section clock - /// until video starts `last_pts`, so a cue ahead of the first picture lands on the media - /// timeline, not at zero. + /// until `last_pts` starts, so a cue ahead of the first picture lands on the media timeline, + /// not at zero. start_pts: Option, /// The program number chosen by [`with_program`](Self::with_program). `None` imports the /// multiplex's only program and refuses a PAT that lists more than one. @@ -748,6 +748,10 @@ impl Import { let is_video = matches!(stream, Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock); let is_clock = matches!(stream, Stream::Clock); + // Audio keeps the clock of a program with no video, whose data tracks would otherwise + // stamp every section where the import started and never pass a second. + let keeps_clock = + is_video || (matches!(stream, Stream::Aac(_) | Stream::Opus(_) | Stream::Legacy(_)) && !self.has_video()); // Verbatim streams with no cadence (SCTE-35, DVB subtitles) are left ungraded // rather than watched against an invented one. if pes.pts.is_some() && !matches!(stream, Stream::Verbatim(_) | Stream::Ignored) { @@ -775,14 +779,14 @@ impl Import { audio.burst = std::time::Duration::ZERO; } } - // Advance the media clock here, not at flush: unbounded video only - // flushes on the next PES, so a SCTE-35 section arriving during this - // frame must be timestamped with this frame's PTS ("now"), not the - // previous one's. - if pts.is_some() { - self.last_pts = on_clock(pts, offset)?; - self.advance_data()?; - } + } + // Advance the media clock here, not at flush: unbounded video only + // flushes on the next PES, so a SCTE-35 section arriving during this + // frame must be timestamped with this frame's PTS ("now"), not the + // previous one's. + if keeps_clock && pts.is_some() { + self.last_pts = on_clock(pts, offset)?; + self.advance_data()?; } if pts.is_some() { self.release_sections(offset)?; @@ -821,6 +825,13 @@ impl Import { Ok(()) } + /// Whether the program has a video stream, decoded or kept only for its clock. + fn has_video(&self) -> bool { + self.streams + .values() + .any(|stream| matches!(stream, Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock)) + } + /// Carry the media clock to the data tracks, which with aligned groups group on its grid /// whether or not they carry anything. fn advance_data(&mut self) -> anyhow::Result<()> { @@ -888,10 +899,7 @@ impl Import { return Ok(()); }; - let batched = self - .streams - .values() - .any(|stream| matches!(stream, Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock)); + let batched = self.has_video(); let Some(stream) = self.streams.get_mut(&pid) else { return Ok(()); }; @@ -7174,6 +7182,77 @@ pub(super) mod test { } } + /// On a shared epoch's timebase, audio keeps the clock of a program with no video: every cue + /// after the second the import started in publishes at the PTS of the audio before it, in a + /// group a second. + #[tokio::test(start_paused = true)] + async fn an_aligned_audio_only_program_publishes_its_cues() { + const CUE_PID: u16 = 0x0021; + let epoch = epoch_time(); + crate::clock::fake_host_wall(epoch + Duration::from_secs(600)); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let config = crate::catalog::Config::default() + .with_catalog(crate::catalog::hang::Catalog::::default()) + .with_max_age(Duration::from_secs(86_400)); + let catalog = crate::catalog::Producer::new(&mut broadcast, config).unwrap(); + let timebase = catalog.timebase(); + timebase + .place(Timestamp::ZERO, epoch - Duration::from_micros(WRAP_MICROS)) + .unwrap(); + timebase.align_groups(); + let mut import = super::Import::new(broadcast, timebase.reserve()); + let mut bytes = synth_pmt( + &[ + (StreamType::Mpeg1Audio, PCR_PID), + (StreamType::Dts8ChannelLosslessAudio, CUE_PID), + ], + true, + ); + // A cue every seven 144 ms steps, after the audio PES whose first frame it follows. + let mut cues = Vec::new(); + for i in 0..50 { + let n = i as u8; + bytes.extend(mp2_pes(PCR_PID, n, 945_000 + i * STEP, [2 * n + 1, 2 * n + 2])); + if i % 7 == 0 { + bytes.extend(packet(true, cues.len() as u8, 0, &CUE)); + cues.push(2 * n + 1); + } + } + import.decode(&bytes).unwrap(); + import.finish().unwrap(); + + let snapshot = catalog.snapshot(); + let audio = snapshot.audio.renditions.keys().next().unwrap(); + let audio: BTreeMap = read_track(&consumer, audio, crate::container::Kind::Audio) + .await + .into_iter() + .map(|frame| (frame.payload[4], frame.timestamp)) + .collect(); + let expected: Vec = cues.iter().filter_map(|fill| audio.get(fill).copied()).collect(); + assert!( + expected.len() >= 6, + "the audio published from its second second: {audio:?}" + ); + let name = snapshot.ext.mpegts.tracks.keys().next().unwrap(); + let published: Vec = read_track(&consumer, name, crate::container::Kind::Data) + .await + .into_iter() + .map(|frame| frame.timestamp) + .collect(); + assert_eq!(published, expected); + let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(3600)); + let mut track = consumer.track(name).unwrap().subscribe(subscription).await.unwrap(); + let mut sequences = Vec::new(); + while let Ok(Ok(Some(group))) = tokio::time::timeout(Duration::from_millis(50), track.recv_group()).await { + sequences.push(group.sequence); + } + assert!( + sequences.len() >= 6 && sequences.windows(2).all(|pair| pair[1] == pair[0] + 1), + "{sequences:?}" + ); + } + #[test] fn pes_header_duplicate_and_discontinuity_follow_the_packet_chain() { let pid = Pid::new(0x50).unwrap(); From 7f298600b92f3e3137374d6ae6a3b61fd7b46d7e Mon Sep 17 00:00:00 2001 From: t0ms Date: Mon, 19 Oct 2026 12:04:49 +0100 Subject: [PATCH 11/11] fix(mux): bound grid fills after long GOPs; let audio clock cues until video does - A start after a run of frames that start nothing fills at most the 60 seconds before it. Frames that join the open group move the grid on without filling, so an endless GOP could otherwise open a group for every second it lasted at its next keyframe. A producer that watched fewer seconds fills a suffix of what one running longer does. - Audio keeps the TS section clock until a video PTS arrives, rather than only in a program with no video stream, so a program whose video sends nothing still moves its cues on. Cues ahead of the first picture now land at the audio's PTS rather than the first PES's. - The upgrade guide says publishers sharing an `--epoch` mark no break. Co-authored-by: Claude Opus 5.5 --- doc/setup/upgrade.md | 3 +- rs/moq-mux/src/container/grid.rs | 29 ++++++++++--- rs/moq-mux/src/container/ts/import.rs | 60 +++++++++++++++------------ 3 files changed, 60 insertions(+), 32 deletions(-) diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index db894231a00..c86470581bf 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -34,7 +34,8 @@ These land with the next breaking release. - **Data-track breaks are an object with no bytes** (#5815). A hang data track (TS verbatim PES, sections, `m2ts`, text) marks a discontinuity with a - group of one empty object instead of a skipped group. Older consumers fail + group of one empty object instead of a skipped group. Publishers sharing an + `--epoch` mark none on any track: their timestamps jump. Older consumers fail to decode it, so upgrade exporters and players before publishers. - **Auth requests name `webtransport`** (#5293). Relays report a WebTransport session as `webtransport` instead of `quic`, and older auth servers reject the diff --git a/rs/moq-mux/src/container/grid.rs b/rs/moq-mux/src/container/grid.rs index 64f7f67d284..3460b1ed4b4 100644 --- a/rs/moq-mux/src/container/grid.rs +++ b/rs/moq-mux/src/container/grid.rs @@ -17,9 +17,10 @@ use std::ops::Range; /// The spacing of the grid, in microseconds of the catalog clock. pub(crate) const GRID_MICROS: u64 = 1_000_000; -/// The most empty cells one step of the timeline fills. A longer step between two frames (or two -/// [`advance`](Grid::advance)s) starts a new watch instead, as a break does, so a source that leaps -/// hours ahead cannot open a group for every second it skipped. +/// The most empty cells the grid fills at once. A longer step between two frames (or two +/// [`advance`](Grid::advance)s) starts a new watch instead, as a break does, and a start after a +/// longer run of frames that start nothing fills only the cells just before it, so neither a source +/// that leaps hours ahead nor an endless GOP opens a group for every second it passed. const FILL_MAX: u64 = 60; /// The cell of the grid `timestamp` falls in. @@ -105,10 +106,12 @@ impl Grid { at } - /// Claim every cell before `at` that was watched whole and holds no group yet. + /// Claim every cell before `at` that was watched whole and holds no group yet, up to + /// [`FILL_MAX`] of them: a producer that watched fewer claims a suffix of what one running longer + /// does. fn empty(&mut self, at: u64) -> Range { let origin = self.origin.expect("a reached grid has an origin"); - let from = self.last.map_or(origin, |last| last.max(origin)) + 1; + let from = (self.last.map_or(origin, |last| last.max(origin)) + 1).max(at.saturating_sub(FILL_MAX)); if from >= at { return at..at; } @@ -181,6 +184,22 @@ mod tests { assert_eq!(grid.place(at(3_600_000), true, true), opens(3_600, 3_600..3_600)); } + #[test] + fn a_start_after_a_long_run_without_one_fills_at_most_the_bound() { + let run = |from: u64| { + let mut grid = Grid::default(); + let mut open = false; + for second in from..1_000 { + let start = second <= 11; + open |= matches!(grid.place(at(second * 1_000), start, open), Place::Open { .. }); + } + grid.place(at(1_000_000), true, open) + }; + assert_eq!(run(10), opens(1_000, 940..1_000)); + assert_eq!(run(900), opens(1_000, 940..1_000)); + assert_eq!(run(980), opens(1_000, 981..1_000)); + } + #[test] fn advance_fills_cells_as_they_pass() { let mut grid = Grid::default(); diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index e1d2c57f1cf..78f61b1988c 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -162,11 +162,13 @@ pub struct Import { /// Whether the service identity (TSID/service_id/PMT PID) has been captured from /// the PAT into the catalog service record yet (set once; the PAT is stable). identity_recorded: bool, - /// Latest video PTS, or audio's in a program without video: the media clock used to + /// Latest video PTS, or audio's until video has one: the media clock used to /// timestamp private sections, which carry no PES PTS of their own. /// SPTS scope: one clock for the whole input. Under MPTS every program's video /// advances it, so a cue could be stamped with another program's PTS. last_pts: Option, + /// Whether video has set `last_pts` since the last timebase break. + video_clock: bool, /// The first PES's PTS on the catalog clock since the last timebase break: the section clock /// until `last_pts` starts, so a cue ahead of the first picture lands on the media timeline, /// not at zero. @@ -227,6 +229,7 @@ impl Import { si, identity_recorded: false, last_pts: None, + video_clock: false, start_pts: None, program: None, } @@ -748,10 +751,10 @@ impl Import { let is_video = matches!(stream, Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock); let is_clock = matches!(stream, Stream::Clock); - // Audio keeps the clock of a program with no video, whose data tracks would otherwise - // stamp every section where the import started and never pass a second. + // Audio keeps the clock until video brings a PTS, else the data tracks of a program with no + // video, or a silent one, would stamp every section where the import started. let keeps_clock = - is_video || (matches!(stream, Stream::Aac(_) | Stream::Opus(_) | Stream::Legacy(_)) && !self.has_video()); + is_video || (matches!(stream, Stream::Aac(_) | Stream::Opus(_) | Stream::Legacy(_)) && !self.video_clock); // Verbatim streams with no cadence (SCTE-35, DVB subtitles) are left ungraded // rather than watched against an invented one. if pes.pts.is_some() && !matches!(stream, Stream::Verbatim(_) | Stream::Ignored) { @@ -785,6 +788,7 @@ impl Import { // frame must be timestamped with this frame's PTS ("now"), not the // previous one's. if keeps_clock && pts.is_some() { + self.video_clock |= is_video; self.last_pts = on_clock(pts, offset)?; self.advance_data()?; } @@ -825,13 +829,6 @@ impl Import { Ok(()) } - /// Whether the program has a video stream, decoded or kept only for its clock. - fn has_video(&self) -> bool { - self.streams - .values() - .any(|stream| matches!(stream, Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock)) - } - /// Carry the media clock to the data tracks, which with aligned groups group on its grid /// whether or not they carry anything. fn advance_data(&mut self) -> anyhow::Result<()> { @@ -899,7 +896,10 @@ impl Import { return Ok(()); }; - let batched = self.has_video(); + let batched = self + .streams + .values() + .any(|stream| matches!(stream, Stream::H264 { .. } | Stream::H265 { .. } | Stream::Clock)); let Some(stream) = self.streams.get_mut(&pid) else { return Ok(()); }; @@ -994,6 +994,7 @@ impl Import { section.discontinuity(self.published)?; } self.last_pts = None; + self.video_clock = false; self.start_pts = None; self.held_edge = None; self.published = false; @@ -7182,11 +7183,19 @@ pub(super) mod test { } } - /// On a shared epoch's timebase, audio keeps the clock of a program with no video: every cue - /// after the second the import started in publishes at the PTS of the audio before it, in a - /// group a second. + /// On a shared epoch's timebase, audio keeps the clock until video brings a PTS: in a program + /// with no video, and in one whose video sends nothing, every cue after the second the import + /// started in publishes at the PTS of the audio before it, in a group a second. #[tokio::test(start_paused = true)] - async fn an_aligned_audio_only_program_publishes_its_cues() { + async fn an_aligned_program_without_video_pts_publishes_its_cues() { + for video in [&[][..], &[(StreamType::H264, 0x0050)]] { + audio_clocked_cues(video).await; + } + } + + /// Import an aligned program of MP2 audio, a SCTE-35 PID and `video`, which sends nothing, and + /// check its cues land as [`an_aligned_program_without_video_pts_publishes_its_cues`] says. + async fn audio_clocked_cues(video: &[(StreamType, u16)]) { const CUE_PID: u16 = 0x0021; let epoch = epoch_time(); crate::clock::fake_host_wall(epoch + Duration::from_secs(600)); @@ -7202,13 +7211,12 @@ pub(super) mod test { .unwrap(); timebase.align_groups(); let mut import = super::Import::new(broadcast, timebase.reserve()); - let mut bytes = synth_pmt( - &[ - (StreamType::Mpeg1Audio, PCR_PID), - (StreamType::Dts8ChannelLosslessAudio, CUE_PID), - ], - true, - ); + let mut streams = vec![ + (StreamType::Mpeg1Audio, PCR_PID), + (StreamType::Dts8ChannelLosslessAudio, CUE_PID), + ]; + streams.extend_from_slice(video); + let mut bytes = synth_pmt(&streams, true); // A cue every seven 144 ms steps, after the audio PES whose first frame it follows. let mut cues = Vec::new(); for i in 0..50 { @@ -7232,7 +7240,7 @@ pub(super) mod test { let expected: Vec = cues.iter().filter_map(|fill| audio.get(fill).copied()).collect(); assert!( expected.len() >= 6, - "the audio published from its second second: {audio:?}" + "{video:?}: the audio published from its second second: {audio:?}" ); let name = snapshot.ext.mpegts.tracks.keys().next().unwrap(); let published: Vec = read_track(&consumer, name, crate::container::Kind::Data) @@ -7240,7 +7248,7 @@ pub(super) mod test { .into_iter() .map(|frame| frame.timestamp) .collect(); - assert_eq!(published, expected); + assert_eq!(published, expected, "{video:?}"); let subscription = moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(3600)); let mut track = consumer.track(name).unwrap().subscribe(subscription).await.unwrap(); let mut sequences = Vec::new(); @@ -7249,7 +7257,7 @@ pub(super) mod test { } assert!( sequences.len() >= 6 && sequences.windows(2).all(|pair| pair[1] == pair[0] + 1), - "{sequences:?}" + "{video:?}: {sequences:?}" ); }