From 786899df21674e0a5d197582a054c42a17efdca0 Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 13 Oct 2026 11:59:10 +0100 Subject: [PATCH 1/5] fix(ts): admit leak-bound video at its full Rbx The schedule charged video whole 188-byte packets at the lesser of Rx and Rbx, plus the clock packet, so a source filling its level's leak, as a full-rate feed with no HRD does, fell behind and missed a decode deadline. The transport buffer now drains whole packets at Rx, the clock packet among them, while MB passes only elementary bytes on into EB at Rbx. Unpadded, a slot carries the lesser of the two, fractions carried from slot to slot, without the one-packet slack. `just test ts --leak` round-trips 1080p video with no HRD at level 4.0's 24 Mb/s leak. With the schedule this replaces, its export stops about 9 s into the 30 s clip. Retires quest/m1/tstd/video-leak.md. Co-authored-by: Claude Opus 5.5 --- .github/workflows/ts.yml | 9 ++ quest/m1/tstd/README.md | 1 - quest/m1/tstd/mux-overhead.md | 4 - quest/m1/tstd/video-leak.md | 31 ---- rs/moq-mux/benches/ts_schedule.rs | 1 + rs/moq-mux/src/container/ts/export.rs | 26 +++- rs/moq-mux/src/container/ts/schedule.rs | 191 ++++++++++++++++++++---- test/ts/README.md | 3 +- test/ts/run.sh | 31 +++- 9 files changed, 223 insertions(+), 74 deletions(-) delete mode 100644 quest/m1/tstd/video-leak.md diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 85b01c848a6..33097cd9801 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -106,6 +106,15 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs shell: bash -leo pipefail {0} + # 1080p video with no HRD at its level's 24 Mb/s leak, so the export has to admit + # the leak's whole rate of payload to keep up (moq-dev/moq#5362). + - name: TS video at its full leak + if: ${{ !cancelled() }} + run: nix develop --command just test ts --leak + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + # The same video beside four audio PIDs in a multiplex faster than its Rx: each slot # has to interleave the video with the rest, or its transport buffer overflows. - name: TS multiplex above the video's Rx diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index f4e60e36e6d..8aa6bb83393 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -51,7 +51,6 @@ MB/AAC controls. Do not relabel the current subset a full standards check. - [Send-ahead within the delay](/quest/m1/tstd/send-ahead.md) - total lag is `--delay`, send-ahead included, with a 1 s default - [PSI pacing](/quest/m1/tstd/psi-runs.md) - padded output paces the PAT and PMT so TBsys and Bsys never overflow -- [Video at its full leak](/quest/m1/tstd/video-leak.md) - a leak-bound video PID is admitted at its full Rbx, not about 3 % below it - [Mux overhead](/quest/m1/tstd/mux-overhead.md) - the PCR rides in the video's adaptation field and audio PES carry several frames, so a full CBR source re-muxes at its own rate ## Related diff --git a/quest/m1/tstd/mux-overhead.md b/quest/m1/tstd/mux-overhead.md index 6226dbb3c92..e65a967ada0 100644 --- a/quest/m1/tstd/mux-overhead.md +++ b/quest/m1/tstd/mux-overhead.md @@ -36,7 +36,3 @@ Decided 2026-10-12: ## Closes - [#5362](https://github.com/moq-dev/moq/issues/5362) - a full-rate CBR contribution feed exports at its own rate - -## Required - -- [TS export admits video at its full leak](/quest/m1/tstd/video-leak.md) - #5362 point 2 diff --git a/quest/m1/tstd/video-leak.md b/quest/m1/tstd/video-leak.md deleted file mode 100644 index 9e52be30df7..00000000000 --- a/quest/m1/tstd/video-leak.md +++ /dev/null @@ -1,31 +0,0 @@ -# [S] TS export admits video at its full leak - -## Goal - -`moq export ts` admits a leak-bound video PID at its full Rbx rate, so a -source that fills its level's leak (no HRD, 1080i25 High@4.0 at Rbx 24 Mb/s) -exports without falling behind and missing a decode deadline. - -## Plan - -Reported in [#5362](https://github.com/moq-dev/moq/issues/5362) point 2 and -verified against `rs/moq-mux/src/container/ts/`: `Buffer::per_slot` -(`schedule.rs`) floors the rate to whole 188-byte packets per 25 ms slot and -subtracts one packet of slack, and `admit` charges each slot's clock packet to -the PCR PID, which is the video. Together that admits about 2.9 % below what -Rbx carries in payload bytes. The slack's only rationale is a comment about -bunching at slot edges; no commit or quest calls it a deliberate margin. - -Decided 2026-10-12: - -- **Exact accounting.** Cap a leak-bound PID at the lesser of Rx in packets - and Rbx in payload bytes, since Rbx governs payload into EB while TB drains - at Rx. Carry fractional packets from slot to slot and drop the one-packet - slack. A standalone PCR packet still enters the video's transport buffer, - so it counts against the Rx packet budget but not the Rbx payload budget. -- Test: a generated source whose video fills Rbx on average exports for the - whole clip and passes the strict T-STD check that `just test ts --hrd` runs. - -## Related - -- [Lower TS mux overhead](/quest/m1/tstd/mux-overhead.md) - #5362 point 3, which closes the issue diff --git a/rs/moq-mux/benches/ts_schedule.rs b/rs/moq-mux/benches/ts_schedule.rs index d7ad661178b..1ca34ec5ea5 100644 --- a/rs/moq-mux/benches/ts_schedule.rs +++ b/rs/moq-mux/benches/ts_schedule.rs @@ -35,6 +35,7 @@ fn admission(c: &mut Criterion) { pid, schedule::Buffer { rate: 2_000_000, + leak: None, size: Some(1_000_000), transport: 512, }, diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 613e99ddbce..7da3fec5737 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -367,8 +367,7 @@ impl Track { /// a unit whose DTS sits on a PCR slot boundary, as a source's own 25 fps timestamps put /// every fifth frame, arrives complete only after it decodes. fn drain(&self) -> Duration { - let rate = self.buffer().map_or(Buffer::SYSTEM.rate, |buffer| buffer.rate); - Duration::from_nanos(TsPacket::SIZE as u64 * 8 * 1_000_000_000 / rate.max(1)) + self.buffer().unwrap_or(Buffer::SYSTEM).drain() } /// Queue every held video frame whose DTS is settled, or all of them when `flush`. @@ -469,16 +468,17 @@ impl Timing { } /// The receiver's buffers (H.222.0 2.14.3.1, 2.17.2): EB is the HRD's CPB, else the - /// level's, and packets go no faster than the slower of the transport buffer's Rx and the - /// leak from MB into EB, so neither MB nor TB fills. + /// level's; the transport buffer drains at Rx, scaled from the HRD's BitRate or else the + /// level's; and MB passes payload on into EB at the level's Rbx whatever the HRD declares. fn buffer(&self) -> Option { let level = self.level?; - let (rate, cpb) = match self.hrd { - Some(hrd) => (hrd.bit_rate.saturating_mul(level.factor) / 1_000, hrd.cpb_size), - None => (level.leak, level.cpb), + let (bit_rate, cpb) = match self.hrd { + Some(hrd) => (hrd.bit_rate, hrd.cpb_size), + None => (level.bit_rate, level.cpb), }; Some(Buffer { - rate: rate.min(level.leak), + rate: bit_rate.saturating_mul(level.factor) / 1_000, + leak: Some(level.leak), size: Some(usize::try_from(cpb / 8).unwrap_or(usize::MAX)), transport: 512, }) @@ -2409,6 +2409,9 @@ fn at_edge(source: ExportSource, replay: bool) -> ExportSource { struct Level { /// The leak from MB into EB, Rbx: CpbBrNalFactor * MaxBR. leak: u64, + /// BitRate without an HRD, which `factor` scales to Rx: 1200 * MaxBR for H.264, the + /// least cpbBrNalFactor of any profile, and CpbBrVclFactor * MaxBR for H.265. + bit_rate: u64, /// EB without an HRD: CpbBrNalFactor * MaxCPB. cpb: u64, /// Rx over an HRD's BitRate, in thousandths: 1.2 for H.264, CpbBrNalFactor / @@ -2445,6 +2448,7 @@ fn video_level(config: &VideoConfig, name: &str) -> Level { }; limits.map(|(max_br, max_cpb)| Level { leak: 1_200 * max_br, + bit_rate: 1_200 * max_br, cpb: 1_200 * max_cpb, factor: 1_200, }) @@ -2474,6 +2478,7 @@ fn video_level(config: &VideoConfig, name: &str) -> Level { }; limits.map(|(max_br, max_cpb)| Level { leak: 1_100 * max_br, + bit_rate: 1_000 * max_br, cpb: 1_100 * max_cpb, factor: 1_100, }) @@ -2484,6 +2489,7 @@ fn video_level(config: &VideoConfig, name: &str) -> Level { tracing::warn!(track = %name, codec = %config.codec, "no T-STD limits for this level; sending its units early"); Level { leak: 1_200 * 64, + bit_rate: 1_200 * 64, cpb: 1_200 * 175, factor: 1_200, } @@ -2509,6 +2515,7 @@ fn audio_buffer(config: &AudioConfig, kind: &Kind) -> Buffer { }; Buffer { rate, + leak: None, size: Some(size), transport: 512, } @@ -2535,11 +2542,13 @@ fn verbatim_buffer(kind: &Kind, descriptors: &[catalog::Descriptor]) -> Option Some(Buffer { rate: 2_000_000, + leak: None, size: Some(5_696), transport: 512, }), TELETEXT_DESCRIPTOR => Some(Buffer { rate: 6_750_000, + leak: None, size: None, transport: 480, }), @@ -3012,6 +3021,7 @@ mod tests { verbatim_buffer(&kind, &[descriptor(0x56)]), Some(Buffer { rate: 6_750_000, + leak: None, size: None, transport: 480, }) diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 7c69c9318d4..fd32fcea806 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -67,9 +67,11 @@ fn rate_pcr(index: u128, rate: u64) -> u128 { /// A PID's buffers in a receiver, as the schedule respects them (ISO 13818-1 2.4.2). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct Buffer { - /// How fast the receiver takes the PID's packets on, in bits per second: its transport - /// buffer's Rx, or for video the slower leak from MB into EB. + /// How fast the receiver's transport buffer drains whole packets, Rx, in bits per second. pub rate: u64, + /// For video, how fast MB passes payload on into EB, Rbx, in bits per second (ISO + /// 13818-1 2.14.3.1). `None` where the transport buffer feeds the decoder buffer directly. + pub leak: Option, /// The decoder buffer (EB for video, B for audio) in bytes, which a unit fills from its /// first packet until it decodes. `None` bounds nothing, so the PID's units go out only /// in the slot before their own. @@ -83,16 +85,34 @@ impl Buffer { /// The program tables' (ISO 13818-1 2.4.2.3, Rxsys). pub const SYSTEM: Self = Self { rate: 1_000_000, + leak: None, size: None, transport: 512, }; - /// The packets one slot may carry on the PID: what its transport buffer passes on within - /// the slot, less one, but at least one. Even laid out evenly, the packets bunch a little - /// where one slot meets the next, and the slack drains that off. - fn per_slot(&self) -> usize { - let bits = u128::from(self.rate) * PCR_INTERVAL.as_nanos() / 1_000_000_000; - (bits / (TsPacket::SIZE as u128 * 8)).saturating_sub(1).max(1) as usize + /// How long the receiver takes to pass one packet on into the decoder buffer: through the + /// transport buffer at Rx and, for video, its payload through MB at Rbx, which overlap. + pub fn drain(&self) -> Duration { + let through = |bytes: usize, rate: u64| Duration::from_nanos(bytes as u64 * 8_000_000_000 / rate.max(1)); + let transport = through(TsPacket::SIZE, self.rate); + self.leak + .map_or(transport, |leak| transport.max(through(PAYLOAD, leak))) + } + + /// The media packets slot `index` may carry on the PID unpadded: what its transport buffer + /// passes on at Rx, less the slot's `clock` packet when it rides the PID, and for video no + /// more payload than MB passes on at Rbx. Both count from the start of the timeline, so + /// no slot rounds on its own. At least one, or a PID too slow for both the clock packet and + /// one of its own would never finish a unit. + fn per_slot(&self, index: u128, clock: bool) -> usize { + let through = |rate: u64, bytes: usize| { + let before = + |index: u128| index * u128::from(rate) * PCR_INTERVAL.as_nanos() / (bytes as u128 * 8_000_000_000); + before(index + 1) - before(index) + }; + let packets = through(self.rate, TsPacket::SIZE).saturating_sub(u128::from(clock)); + let packets = self.leak.map_or(packets, |leak| packets.min(through(leak, PAYLOAD))); + packets.max(1) as usize } } @@ -204,6 +224,16 @@ fn payload(packet: &[u8]) -> &[u8] { &packet[start..] } +/// The bytes of `packet` that video's MB passes on into EB: its payload less any PES header, +/// which MB discards (ISO 13818-1 2.14.3.1). +fn elementary(packet: &[u8]) -> &[u8] { + let body = payload(packet); + match packet[1] & 0x40 != 0 && body.starts_with(&[0, 0, 1]) && body.len() >= 9 { + true => body.get(9 + usize::from(body[8])..).unwrap_or_default(), + false => body, + } +} + /// One grid slot's worth of output. pub(super) struct Slot { pub index: u128, @@ -414,6 +444,8 @@ pub(super) struct Schedule { /// When each transport buffer's last admitted packet finishes leaving it, and when it /// last started filling from empty. transport: HashMap, + /// When each video PID's MB finishes passing its last admitted payload on into EB. + mb: HashMap, /// Every source has ended, so a unit that misses its deadline is the tail going out late /// rather than an output that cannot keep up. ended: bool, @@ -433,6 +465,7 @@ impl Schedule { pmt: None, decoding: VecDeque::new(), transport: HashMap::new(), + mb: HashMap::new(), ended: false, } } @@ -542,6 +575,7 @@ impl Schedule { self.units.clear(); self.decoding.clear(); self.transport.clear(); + self.mb.clear(); self.next = None; self.horizon = None; self.last = None; @@ -643,7 +677,7 @@ impl Schedule { caps: self .buffers .iter() - .map(|(&pid, buffer)| (pid, buffer.per_slot())) + .map(|(&pid, buffer)| (pid, buffer.per_slot(index, self.clock == Some(pid)))) .collect(), positions: None, })) @@ -756,13 +790,31 @@ impl Schedule { continue; } } + // Video's payload goes on from MB into EB at Rbx as it leaves the transport + // buffer, its PES header discarded, so only payload counts against the leak. + // Holding MB to a transport buffer's worth, below any MB size, still carries the + // leak's whole rate across packet positions. + let leak = buffer.and_then(|buffer| buffer.leak).map(|leak| { + let free = last.max(now); + let from = self.mb.get(&pid).copied().unwrap_or(free).max(free); + let bits = elementary(packet).len() as u128 * 8 * 1_000_000_000; + let leaked = from + bits.div_ceil(u128::from(leak)) as i128; + (leaked, (512 * 8 * 1_000_000_000 / u128::from(leak)) as i128) + }); + if let Some((leaked, capacity)) = leak + && leaked - now > capacity + { + waiting.push(Reverse((leaked - capacity, lane))); + continue; + } // No byte leaves the transport buffer before it has arrived: below Rx, the // multiplex rate decides when the packet's last byte gets there. - let done = drained.max(time(position + 1)); + let arrived = drained.max(time(position + 1)); + let done = leak.map_or(arrived, |(leaked, _)| arrived.max(leaked)); // A transport buffer must empty at least once a second (2.4.2.3), so a PID kept // busy that long waits for it to. Every later slot's clock packet that arrives // before the buffer empties extends the stretch without the chance to wait. - let mut busy = done; + let mut busy = arrived; if let Some((_, duration)) = clock.filter(|&(clock, _)| clock == tb) { let mut next = index + 1; while busy - since <= TB_EMPTY.as_nanos() as i128 { @@ -797,7 +849,10 @@ impl Schedule { "MPEG-TS output missed a decode deadline on PID {pid} at {rate} b/s; raise the delay or the multiplex rate" ); } - self.transport.insert(tb, (done, since)); + self.transport.insert(tb, (arrived, since)); + if let Some((leaked, _)) = leak { + self.mb.insert(pid, leaked); + } out.positions.as_mut().unwrap().push(position); out.packets.extend_from_slice(packet); out.nulls -= 1; @@ -876,14 +931,7 @@ impl Schedule { let mut order: Vec = (0..self.units.len()).collect(); order.sort_by_key(|&i| self.units[i].due); let mut take = vec![0; self.units.len()]; - // The clock packet counts against its PID unless that would leave the PID's media no - // packet at all, which would never finish a unit however sparse its frames. - let mut carried: HashMap = self - .clock - .filter(|pid| self.buffers.get(pid).is_some_and(|buffer| buffer.per_slot() > 1)) - .map(|pid| (pid, 1)) - .into_iter() - .collect(); + let mut carried: HashMap = HashMap::new(); let mut stopped: Vec = Vec::new(); for i in order { let unit = &self.units[i]; @@ -898,7 +946,10 @@ impl Schedule { if self.release(unit) <= index { while n < unit.remaining() && budget > 0 { let pid = unit.pid_at(unit.sent + n); - let cap = self.buffers.get(&pid).map_or(usize::MAX, Buffer::per_slot); + let cap = self + .buffers + .get(&pid) + .map_or(usize::MAX, |buffer| buffer.per_slot(index, self.clock == Some(pid))); let count = carried.entry(pid).or_default(); if *count >= cap { break; @@ -962,6 +1013,7 @@ mod tests { /// A video-like buffer: far more than a slot passes on, and a large decoder buffer. const VIDEO: Buffer = Buffer { rate: 100_000_000, + leak: None, size: Some(1_000_000), transport: 512, }; @@ -1087,6 +1139,7 @@ mod tests { let rate = 50_000_000; let teletext = Buffer { rate: 6_750_000, + leak: None, size: None, transport: 480, }; @@ -1123,6 +1176,7 @@ mod tests { for (rx, count, clock) in [(1_000_000, 2_000, 7), (150_000, 150, 9), (120_000, 150, 9)] { let slow = Buffer { rate: rx, + leak: None, size: Some(1_000_000), transport: 512, }; @@ -1224,6 +1278,7 @@ mod tests { // Room for ten packets. let small = Buffer { rate: 100_000_000, + leak: None, size: Some(10 * PAYLOAD), transport: 512, }; @@ -1271,9 +1326,10 @@ mod tests { fn a_pid_takes_what_its_transport_buffer_passes_on() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - // 2 Mb/s: 33 packets a slot, less one. + // 2 Mb/s: 33.2 packets a slot. let audio = Buffer { rate: 2_000_000, + leak: None, size: Some(100_000), transport: 512, }; @@ -1290,9 +1346,10 @@ mod tests { fn the_clock_packet_counts_against_its_pid() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - // 2 Mb/s: 33 packets a slot, less one, the clock packet among them. + // 2 Mb/s: 33.2 packets a slot, the clock packet among them. let audio = Buffer { rate: 2_000_000, + leak: None, size: Some(100_000), transport: 512, }; @@ -1304,6 +1361,78 @@ mod tests { assert_eq!(checked_transport(&mut schedule, 1), 80); } + /// Video whose payload fills its level's leak keeps up: its transport buffer drains whole + /// packets at Rx, the clock packet among them, while MB passes only payload on at Rbx + /// (moq-dev/moq#5362). Charging whole packets, or the clock packet, to Rbx falls behind. + #[test] + fn video_keeps_up_with_its_full_leak() { + // Level 4.0 without an HRD: Rx 1.2 x 24 Mb/s, Rbx 24 Mb/s, BSmux + BSoh 16,000 bytes. + const RX: f64 = 28_800_000.0; + const RBX: f64 = 24_000_000.0; + let video = Buffer { + rate: RX as u64, + leak: Some(RBX as u64), + size: Some(3_750_000), + transport: 512, + }; + let rate = 26_000_000; + let mut schedule = Schedule::new(Duration::from_millis(200)); + schedule.set_rate(Some(rate)); + schedule.set_buffer(1, video); + schedule.set_clock(1); + // 20 s of 40 ms frames, each 652 packets of payload: 23.99 Mb/s. + for frame in 0..500 { + schedule + .push(1, timing(1_000 + 40 * frame), unit(1, 652), frame % 25 == 0) + .unwrap(); + } + let mut null = unit(0, 1); + (null[1], null[2]) = (0x1f, 0xff); + let (mut tb, mut mb, mut previous, mut sent) = (0.0f64, 0.0f64, None::, 0); + while let Some(slot) = schedule.next(None).unwrap() { + let mut clock = unit(1, 1); + clock[3] = 0x20; + for (position, packet) in slot.layout(&clock, 100, &null).chunks(188).enumerate() { + let at = slot.pcr as f64 / SYSTEM_CLOCK as f64 + position as f64 * 1504.0 / rate as f64; + if let Some(previous) = previous { + tb = (tb - (at - previous) * RX / 8.0).max(0.0); + mb = (mb - (at - previous) * RBX / 8.0).max(0.0); + } + previous = Some(at); + if pid(packet) == 1 { + tb += 188.0; + mb += elementary(packet).len() as f64; + sent += usize::from(packet[3] & 0x10 != 0); + } + assert!(tb <= 512.001, "TB holds {tb} bytes at {at}"); + assert!(mb <= 16_000.0, "MB holds {mb} bytes at {at}"); + } + } + assert_eq!(sent, 500 * 652, "every frame goes out"); + } + + /// Unpadded, a video PID's slot carries the lesser of Rx in whole packets and Rbx in + /// payload, fractions carried from slot to slot, and the clock packet counts against Rx + /// alone. + #[test] + fn unpadded_video_takes_the_lesser_of_rx_and_rbx() { + let second = |buffer: Buffer| (0..40).map(|index| buffer.per_slot(index, true)).sum::(); + // Rbx binds: 24 Mb/s of 184-byte payloads, 16,304.3 a second. + let leak = Buffer { + rate: 28_800_000, + leak: Some(24_000_000), + size: None, + transport: 512, + }; + assert_eq!(second(leak), 16_304); + // Rx binds: 20 Mb/s of 188-byte packets, 13,297.9 a second, less 40 clock packets. + let transport = Buffer { + rate: 20_000_000, + ..leak + }; + assert_eq!(second(transport), 13_297 - 40); + } + /// A PID whose transport buffer passes on a single packet a slot still carries one of its /// own beside the clock packet, so its units finish on time and the stream ends. #[test] @@ -1313,6 +1442,7 @@ mod tests { // 76.8 kb/s: one packet a slot. let slow = Buffer { rate: 76_800, + leak: None, size: Some(100_000), transport: 512, }; @@ -1333,6 +1463,7 @@ mod tests { 1, Buffer { rate: 76_800, + leak: None, size: Some(100_000), transport: 512, }, @@ -1354,6 +1485,7 @@ mod tests { // Room for ten packets. let small = Buffer { rate: 100_000_000, + leak: None, size: Some(10 * PAYLOAD), transport: 512, }; @@ -1394,6 +1526,7 @@ mod tests { 1, Buffer { rate: 2_000_000, + leak: None, size: Some(2_592), transport: 512, }, @@ -1438,6 +1571,7 @@ mod tests { 1, Buffer { rate: 2_000_000, + leak: None, size: Some(2 * PAYLOAD), transport: 512, }, @@ -1485,6 +1619,7 @@ mod tests { 1, Buffer { rate: 2_000_000, + leak: None, size: Some(2_592), transport: 512, }, @@ -1613,11 +1748,12 @@ mod tests { #[test] fn unpadded_units_go_out_by_their_decode_time() { let mut schedule = Schedule::new(Duration::ZERO); - // 2 Mb/s passes 32 packets a slot. + // 2 Mb/s passes 33 packets a slot. schedule.set_buffer( 1, Buffer { rate: 2_000_000, + leak: None, size: Some(100_000), transport: 512, }, @@ -1752,10 +1888,11 @@ mod tests { video.into(), Buffer { rate: RX, + leak: None, size: None, transport: 512, } - .per_slot(), + .per_slot(0, true), )]), positions: None, }; @@ -1794,11 +1931,13 @@ mod tests { let (a, b) = ( Buffer { rate: 7_833_600, + leak: None, size: None, transport: 512, }, Buffer { rate: 8_160_000, + leak: None, size: None, transport: 512, }, @@ -1810,7 +1949,7 @@ mod tests { nulls: 88, keyframe: true, units: vec![1, 2], - caps: HashMap::from([(1, a.per_slot()), (2, b.per_slot())]), + caps: HashMap::from([(1, a.per_slot(0, true)), (2, b.per_slot(0, false))]), positions: None, }; let mut null = unit(0, 1); diff --git a/test/ts/README.md b/test/ts/README.md index 997c065a9a2..084360421c0 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -34,6 +34,7 @@ just test ts --pair # two exporters of one broadcast, grade table an just test ts --open-gop # open-GOP clip; its leading pictures must survive just test ts --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB just test ts --headroom # the same with four audio PIDs, muxed above the video's Rx +just test ts --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak just test ts --delay 1s # pass the exporter's --delay ``` @@ -602,7 +603,7 @@ outcomes deterministically in Rust CI, without retries or implicit `--stitch`. [`.github/workflows/ts.yml`](../../.github/workflows/ts.yml) is independent of browser interop. Relevant mux, Hang, CLI, transport, harness and toolchain PRs run -`just test ts-controls`, `just test ts-pair-controls`, the default and 2 Mb/s round trips, HRD, headroom, +`just test ts-controls`, `just test ts-pair-controls`, the default and 2 Mb/s round trips, HRD, leak, headroom, open-GOP, EIT, and T-STD controls. Each step runs even after another test fails. All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. diff --git a/test/ts/run.sh b/test/ts/run.sh index 9e5b49a9ffd..215600f374d 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -21,6 +21,7 @@ # ./run.sh --open-gop # open-GOP clip; its leading pictures must survive # ./run.sh --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB # ./run.sh --headroom # the same with four audio PIDs, muxed above the video's Rx +# ./run.sh --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak # ./run.sh --delay 1s # pass the exporter's --delay # `--live` swaps the analyzer, not the rig. compliance.py grades a captured file @@ -73,6 +74,7 @@ PAIR="" # subscribe twice and grade the two captures against each other OPEN_GOP="" # publish open GOP with leading pictures, and grade them through the round-trip HRD="" # publish video that fills a broadcast-sized CPB, as a contribution encoder does HEADROOM="" # the same plus four audio PIDs, in a multiplex faster than the video's Rx +LEAK="" # publish video with no HRD at its level's leak, as a full-rate feed does EXPORT=() # extra `export ts` flags (--delay) # How far into the run the second subscriber joins. A late join is the point: two # exporters started together can share a cadence by starting together, which is @@ -81,8 +83,8 @@ PAIR_JOIN="${TSC_PAIR_JOIN:-5}" # Shortest overlap worth a verdict, in seconds. Below this the slower tables fall under # the analyzer's emission floor and go report-only, which reads as a pass. PAIR_MIN_OVERLAP="${TSC_PAIR_MIN_OVERLAP:-25}" -DURATION_SET="" # so pair mode can raise the default without overriding an explicit --duration -BITRATE_SET="" # so headroom mode can raise the default without overriding an explicit --bitrate +DURATION_SET="" # so pair and leak modes can raise the default without overriding an explicit --duration +BITRATE_SET="" # so headroom and leak modes can raise the default without overriding an explicit --bitrate PASSTHRU=() # forwarded to compliance.py (thresholds, --report-json, ...) while [[ $# -gt 0 ]]; do @@ -145,6 +147,10 @@ while [[ $# -gt 0 ]]; do HEADROOM=1 shift ;; + --leak) + LEAK=1 + shift + ;; --delay) EXPORT=(--delay "$2") shift 2 @@ -186,6 +192,14 @@ if [[ -n "$HEADROOM" && -n "$HRD$OPEN_GOP$SOURCE$ANALYZE_ONLY" ]]; then fi # Room for the audio, with the video still taking up to 86 % of the multiplex. [[ -n "$HEADROOM" && -z "$BITRATE_SET" && -z "${TSC_BITRATE:-}" ]] && BITRATE=12500000 +if [[ -n "$LEAK" && -n "$HRD$HEADROOM$OPEN_GOP$SOURCE$ANALYZE_ONLY" ]]; then + echo "error: --leak generates its own clip and cannot be combined with --hrd, --headroom, --open-gop, --source, or --analyze-only" >&2 + exit 1 +fi +# Room for the video's packet headers, and long enough that an export admitting video a +# few percent below the leak exhausts the delay's lead and misses a deadline. +[[ -n "$LEAK" && -z "$BITRATE_SET" && -z "${TSC_BITRATE:-}" ]] && BITRATE=26000000 +[[ -n "$LEAK" && -z "$DURATION_SET" && -z "${TSC_DURATION:-}" ]] && DURATION=30 if [[ -n "$OPEN_GOP" && -n "$ANALYZE_ONLY$LIVE$PAIR" ]]; then echo "error: --open-gop cannot be combined with --analyze-only, --live, or --pair" >&2 exit 1 @@ -304,6 +318,17 @@ elif [[ -n "$HRD$HEADROOM" ]]; then -c:v libx264 -profile:v high -level 4.0 -preset veryfast -pix_fmt yuv420p \ -x264-params "keyint=25:min-keyint=25:scenecut=0:nal-hrd=cbr" -b:v 9M -maxrate 9M -bufsize 9M \ -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" +elif [[ -n "$LEAK" ]]; then + echo "### generating ~${DURATION}s 1080p clip with no HRD at its level's 24 Mb/s leak with ffmpeg" + # A full-rate feed's shape: with no HRD, level 4.0's leak from MB into EB (Rbx, 24 Mb/s) + # is the video's only limit, and a target above the VBV's 24 Mb/s ceiling holds the + # video at it, so the export has to admit the leak's whole rate (moq-dev/moq#5362). + ffmpeg -y -hide_banner -loglevel error \ + -f lavfi -i "testsrc2=size=1920x1080:rate=25,noise=alls=24:allf=t" \ + -t "$DURATION" -an \ + -c:v libx264 -profile:v high -level 4.0 -preset veryfast -pix_fmt yuv420p \ + -x264-params "keyint=25:min-keyint=25:scenecut=0" -b:v 30M -maxrate 24M -bufsize 24M \ + -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" else echo "### generating ~${DURATION}s broadcast-like ${OPEN_GOP:+open-GOP }clip with ffmpeg" # CBR with a 20 ms PCR, like a contribution feed. Not cosmetic: `regulate` @@ -535,7 +560,7 @@ echo "### captured $(wc -c <"$SUB_TS" | tr -d ' ') bytes -> analyzing" # The initial decoded-content profile is the generated closed-GOP H.264/AAC # pair. Other arms retain their specific structural/model oracles until their # acquisition and decoder-priming contracts are covered by fixtures. -if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM" ]]; then +if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK" ]]; then python3 "$DIR/decode.py" "$SRC_TS" "$SUB_TS" --output "$HARNESS_RUN/decode" else echo "decoded-content comparison: not covered for this profile" From 20c1d2b315160fb5f55791215da4bec4a4bce1ea Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 13 Oct 2026 12:23:50 +0100 Subject: [PATCH 2/5] fix(ts): let video's MB run BSmux ahead of its leak Held to a transport buffer's worth ahead of the leak, MB emptied whenever audio due sooner took a position or two between video packets, and the leak idled. On a feed whose video fills Rbx, the export carried about 1 % less video than the source and fell behind. H.222.0 2.14.3.1 gives MB BSmux for this, 4 ms of the larger of Rbx and 2 Mb/s, and BSoh for PES headers. Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/schedule.rs | 54 +++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index fd32fcea806..d4728bad6c0 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -791,15 +791,17 @@ impl Schedule { } } // Video's payload goes on from MB into EB at Rbx as it leaves the transport - // buffer, its PES header discarded, so only payload counts against the leak. - // Holding MB to a transport buffer's worth, below any MB size, still carries the - // leak's whole rate across packet positions. + // buffer, its PES header discarded, so only payload counts against the leak. MB + // runs up to BSmux ahead of the leak, 4 ms of the larger of Rbx and 2 Mb/s: the + // room 2.14.3.1 gives it for the multiplex's other PIDs, so the leak never idles + // while they take a few positions, and BSoh is left for the PES headers. let leak = buffer.and_then(|buffer| buffer.leak).map(|leak| { let free = last.max(now); let from = self.mb.get(&pid).copied().unwrap_or(free).max(free); let bits = elementary(packet).len() as u128 * 8 * 1_000_000_000; let leaked = from + bits.div_ceil(u128::from(leak)) as i128; - (leaked, (512 * 8 * 1_000_000_000 / u128::from(leak)) as i128) + let room = 4_000_000 * u128::from(leak.max(2_000_000)) / u128::from(leak); + (leaked, room as i128) }); if let Some((leaked, capacity)) = leak && leaked - now > capacity @@ -1411,6 +1413,50 @@ mod tests { assert_eq!(sent, 500 * 652, "every frame goes out"); } + /// Audio due before the video takes a position or a few between video packets, and MB + /// holds enough payload ahead of the leak that the leak runs on through them: video with + /// frames queued passes on its whole Rbx. With only a transport buffer's worth ahead, the + /// leak idles after the audio's packets and video falls about 1 % below it. + #[test] + fn audio_between_video_packets_leaves_the_leak_running() { + const RBX: u64 = 24_000_000; + let video = Buffer { + rate: 28_800_000, + leak: Some(RBX), + size: Some(3_750_000), + }; + let audio = Buffer { + rate: 2_000_000, + leak: None, + size: Some(3_584), + }; + let rate = 26_000_000; + let mut schedule = Schedule::new(Duration::from_secs(2)); + schedule.set_rate(Some(rate)); + schedule.set_buffer(1, video); + schedule.set_buffer(2, audio); + schedule.set_buffer(3, audio); + schedule.set_clock(1); + // Two seconds of video due from 2 s on, so it is queued from the first slot, and audio + // due within the first second: MP2's 24 ms frames and AC-3's 32 ms, four packets each. + let mut units: Vec<(u128, u8, usize)> = (0..50).map(|frame| (2_000 + 40 * frame, 1, 652)).collect(); + units.extend((0..40).map(|frame| (24 * frame, 2, 4))); + units.extend((0..30).map(|frame| (32 * frame, 3, 4))); + units.sort(); + for (ms, pid, packets) in units { + schedule + .push(pid.into(), timing(ms), unit(pid, packets), false) + .unwrap(); + } + let mut video = 0; + for _ in 0..40 { + let slot = schedule.next(None).unwrap().unwrap(); + video += slot.packets.chunks(188).filter(|packet| pid(packet) == 1).count(); + } + // A second of the leak is 16,304 packets of payload. + assert!(video >= 16_290, "{video} video packets in a second"); + } + /// Unpadded, a video PID's slot carries the lesser of Rx in whole packets and Rbx in /// payload, fractions carried from slot to slot, and the clock packet counts against Rx /// alone. From 4c34f2953fd5a1b5910fefa15b4f4e1d39eb9ea2 Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 13 Oct 2026 16:08:25 +0100 Subject: [PATCH 3/5] feat(ts): carry the PCR in the clock PID and pack audio frames per PES A padded export spent a whole clock packet on every 25 ms slot and a PES on every audio frame. A source multiplex with no null packets has no room for either, so an export at the source's own rate fell behind it until its lead ran out and it missed a deadline (moq-dev/moq#5362). - Each slot's PCR rides the adaptation field of the clock PID's first packet in the slot when one goes out there, reflowing that PES's payload, and falls back to a clock packet of its own otherwise. - Consecutive frames of every audio codec (AAC, MP2, AC-3, E-AC-3 and AC-3 carried in a private stream) share a PES while they run on without a gap and fit B whole with their header. Each frame leaves B at its own decode time. PESs start on a grid of the media timeline, so exporters that joined at different frames cut the same PESs. - Frames and grid slots follow each other in decode order however often the exporter is polled, and tables muxed while frames are still packed go out with the next unit, in the order they were written. `just test ts --full` exports a generated 30 s clip at its own rate with no nulls: an x264 video at its level's leak beside MP2 and DVB AC-3, with the PCR in the video PID and several audio frames per PES. The new `overhead.py` fails the export if it spends more packets than the source on clocks and audio framing; CI runs it in the TS workflow. Retires quest/m1/tstd/mux-overhead.md. Co-authored-by: Claude Opus 5.5 --- .github/workflows/ts.yml | 10 + quest/m1/tstd/README.md | 1 - quest/m1/tstd/mux-overhead.md | 38 -- rs/moq-mux/benches/ts_schedule.rs | 3 +- rs/moq-mux/src/container/ts/export.rs | 259 +++++++++-- rs/moq-mux/src/container/ts/export_test.rs | 193 ++++++-- rs/moq-mux/src/container/ts/schedule.rs | 491 ++++++++++++++++++--- test/ts/README.md | 21 +- test/ts/overhead.py | 86 ++++ test/ts/run.sh | 67 ++- 10 files changed, 998 insertions(+), 171 deletions(-) delete mode 100644 quest/m1/tstd/mux-overhead.md create mode 100755 test/ts/overhead.py diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 33097cd9801..46c6795ac0e 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -115,6 +115,16 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs shell: bash -leo pipefail {0} + # The same video beside two audio PIDs in a multiplex the source fills with no nulls, + # at the source's own rate: the export has no room to spend more packets than the + # source on clocks or audio framing (moq-dev/moq#5362). + - name: TS full multiplex at its own rate + if: ${{ !cancelled() }} + run: nix develop --command just test ts --full + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + shell: bash -leo pipefail {0} + # The same video beside four audio PIDs in a multiplex faster than its Rx: each slot # has to interleave the video with the rest, or its transport buffer overflows. - name: TS multiplex above the video's Rx diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index 8aa6bb83393..5060f62cdd3 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -51,7 +51,6 @@ MB/AAC controls. Do not relabel the current subset a full standards check. - [Send-ahead within the delay](/quest/m1/tstd/send-ahead.md) - total lag is `--delay`, send-ahead included, with a 1 s default - [PSI pacing](/quest/m1/tstd/psi-runs.md) - padded output paces the PAT and PMT so TBsys and Bsys never overflow -- [Mux overhead](/quest/m1/tstd/mux-overhead.md) - the PCR rides in the video's adaptation field and audio PES carry several frames, so a full CBR source re-muxes at its own rate ## Related diff --git a/quest/m1/tstd/mux-overhead.md b/quest/m1/tstd/mux-overhead.md deleted file mode 100644 index e65a967ada0..00000000000 --- a/quest/m1/tstd/mux-overhead.md +++ /dev/null @@ -1,38 +0,0 @@ -# [M] TS export fits a full source at its own rate - -## Goal - -`moq export ts` spends no more packets than the source on clocks and audio -framing, so a CBR source that fills its multiplex with no null packets -re-muxes at its own `--mux-rate` instead of falling behind. - -## Plan - -Reported in [#5362](https://github.com/moq-dev/moq/issues/5362) point 3 and -verified against `rs/moq-mux/src/container/ts/export.rs`: with points 1 and 2 -fixed, the export of a 25 Mb/s broadcast feed averages 25.017 Mb/s of -non-null packets. Against the source it spends about 40 packets/s on -separate PCR packets (the source carries its PCR in video packets) and about -60 on one audio frame per PES (the source packs about twelve MP2 and nine -AC-3 frames per PES). - -Decided 2026-10-12: - -- **PCR in the PCR PID's adaptation field.** When the clock is due in a slot - that sends a PCR-PID packet, carry the PCR in that packet's adaptation - field, sampled at its byte position as today. Fall back to the standalone - `pcr_packet` only when the slot has none. The 25 ms grid stays; the - comment at `lay` explains why the PCR is not sampled from the DTS. -- **Several audio frames per PES.** Pack consecutive frames of an audio PID - into one PES while the PES fits the PID's B, timed by the first frame's - PTS. This applies to every audio stream the export frames (AAC, MP2, AC-3, - E-AC-3), not only DVB AC-3. -- [#5363](https://github.com/moq-dev/moq/pull/5363) landed the DVB AC-3 framing - this packs. Video leak accounting must land first too, so the full-rate - proof covers every part of #5362 before this quest closes it. -- Test: a generated CBR source with no nulls, at the source's own - `--mux-rate`, exports for the whole clip and passes the strict T-STD check. - -## Closes - -- [#5362](https://github.com/moq-dev/moq/issues/5362) - a full-rate CBR contribution feed exports at its own rate diff --git a/rs/moq-mux/benches/ts_schedule.rs b/rs/moq-mux/benches/ts_schedule.rs index 1ca34ec5ea5..09e693a0b1d 100644 --- a/rs/moq-mux/benches/ts_schedule.rs +++ b/rs/moq-mux/benches/ts_schedule.rs @@ -45,7 +45,7 @@ fn admission(c: &mut Criterion) { for pid in 1..=pids { let at = (999 + frame * 40) * 1_000_000; schedule - .push( + .push_frames( pid, schedule::Timing { by: at, @@ -53,6 +53,7 @@ fn admission(c: &mut Criterion) { }, packet(pid).repeat(10), false, + Vec::new(), ) .unwrap(); } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 7da3fec5737..b10abfac3be 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -124,6 +124,9 @@ pub struct Export { generation: u64, tracks: HashMap, + /// Tables muxed with frames still packed, which ride ahead of the next unit queued, so + /// they go out in the order they were written. + tables: Vec, /// The next continuity counter per PID, numbered as packets go out. counters: HashMap, /// PMT program-level descriptors captured on import, re-emitted in the PMT. @@ -242,6 +245,8 @@ struct Track { buffer: Option, /// The AC-3 sync frame a passed-through PES left unfinished, for the next PES to complete. ac3_carry: Option, + /// The audio frames muxed into a PES that has yet to go out. + pack: Option, } impl Track { @@ -261,6 +266,7 @@ impl Track { timing: Timing::default(), buffer: None, ac3_carry: None, + pack: None, } } @@ -353,6 +359,23 @@ impl Track { .any(|descriptor| descriptor.tag == AC3_DESCRIPTOR) } + /// The samples an audio frame of `payload` lasts and their rate, if it may share a PES with + /// the frames around it: one whole frame of audio whose receiver removes each frame from + /// its buffer at the frame's own decode time. + fn samples(&self, payload: &[u8]) -> Option<(u64, u32)> { + let parse = match &self.kind { + Kind::Aac { config, .. } => return (config.sample_rate > 0).then_some((1024, config.sample_rate)), + Kind::Mp2 { .. } => crate::codec::mp2::parse_header, + Kind::Ac3 => crate::codec::ac3::parse_header, + Kind::Eac3 => crate::codec::eac3::parse_header, + _ if self.carries_ac3() => crate::codec::ac3::parse_header, + _ => return None, + }; + let header = parse(payload).ok()?; + (header.len == payload.len() && header.sample_rate > 0 && header.samples > 0) + .then_some((header.samples, header.sample_rate)) + } + /// The receiver's buffers for the track's PID, if the T-STD gives it any. fn buffer(&self) -> Option { match self.kind { @@ -584,6 +607,29 @@ struct PesUnit { stream_id: Option, } +/// Consecutive audio frames muxed into one PES still to go out ([`Export::pack`]). +struct Pack { + /// The first frame's PES fields: its PTS times the frames after it too. + unit: PesUnit, + /// When the first frame decodes, the PES's deadline. + decode: Timestamp, + /// The frames' elementary stream bytes. + payload: Vec, + /// Each frame's decode nanoseconds and bytes, which leave the decoder buffer as it decodes. + frames: Vec<(u128, usize)>, + /// The frames' samples so far, and their sample rate. + samples: u64, + rate: u32, +} + +impl Pack { + /// When the frame after the last presents, in microseconds, if the frames run on without + /// a gap. + fn end(&self) -> u128 { + self.unit.timestamp.as_micros() + u128::from(self.samples) * 1_000_000 / u128::from(self.rate) + } +} + /// One SI entry's subscription: resolves the snapshot track named by the catalog, /// reduces its newest complete group into the current section set, and remembers /// when it last hit the wire so it re-emits on its own cadence. @@ -825,6 +871,7 @@ impl Export { stale: HashSet::new(), version: VersionNumber::default(), flagged: None, + tables: Vec::new(), delay: Duration::ZERO, jitter: jitter::Buffer::new(Duration::ZERO), held: None, @@ -1027,7 +1074,7 @@ impl Export { }, }; if ready.generation > self.generation { - if !self.schedule.is_empty() { + if !self.schedule.is_empty() || self.packing() { // A boundary ends valid media rather than reneging it. // Return that tail under the old generation before adopting the new one. self.lay(None)?; @@ -1049,6 +1096,7 @@ impl Export { .context("frame for an unknown PID")?; self.last_timestamp = Some(ready.item.frame.timestamp); let decode = ready.item.decode.as_nanos(); + self.lay_before(decode)?; self.mux(&name, ready.item)?; if self.delay.is_zero() { // Without a clock to lay slots on, a frame settles every slot before its own: a @@ -1497,7 +1545,7 @@ impl Export { let finished = !self.tracks.is_empty() && self.tracks.values().all(|track| track.finished); let slot = self .slot_due() - .filter(|_| self.schedule.queued() || finished) + .filter(|_| self.schedule.queued() || self.packing() || finished) .map(|(_, at)| at); self.jitter.next_deadline().into_iter().chain(slot).min() } @@ -1620,6 +1668,10 @@ impl Export { } self.video_start = None; self.pcr_discontinuity = true; + self.tables.clear(); + for track in self.tracks.values_mut() { + track.pack = None; + } } /// Header is ready when every track's [`ExportSource`] has resolved its @@ -1651,8 +1703,8 @@ impl Export { } /// Whether `pid`'s next packet is its first since a switch to another instance, so it has - /// to flag the break. The PCR PID's first packet is always a clock packet, flagged on its - /// own ([`Self::lay`]). + /// to flag the break. The PCR PID's first packet always opens a slot with its PCR, flagged + /// on its own ([`Self::lay`]). fn breaks(&mut self, pid: u16) -> bool { let pcr = self.psi.as_ref().map(|psi| psi.pcr_pid); match &mut self.flagged { @@ -1958,13 +2010,120 @@ impl Export { dts, stream_id, }; + // Only a delay leaves the frames time to wait for the ones after them. + let track = self.tracks.get(name).context("missing track")?; + if let Some(samples) = track.samples(&es_payload).filter(|_| !self.delay.is_zero()) { + return self.pack(name, unit, decode, out, es_payload, samples); + } + self.flush(name)?; self.write_pes(&mut out, &unit, &es_payload)?; } } - self.push_unit(name, pid, decode, out, keyframe) + self.push_unit(name, pid, decode, out, keyframe, Vec::new()) + } + + /// Mux an audio frame into `name`'s PES, sending the PES on first if the frame cannot join + /// it, and leave the tables muxed ahead of it to ride the next unit queued. A PES carries + /// frames while they run on from its PTS without a gap and fit the PID's decoder buffer + /// whole with its header, so it arrives by its first frame's decode time as one frame + /// would, and they span at most half the delay, so the PES has at least that long to go + /// out in. + /// + /// A PES starts at every frame whose place on the media timeline, counted in frames, is a + /// multiple of the frames the buffer holds, so exporters of the broadcast that joined at + /// different frames still cut the same PESs from the first such frame, like [`due`]. + fn pack( + &mut self, + name: &str, + unit: PesUnit, + decode: Timestamp, + tables: Vec, + payload: Vec, + (samples, rate): (u64, u32), + ) -> anyhow::Result<()> { + let budget = self.budget().as_nanos(); + let track = self.tracks.get(name).context("missing track")?; + // B holds the PES header with the frames (ISO 13818-1 2.4.2.3). + let size = track + .buffer() + .and_then(|buffer| buffer.size) + .unwrap_or(0) + .saturating_sub(PES_PREFIX_LEN + PES_OPTIONAL_LEN); + let fit = (size / payload.len().max(1)).max(1) as u128; + // Rounded, since a frame need not last whole microseconds: AAC's 1,024 samples at 48 kHz. + let frame = 1_000_000 * u128::from(samples); + let place = (unit.timestamp.as_micros() * u128::from(rate) + frame / 2) / frame; + // A PTS is a whole 90 kHz tick, so the frames run on within one of where they end. + let joins = !place.is_multiple_of(fit) + && track.pack.as_ref().is_some_and(|pack| { + pack.rate == rate + && pack.payload.len() + payload.len() <= size + && decode.as_nanos().saturating_sub(pack.decode.as_nanos()) <= budget + && pack.end().abs_diff(unit.timestamp.as_micros()) <= 12 + }); + if !joins { + self.flush(name)?; + } + self.tables.extend_from_slice(&tables); + let track = self.tracks.get_mut(name).context("missing track")?; + let pack = track.pack.get_or_insert(Pack { + unit, + decode, + payload: Vec::new(), + frames: Vec::new(), + samples: 0, + rate, + }); + pack.frames.push((decode.as_nanos(), payload.len())); + pack.payload.extend_from_slice(&payload); + pack.samples += samples; + // The next frame starts a PES of its own anyway, and without slots running on an + // empty schedule, nothing would send this one on before it is due. + if (place + 1).is_multiple_of(fit) || self.schedule.is_empty() { + self.flush(name)?; + } + Ok(()) + } + + /// Send `name`'s packed frames on as one PES. + fn flush(&mut self, name: &str) -> anyhow::Result<()> { + let Some(pack) = self.tracks.get_mut(name).and_then(|track| track.pack.take()) else { + return Ok(()); + }; + let Pack { + unit, + decode, + payload, + frames, + .. + } = pack; + let mut out = Vec::new(); + self.write_pes(&mut out, &unit, &payload)?; + self.push_unit(name, unit.pid, decode, out, unit.keyframe, frames) } - /// Queue one unit of `name`'s packets on `pid`, decoding at `decode`, on the [`Schedule`]. + /// Send on, in decode order, the packed frames of each track for which `done` holds, given + /// the slot its first frame is due in. + fn flush_packs(&mut self, done: impl Fn(u128) -> bool) -> anyhow::Result<()> { + let mut due: Vec<(Timestamp, String)> = self + .tracks + .iter() + .filter_map(|(name, track)| { + let pack = track.pack.as_ref()?; + let by = pack.decode.as_nanos().saturating_sub(track.drain().as_nanos()); + done(schedule::slot(by)).then(|| (pack.decode, name.clone())) + }) + .collect(); + due.sort(); + for (_, name) in due { + self.flush(&name)?; + } + Ok(()) + } + + /// Queue one unit of `name`'s packets on `pid`, decoding at `decode`, on the [`Schedule`], + /// behind the tables still waiting: with `frames`, each packed frame's decode nanoseconds + /// and bytes. fn push_unit( &mut self, name: &str, @@ -1972,13 +2131,22 @@ impl Export { decode: Timestamp, out: Vec, keyframe: bool, + frames: Vec<(u128, usize)>, ) -> anyhow::Result<()> { + let mut tables = std::mem::take(&mut self.tables); + let out = match tables.is_empty() { + true => out, + false => { + tables.extend_from_slice(&out); + tables + } + }; let track = self.tracks.get(name).context("missing track")?; if let Some(buffer) = track.buffer() { self.schedule.set_buffer(pid, buffer); } let by = decode.as_nanos().saturating_sub(track.drain().as_nanos()); - self.schedule.push( + self.schedule.push_frames( pid, super::schedule::Timing { by, @@ -1986,6 +2154,7 @@ impl Export { }, out, keyframe, + frames, )?; Ok(()) } @@ -2000,13 +2169,33 @@ impl Export { /// never on when they arrived, and the output keeps the clock's pace while media is queued /// or still to decode. `None` without a delay, where frames settle the slots instead. fn slot_due(&self) -> Option<(u128, web_async::time::Instant)> { - if self.delay.is_zero() || self.psi.is_none() || self.schedule.is_empty() { + let (known, settled) = self.slot_settled()?; + let settled = Timestamp::from_micros(u64::try_from(settled / 1_000).ok()?).ok()?; + Some((known, self.jitter.at(settled)?)) + } + + /// The `known` that settles the next grid slot, and the decode time, in nanoseconds, by + /// which every frame that could ride it has gone out of the jitter buffer. + fn slot_settled(&self) -> Option<(u128, u128)> { + if self.delay.is_zero() || self.psi.is_none() || (self.schedule.is_empty() && !self.packing()) { return None; } let (_, known) = self.schedule.upcoming()?; - let settled = known * PCR_INTERVAL.as_nanos() + self.max_drain().as_nanos(); - let settled = Timestamp::from_micros(u64::try_from(settled / 1_000).ok()?).ok()?; - Some((known, self.jitter.at(settled)?)) + Some((known, known * PCR_INTERVAL.as_nanos() + self.max_drain().as_nanos())) + } + + /// Whether a track holds frames packed for a PES still to go out. + fn packing(&self) -> bool { + self.tracks.values().any(|track| track.pack.is_some()) + } + + /// Lay out every grid slot settled before a frame decoding at `decode` goes out, so frames + /// and slots follow each other in the same order however often the exporter is polled. + fn lay_before(&mut self, decode: u128) -> anyhow::Result<()> { + while let Some((known, _)) = self.slot_settled().filter(|&(_, settled)| settled < decode) { + self.lay(Some(known))?; + } + Ok(()) } /// How far a stalled group may fall behind the newest content before each source skips it: @@ -2040,13 +2229,14 @@ impl Export { /// the first slot a frame still to be muxed could be due in, or `None` when nothing /// more is coming. /// - /// Each frame opens with its slot's clock packet. At a multiplex rate its value is the - /// time of its own byte position at the rate, so a consumer holding only the byte stream - /// (which is every MPEG-TS tool) recovers the same clock the values assert. Each frame is - /// stamped at its slot boundary, so a pacing caller releases the clock at the instant it - /// asserts. + /// Each frame opens with its slot's PCR: at a multiplex rate, in the adaptation field of + /// the clock PID's packet when one can go out there, else in a clock packet of its own. + /// At a multiplex rate its value is the time of its own byte position at the rate, so a + /// consumer holding only the byte stream (which is every MPEG-TS tool) recovers the same + /// clock the values assert. Each frame is stamped at its slot boundary, so a pacing + /// caller releases the clock at the instant it asserts. /// - /// The PES units cannot carry the clock themselves: a PCR sampled from the DTS would + /// The PCR's value never comes from the PES units: a PCR sampled from the DTS would /// step with the frame cadence, and no downstream CBR stage can repair that, because a /// groomer can only place the clock samples it receives. So the PCR follows its own /// grid instead: slots on the media timeline, shared by every exporter of the @@ -2056,13 +2246,29 @@ impl Export { return Ok(()); }; let (pcr_pid, pmt_pid) = (psi.pcr_pid, psi.pmt_pid); - while let Some(slot) = self.schedule.next(known)? { - let mut clock = pcr_packet(pcr_pid, slot.pcr)?; + if known.is_none() { + self.flush_packs(|_| true)?; + } + let lead = schedule::slot(self.budget().as_nanos()); + loop { + // A PES of packed frames goes out half the delay before its first is due, so it + // has that long to reach the decoder. + if known.is_some() + && let Some((index, _)) = self.schedule.upcoming() + { + self.flush_packs(|due| due <= index + lead)?; + } + let Some(slot) = self.schedule.next(known)? else { + break; + }; + let clock = pcr_packet(pcr_pid, slot.pcr)?; + let mut payload = slot.layout(&clock, pmt_pid, &NULL_PACKET); + // The slot opens with its PCR, in the clock packet or the clock PID's packet that + // carries it, whose adaptation field flags the break. let discontinuity = std::mem::take(&mut self.pcr_discontinuity); if discontinuity { - clock[5] |= 0x80; + payload[5] |= 0x80; } - let mut payload = slot.layout(&clock, pmt_pid, &NULL_PACKET); self.number(&mut payload); let frame = Frame { timestamp: slot_stamp(slot.index)?, @@ -2253,7 +2459,6 @@ impl Export { /// is numbered when it goes out ([`Export::number`]). fn pcr_packet(pid: u16, pcr: u64) -> anyhow::Result> { anyhow::ensure!(pid <= 0x1FFF, "PID out of range: {pid}"); - let (base, extension) = ((pcr / 300) & TS_TIMESTAMP_MASK, pcr % 300); let mut p = Vec::with_capacity(TsPacket::SIZE); p.push(0x47); p.push((pid >> 8) as u8); @@ -2264,17 +2469,13 @@ fn pcr_packet(pid: u16, pcr: u64) -> anyhow::Result> { p.push(183); // PCR_flag alone. p.push(0x10); - // program_clock_reference_base (33 bits), 6 reserved '1' bits, and the 9-bit extension. - p.push((base >> 25) as u8); - p.push((base >> 17) as u8); - p.push((base >> 9) as u8); - p.push((base >> 1) as u8); - p.push(((base as u8) << 7) | 0x7e | (extension >> 8) as u8); - p.push(extension as u8); + p.extend_from_slice(&super::schedule::pcr_field(pcr)); p.resize(TsPacket::SIZE, 0xff); Ok(p) } +/// PES header bytes ahead of the optional region: start code, stream id and packet length. +const PES_PREFIX_LEN: usize = 6; /// Optional PES header region carrying PTS only: 2 flag bytes + 1 length byte + 5 PTS bytes. const PES_OPTIONAL_LEN: usize = 3 + 5; /// Extra bytes when the optional region also carries a DTS (5 DTS bytes). diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index ec93bd40551..a9b5e4fd0f8 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -256,6 +256,72 @@ async fn export_skips_a_disabled_rendition() { assert_eq!(payloads, vec![vec![0x01, 0x02, 0x03]; 3]); } +/// AAC frames that run on from each other share a PES, as many as B holds with its header +/// (ISO 13818-1 2.4.2.3), each frame whole and in order, though none lasts whole microseconds. +#[tokio::test(start_paused = true)] +async fn export_packs_aac_frames_up_to_their_buffer() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track( + broadcast.unique_name(".aac"), + hang::container::track_info(hang::catalog::PRIORITY.audio), + ) + .unwrap(); + let name = track.name().to_string(); + { + let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 2); + cfg.container = Container::Legacy; + catalog.modify().unwrap().audio.renditions.insert(name.clone(), cfg); + } + let mut producer = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); + // 1,024 samples at 48 kHz: 21.33 ms each, 307 bytes with ADTS, 11 to B's 3,584. + let frames: Vec = (0..48u8).map(|k| Bytes::from(vec![k; 300])).collect(); + for (i, payload) in frames.iter().enumerate() { + producer + .write(Frame { + timestamp: Timestamp::from_micros(i as u64 * 1_024 * 1_000_000 / 48_000).unwrap(), + duration: None, + payload: payload.clone(), + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + } + producer.finish().unwrap(); + let ts = drain(consumer).await; + + let mut pes = PesPacketReader::new(TsPacketReader::new(Cursor::new(ts.as_ref()))); + let (mut count, mut recovered) = (0, Vec::new()); + while let Some(packet) = pes.read_pes_packet().unwrap() { + count += 1; + assert!( + 14 + packet.data.len() <= 3_584, + "a {}-byte PES overflows B", + packet.data.len() + ); + let mut pts = packet.header.pts.expect("PES carried no PTS").as_u64(); + let mut data = &packet.data[..]; + while !data.is_empty() { + let len = usize::from(data[3] & 3) << 11 | usize::from(data[4]) << 3 | usize::from(data[5] >> 5); + recovered.push((pts, data[7..len].to_vec())); + data = &data[len..]; + pts += 1_920; + } + } + assert!( + count <= frames.len().div_ceil(11) + 1, + "{count} PES for {} frames", + frames.len() + ); + assert_eq!(recovered.len(), frames.len()); + for (i, (pts, raw)) in recovered.iter().enumerate() { + assert!(pts.abs_diff(i as u64 * 1_920) <= 1, "frame {i} at {pts}"); + assert_eq!(raw.as_slice(), frames[i].as_ref(), "frame {i}"); + } +} + /// Collect PES presentation timestamps per elementary stream (video H.264, audio AAC), /// keyed off the PMT's PID assignments. fn collect_pes_pts(ts: &[u8]) -> (Vec, Vec) { @@ -1119,17 +1185,17 @@ async fn export_pcr_backfills_a_coarse_cadence() { /// AC-3 passed through as DVB private data (stream_type 0x06 with the AC-3 descriptor) can /// carry several sync frames in one PES, more than a receiver's 5,696-byte buffer holds at -/// once. The export splits it a frame a PES, each presented a frame after the last, so each -/// is due and decoded on its own. +/// once. The export splits it into its sync frames, each presented a frame after the last, so +/// each is due and decoded on its own. #[tokio::test(start_paused = true)] async fn export_splits_a_multi_frame_ac3_pes() { - let pes = exported_ac3(ac3_sync().repeat(3)).await; - assert_eq!(pes.len(), 3, "one PES a sync frame: {pes:?}"); - for (k, &(pts, bytes)) in pes.iter().enumerate() { + let frames = exported_ac3(ac3_sync().repeat(3)).await; + assert_eq!(frames.len(), 3, "every sync frame: {frames:?}"); + for (k, &(pts, bytes)) in frames.iter().enumerate() { assert_eq!(bytes, 128, "frame {k} carries one sync frame"); assert_eq!( pts, - pes[0].0 + k as u64 * 2_880, + frames[0].0 + k as u64 * 2_880, "frame {k} is presented 32 ms after the last" ); } @@ -1142,33 +1208,33 @@ async fn export_splits_a_multi_frame_ac3_pes() { async fn export_drops_a_trailing_partial_ac3_frame() { let mut payload = ac3_sync().repeat(2); payload.extend_from_slice(&ac3_sync()[..60]); - let pes = exported_ac3(payload).await; - assert_eq!(pes.len(), 2, "the whole frames, each its own PES: {pes:?}"); - assert!(pes.iter().all(|&(_, bytes)| bytes == 128), "{pes:?}"); + let frames = exported_ac3(payload).await; + assert_eq!(frames.len(), 2, "the whole frames: {frames:?}"); + assert!(frames.iter().all(|&(_, bytes)| bytes == 128), "{frames:?}"); - let pes = exported_ac3(ac3_sync()[..60].to_vec()).await; - assert!(pes.is_empty(), "a lone cut frame goes nowhere: {pes:?}"); + let frames = exported_ac3(ac3_sync()[..60].to_vec()).await; + assert!(frames.is_empty(), "a lone cut frame goes nowhere: {frames:?}"); } /// A source that leaves `data_alignment_indicator` clear splits AC-3 sync frames across PES, /// each PES's PTS timing the first frame that begins in it. The export puts each frame back -/// together, a frame a PES, byte for byte and presented a frame after the last. +/// together, byte for byte and presented a frame after the last. #[tokio::test(start_paused = true)] async fn export_reassembles_ac3_frames_split_across_pes() { let stream: Vec = (0..6).flat_map(ac3_frame).collect(); // Cut mid-frame into frames 0-1, 1-3 and 3-5, timed by frames 0, 2 and 4. - let pes = exported_ac3_pes(vec![ + let frames = exported_ac3_frames(vec![ Ac3Write::Pes(0, stream[..200].to_vec()), Ac3Write::Pes(64, stream[200..500].to_vec()), Ac3Write::Pes(128, stream[500..].to_vec()), ]) .await; - assert_eq!(pes.len(), 6, "one PES a sync frame"); - for (k, (pts, bytes)) in pes.iter().enumerate() { + assert_eq!(frames.len(), 6, "every sync frame"); + for (k, (pts, bytes)) in frames.iter().enumerate() { assert_eq!(*bytes, ac3_frame(k as u8), "frame {k} whole and in order"); assert_eq!( *pts, - pes[0].0 + k as u64 * 2_880, + frames[0].0 + k as u64 * 2_880, "frame {k} is presented 32 ms after the last" ); } @@ -1191,10 +1257,10 @@ async fn export_reassembles_an_ac3_header_split_across_pes() { ], ]; for input in cuts { - let pes = exported_ac3_pes(input).await; - let frames: Vec<&Vec> = pes.iter().map(|(_, bytes)| bytes).collect(); + let timed = exported_ac3_frames(input).await; + let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); assert_eq!(frames, [&ac3_frame(0), &ac3_frame(1)]); - assert_eq!(pes[1].0 - pes[0].0, 2_880); + assert_eq!(timed[1].0 - timed[0].0, 2_880); } } @@ -1205,15 +1271,15 @@ async fn export_reassembles_an_ac3_header_split_across_pes() { async fn export_drops_ac3_fragments_at_a_join_or_a_gap() { let stream: Vec = (0..6).flat_map(ac3_frame).collect(); // Joined 60 bytes into frame 0; the PES carrying the rest of frame 2 and most of 3 is lost. - let pes = exported_ac3_pes(vec![ + let timed = exported_ac3_frames(vec![ Ac3Write::Pes(32, stream[60..300].to_vec()), Ac3Write::Pes(128, stream[500..].to_vec()), ]) .await; - let frames: Vec<&Vec> = pes.iter().map(|(_, bytes)| bytes).collect(); + let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); assert_eq!(frames, [&ac3_frame(1), &ac3_frame(4), &ac3_frame(5)]); - assert_eq!(pes[1].0 - pes[0].0, 3 * 2_880, "frame 4 at its PES's PTS"); - assert_eq!(pes[2].0 - pes[1].0, 2_880); + assert_eq!(timed[1].0 - timed[0].0, 3 * 2_880, "frame 4 at its PES's PTS"); + assert_eq!(timed[2].0 - timed[1].0, 2_880); } /// A sync word in the bytes ahead of the first frame at a join can pass for a header whose @@ -1225,8 +1291,8 @@ async fn export_skips_an_unconfirmed_ac3_sync_word_at_a_join() { let mut payload = vec![0x0b, 0x77, 0x00, 0x00, 0x25, 0x40, 0x40]; payload.resize(40, 0x00); payload.extend((1..3).flat_map(ac3_frame)); - let pes = exported_ac3_pes(vec![Ac3Write::Pes(0, payload)]).await; - let frames: Vec<&Vec> = pes.iter().map(|(_, bytes)| bytes).collect(); + let timed = exported_ac3_frames(vec![Ac3Write::Pes(0, payload)]).await; + let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); assert_eq!(frames, [&ac3_frame(1), &ac3_frame(2)]); } @@ -1235,16 +1301,40 @@ async fn export_skips_an_unconfirmed_ac3_sync_word_at_a_join() { #[tokio::test(start_paused = true)] async fn export_drops_an_ac3_frame_cut_by_a_marker() { let stream: Vec = (0..4).flat_map(ac3_frame).collect(); - let pes = exported_ac3_pes(vec![ + let timed = exported_ac3_frames(vec![ Ac3Write::Pes(0, stream[..200].to_vec()), Ac3Write::Marker, Ac3Write::Pes(64, stream[256..].to_vec()), ]) .await; - let frames: Vec<&Vec> = pes.iter().map(|(_, bytes)| bytes).collect(); + let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); assert_eq!(frames, [&ac3_frame(0), &ac3_frame(2), &ac3_frame(3)]); } +/// AC-3 sync frames that run on share a PES, as many as DVB's 5,696-byte B holds with the +/// PES header (ISO 13818-1 2.4.2.3), and still decode in order 32 ms apart. +#[tokio::test(start_paused = true)] +async fn export_packs_ac3_frames_up_to_their_buffer() { + let stream: Vec = (0..100).flat_map(ac3_frame).collect(); + let pes = exported_ac3_pes(vec![Ac3Write::Pes(0, stream.clone())]).await; + // 44 of the 128-byte frames to a PES, and the first frame alone, sent as it starts. + assert!( + pes.len() <= 100usize.div_ceil(44) + 1, + "{} PES for 100 frames", + pes.len() + ); + for (_, payload) in &pes { + assert!(14 + payload.len() <= 5_696, "a {}-byte PES overflows B", payload.len()); + } + let bytes: Vec = pes.into_iter().flat_map(|(_, payload)| payload).collect(); + assert_eq!(bytes, stream, "every frame whole and in order"); + let timed = exported_ac3_frames(vec![Ac3Write::Pes(0, stream)]).await; + assert!( + timed.windows(2).all(|pair| pair[1].0 - pair[0].0 == 2_880), + "32 ms apart" + ); +} + /// A 32 kb/s 48 kHz 2/0 AC-3 sync frame: 128 bytes, 1,536 samples (32 ms). fn ac3_sync() -> Vec { let mut sync = vec![0x0b, 0x77, 0x00, 0x00, 0x00, 0x40, 0x40]; @@ -1260,9 +1350,9 @@ fn ac3_frame(k: u8) -> Vec { } /// Export one PES of `payload` on an AC-3 passthrough PID beside bbb's media, and return the -/// PTS of every PES on that PID and how many bytes each carries. +/// PTS of every sync frame on that PID and how many bytes each carries. async fn exported_ac3(payload: Vec) -> Vec<(u64, usize)> { - exported_ac3_pes(vec![Ac3Write::Pes(0, payload)]) + exported_ac3_frames(vec![Ac3Write::Pes(0, payload)]) .await .into_iter() .map(|(pts, bytes)| (pts, bytes.len())) @@ -1369,6 +1459,21 @@ async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec)> { pes } +/// Export `input` as [`exported_ac3_pes`] does, and return each sync frame on the AC-3 PID, +/// timed by its PES's PTS and the frames ahead of it there. +async fn exported_ac3_frames(input: Vec) -> Vec<(u64, Vec)> { + let mut frames = Vec::new(); + for (mut pts, mut payload) in exported_ac3_pes(input).await { + while !payload.is_empty() { + let header = crate::codec::ac3::parse_header(&payload).expect("the PES carries whole sync frames"); + let rest = payload.split_off(header.len); + frames.push((pts, std::mem::replace(&mut payload, rest))); + pts += header.samples * 90_000 / u64::from(header.sample_rate); + } + } + frames +} + /// Full SCTE-35 round-trip: import `bbb.ts` (real H.264 + AAC) into a broadcast /// that also carries a `.scte35` cue track, export to TS, re-import, and assert /// the splice_info_section came back byte-for-byte. The PMT must advertise the @@ -4901,6 +5006,26 @@ fn pes_decode_in_order(frames: &[Frame]) -> Vec { out } +/// How many access units the stream carries: one a PES, or each ADTS frame of an AAC PES. +fn access_units(frames: &[Frame]) -> usize { + let bytes: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); + let mut reader = PesPacketReader::new(TsPacketReader::new(Cursor::new(bytes))); + let mut units = 0; + while let Some(pes) = reader.read_pes_packet().unwrap() { + let mut data = &pes.data[..]; + if !(data.len() > 6 && data[0] == 0xff && data[1] & 0xf0 == 0xf0) { + units += 1; + continue; + } + while data.len() > 6 { + let len = usize::from(data[3] & 3) << 11 | usize::from(data[4]) << 3 | usize::from(data[5] >> 5); + data = &data[len.clamp(7, data.len())..]; + units += 1; + } + } + units +} + /// Whether every PES start decodes no earlier than those ahead of it, up to one grid slot: /// a slot spreads each PID's packets among the others, so units sharing one may swap. fn in_decode_order(decode: &[u64]) -> bool { @@ -5005,7 +5130,8 @@ async fn lagging_audio(delay: Duration) -> Vec { #[tokio::test(start_paused = true)] async fn late_audio_still_arrives_on_time() { let frames = lagging_audio(Duration::from_millis(500)).await; - assert!(assert_on_time(&frames) > 100, "too little output to judge"); + // 75 video frames, and the audio packed several frames a PES. + assert!(assert_on_time(&frames) > 80, "too little output to judge"); } /// A zero delay holds nothing, so the interleave stays in arrival order. @@ -5065,10 +5191,9 @@ async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { out.extend(drain_frames(&mut export).await); let written = rig.audio_index + TICKS / 2; - let decode = pes_decode_in_order(&out); assert!(export.dropped() > 0, "the stalled audio missed its deadline"); assert_eq!( - decode.len() as u64, + access_units(&out) as u64, written - export.dropped(), "only the late frames were dropped" ); @@ -5802,8 +5927,8 @@ fn collect_pcrs(frames: &[Frame]) -> Vec<(usize, u64, u128)> { let mut pcrs = Vec::new(); for frame in frames { for packet in frame.payload.chunks(188) { - // adaptation-field-only packets (adaptation_field_control == 0b10) are the clock. - if packet[3] & 0x30 == 0x20 && packet[5] & 0x10 != 0 { + // A clock packet, or the clock PID's packet that carries the slot's PCR. + if packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x10 != 0 { let base = (u64::from(packet[6]) << 25) | (u64::from(packet[7]) << 17) | (u64::from(packet[8]) << 9) diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index d4728bad6c0..42dae3e9fde 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -175,6 +175,10 @@ struct Unit { pid: u16, /// The coarse deadline used by release windows and unpadded admission. due: u128, + /// The due slot of its last frame. A PES that packs several is pushed once that frame is + /// muxed, so its window runs from there, as one frame's runs from when it is pushed: every + /// slot it may ride is laid out after it is queued, whenever the caller lays them. + settled: u128, timing: Timing, /// The slot after the unit pushed before it, which it may go out from even without a /// window: nothing pushed later can be due sooner. @@ -187,6 +191,9 @@ struct Unit { late: bool, /// Its bytes in the PID's decoder buffer, until it decodes. held: usize, + /// The decode time and bytes of each frame of a PES that packs several, which leave the + /// decoder buffer one by one; empty for a PES of one frame, which leaves at once. + frames: Vec<(u128, usize)>, } impl Unit { @@ -194,6 +201,33 @@ impl Unit { self.packets.len() / TsPacket::SIZE } + /// When its last bytes leave the decoder buffer. + fn last_decode(&self) -> u128 { + self.frames.last().map_or(self.timing.decode, |&(decode, _)| decode) + } + + /// When each of its frames leaves the decoder buffer, with the bytes it takes: all it + /// holds, for a unit of one frame. + fn frames(&self) -> impl Iterator + '_ { + let single = self.frames.is_empty().then_some((self.timing.decode, usize::MAX)); + self.frames.iter().copied().chain(single) + } + + /// What it leaves in the decoder buffer once sent: each frame's PID, decode time and + /// bytes, of the bytes it holds, which the frames decoded already have taken first. + fn leaving(&self) -> Vec<(u16, u128, usize)> { + let mut held = self.held; + let mut out: Vec<_> = self.frames().collect(); + for (_, bytes) in out.iter_mut().rev() { + *bytes = (*bytes).min(held); + held -= *bytes; + } + out.into_iter() + .filter(|&(_, bytes)| bytes > 0) + .map(|(decode, bytes)| (self.pid, decode, bytes)) + .collect() + } + fn remaining(&self) -> usize { self.count() - self.sent } @@ -234,6 +268,90 @@ fn elementary(packet: &[u8]) -> &[u8] { } } +/// A PCR of `ticks` as an adaptation field carries it: the 33-bit base, six reserved bits, +/// all ones as ISO 13818-1 requires, and the 9-bit extension. +pub(super) fn pcr_field(ticks: u64) -> [u8; 6] { + let (base, extension) = ((ticks / 300) & ((1 << 33) - 1), ticks % 300); + [ + (base >> 25) as u8, + (base >> 17) as u8, + (base >> 9) as u8, + (base >> 1) as u8, + ((base as u8) << 7) | 0x7e | (extension >> 8) as u8, + extension as u8, + ] +} + +/// `packets`, the rest of a PES on one PID, rewritten so the first carries the PCR `ticks` in +/// its adaptation field: the payload moves on by the field's bytes, into the stuffing of the +/// last packet or one more packet. `None` unless every packet is the PID's with payload, and +/// its adaptation field, if any, carries nothing but flags and stuffing, as the export +/// writes them. +fn with_pcr(packets: &[u8], ticks: u64) -> Option> { + let packets = packets.as_chunks::<{ TsPacket::SIZE }>().0; + let first = packets.first()?; + let mut body = Vec::with_capacity(packets.len() * PAYLOAD); + let mut flags = Vec::with_capacity(packets.len()); + for packet in packets { + if pid(packet) != pid(first) || packet[3] & 0x10 == 0 { + return None; + } + // Only the PES's first packet may start it, and only the flags byte's random access + // and discontinuity indicators may be set. + if packet != first && packet[1] & 0x40 != 0 { + return None; + } + let field = (packet[3] & 0x20 != 0).then(|| &packet[5..5 + usize::from(packet[4])]); + let flag = field.and_then(|field| field.first().copied()).unwrap_or(0); + if flag & !0xc0 != 0 { + return None; + } + flags.push(flag); + body.extend_from_slice(payload(packet)); + } + if body.is_empty() { + return None; + } + let mut out = Vec::with_capacity((packets.len() + 1) * TsPacket::SIZE); + let mut offset = 0; + for k in 0.. { + let start = if k == 0 { first[1] & 0x40 } else { 0 }; + let flag = flags.get(k).copied().unwrap_or(0); + let mut field = Vec::new(); + if k == 0 { + field.push(flag | 0x10); + field.extend_from_slice(&pcr_field(ticks)); + } else if flag != 0 { + field.push(flag); + } + let room = PAYLOAD - if field.is_empty() { 0 } else { 1 + field.len() }; + let take = room.min(body.len() - offset); + if take < room { + // Stuffing fills the field out to the packet's end; a field of length 0 is the + // single byte of stuffing. + if field.is_empty() && take + 1 < PAYLOAD { + field.push(0); + } + field.resize(PAYLOAD - 1 - take, 0xff); + } + out.extend_from_slice(&[first[0], first[1] & !0x40 | start, first[2]]); + match field.is_empty() && take == PAYLOAD { + true => out.push(0x10), + false => { + out.push(0x30); + out.push(field.len() as u8); + out.extend_from_slice(&field); + } + } + out.extend_from_slice(&body[offset..offset + take]); + offset += take; + if offset == body.len() { + break; + } + } + Some(out) +} + /// One grid slot's worth of output. pub(super) struct Slot { pub index: u128, @@ -251,14 +369,17 @@ pub(super) struct Slot { pub caps: HashMap, /// Exact positions of the media packets in a padded slot, counting the opening clock. positions: Option>, + /// The padded slot opens with a packet of the clock's PID carrying its PCR, so it needs + /// no clock packet of its own. + pub clocked: bool, } impl Slot { - /// The whole slot, opening with its `clock` packet. Padded slots use the exact - /// positions planned against receiver buffers. Unpadded slots use smooth weighted - /// round-robin: each PID (the clock packet counted on its own) and the null packets take - /// positions in proportion to their count, as evenly as they can, every PID keeping its - /// own order. + /// The whole slot, opening with its `clock` packet unless it is [`Self::clocked`]. Padded + /// slots use the exact positions planned against receiver buffers. Unpadded slots use + /// smooth weighted round-robin: each PID (the clock packet counted on its own) and the null + /// packets take positions in proportion to their count, as evenly as they can, every PID + /// keeping its own order. /// /// A PID's packets in a run would arrive at the multiplex's full rate and overflow its /// 512-byte transport buffer, which drains only at its Rx (ISO 13818-1 2.4.2.3): an audio @@ -274,15 +395,18 @@ impl Slot { pub fn layout(&self, clock: &[u8], pmt_pid: u16, null: &[u8]) -> Vec { let packets = self.packets.as_chunks::<{ TsPacket::SIZE }>().0; if let Some(positions) = &self.positions { - let mut out = Vec::with_capacity((1 + packets.len() + self.nulls) * TsPacket::SIZE); - out.extend_from_slice(clock); + let count = usize::from(!self.clocked) + packets.len() + self.nulls; + let mut out = Vec::with_capacity(count * TsPacket::SIZE); + if !self.clocked { + out.extend_from_slice(clock); + } for (&position, packet) in positions.iter().zip(packets) { while out.len() / TsPacket::SIZE < position { out.extend_from_slice(null); } out.extend_from_slice(packet); } - while out.len() / TsPacket::SIZE < 1 + packets.len() + self.nulls { + while out.len() / TsPacket::SIZE < count { out.extend_from_slice(null); } return out; @@ -508,12 +632,30 @@ impl Schedule { self.clock = Some(pid); } - /// Queue an access unit on `pid` (whole TS packets, the tables muxed ahead of it first) - /// with its exact transport deadline and decoder removal time. + /// Queue an access unit of one frame. + #[cfg(test)] pub fn push(&mut self, pid: u16, timing: Timing, packets: Vec, keyframe: bool) -> anyhow::Result<()> { + self.push_frames(pid, timing, packets, keyframe, Vec::new()) + } + + /// Queue an access unit on `pid` (whole TS packets, the tables muxed ahead of it first) + /// with its exact transport deadline and decoder removal time. A PES that packs several + /// frames lists each one's decode time and elementary bytes, in decode order, which leave + /// the decoder buffer one by one; `timing` is then the first frame's. + pub fn push_frames( + &mut self, + pid: u16, + timing: Timing, + packets: Vec, + keyframe: bool, + mut frames: Vec<(u128, usize)>, + ) -> anyhow::Result<()> { if packets.is_empty() { return Ok(()); } + if frames.len() < 2 { + frames.clear(); + } let due = slot(timing.by); // An ended source relaxes deadlines, never buffer sizes: such a unit never fits. if let Some(size) = self.buffers.get(&pid).and_then(|buffer| buffer.size) { @@ -538,13 +680,23 @@ impl Schedule { "MPEG-TS access unit on PID {pid} has {elementary} payload bytes ({bytes} bytes with PES headers), exceeding its {size}-byte decoder buffer" ); } + // The PES header leaves the decoder buffer with the first frame after it. + if let Some(first) = frames.first_mut() { + first.1 += bytes - on_pid().map(|packet| elementary(packet).len()).sum::(); + } } - self.horizon = self.horizon.max(Some(timing.by.div_ceil(PCR_INTERVAL.as_nanos()))); + // The deadline its last frame would have on its own. + let last = timing.by + + frames + .last() + .map_or(0, |&(decode, _)| decode.saturating_sub(timing.decode)); + self.horizon = self.horizon.max(Some(last.div_ceil(PCR_INTERVAL.as_nanos()))); let start = self.last.map_or(due, |last| (last + 1).min(due)); self.last = Some(due); self.units.push_back(Unit { pid, due, + settled: slot(last), timing, start, packets, @@ -552,6 +704,7 @@ impl Schedule { keyframe, late: self.next.is_some_and(|next| due < next), held: 0, + frames, }); Ok(()) } @@ -582,16 +735,16 @@ impl Schedule { self.ended = false; } - /// The first slot `unit` may go out in: a window ahead of its due slot, or only the slot - /// before it for a PID whose decoder buffer bounds nothing, and never before the slot - /// after the unit pushed ahead of it needs. + /// The first slot `unit` may go out in: a window ahead of its last frame's due slot, or + /// only the slot before it for a PID whose decoder buffer bounds nothing, and never before + /// the slot after the unit pushed ahead of it needs. fn release(&self, unit: &Unit) -> u128 { let ahead = match self.buffers.get(&unit.pid).and_then(|buffer| buffer.size) { _ if unit.late => return 0, Some(_) => self.window, None => 1, }; - unit.due.saturating_sub(ahead).min(unit.start) + unit.settled.saturating_sub(ahead).min(unit.start) } /// The slot [`Self::next`] lays out next, with `known` as it would be given. @@ -656,15 +809,15 @@ impl Schedule { keyframe |= unit.keyframe; // Only the tables muxed ahead of a frame that wrote nothing are no access unit. if unit.packets_on(0..unit.count(), unit.pid) > 0 { - units.push(unit.pid); + units.extend(std::iter::repeat_n(unit.pid, unit.frames.len().max(1))); } } let from = unit.sent * TsPacket::SIZE; packets.extend_from_slice(&unit.packets[from..from + take * TsPacket::SIZE]); unit.sent += take; } - for unit in self.units.iter().filter(|unit| unit.remaining() == 0 && unit.held > 0) { - self.decoding.push_back((unit.pid, unit.timing.decode, unit.held)); + for unit in self.units.iter().filter(|unit| unit.remaining() == 0) { + self.decoding.extend(unit.leaving()); } self.units.retain(|unit| unit.remaining() > 0); Ok(Some(Slot { @@ -680,6 +833,7 @@ impl Schedule { .map(|(&pid, buffer)| (pid, buffer.per_slot(index, self.clock == Some(pid)))) .collect(), positions: None, + clocked: false, })) } @@ -701,22 +855,16 @@ impl Schedule { units: Vec::new(), caps: HashMap::new(), positions: Some(Vec::new()), + clocked: false, }; - // The transport buffer the clock packet rides and how long it holds it, which every - // slot's clock packet may add to a busy stretch without the chance to wait. - let mut clock = None; - if let Some(pid) = self.clock - && let Some(buffer) = self.buffers.get(&pid) - { - let duration = (TsPacket::SIZE as u128 * 8 * 1_000_000_000).div_ceil(u128::from(buffer.rate)) as i128; - let tb = self.transport_buffer(pid); - let (free, since) = self.transport.entry(tb).or_insert((time(0), time(0))); - if *free + TICK <= time(0) { - *since = time(0); - } - *free = ((*free).max(time(0)) + duration).max(time(1)); - clock = Some((tb, duration)); - } + let clock = self.clock.filter(|pid| self.buffers.contains_key(pid)); + // The transport buffer every slot's PCR rides and how long it holds the packet carrying + // it, which each later slot's PCR may add to a busy stretch without the chance to wait. + let clock_buffer = clock.map(|pid| { + let rate = u128::from(self.buffers[&pid].rate); + let duration = (TsPacket::SIZE as u128 * 8 * 1_000_000_000).div_ceil(rate) as i128; + (self.transport_buffer(pid), duration) + }); // One head per PID preserves decode order without walking all future units at every // transport position. Removal events keep occupancy work proportional to units. let mut lanes: Vec> = Vec::new(); @@ -734,9 +882,11 @@ impl Schedule { lanes.len() - 1 }); lanes[lane].push_back(i); - if unit.timing.decode as i128 > time(0) { + if unit.last_decode() as i128 > time(0) { *occupancy.entry(unit.pid).or_default() += unit.held; - removals.push((unit.timing.decode, unit.pid, Some(i), 0)); + for (decode, bytes) in unit.frames().filter(|&(decode, _)| decode as i128 > time(0)) { + removals.push((decode, unit.pid, Some(i), bytes)); + } } } removals.sort_by_key(|event| event.0); @@ -750,11 +900,16 @@ impl Schedule { }) .collect(); let mut waiting: BinaryHeap> = BinaryHeap::new(); - let mut position = 1; + let mut position = 0; while position < count { let now = time(position); while let Some((_, pid, unit, held)) = removals.next_if(|event| event.0 as i128 <= now) { - let held = unit.map_or(held, |i| std::mem::take(&mut self.units[i].held)); + let held = unit.map_or(held, |i| { + let unit = &mut self.units[i]; + let take = held.min(unit.held); + unit.held -= take; + take + }); *occupancy.get_mut(&pid).unwrap() -= held; } while waiting.peek().is_some_and(|Reverse((at, _))| *at <= now) { @@ -762,10 +917,26 @@ impl Schedule { ready.push(Reverse((self.units[*lanes[lane].front().unwrap()].timing.by, lane))); } let mut emitted = false; - while let Some(Reverse((_, lane))) = ready.pop() { + // The slot's PCR opens it. Its PID's next packet carries it there when it can go + // out at once, which spares the slot a clock packet; the other PIDs wait a position. + let mut passed = Vec::new(); + while let Some(Reverse((by, lane))) = ready.pop() { let i = *lanes[lane].front().unwrap(); let unit = &self.units[i]; - let packet = &unit.packets[unit.sent * TsPacket::SIZE..(unit.sent + 1) * TsPacket::SIZE]; + let rest = &unit.packets[unit.sent * TsPacket::SIZE..]; + let opening = match position { + 0 => match clock.filter(|&clock| self::pid(rest) == clock) { + Some(_) => with_pcr(rest, out.pcr), + None => None, + }, + _ => None, + }; + if position == 0 && opening.is_none() { + passed.push(Reverse((by, lane))); + continue; + } + let packet = opening.as_deref().unwrap_or(rest); + let packet = &packet[..TsPacket::SIZE]; let pid = self::pid(packet); let buffer = self.buffers.get(&pid); let duration = buffer.map_or(0, |buffer| { @@ -817,7 +988,7 @@ impl Schedule { // busy that long waits for it to. Every later slot's clock packet that arrives // before the buffer empties extends the stretch without the chance to wait. let mut busy = arrived; - if let Some((_, duration)) = clock.filter(|&(clock, _)| clock == tb) { + if let Some((_, duration)) = clock_buffer.filter(|&(clock, _)| clock == tb) { let mut next = index + 1; while busy - since <= TB_EMPTY.as_nanos() as i128 { let start = (packets_before(next, rate) - first) as usize; @@ -859,14 +1030,21 @@ impl Schedule { out.packets.extend_from_slice(packet); out.nulls -= 1; let unit = &mut self.units[i]; + if let Some(rest) = opening { + unit.packets.truncate(unit.sent * TsPacket::SIZE); + unit.packets.extend_from_slice(&rest); + out.clocked = true; + out.nulls += 1; + } if unit.sent == 0 { out.keyframe |= unit.keyframe; if unit.packets_on(0..unit.count(), unit.pid) > 0 { - out.units.push(unit.pid); + out.units + .extend(std::iter::repeat_n(unit.pid, unit.frames.len().max(1))); } } unit.sent += 1; - if unit.timing.decode as i128 > now { + if unit.last_decode() as i128 > now { unit.held += held; *occupancy.entry(unit.pid).or_default() += held; } @@ -882,7 +1060,19 @@ impl Schedule { emitted = true; break; } - if emitted { + ready.extend(passed); + if position == 0 && !emitted { + // A clock packet of its own opens the slot, and its PID's transport buffer + // takes it first. + if let Some((tb, duration)) = clock_buffer { + let (free, since) = self.transport.entry(tb).or_insert((time(0), time(0))); + if *free + TICK <= time(0) { + *since = time(0); + } + *free = ((*free).max(time(0)) + duration).max(time(1)); + } + position = 1; + } else if emitted { position += 1; } else if let Some(Reverse((next, _))) = waiting.peek() { // Nothing can use intervening null positions. Jump to the first packet at the @@ -906,8 +1096,8 @@ impl Schedule { unit.pid ); } - for unit in self.units.iter().filter(|unit| unit.remaining() == 0 && unit.held > 0) { - self.decoding.push_back((unit.pid, unit.timing.decode, unit.held)); + for unit in self.units.iter().filter(|unit| unit.remaining() == 0) { + self.decoding.extend(unit.leaving()); } self.units.retain(|unit| unit.remaining() > 0); Ok(out) @@ -1033,7 +1223,8 @@ mod tests { slots } - /// Check actual wire positions against the receiver's transport buffer, clock included. + /// Check actual wire positions against the receiver's transport buffer, clock included, + /// and count the packets' worth of payload PID 1 sent. fn checked_transport(schedule: &mut Schedule, clock: u8) -> usize { let rate = schedule.rate.unwrap(); let rx = schedule.buffers[&1].rate; @@ -1047,7 +1238,8 @@ mod tests { .0 .iter() .filter(|packet| pid(*packet) == 1) - .count(); + .map(|packet| payload(packet).len()) + .sum::(); for (position, packet) in slot.layout(&unit(clock, 1), 100, &null).chunks(188).enumerate() { let at = slot.pcr as f64 / SYSTEM_CLOCK as f64 + position as f64 * 1504.0 / rate as f64; if let Some(previous) = previous { @@ -1060,7 +1252,7 @@ mod tests { assert!(fill <= 512.001, "PID1 transport buffer holds {fill} bytes"); } } - sent + sent / PAYLOAD } /// Distinct tune-in tables keep their copies and still fit TBsys, which the PAT and PMT @@ -1342,8 +1534,8 @@ mod tests { assert_eq!(checked_transport(&mut schedule, 9), 80); } - /// The clock packet opening every slot rides its PID's transport buffer too, so that PID - /// takes one packet fewer of its own. + /// The PCR opening every slot rides its PID's transport buffer too, in a clock packet or + /// one of the PID's own. #[test] fn the_clock_packet_counts_against_its_pid() { let mut schedule = Schedule::new(Duration::from_millis(100)); @@ -1393,7 +1585,7 @@ mod tests { let (mut tb, mut mb, mut previous, mut sent) = (0.0f64, 0.0f64, None::, 0); while let Some(slot) = schedule.next(None).unwrap() { let mut clock = unit(1, 1); - clock[3] = 0x20; + (clock[3], clock[4]) = (0x20, 183); for (position, packet) in slot.layout(&clock, 100, &null).chunks(188).enumerate() { let at = slot.pcr as f64 / SYSTEM_CLOCK as f64 + position as f64 * 1504.0 / rate as f64; if let Some(previous) = previous { @@ -1404,13 +1596,13 @@ mod tests { if pid(packet) == 1 { tb += 188.0; mb += elementary(packet).len() as f64; - sent += usize::from(packet[3] & 0x10 != 0); + sent += payload(packet).len(); } assert!(tb <= 512.001, "TB holds {tb} bytes at {at}"); assert!(mb <= 16_000.0, "MB holds {mb} bytes at {at}"); } } - assert_eq!(sent, 500 * 652, "every frame goes out"); + assert_eq!(sent, 500 * 652 * PAYLOAD, "every frame goes out"); } /// Audio due before the video takes a position or a few between video packets, and MB @@ -1424,24 +1616,26 @@ mod tests { rate: 28_800_000, leak: Some(RBX), size: Some(3_750_000), + transport: 512, }; let audio = Buffer { rate: 2_000_000, leak: None, size: Some(3_584), + transport: 512, }; let rate = 26_000_000; let mut schedule = Schedule::new(Duration::from_secs(2)); schedule.set_rate(Some(rate)); - schedule.set_buffer(1, video); - schedule.set_buffer(2, audio); - schedule.set_buffer(3, audio); - schedule.set_clock(1); + schedule.set_buffer(7, video); + schedule.set_buffer(8, audio); + schedule.set_buffer(9, audio); + schedule.set_clock(7); // Two seconds of video due from 2 s on, so it is queued from the first slot, and audio // due within the first second: MP2's 24 ms frames and AC-3's 32 ms, four packets each. - let mut units: Vec<(u128, u8, usize)> = (0..50).map(|frame| (2_000 + 40 * frame, 1, 652)).collect(); - units.extend((0..40).map(|frame| (24 * frame, 2, 4))); - units.extend((0..30).map(|frame| (32 * frame, 3, 4))); + let mut units: Vec<(u128, u8, usize)> = (0..50).map(|frame| (2_000 + 40 * frame, 7, 652)).collect(); + units.extend((0..40).map(|frame| (24 * frame, 8, 4))); + units.extend((0..30).map(|frame| (32 * frame, 9, 4))); units.sort(); for (ms, pid, packets) in units { schedule @@ -1451,7 +1645,7 @@ mod tests { let mut video = 0; for _ in 0..40 { let slot = schedule.next(None).unwrap().unwrap(); - video += slot.packets.chunks(188).filter(|packet| pid(packet) == 1).count(); + video += slot.packets.chunks(188).filter(|packet| pid(packet) == 7).count(); } // A second of the leak is 16,304 packets of payload. assert!(video >= 16_290, "{video} video packets in a second"); @@ -1514,8 +1708,9 @@ mod tests { transport: 512, }, ); - schedule.push(1, timing(1_000), unit(1, 4), false).unwrap(); - schedule.push(1, timing(1_200), unit(1, 4), false).unwrap(); + // 76.8 kb/s passes on five packets in the window, the slots' PCRs among them. + schedule.push(1, timing(1_000), unit(1, 6), false).unwrap(); + schedule.push(1, timing(1_200), unit(1, 6), false).unwrap(); let error = std::iter::from_fn(|| schedule.next(None).transpose()) .find_map(Result::err) .unwrap(); @@ -1547,10 +1742,11 @@ mod tests { assert_eq!(sent, [(32, 8), (36, 2), (41, 6)]); } - /// Packetize a PES with exactly `bytes` elementary bytes and adaptation stuffing. + /// Packetize a PES with exactly `bytes` elementary bytes, each its offset's low byte, and + /// adaptation stuffing. fn audio(bytes: usize) -> Vec { - let mut pes = vec![0; 14 + bytes]; - pes[..9].copy_from_slice(&[0, 0, 1, 0xbd, 0, 0, 0x80, 0x80, 5]); + let mut pes: Vec = (0..14 + bytes).map(|i| i as u8).collect(); + pes[..14].copy_from_slice(&[0, 0, 1, 0xbd, 0, 0, 0x80, 0x80, 5, 0, 0, 0, 0, 0]); let mut out = Vec::new(); for (index, body) in pes.chunks(PAYLOAD).enumerate() { let mut packet = [0xff; TsPacket::SIZE]; @@ -1558,6 +1754,9 @@ mod tests { if body.len() < PAYLOAD { packet[3] |= 0x20; packet[4] = (PAYLOAD - body.len() - 1) as u8; + if packet[4] > 0 { + packet[5] = 0; + } } packet[TsPacket::SIZE - body.len()..].copy_from_slice(body); out.extend(packet); @@ -1835,6 +2034,7 @@ mod tests { units: vec![], caps: HashMap::new(), positions: None, + clocked: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -1873,6 +2073,7 @@ mod tests { units: vec![1, 3], caps: HashMap::new(), positions: None, + clocked: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -1941,6 +2142,7 @@ mod tests { .per_slot(0, true), )]), positions: None, + clocked: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -1997,6 +2199,7 @@ mod tests { units: vec![1, 2], caps: HashMap::from([(1, a.per_slot(0, true)), (2, b.per_slot(0, false))]), positions: None, + clocked: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -2016,4 +2219,160 @@ mod tests { assert!(peak <= 512.0, "PID {media} TB peak {peak:.0} B"); } } + + /// The PCR `packet`'s adaptation field carries, in system clock ticks. + fn pcr_of(packet: &[u8]) -> Option { + let field = (packet[3] & 0x20 != 0 && packet[4] >= 7 && packet[5] & 0x10 != 0).then(|| &packet[6..12])?; + let base = u64::from(field[0]) << 25 + | u64::from(field[1]) << 17 + | u64::from(field[2]) << 9 + | u64::from(field[3]) << 1 + | u64::from(field[4]) >> 7; + Some(base * 300 + (u64::from(field[4] & 1) << 8 | u64::from(field[5]))) + } + + /// A PCR moves the PES it rides on by its field's bytes: into the last packet's stuffing, + /// or into one more packet where that has too little. + #[test] + fn a_pcr_reflows_the_pes_it_rides() { + let body = |packets: &[u8]| -> Vec { packets.chunks(TsPacket::SIZE).flat_map(payload).copied().collect() }; + // With its 14-byte header: one packet with room, two with 7 bytes of stuffing, two full. + for (bytes, count) in [(100, 1), (346, 2), (354, 3)] { + let pes = audio(bytes); + let ticks = 27_000_000 * 5 + 299; + let out = with_pcr(&pes, ticks).expect("an exported PES takes a PCR"); + let packets: Vec<&[u8]> = out.chunks(TsPacket::SIZE).collect(); + assert_eq!(packets.len(), count, "{bytes} bytes"); + assert_eq!(out.len(), count * TsPacket::SIZE, "{bytes} bytes"); + assert_eq!(pcr_of(packets[0]), Some(ticks), "{bytes} bytes"); + assert!( + packets[0][1] & 0x40 != 0, + "{bytes} bytes: the first packet still starts the PES" + ); + assert!( + packets[1..] + .iter() + .all(|packet| packet[1] & 0x40 == 0 && pcr_of(packet).is_none()), + "{bytes} bytes: the PCR and the PES start stay in the first packet" + ); + assert_eq!(body(&out), body(&pes), "{bytes} bytes: the PES is unchanged"); + assert!( + with_pcr(&out, ticks).is_none(), + "a PES carrying a PCR already is left alone" + ); + } + } + + /// A slot's PCR rides the clock PID's packet when that PID can go out at once, sparing the + /// slot a clock packet; a slot with only another PID's packets opens with one of its own. + #[test] + fn the_pcr_rides_the_clock_pid_when_it_can() { + let mut schedule = Schedule::new(Duration::from_millis(200)); + schedule.set_rate(Some(RATE)); + schedule.set_buffer(1, VIDEO); + schedule.set_buffer(2, VIDEO); + schedule.set_clock(1); + schedule.push(1, timing(1_000), unit(1, 4), true).unwrap(); + schedule.push(2, timing(2_000), unit(2, 4), false).unwrap(); + let mut clock = unit(1, 1); + (clock[3], clock[4]) = (0x20, 183); + let mut null = unit(0, 1); + (null[1], null[2]) = (0x1f, 0xff); + let (mut sent, mut clocked) = (HashMap::::new(), Vec::new()); + while let Some(slot) = schedule.next(None).unwrap() { + let laid = slot.layout(&clock, 100, &null); + assert_eq!(laid.len(), 40 * TsPacket::SIZE, "slot {} keeps its size", slot.index); + let first = &laid[..TsPacket::SIZE]; + match slot.clocked { + true => { + assert_eq!(pid(first), 1, "slot {}", slot.index); + assert_eq!(pcr_of(first), Some(slot.pcr), "slot {}", slot.index); + } + false => assert_eq!(first, &clock[..], "slot {} opens with a clock packet", slot.index), + } + for packet in laid.chunks(TsPacket::SIZE).skip(usize::from(!slot.clocked)) { + if [1, 2].contains(&pid(packet)) { + *sent.entry(pid(packet)).or_default() += payload(packet).len(); + clocked.push((pid(packet), slot.clocked)); + } + } + } + assert_eq!( + sent, + HashMap::from([(1, 4 * PAYLOAD), (2, 4 * PAYLOAD)]), + "no payload is lost" + ); + assert_eq!( + clocked.first(), + Some(&(1, true)), + "the clock PID's unit carries its slot's PCR" + ); + assert!( + clocked + .iter() + .filter(|&&(pid, _)| pid == 2) + .all(|&(_, clocked)| !clocked), + "another PID's slot has its own clock packet" + ); + } + + /// Each frame of a PES that packs several leaves B at its own decode time, the PES header + /// with the first, so the next PES goes in once the first frame is removed rather than the + /// last. + #[test] + fn packed_frames_leave_the_decoder_buffer_one_by_one() { + let mut schedule = Schedule::new(Duration::from_millis(200)); + schedule.set_rate(Some(RATE)); + // The packed PES's 1,514 bytes leave too little for the next one's 414 until its first + // frame's 514 leave. + schedule.set_buffer( + 1, + Buffer { + rate: 2_000_000, + leak: None, + size: Some(1_600), + transport: 512, + }, + ); + let frames = vec![(ms(1_000), 500), (ms(1_024), 500), (ms(1_048), 500)]; + schedule + .push_frames( + 1, + Timing { + by: ms(999), + decode: ms(1_000), + }, + audio(1_500), + false, + frames, + ) + .unwrap(); + schedule + .push( + 1, + Timing { + by: ms(1_099), + decode: ms(1_100), + }, + audio(400), + false, + ) + .unwrap(); + let mut null = unit(0, 1); + (null[1], null[2]) = (0x1f, 0xff); + let mut starts = Vec::new(); + while let Some(slot) = schedule.next(None).unwrap() { + for (position, packet) in slot.layout(&unit(9, 1), 100, &null).chunks(TsPacket::SIZE).enumerate() { + if pid(packet) == 1 && packet[1] & 0x40 != 0 { + starts.push(slot.pcr as f64 / SYSTEM_CLOCK as f64 + position as f64 * 1504.0 / RATE as f64); + } + } + } + assert_eq!(starts.len(), 2); + assert!( + starts[1] >= 1.000 && starts[1] < 1.024, + "the next PES starts at {}", + starts[1] + ); + } } diff --git a/test/ts/README.md b/test/ts/README.md index 084360421c0..0787694060f 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -18,6 +18,8 @@ software profiles below; it does not certify a hardware IRD or every TS layout. timing from a live pipe. - `table-anchor.py` compares table emission points between two subscribers. - `open-gop.py` checks the coded access units and leading pictures of open GOPs. +- `overhead.py` counts the packets a capture spends on clocks and audio framing + against its source. ## Running @@ -35,6 +37,7 @@ just test ts --open-gop # open-GOP clip; its leading pictures must survi just test ts --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB just test ts --headroom # the same with four audio PIDs, muxed above the video's Rx just test ts --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak +just test ts --full # the same with two audio PIDs, filling its multiplex with no nulls just test ts --delay 1s # pass the exporter's --delay ``` @@ -466,6 +469,22 @@ rebase. | `recovery-point-sei` | shape | each recovery point keeps its SEI | | `dts-before-pts` | shape | no access unit is stamped to decode after it presents | +## Mux overhead (`overhead.py`) + +A source that fills its multiplex with no null packets leaves the export no room +to spend more than it does on PES headers, part-filled audio packets, and PCRs +outside media packets (moq-dev/moq#5362). `--full` exports at 900 ms, above its +source's lead of about 0.65 s, which absorbs a small excess for a while, so it also +runs `overhead.py` on the source and capture before the compliance report. + +```bash +./overhead.py source.ts capture.ts +``` + +It counts the packets on each audio PID and the PCR-only packets on each PID, +scaling the capture's counts by how much of the source's elementary stream on the +PID it carries, and fails when their total exceeds the source's. + ## EIT fixtures No capture in this repository carries EIT (PID 0x0012), so nothing exercises the @@ -603,7 +622,7 @@ outcomes deterministically in Rust CI, without retries or implicit `--stitch`. [`.github/workflows/ts.yml`](../../.github/workflows/ts.yml) is independent of browser interop. Relevant mux, Hang, CLI, transport, harness and toolchain PRs run -`just test ts-controls`, `just test ts-pair-controls`, the default and 2 Mb/s round trips, HRD, leak, headroom, +`just test ts-controls`, `just test ts-pair-controls`, the default and 2 Mb/s round trips, HRD, leak, full, headroom, open-GOP, EIT, and T-STD controls. Each step runs even after another test fails. All recipes use the same pinned Nix FFmpeg and TSDuck locally and in CI. diff --git a/test/ts/overhead.py b/test/ts/overhead.py new file mode 100755 index 00000000000..e85f3fbd613 --- /dev/null +++ b/test/ts/overhead.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Grade whether a round-trip spends no more packets than its source on clocks and audio. + + ./overhead.py source.ts capture.ts + +A source that fills its multiplex with no null packets leaves the export no room to spend +more than it does. Each audio PES costs a header and a part-filled last packet, and a PCR +outside a media packet costs a packet of its own (moq-dev/moq#5362). + +Both are counted on each stream: the packets on every PES PID that is not video (stream ids +0xC0-0xDF, and private stream 1, which DVB AC-3 uses), and the packets on any PID that carry +a PCR and no payload. The capture joins after the source starts, so each PID's count is scaled +by how much of the source's elementary stream on that PID the capture carries. + +Exit status is 0 when the capture spends no more than the source, 1 otherwise. +""" + +import sys +from collections import defaultdict + +PKT = 188 +SYNC = 0x47 +NULL_PID = 0x1FFF + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + sys.exit(1) + + +def count(path: str) -> tuple[dict[int, int], dict[int, int], dict[int, int], set[int]]: + """Per PID: its packets, its elementary bytes, and its packets carrying only a PCR; and + the PIDs whose PES are audio.""" + packets: dict[int, int] = defaultdict(int) + elementary: dict[int, int] = defaultdict(int) + clocks: dict[int, int] = defaultdict(int) + audio: set[int] = set() + with open(path, "rb") as f: + data = f.read() + if not data or len(data) % PKT: + fail(f"{path} is not a whole number of {PKT}-byte packets") + for offset in range(0, len(data), PKT): + packet = data[offset : offset + PKT] + if packet[0] != SYNC: + fail(f"{path} loses sync at byte {offset}") + pid = (packet[1] & 0x1F) << 8 | packet[2] + if pid == NULL_PID: + continue + packets[pid] += 1 + control = packet[3] >> 4 & 3 + field = 1 + packet[4] if control & 2 else 0 + if control & 2 and packet[4] and packet[5] & 0x10 and not control & 1: + clocks[pid] += 1 + if not control & 1: + continue + body = packet[4 + field :] + if packet[1] & 0x40 and body[:3] == b"\0\0\1" and len(body) >= 9: + if 0xC0 <= body[3] <= 0xDF or body[3] == 0xBD: + audio.add(pid) + body = body[9 + body[8] :] + elementary[pid] += len(body) + return packets, elementary, clocks, audio + + +def main() -> int: + if len(sys.argv) != 3: + fail("usage: overhead.py source.ts capture.ts") + source, capture = count(sys.argv[1]), count(sys.argv[2]) + rows = [(pid, "audio", source[0][pid], capture[0][pid]) for pid in sorted(source[3] | capture[3])] + rows += [(pid, "PCR only", source[2][pid], capture[2][pid]) for pid in sorted(source[2].keys() | capture[2].keys())] + spent = [0.0, 0.0] + print("### packets spent on clocks and audio (source -> capture, scaled to the source)") + for pid, kind, before, after in rows: + if not source[1][pid] or not capture[1][pid]: + fail(f"PID {pid} carries no elementary stream in both the source and the capture") + share = capture[1][pid] / source[1][pid] + print(f" PID {pid:<5} {kind:<9} {before:>8} {after / share:>10.0f} ({after} for {share:.1%} of it)") + spent[0] += before + spent[1] += after / share + verdict = "PASS" if spent[1] <= spent[0] else "FAIL" + print(f" {verdict} {spent[1]:.0f} packets against the source's {spent[0]:.0f}") + return 0 if verdict == "PASS" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/ts/run.sh b/test/ts/run.sh index 215600f374d..e81dcf6c878 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -22,6 +22,7 @@ # ./run.sh --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB # ./run.sh --headroom # the same with four audio PIDs, muxed above the video's Rx # ./run.sh --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak +# ./run.sh --full # the same with two audio PIDs, filling its multiplex with no nulls # ./run.sh --delay 1s # pass the exporter's --delay # `--live` swaps the analyzer, not the rig. compliance.py grades a captured file @@ -75,6 +76,7 @@ OPEN_GOP="" # publish open GOP with leading pictures, and grade them through the HRD="" # publish video that fills a broadcast-sized CPB, as a contribution encoder does HEADROOM="" # the same plus four audio PIDs, in a multiplex faster than the video's Rx LEAK="" # publish video with no HRD at its level's leak, as a full-rate feed does +FULL="" # the same with audio, in a multiplex the source fills with no nulls EXPORT=() # extra `export ts` flags (--delay) # How far into the run the second subscriber joins. A late join is the point: two # exporters started together can share a cadence by starting together, which is @@ -151,6 +153,10 @@ while [[ $# -gt 0 ]]; do LEAK=1 shift ;; + --full) + FULL=1 + shift + ;; --delay) EXPORT=(--delay "$2") shift 2 @@ -200,6 +206,20 @@ fi # few percent below the leak exhausts the delay's lead and misses a deadline. [[ -n "$LEAK" && -z "$BITRATE_SET" && -z "${TSC_BITRATE:-}" ]] && BITRATE=26000000 [[ -n "$LEAK" && -z "$DURATION_SET" && -z "${TSC_DURATION:-}" ]] && DURATION=30 +if [[ -n "$FULL" && -n "$LEAK$HRD$HEADROOM$OPEN_GOP$SOURCE$ANALYZE_ONLY" ]]; then + echo "error: --full generates its own clip and cannot be combined with --leak, --hrd, --headroom, --open-gop, --source, or --analyze-only" >&2 + exit 1 +fi +# The clip's rate is whatever its packets fill, measured once it is generated. +if [[ -n "$FULL" && -n "$BITRATE_SET${TSC_BITRATE:-}" ]]; then + echo "error: --full runs at the rate its clip fills and takes no --bitrate" >&2 + exit 1 +fi +[[ -n "$FULL" && -z "$DURATION_SET" && -z "${TSC_DURATION:-}" ]] && DURATION=30 +# Its video arrives up to about 0.65 s ahead of decoding with no room to catch up, so the +# export needs a longer lead than the default delay gives, yet one a unit's window and +# slot keep under the T-STD's 1 s. +[[ -n "$FULL" && ${#EXPORT[@]} -eq 0 ]] && EXPORT=(--delay 900ms) if [[ -n "$OPEN_GOP" && -n "$ANALYZE_ONLY$LIVE$PAIR" ]]; then echo "error: --open-gop cannot be combined with --analyze-only, --live, or --pair" >&2 exit 1 @@ -329,6 +349,39 @@ elif [[ -n "$LEAK" ]]; then -c:v libx264 -profile:v high -level 4.0 -preset veryfast -pix_fmt yuv420p \ -x264-params "keyint=25:min-keyint=25:scenecut=0" -b:v 30M -maxrate 24M -bufsize 24M \ -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" +elif [[ -n "$FULL" ]]; then + echo "### generating ~${DURATION}s 1080p clip at its level's leak with two audio PIDs, filling its multiplex with no nulls" + # A full-rate contribution feed's shape: the --leak video beside an MP2 and a DVB AC-3 + # PID, with its PCR in the video's adaptation field and several audio frames a PES, and + # no null packets, so the export has no room to spend more packets than the source on + # clocks or audio framing (moq-dev/moq#5362). ffmpeg pads any multiplex rate it is + # given, so mux with room to spare, drop the nulls, and stamp the clock at the rate the + # rest fill. The PTS and DTS stay: moving them with each PCR would move B pictures + # against their group's keyframe, and the video goes out far enough ahead of its DTS + # for its packets to arrive a little earlier or later. Muxed at most 0.5 s ahead, it + # arrives up to about 0.65 s ahead once the nulls are gone. + PADDED_TS="$HARNESS_RUN/padded.ts" + PADDED_RATE=27000000 + ffmpeg -y -hide_banner -loglevel error \ + -f lavfi -i "testsrc2=size=1920x1080:rate=25,noise=alls=24:allf=t" \ + -f lavfi -i "sine=frequency=440:sample_rate=48000" \ + -f lavfi -i "sine=frequency=880:sample_rate=48000" \ + -t "$DURATION" -map 0:v -map 1:a -map 2:a -ac 2 \ + -c:v libx264 -profile:v high -level 4.0 -preset veryfast -pix_fmt yuv420p \ + -x264-params "keyint=25:min-keyint=25:scenecut=0" -b:v 30M -maxrate 24M -bufsize 24M \ + -c:a:0 mp2 -b:a:0 192k -c:a:1 ac3 -b:a:1 192k \ + -f mpegts -mpegts_flags system_b -muxrate "$PADDED_RATE" -pcr_period 20 -max_delay 500000 "$PADDED_TS" + PACKETS=$(($(wc -c <"$PADDED_TS") / 188)) + NULLS=$(tsp -I file "$PADDED_TS" -P count --pid 0x1FFF --total -O drop 2>&1 | + sed -n 's/.*counted \([0-9,]*\) packets.*/\1/p' | tr -d ,) + [[ -n "$NULLS" ]] || { + echo "error: could not count the generated clip's null packets" >&2 + exit 1 + } + BITRATE=$((PADDED_RATE * (PACKETS - NULLS) / PACKETS)) + tsp -I file "$PADDED_TS" -P filter --negate --pid 0x1FFF \ + -P pcradjust --bitrate "$BITRATE" --ignore-pts --ignore-dts -O file "$SRC_TS" + echo " ${NULLS} null packets dropped of ${PACKETS}: ${BITRATE} b/s" else echo "### generating ~${DURATION}s broadcast-like ${OPEN_GOP:+open-GOP }clip with ffmpeg" # CBR with a 20 ms PCR, like a contribution feed. Not cosmetic: `regulate` @@ -560,7 +613,7 @@ echo "### captured $(wc -c <"$SUB_TS" | tr -d ' ') bytes -> analyzing" # The initial decoded-content profile is the generated closed-GOP H.264/AAC # pair. Other arms retain their specific structural/model oracles until their # acquisition and decoder-priming contracts are covered by fixtures. -if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK" ]]; then +if [[ -z "$SOURCE$OPEN_GOP$HRD$HEADROOM$LEAK$FULL" ]]; then python3 "$DIR/decode.py" "$SRC_TS" "$SUB_TS" --output "$HARNESS_RUN/decode" else echo "decoded-content comparison: not covered for this profile" @@ -595,6 +648,18 @@ if [[ -n "$OPEN_GOP" ]]; then exit 1 fi fi + +# The delay's lead over the source's hides packets the export spends beyond the source's +# until it runs out, so they are counted outright. +if [[ -n "$FULL" ]]; then + echo + if ! python3 "$DIR/overhead.py" "$SRC_TS" "$SUB_TS"; then + echo >&2 + echo "error: the export spends more packets than its source on clocks and audio" >&2 + dump_logs + exit 1 + fi +fi echo # Pass the source so duration-fidelity can pin the exported stream's rate. A tiny # capture still parses, so the round-trip can fail here with a non-empty file; From 3b69dadda2755a0a9aad56b23312bfb5fe6f4de2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 14 Oct 2026 10:03:07 -0700 Subject: [PATCH 4/5] fix(ts): address review of the full-rate CBR export - A packet carrying the slot's PCR takes the clock packet's position without first spending a null, so a one-packet slot no longer underflows `nulls` in overflow-checked builds. - A frame bearing the tables starts its own audio PES, so a receiver tuning in at a PAT/PMT reads AAC's program config element before any older raw block. - The module header says the PCR rides the clock PID's packet when it can, rather than always a packet of its own. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 24 ++++--- rs/moq-mux/src/container/ts/export_test.rs | 82 +++++++++++++++++++--- rs/moq-mux/src/container/ts/schedule.rs | 33 ++++++++- 3 files changed, 117 insertions(+), 22 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index b10abfac3be..759aa83f313 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1,11 +1,13 @@ //! MPEG-TS muxer. //! //! [`Export`] subscribes to a MoQ broadcast and produces MPEG-TS: PAT/PMT program -//! tables and PES packets, packetized into 188-byte TS packets, with the PCR -//! riding its own adaptation-field-only packets on a fixed media-time grid. -//! Output is sliced on that grid rather than per media frame (the schedule): -//! each [`Frame`] is one slot's clock packet plus the bytes belonging to it, -//! stamped at the slot boundary, so the clock a receiver recovers from byte +//! tables and PES packets, packetized into 188-byte TS packets, with a PCR on a +//! fixed media-time grid. Output is sliced on that grid rather than per media +//! frame (the schedule): each [`Frame`] is one slot, opened by its PCR, plus the +//! bytes belonging to it. At a multiplex rate the PCR rides the adaptation field +//! of the clock PID's packet when one can open the slot; otherwise it takes an +//! adaptation-field-only packet of its own. Each frame is stamped at the slot +//! boundary, so the clock a receiver recovers from byte //! position agrees with the values, and a pacing caller releases each slot at //! the instant it asserts. Video is carried as Annex-B, audio as ADTS AAC. //! @@ -2030,8 +2032,9 @@ impl Export { /// out in. /// /// A PES starts at every frame whose place on the media timeline, counted in frames, is a - /// multiple of the frames the buffer holds, so exporters of the broadcast that joined at - /// different frames still cut the same PESs from the first such frame, like [`due`]. + /// multiple of the frames the buffer holds, and at every frame the tables ride on, so + /// exporters of the broadcast that joined at different frames still cut the same PESs from + /// the first such frame, like [`due`]. fn pack( &mut self, name: &str, @@ -2053,8 +2056,11 @@ impl Export { // Rounded, since a frame need not last whole microseconds: AAC's 1,024 samples at 48 kHz. let frame = 1_000_000 * u128::from(samples); let place = (unit.timestamp.as_micros() * u128::from(rate) + frame / 2) / frame; - // A PTS is a whole 90 kHz tick, so the frames run on within one of where they end. - let joins = !place.is_multiple_of(fit) + // A PTS is a whole 90 kHz tick, so the frames run on within one of where they end. A + // frame bearing the tables starts a PES, since a receiver tunes in there and AAC's + // program config element rides that frame only. + let joins = tables.is_empty() + && !place.is_multiple_of(fit) && track.pack.as_ref().is_some_and(|pack| { pack.rate == rate && pack.payload.len() + payload.len() <= size diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index a9b5e4fd0f8..31af813a2d8 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -310,8 +310,15 @@ async fn export_packs_aac_frames_up_to_their_buffer() { pts += 1_920; } } + // And one more from each frame the tables ride on, which starts a PES of its own. + let tables = ts + .as_chunks::<188>() + .0 + .iter() + .filter(|packet| packet[1] & 0x1f == 0 && packet[2] == 0) + .count(); assert!( - count <= frames.len().div_ceil(11) + 1, + count <= frames.len().div_ceil(11) + 1 + tables, "{count} PES for {} frames", frames.len() ); @@ -1317,16 +1324,18 @@ async fn export_drops_an_ac3_frame_cut_by_a_marker() { async fn export_packs_ac3_frames_up_to_their_buffer() { let stream: Vec = (0..100).flat_map(ac3_frame).collect(); let pes = exported_ac3_pes(vec![Ac3Write::Pes(0, stream.clone())]).await; - // 44 of the 128-byte frames to a PES, and the first frame alone, sent as it starts. + // 44 of the 128-byte frames to a PES, the first frame alone, sent as it starts, and a PES + // from each frame the tables ride on. + let tables = pes.iter().filter(|(_, _, tables)| *tables).count(); assert!( - pes.len() <= 100usize.div_ceil(44) + 1, + pes.len() <= 100usize.div_ceil(44) + 1 + tables, "{} PES for 100 frames", pes.len() ); - for (_, payload) in &pes { + for (_, payload, _) in &pes { assert!(14 + payload.len() <= 5_696, "a {}-byte PES overflows B", payload.len()); } - let bytes: Vec = pes.into_iter().flat_map(|(_, payload)| payload).collect(); + let bytes: Vec = pes.into_iter().flat_map(|(_, payload, _)| payload).collect(); assert_eq!(bytes, stream, "every frame whole and in order"); let timed = exported_ac3_frames(vec![Ac3Write::Pes(0, stream)]).await; assert!( @@ -1368,8 +1377,8 @@ enum Ac3Write { } /// Export `input` on an AC-3 passthrough PID beside bbb's media, and return the PTS and -/// payload of every PES on that PID. -async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec)> { +/// payload of every PES on that PID, and whether a PAT went out since the PES before it. +async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec, bool)> { const AC3_PID: u16 = 0x104; let data = include_bytes!("test_data/bbb.ts"); let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -1431,10 +1440,12 @@ async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec)> { ) .await; - // The PTS of every PES on the AC-3 PID, and its payload. - let mut pes: Vec<(u64, Vec)> = Vec::new(); + // The PTS of every PES on the AC-3 PID, its payload, and whether a PAT leads it. + let mut pes: Vec<(u64, Vec, bool)> = Vec::new(); + let mut tables = false; for packet in ts.as_chunks::<188>().0 { let pid = (u16::from(packet[1] & 0x1f) << 8) | u16::from(packet[2]); + tables |= pid == 0; if pid != AC3_PID || packet[3] & 0x10 == 0 { continue; } @@ -1451,7 +1462,11 @@ async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec)> { | u64::from(b[2] >> 1) << 15 | u64::from(b[3]) << 7 | u64::from(b[4] >> 1); - pes.push((pts, body[9 + usize::from(body[8])..].to_vec())); + pes.push(( + pts, + body[9 + usize::from(body[8])..].to_vec(), + std::mem::take(&mut tables), + )); } else if let Some(last) = pes.last_mut() { last.1.extend_from_slice(body); } @@ -1463,7 +1478,7 @@ async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec)> { /// timed by its PES's PTS and the frames ahead of it there. async fn exported_ac3_frames(input: Vec) -> Vec<(u64, Vec)> { let mut frames = Vec::new(); - for (mut pts, mut payload) in exported_ac3_pes(input).await { + for (mut pts, mut payload, _) in exported_ac3_pes(input).await { while !payload.is_empty() { let header = crate::codec::ac3::parse_header(&payload).expect("the PES carries whole sync frames"); let rest = payload.split_off(header.len); @@ -2118,6 +2133,51 @@ async fn aac_program_config_follows_each_table() { ); } +/// AAC frames that run on share a PES, but the frame the tables ride on starts one: a receiver +/// tuning in at a PAT/PMT reads the program config element first, not earlier raw blocks. +#[tokio::test(start_paused = true)] +async fn aac_program_config_leads_a_packed_pes() { + let mut quad = vec![0x11, 0x80, 0x04, 0xC4, 0x04, 0x00, 0x21, 0x10, 0x0C]; + quad.extend_from_slice(b"Lavc63.1.101"); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("a.aac", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 4); + cfg.container = Container::Legacy; + cfg.description = Some(Bytes::from(quad)); + catalog + .modify() + .unwrap() + .audio + .renditions + .insert("a.aac".to_string(), cfg); + let mut producer = Producer::new(track, HangContainer::Legacy(Kind::Audio)); + // 1,024 samples at 48 kHz: 21.33 ms each, so the 500 ms table cadence lands mid-PES. + for i in 0..96u64 { + producer + .write(Frame { + timestamp: Timestamp::from_micros(i * 1_024 * 1_000_000 / 48_000).unwrap(), + duration: None, + payload: Bytes::from_static(&[0x20; 10]), + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + } + producer.finish().unwrap(); + let ts = drain(consumer).await; + + let units = aac_program_configs(&ts); + assert!(units.len() < 96, "the frames share PESs: {}", units.len()); + for (i, (after_pat, pce)) in units.iter().enumerate() { + assert_eq!(after_pat, pce, "PES {i}: the element leads the PES after the tables"); + } +} + /// GStreamer 1.28 `fdkaacenc` output, 48 kHz stereo, remuxed to FLV by ffmpeg 9.0.1. Its /// AudioSpecificConfig signals SBR (and PS for v2) explicitly: object type 5 or 29 over an LC core /// at 24 kHz, stereo for v1 and mono for v2. ADTS has two bits for the object type, so export diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 42dae3e9fde..5d6092ec0bd 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -1028,13 +1028,14 @@ impl Schedule { } out.positions.as_mut().unwrap().push(position); out.packets.extend_from_slice(packet); - out.nulls -= 1; let unit = &mut self.units[i]; + // A packet carrying the PCR takes the clock packet's position, any other a null's. if let Some(rest) = opening { unit.packets.truncate(unit.sent * TsPacket::SIZE); unit.packets.extend_from_slice(&rest); out.clocked = true; - out.nulls += 1; + } else { + out.nulls -= 1; } if unit.sent == 0 { out.keyframe |= unit.keyframe; @@ -2316,6 +2317,34 @@ mod tests { ); } + /// At 90 kb/s a slot alternates between one and two packets. In a one-packet slot, the + /// clock PID's packet carrying the PCR takes the only position, the one its clock packet + /// would have. + #[test] + fn the_pcr_rides_the_only_packet_of_a_one_packet_slot() { + let mut schedule = Schedule::new(Duration::from_millis(500)); + schedule.set_rate(Some(90_000)); + schedule.set_buffer(1, VIDEO); + schedule.set_clock(1); + for at in 0..8 { + schedule.push(1, timing(1_000 + 125 * at), audio(100), at == 0).unwrap(); + } + let mut clock = unit(1, 1); + (clock[3], clock[4]) = (0x20, 183); + let mut null = unit(0, 1); + (null[1], null[2]) = (0x1f, 0xff); + let mut carried = Vec::new(); + while let Some(slot) = schedule.next(None).unwrap() { + let laid = slot.layout(&clock, 100, &null); + if slot.clocked { + assert_eq!(pcr_of(&laid), Some(slot.pcr), "slot {}", slot.index); + carried.push(laid.len() / TsPacket::SIZE); + } + } + assert_eq!(carried.len(), 8, "every PES carries its slot's PCR"); + assert!(carried.contains(&1), "some ride a one-packet slot: {carried:?}"); + } + /// Each frame of a PES that packs several leaves B at its own decode time, the PES header /// with the first, so the next PES goes in once the first frame is removed rather than the /// last. From 8223fb3b196129d47b4866e560fd636765bd356f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 14 Oct 2026 10:45:56 -0700 Subject: [PATCH 5/5] fix(ts): send configured AAC packs on before the tables A frame bearing the tables started its own PES, but the tables also mark every other AAC PID to repeat its program config element on its next frame, and that frame could still join an open PES of older raw blocks. When PSI fires, send on the open PES of every AAC PID that carries the element instead, so each PID's repeating frame starts a PES whichever PID's frame the tables ride. MP2, AC-3 and AAC with a channel configuration have no element to repeat, so their PESs are no longer cut at the tables. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 28 +++-- rs/moq-mux/src/container/ts/export_test.rs | 132 ++++++++++++--------- 2 files changed, 91 insertions(+), 69 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 759aa83f313..5bdbd75fc55 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1885,6 +1885,12 @@ impl Export { // Refresh PSI at keyframes or after the interval lapses. let psi = (is_video && frame.keyframe) || due(frame.timestamp, self.last_psi, PSI_INTERVAL); if psi { + // A receiver tunes in at the tables, so each AAC PID's next frame repeats its program + // config element. Its older frames go on ahead of the tables, so that frame starts a + // PES rather than following raw blocks that lack the element. + self.flush_packs( + |track, _| matches!(&track.kind, Kind::Aac { config, .. } if config.program_config.is_some()), + )?; for track in self.tracks.values_mut() { if let Kind::Aac { repeat, .. } = &mut track.kind { *repeat = true; @@ -2032,9 +2038,8 @@ impl Export { /// out in. /// /// A PES starts at every frame whose place on the media timeline, counted in frames, is a - /// multiple of the frames the buffer holds, and at every frame the tables ride on, so - /// exporters of the broadcast that joined at different frames still cut the same PESs from - /// the first such frame, like [`due`]. + /// multiple of the frames the buffer holds, so exporters of the broadcast that joined at + /// different frames still cut the same PESs from the first such frame, like [`due`]. fn pack( &mut self, name: &str, @@ -2056,11 +2061,8 @@ impl Export { // Rounded, since a frame need not last whole microseconds: AAC's 1,024 samples at 48 kHz. let frame = 1_000_000 * u128::from(samples); let place = (unit.timestamp.as_micros() * u128::from(rate) + frame / 2) / frame; - // A PTS is a whole 90 kHz tick, so the frames run on within one of where they end. A - // frame bearing the tables starts a PES, since a receiver tunes in there and AAC's - // program config element rides that frame only. - let joins = tables.is_empty() - && !place.is_multiple_of(fit) + // A PTS is a whole 90 kHz tick, so the frames run on within one of where they end. + let joins = !place.is_multiple_of(fit) && track.pack.as_ref().is_some_and(|pack| { pack.rate == rate && pack.payload.len() + payload.len() <= size @@ -2109,15 +2111,15 @@ impl Export { } /// Send on, in decode order, the packed frames of each track for which `done` holds, given - /// the slot its first frame is due in. - fn flush_packs(&mut self, done: impl Fn(u128) -> bool) -> anyhow::Result<()> { + /// the track and the slot its first frame is due in. + fn flush_packs(&mut self, done: impl Fn(&Track, u128) -> bool) -> anyhow::Result<()> { let mut due: Vec<(Timestamp, String)> = self .tracks .iter() .filter_map(|(name, track)| { let pack = track.pack.as_ref()?; let by = pack.decode.as_nanos().saturating_sub(track.drain().as_nanos()); - done(schedule::slot(by)).then(|| (pack.decode, name.clone())) + done(track, schedule::slot(by)).then(|| (pack.decode, name.clone())) }) .collect(); due.sort(); @@ -2253,7 +2255,7 @@ impl Export { }; let (pcr_pid, pmt_pid) = (psi.pcr_pid, psi.pmt_pid); if known.is_none() { - self.flush_packs(|_| true)?; + self.flush_packs(|_, _| true)?; } let lead = schedule::slot(self.budget().as_nanos()); loop { @@ -2262,7 +2264,7 @@ impl Export { if known.is_some() && let Some((index, _)) = self.schedule.upcoming() { - self.flush_packs(|due| due <= index + lead)?; + self.flush_packs(|_, due| due <= index + lead)?; } let Some(slot) = self.schedule.next(known)? else { break; diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 31af813a2d8..9d963448d8c 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -310,15 +310,8 @@ async fn export_packs_aac_frames_up_to_their_buffer() { pts += 1_920; } } - // And one more from each frame the tables ride on, which starts a PES of its own. - let tables = ts - .as_chunks::<188>() - .0 - .iter() - .filter(|packet| packet[1] & 0x1f == 0 && packet[2] == 0) - .count(); assert!( - count <= frames.len().div_ceil(11) + 1 + tables, + count <= frames.len().div_ceil(11) + 1, "{count} PES for {} frames", frames.len() ); @@ -1324,18 +1317,16 @@ async fn export_drops_an_ac3_frame_cut_by_a_marker() { async fn export_packs_ac3_frames_up_to_their_buffer() { let stream: Vec = (0..100).flat_map(ac3_frame).collect(); let pes = exported_ac3_pes(vec![Ac3Write::Pes(0, stream.clone())]).await; - // 44 of the 128-byte frames to a PES, the first frame alone, sent as it starts, and a PES - // from each frame the tables ride on. - let tables = pes.iter().filter(|(_, _, tables)| *tables).count(); + // 44 of the 128-byte frames to a PES, and the first frame alone, sent as it starts. assert!( - pes.len() <= 100usize.div_ceil(44) + 1 + tables, + pes.len() <= 100usize.div_ceil(44) + 1, "{} PES for 100 frames", pes.len() ); - for (_, payload, _) in &pes { + for (_, payload) in &pes { assert!(14 + payload.len() <= 5_696, "a {}-byte PES overflows B", payload.len()); } - let bytes: Vec = pes.into_iter().flat_map(|(_, payload, _)| payload).collect(); + let bytes: Vec = pes.into_iter().flat_map(|(_, payload)| payload).collect(); assert_eq!(bytes, stream, "every frame whole and in order"); let timed = exported_ac3_frames(vec![Ac3Write::Pes(0, stream)]).await; assert!( @@ -1377,8 +1368,8 @@ enum Ac3Write { } /// Export `input` on an AC-3 passthrough PID beside bbb's media, and return the PTS and -/// payload of every PES on that PID, and whether a PAT went out since the PES before it. -async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec, bool)> { +/// payload of every PES on that PID. +async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec)> { const AC3_PID: u16 = 0x104; let data = include_bytes!("test_data/bbb.ts"); let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -1440,12 +1431,10 @@ async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec, bool)> { ) .await; - // The PTS of every PES on the AC-3 PID, its payload, and whether a PAT leads it. - let mut pes: Vec<(u64, Vec, bool)> = Vec::new(); - let mut tables = false; + // The PTS of every PES on the AC-3 PID, and its payload. + let mut pes: Vec<(u64, Vec)> = Vec::new(); for packet in ts.as_chunks::<188>().0 { let pid = (u16::from(packet[1] & 0x1f) << 8) | u16::from(packet[2]); - tables |= pid == 0; if pid != AC3_PID || packet[3] & 0x10 == 0 { continue; } @@ -1462,11 +1451,7 @@ async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec, bool)> { | u64::from(b[2] >> 1) << 15 | u64::from(b[3]) << 7 | u64::from(b[4] >> 1); - pes.push(( - pts, - body[9 + usize::from(body[8])..].to_vec(), - std::mem::take(&mut tables), - )); + pes.push((pts, body[9 + usize::from(body[8])..].to_vec())); } else if let Some(last) = pes.last_mut() { last.1.extend_from_slice(body); } @@ -1478,7 +1463,7 @@ async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec, bool)> { /// timed by its PES's PTS and the frames ahead of it there. async fn exported_ac3_frames(input: Vec) -> Vec<(u64, Vec)> { let mut frames = Vec::new(); - for (mut pts, mut payload, _) in exported_ac3_pes(input).await { + for (mut pts, mut payload) in exported_ac3_pes(input).await { while !payload.is_empty() { let header = crate::codec::ac3::parse_header(&payload).expect("the PES carries whole sync frames"); let rest = payload.split_off(header.len); @@ -2133,8 +2118,8 @@ async fn aac_program_config_follows_each_table() { ); } -/// AAC frames that run on share a PES, but the frame the tables ride on starts one: a receiver -/// tuning in at a PAT/PMT reads the program config element first, not earlier raw blocks. +/// AAC frames that run on share a PES, but every PID's frame repeating the program config +/// element after the tables starts one, whichever PID's frame the tables rode on. #[tokio::test(start_paused = true)] async fn aac_program_config_leads_a_packed_pes() { let mut quad = vec![0x11, 0x80, 0x04, 0xC4, 0x04, 0x00, 0x21, 0x10, 0x0C]; @@ -2143,38 +2128,73 @@ async fn aac_program_config_leads_a_packed_pes() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let track = broadcast - .create_track("a.aac", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 4); - cfg.container = Container::Legacy; - cfg.description = Some(Bytes::from(quad)); - catalog - .modify() - .unwrap() - .audio - .renditions - .insert("a.aac".to_string(), cfg); - let mut producer = Producer::new(track, HangContainer::Legacy(Kind::Audio)); - // 1,024 samples at 48 kHz: 21.33 ms each, so the 500 ms table cadence lands mid-PES. - for i in 0..96u64 { - producer - .write(Frame { - timestamp: Timestamp::from_micros(i * 1_024 * 1_000_000 / 48_000).unwrap(), - duration: None, - payload: Bytes::from_static(&[0x20; 10]), - keyframe: true, - }) + // Offset, so the tables ride one PID's frame while the other's PES is open. + let mut producers = Vec::new(); + for (name, bytes, offset) in [("a.aac", 10, 0), ("b.aac", 40, 7_000)] { + let track = broadcast + .create_track(name, hang::container::track_info(hang::catalog::PRIORITY.audio)) .unwrap(); - producer.cut(None).unwrap(); + let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 4); + cfg.container = Container::Legacy; + cfg.description = Some(Bytes::from(quad.clone())); + catalog.modify().unwrap().audio.renditions.insert(name.to_string(), cfg); + producers.push((Producer::new(track, HangContainer::Legacy(Kind::Audio)), bytes, offset)); + } + // 1,024 samples at 48 kHz: 21.33 ms each, so the 500 ms table cadence lands mid-PES. + for (producer, bytes, offset) in &mut producers { + for i in 0..96u64 { + producer + .write(Frame { + timestamp: Timestamp::from_micros(*offset + i * 1_024 * 1_000_000 / 48_000).unwrap(), + duration: None, + payload: Bytes::from(vec![0x20; *bytes]), + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + } + producer.finish().unwrap(); } - producer.finish().unwrap(); let ts = drain(consumer).await; - let units = aac_program_configs(&ts); - assert!(units.len() < 96, "the frames share PESs: {}", units.len()); - for (i, (after_pat, pce)) in units.iter().enumerate() { - assert_eq!(after_pat, pce, "PES {i}: the element leads the PES after the tables"); + // Each PID's PESs, and each PES's frames: whether the element leads its raw data block. + let mut pes: std::collections::HashMap>> = Default::default(); + let mut reader = TsPacketReader::new(Cursor::new(ts.as_ref())); + while let Some(packet) = reader.read_ts_packet().unwrap() { + let pid = packet.header.pid.as_u16(); + match packet.payload { + Some(TsPayload::PesStart(start)) => pes.entry(pid).or_default().push(start.data.to_vec()), + Some(TsPayload::PesContinuation(rest)) => { + pes.get_mut(&pid).unwrap().last_mut().unwrap().extend_from_slice(&rest) + } + _ => {} + } + } + assert_eq!(pes.len(), 2); + for (pid, pes) in pes { + let frames: Vec> = pes + .iter() + .map(|data| { + let (mut data, mut frames) = (&data[..], Vec::new()); + while !data.is_empty() { + let header = super::adts::Header::parse(data).unwrap(); + // ID_PCE in the first three bits of the raw data block. + frames.push(data[header.header_len] >> 5 == 5); + data = &data[header.frame_len..]; + } + frames + }) + .collect(); + assert_eq!(frames.iter().map(Vec::len).sum::(), 96, "PID {pid}"); + assert!(frames.len() < 96, "PID {pid}: the frames share PESs"); + assert!( + frames.iter().filter(|frames| frames[0]).count() > 2, + "PID {pid}: the element repeats after the tables" + ); + assert!( + frames.iter().all(|frames| !frames[1..].contains(&true)), + "PID {pid}: a PES carries the element after older raw blocks: {frames:?}" + ); } }