From 18ea3a5189713ea3b5f929480ef2f7ef7085e030 Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 13 Oct 2026 11:59:10 +0100 Subject: [PATCH 1/4] fix(ts): admit leak-bound video at its full Rbx The schedule charged video whole 188-byte packets at the lesser of Rx and Rbx, plus the clock packet, so a source filling its level's leak, as a full-rate feed with no HRD does, fell behind and missed a decode deadline. The transport buffer now drains whole packets at Rx, the clock packet among them, while MB passes only elementary bytes on into EB at Rbx. Unpadded, a slot carries the lesser of the two, fractions carried from slot to slot, without the one-packet slack. `just test ts --leak` round-trips 1080p video with no HRD at level 4.0's 24 Mb/s leak. Before, its export stopped 12.7 s into the 30 s clip. Retires quest/m1/tstd/video-leak.md. Co-authored-by: Claude Opus 5.5 --- .github/workflows/interop.yml | 9 ++ quest/m1/tstd/README.md | 1 - quest/m1/tstd/mux-overhead.md | 4 - quest/m1/tstd/video-leak.md | 31 ---- rs/moq-mux/benches/ts_schedule.rs | 1 + rs/moq-mux/src/container/ts/export.rs | 31 ++-- rs/moq-mux/src/container/ts/schedule.rs | 181 ++++++++++++++++++++---- test/ts/README.md | 3 +- test/ts/run.sh | 29 +++- 9 files changed, 217 insertions(+), 73 deletions(-) delete mode 100644 quest/m1/tstd/video-leak.md diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index e74ed2fb484..8c1f0eb4255 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -183,6 +183,15 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # 1080p video with no HRD at its level's 24 Mb/s leak, so the export has to admit + # the leak's whole rate of payload to keep up (moq-dev/moq#5362). + - name: TS video at its full leak + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: nix develop --command just test ts --leak + shell: bash -leo pipefail {0} + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The same video beside four audio PIDs in a multiplex faster than its Rx: each slot # has to interleave the video with the rest, or its transport buffer overflows. - name: TS multiplex above the video's Rx diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index 155cfbd6f8f..8afd802a252 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -51,7 +51,6 @@ MB/AAC controls. Do not relabel the current subset a full standards check. - [Send-ahead within the delay](/quest/m1/tstd/send-ahead.md) - total lag is `--delay`, send-ahead included, with a 1 s default - [PSI runs](/quest/m1/tstd/psi-runs.md) - the PAT and PMT are deduplicated and paced so the system buffers never overflow -- [Video at its full leak](/quest/m1/tstd/video-leak.md) - a leak-bound video PID is admitted at its full Rbx, not about 3 % below it - [Mux overhead](/quest/m1/tstd/mux-overhead.md) - the PCR rides in the video's adaptation field and audio PES carry several frames, so a full CBR source re-muxes at its own rate ## Related diff --git a/quest/m1/tstd/mux-overhead.md b/quest/m1/tstd/mux-overhead.md index f1b604c7197..d49c1e53fe4 100644 --- a/quest/m1/tstd/mux-overhead.md +++ b/quest/m1/tstd/mux-overhead.md @@ -35,7 +35,3 @@ Decided 2026-10-12: ## Closes - [#5362](https://github.com/moq-dev/moq/issues/5362) - a full-rate CBR contribution feed exports at its own rate - -## Related - -- [TS export admits video at its full leak](/quest/m1/tstd/video-leak.md) - #5362 point 2 diff --git a/quest/m1/tstd/video-leak.md b/quest/m1/tstd/video-leak.md deleted file mode 100644 index 9e52be30df7..00000000000 --- a/quest/m1/tstd/video-leak.md +++ /dev/null @@ -1,31 +0,0 @@ -# [S] TS export admits video at its full leak - -## Goal - -`moq export ts` admits a leak-bound video PID at its full Rbx rate, so a -source that fills its level's leak (no HRD, 1080i25 High@4.0 at Rbx 24 Mb/s) -exports without falling behind and missing a decode deadline. - -## Plan - -Reported in [#5362](https://github.com/moq-dev/moq/issues/5362) point 2 and -verified against `rs/moq-mux/src/container/ts/`: `Buffer::per_slot` -(`schedule.rs`) floors the rate to whole 188-byte packets per 25 ms slot and -subtracts one packet of slack, and `admit` charges each slot's clock packet to -the PCR PID, which is the video. Together that admits about 2.9 % below what -Rbx carries in payload bytes. The slack's only rationale is a comment about -bunching at slot edges; no commit or quest calls it a deliberate margin. - -Decided 2026-10-12: - -- **Exact accounting.** Cap a leak-bound PID at the lesser of Rx in packets - and Rbx in payload bytes, since Rbx governs payload into EB while TB drains - at Rx. Carry fractional packets from slot to slot and drop the one-packet - slack. A standalone PCR packet still enters the video's transport buffer, - so it counts against the Rx packet budget but not the Rbx payload budget. -- Test: a generated source whose video fills Rbx on average exports for the - whole clip and passes the strict T-STD check that `just test ts --hrd` runs. - -## Related - -- [Lower TS mux overhead](/quest/m1/tstd/mux-overhead.md) - #5362 point 3, which closes the issue diff --git a/rs/moq-mux/benches/ts_schedule.rs b/rs/moq-mux/benches/ts_schedule.rs index 10dc7fc4cb5..cb10332f2bf 100644 --- a/rs/moq-mux/benches/ts_schedule.rs +++ b/rs/moq-mux/benches/ts_schedule.rs @@ -35,6 +35,7 @@ fn admission(c: &mut Criterion) { pid, schedule::Buffer { rate: 2_000_000, + leak: None, size: Some(1_000_000), }, ); diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 2f7ff93cb4b..ad36574ed28 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -366,8 +366,7 @@ impl Track { /// a unit whose DTS sits on a PCR slot boundary, as a source's own 25 fps timestamps put /// every fifth frame, arrives complete only after it decodes. fn drain(&self) -> Duration { - let rate = self.buffer().map_or(Buffer::SYSTEM.rate, |buffer| buffer.rate); - Duration::from_nanos(TsPacket::SIZE as u64 * 8 * 1_000_000_000 / rate.max(1)) + self.buffer().unwrap_or(Buffer::SYSTEM).drain() } /// Queue every held video frame whose DTS is settled, or all of them when `flush`. @@ -468,16 +467,17 @@ impl Timing { } /// The receiver's buffers (H.222.0 2.14.3.1, 2.17.2): EB is the HRD's CPB, else the - /// level's, and packets go no faster than the slower of the transport buffer's Rx and the - /// leak from MB into EB, so neither MB nor TB fills. + /// level's; the transport buffer drains at Rx, scaled from the HRD's BitRate or else the + /// level's; and MB passes payload on into EB at the level's Rbx whatever the HRD declares. fn buffer(&self) -> Option { let level = self.level?; - let (rate, cpb) = match self.hrd { - Some(hrd) => (hrd.bit_rate.saturating_mul(level.factor) / 1_000, hrd.cpb_size), - None => (level.leak, level.cpb), + let (bit_rate, cpb) = match self.hrd { + Some(hrd) => (hrd.bit_rate, hrd.cpb_size), + None => (level.bit_rate, level.cpb), }; Some(Buffer { - rate: rate.min(level.leak), + rate: bit_rate.saturating_mul(level.factor) / 1_000, + leak: Some(level.leak), size: Some(usize::try_from(cpb / 8).unwrap_or(usize::MAX)), }) } @@ -2420,6 +2420,9 @@ fn at_edge(source: ExportSource, replay: bool) -> ExportSource { struct Level { /// The leak from MB into EB, Rbx: CpbBrNalFactor * MaxBR. leak: u64, + /// BitRate without an HRD, which `factor` scales to Rx: 1200 * MaxBR for H.264, the + /// least cpbBrNalFactor of any profile, and CpbBrVclFactor * MaxBR for H.265. + bit_rate: u64, /// EB without an HRD: CpbBrNalFactor * MaxCPB. cpb: u64, /// Rx over an HRD's BitRate, in thousandths: 1.2 for H.264, CpbBrNalFactor / @@ -2456,6 +2459,7 @@ fn video_level(config: &VideoConfig, name: &str) -> Level { }; limits.map(|(max_br, max_cpb)| Level { leak: 1_200 * max_br, + bit_rate: 1_200 * max_br, cpb: 1_200 * max_cpb, factor: 1_200, }) @@ -2485,6 +2489,7 @@ fn video_level(config: &VideoConfig, name: &str) -> Level { }; limits.map(|(max_br, max_cpb)| Level { leak: 1_100 * max_br, + bit_rate: 1_000 * max_br, cpb: 1_100 * max_cpb, factor: 1_100, }) @@ -2495,6 +2500,7 @@ fn video_level(config: &VideoConfig, name: &str) -> Level { tracing::warn!(track = %name, codec = %config.codec, "no T-STD limits for this level; sending its units early"); Level { leak: 1_200 * 64, + bit_rate: 1_200 * 64, cpb: 1_200 * 175, factor: 1_200, } @@ -2518,7 +2524,11 @@ fn audio_buffer(config: &AudioConfig, kind: &Kind) -> Buffer { Kind::Eac3 => (2_000_000, 12_896), _ => (2_000_000, 3_584), }; - Buffer { rate, size: Some(size) } + Buffer { + rate, + leak: None, + size: Some(size), + } } /// The DVB AC-3 descriptor's tag (ETSI EN 300 468 6.2.1), marking AC-3 as private data. @@ -2542,10 +2552,12 @@ fn verbatim_buffer(kind: &Kind, descriptors: &[catalog::Descriptor]) -> Option Some(Buffer { rate: 2_000_000, + leak: None, size: Some(5_696), }), TELETEXT_DESCRIPTOR => Some(Buffer { rate: 6_750_000, + leak: None, size: None, }), _ => None, @@ -3112,6 +3124,7 @@ mod tests { verbatim_buffer(&kind, &[descriptor(0x56)]), Some(Buffer { rate: 6_750_000, + leak: None, size: None, }) ); diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index ef3c8dedbd6..142d8153221 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -63,9 +63,11 @@ fn rate_pcr(index: u128, rate: u64) -> u128 { /// A PID's buffers in a receiver, as the schedule respects them (ISO 13818-1 2.4.2). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct Buffer { - /// How fast the receiver takes the PID's packets on, in bits per second: its transport - /// buffer's Rx, or for video the slower leak from MB into EB. + /// How fast the receiver's transport buffer drains whole packets, Rx, in bits per second. pub rate: u64, + /// For video, how fast MB passes payload on into EB, Rbx, in bits per second (ISO + /// 13818-1 2.14.3.1). `None` where the transport buffer feeds the decoder buffer directly. + pub leak: Option, /// The decoder buffer (EB for video, B for audio) in bytes, which a unit fills from its /// first packet until it decodes. `None` bounds nothing, so the PID's units go out only /// in the slot before their own. @@ -76,15 +78,33 @@ impl Buffer { /// The program tables' (ISO 13818-1 2.4.2.3, Rxsys). pub const SYSTEM: Self = Self { rate: 1_000_000, + leak: None, size: None, }; - /// The packets one slot may carry on the PID: what its transport buffer passes on within - /// the slot, less one, but at least one. Even laid out evenly, the packets bunch a little - /// where one slot meets the next, and the slack drains that off. - fn per_slot(&self) -> usize { - let bits = u128::from(self.rate) * PCR_INTERVAL.as_nanos() / 1_000_000_000; - (bits / (TsPacket::SIZE as u128 * 8)).saturating_sub(1).max(1) as usize + /// How long the receiver takes to pass one packet on into the decoder buffer: through the + /// transport buffer at Rx and, for video, its payload through MB at Rbx, which overlap. + pub fn drain(&self) -> Duration { + let through = |bytes: usize, rate: u64| Duration::from_nanos(bytes as u64 * 8_000_000_000 / rate.max(1)); + let transport = through(TsPacket::SIZE, self.rate); + self.leak + .map_or(transport, |leak| transport.max(through(PAYLOAD, leak))) + } + + /// The media packets slot `index` may carry on the PID unpadded: what its transport buffer + /// passes on at Rx, less the slot's `clock` packet when it rides the PID, and for video no + /// more payload than MB passes on at Rbx. Both count from the start of the timeline, so + /// no slot rounds on its own. At least one, or a PID too slow for both the clock packet and + /// one of its own would never finish a unit. + fn per_slot(&self, index: u128, clock: bool) -> usize { + let through = |rate: u64, bytes: usize| { + let before = + |index: u128| index * u128::from(rate) * PCR_INTERVAL.as_nanos() / (bytes as u128 * 8_000_000_000); + before(index + 1) - before(index) + }; + let packets = through(self.rate, TsPacket::SIZE).saturating_sub(u128::from(clock)); + let packets = self.leak.map_or(packets, |leak| packets.min(through(leak, PAYLOAD))); + packets.max(1) as usize } } @@ -415,6 +435,8 @@ pub(super) struct Schedule { decoding: VecDeque<(u16, u128, usize)>, /// When each transport buffer's last admitted packet finishes leaving it. transport: HashMap, + /// When each video PID's MB finishes passing its last admitted payload on into EB. + mb: HashMap, /// Every source has ended, so a unit that misses its deadline is the tail going out late /// rather than an output that cannot keep up. ended: bool, @@ -434,6 +456,7 @@ impl Schedule { pmt: None, decoding: VecDeque::new(), transport: HashMap::new(), + mb: HashMap::new(), ended: false, } } @@ -533,6 +556,7 @@ impl Schedule { self.units.clear(); self.decoding.clear(); self.transport.clear(); + self.mb.clear(); self.next = None; self.horizon = None; self.last = None; @@ -634,7 +658,7 @@ impl Schedule { caps: self .buffers .iter() - .map(|(&pid, buffer)| (pid, buffer.per_slot())) + .map(|(&pid, buffer)| (pid, buffer.per_slot(index, self.clock == Some(pid)))) .collect(), positions: None, })) @@ -735,9 +759,26 @@ impl Schedule { continue; } } + // Video's payload goes on from MB into EB at Rbx as it leaves the transport + // buffer, its PES header discarded, so only payload counts against the leak. + // Holding MB to a transport buffer's worth, below any MB size, still carries the + // leak's whole rate across packet positions. + let leak = buffer.and_then(|buffer| buffer.leak).map(|leak| { + let from = self.mb.get(&pid).copied().unwrap_or(free).max(free); + let bits = elementary(packet).len() as u128 * 8 * 1_000_000_000; + let leaked = from + bits.div_ceil(u128::from(leak)) as i128; + (leaked, (512 * 8 * 1_000_000_000 / u128::from(leak)) as i128) + }); + if let Some((leaked, capacity)) = leak + && leaked - now > capacity + { + waiting.push(Reverse((leaked - capacity, lane))); + continue; + } // No byte leaves the transport buffer before it has arrived: below Rx, the // multiplex rate decides when the packet's last byte gets there. - let done = drained.max(time(position + 1)); + let arrived = drained.max(time(position + 1)); + let done = leak.map_or(arrived, |(leaked, _)| arrived.max(leaked)); let held = if pid == unit.pid && buffer.is_some_and(|buffer| buffer.size.is_some()) { elementary(packet).len() } else { @@ -757,7 +798,10 @@ impl Schedule { "MPEG-TS output missed a decode deadline on PID {pid} at {rate} b/s; raise the delay or the multiplex rate" ); } - self.transport.insert(tb, done); + self.transport.insert(tb, arrived); + if let Some((leaked, _)) = leak { + self.mb.insert(pid, leaked); + } out.positions.as_mut().unwrap().push(position); out.packets.extend_from_slice(packet); out.nulls -= 1; @@ -836,14 +880,7 @@ impl Schedule { let mut order: Vec = (0..self.units.len()).collect(); order.sort_by_key(|&i| self.units[i].due); let mut take = vec![0; self.units.len()]; - // The clock packet counts against its PID unless that would leave the PID's media no - // packet at all, which would never finish a unit however sparse its frames. - let mut carried: HashMap = self - .clock - .filter(|pid| self.buffers.get(pid).is_some_and(|buffer| buffer.per_slot() > 1)) - .map(|pid| (pid, 1)) - .into_iter() - .collect(); + let mut carried: HashMap = HashMap::new(); let mut stopped: Vec = Vec::new(); for i in order { let unit = &self.units[i]; @@ -858,7 +895,10 @@ impl Schedule { if self.release(unit) <= index { while n < unit.remaining() && budget > 0 { let pid = unit.pid_at(unit.sent + n); - let cap = self.buffers.get(&pid).map_or(usize::MAX, Buffer::per_slot); + let cap = self + .buffers + .get(&pid) + .map_or(usize::MAX, |buffer| buffer.per_slot(index, self.clock == Some(pid))); let count = carried.entry(pid).or_default(); if *count >= cap { break; @@ -922,6 +962,7 @@ mod tests { /// A video-like buffer: far more than a slot passes on, and a large decoder buffer. const VIDEO: Buffer = Buffer { rate: 100_000_000, + leak: None, size: Some(1_000_000), }; @@ -1075,6 +1116,7 @@ mod tests { // Room for ten packets. let small = Buffer { rate: 100_000_000, + leak: None, size: Some(10 * PAYLOAD), }; schedule.set_buffer(1, small); @@ -1121,9 +1163,10 @@ mod tests { fn a_pid_takes_what_its_transport_buffer_passes_on() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - // 2 Mb/s: 33 packets a slot, less one. + // 2 Mb/s: 33.2 packets a slot. let audio = Buffer { rate: 2_000_000, + leak: None, size: Some(100_000), }; schedule.set_buffer(1, audio); @@ -1139,9 +1182,10 @@ mod tests { fn the_clock_packet_counts_against_its_pid() { let mut schedule = Schedule::new(Duration::from_millis(100)); schedule.set_rate(Some(RATE)); - // 2 Mb/s: 33 packets a slot, less one, the clock packet among them. + // 2 Mb/s: 33.2 packets a slot, the clock packet among them. let audio = Buffer { rate: 2_000_000, + leak: None, size: Some(100_000), }; schedule.set_buffer(1, audio); @@ -1152,6 +1196,76 @@ mod tests { assert_eq!(checked_transport(&mut schedule, 1), 80); } + /// Video whose payload fills its level's leak keeps up: its transport buffer drains whole + /// packets at Rx, the clock packet among them, while MB passes only payload on at Rbx + /// (moq-dev/moq#5362). Charging whole packets, or the clock packet, to Rbx falls behind. + #[test] + fn video_keeps_up_with_its_full_leak() { + // Level 4.0 without an HRD: Rx 1.2 x 24 Mb/s, Rbx 24 Mb/s, BSmux + BSoh 16,000 bytes. + const RX: f64 = 28_800_000.0; + const RBX: f64 = 24_000_000.0; + let video = Buffer { + rate: RX as u64, + leak: Some(RBX as u64), + size: Some(3_750_000), + }; + let rate = 26_000_000; + let mut schedule = Schedule::new(Duration::from_millis(200)); + schedule.set_rate(Some(rate)); + schedule.set_buffer(1, video); + schedule.set_clock(1); + // 20 s of 40 ms frames, each 652 packets of payload: 23.99 Mb/s. + for frame in 0..500 { + schedule + .push(1, timing(1_000 + 40 * frame), unit(1, 652), frame % 25 == 0) + .unwrap(); + } + let mut null = unit(0, 1); + (null[1], null[2]) = (0x1f, 0xff); + let (mut tb, mut mb, mut previous, mut sent) = (0.0f64, 0.0f64, None::, 0); + while let Some(slot) = schedule.next(None).unwrap() { + let mut clock = unit(1, 1); + clock[3] = 0x20; + for (position, packet) in slot.layout(&clock, 100, &null).chunks(188).enumerate() { + let at = slot.pcr as f64 / SYSTEM_CLOCK as f64 + position as f64 * 1504.0 / rate as f64; + if let Some(previous) = previous { + tb = (tb - (at - previous) * RX / 8.0).max(0.0); + mb = (mb - (at - previous) * RBX / 8.0).max(0.0); + } + previous = Some(at); + if pid(packet) == 1 { + tb += 188.0; + mb += elementary(packet).len() as f64; + sent += usize::from(packet[3] & 0x10 != 0); + } + assert!(tb <= 512.001, "TB holds {tb} bytes at {at}"); + assert!(mb <= 16_000.0, "MB holds {mb} bytes at {at}"); + } + } + assert_eq!(sent, 500 * 652, "every frame goes out"); + } + + /// Unpadded, a video PID's slot carries the lesser of Rx in whole packets and Rbx in + /// payload, fractions carried from slot to slot, and the clock packet counts against Rx + /// alone. + #[test] + fn unpadded_video_takes_the_lesser_of_rx_and_rbx() { + let second = |buffer: Buffer| (0..40).map(|index| buffer.per_slot(index, true)).sum::(); + // Rbx binds: 24 Mb/s of 184-byte payloads, 16,304.3 a second. + let leak = Buffer { + rate: 28_800_000, + leak: Some(24_000_000), + size: None, + }; + assert_eq!(second(leak), 16_304); + // Rx binds: 20 Mb/s of 188-byte packets, 13,297.9 a second, less 40 clock packets. + let transport = Buffer { + rate: 20_000_000, + ..leak + }; + assert_eq!(second(transport), 13_297 - 40); + } + /// A PID whose transport buffer passes on a single packet a slot still carries one of its /// own beside the clock packet, so its units finish on time and the stream ends. #[test] @@ -1161,6 +1275,7 @@ mod tests { // 76.8 kb/s: one packet a slot. let slow = Buffer { rate: 76_800, + leak: None, size: Some(100_000), }; schedule.set_buffer(1, slow); @@ -1180,6 +1295,7 @@ mod tests { 1, Buffer { rate: 76_800, + leak: None, size: Some(100_000), }, ); @@ -1200,6 +1316,7 @@ mod tests { // Room for ten packets. let small = Buffer { rate: 100_000_000, + leak: None, size: Some(10 * PAYLOAD), }; schedule.set_buffer(1, small); @@ -1239,6 +1356,7 @@ mod tests { 1, Buffer { rate: 2_000_000, + leak: None, size: Some(2_592), }, ); @@ -1263,6 +1381,7 @@ mod tests { 1, Buffer { rate: 2_000_000, + leak: None, size: Some(2 * PAYLOAD), }, ); @@ -1309,6 +1428,7 @@ mod tests { 1, Buffer { rate: 2_000_000, + leak: None, size: Some(2_592), }, ); @@ -1436,11 +1556,12 @@ mod tests { #[test] fn unpadded_units_go_out_by_their_decode_time() { let mut schedule = Schedule::new(Duration::ZERO); - // 2 Mb/s passes 32 packets a slot. + // 2 Mb/s passes 33 packets a slot. schedule.set_buffer( 1, Buffer { rate: 2_000_000, + leak: None, size: Some(100_000), }, ); @@ -1570,7 +1691,15 @@ mod tests { packets, keyframe: true, units: vec![video.into()], - caps: HashMap::from([(video.into(), Buffer { rate: RX, size: None }.per_slot())]), + caps: HashMap::from([( + video.into(), + Buffer { + rate: RX, + leak: None, + size: None, + } + .per_slot(0, true), + )]), positions: None, }; let mut null = unit(0, 1); @@ -1608,10 +1737,12 @@ mod tests { let (a, b) = ( Buffer { rate: 7_833_600, + leak: None, size: None, }, Buffer { rate: 8_160_000, + leak: None, size: None, }, ); @@ -1622,7 +1753,7 @@ mod tests { nulls: 88, keyframe: true, units: vec![1, 2], - caps: HashMap::from([(1, a.per_slot()), (2, b.per_slot())]), + caps: HashMap::from([(1, a.per_slot(0, true)), (2, b.per_slot(0, false))]), positions: None, }; let mut null = unit(0, 1); diff --git a/test/ts/README.md b/test/ts/README.md index 583ac1a7223..07e909fd0bd 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -35,6 +35,7 @@ just test ts --pair # two exporters of one broadcast, grade table an just test ts --open-gop # open-GOP clip; its leading pictures must survive just test ts --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB just test ts --headroom # the same with four audio PIDs, muxed above the video's Rx +just test ts --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak just test ts --delay 1s # pass the exporter's --delay ``` @@ -563,7 +564,7 @@ exporter re-emits SI on its own repetition cadence rather than the source's. ## CI `.github/workflows/interop.yml` runs `just test ts`, `just test ts --bitrate -2000000`, `just test ts --hrd`, `just test ts --headroom`, +2000000`, `just test ts --hrd`, `just test ts --leak`, `just test ts --headroom`, `just test ts --open-gop`, `just test ts-eit`, and `just test ts-tstd` after the interop matrix (nightly, on demand, and on PRs touching `test/ts/`). `ts-eit` is `eit-roundtrip.sh`: it builds the sparse-schedule and diff --git a/test/ts/run.sh b/test/ts/run.sh index 5ac319a7c03..d8b8dca52a7 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -21,6 +21,7 @@ # ./run.sh --open-gop # open-GOP clip; its leading pictures must survive # ./run.sh --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB # ./run.sh --headroom # the same with four audio PIDs, muxed above the video's Rx +# ./run.sh --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak # ./run.sh --delay 1s # pass the exporter's --delay # `--live` swaps the analyzer, not the rig. compliance.py grades a captured file @@ -67,6 +68,7 @@ PAIR="" # subscribe twice and grade the two captures against each other OPEN_GOP="" # publish open GOP with leading pictures, and grade them through the round-trip HRD="" # publish video that fills a broadcast-sized CPB, as a contribution encoder does HEADROOM="" # the same plus four audio PIDs, in a multiplex faster than the video's Rx +LEAK="" # publish video with no HRD at its level's leak, as a full-rate feed does EXPORT=() # extra `export ts` flags (--delay) # How far into the run the second subscriber joins. A late join is the point: two # exporters started together can share a cadence by starting together, which is @@ -75,8 +77,8 @@ PAIR_JOIN="${TSC_PAIR_JOIN:-5}" # Shortest overlap worth a verdict, in seconds. Below this the slower tables fall under # the analyzer's emission floor and go report-only, which reads as a pass. PAIR_MIN_OVERLAP="${TSC_PAIR_MIN_OVERLAP:-25}" -DURATION_SET="" # so pair mode can raise the default without overriding an explicit --duration -BITRATE_SET="" # so headroom mode can raise the default without overriding an explicit --bitrate +DURATION_SET="" # so pair and leak modes can raise the default without overriding an explicit --duration +BITRATE_SET="" # so headroom and leak modes can raise the default without overriding an explicit --bitrate PASSTHRU=() # forwarded to compliance.py (thresholds, --report-json, ...) while [[ $# -gt 0 ]]; do @@ -139,6 +141,10 @@ while [[ $# -gt 0 ]]; do HEADROOM=1 shift ;; + --leak) + LEAK=1 + shift + ;; --delay) EXPORT=(--delay "$2") shift 2 @@ -180,6 +186,14 @@ if [[ -n "$HEADROOM" && -n "$HRD$OPEN_GOP$SOURCE$ANALYZE_ONLY" ]]; then fi # Room for the audio, with the video still taking up to 86 % of the multiplex. [[ -n "$HEADROOM" && -z "$BITRATE_SET" && -z "${TSC_BITRATE:-}" ]] && BITRATE=12500000 +if [[ -n "$LEAK" && -n "$HRD$HEADROOM$OPEN_GOP$SOURCE$ANALYZE_ONLY" ]]; then + echo "error: --leak generates its own clip and cannot be combined with --hrd, --headroom, --open-gop, --source, or --analyze-only" >&2 + exit 1 +fi +# Room for the video's packet headers, and long enough that an export admitting video a +# few percent below the leak exhausts the delay's lead and misses a deadline. +[[ -n "$LEAK" && -z "$BITRATE_SET" && -z "${TSC_BITRATE:-}" ]] && BITRATE=26000000 +[[ -n "$LEAK" && -z "$DURATION_SET" && -z "${TSC_DURATION:-}" ]] && DURATION=30 if [[ -n "$OPEN_GOP" && -n "$ANALYZE_ONLY$LIVE$PAIR" ]]; then echo "error: --open-gop cannot be combined with --analyze-only, --live, or --pair" >&2 exit 1 @@ -294,6 +308,17 @@ elif [[ -n "$HRD$HEADROOM" ]]; then -c:v libx264 -profile:v high -level 4.0 -preset veryfast -pix_fmt yuv420p \ -x264-params "keyint=25:min-keyint=25:scenecut=0:nal-hrd=cbr" -b:v 9M -maxrate 9M -bufsize 9M \ -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" +elif [[ -n "$LEAK" ]]; then + echo "### generating ~${DURATION}s 1080p clip with no HRD at its level's 24 Mb/s leak with ffmpeg" + # A full-rate feed's shape: with no HRD, level 4.0's leak from MB into EB (Rbx, 24 Mb/s) + # is the video's only limit, and a target above the VBV's 24 Mb/s ceiling holds the + # video at it, so the export has to admit the leak's whole rate (moq-dev/moq#5362). + ffmpeg -y -hide_banner -loglevel error \ + -f lavfi -i "testsrc2=size=1920x1080:rate=25,noise=alls=24:allf=t" \ + -t "$DURATION" -an \ + -c:v libx264 -profile:v high -level 4.0 -preset veryfast -pix_fmt yuv420p \ + -x264-params "keyint=25:min-keyint=25:scenecut=0" -b:v 30M -maxrate 24M -bufsize 24M \ + -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" else echo "### generating ~${DURATION}s broadcast-like ${OPEN_GOP:+open-GOP }clip with ffmpeg" # CBR with a 20 ms PCR, like a contribution feed. Not cosmetic: `regulate` From 02a47615b01420e87c76b7d19825c881e6c0dc77 Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 13 Oct 2026 12:23:50 +0100 Subject: [PATCH 2/4] fix(ts): let video's MB run BSmux ahead of its leak Held to a transport buffer's worth ahead of the leak, MB emptied whenever audio due sooner took a position or two between video packets, and the leak idled. On a feed whose video fills Rbx, the export carried about 1 % less video than the source and stopped 105 s in. H.222.0 2.14.3.1 gives MB BSmux for this, 4 ms of the larger of Rbx and 2 Mb/s, and BSoh for PES headers. Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/schedule.rs | 54 +++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 142d8153221..d302443647d 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -760,14 +760,16 @@ impl Schedule { } } // Video's payload goes on from MB into EB at Rbx as it leaves the transport - // buffer, its PES header discarded, so only payload counts against the leak. - // Holding MB to a transport buffer's worth, below any MB size, still carries the - // leak's whole rate across packet positions. + // buffer, its PES header discarded, so only payload counts against the leak. MB + // runs up to BSmux ahead of the leak, 4 ms of the larger of Rbx and 2 Mb/s: the + // room 2.14.3.1 gives it for the multiplex's other PIDs, so the leak never idles + // while they take a few positions, and BSoh is left for the PES headers. let leak = buffer.and_then(|buffer| buffer.leak).map(|leak| { let from = self.mb.get(&pid).copied().unwrap_or(free).max(free); let bits = elementary(packet).len() as u128 * 8 * 1_000_000_000; let leaked = from + bits.div_ceil(u128::from(leak)) as i128; - (leaked, (512 * 8 * 1_000_000_000 / u128::from(leak)) as i128) + let room = 4_000_000 * u128::from(leak.max(2_000_000)) / u128::from(leak); + (leaked, room as i128) }); if let Some((leaked, capacity)) = leak && leaked - now > capacity @@ -1245,6 +1247,50 @@ mod tests { assert_eq!(sent, 500 * 652, "every frame goes out"); } + /// Audio due before the video takes a position or a few between video packets, and MB + /// holds enough payload ahead of the leak that the leak runs on through them: video with + /// frames queued passes on its whole Rbx. With only a transport buffer's worth ahead, the + /// leak idles after the audio's packets and video falls about 1 % below it. + #[test] + fn audio_between_video_packets_leaves_the_leak_running() { + const RBX: u64 = 24_000_000; + let video = Buffer { + rate: 28_800_000, + leak: Some(RBX), + size: Some(3_750_000), + }; + let audio = Buffer { + rate: 2_000_000, + leak: None, + size: Some(3_584), + }; + let rate = 26_000_000; + let mut schedule = Schedule::new(Duration::from_secs(2)); + schedule.set_rate(Some(rate)); + schedule.set_buffer(1, video); + schedule.set_buffer(2, audio); + schedule.set_buffer(3, audio); + schedule.set_clock(1); + // Two seconds of video due from 2 s on, so it is queued from the first slot, and audio + // due within the first second: MP2's 24 ms frames and AC-3's 32 ms, four packets each. + let mut units: Vec<(u128, u8, usize)> = (0..50).map(|frame| (2_000 + 40 * frame, 1, 652)).collect(); + units.extend((0..40).map(|frame| (24 * frame, 2, 4))); + units.extend((0..30).map(|frame| (32 * frame, 3, 4))); + units.sort(); + for (ms, pid, packets) in units { + schedule + .push(pid.into(), timing(ms), unit(pid, packets), false) + .unwrap(); + } + let mut video = 0; + for _ in 0..40 { + let slot = schedule.next(None).unwrap().unwrap(); + video += slot.packets.chunks(188).filter(|packet| pid(packet) == 1).count(); + } + // A second of the leak is 16,304 packets of payload. + assert!(video >= 16_290, "{video} video packets in a second"); + } + /// Unpadded, a video PID's slot carries the lesser of Rx in whole packets and Rbx in /// payload, fractions carried from slot to slot, and the clock packet counts against Rx /// alone. From 79e67a7fd9c8a09aa4a73577ae0453b2ca56cc71 Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 13 Oct 2026 16:07:08 +0100 Subject: [PATCH 3/4] fix(ts): count audio PES headers in their decoder buffer The T-STD discards a video PES header in MB, before EB, but an audio or system PID's transport buffer feeds B directly: B holds the PES header until the access unit after it is removed (ISO 13818-1 2.4.2.3). The schedule counted only elementary bytes in every decoder buffer, so an audio PES could fill B with its frame and then overflow it by its header. Count the whole payload for PIDs without a leak, and keep video's EB elementary-only. The admission check now rejects a frame that fills B with no room for its header. Co-authored-by: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/schedule.rs | 38 ++++++++++++++++++------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index d302443647d..979d010ee90 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -106,6 +106,17 @@ impl Buffer { let packets = self.leak.map_or(packets, |leak| packets.min(through(leak, PAYLOAD))); packets.max(1) as usize } + + /// The bytes of `packet` the decoder buffer holds: the payload less any PES header for + /// video, whose MB discards it, and the whole payload where the transport buffer feeds the + /// decoder buffer, which holds a PES header until the access unit after it is removed + /// (ISO 13818-1 2.4.2.3). + fn holds(&self, packet: &[u8]) -> usize { + match self.leak { + Some(_) => elementary(packet).len(), + None => payload(packet).len(), + } + } } /// TBsys in a receiver over one slot: the single 512-byte transport buffer that PIDs 0-3 and @@ -216,8 +227,7 @@ fn payload(packet: &[u8]) -> &[u8] { &packet[start..] } -/// The bytes of `packet` that reach a decoder buffer: its payload less any PES header, which -/// the T-STD discards on the way out of the transport buffer (ISO 13818-1 2.4.2.3). +/// The elementary bytes of `packet`: its payload less any PES header. fn elementary(packet: &[u8]) -> &[u8] { let body = payload(packet); match packet[1] & 0x40 != 0 && body.starts_with(&[0, 0, 1]) && body.len() >= 9 { @@ -504,7 +514,9 @@ impl Schedule { pub fn push(&mut self, pid: u16, timing: Timing, packets: Vec, keyframe: bool) -> anyhow::Result<()> { let due = slot(timing.by); // An ended source relaxes deadlines, never buffer sizes: such a unit never fits. - if let Some(size) = self.buffers.get(&pid).and_then(|buffer| buffer.size) { + if let Some(buffer) = self.buffers.get(&pid) + && let Some(size) = buffer.size + { let on_pid = || { packets .as_chunks::<{ TsPacket::SIZE }>() @@ -512,8 +524,9 @@ impl Schedule { .iter() .filter(|packet| self::pid(*packet) == pid) }; + let held: usize = on_pid().map(|packet| buffer.holds(packet)).sum(); let bytes: usize = on_pid().map(|packet| elementary(packet).len()).sum(); - if bytes > size { + if held > size { let pes: usize = on_pid().map(|packet| payload(packet).len()).sum(); anyhow::bail!( "MPEG-TS access unit on PID {pid} has {bytes} payload bytes ({pes} bytes with PES headers), exceeding its {size}-byte decoder buffer" @@ -781,10 +794,9 @@ impl Schedule { // multiplex rate decides when the packet's last byte gets there. let arrived = drained.max(time(position + 1)); let done = leak.map_or(arrived, |(leaked, _)| arrived.max(leaked)); - let held = if pid == unit.pid && buffer.is_some_and(|buffer| buffer.size.is_some()) { - elementary(packet).len() - } else { - 0 + let held = match buffer { + Some(buffer) if pid == unit.pid && buffer.size.is_some() => buffer.holds(packet), + _ => 0, }; if let Some(size) = buffer.and_then(|buffer| buffer.size) && occupancy.get(&pid).copied().unwrap_or(0) + held > size @@ -1413,10 +1425,14 @@ mod tests { assert!(error.contains("2688"), "{error}"); assert!(error.contains("2592"), "{error}"); assert!(!schedule.queued(), "refusal must not partially queue a unit"); - // Only the elementary bytes count: the PES header never reaches the decoder buffer. - for bytes in [2_304, 2_436, 2_592] { + // B holds the PES header too, until the frame after it is removed. + for bytes in [2_304, 2_436, 2_578] { schedule.push(1, timing(1_000), audio(bytes), false).unwrap(); } + assert!( + schedule.push(1, timing(1_000), audio(2_579), false).is_err(), + "a frame filling B leaves no room for its PES header" + ); } #[test] @@ -1516,7 +1532,7 @@ mod tests { free = free.max(at) + 1504.0 / 2_000_000.0; let decode = 1.0 + frame as f64 * 0.032; assert!(free <= decode + 1e-7, "{bitrate} frame{frame}: {free} > {decode}"); - held[frame] += elementary(packet).len(); + held[frame] += payload(packet).len(); let occupancy: usize = held .iter() .enumerate() From 4c3fae66092e4ebaf63a1a957140448234eb1a5e Mon Sep 17 00:00:00 2001 From: t0ms Date: Tue, 13 Oct 2026 16:08:25 +0100 Subject: [PATCH 4/4] feat(ts): carry the PCR in the clock PID and pack audio frames per PES A padded export spent a whole clock packet on every 25 ms slot and a PES on every audio frame. A source multiplex with no null packets has no room for either, so an export at the source's own rate fell behind it until its lead ran out and it missed a deadline (moq-dev/moq#5362). - Each slot's PCR rides the adaptation field of the clock PID's first packet in the slot when one goes out there, reflowing that PES's payload, and falls back to a clock packet of its own otherwise. - Consecutive frames of every audio codec (AAC, MP2, AC-3, E-AC-3 and AC-3 carried in a private stream) share a PES while they run on without a gap and fit B whole with their header. Each frame leaves B at its own decode time. PESs start on a grid of the media timeline, so exporters that joined at different frames cut the same PESs. - Frames and grid slots follow each other in decode order however often the exporter is polled, and tables muxed while frames are still packed go out with the next unit, in the order they were written. `just test ts --full` exports a generated 30 s clip at its own rate with no nulls: an x264 video at its level's leak beside MP2 and DVB AC-3, with the PCR in the video PID and several audio frames per PES. The new `overhead.py` fails the export if it spends more packets than the source on clocks and audio framing; CI runs it in the interop workflow. Retires quest/m1/tstd/mux-overhead.md. Co-authored-by: Claude Opus 5.5 --- .github/workflows/interop.yml | 10 + quest/m1/tstd/README.md | 1 - quest/m1/tstd/mux-overhead.md | 37 -- rs/moq-mux/benches/ts_schedule.rs | 3 +- rs/moq-mux/src/container/ts/export.rs | 259 ++++++++++-- rs/moq-mux/src/container/ts/export_test.rs | 193 +++++++-- rs/moq-mux/src/container/ts/schedule.rs | 454 ++++++++++++++++++--- test/ts/README.md | 27 +- test/ts/overhead.py | 86 ++++ test/ts/run.sh | 65 +++ 10 files changed, 980 insertions(+), 155 deletions(-) delete mode 100644 quest/m1/tstd/mux-overhead.md create mode 100755 test/ts/overhead.py diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 8c1f0eb4255..81d43eee246 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -192,6 +192,16 @@ jobs: env: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The same video beside two audio PIDs in a multiplex the source fills with no nulls, + # at the source's own rate: the export has no room to spend more packets than the + # source on clocks or audio framing (moq-dev/moq#5362). + - name: TS full multiplex at its own rate + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} + run: nix develop --command just test ts --full + shell: bash -leo pipefail {0} + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The same video beside four audio PIDs in a multiplex faster than its Rx: each slot # has to interleave the video with the rest, or its transport buffer overflows. - name: TS multiplex above the video's Rx diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index 8afd802a252..dbc5cf63012 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -51,7 +51,6 @@ MB/AAC controls. Do not relabel the current subset a full standards check. - [Send-ahead within the delay](/quest/m1/tstd/send-ahead.md) - total lag is `--delay`, send-ahead included, with a 1 s default - [PSI runs](/quest/m1/tstd/psi-runs.md) - the PAT and PMT are deduplicated and paced so the system buffers never overflow -- [Mux overhead](/quest/m1/tstd/mux-overhead.md) - the PCR rides in the video's adaptation field and audio PES carry several frames, so a full CBR source re-muxes at its own rate ## Related diff --git a/quest/m1/tstd/mux-overhead.md b/quest/m1/tstd/mux-overhead.md deleted file mode 100644 index d49c1e53fe4..00000000000 --- a/quest/m1/tstd/mux-overhead.md +++ /dev/null @@ -1,37 +0,0 @@ -# [M] TS export fits a full source at its own rate - -## Goal - -`moq export ts` spends no more packets than the source on clocks and audio -framing, so a CBR source that fills its multiplex with no null packets -re-muxes at its own `--mux-rate` instead of falling behind. - -## Plan - -Reported in [#5362](https://github.com/moq-dev/moq/issues/5362) point 3 and -verified against `rs/moq-mux/src/container/ts/export.rs`: with points 1 and 2 -fixed, the export of a 25 Mb/s broadcast feed averages 25.017 Mb/s of -non-null packets. Against the source it spends about 40 packets/s on -separate PCR packets (the source carries its PCR in video packets) and about -60 on one audio frame per PES (the source packs about twelve MP2 and nine -AC-3 frames per PES). - -Decided 2026-10-12: - -- **PCR in the PCR PID's adaptation field.** When the clock is due in a slot - that sends a PCR-PID packet, carry the PCR in that packet's adaptation - field, sampled at its byte position as today. Fall back to the standalone - `pcr_packet` only when the slot has none. The 25 ms grid stays; the - comment at `lay` explains why the PCR is not sampled from the DTS. -- **Several audio frames per PES.** Pack consecutive frames of an audio PID - into one PES while the PES fits the PID's B, timed by the first frame's - PTS. This applies to every audio stream the export frames (AAC, MP2, AC-3, - E-AC-3), not only DVB AC-3. -- Land after [#5363](https://github.com/moq-dev/moq/pull/5363), which reworks - the DVB AC-3 framing this packs. -- Test: a generated CBR source with no nulls, at the source's own - `--mux-rate`, exports for the whole clip and passes the strict T-STD check. - -## Closes - -- [#5362](https://github.com/moq-dev/moq/issues/5362) - a full-rate CBR contribution feed exports at its own rate diff --git a/rs/moq-mux/benches/ts_schedule.rs b/rs/moq-mux/benches/ts_schedule.rs index cb10332f2bf..9566d2f87d7 100644 --- a/rs/moq-mux/benches/ts_schedule.rs +++ b/rs/moq-mux/benches/ts_schedule.rs @@ -44,7 +44,7 @@ fn admission(c: &mut Criterion) { for pid in 1..=pids { let at = (999 + frame * 40) * 1_000_000; schedule - .push( + .push_frames( pid, schedule::Timing { by: at, @@ -52,6 +52,7 @@ fn admission(c: &mut Criterion) { }, packet(pid).repeat(10), false, + Vec::new(), ) .unwrap(); } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index ad36574ed28..dc234113549 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -123,6 +123,9 @@ pub struct Export { generation: u64, tracks: HashMap, + /// Tables muxed with frames still packed, which ride ahead of the next unit queued, so + /// they go out in the order they were written. + tables: Vec, /// The next continuity counter per PID, numbered as packets go out. counters: HashMap, /// PMT program-level descriptors captured on import, re-emitted in the PMT. @@ -241,6 +244,8 @@ struct Track { buffer: Option, /// The AC-3 sync frame a passed-through PES left unfinished, for the next PES to complete. ac3_carry: Option, + /// The audio frames muxed into a PES that has yet to go out. + pack: Option, } impl Track { @@ -260,6 +265,7 @@ impl Track { timing: Timing::default(), buffer: None, ac3_carry: None, + pack: None, } } @@ -352,6 +358,23 @@ impl Track { .any(|descriptor| descriptor.tag == AC3_DESCRIPTOR) } + /// The samples an audio frame of `payload` lasts and their rate, if it may share a PES with + /// the frames around it: one whole frame of audio whose receiver removes each frame from + /// its buffer at the frame's own decode time. + fn samples(&self, payload: &[u8]) -> Option<(u64, u32)> { + let parse = match &self.kind { + Kind::Aac { config, .. } => return (config.sample_rate > 0).then_some((1024, config.sample_rate)), + Kind::Mp2 { .. } => crate::codec::mp2::parse_header, + Kind::Ac3 => crate::codec::ac3::parse_header, + Kind::Eac3 => crate::codec::eac3::parse_header, + _ if self.carries_ac3() => crate::codec::ac3::parse_header, + _ => return None, + }; + let header = parse(payload).ok()?; + (header.len == payload.len() && header.sample_rate > 0 && header.samples > 0) + .then_some((header.samples, header.sample_rate)) + } + /// The receiver's buffers for the track's PID, if the T-STD gives it any. fn buffer(&self) -> Option { match self.kind { @@ -590,6 +613,29 @@ struct PesUnit { stream_id: Option, } +/// Consecutive audio frames muxed into one PES still to go out ([`Export::pack`]). +struct Pack { + /// The first frame's PES fields: its PTS times the frames after it too. + unit: PesUnit, + /// When the first frame decodes, the PES's deadline. + decode: Timestamp, + /// The frames' elementary stream bytes. + payload: Vec, + /// Each frame's decode nanoseconds and bytes, which leave the decoder buffer as it decodes. + frames: Vec<(u128, usize)>, + /// The frames' samples so far, and their sample rate. + samples: u64, + rate: u32, +} + +impl Pack { + /// When the frame after the last presents, in microseconds, if the frames run on without + /// a gap. + fn end(&self) -> u128 { + self.unit.timestamp.as_micros() + u128::from(self.samples) * 1_000_000 / u128::from(self.rate) + } +} + /// One SI entry's subscription: resolves the snapshot track named by the catalog, /// reduces its newest complete group into the current section set, and remembers /// when it last hit the wire so it re-emits on its own cadence. @@ -831,6 +877,7 @@ impl Export { stale: HashSet::new(), version: VersionNumber::default(), flagged: None, + tables: Vec::new(), delay: Duration::ZERO, jitter: jitter::Buffer::new(Duration::ZERO), held: None, @@ -1033,7 +1080,7 @@ impl Export { }, }; if ready.generation > self.generation { - if !self.schedule.is_empty() { + if !self.schedule.is_empty() || self.packing() { // A boundary ends valid media rather than reneging it. // Return that tail under the old generation before adopting the new one. self.lay(None)?; @@ -1055,6 +1102,7 @@ impl Export { .context("frame for an unknown PID")?; self.last_timestamp = Some(ready.item.frame.timestamp); let decode = ready.item.decode.as_nanos(); + self.lay_before(decode)?; self.mux(&name, ready.item)?; if self.delay.is_zero() { // Without a clock to lay slots on, a frame settles every slot before its own: a @@ -1503,7 +1551,7 @@ impl Export { let finished = !self.tracks.is_empty() && self.tracks.values().all(|track| track.finished); let slot = self .slot_due() - .filter(|_| self.schedule.queued() || finished) + .filter(|_| self.schedule.queued() || self.packing() || finished) .map(|(_, at)| at); self.jitter.next_deadline().into_iter().chain(slot).min() } @@ -1626,6 +1674,10 @@ impl Export { } self.video_start = None; self.pcr_discontinuity = true; + self.tables.clear(); + for track in self.tracks.values_mut() { + track.pack = None; + } } /// Header is ready when every track's [`ExportSource`] has resolved its @@ -1657,8 +1709,8 @@ impl Export { } /// Whether `pid`'s next packet is its first since a switch to another instance, so it has - /// to flag the break. The PCR PID's first packet is always a clock packet, flagged on its - /// own ([`Self::lay`]). + /// to flag the break. The PCR PID's first packet always opens a slot with its PCR, flagged + /// on its own ([`Self::lay`]). fn breaks(&mut self, pid: u16) -> bool { let pcr = self.psi.as_ref().map(|psi| psi.pcr_pid); match &mut self.flagged { @@ -1964,13 +2016,120 @@ impl Export { dts, stream_id, }; + // Only a delay leaves the frames time to wait for the ones after them. + let track = self.tracks.get(name).context("missing track")?; + if let Some(samples) = track.samples(&es_payload).filter(|_| !self.delay.is_zero()) { + return self.pack(name, unit, decode, out, es_payload, samples); + } + self.flush(name)?; self.write_pes(&mut out, &unit, &es_payload)?; } } - self.push_unit(name, pid, decode, out, keyframe) + self.push_unit(name, pid, decode, out, keyframe, Vec::new()) + } + + /// Mux an audio frame into `name`'s PES, sending the PES on first if the frame cannot join + /// it, and leave the tables muxed ahead of it to ride the next unit queued. A PES carries + /// frames while they run on from its PTS without a gap and fit the PID's decoder buffer + /// whole with its header, so it arrives by its first frame's decode time as one frame + /// would, and they span at most half the delay, so the PES has at least that long to go + /// out in. + /// + /// A PES starts at every frame whose place on the media timeline, counted in frames, is a + /// multiple of the frames the buffer holds, so exporters of the broadcast that joined at + /// different frames still cut the same PESs from the first such frame, like [`due`]. + fn pack( + &mut self, + name: &str, + unit: PesUnit, + decode: Timestamp, + tables: Vec, + payload: Vec, + (samples, rate): (u64, u32), + ) -> anyhow::Result<()> { + let budget = self.budget().as_nanos(); + let track = self.tracks.get(name).context("missing track")?; + // B holds the PES header with the frames (ISO 13818-1 2.4.2.3). + let size = track + .buffer() + .and_then(|buffer| buffer.size) + .unwrap_or(0) + .saturating_sub(PES_PREFIX_LEN + PES_OPTIONAL_LEN); + let fit = (size / payload.len().max(1)).max(1) as u128; + // Rounded, since a frame need not last whole microseconds: AAC's 1,024 samples at 48 kHz. + let frame = 1_000_000 * u128::from(samples); + let place = (unit.timestamp.as_micros() * u128::from(rate) + frame / 2) / frame; + // A PTS is a whole 90 kHz tick, so the frames run on within one of where they end. + let joins = !place.is_multiple_of(fit) + && track.pack.as_ref().is_some_and(|pack| { + pack.rate == rate + && pack.payload.len() + payload.len() <= size + && decode.as_nanos().saturating_sub(pack.decode.as_nanos()) <= budget + && pack.end().abs_diff(unit.timestamp.as_micros()) <= 12 + }); + if !joins { + self.flush(name)?; + } + self.tables.extend_from_slice(&tables); + let track = self.tracks.get_mut(name).context("missing track")?; + let pack = track.pack.get_or_insert(Pack { + unit, + decode, + payload: Vec::new(), + frames: Vec::new(), + samples: 0, + rate, + }); + pack.frames.push((decode.as_nanos(), payload.len())); + pack.payload.extend_from_slice(&payload); + pack.samples += samples; + // The next frame starts a PES of its own anyway, and without slots running on an + // empty schedule, nothing would send this one on before it is due. + if (place + 1).is_multiple_of(fit) || self.schedule.is_empty() { + self.flush(name)?; + } + Ok(()) + } + + /// Send `name`'s packed frames on as one PES. + fn flush(&mut self, name: &str) -> anyhow::Result<()> { + let Some(pack) = self.tracks.get_mut(name).and_then(|track| track.pack.take()) else { + return Ok(()); + }; + let Pack { + unit, + decode, + payload, + frames, + .. + } = pack; + let mut out = Vec::new(); + self.write_pes(&mut out, &unit, &payload)?; + self.push_unit(name, unit.pid, decode, out, unit.keyframe, frames) } - /// Queue one unit of `name`'s packets on `pid`, decoding at `decode`, on the [`Schedule`]. + /// Send on, in decode order, the packed frames of each track for which `done` holds, given + /// the slot its first frame is due in. + fn flush_packs(&mut self, done: impl Fn(u128) -> bool) -> anyhow::Result<()> { + let mut due: Vec<(Timestamp, String)> = self + .tracks + .iter() + .filter_map(|(name, track)| { + let pack = track.pack.as_ref()?; + let by = pack.decode.as_nanos().saturating_sub(track.drain().as_nanos()); + done(schedule::slot(by)).then(|| (pack.decode, name.clone())) + }) + .collect(); + due.sort(); + for (_, name) in due { + self.flush(&name)?; + } + Ok(()) + } + + /// Queue one unit of `name`'s packets on `pid`, decoding at `decode`, on the [`Schedule`], + /// behind the tables still waiting: with `frames`, each packed frame's decode nanoseconds + /// and bytes. fn push_unit( &mut self, name: &str, @@ -1978,13 +2137,22 @@ impl Export { decode: Timestamp, out: Vec, keyframe: bool, + frames: Vec<(u128, usize)>, ) -> anyhow::Result<()> { + let mut tables = std::mem::take(&mut self.tables); + let out = match tables.is_empty() { + true => out, + false => { + tables.extend_from_slice(&out); + tables + } + }; let track = self.tracks.get(name).context("missing track")?; if let Some(buffer) = track.buffer() { self.schedule.set_buffer(pid, buffer); } let by = decode.as_nanos().saturating_sub(track.drain().as_nanos()); - self.schedule.push( + self.schedule.push_frames( pid, super::schedule::Timing { by, @@ -1992,6 +2160,7 @@ impl Export { }, out, keyframe, + frames, )?; Ok(()) } @@ -2006,13 +2175,33 @@ impl Export { /// never on when they arrived, and the output keeps the clock's pace while media is queued /// or still to decode. `None` without a delay, where frames settle the slots instead. fn slot_due(&self) -> Option<(u128, web_async::time::Instant)> { - if self.delay.is_zero() || self.psi.is_none() || self.schedule.is_empty() { + let (known, settled) = self.slot_settled()?; + let settled = Timestamp::from_micros(u64::try_from(settled / 1_000).ok()?).ok()?; + Some((known, self.jitter.at(settled)?)) + } + + /// The `known` that settles the next grid slot, and the decode time, in nanoseconds, by + /// which every frame that could ride it has gone out of the jitter buffer. + fn slot_settled(&self) -> Option<(u128, u128)> { + if self.delay.is_zero() || self.psi.is_none() || (self.schedule.is_empty() && !self.packing()) { return None; } let (_, known) = self.schedule.upcoming()?; - let settled = known * PCR_INTERVAL.as_nanos() + self.max_drain().as_nanos(); - let settled = Timestamp::from_micros(u64::try_from(settled / 1_000).ok()?).ok()?; - Some((known, self.jitter.at(settled)?)) + Some((known, known * PCR_INTERVAL.as_nanos() + self.max_drain().as_nanos())) + } + + /// Whether a track holds frames packed for a PES still to go out. + fn packing(&self) -> bool { + self.tracks.values().any(|track| track.pack.is_some()) + } + + /// Lay out every grid slot settled before a frame decoding at `decode` goes out, so frames + /// and slots follow each other in the same order however often the exporter is polled. + fn lay_before(&mut self, decode: u128) -> anyhow::Result<()> { + while let Some((known, _)) = self.slot_settled().filter(|&(_, settled)| settled < decode) { + self.lay(Some(known))?; + } + Ok(()) } /// How far a stalled group may fall behind the newest content before each source skips it: @@ -2046,13 +2235,14 @@ impl Export { /// the first slot a frame still to be muxed could be due in, or `None` when nothing /// more is coming. /// - /// Each frame opens with its slot's clock packet. At a multiplex rate its value is the - /// time of its own byte position at the rate, so a consumer holding only the byte stream - /// (which is every MPEG-TS tool) recovers the same clock the values assert. Each frame is - /// stamped at its slot boundary, so a pacing caller releases the clock at the instant it - /// asserts. + /// Each frame opens with its slot's PCR: at a multiplex rate, in the adaptation field of + /// the clock PID's packet when one can go out there, else in a clock packet of its own. + /// At a multiplex rate its value is the time of its own byte position at the rate, so a + /// consumer holding only the byte stream (which is every MPEG-TS tool) recovers the same + /// clock the values assert. Each frame is stamped at its slot boundary, so a pacing + /// caller releases the clock at the instant it asserts. /// - /// The PES units cannot carry the clock themselves: a PCR sampled from the DTS would + /// The PCR's value never comes from the PES units: a PCR sampled from the DTS would /// step with the frame cadence, and no downstream CBR stage can repair that, because a /// groomer can only place the clock samples it receives. So the PCR follows its own /// grid instead: slots on the media timeline, shared by every exporter of the @@ -2062,13 +2252,29 @@ impl Export { return Ok(()); }; let (pcr_pid, pmt_pid) = (psi.pcr_pid, psi.pmt_pid); - while let Some(slot) = self.schedule.next(known)? { - let mut clock = pcr_packet(pcr_pid, slot.pcr)?; + if known.is_none() { + self.flush_packs(|_| true)?; + } + let lead = schedule::slot(self.budget().as_nanos()); + loop { + // A PES of packed frames goes out half the delay before its first is due, so it + // has that long to reach the decoder. + if known.is_some() + && let Some((index, _)) = self.schedule.upcoming() + { + self.flush_packs(|due| due <= index + lead)?; + } + let Some(slot) = self.schedule.next(known)? else { + break; + }; + let clock = pcr_packet(pcr_pid, slot.pcr)?; + let mut payload = slot.layout(&clock, pmt_pid, &NULL_PACKET); + // The slot opens with its PCR, in the clock packet or the clock PID's packet that + // carries it, whose adaptation field flags the break. let discontinuity = std::mem::take(&mut self.pcr_discontinuity); if discontinuity { - clock[5] |= 0x80; + payload[5] |= 0x80; } - let mut payload = slot.layout(&clock, pmt_pid, &NULL_PACKET); self.number(&mut payload); let frame = Frame { timestamp: slot_stamp(slot.index)?, @@ -2259,7 +2465,6 @@ impl Export { /// is numbered when it goes out ([`Export::number`]). fn pcr_packet(pid: u16, pcr: u64) -> anyhow::Result> { anyhow::ensure!(pid <= 0x1FFF, "PID out of range: {pid}"); - let (base, extension) = ((pcr / 300) & TS_TIMESTAMP_MASK, pcr % 300); let mut p = Vec::with_capacity(TsPacket::SIZE); p.push(0x47); p.push((pid >> 8) as u8); @@ -2270,17 +2475,13 @@ fn pcr_packet(pid: u16, pcr: u64) -> anyhow::Result> { p.push(183); // PCR_flag alone. p.push(0x10); - // program_clock_reference_base (33 bits), 6 reserved '1' bits, and the 9-bit extension. - p.push((base >> 25) as u8); - p.push((base >> 17) as u8); - p.push((base >> 9) as u8); - p.push((base >> 1) as u8); - p.push(((base as u8) << 7) | 0x7e | (extension >> 8) as u8); - p.push(extension as u8); + p.extend_from_slice(&super::schedule::pcr_field(pcr)); p.resize(TsPacket::SIZE, 0xff); Ok(p) } +/// PES header bytes ahead of the optional region: start code, stream id and packet length. +const PES_PREFIX_LEN: usize = 6; /// Optional PES header region carrying PTS only: 2 flag bytes + 1 length byte + 5 PTS bytes. const PES_OPTIONAL_LEN: usize = 3 + 5; /// Extra bytes when the optional region also carries a DTS (5 DTS bytes). diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 214d4b870c1..4ef261e6301 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -193,6 +193,72 @@ async fn export_aac_roundtrip() { } } +/// AAC frames that run on from each other share a PES, as many as B holds with its header +/// (ISO 13818-1 2.4.2.3), each frame whole and in order, though none lasts whole microseconds. +#[tokio::test(start_paused = true)] +async fn export_packs_aac_frames_up_to_their_buffer() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track( + broadcast.unique_name(".aac"), + hang::container::track_info(hang::catalog::PRIORITY.audio), + ) + .unwrap(); + let name = track.name().to_string(); + { + let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 2); + cfg.container = Container::Legacy; + catalog.modify().unwrap().audio.renditions.insert(name.clone(), cfg); + } + let mut producer = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); + // 1,024 samples at 48 kHz: 21.33 ms each, 307 bytes with ADTS, 11 to B's 3,584. + let frames: Vec = (0..48u8).map(|k| Bytes::from(vec![k; 300])).collect(); + for (i, payload) in frames.iter().enumerate() { + producer + .write(Frame { + timestamp: Timestamp::from_micros(i as u64 * 1_024 * 1_000_000 / 48_000).unwrap(), + duration: None, + payload: payload.clone(), + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + } + producer.finish().unwrap(); + let ts = drain(consumer).await; + + let mut pes = PesPacketReader::new(TsPacketReader::new(Cursor::new(ts.as_ref()))); + let (mut count, mut recovered) = (0, Vec::new()); + while let Some(packet) = pes.read_pes_packet().unwrap() { + count += 1; + assert!( + 14 + packet.data.len() <= 3_584, + "a {}-byte PES overflows B", + packet.data.len() + ); + let mut pts = packet.header.pts.expect("PES carried no PTS").as_u64(); + let mut data = &packet.data[..]; + while !data.is_empty() { + let len = usize::from(data[3] & 3) << 11 | usize::from(data[4]) << 3 | usize::from(data[5] >> 5); + recovered.push((pts, data[7..len].to_vec())); + data = &data[len..]; + pts += 1_920; + } + } + assert!( + count <= frames.len().div_ceil(11) + 1, + "{count} PES for {} frames", + frames.len() + ); + assert_eq!(recovered.len(), frames.len()); + for (i, (pts, raw)) in recovered.iter().enumerate() { + assert!(pts.abs_diff(i as u64 * 1_920) <= 1, "frame {i} at {pts}"); + assert_eq!(raw.as_slice(), frames[i].as_ref(), "frame {i}"); + } +} + /// Collect PES presentation timestamps per elementary stream (video H.264, audio AAC), /// keyed off the PMT's PID assignments. fn collect_pes_pts(ts: &[u8]) -> (Vec, Vec) { @@ -1056,17 +1122,17 @@ async fn export_pcr_backfills_a_coarse_cadence() { /// AC-3 passed through as DVB private data (stream_type 0x06 with the AC-3 descriptor) can /// carry several sync frames in one PES, more than a receiver's 5,696-byte buffer holds at -/// once. The export splits it a frame a PES, each presented a frame after the last, so each -/// is due and decoded on its own. +/// once. The export splits it into its sync frames, each presented a frame after the last, so +/// each is due and decoded on its own. #[tokio::test(start_paused = true)] async fn export_splits_a_multi_frame_ac3_pes() { - let pes = exported_ac3(ac3_sync().repeat(3)).await; - assert_eq!(pes.len(), 3, "one PES a sync frame: {pes:?}"); - for (k, &(pts, bytes)) in pes.iter().enumerate() { + let frames = exported_ac3(ac3_sync().repeat(3)).await; + assert_eq!(frames.len(), 3, "every sync frame: {frames:?}"); + for (k, &(pts, bytes)) in frames.iter().enumerate() { assert_eq!(bytes, 128, "frame {k} carries one sync frame"); assert_eq!( pts, - pes[0].0 + k as u64 * 2_880, + frames[0].0 + k as u64 * 2_880, "frame {k} is presented 32 ms after the last" ); } @@ -1079,33 +1145,33 @@ async fn export_splits_a_multi_frame_ac3_pes() { async fn export_drops_a_trailing_partial_ac3_frame() { let mut payload = ac3_sync().repeat(2); payload.extend_from_slice(&ac3_sync()[..60]); - let pes = exported_ac3(payload).await; - assert_eq!(pes.len(), 2, "the whole frames, each its own PES: {pes:?}"); - assert!(pes.iter().all(|&(_, bytes)| bytes == 128), "{pes:?}"); + let frames = exported_ac3(payload).await; + assert_eq!(frames.len(), 2, "the whole frames: {frames:?}"); + assert!(frames.iter().all(|&(_, bytes)| bytes == 128), "{frames:?}"); - let pes = exported_ac3(ac3_sync()[..60].to_vec()).await; - assert!(pes.is_empty(), "a lone cut frame goes nowhere: {pes:?}"); + let frames = exported_ac3(ac3_sync()[..60].to_vec()).await; + assert!(frames.is_empty(), "a lone cut frame goes nowhere: {frames:?}"); } /// A source that leaves `data_alignment_indicator` clear splits AC-3 sync frames across PES, /// each PES's PTS timing the first frame that begins in it. The export puts each frame back -/// together, a frame a PES, byte for byte and presented a frame after the last. +/// together, byte for byte and presented a frame after the last. #[tokio::test(start_paused = true)] async fn export_reassembles_ac3_frames_split_across_pes() { let stream: Vec = (0..6).flat_map(ac3_frame).collect(); // Cut mid-frame into frames 0-1, 1-3 and 3-5, timed by frames 0, 2 and 4. - let pes = exported_ac3_pes(vec![ + let frames = exported_ac3_frames(vec![ Ac3Write::Pes(0, stream[..200].to_vec()), Ac3Write::Pes(64, stream[200..500].to_vec()), Ac3Write::Pes(128, stream[500..].to_vec()), ]) .await; - assert_eq!(pes.len(), 6, "one PES a sync frame"); - for (k, (pts, bytes)) in pes.iter().enumerate() { + assert_eq!(frames.len(), 6, "every sync frame"); + for (k, (pts, bytes)) in frames.iter().enumerate() { assert_eq!(*bytes, ac3_frame(k as u8), "frame {k} whole and in order"); assert_eq!( *pts, - pes[0].0 + k as u64 * 2_880, + frames[0].0 + k as u64 * 2_880, "frame {k} is presented 32 ms after the last" ); } @@ -1128,10 +1194,10 @@ async fn export_reassembles_an_ac3_header_split_across_pes() { ], ]; for input in cuts { - let pes = exported_ac3_pes(input).await; - let frames: Vec<&Vec> = pes.iter().map(|(_, bytes)| bytes).collect(); + let timed = exported_ac3_frames(input).await; + let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); assert_eq!(frames, [&ac3_frame(0), &ac3_frame(1)]); - assert_eq!(pes[1].0 - pes[0].0, 2_880); + assert_eq!(timed[1].0 - timed[0].0, 2_880); } } @@ -1142,15 +1208,15 @@ async fn export_reassembles_an_ac3_header_split_across_pes() { async fn export_drops_ac3_fragments_at_a_join_or_a_gap() { let stream: Vec = (0..6).flat_map(ac3_frame).collect(); // Joined 60 bytes into frame 0; the PES carrying the rest of frame 2 and most of 3 is lost. - let pes = exported_ac3_pes(vec![ + let timed = exported_ac3_frames(vec![ Ac3Write::Pes(32, stream[60..300].to_vec()), Ac3Write::Pes(128, stream[500..].to_vec()), ]) .await; - let frames: Vec<&Vec> = pes.iter().map(|(_, bytes)| bytes).collect(); + let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); assert_eq!(frames, [&ac3_frame(1), &ac3_frame(4), &ac3_frame(5)]); - assert_eq!(pes[1].0 - pes[0].0, 3 * 2_880, "frame 4 at its PES's PTS"); - assert_eq!(pes[2].0 - pes[1].0, 2_880); + assert_eq!(timed[1].0 - timed[0].0, 3 * 2_880, "frame 4 at its PES's PTS"); + assert_eq!(timed[2].0 - timed[1].0, 2_880); } /// A sync word in the bytes ahead of the first frame at a join can pass for a header whose @@ -1162,8 +1228,8 @@ async fn export_skips_an_unconfirmed_ac3_sync_word_at_a_join() { let mut payload = vec![0x0b, 0x77, 0x00, 0x00, 0x25, 0x40, 0x40]; payload.resize(40, 0x00); payload.extend((1..3).flat_map(ac3_frame)); - let pes = exported_ac3_pes(vec![Ac3Write::Pes(0, payload)]).await; - let frames: Vec<&Vec> = pes.iter().map(|(_, bytes)| bytes).collect(); + let timed = exported_ac3_frames(vec![Ac3Write::Pes(0, payload)]).await; + let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); assert_eq!(frames, [&ac3_frame(1), &ac3_frame(2)]); } @@ -1172,16 +1238,40 @@ async fn export_skips_an_unconfirmed_ac3_sync_word_at_a_join() { #[tokio::test(start_paused = true)] async fn export_drops_an_ac3_frame_cut_by_a_marker() { let stream: Vec = (0..4).flat_map(ac3_frame).collect(); - let pes = exported_ac3_pes(vec![ + let timed = exported_ac3_frames(vec![ Ac3Write::Pes(0, stream[..200].to_vec()), Ac3Write::Marker, Ac3Write::Pes(64, stream[256..].to_vec()), ]) .await; - let frames: Vec<&Vec> = pes.iter().map(|(_, bytes)| bytes).collect(); + let frames: Vec<&Vec> = timed.iter().map(|(_, bytes)| bytes).collect(); assert_eq!(frames, [&ac3_frame(0), &ac3_frame(2), &ac3_frame(3)]); } +/// AC-3 sync frames that run on share a PES, as many as DVB's 5,696-byte B holds with the +/// PES header (ISO 13818-1 2.4.2.3), and still decode in order 32 ms apart. +#[tokio::test(start_paused = true)] +async fn export_packs_ac3_frames_up_to_their_buffer() { + let stream: Vec = (0..100).flat_map(ac3_frame).collect(); + let pes = exported_ac3_pes(vec![Ac3Write::Pes(0, stream.clone())]).await; + // 44 of the 128-byte frames to a PES, and the first frame alone, sent as it starts. + assert!( + pes.len() <= 100usize.div_ceil(44) + 1, + "{} PES for 100 frames", + pes.len() + ); + for (_, payload) in &pes { + assert!(14 + payload.len() <= 5_696, "a {}-byte PES overflows B", payload.len()); + } + let bytes: Vec = pes.into_iter().flat_map(|(_, payload)| payload).collect(); + assert_eq!(bytes, stream, "every frame whole and in order"); + let timed = exported_ac3_frames(vec![Ac3Write::Pes(0, stream)]).await; + assert!( + timed.windows(2).all(|pair| pair[1].0 - pair[0].0 == 2_880), + "32 ms apart" + ); +} + /// A 32 kb/s 48 kHz 2/0 AC-3 sync frame: 128 bytes, 1,536 samples (32 ms). fn ac3_sync() -> Vec { let mut sync = vec![0x0b, 0x77, 0x00, 0x00, 0x00, 0x40, 0x40]; @@ -1197,9 +1287,9 @@ fn ac3_frame(k: u8) -> Vec { } /// Export one PES of `payload` on an AC-3 passthrough PID beside bbb's media, and return the -/// PTS of every PES on that PID and how many bytes each carries. +/// PTS of every sync frame on that PID and how many bytes each carries. async fn exported_ac3(payload: Vec) -> Vec<(u64, usize)> { - exported_ac3_pes(vec![Ac3Write::Pes(0, payload)]) + exported_ac3_frames(vec![Ac3Write::Pes(0, payload)]) .await .into_iter() .map(|(pts, bytes)| (pts, bytes.len())) @@ -1306,6 +1396,21 @@ async fn exported_ac3_pes(input: Vec) -> Vec<(u64, Vec)> { pes } +/// Export `input` as [`exported_ac3_pes`] does, and return each sync frame on the AC-3 PID, +/// timed by its PES's PTS and the frames ahead of it there. +async fn exported_ac3_frames(input: Vec) -> Vec<(u64, Vec)> { + let mut frames = Vec::new(); + for (mut pts, mut payload) in exported_ac3_pes(input).await { + while !payload.is_empty() { + let header = crate::codec::ac3::parse_header(&payload).expect("the PES carries whole sync frames"); + let rest = payload.split_off(header.len); + frames.push((pts, std::mem::replace(&mut payload, rest))); + pts += header.samples * 90_000 / u64::from(header.sample_rate); + } + } + frames +} + /// Full SCTE-35 round-trip: import `bbb.ts` (real H.264 + AAC) into a broadcast /// that also carries a `.scte35` cue track, export to TS, re-import, and assert /// the splice_info_section came back byte-for-byte. The PMT must advertise the @@ -4838,6 +4943,26 @@ fn pes_decode_in_order(frames: &[Frame]) -> Vec { out } +/// How many access units the stream carries: one a PES, or each ADTS frame of an AAC PES. +fn access_units(frames: &[Frame]) -> usize { + let bytes: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); + let mut reader = PesPacketReader::new(TsPacketReader::new(Cursor::new(bytes))); + let mut units = 0; + while let Some(pes) = reader.read_pes_packet().unwrap() { + let mut data = &pes.data[..]; + if !(data.len() > 6 && data[0] == 0xff && data[1] & 0xf0 == 0xf0) { + units += 1; + continue; + } + while data.len() > 6 { + let len = usize::from(data[3] & 3) << 11 | usize::from(data[4]) << 3 | usize::from(data[5] >> 5); + data = &data[len.clamp(7, data.len())..]; + units += 1; + } + } + units +} + /// Whether every PES start decodes no earlier than those ahead of it, up to one grid slot: /// a slot spreads each PID's packets among the others, so units sharing one may swap. fn in_decode_order(decode: &[u64]) -> bool { @@ -4942,7 +5067,8 @@ async fn lagging_audio(delay: Duration) -> Vec { #[tokio::test(start_paused = true)] async fn late_audio_still_arrives_on_time() { let frames = lagging_audio(Duration::from_millis(500)).await; - assert!(assert_on_time(&frames) > 100, "too little output to judge"); + // 75 video frames, and the audio packed several frames a PES. + assert!(assert_on_time(&frames) > 80, "too little output to judge"); } /// A zero delay holds nothing, so the interleave stays in arrival order. @@ -5002,10 +5128,9 @@ async fn a_late_frame_is_dropped_and_the_rest_keep_their_order() { out.extend(drain_frames(&mut export).await); let written = rig.audio_index + TICKS / 2; - let decode = pes_decode_in_order(&out); assert!(export.dropped() > 0, "the stalled audio missed its deadline"); assert_eq!( - decode.len() as u64, + access_units(&out) as u64, written - export.dropped(), "only the late frames were dropped" ); @@ -5739,8 +5864,8 @@ fn collect_pcrs(frames: &[Frame]) -> Vec<(usize, u64, u128)> { let mut pcrs = Vec::new(); for frame in frames { for packet in frame.payload.chunks(188) { - // adaptation-field-only packets (adaptation_field_control == 0b10) are the clock. - if packet[3] & 0x30 == 0x20 && packet[5] & 0x10 != 0 { + // A clock packet, or the clock PID's packet that carries the slot's PCR. + if packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x10 != 0 { let base = (u64::from(packet[6]) << 25) | (u64::from(packet[7]) << 17) | (u64::from(packet[8]) << 9) diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 979d010ee90..0b0d3bf8ee5 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -178,6 +178,10 @@ struct Unit { pid: u16, /// The coarse deadline used by release windows and unpadded admission. due: u128, + /// The due slot of its last frame. A PES that packs several is pushed once that frame is + /// muxed, so its window runs from there, as one frame's runs from when it is pushed: every + /// slot it may ride is laid out after it is queued, whenever the caller lays them. + settled: u128, timing: Timing, /// The slot after the unit pushed before it, which it may go out from even without a /// window: nothing pushed later can be due sooner. @@ -190,6 +194,9 @@ struct Unit { late: bool, /// Its bytes in the PID's decoder buffer, until it decodes. held: usize, + /// The decode time and bytes of each frame of a PES that packs several, which leave the + /// decoder buffer one by one; empty for a PES of one frame, which leaves at once. + frames: Vec<(u128, usize)>, } impl Unit { @@ -197,6 +204,33 @@ impl Unit { self.packets.len() / TsPacket::SIZE } + /// When its last bytes leave the decoder buffer. + fn last_decode(&self) -> u128 { + self.frames.last().map_or(self.timing.decode, |&(decode, _)| decode) + } + + /// When each of its frames leaves the decoder buffer, with the bytes it takes: all it + /// holds, for a unit of one frame. + fn frames(&self) -> impl Iterator + '_ { + let single = self.frames.is_empty().then_some((self.timing.decode, usize::MAX)); + self.frames.iter().copied().chain(single) + } + + /// What it leaves in the decoder buffer once sent: each frame's PID, decode time and + /// bytes, of the bytes it holds, which the frames decoded already have taken first. + fn leaving(&self) -> Vec<(u16, u128, usize)> { + let mut held = self.held; + let mut out: Vec<_> = self.frames().collect(); + for (_, bytes) in out.iter_mut().rev() { + *bytes = (*bytes).min(held); + held -= *bytes; + } + out.into_iter() + .filter(|&(_, bytes)| bytes > 0) + .map(|(decode, bytes)| (self.pid, decode, bytes)) + .collect() + } + fn remaining(&self) -> usize { self.count() - self.sent } @@ -236,6 +270,90 @@ fn elementary(packet: &[u8]) -> &[u8] { } } +/// A PCR of `ticks` as an adaptation field carries it: the 33-bit base, six reserved bits, +/// all ones as ISO 13818-1 requires, and the 9-bit extension. +pub(super) fn pcr_field(ticks: u64) -> [u8; 6] { + let (base, extension) = ((ticks / 300) & ((1 << 33) - 1), ticks % 300); + [ + (base >> 25) as u8, + (base >> 17) as u8, + (base >> 9) as u8, + (base >> 1) as u8, + ((base as u8) << 7) | 0x7e | (extension >> 8) as u8, + extension as u8, + ] +} + +/// `packets`, the rest of a PES on one PID, rewritten so the first carries the PCR `ticks` in +/// its adaptation field: the payload moves on by the field's bytes, into the stuffing of the +/// last packet or one more packet. `None` unless every packet is the PID's with payload, and +/// its adaptation field, if any, carries nothing but flags and stuffing, as the export +/// writes them. +fn with_pcr(packets: &[u8], ticks: u64) -> Option> { + let packets = packets.as_chunks::<{ TsPacket::SIZE }>().0; + let first = packets.first()?; + let mut body = Vec::with_capacity(packets.len() * PAYLOAD); + let mut flags = Vec::with_capacity(packets.len()); + for packet in packets { + if pid(packet) != pid(first) || packet[3] & 0x10 == 0 { + return None; + } + // Only the PES's first packet may start it, and only the flags byte's random access + // and discontinuity indicators may be set. + if packet != first && packet[1] & 0x40 != 0 { + return None; + } + let field = (packet[3] & 0x20 != 0).then(|| &packet[5..5 + usize::from(packet[4])]); + let flag = field.and_then(|field| field.first().copied()).unwrap_or(0); + if flag & !0xc0 != 0 { + return None; + } + flags.push(flag); + body.extend_from_slice(payload(packet)); + } + if body.is_empty() { + return None; + } + let mut out = Vec::with_capacity((packets.len() + 1) * TsPacket::SIZE); + let mut offset = 0; + for k in 0.. { + let start = if k == 0 { first[1] & 0x40 } else { 0 }; + let flag = flags.get(k).copied().unwrap_or(0); + let mut field = Vec::new(); + if k == 0 { + field.push(flag | 0x10); + field.extend_from_slice(&pcr_field(ticks)); + } else if flag != 0 { + field.push(flag); + } + let room = PAYLOAD - if field.is_empty() { 0 } else { 1 + field.len() }; + let take = room.min(body.len() - offset); + if take < room { + // Stuffing fills the field out to the packet's end; a field of length 0 is the + // single byte of stuffing. + if field.is_empty() && take + 1 < PAYLOAD { + field.push(0); + } + field.resize(PAYLOAD - 1 - take, 0xff); + } + out.extend_from_slice(&[first[0], first[1] & !0x40 | start, first[2]]); + match field.is_empty() && take == PAYLOAD { + true => out.push(0x10), + false => { + out.push(0x30); + out.push(field.len() as u8); + out.extend_from_slice(&field); + } + } + out.extend_from_slice(&body[offset..offset + take]); + offset += take; + if offset == body.len() { + break; + } + } + Some(out) +} + /// One grid slot's worth of output. pub(super) struct Slot { pub index: u128, @@ -253,14 +371,17 @@ pub(super) struct Slot { pub caps: HashMap, /// Exact positions of the media packets in a padded slot, counting the opening clock. positions: Option>, + /// The padded slot opens with a packet of the clock's PID carrying its PCR, so it needs + /// no clock packet of its own. + pub clocked: bool, } impl Slot { - /// The whole slot, opening with its `clock` packet. Padded slots use the exact - /// positions planned against receiver buffers. Unpadded slots use smooth weighted - /// round-robin: each PID (the clock packet counted on its own) and the null packets take - /// positions in proportion to their count, as evenly as they can, every PID keeping its - /// own order. + /// The whole slot, opening with its `clock` packet unless it is [`Self::clocked`]. Padded + /// slots use the exact positions planned against receiver buffers. Unpadded slots use + /// smooth weighted round-robin: each PID (the clock packet counted on its own) and the null + /// packets take positions in proportion to their count, as evenly as they can, every PID + /// keeping its own order. /// /// A PID's packets in a run would arrive at the multiplex's full rate and overflow its /// 512-byte transport buffer, which drains only at its Rx (ISO 13818-1 2.4.2.3): an audio @@ -276,15 +397,18 @@ impl Slot { pub fn layout(&self, clock: &[u8], pmt_pid: u16, null: &[u8]) -> Vec { let packets = self.packets.as_chunks::<{ TsPacket::SIZE }>().0; if let Some(positions) = &self.positions { - let mut out = Vec::with_capacity((1 + packets.len() + self.nulls) * TsPacket::SIZE); - out.extend_from_slice(clock); + let count = usize::from(!self.clocked) + packets.len() + self.nulls; + let mut out = Vec::with_capacity(count * TsPacket::SIZE); + if !self.clocked { + out.extend_from_slice(clock); + } for (&position, packet) in positions.iter().zip(packets) { while out.len() / TsPacket::SIZE < position { out.extend_from_slice(null); } out.extend_from_slice(packet); } - while out.len() / TsPacket::SIZE < 1 + packets.len() + self.nulls { + while out.len() / TsPacket::SIZE < count { out.extend_from_slice(null); } return out; @@ -509,9 +633,27 @@ impl Schedule { self.clock = Some(pid); } - /// Queue an access unit on `pid` (whole TS packets, the tables muxed ahead of it first) - /// with its exact transport deadline and decoder removal time. + /// Queue an access unit of one frame. + #[cfg(test)] pub fn push(&mut self, pid: u16, timing: Timing, packets: Vec, keyframe: bool) -> anyhow::Result<()> { + self.push_frames(pid, timing, packets, keyframe, Vec::new()) + } + + /// Queue an access unit on `pid` (whole TS packets, the tables muxed ahead of it first) + /// with its exact transport deadline and decoder removal time. A PES that packs several + /// frames lists each one's decode time and elementary bytes, in decode order, which leave + /// the decoder buffer one by one; `timing` is then the first frame's. + pub fn push_frames( + &mut self, + pid: u16, + timing: Timing, + packets: Vec, + keyframe: bool, + mut frames: Vec<(u128, usize)>, + ) -> anyhow::Result<()> { + if frames.len() < 2 { + frames.clear(); + } let due = slot(timing.by); // An ended source relaxes deadlines, never buffer sizes: such a unit never fits. if let Some(buffer) = self.buffers.get(&pid) @@ -532,13 +674,23 @@ impl Schedule { "MPEG-TS access unit on PID {pid} has {bytes} payload bytes ({pes} bytes with PES headers), exceeding its {size}-byte decoder buffer" ); } + // The PES header leaves the decoder buffer with the first frame after it. + if let Some(first) = frames.first_mut() { + first.1 += held - bytes; + } } - self.horizon = self.horizon.max(Some(timing.by.div_ceil(PCR_INTERVAL.as_nanos()))); + // The deadline its last frame would have on its own. + let last = timing.by + + frames + .last() + .map_or(0, |&(decode, _)| decode.saturating_sub(timing.decode)); + self.horizon = self.horizon.max(Some(last.div_ceil(PCR_INTERVAL.as_nanos()))); let start = self.last.map_or(due, |last| (last + 1).min(due)); self.last = Some(due); self.units.push_back(Unit { pid, due, + settled: slot(last), timing, start, packets, @@ -546,6 +698,7 @@ impl Schedule { keyframe, late: self.next.is_some_and(|next| due < next), held: 0, + frames, }); Ok(()) } @@ -576,16 +729,16 @@ impl Schedule { self.ended = false; } - /// The first slot `unit` may go out in: a window ahead of its due slot, or only the slot - /// before it for a PID whose decoder buffer bounds nothing, and never before the slot - /// after the unit pushed ahead of it needs. + /// The first slot `unit` may go out in: a window ahead of its last frame's due slot, or + /// only the slot before it for a PID whose decoder buffer bounds nothing, and never before + /// the slot after the unit pushed ahead of it needs. fn release(&self, unit: &Unit) -> u128 { let ahead = match self.buffers.get(&unit.pid).and_then(|buffer| buffer.size) { _ if unit.late => return 0, Some(_) => self.window, None => 1, }; - unit.due.saturating_sub(ahead).min(unit.start) + unit.settled.saturating_sub(ahead).min(unit.start) } /// The slot [`Self::next`] lays out next, with `known` as it would be given. @@ -650,15 +803,15 @@ impl Schedule { keyframe |= unit.keyframe; // Only the tables muxed ahead of a frame that wrote nothing are no access unit. if unit.packets_on(0..unit.count(), unit.pid) > 0 { - units.push(unit.pid); + units.extend(std::iter::repeat_n(unit.pid, unit.frames.len().max(1))); } } let from = unit.sent * TsPacket::SIZE; packets.extend_from_slice(&unit.packets[from..from + take * TsPacket::SIZE]); unit.sent += take; } - for unit in self.units.iter().filter(|unit| unit.remaining() == 0 && unit.held > 0) { - self.decoding.push_back((unit.pid, unit.timing.decode, unit.held)); + for unit in self.units.iter().filter(|unit| unit.remaining() == 0) { + self.decoding.extend(unit.leaving()); } self.units.retain(|unit| unit.remaining() > 0); Ok(Some(Slot { @@ -674,6 +827,7 @@ impl Schedule { .map(|(&pid, buffer)| (pid, buffer.per_slot(index, self.clock == Some(pid)))) .collect(), positions: None, + clocked: false, })) } @@ -695,14 +849,9 @@ impl Schedule { units: Vec::new(), caps: HashMap::new(), positions: Some(Vec::new()), + clocked: false, }; - if let Some(pid) = self.clock - && let Some(buffer) = self.buffers.get(&pid) - { - let duration = (TsPacket::SIZE as u128 * 8 * 1_000_000_000).div_ceil(u128::from(buffer.rate)); - let free = self.transport.entry(pid).or_insert(time(0)); - *free = ((*free).max(time(0)) + duration as i128).max(time(1)); - } + let clock = self.clock.filter(|pid| self.buffers.contains_key(pid)); // One head per PID preserves decode order without walking all future units at every // transport position. Removal events keep occupancy work proportional to units. let mut lanes: Vec> = Vec::new(); @@ -720,9 +869,11 @@ impl Schedule { lanes.len() - 1 }); lanes[lane].push_back(i); - if unit.timing.decode as i128 > time(0) { + if unit.last_decode() as i128 > time(0) { *occupancy.entry(unit.pid).or_default() += unit.held; - removals.push((unit.timing.decode, unit.pid, Some(i), 0)); + for (decode, bytes) in unit.frames().filter(|&(decode, _)| decode as i128 > time(0)) { + removals.push((decode, unit.pid, Some(i), bytes)); + } } } removals.sort_by_key(|event| event.0); @@ -736,11 +887,16 @@ impl Schedule { }) .collect(); let mut waiting: BinaryHeap> = BinaryHeap::new(); - let mut position = 1; + let mut position = 0; while position < count { let now = time(position); while let Some((_, pid, unit, held)) = removals.next_if(|event| event.0 as i128 <= now) { - let held = unit.map_or(held, |i| std::mem::take(&mut self.units[i].held)); + let held = unit.map_or(held, |i| { + let unit = &mut self.units[i]; + let take = held.min(unit.held); + unit.held -= take; + take + }); *occupancy.get_mut(&pid).unwrap() -= held; } while waiting.peek().is_some_and(|Reverse((at, _))| *at <= now) { @@ -748,10 +904,26 @@ impl Schedule { ready.push(Reverse((self.units[*lanes[lane].front().unwrap()].timing.by, lane))); } let mut emitted = false; - while let Some(Reverse((_, lane))) = ready.pop() { + // The slot's PCR opens it. Its PID's next packet carries it there when it can go + // out at once, which spares the slot a clock packet; the other PIDs wait a position. + let mut passed = Vec::new(); + while let Some(Reverse((by, lane))) = ready.pop() { let i = *lanes[lane].front().unwrap(); let unit = &self.units[i]; - let packet = &unit.packets[unit.sent * TsPacket::SIZE..(unit.sent + 1) * TsPacket::SIZE]; + let rest = &unit.packets[unit.sent * TsPacket::SIZE..]; + let opening = match position { + 0 => match clock.filter(|&clock| self::pid(rest) == clock) { + Some(_) => with_pcr(rest, out.pcr), + None => None, + }, + _ => None, + }; + if position == 0 && opening.is_none() { + passed.push(Reverse((by, lane))); + continue; + } + let packet = opening.as_deref().unwrap_or(rest); + let packet = &packet[..TsPacket::SIZE]; let pid = self::pid(packet); let buffer = self.buffers.get(&pid); let duration = buffer.map_or(0, |buffer| { @@ -820,14 +992,21 @@ impl Schedule { out.packets.extend_from_slice(packet); out.nulls -= 1; let unit = &mut self.units[i]; + if let Some(rest) = opening { + unit.packets.truncate(unit.sent * TsPacket::SIZE); + unit.packets.extend_from_slice(&rest); + out.clocked = true; + out.nulls += 1; + } if unit.sent == 0 { out.keyframe |= unit.keyframe; if unit.packets_on(0..unit.count(), unit.pid) > 0 { - out.units.push(unit.pid); + out.units + .extend(std::iter::repeat_n(unit.pid, unit.frames.len().max(1))); } } unit.sent += 1; - if unit.timing.decode as i128 > now { + if unit.last_decode() as i128 > now { unit.held += held; *occupancy.entry(unit.pid).or_default() += held; } @@ -843,7 +1022,18 @@ impl Schedule { emitted = true; break; } - if emitted { + ready.extend(passed); + if position == 0 && !emitted { + // A clock packet of its own opens the slot, and its PID's transport buffer + // takes it first. + if let Some(pid) = clock { + let rate = self.buffers[&pid].rate; + let duration = (TsPacket::SIZE as u128 * 8 * 1_000_000_000).div_ceil(u128::from(rate)); + let free = self.transport.entry(pid).or_insert(time(0)); + *free = ((*free).max(time(0)) + duration as i128).max(time(1)); + } + position = 1; + } else if emitted { position += 1; } else if let Some(Reverse((next, _))) = waiting.peek() { // Nothing can use intervening null positions. Jump to the first packet at the @@ -867,8 +1057,8 @@ impl Schedule { unit.pid ); } - for unit in self.units.iter().filter(|unit| unit.remaining() == 0 && unit.held > 0) { - self.decoding.push_back((unit.pid, unit.timing.decode, unit.held)); + for unit in self.units.iter().filter(|unit| unit.remaining() == 0) { + self.decoding.extend(unit.leaving()); } self.units.retain(|unit| unit.remaining() > 0); Ok(out) @@ -993,7 +1183,8 @@ mod tests { slots } - /// Check actual wire positions against the receiver's transport buffer, clock included. + /// Check actual wire positions against the receiver's transport buffer, clock included, + /// and count the packets' worth of payload PID 1 sent. fn checked_transport(schedule: &mut Schedule, clock: u8) -> usize { let rate = schedule.rate.unwrap(); let rx = schedule.buffers[&1].rate; @@ -1007,7 +1198,8 @@ mod tests { .0 .iter() .filter(|packet| pid(*packet) == 1) - .count(); + .map(|packet| payload(packet).len()) + .sum::(); for (position, packet) in slot.layout(&unit(clock, 1), 100, &null).chunks(188).enumerate() { let at = slot.pcr as f64 / SYSTEM_CLOCK as f64 + position as f64 * 1504.0 / rate as f64; if let Some(previous) = previous { @@ -1020,7 +1212,7 @@ mod tests { assert!(fill <= 512.001, "PID1 transport buffer holds {fill} bytes"); } } - sent + sent / PAYLOAD } /// Distinct tune-in tables keep their copies and still fit TBsys, which the PAT and PMT @@ -1190,8 +1382,8 @@ mod tests { assert_eq!(checked_transport(&mut schedule, 9), 80); } - /// The clock packet opening every slot rides its PID's transport buffer too, so that PID - /// takes one packet fewer of its own. + /// The PCR opening every slot rides its PID's transport buffer too, in a clock packet or + /// one of the PID's own. #[test] fn the_clock_packet_counts_against_its_pid() { let mut schedule = Schedule::new(Duration::from_millis(100)); @@ -1239,7 +1431,7 @@ mod tests { let (mut tb, mut mb, mut previous, mut sent) = (0.0f64, 0.0f64, None::, 0); while let Some(slot) = schedule.next(None).unwrap() { let mut clock = unit(1, 1); - clock[3] = 0x20; + (clock[3], clock[4]) = (0x20, 183); for (position, packet) in slot.layout(&clock, 100, &null).chunks(188).enumerate() { let at = slot.pcr as f64 / SYSTEM_CLOCK as f64 + position as f64 * 1504.0 / rate as f64; if let Some(previous) = previous { @@ -1250,13 +1442,13 @@ mod tests { if pid(packet) == 1 { tb += 188.0; mb += elementary(packet).len() as f64; - sent += usize::from(packet[3] & 0x10 != 0); + sent += payload(packet).len(); } assert!(tb <= 512.001, "TB holds {tb} bytes at {at}"); assert!(mb <= 16_000.0, "MB holds {mb} bytes at {at}"); } } - assert_eq!(sent, 500 * 652, "every frame goes out"); + assert_eq!(sent, 500 * 652 * PAYLOAD, "every frame goes out"); } /// Audio due before the video takes a position or a few between video packets, and MB @@ -1357,8 +1549,9 @@ mod tests { size: Some(100_000), }, ); - schedule.push(1, timing(1_000), unit(1, 4), false).unwrap(); - schedule.push(1, timing(1_200), unit(1, 4), false).unwrap(); + // 76.8 kb/s passes on five packets in the window, the slots' PCRs among them. + schedule.push(1, timing(1_000), unit(1, 6), false).unwrap(); + schedule.push(1, timing(1_200), unit(1, 6), false).unwrap(); let error = std::iter::from_fn(|| schedule.next(None).transpose()) .find_map(Result::err) .unwrap(); @@ -1389,10 +1582,11 @@ mod tests { assert_eq!(sent, [(32, 8), (36, 2), (41, 6)]); } - /// Packetize a PES with exactly `bytes` elementary bytes and adaptation stuffing. + /// Packetize a PES with exactly `bytes` elementary bytes, each its offset's low byte, and + /// adaptation stuffing. fn audio(bytes: usize) -> Vec { - let mut pes = vec![0; 14 + bytes]; - pes[..9].copy_from_slice(&[0, 0, 1, 0xbd, 0, 0, 0x80, 0x80, 5]); + let mut pes: Vec = (0..14 + bytes).map(|i| i as u8).collect(); + pes[..14].copy_from_slice(&[0, 0, 1, 0xbd, 0, 0, 0x80, 0x80, 5, 0, 0, 0, 0, 0]); let mut out = Vec::new(); for (index, body) in pes.chunks(PAYLOAD).enumerate() { let mut packet = [0xff; TsPacket::SIZE]; @@ -1400,6 +1594,9 @@ mod tests { if body.len() < PAYLOAD { packet[3] |= 0x20; packet[4] = (PAYLOAD - body.len() - 1) as u8; + if packet[4] > 0 { + packet[5] = 0; + } } packet[TsPacket::SIZE - body.len()..].copy_from_slice(body); out.extend(packet); @@ -1658,6 +1855,7 @@ mod tests { units: vec![], caps: HashMap::new(), positions: None, + clocked: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -1696,6 +1894,7 @@ mod tests { units: vec![1, 3], caps: HashMap::new(), positions: None, + clocked: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -1763,6 +1962,7 @@ mod tests { .per_slot(0, true), )]), positions: None, + clocked: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -1817,6 +2017,7 @@ mod tests { units: vec![1, 2], caps: HashMap::from([(1, a.per_slot(0, true)), (2, b.per_slot(0, false))]), positions: None, + clocked: false, }; let mut null = unit(0, 1); (null[1], null[2]) = (0x1f, 0xff); @@ -1836,4 +2037,159 @@ mod tests { assert!(peak <= 512.0, "PID {media} TB peak {peak:.0} B"); } } + + /// The PCR `packet`'s adaptation field carries, in system clock ticks. + fn pcr_of(packet: &[u8]) -> Option { + let field = (packet[3] & 0x20 != 0 && packet[4] >= 7 && packet[5] & 0x10 != 0).then(|| &packet[6..12])?; + let base = u64::from(field[0]) << 25 + | u64::from(field[1]) << 17 + | u64::from(field[2]) << 9 + | u64::from(field[3]) << 1 + | u64::from(field[4]) >> 7; + Some(base * 300 + (u64::from(field[4] & 1) << 8 | u64::from(field[5]))) + } + + /// A PCR moves the PES it rides on by its field's bytes: into the last packet's stuffing, + /// or into one more packet where that has too little. + #[test] + fn a_pcr_reflows_the_pes_it_rides() { + let body = |packets: &[u8]| -> Vec { packets.chunks(TsPacket::SIZE).flat_map(payload).copied().collect() }; + // With its 14-byte header: one packet with room, two with 7 bytes of stuffing, two full. + for (bytes, count) in [(100, 1), (346, 2), (354, 3)] { + let pes = audio(bytes); + let ticks = 27_000_000 * 5 + 299; + let out = with_pcr(&pes, ticks).expect("an exported PES takes a PCR"); + let packets: Vec<&[u8]> = out.chunks(TsPacket::SIZE).collect(); + assert_eq!(packets.len(), count, "{bytes} bytes"); + assert_eq!(out.len(), count * TsPacket::SIZE, "{bytes} bytes"); + assert_eq!(pcr_of(packets[0]), Some(ticks), "{bytes} bytes"); + assert!( + packets[0][1] & 0x40 != 0, + "{bytes} bytes: the first packet still starts the PES" + ); + assert!( + packets[1..] + .iter() + .all(|packet| packet[1] & 0x40 == 0 && pcr_of(packet).is_none()), + "{bytes} bytes: the PCR and the PES start stay in the first packet" + ); + assert_eq!(body(&out), body(&pes), "{bytes} bytes: the PES is unchanged"); + assert!( + with_pcr(&out, ticks).is_none(), + "a PES carrying a PCR already is left alone" + ); + } + } + + /// A slot's PCR rides the clock PID's packet when that PID can go out at once, sparing the + /// slot a clock packet; a slot with only another PID's packets opens with one of its own. + #[test] + fn the_pcr_rides_the_clock_pid_when_it_can() { + let mut schedule = Schedule::new(Duration::from_millis(200)); + schedule.set_rate(Some(RATE)); + schedule.set_buffer(1, VIDEO); + schedule.set_buffer(2, VIDEO); + schedule.set_clock(1); + schedule.push(1, timing(1_000), unit(1, 4), true).unwrap(); + schedule.push(2, timing(2_000), unit(2, 4), false).unwrap(); + let mut clock = unit(1, 1); + (clock[3], clock[4]) = (0x20, 183); + let mut null = unit(0, 1); + (null[1], null[2]) = (0x1f, 0xff); + let (mut sent, mut clocked) = (HashMap::::new(), Vec::new()); + while let Some(slot) = schedule.next(None).unwrap() { + let laid = slot.layout(&clock, 100, &null); + assert_eq!(laid.len(), 40 * TsPacket::SIZE, "slot {} keeps its size", slot.index); + let first = &laid[..TsPacket::SIZE]; + match slot.clocked { + true => { + assert_eq!(pid(first), 1, "slot {}", slot.index); + assert_eq!(pcr_of(first), Some(slot.pcr), "slot {}", slot.index); + } + false => assert_eq!(first, &clock[..], "slot {} opens with a clock packet", slot.index), + } + for packet in laid.chunks(TsPacket::SIZE).skip(usize::from(!slot.clocked)) { + if [1, 2].contains(&pid(packet)) { + *sent.entry(pid(packet)).or_default() += payload(packet).len(); + clocked.push((pid(packet), slot.clocked)); + } + } + } + assert_eq!( + sent, + HashMap::from([(1, 4 * PAYLOAD), (2, 4 * PAYLOAD)]), + "no payload is lost" + ); + assert_eq!( + clocked.first(), + Some(&(1, true)), + "the clock PID's unit carries its slot's PCR" + ); + assert!( + clocked + .iter() + .filter(|&&(pid, _)| pid == 2) + .all(|&(_, clocked)| !clocked), + "another PID's slot has its own clock packet" + ); + } + + /// Each frame of a PES that packs several leaves B at its own decode time, the PES header + /// with the first, so the next PES goes in once the first frame is removed rather than the + /// last. + #[test] + fn packed_frames_leave_the_decoder_buffer_one_by_one() { + let mut schedule = Schedule::new(Duration::from_millis(200)); + schedule.set_rate(Some(RATE)); + // The packed PES's 1,514 bytes leave too little for the next one's 414 until its first + // frame's 514 leave. + schedule.set_buffer( + 1, + Buffer { + rate: 2_000_000, + leak: None, + size: Some(1_600), + }, + ); + let frames = vec![(ms(1_000), 500), (ms(1_024), 500), (ms(1_048), 500)]; + schedule + .push_frames( + 1, + Timing { + by: ms(999), + decode: ms(1_000), + }, + audio(1_500), + false, + frames, + ) + .unwrap(); + schedule + .push( + 1, + Timing { + by: ms(1_099), + decode: ms(1_100), + }, + audio(400), + false, + ) + .unwrap(); + let mut null = unit(0, 1); + (null[1], null[2]) = (0x1f, 0xff); + let mut starts = Vec::new(); + while let Some(slot) = schedule.next(None).unwrap() { + for (position, packet) in slot.layout(&unit(9, 1), 100, &null).chunks(TsPacket::SIZE).enumerate() { + if pid(packet) == 1 && packet[1] & 0x40 != 0 { + starts.push(slot.pcr as f64 / SYSTEM_CLOCK as f64 + position as f64 * 1504.0 / RATE as f64); + } + } + } + assert_eq!(starts.len(), 2); + assert!( + starts[1] >= 1.000 && starts[1] < 1.024, + "the next PES starts at {}", + starts[1] + ); + } } diff --git a/test/ts/README.md b/test/ts/README.md index 07e909fd0bd..d54866c96b6 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -13,14 +13,15 @@ puts a PCR on its own packet every 25 ms of media time, so several broadcast-shape checks are expected to flag. The report quantifies exactly where and by how much. -Four instruments live here. `compliance.py` (via `run.sh`) grades a captured file +Five instruments live here. `compliance.py` (via `run.sh`) grades a captured file against the IRD model. [`pcr-timing.py`](#pcr-timing-pcr-timingpy) grades a live pipe, which is the only way to see *when* the exporter released each PCR. [`table-anchor.py`](#table-anchoring-table-anchorpy) grades two exporters of one broadcast against each other, which is the only way to see whether a table's emission points come from the media or from the exporter's own clock. [`open-gop.py`](#open-gop-open-goppy) grades an open-GOP capture against its -source, access unit by access unit. +source, access unit by access unit. [`overhead.py`](#mux-overhead-overheadpy) +counts the packets a capture spends on clocks and audio framing against its source. ## Running @@ -36,6 +37,7 @@ just test ts --open-gop # open-GOP clip; its leading pictures must survi just test ts --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB just test ts --headroom # the same with four audio PIDs, muxed above the video's Rx just test ts --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak +just test ts --full # the same with two audio PIDs, filling its multiplex with no nulls just test ts --delay 1s # pass the exporter's --delay ``` @@ -466,6 +468,22 @@ rebase. | `recovery-point-sei` | shape | each recovery point keeps its SEI | | `dts-before-pts` | shape | no access unit is stamped to decode after it presents | +## Mux overhead (`overhead.py`) + +A source that fills its multiplex with no null packets leaves the export no room +to spend more than it does on PES headers, part-filled audio packets, and PCRs +outside media packets (moq-dev/moq#5362). `--full` exports at 900 ms, above its +source's lead of about 0.65 s, which absorbs a small excess for a while, so it also +runs `overhead.py` on the source and capture before the compliance report. + +```bash +./overhead.py source.ts capture.ts +``` + +It counts the packets on each audio PID and the PCR-only packets on each PID, +scaling the capture's counts by how much of the source's elementary stream on the +PID it carries, and fails when their total exceeds the source's. + ## EIT fixtures No capture in this repository carries EIT (PID 0x0012), so nothing exercises the @@ -564,8 +582,9 @@ exporter re-emits SI on its own repetition cadence rather than the source's. ## CI `.github/workflows/interop.yml` runs `just test ts`, `just test ts --bitrate -2000000`, `just test ts --hrd`, `just test ts --leak`, `just test ts --headroom`, -`just test ts --open-gop`, `just test ts-eit`, and `just test ts-tstd` after the +2000000`, `just test ts --hrd`, `just test ts --leak`, `just test ts --full`, +`just test ts --headroom`, `just test ts --open-gop`, `just test ts-eit`, and +`just test ts-tstd` after the interop matrix (nightly, on demand, and on PRs touching `test/ts/`). `ts-eit` is `eit-roundtrip.sh`: it builds the sparse-schedule and pending-version fixtures from a generated clip, round-trips them through a diff --git a/test/ts/overhead.py b/test/ts/overhead.py new file mode 100755 index 00000000000..e85f3fbd613 --- /dev/null +++ b/test/ts/overhead.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Grade whether a round-trip spends no more packets than its source on clocks and audio. + + ./overhead.py source.ts capture.ts + +A source that fills its multiplex with no null packets leaves the export no room to spend +more than it does. Each audio PES costs a header and a part-filled last packet, and a PCR +outside a media packet costs a packet of its own (moq-dev/moq#5362). + +Both are counted on each stream: the packets on every PES PID that is not video (stream ids +0xC0-0xDF, and private stream 1, which DVB AC-3 uses), and the packets on any PID that carry +a PCR and no payload. The capture joins after the source starts, so each PID's count is scaled +by how much of the source's elementary stream on that PID the capture carries. + +Exit status is 0 when the capture spends no more than the source, 1 otherwise. +""" + +import sys +from collections import defaultdict + +PKT = 188 +SYNC = 0x47 +NULL_PID = 0x1FFF + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + sys.exit(1) + + +def count(path: str) -> tuple[dict[int, int], dict[int, int], dict[int, int], set[int]]: + """Per PID: its packets, its elementary bytes, and its packets carrying only a PCR; and + the PIDs whose PES are audio.""" + packets: dict[int, int] = defaultdict(int) + elementary: dict[int, int] = defaultdict(int) + clocks: dict[int, int] = defaultdict(int) + audio: set[int] = set() + with open(path, "rb") as f: + data = f.read() + if not data or len(data) % PKT: + fail(f"{path} is not a whole number of {PKT}-byte packets") + for offset in range(0, len(data), PKT): + packet = data[offset : offset + PKT] + if packet[0] != SYNC: + fail(f"{path} loses sync at byte {offset}") + pid = (packet[1] & 0x1F) << 8 | packet[2] + if pid == NULL_PID: + continue + packets[pid] += 1 + control = packet[3] >> 4 & 3 + field = 1 + packet[4] if control & 2 else 0 + if control & 2 and packet[4] and packet[5] & 0x10 and not control & 1: + clocks[pid] += 1 + if not control & 1: + continue + body = packet[4 + field :] + if packet[1] & 0x40 and body[:3] == b"\0\0\1" and len(body) >= 9: + if 0xC0 <= body[3] <= 0xDF or body[3] == 0xBD: + audio.add(pid) + body = body[9 + body[8] :] + elementary[pid] += len(body) + return packets, elementary, clocks, audio + + +def main() -> int: + if len(sys.argv) != 3: + fail("usage: overhead.py source.ts capture.ts") + source, capture = count(sys.argv[1]), count(sys.argv[2]) + rows = [(pid, "audio", source[0][pid], capture[0][pid]) for pid in sorted(source[3] | capture[3])] + rows += [(pid, "PCR only", source[2][pid], capture[2][pid]) for pid in sorted(source[2].keys() | capture[2].keys())] + spent = [0.0, 0.0] + print("### packets spent on clocks and audio (source -> capture, scaled to the source)") + for pid, kind, before, after in rows: + if not source[1][pid] or not capture[1][pid]: + fail(f"PID {pid} carries no elementary stream in both the source and the capture") + share = capture[1][pid] / source[1][pid] + print(f" PID {pid:<5} {kind:<9} {before:>8} {after / share:>10.0f} ({after} for {share:.1%} of it)") + spent[0] += before + spent[1] += after / share + verdict = "PASS" if spent[1] <= spent[0] else "FAIL" + print(f" {verdict} {spent[1]:.0f} packets against the source's {spent[0]:.0f}") + return 0 if verdict == "PASS" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/ts/run.sh b/test/ts/run.sh index d8b8dca52a7..998e2cbc680 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -22,6 +22,7 @@ # ./run.sh --hrd # 1080p video filling a broadcast-sized 9 Mbit CPB # ./run.sh --headroom # the same with four audio PIDs, muxed above the video's Rx # ./run.sh --leak # 1080p video with no HRD, filling its level's 24 Mb/s leak +# ./run.sh --full # the same with two audio PIDs, filling its multiplex with no nulls # ./run.sh --delay 1s # pass the exporter's --delay # `--live` swaps the analyzer, not the rig. compliance.py grades a captured file @@ -69,6 +70,7 @@ OPEN_GOP="" # publish open GOP with leading pictures, and grade them through the HRD="" # publish video that fills a broadcast-sized CPB, as a contribution encoder does HEADROOM="" # the same plus four audio PIDs, in a multiplex faster than the video's Rx LEAK="" # publish video with no HRD at its level's leak, as a full-rate feed does +FULL="" # the same with audio, in a multiplex the source fills with no nulls EXPORT=() # extra `export ts` flags (--delay) # How far into the run the second subscriber joins. A late join is the point: two # exporters started together can share a cadence by starting together, which is @@ -145,6 +147,10 @@ while [[ $# -gt 0 ]]; do LEAK=1 shift ;; + --full) + FULL=1 + shift + ;; --delay) EXPORT=(--delay "$2") shift 2 @@ -194,6 +200,20 @@ fi # few percent below the leak exhausts the delay's lead and misses a deadline. [[ -n "$LEAK" && -z "$BITRATE_SET" && -z "${TSC_BITRATE:-}" ]] && BITRATE=26000000 [[ -n "$LEAK" && -z "$DURATION_SET" && -z "${TSC_DURATION:-}" ]] && DURATION=30 +if [[ -n "$FULL" && -n "$LEAK$HRD$HEADROOM$OPEN_GOP$SOURCE$ANALYZE_ONLY" ]]; then + echo "error: --full generates its own clip and cannot be combined with --leak, --hrd, --headroom, --open-gop, --source, or --analyze-only" >&2 + exit 1 +fi +# The clip's rate is whatever its packets fill, measured once it is generated. +if [[ -n "$FULL" && -n "$BITRATE_SET${TSC_BITRATE:-}" ]]; then + echo "error: --full runs at the rate its clip fills and takes no --bitrate" >&2 + exit 1 +fi +[[ -n "$FULL" && -z "$DURATION_SET" && -z "${TSC_DURATION:-}" ]] && DURATION=30 +# Its video arrives up to about 0.65 s ahead of decoding with no room to catch up, so the +# export needs a longer lead than the default delay gives, yet one a unit's window and +# slot keep under the T-STD's 1 s. +[[ -n "$FULL" && ${#EXPORT[@]} -eq 0 ]] && EXPORT=(--delay 900ms) if [[ -n "$OPEN_GOP" && -n "$ANALYZE_ONLY$LIVE$PAIR" ]]; then echo "error: --open-gop cannot be combined with --analyze-only, --live, or --pair" >&2 exit 1 @@ -319,6 +339,39 @@ elif [[ -n "$LEAK" ]]; then -c:v libx264 -profile:v high -level 4.0 -preset veryfast -pix_fmt yuv420p \ -x264-params "keyint=25:min-keyint=25:scenecut=0" -b:v 30M -maxrate 24M -bufsize 24M \ -f mpegts -muxrate "$BITRATE" -pcr_period 20 -pes_payload_size 0 "$SRC_TS" +elif [[ -n "$FULL" ]]; then + echo "### generating ~${DURATION}s 1080p clip at its level's leak with two audio PIDs, filling its multiplex with no nulls" + # A full-rate contribution feed's shape: the --leak video beside an MP2 and a DVB AC-3 + # PID, with its PCR in the video's adaptation field and several audio frames a PES, and + # no null packets, so the export has no room to spend more packets than the source on + # clocks or audio framing (moq-dev/moq#5362). ffmpeg pads any multiplex rate it is + # given, so mux with room to spare, drop the nulls, and stamp the clock at the rate the + # rest fill. The PTS and DTS stay: moving them with each PCR would move B pictures + # against their group's keyframe, and the video goes out far enough ahead of its DTS + # for its packets to arrive a little earlier or later. Muxed at most 0.5 s ahead, it + # arrives up to about 0.65 s ahead once the nulls are gone. + PADDED_TS="$HARNESS_RUN/padded.ts" + PADDED_RATE=27000000 + ffmpeg -y -hide_banner -loglevel error \ + -f lavfi -i "testsrc2=size=1920x1080:rate=25,noise=alls=24:allf=t" \ + -f lavfi -i "sine=frequency=440:sample_rate=48000" \ + -f lavfi -i "sine=frequency=880:sample_rate=48000" \ + -t "$DURATION" -map 0:v -map 1:a -map 2:a -ac 2 \ + -c:v libx264 -profile:v high -level 4.0 -preset veryfast -pix_fmt yuv420p \ + -x264-params "keyint=25:min-keyint=25:scenecut=0" -b:v 30M -maxrate 24M -bufsize 24M \ + -c:a:0 mp2 -b:a:0 192k -c:a:1 ac3 -b:a:1 192k \ + -f mpegts -mpegts_flags system_b -muxrate "$PADDED_RATE" -pcr_period 20 -max_delay 500000 "$PADDED_TS" + PACKETS=$(($(wc -c <"$PADDED_TS") / 188)) + NULLS=$(tsp -I file "$PADDED_TS" -P count --pid 0x1FFF --total -O drop 2>&1 | + sed -n 's/.*counted \([0-9,]*\) packets.*/\1/p' | tr -d ,) + [[ -n "$NULLS" ]] || { + echo "error: could not count the generated clip's null packets" >&2 + exit 1 + } + BITRATE=$((PADDED_RATE * (PACKETS - NULLS) / PACKETS)) + tsp -I file "$PADDED_TS" -P filter --negate --pid 0x1FFF \ + -P pcradjust --bitrate "$BITRATE" --ignore-pts --ignore-dts -O file "$SRC_TS" + echo " ${NULLS} null packets dropped of ${PACKETS}: ${BITRATE} b/s" else echo "### generating ~${DURATION}s broadcast-like ${OPEN_GOP:+open-GOP }clip with ffmpeg" # CBR with a 20 ms PCR, like a contribution feed. Not cosmetic: `regulate` @@ -589,6 +642,18 @@ if [[ -n "$OPEN_GOP" ]]; then exit 1 fi fi + +# The delay's lead over the source's hides packets the export spends beyond the source's +# until it runs out, so they are counted outright. +if [[ -n "$FULL" ]]; then + echo + if ! python3 "$DIR/overhead.py" "$SRC_TS" "$SUB_TS"; then + echo >&2 + echo "error: the export spends more packets than its source on clocks and audio" >&2 + dump_logs + exit 1 + fi +fi echo # Pass the source so duration-fidelity can pin the exported stream's rate. A tiny # capture still parses, so the round-trip can fail here with a non-empty file;