diff --git a/Cargo.lock b/Cargo.lock index ee815167cb4..8f11ed1f516 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5008,6 +5008,7 @@ dependencies = [ "criterion", "futures", "h264-parser", + "h264-reader 0.9.0", "hang", "kio 0.6.2", "memchr", diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 961ce45d632..4c0718f4d27 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -93,7 +93,11 @@ or use [`import srt`](/bin/srt). MPEG-TS import carries H.264/H.265 and AAC/MP2/AC-3/E-AC-3/Opus, and passes SCTE-35 and every other stream it does not decode (KLV, subtitles, DVB AC-3) -through as tracks. Each such PES goes through whole, at its place on the program clock +through as tracks. Video AUs can span PES, and a PES can contain several. Supply +a PTS for each AU or declare a proportional POC clock: AVC needs +`fixed_frame_rate_flag` and `temporal_poc_flag` in the TS timing descriptor plus +fixed-rate SPS VUI; HEVC needs VUI's `vui_poc_proportional_to_timing_flag`. Missing timing +ends the import instead of repeating a timestamp. Private and undecoded PES go through whole, at their place on the program clock (PCR); one before the first PCR is dropped. A stream it cannot carry unaltered is refused with a warning: any in a program with no PCR, and one whose PES carry an ESCR, pack header or TREF, are scrambled, leave a non-video PES unbounded, diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index 50f07919b6b..0952ea7bad8 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -62,7 +62,7 @@ refused means an error or a counted drop. | Mechanism | Contract | Owner | |:--|:--|:--| -| H.264 (0x1B), H.265 (0x24) | Preserved: video rendition, PID and ES descriptors in `mpegts.tracks`; DTS normalized from decode order. Today one access unit per PES | [access units](/quest/m1/ts-reliability/pes-access-units.md) | +| H.264 (0x1B), H.265 (0x24) | Preserved: video rendition, PID and ES descriptors in `mpegts.tracks`; AUs cross PES boundaries with source or explicitly proportional POC-derived PTS; DTS normalized from decode order | landed | | AAC ADTS (0x0F), MP2, AC-3, E-AC-3, Opus (0x06 + `Opus`) | Preserved: audio rendition; Opus export labels mono and stereo, and 3 to 8 channels with an OpusHead in the Vorbis layout. Other mapping families (such as 255) refuse | landed | | Private and unknown PES types, LATM (0x11), metadata (0x15) | Preserved: `pesPacket`, PTS and DTS rebased | landed | | PES clock fields (ESCR, pack header, TREF), PES scrambling | Refused, counted in `refused` | landed | @@ -99,7 +99,6 @@ catalog linger (#5305). ## Required - [State bound](/quest/m1/ts-reliability/state-bound.md) - SI and revision state retire with what the source stops repeating -- [Access units](/quest/m1/ts-reliability/pes-access-units.md) - a PES carrying several access units publishes each at its own time - [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a revised PAT never leaves a broadcast meaning another program - [Stream routes](/quest/m1/ts-reliability/stream-routes.md) - every PID gets a declared route, and what is not carried is counted diff --git a/quest/m1/ts-reliability/pes-access-units.md b/quest/m1/ts-reliability/pes-access-units.md deleted file mode 100644 index f8cd4cd42ba..00000000000 --- a/quest/m1/ts-reliability/pes-access-units.md +++ /dev/null @@ -1,32 +0,0 @@ -# [M] TS video import assembles access units across PES boundaries - -## Goal - -An H.264 or H.265 PES carrying several access units publishes each as its -own frame at its own presentation time, and an access unit split across PES -packets publishes once. A stream whose later access units' timing cannot be -derived is refused loudly, never published with duplicate timestamps. - -## Plan - -Today `Import` flushes the codec splitter after every video PES ("Each PES -is one access unit", `rs/moq-mux/src/container/ts/import.rs`), so two access -units in one PES publish two frames at the first one's PTS (reproduced in -the 2026-10-22 audit with the existing fixture). - -Decided 2026-10-22 by the maintainer: - -- Keep one elementary-stream assembler per PID across PES boundaries; a PES - PTS stamps the access unit that starts in it. -- A later access unit without a PTS of its own derives it from the POC and - the SPS's VUI timing when the SPS declares both; otherwise the stream is - refused. Never repeat the previous timestamp. - -Extend the T-STD harness's video profile beyond one access unit per PES. - -Tests: the same elementary stream repartitioned into one access unit per -PES, several per PES, and access units split across PES, through import and -export, each publishing identical frames and timestamps; a multi-unit PES -without VUI timing is refused. - -Public API: none. Wire: none. diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index ae1d50866fc..02e306534df 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -18,6 +18,7 @@ base64 = { workspace = true } bytes = { workspace = true } crc = "3" h264-parser = { version = "0.4.2" } +h264-reader = "0.9.0" hang = { workspace = true } kio = { workspace = true } memchr = "2" @@ -77,5 +78,9 @@ harness = false name = "containers" harness = false +[[bench]] +name = "ts_video" +harness = false + [lints] workspace = true diff --git a/rs/moq-mux/benches/ts_video.rs b/rs/moq-mux/benches/ts_video.rs new file mode 100644 index 00000000000..72b89c78fd0 --- /dev/null +++ b/rs/moq-mux/benches/ts_video.rs @@ -0,0 +1,93 @@ +//! Video assembly across access-unit size and PES fragmentation. +use bytes::Bytes; +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use mpeg2ts::{ + pes::{PesPacketReader, ReadPesPacket}, + ts::TsPacketReader, +}; +use std::{io::Cursor, time::Duration}; + +fn input(size: usize, payload: usize) -> Vec { + let fixture = include_bytes!("../src/container/ts/test_data/poc-avc.ts"); + let mut reader = PesPacketReader::new(TsPacketReader::new(Cursor::new(fixture))); + // A real SPS/PPS/IDR exercises the timing parser rather than its explicit-PTS + // fallback for unsupported syntax. Padding stays in the final slice NAL. + let mut au = reader.read_pes_packet().unwrap().unwrap().data; + assert!(au.len() <= size); + au.resize(size, 0xff); + // The fixture's SDT/PAT/PMT declare one AVC stream on PID 0x100. + let mut out = include_bytes!("../src/container/ts/test_data/poc-avc.ts")[..188 * 3].to_vec(); + let mut cc = 0; + for (n, data) in au.chunks(payload).enumerate() { + let len = data.len() + if n == 0 { 8 } else { 3 }; + let mut pes = vec![ + 0, + 0, + 1, + 0xe0, + (len >> 8) as u8, + len as u8, + 0x80, + if n == 0 { 0x80 } else { 0 }, + if n == 0 { 5 } else { 0 }, + ]; + if n == 0 { + pes.extend([0x21, 0, 1, 0, 1]); + } + pes.extend(data); + for (packet, chunk) in pes.chunks(184).enumerate() { + let mut ts = vec![ + 0x47, + if packet == 0 { 0x41 } else { 1 }, + 0, + if chunk.len() == 184 { 0x10 | cc } else { 0x30 | cc }, + ]; + cc = (cc + 1) & 15; + if chunk.len() < 184 { + let field = 183 - chunk.len(); + ts.push(field as u8); + if field > 0 { + ts.push(0); + ts.resize(5 + field, 0xff); + } + } + ts.extend(chunk); + out.extend(ts); + } + } + out +} + +fn assembly(c: &mut Criterion) { + let mut group = c.benchmark_group("ts_video"); + group.measurement_time(Duration::from_secs(1)); + group.warm_up_time(Duration::from_millis(200)); + group.sample_size(10); + for size in [4_096, 65_536, 1_048_576] { + group.throughput(Throughput::Bytes(size as u64)); + for payload in [64, 1_024, 65_000] { + let data = Bytes::from(input(size, payload)); + group.bench_function(BenchmarkId::new(size.to_string(), payload), |b| { + b.iter_batched( + || { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + ( + moq_mux::container::ts::Import::new(broadcast, catalog.reserve()), + catalog, + ) + }, + |(mut import, _catalog)| { + import.decode(&data).unwrap(); + import.finish().unwrap(); + std::hint::black_box(import); + }, + BatchSize::SmallInput, + ) + }); + } + } + group.finish(); +} +criterion_group!(benches, assembly); +criterion_main!(benches); diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs index 79768a07cf2..adfad6b1342 100644 --- a/rs/moq-mux/src/codec/h264/split.rs +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -48,6 +48,12 @@ pub(crate) struct Params { pps: Vec, } +impl Params { + pub(crate) fn nals(&self) -> impl Iterator { + self.sps.iter().chain(&self.pps) + } +} + #[derive(Default)] struct Avc3Frame { chunks: BytesMut, diff --git a/rs/moq-mux/src/codec/h265/split.rs b/rs/moq-mux/src/codec/h265/split.rs index dad657f8d2e..83a84750494 100644 --- a/rs/moq-mux/src/codec/h265/split.rs +++ b/rs/moq-mux/src/codec/h265/split.rs @@ -47,6 +47,12 @@ pub(crate) struct Params { pps: Vec, } +impl Params { + pub(crate) fn nals(&self) -> impl Iterator { + self.vps.iter().chain(&self.sps).chain(&self.pps) + } +} + #[derive(Default)] struct Au { chunks: BytesMut, diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 85960ce6019..1a6a85dce64 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -36,6 +36,9 @@ use crate::catalog::hang::CatalogExt; use crate::codec::{aac, ac3, eac3, h264, h265, legacy, mp2, opus}; use moq_net::Timestamp; +#[path = "video.rs"] +mod video; + /// Demuxes an MPEG-TS byte stream into a MoQ broadcast. /// /// Supports H.264 (stream type 0x1B), H.265 (0x24), ADTS AAC (0x0F), MP2 @@ -43,9 +46,16 @@ use moq_net::Timestamp; /// ADTS-framed and is dropped. Each codec stream is fed to its importer, which /// manages the track, catalog config, and keyframe-based group boundaries. /// -/// Partial header contents hold at most 264 bytes per PID. PES payloads and sections awaiting a -/// timestamp share half a MoQ group's byte budget; held sections also share half its frame -/// budget. Exceeding either limit refuses the input instead of retaining more bytes. +/// Video access units can span PES packets, and one PES can contain several. Its PTS +/// stamps the AU whose required start-code prefix begins there, once. Later AUs need +/// anchored POC and proportional timing: AVC's TS timing descriptor plus fixed-rate +/// SPS VUI, or HEVC's `poc_proportional_to_timing_flag`. Missing timing ends the import; +/// it is not recoverable packet damage. Raw elementary-stream import keeps its +/// caller-supplied timestamp mode. +/// +/// Partial header contents hold at most 264 bytes per PID. PES payloads, retained video +/// AUs, and sections awaiting a timestamp share half a MoQ group's byte budget; +/// buffered units also share half its frame budget. Exceeding either limit refuses the input instead of retaining more bytes. /// /// Elementary streams we don't decode are carried verbatim, one MoQ track per /// PID, when the catalog `E` carries the [`mpegts`](super::Mpegts) section: section-framed @@ -764,6 +774,15 @@ impl Import { } fn ensure_stream(&mut self, pid: Pid, stream_type: u8, descriptors: &[catalog::Descriptor]) -> anyhow::Result<()> { + let temporal_poc = if stream_type == StreamType::H264 as u8 { + video::temporal_poc(descriptors)? + } else { + false + }; + if let Some(Stream::H264 { assembly, .. }) = self.streams.get_mut(&pid) { + assembly.clock.temporal_poc = temporal_poc; + } + // A later PMT can remap a PID that was section-framed (intercepted in // `decode`) to a PES codec/verbatim stream. Drop the stale section route first, // or it would keep intercepting the PID and the new stream would never get data. @@ -803,6 +822,11 @@ impl Import { .unique_track(".avc3", self.catalog.track_info(hang::catalog::PRIORITY.video))?; Stream::H264 { split: h264::Split::new(), + assembly: { + let mut assembly = video::Assembly::new(video::Codec::Avc); + assembly.clock.temporal_poc = temporal_poc; + assembly + }, import: Box::new(h264::Import::new(track, self.reserve(), self.video_hint())?), } } @@ -812,6 +836,7 @@ impl Import { .unique_track(".hev1", self.catalog.track_info(hang::catalog::PRIORITY.video))?; Stream::H265 { split: h265::Split::new(), + assembly: video::Assembly::new(video::Codec::Hevc), import: Box::new(h265::Import::new(track, self.reserve(), self.video_hint())?), } } @@ -1052,17 +1077,20 @@ impl Import { self.keep_stats(pid, Some(stats::Stream::new(".ts", stats::Class::Data))); } self.unclocked.remove(&pid); - if let Some(stream) = self.streams.get_mut(&pid) { - if let Stream::PesPacket(stream) = &mut *stream { - stream.settle(&self.placement); - let (published, dropped) = stream.release(&self.timebase, &self.pts, generation)?; - units += published; - unplaced += dropped; - anyhow::ensure!( - stream.held.is_empty(), - "TS private PES reached a PMT revision without a media timestamp" - ); - } + if let Some(stream) = self.streams.get_mut(&pid) + && let Stream::PesPacket(stream) = &mut *stream + { + stream.settle(&self.placement); + let (published, dropped) = stream.release(&self.timebase, &self.pts, generation)?; + units += published; + unplaced += dropped; + anyhow::ensure!( + stream.held.is_empty(), + "TS private PES reached a PMT revision without a media timestamp" + ); + } + units += self.finish_stream(pid)?; + if let Some(stream) = self.streams.get(&pid) { // A verbatim entry leaves with its stream; a decoded track's is recorded here. if self.recorded_media.remove(&pid) && let Some(name) = stream.media_track_name() @@ -1070,7 +1098,6 @@ impl Import { { mpegts.tracks.remove(&name); } - units += stream.finish()?; } if let Some(stream) = self.streams.remove(&pid) { // Finishing can publish a held frame, which its counters then include. @@ -1222,9 +1249,35 @@ impl Import { Ok(()) } + fn video_damage(&mut self, pid: Pid) { + let damage = match self.streams.get_mut(&pid) { + Some(Stream::H264 { assembly, .. } | Stream::H265 { assembly, .. }) => assembly.take_damage(), + _ => None, + }; + if let Some((count, err)) = damage { + *self.damaged.entry(pid.as_u16()).or_default() += count; + self.damage_log.record(pid.as_u16(), &err); + } + } + /// Refuse one damaged unit without changing any other PID or the program clock. fn damage(&mut self, pid: Pid, err: &anyhow::Error) -> anyhow::Result<()> { - self.pending.remove(&pid); + // Complete AUs ahead of the damaged packet remain valid, including those + // in an unbounded PES. Its final NAL/AU is never flushed at a loss boundary. + if let Some(pending) = self.pending.remove(&pid) + && let Some(stream @ (Stream::H264 { .. } | Stream::H265 { .. })) = self.streams.get_mut(&pid) + { + let result = stream.write(pending, false); + self.video_damage(pid); + let units = match result { + Ok(units) => units, + Err(err) if err.is::() => video::published(&err), + Err(err) => return Err(err), + }; + self.published |= units > 0; + self.liveness.delivered(pid.as_u16(), units); + self.record_media_track(pid)?; + } self.headers.remove(&pid); match self.streams.get_mut(&pid) { // Nothing is imported from it, so there is nothing to refuse or report. @@ -1264,13 +1317,24 @@ impl Import { if pending.pts.is_some() { self.initial_reservation = None; } - let units = match stream.write(pending, batched) { - Ok(units) => units, - Err(err) if err.is::() => return self.damage(pid, &err), + let result = stream.write(pending, batched); + self.video_damage(pid); + let (units, damage) = match result { + Ok(units) => (units, None), + Err(err) if err.is::() => (video::published(&err), Some(err)), Err(err) => return Err(err), }; - if truncated { - stream.lost()?; + if let Some(err) = damage { + self.damage(pid, &err)?; + } else if truncated { + if matches!(self.streams.get(&pid), Some(Stream::H264 { .. } | Stream::H265 { .. })) { + self.damage( + pid, + &anyhow::anyhow!("PES payload truncated before its declared length"), + )?; + } else { + self.streams.get_mut(&pid).expect("existing route").lost()?; + } } self.published |= units > 0; self.liveness.delivered(pid.as_u16(), units); @@ -1605,6 +1669,27 @@ impl Import { Ok(()) } + fn finish_stream(&mut self, pid: Pid) -> anyhow::Result { + let Some(stream) = self.streams.get_mut(&pid) else { + return Ok(0); + }; + let result = stream.finish(); + self.video_damage(pid); + match result { + Ok(units) => Ok(units), + Err(err) if err.is::() => { + let units = video::published(&err); + self.damage(pid, &err)?; + self.streams + .get_mut(&pid) + .expect("damage preserves the route") + .finish()?; + Ok(units) + } + Err(err) => Err(err), + } + } + /// Flush any buffered PES and finish every track. pub fn finish(&mut self) -> anyhow::Result<()> { self.demux.finish(); @@ -1627,9 +1712,11 @@ impl Import { // EOF releases an unmeasured short source as well as its decoder reservations. self.mux_rate_reservation = None; self.initial_reservation = None; - for (pid, stream) in &mut self.streams { - let units = stream.finish()?; + for pid in self.streams.keys().copied().collect::>() { + let units = self.finish_stream(pid)?; + self.published |= units > 0; self.liveness.delivered(pid.as_u16(), units); + self.record_media_track(pid)?; } for section in self.sections.values_mut() { section.finish()?; @@ -1799,7 +1886,7 @@ impl Retained { if !reserve(&self.staging.units, units, STAGING_UNITS_MAX) { self.staging.bytes.fetch_sub(bytes, Ordering::Relaxed); anyhow::bail!( - "TS staging exceeds {STAGING_UNITS_MAX} sections and whole PES while waiting for a timestamp" + "TS staging exceeds {STAGING_UNITS_MAX} buffered units while waiting for a boundary or timestamp" ); } self.bytes += bytes; @@ -1807,6 +1894,25 @@ impl Retained { Ok(()) } + fn resize(&mut self, bytes: usize, units: usize) -> anyhow::Result<()> { + self.grow(bytes.saturating_sub(self.bytes), units.saturating_sub(self.units))?; + self.staging.bytes.fetch_sub(self.bytes - bytes, Ordering::Relaxed); + self.staging.units.fetch_sub(self.units - units, Ordering::Relaxed); + self.bytes = bytes; + self.units = units; + Ok(()) + } + + fn take(&mut self, bytes: usize, units: usize) -> Self { + self.bytes -= bytes; + self.units -= units; + Self { + staging: self.staging.clone(), + bytes, + units, + } + } + fn clear(&mut self) { self.staging.bytes.fetch_sub(self.bytes, Ordering::Relaxed); self.staging.units.fetch_sub(self.units, Ordering::Relaxed); @@ -2829,10 +2935,12 @@ fn unit_error(err: crate::Error) -> anyhow::Error { enum Stream { H264 { split: h264::Split, + assembly: video::Assembly, import: Box, }, H265 { split: h265::Split, + assembly: video::Assembly, import: Box, }, Aac(Box>), @@ -2850,58 +2958,21 @@ impl Stream { /// Route one reassembled PES, returning how many access units it published. fn write(&mut self, pending: Pending, batched: bool) -> anyhow::Result { match self { - Stream::H264 { split, import } => { - let reorder = reorder_delay(pending.pts, pending.dts); - let pts = on_clock(pending.pts, pending.offset)?; - let params = split.params(); - // Each PES is one access unit, so flush to emit it immediately. - let published = (|| { - let mut frames = split.decode(&pending.data, pts).map_err(unit_error)?; - frames.extend(split.flush(pts).map_err(unit_error)?); - let mut published = 0; - for frame in frames { - published += u64::from(skip_missing_keyframe(import.decode([frame]))?); - } - anyhow::Ok(published) - })(); - // A refused unit must not leave its parameter sets for a bare keyframe to re-inject. - // The snapshot is per PES, so if one carries several AUs, a later damaged AU also rolls - // back new parameter sets an earlier, already published AU brought. - if published.as_ref().is_err_and(|err| err.is::()) { - split.restore(params); - } - let published = published?; - // After decode, so the track (and its catalog rendition) exists. - if let Some(reorder) = reorder { - import.observe_reorder(reorder)?; - } - Ok(published) - } - Stream::H265 { split, import } => { - let reorder = reorder_delay(pending.pts, pending.dts); - let pts = on_clock(pending.pts, pending.offset)?; - let params = split.params(); - // Each PES is one access unit, so flush to emit it immediately. - let published = (|| { - let mut frames = split.decode(&pending.data, pts).map_err(unit_error)?; - frames.extend(split.flush(pts).map_err(unit_error)?); - let mut published = 0; - for frame in frames { - published += u64::from(skip_missing_keyframe(import.decode([frame]))?); - } - anyhow::Ok(published) - })(); - // A refused unit must not leave its parameter sets for a bare keyframe to re-inject. - // The snapshot is per PES, so if one carries several AUs, a later damaged AU also rolls - // back new parameter sets an earlier, already published AU brought. - if published.as_ref().is_err_and(|err| err.is::()) { - split.restore(params); - } - let published = published?; - if let Some(reorder) = reorder { - import.observe_reorder(reorder)?; - } - Ok(published) + Stream::H264 { + split, + assembly, + import, + } => { + assembly.push(pending)?; + video::avc(split, import, assembly, false) + } + Stream::H265 { + split, + assembly, + import, + } => { + assembly.push(pending)?; + video::hevc(split, import, assembly, false) } Stream::Aac(stream) => stream.write(pending, batched), Stream::Opus(stream) => stream.write(pending), @@ -2925,19 +2996,18 @@ impl Stream { /// Whether a PES cut short by a break is still worth publishing. /// - /// True where one PES carries many independently decodable units, so the ones ahead of - /// the cut are whole and correct on their own. False where it carries exactly one: half - /// an access unit is a picture with missing slices, and half a keyframe stays wrong for - /// every picture that references it. Verbatim payloads are all-or-nothing the same way. + /// True where codec framing proves complete units ahead of the cut. A video + /// assembler publishes only pictures closed by a following codec boundary; + /// its unconfirmed tail is discarded by `lost`. Verbatim PES are all-or-nothing. fn salvages_partial_pes(&self) -> bool { match self { - Stream::Aac(_) | Stream::Legacy(_) => true, + Stream::Aac(_) | Stream::Legacy(_) | Stream::H264 { .. } | Stream::H265 { .. } => true, // Opus carries many packets per PES like the audio above, but its framing is // declared rather than self-describing: a trailing packet cut short of the length // its control header promises is a parse error, and that error would travel up out // of `decode` and end the session. Losing the PES beats losing the broadcast. Stream::Opus(_) => false, - Stream::H264 { .. } | Stream::H265 { .. } | Stream::PesPacket(_) => false, + Stream::PesPacket(_) => false, Stream::Clock | Stream::Ignored => false, } } @@ -2947,8 +3017,14 @@ impl Stream { /// break the stream recovers from in place. fn desync(&mut self) { match self { - Stream::H264 { split, .. } => split.reset(), - Stream::H265 { split, .. } => split.reset(), + Stream::H264 { split, assembly, .. } => { + split.reset(); + assembly.reset(); + } + Stream::H265 { split, assembly, .. } => { + split.reset(); + assembly.reset(); + } Stream::Aac(stream) => stream.desync(), Stream::Legacy(stream) => stream.desync(), Stream::Opus(_) | Stream::PesPacket(_) | Stream::Clock | Stream::Ignored => {} @@ -2994,11 +3070,23 @@ impl Stream { fn seek(&mut self, sequence: u64) -> anyhow::Result<()> { match self { - Stream::H264 { split, import, .. } => { + Stream::H264 { + split, + assembly, + import, + .. + } => { + assembly.reset(); split.reset(); Ok(import.seek(sequence)?) } - Stream::H265 { split, import, .. } => { + Stream::H265 { + split, + assembly, + import, + .. + } => { + assembly.reset(); split.reset(); Ok(import.seek(sequence)?) } @@ -3016,14 +3104,29 @@ impl Stream { // Only the self-describing audio holds a frame back for a successor to confirm. Stream::Aac(stream) => return stream.finish(), Stream::Legacy(stream) => return stream.finish(), - // Each PES flushed its access unit, so the splitter holds only input storage. - Stream::H264 { split, import } => { + // No input follows: drain the clean tail, then release the splitter and + // assembler outright, including the parameter caches a loss reset keeps. + Stream::H264 { + split, + assembly, + import, + } => { + let published = video::avc(split, import, assembly, true)?; *split = Default::default(); - import.finish()? - } - Stream::H265 { split, import } => { + assembly.finish(); + import.finish()?; + return Ok(published); + } + Stream::H265 { + split, + assembly, + import, + } => { + let published = video::hevc(split, import, assembly, true)?; *split = Default::default(); - import.finish()? + assembly.finish(); + import.finish()?; + return Ok(published); } Stream::Opus(stream) => stream.finish()?, Stream::PesPacket(stream) => stream.finish()?, @@ -4274,6 +4377,7 @@ pub(super) mod test { use std::time::Duration; use moq_net::Timestamp; + use mpeg2ts::es::StreamType; use super::{Continuation, Continuity, SectionReassembler, Stream}; @@ -5657,10 +5761,8 @@ pub(super) mod test { out } - // A break mid-picture is not the same as a break mid-audio. One video PES is exactly one - // access unit, so there is no whole unit ahead of the cut to salvage: publishing what - // arrived would hand the decoder a picture with missing slices, and a keyframe missing - // slices stays wrong for every picture that references it. Drop it and wait for the next. + // A PES ending mid-picture cannot prove that picture complete. Keep only AUs + // closed by a codec boundary ahead of a cut, and recover at the next keyframe. #[tokio::test(start_paused = true)] async fn video_drops_a_partial_access_unit_across_a_continuity_break() { const VIDEO_PID: u16 = 0x0050; @@ -8144,7 +8246,10 @@ pub(super) mod test { let err = import .decode(&packet(true, (super::STAGING_UNITS_MAX % 16) as u8, 0, &CUE)) .unwrap_err(); - assert!(err.to_string().contains("whole PES while waiting for a timestamp")); + assert!(err.to_string().contains(&format!( + "TS staging exceeds {} buffered units", + super::STAGING_UNITS_MAX + ))); assert_eq!(import.sections[&0x21].held.len(), super::STAGING_UNITS_MAX); assert_eq!( import.staging.bytes.load(super::Ordering::Relaxed), @@ -8439,6 +8544,208 @@ pub(super) mod test { const VIDEO: u16 = 0x0050; + #[tokio::test(start_paused = true)] + async fn video_pes_continuation_keeps_the_access_unit_timestamp() { + let au = annexb_au(true); + // Split inside the IDR, so neither PES is an independently decodable picture. + let cut = au.len() - 2; + let mut data = synth_pmt(&[(StreamType::H264, VIDEO)], false); + data.extend(video_pes(VIDEO, 0, FRAME, None, &au[..cut])); + data.extend(video_pes(VIDEO, 1, FRAME * 9, None, &au[cut..])); + let frames = video_frames(&data).await; + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].payload.as_ref(), au); + assert_eq!(frames[0].timestamp.as_micros(), u128::from(FRAME) * 1_000_000 / 90_000); + } + + #[test] + fn video_pes_does_not_reuse_its_pts_for_an_untimestampable_successor() { + let mut payload = annexb_au(true); + payload.extend(annexb_au(false)); + let mut data = synth_pmt(&[(StreamType::H264, VIDEO)], false); + data.extend(video_pes(VIDEO, 0, FRAME, None, &payload)); + let err = match import_all(&data) { + Ok(_) => panic!("a second AU cannot reuse the first AU's PES PTS"), + Err(err) => err, + }; + assert!(!err.is::(), "timing refusal must end the import"); + assert!(err.to_string().contains("timestamp"), "{err:#}"); + } + + /// Repacketize actual encoder-produced POCs, retaining an independent PTS + /// oracle from its original TS rather than deriving the expected values. + #[tokio::test(start_paused = true)] + async fn video_poc_repartition_preserves_frames_timestamps_and_export() { + use crate::container::ts::fixture; + use mpeg2ts::pes::{PesPacketReader, ReadPesPacket}; + use mpeg2ts::ts::TsPacketReader; + use std::io::Cursor; + for (kind, fixture) in [ + (StreamType::H264, &include_bytes!("test_data/poc-avc.ts")[..]), + (StreamType::H265, &include_bytes!("test_data/poc-hevc.ts")[..]), + ] { + let mut reader = PesPacketReader::new(TsPacketReader::new(Cursor::new(fixture))); + let mut pictures = Vec::new(); + while let Some(pes) = reader.read_pes_packet().unwrap() { + pictures.push((pes.header.pts.unwrap().as_u64(), pes.data)); + } + assert_eq!(pictures.len(), 12); + let mut baseline = None; + for layout in ["single", "combined", "split", "no-clock"] { + if layout == "no-clock" && kind != StreamType::H264 { + continue; + } + let mut mux = fixture::Mux::default(); + mux.pat(&[(1, 0x1000)]); + // fixed_frame_rate_flag + temporal_poc_flag. The AVC VUI alone + // deliberately does not authorize multiplying POC by a clock tick. + let descriptor = if kind == StreamType::H264 && layout != "no-clock" { + &[0x2a, 2, 0x7e, 0xdf][..] + } else { + &[] + }; + mux.pmt_with(0x1000, 1, VIDEO, &[(kind as u8, VIDEO, descriptor)], 0); + let push = |mux: &mut fixture::Mux, pts: Option, payload: &[u8]| { + let header_len = if pts.is_some() { 8 } else { 3 }; + let len = header_len + payload.len(); + let mut pes = vec![ + 0, + 0, + 1, + 0xe0, + (len >> 8) as u8, + len as u8, + 0x80, + if pts.is_some() { 0x80 } else { 0 }, + if pts.is_some() { 5 } else { 0 }, + ]; + if let Some(pts) = pts { + pes.extend(pes_timestamp(2, pts)); + } + pes.extend(payload); + mux.whole(VIDEO, &pes); + }; + match layout { + "single" => { + for (pts, data) in &pictures { + push(&mut mux, Some(*pts), data); + } + } + "combined" | "no-clock" => { + for chunk in pictures.chunks(4) { + let data: Vec = chunk.iter().flat_map(|(_, data)| data.iter().copied()).collect(); + push(&mut mux, Some(chunk[0].0), &data); + } + } + "split" => { + for (pts, data) in &pictures { + // All three required prefix bytes and the final slice body + // are exercised on opposite sides of a PES boundary. + for (n, chunk) in data.chunks(2).enumerate() { + push(&mut mux, if n == 0 { Some(*pts) } else { None }, chunk); + } + } + } + _ => unreachable!(), + } + if layout == "no-clock" { + // Valid AVC with fixed-rate SPS VUI still needs the TS + // temporal-POC promise before numeric POC can mean time. + let err = match import_all(&mux.out) { + Ok(_) => panic!("SPS fixed frame rate alone cannot timestamp a later AU"), + Err(err) => err, + }; + assert!(!err.is::()); + assert!(err.to_string().contains("proportional"), "{err:#}"); + continue; + } + let (consumer, catalog, _import) = + import_all(&mux.out).unwrap_or_else(|err| panic!("{kind:?} {layout}: {err:#}")); + let name = catalog.snapshot().video.renditions.keys().next().unwrap().clone(); + let frames = read_track(&consumer, &name, crate::container::Kind::Video).await; + let observed: Vec<_> = frames + .iter() + .map(|f| (f.timestamp.as_micros(), f.keyframe, f.payload.clone())) + .collect(); + assert_eq!(frames.len(), pictures.len(), "{kind:?} {layout}"); + assert_eq!( + frames.iter().map(|f| f.timestamp.as_micros()).collect::>(), + pictures + .iter() + .map(|(pts, _)| u128::from(*pts) * 1_000_000 / 90_000) + .collect::>(), + "{kind:?} {layout}" + ); + match &baseline { + None => baseline = Some(observed.clone()), + Some(expected) => assert_eq!(&observed, expected, "{kind:?} {layout}"), + } + let mut export = + crate::container::ts::export_test::open_export(crate::source::announced(&consumer).await) + .await + .unwrap() + .with_delay(Duration::from_secs(30)) + .with_replay(); + let mut ts = Vec::new(); + while let Ok(frame) = tokio::time::timeout(Duration::from_secs(100), export.next()).await { + let Some(frame) = frame.unwrap() else { + break; + }; + ts.extend(frame.payload); + } + let roundtrip = video_frames(&ts).await; + assert_eq!( + roundtrip + .iter() + .map(|f| (f.timestamp.as_micros(), f.keyframe, f.payload.clone())) + .collect::>(), + observed, + "export {kind:?} {layout}" + ); + } + } + } + + #[tokio::test(start_paused = true)] + async fn video_parameter_sets_commit_per_au_instead_of_per_pes() { + use crate::container::test_util::{IDR, PPS, SPS}; + let extra_pps = [0x68, 0xce, 0x3c, 0x81]; + let mut payload = Vec::new(); + for nal in [SPS, PPS, &extra_pps, IDR, &[0x67, 0x42], &[0x65, 0x80]] { + payload.extend([0, 0, 0, 1]); + payload.extend(nal); + } + let mut data = synth_pmt(&[(StreamType::H264, VIDEO)], false); + data.extend(video_pes(VIDEO, 0, 90_000, None, &payload)); + // A later keyframe proves the damaged AU ended and must recover with + // the last committed parameter sets, including the extra PPS above. + let mut bare = vec![0, 0, 0, 1]; + bare.extend(IDR); + data.extend(video_pes(VIDEO, 1, 180_000, None, &bare)); + let (consumer, catalog, import) = import_all(&data).unwrap(); + let name = catalog.snapshot().video.renditions.keys().next().unwrap().clone(); + let frames = read_track(&consumer, &name, crate::container::Kind::Video).await; + assert_eq!(frames.len(), 2); + assert_eq!(import.stats().streams[&VIDEO].damaged, 1); + assert_eq!(import.stats().streams[&VIDEO].units, 2); + for frame in &frames { + assert!(frame.payload.windows(extra_pps.len()).any(|nal| nal == extra_pps)); + } + } + + #[test] + fn video_damage_in_the_final_access_unit_is_counted() { + let mut data = synth_pmt(&[(StreamType::H264, VIDEO)], false); + data.extend(video_pes(VIDEO, 0, 90_000, None, &annexb_au(true))); + // Only end of input ends this AU, so finish() refuses it: its truncated + // SPS assembles but fails to parse. + let au = [0, 0, 0, 1, 0x67, 0x42, 0, 0, 0, 1, 0x65, 0x80]; + data.extend(video_pes(VIDEO, 1, 90_000 + FRAME, None, &au)); + let (_consumer, _catalog, import) = import_all(&data).unwrap(); + assert_eq!(import.stats().streams[&VIDEO].units, 1); + assert_eq!(import.stats().streams[&VIDEO].damaged, 1); + } + /// Damage on one PID refuses that unit and its dependent pictures while another PID /// carries on. A later decode call resumes from the next keyframe. async fn damaged_video_recovers(damage: &str) { @@ -8448,6 +8755,15 @@ pub(super) mod test { ..Default::default() }; mux.gop(VIDEO, 90_000); + if matches!(damage, "pes" | "adaptation" | "adaptation-clock" | "tei") { + // These faults hide the next PES entirely. An AUD received ahead of + // the fault proves the preceding picture complete; the AUD's own + // picture remains unconfirmed and is discarded at the loss. + let mut au = annexb_au(false); + au.extend([0, 0, 1, 9, 0xf0]); + let last = mux.out.len() - 188; + mux.out[last..].copy_from_slice(&video_pes(VIDEO, 3, 90_000 + 3 * FRAME, None, &au)); + } mux.gop(peer, 90_000); let damaged_at = mux.out.len(); mux.gop(VIDEO, 90_000 + 4 * FRAME); @@ -8469,8 +8785,8 @@ pub(super) mod test { packet.copy_from_slice(&video_pes(VIDEO, 4, 90_000 + 5 * FRAME, Some(90_000 + 4 * FRAME), &au)); } "queued" => { - // The AUD completes the IDR before the forbidden NAL fails, so the splitter - // has already queued it when the unit is refused. + // The AUD closes a valid IDR before the forbidden NAL in the next + // AU fails. That already-admitted IDR must remain committed. let mut au = annexb_au(true); au.extend_from_slice(&[0, 0, 0, 1, 0x09, 0xf0, 0, 0, 0, 1, 0xe1, 0x80, 0, 0, 0, 1, 0x41, 0x9a]); packet.copy_from_slice(&video_pes(VIDEO, 4, 90_000 + 5 * FRAME, Some(90_000 + 4 * FRAME), &au)); @@ -8511,16 +8827,35 @@ pub(super) mod test { import.finish().unwrap(); assert_eq!(import.stats().streams[&VIDEO].damaged, u64::from(damage != "clean")); assert_eq!(import.stats().streams[&peer].damaged, 0); - for (pid, expected) in [(VIDEO, if damage == "clean" { 12 } else { 8 }), (peer, 12)] { + for (pid, expected) in [ + ( + VIDEO, + if damage == "clean" { + 12 + } else if damage == "queued" { + 9 + } else { + 8 + }, + ), + (peer, 12), + ] { let Stream::H264 { import: video, .. } = &import.streams[&Pid::new(pid).unwrap()] else { panic!("video route"); }; let frames = read_track(&consumer, video.name(), crate::container::Kind::Video).await; assert_eq!(frames.len(), expected, "published pictures on PID {pid}"); if pid == VIDEO && damage != "clean" { - assert!(frames[4].keyframe, "recovery starts at a keyframe"); + let recovery = if damage == "queued" { 5 } else { 4 }; + if damage == "queued" { + assert!( + frames[4].keyframe, + "the valid AU before the damaged tail remains committed" + ); + } + assert!(frames[recovery].keyframe, "recovery starts at a keyframe"); assert_eq!( - frames[4].timestamp.as_micros(), + frames[recovery].timestamp.as_micros(), u128::from(90_000 + 9 * FRAME) * 1_000_000 / 90_000 ); } @@ -8543,7 +8878,7 @@ pub(super) mod test { } #[tokio::test(start_paused = true)] - async fn damaged_unit_drops_a_picture_the_splitter_queued() { + async fn damaged_tail_preserves_an_access_unit_already_admitted() { damaged_video_recovers("queued").await; } @@ -8721,15 +9056,19 @@ pub(super) mod test { ); } - /// A damaged PES start refuses only its own unit: the unbounded PES before it ends there, - /// whole, and still publishes. + /// A damaged PES start does not revoke the AU a codec boundary already + /// completed in the preceding unbounded PES. #[test] fn damaged_pes_start_flushes_the_unit_before_it() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); let mut import = super::Import::new(broadcast, catalog.reserve()); let unbounded = |cc, pts, keyframe| { - let mut packet = video_pes(VIDEO, cc, pts, None, &annexb_au(keyframe)); + let mut au = annexb_au(keyframe); + if keyframe { + au.extend([0, 0, 1, 9, 0xf0]); + } + let mut packet = video_pes(VIDEO, cc, pts, None, &au); let start = 5 + usize::from(packet[4]); packet[start + 4..start + 6].fill(0); packet @@ -8758,7 +9097,11 @@ pub(super) mod test { let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); let mut import = super::Import::new(broadcast, catalog.reserve()); let unbounded = |cc, pts, keyframe| { - let mut packet = video_pes(VIDEO, cc, pts, None, &annexb_au(keyframe)); + let mut au = annexb_au(keyframe); + if keyframe { + au.extend([0, 0, 1, 9, 0xf0]); + } + let mut packet = video_pes(VIDEO, cc, pts, None, &au); let start = 5 + usize::from(packet[4]); packet[start + 4..start + 6].fill(0); packet @@ -8775,33 +9118,35 @@ pub(super) mod test { assert_eq!( import.stats().streams[&VIDEO].units, 1, - "only the keyframe the second start proved whole" + "only the keyframe the AUD proved whole" ); } /// A malformed unbounded PES drained at EOF is still refused and counted. #[test] fn damaged_final_access_unit_is_refused() { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut import = super::Import::new(broadcast, catalog.reserve()); let mut au = annexb_au(true); let nal = au .windows(5) .position(|bytes| bytes[..4] == [0, 0, 0, 1] && bytes[4] & 0x1f == 5) .unwrap(); au[nal + 4] |= 0x80; - let mut packet = video_pes(VIDEO, 0, 90_000, None, &au); - let start = 5 + usize::from(packet[4]); - packet[start + 4..start + 6].fill(0); - let mut data = synth_pmt(&[(StreamType::H264, VIDEO)], false); - data.extend(packet); - import.decode(&data).unwrap(); - assert_eq!(import.stats().streams[&VIDEO].damaged, 0, "PES has not ended yet"); - import.finish().unwrap(); - assert_eq!(import.stats().streams[&VIDEO].damaged, 1); - assert_eq!(import.stats().streams[&VIDEO].units, 0); - assert!(catalog.snapshot().video.renditions.is_empty()); + for au in [au, vec![0], vec![0, 0], vec![0, 0, 0]] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + let mut packet = video_pes(VIDEO, 0, 90_000, None, &au); + let start = 5 + usize::from(packet[4]); + packet[start + 4..start + 6].fill(0); + let mut data = synth_pmt(&[(StreamType::H264, VIDEO)], false); + data.extend(packet); + import.decode(&data).unwrap(); + assert_eq!(import.stats().streams[&VIDEO].damaged, 0, "PES has not ended yet"); + import.finish().unwrap(); + assert_eq!(import.stats().streams[&VIDEO].damaged, 1); + assert_eq!(import.stats().streams[&VIDEO].units, 0); + assert!(catalog.snapshot().video.renditions.is_empty()); + } } /// An invalid trailing Opus control header refuses the complete PES before its first @@ -8852,6 +9197,7 @@ pub(super) mod test { Pid::new(VIDEO).unwrap(), Stream::H264 { split: crate::codec::h264::Split::new(), + assembly: super::video::Assembly::new(super::video::Codec::Avc), import: Box::new(video), }, ); @@ -8906,8 +9252,13 @@ pub(super) mod test { damaged_at = mux.out.len(); } let cc = mux.cc(VIDEO); - mux.out - .extend_from_slice(&video_pes(VIDEO, cc, pts, dts, &annexb_au(keyframe))); + mux.out.extend_from_slice(&video_pes(VIDEO, cc, pts, dts, &{ + let mut au = annexb_au(keyframe); + if i == 1 { + au.extend([0, 0, 1, 9, 0xf0]); + } + au + })); } mux.out[damaged_at + 1] |= 0x80; @@ -9013,6 +9364,12 @@ pub(super) mod test { ..Default::default() }; mux.gops(VIDEO, 90_000, 2); + // A codec boundary, not the following clock marker, vouches for the + // final picture on the old timeline. + let mut last = annexb_au(false); + last.extend([0, 0, 1, 9, 0xf0]); + let at = mux.out.len() - 188; + mux.out[at..].copy_from_slice(&video_pes(VIDEO, 7, 90_000 + 7 * FRAME, None, &last)); mux.out.extend_from_slice(&clock_break_packet(VIDEO)); // Inside the track's 30 s retention, so the reader still reaches the first pass. mux.gops(VIDEO, 20 * 90_000, 2); diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index 5e3bf20cb11..a0482678198 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -750,7 +750,8 @@ fn tiny_whole_pes_held_for_a_timestamp_are_bounded_by_count() { let mut imported = start(false); let err = imported.import.decode(&feed.bytes()).expect_err("the flood is refused"); assert!( - err.to_string().contains("whole PES while waiting for a timestamp"), + err.to_string() + .contains(&format!("TS staging exceeds {units} buffered units")), "{err:#}" ); } diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs index 706ce7c4dca..4abf4b85735 100644 --- a/rs/moq-mux/src/container/ts/recovery_test.rs +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -328,6 +328,25 @@ fn splice(a: Vec, b: Vec, declare: bool) -> Clip { _ => out.push(*packet), } } + if declare { + // Receive an AU boundary ahead of the discontinuity. The preceding + // unbounded video PES is now provably complete; the delimiter's own + // unfinished AU is discarded when the clock breaks. + let pid = payload(&a_packets[from]).0; + let cc = out + .iter() + .rev() + .find(|packet| payload(packet).0 == pid && packet[3] & 0x10 != 0) + .unwrap()[3] + .wrapping_add(1) + & 15; + let pes = [0, 0, 1, 0xe0, 0, 8, 0x80, 0, 0, 0, 0, 1, 9, 0xf0]; + let mut packet = [0xff; 188]; + packet[..6].copy_from_slice(&[0x47, 0x40 | (pid >> 8) as u8, pid as u8, 0x30 | cc, 169, 0]); + packet[174..].copy_from_slice(&pes); + out.push(packet); + } + let mut counters = std::collections::HashMap::new(); for packet in &out { if packet[3] & 0x10 != 0 { diff --git a/rs/moq-mux/src/container/ts/test_data/README.md b/rs/moq-mux/src/container/ts/test_data/README.md new file mode 100644 index 00000000000..acef3171aa0 --- /dev/null +++ b/rs/moq-mux/src/container/ts/test_data/README.md @@ -0,0 +1,18 @@ +# POC fixtures + +`poc-avc.ts` and `poc-hevc.ts` contain twelve 32x32 pictures at 25 fps, +including B pictures. The original PES PTS values are an independent oracle for +repartitioning the identical elementary bytes into single, combined, and split +PES, then exporting and reimporting them. SEI is removed here because the test +uses the declared proportional POC clock, not HRD picture timing. T-STD's +separate generated controls retain HRD to prove decode-time derivation too. + +Generated in the repository Nix shell with: + +```bash +ffmpeg -hide_banner -loglevel error -f lavfi -i testsrc2=size=32x32:rate=25 -frames:v 12 -an -c:v libx264 -x264-params bframes=3:force-cfr=1:keyint=12:aud=1:threads=1 -bsf:v filter_units=remove_types=6,h264_metadata=fixed_frame_rate_flag=1 -f mpegts poc-avc.ts +ffmpeg -hide_banner -loglevel error -f lavfi -i testsrc2=size=32x32:rate=25 -frames:v 12 -an -c:v libx265 -x265-params bframes=3:keyint=12:aud=1:pools=none:frame-threads=1 -bsf:v filter_units=remove_types=39,hevc_metadata=tick_rate=25:num_ticks_poc_diff_one=1 -f mpegts poc-hevc.ts +``` + +The test adds AVC's MPEG-TS timing descriptor to its repacketized PMT: fixed +frame rate plus temporal POC explicitly permits scaling POC by the VUI tick. diff --git a/rs/moq-mux/src/container/ts/test_data/poc-avc.ts b/rs/moq-mux/src/container/ts/test_data/poc-avc.ts new file mode 100644 index 00000000000..455934b9e1c Binary files /dev/null and b/rs/moq-mux/src/container/ts/test_data/poc-avc.ts differ diff --git a/rs/moq-mux/src/container/ts/test_data/poc-hevc.ts b/rs/moq-mux/src/container/ts/test_data/poc-hevc.ts new file mode 100644 index 00000000000..f390129fbbb Binary files /dev/null and b/rs/moq-mux/src/container/ts/test_data/poc-hevc.ts differ diff --git a/rs/moq-mux/src/container/ts/video.rs b/rs/moq-mux/src/container/ts/video.rs new file mode 100644 index 00000000000..d7172b16646 --- /dev/null +++ b/rs/moq-mux/src/container/ts/video.rs @@ -0,0 +1,576 @@ +//! PES-independent access-unit ownership for the TS importer. +//! +//! This assembler owns PES stamps at the byte where an AU begins. The public +//! elementary splitters keep their caller-supplied complete-frame timestamp mode. + +use std::collections::VecDeque; + +use bytes::{Buf, Bytes, BytesMut}; +use moq_net::Timestamp; + +use super::{Pending, Retained, on_clock, reorder_delay, unit_error}; +use crate::codec::annexb::{START_CODE, after_start_code, find_start_code}; + +#[derive(Clone, Copy)] +pub(super) enum Codec { + Avc, + Hevc, +} + +impl Codec { + fn end(self, nal: &[u8]) -> bool { + match self { + Self::Avc => matches!(nal[0] & 31, 10 | 11), + Self::Hevc => matches!((nal[0] >> 1) & 63, 36 | 37), + } + } + + /// Whether this NAL begins a picture or its leading non-VCL data + /// (H.264 7.4.1.2.3, H.265 7.4.2.4.4), and whether it is a slice. + fn boundary(self, nal: &[u8]) -> anyhow::Result<(bool, bool)> { + match self { + Self::Avc => { + let header = *nal + .first() + .ok_or(crate::codec::h264::Error::NalTooShort) + .map_err(crate::Error::from) + .map_err(unit_error)?; + if header & 0x80 != 0 { + return Err(unit_error(crate::codec::h264::Error::ForbiddenZeroBit.into())); + } + let kind = header & 31; + let slice = matches!(kind, 1..=5); + let first = if slice { + *nal.get(1) + .ok_or(crate::codec::h264::Error::NalTooShort) + .map_err(crate::Error::from) + .map_err(unit_error)? + & 0x80 != 0 + } else { + matches!(kind, 6..=9 | 14..=18) + }; + Ok((first, slice)) + } + Self::Hevc => { + let header = *nal + .first() + .ok_or(crate::codec::h265::Error::NalTooShort) + .map_err(crate::Error::from) + .map_err(unit_error)?; + if header & 0x80 != 0 { + return Err(unit_error(crate::codec::h265::Error::ForbiddenZeroBit.into())); + } + let kind = (header >> 1) & 63; + let slice = kind <= 31; + let first = if slice { + *nal.get(2) + .ok_or(crate::codec::h265::Error::NalTooShort) + .map_err(crate::Error::from) + .map_err(unit_error)? + & 0x80 != 0 + } else { + matches!(kind, 32..=35 | 39 | 41..=44 | 48..=55) + }; + Ok((first, slice)) + } + } + } +} + +struct Span { + len: usize, + pts: Option, + reorder: Option, +} + +pub(super) struct Unit { + pub data: Bytes, + pub pts: Option, + pub reorder: Option, + end_sequence: bool, + parameters: bool, + // Charges remain live while the caller parses and publishes this AU. + _retained: Retained, +} + +#[path = "video_timing.rs"] +mod timing; + +pub(super) struct Assembly { + pub clock: timing::Clock, + damage: Option<(u64, anyhow::Error)>, + codec: Codec, + tail: BytesMut, + searched: usize, + spans: VecDeque, + current: BytesMut, + stamp: Option<(Option, Option)>, + has_slice: bool, + ending: bool, + parameters: bool, + retained: Option, +} + +impl Assembly { + pub fn new(codec: Codec) -> Self { + Self { + clock: timing::Clock::default(), + damage: None, + codec, + tail: BytesMut::new(), + searched: 0, + spans: VecDeque::new(), + current: BytesMut::new(), + stamp: None, + has_slice: false, + ending: false, + parameters: false, + retained: None, + } + } + + pub fn push(&mut self, mut pending: Pending) -> anyhow::Result<()> { + let pts = on_clock(pending.pts, pending.offset)?; + let reorder = reorder_delay(pending.pts, pending.dts); + if pending.data.is_empty() { + return Ok(()); + } + if let Some(retained) = &mut self.retained { + debug_assert!(std::sync::Arc::ptr_eq(&retained.staging, &pending.retained.staging)); + retained.bytes += pending.retained.bytes; + retained.units += pending.retained.units; + pending.retained.bytes = 0; + pending.retained.units = 0; + } else { + self.retained = Some(pending.retained); + } + self.spans.push_back(Span { + len: pending.data.len(), + pts, + reorder, + }); + self.tail.extend_from_slice(&pending.data); + self.account() + } + + fn account(&mut self) -> anyhow::Result<()> { + if let Some(retained) = &mut self.retained { + retained.resize( + self.tail.len() + self.current.len(), + self.spans.len() + usize::from(self.stamp.is_some()), + )?; + } + Ok(()) + } + + /// Return one AU at a time so later parameter sets cannot precede its validation. + pub fn next(&mut self, eof: bool) -> anyhow::Result> { + loop { + if self.tail.is_empty() { + return if eof { self.emit() } else { Ok(None) }; + } + let start = after_start_code(&self.tail) + .map_err(crate::Error::from) + .map_err(unit_error)?; + let Some(start) = start else { + if eof { + return Err(unit_error(crate::codec::annexb::Error::MissingStartCode.into())); + } + return Ok(None); + }; + // A new picture's header closes the preceding AU even while its final + // NAL is still arriving. Never treat a PES boundary as this evidence. + if self.has_slice { + let header = match self.codec { + Codec::Avc => 2, + Codec::Hevc => 3, + }; + if self.tail.len() < start + header { + // Until the boundary is decided, this prefix may own the next AU's + // stamp, so it must not be scanned into the spent spans below. + if !eof { + return Ok(None); + } + } else if self.codec.boundary(&self.tail[start..])?.0 { + return self.emit(); + } + } + if self.stamp.is_none() { + // The optional zero_byte may be a trailing zero from the preceding + // PES. Ownership follows the required three-byte start_code_prefix. + let mut skip = start - 3; + let span = self + .spans + .iter_mut() + .find(|span| { + if skip < span.len { + true + } else { + skip -= span.len; + false + } + }) + .expect("every input byte has an owner"); + self.stamp = Some((span.pts.take(), span.reorder.take())); + } + self.account()?; + let from = self.searched.max(start); + let end = match find_start_code(&self.tail[from..]) { + Some((offset, _)) => from + offset, + None if eof => self.tail.len(), + None => { + // A hostile stream can split one large NAL across tiny PES. + // Scan each byte once, retaining just the possible split prefix. + self.searched = self.tail.len().saturating_sub(3).max(start); + // Only the possible split prefix can start another AU. Earlier + // PES stamps are spent; retain byte coverage without one entry + // per fragment of an arbitrarily large NAL. + let mut scanned = 0; + while self + .spans + .front() + .is_some_and(|span| scanned + span.len <= self.searched) + { + scanned += self.spans.pop_front().unwrap().len; + } + if scanned > 0 { + self.spans.push_front(Span { + len: scanned, + pts: None, + reorder: None, + }); + } + self.account()?; + return Ok(None); + } + }; + self.tail.advance(start); + let nal = self.tail.copy_to_bytes(end - start); + let consumed = end; + self.searched = 0; + let (_, slice) = self.codec.boundary(&nal)?; + self.has_slice |= slice; + self.ending = self.codec.end(&nal); + self.parameters |= match self.codec { + Codec::Avc => matches!(nal[0] & 31, 7 | 8), + Codec::Hevc => matches!((nal[0] >> 1) & 63, 32..=34), + }; + self.current.extend_from_slice(&START_CODE); + self.current.extend_from_slice(&nal); + let mut remaining = consumed; + while remaining > 0 { + let span = self.spans.front_mut().expect("consumed bytes have an owner"); + let take = remaining.min(span.len); + span.len -= take; + remaining -= take; + if span.len == 0 { + self.spans.pop_front(); + } + } + self.account()?; + if self.ending { + if self.has_slice { + return self.emit(); + } + self.current.clear(); + self.stamp = None; + self.ending = false; + self.clock.reset(); + self.account()?; + } + } + } + + fn emit(&mut self) -> anyhow::Result> { + if !self.has_slice { + return Ok(None); + } + let (pts, reorder) = self.stamp.take().expect("a picture has a first byte"); + let data = std::mem::take(&mut self.current).freeze(); + self.has_slice = false; + let retained = self + .retained + .as_mut() + .expect("buffered bytes are charged") + .take(data.len(), 1); + Ok(Some(Unit { + data, + pts, + reorder, + end_sequence: std::mem::take(&mut self.ending), + parameters: std::mem::take(&mut self.parameters), + _retained: retained, + })) + } + + pub fn take_damage(&mut self) -> Option<(u64, anyhow::Error)> { + self.damage.take() + } + + /// Release everything buffered at end of input, keeping damage the final drain refused. + pub fn finish(&mut self) { + let damage = self.damage.take(); + *self = Self::new(self.codec); + self.damage = damage; + } + + pub fn reset(&mut self) { + self.clock.reset(); + self.tail = BytesMut::new(); + self.searched = 0; + self.spans.clear(); + self.current = BytesMut::new(); + self.stamp = None; + self.has_slice = false; + self.ending = false; + self.parameters = false; + self.retained = None; + } +} + +/// One proportional AVC clock must be declared explicitly by the TS descriptor; +/// an SPS fixed frame rate alone says nothing about numeric POC distance. +pub(super) fn temporal_poc(descriptors: &[super::catalog::Descriptor]) -> anyhow::Result { + let mut proportional = false; + for descriptor in descriptors.iter().filter(|d| d.tag == 0x2a) { + let data = &descriptor.data; + let first = *data + .first() + .ok_or_else(|| anyhow::anyhow!("truncated AVC timing descriptor"))?; + let len = if first & 1 == 0 { + 2 + } else { + let clock = *data + .get(1) + .ok_or_else(|| anyhow::anyhow!("truncated AVC timing descriptor"))?; + if clock & 0x80 != 0 { 7 } else { 15 } + }; + anyhow::ensure!(data.len() == len, "invalid AVC timing descriptor length"); + proportional |= data[len - 1] & 0xc0 == 0xc0; + } + Ok(proportional) +} + +// Both codecs publish one AU before inspecting the next, committing parameter +// sets only after that AU is accepted. EOF and PMT retirement use the same path. +macro_rules! publish { + ($name:ident, $codec:ident) => { + pub(super) fn $name( + split: &mut crate::codec::$codec::Split, + import: &mut crate::codec::$codec::Import, + assembly: &mut Assembly, + eof: bool, + ) -> anyhow::Result { + let mut published = 0; + let result = (|| { + while let Some(unit) = assembly.next(eof)? { + let params = unit.parameters.then(|| split.params()); + let result = (|| { + // Structural damage is local even when it prevents timing + // inference. No provisional timestamp reaches the producer. + let provisional = unit.pts.unwrap_or(Timestamp::ZERO); + let mut frames = split.decode(&unit.data, provisional).map_err(unit_error)?; + frames.extend(split.flush(provisional).map_err(unit_error)?); + anyhow::ensure!(frames.len() == 1, "unsupported TS video access-unit framing"); + for frame in &frames { + if frame.keyframe { + crate::codec::$codec::config(&frame.payload) + .map_err(crate::Error::from) + .map_err(unit_error)?; + } + } + let pts = assembly.clock.$name( + &unit, + params.as_ref().into_iter().flat_map(|params| params.nals()), + )?; + let mut published = 0; + for mut frame in frames { + frame.timestamp = pts; + published += u64::from(super::skip_missing_keyframe(import.decode([frame]))?); + } + anyhow::Ok(published) + })(); + match result { + Ok(units) => { + published += units; + assembly.clock.commit(); + } + Err(err) if err.is::() => { + // This AU has a known end. Keep the next AU's bytes, + // which may already be here in the same or following PES. + if let Some(params) = params { + split.restore(params); + } + split.reset(); + assembly.clock.reset(); + import.cut(None)?; + let count = assembly.damage.take().map_or(1, |(count, _)| count + 1); + assembly.damage = Some((count, err)); + continue; + } + Err(err) => return Err(err), + } + if let Some(reorder) = unit.reorder { + import.observe_reorder(reorder)?; + } + if unit.end_sequence { + assembly.clock.reset(); + } + } + anyhow::Ok(()) + })(); + match result { + Ok(()) => Ok(published), + Err(err) if err.is::() => Err(err.context(Published(published))), + Err(err) => Err(err), + } + } + }; +} +publish!(avc, h264); +publish!(hevc, h265); + +#[derive(Debug, thiserror::Error)] +#[error("damaged TS video after {0} published access units")] +struct Published(u64); + +pub(super) fn published(err: &anyhow::Error) -> u64 { + err.downcast_ref::().map_or(0, |published| published.0) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, atomic::Ordering}; + + fn pending(staging: &Arc, data: &[u8], pts: Option) -> Pending { + let mut retained = Retained::new(staging.clone()); + retained.grow(data.len(), 0).unwrap(); + Pending { + retained, + data: data.to_vec(), + pts, + ..Pending::empty() + } + } + + fn drain(assembly: &mut Assembly, eof: bool, units: &mut Vec) { + while let Some(unit) = assembly.next(eof).unwrap() { + units.push(unit); + } + } + + #[test] + fn prefix_ownership_and_staging_follow_the_bytes() { + // Every cut of the required prefix and slice, for both 3/4-byte prefixes, + // with and without a preceding AU awaiting its boundary. A continuation's + // unrelated PTS never restamps this AU. + for lead in [false, true] { + for prefix in [&[0, 0, 1][..], &[0, 0, 0, 1][..]] { + let data = [prefix, &[0x65, 0x80, 0xab, 0xcd]].concat(); + for cut in 1..data.len() { + let staging = Arc::default(); + let mut assembly = Assembly::new(Codec::Avc); + let mut units = Vec::new(); + if lead { + assembly + .push(pending(&staging, &[0, 0, 1, 0x65, 0x80, 0x11], None)) + .unwrap(); + } + assembly.push(pending(&staging, &data[..cut], Some(90_000))).unwrap(); + drain(&mut assembly, false, &mut units); + if !lead { + assert_eq!(staging.bytes.load(Ordering::Relaxed), cut); + } + assembly.push(pending(&staging, &data[cut..], Some(180_000))).unwrap(); + drain(&mut assembly, false, &mut units); + drain(&mut assembly, true, &mut units); + assert_eq!(units.len(), 1 + usize::from(lead)); + let unit = units.last().unwrap(); + let owner = if prefix.len() == 4 && cut == 1 { 2 } else { 1 }; + assert_eq!(unit.pts.unwrap().as_secs(), owner, "lead {lead} cut {cut}"); + assert_eq!( + unit.data.as_ref(), + [START_CODE.as_ref(), &[0x65, 0x80, 0xab, 0xcd]].concat() + ); + if !lead { + assert_eq!(staging.bytes.load(Ordering::Relaxed), unit.data.len()); + } + drop(units); + assembly.reset(); + assert_eq!(staging.bytes.load(Ordering::Relaxed), 0); + assert_eq!(staging.units.load(Ordering::Relaxed), 0); + } + } + } + } + + #[test] + fn every_standard_opener_starts_the_next_au() { + let avc = [6, 7, 8, 9, 14, 15, 16, 17, 18].map(|kind| vec![kind]); + let hevc = [32, 33, 34, 35, 39, 41, 44, 48, 55].map(|kind| vec![kind << 1, 0x01]); + let cases = [ + (Codec::Avc, vec![0x65, 0x80], vec![0x61, 0x80], avc), + (Codec::Hevc, vec![0x26, 0x01, 0x80], vec![0x02, 0x01, 0x80], hevc), + ]; + for (codec, first, next, openers) in cases { + for opener in openers { + let staging = Arc::default(); + let mut assembly = Assembly::new(codec); + let data = [&[0, 0, 1][..], &first, &[0, 0, 1], &opener, &[0, 0, 1], &next].concat(); + assembly.push(pending(&staging, &data, Some(90_000))).unwrap(); + let mut units = Vec::new(); + drain(&mut assembly, true, &mut units); + assert_eq!(units.len(), 2, "opener {opener:?}"); + assert_eq!( + &units[1].data[..START_CODE.len() + opener.len()], + [START_CODE.as_ref(), &opener].concat() + ); + } + } + } + + #[test] + fn tiny_pes_fragments_release_spent_stamp_metadata() { + let staging = Arc::default(); + let mut assembly = Assembly::new(Codec::Avc); + let data = [&[0, 0, 1, 0x65, 0x80][..], &vec![0xff; 4096]].concat(); + for (i, byte) in data.iter().enumerate() { + assembly + .push(pending(&staging, &[*byte], Some(if i == 0 { 90_000 } else { 180_000 }))) + .unwrap(); + assert!(assembly.next(false).unwrap().is_none()); + // At most three possible prefix bytes, one scanned byte range, + // and the current AU's stamp remain independently owned. + assert!(staging.units.load(Ordering::Relaxed) <= 5); + } + assembly + .push(pending(&staging, &[0, 0, 1, 0x65, 0x80], Some(270_000))) + .unwrap(); + let first = assembly.next(false).unwrap().unwrap(); + assert_eq!(first.pts.unwrap().as_secs(), 1); + assert_eq!(&first.data[4..], &data[3..]); + let second = assembly.next(true).unwrap().unwrap(); + assert_eq!(second.pts.unwrap().as_secs(), 3); + drop((first, second)); + assembly.reset(); + assert_eq!(staging.bytes.load(Ordering::Relaxed), 0); + assert_eq!(staging.units.load(Ordering::Relaxed), 0); + } + + #[test] + fn lost_tail_releases_bytes_stamps_and_clock() { + let staging = Arc::default(); + let mut assembly = Assembly::new(Codec::Avc); + assembly + .push(pending(&staging, &[0, 0, 1, 0x65, 0x80], Some(90_000))) + .unwrap(); + assert!(assembly.next(false).unwrap().is_none()); + assembly.reset(); + assert_eq!(staging.bytes.load(Ordering::Relaxed), 0); + assert_eq!(staging.units.load(Ordering::Relaxed), 0); + assembly.push(pending(&staging, &[0, 0, 1, 0x65, 0x80], None)).unwrap(); + let unit = assembly.next(true).unwrap().unwrap(); + assert!(unit.pts.is_none()); + } +} diff --git a/rs/moq-mux/src/container/ts/video_timing.rs b/rs/moq-mux/src/container/ts/video_timing.rs new file mode 100644 index 00000000000..a866d2faa4a --- /dev/null +++ b/rs/moq-mux/src/container/ts/video_timing.rs @@ -0,0 +1,746 @@ +//! POC is a timestamp only when the stream declares a proportional clock. +//! +//! h264-reader owns AVC syntax; scuffle-h265 owns HEVC SPS/VUI syntax. The +//! private HEVC reader stops at the first independent slice's POC prefix. +//! Only owned POC state survives between calls. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use bytes::Bytes; +use h264_reader::nal::slice::{ + DecRefPicMarking, Field, FieldPic, MemoryManagementControlOperation, PicOrderCountLsb, SliceHeader, +}; +use h264_reader::nal::sps::PicOrderCntType; +use h264_reader::nal::{Nal, RefNal, UnitType}; +use h264_reader::nal::{pps::PicParameterSet, sps::SeqParameterSet}; +use moq_net::Timestamp; + +use super::Unit; + +#[derive(Clone, Copy, PartialEq, Eq)] +struct Rate { + numerator: u64, + denominator: u32, +} + +#[derive(Clone, Copy)] +struct Anchor { + poc: i64, + pts: Timestamp, + rate: Rate, +} + +#[derive(Default)] +pub(crate) struct Clock { + pub temporal_poc: bool, + avc: AvcPoc, + hevc: Option, + anchor: Option, + avc_ctx: Option>, + avc_update: Option>>, + hevc_ctx: Option>, + hevc_update: Option>>, +} + +impl Clock { + pub fn reset(&mut self) { + self.avc = AvcPoc::default(); + self.hevc = None; + self.anchor = None; + self.avc_update = None; + self.hevc_update = None; + } + + pub fn commit(&mut self) { + if let Some(ctx) = self.avc_update.take() { + self.avc_ctx = ctx; + } + if let Some(ctx) = self.hevc_update.take() { + self.hevc_ctx = ctx; + } + } + + fn stamp( + &mut self, + explicit: Option, + poc: i64, + rate: Option, + reset: bool, + ) -> anyhow::Result { + if reset { + self.anchor = None; + } + let Some(rate) = rate else { + self.anchor = None; + return explicit.ok_or_else(|| { + anyhow::anyhow!("TS video access unit has no timestamp or signalled proportional POC clock") + }); + }; + if let Some(pts) = explicit { + self.anchor = Some(Anchor { poc, pts, rate }); + return Ok(pts); + } + let anchor = self + .anchor + .filter(|anchor| anchor.rate == rate) + .ok_or_else(|| anyhow::anyhow!("TS video access unit has no timestamp anchor for its POC clock"))?; + // Always measure from the explicit anchor, retaining fractional ticks rather + // than accumulating the rounding error of a frame-by-frame increment. + let scale = i128::from(anchor.pts.scale().as_u64()); + let denominator = scale + .checked_mul(rate.denominator.into()) + .ok_or_else(|| anyhow::anyhow!("TS video timestamp overflow"))?; + let delta = (i128::from(poc) - i128::from(anchor.poc)) + .checked_mul(i128::from(rate.numerator)) + .and_then(|n| n.checked_mul(scale)); + let micros = i128::from(anchor.pts.value()) + .checked_mul(rate.denominator.into()) + .and_then(|n| n.checked_add(delta?)) + .and_then(|n| n.checked_mul(1_000_000)) + .and_then(|n| u64::try_from(n / denominator).ok()) + .ok_or_else(|| anyhow::anyhow!("TS video timestamp outside the media clock"))?; + Ok(Timestamp::from_micros(micros)?) + } + + pub fn avc<'a>(&mut self, unit: &Unit, params: impl Iterator) -> anyhow::Result { + let parsed = (|| { + let mut ctx = self.avc_ctx.clone().unwrap_or_default(); + let bootstrap: Vec<_> = if self.avc_ctx.is_none() { + params.cloned().collect() + } else { + Vec::new() + }; + for bytes in bootstrap.into_iter().chain(nals(&unit.data)?) { + let nal = RefNal::new(&bytes, &[], true); + let header = nal.header().map_err(|err| format!("{err:?}"))?; + match header.nal_unit_type() { + UnitType::SeqParameterSet => Arc::make_mut(&mut ctx).0.put_seq_param_set( + SeqParameterSet::from_bits(nal.rbsp_bits()).map_err(|err| format!("{err:?}"))?, + ), + UnitType::PicParameterSet => { + let pps = + PicParameterSet::from_bits(&ctx.0, nal.rbsp_bits()).map_err(|err| format!("{err:?}"))?; + Arc::make_mut(&mut ctx).0.put_pic_param_set(pps); + } + UnitType::SliceLayerWithoutPartitioningNonIdr + | UnitType::SliceLayerWithoutPartitioningIdr + | UnitType::SliceDataPartitionALayer => { + let (hdr, sps, _) = SliceHeader::from_bits(&ctx.0, &mut nal.rbsp_bits(), header, None) + .map_err(|err| format!("{err:?}"))?; + let info = sps.vui_parameters.as_ref().and_then(|vui| vui.timing_info.as_ref()); + let rate = info + .filter(|info| { + self.temporal_poc + && info.fixed_frame_rate_flag + && info.num_units_in_tick != 0 + && info.time_scale != 0 + }) + .map(|info| Rate { + numerator: info.num_units_in_tick.into(), + denominator: info.time_scale, + }); + let mut picture = AvcSlice { + idr: hdr.idr_pic_id.is_some(), + reference: header.nal_ref_idc() != 0, + ..Default::default() + }; + picture.header.frame_num = hdr.frame_num; + picture.header.field_pic_flag = hdr.field_pic != FieldPic::Frame; + picture.header.bottom_field_flag = hdr.field_pic == FieldPic::Field(Field::Bottom); + match hdr.pic_order_cnt_lsb { + Some(PicOrderCountLsb::Frame(lsb)) => picture.header.pic_order_cnt_lsb = lsb, + Some(PicOrderCountLsb::FieldsAbsolute { + pic_order_cnt_lsb, + delta_pic_order_cnt_bottom, + }) => { + picture.header.pic_order_cnt_lsb = pic_order_cnt_lsb; + picture.header.delta_pic_order_cnt_bottom = delta_pic_order_cnt_bottom; + } + Some(PicOrderCountLsb::FieldsDelta(delta)) => picture.header.delta_pic_order_cnt = delta, + None => {} + } + picture.header.mmco5 = matches!(hdr.dec_ref_pic_marking, Some(DecRefPicMarking::Adaptive(ref ops)) if ops.iter().any(|op| matches!(op, MemoryManagementControlOperation::AllRefPicturesUnused))); + let sps = AvcSps::from(sps); + let (poc, mmco5) = self.avc.compute(&picture, &sps)?; + let idr = picture.idr; + self.avc_update = Some(Some(ctx)); + return Ok((poc, rate, idr, mmco5)); + } + _ => {} + } + } + Err("missing AVC picture".to_owned()) + })(); + let (poc, rate, reset, mmco5) = match parsed { + Ok(parsed) => parsed, + Err(err) => { + self.reset(); + self.avc_update = Some(None); + return unit + .pts + .ok_or_else(|| anyhow::anyhow!("TS video timestamp cannot be derived: {err}")); + } + }; + let pts = self.stamp(unit.pts, poc, rate, reset)?; + // MMCO 5 renumbers the current picture to zero for subsequent POC + // derivation. Its presentation time is still the one just calculated. + if mmco5 { + self.anchor = rate.map(|rate| Anchor { poc: 0, pts, rate }); + } + Ok(pts) + } + + pub fn hevc<'a>(&mut self, unit: &Unit, params: impl Iterator) -> anyhow::Result { + let parsed = (|| { + let mut ctx = self.hevc_ctx.clone().unwrap_or_default(); + let bootstrap: Vec<_> = if self.hevc_ctx.is_none() { + params.cloned().collect() + } else { + Vec::new() + }; + for bytes in bootstrap.into_iter().chain(nals(&unit.data)?) { + let kind = (bytes[0] >> 1) & 63; + match kind { + 33 => { + let sps = scuffle_h265::SpsNALUnit::parse(&bytes[..]) + .map_err(|err| err.to_string())? + .rbsp; + Arc::make_mut(&mut ctx) + .sets + .insert(sps.sps_seq_parameter_set_id, Arc::new(sps)); + } + 34 => { + let rbsp = h264_parser::nal::ebsp_to_rbsp(bytes.get(2..).ok_or("short HEVC PPS")?); + let pps = HevcPps::parse(&rbsp)?; + Arc::make_mut(&mut ctx).pictures.insert(pps.id, pps); + } + 0..=31 => { + let second = *bytes.get(1).ok_or("short HEVC NAL header")?; + if bytes[0] & 1 != 0 || second & 0xf8 != 0 { + return Err("multilayer HEVC timing is unsupported".to_owned()); + } + let rbsp = h264_parser::nal::ebsp_to_rbsp(bytes.get(2..).ok_or("short HEVC slice")?); + let (lsb, width, rate) = hevc_prefix(&rbsp, kind, &ctx.pictures, &ctx.sets)?; + let idr = matches!(kind, 19 | 20); + let reset = idr || matches!(kind, 16..=18) || (kind == 21 && self.hevc.is_none()); + let max = 1i64 << width; + let prev = self.hevc.unwrap_or_default(); + let msb = if reset { + Some(0) + } else if lsb < prev.lsb && prev.lsb - lsb >= max / 2 { + prev.msb.checked_add(max) + } else if lsb > prev.lsb && lsb - prev.lsb > max / 2 { + prev.msb.checked_sub(max) + } else { + Some(prev.msb) + } + .ok_or("HEVC POC overflow")?; + let poc = msb.checked_add(lsb).ok_or("HEVC POC overflow")?; + if second & 7 == 1 && !matches!(kind, 0 | 2 | 4 | 6..=9) { + self.hevc = Some(HevcPoc { msb, lsb }); + } + self.hevc_update = Some(Some(ctx)); + return Ok((poc, rate, reset)); + } + _ => {} + } + } + Err("missing HEVC picture".to_owned()) + })(); + match parsed { + Ok((poc, rate, reset)) => self.stamp(unit.pts, poc, rate, reset), + Err(err) => { + self.reset(); + self.hevc_update = Some(None); + unit.pts + .ok_or_else(|| anyhow::anyhow!("TS video timestamp cannot be derived: {err}")) + } + } + } +} + +#[derive(Clone)] +struct HevcPps { + id: u32, + sps: u32, + output: bool, + extra: u8, +} + +impl HevcPps { + fn parse(rbsp: &[u8]) -> Result { + let parsed = (|| { + let mut r = h264_parser::bitreader::BitReader::new(rbsp); + let id = h264_parser::eg::read_ue(&mut r)?; + let sps = h264_parser::eg::read_ue(&mut r)?; + r.skip_bits(1)?; // dependent_slice_segments_enabled_flag, absent on the first slice + let output = r.read_flag()?; + let extra = r.read_bits(3)? as u8; + h264_parser::Result::Ok(Self { id, sps, output, extra }) + })(); + let pps = parsed.map_err(|err| err.to_string())?; + if pps.id > 63 || pps.sps > 15 { + return Err("invalid HEVC parameter-set id".to_owned()); + } + Ok(pps) + } +} + +fn hevc_prefix( + rbsp: &[u8], + kind: u8, + pictures: &BTreeMap, + sets: &BTreeMap>, +) -> Result<(i64, u32, Option), String> { + let mut r = h264_parser::bitreader::BitReader::new(rbsp); + if !r.read_flag().map_err(|err| err.to_string())? { + return Err("missing first independent HEVC slice".to_owned()); + } + if (16..=23).contains(&kind) { + r.skip_bits(1).map_err(|err| err.to_string())?; + } + let id = h264_parser::eg::read_ue(&mut r).map_err(|err| err.to_string())?; + let pps = pictures.get(&id).ok_or("missing HEVC PPS")?; + let sps = sets.get(&u64::from(pps.sps)).ok_or("missing HEVC SPS")?; + // The first independent slice has no segment address or inherited header. + // Stop as soon as its POC is available, before reference lists and slice data. + r.skip_bits(pps.extra.into()).map_err(|err| err.to_string())?; + let slice_type = h264_parser::eg::read_ue(&mut r).map_err(|err| err.to_string())?; + if slice_type > 2 { + return Err("invalid HEVC slice type".to_owned()); + } + if pps.output { + r.skip_bits(1).map_err(|err| err.to_string())?; + } + if sps.separate_colour_plane_flag { + r.skip_bits(2).map_err(|err| err.to_string())?; + } + let width = u32::from(sps.log2_max_pic_order_cnt_lsb_minus4) + 4; + let lsb = if matches!(kind, 19 | 20) { + 0 + } else { + i64::from(r.read_bits(width).map_err(|err| err.to_string())?) + }; + let info = sps.vui_parameters.as_ref().and_then(|vui| vui.vui_timing_info.as_ref()); + let rate = info + .filter(|info| info.poc_proportional_to_timing_flag) + .map(|info| Rate { + numerator: u64::from(info.num_units_in_tick.get()) + * (u64::from(info.num_ticks_poc_diff_one_minus1.unwrap_or(0)) + 1), + denominator: info.time_scale.get(), + }); + Ok((lsb, width, rate)) +} + +fn nals(data: &[u8]) -> Result, String> { + let mut data = data; + let mut nals = Vec::new(); + let mut iter = crate::codec::annexb::NalIterator::new(&mut data); + for nal in iter.by_ref() { + nals.push(nal.map_err(|err| err.to_string())?); + } + if let Some(nal) = iter.flush().map_err(|err| err.to_string())? { + nals.push(nal); + } + Ok(nals) +} + +#[derive(Default)] +struct AvcPoc { + frame_num: i64, + frame_offset: i64, + ref_msb: i64, + ref_lsb: i64, + ref_top: i64, + ref_bottom_field: bool, + ref_mmco5: bool, + mmco5: bool, +} + +impl AvcPoc { + fn compute(&mut self, slice: &AvcSlice, sps: &AvcSps) -> Result<(i64, bool), String> { + let hdr = &slice.header; + let idr = slice.idr; + let reference = slice.reference; + let frame = i64::from(hdr.frame_num); + let offset = if idr || self.mmco5 { + 0 + } else if frame < self.frame_num { + self.frame_offset + .checked_add(1 << (sps.log2_max_frame_num_minus4 + 4)) + .ok_or("AVC frame number overflow")? + } else { + self.frame_offset + }; + let (top, bottom, msb) = match sps.pic_order_cnt_type { + 0 => { + let (prev_msb, prev_lsb) = if idr { + (0, 0) + } else if self.ref_mmco5 { + (0, if self.ref_bottom_field { 0 } else { self.ref_top }) + } else { + (self.ref_msb, self.ref_lsb) + }; + let lsb = i64::from(hdr.pic_order_cnt_lsb); + let max = 1 << (sps.log2_max_pic_order_cnt_lsb_minus4 + 4); + let msb = if lsb < prev_lsb && prev_lsb - lsb >= max / 2 { + prev_msb.checked_add(max) + } else if lsb > prev_lsb && lsb - prev_lsb > max / 2 { + prev_msb.checked_sub(max) + } else { + Some(prev_msb) + } + .ok_or("AVC POC overflow")?; + let top = msb.checked_add(lsb).ok_or("AVC POC overflow")?; + let bottom = top + .checked_add(if hdr.field_pic_flag { + 0 + } else { + i64::from(hdr.delta_pic_order_cnt_bottom) + }) + .ok_or("AVC POC overflow")?; + (top, bottom, msb) + } + 1 | 2 => { + let mut abs = offset.checked_add(frame).ok_or("AVC POC overflow")?; + let base = if sps.pic_order_cnt_type == 2 { + if idr { + 0 + } else { + abs.checked_mul(2) + .and_then(|n| n.checked_sub(i64::from(!reference))) + .ok_or("AVC POC overflow")? + } + } else { + let cycle = i64::from(sps.num_ref_frames_in_pic_order_cnt_cycle); + if cycle == 0 { + abs = 0; + } + if !reference && abs > 0 { + abs -= 1; + } + let mut expected = 0i64; + if abs > 0 { + let offsets = &sps.offset_for_ref_frame[..cycle as usize]; + let sum: i64 = offsets.iter().map(|&n| i64::from(n)).sum(); + expected = ((abs - 1) / cycle).checked_mul(sum).ok_or("AVC POC overflow")?; + for &n in &offsets[..=((abs - 1) % cycle) as usize] { + expected = expected.checked_add(n.into()).ok_or("AVC POC overflow")?; + } + } + if !reference { + expected = expected + .checked_add(sps.offset_for_non_ref_pic.into()) + .ok_or("AVC POC overflow")?; + } + expected + }; + let top = base + .checked_add(i64::from(hdr.delta_pic_order_cnt[0])) + .ok_or("AVC POC overflow")?; + let bottom = if sps.pic_order_cnt_type == 2 { + Some(top) + } else if hdr.field_pic_flag { + top.checked_add(sps.offset_for_top_to_bottom_field.into()) + } else { + top.checked_add(sps.offset_for_top_to_bottom_field.into()) + .and_then(|n| n.checked_add(hdr.delta_pic_order_cnt[1].into())) + } + .ok_or("AVC POC overflow")?; + (top, bottom, 0) + } + _ => return Err("invalid AVC POC type".to_owned()), + }; + let poc = if hdr.field_pic_flag { + if hdr.bottom_field_flag { bottom } else { top } + } else { + top.min(bottom) + }; + let mmco5 = hdr.mmco5; + self.frame_num = if mmco5 { 0 } else { frame }; + self.frame_offset = if mmco5 { 0 } else { offset }; + self.mmco5 = mmco5; + if reference { + self.ref_msb = msb; + self.ref_lsb = hdr.pic_order_cnt_lsb.into(); + self.ref_top = top - if mmco5 { poc } else { 0 }; + self.ref_bottom_field = hdr.field_pic_flag && hdr.bottom_field_flag; + self.ref_mmco5 = mmco5; + } + Ok((poc, mmco5)) + } +} + +#[derive(Clone, Copy, Default)] +struct HevcPoc { + msb: i64, + lsb: i64, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn slice(frame: u16, lsb: u16, reference: bool, idr: bool) -> AvcSlice { + AvcSlice { + header: AvcHeader { + frame_num: frame, + pic_order_cnt_lsb: lsb.into(), + ..Default::default() + }, + idr, + reference, + } + } + + #[test] + fn sparse_reordered_pocs_and_wrap_use_the_previous_reference() { + let mut state = AvcPoc::default(); + let sps = AvcSps { + pic_order_cnt_type: 0, + log2_max_pic_order_cnt_lsb_minus4: 0, + ..Default::default() + }; + for (frame, lsb, reference, expected) in [ + (0, 0, true, 0), + (1, 6, true, 6), + (2, 2, false, 2), + (2, 7, false, 7), + (2, 14, true, 14), + (3, 2, true, 18), + (4, 15, false, 15), + (4, 4, true, 20), + ] { + assert_eq!( + state + .compute(&slice(frame, lsb, reference, frame == 0), &sps) + .unwrap() + .0, + expected + ); + } + } + + #[test] + fn poc_types_one_and_two_keep_cycle_offsets_and_nonreference_pictures() { + let mut sps = AvcSps { + pic_order_cnt_type: 1, + num_ref_frames_in_pic_order_cnt_cycle: 2, + offset_for_non_ref_pic: -1, + ..Default::default() + }; + sps.offset_for_ref_frame = vec![2, 4]; + let mut state = AvcPoc::default(); + for (frame, reference, expected) in [(0, true, 0), (1, true, 2), (2, true, 6), (3, false, 5), (3, true, 8)] { + assert_eq!( + state.compute(&slice(frame, 0, reference, frame == 0), &sps).unwrap().0, + expected + ); + } + sps.pic_order_cnt_type = 2; + let mut state = AvcPoc::default(); + for (frame, reference, expected) in [ + (0, true, 0), + (15, true, 30), + (0, false, 31), + (0, true, 32), + (1, true, 34), + ] { + assert_eq!( + state + .compute(&slice(frame, 0, reference, frame == 0 && expected == 0), &sps) + .unwrap() + .0, + expected + ); + } + } + + #[test] + fn mmco_five_renumbers_future_pictures_without_reusing_the_old_clock() { + let sps = AvcSps { + pic_order_cnt_type: 0, + log2_max_pic_order_cnt_lsb_minus4: 0, + ..Default::default() + }; + let mut state = AvcPoc::default(); + state.compute(&slice(0, 0, true, true), &sps).unwrap(); + let mut reset = slice(1, 6, true, false); + reset.header.mmco5 = true; + assert_eq!(state.compute(&reset, &sps).unwrap(), (6, true)); + assert_eq!(state.compute(&slice(1, 2, true, false), &sps).unwrap().0, 2); + } + + #[test] + fn inferred_clock_rounds_only_after_combining_anchor_and_signed_poc_distance() { + let mut clock = Clock::default(); + let rate = Some(Rate { + numerator: 1001, + denominator: 60_000, + }); + let anchor = Timestamp::from_scale(123_123, 90_000).unwrap(); + clock.stamp(Some(anchor), 6, rate, true).unwrap(); + for poc in [2, 7, 12, 60_006] { + let got = clock.stamp(None, poc, rate, false).unwrap(); + let expected = (123_123i128 * 2 + (i128::from(poc) - 6) * 3003) * 1_000_000 / 180_000; + assert_eq!(got.as_micros(), expected as u128); + } + assert!( + clock.stamp(None, 0, rate, true).is_err(), + "a new POC epoch needs its own anchor" + ); + assert!( + clock.stamp(None, 1, None, false).is_err(), + "a missing clock never repeats an earlier PTS" + ); + } + fn bits(input: &str) -> Vec { + input + .as_bytes() + .chunks(8) + .map(|chunk| { + chunk + .iter() + .enumerate() + .fold(0, |byte, (n, &bit)| byte | u8::from(bit == b'1') << (7 - n)) + }) + .collect() + } + + #[test] + fn hevc_poc_prefix_consumes_only_declared_fields_and_refuses_truncation() { + for irap in [false, true] { + for output in [false, true] { + for separate in [false, true] { + for extra in [0, 7] { + let mut sps = scuffle_h265::SpsNALUnit::parse(crate::codec::h265::fixtures::SPS) + .unwrap() + .rbsp; + sps.separate_colour_plane_flag = separate; + sps.log2_max_pic_order_cnt_lsb_minus4 = 12; + let sets = BTreeMap::from([(0, Arc::new(sps))]); + let pictures = BTreeMap::from([( + 0, + HevcPps { + id: 0, + sps: 0, + output, + extra, + }, + )]); + let head = format!( + "1{}1{}011{}{}", + if irap { "0" } else { "" }, + "0".repeat(extra as usize), + if output { "1" } else { "" }, + if separate { "10" } else { "" } + ); + let data = bits(&format!("{head}1010101010101010")); + assert_eq!( + hevc_prefix(&data, if irap { 21 } else { 1 }, &pictures, &sets) + .unwrap() + .0, + 0xaaaa + ); + for len in 0..data.len() { + assert!(hevc_prefix(&data[..len], if irap { 21 } else { 1 }, &pictures, &sets).is_err()); + } + // IDR still consumes the IRAP/output/colour-plane fields, + // but has no POC LSB in the slice header. + if irap { + assert_eq!(hevc_prefix(&bits(&head), 19, &pictures, &sets).unwrap().0, 0); + } + } + } + } + } + } + + #[test] + fn hevc_missing_parameter_sets_dependent_slices_and_unbounded_ue_are_refused() { + let pictures = BTreeMap::new(); + let sets = BTreeMap::new(); + for input in [vec![], vec![0; 64], bits("0"), bits("11")] { + assert!(hevc_prefix(&input, 1, &pictures, &sets).is_err()); + } + for input in [vec![], vec![0; 64], bits("00000010000001")] { + assert!(HevcPps::parse(&input).is_err()); + } + } +} + +#[derive(Default)] +struct AvcSlice { + header: AvcHeader, + idr: bool, + reference: bool, +} +#[derive(Default)] +struct AvcHeader { + frame_num: u16, + pic_order_cnt_lsb: u32, + field_pic_flag: bool, + bottom_field_flag: bool, + delta_pic_order_cnt_bottom: i32, + delta_pic_order_cnt: [i32; 2], + mmco5: bool, +} +#[derive(Default)] +struct AvcSps { + pic_order_cnt_type: u8, + log2_max_frame_num_minus4: u8, + log2_max_pic_order_cnt_lsb_minus4: u8, + offset_for_non_ref_pic: i32, + offset_for_top_to_bottom_field: i32, + num_ref_frames_in_pic_order_cnt_cycle: u8, + offset_for_ref_frame: Vec, +} +impl From<&SeqParameterSet> for AvcSps { + fn from(sps: &SeqParameterSet) -> Self { + let mut out = Self { + log2_max_frame_num_minus4: sps.log2_max_frame_num_minus4, + ..Default::default() + }; + match &sps.pic_order_cnt { + PicOrderCntType::TypeZero { + log2_max_pic_order_cnt_lsb_minus4, + } => out.log2_max_pic_order_cnt_lsb_minus4 = *log2_max_pic_order_cnt_lsb_minus4, + PicOrderCntType::TypeOne { + offset_for_non_ref_pic, + offset_for_top_to_bottom_field, + offsets_for_ref_frame, + .. + } => { + out.pic_order_cnt_type = 1; + out.offset_for_non_ref_pic = *offset_for_non_ref_pic; + out.offset_for_top_to_bottom_field = *offset_for_top_to_bottom_field; + out.num_ref_frames_in_pic_order_cnt_cycle = offsets_for_ref_frame.len() as u8; + out.offset_for_ref_frame = offsets_for_ref_frame.clone(); + } + PicOrderCntType::TypeTwo => out.pic_order_cnt_type = 2, + } + out + } +} + +#[derive(Default)] +struct AvcContext(h264_reader::Context); +impl Clone for AvcContext { + fn clone(&self) -> Self { + let mut ctx = h264_reader::Context::new(); + for sps in self.0.sps() { + ctx.put_seq_param_set(sps.clone()); + } + for pps in self.0.pps() { + ctx.put_pic_param_set(pps.clone()); + } + Self(ctx) + } +} + +#[derive(Clone, Default)] +struct HevcContext { + sets: BTreeMap>, + pictures: BTreeMap, +} diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index d1cefc9f0a8..62cae832ebb 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -120,7 +120,7 @@ case "$1" in # only Criterion estimates enter the trend report. criterion extra --no-fail-fast -p moq-e2ee -p moq-json -p moq-archive -p moq-cli -p moq-mux \ --bench protect --bench codec --bench table --bench allocations --bench window \ - --bench producer --bench play_buffer --bench jitter --bench decode_clock --bench ts_programs + --bench producer --bench play_buffer --bench jitter --bench decode_clock --bench ts_programs --bench ts_video ;; rust-uring) # One run per target makes an unsupported/skipped ring fail report diff --git a/test/ts/README.md b/test/ts/README.md index 7c9786214ce..52f3c1499b8 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -215,9 +215,14 @@ payload twice); only PES bytes of a first copy go on to MB/B. Video access units are read from the ES rather than taken from PES boundaries: one opens at the first delimiter, parameter set or SEI after the previous picture's slices (H.264 7.4.1.2.3, H.265 7.4.2.4.4), and H.222.0 requires a delimiter in each. A PES may -carry several, but only the first takes its timestamp, and deriving the rest from -the stream's own timing is not modelled, so that layout is refused. Video decode -times must strictly increase. +carry several, but only the AU starting in it takes its timestamp once. The model +can derive later presentation times from anchored POC and explicitly proportional +VUI timing; AVC also needs the TS timing descriptor's fixed-rate and temporal-POC +flags. Its frame-only profile supports AVC POC types 0/2 and HEVC without sub-picture +HRD. Inferred decode times additionally need each picture's HRD DPB output delay; +PTS is never substituted for DTS. Missing signaling, field/pic-struct timing, AVC +POC type 1 or MMCO5 with missing timestamps are refused. Video decode times must +strictly increase. Audio bytes enter B as they leave TB, so a frame that ends partway through a packet is complete once its own last byte has left, not the packet's, and B is checked @@ -257,6 +262,11 @@ An 8 s FFmpeg AVC Baseline@2.0 clip has no declared NAL HRD, so its MB uses the much smaller level defaults. It passes as encoded; restamping only its PCRs at 4x overflows MB (and TB), closing the separate MB negative-control gap. +Generated AVC and HEVC controls put two AUs in each PES, preserving every ES +byte's original arrival position. Proportional POC timing and HRD picture timing +make both pass; removing HRD picture timing makes both refuse their unknown decode +times. Their no-reordering and B-picture variants exercise separate PTS and DTS. + The packet layouts a capture cannot be edited into are built synthetically: a 10 Mb/s single-video stream carrying the Kyrion SPS, one access unit per PES, with a PCR packet between them. Each case fails without the handling it names: @@ -264,7 +274,7 @@ a PCR packet between them. Each case fails without the handling it names: | Case | Expected | |---|---| | as built | pass | -| two access units in one PES | refused | +| two access units in one PES without a declared clock | refused | | four adaptation-only packets after an access unit | TB overflow | | an access unit's first packet sent twice | pass | | the PMT declares a PCR PID that carries no PCR | `pcr-presence` fails | diff --git a/test/ts/compliance.py b/test/ts/compliance.py index 083a7c465ee..11c048b2fbe 100755 --- a/test/ts/compliance.py +++ b/test/ts/compliance.py @@ -29,6 +29,8 @@ import argparse import bisect import json +import re +from fractions import Fraction import subprocess import sys from collections import deque @@ -556,6 +558,8 @@ def read_programs(ts_path: str) -> list[tuple[int, list[Stream]]]: names = {d["#name"] for d in descriptors} names |= {f"registration:{d['format_identifier']}" for d in descriptors if "format_identifier" in d} names |= {"hrd_management_valid" for d in descriptors if d.get("hrd_management_valid") is True} + names |= {"temporal_poc_clock" for d in descriptors + if d.get("fixed_frame_rate") is True and d.get("temporal_poc") is True} names |= { f"tag{d['tag']}:" + "".join(d.get("#nodes", [])).replace(" ", "").lower() for d in descriptors @@ -884,18 +888,130 @@ def stream_params(kind: str, stream: Stream) -> tuple[Params, object]: DELIMITER = {"avc": 9, "hevc": 35} +def proportional_video_clock(stream: Stream, kind: str, count: int) -> list[tuple[int, Fraction, bool, tuple[int, Fraction] | None]]: + """Frame-only POC clocks and HRD output delays, parsed by FFmpeg. + + POC plus VUI supplies presentation time; picture-timing SEI's DPB output + delay supplies its distance from decode time (H.264 C.2, H.265 D.3.3). + Never let FFmpeg's guessed packet/frame timestamps become the timing oracle. + """ + if kind == "avc" and "temporal_poc_clock" not in stream.descriptors: + raise Refused("several access units have no declared proportional POC timestamp clock") + proc = subprocess.run(["ffmpeg", "-hide_banner", "-loglevel", "trace", "-f", "h264" if kind == "avc" else "hevc", + "-i", "pipe:0", "-map", "0:v:0", "-c", "copy", "-bsf:v", "trace_headers", "-f", "null", "-"], + input=bytes(stream.es), capture_output=True, check=False) + if proc.returncode: + raise Refused("cannot parse the video timing headers") + blocks: list[tuple[str, dict[str, int]]] = [] + for line in proc.stderr.decode(errors="replace").splitlines(): + if not line.startswith("[trace_headers @"): + continue + text = line.split("] ", 1)[1] + if text in ("Sequence Parameter Set", "Slice Header", "Slice Segment Header"): + blocks.append((text, {})) + elif text and text[0].isupper(): + blocks.append((text, {})) + elif blocks: + match = re.match(r"\d+\s+(\S+)\s+\S+\s+=\s+(-?\d+)$", text) + if match: + blocks[-1][1][match[1]] = int(match[2]) + sps: dict[str, int] = {} + previous_lsb = previous_msb = frame_num = frame_offset = 0 + seen = False + clocks = [] + timing = None + for name, fields in blocks: + if name == "Picture Timing": + timing = fields + continue + if name == "Sequence Parameter Set": + sps = fields + continue + if name not in ("Slice Header", "Slice Segment Header"): + continue + if kind == "avc": + if fields.get("first_mb_in_slice") != 0: + continue + if (sps.get("frame_mbs_only_flag") != 1 or sps.get("pic_struct_present_flag", 0) + or sps.get("fixed_frame_rate_flag") != 1): + raise Refused("missing DTS outside the frame-only video clock profile") + units, scale = sps.get("num_units_in_tick", 0), sps.get("time_scale", 0) + if units <= 0 or scale <= 0: + raise Refused("several access units have no proportional timestamp clock") + rate = tick = Fraction(units, scale) + output = None if timing is None else timing.get("dpb_output_delay") + reset = fields.get("nal_unit_type") == 5 + if 5 in [value for key, value in fields.items() if key.startswith("memory_management_control_operation")]: + raise Refused("missing DTS across an MMCO-5 reset is outside the model profile") + mode = sps.get("pic_order_cnt_type") + if reset: + previous_lsb = previous_msb = frame_num = frame_offset = 0 + if mode == 0: + maximum = 1 << (sps["log2_max_pic_order_cnt_lsb_minus4"] + 4) + lsb = fields["pic_order_cnt_lsb"] + msb = previous_msb + if lsb < previous_lsb and previous_lsb - lsb >= maximum // 2: + msb += maximum + elif lsb > previous_lsb and lsb - previous_lsb > maximum // 2: + msb -= maximum + poc = msb + lsb + if fields.get("nal_ref_idc"): + previous_lsb, previous_msb = lsb, msb + elif mode == 2: + if fields["frame_num"] < frame_num: + frame_offset += 1 << (sps["log2_max_frame_num_minus4"] + 4) + frame_num = fields["frame_num"] + poc = 0 if reset else 2 * (frame_offset + frame_num) - (fields.get("nal_ref_idc") == 0) + else: + raise Refused("missing DTS with AVC POC type 1 is outside the model profile") + else: + if fields.get("first_slice_segment_in_pic_flag") != 1: + continue + if (sps.get("field_seq_flag", 0) or sps.get("sub_pic_hrd_params_present_flag", 0) + or sps.get("vui_poc_proportional_to_timing_flag") != 1): + raise Refused("missing DTS outside the frame-only proportional HEVC clock profile") + units, scale = sps.get("vui_num_units_in_tick", 0), sps.get("vui_time_scale", 0) + if units <= 0 or scale <= 0: + raise Refused("several access units have no proportional timestamp clock") + tick = Fraction(units, scale) + rate = tick * (sps.get("vui_num_ticks_poc_diff_one_minus1", 0) + 1) + output = None if timing is None else timing.get("pic_dpb_output_delay") + nal = fields["nal_unit_type"] + reset = nal in range(16, 21) or (nal == 21 and not seen) + lsb = fields.get("slice_pic_order_cnt_lsb", 0) + maximum = 1 << (sps["log2_max_pic_order_cnt_lsb_minus4"] + 4) + msb = 0 if reset else previous_msb + if not reset: + if lsb < previous_lsb and previous_lsb - lsb >= maximum // 2: + msb += maximum + elif lsb > previous_lsb and lsb - previous_lsb > maximum // 2: + msb -= maximum + poc = msb + lsb + if fields.get("nuh_temporal_id_plus1") == 1 and nal not in (0, 2, 4, 6, 7, 8, 9): + previous_lsb, previous_msb = lsb, msb + if not rate: + raise Refused("several access units have no proportional timestamp clock") + clocks.append((poc, rate, reset, None if output is None else (output, tick))) + timing = None + seen = True + if len(clocks) != count: + raise Refused("video timing headers do not identify every access unit") + return clocks + + def video_units(stream: Stream, kind: str) -> list[AccessUnit]: """One access unit per coded picture, each decoded at its PES's DTS (or PTS). An access unit opens at the first delimiter, parameter set or SEI after the previous picture's last slice (H.264 7.4.1.2.3, H.265 7.4.2.4.4); H.222.0 2.14.1 and 2.17.1 also put a delimiter in every one, so a stream without them is refused. A PES may - carry several access units, but only the first takes its timestamp; the rest would - need decoding times derived from the stream's own timing (2.4.2.3), which this model - does not do, so such a layout is refused rather than graded as one unit. + carry several access units, but only the first takes its timestamp. Missing times + use a declared proportional POC clock and HRD picture-timing output delay + in the frame-only profile; absent signaling is refused. """ es = bytes(stream.es) starts = [] + owners = [] delimited = False after_vcl = True # the first opener starts the first whole access unit pos = es.find(b"\x00\x00\x01") @@ -906,6 +1022,9 @@ def video_units(stream: Stream, kind: str) -> list[AccessUnit]: # The zero_byte ahead of the start code belongs to this NAL unit (H.264 B.1.1, # H.265 B.2.1), so to this access unit rather than the last. starts.append(pos - 1 if pos and es[pos - 1] == 0 else pos) + # An optional zero_byte can be a preceding PES's trailing zero. + # The required start_code_prefix decides timestamp ownership. + owners.append(pos) after_vcl = False elif nal in VCL[kind]: after_vcl = True @@ -918,13 +1037,30 @@ def video_units(stream: Stream, kind: str) -> list[AccessUnit]: packets = [index for index, _h, body, _o in stream.packets if body] units: list[AccessUnit] = [] stamped: set[int] = set() + owner_pes = [bisect.bisect_right(pes_at, owner) - 1 for owner in owners] + needs_clock = len(set(owner_pes)) != len(owner_pes) or any(pes[k][1] is None for k in owner_pes) + clocks = proportional_video_clock(stream, kind, len(starts)) if needs_clock else [] + anchor = None for n, start in enumerate(starts): - k = bisect.bisect_right(pes_at, start) - 1 + k = owner_pes[n] _offset, pts, dts, index = pes[k] + explicit = pts is not None and k not in stamped + if clocks: + poc, rate, reset, output = clocks[n] + if reset: + anchor = None + if explicit: + anchor = (poc, Fraction(pts), rate) + elif anchor is not None and anchor[2] == rate: + pts = anchor[1] + (poc - anchor[0]) * rate * PTS_HZ + if output is None: + raise Refused("an inferred access unit has no HRD picture timing for its decode time") + delay, tick = output + dts = pts - delay * tick * PTS_HZ + else: + raise Refused("an access unit has no timestamp anchor for its POC clock") if pts is None: raise Refused(f"an access unit starts in the untimestamped PES at packet {index}") - if k in stamped: - raise Refused(f"the PES at packet {index} carries several access units, whose decode times are not derived") stamped.add(k) end = starts[n + 1] if n + 1 < len(starts) else stream.es_len first = packets[bisect.bisect_right(packet_at, start) - 1] diff --git a/test/ts/tstd-controls.py b/test/ts/tstd-controls.py index f915be75785..3be0504c82f 100755 --- a/test/ts/tstd-controls.py +++ b/test/ts/tstd-controls.py @@ -27,6 +27,7 @@ synthetic as built must pass two AUs, one PES an AU without its own timestamp refused + AVC/HEVC timed pairs two AUs per PES with proportional POC timing must pass adaptation burst four payload-less packets after an AU TB overflow (they cost TB) duplicate packet one AU's first packet sent twice (2.4.3.3) must pass (TB only, not MB) teletext burst eight teletext packets back to back TB overflow (EN 300 472's 480 B, @@ -487,6 +488,55 @@ def build(path: str) -> str: return build +def timed_video_pairs(kind: str, hrd: bool = True, reordered: bool = False): + """Two pictures per PES, with a declared proportional POC clock. + + Removing alternate PES headers preserves every ES byte at its original + packet/byte arrival position: the removed header becomes adaptation stuffing. + """ + def build(path: str) -> str: + source = path + ".source.ts" + command = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", + "testsrc2=size=32x32:rate=25", "-t", "2", "-an", "-c:v", "libx264" if kind == "avc" else "libx265", + "-preset", "ultrafast", "-tune", "zerolatency", "-threads", "1", "-b:v", "100k", "-maxrate", "1M", "-bufsize", "1M"] + if kind == "avc": + command += ["-profile:v", "high" if reordered else "baseline", "-level:v", "2.0", "-x264-params", + "keyint=50:min-keyint=50:scenecut=0:aud=1:force-cfr=1" + (":bframes=3:b-adapt=0" if reordered else ":bframes=0") + (":nal-hrd=vbr" if hrd else ""), + "-bsf:v", "h264_metadata=fixed_frame_rate_flag=1"] + else: + command += ["-x265-params", "keyint=50:min-keyint=50:scenecut=0:aud=1:level-idc=2:pools=none:frame-threads=1:log-level=error" + (":bframes=3:b-adapt=0:rc-lookahead=4" if reordered else ":bframes=0") + (":hrd=1" if hrd else ""), + "-bsf:v", "hevc_metadata=tick_rate=25:num_ticks_poc_diff_one=1"] + command += ["-f", "mpegts", "-muxrate", "500000", "-pcr_period", "20", "-muxdelay", "0.1", "-muxpreload", "0.1", source] + subprocess.run(command, check=True) + with open(source, "rb") as handle: + data = handle.read() + out = bytearray() + pictures = 0 + for at in range(0, len(data), 188): + pkt = data[at:at + 188] + pid = (pkt[1] & 31) * 256 + pkt[2] + start = 4 + (1 + pkt[4] if pkt[3] & 0x20 else 0) + if kind == "avc" and pid == 0x1000 and pkt[1] & 0x40: + descriptor = bytes.fromhex("2a027edf") + body = bytes.fromhex("e100f000 1be100f004") + descriptor + pmt = section(2, 1, body) + pkt = packet(pid, pkt[3] & 15, b"\0" + pmt, pusi=True) + elif pid == 0x100 and pkt[1] & 0x40 and pkt[3] & 0x10: + if pictures % 2: + body = pkt[start + 9 + pkt[start + 8]:] + original_af = pkt[5:start] if pkt[3] & 0x20 else b"\0" + field = 183 - len(body) + af = original_af.ljust(field, b"\xff") + pkt = bytes([pkt[0], pkt[1] & ~0x40, pkt[2], 0x30 | (pkt[3] & 15), field]) + af + body + pictures += 1 + out.extend(pkt) + assert pictures == 50 + with open(path, "wb") as handle: + handle.write(out) + return path + return build + + # A packet's byte j leaves an empty TB (j + 1) x 4 us after the packet starts arriving. BYTE_S = 8 / compliance.AUDIO_RX # Frame 0 ends at PES byte 14 + 576 = 590, byte 41 of the fourth packet: decode it 0.3 ms @@ -512,6 +562,12 @@ def build(path: str) -> str: ("AAC reference", encoded(True), "pass"), ("synthetic", built("synthetic"), "pass"), ("two AUs, one PES", built("two AUs, one PES"), "several access units"), + ("AVC timed pairs", timed_video_pairs("avc"), "pass"), + ("HEVC timed pairs", timed_video_pairs("hevc"), "pass"), + ("AVC reordered pairs", timed_video_pairs("avc", reordered=True), "pass"), + ("HEVC reordered pairs", timed_video_pairs("hevc", reordered=True), "pass"), + ("AVC no removal time", timed_video_pairs("avc", False), "HRD picture timing"), + ("HEVC no removal time", timed_video_pairs("hevc", False), "HRD picture timing"), ("adaptation burst", built("adaptation burst"), {"TB overflow"}), ("duplicate packet", built("duplicate packet"), "pass"), ("teletext burst", built("teletext burst"), {"TB overflow"}),