diff --git a/Cargo.lock b/Cargo.lock index 9eba9b2373e..1bcc82217a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3612,12 +3612,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "h264-parser" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c16ee729d5d4a75be0f5181682a17cf0fce29497d0fa5273d57fe477409f02e6" - [[package]] name = "h264-reader" version = "0.8.0" @@ -5352,6 +5346,7 @@ name = "moq-audio" version = "0.1.10" dependencies = [ "bench-harness", + "bitstream-io 4.10.0", "block2 0.6.2", "bytes", "cpal", @@ -5736,11 +5731,11 @@ dependencies = [ "anyhow", "base64 0.23.1", "bench-harness", + "bitstream-io 4.10.0", "bytes", "crc", "ebml-iterable", "futures", - "h264-parser", "h264-reader 0.9.0", "hang", "kio 0.6.2", diff --git a/Cargo.toml b/Cargo.toml index 9e8d4da072c..5420f962874 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,8 @@ axum = { version = "0.8", default-features = false, features = [ axum-server = "0.8" base64 = "0.23" bitflags = "2" +# MSB-first bit reading for codec configs; the version h264-reader builds on. +bitstream-io = "4.10" block2 = "0.6.2" bytes = "1" clap = { version = "4", features = ["derive"] } diff --git a/quest/m2/README.md b/quest/m2/README.md index fd1acc4fc4b..efcf71a4853 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -193,7 +193,6 @@ lite-07; bitrate enforcement is not part of this release. - [Auth session caps cost O(1)](/quest/m2/auth-session-index.md) - per-token and per-remote counts and a time-ordered expiry queue replace the scans, after #7211 - [The relay refuses a cluster peer it cannot dial](/quest/m2/relay-undialable-peers.md) - moq-tokio reports the schemes it can dial, and the relay refuses an undialable peer at startup instead of retrying forever - [Drop moq-auth's iroh transport](/quest/m2/auth-drop-iroh.md) - the next breaking moq-auth release drops `Transport::Iroh` and the JS `"iroh"` value -- [H.264 parsing with maintained crates](/quest/m2/h264-reader.md) - moq-mux parses SPS VUI with h264-reader and shares one bit reader across codec config parsers - [moq-transport finish interop](/quest/m2/ietf-finish.md) - the interop `finish` case runs on every even moq-transport draft instead of being skipped - [JS joining FETCH](/quest/m2/ietf-js-joining-fetch.md) - the JS publisher answers a joining FETCH that arrives after a quick finish, mirroring the Rust bound - [ashpd uses maintained D-Bus crates](/quest/m2/ashpd-maintained.md) - wait for a stable zbus 6 port, then remove the portal dependency exceptions diff --git a/quest/m2/aac-sbr-config.md b/quest/m2/aac-sbr-config.md index a90ea5606ec..3e3cdd25a44 100644 --- a/quest/m2/aac-sbr-config.md +++ b/quest/m2/aac-sbr-config.md @@ -11,7 +11,3 @@ Follow-up of [moq-dev/moq#7398](https://github.com/moq-dev/moq/pull/7398), which Decided 2026-10-30 (maintainer, follow-up planning after the /quest-afk run). Add the field to moq-mux's public config and delete moq-audio's own parse; test an implicit and an explicit SBR config. Public API: an additive field on `moq_mux::codec::aac::Config`. Wire: none. - -## Required - -- [Parse H.264 and codec config bits with maintained crates](/quest/m2/h264-reader.md) - its PR, #7398, rewrites both readers diff --git a/quest/m2/audit-deps.md b/quest/m2/audit-deps.md index 05203a80c1b..a06873de6cb 100644 --- a/quest/m2/audit-deps.md +++ b/quest/m2/audit-deps.md @@ -144,10 +144,9 @@ locks have none. Most duplicates are ecosystem transitions owned upstream: ### Hand-rolled code -Planned: [H.264 parsing](/quest/m2/h264-reader.md); moq-mux's avcC and -hvcC records go through mp4-atom, and relay metrics now render through -prometheus-client. Reviewed and kept, -each small or cold: about eleven `(d * 2).min(max)` backoff loops (moq-tokio +moq-mux's avcC and hvcC records go through mp4-atom, its H.264 SPS through +h264-reader, and relay metrics now render through prometheus-client. Reviewed +and kept, each small or cold: about eleven `(d * 2).min(max)` backoff loops (moq-tokio already has a `Backoff` that moq-relay's cluster loop could reuse), per-byte hex formatting in moq-auth and moq-hls, the STUN token bucket, the JS deep-equal pair, and duplicated WebVTT stamps and `jwt=` lookups. diff --git a/quest/m2/h264-reader.md b/quest/m2/h264-reader.md deleted file mode 100644 index 02ec5a8169d..00000000000 --- a/quest/m2/h264-reader.md +++ /dev/null @@ -1,33 +0,0 @@ -# [M] Parse H.264 and codec config bits with maintained crates - -## Goal - -moq-mux parses H.264 parameter sets with `h264-reader`, which moq-video and -moq-mux's TS video timing (`container/ts/video_timing.rs`) already use, instead of `h264-parser` plus a hand-written VUI and HRD walker, -and its codec config readers share one maintained bit reader. Catalog output -is unchanged. - -## Plan - -Found by the 2026-10-19 dependency audit. `h264-parser` stops at -`vui_parameters_present_flag`, so `rs/moq-mux/src/codec/h264/mod.rs` re-reads -the SPS bit by bit (about 230 lines of `Vui`, `sps_vui`, and `read_hrd`). -`h264-reader` 0.9 exposes the VUI, HRD, and bitstream restriction, and -`rs/moq-video/src/decode/vui.rs` duplicates moq-mux's `sps_color` on top of it. -The H.265 SPS fallback in `codec/h265/sps.rs` borrows `h264-parser`'s reader. - -Check first whether `h264-reader` covers the SEI recovery point and mastering -display parsing moq-mux does (`split.rs`, `color.rs`); keep those small -pieces hand-written if not. [Intra refresh import](/quest/m2/intra-refresh/h264-import.md) -reads `recovery_frame_cnt` through `h264-parser` today, so whichever lands -second adapts. - -Four MSB-first bit readers repeat the same job for codec config: VP9 -(`codec/vp9/mod.rs`), AAC (`codec/aac/mod.rs`, with a recording writer), the -TS Opus channel config (`container/ts/import.rs`), and moq-audio's own -AudioSpecificConfig scan (`rs/moq-audio/src/aac.rs`), which could call -moq-mux's parser instead. `bitstream-io` is already in the lock through -`h264-reader`. These are cold paths, so pick by API fit, not speed. - -The existing codec tests and catalog fixtures are the regression gate. No -public API or wire change. diff --git a/quest/m2/intra-refresh/h264-import.md b/quest/m2/intra-refresh/h264-import.md index 677340d4317..621d7a61eb0 100644 --- a/quest/m2/intra-refresh/h264-import.md +++ b/quest/m2/intra-refresh/h264-import.md @@ -19,8 +19,9 @@ follows the hang draft, which rounds up ([#5212](https://github.com/moq-dev/moq/pull/5212)). - `rs/moq-mux/src/codec/h264/split.rs`: `sei_has_recovery_point` returns the - `recovery_frame_cnt` the `h264-parser` crate already exposes on - `SeiPayload::RecoveryPoint`, and the split frame carries it. A count of zero + `recovery_frame_cnt` that `h264-reader`'s + `nal::sei::recovery_point::RecoveryPoint::read` parses from the message, and + the split frame carries it. A count of zero stays a plain keyframe. Recovery points with `broken_link_flag` set remain out of scope, as before. - The length-prefixed path bypasses the splitter: `rs/moq-mux/src/import/track.rs` diff --git a/quest/m2/sps-fuzz-h264-reader.md b/quest/m2/sps-fuzz-h264-reader.md index 8fb06b343fc..ee6e864b085 100644 --- a/quest/m2/sps-fuzz-h264-reader.md +++ b/quest/m2/sps-fuzz-h264-reader.md @@ -12,5 +12,4 @@ Public API: none. Wire: none. ## Required -- [Parse H.264 with maintained crates](/quest/m2/h264-reader.md) - its PR, #7398, moves SPS parsing to h264-reader - [Media parsers have fuzz targets](/quest/m2/media-fuzz.md) - its PR, #7313, adds the SPS target and the fuzz-findings epic diff --git a/rs/moq-audio/Cargo.toml b/rs/moq-audio/Cargo.toml index 89e4140570c..8a56745ae6e 100644 --- a/rs/moq-audio/Cargo.toml +++ b/rs/moq-audio/Cargo.toml @@ -68,6 +68,7 @@ pulseaudio = ["cpal?/pulseaudio"] aec = ["capture", "playback"] [dependencies] +bitstream-io = { workspace = true } bytes = { workspace = true } # Pure-Rust cross-platform audio I/O. Optional: only the `capture` and # `playback` features need it. Encode stays on unsafe-libopus, so a build diff --git a/rs/moq-audio/src/aac.rs b/rs/moq-audio/src/aac.rs index 1dab4c9efa5..d30d3b78eef 100644 --- a/rs/moq-audio/src/aac.rs +++ b/rs/moq-audio/src/aac.rs @@ -141,7 +141,7 @@ fn declares_sbr(description: &[u8]) -> bool { // The sync extension itself. Padding can't be mistaken for it: the pattern // has to match, name SBR or PS, and then set the flag. - if bits.left() < 16 || bits.read(11)? != SYNC_EXTENSION { + if bits.read(11)? != SYNC_EXTENSION { return Some(false); } if !matches!(object_type(bits)?, OBJECT_TYPE_SBR | OBJECT_TYPE_PS) { @@ -151,7 +151,7 @@ fn declares_sbr(description: &[u8]) -> bool { Some(bits.read(1)? == 1) } - scan(&mut Bits::new(description)).unwrap_or(false) + scan(&mut Bits::endian(description, bitstream_io::BigEndian)).unwrap_or(false) } /// An audioObjectType, which escapes to a second field once it runs out of room. @@ -163,34 +163,16 @@ fn object_type(bits: &mut Bits) -> Option { } /// A big-endian bit cursor, which is how an AudioSpecificConfig is packed. -struct Bits<'a> { - data: &'a [u8], - pos: usize, -} - -impl<'a> Bits<'a> { - fn new(data: &'a [u8]) -> Self { - Self { data, pos: 0 } - } - - fn left(&self) -> usize { - self.data.len() * 8 - self.pos - } +type Bits<'a> = bitstream_io::BitReader<&'a [u8], bitstream_io::BigEndian>; - /// The next `count` bits, or `None` when the config is shorter than that. - fn read(&mut self, count: usize) -> Option { - if count > 32 || self.left() < count { - return None; - } - - let mut value = 0; - for _ in 0..count { - let bit = (self.data[self.pos / 8] >> (7 - self.pos % 8)) & 1; - value = (value << 1) | u32::from(bit); - self.pos += 1; - } +/// Reads the next `count` bits, or `None` when the config is shorter than that. +trait ReadBits { + fn read(&mut self, count: u32) -> Option; +} - Some(value) +impl ReadBits for Bits<'_> { + fn read(&mut self, count: u32) -> Option { + bitstream_io::BitRead::read_var(self, count).ok() } } diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 4b24fcbd150..da6d091fa2e 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -15,10 +15,10 @@ categories = ["multimedia", "network-programming", "web-programming"] [dependencies] anyhow = { workspace = true } base64 = { workspace = true } +bitstream-io = { workspace = true } bytes = { workspace = true } crc = "3" ebml-iterable = { workspace = true } -h264-parser = { version = "0.4.2" } h264-reader = "0.9.0" hang = { workspace = true } kio = { workspace = true, features = ["runtime"] } diff --git a/rs/moq-mux/src/codec/aac/mod.rs b/rs/moq-mux/src/codec/aac/mod.rs index b11cb599fdc..50f7175b2c7 100644 --- a/rs/moq-mux/src/codec/aac/mod.rs +++ b/rs/moq-mux/src/codec/aac/mod.rs @@ -162,7 +162,7 @@ impl Config { /// Read an AudioSpecificConfig's leading fields: the audioObjectType, sample rate, and /// channelConfiguration. fn read_header(reader: &mut BitReader) -> Result<(u8, u32, u8)> { - if reader.buf.remaining() < 2 { + if reader.remaining().is_none_or(|remaining| remaining < 2) { return Err(Error::ConfigTooShort); } let object_type = read_object_type(reader)?; @@ -370,15 +370,14 @@ const SAMPLE_RATES: [u32; 13] = [ 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350, ]; -/// MSB-first bit reader that pulls bytes from a [`Buf`] on demand. +/// MSB-first bit reader that pulls bytes from a [`Buf`] on demand, so the buffer advances only +/// past the bytes it reads. /// /// AudioSpecificConfig is bit-packed: an explicit 24-bit sample rate pushes the /// following channelConfiguration off byte boundaries, so the fields can't be /// read a whole byte at a time. struct BitReader<'a, T: Buf> { - buf: &'a mut T, - current: u8, - bits_left: u8, + bits: bitstream_io::BitReader, bitstream_io::BigEndian>, /// Every bit read is copied here when set, with alignment redone on the writer's own bytes. record: Option, } @@ -386,16 +385,19 @@ struct BitReader<'a, T: Buf> { impl<'a, T: Buf> BitReader<'a, T> { fn new(buf: &'a mut T) -> Self { Self { - buf, - current: 0, - bits_left: 0, + bits: bitstream_io::BitReader::new(buf.reader()), record: None, } } + /// Bytes left in the buffer, `None` partway through one. + fn remaining(&mut self) -> Option { + Some(self.bits.reader()?.get_ref().remaining()) + } + /// Skip to the next byte boundary. fn align(&mut self) { - self.bits_left = 0; + bitstream_io::BitRead::byte_align(&mut self.bits); if let Some(record) = &mut self.record { record.align(); } @@ -403,18 +405,7 @@ impl<'a, T: Buf> BitReader<'a, T> { /// Read `n` bits (n <= 32) MSB-first, returning `short` if the buffer runs dry. fn read(&mut self, n: u8, short: Error) -> Result { - let mut value = 0u32; - for _ in 0..n { - if self.bits_left == 0 { - if !self.buf.has_remaining() { - return Err(short); - } - self.current = self.buf.get_u8(); - self.bits_left = 8; - } - self.bits_left -= 1; - value = (value << 1) | u32::from((self.current >> self.bits_left) & 1); - } + let value = bitstream_io::BitRead::read_var(&mut self.bits, n.into()).map_err(|_| short)?; if let Some(record) = &mut self.record { record.write(n, value); } diff --git a/rs/moq-mux/src/codec/annexb.rs b/rs/moq-mux/src/codec/annexb.rs index bce9437f9d0..5c8e51ac9bb 100644 --- a/rs/moq-mux/src/codec/annexb.rs +++ b/rs/moq-mux/src/codec/annexb.rs @@ -6,6 +6,63 @@ use std::collections::BTreeMap; use bytes::{Buf, Bytes, BytesMut}; +/// A bit reader over an H.264 or H.265 parameter set or SEI NAL unit's RBSP: the bytes after its +/// `header`-byte NAL header, with emulation prevention removed as they are read. +/// +/// These NAL units end in `rbsp_trailing_bits`, so a trailing zero byte is Annex B's +/// `trailing_zero_8bits` left behind by the splitter, and is dropped rather than refused as an +/// emulated start code. +pub(crate) fn rbsp(nal: &[u8], header: usize) -> h264_reader::rbsp::BitReader> { + h264_reader::rbsp::BitReader::new(rbsp_bytes_reader(nal, header)) +} + +/// The RBSP bytes of an H.264 or H.265 parameter set or SEI NAL unit, as [`rbsp`] reads them. +/// `None` when the payload holds a byte sequence emulation prevention forbids. +pub(crate) fn rbsp_bytes(nal: &[u8], header: usize) -> Option> { + use std::io::Read; + + let mut out = Vec::with_capacity(nal.len()); + rbsp_bytes_reader(nal, header).read_to_end(&mut out).ok()?; + Some(out) +} + +/// The `(payloadType, payload)` of each message in an H.264 or H.265 SEI RBSP, which share the +/// `sei_message` syntax. A malformed message ends the iteration. Unlike h264-reader's +/// `SeiReader`, a payload of any size is skipped without a copy. +pub(crate) fn sei_messages(rbsp: &[u8]) -> impl Iterator { + let mut buf = rbsp; + std::iter::from_fn(move || { + // Each message needs at least a type and size byte; the last byte is the RBSP stop bit. + if buf.len() <= 2 { + return None; + } + let kind = sei_value(&mut buf)?; + let size = sei_value(&mut buf)?; + let (payload, rest) = buf.split_at_checked(size as usize)?; + buf = rest; + Some((kind, payload)) + }) +} + +/// An SEI `payloadType` or `payloadSize`: a run of 0xFF bytes, each adding 255, then the remainder. +fn sei_value(buf: &mut &[u8]) -> Option { + let mut value = 0u32; + loop { + let (&byte, rest) = buf.split_first()?; + *buf = rest; + value = value.checked_add(byte.into())?; + if byte != 0xFF { + return Some(value); + } + } +} + +fn rbsp_bytes_reader(nal: &[u8], header: usize) -> h264_reader::rbsp::ByteReader<&[u8]> { + let payload = nal.get(header..).unwrap_or_default(); + let end = payload.iter().rposition(|&b| b != 0).map_or(0, |last| last + 1); + h264_reader::rbsp::ByteReader::without_skip(&payload[..end]) +} + /// The four-byte Annex B start code preceding a NAL unit. pub const START_CODE: Bytes = Bytes::from_static(&[0, 0, 0, 1]); @@ -189,21 +246,22 @@ impl InBandCodec { /// Insert parameter set `nal` (header included) under the id it defines, replacing any /// earlier set with that id. pub(crate) fn insert_parameter_set(self, sets: &mut ParameterSets, nal: &Bytes) -> Result<()> { - use h264_parser::{bitreader::BitReader, eg::read_ue}; + use h264_reader::rbsp::BitRead; let nal_type = self.nal_type(nal); let header = match self { Self::H264 => 1, Self::H265 => 2, }; - let rbsp = h264_parser::nal::ebsp_to_rbsp(nal.get(header..).unwrap_or_default()); - let r = &mut BitReader::new(&rbsp); + let r = &mut rbsp(nal, header); let id = match (self, nal_type) { // seq_parameter_set_id follows profile_idc, the constraint flags, and level_idc. - (Self::H264, 7) => r.skip_bits(24).and_then(|()| read_ue(r)), + (Self::H264, 7) => r + .skip(24, "profile_idc..level_idc") + .and_then(|()| r.read_ue("seq_parameter_set_id")), // pic_parameter_set_id opens a PPS in both codecs. - (Self::H264, 8) | (Self::H265, 34) => read_ue(r), - (Self::H265, 32) => r.read_bits(4), // vps_video_parameter_set_id + (Self::H264, 8) | (Self::H265, 34) => r.read_ue("pic_parameter_set_id"), + (Self::H265, 32) => r.read::<4, u32>("vps_video_parameter_set_id"), (Self::H265, 33) => crate::codec::h265::sps_id(r), _ => return Err(Error::ParameterSetId(nal_type)), }; diff --git a/rs/moq-mux/src/codec/color.rs b/rs/moq-mux/src/codec/color.rs index 92da681ed3f..133ee179acc 100644 --- a/rs/moq-mux/src/codec/color.rs +++ b/rs/moq-mux/src/codec/color.rs @@ -104,8 +104,11 @@ pub(crate) fn access_unit( ) -> Hdr { let mut hdr = Hdr::default(); let mut scan = |nal: &[u8]| { - if nal.len() > header && sei(nal[0]) { - hdr.update(self::sei(&h264_parser::nal::ebsp_to_rbsp(&nal[header..]))); + if nal.len() > header + && sei(nal[0]) + && let Some(rbsp) = crate::codec::annexb::rbsp_bytes(nal, header) + { + hdr.update(self::sei(&rbsp)); } }; match length_size { @@ -137,15 +140,7 @@ const SEI_CONTENT_LIGHT: u32 = 144; /// (ITU-T H.264 D.1.29 and D.1.31, H.265 D.2.28 and D.2.35). A malformed message ends the scan. pub(crate) fn sei(rbsp: &[u8]) -> Hdr { let mut hdr = Hdr::default(); - let mut buf = rbsp; - // Each message needs at least a type and size byte; the last byte is the RBSP stop bit. - while buf.len() > 2 { - let Some(kind) = sei_value(&mut buf) else { break }; - let Some(size) = sei_value(&mut buf) else { break }; - let Some((payload, rest)) = buf.split_at_checked(size as usize) else { - break; - }; - buf = rest; + for (kind, payload) in crate::codec::annexb::sei_messages(rbsp) { match kind { SEI_MASTERING_DISPLAY if payload.len() >= 24 => hdr.mastering = Some(sei_mastering(payload)), SEI_CONTENT_LIGHT if payload.len() >= 4 => { @@ -214,19 +209,6 @@ fn leb128(buf: &mut &[u8]) -> Option { None } -/// An SEI `payloadType` or `payloadSize`: a run of 0xFF bytes, each adding 255, then the remainder. -fn sei_value(buf: &mut &[u8]) -> Option { - let mut value = 0u32; - loop { - let (&byte, rest) = buf.split_first()?; - *buf = rest; - value = value.checked_add(byte.into())?; - if byte != 0xFF { - return Some(value); - } - } -} - /// A `mastering_display_colour_volume` payload. Chromaticities count 0.00002 and luminance /// 0.0001 cd/m². The primaries are conventionally green, blue, red, which libavcodec assumes too. fn sei_mastering(p: &[u8]) -> Mastering { @@ -311,7 +293,7 @@ pub(crate) mod test { use crate::codec::h264::fixtures::{SEI_CONTENT_LIGHT, SEI_MASTERING}; let mut hdr = Hdr::default(); for nal in [SEI_MASTERING, SEI_CONTENT_LIGHT] { - hdr.update(sei(&h264_parser::nal::ebsp_to_rbsp(&nal[1..]))); + hdr.update(sei(&crate::codec::annexb::rbsp_bytes(nal, 1).unwrap())); } assert_eq!(hdr, hdr10()); } diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 407420cd792..d5afd3ed75e 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -714,7 +714,7 @@ mod tests { ]; let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; // recovery-point SEI (payload type 6), then a non-IDR I-slice (type 1). - let sei: &[u8] = &[0x06, 0x06, 0x02, 0x00, 0x40, 0x80]; + let sei: &[u8] = &[0x06, 0x06, 0x01, 0x84, 0x80]; let islice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; let mut annexb = BytesMut::new(); diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index c0bfd689625..ebbd7f42293 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -19,6 +19,8 @@ pub use import::*; pub use split::*; use bytes::{Buf, Bytes, BytesMut}; +use h264_reader::nal::sps::{FrameMbsFlags, SeqParameterSet}; +use h264_reader::rbsp::{BitRead, BitReaderError}; use crate::codec::annexb::{InBandCodec, ParameterSets}; @@ -146,7 +148,7 @@ pub type Result = std::result::Result; /// Parsed H.264 SPS (Sequence Parameter Set) NAL. /// -/// Wraps [`h264_parser::Sps`] with the codec-config fields that the hang +/// Reads the SPS with [`h264_reader`] and keeps the codec-config fields that the hang /// catalog records: profile_idc, level_idc, and the packed constraint_set /// flags. The first byte of `nal` must be the NAL header. #[derive(Debug, Clone)] @@ -169,18 +171,99 @@ impl Sps { if nal.len() < 4 { return Err(Error::SpsTooShort); } - let rbsp = h264_parser::nal::ebsp_to_rbsp(&nal[1..]); - let sps = h264_parser::Sps::parse(&rbsp).map_err(|_| Error::SpsParse)?; + let sps = read_sps(nal).ok_or(Error::SpsParse)?; + let (coded_width, coded_height) = sps.pixel_dimensions().map_err(|_| Error::SpsParse)?; Ok(Self { - profile: sps.profile_idc, - constraints: pack_constraint_flags(&sps), + profile: sps.profile_idc.into(), + // constraint_set0..5_flag; the two reserved bits after them stay clear. + constraints: u8::from(sps.constraint_flags) & 0xfc, level: sps.level_idc, - coded_width: sps.width, - coded_height: sps.height, + coded_width, + coded_height, }) } } +/// Parse an SPS NAL unit (header included), `None` when it is malformed. +/// +/// h264-reader refuses the whole SPS when its VUI is cut short or breaks a constraint, but the +/// codec string and coded size only need the fields before it. So a refused SPS is read again +/// without its VUI, as though `vui_parameters_present_flag` were 0, and declares nothing the VUI +/// would have. +fn read_sps(nal: &[u8]) -> Option { + use crate::codec::annexb::rbsp; + + SeqParameterSet::from_bits(rbsp(nal, 1)).ok().or_else(|| { + let sps = SeqParameterSet::from_bits(WithoutVui(rbsp(nal, 1))).ok()?; + tracing::debug!("H.264 SPS VUI refused by h264-reader; using the SPS without it"); + Some(sps) + }) +} + +/// A reader that reports `vui_parameters_present_flag` as 0 and leaves the rest of the RBSP +/// unread, so h264-reader parses an SPS up to its VUI. +struct WithoutVui(R); + +impl BitRead for WithoutVui { + fn read_ue(&mut self, name: &'static str) -> std::result::Result { + self.0.read_ue(name) + } + + fn read_se(&mut self, name: &'static str) -> std::result::Result { + self.0.read_se(name) + } + + fn read_bit(&mut self, name: &'static str) -> std::result::Result { + match name { + "vui_parameters_present_flag" => Ok(false), + _ => self.0.read_bit(name), + } + } + + fn read( + &mut self, + name: &'static str, + ) -> std::result::Result { + self.0.read::(name) + } + + fn read_var( + &mut self, + bit_count: u32, + name: &'static str, + ) -> std::result::Result { + self.0.read_var(bit_count, name) + } + + fn read_to( + &mut self, + name: &'static str, + ) -> std::result::Result { + self.0.read_to(name) + } + + fn skip(&mut self, bit_count: u32, name: &'static str) -> std::result::Result<(), BitReaderError> { + self.0.skip(bit_count, name) + } + + fn byte_aligned(&self) -> bool { + self.0.byte_aligned() + } + + fn has_more_rbsp_data(&mut self, name: &'static str) -> std::result::Result { + self.0.has_more_rbsp_data(name) + } + + /// The VUI and the trailing bits after it go unread. + fn finish_rbsp(self) -> std::result::Result<(), BitReaderError> { + Ok(()) + } + + fn finish_sei_payload(self) -> std::result::Result<(), BitReaderError> { + self.0.finish_sei_payload() + } +} + /// The reordering an SPS NAL unit declares in its VUI: `max_num_reorder_frames` from the /// bitstream restriction, and the frame period when `fixed_frame_rate_flag` is set. /// @@ -200,12 +283,8 @@ pub(crate) fn sps_hrd(nal: &[u8]) -> Option { /// Whether an SPS NAL unit codes field pictures or field macroblock pairs (`frame_mbs_only_flag` 0). pub(crate) fn sps_interlaced(nal: &[u8]) -> Option { - if nal.len() < 4 { - return None; - } - let rbsp = h264_parser::nal::ebsp_to_rbsp(&nal[1..]); - let sps = h264_parser::Sps::parse(&rbsp).ok()?; - Some(!sps.frame_mbs_only_flag) + let sps = read_sps(nal)?; + Some(!matches!(sps.frame_mbs_flags, FrameMbsFlags::Frames)) } /// What an SPS VUI declares about picture timing, sample shape, and colour. @@ -225,6 +304,45 @@ struct Vui { } impl Vui { + /// Read the VUI of a parsed SPS (ITU-T H.264 E.1.1), `None` when it carries none. + fn new(sps: &SeqParameterSet) -> Option { + let vui = sps.vui_parameters.as_ref()?; + let sample_aspect = vui + .aspect_ratio_info + .as_ref() + .and_then(|info| info.get()) + .map(|(width, height)| (u32::from(width), u32::from(height))); + let color = vui.video_signal_type.as_ref().and_then(|signal| { + let (primaries, transfer, matrix) = match &signal.colour_description { + Some(c) => (c.colour_primaries, c.transfer_characteristics, c.matrix_coefficients), + None => (2, 2, 2), + }; + crate::codec::color::cicp(primaries, transfer, matrix, Some(signal.video_full_range_flag)) + }); + // With a fixed rate, a frame lasts two ticks (E.2.1). + let period = vui + .timing_info + .as_ref() + .filter(|t| t.fixed_frame_rate_flag && t.num_units_in_tick > 0 && t.time_scale > 0) + .map(|t| (2 * u64::from(t.num_units_in_tick), u64::from(t.time_scale))); + // The last schedule is the one a decoder conforms to (E.2.2). + let hrd = vui.nal_hrd_parameters.as_ref().and_then(|hrd| { + let spec = hrd.cpb_specs.last()?; + Some(crate::codec::video::Hrd { + bit_rate: (u64::from(spec.bit_rate_value_minus1) + 1) << (6 + hrd.bit_rate_scale), + cpb_size: (u64::from(spec.cpb_size_value_minus1) + 1) << (4 + hrd.cpb_size_scale), + }) + }); + Some(Self { + sample_aspect, + color, + frame_only: matches!(sps.frame_mbs_flags, FrameMbsFlags::Frames), + period, + depth: vui.bitstream_restrictions.as_ref().map(|b| b.max_num_reorder_frames), + hrd, + }) + } + /// Only a fixed frame rate on a frame-only SPS fixes the picture period; field pictures /// need their slice header's `pic_struct` (ITU-T H.264 E.2.1, Table E-6). fn framerate(&self) -> Option { @@ -254,187 +372,7 @@ impl Vui { /// `None` when the SPS carries no VUI or fails to parse. fn sps_vui(nal: &[u8]) -> Option { - if nal.len() < 4 { - return None; - } - let rbsp = h264_parser::nal::ebsp_to_rbsp(&nal[1..]); - let sps = h264_parser::Sps::parse(&rbsp).ok()?; - if !sps.vui_parameters_present_flag { - return None; - } - vui(&rbsp).ok().flatten() -} - -/// Walk an SPS RBSP to its VUI (ITU-T H.264 7.3.2.1.1, E.1.1). `h264_parser` stops at -/// `vui_parameters_present_flag` without exposing its position, so this re-reads the fields -/// before it. -fn vui(rbsp: &[u8]) -> h264_parser::Result> { - use h264_parser::bitreader::BitReader; - use h264_parser::eg::{read_se, read_ue}; - - let mut r = BitReader::new(rbsp); - let profile_idc = r.read_u8()?; - r.skip_bits(16)?; // constraint flags, reserved bits, level_idc - read_ue(&mut r)?; // seq_parameter_set_id - if matches!( - profile_idc, - 100 | 110 | 122 | 244 | 44 | 83 | 86 | 118 | 128 | 138 | 139 | 134 | 135 - ) { - let chroma_format_idc = read_ue(&mut r)?; - if chroma_format_idc == 3 { - r.skip_bits(1)?; // separate_colour_plane_flag - } - read_ue(&mut r)?; // bit_depth_luma_minus8 - read_ue(&mut r)?; // bit_depth_chroma_minus8 - r.skip_bits(1)?; // qpprime_y_zero_transform_bypass_flag - if r.read_flag()? { - let lists = if chroma_format_idc == 3 { 12 } else { 8 }; - for i in 0..lists { - if r.read_flag()? { - let size = if i < 6 { 16 } else { 64 }; - let (mut last, mut next) = (8i32, 8i32); - for _ in 0..size { - if next != 0 { - next = (last + read_se(&mut r)?).rem_euclid(256); - } - if next != 0 { - last = next; - } - } - } - } - } - } - read_ue(&mut r)?; // log2_max_frame_num_minus4 - match read_ue(&mut r)? { - 0 => { - read_ue(&mut r)?; // log2_max_pic_order_cnt_lsb_minus4 - } - 1 => { - r.skip_bits(1)?; // delta_pic_order_always_zero_flag - read_se(&mut r)?; // offset_for_non_ref_pic - read_se(&mut r)?; // offset_for_top_to_bottom_field - for _ in 0..read_ue(&mut r)? { - read_se(&mut r)?; // offset_for_ref_frame - } - } - _ => {} - } - read_ue(&mut r)?; // max_num_ref_frames - r.skip_bits(1)?; // gaps_in_frame_num_value_allowed_flag - read_ue(&mut r)?; // pic_width_in_mbs_minus1 - read_ue(&mut r)?; // pic_height_in_map_units_minus1 - let frame_only = r.read_flag()?; // frame_mbs_only_flag - if !frame_only { - r.skip_bits(1)?; // mb_adaptive_frame_field_flag - } - r.skip_bits(1)?; // direct_8x8_inference_flag - if r.read_flag()? { - for _ in 0..4 { - read_ue(&mut r)?; // frame_crop_*_offset - } - } - if !r.read_flag()? { - return Ok(None); - } - - // vui_parameters() - let sample_aspect = match r.read_flag()? { - true => match r.read_u8()? { - 255 => Some((u32::from(r.read_u16()?), u32::from(r.read_u16()?))), - idc => crate::codec::video::sample_aspect(idc), - }, - false => None, - }; - if r.read_flag()? { - r.skip_bits(1)?; // overscan_appropriate_flag - } - let mut color = None; - if r.read_flag()? { - r.skip_bits(3)?; // video_format - let full_range = r.read_flag()?; - let (primaries, transfer, matrix) = match r.read_flag()? { - true => (r.read_u8()?, r.read_u8()?, r.read_u8()?), - false => (2, 2, 2), - }; - color = crate::codec::color::cicp(primaries, transfer, matrix, Some(full_range)); - } - if r.read_flag()? { - read_ue(&mut r)?; // chroma_sample_loc_type_top_field - read_ue(&mut r)?; // chroma_sample_loc_type_bottom_field - } - let mut period = None; - if r.read_flag()? { - let units = r.read_bits(32)?; - let scale = r.read_bits(32)?; - // With a fixed rate, a frame lasts two ticks (E.2.1). - if r.read_flag()? && units > 0 && scale > 0 { - period = Some((2 * u64::from(units), u64::from(scale))); - } - } - let nal_hrd = r.read_flag()?; - let hrd = match nal_hrd { - true => Some(read_hrd(&mut r)?), - false => None, - }; - let vcl_hrd = r.read_flag()?; - if vcl_hrd { - read_hrd(&mut r)?; - } - if nal_hrd || vcl_hrd { - r.skip_bits(1)?; // low_delay_hrd_flag - } - r.skip_bits(1)?; // pic_struct_present_flag - if !r.read_flag()? { - return Ok(Some(Vui { - sample_aspect, - color, - frame_only, - period, - depth: None, - hrd, - })); - } - r.skip_bits(1)?; // motion_vectors_over_pic_boundaries_flag - for _ in 0..4 { - read_ue(&mut r)?; // max_bytes_per_pic_denom .. log2_max_mv_length_vertical - } - let depth = read_ue(&mut r)?; // max_num_reorder_frames - Ok(Some(Vui { - sample_aspect, - color, - frame_only, - period, - depth: Some(depth), - hrd, - })) -} - -/// Read `hrd_parameters()` (ITU-T H.264 E.1.2) down to its last schedule (E.2.2). -fn read_hrd(r: &mut h264_parser::bitreader::BitReader) -> h264_parser::Result { - use h264_parser::eg::read_ue; - - let cpb_cnt_minus1 = read_ue(r)?; - if cpb_cnt_minus1 > 31 { - return Err(h264_parser::Error::MalformedSps("cpb_cnt_minus1 out of range".into())); - } - let bit_rate_scale = r.read_bits(4)?; - let cpb_size_scale = r.read_bits(4)?; - let mut hrd = crate::codec::video::Hrd { - bit_rate: 0, - cpb_size: 0, - }; - for _ in 0..=cpb_cnt_minus1 { - let bit_rate = u64::from(read_ue(r)?) + 1; - let cpb_size = u64::from(read_ue(r)?) + 1; - r.skip_bits(1)?; // cbr_flag - hrd = crate::codec::video::Hrd { - bit_rate: bit_rate << (6 + bit_rate_scale), - cpb_size: cpb_size << (4 + cpb_size_scale), - }; - } - r.skip_bits(20)?; // four delay/offset lengths - Ok(hrd) + Vui::new(&read_sps(nal)?) } /// Parsed AVCDecoderConfigurationRecord (ISO/IEC 14496-15 §5.3.3.1.2). @@ -519,15 +457,6 @@ fn encode_avcc(record: &mp4_atom::Avcc) -> Bytes { out.freeze() } -fn pack_constraint_flags(sps: &h264_parser::Sps) -> u8 { - ((sps.constraint_set0_flag as u8) << 7) - | ((sps.constraint_set1_flag as u8) << 6) - | ((sps.constraint_set2_flag as u8) << 5) - | ((sps.constraint_set3_flag as u8) << 4) - | ((sps.constraint_set4_flag as u8) << 3) - | ((sps.constraint_set5_flag as u8) << 2) -} - /// Build an AVCDecoderConfigurationRecord (ISO/IEC 14496-15 §5.3.3.1.2) from the /// given SPS and PPS NALs. At least one SPS is required; the profile/level fields /// are read from the first SPS. A stream may legitimately carry several distinct @@ -841,6 +770,30 @@ mod tests { assert_eq!(sps_color(fixtures::SPS_IPB), None, "a VUI without a video signal type"); } + /// An SPS whose VUI ends early still yields its codec string and size, and declares no VUI. + #[test] + fn sps_with_a_short_vui_keeps_the_fields_before_it() { + let sps: &[u8] = &[ + 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, + 0x01, 0xd4, 0xc0, 0x80, + ]; + assert!(SeqParameterSet::from_bits(crate::codec::annexb::rbsp(sps, 1)).is_err()); + let parsed = Sps::parse(sps).unwrap(); + assert_eq!((parsed.profile, parsed.constraints, parsed.level), (0x42, 0xc0, 0x1f)); + assert_eq!((parsed.coded_width, parsed.coded_height), (1280, 720)); + assert!(sps_vui(sps).is_none()); + assert_eq!(sps_interlaced(sps), Some(false)); + } + + /// Annex B's `trailing_zero_8bits` left on an SPS is not an emulated start code. + #[test] + fn sps_ignores_trailing_zero_bytes() { + let mut padded = fixtures::SPS_IPB.to_vec(); + padded.extend_from_slice(&[0, 0, 0, 0]); + assert_eq!(sps_reorder(&padded), sps_reorder(fixtures::SPS_IPB)); + assert!(sps_reorder(&padded).is_some()); + } + const SC4: &[u8] = &[0, 0, 0, 1]; #[test] diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs index 1e687cfa04e..46b09158ab6 100644 --- a/rs/moq-mux/src/codec/h264/split.rs +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -272,19 +272,33 @@ impl Split { } } -/// True if an SEI NAL carries a recovery-point message (payload type 6), the +/// True if an SEI NAL carries a well-formed recovery-point message (payload type 6), the /// open-GOP random-access marker. The NAL header byte precedes the SEI RBSP. fn sei_has_recovery_point(nal: &[u8]) -> bool { - let Some(ebsp) = nal.get(1..) else { - return false; + use h264_reader::nal::sei::{ + HeaderType, SeiMessage, + recovery_point::{RecoveryPoint, RecoveryPointError}, }; - let rbsp = h264_parser::nal::ebsp_to_rbsp(ebsp); - let Ok(messages) = h264_parser::sei::SeiMessage::parse(&rbsp) else { + use h264_reader::rbsp::BitReaderError; + + const SEI_RECOVERY_POINT: u32 = 6; + + let Some(rbsp) = crate::codec::annexb::rbsp_bytes(nal, 1) else { return false; }; - messages - .iter() - .any(|m| matches!(m.payload, h264_parser::sei::SeiPayload::RecoveryPoint { .. })) + crate::codec::annexb::sei_messages(&rbsp) + .filter(|&(kind, _)| kind == SEI_RECOVERY_POINT) + .any(|(_, payload)| { + let msg = SeiMessage { + payload_type: HeaderType::RecoveryPoint, + payload, + }; + // Data after the fields is a payload extension, which a later spec revision may define. + matches!( + RecoveryPoint::read(&msg), + Ok(_) | Err(RecoveryPointError::ReaderError(BitReaderError::RemainingData)) + ) + }) } #[derive(Debug, Clone, Copy, PartialEq, Eq, num_enum::TryFromPrimitive)] @@ -460,9 +474,9 @@ mod tests { assert_eq!(tail[0].payload.as_ref(), annexb(&[sps, pps, idr]).freeze().as_ref()); } - /// A recovery-point SEI (payload type 6) with `recovery_frame_cnt == 0`, - /// followed by the recovery point's flag byte, then the RBSP stop bit. - const RECOVERY_SEI: &[u8] = &[0x06, 0x06, 0x02, 0x00, 0x40, 0x80]; + /// A recovery-point SEI (payload type 6, size 1): `recovery_frame_cnt == 0`, clear flags, + /// and payload alignment in `0x84`, then the RBSP stop bit. + const RECOVERY_SEI: &[u8] = &[0x06, 0x06, 0x01, 0x84, 0x80]; /// Open-GOP broadcast H.264: the random-access point is a non-IDR I-slice /// flagged by a recovery-point SEI, not an IDR. The splitter must still flag @@ -509,6 +523,46 @@ mod tests { ); } + /// A recovery point after a payload larger than h264-reader's 1 MiB `SeiReader` cap is still found. + #[test] + fn recovery_point_after_a_large_payload() { + let size = (1 << 20) + 1; + let mut sei = vec![0x06, 0x05]; // user_data_unregistered + sei.extend(std::iter::repeat_n(0xff, size / 255)); + sei.push((size % 255) as u8); + sei.extend(std::iter::repeat_n(0x01, size)); + sei.extend_from_slice(&RECOVERY_SEI[1..]); + assert!(sei_has_recovery_point(&sei)); + assert!(!sei_has_recovery_point(&sei[..sei.len() - RECOVERY_SEI.len() + 1])); + } + + /// A type-6 SEI without the recovery-point fields is not a random-access marker. + #[test] + fn malformed_recovery_point_is_ignored() { + // Empty payload, then a payload cut off inside `recovery_frame_cnt`. + assert!(!sei_has_recovery_point(&[0x06, 0x06, 0x00, 0x80])); + assert!(!sei_has_recovery_point(&[0x06, 0x06, 0x01, 0x00, 0x80])); + // A malformed message before a valid one doesn't hide it. + assert!(sei_has_recovery_point(&[0x06, 0x06, 0x00, 0x06, 0x01, 0x84, 0x80])); + } + + /// An empty recovery-point SEI before a non-IDR slice leaves that slice a delta frame. + #[tokio::test(start_paused = true)] + async fn empty_recovery_point_slice_is_delta() { + let aud: &[u8] = &[0x09, 0x10]; + let sei: &[u8] = &[0x06, 0x06, 0x00, 0x80]; + let pslice: &[u8] = &[0x61, 0xe0, 0x12, 0x34]; + + let mut split = Split::new(); + let frames = decode_one(&mut split, &mut annexb(&[aud, sei, pslice]), ts()); + + assert_eq!(frames.len(), 1); + assert!( + !frames[0].keyframe, + "an empty recovery-point SEI must not flag a keyframe" + ); + } + /// An access unit whose SEI is not a recovery point (e.g. pic_timing) stays a /// delta frame: only recovery-point SEIs mark open-GOP random access. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/codec/h265/sps.rs b/rs/moq-mux/src/codec/h265/sps.rs index b67f9dabbb4..b071b0b7ecf 100644 --- a/rs/moq-mux/src/codec/h265/sps.rs +++ b/rs/moq-mux/src/codec/h265/sps.rs @@ -7,7 +7,7 @@ //! `bit_depth_chroma_minus8`. So when scuffle_h265 refuses a value (`InvalidData`), [`Sps::parse`] //! reads the head itself. A truncated SPS, or one whose head is invalid, is still an error. -use h264_parser::{bitreader::BitReader, eg::read_ue}; +use h264_reader::rbsp::{BitRead, BitReaderError}; use scuffle_h265::{NALUnitType, SpsNALUnit}; use super::{Error, Result}; @@ -118,56 +118,60 @@ impl SpsHead { /// Read the head of an SPS NAL unit (header included). `None` when the NAL is not an SPS, ends /// before `bit_depth_chroma_minus8`, or breaks a constraint on the head. fn parse(nal: &[u8]) -> Option { - let [header, header2, payload @ ..] = nal else { + let [header, header2, ..] = nal else { return None; }; // forbidden_zero_bit, nal_unit_type, and TemporalId 0 for an SPS (7.4.2.2). if header & 0x80 != 0 || super::split::nal_unit_type(*header) != NALUnitType::SpsNut || header2 & 0x07 != 1 { return None; } - let rbsp = h264_parser::nal::ebsp_to_rbsp(payload); - Self::read(&mut BitReader::new(&rbsp)).ok()? + Self::read(&mut crate::codec::annexb::rbsp(nal, 2)).ok()? } /// Read the head from the SPS RBSP: `Ok(None)` for an out-of-range value. - fn read(r: &mut BitReader<'_>) -> h264_parser::Result> { - r.skip_bits(4)?; // sps_video_parameter_set_id - let max_sub_layers_minus1 = r.read_bits(3)? as u8; - let temporal_id_nesting_flag = r.read_flag()?; + fn read(r: &mut impl BitRead) -> std::result::Result, BitReaderError> { + r.skip(4, "sps_video_parameter_set_id")?; + let max_sub_layers_minus1 = r.read::<3, u8>("sps_max_sub_layers_minus1")?; + let temporal_id_nesting_flag = r.read_bit("sps_temporal_id_nesting_flag")?; // 7.4.3.2.1: at most 7 sub-layers, and a single sub-layer is nested. if max_sub_layers_minus1 > 6 || (max_sub_layers_minus1 == 0 && !temporal_id_nesting_flag) { return Ok(None); } // profile_tier_level(1, sps_max_sub_layers_minus1), 7.3.3 - let profile_space = r.read_bits(2)? as u8; - let tier_flag = r.read_flag()?; - let profile_idc = r.read_bits(5)? as u8; - let profile_compatibility_flags = r.read_bits(32)?.to_be_bytes(); + let profile_space = r.read::<2, u8>("general_profile_space")?; + let tier_flag = r.read_bit("general_tier_flag")?; + let profile_idc = r.read::<5, u8>("general_profile_idc")?; + let profile_compatibility_flags = r.read::<32, u32>("general_profile_compatibility_flags")?.to_be_bytes(); // progressive_source, interlaced_source, non_packed_constraint and frame_only_constraint. - let constraint_flags = [(r.read_bits(4)? as u8) << 4, 0, 0, 0, 0, 0]; - r.skip_bits(44)?; // the other 43 constraint and reserved bits, and general_inbld_flag - let level_idc = r.read_u8()?; + let constraint_flags = [r.read::<4, u8>("general_constraint_flags")? << 4, 0, 0, 0, 0, 0]; + r.skip(44, "general_reserved_zero_43bits")?; // and general_inbld_flag + let level_idc = r.read::<8, u8>("general_level_idc")?; skip_sub_layers(r, max_sub_layers_minus1)?; - let sps_seq_parameter_set_id = read_ue(r)?; - let chroma_format_idc = read_ue(r)?; + let sps_seq_parameter_set_id = r.read_ue("sps_seq_parameter_set_id")?; + let chroma_format_idc = r.read_ue("chroma_format_idc")?; if sps_seq_parameter_set_id > 15 || chroma_format_idc > 3 { return Ok(None); } if chroma_format_idc == 3 { - r.skip_bits(1)?; // separate_colour_plane_flag + r.skip(1, "separate_colour_plane_flag")?; } - let pic_width = read_ue(r)?; - let pic_height = read_ue(r)?; + let pic_width = r.read_ue("pic_width_in_luma_samples")?; + let pic_height = r.read_ue("pic_height_in_luma_samples")?; // conformance_window_flag, then the left, right, top and bottom offsets. - let [left, right, top, bottom] = if r.read_flag()? { - [read_ue(r)?, read_ue(r)?, read_ue(r)?, read_ue(r)?] + let [left, right, top, bottom] = if r.read_bit("conformance_window_flag")? { + [ + r.read_ue("conf_win_left_offset")?, + r.read_ue("conf_win_right_offset")?, + r.read_ue("conf_win_top_offset")?, + r.read_ue("conf_win_bottom_offset")?, + ] } else { [0; 4] }; - let bit_depth_luma_minus8 = read_ue(r)?; - let bit_depth_chroma_minus8 = read_ue(r)?; + let bit_depth_luma_minus8 = r.read_ue("bit_depth_luma_minus8")?; + let bit_depth_chroma_minus8 = r.read_ue("bit_depth_chroma_minus8")?; if bit_depth_luma_minus8 > 8 || bit_depth_chroma_minus8 > 8 { return Ok(None); } @@ -200,34 +204,41 @@ impl SpsHead { } /// Read `sps_seq_parameter_set_id` from an SPS RBSP (the payload after the NAL header). -pub(crate) fn id(r: &mut BitReader<'_>) -> h264_parser::Result { - r.skip_bits(4)?; // sps_video_parameter_set_id - let max_sub_layers_minus1 = r.read_bits(3)? as u8; - r.skip_bits(1)?; // sps_temporal_id_nesting_flag - r.skip_bits(96)?; // the general profile's 88 bits and general_level_idc +pub(crate) fn id(r: &mut impl BitRead) -> std::result::Result { + r.skip(4, "sps_video_parameter_set_id")?; + let max_sub_layers_minus1 = r.read::<3, u8>("sps_max_sub_layers_minus1")?; + r.skip(1, "sps_temporal_id_nesting_flag")?; + r.skip(96, "general_profile_and_level")?; // the general profile's 88 bits and general_level_idc skip_sub_layers(r, max_sub_layers_minus1)?; - read_ue(r) + r.read_ue("sps_seq_parameter_set_id") } /// Skip the sub-layer half of profile_tier_level (7.3.3), which follows `general_level_idc`. -fn skip_sub_layers(r: &mut BitReader<'_>, max_sub_layers_minus1: u8) -> h264_parser::Result<()> { +fn skip_sub_layers(r: &mut impl BitRead, max_sub_layers_minus1: u8) -> std::result::Result<(), BitReaderError> { // 7.4.3.2.1: at most 7 sub-layers. if max_sub_layers_minus1 > 6 { - return Err(h264_parser::Error::MalformedSps( - "sps_max_sub_layers_minus1 out of range".into(), + return Err(BitReaderError::ReaderError( + "sps_max_sub_layers_minus1", + std::io::ErrorKind::InvalidData.into(), )); } let sub_layers = usize::from(max_sub_layers_minus1); let mut present = [(false, false); 6]; for flags in &mut present[..sub_layers] { - *flags = (r.read_flag()?, r.read_flag()?); + *flags = ( + r.read_bit("sub_layer_profile_present_flag")?, + r.read_bit("sub_layer_level_present_flag")?, + ); } if sub_layers > 0 { - r.skip_bits(2 * (8 - u32::from(max_sub_layers_minus1)))?; // reserved_zero_2bits + r.skip(2 * (8 - u32::from(max_sub_layers_minus1)), "reserved_zero_2bits")?; } for &(profile, level) in &present[..sub_layers] { // A sub-layer profile is the same 88 bits as the general one; sub_layer_level_idc is 8. - r.skip_bits(88 * u32::from(profile) + 8 * u32::from(level))?; + r.skip( + 88 * u32::from(profile) + 8 * u32::from(level), + "sub_layer_profile_and_level", + )?; } Ok(()) } @@ -270,14 +281,15 @@ mod tests { fn id_matches_the_full_parse() { for nal in [fixtures::SPS, fixtures::SPS_PYRAMID, SPS_SUB_LAYERS] { let full = SpsNALUnit::parse(&mut &nal[..]).unwrap().rbsp; - let rbsp = h264_parser::nal::ebsp_to_rbsp(&nal[2..]); assert_eq!( - id(&mut BitReader::new(&rbsp)).unwrap(), + id(&mut crate::codec::annexb::rbsp(nal, 2)).unwrap(), full.sps_seq_parameter_set_id as u32 ); } - let rbsp = h264_parser::nal::ebsp_to_rbsp(&SPS_SUB_LAYER_PROFILE_AND_LEVEL[2..]); - assert_eq!(id(&mut BitReader::new(&rbsp)).unwrap(), 0); + assert_eq!( + id(&mut crate::codec::annexb::rbsp(SPS_SUB_LAYER_PROFILE_AND_LEVEL, 2)).unwrap(), + 0 + ); } /// scuffle_h265 0.2.2 reads that `sub_layer_level_idc` twice and refuses the SPS; the head diff --git a/rs/moq-mux/src/codec/vp9/mod.rs b/rs/moq-mux/src/codec/vp9/mod.rs index 76e899a4e0b..4679a94fce6 100644 --- a/rs/moq-mux/src/codec/vp9/mod.rs +++ b/rs/moq-mux/src/codec/vp9/mod.rs @@ -71,30 +71,33 @@ impl FrameHeader { /// Reads only as far as the frame size (the last field we care about); /// everything after it is left untouched. pub fn parse(data: &[u8]) -> Result { - let mut r = BitReader::new(data); + use bitstream_io::BitRead; - if r.read(2)? != 0b10 { + let mut r = bitstream_io::BitReader::endian(data, bitstream_io::BigEndian); + let mut read = |bits: u32| r.read_var::(bits).map_err(|_| Error::Truncated); + + if read(2)? != 0b10 { return Err(Error::InvalidFrameMarker); } - let profile_low = r.read(1)?; - let profile_high = r.read(1)?; + let profile_low = read(1)?; + let profile_high = read(1)?; let profile = ((profile_high << 1) | profile_low) as u8; if profile == 3 { - r.skip(1)?; // reserved_zero + read(1)?; // reserved_zero } // show_existing_frame: displays a previously decoded frame, no header follows. - if r.read(1)? == 1 { - r.skip(3)?; // frame_to_show_map_idx + if read(1)? == 1 { + read(3)?; // frame_to_show_map_idx return Ok(Self { keyframe: false, key: None, }); } - let keyframe = r.read(1)? == 0; // frame_type: 0 = KEY_FRAME - r.skip(2)?; // show_frame, error_resilient_mode + let keyframe = read(1)? == 0; // frame_type: 0 = KEY_FRAME + read(2)?; // show_frame, error_resilient_mode if !keyframe { return Ok(Self { @@ -103,26 +106,26 @@ impl FrameHeader { }); } - if r.read(24)? != SYNC_CODE { + if read(24)? != SYNC_CODE { return Err(Error::InvalidSyncCode); } // color_config (VP9 spec §6.2.2). let bit_depth = if profile >= 2 { - if r.read(1)? == 1 { 12 } else { 10 } + if read(1)? == 1 { 12 } else { 10 } } else { 8 }; - let color_space = r.read(3)? as u8; + let color_space = read(3)? as u8; const CS_RGB: u8 = 7; let (subsampling_x, subsampling_y, full_range); if color_space != CS_RGB { - full_range = r.read(1)? == 1; + full_range = read(1)? == 1; if profile == 1 || profile == 3 { - subsampling_x = r.read(1)? == 1; - subsampling_y = r.read(1)? == 1; - r.skip(1)?; // reserved_zero + subsampling_x = read(1)? == 1; + subsampling_y = read(1)? == 1; + read(1)?; // reserved_zero } else { // Profiles 0 and 2 are 4:2:0 only. (subsampling_x, subsampling_y) = (true, true); @@ -132,13 +135,13 @@ impl FrameHeader { full_range = true; (subsampling_x, subsampling_y) = (false, false); if profile == 1 || profile == 3 { - r.skip(1)?; // reserved_zero + read(1)?; // reserved_zero } } // frame_size (VP9 spec §6.2.3): each field is `value - 1`. - let width = r.read(16)? + 1; - let height = r.read(16)? + 1; + let width = read(16)? + 1; + let height = read(16)? + 1; Ok(Self { keyframe: true, @@ -275,36 +278,6 @@ pub(crate) fn vpcc(vp9: &VP9) -> mp4_atom::VpcC { } } -/// Minimal MSB-first bit reader over a byte slice. -struct BitReader<'a> { - data: &'a [u8], - bit: usize, -} - -impl<'a> BitReader<'a> { - fn new(data: &'a [u8]) -> Self { - Self { data, bit: 0 } - } - - fn read(&mut self, n: u32) -> Result { - let mut value = 0; - for _ in 0..n { - let byte = self.bit / 8; - if byte >= self.data.len() { - return Err(Error::Truncated); - } - let shift = 7 - (self.bit % 8); - value = (value << 1) | u32::from((self.data[byte] >> shift) & 1); - self.bit += 1; - } - Ok(value) - } - - fn skip(&mut self, n: u32) -> Result<()> { - self.read(n).map(|_| ()) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 621832497f7..e01d3790dbc 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -4182,6 +4182,8 @@ fn mapped(channels: u32, family: u8, streams: u8, coupled: u8, table: &[u8]) -> /// are refused. Retrying with gstreamer's widths could misread a descriptor written to /// the draft, so there is no fallback. fn explicit_layout(data: &[u8]) -> anyhow::Result { + use bitstream_io::BitRead; + let (&channel_count, rest) = data.split_first().context("truncated Opus channel configuration")?; let (&family, rest) = rest.split_first().context("truncated Opus channel configuration")?; anyhow::ensure!(channel_count > 0, "Opus channel_count is zero"); @@ -4197,13 +4199,13 @@ fn explicit_layout(data: &[u8]) -> anyhow::Result { return Ok(opus::Config::new(48_000, u32::from(channel_count))); } - let mut bits = Bits::new(rest); - let stream_count = bits.read(ceil_log2(u32::from(channel_count)))? + 1; + let mut bits = bitstream_io::BitReader::endian(rest, bitstream_io::BigEndian); + let stream_count = read_bits(&mut bits, ceil_log2(u32::from(channel_count)))? + 1; anyhow::ensure!( stream_count <= u32::from(channel_count), "Opus stream_count {stream_count} exceeds channel_count {channel_count}" ); - let coupled = bits.read(ceil_log2(stream_count + 1))?; + let coupled = read_bits(&mut bits, ceil_log2(stream_count + 1))?; anyhow::ensure!( coupled <= stream_count, "Opus coupled_stream_count {coupled} exceeds stream_count {stream_count}" @@ -4217,15 +4219,17 @@ fn explicit_layout(data: &[u8]) -> anyhow::Result { let silence = (1u32 << width) - 1; let mut table = Vec::with_capacity(channel_count as usize); for _ in 0..channel_count { - let entry = bits.read(width)?; + let entry = read_bits(&mut bits, width)?; table.push(if entry == silence { 255 } else { entry as u8 }); } - let pad = (8 - (bits.taken % 8)) % 8; - if pad > 0 { - let reserved = bits.read(pad as u8)?; + while !bits.byte_aligned() { + let reserved = read_bits(&mut bits, 1)?; anyhow::ensure!(reserved == 0, "nonzero reserved bits in Opus channel configuration"); } - bits.finish()?; + anyhow::ensure!( + bits.into_reader().is_empty(), + "trailing bytes in Opus channel configuration" + ); mapped( u32::from(channel_count), @@ -4242,53 +4246,10 @@ fn ceil_log2(n: u32) -> u8 { (u32::BITS - (n - 1).leading_zeros()) as u8 } -/// MSB-first reader over the bit-packed tail of an explicit Opus channel configuration. -struct Bits<'a> { - data: &'a [u8], - index: usize, - current: u8, - left: u8, - /// Bits returned so far, padding included, so the caller can byte-align. - taken: u32, -} - -impl Bits<'_> { - fn new(data: &[u8]) -> Bits<'_> { - Bits { - data, - index: 0, - current: 0, - left: 0, - taken: 0, - } - } - - fn read(&mut self, n: u8) -> anyhow::Result { - let mut value = 0u32; - for _ in 0..n { - if self.left == 0 { - self.current = *self - .data - .get(self.index) - .context("truncated Opus channel configuration")?; - self.index += 1; - self.left = 8; - } - self.left -= 1; - value = (value << 1) | u32::from((self.current >> self.left) & 1); - } - self.taken += u32::from(n); - Ok(value) - } - - /// The descriptor ended on the byte the layout consumed, with nothing after it. - fn finish(self) -> anyhow::Result<()> { - anyhow::ensure!( - self.left == 0 && self.index == self.data.len(), - "trailing bytes in Opus channel configuration" - ); - Ok(()) - } +/// Read an `n`-bit field, MSB-first, from the bit-packed tail of an explicit Opus channel +/// configuration. A zero-width field reads 0. +fn read_bits(bits: &mut impl bitstream_io::BitRead, n: u8) -> anyhow::Result { + bits.read_var(n.into()).context("truncated Opus channel configuration") } /// Validate the complete PES before exposing any of its declared Opus units. diff --git a/rs/moq-mux/src/container/ts/video_timing.rs b/rs/moq-mux/src/container/ts/video_timing.rs index a866d2faa4a..81684a9275f 100644 --- a/rs/moq-mux/src/container/ts/video_timing.rs +++ b/rs/moq-mux/src/container/ts/video_timing.rs @@ -14,6 +14,7 @@ use h264_reader::nal::slice::{ use h264_reader::nal::sps::PicOrderCntType; use h264_reader::nal::{Nal, RefNal, UnitType}; use h264_reader::nal::{pps::PicParameterSet, sps::SeqParameterSet}; +use h264_reader::rbsp::{BitRead, BitReader, BitReaderError}; use moq_net::Timestamp; use super::Unit; @@ -211,7 +212,7 @@ impl Clock { .insert(sps.sps_seq_parameter_set_id, Arc::new(sps)); } 34 => { - let rbsp = h264_parser::nal::ebsp_to_rbsp(bytes.get(2..).ok_or("short HEVC PPS")?); + let rbsp = crate::codec::annexb::rbsp_bytes(&bytes, 2).ok_or("invalid HEVC PPS")?; let pps = HevcPps::parse(&rbsp)?; Arc::make_mut(&mut ctx).pictures.insert(pps.id, pps); } @@ -220,7 +221,7 @@ impl Clock { if bytes[0] & 1 != 0 || second & 0xf8 != 0 { return Err("multilayer HEVC timing is unsupported".to_owned()); } - let rbsp = h264_parser::nal::ebsp_to_rbsp(bytes.get(2..).ok_or("short HEVC slice")?); + let rbsp = crate::codec::annexb::rbsp_bytes(&bytes, 2).ok_or("invalid HEVC slice")?; let (lsb, width, rate) = hevc_prefix(&rbsp, kind, &ctx.pictures, &ctx.sets)?; let idr = matches!(kind, 19 | 20); let reset = idr || matches!(kind, 16..=18) || (kind == 21 && self.hevc.is_none()); @@ -271,15 +272,16 @@ struct HevcPps { impl HevcPps { fn parse(rbsp: &[u8]) -> Result { let parsed = (|| { - let mut r = h264_parser::bitreader::BitReader::new(rbsp); - let id = h264_parser::eg::read_ue(&mut r)?; - let sps = h264_parser::eg::read_ue(&mut r)?; - r.skip_bits(1)?; // dependent_slice_segments_enabled_flag, absent on the first slice - let output = r.read_flag()?; - let extra = r.read_bits(3)? as u8; - h264_parser::Result::Ok(Self { id, sps, output, extra }) + let mut r = BitReader::new(rbsp); + let id = r.read_ue("pps_pic_parameter_set_id")?; + let sps = r.read_ue("pps_seq_parameter_set_id")?; + // Absent on the first slice. + r.skip(1, "dependent_slice_segments_enabled_flag")?; + let output = r.read_bit("output_flag_present_flag")?; + let extra = r.read::<3, u8>("num_extra_slice_header_bits")?; + Ok::<_, BitReaderError>(Self { id, sps, output, extra }) })(); - let pps = parsed.map_err(|err| err.to_string())?; + let pps = parsed.map_err(|err| format!("{err:?}"))?; if pps.id > 63 || pps.sps > 15 { return Err("invalid HEVC parameter-set id".to_owned()); } @@ -293,34 +295,35 @@ fn hevc_prefix( pictures: &BTreeMap, sets: &BTreeMap>, ) -> Result<(i64, u32, Option), String> { - let mut r = h264_parser::bitreader::BitReader::new(rbsp); - if !r.read_flag().map_err(|err| err.to_string())? { + let mut r = BitReader::new(rbsp); + let err = |err: BitReaderError| format!("{err:?}"); + if !r.read_bit("first_slice_segment_in_pic_flag").map_err(err)? { return Err("missing first independent HEVC slice".to_owned()); } if (16..=23).contains(&kind) { - r.skip_bits(1).map_err(|err| err.to_string())?; + r.skip(1, "no_output_of_prior_pics_flag").map_err(err)?; } - let id = h264_parser::eg::read_ue(&mut r).map_err(|err| err.to_string())?; + let id = r.read_ue("slice_pic_parameter_set_id").map_err(err)?; let pps = pictures.get(&id).ok_or("missing HEVC PPS")?; let sps = sets.get(&u64::from(pps.sps)).ok_or("missing HEVC SPS")?; // The first independent slice has no segment address or inherited header. // Stop as soon as its POC is available, before reference lists and slice data. - r.skip_bits(pps.extra.into()).map_err(|err| err.to_string())?; - let slice_type = h264_parser::eg::read_ue(&mut r).map_err(|err| err.to_string())?; + r.skip(pps.extra.into(), "slice_reserved_flag").map_err(err)?; + let slice_type = r.read_ue("slice_type").map_err(err)?; if slice_type > 2 { return Err("invalid HEVC slice type".to_owned()); } if pps.output { - r.skip_bits(1).map_err(|err| err.to_string())?; + r.skip(1, "pic_output_flag").map_err(err)?; } if sps.separate_colour_plane_flag { - r.skip_bits(2).map_err(|err| err.to_string())?; + r.skip(2, "colour_plane_id").map_err(err)?; } let width = u32::from(sps.log2_max_pic_order_cnt_lsb_minus4) + 4; let lsb = if matches!(kind, 19 | 20) { 0 } else { - i64::from(r.read_bits(width).map_err(|err| err.to_string())?) + i64::from(r.read_var::(width, "slice_pic_order_cnt_lsb").map_err(err)?) }; let info = sps.vui_parameters.as_ref().and_then(|vui| vui.vui_timing_info.as_ref()); let rate = info diff --git a/rs/moq-rtc/src/codec/bitstream_test.rs b/rs/moq-rtc/src/codec/bitstream_test.rs index 005a06993b7..3f1e76df018 100644 --- a/rs/moq-rtc/src/codec/bitstream_test.rs +++ b/rs/moq-rtc/src/codec/bitstream_test.rs @@ -31,7 +31,7 @@ fn annexb(nals: &[&[u8]]) -> Bytes { #[tokio::test(start_paused = true)] async fn h264_annexb_frame_publishes_catalog_entry() { // Real SPS+PPS pair lifted from moq-mux's avc3 catalog test. Anything - // shorter and h264-parser's RBSP decoder runs out of bytes parsing + // shorter and the SPS parser runs out of bytes reading // vui_parameters; not worth synthesizing a smaller one by hand. let sps: &[u8] = &[ 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, diff --git a/rs/moq-rtsp/src/fake.rs b/rs/moq-rtsp/src/fake.rs index 862c22e61d1..848390445be 100644 --- a/rs/moq-rtsp/src/fake.rs +++ b/rs/moq-rtsp/src/fake.rs @@ -21,7 +21,7 @@ pub const H264_PPS: &[u8] = &[0x68, 0xcb, 0x83, 0xcb, 0x20]; /// An IDR slice; its contents are never decoded. pub const H264_IDR: &[u8] = &[0x65, 0x88, 0x84, 0x21, 0xa0]; /// A recovery-point SEI: the non-IDR I-picture after it is an open-GOP keyframe. -pub const H264_RECOVERY: &[u8] = &[0x06, 0x06, 0x02, 0x00, 0x40, 0x80]; +pub const H264_RECOVERY: &[u8] = &[0x06, 0x06, 0x01, 0x84, 0x80]; /// A non-IDR slice. pub const H264_SLICE: &[u8] = &[0x41, 0x9a, 0x02, 0x04, 0x80];