diff --git a/doc/concept/hang.md b/doc/concept/hang.md index ecfb5abadd3..614b7436755 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -214,11 +214,14 @@ independent, so an audio group can end at any frame; the first-party encoders put one packet in each unless told otherwise. The `description` field carries out-of-band codec setup (an `avcC` box for -H.264). When it is absent, the parameter sets ride inline before each keyframe, -which is what `avc3`/`hev1` tracks do. Decoders should handle both. CMAF is the -exception: its samples are always length-prefixed, so an `avc3`/`hev1` CMAF -track keeps the configuration record as its `description` for the NAL length -size, even when the parameter sets ride in the samples. +H.264) and decides the framing. With one, H.264 and H.265 frames are +length-prefixed by its NAL length size; without one, they are Annex-B and carry +their parameter sets before each keyframe. The codec string only says where the +parameter sets live: `avc1`/`hvc1` keep them in the `description`, while +`avc3`/`hev1` may repeat them in band, with or without one. Decoders should +handle every combination. CMAF samples are always length-prefixed, so a CMAF +track always carries the configuration record, even when the parameter sets +ride in the samples. ## Your own format diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index 9a26e369406..504a6b667f6 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -166,7 +166,7 @@ for (const stage of ["support", "flush"] as const) { if (stage === "flush") expect(FakeVideoEncoder.instances[0]?.state).toBe("closed"); enabled.set(true); await settle(); - expect(encoder.out.catalog.peek()?.codec).toStartWith("avc1"); + expect(encoder.out.catalog.peek()?.codec).toStartWith("avc3"); // The abandoned browser promise still has not resolved. expect(held.mock.calls.length).toBeGreaterThan(1); } finally { @@ -949,10 +949,12 @@ test("the catalog advertises the codec string a probe encode reports", async () encoder.config.set({ codec: "avc1" }); await settle(); expect(encoder.out.resolved.peek()?.codec).toBe("avc1.640028"); - expect(encoder.out.catalog.peek()?.codec).toBe("avc1.64001F"); + // Annex B carries its parameter sets in band, so the catalog says avc3 rather than the + // avc1 the encoder reports. + expect(encoder.out.catalog.peek()?.codec).toBe("avc3.64001F"); // The configured strings were never advertised, not even briefly. - expect([...new Set(advertised)]).toEqual(["vp09.00.31.08", "avc1.64001F"]); + expect([...new Set(advertised)]).toEqual(["vp09.00.31.08", "avc3.64001F"]); } finally { unsubscribe(); encoder.close(); diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index 9c32d86ecd4..3e465632c07 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -973,7 +973,8 @@ type Probe = { // The config the browser accepted and encoded a probe frame with. Its bitrate is the ceiling, // before any bandwidth grant. config: VideoEncoderConfig & { framerate: number; bitrate: number }; - // The codec string the encoder reported for it, which the catalog advertises. + // The codec string the encoder reported for it, as the catalog advertises it: `avc3`/`hev1` for + // Annex B. reported: string; }; @@ -1102,7 +1103,17 @@ async function probeEncoder( if (requestsAnnexB(config.codec) && (described || !(first && startsWithStartCode(first)))) { return { refused: "its encoder ignored the requested Annex B format" }; } - return { reported }; + return { reported: requestsAnnexB(config.codec) ? inBand(reported) : reported }; +} + +// The codec string for Annex B output, which carries its parameter sets in band: `avc3`/`hev1` +// rather than the `avc1`/`hvc1` an encoder reports, which would say they live in a description the +// catalog doesn't carry. Only the catalog changes: Safari refuses `avc3` as an encoder string, and the +// watcher opens an `avc3` rendition as `avc1` where it must. +function inBand(codec: string): string { + if (codec.startsWith("avc1.")) return `avc3.${codec.slice(5)}`; + if (codec.startsWith("hvc1.")) return `hev1.${codec.slice(5)}`; + return codec; } // The codecs the probe configures for Annex B output. diff --git a/quest/m1/video-framing.md b/quest/m1/video-framing.md deleted file mode 100644 index a125a945c8f..00000000000 --- a/quest/m1/video-framing.md +++ /dev/null @@ -1,81 +0,0 @@ -# [M] H.264 and H.265 framing follows the description, and the codec says where parameter sets live - -## Goal - -Every producer reports the codec string that matches its frames, and every -consumer parses frames by the catalog's `description`, never by the codec -prefix: - -- No `description`: frames are Annex B and carry their parameter sets, so - the codec is `avc3`/`hev1`. -- A `description`: frames are length-prefixed by the record's length size, - and the record seeds the parameter sets. The codec is `avc1`/`hvc1` - unless parameter sets also appear in-band, then `avc3`/`hev1`. -- CMAF frames are always length-prefixed. - -A Legacy `hev1`/`avc3` track with a record, as `moq publish fmp4` and the -HLS import produce, plays natively (`moq play`, every binding, -moq-transcode) as it does in the browser. - -## Plan - -Decided 2026-11-17 (maintainer, `/quest-import` of -[moq-dev/moq#9011](https://github.com/moq-dev/moq/issues/9011)): - -- ✅ Consumers take framing from `description` (rejected: make the - fMP4/HLS import rewrite length prefixes to start codes and drop the record, - so `avc3` always means Annex B; it costs a rewrite per frame and still - breaks on any other publisher that sends `avc3` with a record). This - matches the hang draft (`drafts/draft-lcurley-moq-hang.md`, framing by - `description`), the browser, and every exporter. -- ✅ The browser publisher reports `avc3` for its Annex B H.264 (rejected: - keep `avc1`). `avc1` with no record contradicts itself, and it makes the - fMP4 export pull SPS/PPS into a fixed avcC, so a resolution change at a - keyframe likely fails with `TrackChanged`; reproduce that first. Safari - rejects `avc3` as an encoder string, so only the catalog's codec changes; - the watcher already retries `avc3` as `avc1`. -- The maintainer asked that importers detect `avc1` vs `avc3` from the - bytes, as WebCodecs does: check that an avcC/hvcC input whose stream - repeats parameter sets in-band reports `avc3`/`hev1`. -- The import format names move in their own breaking quest, - [video-format-names](/quest/m2/video-format-names.md); this one only fixes - behavior and wording. - -Bugs found while importing (reproduce each first): - -1. `rs/moq-video/src/decode/decoder.rs` picks framing from `inline` / - `in_band` (the `avc3`/`hev1` prefix): a Legacy `hev1` track with an hvcC - and length-prefixed frames is handed to the backend as Annex B. ffmpeg - writes `hev1` for HEVC in MP4 by default, so - `ffmpeg … -c:v libx265 -f mp4 -movflags frag_keyframe+empty_moov | moq publish fmp4` - then `moq play` fails. Since - [moq-dev/moq#6250](https://github.com/moq-dev/moq/pull/6250) made the - Legacy unwrap the default, this is the common fMP4/HLS path. An `hvc1` - track with no record is refused, where H.264 passes it through as Annex B. -2. `rs/moq-webrtc/src/codec/mod.rs` treats any record as complete: an - `avc3`/`hev1` record with empty parameter-set lists (what our fMP4 export - writes) fails with `MissingParamSets` instead of taking them in-band. - -Wording to fix (comments and docs that call `avc3`/`hev1` Annex B): -`rs/moq-mux/src/codec/h264/mod.rs`, `h264/import.rs`, `h264/split.rs`, -`h265/import.rs`, `import/track.rs`, `container/source.rs`, -`ts/export.rs`, `flv/export.rs`, `mkv/export.rs`, -`rs/moq-cli/src/subscribe.rs`, `rs/moq-video/src/encode/encoder.rs`, -`rs/moq-webrtc/src/codec/`, `rs/hang/src/catalog/video/h265.rs:14`, and -`doc/concept/hang.md` near line 216 (an absent description is not "what -avc3/hev1 tracks do"). `grep -rniE 'annex.?b' rs js doc` finds the rest. - -Public API: none. Wire: none; the browser's catalog codec string changes -from `avc1.*` to `avc3.*`. - -Caused by, for the fix PR's `## Caused by`: - -- [moq-dev/moq#6250](https://github.com/moq-dev/moq/pull/6250) - made the Legacy unwrap the default, so fMP4/HLS tracks reached the prefix-framed native decoder - -## Closes - -- [#9011](https://github.com/moq-dev/moq/issues/9011) - avc3/hev1 conflated with Annex B - -## Related - -- [Import format names](/quest/m2/video-format-names.md) - renames the Annex B input formats away from `avc3`/`hev1` diff --git a/quest/m2/video-format-names.md b/quest/m2/video-format-names.md index 6b5bf9d0c9d..21f4e17431c 100644 --- a/quest/m2/video-format-names.md +++ b/quest/m2/video-format-names.md @@ -16,8 +16,8 @@ Decided 2026-11-17 (maintainer, `/quest-import` of `avc3`/`hev1` say where parameter sets live, not how NAL units are framed: inside a frame, NAL units are still split by start codes or length prefixes. Naming the Annex B input after them taught the conflation that -became the native decoder bug in -[video-framing](/quest/m1/video-framing.md). The names follow ffmpeg's raw +became the native decoder bug fixed in +[moq-dev/moq#9066](https://github.com/moq-dev/moq/pull/9066). The names follow ffmpeg's raw `h264` / `hevc` formats; `h264` already aliases `avc3`. Where the names live: `rs/moq-mux/src/import/format.rs` (the enum, its doc @@ -30,7 +30,3 @@ No alias for the old names. The catalog's codec strings are not import formats and keep `avc3`/`hev1`. Public API: breaking, as above. Wire: none. - -## Required - -- [Video framing](/quest/m1/video-framing.md) - settles the framing contract the new names and docs describe diff --git a/rs/hang/src/catalog/video/h264.rs b/rs/hang/src/catalog/video/h264.rs index 3f6095eb774..6d5e85e0cc1 100644 --- a/rs/hang/src/catalog/video/h264.rs +++ b/rs/hang/src/catalog/video/h264.rs @@ -12,8 +12,11 @@ use crate::Error; #[serde_with::serde_as] #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct H264 { - /// If true, SPS/PPS are inline in the bitstream (avc3). - /// If false, they must be in the description (avc1). + /// If true (avc3), SPS/PPS may also ride in the bitstream, beside any description. + /// If false (avc1), they live only in the description. + /// + /// Says nothing about the NAL framing, which the description decides: with one the + /// NAL units are length-prefixed, without one they are Annex-B. #[serde(default)] pub inline: bool, /// The H.264 profile (e.g., 0x42 for Baseline, 0x4D for Main, 0x64 for High) diff --git a/rs/hang/src/catalog/video/h265.rs b/rs/hang/src/catalog/video/h265.rs index 38aaf105f95..435c9b12d6e 100644 --- a/rs/hang/src/catalog/video/h265.rs +++ b/rs/hang/src/catalog/video/h265.rs @@ -7,12 +7,13 @@ use crate::Error; /// H.265/HEVC codec mimetype. /// /// This struct contains the profile, tier, level, and constraint information -/// needed to identify a specific H.265 variant. The `in_band` flag determines -/// whether parameter sets are included in-band (hev1) or out-of-band (hvc1). +/// needed to identify a specific H.265 variant. The `in_band` flag says whether +/// parameter sets may ride in band (hev1) or live only in the description (hvc1); +/// the description, not the flag, decides the NAL framing. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct H265 { - /// If true (hev1), then the SPS/PPS/etc are in the same NAL unit as the IDR. - /// If false (hvc1), then the SPS/PPS/etc are in the description. + /// If true (hev1), the VPS/SPS/PPS may also ride in the bitstream ahead of a keyframe. + /// If false (hvc1), they live only in the description. pub in_band: bool, /// Profile space (0 for main profile space, 1-3 for other spaces) diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 961a8f585c6..55f1a0938dd 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -501,7 +501,8 @@ impl Subscribe { /// Write the selected TS media to a writer after resolving its route. pub(super) async fn write_ts(self, mut stdout: impl tokio::io::AsyncWrite + Unpin) -> anyhow::Result<()> { // TS emits PAT/PMT then a continuous PES stream (re-emitting PAT/PMT at - // keyframes for tune-in). Avc3/Hev1 sources pass through as Annex-B; AAC + // keyframes for tune-in). Annex-B sources (no description) pass through; + // length-prefixed ones are rewritten to Annex-B; AAC // is re-framed as ADTS. `fragment_duration` does not apply to TS. The catalog stream // selects the `mpegts` catalog extension so undecoded elementary streams // (SCTE-35, teletext, DVB AC-3, ...) are re-emitted verbatim on their PIDs. diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index d4ac78e218e..fda6b10f6a9 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -10052,10 +10052,11 @@ async fn video_subscribe_configuration_boundary() { let consumer = broadcast.consume().unwrap(); for (codec, description, supported) in [ ("avc1.42001e", None, true), - ("avc1.42001e", Some(vec![]), false), + // An empty description is no description: Annex-B framing. + ("avc1.42001e", Some(vec![]), true), ("avc1.42001e", Some(vec![1, 0x42, 0, 0x1e]), false), - // hvc1 requires hvcC before any backend is opened, even on HEVC-capable hosts. - ("hvc1.1.6.L93.B0", None, false), + // A malformed hvcC is refused before any backend is opened, even on HEVC-capable hosts. + ("hvc1.1.6.L93.B0", Some(vec![1, 2, 3]), false), ("vp09.02.10.10", None, false), ("avc1.invalid", None, false), ("unknown", None, false), diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index a07e08dc507..231a22861e0 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -6,14 +6,17 @@ //! Frames arrive via [`decode`](Import::decode). //! //! The codec config comes from exactly one of two places: an avcC handed to -//! [`initialize`](Import::initialize) (the "avc1" shape), or the SPS the splitter -//! packages into the first keyframe (the "avc3" shape, scanned out of the frame -//! here). A keyframe that can't be configured from either is an error; -//! non-keyframes before the first config are tolerated (mid-stream joins). +//! [`initialize`](Import::initialize), for length-prefixed frames, or the SPS the +//! splitter packages into the first Annex-B keyframe (scanned out of the frame +//! here). The codec string says where the parameter sets live, never the +//! framing: Annex-B is always avc3, and an avcC is avc3 when its first keyframe +//! carries a parameter set in band and avc1 otherwise. A keyframe that can't be +//! configured from either is an error; non-keyframes before the first config are +//! tolerated (mid-stream joins). use bytes::Bytes; -use super::{Error, NAL_TYPE_SEI, NAL_TYPE_SPS, Sps}; +use super::{Error, NAL_TYPE_PPS, NAL_TYPE_SEI, NAL_TYPE_SPS, Sps}; use crate::Result; use crate::catalog::hang::CatalogExt; use crate::codec::annexb::NalIterator; @@ -24,8 +27,9 @@ use crate::container::Frame; /// Build it with [`new`](Self::new), passing the track producer and the /// [`catalog::Reserved`](crate::catalog::Reserved) it reserves its rendition from. /// Feed it frames a [`Split`](super::Split) produced via [`decode`](Self::decode). -/// The catalog rendition fills in lazily once the codec config is known (avcC via -/// [`initialize`](Self::initialize) for avc1, the first SPS for avc3). +/// The catalog rendition fills in lazily once the codec config is known (at the +/// first keyframe after an avcC via [`initialize`](Self::initialize), or the first +/// Annex-B SPS). pub struct Import { /// The NALU length size for the avc1 shape, whose codec config is out-of-band (avcC), so /// keyframes are not scanned for an inline SPS. `None` for Annex-B. @@ -33,6 +37,9 @@ pub struct Import { track: crate::container::Producer, catalog: crate::codec::video::Catalog, last_sps: Option, + /// The avcC's config, held until the first keyframe says whether the stream also repeats its + /// parameter sets in band (avc3) or keeps them in the record only (avc1). + pending: Option, } impl Import { @@ -56,6 +63,7 @@ impl Import { track, catalog, last_sps: None, + pending: None, }; if let Some(config) = import.catalog.initial_config() { import.apply_config(config)?; @@ -65,11 +73,12 @@ impl Import { /// Resolve the codec config from the codec's leading bytes. /// - /// - **avc1** (no leading start code): parsed as an `AVCDecoderConfigurationRecord`, + /// - **avcC** (no leading start code): parsed as an `AVCDecoderConfigurationRecord`, /// which resolves the config and is stored as the catalog `description`. Required - /// for avc1. - /// - **avc3** (leading start code): parsed as Annex-B; any SPS resolves the config. - /// Optional, since avc3 also self-initializes from the first keyframe. + /// for length-prefixed frames. The rendition publishes at the first keyframe, as avc3 + /// when that keyframe carries a parameter set in band and avc1 otherwise. + /// - **Annex-B** (leading start code): any SPS resolves the config, as avc3. + /// Optional, since Annex-B also self-initializes from the first keyframe. /// /// Takes a read-only slice: the dispatcher-owned [`Split`](super::Split) is what /// consumes the stream (and reads the same avcC for the NALU length size). The @@ -83,13 +92,12 @@ impl Import { } fn initialize_avc1(&mut self, avcc_bytes: &[u8]) -> Result<()> { - // Only switch to avc1 mode once the avcC actually parses, so a parse failure leaves the - // importer in avc3 mode where inline-SPS keyframes still self-initialize. + // Only switch to length-prefixed mode once the avcC actually parses, so a parse failure + // leaves the importer in Annex-B mode where inline-SPS keyframes still self-initialize. let config = config_from_avcc(avcc_bytes)?; - // A new record starts a new coded sequence, whose HDR metadata the samples will carry. - self.catalog.hdr(&mut self.track, Default::default(), true)?; self.length_size = Some(super::Avcc::parse(avcc_bytes)?.length_size); - self.apply_config(config)?; + // A new record waits for its keyframe; the record already published holds nothing back. + self.pending = (self.catalog.description() != Some(avcc_bytes)).then_some(config); Ok(()) } @@ -134,16 +142,42 @@ impl Import { /// Finish the track, flushing the current group, and return the [`Finished`](crate::container::Finished) /// track, which keeps it and its catalog rendition published. /// - /// A finish that fails ends the track with the cause instead, so it can't be retried. + /// A record no keyframe followed publishes as it stands. A finish that fails ends the track with + /// the cause instead, so it can't be retried. pub fn finish( - self, + mut self, ) -> crate::Result> { - self.track.finish() + match self.publish_pending() { + Ok(()) => self.track.finish(), + Err(err) => { + drop(self.track.fail(err.reason())); + Err(err) + } + } } /// [`finish`](Self::finish) the track in place, for an owner that keeps this importer. pub(crate) fn finish_in_place(&mut self) -> crate::Result<()> { - self.track.end_with(None) + let published = self.publish_pending(); + let ended = self.track.end_with(published.as_ref().err().map(crate::Error::reason)); + published.and(ended) + } + + /// End the track unless it already ended, dropping a record no keyframe followed, for an owner + /// replacing this importer with a successor. + pub(crate) fn end(&mut self) -> crate::Result<()> { + self.pending = None; + self.track.end() + } + + /// Publish a held record as it stands: no keyframe follows to say where the parameter sets live. + /// It starts a coded sequence, so the last one's in-band HDR metadata goes first. + fn publish_pending(&mut self) -> crate::Result<()> { + let Some(config) = self.pending.take() else { + return Ok(()); + }; + self.catalog.hdr(&mut self.track, Default::default(), true)?; + self.apply_config(config) } /// Abort the track with `err` instead of finishing it cleanly, so subscribers @@ -235,6 +269,11 @@ impl Import { /// keyframe's inline SPS and refining the catalog jitter as it goes. fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { + // A delta between a new record and its keyframe decodes against neither, so it never + // reaches the open group, whose timestamps it is no part of. + if self.pending.is_some() && !frame.keyframe { + return Err(crate::container::MissingKeyframe.into()); + } self.configure(&frame)?; if !self.catalog.configured() { @@ -268,8 +307,14 @@ impl Import { let sps = self.length_size.is_none().then(|| find_sps(&frame.payload)).flatten(); // A new SPS starts a coded sequence whose HDR metadata replaces the last one's. It goes // first, so a keyframe that also configures publishes once. - let sequence = sps.as_ref().is_some_and(|sps| self.last_sps.as_ref() != Some(sps)); + // So does a new record, held for this keyframe. + let pending = self.pending.take(); + let sequence = pending.is_some() || sps.as_ref().is_some_and(|sps| self.last_sps.as_ref() != Some(sps)); self.catalog.hdr(&mut self.track, hdr, sequence)?; + if let Some(mut config) = pending { + crate::codec::video::detect_in_band(&mut config, &frame.payload)?; + self.apply_config(config)?; + } if let Some(sps) = sps { self.configure_from_sps(&sps)?; } @@ -284,7 +329,7 @@ impl Import { } /// The catalog config a stream's codec bytes describe, resolved without publishing anything: an -/// `AVCDecoderConfigurationRecord` (the avc1 shape) or Annex-B carrying an inline SPS (avc3). +/// `AVCDecoderConfigurationRecord` (avc1) or Annex-B carrying an inline SPS (avc3). /// /// The video counterpart of [`opus::config`](crate::codec::opus::config), for a caller holding the /// bytes but no track: a publisher that wants its catalog rendition to say what its encoder really @@ -338,6 +383,11 @@ fn config_from_sps(sps_nal: &[u8]) -> Result { Ok(config) } +/// Whether a NAL unit header names a parameter set (SPS or PPS). +pub(crate) fn is_parameter_set(header: u8) -> bool { + matches!(header & 0x1f, NAL_TYPE_SPS | NAL_TYPE_PPS) +} + fn is_sps(nal: &[u8]) -> bool { nal.first().is_some_and(|h| h & 0x1f == NAL_TYPE_SPS) } @@ -542,14 +592,45 @@ mod tests { let color = || catalog.snapshot().video.renditions["video"].color.clone().unwrap(); assert!(color().mastering.is_some()); - // The new avcC is a new description, so it publishes a new track. + // The new avcC is a new description, so its next keyframe publishes a new track. import.initialize(&avcc(SPS_BT709)).unwrap(); + import.decode([idr_frame(33)]).unwrap(); assert_eq!(import.name(), "video/1"); let color = || catalog.snapshot().video.renditions["video/1"].color.clone().unwrap(); assert_eq!(color().primaries, Some(1)); assert_eq!((color().mastering, color().content_light), (None, None)); } + /// A new avcC that no keyframe follows still starts a new coded sequence when it publishes at + /// finish, so the last one's in-band HDR metadata is dropped. + #[tokio::test(start_paused = true)] + async fn a_record_published_at_finish_drops_stale_hdr_metadata() { + use crate::codec::h264::fixtures::{PPS, SEI_CONTENT_LIGHT, SEI_MASTERING, SPS_BT709, SPS_HDR10}; + + let avcc = |sps: &'static [u8]| { + crate::codec::h264::build_avcc(&[Bytes::from_static(sps)], &[Bytes::from_static(PPS)]).unwrap() + }; + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + import.initialize(&avcc(SPS_HDR10)).unwrap(); + let mut au = Vec::new(); + for nal in [SEI_MASTERING, SEI_CONTENT_LIGHT, &[0x65, 0x88, 0x84, 0x21][..]] { + au.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + au.extend_from_slice(nal); + } + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + import + .decode([crate::codec::h264::avc1_frame(au, 4, pts).unwrap()]) + .unwrap(); + + import.initialize(&avcc(SPS_BT709)).unwrap(); + let _finished = import.finish().unwrap(); + let snapshot = catalog.snapshot(); + let color = snapshot.video.renditions["video/1"].color.clone().unwrap(); + assert_eq!(color.primaries, Some(1)); + assert_eq!((color.mastering, color.content_light), (None, None)); + } + /// Seeding an Annex-B importer with a new SPS starts a new coded sequence, dropping the last /// one's HDR metadata before the new config publishes. #[tokio::test(start_paused = true)] @@ -599,6 +680,7 @@ mod tests { let avcc = crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_BT709)], &[Bytes::from_static(PPS)]).unwrap(); import.initialize(&avcc).unwrap(); + import.decode([idr_frame(0)]).unwrap(); let color = catalog.snapshot().video.renditions["video"] .color @@ -686,18 +768,87 @@ mod tests { let buf = bytes::BytesMut::from(avcc.as_slice()); import.initialize(&buf).expect("initialize avc1"); assert_eq!(buf.len(), avcc.len(), "initialize must not consume the buffer"); + assert!( + catalog.snapshot().video.renditions.is_empty(), + "the first keyframe decides where the parameter sets live" + ); + import.decode([idr_frame(0)]).unwrap(); let snapshot = catalog.snapshot(); let cfg = snapshot.video.renditions.get("video").expect("rendition"); let hang::catalog::VideoCodec::H264(h264) = &cfg.codec else { panic!("expected H.264 codec") }; - assert!(!h264.inline, "avc1 source should land as inline=false"); + assert!(!h264.inline, "parameter sets only in the avcC land as avc1"); assert_eq!(h264.profile, 0x42); assert_eq!(h264.level, 0x1f); assert_eq!(cfg.description.as_ref().expect("description").as_ref(), avcc.as_slice()); } + /// A length-prefixed IDR slice at `ms`, carrying no parameter sets. + fn idr_frame(ms: u64) -> Frame { + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + let mut au = (idr.len() as u32).to_be_bytes().to_vec(); + au.extend_from_slice(idr); + crate::codec::h264::avc1_frame(au, 4, moq_net::Timestamp::from_millis(ms).unwrap()).unwrap() + } + + /// An avcC stream whose keyframes carry any parameter set in band lands as avc3, since a keyframe + /// may change or supply parameter sets the record would otherwise fix: a PPS alone counts, as for + /// a record that omits it. The record still frames the stream. + #[tokio::test(start_paused = true)] + async fn avcc_with_in_band_parameter_sets_lands_as_avc3() { + use crate::codec::h264::fixtures::{PPS, SPS_BT709}; + + let full = + crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_BT709)], &[Bytes::from_static(PPS)]).unwrap(); + let sps_only = crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_BT709)], &[]).unwrap(); + for (avcc, in_band) in [(full, &[SPS_BT709, PPS][..]), (sps_only, &[PPS][..])] { + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + import.initialize(&avcc).unwrap(); + + let mut au = Vec::new(); + for nal in in_band.iter().chain([&&[0x65, 0x88, 0x84, 0x21][..]]) { + au.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + au.extend_from_slice(nal); + } + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + import + .decode([crate::codec::h264::avc1_frame(au, 4, pts).unwrap()]) + .unwrap(); + + let snapshot = catalog.snapshot(); + let cfg = &snapshot.video.renditions["video"]; + assert_eq!( + cfg.codec.to_string().get(..4), + Some("avc3"), + "{} in band", + in_band.len() + ); + assert_eq!(cfg.description.as_deref(), Some(avcc.as_ref())); + } + } + + /// A record no keyframe followed publishes as it stands when the track finishes. + #[tokio::test(start_paused = true)] + async fn a_record_without_a_keyframe_publishes_at_finish() { + use crate::codec::h264::fixtures::{PPS, SPS_BT709}; + + let avcc = + crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_BT709)], &[Bytes::from_static(PPS)]).unwrap(); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + import.initialize(&avcc).unwrap(); + assert!(catalog.snapshot().video.renditions.is_empty()); + let _finished = import.finish().unwrap(); + let snapshot = catalog.snapshot(); + assert_eq!( + snapshot.video.renditions["video"].codec.to_string().get(..4), + Some("avc1") + ); + } + /// An avc3 stream self-initializes: the config is resolved from the SPS the /// splitter packages into the first keyframe. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index cd472fd590d..607ce60bb28 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -5,10 +5,14 @@ //! (inline SPS/PPS) as length-prefixed NALU + out-of-band avcC, which is //! what every CMAF and MKV consumer expects. [`Export`] subscribes to a //! catalog-narrowed H.264 rendition and emits an Annex-B elementary -//! stream; [`Split`] does the byte-level framing for the Annex-B (avc3) -//! wire shape and [`Import`] is the pure frame publisher that resolves the -//! catalog. avc1 (length-prefixed NALU) has no stream framing; wrap one -//! access unit with `avc1_frame`. +//! stream; [`Split`] does the byte-level framing of an Annex-B stream and +//! [`Import`] is the pure frame publisher that resolves the catalog. +//! Length-prefixed NALU has no stream framing; wrap one access unit with +//! `avc1_frame`. +//! +//! The catalog's `description` decides the framing (an avcC means +//! length-prefixed, none means Annex-B); avc1 and avc3 only say whether the +//! parameter sets may also ride in band. mod export; mod import; @@ -548,8 +552,8 @@ pub(crate) fn avcc_params(avcc: &[u8]) -> Result<(usize, Vec)> { Ok((record.length_size.into(), params)) } -/// Transform H.264 frames from Annex-B (inline SPS/PPS, "avc3") to -/// length-prefixed NALU (out-of-band AVCDecoderConfigurationRecord, "avc1"). +/// Transform H.264 frames from Annex-B (inline SPS/PPS) to length-prefixed +/// NALU with an out-of-band AVCDecoderConfigurationRecord ("avc1"). /// /// The avcC is synthesized from the active SPS+PPS and exposed via /// [`Self::avcc`]. Once it returns `Some`, all subsequent calls to diff --git a/rs/moq-mux/src/codec/h264/split.rs b/rs/moq-mux/src/codec/h264/split.rs index 65d92507ad4..aa5c31f62de 100644 --- a/rs/moq-mux/src/codec/h264/split.rs +++ b/rs/moq-mux/src/codec/h264/split.rs @@ -1,7 +1,7 @@ //! H.264 Annex-B stream splitter. //! -//! [`Split`] turns a raw H.264 Annex-B byte stream (inline SPS/PPS, the "avc3" -//! wire shape) into [`crate::container::Frame`]s. It finds access-unit +//! [`Split`] turns a raw H.264 Annex-B byte stream (inline SPS/PPS, published +//! as avc3 with no description) into [`crate::container::Frame`]s. It finds access-unit //! boundaries, caches SPS/PPS and re-inserts them ahead of each keyframe so //! every keyframe is self-contained, and stamps wall-clock timestamps when the //! caller has none (stdin). @@ -10,8 +10,7 @@ //! track, catalog, or codec config (no [`VideoConfig`](hang::catalog::VideoConfig)). //! The importer parses the codec config out of the frames it emits. //! -//! avc1 (length-prefixed NALU + out-of-band avcC) is not a stream and has no -//! splitter; wrap one access unit with `super::avc1_frame`. +//! Length-prefixed NALU (with an avcC) is not a stream and has no splitter; wrap one access unit with `super::avc1_frame`. use bytes::{Bytes, BytesMut}; diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 35753f1c809..756bdc21dbc 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -1,14 +1,15 @@ //! H.265 importer. //! //! Publishes H.265 frames on a single moq track and resolves the catalog -//! rendition, in either wire shape: Annex-B with inline VPS/SPS/PPS ("hev1"), or -//! length-prefixed NALU with an out-of-band hvcC ("hvc1"). A layered stream +//! rendition, in either framing: Annex-B with inline VPS/SPS/PPS (always hev1), or +//! length-prefixed NALU with an out-of-band hvcC (hev1 when its first keyframe +//! carries a parameter set in band, hvc1 otherwise). A layered stream //! (MV-HEVC, SHVC) imports as its base layer: only a base-layer SPS configures the //! rendition, while every layer's NAL units pass through (VPS is cached but not parsed). //! //! The codec config comes from exactly one of two places: an hvcC handed to -//! [`initialize`](Import::initialize) (the hvc1 shape), or the SPS the splitter -//! packages into the first keyframe (hev1). A keyframe +//! [`initialize`](Import::initialize), or the SPS the splitter packages into the +//! first Annex-B keyframe. A keyframe //! that can't be configured is an error; non-keyframes before the first config //! are written through to the producer, which reports //! [`MissingKeyframe`](crate::container::MissingKeyframe) for a mid-stream join. @@ -28,8 +29,9 @@ use crate::container::Frame; /// Build it with [`new`](Self::new), passing the track producer and the /// [`catalog::Reserved`](crate::catalog::Reserved) it reserves its rendition from, and feed it /// frames a [`Split`](super::Split) produced via [`decode`](Self::decode). The -/// catalog rendition fills in lazily once the codec config is known (hvcC via -/// [`initialize`](Self::initialize) for hvc1, the first base-layer SPS for hev1). +/// catalog rendition fills in lazily once the codec config is known (at the first +/// keyframe after an hvcC via [`initialize`](Self::initialize), or the first +/// Annex-B base-layer SPS). pub struct Import { /// The NALU length size for the hvc1 shape, whose codec config is out-of-band (hvcC), so /// frame payloads are length-prefixed rather than Annex-B. `None` for Annex-B. @@ -37,6 +39,9 @@ pub struct Import { track: crate::container::Producer, catalog: crate::codec::video::Catalog, last_sps: Option, + /// The hvcC's config, held until the first keyframe says whether the stream also repeats its + /// parameter sets in band (hev1) or keeps them in the record only (hvc1). + pending: Option, } impl Import { @@ -60,6 +65,7 @@ impl Import { track, catalog, last_sps: None, + pending: None, }; if let Some(config) = import.catalog.initial_config() { import.apply_config(config)?; @@ -69,11 +75,12 @@ impl Import { /// Resolve the codec config from the codec's leading bytes. /// - /// - **hvc1** (no leading start code): parsed as an `HEVCDecoderConfigurationRecord`, + /// - **hvcC** (no leading start code): parsed as an `HEVCDecoderConfigurationRecord`, /// which resolves the config and is stored as the catalog `description`. Required - /// for hvc1. - /// - **hev1** (leading start code): parsed as Annex-B; the first base-layer SPS resolves the config. - /// Optional, since hev1 also self-initializes from the first keyframe. + /// for length-prefixed frames. The rendition publishes at the first keyframe, as hev1 + /// when that keyframe carries a parameter set in band and hvc1 otherwise. + /// - **Annex-B** (leading start code): the first base-layer SPS resolves the config, as hev1. + /// Optional, since Annex-B also self-initializes from the first keyframe. /// /// Takes a read-only slice: the dispatcher-owned [`Split`](super::Split) is what /// consumes the stream (and reads the same hvcC for the NALU length size). The @@ -90,10 +97,9 @@ impl Import { // Only switch to hvc1 mode once the hvcC actually parses, so a parse failure leaves the // importer in hev1 mode where inline-SPS keyframes still self-initialize. let config = super::config_from_hvcc(hvcc_bytes)?; - // A new record starts a new coded sequence, whose HDR metadata the samples will carry. - self.catalog.hdr(&mut self.track, Default::default(), true)?; self.length_size = Some(super::Hvcc::parse(hvcc_bytes)?.length_size); - self.apply_config(config)?; + // A new record waits for its keyframe; the record already published holds nothing back. + self.pending = (self.catalog.description() != Some(hvcc_bytes)).then_some(config); Ok(()) } @@ -137,16 +143,42 @@ impl Import { /// Finish the track, flushing the current group, and return the [`Finished`](crate::container::Finished) /// track, which keeps it and its catalog rendition published. /// - /// A finish that fails ends the track with the cause instead, so it can't be retried. + /// A record no keyframe followed publishes as it stands. A finish that fails ends the track with + /// the cause instead, so it can't be retried. pub fn finish( - self, + mut self, ) -> crate::Result> { - self.track.finish() + match self.publish_pending() { + Ok(()) => self.track.finish(), + Err(err) => { + drop(self.track.fail(err.reason())); + Err(err) + } + } } /// [`finish`](Self::finish) the track in place, for an owner that keeps this importer. pub(crate) fn finish_in_place(&mut self) -> crate::Result<()> { - self.track.end_with(None) + let published = self.publish_pending(); + let ended = self.track.end_with(published.as_ref().err().map(crate::Error::reason)); + published.and(ended) + } + + /// End the track unless it already ended, dropping a record no keyframe followed, for an owner + /// replacing this importer with a successor. + pub(crate) fn end(&mut self) -> crate::Result<()> { + self.pending = None; + self.track.end() + } + + /// Publish a held record as it stands: no keyframe follows to say where the parameter sets live. + /// It starts a coded sequence, so the last one's in-band HDR metadata goes first. + fn publish_pending(&mut self) -> crate::Result<()> { + let Some(config) = self.pending.take() else { + return Ok(()); + }; + self.catalog.hdr(&mut self.track, Default::default(), true)?; + self.apply_config(config) } /// Abort the track with `err` instead of finishing it cleanly, so subscribers @@ -237,6 +269,11 @@ impl Import { /// keyframe's inline SPS and refining the catalog jitter as it goes. fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { + // A delta between a new record and its keyframe decodes against neither, so it never + // reaches the open group, whose timestamps it is no part of. + if self.pending.is_some() && !frame.keyframe { + return Err(crate::container::MissingKeyframe.into()); + } self.configure(&frame)?; // A keyframe we still can't configure (no SPS) is undecodable. @@ -274,8 +311,14 @@ impl Import { }; // A new SPS starts a coded sequence whose HDR metadata replaces the last one's. It goes // first, so a keyframe that also configures publishes once. - let sequence = sps.as_ref().is_some_and(|sps| self.last_sps.as_ref() != Some(sps)); + // So does a new record, held for this keyframe. + let pending = self.pending.take(); + let sequence = pending.is_some() || sps.as_ref().is_some_and(|sps| self.last_sps.as_ref() != Some(sps)); self.catalog.hdr(&mut self.track, hdr, sequence)?; + if let Some(mut config) = pending { + crate::codec::video::detect_in_band(&mut config, &frame.payload)?; + self.apply_config(config)?; + } if let Some(sps) = sps { self.configure_from_sps(&sps)?; } @@ -301,6 +344,11 @@ pub fn config(buf: &[u8]) -> Result { Ok(super::config_from_sps(&sps, true)?) } +/// Whether a NAL unit header names a parameter set (VPS, SPS, or PPS). +pub(crate) fn is_parameter_set(header: u8) -> bool { + matches!(NalType::of(header), NalType::VPS | NalType::SPS | NalType::PPS) +} + fn is_sps(nal: &[u8]) -> bool { nal.first().is_some_and(|h| NalType::of(*h) == NalType::SPS) } @@ -491,18 +539,67 @@ mod tests { let buf = BytesMut::from(hvcc.as_ref()); import.initialize(&buf).expect("initialize hvcC"); assert_eq!(buf.len(), hvcc.len(), "initialize must not consume the buffer"); + assert!( + catalog.snapshot().video.renditions.is_empty(), + "the first keyframe decides where the parameter sets live" + ); + import.decode([idr_frame(&[])]).unwrap(); let snapshot = catalog.snapshot(); let cfg = snapshot.video.renditions.get("video").expect("rendition"); let hang::catalog::VideoCodec::H265(h265) = &cfg.codec else { panic!("expected H.265 codec") }; - assert!(!h265.in_band, "hvc1 source should land as in_band=false"); + assert!(!h265.in_band, "parameter sets only in the hvcC land as hvc1"); assert_eq!(cfg.coded_width, Some(1280)); assert_eq!(cfg.coded_height, Some(720)); assert_eq!(cfg.description.as_deref(), Some(hvcc.as_ref())); } + /// A length-prefixed IDR access unit, behind `params` (VPS, SPS, PPS NAL units) if any. + fn idr_frame(params: &[&[u8]]) -> Frame { + let idr: &[u8] = &[0x26, 0x01, 0x80, 0xaa]; // IdrWRadl (19) + let mut au = Vec::new(); + for nal in params.iter().chain([&idr]) { + au.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + au.extend_from_slice(nal); + } + crate::codec::h265::hvc1_frame(au, 4, moq_net::Timestamp::from_micros(0).unwrap()).unwrap() + } + + /// An hvcC stream whose keyframes carry any parameter set in band lands as hev1, a VPS alone + /// included, or a PPS the record omits; the record still frames it. + #[tokio::test(start_paused = true)] + async fn hvcc_with_in_band_parameter_sets_lands_as_hev1() { + let full = fixtures::hvcc(); + let no_pps = super::super::build_hvcc( + &[Bytes::from_static(fixtures::VPS)], + &[Bytes::from_static(fixtures::SPS)], + &[], + ) + .unwrap(); + for (hvcc, in_band) in [ + (full.clone(), &[fixtures::VPS, fixtures::SPS, fixtures::PPS][..]), + (full, &[fixtures::VPS][..]), + (no_pps, &[fixtures::PPS][..]), + ] { + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + import.initialize(&hvcc).unwrap(); + import.decode([idr_frame(in_band)]).unwrap(); + + let snapshot = catalog.snapshot(); + let cfg = &snapshot.video.renditions["video"]; + assert_eq!( + cfg.codec.to_string().get(..4), + Some("hev1"), + "{} in band", + in_band.len() + ); + assert_eq!(cfg.description.as_deref(), Some(hvcc.as_ref())); + } + } + /// An hvc1 payload is never scanned for an inline SPS. A NAL of exactly 322 bytes /// carries the 4-byte length prefix `00 00 01 42`, which the Annex-B scanner reads /// as a 3-byte start code followed by an SPS NAL header (0x42 is type 33), so @@ -615,6 +712,7 @@ mod tests { let (track, catalog) = setup("video"); let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); import.initialize(&hvcc).unwrap(); + import.decode([idr_frame(&[])]).unwrap(); assert_eq!(size(&catalog), (Some(128), Some(64))); let (track, catalog) = setup("video"); diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index 81697a25890..9ee66d94259 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -8,6 +8,104 @@ use crate::catalog::VideoHint; +/// Mark an H.264 or H.265 config in band when the first keyframe after its record carries any +/// parameter set. +/// +/// The record's parameter sets seed every keyframe; any one in band (an SPS, a PPS, or an H.265 +/// VPS) means a keyframe may also change or supply them, which only the `avc3`/`hev1` codec string +/// allows. `keyframe` is length-prefixed by the record's length size. A config without a record is +/// left alone. +pub(crate) fn detect_in_band(config: &mut hang::catalog::VideoConfig, keyframe: &[u8]) -> crate::Result<()> { + let mut scan = InBand::new(config)?; + scan.scan(keyframe); + scan.apply(config); + Ok(()) +} + +/// [`detect_in_band`] over a keyframe that arrives in pieces, holding only the current NAL unit's +/// length prefix and how much of it is left. +pub(crate) struct InBand { + /// How the record frames the keyframe, or `None` for a config this leaves alone. + framing: Option, + /// The length prefix read so far, and how many of its bytes. + length: usize, + read: usize, + /// The bytes left in the current NAL unit, whose first is its header when `header` is set. + remaining: usize, + header: bool, + found: bool, +} + +/// A record's NALU length size and its codec's check for a parameter-set NAL header. +#[derive(Clone, Copy)] +struct Framing { + length_size: usize, + is_parameter_set: fn(u8) -> bool, +} + +impl InBand { + /// A scan for the record `config` carries. + pub(crate) fn new(config: &hang::catalog::VideoConfig) -> crate::Result { + let codec = match (&config.codec, config.description.as_deref()) { + (hang::catalog::VideoCodec::H264(_), Some(record)) => Some(Framing { + length_size: super::h264::Avcc::parse(record)?.length_size, + is_parameter_set: super::h264::is_parameter_set, + }), + (hang::catalog::VideoCodec::H265(_), Some(record)) => Some(Framing { + length_size: super::h265::Hvcc::parse(record)?.length_size, + is_parameter_set: super::h265::is_parameter_set, + }), + _ => None, + }; + Ok(Self { + framing: codec, + length: 0, + read: 0, + remaining: 0, + header: false, + found: false, + }) + } + + /// Read the next bytes of the keyframe. + pub(crate) fn scan(&mut self, mut bytes: &[u8]) { + let Some(Framing { + length_size, + is_parameter_set, + }) = self.framing + else { + return; + }; + while !self.found && !bytes.is_empty() { + if self.remaining > 0 { + if std::mem::take(&mut self.header) { + self.found = is_parameter_set(bytes[0]); + } + let skip = self.remaining.min(bytes.len()); + self.remaining -= skip; + bytes = &bytes[skip..]; + } else { + self.length = (self.length << 8) | usize::from(bytes[0]); + self.read += 1; + bytes = &bytes[1..]; + if self.read == length_size { + (self.remaining, self.header) = (self.length, self.length > 0); + (self.length, self.read) = (0, 0); + } + } + } + } + + /// Mark `config` in band if the bytes read so far carried a parameter set. + pub(crate) fn apply(&self, config: &mut hang::catalog::VideoConfig) { + match &mut config.codec { + hang::catalog::VideoCodec::H264(codec) => codec.inline |= self.found, + hang::catalog::VideoCodec::H265(codec) => codec.in_band |= self.found, + _ => {} + } + } +} + /// The predefined sample aspects shared by H.264 and H.265 (their VUI Table E-1). pub(super) fn sample_aspect(idc: u8) -> Option<(u32, u32)> { Some(match idc { @@ -236,6 +334,11 @@ impl Catalog { self.last.is_some() } + /// The out-of-band record the last published config carries, if any. + pub(crate) fn description(&self) -> Option<&[u8]> { + self.last.as_ref()?.description.as_deref() + } + /// Overlay the hint onto `config` and publish it to `rendition`, unless it matches the last /// publish. A decoder identity change finishes the previous track and mints a fresh name; /// anything else updates the rendition in place. @@ -301,6 +404,49 @@ impl Catalog { #[cfg(test)] mod tests { + /// The in-band scan reads NAL headers only, never their bodies, and finds the same answer however + /// the keyframe is split. + #[test] + fn in_band_scan_matches_across_any_split() { + use crate::codec::h264::fixtures::{PPS, SPS_BT709}; + use bytes::Bytes; + + let avcc = + crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_BT709)], &[Bytes::from_static(PPS)]).unwrap(); + let config = crate::codec::h264::config(&avcc).unwrap(); + // An IDR whose body is full of bytes that would read as an SPS header. + let idr: &[u8] = &[0x65, 0x67, 0x67, 0x68, 0x67]; + let au = |nals: &[&[u8]]| -> Vec { + nals.iter() + .flat_map(|nal| [&(nal.len() as u32).to_be_bytes()[..], nal].concat()) + .collect() + }; + for (keyframe, expected) in [ + (au(&[idr]), false), + (au(&[idr, idr]), false), + (au(&[SPS_BT709, PPS, idr]), true), + (au(&[idr, PPS]), true), + (au(&[&[], idr]), false), + ] { + let detected = |pieces: &[&[u8]]| { + let mut scan = InBand::new(&config).unwrap(); + for piece in pieces { + scan.scan(piece); + } + let mut config = config.clone(); + scan.apply(&mut config); + config.codec.to_string().starts_with("avc3") + }; + assert_eq!(detected(&[&keyframe]), expected, "{keyframe:?}"); + for at in 0..=keyframe.len() { + let (head, tail) = keyframe.split_at(at); + assert_eq!(detected(&[head, tail]), expected, "{keyframe:?} split at {at}"); + } + let bytes: Vec<&[u8]> = keyframe.chunks(1).collect(); + assert_eq!(detected(&bytes), expected, "{keyframe:?} byte at a time"); + } + } + use super::*; use crate::container::Frame; diff --git a/rs/moq-mux/src/container/flv/export.rs b/rs/moq-mux/src/container/flv/export.rs index 9c3d03e77f6..071c5b65510 100644 --- a/rs/moq-mux/src/container/flv/export.rs +++ b/rs/moq-mux/src/container/flv/export.rs @@ -134,9 +134,9 @@ impl Flavor { /// tag for one media frame. Returns `None` when the broadcast ends. Narrow the /// renditions first with [`Stream::select`](crate::catalog::Stream::select). /// -/// ## Avc3 sources +/// ## Annex-B sources /// -/// Annex-B H.264 (`H264 { inline: true }`, empty `description`) is accepted: an +/// Annex-B H.264 (an empty `description`, whatever the codec string) is accepted: an /// [`Avc1`](crate::codec::h264::Avc1) transform caches the inline SPS/PPS, builds /// the avcC for the sequence-header tag, and length-prefixes each sample. The /// header is deferred until that codec config is available (typically the first diff --git a/rs/moq-mux/src/container/flv/import.rs b/rs/moq-mux/src/container/flv/import.rs index bb8db3c8f63..99f9fa8049d 100644 --- a/rs/moq-mux/src/container/flv/import.rs +++ b/rs/moq-mux/src/container/flv/import.rs @@ -98,13 +98,74 @@ pub struct Import { input: crate::container::Input, } -/// The demuxed video track plus its current catalog config, so a repeated -/// (identical) sequence header is a no-op rather than a track rebuild. -struct VideoStream { +/// A demuxed video track, holding what it was configured from so a repeated (identical) sequence +/// header is a no-op rather than a track rebuild. +enum VideoStream { + /// H.264 through its importer, which holds the avcC until the next keyframe says whether the + /// parameter sets also ride in band. + H264 { + import: Box, + record: Bytes, + }, + /// H.265 through its importer, which holds the hvcC the same way. + H265 { + import: Box, + record: Bytes, + }, + /// AV1 or VP9, configured from its av1C or keyframe header. + Plain(Box), +} + +/// An AV1 or VP9 track plus the config it was built from. +struct PlainVideo { track: crate::container::Producer, config: VideoConfig, } +impl VideoStream { + fn write(&mut self, frame: Frame) -> crate::Result<()> { + match self { + Self::H264 { import, .. } => import.decode([frame]), + Self::H265 { import, .. } => import.decode([frame]), + Self::Plain(plain) => plain.track.write(frame), + } + } + + fn seek(&mut self, start: u64) -> crate::Result<()> { + match self { + Self::H264 { import, .. } => import.seek(start), + Self::H265 { import, .. } => import.seek(start), + Self::Plain(plain) => plain.track.seek(start), + } + } + + /// End the track for a successor or a drop, leaving a record no keyframe followed unpublished. + fn end(&mut self) -> crate::Result<()> { + match self { + Self::H264 { import, .. } => import.end(), + Self::H265 { import, .. } => import.end(), + Self::Plain(plain) => plain.track.end(), + } + } + + /// End the track as the input ends, publishing a record no keyframe followed as it stands. + fn finish(&mut self) -> crate::Result<()> { + match self { + Self::H264 { import, .. } => import.finish_in_place(), + Self::H265 { import, .. } => import.finish_in_place(), + Self::Plain(plain) => plain.track.end(), + } + } + + fn abort(self, err: moq_net::Error) { + match self { + Self::H264 { import, .. } => import.abort(err), + Self::H265 { import, .. } => import.abort(err), + Self::Plain(plain) => plain.track.abort(err), + } + } +} + /// The demuxed audio track plus its current catalog config. struct AudioStream { track: crate::container::Producer, @@ -238,7 +299,7 @@ impl Import { let data = &body[5..]; match avc_packet_type { - AVC_SEQUENCE_HEADER => self.init_video(DEFAULT_TRACK_ID, crate::codec::h264::config(data)?), + AVC_SEQUENCE_HEADER => self.video_record(DEFAULT_TRACK_ID, b"avc1", data), AVC_NALU => self.write_video( DEFAULT_TRACK_ID, data, @@ -311,8 +372,7 @@ impl Import { match packet_type { VIDEO_PACKET_SEQUENCE_START => { let config = match fourcc { - b"avc1" => crate::codec::h264::config(payload)?, - b"hvc1" => crate::codec::h265::config_from_hvcc(payload)?, + b"avc1" | b"hvc1" => return self.video_record(track_id, fourcc, payload), b"av01" => crate::codec::av1::config_from_av1c(payload)?, // VP9 carries its config in band; the SequenceStart vpcC is // redundant with the key-frame header we configure from. @@ -502,10 +562,11 @@ impl Import { let shifted = self.timebase.shift(Timestamp::from_millis(pts_ms as u64)?); let timestamp = self.refuse(shifted)?; // A media frame means every sequence header has arrived (FLV sends config before data), so - // the track set is declared; release the reservation to publish. + // the track set is declared; release the reservation to publish. An H.264/H.265 rendition + // still waiting for its first keyframe holds the catalog back on its own. self.initial_reservation = None; let stream = self.video.get_mut(&track_id).expect("checked above"); - let result = match stream.track.write(Frame { + let result = match stream.write(Frame { timestamp, duration: None, payload: Bytes::copy_from_slice(data), @@ -547,7 +608,71 @@ impl Import { result.inspect_err(|err| self.input = crate::container::Input::Refused(err.clone())) } - /// (Re)build the video track `track_id` for `config`, unless it matches the current one. + /// Hand an H.264 (`avc1`) or H.265 (`hvc1`) sequence header to the track's codec importer, + /// which holds it until the next keyframe says whether the parameter sets also ride in band. + /// A header for another codec replaces the track. + fn video_record(&mut self, track_id: u8, fourcc: &[u8; 4], record: &[u8]) -> crate::Result<()> { + // Validate before touching the track, so a malformed header leaves it as it was. + match fourcc { + b"hvc1" => drop(crate::codec::h265::config_from_hvcc(record)?), + _ => drop(crate::codec::h264::config_from_avcc(record)?), + } + let last = match (self.video.get_mut(&track_id), fourcc) { + (Some(VideoStream::H264 { import, record: last }), b"avc1") if last.as_ref() != record => { + import.initialize(record)?; + last + } + (Some(VideoStream::H265 { import, record: last }), b"hvc1") if last.as_ref() != record => { + import.initialize(record)?; + last + } + // Unchanged, the importer would only re-detect what its last keyframe already settled. + (Some(VideoStream::H264 { .. }), b"avc1") | (Some(VideoStream::H265 { .. }), b"hvc1") => return Ok(()), + _ => { + let result = self.build_record(track_id, fourcc, record); + return self.refuse(result); + } + }; + *last = Bytes::copy_from_slice(record); + Ok(()) + } + + fn build_record(&mut self, track_id: u8, fourcc: &[u8; 4], record: &[u8]) -> crate::Result<()> { + let track = self.replace_video(track_id)?; + // Past the initial reservation, a fresh one still holds an unpublished catalog for this + // track's first keyframe, and holds nothing once the catalog went out. + let reserved = match &self.initial_reservation { + Some(reserved) => reserved.clone(), + None => self.timebase.reserve(), + }; + let hint = crate::catalog::VideoHint { + container: self.container.clone(), + ..Default::default() + }; + let record = Bytes::copy_from_slice(record); + let stream = match fourcc { + b"hvc1" => { + let mut import = crate::codec::h265::Import::new(track, reserved, hint)?; + import.initialize(&record)?; + VideoStream::H265 { + import: Box::new(import), + record, + } + } + _ => { + let mut import = crate::codec::h264::Import::new(track, reserved, hint)?; + import.initialize(&record)?; + VideoStream::H264 { + import: Box::new(import), + record, + } + } + }; + self.video.insert(track_id, stream); + Ok(()) + } + + /// (Re)build the AV1/VP9 track `track_id` for `config`, unless it matches the current one. fn init_video(&mut self, track_id: u8, config: VideoConfig) -> crate::Result<()> { let result = self.build_video(track_id, config); self.refuse(result) @@ -555,7 +680,9 @@ impl Import { fn build_video(&mut self, track_id: u8, mut config: VideoConfig) -> crate::Result<()> { config.container = self.container.clone(); - if self.video.get(&track_id).is_some_and(|s| s.config == config) { + if let Some(VideoStream::Plain(current)) = self.video.get(&track_id) + && current.config == config + { return Ok(()); } @@ -570,12 +697,12 @@ impl Import { }; self.video.insert( track_id, - VideoStream { + VideoStream::Plain(Box::new(PlainVideo { // Leading deltas before the first keyframe are skipped at the write // site (the producer reports MissingKeyframe), so a mid-GOP join works. track: media, config, - }, + })), ); Ok(()) } @@ -607,8 +734,8 @@ impl Import { /// Retire any existing video track `track_id` and allocate a fresh one. fn replace_video(&mut self, track_id: u8) -> crate::Result { - if let Some(old) = self.video.remove(&track_id) { - self.retire(old.track)?; + if let Some(mut old) = self.video.remove(&track_id) { + old.end()?; } Ok(self .broadcast @@ -617,28 +744,19 @@ impl Import { /// Retire any existing audio track `track_id` and allocate a fresh one. fn replace_audio(&mut self, track_id: u8) -> crate::Result { - if let Some(old) = self.audio.remove(&track_id) { - self.retire(old.track)?; + if let Some(mut old) = self.audio.remove(&track_id) { + old.track.end()?; } Ok(self .broadcast .unique_track(".flv-a", self.catalog.track_info(hang::catalog::PRIORITY.audio))?) } - /// End a track this importer is done with, clearing its catalog rendition. Every track takes a - /// fresh name, so a publisher changing its config holds only its live tracks. - fn retire( - &self, - mut track: crate::container::Producer, - ) -> crate::Result<()> { - track.end() - } - /// Close the current group on every track and reopen at `start`. pub fn seek(&mut self, start: u64) -> crate::Result<()> { self.input.check()?; for stream in self.video.values_mut() { - stream.track.seek(start)?; + stream.seek(start)?; } for stream in self.audio.values_mut() { stream.track.seek(start)?; @@ -663,11 +781,12 @@ impl Import { } // No input follows, so the buffer that waited for the rest of a tag is done. self.buffer = BytesMut::new(); - // No frame follows to anchor the clock, so publish the declared track set now. + // No frame follows to anchor the clock, so publish the declared track set now, with a track + // whose record no keyframe followed as the record stands. self.initial_reservation = None; let mut result = Ok(()); for stream in self.video.values_mut() { - result = result.and(stream.track.end()); + result = result.and(stream.finish()); } for stream in self.audio.values_mut() { result = result.and(stream.track.end()); @@ -690,7 +809,7 @@ impl Import { /// rather than ending them as a complete recording. pub fn abort(mut self, err: moq_net::Error) { for stream in std::mem::take(&mut self.video).into_values() { - stream.track.abort(err.clone()); + stream.abort(err.clone()); } for stream in std::mem::take(&mut self.audio).into_values() { stream.track.abort(err.clone()); @@ -724,8 +843,8 @@ impl Finished { impl Drop for Import { fn drop(&mut self) { let (video, audio) = (std::mem::take(&mut self.video), std::mem::take(&mut self.audio)); - let video = video.into_values().map(|stream| self.retire(stream.track)); - let audio = audio.into_values().map(|stream| self.retire(stream.track)); + let video = video.into_values().map(|mut stream| stream.end()); + let audio = audio.into_values().map(|mut stream| stream.track.end()); for result in video.chain(audio) { if let Err(err) = result { tracing::warn!(%err, "failed to end a dropped FLV track"); diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index 12987b3484d..c19570050d4 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -459,6 +459,17 @@ async fn import_multitrack_video_many_codecs() { ); write_tag(&mut out, super::TAG_VIDEO, 0, &seq); + // A record publishes at its track's first keyframe. + let idr = vec![0, 0, 0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]; + let frames = multitrack_video_body( + super::FRAME_TYPE_KEY, + super::MULTITRACK_MANY_TRACKS_MANY_CODECS, + super::VIDEO_PACKET_CODED_FRAMES, + None, + &[(0, *b"avc1", idr.clone()), (1, *b"avc1", idr)], + ); + write_tag(&mut out, super::TAG_VIDEO, 0, &frames); + let mut producer = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut producer, crate::catalog::Config::default()).unwrap(); let mut importer = Import::new(producer, catalog.reserve()); @@ -468,6 +479,82 @@ async fn import_multitrack_video_many_codecs() { assert_eq!(catalog.snapshot().video.renditions.len(), 2); } +/// FLV video tags: an AVC sequence header carrying `avcc()`, then one keyframe of `nalus`. +fn avc_session(nalus: &[u8]) -> (Vec, Vec) { + let mut headers = flv_header(0x05); + let mut vseq = vec![ + (super::FRAME_TYPE_KEY << 4) | super::VIDEO_CODEC_AVC, + super::AVC_SEQUENCE_HEADER, + 0, + 0, + 0, + ]; + vseq.extend_from_slice(&avcc()); + write_tag(&mut headers, super::TAG_VIDEO, 0, &vseq); + let mut aseq = vec![super::AAC_AUDIO_TAG_HEADER, super::AAC_SEQUENCE_HEADER]; + aseq.extend_from_slice(&ASC); + write_tag(&mut headers, super::TAG_AUDIO, 0, &aseq); + + let mut keyframe = Vec::new(); + let mut vframe = vec![ + (super::FRAME_TYPE_KEY << 4) | super::VIDEO_CODEC_AVC, + super::AVC_NALU, + 0, + 0, + 0, + ]; + vframe.extend_from_slice(nalus); + write_tag(&mut keyframe, super::TAG_VIDEO, 0, &vframe); + (headers, keyframe) +} + +/// An avcC whose keyframes repeat the SPS in band lands as `avc3`; one whose keyframes don't stays +/// `avc1`. The record holds the catalog until the keyframe, even past an earlier audio frame. +#[tokio::test(start_paused = true)] +async fn avc_keyframe_decides_where_parameter_sets_live() { + let idr = [0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]; + let mut in_band = vec![0, 0, 0, 4, 0x67, 0x42, 0xc0, 0x1f]; + in_band.extend_from_slice(&idr); + + for (nalus, inline) in [(idr.to_vec(), false), (in_band, true)] { + let (headers, keyframe) = avc_session(&nalus); + let mut audio = Vec::new(); + write_tag( + &mut audio, + super::TAG_AUDIO, + 0, + &[super::AAC_AUDIO_TAG_HEADER, super::AAC_RAW, 0xde, 0xad], + ); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut clocks = crate::container::test_util::Clocks::subscribe(&consumer).await; + let mut importer = Import::new(broadcast, catalog.reserve()); + + importer.decode(&headers).unwrap(); + importer.decode(&audio).unwrap(); + assert_eq!( + clocks.drain(), + vec![], + "the record withholds the catalog until its keyframe" + ); + + importer.decode(&keyframe).unwrap(); + assert!(!clocks.drain().is_empty(), "the keyframe publishes the catalog"); + let _finished = importer.finish().unwrap(); + + let snap = catalog.snapshot(); + assert_eq!(snap.audio.renditions.len(), 1); + let video = snap.video.renditions.values().next().expect("a video rendition"); + let VideoCodec::H264(codec) = &video.codec else { + panic!("{:?}", video.codec); + }; + assert_eq!(codec.inline, inline, "{}", video.codec); + assert_eq!(video.description.as_deref(), Some(avcc().as_slice())); + } +} + /// A publisher that keeps changing its audio and video config replaces each track under a fresh /// name. A replaced track flushes its last frame and ends, and the importer retires it, so the HLS /// records it holds stay those of the live renditions. Dropping the importer ends and retires those. @@ -1038,3 +1125,137 @@ async fn an_unmappable_restart_refuses_later_input() { "refused with the cause: {err:?}" ); } + +/// A newer sequence header replaces an AVC record still waiting for its keyframe, so the stale +/// record never claims the new codec's keyframe. +#[tokio::test(start_paused = true)] +async fn a_newer_sequence_header_supersedes_a_held_record() { + let (mut out, _) = avc_session(&[]); + let mut seq = vec![super::VIDEO_EX_HEADER | (super::FRAME_TYPE_KEY << 4) | super::VIDEO_PACKET_SEQUENCE_START]; + seq.extend_from_slice(b"av01"); + seq.extend_from_slice(&AV1C); + write_tag(&mut out, super::TAG_VIDEO, 0, &seq); + let mut frame = vec![super::VIDEO_EX_HEADER | (super::FRAME_TYPE_KEY << 4) | super::VIDEO_PACKET_CODED_FRAMES_X]; + frame.extend_from_slice(b"av01"); + frame.extend_from_slice(&[0x12, 0x00, 0x34, 0x56]); + write_tag(&mut out, super::TAG_VIDEO, 0, &frame); + + let mut producer = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut producer, crate::catalog::Config::default()).unwrap(); + let mut importer = Import::new(producer, catalog.reserve()); + importer.decode(&out).unwrap(); + let _finished = importer.finish().unwrap(); + + let codecs: Vec<_> = catalog + .snapshot() + .video + .renditions + .into_values() + .map(|v| v.codec) + .collect(); + assert!(matches!(codecs[..], [VideoCodec::AV1(_)]), "{codecs:?}"); +} + +/// A seek made while a record waits for its keyframe keys the group that keyframe opens, even when +/// the sequence header repeats in between. +#[tokio::test(start_paused = true)] +async fn a_seek_reaches_a_track_waiting_for_its_keyframe() { + let (headers, keyframe) = avc_session(&[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut importer = Import::new(broadcast, catalog.reserve()); + importer.decode(&headers).unwrap(); + importer.seek(100).unwrap(); + // The same sequence headers again, past the FLV file header. + importer.decode(&headers[flv_header(0).len()..]).unwrap(); + importer.decode(&keyframe).unwrap(); + let _finished = importer.finish().unwrap(); + + let snapshot = catalog.snapshot(); + let name = snapshot.video.renditions.keys().next().unwrap(); + let replay = moq_net::track::Subscription::default() + .with_delay(moq_net::track::Delay::default().with_max(Duration::from_secs(3600))); + let mut track = consumer.track(name).unwrap().subscribe(replay).await.unwrap(); + let mut starts = Vec::new(); + while let Some(group) = track.recv_group().await.unwrap() { + starts.push(group.start); + } + assert_eq!(starts, [100]); +} + +/// A changed sequence header on a live track skips the deltas before its keyframe, which can't +/// decode against the old record and whose timestamps the old track never checks, and a seek in +/// between keys the group that keyframe opens. +#[tokio::test(start_paused = true)] +async fn a_changed_sequence_header_waits_for_its_keyframe() { + let video = |frame_type: u8, nalus: &[u8], ms: u32, cts: i32| { + let cts = cts.to_be_bytes(); + let mut body = vec![ + (frame_type << 4) | super::VIDEO_CODEC_AVC, + super::AVC_NALU, + cts[1], + cts[2], + cts[3], + ]; + body.extend_from_slice(nalus); + let mut out = Vec::new(); + write_tag(&mut out, super::TAG_VIDEO, ms, &body); + out + }; + let idr = [0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]; + let delta = [0, 0, 0, 5, 0x41, 0x9a, 0x02, 0x03, 0x04]; + let (headers, keyframe) = avc_session(&idr); + let mut changed = vec![ + (super::FRAME_TYPE_KEY << 4) | super::VIDEO_CODEC_AVC, + super::AVC_SEQUENCE_HEADER, + 0, + 0, + 0, + ]; + changed.extend_from_slice(&avcc_level(0x28)); + let mut header = Vec::new(); + write_tag(&mut header, super::TAG_VIDEO, 133, &changed); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut importer = Import::new(broadcast, catalog.reserve()); + let replay = || { + moq_net::track::Subscription::default() + .with_delay(moq_net::track::Delay::default().with_max(Duration::from_secs(3600))) + }; + importer.decode(&headers).unwrap(); + importer.decode(&keyframe).unwrap(); + importer.decode(&video(2, &delta, 33, 0)).unwrap(); + importer.decode(&video(super::FRAME_TYPE_KEY, &idr, 80, 0)).unwrap(); + importer.decode(&video(super::FRAME_TYPE_KEY, &idr, 100, 0)).unwrap(); + let old = catalog.snapshot().video.renditions.keys().next().unwrap().clone(); + let mut old = consumer.track(&old).unwrap().subscribe(replay()).await.unwrap(); + + importer.decode(&header).unwrap(); + // Presented before the old track's 80 ms group, the floor a delta is checked against, which + // would refuse it as a rewind. + importer.decode(&video(2, &delta, 133, -60)).unwrap(); + importer.seek(200).unwrap(); + importer.decode(&video(super::FRAME_TYPE_KEY, &idr, 200, 0)).unwrap(); + let _finished = importer.finish().unwrap(); + + let mut deltas = 0; + while let Some(mut group) = old.recv_group().await.unwrap() { + while let Some(frame) = group.read_frame().await.unwrap() { + deltas += usize::from(frame.payload.ends_with(&delta)); + } + } + assert_eq!(deltas, 1, "the delta after the new header never reaches the old track"); + + let snapshot = catalog.snapshot(); + let name = snapshot.video.renditions.keys().next().unwrap(); + let mut track = consumer.track(name).unwrap().subscribe(replay()).await.unwrap(); + let mut starts = Vec::new(); + while let Some(group) = track.recv_group().await.unwrap() { + starts.push(group.start); + } + assert_eq!(starts, [200]); +} diff --git a/rs/moq-mux/src/container/fmp4/export_test.rs b/rs/moq-mux/src/container/fmp4/export_test.rs index 8711189d5b1..b8d9f3bfabe 100644 --- a/rs/moq-mux/src/container/fmp4/export_test.rs +++ b/rs/moq-mux/src/container/fmp4/export_test.rs @@ -2339,6 +2339,47 @@ async fn a_changed_catalog_profile_fails() { ); } +/// A Legacy avc3 track with a full avcC, as the fMP4 import publishes an avc3 sample entry, +/// passes its samples through with their in-band parameter sets, so it exports as avc3 again. +#[tokio::test(start_paused = true)] +async fn avc3_with_a_record_exports_as_avc3() { + use hang::catalog::{Container, H264, VideoConfig}; + + let avcc = crate::codec::h264::build_avcc(&[Bytes::from_static(SPS)], &[Bytes::from_static(PPS)]).unwrap(); + let description = avcc.clone(); + let mut live = Live::new(".avc3", move |catalog, name| { + let mut config = VideoConfig::new(H264 { + profile: 0x42, + constraints: 0xc0, + level: 0x1f, + inline: true, + }); + config.coded_width = Some(320); + config.coded_height = Some(240); + config.framerate = Some(30.0); + config.container = Container::Legacy; + config.description = Some(description); + catalog.modify().unwrap().video.renditions.insert(name, config); + }); + live.track + .write(crate::container::Frame { + timestamp: moq_net::Timestamp::from_micros(0).unwrap(), + payload: length_prefixed(&[SPS, PPS, IDR]).into(), + keyframe: true, + duration: None, + }) + .unwrap(); + live.track.end_with(None).unwrap(); + + let mut exporter = crate::container::fmp4::Export::new(live.source().await, live.catalog_stream().await) + .with_delay(moq_net::track::Delay::default().with_max(RECORDING_MAX_DELAY)); + let init = chunk_now(&mut exporter).await.init().expect("init"); + let mp4_atom::Codec::Avc3(avc3) = video_codec(&init) else { + panic!("expected an avc3 sample entry"); + }; + assert_eq!(avc3.avcc.sequence_parameter_sets[0].as_slice(), SPS); +} + /// High 10 cannot be described from the codec string, so the init waits for the /// SPS and then declares avc1 with that SPS inside the avcC. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 048bce84069..5bae1b119dc 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -1022,9 +1022,13 @@ pub(crate) fn synthesize_video_trak( VideoCodec::H264(h264) => { let mut cursor = std::io::Cursor::new(require_description()?); let avcc = mp4_atom::Avcc::decode_body(&mut cursor).map_err(Error::mp4)?; - // Empty parameter-set lists are the in-band sample entry. A record that - // still carries them, including one built from the first keyframe, stays avc1. - if h264.inline && avcc.sequence_parameter_sets.is_empty() && avcc.picture_parameter_sets.is_empty() { + // avc3 when the samples may still carry parameter sets: a catalog record + // passes the samples through as they are, and empty lists are the in-band + // entry. A record built from the first keyframe strips them from the + // samples, so it stays avc1. + let catalog_record = config.description.as_ref().is_some_and(|d| !d.is_empty()); + let empty = avcc.sequence_parameter_sets.is_empty() && avcc.picture_parameter_sets.is_empty(); + if h264.inline && (catalog_record || empty) { mp4_atom::Codec::from(mp4_atom::Avc3 { visual, avcc, diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 7d5b3ac6181..096fdb051c0 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -34,10 +34,10 @@ const TIMESTAMP_SCALE_NS: u64 = 1_000_000; /// duration for downstream consumers that throttle by fragment rate. /// Returns `None` when the broadcast ends. /// -/// ## Avc3 / Hev1 sources +/// ## Annex-B sources /// -/// `Export` accepts Annex-B sources (`H264 { inline: true }`, `H265 { in_band: true }`, -/// catalog `description` empty) by attaching a [`crate::codec::h264::Avc1`] / +/// `Export` accepts Annex-B sources (H.264 or H.265 with an empty catalog +/// `description`, whatever the codec string) by attaching a [`crate::codec::h264::Avc1`] / /// [`crate::codec::h265::Hvc1`] to each affected track. The transform caches /// parameter sets, builds the out-of-band `AVCDecoderConfigurationRecord` / /// `HEVCDecoderConfigurationRecord`, and length-prefixes sample NALs. Header @@ -839,7 +839,7 @@ fn build_video_track_entry( description: Option<&Bytes>, ) -> Result { // The description came from either the catalog (avc1/hvc1 sources) or - // the codec transform (Avc3/Hev1 sources synthesizing it from inline params). + // the codec transform (Annex-B sources synthesizing it from inline params). let codec_private = description.map(|b| b.to_vec()); let (codec_id, codec_private) = match &config.codec { diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index 46f0a6e8539..ee3431f58a8 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -155,6 +155,16 @@ struct MkvTrack { default_duration: Option, kind: TrackKind, track: Media, + /// An H.264/H.265 record held unpublished until the track's first keyframe commits, whose + /// bytes say whether the parameter sets also ride in band. The rendition withholds the catalog + /// meanwhile. + record: Option, +} + +struct Record { + config: VideoConfig, + /// The scan of the first keyframe's bytes so far, once it opens. + keyframe: Option, } enum Media { @@ -210,6 +220,14 @@ impl Media { Self::Audio(track) => track.abort(err), } } + + /// Publish a video track's config. + fn configure(&mut self, config: VideoConfig) -> crate::Result<()> { + match self { + Self::Video(track) => track.set(config), + Self::Audio(_) => unreachable!("only a video track holds a record"), + } + } } /// How the importer consumes an element. @@ -651,16 +669,22 @@ impl Import { TrackKind::Audio => crate::container::Kind::Audio, }, )?; - let media = match kind { + let (media, record) = match kind { TrackKind::Video => { let mut config = build_video_config(&codec_id, codec_private.as_ref(), video_children.as_deref())?; config.container = self.container.clone(); - Media::Video(reserved.video(track, wire, config)?) + match config.codec { + VideoCodec::H264(_) | VideoCodec::H265(_) => { + let record = Record { config, keyframe: None }; + (Media::Video(reserved.video(track, wire, None)?), Some(record)) + } + _ => (Media::Video(reserved.video(track, wire, config)?), None), + } } TrackKind::Audio => { let mut config = build_audio_config(&codec_id, codec_private.as_ref(), audio_children.as_deref())?; config.container = self.container.clone(); - Media::Audio(reserved.audio(track, wire, config)?) + (Media::Audio(reserved.audio(track, wire, config)?), None) } }; @@ -670,6 +694,7 @@ impl Import { default_duration, kind, track: media, + record, }, ); @@ -835,14 +860,23 @@ impl Import { }; let size = usize::try_from(size).map_err(|_| moq_net::Error::FrameTooLarge)?; frames.index += 1; - frames - .open - .insert(track.track.open(timestamp, keyframe, frames.duration, size)?) + let frame = track.track.open(timestamp, keyframe, frames.duration, size)?; + if keyframe && let Some(record) = &mut track.record { + // A keyframe a failed block cut short starts over. + record.keyframe = Some(crate::codec::video::InBand::new(&record.config)?); + } + frames.open.insert(frame) } }; let (bytes, rest) = chunk.split_at(frame.remaining().min(chunk.len())); if !bytes.is_empty() { frame.write(bytes)?; + if let Some(Record { + keyframe: Some(scan), .. + }) = &mut track.record + { + scan.scan(bytes); + } } chunk = rest; if frame.remaining() > 0 { @@ -850,6 +884,14 @@ impl Import { } let frame = frames.open.take().expect("a frame is open"); track.track.commit(frame)?; + if let Some(Record { + mut config, + keyframe: Some(scan), + }) = track.record.take_if(|record| record.keyframe.is_some()) + { + scan.apply(&mut config); + track.track.configure(config)?; + } } Ok(total - chunk.len()) } @@ -928,7 +970,13 @@ impl Import { // A frame cut short fails its group with the reason, rather than waiting for the rest. self.abort_block(err.reason()); } - // No frame follows to anchor the clock, so publish the declared track set now. + // No frame follows to anchor the clock, so publish the declared track set now, with a track + // whose record no keyframe followed as the record stands. + for track in self.tracks.values_mut() { + if let Some(record) = track.record.take() { + result = result.and_then(|()| track.track.configure(record.config)); + } + } self.initial_reservation = None; for track in self.tracks.values_mut() { // End every track even on refusal, keeping the first error. A track whose finish fails @@ -2024,6 +2072,8 @@ mod tests { assert!(hvcc.retained() <= 2 * canonical.len(), "{}", hvcc.retained()); } } + // No keyframe follows, so the record publishes as it stands at finish. + let _finished = import.finish().unwrap(); let config = catalog.snapshot().video.renditions.into_values().next().unwrap(); assert_eq!(config.description.as_deref(), Some(canonical.as_slice())); } diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index 0de79ef1add..ddef36ec993 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -2147,3 +2147,107 @@ fn repeated_reinit_keeps_flat_state() { assert!(snapshot.audio.renditions.is_empty(), "{label}"); } } + +/// An avcC or hvcC track whose first keyframe carries a parameter set in band lands as +/// `avc3`/`hev1`; one whose keyframe doesn't stays `avc1`/`hvc1`. The keyframe decides even when it +/// streams in a byte at a time behind an earlier audio block, and the catalog waits for it: the +/// first one published already lists the video. +#[tokio::test(start_paused = true)] +async fn keyframe_decides_where_parameter_sets_live() { + use crate::codec::{h264, h265}; + use bytes::Bytes; + + let avcc = h264::build_avcc( + &[Bytes::from_static(h264::fixtures::SPS_IPB)], + &[Bytes::from_static(h264::fixtures::PPS)], + ) + .unwrap(); + let hvcc = h265::build_hvcc( + &[Bytes::from_static(h265::fixtures::VPS)], + &[Bytes::from_static(h265::fixtures::SPS_NVENC)], + &[Bytes::from_static(h265::fixtures::PPS)], + ) + .unwrap(); + let prefixed = |nals: &[&[u8]]| -> Vec { + nals.iter() + .flat_map(|nal| [&(nal.len() as u32).to_be_bytes()[..], nal].concat()) + .collect() + }; + let h264_idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + let h265_idr: &[u8] = &[0x26, 0x01, 0xaf, 0x10]; + + for (codec_id, private, sps, idr) in [ + ("V_MPEG4/ISO/AVC", avcc, h264::fixtures::SPS_IPB, h264_idr), + ("V_MPEGH/ISO/HEVC", hvcc, h265::fixtures::SPS_NVENC, h265_idr), + ] { + for in_band in [false, true] { + let keyframe = match in_band { + true => prefixed(&[sps, idr]), + false => prefixed(&[idr]), + }; + let data = MkvBuilder::new() + .header("matroska") + .segment_start() + .info(1_000_000) + .tracks(vec![ + track_entry_audio_opus(1, 48_000.0, 2), + MatroskaSpec::TrackEntry(Master::Full(vec![ + MatroskaSpec::TrackNumber(2), + MatroskaSpec::TrackUID(2), + MatroskaSpec::TrackType(1), + MatroskaSpec::CodecID(codec_id.into()), + MatroskaSpec::CodecPrivate(private.to_vec()), + ])), + ]) + .cluster(0, || { + vec![ + simple_block(1, 0, true, &[0xfc, 0x00]), + simple_block(2, 0, true, &keyframe), + ] + }) + .segment_end() + .build(); + + for chunk in [1, data.len()] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut published = crate::catalog::hang::Consumer::<()>::new( + broadcast.consume().track(hang::Catalog::DEFAULT_NAME).unwrap(), + ); + let mut mkv = crate::container::mkv::Import::new(broadcast, catalog.reserve()); + let waiter = kio::Waiter::noop(); + let mut first = None; + for chunk in data.chunks(chunk) { + mkv.decode(chunk).unwrap(); + if first.is_none() + && let std::task::Poll::Ready(snapshot) = published.poll_next(&waiter) + { + first = snapshot.unwrap(); + } + } + let first = first.expect("the keyframe publishes the catalog"); + assert_eq!( + first.video.renditions.len(), + 1, + "{codec_id}: the first catalog lists the video" + ); + let snapshot = catalog.snapshot(); + assert_eq!(snapshot.audio.renditions.len(), 1); + let config = snapshot + .video + .renditions + .values() + .next() + .expect("the keyframe publishes the record"); + let detected = match &config.codec { + VideoCodec::H264(codec) => codec.inline, + VideoCodec::H265(codec) => codec.in_band, + other => panic!("{other:?}"), + }; + assert_eq!(detected, in_band, "{codec_id} {}", config.codec); + assert_eq!(config.description.as_deref(), Some(private.as_ref())); + let _finished = mkv.finish().unwrap(); + } + } + } +} diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 3e5696cc1f3..ea869644cfb 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -405,6 +405,8 @@ impl Producer { let interval = self.member.as_ref().and_then(|member| member.interval()); let mut next = rendition.fresh(container, self.inner.info().clone(), interval)?; next.buffer_duration = self.buffer_duration; + // A seek keys the next group, which the successor opens. + next.pending_start = self.pending_start; next.set(config)?; self.end_with(None)?; *self = next; @@ -1906,6 +1908,23 @@ mod tests { assert_eq!(collect_starts(consumer).await, vec![0, 42]); } + /// A seek made before a decoder identity change keys the successor's first group. + #[tokio::test] + async fn a_seek_carries_across_a_new_identity() { + let (_broadcast, _catalog, mut tracks) = renditions(&["v"]); + let track = &mut tracks[0]; + track.seek(42).unwrap(); + let config = hang::catalog::VideoConfig::new("avc3.42c01e".parse::().unwrap()); + track.reconfigure(config).unwrap(); + assert_ne!(track.name(), "v", "a new identity mints a new track"); + let consumer = track.inner.subscribe(replay()); + track.write(frame(10_000, true)).unwrap(); + track.cut(None).unwrap(); + drop(track.inner.clone().finish().unwrap()); + + assert_eq!(collect_starts(consumer).await, vec![42]); + } + /// `seek` is consumed on the next group creation; subsequent groups start at their keyframes. #[tokio::test] async fn seek_clears_pending_after_use() { diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index dc191baafeb..ef250895420 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -3,8 +3,8 @@ //! //! Exporters declare what wire shape they want their frames in (currently: //! length-prefixed NAL units for H.264/H.265) and call [`ExportSource::poll_read`] -//! to pull normalized frames. For Annex-B sources (catalog codec marked -//! `inline: true` / `in_band: true`, empty `description`) the source attaches +//! to pull normalized frames. For Annex-B sources (no catalog `description`, +//! whatever the codec string says) the source attaches //! an [`Avc1`] / [`Hvc1`] transform that strips parameter sets into an avcC or //! hvcC and length-prefixes the rest; frames wait until that record exists. fMP4 //! instead attaches an [`InBand`] transform, keeping the sets in the samples, when @@ -237,7 +237,7 @@ impl ExportSource { /// Subscribe to a video rendition without attaching any codec-shape /// transform. Payloads pass through untouched (Annex-B stays Annex-B, - /// avc1 length-prefixed stays length-prefixed). The Annex-B exporter + /// length-prefixed stays length-prefixed). The Annex-B exporter /// uses this to keep parameter sets in-band. /// /// Rejects an escaping `broadcast` reference like [`Self::for_video`]. diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index d92e496d6a2..4a5f0a4ac4a 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -16,9 +16,9 @@ //! arrival skew between them, and the output keeps the source's pace. //! //! Video flows through `ExportSource`, which normalizes every H.264/H.265 -//! source to length-prefixed NALU plus a resolved avcC/hvcC (parsing in-band -//! avc3/hev1 parameter sets out of the bitstream, or taking the catalog -//! `description` for out-of-band avc1/hvc1). The muxer then does one +//! source to length-prefixed NALU plus a resolved avcC/hvcC (parsing the +//! parameter sets out of an Annex-B bitstream, or taking the catalog +//! `description` for a length-prefixed one). The muxer then does one //! length-prefixed -> Annex-B conversion, re-injecting the parameter sets as //! inline NALs on every keyframe. CMAF tracks are rejected with a clear error. @@ -2355,7 +2355,7 @@ where /// and slices that arrive before their codec config resolves are dropped: a receiver /// joining mid-GOP can't use them, and parking them would stop us polling for the /// keyframe that carries the parameter sets. [`ExportSource`] has already transformed - /// Annex-B avc3/hev1 into length-prefixed form and resolved the avcC/hvcC. + /// Annex-B input into length-prefixed form and resolved the avcC/hvcC. fn fill(&mut self, waiter: &kio::Waiter) -> crate::Result<()> { let mut input = Input { waiting_for_header: self.program.psi.is_none(), diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index f5d4e674852..f1c87b14f5d 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -148,13 +148,13 @@ fn not_video(format: AudioFormat) -> crate::Error { } enum TrackKind { - /// H.264 avc3 (Annex-B, inline SPS/PPS). The split owns byte parsing; the - /// import publishes. + /// H.264 Annex-B with inline SPS/PPS, published as avc3. The split owns byte + /// parsing; the import publishes. Avc3 { split: crate::codec::h264::Split, import: crate::codec::h264::Import, }, - /// H.264 avc1 (length-prefixed NALU, out-of-band avcC). No splitter: each + /// H.264 length-prefixed NALU with an avcC, published as avc1 or avc3. No splitter: each /// access unit is wrapped directly. `length_size` is the NALU length prefix /// width read from the avcC. Avc1 { @@ -165,7 +165,7 @@ enum TrackKind { split: crate::codec::h265::Split, import: crate::codec::h265::Import, }, - /// H.265 hvc1 (length-prefixed NALU, out-of-band hvcC). No splitter: each + /// H.265 length-prefixed NALU with an hvcC, published as hvc1 or hev1. No splitter: each /// access unit is wrapped directly. `length_size` is the NALU length prefix /// width read from the hvcC. Hvc1 { @@ -614,7 +614,7 @@ impl From for Track { } enum TrackStreamKind { - /// H.264 in avc3 wire shape (Annex-B with inline SPS/PPS). The split owns + /// H.264 Annex-B with inline SPS/PPS, published as avc3. The split owns /// byte parsing; the import publishes. Avc3 { split: crate::codec::h264::Split, @@ -1523,24 +1523,30 @@ mod tests { assert_eq!(video.label.as_deref(), Some("Main camera")); } - /// hvc1 publishes the catalog up front from the out-of-band hvcC: dimensions - /// from the parsed SPS and the record itself as the `description`. + /// An hvcC publishes the catalog at the first keyframe, which says whether the + /// parameter sets also ride in band: dimensions from the parsed SPS and the record + /// itself as the `description`. #[tokio::test(start_paused = true)] async fn existing_track_hvc1_uses_existing_name_in_catalog() { let hvcc = crate::codec::h265::fixtures::hvcc(); let (broadcast, catalog) = new_broadcast(); let request = broadcast.reserve_track("camera").unwrap(); - let import = Track::video( + let mut import = Track::video( request, catalog.reserve(), VideoInit::new(VideoFormat::Hvc1, hvcc.clone()), ) .unwrap(); + let idr: &[u8] = &[0x26, 0x01, 0x80, 0xaa]; // IdrWRadl (19) + let mut au = (idr.len() as u32).to_be_bytes().to_vec(); + au.extend_from_slice(idr); + import.decode(&au, Some(Timestamp::from_micros(0).unwrap())).unwrap(); assert_eq!(import.name(), "camera"); let snapshot = catalog.snapshot(); let video = snapshot.video.renditions.get("camera").unwrap(); + assert_eq!(video.codec.to_string().get(..4), Some("hvc1")); assert_eq!(video.coded_width, Some(1280)); assert_eq!(video.coded_height, Some(720)); assert_eq!(video.description.as_deref(), Some(hvcc.as_ref())); diff --git a/rs/moq-rtmp/src/server.rs b/rs/moq-rtmp/src/server.rs index 9dfa646f995..336e75bcbbd 100644 --- a/rs/moq-rtmp/src/server.rs +++ b/rs/moq-rtmp/src/server.rs @@ -2208,6 +2208,11 @@ mod tests { ) .unwrap(); publisher.push(flv::TAG_VIDEO, 0, &vseq).unwrap(); + // The track is minted at the first keyframe, which says where the parameter sets live. + let keyframe = [ + 0x17, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x65, 0x88, 0x84, 0x21, 0x00, + ]; + publisher.push(flv::TAG_VIDEO, 0, &keyframe).unwrap(); let consumer = origin.consume(); let broadcast = consumer diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 5516d65d3ae..a1ef9fe84b3 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -6,19 +6,19 @@ //! AV1 OBU temporal units, and VP8 / VP9 frames untouched. Gates output until the //! first keyframe so the backend never sees a delta frame it can't decode. //! -//! The container decides the NAL framing. CMAF samples are always -//! length-prefixed (ISO/IEC 14496-15), so even an avc3 / hev1 track, whose -//! parameter sets ride in the samples, converts with the length size from its -//! description. A Legacy or LOC track is Annex-B when the codec says its -//! parameter sets are in band (avc3 / hev1), and length-prefixed otherwise -//! (avc1 / hvc1). +//! The catalog's `description` decides the NAL framing, never the codec +//! prefix. A configuration record (avcC / hvcC) means length-prefixed NAL units +//! of its length size, its parameter sets seeding every keyframe; no record +//! means Annex-B carrying its own parameter sets. The prefix only says where +//! the parameter sets live: avc3 / hev1 may repeat them in band beside a +//! record, as the fMP4 and HLS imports publish. CMAF samples are always +//! length-prefixed, so a CMAF track needs its record. //! -//! A track that says avc1 and carries no description is read as Annex-B rather -//! than refused. A browser encoding with WebCodecs' `annexb` output keeps the -//! avc1 label while putting its parameter sets in band, which is what -//! `@moq/publish` does today. Length-prefixed payloads without their parameter -//! sets could not be decoded anyway, so the lenient reading only ever turns an -//! error into a picture. +//! A Legacy or LOC avc1 / hvc1 track with no record contradicts itself and is +//! read as Annex-B rather than refused, as older `@moq/publish` builds labelled +//! their Annex-B output: length-prefixed payloads without their parameter sets +//! could not be decoded anyway, so the lenient reading only ever turns an error +//! into a picture. use std::collections::VecDeque; use std::marker::PhantomData; @@ -105,11 +105,11 @@ impl Config { /// How to turn a container payload into a backend access unit. enum Conversion { - /// The payload is already in the backend's input framing: Annex-B for a - /// Legacy or LOC avc3 / hev1, OBU temporal units for AV1, one coded frame for - /// VP8 / VP9. + /// The payload is already in the backend's input framing: Annex-B for an + /// H.264 / H.265 track with no record, OBU temporal units for AV1, one coded + /// frame for VP8 / VP9. Passthrough, - /// avc1 / hvc1, and every CMAF H.264 / H.265 track: length-prefixed NALs. + /// An H.264 / H.265 track with a record: length-prefixed NALs. /// Replace the length prefixes with start codes and prepend `keyframe_prefix` /// (the description's parameter sets, empty when they are all in band) ahead /// of every keyframe. `pins_size` is whether the description carries an @@ -187,14 +187,19 @@ impl Decoder { /// for another reason, such as a missing driver, the result is /// [`Error::NoDecoder`], a fault rather than a property of the track. pub fn new(catalog: &VideoConfig, config: &Config) -> Result { - // CMAF samples are length-prefixed whatever the sample entry; Legacy and LOC - // carry Annex-B exactly when the parameter sets are in band. + // The description decides the framing, never the codec prefix: with a + // configuration record the NAL units are length-prefixed by its length size, + // without one they are Annex-B. CMAF samples are always length-prefixed, so + // they need the record. let cmaf = matches!(catalog.container, Container::Cmaf { .. }); + let description = catalog + .description + .as_ref() + .filter(|description| !description.is_empty()); let (codec, conversion) = match &catalog.codec { VideoCodec::H264(h264) => { - let conversion = match (cmaf || !h264.inline, catalog.description.as_ref()) { - (false, _) => Conversion::Passthrough, - (true, Some(avcc)) => { + let conversion = match description { + Some(avcc) => { let params = h264::Avcc::parse(avcc) .map_err(|err| Error::UnsupportedCodec(format!("invalid avcC description: {err}")))?; let keyframe_prefix = annexb::build_prefix(params.sps.iter().chain(params.pps.iter())); @@ -204,34 +209,24 @@ impl Decoder { pins_size: !params.sps.is_empty(), } } - (true, None) if cmaf => { - return Err(Error::UnsupportedCodec( - "CMAF H.264 track is missing its avcC description".into(), - )); - } - (true, None) => { - tracing::warn!("avc1 track has no avcC description; reading it as Annex-B"); - Conversion::Passthrough - } + None => annexb_conversion(cmaf, h264.inline, "H.264", "avcC")?, }; (Codec::H264, conversion) } VideoCodec::H265(h265) => { - let conversion = if !cmaf && h265.in_band { - Conversion::Passthrough - } else { - let hvcc = catalog.description.as_ref().ok_or_else(|| { - Error::UnsupportedCodec("length-prefixed H.265 track is missing its hvcC description".into()) - })?; - let params = h265::Hvcc::parse(hvcc) - .map_err(|err| Error::UnsupportedCodec(format!("invalid hvcC description: {err}")))?; - let keyframe_prefix = - annexb::build_prefix(params.vps.iter().chain(params.sps.iter()).chain(params.pps.iter())); - Conversion::LengthPrefixed { - length_size: params.length_size, - keyframe_prefix, - pins_size: !params.sps.is_empty(), + let conversion = match description { + Some(hvcc) => { + let params = h265::Hvcc::parse(hvcc) + .map_err(|err| Error::UnsupportedCodec(format!("invalid hvcC description: {err}")))?; + let keyframe_prefix = + annexb::build_prefix(params.vps.iter().chain(params.sps.iter()).chain(params.pps.iter())); + Conversion::LengthPrefixed { + length_size: params.length_size, + keyframe_prefix, + pins_size: !params.sps.is_empty(), + } } + None => annexb_conversion(cmaf, h265.in_band, "H.265", "hvcC")?, }; (Codec::H265, conversion) } @@ -430,6 +425,27 @@ fn declared(catalog: &VideoConfig) -> Option { Color::from_code_points(Some(codes), range, coded.unwrap_or_default()) } +/// The conversion for an H.264 / H.265 track with no configuration record: +/// its frames are Annex-B and carry their own parameter sets. +/// +/// A CMAF track has no Annex-B framing, so one without a record is refused. A +/// Legacy or LOC track whose codec says its parameter sets are out of band +/// (avc1 / hvc1) contradicts itself, but its frames can only decode if they +/// carry them, so it is read as Annex-B rather than refused. +fn annexb_conversion(cmaf: bool, in_band: bool, codec: &str, record: &str) -> Result { + if cmaf { + return Err(Error::UnsupportedCodec(format!( + "CMAF {codec} track is missing its {record} description" + ))); + } + if !in_band { + tracing::warn!( + "{codec} track says its parameter sets are out of band but has no {record}; reading it as Annex-B" + ); + } + Ok(Conversion::Passthrough) +} + /// What the catalog says about the stream before its first picture, for the /// backends to ask their devices with. fn stream(catalog: &VideoConfig, codec: Codec, conversion: &Conversion) -> backend::Stream { @@ -706,19 +722,6 @@ mod tests { /// framing from the container and hands the backend start-coded NAL units. #[tokio::test] async fn hev1_cmaf_round_trip_reaches_the_backend_as_annexb() { - // One keyframe of a 64x64 HEVC stream from x265. - const VPS: &[u8] = &[ - 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, - 0x03, 0x00, 0x1e, 0x95, 0x98, 0x09, - ]; - const SPS: &[u8] = &[ - 0x42, 0x01, 0x01, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x1e, - 0xa0, 0x20, 0x81, 0x04, 0xd9, 0x65, 0x66, 0x92, 0x4c, 0xaf, 0x01, 0x68, 0x08, 0x00, 0x00, 0x03, 0x00, 0x08, - 0x00, 0x00, 0x03, 0x00, 0xf0, 0x40, - ]; - const PPS: &[u8] = &[0x44, 0x01, 0xc1, 0x72, 0xb4, 0x22, 0x40]; - const IDR: &[u8] = &[0x28, 0x01, 0xaf, 0x08, 0x60, 0xf9, 0x2a, 0x5c, 0xf3, 0x65, 0x62, 0xe8]; - let (origin, driver) = moq_net::origin::Producer::new(Default::default()); tokio::spawn(moq_net::time::run(driver)); let source = origin.route("test", Default::default()).unwrap(); @@ -729,15 +732,7 @@ mod tests { .broadcast() .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let mut config = hang::catalog::VideoConfig::new(hang::catalog::H265 { - in_band: true, - profile_space: 0, - profile_idc: 2, - profile_compatibility_flags: [0x20, 0, 0, 0], - tier_flag: false, - level_idc: 0x1e, - constraint_flags: [0x90, 0, 0, 0, 0, 0], - }); + let mut config = hev1_catalog(true); config.coded_width = Some(64); config.coded_height = Some(64); source_catalog @@ -750,16 +745,11 @@ mod tests { track, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), ); - let mut keyframe = Vec::new(); - for nal in [VPS, SPS, PPS, IDR] { - keyframe.extend_from_slice(&[0, 0, 0, 1]); - keyframe.extend_from_slice(nal); - } producer .write(moq_mux::container::Frame { timestamp: Timestamp::from_micros(0).unwrap(), duration: Some(Timestamp::from_micros(33_333).unwrap()), - payload: keyframe.into(), + payload: hevc_keyframe(), keyframe: true, }) .unwrap(); @@ -815,6 +805,37 @@ mod tests { .conversion .access_unit(&sample.payload, sample.keyframe) .unwrap(); + let types = hevc_nal_types(&access_unit); + // VPS (32), SPS (33), PPS (34), then the IDR_W_RADL slice (20): every NAL + // found by its start code, and the parameter sets ahead of the slice. + assert_eq!(types.first(), Some(&32), "{types:?}"); + assert!(types.contains(&33) && types.contains(&34), "{types:?}"); + assert_eq!(types.last(), Some(&20), "{types:?}"); + } + + /// One keyframe of a 64x64 HEVC stream from x265, as Annex-B. + fn hevc_keyframe() -> bytes::Bytes { + const VPS: &[u8] = &[ + 0x40, 0x01, 0x0c, 0x01, 0xff, 0xff, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, + 0x03, 0x00, 0x1e, 0x95, 0x98, 0x09, + ]; + const SPS: &[u8] = &[ + 0x42, 0x01, 0x01, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x1e, + 0xa0, 0x20, 0x81, 0x04, 0xd9, 0x65, 0x66, 0x92, 0x4c, 0xaf, 0x01, 0x68, 0x08, 0x00, 0x00, 0x03, 0x00, 0x08, + 0x00, 0x00, 0x03, 0x00, 0xf0, 0x40, + ]; + const PPS: &[u8] = &[0x44, 0x01, 0xc1, 0x72, 0xb4, 0x22, 0x40]; + const IDR: &[u8] = &[0x28, 0x01, 0xaf, 0x08, 0x60, 0xf9, 0x2a, 0x5c, 0xf3, 0x65, 0x62, 0xe8]; + let mut keyframe = Vec::new(); + for nal in [VPS, SPS, PPS, IDR] { + keyframe.extend_from_slice(&[0, 0, 0, 1]); + keyframe.extend_from_slice(nal); + } + keyframe.into() + } + + /// The H.265 NAL unit types of an Annex-B access unit, in order. + fn hevc_nal_types(access_unit: &bytes::Bytes) -> Vec { let mut buf = access_unit.clone(); let mut nals = moq_mux::codec::annexb::NalIterator::new(&mut buf); let mut types = Vec::new(); @@ -822,11 +843,67 @@ mod tests { types.push(nal.unwrap()[0] >> 1); } types.extend(nals.flush().unwrap().map(|nal| nal[0] >> 1)); - // VPS (32), SPS (33), PPS (34), then the IDR_W_RADL slice (20): every NAL - // found by its start code, and the parameter sets ahead of the slice. - assert_eq!(types.first(), Some(&32), "{types:?}"); - assert!(types.contains(&33) && types.contains(&34), "{types:?}"); - assert_eq!(types.last(), Some(&20), "{types:?}"); + types + } + + fn hev1_catalog(in_band: bool) -> hang::catalog::VideoConfig { + hang::catalog::VideoConfig::new(hang::catalog::H265 { + in_band, + profile_space: 0, + profile_idc: 2, + profile_compatibility_flags: [0x20, 0, 0, 0], + tier_flag: false, + level_idc: 0x1e, + constraint_flags: [0x90, 0, 0, 0, 0, 0], + }) + } + + /// A Legacy hev1 track with an hvcC, as `moq publish fmp4` and the HLS import + /// publish ffmpeg's default HEVC MP4: the record says the frames are + /// length-prefixed, whatever the codec prefix says about the parameter sets. + #[test] + fn legacy_hev1_with_a_record_is_length_prefixed() { + let mut hvc1 = moq_mux::codec::h265::Hvc1::new(); + let sample = hvc1 + .transform(hevc_keyframe()) + .unwrap() + .expect("a length-prefixed sample"); + let mut catalog = hev1_catalog(true); + catalog.description = Some(hvc1.hvcc().expect("an hvcC").clone()); + + let decoder = super::Decoder::new( + &catalog, + &decode_config(super::Kind::Named(probe::BUFFERED_NAME.into())), + ) + .unwrap(); + let access_unit = decoder.conversion.access_unit(&sample, true).unwrap(); + // VPS (32), SPS (33), PPS (34) from the record, then the IDR_W_RADL slice (20). + assert_eq!(hevc_nal_types(&access_unit), [32, 33, 34, 20]); + } + + /// An hvc1 track with no record carries Annex-B with its parameter sets, the + /// only framing that decodes without one, as H.264 reads a record-less avc1. + #[test] + fn hvc1_without_a_record_is_annexb() { + let decoder = super::Decoder::new( + &hev1_catalog(false), + &decode_config(super::Kind::Named(probe::BUFFERED_NAME.into())), + ) + .unwrap(); + let keyframe = hevc_keyframe(); + let access_unit = decoder.conversion.access_unit(&keyframe, true).unwrap(); + assert_eq!(hevc_nal_types(&access_unit), [32, 33, 34, 20]); + } + + /// A CMAF track has no Annex-B framing, so one with no record is refused. + #[test] + fn cmaf_hev1_without_a_record_is_refused() { + let mut catalog = hev1_catalog(true); + catalog.container = hang::catalog::Container::Cmaf { + init: bytes::Bytes::new(), + }; + let config = decode_config(super::Kind::Named(probe::BUFFERED_NAME.into())); + assert!(super::Decoder::new(&catalog, &config).is_err()); } /// A reordering backend still holds the old sequence's last picture when a diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index ffad1a733f4..8343e9081cb 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -22,11 +22,13 @@ use crate::{Color, Device, Error, Frame, FrameRate, Size}; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[non_exhaustive] pub enum Codec { - /// H.264 / AVC, Annex-B with in-band SPS/PPS (the "avc3" shape). The widest + /// H.264 / AVC, Annex-B with in-band SPS/PPS, published as avc3 with no + /// description. The widest /// support and the default. #[default] H264, - /// H.265 / HEVC, Annex-B with in-band VPS/SPS/PPS (the "hev1" shape). + /// H.265 / HEVC, Annex-B with in-band VPS/SPS/PPS, published as hev1 + /// with no description. H265, } @@ -823,7 +825,7 @@ mod tests { "encoded timestamp outside the fed set: {micros:?}" ); - // The first packet must start with an Annex-B start code so the avc3 + // The first packet must start with an Annex-B start code so the Annex-B // importer can find the inline SPS/PPS. let packets = payloads(&frames); let first = &packets[0]; diff --git a/rs/moq-webrtc/src/codec/mod.rs b/rs/moq-webrtc/src/codec/mod.rs index bfa5bf11908..d64f6196d28 100644 --- a/rs/moq-webrtc/src/codec/mod.rs +++ b/rs/moq-webrtc/src/codec/mod.rs @@ -244,13 +244,13 @@ pub struct Track { /// Codec-specific per-frame transform. enum TrackConvert { - /// Opus / VP8 / VP9 / AV1, plus inline-parameter H.264 (avc3) and H.265 - /// (hev1): the stored bitstream is already in the shape str0m's - /// packetizer wants, so it passes through untouched. + /// Opus / VP8 / VP9 / AV1, plus H.264 / H.265 with no configuration record + /// (Annex-B carrying its parameter sets): the stored bitstream is already in + /// the shape str0m's packetizer wants, so it passes through untouched. Passthrough, - /// Out-of-band-parameter H.264 (avc1) and H.265 (hvc1): length-prefixed - /// NALU rewritten to Annex-B, with the cached parameter sets (SPS+PPS, - /// plus VPS for H.265) prepended to every keyframe. Both codecs share this + /// H.264 / H.265 with a configuration record: length-prefixed NALU + /// rewritten to Annex-B, with the record's parameter sets (SPS+PPS, plus VPS + /// for H.265, empty when they all ride in band) prepended to every keyframe. Both codecs share this /// path; only the config record parsed to build it differs (avcC vs hvcC). /// Mirrors moq-mux's `h264::Export` / `h265::Export`. LengthPrefixed { length_size: usize, keyframe_prefix: Bytes }, @@ -270,8 +270,10 @@ impl Track { } /// Video track from a subscribed `track` consumer. Codec inferred from - /// `config.codec`; for H.264 / H.265 the bitstream shape (inline vs out-of-band - /// parameter sets) is inferred from `config.description` (avc1/hvc1 vs avc3/hev1). + /// `config.codec`; for H.264 / H.265 the framing comes from `config.description` + /// (a record means length-prefixed, none means Annex-B), and the codec prefix + /// says whether the parameter sets may ride in band (avc3/hev1) or only in the + /// record (avc1/hvc1). pub fn video(mut track: moq_net::track::Subscriber, config: &VideoConfig) -> Result { let container: moq_mux::catalog::hang::Container = config.try_into()?; let bill = track.bill(); @@ -281,8 +283,8 @@ impl Track { hang::catalog::VideoCodec::VP8 => TrackConvert::Passthrough, hang::catalog::VideoCodec::VP9(_) => TrackConvert::Passthrough, hang::catalog::VideoCodec::AV1(_) => TrackConvert::Passthrough, - hang::catalog::VideoCodec::H264(_) => h264_convert(config)?, - hang::catalog::VideoCodec::H265(_) => h265_convert(config)?, + hang::catalog::VideoCodec::H264(h264) => h264_convert(config, h264.inline)?, + hang::catalog::VideoCodec::H265(h265) => h265_convert(config, h265.in_band)?, other => return Err(crate::Error::UnsupportedCodec(format!("{other:?}"))), }; @@ -329,18 +331,17 @@ impl Track { /// Build the per-frame transform for an H.264 rendition. /// -/// avc3 (inline SPS/PPS, empty `description`) passes through. avc1 (out-of-band -/// avcC in `description`) parses the avcC and prebuilds the Annex-B SPS+PPS -/// prefix to prepend ahead of every keyframe. -fn h264_convert(config: &VideoConfig) -> Result { +/// No `description` passes through as Annex-B. An avcC parses into the Annex-B +/// SPS+PPS prefix to prepend ahead of every keyframe; `in_band` (avc3) lets it +/// list none, as an avc3 track's record may, since the samples carry them. +fn h264_convert(config: &VideoConfig, in_band: bool) -> Result { let Some(avcc) = config.description.as_ref().filter(|d| !d.is_empty()) else { return Ok(TrackConvert::Passthrough); }; let params = moq_mux::codec::h264::Avcc::parse(avcc).map_err(moq_mux::Error::from)?; - // Without SPS+PPS the keyframe prefix would be empty and every keyframe - // would reach the peer without inline parameter sets, i.e. undecodable. - // Fail loudly instead, matching moq-mux's `h264::Export`. - if params.sps.is_empty() || params.pps.is_empty() { + // An avc1 record without SPS+PPS leaves every keyframe without parameter + // sets, i.e. undecodable. Fail loudly instead, matching moq-mux's `h264::Export`. + if !in_band && (params.sps.is_empty() || params.pps.is_empty()) { return Err(moq_mux::Error::H264(moq_mux::codec::h264::Error::MissingParamSets { name: "WebRTC rendition".to_string(), sps: params.sps.len(), @@ -357,17 +358,16 @@ fn h264_convert(config: &VideoConfig) -> Result { /// Build the per-frame transform for an H.265 rendition. /// -/// The H.265 analogue of [`h264_convert`]: hev1 (inline VPS/SPS/PPS) passes -/// through; hvc1 (out-of-band hvcC) parses the hvcC and prebuilds the Annex-B -/// VPS+SPS+PPS prefix to prepend ahead of every keyframe. -fn h265_convert(config: &VideoConfig) -> Result { +/// The H.265 analogue of [`h264_convert`], with a VPS+SPS+PPS prefix; `in_band` +/// (hev1) lets the hvcC list none. +fn h265_convert(config: &VideoConfig, in_band: bool) -> Result { let Some(hvcc) = config.description.as_ref().filter(|d| !d.is_empty()) else { return Ok(TrackConvert::Passthrough); }; let params = moq_mux::codec::h265::Hvcc::parse(hvcc).map_err(moq_mux::Error::from)?; - // Same reasoning as `h264_convert`: a keyframe with no inline VPS/SPS/PPS - // is undecodable, so reject an hvcC that omits any of them. - if params.vps.is_empty() || params.sps.is_empty() || params.pps.is_empty() { + // Same reasoning as `h264_convert`: an hvc1 keyframe with no VPS/SPS/PPS is + // undecodable, so reject an hvc1 hvcC that omits any of them. + if !in_band && (params.vps.is_empty() || params.sps.is_empty() || params.pps.is_empty()) { return Err(moq_mux::Error::H265(moq_mux::codec::h265::Error::MissingParamSets { name: "WebRTC rendition".to_string(), vps: params.vps.len(), @@ -446,7 +446,7 @@ mod tests { #[test] fn h264_avc3_passthrough() { let cfg = config(h264(true), None); - assert!(matches!(h264_convert(&cfg).unwrap(), TrackConvert::Passthrough)); + assert!(matches!(h264_convert(&cfg, true).unwrap(), TrackConvert::Passthrough)); } #[test] @@ -458,7 +458,7 @@ mod tests { let TrackConvert::LengthPrefixed { length_size, keyframe_prefix, - } = h264_convert(&cfg).unwrap() + } = h264_convert(&cfg, false).unwrap() else { panic!("expected LengthPrefixed"); }; @@ -471,7 +471,7 @@ mod tests { #[test] fn h265_hev1_passthrough() { let cfg = config(h265(true), None); - assert!(matches!(h265_convert(&cfg).unwrap(), TrackConvert::Passthrough)); + assert!(matches!(h265_convert(&cfg, true).unwrap(), TrackConvert::Passthrough)); } #[test] @@ -484,7 +484,7 @@ mod tests { let TrackConvert::LengthPrefixed { length_size, keyframe_prefix, - } = h265_convert(&cfg).unwrap() + } = h265_convert(&cfg, false).unwrap() else { panic!("expected LengthPrefixed"); }; @@ -503,6 +503,33 @@ mod tests { // 6-byte header (numSPS = 0 in the low 5 bits of byte 5) + a zero PPS count. let avcc = Bytes::from(vec![1, 0x42, 0, 0x1f, 0xff, 0xe0, 0x00]); let cfg = config(h264(false), Some(avcc)); - assert!(h264_convert(&cfg).is_err(), "missing SPS/PPS must error"); + assert!(h264_convert(&cfg, false).is_err(), "missing SPS/PPS must error"); + } + + /// An avc3 / hev1 record may list no parameter sets, as the fMP4 export + /// writes one: the samples carry them, so the record only sets the framing. + #[test] + fn in_band_record_without_param_sets_is_length_prefixed() { + let avcc = Bytes::from(vec![1, 0x42, 0, 0x1f, 0xff, 0xe0, 0x00]); + let TrackConvert::LengthPrefixed { + length_size, + keyframe_prefix, + } = h264_convert(&config(h264(true), Some(avcc)), true).unwrap() + else { + panic!("expected LengthPrefixed"); + }; + assert_eq!(length_size, 4); + assert!(keyframe_prefix.is_empty()); + + // configurationVersion 1, a zeroed profile block, 4-byte lengths, no arrays. + let mut hvcc = vec![0u8; 23]; + hvcc[0] = 1; + hvcc[21] = 0xff; + let TrackConvert::LengthPrefixed { keyframe_prefix, .. } = + h265_convert(&config(h265(true), Some(Bytes::from(hvcc))), true).unwrap() + else { + panic!("expected LengthPrefixed"); + }; + assert!(keyframe_prefix.is_empty()); } }