From dd558c1c83c6ff4e7847914b4985734a5417692e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 20:51:39 -0700 Subject: [PATCH 1/7] quest: claim m1/fmp4-import-unwrap Co-Authored-By: Claude Opus 5.5 From 40beec01a6662dccc72dedd2ad95594a0b83b7c4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:10:53 -0700 Subject: [PATCH 2/7] feat(moq-mux)!: unwrap fMP4 import to Legacy by default fmp4::Import splits each fragment into one Legacy frame per sample; with_passthrough(true), ContainerFormat::Cmaf, and --passthrough on import fmp4 / import hls keep publishing CMAF. Co-Authored-By: Claude Opus 5.5 --- cpp/moq/MIGRATE.md | 5 + dart/moq/MIGRATE.md | 5 + doc/bin/cli.md | 5 + doc/bin/hls.md | 3 +- doc/lib/c/index.md | 1 + doc/lib/cpp/index.md | 1 + doc/lib/dart/index.md | 1 + doc/lib/go/index.md | 1 + doc/lib/kt/index.md | 1 + doc/lib/py/index.md | 1 + doc/lib/rs/moq-mux.md | 2 +- doc/lib/swift/index.md | 1 + go/wrapper/MIGRATE.md | 5 + go/wrapper/media/media.go | 4 +- kt/moq/MIGRATE.md | 5 + py/moq-rs/MIGRATE.md | 5 + rs/moq-c/MIGRATE.md | 5 + rs/moq-c/src/api.rs | 5 +- rs/moq-cli/MIGRATE.md | 5 + rs/moq-cli/src/args.rs | 18 +- rs/moq-cli/src/complete.rs | 2 +- rs/moq-cli/src/hls.rs | 12 +- rs/moq-cli/src/main.rs | 2 +- rs/moq-cli/src/publish.rs | 11 +- rs/moq-ffi/MIGRATE.md | 5 + rs/moq-ffi/src/media.rs | 5 +- rs/moq-hls/MIGRATE.md | 5 + rs/moq-hls/src/import.rs | 25 +- rs/moq-mux/MIGRATE.md | 6 + rs/moq-mux/benches/containers.rs | 28 +- rs/moq-mux/src/container/fmp4/import.rs | 858 ++++++++++++------- rs/moq-mux/src/container/fmp4/import_test.rs | 298 +++++-- rs/moq-mux/src/container/producer.rs | 8 + rs/moq-mux/src/import/container.rs | 8 +- rs/moq-mux/src/import/format.rs | 12 +- swift/MIGRATE.md | 5 + 36 files changed, 943 insertions(+), 426 deletions(-) diff --git a/cpp/moq/MIGRATE.md b/cpp/moq/MIGRATE.md index ec0b7243cd4..b2bb53aef50 100644 --- a/cpp/moq/MIGRATE.md +++ b/cpp/moq/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `moq::ContainerFormat::kFmp4` unwraps to Legacy (#PR) + +A container import with `moq::ContainerFormat::kFmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. +Pass the new `moq::ContainerFormat::kCmaf` to keep publishing CMAF. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer::finish` lets go. diff --git a/dart/moq/MIGRATE.md b/dart/moq/MIGRATE.md index 7a1f1be6ca3..97cd44ea9e6 100644 --- a/dart/moq/MIGRATE.md +++ b/dart/moq/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `ContainerFormat.fmp4` unwraps to Legacy (#PR) + +A container import with `ContainerFormat.fmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. +Pass the new `ContainerFormat.cmaf` to keep publishing CMAF. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.finish()` lets go. diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 6c6ce75edec..299f5988017 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -124,6 +124,11 @@ program aboard, for what demultiplexing cannot carry, such as a scrambled service; `export ts --passthrough` writes it back out. FLV accepts the same legacy and enhanced-RTMP codecs as the [RTMP gateway](/bin/rtmp). +fMP4 import splits each fragment into one frame per sample, so players need no +MP4 parser. `import fmp4 --passthrough` publishes each fragment whole as +[`cmaf`](/concept/hang#container) instead, keeping the source's exact sample +timing. + `import rtsp` pulls an IP camera's H.264 or H.265 video and AAC audio without re-encoding. RTP rides the RTSP connection, so run it on the camera's network and the camera needs no inbound port. Credentials go in the URL diff --git a/doc/bin/hls.md b/doc/bin/hls.md index 33ac9a7e15a..3242a556033 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -63,7 +63,8 @@ and `--cors-origin` opens it to browsers. See `moq export hls --help`. Import handles classic HLS; LL-HLS parts are not implemented. It publishes the playlist's own media times, and anchors the catalog clock on the first imported -segment. +segment. Each fMP4 segment is split into one frame per sample; `--passthrough` +publishes each fragment whole as CMAF instead, keeping its exact sample timing. The library, including the recorder cursors for mirroring a broadcast to storage, is [`moq-hls`](https://docs.rs/moq-hls). diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index c498dbe57cc..19d163948b2 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -63,6 +63,7 @@ inside. The header is the reference: each function carries a doc comment. - **Threading.** Call any function from any thread. Raw publish calls block until the codec takes the frame, which paces a publisher. - **Zeroed config is the default.** A zeroed `moq_client_config` means the default for every field, so new fields don't break callers. A field whose default isn't zero has a `has_*` flag: `backoff_timeout_us = 0` means "retry forever" only with `has_backoff_timeout = true`. - **Compressed tracks.** `moq_track_info.compress` on a raw track sends each group DEFLATE-compressed to opt-in `moq-lite-07-wip` peers; other peers, and every reader here, get the original frames. `@moq/net` cannot read a compressed track yet. +- **fMP4 import.** `MOQ_CONTAINER_FORMAT_FMP4` splits each fragment into one frame per sample; `MOQ_CONTAINER_FORMAT_CMAF` republishes it whole. - **Audio needs cuts.** Video groups at its keyframes, but audio forms a group only where you call `moq_publish_media_cut`: after every frame, or at a segment cadence to align with video. - **Ending cleanly.** When the input ends, finish every track, then call `moq_publish_finish()`. It consumes the broadcast and returns a finish handle that keeps it announced and serving the finished tracks, writing the linger, their longest max age, to `linger_us`. Keep the handle for that long on your own clock so a subscriber still catching up can drain them, then `moq_publish_finish_close()` it; closing early is the interrupt. `moq_publish_close()` stays the abrupt end. - **Live encoder timing.** After writing a frame you encoded yourself, call `moq_publish_media_flush` with the same timestamp so the catalog advertises your jitter. Skip it for file and network imports. On a seek or pause, call `moq_publish_media_discontinuity`, then keep timestamps moving forward and resume video on a keyframe. diff --git a/doc/lib/cpp/index.md b/doc/lib/cpp/index.md index bf6baaa3cd1..725a713349c 100644 --- a/doc/lib/cpp/index.md +++ b/doc/lib/cpp/index.md @@ -109,6 +109,7 @@ Share `moq::MediaTimebase::init(broadcast)` through `ContainerInit` when several use one source timeline; omitting it keeps an independent offset. A timebase belongs to its broadcast and does not keep it open. `MediaContainerProducer` takes chunks split anywhere, so a raw byte stream needs no other importer. +`moq::ContainerFormat::kFmp4` splits each fragment into one frame per sample; `kCmaf` republishes it whole. ```cpp // Publish encoded frames, or raw pixels with the codec inside the binding. diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index 92146a65564..6565953d44d 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -30,6 +30,7 @@ Share `media.Timebase(broadcast: ...)` through `ContainerInit` when several impo one source timeline; omitting it keeps an independent offset. A timebase belongs to its broadcast and does not keep it open. `ContainerProducer` takes chunks split anywhere, so a raw byte stream needs no other importer. +`ContainerFormat.fmp4` splits each fragment into one frame per sample; `ContainerFormat.cmaf` republishes it whole. `media.CatalogConsumer.subscribe`, `media.TrackConsumer.subscribe`, and `media.GroupConsumer.fetch` construct the read side from a broadcast; the last two take a catalog `media.Rendition` and follow its broadcast and track themselves. For arrival-order reads, decode each group from diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 545793d4dbd..50534897fde 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -31,6 +31,7 @@ Share `media.NewTimebase(broadcast)` through `ContainerInit` when several import source timeline; omitting it keeps an independent offset. Timebases belong to one broadcast and do not keep it open; release the handle with `Close` on the calling goroutine. `NewContainerProducer` takes chunks split anywhere, so a raw byte stream needs no other importer. +`ContainerFormatFmp4` splits each fragment into one frame per sample; `ContainerFormatCmaf` republishes it whole. For arrival-order reads, decode each group from `moq.TrackConsumer.RecvGroup` with `media.NewGroupConsumer`; frames carry container media time as they land. `media.FetchGroup` fetches and decodes one past group. diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 5632dddf2c4..802721741f9 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -29,6 +29,7 @@ Share `Timebase(broadcast)` through `ContainerInit` when several imports use one timeline; omitting it keeps an independent offset. A timebase belongs to its broadcast and does not keep it open. `ContainerProducer` takes chunks split anywhere, so a raw byte stream needs no other importer. +`ContainerFormat.FMP4` splits each fragment into one frame per sample; `ContainerFormat.CMAF` republishes it whole. Construct readers with `CatalogConsumer.subscribe`, `TrackConsumer.subscribe`, or `GroupConsumer.fetch`; the last two take a catalog `Rendition` and follow its broadcast and track themselves. For arrival-order reads, decode each group from diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 3170d27e35f..abeb6c4a8ce 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -21,6 +21,7 @@ Share `media.Timebase(broadcast)` through `ContainerInit` when several imports u source timeline; omitting it keeps an independent offset. A timebase belongs to its broadcast and does not keep it open. `ContainerProducer` takes chunks split anywhere, so a raw byte stream needs no other importer. +`ContainerFormat.FMP4` splits each fragment into one frame per sample; `ContainerFormat.CMAF` republishes it whole. `MediaFrame.timestamp`, media writes, and `flush` use `datetime.timedelta`. A missing data timestamp stays untimed; the library never substitutes zero or now. JSON consumers yield `moq.json.Timed(value, timestamp)`; a snapshot consumer yields each state in order, diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 51aeb9868ec..9e7c3494d37 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -13,7 +13,7 @@ back. This is what `moq import` / `export` and the gateways are built on. | Format | Import | Export | Notes | | --- | --- | --- | --- | -| fMP4 / CMAF | yes | yes | Audio/video passthrough as `cmaf` or repackaged as `legacy`; `wvtt` subtitles import as timed WebVTT text and `stpp` TTML (IMSC timing profile) as `ttml` documents on the broadcast clock, preserving source language and label when present; other subtitle codecs are skipped. | +| fMP4 / CMAF | yes | yes | Audio/video unwrapped to `legacy` by default, or passed through as `cmaf` with `with_passthrough`; `wvtt` subtitles import as timed WebVTT text and `stpp` TTML (IMSC timing profile) as `ttml` documents on the broadcast clock, preserving source language and label when present; other subtitle codecs are skipped. | | MPEG-TS | yes | yes | H.264/H.265; AAC, MP2, AC-3, E-AC-3, Opus up to 7.1; SCTE-35 sections and every other undecoded PES stream (KLV, subtitles, DVB AC-3) as tracks; service tables round-trip. | | FLV / RTMP | yes | yes | Legacy H.264 + AAC + MP3, plus enhanced-RTMP HEVC, AV1, VP9, Opus, AC-3, E-AC-3, and multitrack. | | Matroska / WebM | yes | yes | Timed SRT and WebVTT subtitles import as WebVTT text renditions, preserving source language and label when present; other subtitle codecs are skipped. | diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index 58eeea26118..9af56d5a1da 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -30,6 +30,7 @@ Share `Media.Timebase(broadcast:)` through `ContainerInit` when several imports source timeline; omitting it keeps an independent offset. A timebase belongs to its broadcast and does not keep it open. `ContainerProducer` takes chunks split anywhere, so a raw byte stream needs no other importer. +`.fmp4` splits each fragment into one frame per sample; `.cmaf` republishes it whole. For arrival-order reads, decode each group from `TrackConsumer.recvGroup()` with `Media.GroupConsumer(group:container:)`; frames carry container media time as they land. diff --git a/go/wrapper/MIGRATE.md b/go/wrapper/MIGRATE.md index b5e60cf4cc7..99e544d1495 100644 --- a/go/wrapper/MIGRATE.md +++ b/go/wrapper/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `media.ContainerFormatFmp4` unwraps to Legacy (#PR) + +A container import with `media.ContainerFormatFmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. +Pass the new `media.ContainerFormatCmaf` to keep publishing CMAF. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.Finish()` lets go. diff --git a/go/wrapper/media/media.go b/go/wrapper/media/media.go index 8af12738b72..053b24a367b 100644 --- a/go/wrapper/media/media.go +++ b/go/wrapper/media/media.go @@ -328,8 +328,10 @@ const ( // ContainerFormat values: a container that publishes its own tracks. const ( - // ContainerFormatFmp4 is fragmented MP4 / CMAF. + // ContainerFormatFmp4 is fragmented MP4 / CMAF, split into one frame per sample. ContainerFormatFmp4 = ffi.MoqContainerFormatFmp4 + // ContainerFormatCmaf is fragmented MP4 / CMAF, republished whole as one CMAF frame per fragment. + ContainerFormatCmaf = ffi.MoqContainerFormatCmaf // ContainerFormatMkv is Matroska / WebM. ContainerFormatMkv = ffi.MoqContainerFormatMkv // ContainerFormatTs is an MPEG-2 transport stream. diff --git a/kt/moq/MIGRATE.md b/kt/moq/MIGRATE.md index 2ff652f9212..aec077ecf1d 100644 --- a/kt/moq/MIGRATE.md +++ b/kt/moq/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `ContainerFormat.FMP4` unwraps to Legacy (#PR) + +A container import with `ContainerFormat.FMP4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. +Pass the new `ContainerFormat.CMAF` to keep publishing CMAF. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast ends or the new `BroadcastProducer.finish()` lets go. diff --git a/py/moq-rs/MIGRATE.md b/py/moq-rs/MIGRATE.md index 4175ecd3ec6..03f5c618802 100644 --- a/py/moq-rs/MIGRATE.md +++ b/py/moq-rs/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `ContainerFormat.FMP4` unwraps to Legacy (#PR) + +A container import with `ContainerFormat.FMP4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. +Pass the new `ContainerFormat.CMAF` to keep publishing CMAF. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.finish()` lets go. diff --git a/rs/moq-c/MIGRATE.md b/rs/moq-c/MIGRATE.md index 45a2f272283..589402ebab1 100644 --- a/rs/moq-c/MIGRATE.md +++ b/rs/moq-c/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `MOQ_CONTAINER_FORMAT_FMP4` unwraps to Legacy (#PR) + +A container import with `MOQ_CONTAINER_FORMAT_FMP4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. +Pass the new `MOQ_CONTAINER_FORMAT_CMAF` to keep publishing CMAF. + ### Finished tracks stay until the broadcast ends (#6157) `moq_publish_track_finish`, `moq_publish_media_finish`, `moq_publish_container_finish`, and the encoder finishes now release the handle while the broadcast keeps the finished track listed, with its catalog rendition and demand, until `moq_publish_close` or the new `moq_publish_finish_close`. diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 2ed45b0811b..d01021e42fb 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -148,7 +148,7 @@ pub enum moq_video_format { #[allow(non_camel_case_types)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum moq_container_format { - /// Fragmented MP4 / CMAF. + /// Fragmented MP4 / CMAF, split into one frame per sample. MOQ_CONTAINER_FORMAT_FMP4 = 0, /// Matroska / WebM. MOQ_CONTAINER_FORMAT_MKV = 1, @@ -156,6 +156,8 @@ pub enum moq_container_format { MOQ_CONTAINER_FORMAT_TS = 2, /// Flash Video, as used by RTMP. MOQ_CONTAINER_FORMAT_FLV = 3, + /// Fragmented MP4 / CMAF, republished whole as one CMAF frame per fragment. + MOQ_CONTAINER_FORMAT_CMAF = 4, } /// Configuration for [moq_publish_audio]. @@ -312,6 +314,7 @@ fn container_format_from_u32(value: u32) -> Result ContainerFormat::Mkv, v if v == moq_container_format::MOQ_CONTAINER_FORMAT_TS as u32 => ContainerFormat::Ts, v if v == moq_container_format::MOQ_CONTAINER_FORMAT_FLV as u32 => ContainerFormat::Flv, + v if v == moq_container_format::MOQ_CONTAINER_FORMAT_CMAF as u32 => ContainerFormat::Cmaf, _ => return Err(Error::InvalidCode), }) } diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index c52537abb08..a5ae57e8842 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,6 +4,11 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased +### `import fmp4` and `import hls` unwrap to Legacy (#PR) + +Both now publish one `legacy` frame per sample instead of one `cmaf` frame per fragment. +Add `--passthrough` to keep publishing CMAF. + ### `export archive` records a prefix (#6160) `moq export archive ` records every broadcast announced at or under `--broadcast`, each instance under `//@`, until the connection ends, instead of one broadcast at ``. diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 81d8c719aa9..18636cac615 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -794,7 +794,7 @@ pub enum ImportSource { /// Raw H.264 Annex-B from stdin. Avc3, /// Fragmented MP4 / CMAF from stdin. - Fmp4, + Fmp4(Fmp4Import), /// MPEG-TS from stdin. Ts(TsImport), /// FLV / RTMP container from stdin. @@ -822,7 +822,9 @@ impl ImportSource { pub fn stdin_format(&self) -> Option { Some(match self { Self::Avc3 => PublishFormat::Avc3, - Self::Fmp4 => PublishFormat::Fmp4, + Self::Fmp4(args) => PublishFormat::Fmp4 { + passthrough: args.passthrough, + }, Self::Ts(args) if args.passthrough => PublishFormat::TsPassthrough { pcr_pid: args.pcr_pid.map(|pid| pid.0), }, @@ -846,11 +848,21 @@ impl ImportSource { } #[cfg(feature = "capture")] Self::Capture(_) => Some("`import capture` encodes, and its encoders do not count yet"), - Self::Avc3 | Self::Fmp4 | Self::Ts(_) | Self::Flv => None, + Self::Avc3 | Self::Fmp4(_) | Self::Ts(_) | Self::Flv => None, } } } +/// The fMP4 stdin container: whether to unwrap its fragments. +#[derive(usage_rs::Args, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct Fmp4Import { + /// Publish each fragment whole as CMAF instead of splitting it into one frame per sample. + /// Keeps the source's exact sample timing, but every player has to parse ISO BMFF. + #[usage(long)] + pub passthrough: bool, +} + /// The MPEG-TS stdin container: which programs of a multiplex to publish. #[derive(usage_rs::Args, Clone)] #[usage(unknown_flags = "error", args_override_self = false)] diff --git a/rs/moq-cli/src/complete.rs b/rs/moq-cli/src/complete.rs index fb14eb3c21a..a8ed9b2c206 100644 --- a/rs/moq-cli/src/complete.rs +++ b/rs/moq-cli/src/complete.rs @@ -359,7 +359,7 @@ mod tests { ); // A cursor sitting on the separator is typing `--`, not inside a stage. - assert!(complete("moq import fmp4 --").await.is_empty()); + assert!(complete("moq import flv --").await.is_empty()); } /// Every overlay is scoped to a command that declares the value it answers for. diff --git a/rs/moq-cli/src/hls.rs b/rs/moq-cli/src/hls.rs index aee6bb604fb..9cc2bee9475 100644 --- a/rs/moq-cli/src/hls.rs +++ b/rs/moq-cli/src/hls.rs @@ -18,6 +18,11 @@ pub struct ImportArgs { /// Playlist URL (http/https) or local file path. #[usage(value_hint = usage_rs::ValueHint::AnyPath, extensions("m3u8", "m3u"))] pub playlist: String, + + /// Publish each fMP4 fragment whole as CMAF instead of splitting it into one frame per + /// sample. Keeps the source's exact sample timing, but every player has to parse ISO BMFF. + #[usage(long)] + pub passthrough: bool, } /// HLS export (serve over HTTP) args. @@ -45,7 +50,7 @@ pub struct ExportArgs { /// Pull a remote HLS/LL-HLS playlist (URL or file path) into the Origin under `target.name`, /// announced under `epoch`. -pub async fn import(target: ImportTarget, playlist: String, epoch: moq_net::Epoch) -> anyhow::Result<()> { +pub async fn import(target: ImportTarget, args: ImportArgs, epoch: moq_net::Epoch) -> anyhow::Result<()> { let ImportTarget { origin, name, @@ -63,8 +68,9 @@ pub async fn import(target: ImportTarget, playlist: String, epoch: moq_net::Epoc // publisher that cannot serve its declared tracks yet. let mut listed = catalog.consume()?; - let playlist = playlist_url(&playlist)?; - let mut importer = moq_hls::import::Import::new(producer.clone(), catalog, moq_hls::import::Config::new(playlist))?; + let mut config = moq_hls::import::Config::new(playlist_url(&args.playlist)?); + config.passthrough = args.passthrough; + let mut importer = moq_hls::import::Import::new(producer.clone(), catalog, config)?; tracing::info!(%name, "importing HLS"); diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 196863afe11..1772230582a 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -750,7 +750,7 @@ fn spawn_import( match import.source { ImportSource::Hls(hls) => { warn_if_missing_format(&name); - tasks.spawn(hls::import(target(name), hls.playlist, epoch)); + tasks.spawn(hls::import(target(name), hls, epoch)); } ImportSource::Rtmp(rtmp) => { if let Some(addr) = rtmp.listen { diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 66651054d18..6da7b2aab71 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -7,8 +7,8 @@ use moq_mux::container::{flv, fmp4, ts}; pub enum PublishFormat { /// Raw AVC (H.264) Annex B elementary stream. Avc3, - /// Fragmented MP4 (CMAF). - Fmp4, + /// Fragmented MP4 (CMAF), republished whole as CMAF when `passthrough`. + Fmp4 { passthrough: bool }, /// MPEG-TS (transport stream), limited to one `program` of a multiplex when given. Ts { program: Option }, /// MPEG-TS published whole, paced on `pcr_pid` when given rather than the PMT's clock. @@ -381,9 +381,9 @@ impl Publish { import: Box::new(import), } } - PublishFormat::Fmp4 => { + PublishFormat::Fmp4 { passthrough } => { let timebase = catalog.timebase(); - let fmp4 = fmp4::Import::new(broadcast.clone(), timebase.reserve()); + let fmp4 = fmp4::Import::new(broadcast.clone(), timebase.reserve()).with_passthrough(*passthrough); PublishDecoder::Fmp4(Box::new(fmp4)) } PublishFormat::Ts { .. } => unreachable!("TS is handled above with the mpegts catalog extension"), @@ -1565,7 +1565,8 @@ pub(crate) mod tests { for (format, input) in [ (PublishFormat::Ts { program: None }, TS), (PublishFormat::TsPassthrough { pcr_pid: None }, TS), - (PublishFormat::Fmp4, FMP4), + (PublishFormat::Fmp4 { passthrough: false }, FMP4), + (PublishFormat::Fmp4 { passthrough: true }, FMP4), ] { let origin = moq_tokio::origin::spawn(); let consumer = origin.consume(); diff --git a/rs/moq-ffi/MIGRATE.md b/rs/moq-ffi/MIGRATE.md index 59e13b01410..8e19e0c033f 100644 --- a/rs/moq-ffi/MIGRATE.md +++ b/rs/moq-ffi/MIGRATE.md @@ -5,6 +5,11 @@ These are the UniFFI names; each language wrapper has its own `MIGRATE.md` in it ## Unreleased +### `MoqContainerFormat::Fmp4` unwraps to Legacy (#PR) + +A container import with `MoqContainerFormat::Fmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. +Pass the new `MoqContainerFormat::Cmaf` to keep publishing CMAF. + ### Finished tracks stay until the broadcast ends (#6157) A finished track, raw or media, now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `MoqBroadcastProducer::finish` lets go, instead of unlisting when its producer finishes or is released. diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index a753e094054..5c147d3a6eb 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -275,8 +275,10 @@ pub enum MoqVideoFormat { /// A container that publishes its own tracks, which may be more than one. #[derive(Clone, Copy, Debug, PartialEq, Eq, uniffi::Enum)] pub enum MoqContainerFormat { - /// Fragmented MP4 / CMAF. + /// Fragmented MP4 / CMAF, split into one frame per sample. Fmp4, + /// Fragmented MP4 / CMAF, republished whole as one CMAF frame per fragment. + Cmaf, /// Matroska / WebM. Mkv, /// MPEG-2 transport stream. @@ -462,6 +464,7 @@ impl From for moq_mux::import::ContainerFormat { fn from(format: MoqContainerFormat) -> Self { match format { MoqContainerFormat::Fmp4 => Self::Fmp4, + MoqContainerFormat::Cmaf => Self::Cmaf, MoqContainerFormat::Mkv => Self::Mkv, MoqContainerFormat::Ts => Self::Ts, MoqContainerFormat::Flv => Self::Flv, diff --git a/rs/moq-hls/MIGRATE.md b/rs/moq-hls/MIGRATE.md index dec2b185fe8..7ffbccb0da1 100644 --- a/rs/moq-hls/MIGRATE.md +++ b/rs/moq-hls/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### HLS import unwraps fMP4 to Legacy (#PR) + +`import::Import` now publishes one `Legacy` frame per sample instead of one `Cmaf` frame per fragment. +Set `import::Config::passthrough` to keep publishing CMAF. + ### A failed timeline fails its listings `export::Broadcaster::manifest` and `export::Rendition::media_playlist` return `Result>`. diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index e7a889bf2a4..9d6cb97e3b1 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -65,12 +65,20 @@ pub struct Config { /// /// This is a [`reqwest::Client`]. pub client: Option, + + /// Republish each fMP4 fragment whole as CMAF instead of splitting it into one Legacy frame per + /// sample. See [`fmp4::Import::with_passthrough`](moq_mux::container::fmp4::Import::with_passthrough). + pub passthrough: bool, } impl Config { /// Create an import configuration for `playlist` using the default HTTP client. pub fn new(playlist: Url) -> Self { - Self { playlist, client: None } + Self { + playlist, + client: None, + passthrough: false, + } } } @@ -314,15 +322,18 @@ struct Sink { /// new `EXT-X-MAP`, shifts by its offset: separate offsets would shift each by its own first /// fragment's arrival. timebase: moq_mux::catalog::Timebase, + /// Republish fragments whole rather than unwrapping their samples. + passthrough: bool, } impl Sink { - fn new(broadcast: moq_net::broadcast::Producer, catalog: CatalogProducer) -> Self { + fn new(broadcast: moq_net::broadcast::Producer, catalog: CatalogProducer, passthrough: bool) -> Self { let timebase = catalog.timebase(); Self { broadcast, catalog, timebase, + passthrough, } } @@ -331,7 +342,9 @@ impl Sink { // A reservation (not a bare clone) so this init's tracks publish together. // The pass holds a separate reservation across every rendition; this one // only covers the tracks this init segment declares. - Fmp4::new(self.broadcast.clone(), self.timebase.reserve()).with_select(select.clone()) + Fmp4::new(self.broadcast.clone(), self.timebase.reserve()) + .with_select(select.clone()) + .with_passthrough(self.passthrough) } } @@ -611,7 +624,7 @@ impl Import { pub fn new(broadcast: moq_net::broadcast::Producer, catalog: CatalogProducer, cfg: Config) -> Result { let base_url = cfg.playlist; Ok(Self { - sink: Sink::new(broadcast, catalog), + sink: Sink::new(broadcast, catalog, cfg.passthrough), fetcher: Fetcher::new(cfg.client)?, base_url, video: Vec::new(), @@ -1130,7 +1143,7 @@ mod tests { fn sink() -> Sink { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = CatalogProducer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); - Sink::new(broadcast, catalog) + Sink::new(broadcast, catalog, false) } fn track_state() -> TrackState { @@ -1578,7 +1591,7 @@ mod tests { let consumer = broadcast.consume(); let catalog = CatalogProducer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); let mut catalogs = catalog.consume().unwrap(); - let sink = Sink::new(broadcast, catalog.clone()); + let sink = Sink::new(broadcast, catalog.clone(), false); let mut first = sink.importer(&select_muxed()); first.decode(&init).unwrap(); diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index 0dcc889ce36..e5bf39c00cf 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,6 +4,12 @@ How to update your code for each breaking change, newest first. ## Unreleased +### fMP4 import unwraps to Legacy (#PR) + +`container::fmp4::Import` now publishes one `Legacy` frame per sample instead of one `Cmaf` frame per fragment. +Call `.with_passthrough(true)` to keep publishing CMAF. +`import::ContainerFormat::Fmp4` follows, and `"cmaf"` no longer parses as its alias but as the new `ContainerFormat::Cmaf`, which keeps CMAF. + ### `Source` resolution takes the reference's epoch (#5699) `Source::bind(rel)`, `resolve(rel)`, and `subscribe_track(rel, name)` take the catalog entry's optional epoch after `rel`. diff --git a/rs/moq-mux/benches/containers.rs b/rs/moq-mux/benches/containers.rs index 48fe61619b8..62fbe1e97bb 100644 --- a/rs/moq-mux/benches/containers.rs +++ b/rs/moq-mux/benches/containers.rs @@ -120,10 +120,12 @@ async fn export(format: &str, source: moq_mux::Source, stream: impl Stream, coll data.freeze() } -fn cmaf_import() -> (moq_mux::catalog::Producer, moq_mux::container::fmp4::Import) { +/// An fMP4 importer, republishing fragments whole when `passthrough` and unwrapping their samples +/// otherwise. +fn cmaf_import(passthrough: bool) -> (moq_mux::catalog::Producer, moq_mux::container::fmp4::Import) { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); - let import = moq_mux::container::fmp4::Import::new(broadcast, catalog.reserve()); + let import = moq_mux::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); (catalog, import) } @@ -184,10 +186,12 @@ fn containers(c: &mut Criterion) { let data = runtime.block_on(export(format, source, stream, true)); drop(fixture); if format == "cmaf" { - let (catalog, mut import) = cmaf_import(); - import.decode(&data).unwrap(); - import.finish().unwrap(); - assert_eq!(catalog.snapshot().video.renditions.len(), *tracks); + for passthrough in [false, true] { + let (catalog, mut import) = cmaf_import(passthrough); + import.decode(&data).unwrap(); + import.finish().unwrap(); + assert_eq!(catalog.snapshot().video.renditions.len(), *tracks); + } use mp4_atom::DecodeMaybe; let mut cursor = std::io::Cursor::new(data.as_ref()); let mut fragments = 0; @@ -212,16 +216,22 @@ fn containers(c: &mut Criterion) { } else { Throughput::Bytes(data.len() as u64) }; - for direction in ["import", "export"] { + // `unwrap` imports the same fragments one frame per sample; `import` passes them through. + let directions: &[&str] = match format { + "cmaf" => &["import", "unwrap", "export"], + _ => &["import", "export"], + }; + for &direction in directions { let mut group = c.benchmark_group(format!("{format}-{direction}-{unit}")); group.throughput(throughput.clone()); group.bench_function( BenchmarkId::new(format!("{tracks}tracks-{frames}frames"), size), |b| { - if direction == "import" { + if direction != "export" { if format == "cmaf" { + let passthrough = direction == "import"; b.iter_batched( - cmaf_import, + || cmaf_import(passthrough), |import| import_cmaf(import, std::hint::black_box(&data)), BatchSize::PerIteration, ); diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 5f477738da6..fb411830068 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -8,10 +8,13 @@ use super::{Error, Kind, sample}; use crate::Result; use crate::catalog::Estimator; -/// Converts fMP4/CMAF files into MoQ broadcast streams using CMAF passthrough. +/// Imports fragmented MP4 (fMP4/CMAF) into a MoQ broadcast. /// -/// This struct processes fragmented MP4 (fMP4) files and transports complete -/// moof+mdat fragments directly as MoQ frames, preserving the CMAF container format. +/// By default each fragment is unwrapped: every sample becomes one +/// [`Legacy`](Container::Legacy) frame carrying the codec payload, timed at its presentation time, +/// so a consumer needs no ISO BMFF parser. [`with_passthrough`](Self::with_passthrough) instead +/// republishes each fragment whole as one [`Cmaf`](Container::Cmaf) frame, for a source whose exact +/// `trun` timing must survive. /// /// ## Supported Codecs /// @@ -41,13 +44,17 @@ pub struct Import { /// importers, until they release theirs too). initial_reservation: Option>, - /// The source's timestamp base: the first fragment anchors it, and every fragment's timestamp - /// shifts by its offset onto the catalog clock. The `tfdt` inside a fragment stays verbatim. + /// The source's timestamp base: the first fragment anchors it, and every sample's timestamp + /// shifts by its offset onto the catalog clock. The `tfdt` inside a passthrough fragment stays + /// verbatim. timebase: crate::catalog::Timebase, // Which track roles to publish. `None` imports every supported track. select: Option, + // Republish each fragment whole as CMAF instead of unwrapping its samples. + passthrough: bool, + // A lookup to tracks in the broadcast tracks: HashMap>, text: HashMap, @@ -80,7 +87,7 @@ pub struct Import { dropped_metadata: u64, } -/// The catalog entry for one imported track, whichever section it lives in. +/// The catalog entry for one passthrough track, whichever section it lives in. /// /// Both arms own the private catalog entry, so publishing what the estimator measured and retiring /// the entry on drop is the same code either way. @@ -104,6 +111,34 @@ struct Fmp4Track { /// The `trex` values this track's samples fall back to. defaults: sample::Defaults, + /// The `mdhd` timescale the samples count in. + timescale: moq_net::Timescale, + + // The decode time of the last fragment, which the next one has to advance past. + last_decode_time: Option, + + // The shortest declared sample duration, used when a final sample leaves its end open. + sample_duration: Option, + + // The segment this track's open group belongs to. A mismatch with `Import::segment` rolls the + // group, which is what keeps audio on the same boundaries as video. + segment: Option, + + output: Output, +} + +/// Where a track's samples are published. +enum Output { + /// One CMAF frame per fragment. + Passthrough(Box>), + /// One Legacy frame per sample. + Video(crate::container::Producer), + /// One Legacy frame per sample. + Audio(crate::container::Producer), +} + +/// A track republishing its source fragments whole, writing groups by hand. +struct Passthrough { /// The catalog entry, which owns the published bitrate and jitter and removes itself on drop. rendition: Rendition, @@ -114,19 +149,9 @@ struct Fmp4Track { // `container::Producer`), so the recorder is fed directly rather than through `with_recorder`. recorder: Option, - // The decode time of the last fragment, which the next one has to advance past. - last_decode_time: Option, - - // The shortest declared sample duration, used when a final sample leaves its end open. - sample_duration: Option, - // Sequence to use for the next group, set by `Import::seek`. pending_sequence: Option, - // The segment this track's open group belongs to. A mismatch with `Import::segment` rolls the - // group, which is what keeps audio on the same boundaries as video. - segment: Option, - // Measures the track's catalog jitter and bitrate from the fragments; the descriptor's own // bitrate, when it declares one, still wins over what this measures. estimator: Estimator, @@ -140,16 +165,63 @@ struct Fmp4Track { sent: std::sync::Arc, } -impl Fmp4Track { +impl Passthrough { fn publish_estimate(&mut self) -> crate::Result<()> { let estimate = self.estimator.estimate(); self.claim.update(&self.track.demand(), estimate.bitrate); self.rendition.estimate(estimate) } + + /// Close the open group, so the next fragment opens one at `pending_sequence` or the next. + fn close(&mut self) -> crate::Result<()> { + self.estimator.cut(None); + self.publish_estimate()?; + if let Some(mut group) = self.group.take() { + group.finish()?; + if let Some(recorder) = self.recorder.as_mut() { + recorder.finish_group(group.sequence); + } + } + Ok(()) + } +} + +/// Forward a call to whichever unwrapped producer a track writes. +macro_rules! media { + ($output:expr, $producer:ident => $body:expr, $passthrough:pat => $other:expr) => { + match $output { + Output::Video($producer) => $body, + Output::Audio($producer) => $body, + Output::Passthrough($passthrough) => $other, + } + }; +} + +/// One sample of a fragment, resolved against the track's defaults. +struct Sample { + /// Decode time, in track ticks. + dts: u64, + /// Presentation time, in track ticks. + pts: u64, + duration: Option, + sync: bool, + /// The sample's bytes within the `mdat` payload. + range: std::ops::Range, +} + +/// The samples one `traf` places in its `mdat`. +struct Run { + samples: Vec, + /// The earliest presentation time, which a passthrough fragment is stamped with. + earliest: Timestamp, + /// The furthest presentation end any sample reaches. + end: Timestamp, + /// The track's bytes within the `mdat` payload. + range: std::ops::Range, } impl Import { - /// Create a new CMAF importer that will write to the given broadcast. + /// Create a new fMP4 importer that will write to the given broadcast. /// /// The broadcast will be populated with tracks as they're discovered in the fMP4 file. pub fn new(broadcast: moq_net::broadcast::Producer, reserved: crate::catalog::Reserved) -> Self { @@ -158,6 +230,7 @@ impl Import { timebase: reserved.timebase(), initial_reservation: Some(reserved), select: None, + passthrough: false, tracks: HashMap::default(), text: HashMap::default(), skipped: HashSet::default(), @@ -186,8 +259,9 @@ impl Import { /// opens its group for the new segment, following the source's segmentation instead of the /// minimum-duration pacing. /// - /// This is where groups are drawn too: every track rolls its group at a segment boundary, so - /// a group is a segment and each fragment inside it is a frame. + /// This is where audio groups are drawn too: audio rolls its group at a segment boundary, so + /// it follows the same segmentation as video. A passthrough video track rolls there as well, + /// while an unwrapped one opens a group at each sync sample. pub fn cut(&mut self) { self.pending_cut = true; } @@ -204,6 +278,16 @@ impl Import { self } + /// Republish each fragment whole as one [`Cmaf`](Container::Cmaf) frame instead of + /// unwrapping it into one [`Legacy`](Container::Legacy) frame per sample. + /// + /// Keeps the source's exact `trun` timing (sample durations, composition offsets) at the cost + /// of every consumer parsing ISO BMFF. Text tracks are reconstructed as timed text either way. + pub fn with_passthrough(mut self, passthrough: bool) -> Self { + self.passthrough = passthrough; + self + } + /// Whether `kind` is selected for import (every role when unset). fn selects(&self, kind: &Kind) -> bool { match (&self.select, kind) { @@ -269,7 +353,7 @@ impl Import { } Any::Mdat(mdat) => { let raw = consumed.slice(start..start + size); - self.extract(mdat, &raw)?; + self.extract(mdat, raw)?; } _ => { // Skip unknown atoms (e.g., sidx, which is optional and used for segment indexing) @@ -292,7 +376,6 @@ impl Import { for trak in &moov.trak { let track_id = trak.tkhd.track_id; let handler = &trak.mdia.hdlr.handler; - let suffix = ".m4s"; if matches!(handler.as_ref(), b"sbtl" | b"subt" | b"text") { if self.init_text(trak, &moov, &reserved)? { @@ -312,54 +395,22 @@ impl Import { continue; } - // Declare the track at the fMP4's native timescale, which the moq-hang draft requires - // of a CMAF track: frame timestamps are emitted at this same scale (see below), so - // each one carries its fragment's earliest presentation time exactly. let timescale = moq_net::Timescale::new(trak.mdia.mdhd.timescale as u64)?; - let track = self.broadcast.create_track( - self.broadcast.unique_name(suffix), - self.catalog.track_info(kind.priority()).with_timescale(timescale), - )?; - - // Enroll every track in the broadcast's timelines: passthrough writes groups by hand - // (no `container::Producer`), so the recorder is fed directly at each fragment. The - // reservation holds the archive entry back until the whole track set is declared. - let recorder = self.catalog.enroll(track.name(), None)?; - - // Whatever the descriptor declared (a bitrate) is authoritative; the rest is filled by - // `estimate` as fragments arrive. - let rendition = match kind { - Kind::Video => { - let config = self.init_video(trak, &moov)?; - let mut rendition = reserved.init(track.name())?; - rendition.set(config)?; - Rendition::Video(rendition) - } - Kind::Audio => { - let config = self.init_audio(trak, &moov)?; - let mut rendition = reserved.init(track.name())?; - rendition.set(config)?; - Rendition::Audio(rendition) - } + let output = match self.passthrough { + true => Output::Passthrough(Box::new(self.init_passthrough(trak, &moov, &reserved, kind, timescale)?)), + false => self.init_unwrap(trak, &moov, &reserved, kind)?, }; - let sent = self.catalog.sent(track.name()); self.tracks.insert( track_id, Fmp4Track { kind, defaults: sample::Defaults::from_moov(&moov, track_id), - rendition, - track, - group: None, - segment: None, - recorder: Some(recorder), + timescale, last_decode_time: None, sample_duration: None, - pending_sequence: None, - estimator: Estimator::new(), - claim: crate::catalog::Claim::new(self.catalog.bandwidth()), - sent, + segment: None, + output, }, ); } @@ -375,6 +426,79 @@ impl Import { Ok(()) } + fn init_passthrough( + &mut self, + trak: &Trak, + moov: &Moov, + reserved: &crate::catalog::Reserved, + kind: Kind, + timescale: moq_net::Timescale, + ) -> Result> { + // Declare the track at the fMP4's native timescale, which the moq-hang draft requires + // of a CMAF track: frame timestamps are emitted at this same scale (see `extract`), so + // each one carries its fragment's earliest presentation time exactly. + let track = self.broadcast.create_track( + self.broadcast.unique_name(".m4s"), + self.catalog.track_info(kind.priority()).with_timescale(timescale), + )?; + + // Enroll every track in the broadcast's timelines: passthrough writes groups by hand + // (no `container::Producer`), so the recorder is fed directly at each fragment. The + // reservation holds the archive entry back until the whole track set is declared. + let recorder = self.catalog.enroll(track.name(), None)?; + + // Whatever the descriptor declared (a bitrate) is authoritative; the rest is filled by + // `estimate` as fragments arrive. + let rendition = match kind { + Kind::Video => { + let config = self.init_video(trak, moov)?; + let mut rendition = reserved.init(track.name())?; + rendition.set(config)?; + Rendition::Video(rendition) + } + Kind::Audio => { + let config = self.init_audio(trak, moov)?; + let mut rendition = reserved.init(track.name())?; + rendition.set(config)?; + Rendition::Audio(rendition) + } + }; + + Ok(Passthrough { + sent: self.catalog.sent(track.name()), + rendition, + track, + group: None, + recorder: Some(recorder), + pending_sequence: None, + estimator: Estimator::new(), + claim: crate::catalog::Claim::new(self.catalog.bandwidth()), + }) + } + + fn init_unwrap( + &mut self, + trak: &Trak, + moov: &Moov, + reserved: &crate::catalog::Reserved, + kind: Kind, + ) -> Result> { + // Legacy frames carry microseconds, which the track's default timescale already is. + let (suffix, wire_kind) = match kind { + Kind::Video => (".mp4-v", crate::container::Kind::Video), + Kind::Audio => (".mp4-a", crate::container::Kind::Audio), + }; + let track = self.broadcast.create_track( + self.broadcast.unique_name(suffix), + self.catalog.track_info(kind.priority()), + )?; + let wire = crate::catalog::hang::Container::new(&Container::Legacy, wire_kind)?; + Ok(match kind { + Kind::Video => Output::Video(reserved.video(track, wire, self.init_video(trak, moov)?)?), + Kind::Audio => Output::Audio(reserved.audio(track, wire, self.init_audio(trak, moov)?)?), + }) + } + fn init_text(&mut self, trak: &Trak, moov: &Moov, reserved: &crate::catalog::Reserved) -> Result { let (format, mut config) = match trak.mdia.minf.stbl.stsd.codecs.as_slice() { [mp4_atom::Codec::Wvtt(codec)] => { @@ -415,7 +539,13 @@ impl Import { Ok(true) } + /// The container a rendition advertises: unwrapped samples are Legacy frames, while a + /// passthrough track carries its own single-track init segment. fn container(&self, trak: &Trak, moov: &Moov) -> Result { + if !self.passthrough { + return Ok(Container::Legacy); + } + // Build a single-track init segment (ftyp+moov) for this track. { let ftyp = mp4_atom::Ftyp { @@ -715,11 +845,12 @@ impl Import { Ok(config) } + /// Whether the fragments in `moof` open a new segment, and so roll a group on every track. /// - /// A segment is the unit a group carries: each fragment inside it becomes a frame, which is - /// what lets audio (whose samples are all independently decodable, so it has no boundary of - /// its own) follow the same segmentation as video instead of opening a group per fragment. + /// A segment is the unit a group carries, which is what lets audio (whose samples are all + /// independently decodable, so it has no boundary of its own) follow the same segmentation as + /// video instead of opening a group per fragment. fn starts_segment(&self, moof: &Moof, has_video: bool) -> bool { if self.pending_cut { return true; @@ -760,12 +891,14 @@ impl Import { .all(|track| track.segment == Some(self.segment)) } - // Extract all frames out of an mdat atom using CMAF passthrough. - fn extract(&mut self, mdat: Mdat, mdat_raw: &[u8]) -> Result<()> { + // Publish the samples an mdat carries for each traf of the preceding moof. + fn extract(&mut self, mdat: Mdat, raw: Bytes) -> Result<()> { let has_video = self.tracks.values().any(|t| t.kind == Kind::Video); let moof = self.moof.take().ok_or(Error::NoMoof)?; let moof_size = self.moof_size; - let header_size = mdat_raw.len() - mdat.data.len(); + let header_size = raw.len() - mdat.data.len(); + // The mdat payload as a view of the input, so each unwrapped sample is sliced, not copied. + let payload = raw.slice(header_size..); // Resolve the segment boundary BEFORE the per-track loop. A moof can carry both an audio // and a video traf, and whichever comes first must see the same answer, or the other rolls @@ -778,7 +911,9 @@ impl Import { self.pending_cut = false; } - let moov = self.moov.as_ref().ok_or(Error::NoMoov)?; + if self.moov.is_none() { + return Err(Error::NoMoov.into()); + } // Loop over all of the traf boxes in the moof. for traf in &moof.traf { @@ -787,7 +922,7 @@ impl Import { track.extract( super::text::Fragment { traf, - data: &mdat.data, + data: &payload, moof_size, header_size, }, @@ -803,285 +938,356 @@ impl Import { None => return Err(Error::UnknownTrack(track_id).into()), }; - // Find the track information in the moov - let trak = moov - .trak - .iter() - .find(|trak| trak.tkhd.track_id == track_id) - .ok_or(Error::UnknownTrack(track_id))?; + let run = track.run(traf, moof_size, header_size, payload.len())?; - let tfdt = traf.tfdt.as_ref().ok_or(Error::MissingTfdt)?; - let timescale = moq_net::Timescale::new(trak.mdia.mdhd.timescale as u64)?; - let mut dts = tfdt.base_media_decode_time; + // Shift onto the catalog clock. Anchor before releasing the reservation, so the first + // snapshot carries the final clock; the moov declared every track, so any track's + // fragment releases it. + let offset = self.timebase.anchor(run.earliest)?; + self.initial_reservation = None; - // Every fragment restates its decode time, so a stale one puts two different samples - // on the same timestamp, which reads downstream as an undeclared hole. - if let Some(previous) = track.last_decode_time - && dts <= previous - { - return Err(Error::NonMonotonicDecodeTime { - track: track_id, - decode_time: Timestamp::new(dts, timescale)?, - previous: Timestamp::new(previous, timescale)?, + let new_segment = track.segment != Some(self.segment); + track.segment = Some(self.segment); + let placement = Placement { + offset, + new_segment, + // A group opening for a declared segment is a boundary on this track's timeline. + declared: new_segment && self.declared, + }; + + match &mut track.output { + Output::Passthrough(passthrough) => { + let fragment = fragment(traf, &moof, &run, &payload, &track.defaults)?; + passthrough.write(fragment, &run, track.kind, placement)?; } - .into()); + output => unwrap(output, &run, &payload, track.timescale, placement)?, } + } - track.last_decode_time = Some(dts); + Ok(()) + } +} - let mut offset = traf.tfhd.base_data_offset.unwrap_or_default() as usize; - let mut track_data_start: Option = None; +/// Where a fragment lands on its track. +#[derive(Clone, Copy)] +struct Placement { + /// Shifts the source's timestamps onto the catalog clock. + offset: crate::catalog::Offset, + /// The fragment is this track's first in the current segment. + new_segment: bool, + /// That segment was declared by the source, so it is a timeline boundary too. + declared: bool, +} - if traf.trun.is_empty() { - return Err(Error::MissingTrun.into()); +impl Fmp4Track { + /// Resolve the samples `traf` places in an mdat whose payload is `len` bytes. + fn run(&mut self, traf: &mp4_atom::Traf, moof_size: usize, header_size: usize, len: usize) -> Result { + let tfdt = traf.tfdt.as_ref().ok_or(Error::MissingTfdt)?; + let timescale = self.timescale; + let mut dts = tfdt.base_media_decode_time; + + // Every fragment restates its decode time, so a stale one puts two different samples + // on the same timestamp, which reads downstream as an undeclared hole. + if let Some(previous) = self.last_decode_time + && dts <= previous + { + return Err(Error::NonMonotonicDecodeTime { + track: traf.tfhd.track_id, + decode_time: Timestamp::new(dts, timescale)?, + previous: Timestamp::new(previous, timescale)?, } + .into()); + } - let mut min_timestamp = None; - let mut max_end = None; - let mut contains_keyframe = false; - let mut keyframes = 0u64; - let total_samples: usize = traf.trun.iter().map(|t| t.entries.len()).sum(); - let mut sample_index = 0usize; + self.last_decode_time = Some(dts); - for trun in &traf.trun { - let tfhd = &traf.tfhd; + if traf.trun.is_empty() { + return Err(Error::MissingTrun.into()); + } - if let Some(data_offset) = trun.data_offset { - let base_offset = tfhd.base_data_offset.unwrap_or_default() as usize; - let data_offset: usize = data_offset.try_into().map_err(|_| Error::InvalidDataOffset)?; + let tfhd = &traf.tfhd; + let mut offset = tfhd.base_data_offset.unwrap_or_default() as usize; + let mut start: Option = None; + let mut earliest = None; + let mut end = None; + let total_samples: usize = traf.trun.iter().map(|t| t.entries.len()).sum(); + let mut samples = Vec::with_capacity(total_samples); + + for trun in &traf.trun { + if let Some(data_offset) = trun.data_offset { + let base_offset = tfhd.base_data_offset.unwrap_or_default() as usize; + let data_offset: usize = data_offset.try_into().map_err(|_| Error::InvalidDataOffset)?; + + let relative_offset = data_offset + .checked_sub(moof_size) + .and_then(|v| v.checked_sub(header_size)) + .ok_or(Error::InvalidDataOffset)?; + + offset = base_offset + .checked_add(relative_offset) + .ok_or(Error::InvalidDataOffset)?; + } - let relative_offset = data_offset - .checked_sub(moof_size) - .and_then(|v| v.checked_sub(header_size)) - .ok_or(Error::InvalidDataOffset)?; + // Capture the actual start offset for this traf before consuming samples + start.get_or_insert(offset); - offset = base_offset - .checked_add(relative_offset) - .ok_or(Error::InvalidDataOffset)?; - } + for entry in &trun.entries { + let sample = self.defaults.resolve(tfhd, entry); + let duration = sample.duration; - // Capture the actual start offset for this traf before consuming samples - if track_data_start.is_none() { - track_data_start = Some(offset); + // A non-final sample with no resolvable duration leaves the rest of the + // fragment's DTS ambiguous, so reject it rather than collapse timestamps. + if duration.is_none() && samples.len() + 1 != total_samples { + return Err(Error::MissingSampleDuration.into()); } - for entry in &trun.entries { - let sample = track.defaults.resolve(tfhd, entry); - let duration = sample.duration; - let size = sample.size as usize; - - // A non-final sample with no resolvable duration leaves the rest of the - // fragment's DTS ambiguous, so reject it rather than collapse timestamps. - if duration.is_none() && sample_index + 1 != total_samples { - return Err(Error::MissingSampleDuration.into()); - } - - // Checked: a negative composition offset must not wrap into a huge u64 PTS. - let pts = dts - .checked_add_signed(entry.cts.unwrap_or_default() as i64) - .ok_or(Error::PtsOverflow)?; - // Preserve the fmp4 track's native timescale so a passthrough re-emit - // doesn't go through a lossy microsecond detour. - let timestamp = moq_net::Timestamp::new(pts, timescale)?; - let end = if let Some(duration) = duration { - let duration = Timestamp::new(duration as u64, timescale)?; - track.sample_duration = Some(track.sample_duration.map_or(duration, |min| min.min(duration))); - timestamp.checked_add(duration)? - } else if let Some(duration) = track.sample_duration { - timestamp.checked_add(duration)? - } else { - timestamp - }; - max_end = Some(max_end.map_or(end, |max: Timestamp| max.max(end))); - - let sample_end = offset.checked_add(size).ok_or(Error::InvalidDataOffset)?; - if sample_end > mdat.data.len() { - return Err(Error::InvalidDataOffset.into()); - } - - let keyframe = match track.kind { - Kind::Video => sample.is_sync(), - Kind::Audio => true, - }; + // Checked: a negative composition offset must not wrap into a huge u64 PTS. + let pts = dts + .checked_add_signed(entry.cts.unwrap_or_default() as i64) + .ok_or(Error::PtsOverflow)?; + let timestamp = Timestamp::new(pts, timescale)?; + let sample_end = if let Some(duration) = duration { + let duration = Timestamp::new(duration as u64, timescale)?; + self.sample_duration = Some(self.sample_duration.map_or(duration, |min| min.min(duration))); + timestamp.checked_add(duration)? + } else if let Some(duration) = self.sample_duration { + timestamp.checked_add(duration)? + } else { + timestamp + }; + end = Some(end.map_or(sample_end, |max: Timestamp| max.max(sample_end))); + earliest = Some(earliest.map_or(timestamp, |min: Timestamp| min.min(timestamp))); - contains_keyframe |= keyframe; - keyframes += u64::from(keyframe && track.kind == Kind::Video); + let next = offset.checked_add(sample.size as usize).ok_or(Error::InvalidDataOffset)?; + if next > len { + return Err(Error::InvalidDataOffset.into()); + } - if min_timestamp.is_none_or(|min| timestamp <= min) { - min_timestamp = Some(timestamp); - } + samples.push(Sample { + dts, + pts, + duration, + sync: sample.is_sync(), + range: offset..next, + }); - if let Some(duration) = duration { - dts = dts.checked_add(duration as u64).ok_or(Error::PtsOverflow)?; - } - offset = sample_end; - sample_index += 1; + if let Some(duration) = duration { + dts = dts.checked_add(duration as u64).ok_or(Error::PtsOverflow)?; } + offset = next; } + } - // Build a per-track moof containing only this traf, and a per-track mdat - // with only the samples belonging to this track. - let single_traf_moof = Moof { - mfhd: moof.mfhd.clone(), - traf: vec![traf.clone()], - }; + // The track's bytes must be in bounds of the mdat. If not, the parsed sample sizes and + // offsets disagree with the actual data and no sample can be addressed safely. + let start = start.unwrap_or(0); + if !(start <= offset && offset <= len) { + return Err(Error::SampleRangeOutOfBounds { start, end: offset, len }.into()); + } - // Compute the data range within the original mdat for this traf's samples. - let track_data_start = track_data_start.unwrap_or(0); - let track_data_end = offset; // offset was advanced past all samples above - - // The per-track sample range must be in bounds of the original mdat. - // If not, the parsed sample sizes/offsets disagree with the actual data - // and we cannot safely emit a passthrough fragment with rewritten offsets. - if !(track_data_start <= track_data_end && track_data_end <= mdat.data.len()) { - return Err(Error::SampleRangeOutOfBounds { - start: track_data_start, - end: track_data_end, - len: mdat.data.len(), - } - .into()); - } - let track_mdat_data = &mdat.data[track_data_start..track_data_end]; - - let mut adjusted_moof = single_traf_moof; - - // Apply structural (flag-presence) changes BEFORE measuring the encoded - // size, otherwise the rewritten data_offset values are computed from a - // stale size and the resulting fragment misaddresses sample data. - // In particular: clearing tfhd.base_data_offset removes 8 bytes per traf, - // and ensuring trun.data_offset is Some(...) reserves 4 bytes per trun. - for traf_mut in &mut adjusted_moof.traf { - traf_mut.tfhd.base_data_offset = None; - // A zero default/sample duration is "unknown", not "instantaneous": drop it so - // the re-emitted fragment carries no bogus zero that a decoder would honor. - if traf_mut.tfhd.default_sample_duration == Some(0) { - traf_mut.tfhd.default_sample_duration = None; - } - for trun_mut in &mut traf_mut.trun { - // Reserve the data_offset field; the real value is filled in below. - trun_mut.data_offset = Some(0); - for entry in &mut trun_mut.entries { - if entry.duration == Some(0) { - entry.duration = None; - } - } - } - } + Ok(Run { + samples, + earliest: earliest.ok_or(Error::MissingTrun)?, + end: end.ok_or(Error::MissingTrun)?, + range: start..offset, + }) + } +} - let mut moof_buf = Vec::new(); - adjusted_moof.encode(&mut moof_buf)?; - let new_moof_size = moof_buf.len(); - - // Re-encode moof with corrected per-trun data_offset for the per-track fragment. - // Each trun's data_offset points to the start of that run's data within the new mdat. - let mdat_header_size_new = 8u64; // 4 bytes size + 4 bytes 'mdat' - let mut cumulative_offset = 0u64; - for traf_mut in &mut adjusted_moof.traf { - for trun_mut in &mut traf_mut.trun { - trun_mut.data_offset = - Some((new_moof_size as u64 + mdat_header_size_new + cumulative_offset) as i32); - - // Advance past this trun's sample data - let trun_data_size: u64 = trun_mut - .entries - .iter() - .map(|entry| u64::from(track.defaults.resolve(&traf_mut.tfhd, entry).size)) - .sum(); - cumulative_offset += trun_data_size; +/// Re-encode `traf`'s samples as a standalone moof+mdat fragment addressing only its own bytes. +fn fragment( + traf: &mp4_atom::Traf, + moof: &Moof, + run: &Run, + payload: &Bytes, + defaults: &sample::Defaults, +) -> Result { + // Build a per-track moof containing only this traf, and a per-track mdat + // with only the samples belonging to this track. + let mut adjusted_moof = Moof { + mfhd: moof.mfhd.clone(), + traf: vec![traf.clone()], + }; + + // Apply structural (flag-presence) changes BEFORE measuring the encoded + // size, otherwise the rewritten data_offset values are computed from a + // stale size and the resulting fragment misaddresses sample data. + // In particular: clearing tfhd.base_data_offset removes 8 bytes per traf, + // and ensuring trun.data_offset is Some(...) reserves 4 bytes per trun. + for traf_mut in &mut adjusted_moof.traf { + traf_mut.tfhd.base_data_offset = None; + // A zero default/sample duration is "unknown", not "instantaneous": drop it so + // the re-emitted fragment carries no bogus zero that a decoder would honor. + if traf_mut.tfhd.default_sample_duration == Some(0) { + traf_mut.tfhd.default_sample_duration = None; + } + for trun_mut in &mut traf_mut.trun { + // Reserve the data_offset field; the real value is filled in below. + trun_mut.data_offset = Some(0); + for entry in &mut trun_mut.entries { + if entry.duration == Some(0) { + entry.duration = None; } } + } + } - moof_buf.clear(); - adjusted_moof.encode(&mut moof_buf)?; + let mut moof_buf = Vec::new(); + adjusted_moof.encode(&mut moof_buf)?; + let new_moof_size = moof_buf.len(); + + // Re-encode moof with corrected per-trun data_offset for the per-track fragment. + // Each trun's data_offset points to the start of that run's data within the new mdat. + let mdat_header_size_new = 8u64; // 4 bytes size + 4 bytes 'mdat' + let mut cumulative_offset = 0u64; + for traf_mut in &mut adjusted_moof.traf { + for trun_mut in &mut traf_mut.trun { + trun_mut.data_offset = Some((new_moof_size as u64 + mdat_header_size_new + cumulative_offset) as i32); + + // Advance past this trun's sample data + let trun_data_size: u64 = trun_mut + .entries + .iter() + .map(|entry| u64::from(defaults.resolve(&traf_mut.tfhd, entry).size)) + .sum(); + cumulative_offset += trun_data_size; + } + } - let per_track_mdat = Mdat { - data: track_mdat_data.to_vec(), - }; - per_track_mdat.encode(&mut moof_buf)?; - - let fragment_bytes = Bytes::from(moof_buf); - - // Carry the fragment's earliest presentation time as the frame timestamp, in the - // track's native timescale and shifted onto the catalog clock. Consumers present the - // fragment at it; `tfdt` only places the samples relative to each other. Anchor before - // releasing the reservation, so the first snapshot carries the final clock; the moov - // declared every track, so any track's fragment releases it. - let earliest = min_timestamp.ok_or(Error::MissingTrun)?; - let offset = self.timebase.anchor(earliest)?; - let timestamp = offset.apply(earliest)?; - let max_end = max_end.map(|end| offset.apply(end)).transpose()?; - self.initial_reservation = None; + moof_buf.clear(); + adjusted_moof.encode(&mut moof_buf)?; - // Write the per-track fragment as a single MoQ frame (passthrough). The group rolls - // once per segment, so a group is a segment and the fragments inside it are frames. - // The keyframe bit no longer decides that: audio flags every sample a sync sample, so - // keying off it opened a group (a QUIC stream) per audio fragment. - let new_segment = track.segment != Some(self.segment); - let start_group = track.group.is_none() || new_segment; - let mut g = if start_group { - if let Some(mut prev) = track.group.take() { - prev.finish()?; - if let Some(recorder) = track.recorder.as_mut() { - recorder.finish_group(prev.sequence); - } - } - track.segment = Some(self.segment); - match track.pending_sequence.take() { - Some(sequence) => track.track.create_group(moq_net::group::Info { sequence })?, - None => track.track.append_group()?, - } - } else { - track.group.take().ok_or(Error::NoKeyframe)? - }; + Mdat { + data: payload[run.range.clone()].to_vec(), + } + .encode(&mut moof_buf)?; - let position = hang::timeline::Position::new(g.sequence, g.frame_count() as u64); + Ok(Bytes::from(moof_buf)) +} - if start_group { - // A group opening for a declared segment is a boundary on this track's timeline. A - // group reopened mid-segment (after a `seek`) is not. - if new_segment - && self.declared - && let Some(recorder) = track.recorder.as_mut() - { - recorder.cut(timestamp); +impl Passthrough { + /// Write one per-track `fragment` as a single frame. + fn write(&mut self, fragment: Bytes, run: &Run, kind: Kind, placement: Placement) -> Result<()> { + // Carry the fragment's earliest presentation time as the frame timestamp, in the + // track's native timescale and shifted onto the catalog clock. Consumers present the + // fragment at it; `tfdt` only places the samples relative to each other. + let timestamp = placement.offset.apply(run.earliest)?; + let end = placement.offset.apply(run.end)?; + + // The group rolls once per segment, so a group is a segment and the fragments inside it + // are frames. The keyframe bit does not decide that: audio flags every sample a sync + // sample, so keying off it opened a group (a QUIC stream) per audio fragment. + let start_group = self.group.is_none() || placement.new_segment; + let mut group = if start_group { + if let Some(mut prev) = self.group.take() { + prev.finish()?; + if let Some(recorder) = self.recorder.as_mut() { + recorder.finish_group(prev.sequence); } - - // Close the previous group for the bitrate estimator (used only when the - // descriptor didn't declare a bitrate). - track.estimator.cut(Some(timestamp)); } - let fragment_len = fragment_bytes.len(); - let mut frame = g.create_frame(moq_net::frame::Info { - size: fragment_bytes.len() as u64, - timestamp: Some(timestamp), - })?; - frame.write(fragment_bytes)?; - frame.finish()?; - - // Only once published, so a rejected fragment is never indexed. Audio is always - // independently decodable; video says whether this fragment really begins on an IDR, - // which is what an HLS export reads to bootstrap an init segment. - if let Some(recorder) = track.recorder.as_mut() { - recorder.frame(position, timestamp, contains_keyframe); + match self.pending_sequence.take() { + Some(sequence) => self.track.create_group(moq_net::group::Info { sequence })?, + None => self.track.append_group()?, } + } else { + self.group.take().ok_or(Error::NoKeyframe)? + }; - track.group = Some(g); + let position = hang::timeline::Position::new(group.sequence, group.frame_count() as u64); - track.estimator.write(timestamp, fragment_len); - track.sent.add(total_samples as u64, track_mdat_data.len(), keyframes); - let end = max_end.ok_or(Error::MissingTrun)?; - if let Some(recorder) = track.recorder.as_mut() { - recorder.end(end); + if start_group { + // A group reopened mid-segment (after a `seek`) is no timeline boundary. + if placement.declared + && let Some(recorder) = self.recorder.as_mut() + { + recorder.cut(timestamp); } - let span = end.checked_sub(timestamp)?; - track.estimator.burst(span.into()); - track.publish_estimate()?; + + // Close the previous group for the bitrate estimator (used only when the + // descriptor didn't declare a bitrate). + self.estimator.cut(Some(timestamp)); + } + let fragment_len = fragment.len(); + let mut frame = group.create_frame(moq_net::frame::Info { + size: fragment_len as u64, + timestamp: Some(timestamp), + })?; + frame.write(fragment)?; + frame.finish()?; + + // Only once published, so a rejected fragment is never indexed. Audio is always + // independently decodable; video says whether this fragment really begins on an IDR, + // which is what an HLS export reads to bootstrap an init segment. + let keyframes = match kind { + Kind::Video => run.samples.iter().filter(|sample| sample.sync).count() as u64, + Kind::Audio => 0, + }; + if let Some(recorder) = self.recorder.as_mut() { + recorder.frame(position, timestamp, kind == Kind::Audio || keyframes > 0); } - Ok(()) + self.group = Some(group); + + self.estimator.write(timestamp, fragment_len); + self.sent.add(run.samples.len() as u64, run.range.len(), keyframes); + if let Some(recorder) = self.recorder.as_mut() { + recorder.end(end); + } + self.estimator.burst(end.checked_sub(timestamp)?.into()); + self.publish_estimate() } } +/// Write each sample of `run` as its own frame on an unwrapped track. +fn unwrap( + output: &mut Output, + run: &Run, + payload: &Bytes, + timescale: moq_net::Timescale, + placement: Placement, +) -> Result<()> { + let earliest = placement.offset.apply(run.earliest)?; + let end = placement.offset.apply(run.end)?; + let video = matches!(output, Output::Video(_)); + + media!(output, producer => { + if placement.declared { + producer.boundary(earliest); + } + + for (index, sample) in run.samples.iter().enumerate() { + let keyframe = match video { + // Video groups open on sync samples, the `depends_on` rule the rest of moq-mux applies. + true => sample.sync, + // Packagers flag every audio sample a sync sample, which would open a group per + // packet. Only a segment's first sample opens one instead, so audio keeps the + // source's grouping (and whatever a `seek` left open). + false => (index == 0 && placement.new_segment) || producer.needs_keyframe(), + }; + if sample.pts > sample.dts { + producer.reorder(Timestamp::new(sample.pts - sample.dts, timescale)?); + } + let frame = crate::container::Frame { + timestamp: placement.offset.apply(Timestamp::new(sample.pts, timescale)?)?, + payload: payload.slice(sample.range.clone()), + keyframe, + duration: sample + .duration + .map(|duration| Timestamp::new(duration as u64, timescale)) + .transpose()?, + }; + match producer.write(frame) { + // A source joined mid-GOP: skip ahead to the first sync sample. + Ok(()) | Err(crate::Error::MissingKeyframe(_)) => {} + Err(err) => return Err(err), + } + } + + // The fragment arrived at once, so a consumer has to buffer its whole span. + producer.burst(end.checked_sub(earliest)?.into()) + }, _ => unreachable!("passthrough tracks write whole fragments")) +} + impl Import { /// Finish all tracks, flushing current groups. pub fn finish(&mut self) -> Result<()> { @@ -1091,17 +1297,12 @@ impl Import { // No frame follows to anchor the clock, so publish the declared track set now. self.initial_reservation = None; for track in self.tracks.values_mut() { - track.estimator.cut(None); - track.publish_estimate()?; - if let Some(mut g) = track.group.take() { - g.finish()?; - if let Some(recorder) = track.recorder.as_mut() { - recorder.finish_group(g.sequence); - } - } - track.track.finish()?; - // Dropping the recorder flushes the open record, so the track is advertised. - track.recorder = None; + media!(&mut track.output, producer => producer.finish()?, passthrough => { + passthrough.close()?; + passthrough.track.finish()?; + // Dropping the recorder flushes the open record, so the track is advertised. + passthrough.recorder = None; + }); } Ok(()) } @@ -1113,11 +1314,13 @@ impl Import { track.producer.abort(err.clone()); } // Dropping each track's rendition retires its catalog entry. - for mut track in std::mem::take(&mut self.tracks).into_values() { - if let Some(g) = track.group.take() { - let _ = g.abort(err.clone()); - } - let _ = track.track.abort(err.clone()); + for track in std::mem::take(&mut self.tracks).into_values() { + media!(track.output, producer => producer.abort(err.clone()), mut passthrough => { + if let Some(group) = passthrough.group.take() { + let _ = group.abort(err.clone()); + } + let _ = passthrough.track.abort(err.clone()); + }); } } @@ -1134,15 +1337,10 @@ impl Import { track.producer.seek(sequence.max(next))?; } for track in self.tracks.values_mut() { - track.estimator.cut(None); - track.publish_estimate()?; - if let Some(mut g) = track.group.take() { - g.finish()?; - if let Some(recorder) = track.recorder.as_mut() { - recorder.finish_group(g.sequence); - } - } - track.pending_sequence = Some(sequence); + media!(&mut track.output, producer => producer.seek(sequence)?, passthrough => { + passthrough.close()?; + passthrough.pending_sequence = Some(sequence); + }); } Ok(()) } diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index a4500da0bee..ece97ae3356 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -27,10 +27,19 @@ fn drain_group_sequences(consumer: &mut moq_net::track::Subscriber) -> Vec } fn run_fmp4(data: &[u8]) -> crate::catalog::hang::Catalog { + run_fmp4_with(data, false) +} + +/// Like [`run_fmp4`], republishing each fragment as CMAF. +fn run_fmp4_passthrough(data: &[u8]) -> crate::catalog::hang::Catalog { + run_fmp4_with(data, true) +} + +fn run_fmp4_with(data: &[u8], passthrough: bool) -> crate::catalog::hang::Catalog { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); let buf = bytes::BytesMut::from(data); // Ignore errors from incomplete/malformed trailing fragments in test files. @@ -376,36 +385,13 @@ fn test_bbb_catalog() { assert_eq!(video.codec.to_string(), "avc1.64001f"); assert_eq!(video.coded_width, Some(1280)); assert_eq!(video.coded_height, Some(720)); - assert!(matches!(video.container, Container::Cmaf { .. })); + assert!(matches!(video.container, Container::Legacy)); let audio = catalog.audio.renditions.values().next().unwrap(); assert_eq!(audio.codec.to_string(), "mp4a.40.2"); assert_eq!(audio.sample_rate, 44100); assert_eq!(audio.channel_count, 2); - assert!(matches!(audio.container, Container::Cmaf { .. })); -} - -/// fMP4 is passthrough: it republishes the fragments its source already produced, so it exposes no -/// container selection (unlike the demuxers, which decode to elementary payloads) and every -/// rendition it publishes carries the CMAF init it parsed. -#[test] -fn every_rendition_is_cmaf() { - let data = include_bytes!("test_data/bbb.mp4"); - let mut cursor = std::io::Cursor::new(data.as_slice()); - mp4_atom::Ftyp::decode(&mut cursor).unwrap(); - mp4_atom::Moov::decode(&mut cursor).unwrap(); - let init = &data[..cursor.position() as usize]; - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut import = super::Import::new(broadcast, catalog.reserve()); - import.decode(init).unwrap(); - import.finish().unwrap(); - - let snapshot = catalog.snapshot(); - let video = snapshot.video.renditions.values().next().expect("a video rendition"); - assert!(matches!(video.container, Container::Cmaf { .. })); - let audio = snapshot.audio.renditions.values().next().expect("an audio rendition"); - assert!(matches!(audio.container, Container::Cmaf { .. })); + assert!(matches!(audio.container, Container::Legacy)); } #[test] @@ -517,10 +503,129 @@ fn select_nothing_publishes_nothing() { assert!(catalog.audio.renditions.is_empty()); } +/// Every sample bbb's fragments carry, per role, as a consumer reads it back: its presentation +/// time in microseconds, whether it opens a group, and its bytes. +async fn bbb_samples(passthrough: bool) -> [Vec<(u128, bool, bytes::Bytes)>; 2] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); + // The file ends on a malformed fragment, after every whole one has published. + let _ = fmp4.decode(include_bytes!("test_data/bbb.mp4")); + fmp4.finish().unwrap(); + + let snapshot = catalog.snapshot(); + let video = snapshot.video.renditions.iter().next().unwrap(); + let audio = snapshot.audio.renditions.iter().next().unwrap(); + let containers = [ + (video.0, crate::catalog::hang::Container::try_from(video.1).unwrap()), + (audio.0, crate::catalog::hang::Container::try_from(audio.1).unwrap()), + ]; + + let mut out = [Vec::new(), Vec::new()]; + for ((name, container), samples) in containers.into_iter().zip(&mut out) { + let track = consumer.track(name).unwrap().subscribe(replay()).await.unwrap(); + let mut reader = crate::container::Consumer::new(track, container); + while let Some(frame) = reader.read_frame().await.unwrap() { + samples.push((frame.timestamp.as_micros(), frame.keyframe, frame.payload)); + } + } + out +} + +/// Unwrapping publishes exactly the samples a consumer would have decoded out of the passthrough +/// fragments: the same bytes at the same presentation times, opening groups at the same samples. +#[tokio::test] +async fn unwrapped_samples_match_the_passthrough_fragments() { + let [video, audio] = bbb_samples(false).await; + let [cmaf_video, cmaf_audio] = bbb_samples(true).await; + assert!(!video.is_empty() && !audio.is_empty(), "bbb carries both roles"); + assert_eq!(video, cmaf_video, "video"); + assert_eq!(audio, cmaf_audio, "audio"); +} + +/// An unwrapped sample presents at `tfdt + the durations before it + its composition offset`, so +/// reordered (B-frame) samples keep decode order on the wire with their own presentation times, +/// and only the sync sample opens a group. +#[tokio::test] +async fn unwrapped_samples_present_at_their_composition_time() { + let (init, (video_id, video_scale), _) = bbb_init(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + fmp4.decode(&init).unwrap(); + + let snapshot = catalog.snapshot(); + let (name, config) = snapshot.video.renditions.iter().next().unwrap(); + assert_eq!(config.container, Container::Legacy); + let mut track = consumer.track(name).unwrap().subscribe(replay()).await.unwrap(); + + // I P B in decode order, one frame (1/24 s) apart, the P presenting after the B. + let tick = video_scale.as_u64() / 24; + let non_sync = 0x0001_0000; + let entries = [(tick, 0, 0xA), (2 * tick, non_sync, 0xB), (0, non_sync, 0xC)]; + let moof = mp4_atom::Moof { + mfhd: mp4_atom::Mfhd { sequence_number: 1 }, + traf: vec![mp4_atom::Traf { + tfhd: mp4_atom::Tfhd { + track_id: video_id, + ..Default::default() + }, + tfdt: Some(mp4_atom::Tfdt { + base_media_decode_time: video_scale.as_u64(), + }), + trun: vec![mp4_atom::Trun { + data_offset: None, + entries: entries + .iter() + .map(|&(cts, flags, _)| mp4_atom::TrunEntry { + duration: Some(tick as u32), + size: Some(2), + flags: Some(flags), + cts: Some(cts as i32), + }) + .collect(), + }], + ..Default::default() + }], + }; + let mut fragment = Vec::new(); + moof.encode(&mut fragment).unwrap(); + mp4_atom::Mdat { + data: entries.iter().flat_map(|&(_, _, byte)| [byte; 2]).collect(), + } + .encode(&mut fragment) + .unwrap(); + fmp4.decode(&fragment).unwrap(); + fmp4.finish().unwrap(); + + let mut group = track.recv_group().await.unwrap().unwrap(); + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await.unwrap() { + let frame = hang::container::Frame::decode(frame.payload).unwrap(); + frames.push((frame.timestamp.as_micros(), frame.payload.to_vec())); + } + // 25, 27, and 26 frames at 24 per second, in whole microseconds. + assert_eq!( + frames[..3], + [ + (1_041_666, vec![0xA; 2]), + (1_125_000, vec![0xB; 2]), + (1_083_333, vec![0xC; 2]), + ], + "decode order, each at its composition time" + ); + assert!( + track.recv_group().await.unwrap().is_none(), + "the non-sync samples stay in the sync sample's group" + ); +} + #[test] fn test_bbb_init_roundtrip() { let data = include_bytes!("test_data/bbb.mp4"); - let catalog = run_fmp4(data); + let catalog = run_fmp4_passthrough(data); let video = catalog.video.renditions.values().next().unwrap(); let Container::Cmaf { init, .. } = &video.container else { @@ -560,7 +665,7 @@ fn test_bbb_init_roundtrip() { #[test] fn test_av1_catalog() { let data = include_bytes!("test_data/av1.mp4"); - let catalog = run_fmp4(data); + let catalog = run_fmp4_passthrough(data); assert_eq!(catalog.video.renditions.len(), 1); assert_eq!(catalog.audio.renditions.len(), 0); @@ -573,9 +678,7 @@ fn test_av1_catalog() { (color.primaries, color.range), (None, Some(hang::catalog::Range::Limited)) ); - assert!(matches!(video.container, Container::Cmaf { .. })); - - let Container::Cmaf { init, .. } = &video.container else { + let Container::Cmaf { init, .. } = &video.container else { panic!("expected Cmaf container"); }; let (ftyp, moov) = decode_init(init); @@ -634,22 +737,20 @@ fn avc3_imports_in_band() { assert_eq!(video.codec.to_string(), "avc3.42c01f"); assert_eq!(video.description.as_deref(), Some(AVCC)); assert_eq!((video.coded_width, video.coded_height), (Some(320), Some(240))); - assert!(matches!(video.container, Container::Cmaf { .. })); + assert!(matches!(video.container, Container::Legacy)); } #[test] fn test_vp9_catalog() { let data = include_bytes!("test_data/vp9.mp4"); - let catalog = run_fmp4(data); + let catalog = run_fmp4_passthrough(data); assert_eq!(catalog.video.renditions.len(), 1); assert_eq!(catalog.audio.renditions.len(), 0); let video = catalog.video.renditions.values().next().unwrap(); assert!(video.codec.to_string().starts_with("vp09."), "codec: {}", video.codec); - assert!(matches!(video.container, Container::Cmaf { .. })); - - let Container::Cmaf { init, .. } = &video.container else { + let Container::Cmaf { init, .. } = &video.container else { panic!("expected Cmaf container"); }; let (ftyp, moov) = decode_init(init); @@ -664,12 +765,18 @@ fn test_vp9_catalog() { /// auto-increment from there. #[tokio::test] async fn test_seek_sets_initial_sequence() { + for passthrough in [false, true] { + test_seek_sets_initial_sequence_in(passthrough).await; + } +} + +async fn test_seek_sets_initial_sequence_in(passthrough: bool) { use mp4_atom::{Any, DecodeMaybe}; let mut broadcast = moq_net::broadcast::Info::new().produce(); let broadcast_consumer = broadcast.consume(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); let data = include_bytes!("test_data/bbb.mp4"); @@ -735,7 +842,7 @@ async fn test_msf_catalog_roundtrip() { // MSF catalog track has been created by `catalog::Producer::new`. let consumer = broadcast.consume(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(true); let data = include_bytes!("test_data/bbb.mp4"); let buf = bytes::BytesMut::from(&data[..]); @@ -951,8 +1058,7 @@ fn test_flac_catalog() { assert!(matches!(a.codec, hang::catalog::AudioCodec::Flac)); assert_eq!(a.sample_rate, 96_000); assert_eq!(a.channel_count, 2); - // fmp4 import is CMAF passthrough. - assert!(matches!(a.container, Container::Cmaf { .. })); + assert!(matches!(a.container, Container::Legacy)); // The WebCodecs FLAC description: `fLaC` marker + STREAMINFO. let desc = a.description.as_ref().expect("flac description"); assert_eq!(&desc[..4], b"fLaC"); @@ -1130,12 +1236,18 @@ fn drain_group_frames(consumer: &mut moq_net::track::Subscriber) -> Vec { /// what an HLS/DASH export needs them to agree on, and LL-HLS parts have no consistent unit at all. #[tokio::test] async fn segmented_source_groups_per_segment() { + for passthrough in [false, true] { + segmented_source_groups_per_segment_in(passthrough).await; + } +} + +async fn segmented_source_groups_per_segment_in(passthrough: bool) { let (init, (video_id, video_scale), (audio_id, audio_scale)) = bbb_init(); let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); fmp4.decode(&init).unwrap(); let snapshot = catalog.snapshot(); @@ -1163,9 +1275,11 @@ async fn segmented_source_groups_per_segment() { } fmp4.finish().unwrap(); + // An unwrapped video group also ends on a Legacy duration marker. + let marker = usize::from(!passthrough); assert_eq!( drain_group_frames(&mut video_track), - vec![2, 2, 2], + vec![2 + marker; 3], "one video group per segment, each carrying its two fragments" ); assert_eq!( @@ -1180,12 +1294,18 @@ async fn segmented_source_groups_per_segment() { /// round-trip each. #[tokio::test] async fn segmented_source_indexes_one_group_per_record() { + for passthrough in [false, true] { + segmented_source_indexes_one_group_per_record_in(passthrough).await; + } +} + +async fn segmented_source_indexes_one_group_per_record_in(passthrough: bool) { let (init, (video_id, video_scale), (audio_id, audio_scale)) = bbb_init(); let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); fmp4.decode(&init).unwrap(); let snapshot = catalog.snapshot(); @@ -1235,13 +1355,13 @@ async fn segmented_source_indexes_one_group_per_record() { /// /// `video_offset_us` shifts video relative to audio, so the same source is exercised with audio /// leading and lagging. Returns each track's records as `(start group, end group)`. -async fn records_with_skew(video_offset_us: i64) -> (Vec<(u64, u64)>, Vec<(u64, u64)>) { +async fn records_with_skew(video_offset_us: i64, passthrough: bool) -> (Vec<(u64, u64)>, Vec<(u64, u64)>) { let (init, (video_id, video_scale), (audio_id, audio_scale)) = bbb_init(); let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); fmp4.decode(&init).unwrap(); let snapshot = catalog.snapshot(); @@ -1285,14 +1405,18 @@ async fn records_with_skew(video_offset_us: i64) -> (Vec<(u64, u64)>, Vec<(u64, #[tokio::test] async fn audio_leading_video_still_indexes_one_group_each() { let expected = vec![(0, 1), (1, 2), (2, 3)]; - assert_eq!(records_with_skew(20_000).await, (expected.clone(), expected)); + for passthrough in [false, true] { + assert_eq!(records_with_skew(20_000, passthrough).await, (expected.clone(), expected.clone())); + } } /// Video's IDR lands 20ms BEFORE audio's first sample, the other side of the same skew. #[tokio::test] async fn audio_lagging_video_still_indexes_one_group_each() { let expected = vec![(0, 1), (1, 2), (2, 3)]; - assert_eq!(records_with_skew(-20_000).await, (expected.clone(), expected)); + for passthrough in [false, true] { + assert_eq!(records_with_skew(-20_000, passthrough).await, (expected.clone(), expected.clone())); + } } /// A source that emits one leading `styp` and then never another still segments on its video @@ -1305,12 +1429,18 @@ async fn audio_lagging_video_still_indexes_one_group_each() { /// harmless: a timeline range spans as many groups as the segment needs. #[tokio::test] async fn a_single_leading_styp_still_segments_on_keyframes() { + for passthrough in [false, true] { + a_single_leading_styp_still_segments_on_keyframes_in(passthrough).await; + } +} + +async fn a_single_leading_styp_still_segments_on_keyframes_in(passthrough: bool) { let (init, (video_id, video_scale), (audio_id, audio_scale)) = bbb_init(); let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); fmp4.decode(&init).unwrap(); let snapshot = catalog.snapshot(); @@ -1338,9 +1468,11 @@ async fn a_single_leading_styp_still_segments_on_keyframes() { } fmp4.finish().unwrap(); + // An unwrapped video group also ends on a Legacy duration marker. + let marker = usize::from(!passthrough); assert_eq!( drain_group_frames(&mut video_track), - vec![1, 1, 1, 1], + vec![1 + marker; 4], "each keyframe fragment opens its own segment" ); assert_eq!( @@ -1477,6 +1609,12 @@ fn audio_fragment_samples(base_media_decode_time: u64, duration: u32, size: u32, /// value back down: the publisher can emit a long fragment again at any point. #[test] fn fragment_jitter_never_shrinks() { + for passthrough in [false, true] { + fragment_jitter_never_shrinks_in(passthrough); + } +} + +fn fragment_jitter_never_shrinks_in(passthrough: bool) { let (ftyp, moov) = decode_init(include_bytes!("test_data/bbb.mp4")); let mut init = Vec::new(); ftyp.encode(&mut init).unwrap(); @@ -1484,7 +1622,7 @@ fn fragment_jitter_never_shrinks() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); fmp4.decode(&init).unwrap(); let audio_jitter = |catalog: &crate::catalog::Producer| { @@ -1829,10 +1967,16 @@ async fn import_refuses_a_restart() { /// but the track still refuses a group that starts before the one before it. #[tokio::test] async fn seek_keeps_the_decode_time_until_a_discontinuity() { + for passthrough in [false, true] { + seek_keeps_the_decode_time_until_a_discontinuity_in(passthrough).await; + } +} + +async fn seek_keeps_the_decode_time_until_a_discontinuity_in(passthrough: bool) { let (init, _, _) = bbb_init(); let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); fmp4.decode(&init).unwrap(); fmp4.decode(&live_session(5_000_000, 5)).unwrap(); @@ -1849,8 +1993,13 @@ async fn seek_keeps_the_decode_time_until_a_discontinuity() { fmp4.discontinuity(); fmp4.seek(200).unwrap(); let err = fmp4.decode(&live_session(0, 5)).unwrap_err(); + // Passthrough writes its groups by hand, so the track refuses the rewound group; unwrapped, + // the media producer refuses the rewound keyframe before it gets that far. assert!( - matches!(err, crate::Error::Moq(moq_net::Error::TimestampMismatch)), + match passthrough { + true => matches!(err, crate::Error::Moq(moq_net::Error::TimestampMismatch)), + false => matches!(err, crate::Error::TimestampRewind(_)), + }, "{err:?}" ); } @@ -1966,21 +2115,29 @@ async fn a_deselected_fragment_does_not_publish() { /// The fragments it passes through then decode against its catalog init to the same samples. #[tokio::test] async fn imports_the_sample_defaults_fixtures() { + for passthrough in [false, true] { + imports_the_sample_defaults_fixtures_in(passthrough).await; + } +} + +async fn imports_the_sample_defaults_fixtures_in(passthrough: bool) { use mp4_atom::DecodeMaybe; for fixture in super::fixtures() { let mut broadcast = moq_net::broadcast::Info::new().produce(); let consumer = broadcast.consume(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); fmp4.decode(&fixture.init).unwrap(); let snapshot = catalog.snapshot(); let (name, video) = snapshot.video.renditions.iter().next().unwrap(); - let Container::Cmaf { init } = &video.container else { - panic!("{}: a CMAF rendition", fixture.name); + let wire = match &video.container { + Container::Cmaf { init } => Some(super::Wire::from_init(init).unwrap()), + Container::Legacy => None, + other => panic!("{}: unexpected container {other:?}", fixture.name), }; - let wire = super::Wire::from_init(init).unwrap(); + assert_eq!(wire.is_some(), passthrough, "{}", fixture.name); let mut track = consumer.track(name).unwrap().subscribe(replay()).await.unwrap(); // The same fragment again, one second later, so its first sample can open a second group. @@ -1997,6 +2154,35 @@ async fn imports_the_sample_defaults_fixtures() { fmp4.decode(&later).unwrap(); fmp4.finish().unwrap(); + let Some(wire) = wire else { + // Unwrapped, each sample is its own frame and each sync sample opens a group. Legacy + // carries no duration, so only the timing, bytes, and grouping are compared. + let mut samples = Vec::new(); + while let Some(mut group) = track.recv_group().await.unwrap() { + let mut index = 0; + while let Some(frame) = group.read_frame().await.unwrap() { + let frame = hang::container::Frame::decode(frame.payload).unwrap(); + // The duration marker closing a video group is no sample. + if !frame.payload.is_empty() { + samples.push((frame.timestamp.as_micros() as u64, index == 0, frame.payload.to_vec())); + } + index += 1; + } + } + let expected: Vec<_> = fixture + .samples + .iter() + .map(|sample| (sample.timestamp, sample.keyframe, sample.data.clone())) + .collect(); + assert_eq!(samples[..expected.len()], expected, "{}", fixture.name); + assert!( + samples[expected.len()].1, + "{}: the second fragment opens on a sync sample, so a group", + fixture.name + ); + continue; + }; + let mut first = track.recv_group().await.unwrap().unwrap(); let frame = first.read_frame().await.unwrap().unwrap(); let frames = super::decode(frame.payload, frame.timestamp, wire.track().unwrap()).unwrap(); diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index a52f386824a..fe1bc6ab8e8 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -332,6 +332,14 @@ where self.publish_estimate() } + /// Declare a segment boundary the source drew at `pts`: the timeline closes a record at the + /// first group starting at or after it, instead of pacing records by duration. + pub(crate) fn boundary(&mut self, pts: moq_net::Timestamp) { + if let Some(recorder) = self.recorder.as_mut() { + recorder.cut(pts); + } + } + /// Whether the next [`write`](Self::write) has to be a keyframe, i.e. no group is currently open /// (at the start, or after a [`cut`](Self::cut) / [`seek`](Self::seek)). /// diff --git a/rs/moq-mux/src/import/container.rs b/rs/moq-mux/src/import/container.rs index a8194797c27..9b0ae4295d2 100644 --- a/rs/moq-mux/src/import/container.rs +++ b/rs/moq-mux/src/import/container.rs @@ -26,15 +26,17 @@ impl ContainerImpl { format: ContainerFormat, ) -> Self { match format { - ContainerFormat::Fmp4 => Self::fmp4(broadcast, reserved), + ContainerFormat::Fmp4 => Self::fmp4(broadcast, reserved, false), + ContainerFormat::Cmaf => Self::fmp4(broadcast, reserved, true), ContainerFormat::Mkv => Self::mkv(broadcast, reserved), ContainerFormat::Ts => Self::ts(broadcast, reserved), ContainerFormat::Flv => Self::flv(broadcast, reserved), } } - fn fmp4(broadcast: moq_net::broadcast::Producer, reserved: crate::catalog::Reserved) -> Self { - ContainerImpl::Fmp4(Box::new(crate::container::fmp4::Import::new(broadcast, reserved))) + fn fmp4(broadcast: moq_net::broadcast::Producer, reserved: crate::catalog::Reserved, passthrough: bool) -> Self { + let import = crate::container::fmp4::Import::new(broadcast, reserved).with_passthrough(passthrough); + ContainerImpl::Fmp4(Box::new(import)) } fn mkv(broadcast: moq_net::broadcast::Producer, reserved: crate::catalog::Reserved) -> Self { diff --git a/rs/moq-mux/src/import/format.rs b/rs/moq-mux/src/import/format.rs index 67f28b11152..1e184014775 100644 --- a/rs/moq-mux/src/import/format.rs +++ b/rs/moq-mux/src/import/format.rs @@ -51,8 +51,10 @@ pub enum VideoFormat { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum ContainerFormat { - /// Fragmented MP4 / CMAF. + /// Fragmented MP4 / CMAF, unwrapped into one frame per sample. Fmp4, + /// Fragmented MP4 / CMAF, republished whole as one CMAF frame per fragment. + Cmaf, /// Matroska / WebM. Mkv, /// MPEG-2 transport stream. @@ -129,7 +131,8 @@ impl FromStr for Format { "vp8" | "vp08" => Format::Video(VideoFormat::Vp8), "vp9" | "vp09" => Format::Video(VideoFormat::Vp9), - "fmp4" | "cmaf" => Format::Container(ContainerFormat::Fmp4), + "fmp4" => Format::Container(ContainerFormat::Fmp4), + "cmaf" => Format::Container(ContainerFormat::Cmaf), "mkv" | "webm" | "matroska" => Format::Container(ContainerFormat::Mkv), "ts" | "mpegts" | "mpeg2ts" | "m2ts" => Format::Container(ContainerFormat::Ts), "flv" => Format::Container(ContainerFormat::Flv), @@ -199,7 +202,7 @@ kind_format!(VideoFormat, Video, Av01 => "av01", Vp8 => "vp8", Vp9 => "vp9"); kind_format!(ContainerFormat, Container, - Fmp4 => "fmp4", Mkv => "mkv", Ts => "ts", Flv => "flv"); + Fmp4 => "fmp4", Cmaf => "cmaf", Mkv => "mkv", Ts => "ts", Flv => "flv"); #[cfg(test)] mod tests { @@ -216,7 +219,8 @@ mod tests { (vec!["av01", "av1", "av1c", "av1C"], "av01"), (vec!["vp8", "vp08"], "vp8"), (vec!["vp9", "vp09"], "vp9"), - (vec!["fmp4", "cmaf"], "fmp4"), + (vec!["fmp4"], "fmp4"), + (vec!["cmaf"], "cmaf"), (vec!["mkv", "webm", "matroska"], "mkv"), (vec!["ts", "mpegts", "mpeg2ts", "m2ts"], "ts"), (vec!["opus"], "opus"), diff --git a/swift/MIGRATE.md b/swift/MIGRATE.md index 044aa7fa273..63d32f41214 100644 --- a/swift/MIGRATE.md +++ b/swift/MIGRATE.md @@ -4,6 +4,11 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `ContainerFormat.fmp4` unwraps to Legacy (#PR) + +A container import with `ContainerFormat.fmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. +Pass the new `ContainerFormat.cmaf` to keep publishing CMAF. + ### Finished tracks stay until the broadcast ends (#6157) A finished track now stays listed, with its catalog rendition and demand, until its broadcast closes or the new `BroadcastProducer.finish()` lets go. From bfe28159c6d58557c272a26df8ea994f506ad805 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:35:13 -0700 Subject: [PATCH 3/7] fix: keep MoqContainerFormat discriminants stable and regenerate Dart bindings Co-Authored-By: Claude Opus 5.5 --- dart/moq_ffi/lib/src/moq.dart | 4 +++- rs/moq-ffi/src/media.rs | 4 ++-- rs/moq-mux/src/container/fmp4/import.rs | 16 ++++++++++++---- rs/moq-mux/src/container/fmp4/import_test.rs | 14 ++++++++++---- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 1d0d4121dcd..e4025fce4cd 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -6311,7 +6311,7 @@ class LocMoqContainer extends MoqContainer { } } -enum MoqContainerFormat { fmp4, mkv, ts, flv } +enum MoqContainerFormat { fmp4, mkv, ts, flv, cmaf } class FfiConverterMoqContainerFormat { static LiftRetVal read(Uint8List buf) { @@ -6325,6 +6325,8 @@ class FfiConverterMoqContainerFormat { return LiftRetVal(MoqContainerFormat.ts, 4); case 4: return LiftRetVal(MoqContainerFormat.flv, 4); + case 5: + return LiftRetVal(MoqContainerFormat.cmaf, 4); default: throw UniffiInternalError( UniffiInternalError.unexpectedEnumCase, diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 5c147d3a6eb..f28f005f159 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -277,14 +277,14 @@ pub enum MoqVideoFormat { pub enum MoqContainerFormat { /// Fragmented MP4 / CMAF, split into one frame per sample. Fmp4, - /// Fragmented MP4 / CMAF, republished whole as one CMAF frame per fragment. - Cmaf, /// Matroska / WebM. Mkv, /// MPEG-2 transport stream. Ts, /// Flash Video, as used by RTMP. Flv, + /// Fragmented MP4 / CMAF, republished whole as one CMAF frame per fragment. + Cmaf, } /// What an audio publish needs: a format, its codec init bytes, and an optional label. diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index fb411830068..6ded2d6bc74 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -397,7 +397,9 @@ impl Import { let timescale = moq_net::Timescale::new(trak.mdia.mdhd.timescale as u64)?; let output = match self.passthrough { - true => Output::Passthrough(Box::new(self.init_passthrough(trak, &moov, &reserved, kind, timescale)?)), + true => Output::Passthrough(Box::new( + self.init_passthrough(trak, &moov, &reserved, kind, timescale)?, + )), false => self.init_unwrap(trak, &moov, &reserved, kind)?, }; @@ -845,7 +847,6 @@ impl Import { Ok(config) } - /// Whether the fragments in `moof` open a new segment, and so roll a group on every track. /// /// A segment is the unit a group carries, which is what lets audio (whose samples are all @@ -1058,7 +1059,9 @@ impl Fmp4Track { end = Some(end.map_or(sample_end, |max: Timestamp| max.max(sample_end))); earliest = Some(earliest.map_or(timestamp, |min: Timestamp| min.min(timestamp))); - let next = offset.checked_add(sample.size as usize).ok_or(Error::InvalidDataOffset)?; + let next = offset + .checked_add(sample.size as usize) + .ok_or(Error::InvalidDataOffset)?; if next > len { return Err(Error::InvalidDataOffset.into()); } @@ -1082,7 +1085,12 @@ impl Fmp4Track { // offsets disagree with the actual data and no sample can be addressed safely. let start = start.unwrap_or(0); if !(start <= offset && offset <= len) { - return Err(Error::SampleRangeOutOfBounds { start, end: offset, len }.into()); + return Err(Error::SampleRangeOutOfBounds { + start, + end: offset, + len, + } + .into()); } Ok(Run { diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 05bff5c97b7..4515a797db0 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -678,7 +678,7 @@ fn test_av1_catalog() { (color.primaries, color.range), (None, Some(hang::catalog::Range::Limited)) ); - let Container::Cmaf { init, .. } = &video.container else { + let Container::Cmaf { init, .. } = &video.container else { panic!("expected Cmaf container"); }; let (ftyp, moov) = decode_init(init); @@ -750,7 +750,7 @@ fn test_vp9_catalog() { let video = catalog.video.renditions.values().next().unwrap(); assert!(video.codec.to_string().starts_with("vp09."), "codec: {}", video.codec); - let Container::Cmaf { init, .. } = &video.container else { + let Container::Cmaf { init, .. } = &video.container else { panic!("expected Cmaf container"); }; let (ftyp, moov) = decode_init(init); @@ -1375,7 +1375,10 @@ async fn records_with_skew(video_offset_us: i64, passthrough: bool) -> (Vec<(u64 async fn audio_leading_video_still_indexes_one_group_each() { let expected = vec![(0, 1), (1, 2), (2, 3)]; for passthrough in [false, true] { - assert_eq!(records_with_skew(20_000, passthrough).await, (expected.clone(), expected.clone())); + assert_eq!( + records_with_skew(20_000, passthrough).await, + (expected.clone(), expected.clone()) + ); } } @@ -1384,7 +1387,10 @@ async fn audio_leading_video_still_indexes_one_group_each() { async fn audio_lagging_video_still_indexes_one_group_each() { let expected = vec![(0, 1), (1, 2), (2, 3)]; for passthrough in [false, true] { - assert_eq!(records_with_skew(-20_000, passthrough).await, (expected.clone(), expected.clone())); + assert_eq!( + records_with_skew(-20_000, passthrough).await, + (expected.clone(), expected.clone()) + ); } } From ecb1444cbff8952a4f8a3a23447ef6d6447dbab2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 21:38:46 -0700 Subject: [PATCH 4/7] docs: link MIGRATE entries to #6250 Co-Authored-By: Claude Opus 5.5 --- cpp/moq/MIGRATE.md | 2 +- dart/moq/MIGRATE.md | 2 +- go/wrapper/MIGRATE.md | 2 +- kt/moq/MIGRATE.md | 2 +- py/moq-rs/MIGRATE.md | 2 +- rs/moq-c/MIGRATE.md | 2 +- rs/moq-cli/MIGRATE.md | 2 +- rs/moq-ffi/MIGRATE.md | 2 +- rs/moq-hls/MIGRATE.md | 2 +- rs/moq-mux/MIGRATE.md | 2 +- swift/MIGRATE.md | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cpp/moq/MIGRATE.md b/cpp/moq/MIGRATE.md index b2bb53aef50..17cba0ef04a 100644 --- a/cpp/moq/MIGRATE.md +++ b/cpp/moq/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `moq::ContainerFormat::kFmp4` unwraps to Legacy (#PR) +### `moq::ContainerFormat::kFmp4` unwraps to Legacy (#6250) A container import with `moq::ContainerFormat::kFmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. Pass the new `moq::ContainerFormat::kCmaf` to keep publishing CMAF. diff --git a/dart/moq/MIGRATE.md b/dart/moq/MIGRATE.md index 97cd44ea9e6..79dec59a9b8 100644 --- a/dart/moq/MIGRATE.md +++ b/dart/moq/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `ContainerFormat.fmp4` unwraps to Legacy (#PR) +### `ContainerFormat.fmp4` unwraps to Legacy (#6250) A container import with `ContainerFormat.fmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. Pass the new `ContainerFormat.cmaf` to keep publishing CMAF. diff --git a/go/wrapper/MIGRATE.md b/go/wrapper/MIGRATE.md index 99e544d1495..e77e40507e2 100644 --- a/go/wrapper/MIGRATE.md +++ b/go/wrapper/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `media.ContainerFormatFmp4` unwraps to Legacy (#PR) +### `media.ContainerFormatFmp4` unwraps to Legacy (#6250) A container import with `media.ContainerFormatFmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. Pass the new `media.ContainerFormatCmaf` to keep publishing CMAF. diff --git a/kt/moq/MIGRATE.md b/kt/moq/MIGRATE.md index aec077ecf1d..2b0fd44aa86 100644 --- a/kt/moq/MIGRATE.md +++ b/kt/moq/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `ContainerFormat.FMP4` unwraps to Legacy (#PR) +### `ContainerFormat.FMP4` unwraps to Legacy (#6250) A container import with `ContainerFormat.FMP4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. Pass the new `ContainerFormat.CMAF` to keep publishing CMAF. diff --git a/py/moq-rs/MIGRATE.md b/py/moq-rs/MIGRATE.md index 03f5c618802..196c06b5bdc 100644 --- a/py/moq-rs/MIGRATE.md +++ b/py/moq-rs/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `ContainerFormat.FMP4` unwraps to Legacy (#PR) +### `ContainerFormat.FMP4` unwraps to Legacy (#6250) A container import with `ContainerFormat.FMP4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. Pass the new `ContainerFormat.CMAF` to keep publishing CMAF. diff --git a/rs/moq-c/MIGRATE.md b/rs/moq-c/MIGRATE.md index 589402ebab1..9c8718d1616 100644 --- a/rs/moq-c/MIGRATE.md +++ b/rs/moq-c/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `MOQ_CONTAINER_FORMAT_FMP4` unwraps to Legacy (#PR) +### `MOQ_CONTAINER_FORMAT_FMP4` unwraps to Legacy (#6250) A container import with `MOQ_CONTAINER_FORMAT_FMP4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. Pass the new `MOQ_CONTAINER_FORMAT_CMAF` to keep publishing CMAF. diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index a5ae57e8842..215b3a099ff 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,7 +4,7 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased -### `import fmp4` and `import hls` unwrap to Legacy (#PR) +### `import fmp4` and `import hls` unwrap to Legacy (#6250) Both now publish one `legacy` frame per sample instead of one `cmaf` frame per fragment. Add `--passthrough` to keep publishing CMAF. diff --git a/rs/moq-ffi/MIGRATE.md b/rs/moq-ffi/MIGRATE.md index 8e19e0c033f..5aeeec7d683 100644 --- a/rs/moq-ffi/MIGRATE.md +++ b/rs/moq-ffi/MIGRATE.md @@ -5,7 +5,7 @@ These are the UniFFI names; each language wrapper has its own `MIGRATE.md` in it ## Unreleased -### `MoqContainerFormat::Fmp4` unwraps to Legacy (#PR) +### `MoqContainerFormat::Fmp4` unwraps to Legacy (#6250) A container import with `MoqContainerFormat::Fmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. Pass the new `MoqContainerFormat::Cmaf` to keep publishing CMAF. diff --git a/rs/moq-hls/MIGRATE.md b/rs/moq-hls/MIGRATE.md index 7ffbccb0da1..c1e62870a2f 100644 --- a/rs/moq-hls/MIGRATE.md +++ b/rs/moq-hls/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### HLS import unwraps fMP4 to Legacy (#PR) +### HLS import unwraps fMP4 to Legacy (#6250) `import::Import` now publishes one `Legacy` frame per sample instead of one `Cmaf` frame per fragment. Set `import::Config::passthrough` to keep publishing CMAF. diff --git a/rs/moq-mux/MIGRATE.md b/rs/moq-mux/MIGRATE.md index e5bf39c00cf..f976a6b1559 100644 --- a/rs/moq-mux/MIGRATE.md +++ b/rs/moq-mux/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### fMP4 import unwraps to Legacy (#PR) +### fMP4 import unwraps to Legacy (#6250) `container::fmp4::Import` now publishes one `Legacy` frame per sample instead of one `Cmaf` frame per fragment. Call `.with_passthrough(true)` to keep publishing CMAF. diff --git a/swift/MIGRATE.md b/swift/MIGRATE.md index 63d32f41214..81403ec3109 100644 --- a/swift/MIGRATE.md +++ b/swift/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `ContainerFormat.fmp4` unwraps to Legacy (#PR) +### `ContainerFormat.fmp4` unwraps to Legacy (#6250) A container import with `ContainerFormat.fmp4` now publishes one Legacy frame per sample instead of one CMAF frame per fragment. Pass the new `ContainerFormat.cmaf` to keep publishing CMAF. From 9e91e2a16155e44d647d97cb0a719b331ad79e19 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 22:01:36 -0700 Subject: [PATCH 5/7] quest: finish m1/fmp4-import-unwrap Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/fmp4-import-unwrap.md | 66 ---------------------------------- quest/m1/hang-cmaf-lazy.md | 1 - 3 files changed, 68 deletions(-) delete mode 100644 quest/m1/fmp4-import-unwrap.md diff --git a/quest/m1/README.md b/quest/m1/README.md index b3b330fc544..e7c95e4c1cd 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -117,7 +117,6 @@ lite-07 stream type here. - [Gateway linger](/quest/m1/linger-gateways.md) - HLS, RTMP, SRT and RTC imports linger at a clean end, cut short by a newer epoch at the same path - [Interop finish](/quest/m1/interop-finish.md) - the interop matrix proves a clean broadcast finish across implementations - [Exporters follow a rendition to a new track](/quest/m1/export-rendition-rebind.md) - the fMP4, TS, MKV and FLV exporters rebuild a rendition whose resolved `track` or `broadcast` changes under the same ID -- [fMP4 import unwraps to Legacy](/quest/m1/fmp4-import-unwrap.md) - fMP4 and HLS imports publish per-sample `Legacy` frames unless `--passthrough` asks for CMAF, and no first-party producer defaults to CMAF - [Load CMAF decode on demand](/quest/m1/hang-cmaf-lazy.md) - `@moq/watch` fetches the cml-based `@moq/hang/cmaf` only for CMAF renditions, and the unused public CMAF encoder is deleted - [lite-06 max sequence](/quest/m1/lite06-max-sequence.md) - group sequence 2^62-1 reaches a lite-06 subscriber or is refused loudly, instead of vanishing - [Track live signal](/quest/m1/track-live.md) - a reader can wait for its track's feed to be live, so a player skips old cached groups while a fresh upstream subscription starts diff --git a/quest/m1/fmp4-import-unwrap.md b/quest/m1/fmp4-import-unwrap.md deleted file mode 100644 index 658c75da697..00000000000 --- a/quest/m1/fmp4-import-unwrap.md +++ /dev/null @@ -1,66 +0,0 @@ -# [L] fMP4 import unwraps to Legacy - -## Goal - -First-party importers stop publishing CMAF unless asked. `moq import fmp4` -and `moq import hls` split each fragment into per-sample `Legacy` frames by -default. `--passthrough` keeps today's CMAF passthrough, which advertises -`Container::Cmaf` and sends one fragment per frame, mirroring -`import ts --passthrough`. Every other producer that defaults to CMAF flips -the same way. - -CMAF makes each frame carry a `moof`, makes every player parse ISO BMFF, -and keeps `@svta/cml-iso-bmff` in the browser. Every other importer already -emits `Legacy`. - -## Plan - -Decided with the maintainer on 2026-10-20: - -- The default lives in `moq-mux`, not just the CLI. - `moq_mux::container::fmp4::Import` gains an unwrap mode and makes it the - default. Passthrough becomes an explicit option, so moq-cli, moq-hls, the - `ContainerFormat::Fmp4` path in `import/container.rs`, and FFI callers - all get `Legacy` unless they opt in. Thread the opt-in through every path - that builds an `fmp4::Import`: the importer itself (a builder like its - `with_select`), `ContainerFormat` for `import/container.rs` and the FFI, - and moq-hls's import args. Settle the exact shape with the maintainer in - the PR. The FFI option follows the `rs/moq-ffi` row of the AGENTS.md sync - table: `moq-c`, every hand-written wrapper (`py`, `swift`, `kt`, `dart`, - `go/wrapper`), `cpp/moq`, and their `doc/lib` pages expose it, so no - binding's fMP4 import silently changes without a way back. Breaking: add - `MIGRATE.md` entries for `moq-mux` and each affected binding. -- Unwrapped output is `Legacy`, not LOC (maintainer choice). Each sample - becomes one frame. Sample `i` is stamped `tfdt + sum(durations of samples - before i) + composition_offset_i` in the track's timescale. The existing - passthrough extraction in `fmp4/import.rs` already accumulates `dts` this - way, so reuse it. The stamp is then published through the - importer's `Timebase` like the passthrough path (`timebase.anchor`), so - both modes land on the same catalog clock. Edit lists stay out of scope: - both players already ignore `edts` today, and - [fMP4 edit lists](/quest/m2/fmp4-edit-lists.md) applies them for both - modes. Video keyframes follow the - `depends_on` rule moq-mux already applies ([moq-dev/moq#5078](https://github.com/moq-dev/moq/pull/5078)). - Audio does not: packagers flag every audio sample as sync, which would - open a group per packet. As in `fmp4::decode` (`container/fmp4/mod.rs`), - audio samples carry the bit clear and only the first sample of each source - fragment opens a group, so the source's segment grouping survives. The codec description - comes from `stsd`. -- `moq import hls` takes the same `--passthrough`. HLS and fMP4 export - already re-mux `Legacy` tracks through `fmp4::Muxer`. The `Legacy` wire - carries no sample duration, so the muxer re-derives durations from codec - data or the gap to the next frame, as it already does for TS, MKV and - RTMP sources. A source whose exact `trun` timing must survive keeps - `--passthrough`. -- Audit every CMAF producer and flip any that defaults to it: moq-gst's - source, `moq-ffi`/`moq-c` media setup, `moq-video`/`moq-audio` helpers, - and the demo and `justfile` recipes. Tests that need CMAF opt in - explicitly. -- Update `doc/bin/cli.md`, `doc/bin/hls.md`, and every example invocation - that relies on CMAF output, checked against `--help`. - -## Related - -- [Load CMAF decode on demand](/quest/m1/hang-cmaf-lazy.md) - the player side; this quest is what makes CMAF rare -- [fMP4 edit lists](/quest/m2/fmp4-edit-lists.md) - the unwrap path is where edit lists would apply to frame timestamps -- [CMAF surround Opus](/quest/m2/cmaf-opus-surround.md) - touches the same fMP4 import diff --git a/quest/m1/hang-cmaf-lazy.md b/quest/m1/hang-cmaf-lazy.md index 77e7710e675..afe470f4728 100644 --- a/quest/m1/hang-cmaf-lazy.md +++ b/quest/m1/hang-cmaf-lazy.md @@ -50,5 +50,4 @@ Decided with the maintainer on 2026-10-20: ## Related -- [fMP4 import unwraps to Legacy](/quest/m1/fmp4-import-unwrap.md) - makes CMAF renditions rare, which is why loading on demand pays - [Artifact size audit](/quest/m1/audit-size.md) - tracks the watch first-load baseline this cuts From d5fd1e84d31df697a221b1ab2e5b5aa05b48c3d7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 22:50:04 -0700 Subject: [PATCH 6/7] fix(moq-mux): append ContainerFormat::Cmaf to keep discriminants stable Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/import/format.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-mux/src/import/format.rs b/rs/moq-mux/src/import/format.rs index 1e184014775..1f583ed021c 100644 --- a/rs/moq-mux/src/import/format.rs +++ b/rs/moq-mux/src/import/format.rs @@ -53,14 +53,14 @@ pub enum VideoFormat { pub enum ContainerFormat { /// Fragmented MP4 / CMAF, unwrapped into one frame per sample. Fmp4, - /// Fragmented MP4 / CMAF, republished whole as one CMAF frame per fragment. - Cmaf, /// Matroska / WebM. Mkv, /// MPEG-2 transport stream. Ts, /// Flash Video, as used by RTMP. Flv, + /// Fragmented MP4 / CMAF, republished whole as one CMAF frame per fragment. + Cmaf, } /// Any format an importer handles, and the kind of import it selects. From 74831dd2b67874939d232b2033501cad3e3c6aa4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 02:34:28 -0700 Subject: [PATCH 7/7] fix(moq-mux): open a mid-GOP video segment at its first sync sample Unwrapped video skips samples until a sync sample, but the skipped fragment still marked the segment opened, so the next keyframe rolled a second segment and left audio one group ahead of video. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/fmp4/import.rs | 16 +++++-- rs/moq-mux/src/container/fmp4/import_test.rs | 50 ++++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 6ded2d6bc74..a383e9eebe5 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -948,7 +948,6 @@ impl Import { self.initial_reservation = None; let new_segment = track.segment != Some(self.segment); - track.segment = Some(self.segment); let placement = Placement { offset, new_segment, @@ -956,12 +955,18 @@ impl Import { declared: new_segment && self.declared, }; - match &mut track.output { + let opened = match &mut track.output { Output::Passthrough(passthrough) => { let fragment = fragment(traf, &moof, &run, &payload, &track.defaults)?; passthrough.write(fragment, &run, track.kind, placement)?; + true } output => unwrap(output, &run, &payload, track.timescale, placement)?, + }; + // A video track joined mid-GOP skips samples until a sync sample, so its segment opens + // there; counting a skipped fragment would let the next keyframe roll a second segment. + if opened { + track.segment = Some(self.segment); } } @@ -1247,13 +1252,15 @@ impl Passthrough { } /// Write each sample of `run` as its own frame on an unwrapped track. +/// +/// Returns whether the track now has a group open, which video lacks until its first sync sample. fn unwrap( output: &mut Output, run: &Run, payload: &Bytes, timescale: moq_net::Timescale, placement: Placement, -) -> Result<()> { +) -> Result { let earliest = placement.offset.apply(run.earliest)?; let end = placement.offset.apply(run.end)?; let video = matches!(output, Output::Video(_)); @@ -1292,7 +1299,8 @@ fn unwrap( } // The fragment arrived at once, so a consumer has to buffer its whole span. - producer.burst(end.checked_sub(earliest)?.into()) + producer.burst(end.checked_sub(earliest)?.into())?; + Ok(!producer.needs_keyframe()) }, _ => unreachable!("passthrough tracks write whole fragments")) } diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 4515a797db0..33c6b181c6c 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1457,6 +1457,56 @@ async fn a_single_leading_styp_still_segments_on_keyframes_in(passthrough: bool) ); } +/// A muxed source joined mid-GOP keeps audio and video on the same group sequences. +/// +/// Unwrapped video drops samples until its first sync sample, so its segment opens there, not on +/// the skipped fragment. Counting the skipped fragment as opened rolled the next keyframe into a +/// second segment, leaving audio one group ahead of video for the rest of the stream. +#[tokio::test] +async fn a_mid_gop_join_keeps_tracks_on_the_same_groups() { + for passthrough in [false, true] { + a_mid_gop_join_keeps_tracks_on_the_same_groups_in(passthrough).await; + } +} + +async fn a_mid_gop_join_keeps_tracks_on_the_same_groups_in(passthrough: bool) { + let (init, (video_id, video_scale), (audio_id, audio_scale)) = bbb_init(); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); + fmp4.decode(&init).unwrap(); + + let snapshot = catalog.snapshot(); + let video_name = snapshot.video.renditions.keys().next().unwrap().clone(); + let audio_name = snapshot.audio.renditions.keys().next().unwrap().clone(); + let mut video_track = consumer.track(&video_name).unwrap().subscribe(replay()).await.unwrap(); + let mut audio_track = consumer.track(&audio_name).unwrap().subscribe(replay()).await.unwrap(); + + // No styp, so segments follow video keyframes; the first video fragment is mid-GOP. + for segment in 0..3u64 { + let base = segment * 1_000_000; + let frag = super::encode_fragment( + info(video_id, video_scale, segment as u32), + &[sample(base, segment > 0, Some(1_000_000))], + ) + .unwrap(); + fmp4.decode(&frag).unwrap(); + let frag = super::encode_fragment( + info(audio_id, audio_scale, segment as u32), + &[sample(base, true, Some(1_000_000))], + ) + .unwrap(); + fmp4.decode(&frag).unwrap(); + } + fmp4.finish().unwrap(); + + let video = drain_group_sequences(&mut video_track); + let audio = drain_group_sequences(&mut audio_track); + assert_eq!(audio, video, "audio and video groups line up"); +} + /// One audio fragment for bbb's track 2, carrying a single sample at the track's 44100 timescale. /// /// `trun.data_offset` is left unset so the samples start at the front of the mdat, which is all