From 8bcd48557fc2c44952b5a6ecef602898414f46a5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 05:12:24 -0700 Subject: [PATCH 1/7] chore(quest): claim quest/m2/mp4-export.md Quest-Claim: b0e4fa74a9572876 From 37c690a4734219026879392fab4f58a33ba94ead Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:07:57 -0700 Subject: [PATCH 2/7] feat(cli): `moq export mp4 --output` records a crash-safe regular MP4 The file is fragmented MP4 while recording, with a reserved `free` box after the ftyp. The broadcast's end or the first SIGINT/SIGTERM appends a regular moov, indexed by parsing the written moofs back, and overwrites the reserved box with a 64-bit mdat header, so no media is copied. A second signal exits at once and leaves the fragmented file. Renditions that join mid-recording go into `free` boxes and only the final moov indexes them. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + doc/bin/cli.md | 18 +- quest/m2/README.md | 1 - quest/m2/mp4-export.md | 41 -- rs/moq-cli/Cargo.toml | 2 + rs/moq-cli/src/args.rs | 78 ++- rs/moq-cli/src/main.rs | 212 +++++-- rs/moq-cli/src/report/echo_test.rs | 1 + rs/moq-cli/src/subscribe.rs | 82 ++- rs/moq-cli/tests/integration/export.rs | 66 +++ rs/moq-mux/src/container/fmp4/export.rs | 23 +- rs/moq-mux/src/container/fmp4/mod.rs | 23 +- rs/moq-mux/src/container/mod.rs | 1 + rs/moq-mux/src/container/mp4.rs | 740 ++++++++++++++++++++++++ 14 files changed, 1176 insertions(+), 113 deletions(-) delete mode 100644 quest/m2/mp4-export.md create mode 100644 rs/moq-mux/src/container/mp4.rs diff --git a/Cargo.lock b/Cargo.lock index 220f432804f..9d3400ea9db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5350,6 +5350,7 @@ dependencies = [ "terminal_size", "test-env", "tokio", + "tokio-util", "tower-http 0.7.1", "tracing", "url", diff --git a/doc/bin/cli.md b/doc/bin/cli.md index d69e77e5eb0..506f60eb760 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -22,6 +22,7 @@ Docker; see [Install](/setup/install). | `import` | `rtsp --connect ` | Pull an IP camera's stream. | | `import` | `archive ` | Replay a recording from an object store. | | `export` | `fmp4`, `mkv`, `ts`, `flv`, `h264`, `h265` | Write a container to stdout. | +| `export` | `mp4 --output ` | Record a regular MP4 that stays playable if the recording is cut short. | | `export` | `hls --listen`, `hls --output ` | Serve the broadcast as HLS over HTTP, record it to a directory, or both. | | `export` | `rtmp`, `srt`, `rtc` | Serve plays (`--listen`) or push to a remote (`--connect`). | | `export` | `archive ` | Record every broadcast under the prefix into an object store. | @@ -168,7 +169,7 @@ moq --connect https://relay.example.com/anon --broadcast camera.hang \ ## Export -The `fmp4`, `mkv`, `flv`, `h264`, and `h265` exports pick renditions with +The `fmp4`, `mp4`, `mkv`, `flv`, `h264`, and `h265` exports pick renditions with `--video-name`, `--audio-name`, `--video-codec`, `--audio-codec`, `--no-video`, and `--no-audio`. `ts`, `archive`, and the gateways export everything and refuse these flags. @@ -194,6 +195,17 @@ the samples, so an encoder restarting with a new SPS does not end the export. Each keyframe carries them, so a keyframe whose sets never appeared ends it. Other video waits for its first keyframe. +`mp4 --output ` records to a new file the way OBS's hybrid MP4 does: it is +fragmented MP4 while recording, so a crash leaves a playable file, and the +broadcast's end or the first Ctrl-C (or SIGTERM) finishes it as a regular MP4 with +its index at the end, without copying the media. A second signal exits at once +and leaves it fragmented. Unlike `fmp4`, a rendition that joins mid-recording is +kept, though only the finished file indexes it. + +```bash +moq ... export mp4 --output recording.mp4 +``` + MPEG-TS export pads to the source's constant mux rate when the catalog recorded one, or to `--mux-rate`, on a constant-rate schedule an IRD or groomer can lock to. Each frame goes out as early as the receiver's buffers admit, up @@ -451,8 +463,8 @@ See [Authentication](/bin/relay/auth). ## Retention and latency `import --max-age` (default 30 s) tells relays how long to keep old groups -fetchable, which the [HLS gateway](/bin/hls) depends on. `export fmp4`, `export h264`, -and `export h265` take `--max-delay` (default 500 ms): how far a stalled group +fetchable, which the [HLS gateway](/bin/hls) depends on. `export fmp4`, `export mp4`, +`export h264`, and `export h265` take `--max-delay` (default 500 ms): how far a stalled group may fall behind the live edge before *this* consumer skips it. Raising `--max-age` never delays playback. At stdin EOF, `import` keeps the finished broadcast announced for diff --git a/quest/m2/README.md b/quest/m2/README.md index 4f4f2452b0a..0abd1a229d0 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -151,7 +151,6 @@ lite-07; bitrate enforcement is not part of this release. - [One support probe for watch audio and video](/quest/m2/watch-support-probe.md) - one probe helper with video's cache, and audio reports unsupported renditions - [Shared fullscreen, formatters, and stats cards in @moq/ui](/quest/m2/ui-shared-controls.md) - watch and publish stop keeping copies of their shared controls - [moqsrc A/V sync](/quest/m2/moqsrc-av-sync.md) - a nightly test measures A/V skew from `moqsrc` through clock-synced sinks -- [MP4 export](/quest/m2/mp4-export.md) - `moq export mp4 --output` records crash-safe fragments, then finishes a regular MP4 with moov at the end - [Importers fill the VR fields](/quest/m2/mux-vr-fields.md) - MP4, Matroska, and Opus imports set the catalog's projection, stereo, and ambisonic layout - [fMP4 ambisonic layout](/quest/m2/mux-vr-ambisonic-fmp4.md) - fMP4 Opus import fills `ambisonic` from the `dOps` mapping table - [Linux decoded frames](/quest/m2/obs-decode-linux.md) - present supported native decoded surfaces with visible CPU fallback diff --git a/quest/m2/mp4-export.md b/quest/m2/mp4-export.md deleted file mode 100644 index 2cf878bc561..00000000000 --- a/quest/m2/mp4-export.md +++ /dev/null @@ -1,41 +0,0 @@ -# [M] moq export mp4 records a crash-safe regular MP4 - -## Goal - -`moq export mp4 --output ` records the way OBS's hybrid MP4 does: the -file is a valid fragmented MP4 while recording, so a crash still leaves a -playable file, and a clean end or Ctrl+C finishes it as a regular MP4 with -its moov at the end, without copying media. Renditions that join mid-recording -are included in the finished file. - -## Plan - -Decided (2026-10-04): - -- Layout, as OBS does: `ftyp`, a reserved `free` box, the fragmented moov, - then moof and mdat pairs. On finish, write a full moov at the end and - overwrite the `free` header with an mdat header covering everything before - it. While recording, the reserved box must be `free`: an `mdat` header - there would swallow the moov and fragments, and a crashed file would be - unreadable. Moov-first (faststart) would cost a full copy and is not - planned. -- Renditions that join after the fragmented moov are written into `free` - boxes, which crash-time readers skip, and indexed only by the final moov. -- The final sample tables come from parsing the written moofs back, which - needs no API change and also enables a repair command for crashed files. -- A new `mp4` format that requires `--output` (it must seek); `export fmp4` - stays a stream to stdout. - -- Finishing is required, not best effort: the first SIGINT or SIGTERM - finalizes the file, which needs a graceful-finish path in `moq-cli` (today - SIGINT drops the task set). A second signal aborts at once and leaves the - crash-safe fragmented file. -- No size limit: always write co64 chunk offsets and a 64-bit (largesize) mdat - header. - -Tests: SIGINT during a recording produces a regular MP4 that parses with its -moov at the end; a second SIGINT leaves a playable fragmented file. - -Open for the PR: edit lists or stretched durations for gaps and track start -offsets; whether a returning rendition with a new config gets an extra sample -description. diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 516d1820468..b2dd7caa09a 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -136,6 +136,8 @@ serde = { workspace = true } serde_json = { workspace = true } terminal_size = "0.4.4" tokio = { workspace = true, features = ["full"] } +# Tells a recording to finish its file on the first shutdown signal. +tokio-util = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } url = { workspace = true } diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 049e40ec9e8..4509fe65563 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -31,6 +31,7 @@ //! the separator stays unconditional rather than context-sensitive. use std::ffi::{OsStr, OsString}; +use std::path::PathBuf; use std::time::Duration; use crate::publish::PublishFormat; @@ -383,12 +384,10 @@ impl Invocation { ); } if export.echo.is_some() { - if export.sink.stdout().is_none() + if export.sink.container().is_none() && let Some(sink) = export.sink.ignores_selection() { - anyhow::bail!( - "--echo reports what a stdout container export reads, and `export {sink}` is not one" - ); + anyhow::bail!("--echo reports what a container export reads, and `export {sink}` is not one"); } let msf = match export.catalog_format { Some(format) => matches!(format, CatalogFormatArg::Msf), @@ -402,7 +401,15 @@ impl Invocation { "--echo follows the hang catalog's `echo` section, which the MSF catalog does not carry" ); } - if let Some(stdout) = export.sink.stdout() { + if let Some(output) = export.sink.container().and_then(|container| container.output) { + // Refused before anything dials, rather than once the broadcast arrives. + anyhow::ensure!( + !output.exists(), + "--output {} already exists; pick a new file", + output.display() + ); + } + if let Some(stdout) = export.sink.container() { anyhow::ensure!( stdout.follow.is_zero() || matches!(stdout.format, SubscribeFormat::Ts), "--linger is gone, and following a broadcast's return needs an output that can mark the restart: only `export ts --follow` can" @@ -1065,6 +1072,7 @@ impl Export { } match &self.sink { ExportSink::Fmp4(args) => found.extend(args.container.deprecated()), + ExportSink::Mp4(args) => found.extend(args.fragmented.container.deprecated()), ExportSink::Mkv(args) => found.extend(args.container.deprecated()), ExportSink::Ts(args) => found.extend(args.deprecated()), ExportSink::Flv(args) => found.extend(args.deprecated()), @@ -1078,11 +1086,13 @@ impl Export { } /// The single sink draining the Origin on an export. The container formats write -/// to stdout; the gateways bridge another protocol. +/// to stdout, except `mp4`, which records to a file; the gateways bridge another protocol. #[derive(usage_rs::Subcommands, Clone)] pub enum ExportSink { /// Fragmented MP4 / CMAF to stdout. Fmp4(Fragmented), + /// Regular MP4 recorded to a file, which plays as fragmented MP4 if the recording is cut short. + Mp4(Recorded), /// Matroska / WebM to stdout. Mkv(BufferedFragmented), /// MPEG-TS to stdout. @@ -1113,7 +1123,7 @@ impl ExportSink { /// The sink's name when it doesn't apply selection, so the selection flags would be ignored. fn ignores_selection(&self) -> Option<&'static str> { Some(match self { - Self::Fmp4(_) | Self::Mkv(_) | Self::Flv(_) | Self::H264(_) | Self::H265(_) => return None, + Self::Fmp4(_) | Self::Mp4(_) | Self::Mkv(_) | Self::Flv(_) | Self::H264(_) | Self::H265(_) => return None, Self::Ts(_) => "ts", Self::Hls(_) => "hls", Self::Rtmp(_) => "rtmp", @@ -1134,15 +1144,15 @@ impl ExportSink { } } - /// Whether this sink writes to stdout (the container formats and the report sinks). + /// Whether this sink writes to stdout (the container formats but `mp4`, and the report sinks). pub fn is_stdout(&self) -> bool { - self.stdout().is_some() || self.report().is_some() + self.container().is_some_and(|container| container.output.is_none()) || self.report().is_some() } - /// The stdout container format and its options, when this sink writes to - /// stdout. The fragment cap is fmp4/mkv-only and the mux rate is TS-only. - pub fn stdout(&self) -> Option { - let container = |format, container: &Container| Stdout { + /// The container format and its options, when this sink is one. The fragment cap is + /// fmp4/mp4/mkv-only and the mux rate is TS-only. + pub fn container(&self) -> Option { + let container = |format, container: &Container| ContainerSink { format, max_delay: container.max_delay.into(), follow: container.linger.into(), @@ -1150,8 +1160,9 @@ impl ExportSink { fragment_duration: None, mux_rate: None, passthrough: false, + output: None, }; - let buffered = |format, args: &Buffered| Stdout { + let buffered = |format, args: &Buffered| ContainerSink { format, max_delay: args.delay.into(), follow: args.linger.into(), @@ -1159,17 +1170,23 @@ impl ExportSink { fragment_duration: None, mux_rate: None, passthrough: false, + output: None, }; Some(match self { - Self::Fmp4(args) => Stdout { + Self::Fmp4(args) => ContainerSink { fragment_duration: args.fragment_duration.map(Into::into), ..container(SubscribeFormat::Fmp4, &args.container) }, - Self::Mkv(args) => Stdout { + Self::Mp4(args) => ContainerSink { + fragment_duration: args.fragmented.fragment_duration.map(Into::into), + output: Some(args.output.clone()), + ..container(SubscribeFormat::Mp4, &args.fragmented.container) + }, + Self::Mkv(args) => ContainerSink { fragment_duration: args.fragment_duration.map(Into::into), ..buffered(SubscribeFormat::Mkv, &args.container) }, - Self::Ts(args) => Stdout { + Self::Ts(args) => ContainerSink { format: SubscribeFormat::Ts, max_delay: args.delay.into(), follow: args.follow.into(), @@ -1177,6 +1194,7 @@ impl ExportSink { fragment_duration: None, mux_rate: args.mux_rate, passthrough: args.passthrough, + output: None, }, Self::Flv(args) => buffered(SubscribeFormat::Flv, args), Self::H264(args) => container(SubscribeFormat::H264, args), @@ -1186,8 +1204,8 @@ impl ExportSink { } } -/// A stdout sink's format and the options that apply to it. -pub struct Stdout { +/// A container sink's format and the options that apply to it. +pub struct ContainerSink { pub format: SubscribeFormat, /// The staleness budget, also the FLV/MKV release delay and how far behind the source `ts` /// runs (`--delay`). @@ -1201,6 +1219,8 @@ pub struct Stdout { pub mux_rate: Option, /// Write the `m2ts` passthrough track back out (`ts` only). pub passthrough: bool, + /// The file a recording writes (`mp4` only); the rest write to stdout. + pub output: Option, } /// Staleness options for stdout sinks without a fixed release delay. @@ -1350,6 +1370,20 @@ impl Transport { } } +/// A regular MP4 recording. +#[derive(usage_rs::Args, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct Recorded { + #[usage(flatten)] + pub fragmented: Fragmented, + + /// The file to record to, which must not exist yet. It plays as fragmented MP4 while + /// recording; the broadcast's end or the first SIGINT/SIGTERM finishes it as a regular + /// MP4, and a second signal exits at once, leaving it fragmented. + #[usage(long, value_name = "FILE", value_hint = usage_rs::ValueHint::FilePath)] + pub output: PathBuf, +} + /// The fMP4 stdout container with a fragment cap. #[derive(usage_rs::Args, Clone)] #[usage(unknown_flags = "error", args_override_self = false)] @@ -1424,7 +1458,7 @@ mod tests { let Command::Export(export) = &cli.stages[0] else { panic!("not an export"); }; - export.sink.stdout().unwrap().max_delay + export.sink.container().unwrap().max_delay }; assert_eq!(stdout(&[]), Duration::from_secs(1)); assert_eq!(stdout(&["--delay", "2s"]), Duration::from_secs(2)); @@ -1607,7 +1641,7 @@ mod tests { let Command::Export(export) = &cli.stages[0] else { panic!("not an export"); }; - let stdout = export.sink.stdout().unwrap(); + let stdout = export.sink.container().unwrap(); assert!(stdout.passthrough); assert_eq!(stdout.max_delay, Duration::from_secs(2)); for flags in [&["--stitch"][..], &["--follow", "10s"], &["--mux-rate", "20000000"]] { @@ -2448,7 +2482,7 @@ mod tests { let Command::Export(export) = &cli.stages[0] else { panic!("expected export") }; - export.sink.stdout().unwrap().max_delay + export.sink.container().unwrap().max_delay }; for format in ["fmp4", "h264", "h265"] { diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 43808bc97a1..cb05aaf2672 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -34,6 +34,7 @@ use subscribe::{Subscribe, SubscribeArgs}; use anyhow::Context; use moq_relay::shutdown::Signals; use tokio::task::JoinSet; +use tokio_util::sync::CancellationToken; #[cfg(feature = "jemalloc")] #[global_allocator] @@ -282,6 +283,41 @@ struct Shutdown { observer: moq_relay::shutdown::Observer, } +/// The recordings, which finish their files on shutdown instead of being dropped with the +/// other stages. +#[derive(Default)] +struct Recordings { + /// Cancelled to tell every recording to finish its file. + finish: CancellationToken, + tasks: JoinSet>, +} + +impl Recordings { + /// Finish every recording and wait for them, or `None` when a signal cuts that short and + /// leaves the unfinished files fragmented. + async fn finish(&mut self, signals: &mut Signals) -> Option> { + self.finish.cancel(); + if !self.tasks.is_empty() { + tracing::info!("finishing recordings (signal again to exit immediately)"); + } + let mut result = Ok(()); + loop { + tokio::select! { + biased; + () = signals.recv() => { + tracing::warn!("second shutdown signal; leaving recordings unfinished"); + self.tasks.abort_all(); + return None; + } + joined = self.tasks.join_next() => match joined { + Some(joined) => result = result.and(joined.map_err(anyhow::Error::from).and_then(|ended| ended)), + None => return Some(result), + }, + } + } + } +} + /// Keep all stages and listener tasks alive while admitted sessions drain, the way /// the relay does, or until a second signal. async fn drain(shutdown: Shutdown, signals: &mut Signals) { @@ -544,6 +580,7 @@ async fn run_stages(moq: MoqSide, stages: Vec, net: Net) -> anyhow::Res // The stdin/capture pipelines run on this thread instead of the JoinSet: the // platform capture stream is not Send, so their futures cannot be spawned. let mut locals: Vec = Vec::new(); + let mut recordings = Recordings::default(); // Before anything listens: a client can count its session established as soon // as the listener is up, and a signal then must drain it, not kill the process. @@ -611,17 +648,19 @@ async fn run_stages(moq: MoqSide, stages: Vec, net: Net) -> anyhow::Res claim("stdout", &mut stdout, &name)?; } let monitor = connection.as_ref().map(moq_tokio::Connection::monitor); - spawn_export(&origin, export, name, monitor, &mut tasks)?; + spawn_export(&origin, export, name, monitor, &mut tasks, &mut recordings)?; } other => unreachable!("`{}` is not a stage", other.name()), } } if locals.is_empty() { - drive(&mut tasks, signals, shutdown).await + drive(&mut tasks, &mut recordings, signals, shutdown).await } else { supervise(&local, locals.into_iter().map(Publish::run), &mut tasks); - local.run_until(drive(&mut tasks, signals, shutdown)).await + local + .run_until(drive(&mut tasks, &mut recordings, signals, shutdown)) + .await } } .await; @@ -820,13 +859,15 @@ fn spawn_import( } /// Route the shared Origin OUT to one stage's sink, filling it from the MoQ network. -/// `connection` supplies an `--echo` report's transport measurements. +/// `connection` supplies an `--echo` report's transport measurements. A recording to a file +/// joins `recordings`, the rest `tasks`. fn spawn_export( origin: &moq_net::origin::Producer, export: Export, name: String, connection: Option, tasks: &mut JoinSet>, + recordings: &mut Recordings, ) -> anyhow::Result<()> { if let ExportSink::Rtc(rtc) = &export.sink && rtc.connect.is_some() @@ -834,15 +875,20 @@ fn spawn_export( reject_listener_cors(&rtc.cors, "export rtc")?; } - if let Some(stdout) = export.sink.stdout() { + if let Some(container) = export.sink.container() { + let (tasks, finish) = match container.output { + Some(_) => (&mut recordings.tasks, recordings.finish.clone()), + None => (tasks, CancellationToken::new()), + }; let args = SubscribeArgs { - format: stdout.format, - max_delay: stdout.max_delay, - follow: stdout.follow, - stitch: stdout.stitch, - fragment_duration: stdout.fragment_duration, - mux_rate: stdout.mux_rate, - passthrough: stdout.passthrough, + format: container.format, + max_delay: container.max_delay, + follow: container.follow, + stitch: container.stitch, + fragment_duration: container.fragment_duration, + mux_rate: container.mux_rate, + passthrough: container.passthrough, + output: container.output, catalog: export.catalog_format, select: export.select, }; @@ -852,7 +898,7 @@ fn spawn_export( name, connection, }); - tasks.spawn(async move { run_stdout(consumer, name, args, echo).await }); + tasks.spawn(async move { run_container(consumer, name, args, echo, finish).await }); } else { match export.sink { ExportSink::Hls(args) => { @@ -902,32 +948,34 @@ fn spawn_export( ExportSink::Echo => { tasks.spawn(report::print_echo(origin.consume(), name)); } - _ => unreachable!("container formats are handled by stdout_format above"), + _ => unreachable!("container formats are handled by container() above"), } } Ok(()) } -/// Subscribe to `name` from the Origin and write it to stdout, publishing feedback beside it -/// where the catalog solicits it when `echo` is set. -async fn run_stdout( +/// Subscribe to `name` from the Origin and write it to stdout or record it, publishing feedback +/// beside it where the catalog solicits it when `echo` is set. A recording returns once +/// `finish` is cancelled, after finishing its file. +async fn run_container( consumer: moq_net::origin::Consumer, name: String, args: SubscribeArgs, echo: Option, + finish: CancellationToken, ) -> anyhow::Result<()> { let catalog = args.catalog_format(&name); // Wait for the broadcast to be announced; the export resolves any sibling broadcast a // rendition's `broadcast` field references (e.g. "./source") through the same origin. - let broadcast = consumer - .follow(name.as_str(), None)? - .available() - .await - .with_context(|| format!("failed to resolve broadcast `{name}`"))?; + let mut follow = consumer.follow(name.as_str(), None)?; + let broadcast = tokio::select! { + broadcast = follow.available() => broadcast.with_context(|| format!("failed to resolve broadcast `{name}`"))?, + () = finish.cancelled() => return Ok(()), + }; - let subscribe = Subscribe::new(moq_mux::Source::new(consumer, broadcast), catalog, args); + let subscribe = Subscribe::new(moq_mux::Source::new(consumer, broadcast), catalog, args).with_finish(finish); let Some(echo) = echo else { return subscribe.run().await; }; @@ -943,29 +991,46 @@ async fn run_stdout( /// Run until an endpoint ends, keeping all tasks alive while a signalled listener drains. /// -/// The tasks are left for the caller to shut down once the MoQ side is detached. +/// Either way the recordings finish their files before this returns. The tasks are left for +/// the caller to shut down once the MoQ side is detached. async fn drive( tasks: &mut JoinSet>, + recordings: &mut Recordings, mut signals: Signals, shutdown: Option, ) -> anyhow::Result<()> { - loop { + let result = loop { + if tasks.is_empty() && recordings.tasks.is_empty() { + break Ok(()); + } tokio::select! { biased; () = signals.recv() => { moq::stop(); + // Before the drain, so a second signal cutting the drain short can't cost a + // recording its index. + let Some(finished) = recordings.finish(&mut signals).await else { + return Ok(()); + }; if let Some(shutdown) = shutdown { drain(shutdown, &mut signals).await; } - break Ok(()); + return finished; } - result = tasks.join_next() => match result { - Some(Ok(result)) => break result, - Some(Err(err)) if err.is_cancelled() => continue, - Some(Err(err)) => break Err(err.into()), - None => break Ok(()), + Some(result) = tasks.join_next() => match result { + Ok(result) => break result, + Err(err) if err.is_cancelled() => continue, + Err(err) => break Err(err.into()), + }, + Some(result) = recordings.tasks.join_next() => { + break result.map_err(anyhow::Error::from).and_then(|ended| ended); } } + }; + // The process is ending, so the other recordings finish their files too. + match recordings.finish(&mut signals).await { + Some(finished) => result.and(finished), + None => result, } } @@ -1045,13 +1110,72 @@ mod tests { trigger, observer: observer.clone(), }; - let driving = tokio::spawn(async move { drive(&mut tasks, signals, Some(shutdown)).await }); + let driving = + tokio::spawn(async move { drive(&mut tasks, &mut Recordings::default(), signals, Some(shutdown)).await }); observer.started().await; assert!(!driving.is_finished(), "drain ended under an established session"); drop(session); driving.await.unwrap().unwrap(); } + /// A recording that sets `finished` once told to finish. + fn recording(recordings: &mut Recordings) -> std::sync::Arc { + let finished = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let finish = recordings.finish.clone(); + let done = finished.clone(); + recordings.tasks.spawn(async move { + finish.cancelled().await; + done.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }); + finished + } + + /// The first signal finishes every recording's file before the process exits. + #[cfg(unix)] + #[tokio::test(start_paused = true)] + async fn a_signal_finishes_recordings() { + let _signals = SIGNALS.lock().await; + let signals = Signals::new().unwrap(); + let mut recordings = Recordings::default(); + let first = recording(&mut recordings); + let second = recording(&mut recordings); + let mut tasks = JoinSet::new(); + tasks.spawn(std::future::pending()); + // SAFETY: raise has no memory-safety preconditions. + assert_eq!(unsafe { libc::raise(libc::SIGINT) }, 0); + drive(&mut tasks, &mut recordings, signals, None).await.unwrap(); + assert!(first.load(std::sync::atomic::Ordering::SeqCst)); + assert!(second.load(std::sync::atomic::Ordering::SeqCst)); + } + + /// Another stage ending the process still lets the recordings finish. + #[tokio::test(start_paused = true)] + async fn an_ending_stage_finishes_recordings() { + let _signals = SIGNALS.lock().await; + let mut recordings = Recordings::default(); + let finished = recording(&mut recordings); + let mut tasks = JoinSet::new(); + tasks.spawn(async { Ok(()) }); + drive(&mut tasks, &mut recordings, Signals::new().unwrap(), None) + .await + .unwrap(); + assert!(finished.load(std::sync::atomic::Ordering::SeqCst)); + } + + /// A second signal exits at once, leaving a recording that is still finishing as it is. + #[cfg(unix)] + #[tokio::test(start_paused = true)] + async fn a_second_signal_leaves_recordings_unfinished() { + let _signals = SIGNALS.lock().await; + let mut signals = Signals::new().unwrap(); + let mut recordings = Recordings::default(); + recordings.tasks.spawn(std::future::pending()); + // SAFETY: raise has no memory-safety preconditions. + assert_eq!(unsafe { libc::raise(libc::SIGTERM) }, 0); + assert!(recordings.finish(&mut signals).await.is_none()); + } + #[tokio::test(start_paused = true)] async fn drain_bounds_a_pending_admission() { let _signals = SIGNALS.lock().await; @@ -1307,7 +1431,12 @@ mod tests { supervise(&local, pipelines, &mut tasks); let err = local - .run_until(drive(&mut tasks, Signals::new().unwrap(), None)) + .run_until(drive( + &mut tasks, + &mut Recordings::default(), + Signals::new().unwrap(), + None, + )) .await .unwrap_err(); assert!(err.to_string().contains("pipeline panicked"), "{err}"); @@ -1324,7 +1453,12 @@ mod tests { supervise(&local, pipelines, &mut tasks); local - .run_until(drive(&mut tasks, Signals::new().unwrap(), None)) + .run_until(drive( + &mut tasks, + &mut Recordings::default(), + Signals::new().unwrap(), + None, + )) .await .unwrap(); } @@ -1607,7 +1741,15 @@ mod tests { }; let origin = moq_tokio::origin::spawn(); let mut tasks = JoinSet::new(); - spawn_export(&origin, export, String::new(), None, &mut tasks).expect("the root is a broadcast"); + spawn_export( + &origin, + export, + String::new(), + None, + &mut tasks, + &mut Recordings::default(), + ) + .expect("the root is a broadcast"); tasks.abort_all(); } } diff --git a/rs/moq-cli/src/report/echo_test.rs b/rs/moq-cli/src/report/echo_test.rs index c6eb3a0a2c1..3a1861918a3 100644 --- a/rs/moq-cli/src/report/echo_test.rs +++ b/rs/moq-cli/src/report/echo_test.rs @@ -40,6 +40,7 @@ async fn export_echo_reports_where_the_import_solicits() { fragment_duration: None, mux_rate: None, passthrough: false, + output: None, catalog: None, select: SelectArgs::default(), }; diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 39518500dee..6734fdb1830 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -1,15 +1,19 @@ use std::time::Duration; +use anyhow::Context; use hang::catalog::{AudioCodecKind, VideoCodecKind}; use moq_mux::catalog::{self, CatalogFormat, Stream}; use moq_mux::select; -use tokio::io::AsyncWriteExt; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; +use tokio_util::sync::CancellationToken; -/// Container format written to stdout on the export (sink) side. +/// Container format written on the export (sink) side. #[derive(Clone, Copy)] pub enum SubscribeFormat { /// Fragmented MP4 (CMAF). Fmp4, + /// Regular MP4, recorded to a file. + Mp4, /// Matroska / WebM. Mkv, /// H.264 Annex-B elementary stream (no container). @@ -166,10 +170,10 @@ impl SelectArgs { } } -/// The resolved stdout export settings (built from the `export` flags + format). +/// The resolved container export settings (built from the `export` flags + format). #[derive(Clone)] pub struct SubscribeArgs { - /// The format to write to stdout. + /// The format to write. pub format: SubscribeFormat, /// How far playback may drift from the live edge before skipping groups. TS also runs @@ -193,6 +197,9 @@ pub struct SubscribeArgs { /// Write the `m2ts` passthrough track back out instead of muxing the media (TS only). pub passthrough: bool, + /// The file to record to (MP4 only); the other formats write to stdout. + pub output: Option, + /// Catalog format for track discovery (default: detect from the broadcast suffix). pub catalog: Option, @@ -216,7 +223,11 @@ impl SubscribeArgs { match self.format { SubscribeFormat::H264 => Some(VideoCodecKind::H264), SubscribeFormat::H265 => Some(VideoCodecKind::H265), - SubscribeFormat::Fmp4 | SubscribeFormat::Mkv | SubscribeFormat::Ts | SubscribeFormat::Flv => None, + SubscribeFormat::Fmp4 + | SubscribeFormat::Mp4 + | SubscribeFormat::Mkv + | SubscribeFormat::Ts + | SubscribeFormat::Flv => None, } } @@ -242,13 +253,15 @@ impl SubscribeArgs { } } -/// Exports one broadcast from the Origin to stdout in the requested format. +/// Exports one broadcast from the Origin in the requested format, to stdout or a recording. pub struct Subscribe { source: moq_mux::Source, catalog: CatalogFormat, args: SubscribeArgs, /// Where a TS export sets its MPEG-TS sections, when it reports them. sections: Option, + /// Cancelled to make a recording finish its file and return. + finish: CancellationToken, } /// Log how a TS export's release clock keeps up with the source. @@ -274,9 +287,16 @@ impl Subscribe { catalog, args, sections: None, + finish: CancellationToken::new(), } } + /// Finish a recording's file and return once `finish` is cancelled. + pub fn with_finish(mut self, finish: CancellationToken) -> Self { + self.finish = finish; + self + } + /// The resolved broadcast, whose [`stats`](moq_mux::Source::stats) count what the export reads. pub fn source(&self) -> &moq_mux::Source { &self.source @@ -295,10 +315,11 @@ impl Subscribe { Ok(consumer.select(self.args.selection()?)) } - /// Write the broadcast to stdout until it ends. + /// Write the broadcast until it ends. pub async fn run(self) -> anyhow::Result<()> { match self.args.format { SubscribeFormat::Fmp4 => self.run_fmp4().await, + SubscribeFormat::Mp4 => self.run_mp4().await, SubscribeFormat::Mkv => self.run_mkv().await, SubscribeFormat::H264 => self.run_h264().await, SubscribeFormat::H265 => self.run_h265().await, @@ -325,6 +346,53 @@ impl Subscribe { Ok(()) } + /// Record a regular MP4 until the broadcast ends or `finish` is cancelled, then finish it. + async fn run_mp4(self) -> anyhow::Result<()> { + let path = self.args.output.clone().context("an mp4 export records to --output")?; + let finish = self.finish.clone(); + let stream = tokio::select! { + stream = self.stream() => stream?, + () = finish.cancelled() => return Ok(()), + }; + let mut mp4 = moq_mux::container::mp4::Export::new(self.source, stream) + .with_max_delay(self.args.max_delay) + .with_fragment_duration(self.args.fragment_duration); + let mut file = tokio::fs::File::create_new(&path) + .await + .with_context(|| format!("failed to create {}", path.display()))?; + + // A failed export still finishes what it recorded; a failed write leaves the file as is. + let ended = loop { + let next = tokio::select! { + next = mp4.next() => next, + () = finish.cancelled() => break Ok(()), + }; + match next { + Ok(Some(data)) => file.write_all(&data).await?, + Ok(None) => break Ok(()), + Err(err) => break Err(err), + } + }; + + match mp4.finish()? { + Some(trailer) => { + file.write_all(&trailer.moov).await?; + // The header points readers past the media at the moov, so the moov must be on disk first. + file.sync_data().await?; + file.seek(std::io::SeekFrom::Start(trailer.mdat_offset)).await?; + file.write_all(&trailer.mdat).await?; + file.sync_all().await?; + tracing::info!(path = %path.display(), "recording finished"); + } + // Nothing reached the file, so there is no recording to keep. + None => { + drop(file); + tokio::fs::remove_file(&path).await?; + } + } + Ok(ended?) + } + async fn run_mkv(self) -> anyhow::Result<()> { let mut stdout = tokio::io::stdout(); diff --git a/rs/moq-cli/tests/integration/export.rs b/rs/moq-cli/tests/integration/export.rs index 975f5ed7304..b9aaf418cdd 100644 --- a/rs/moq-cli/tests/integration/export.rs +++ b/rs/moq-cli/tests/integration/export.rs @@ -14,6 +14,7 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use hang::moq_net; +use moq_mux::container::fmp4::mp4_atom; use moq_mux::container::ts; use mpeg2ts::ts::{ReadTsPacket, TsPacketReader, TsPayload}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; @@ -1093,3 +1094,68 @@ async fn a_passthrough_export_ends_with_its_redundant_importer() { second.kill().await.unwrap(); feeding.abort(); } + +/// The top-level boxes of an MP4 file, by kind. +fn boxes(file: &[u8]) -> Vec { + use mp4_atom::DecodeMaybe; + let mut cursor = Cursor::new(file); + let mut boxes = Vec::new(); + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor).expect("decode the recording") { + boxes.push(atom); + } + boxes +} + +/// SIGINT during `export mp4` finishes the recording as a regular MP4 with its moov at the end. +#[tokio::test] +#[ignore = "real-time process test: just rs cli-export"] +async fn an_interrupted_mp4_recording_is_finished() { + let relay = relay().await; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("recording.mp4"); + let mut recording = moq(&relay, &[]) + .args(["export", "mp4", "--output"]) + .arg(&path) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn export"); + let (mut publisher, feeding) = import(&relay); + + tokio::time::timeout(TIMEOUT, async { + while std::fs::metadata(&path).map_or(0, |file| file.len()) < 10_000 { + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("the recording never grew"); + let pid = recording.id().expect("running").to_string(); + let status = std::process::Command::new("kill") + .args(["-INT", &pid]) + .status() + .expect("kill"); + assert!(status.success()); + let status = wait(&mut recording).await; + let mut errors = String::new(); + recording + .stderr + .take() + .unwrap() + .read_to_string(&mut errors) + .await + .unwrap(); + assert!(status.success(), "{errors}"); + + let file = std::fs::read(&path).unwrap(); + let boxes = boxes(&file); + let kinds: Vec = boxes.iter().map(|atom| atom.kind().to_string()).collect(); + assert_eq!(kinds, ["ftyp", "mdat", "moov"], "{errors}"); + let Some(mp4_atom::Any::Moov(moov)) = boxes.last() else { + unreachable!(); + }; + assert!(moov.mvex.is_none()); + assert_eq!(moov.trak.len(), 2, "video and audio"); + + let stdin = feeding.await.unwrap(); + drop(stdin); + assert!(wait(&mut publisher).await.success()); +} diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index 35b87c73ed7..5bd22ba0dd7 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -78,6 +78,10 @@ pub struct Export { /// Set after the init segment has been emitted, which fixes the track set. init_emitted: bool, + /// Keep a rendition that joins after the init segment instead of failing, for a writer + /// that declares it elsewhere. Its fragments carry a track id the init never declared. + late: bool, + /// The `tkhd.track_id` the next new rendition gets. next_track_id: u32, @@ -483,6 +487,7 @@ impl Export { tracks: HashMap::new(), queued: VecDeque::new(), init_emitted: false, + late: false, next_track_id: 1, sequence_number: 1, } @@ -516,6 +521,22 @@ impl Export { self } + /// Keep renditions that join after the init segment, whose fragments then carry track + /// ids the init does not declare. See [`declaration`](Self::declaration). + pub(crate) fn with_late_tracks(mut self) -> Self { + self.late = true; + self + } + + /// The `trak` and `trex` declaring `track_id`, once the track could be described. + pub(crate) fn declaration(&self, track_id: u32) -> Option<(&mp4_atom::Trak, &mp4_atom::Trex)> { + self.tracks + .values() + .filter(|track| track.track_id == track_id) + .find_map(|track| track.declaration.as_ref()) + .map(|declaration| (&declaration.trak, &declaration.trex)) + } + /// Get the next byte chunk. /// /// The first call returns the merged init segment (ftyp + multi-track moov); each @@ -864,7 +885,7 @@ impl Export { let Some(source) = config.subscribe(&self.source, &name, self.max_delay)? else { continue; }; - if self.init_emitted { + if self.init_emitted && !self.late { return Err(Error::TrackAdded(name).into()); } let mut track = Fmp4Track::new(source, config, self.next_track_id)?; diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 11b55d2afa6..6de7cacdbf1 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -316,6 +316,15 @@ pub enum Error { /// A track's edit list does more than delay the track and start its media at one time. #[error("unsupported edit list: {0}")] UnsupportedEditList(&'static str), + + /// A fragment written to a regular MP4 can't be indexed in its final `moov`. + #[error("track {track} can't be indexed: {reason}")] + Unindexable { + /// The fragment's track id. + track: u32, + /// What the sample tables can't express. + reason: &'static str, + }, } impl From for Error { @@ -1119,9 +1128,9 @@ pub(crate) fn encode_init( // dOps alone describes decoder priming; ISO BMFF needs an edit to express that // presentation shift. Rebuild it after the output track timescales are settled. for trak in &mut traks { - if let [mp4_atom::Codec::Opus(opus)] = trak.mdia.minf.stbl.stsd.codecs.as_slice() { - let skip = u64::from(opus.dops.pre_skip) * u64::from(trak.mdia.mdhd.timescale) / 48_000; - trak.edts = (skip != 0).then(|| mp4_atom::Edts { + let skip = priming(trak); + if skip != 0 { + trak.edts = Some(mp4_atom::Edts { elst: Some(mp4_atom::Elst { entries: vec![mp4_atom::ElstEntry { // A live edit's duration is implicit in the fragments that follow. @@ -1162,6 +1171,14 @@ pub(crate) fn encode_init( Ok(Bytes::from(buf)) } +/// An Opus track's pre-skip in its media timescale, or zero for any other track. +pub(crate) fn priming(trak: &mp4_atom::Trak) -> u64 { + match trak.mdia.minf.stbl.stsd.codecs.as_slice() { + [mp4_atom::Codec::Opus(opus)] => u64::from(opus.dops.pre_skip) * u64::from(trak.mdia.mdhd.timescale) / 48_000, + _ => 0, + } +} + /// Narrow a media timescale to the 32-bit `mdhd` field, rejecting what would truncate. /// /// `moq_net::Timescale` permits the whole QUIC varint range, so a caller-supplied scale can be diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 2c11f751514..2de73f7a31f 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -28,6 +28,7 @@ pub mod fmp4; pub mod legacy; pub mod loc; pub mod mkv; +pub mod mp4; pub mod ts; pub use consumer::{Consumer, DEFAULT_MAX_BUFFER}; diff --git a/rs/moq-mux/src/container/mp4.rs b/rs/moq-mux/src/container/mp4.rs new file mode 100644 index 00000000000..53f40c16219 --- /dev/null +++ b/rs/moq-mux/src/container/mp4.rs @@ -0,0 +1,740 @@ +//! Regular MP4 files, recorded crash-safe. +//! +//! [`Export`] writes a file that is a valid fragmented MP4 while it grows, so a crash still +//! leaves a playable file, and [`Export::finish`] turns it into a regular MP4 with its `moov` +//! at the end without moving any media, the way OBS's hybrid MP4 does. +//! +//! The file starts with `ftyp`, a reserved 16-byte `free` box and the fragmented `moov`, then +//! the moof+mdat fragments. Finishing appends a `moov` that indexes every sample where it +//! already lies, then overwrites the reserved box with a 64-bit `mdat` header covering +//! everything before that `moov`. The reserved box must stay `free` until then: an `mdat` +//! header there would swallow the fragmented `moov` and leave a crashed file unreadable. +//! +//! A rendition that joins after the fragmented `moov` is written into `free` boxes, which a +//! reader of the unfinished file skips, and only the final `moov` indexes it. + +use std::collections::{BTreeMap, HashSet}; +use std::task::Poll; +use std::time::Duration; + +use bytes::{BufMut, Bytes, BytesMut}; +use mp4_atom::{Decode, Encode}; + +use crate::Result; +use crate::catalog::Stream; +use crate::container::fmp4::{self, Chunk, Error}; + +/// The reserved box: the size of a 64-bit `mdat` header. +const RESERVED: u64 = 16; + +/// The `sample_is_non_sync_sample` bit of ISO/IEC 14496-12 sample flags. +const NON_SYNC: u32 = 0x0001_0000; + +/// Subscribe to a moq broadcast and record it as a crash-safe regular MP4. +/// +/// [`next`](Self::next) returns the bytes to append to the file, the same fMP4 an +/// [`fmp4::Export`] writes plus the reserved box, so the file plays if the recording never +/// finishes. [`finish`](Self::finish) then returns the trailing `moov` and the `mdat` header +/// that make it a regular MP4. Writing those needs a file that can be overwritten in place, +/// not a stream. +/// +/// Unlike [`fmp4::Export`], a rendition that joins after the fragmented `moov` is recorded. +pub struct Export { + fmp4: fmp4::Export, + + /// Bytes returned so far, which is where the next chunk starts in the file. + written: u64, + + /// The fragmented `moov`, once written. + header: Option
, + + /// What each track wrote, by track id. + tracks: BTreeMap, +} + +/// What the fragmented `moov` fixed. +struct Header { + /// Where the reserved box starts. + reserved: u64, + /// The movie timescale. + timescale: u32, + /// The tracks it declares. Any other track joined later. + declared: HashSet, +} + +/// What one track wrote, in decode order. +#[derive(Default)] +struct Track { + /// One chunk per `trun`, its samples contiguous in the file. + chunks: Vec, + samples: Vec, +} + +struct Run { + /// Where the first sample starts in the file. + offset: u64, + /// The first sample's decode time, in media ticks. + decode_time: u64, + /// How many samples the run holds. + count: u32, +} + +struct Sample { + size: u32, + duration: u32, + cts: i32, + sync: bool, +} + +/// The writes that finish a recording, in order. +#[derive(Clone, Debug)] +pub struct Finish { + /// The regular `moov`, appended after everything [`Export::next`] returned. + pub moov: Bytes, + + /// Where [`mdat`](Self::mdat) overwrites the reserved box, from the start of the file. + pub mdat_offset: u64, + + /// The 64-bit `mdat` header that covers the media, written last. + pub mdat: Bytes, +} + +impl Export { + /// Subscribe to `source` and record it, driving track (un)subscription from `catalog`. + pub fn new(source: crate::Source, catalog: S) -> Self { + Self { + fmp4: fmp4::Export::new(source, catalog).with_late_tracks(), + written: 0, + header: None, + tracks: BTreeMap::new(), + } + } + + /// Set the max delay for each per-track source; see [`fmp4::Export::with_max_delay`]. + pub fn with_max_delay(mut self, max_delay: Duration) -> Self { + self.fmp4 = self.fmp4.with_max_delay(max_delay); + self + } + + /// Cap the fragment duration; see [`fmp4::Export::with_fragment_duration`]. + /// + /// A fragment is what a crash can lose, besides being the unit a `moof` indexes. + pub fn with_fragment_duration(mut self, duration: impl Into>) -> Self { + self.fmp4 = self.fmp4.with_fragment_duration(duration); + self + } + + /// The next bytes to append to the file, or `None` once the broadcast ends. + pub async fn next(&mut self) -> Result> { + kio::wait(|waiter| self.poll_next(waiter)).await + } + + /// Poll-based variant of [`Self::next`]. + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>> { + let Some(chunk) = std::task::ready!(self.fmp4.poll_next_chunk(waiter)?) else { + return Poll::Ready(Ok(None)); + }; + let data = match chunk { + Chunk::Init(init) => self.init(&init)?, + Chunk::Fragment(fragment) => self.fragment(fragment.data)?, + }; + self.written += data.len() as u64; + Poll::Ready(Ok(Some(data))) + } + + /// The writes that turn the file into a regular MP4, or `None` when nothing was written. + /// + /// Media still buffered for a fragment that has not closed is left out. + pub fn finish(self) -> Result> { + let Some(header) = &self.header else { + return Ok(None); + }; + + let mut timelines = Vec::new(); + for (&id, track) in self.tracks.iter().filter(|(_, track)| !track.samples.is_empty()) { + let Some((trak, _)) = self.fmp4.declaration(id) else { + return Err(unindexable(id, "the track has no declaration").into()); + }; + timelines.push((trak, track, Timeline::new(id, trak, track)?)); + } + + let movie = u128::from(header.timescale); + let to_movie = |ticks: u64, timescale: u32| { + u64::try_from(u128::from(ticks) * movie / u128::from(timescale)).map_err(|_| Error::PtsOverflow) + }; + // The earliest track starts the movie; the rest start that much later. + let mut origin = u64::MAX; + for (trak, _, timeline) in &timelines { + origin = origin.min(to_movie(timeline.start, trak.mdia.mdhd.timescale)?); + } + + let mut traks = Vec::new(); + for (trak, track, timeline) in timelines { + let timescale = trak.mdia.mdhd.timescale; + let delay = to_movie(timeline.start, timescale)? - origin; + let presented = to_movie(timeline.duration, timescale)?; + let mut trak = trak.clone(); + trak.mdia.mdhd.duration = timeline.deltas.iter().map(|&delta| u64::from(delta)).sum(); + trak.tkhd.duration = delay + presented; + trak.edts = edits(delay, timeline.media_time, presented); + trak.mdia.minf.stbl = sample_table(&trak.mdia.minf.stbl.stsd, track, &timeline.deltas); + traks.push(trak); + } + traks.sort_by_key(|trak| (trak.mdia.hdlr.handler != b"vide".into(), trak.tkhd.track_id)); + + let moov = mp4_atom::Moov { + mvhd: mp4_atom::Mvhd { + timescale: header.timescale, + duration: traks.iter().map(|trak| trak.tkhd.duration).max().unwrap_or(0), + // Normal playback rate and full volume; both default to 0. + rate: mp4_atom::FixedPoint::from(1), + volume: mp4_atom::FixedPoint::from(1), + next_track_id: traks.iter().map(|trak| trak.tkhd.track_id).max().unwrap_or(0) + 1, + ..Default::default() + }, + trak: traks, + ..Default::default() + }; + let mut buf = Vec::new(); + moov.encode(&mut buf)?; + + // Everything from the reserved box to the moov, the fragmented moov and every moof + // included, becomes one mdat that a reader skips through to the moov. + let mut mdat = BytesMut::with_capacity(RESERVED as usize); + mdat.put_u32(1); + mdat.put_slice(b"mdat"); + mdat.put_u64(self.written - header.reserved); + + Ok(Some(Finish { + moov: Bytes::from(buf), + mdat_offset: header.reserved, + mdat: mdat.freeze(), + })) + } + + /// The file header: the init segment's `ftyp`, the reserved box, then its `moov`. + fn init(&mut self, init: &Bytes) -> Result { + let mut cursor = std::io::Cursor::new(init.as_ref()); + mp4_atom::Ftyp::decode(&mut cursor)?; + let split = cursor.position() as usize; + let moov = mp4_atom::Moov::decode(&mut cursor)?; + + let mut data = BytesMut::with_capacity(init.len() + RESERVED as usize); + data.put_slice(&init[..split]); + data.put_u32(RESERVED as u32); + data.put_slice(b"free"); + data.put_u64(0); + data.put_slice(&init[split..]); + + self.header = Some(Header { + reserved: self.written + split as u64, + timescale: moov.mvhd.timescale, + declared: moov.trak.iter().map(|trak| trak.tkhd.track_id).collect(), + }); + Ok(data.freeze()) + } + + /// Index a moof+mdat fragment where it will land, wrapping a late track's in a `free` box. + fn fragment(&mut self, fragment: Bytes) -> Result { + let header = self.header.as_ref().expect("the init segment comes first"); + let moof = mp4_atom::Moof::decode(&mut std::io::Cursor::new(fragment.as_ref()))?; + + let mut data = BytesMut::new(); + if moof + .traf + .iter() + .any(|traf| !header.declared.contains(&traf.tfhd.track_id)) + { + mp4_atom::Header { + kind: b"free".into(), + size: Some(fragment.len()), + } + .encode(&mut data)?; + } + let start = self.written + data.len() as u64; + for traf in &moof.traf { + self.index(start, traf)?; + } + data.put_slice(&fragment); + Ok(data.freeze()) + } + + /// Record the samples of `traf`, whose moof starts at `moof` in the file. + fn index(&mut self, moof: u64, traf: &mp4_atom::Traf) -> Result<()> { + let id = traf.tfhd.track_id; + let Some((_, trex)) = self.fmp4.declaration(id) else { + return Err(unindexable(id, "the track has no declaration").into()); + }; + let Some(tfdt) = &traf.tfdt else { + return Err(unindexable(id, "a fragment has no decode time").into()); + }; + // The data offsets are then relative to the moof, as fmp4::Export writes them. + if traf.tfhd.base_data_offset.is_some() { + return Err(unindexable(id, "a fragment has an explicit base data offset").into()); + } + let duration = traf + .tfhd + .default_sample_duration + .unwrap_or(trex.default_sample_duration); + let size = traf.tfhd.default_sample_size.unwrap_or(trex.default_sample_size); + let flags = traf.tfhd.default_sample_flags.unwrap_or(trex.default_sample_flags); + + let track = self.tracks.entry(id).or_default(); + let mut decode_time = tfdt.base_media_decode_time; + for trun in traf.trun.iter().filter(|trun| !trun.entries.is_empty()) { + let Some(offset) = trun.data_offset.and_then(|offset| u64::try_from(offset).ok()) else { + return Err(unindexable(id, "a run has no data offset").into()); + }; + track.chunks.push(Run { + offset: moof + offset, + decode_time, + count: trun.entries.len() as u32, + }); + for entry in &trun.entries { + let sample = Sample { + size: entry.size.unwrap_or(size), + duration: entry.duration.unwrap_or(duration), + cts: entry.cts.unwrap_or(0), + sync: entry.flags.unwrap_or(flags) & NON_SYNC == 0, + }; + decode_time = decode_time + .checked_add(u64::from(sample.duration)) + .ok_or(Error::PtsOverflow)?; + track.samples.push(sample); + } + } + Ok(()) + } +} + +fn unindexable(track: u32, reason: &'static str) -> Error { + Error::Unindexable { track, reason } +} + +/// A track's samples laid on one media timeline. +struct Timeline { + /// Each sample's `stts` delta: up to the next sample's decode time, which stretches a + /// sample over any gap after it, and the last sample's own duration. + deltas: Vec, + /// Where presentation starts on the media timeline, past the first decode time. + media_time: u64, + /// When presentation starts, in the media ticks the fragments were written in. + start: u64, + /// How long the track presents, in media ticks. + duration: u64, +} + +impl Timeline { + fn new(id: u32, trak: &mp4_atom::Trak, track: &Track) -> Result { + let mut decode_times = Vec::with_capacity(track.samples.len()); + let mut samples = track.samples.iter(); + for run in &track.chunks { + let mut time = run.decode_time; + for sample in samples.by_ref().take(run.count as usize) { + decode_times.push(time); + time += u64::from(sample.duration); + } + } + + let mut deltas = Vec::with_capacity(decode_times.len()); + for (i, &time) in decode_times.iter().enumerate() { + let delta = match decode_times.get(i + 1) { + Some(&next) if next <= time => return Err(unindexable(id, "decode time stops advancing").into()), + Some(&next) => { + u32::try_from(next - time).map_err(|_| unindexable(id, "a gap overflows a sample delta"))? + } + None => track.samples[i].duration, + }; + deltas.push(delta); + } + + let Some(&first) = decode_times.first() else { + return Ok(Self { + deltas, + media_time: 0, + start: 0, + duration: 0, + }); + }; + let mut presented = i128::MAX; + let mut end = i128::MIN; + for ((&time, sample), &delta) in decode_times.iter().zip(&track.samples).zip(&deltas) { + let pts = i128::from(time) + i128::from(sample.cts); + presented = presented.min(pts); + end = end.max(pts + i128::from(delta)); + } + // An Opus track presents one pre-skip into its first packet, and nothing is presented + // before the first decode time. + let start = (presented + i128::from(fmp4::priming(trak))).max(i128::from(first)); + let start = u64::try_from(start).map_err(|_| Error::PtsOverflow)?; + let duration = u64::try_from((end - i128::from(start)).max(0)).map_err(|_| Error::PtsOverflow)?; + Ok(Self { + deltas, + media_time: start - first, + start, + duration, + }) + } +} + +/// The edit list placing a track `delay` into the movie and its media from `media_time`, both +/// lasting `presented`; `None` when the media plays from its start at the movie's. +fn edits(delay: u64, media_time: u64, presented: u64) -> Option { + if delay == 0 && media_time == 0 { + return None; + } + let mut entries = Vec::new(); + if delay > 0 { + entries.push(mp4_atom::ElstEntry { + segment_duration: delay, + media_time: None, + media_rate: mp4_atom::FixedPoint::from(1), + }); + } + entries.push(mp4_atom::ElstEntry { + segment_duration: presented, + media_time: Some(media_time), + media_rate: mp4_atom::FixedPoint::from(1), + }); + Some(mp4_atom::Edts { + elst: Some(mp4_atom::Elst { entries }), + }) +} + +/// The sample tables indexing `track` where its samples lie, each `trun` one chunk. +fn sample_table(stsd: &mp4_atom::Stsd, track: &Track, deltas: &[u32]) -> mp4_atom::Stbl { + let mut stts: Vec = Vec::new(); + for &delta in deltas { + match stts.last_mut() { + Some(last) if last.sample_delta == delta => last.sample_count += 1, + _ => stts.push(mp4_atom::SttsEntry { + sample_count: 1, + sample_delta: delta, + }), + } + } + + let mut ctts: Vec = Vec::new(); + for sample in &track.samples { + let offset = i64::from(sample.cts); + match ctts.last_mut() { + Some(last) if last.sample_offset == offset => last.sample_count += 1, + _ => ctts.push(mp4_atom::CttsEntry { + sample_count: 1, + sample_offset: offset, + }), + } + } + + let mut stsc: Vec = Vec::new(); + for (i, run) in track.chunks.iter().enumerate() { + if stsc.last().is_none_or(|last| last.samples_per_chunk != run.count) { + stsc.push(mp4_atom::StscEntry { + first_chunk: i as u32 + 1, + samples_per_chunk: run.count, + sample_description_index: 1, + }); + } + } + + let sizes: Vec = track.samples.iter().map(|sample| sample.size).collect(); + let stsz = match sizes.first() { + Some(&size) if sizes.iter().all(|&other| other == size) => mp4_atom::StszSamples::Identical { + count: sizes.len() as u32, + size, + }, + _ => mp4_atom::StszSamples::Different { sizes }, + }; + + let all_sync = track.samples.iter().all(|sample| sample.sync); + mp4_atom::Stbl { + stsd: stsd.clone(), + stts: mp4_atom::Stts { entries: stts }, + ctts: ctts + .iter() + .any(|entry| entry.sample_offset != 0) + .then_some(mp4_atom::Ctts { entries: ctts }), + stss: (!all_sync).then(|| mp4_atom::Stss { + entries: (1..) + .zip(&track.samples) + .filter(|(_, sample)| sample.sync) + .map(|(number, _)| number) + .collect(), + }), + stsc: mp4_atom::Stsc { entries: stsc }, + stsz: mp4_atom::Stsz { samples: stsz }, + co64: Some(mp4_atom::Co64 { + entries: track.chunks.iter().map(|run| run.offset).collect(), + }), + ..Default::default() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use mp4_atom::DecodeMaybe; + + use crate::container::test_util::{Live, raw_frame, video_frame}; + + /// Every retained group, since these tests publish before they record. + const MAX_DELAY: Duration = Duration::from_secs(30); + + /// A 20 ms Opus packet (TOC config 1, one frame). + const OPUS: &[u8] = &[0x08, 0xaa, 0xbb, 0xcc]; + + type Producer = crate::container::Producer; + + fn add_opus(live: &mut Live) -> Producer { + use hang::catalog::{AudioCodec, AudioConfig, Container}; + + live.add_track(".opus", |catalog, name| { + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + config.container = Container::Legacy; + catalog.modify().unwrap().audio.renditions.insert(name, config); + }) + } + + async fn export(live: &Live) -> super::Export { + super::Export::new(live.source().await, live.catalog_stream().await).with_max_delay(MAX_DELAY) + } + + /// Append everything the export writes without waiting for more input. + async fn record(export: &mut super::Export, file: &mut Vec) { + while let Ok(next) = tokio::time::timeout(Duration::from_millis(1), export.next()).await { + match next.expect("export failed") { + Some(data) => file.extend_from_slice(&data), + None => break, + } + } + } + + /// Finish the recording in `file` the way a writer does. + fn finish(export: super::Export, file: &mut Vec) { + let finish = export.finish().expect("finish").expect("something was written"); + file.extend_from_slice(&finish.moov); + let offset = finish.mdat_offset as usize; + file[offset..offset + finish.mdat.len()].copy_from_slice(&finish.mdat); + } + + fn atoms(file: &[u8]) -> Vec { + let mut cursor = std::io::Cursor::new(file); + let mut atoms = Vec::new(); + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor).expect("decode file") { + atoms.push(atom); + } + atoms + } + + fn kinds(atoms: &[mp4_atom::Any]) -> Vec { + atoms.iter().map(|atom| atom.kind().to_string()).collect() + } + + fn moov(atoms: Vec) -> mp4_atom::Moov { + atoms + .into_iter() + .find_map(|atom| match atom { + mp4_atom::Any::Moov(moov) => Some(moov), + _ => None, + }) + .expect("a moov") + } + + fn trak<'a>(moov: &'a mp4_atom::Moov, handler: &[u8; 4]) -> &'a mp4_atom::Trak { + moov.trak + .iter() + .find(|trak| trak.mdia.hdlr.handler == handler.into()) + .expect("a trak") + } + + /// Each sample's bytes, found through the sample tables alone. + fn samples<'a>(file: &'a [u8], trak: &mp4_atom::Trak) -> Vec<&'a [u8]> { + let stbl = &trak.mdia.minf.stbl; + let sizes: Vec = match &stbl.stsz.samples { + mp4_atom::StszSamples::Identical { count, size } => vec![*size; *count as usize], + mp4_atom::StszSamples::Different { sizes } => sizes.clone(), + }; + let offsets = &stbl.co64.as_ref().expect("co64").entries; + let mut sizes = sizes.into_iter(); + let mut samples = Vec::new(); + for (index, &offset) in offsets.iter().enumerate() { + let chunk = index as u32 + 1; + let entry = stbl + .stsc + .entries + .iter() + .rev() + .find(|entry| entry.first_chunk <= chunk) + .expect("stsc"); + let mut offset = offset as usize; + for size in sizes.by_ref().take(entry.samples_per_chunk as usize) { + samples.push(&file[offset..offset + size as usize]); + offset += size as usize; + } + } + assert!(sizes.next().is_none(), "every sample is in a chunk"); + samples + } + + /// An Annex-B frame's NALs with 4-byte lengths for start codes, as the export writes it. + fn length_prefixed(frame: &crate::container::Frame) -> Vec { + let payload = &frame.payload[..]; + let starts: Vec = (0..payload.len().saturating_sub(3)) + .filter(|&i| payload[i..i + 4] == [0, 0, 0, 1]) + .collect(); + let mut out = Vec::new(); + for (i, &start) in starts.iter().enumerate() { + let end = starts.get(i + 1).copied().unwrap_or(payload.len()); + let nal = &payload[start + 4..end]; + out.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + out.extend_from_slice(nal); + } + out + } + + /// A recording that runs to its end is fragmented until finished, and a regular MP4 + /// indexing every sample in place after. + #[tokio::test(start_paused = true)] + async fn a_finished_recording_is_a_regular_mp4() { + let mut live = Live::avc3(); + let mut audio = add_opus(&mut live); + let video: Vec<_> = (0..9u64).map(|i| video_frame(i * 33_000, i % 3 == 0)).collect(); + for frame in &video { + live.track.write(frame.clone()).unwrap(); + } + for i in 0..12u64 { + audio.write(raw_frame(i * 20_000, OPUS, i % 5 == 0)).unwrap(); + } + live.track.finish().unwrap(); + audio.finish().unwrap(); + + let mut export = export(&live).await; + let mut file = Vec::new(); + record(&mut export, &mut file).await; + + // Unfinished, it is the fMP4 a crash would leave. + let unfinished = atoms(&file); + assert_eq!(kinds(&unfinished[..3]), ["ftyp", "free", "moov"]); + assert!( + kinds(&unfinished[3..]) + .iter() + .all(|kind| kind == "moof" || kind == "mdat") + ); + assert!(moov(unfinished).mvex.is_some()); + + let media = file.len(); + finish(export, &mut file); + let finished = atoms(&file); + assert_eq!(kinds(&finished), ["ftyp", "mdat", "moov"]); + let moov = moov(finished); + assert!(moov.mvex.is_none(), "a regular MP4 has no fragments to extend into"); + + let video_trak = trak(&moov, b"vide"); + let written: Vec> = video.iter().map(length_prefixed).collect(); + assert_eq!(samples(&file, video_trak), written); + let stbl = &video_trak.mdia.minf.stbl; + assert_eq!(stbl.stss.as_ref().expect("stss").entries, [1, 4, 7]); + assert_eq!(stbl.stsc.entries.len(), 1, "every GOP is a chunk of three"); + assert!( + stbl.co64 + .as_ref() + .unwrap() + .entries + .iter() + .all(|&offset| (offset as usize) < media) + ); + + let audio_trak = trak(&moov, b"soun"); + assert_eq!(samples(&file, audio_trak), vec![OPUS; 12]); + assert!( + audio_trak.mdia.minf.stbl.stss.is_none(), + "every audio sample is a sync sample" + ); + } + + /// A rendition that joins after the fragmented moov hides in free boxes until the final + /// moov indexes it. + #[tokio::test(start_paused = true)] + async fn a_late_rendition_is_recorded() { + let mut live = Live::avc3(); + live.track.write(video_frame(0, true)).unwrap(); + let mut export = export(&live).await; + let mut file = Vec::new(); + record(&mut export, &mut file).await; + assert_eq!(moov(atoms(&file)).trak.len(), 1); + + let mut audio = add_opus(&mut live); + for i in 0..10u64 { + audio.write(raw_frame(i * 20_000, OPUS, i % 5 == 0)).unwrap(); + } + audio.finish().unwrap(); + live.track.write(video_frame(200_000, true)).unwrap(); + live.track.finish().unwrap(); + record(&mut export, &mut file).await; + + let unfinished = atoms(&file); + let frees = unfinished + .iter() + .filter(|atom| matches!(atom, mp4_atom::Any::Free(_))) + .count(); + assert_eq!(frees, 3, "the reserved box and both audio fragments"); + + finish(export, &mut file); + let moov = moov(atoms(&file)); + assert_eq!(moov.trak.len(), 2); + assert_eq!(samples(&file, trak(&moov, b"soun")), vec![OPUS; 10]); + assert_eq!(samples(&file, trak(&moov, b"vide")).len(), 2); + } + + /// A track that starts later waits behind an empty edit, and a gap stretches the sample + /// before it rather than pulling later samples early. + #[tokio::test(start_paused = true)] + async fn later_starts_and_gaps_keep_their_time() { + let mut live = Live::avc3(); + let mut audio = add_opus(&mut live); + for i in 0..10u64 { + audio.write(raw_frame(i * 20_000, OPUS, i % 5 == 0)).unwrap(); + } + audio.finish().unwrap(); + // Video starts 100 ms in, and its second GOP comes a second after the first. + for (timestamp, keyframe) in [(100_000, true), (133_000, false), (1_133_000, true), (1_166_000, false)] { + live.track.write(video_frame(timestamp, keyframe)).unwrap(); + } + live.track.finish().unwrap(); + + let mut export = export(&live).await; + let mut file = Vec::new(); + record(&mut export, &mut file).await; + finish(export, &mut file); + let moov = moov(atoms(&file)); + + let video = trak(&moov, b"vide"); + let movie = u64::from(moov.mvhd.timescale); + let edits = &video.edts.as_ref().expect("edts").elst.as_ref().expect("elst").entries; + assert_eq!(edits[0].media_time, None, "an empty edit first"); + assert_eq!(edits[0].segment_duration, movie / 10); + assert_eq!(edits[1].media_time, Some(0)); + + let timescale = u64::from(video.mdia.mdhd.timescale); + let deltas: Vec = video + .mdia + .minf + .stbl + .stts + .entries + .iter() + .flat_map(|entry| std::iter::repeat_n(u64::from(entry.sample_delta), entry.sample_count as usize)) + .collect(); + assert_eq!(deltas.len(), 4); + assert_eq!(deltas[1], timescale, "the frame before the gap lasts through it"); + } + + /// Nothing to finish before the header is written. + #[tokio::test(start_paused = true)] + async fn nothing_written_needs_no_finish() { + let live = Live::avc3(); + let export = export(&live).await; + assert!(export.finish().unwrap().is_none()); + } +} From 60fa3370fdd5925a2171f93f810207a6e2d13ab9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 07:09:19 -0700 Subject: [PATCH 3/7] fix(mux): index reordered video cut shorter than a GOP on one decode clock Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/mp4.rs | 87 ++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 12 deletions(-) diff --git a/rs/moq-mux/src/container/mp4.rs b/rs/moq-mux/src/container/mp4.rs index 53f40c16219..a39a21248da 100644 --- a/rs/moq-mux/src/container/mp4.rs +++ b/rs/moq-mux/src/container/mp4.rs @@ -177,7 +177,7 @@ impl Export { trak.mdia.mdhd.duration = timeline.deltas.iter().map(|&delta| u64::from(delta)).sum(); trak.tkhd.duration = delay + presented; trak.edts = edits(delay, timeline.media_time, presented); - trak.mdia.minf.stbl = sample_table(&trak.mdia.minf.stbl.stsd, track, &timeline.deltas); + trak.mdia.minf.stbl = sample_table(&trak.mdia.minf.stbl.stsd, track, &timeline); traks.push(trak); } traks.sort_by_key(|trak| (trak.mdia.hdlr.handler != b"vide".into(), trak.tkhd.track_id)); @@ -316,6 +316,8 @@ struct Timeline { /// Each sample's `stts` delta: up to the next sample's decode time, which stretches a /// sample over any gap after it, and the last sample's own duration. deltas: Vec, + /// Each sample's composition offset from its decode time on this timeline. + offsets: Vec, /// Where presentation starts on the media timeline, past the first decode time. media_time: u64, /// When presentation starts, in the media ticks the fragments were written in. @@ -326,31 +328,47 @@ struct Timeline { impl Timeline { fn new(id: u32, trak: &mp4_atom::Trak, track: &Track) -> Result { - let mut decode_times = Vec::with_capacity(track.samples.len()); + // Each fragment's decode time is its first sample's presentation time, so fragments + // shorter than a GOP of reordered video step back and forth. Decode on one clock that + // a sync sample moves ahead to its fragment's time, which keeps a real gap, and keep + // every presentation time by recomputing the composition offsets against it. + let mut decode_times: Vec = Vec::with_capacity(track.samples.len()); + let mut presentations: Vec = Vec::with_capacity(track.samples.len()); + let mut clock: Option = None; let mut samples = track.samples.iter(); for run in &track.chunks { - let mut time = run.decode_time; + let mut written = run.decode_time; for sample in samples.by_ref().take(run.count as usize) { + presentations.push(i128::from(written) + i128::from(sample.cts)); + let time = match clock { + Some(clock) if sample.sync => clock.max(written), + Some(clock) => clock, + None => written, + }; decode_times.push(time); - time += u64::from(sample.duration); + clock = Some(time.checked_add(u64::from(sample.duration)).ok_or(Error::PtsOverflow)?); + written += u64::from(sample.duration); } } let mut deltas = Vec::with_capacity(decode_times.len()); + let mut offsets = Vec::with_capacity(decode_times.len()); for (i, &time) in decode_times.iter().enumerate() { let delta = match decode_times.get(i + 1) { - Some(&next) if next <= time => return Err(unindexable(id, "decode time stops advancing").into()), Some(&next) => { u32::try_from(next - time).map_err(|_| unindexable(id, "a gap overflows a sample delta"))? } None => track.samples[i].duration, }; deltas.push(delta); + let offset = presentations[i] - i128::from(time); + offsets.push(i32::try_from(offset).map_err(|_| unindexable(id, "a composition offset overflows 32 bits"))?); } let Some(&first) = decode_times.first() else { return Ok(Self { deltas, + offsets, media_time: 0, start: 0, duration: 0, @@ -358,10 +376,9 @@ impl Timeline { }; let mut presented = i128::MAX; let mut end = i128::MIN; - for ((&time, sample), &delta) in decode_times.iter().zip(&track.samples).zip(&deltas) { - let pts = i128::from(time) + i128::from(sample.cts); + for (&pts, sample) in presentations.iter().zip(&track.samples) { presented = presented.min(pts); - end = end.max(pts + i128::from(delta)); + end = end.max(pts + i128::from(sample.duration)); } // An Opus track presents one pre-skip into its first packet, and nothing is presented // before the first decode time. @@ -370,6 +387,7 @@ impl Timeline { let duration = u64::try_from((end - i128::from(start)).max(0)).map_err(|_| Error::PtsOverflow)?; Ok(Self { deltas, + offsets, media_time: start - first, start, duration, @@ -402,9 +420,9 @@ fn edits(delay: u64, media_time: u64, presented: u64) -> Option } /// The sample tables indexing `track` where its samples lie, each `trun` one chunk. -fn sample_table(stsd: &mp4_atom::Stsd, track: &Track, deltas: &[u32]) -> mp4_atom::Stbl { +fn sample_table(stsd: &mp4_atom::Stsd, track: &Track, timeline: &Timeline) -> mp4_atom::Stbl { let mut stts: Vec = Vec::new(); - for &delta in deltas { + for &delta in &timeline.deltas { match stts.last_mut() { Some(last) if last.sample_delta == delta => last.sample_count += 1, _ => stts.push(mp4_atom::SttsEntry { @@ -415,8 +433,8 @@ fn sample_table(stsd: &mp4_atom::Stsd, track: &Track, deltas: &[u32]) -> mp4_ato } let mut ctts: Vec = Vec::new(); - for sample in &track.samples { - let offset = i64::from(sample.cts); + for &offset in &timeline.offsets { + let offset = i64::from(offset); match ctts.last_mut() { Some(last) if last.sample_offset == offset => last.sample_count += 1, _ => ctts.push(mp4_atom::CttsEntry { @@ -730,6 +748,51 @@ mod tests { assert_eq!(deltas[1], timescale, "the frame before the gap lasts through it"); } + /// Fragments shorter than a GOP of reordered video each start their decode time at their + /// own presentation time, which steps backwards; the index still decodes in order and + /// presents every frame at its time. + #[tokio::test(start_paused = true)] + async fn reordered_video_in_short_fragments_keeps_its_times() { + let mut live = Live::avc3(); + let presentations: [i64; 5] = [0, 120_000, 40_000, 80_000, 160_000]; + for (i, ×tamp) in presentations.iter().enumerate() { + live.track.write(video_frame(timestamp as u64, i % 4 == 0)).unwrap(); + } + live.track.finish().unwrap(); + + let mut export = export(&live).await.with_fragment_duration(Duration::ZERO); + let mut file = Vec::new(); + record(&mut export, &mut file).await; + finish(export, &mut file); + let moov = moov(atoms(&file)); + let video = trak(&moov, b"vide"); + let stbl = &video.mdia.minf.stbl; + + let deltas = stbl + .stts + .entries + .iter() + .flat_map(|entry| std::iter::repeat_n(i64::from(entry.sample_delta), entry.sample_count as usize)); + let offsets: Vec = match &stbl.ctts { + Some(ctts) => ctts + .entries + .iter() + .flat_map(|entry| std::iter::repeat_n(entry.sample_offset, entry.sample_count as usize)) + .collect(), + None => vec![0; presentations.len()], + }; + let mut decode = 0; + let mut presented = Vec::new(); + for (delta, offset) in deltas.zip(offsets) { + assert!(delta > 0, "decode time advances"); + presented.push(decode + offset); + decode += delta; + } + let timescale = i64::from(video.mdia.mdhd.timescale); + let expected: Vec = presentations.iter().map(|&us| us * timescale / 1_000_000).collect(); + assert_eq!(presented, expected); + } + /// Nothing to finish before the header is written. #[tokio::test(start_paused = true)] async fn nothing_written_needs_no_finish() { From b7efdd7db425541790967fc05e03f5f7a40b266f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 00:17:54 -0700 Subject: [PATCH 4/7] fix(mux): place Opus at its Hang time and drop drained departed renditions The finished moov placed an Opus track one pre-skip late: the edit's media time counts the priming, but the movie position is the Hang time. A late rendition that leaves is moved out of the polled track set once written out, and dropped when it wrote nothing, so a publisher cycling rendition names holds only what was recorded. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/fmp4/export.rs | 45 +++++++++++--- rs/moq-mux/src/container/mp4.rs | 82 +++++++++++++++++++++---- 2 files changed, 107 insertions(+), 20 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index 5bd22ba0dd7..c1ec519686f 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -67,10 +67,18 @@ pub struct Export { max_delay: Duration, fragment_duration: Option, - /// Every track by rendition name. Before the init a rendition that leaves the catalog - /// is dropped; after it, the track stays so a returning rendition reuses its id. + /// The tracks being read, by rendition name. Before the init a rendition that leaves the + /// catalog is dropped; after it, the track moves to `departed` once written out. tracks: HashMap, + /// Renditions that left the catalog after the init, kept out of the polled set so one + /// returning reuses its track. A late rendition that wrote nothing is dropped instead, + /// so a publisher cycling names holds only what it recorded. + departed: HashMap, + + /// The first track id the init segment does not declare. + first_late_id: u32, + /// Fragments cut before the init segment, written right after it in the order they /// were cut. queued: VecDeque<(String, Fragment)>, @@ -485,6 +493,8 @@ impl Export { max_delay: Duration::ZERO, fragment_duration: None, tracks: HashMap::new(), + departed: HashMap::new(), + first_late_id: u32::MAX, queued: VecDeque::new(), init_emitted: false, late: false, @@ -530,11 +540,17 @@ impl Export { /// The `trak` and `trex` declaring `track_id`, once the track could be described. pub(crate) fn declaration(&self, track_id: u32) -> Option<(&mp4_atom::Trak, &mp4_atom::Trex)> { - self.tracks - .values() - .filter(|track| track.track_id == track_id) - .find_map(|track| track.declaration.as_ref()) - .map(|declaration| (&declaration.trak, &declaration.trex)) + self.declarations() + .find(|(id, _, _)| *id == track_id) + .map(|(_, trak, trex)| (trak, trex)) + } + + /// Every track id with its `trak` and `trex`, the tracks being read first. + pub(crate) fn declarations(&self) -> impl Iterator { + self.tracks.values().chain(self.departed.values()).filter_map(|track| { + let declaration = track.declaration.as_ref()?; + Some((track.track_id, &declaration.trak, &declaration.trex)) + }) } /// Get the next byte chunk. @@ -647,6 +663,7 @@ impl Export { if self.init_ready() { let init = self.build_init()?; self.init_emitted = true; + self.first_late_id = self.next_track_id; return Poll::Ready(Ok(Some(Chunk::Init(init)))); } if self.catalog.is_none() && self.tracks.values().all(|t| t.finished) { @@ -848,6 +865,11 @@ impl Export { if active.contains(&name) { continue; } + if !self.tracks.contains_key(&name) + && let Some(track) = self.departed.remove(&name) + { + self.tracks.insert(name.clone(), track); + } let moved = self .tracks .get(&name) @@ -909,6 +931,15 @@ impl Export { track.described = false; } } + // A departed track still holding media waits for the next catalog to move. + let drained = self + .tracks + .extract_if(|_, track| track.source.is_none() && track.pending.is_none() && track.buffer.is_empty()); + for (name, track) in drained { + if track.written.is_some() || track.track_id < self.first_late_id { + self.departed.insert(name, track); + } + } } else { self.tracks.retain(|name, _| active.contains(name)); self.queued.retain(|(name, _)| active.contains(name)); diff --git a/rs/moq-mux/src/container/mp4.rs b/rs/moq-mux/src/container/mp4.rs index a39a21248da..293e4bfc8e0 100644 --- a/rs/moq-mux/src/container/mp4.rs +++ b/rs/moq-mux/src/container/mp4.rs @@ -13,7 +13,7 @@ //! A rendition that joins after the fragmented `moov` is written into `free` boxes, which a //! reader of the unfinished file skips, and only the final `moov` indexes it. -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::task::Poll; use std::time::Duration; @@ -150,9 +150,11 @@ impl Export { return Ok(None); }; + let declarations: HashMap = + self.fmp4.declarations().map(|(id, trak, _)| (id, trak)).collect(); let mut timelines = Vec::new(); for (&id, track) in self.tracks.iter().filter(|(_, track)| !track.samples.is_empty()) { - let Some((trak, _)) = self.fmp4.declaration(id) else { + let Some(&trak) = declarations.get(&id) else { return Err(unindexable(id, "the track has no declaration").into()); }; timelines.push((trak, track, Timeline::new(id, trak, track)?)); @@ -320,7 +322,8 @@ struct Timeline { offsets: Vec, /// Where presentation starts on the media timeline, past the first decode time. media_time: u64, - /// When presentation starts, in the media ticks the fragments were written in. + /// The Hang time presentation starts at, in media ticks, which places the track in the + /// movie. An Opus track's media runs one pre-skip ahead of it. start: u64, /// How long the track presents, in media ticks. duration: u64, @@ -382,14 +385,15 @@ impl Timeline { } // An Opus track presents one pre-skip into its first packet, and nothing is presented // before the first decode time. - let start = (presented + i128::from(fmp4::priming(trak))).max(i128::from(first)); - let start = u64::try_from(start).map_err(|_| Error::PtsOverflow)?; - let duration = u64::try_from((end - i128::from(start)).max(0)).map_err(|_| Error::PtsOverflow)?; + let priming = fmp4::priming(trak); + let media = (presented + i128::from(priming)).max(i128::from(first)); + let media = u64::try_from(media).map_err(|_| Error::PtsOverflow)?; + let duration = u64::try_from((end - i128::from(media)).max(0)).map_err(|_| Error::PtsOverflow)?; Ok(Self { deltas, offsets, - media_time: start - first, - start, + media_time: media - first, + start: media.checked_sub(priming).ok_or(Error::PtsOverflow)?, duration, }) } @@ -504,12 +508,19 @@ mod tests { type Producer = crate::container::Producer; - fn add_opus(live: &mut Live) -> Producer { + /// A 48 kHz Opus rendition whose OpusHead declares `pre_skip`. + fn add_opus(live: &mut Live, pre_skip: u16) -> Producer { use hang::catalog::{AudioCodec, AudioConfig, Container}; live.add_track(".opus", |catalog, name| { let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 2); config.container = Container::Legacy; + config.description = Some( + crate::codec::opus::Config::new(48_000, 2) + .with_pre_skip(pre_skip) + .encode() + .unwrap(), + ); catalog.modify().unwrap().audio.renditions.insert(name, config); }) } @@ -616,7 +627,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn a_finished_recording_is_a_regular_mp4() { let mut live = Live::avc3(); - let mut audio = add_opus(&mut live); + let mut audio = add_opus(&mut live, 0); let video: Vec<_> = (0..9u64).map(|i| video_frame(i * 33_000, i % 3 == 0)).collect(); for frame in &video { live.track.write(frame.clone()).unwrap(); @@ -682,7 +693,7 @@ mod tests { record(&mut export, &mut file).await; assert_eq!(moov(atoms(&file)).trak.len(), 1); - let mut audio = add_opus(&mut live); + let mut audio = add_opus(&mut live, 0); for i in 0..10u64 { audio.write(raw_frame(i * 20_000, OPUS, i % 5 == 0)).unwrap(); } @@ -705,12 +716,51 @@ mod tests { assert_eq!(samples(&file, trak(&moov, b"vide")).len(), 2); } + /// A late rendition that leaves without writing anything is forgotten, so cycling names + /// holds nothing; one that wrote stays for the final moov. + #[tokio::test(start_paused = true)] + async fn departed_late_renditions_keep_only_what_they_wrote() { + let mut live = Live::avc3(); + live.track.write(video_frame(0, true)).unwrap(); + let mut export = export(&live).await; + let mut file = Vec::new(); + record(&mut export, &mut file).await; + + for _ in 0..3 { + let audio = add_opus(&mut live, 0); + record(&mut export, &mut file).await; + let name = audio.name().to_string(); + live.catalog.modify().unwrap().audio.renditions.remove(&name); + record(&mut export, &mut file).await; + } + assert_eq!(export.fmp4.declarations().count(), 1, "only the video is held"); + + let mut audio = add_opus(&mut live, 0); + record(&mut export, &mut file).await; + for i in 0..5u64 { + audio.write(raw_frame(i * 20_000, OPUS, i == 0)).unwrap(); + } + audio.finish().unwrap(); + record(&mut export, &mut file).await; + let name = audio.name().to_string(); + live.catalog.modify().unwrap().audio.renditions.remove(&name); + live.track.write(video_frame(200_000, true)).unwrap(); + live.track.finish().unwrap(); + record(&mut export, &mut file).await; + assert_eq!(export.fmp4.declarations().count(), 2); + + finish(export, &mut file); + let moov = moov(atoms(&file)); + assert_eq!(samples(&file, trak(&moov, b"soun")), vec![OPUS; 5]); + } + /// A track that starts later waits behind an empty edit, and a gap stretches the sample - /// before it rather than pulling later samples early. + /// before it rather than pulling later samples early. Opus starts at its Hang time, its + /// edit skipping the pre-skip in its media. #[tokio::test(start_paused = true)] async fn later_starts_and_gaps_keep_their_time() { let mut live = Live::avc3(); - let mut audio = add_opus(&mut live); + let mut audio = add_opus(&mut live, 312); for i in 0..10u64 { audio.write(raw_frame(i * 20_000, OPUS, i % 5 == 0)).unwrap(); } @@ -734,6 +784,12 @@ mod tests { assert_eq!(edits[0].segment_duration, movie / 10); assert_eq!(edits[1].media_time, Some(0)); + let audio = trak(&moov, b"soun"); + let skip = 312 * u64::from(audio.mdia.mdhd.timescale) / 48_000; + let edits = &audio.edts.as_ref().expect("edts").elst.as_ref().expect("elst").entries; + assert_eq!(edits.len(), 1, "audio starts the movie"); + assert_eq!(edits[0].media_time, Some(skip)); + let timescale = u64::from(video.mdia.mdhd.timescale); let deltas: Vec = video .mdia From 9fb0bbb6a7e301329f80abfd946398ea8a7f42d6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 00:57:19 -0700 Subject: [PATCH 5/7] quest: unblock the mp4 follow-ups this PR finishes mp4-repair and mp4-sigint-flush required quest/m2/mp4-export.md, which this PR deletes on completion. Co-Authored-By: Claude Opus 5.5 --- quest/m2/mp4-repair.md | 4 ---- quest/m2/mp4-sigint-flush.md | 4 ---- 2 files changed, 8 deletions(-) diff --git a/quest/m2/mp4-repair.md b/quest/m2/mp4-repair.md index 555ddfb351b..ce5ff24bf2f 100644 --- a/quest/m2/mp4-repair.md +++ b/quest/m2/mp4-repair.md @@ -11,7 +11,3 @@ Follow-up of [moq-dev/moq#7357](https://github.com/moq-dev/moq/pull/7357). Its f Decided 2026-10-30 (maintainer, follow-up planning after the /quest-afk run). Late renditions are written inside `free` boxes that only the final `moov` indexes, so they're unrecoverable today: the recorder must also write each late rendition's track description into the file at the time it joins. Test a file cut at several points (mid-fragment, after a late rendition joins) and check the repaired file decodes like a clean finish of the same data. Public API: a CLI subcommand or flag (pick the shape in the PR and list it). Wire: none. - -## Required - -- [moq export mp4](/quest/m2/mp4-export.md) - its PR, #7357, adds the recorder and the moof indexer diff --git a/quest/m2/mp4-sigint-flush.md b/quest/m2/mp4-sigint-flush.md index 37818429f44..e1a0167c63c 100644 --- a/quest/m2/mp4-sigint-flush.md +++ b/quest/m2/mp4-sigint-flush.md @@ -11,7 +11,3 @@ Follow-up of [moq-dev/moq#7357](https://github.com/moq-dev/moq/pull/7357), which Decided 2026-10-30 (maintainer, follow-up planning after the /quest-afk run). Flush through the fMP4 exporter's normal fragment path so the flushed fragment is indexed by the final `moov` like any other; extend #7357's nightly SIGINT test to check the last frames received before the signal are in the file. Public API: none. Wire: none. - -## Required - -- [moq export mp4](/quest/m2/mp4-export.md) - its PR, #7357, adds the recorder From f6a520199c5d9e3e45c73669c7c2f80aacb015cd Mon Sep 17 00:00:00 2001 From: quest fixer Date: Sun, 1 Nov 2026 01:49:38 -0700 Subject: [PATCH 6/7] fix(mux): follow current mp4-atom sample API --- rs/moq-mux/src/container/mp4.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rs/moq-mux/src/container/mp4.rs b/rs/moq-mux/src/container/mp4.rs index 293e4bfc8e0..ca22f633951 100644 --- a/rs/moq-mux/src/container/mp4.rs +++ b/rs/moq-mux/src/container/mp4.rs @@ -82,7 +82,7 @@ struct Run { struct Sample { size: u32, duration: u32, - cts: i32, + cts: i64, sync: bool, } @@ -283,16 +283,16 @@ impl Export { let track = self.tracks.entry(id).or_default(); let mut decode_time = tfdt.base_media_decode_time; - for trun in traf.trun.iter().filter(|trun| !trun.entries.is_empty()) { + for trun in traf.trun.iter().filter(|trun| !trun.samples.is_empty()) { let Some(offset) = trun.data_offset.and_then(|offset| u64::try_from(offset).ok()) else { return Err(unindexable(id, "a run has no data offset").into()); }; track.chunks.push(Run { offset: moof + offset, decode_time, - count: trun.entries.len() as u32, + count: trun.samples.len() as u32, }); - for entry in &trun.entries { + for entry in trun.samples.iter() { let sample = Sample { size: entry.size.unwrap_or(size), duration: entry.duration.unwrap_or(duration), From b078a65575d8a8bbd99eafe53c6f0f9dec6ae8f5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:28:11 -0800 Subject: [PATCH 7/7] test: wait on kio's clock in the export tests main moved moq-clock into kio::clock and bans tokio's timers workspace- wide; the export integration test and moq-mux's record helper still used tokio::time::{timeout,sleep}, which the gate's clippy build denies. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/tests/integration/export.rs | 4 ++-- rs/moq-mux/src/container/mp4.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/moq-cli/tests/integration/export.rs b/rs/moq-cli/tests/integration/export.rs index b611f7e168d..9e7db241973 100644 --- a/rs/moq-cli/tests/integration/export.rs +++ b/rs/moq-cli/tests/integration/export.rs @@ -1136,9 +1136,9 @@ async fn an_interrupted_mp4_recording_is_finished() { .expect("spawn export"); let (mut publisher, feeding) = import(&relay); - tokio::time::timeout(TIMEOUT, async { + kio::clock::timeout(TIMEOUT, async { while std::fs::metadata(&path).map_or(0, |file| file.len()) < 10_000 { - tokio::time::sleep(Duration::from_millis(50)).await; + kio::clock::sleep(Duration::from_millis(50)).await; } }) .await diff --git a/rs/moq-mux/src/container/mp4.rs b/rs/moq-mux/src/container/mp4.rs index ca22f633951..aa955fc8744 100644 --- a/rs/moq-mux/src/container/mp4.rs +++ b/rs/moq-mux/src/container/mp4.rs @@ -531,7 +531,7 @@ mod tests { /// Append everything the export writes without waiting for more input. async fn record(export: &mut super::Export, file: &mut Vec) { - while let Ok(next) = tokio::time::timeout(Duration::from_millis(1), export.next()).await { + while let Ok(next) = kio::clock::timeout(Duration::from_millis(1), export.next()).await { match next.expect("export failed") { Some(data) => file.extend_from_slice(&data), None => break,