diff --git a/quest/m1/README.md b/quest/m1/README.md index 96017b5e0bd..426ec1408fc 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -186,7 +186,6 @@ per-frame CPU growth since #5926. - [JS ordered reads wait only on owed groups](/quest/m1/js-ordered-owed.md) - @moq/net matches Rust's owed-group wait - [Binding announcements always carry captures](/quest/m1/ffi-required-captures.md) - moq-ffi and its generated bindings make `captures` a required list, as #7266 always sets it - [HLS export follows a resolution change](/quest/m1/hls-reinit-same-codec.md) - a decoder-identity change under the same codec string starts a new init and segment tag -- [fMP4 decode times keep increasing](/quest/m1/fmp4-bframe-decode-time.md) - fragments shorter than a GOP of B-frame video never step `tfdt` backwards - [moq-quic deletes its remaining 0-RTT code](/quest/m1/quic-zero-rtt-leftovers.md) - only the code that drops a 0-RTT packet remains, with a test - [Gate the no-em-dash rule](/quest/m1/em-dash-gate.md) - `just check` fails on an em dash in any tracked text file - [Every package shares one minor version](/quest/m1/shared-version.md) - a recipe sets every published package to one minor at the cut (next: 0.22.0), and `just check` enforces it and the `0.0.x` rule for experiments diff --git a/quest/m1/fmp4-bframe-decode-time.md b/quest/m1/fmp4-bframe-decode-time.md deleted file mode 100644 index 30789138dea..00000000000 --- a/quest/m1/fmp4-bframe-decode-time.md +++ /dev/null @@ -1,13 +0,0 @@ -# [M] fMP4 fragments shorter than a GOP keep decode times increasing - -## Goal - -With B-frame video and a fragment duration shorter than a GOP, every fragment the fMP4 exporter writes has a `tfdt` at or after the previous fragment's end, so `moq export fmp4`, HLS packaging, and an unfinished `moq export mp4` file are valid fragmented MP4. - -## Plan - -Found by the agent on [moq-dev/moq#7357](https://github.com/moq-dev/moq/pull/7357): with `--fragment-duration` shorter than a GOP of B-frame video, the exporter writes fragment decode times that step backwards. The finished `export mp4` file is correct because it rebuilds its sample tables, but the fragmented output isn't. - -Decided 2026-10-30 (maintainer, follow-up planning after the /quest-afk run). The fix is in how the exporter infers B-frame sample durations and decode times, which `export fmp4`, HLS packaging, and `export mp4` share; fix it there once rather than per consumer. Test with reordered video cut into fragments shorter than a GOP: decode times never decrease, presentation order and A/V timing are unchanged, and ffprobe reads every fragment without warnings. - -Public API: none. Wire: none. diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index ed9cdad1f93..b52bcaa6660 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -199,6 +199,18 @@ impl Rendition { matches!(self, Rendition::Video(_)) } + /// Whether frames may present out of decode order: what the catalog's `reorder` declares, + /// else whatever the codec allows. + fn may_reorder(&self) -> bool { + match self { + Rendition::Video(config) => match config.reorder { + Some(depth) => depth > 0, + None => crate::codec::video::max_reorder(config) > 0, + }, + Rendition::Audio(_) => false, + } + } + /// Fallback duration for a trailing frame that carries no per-sample duration. fn default_frame(&self) -> Duration { match self { @@ -269,6 +281,14 @@ struct Fmp4Track { /// instead of writing the track's history twice. written: Option, + /// Where the last written fragment's decode timeline ended, in ticks at `timescale`. + /// + /// A fragment cut inside a GOP continues from here rather than anchoring at its first + /// frame's presentation time: with reordered video that frame is often a reference + /// presented after the B-frames decoded next, so its presentation time would put the + /// fragment's `tfdt` ahead of the one that follows. + decode_end: Option, + /// The first presentation time read before the init segment, which bounds how much /// media queues behind it. backlog: Option, @@ -300,6 +320,7 @@ impl Fmp4Track { resume: None, group_finished: false, written: None, + decode_end: None, backlog: None, track_id, timescale, @@ -528,6 +549,12 @@ impl Export { /// marker; audio and samples with explicit durations remain immediate. Otherwise /// the cap applies in addition to the group boundary. /// + /// A fragment cut inside a GOP continues the decode timeline where the previous one + /// ended, so B-frame video keeps increasing decode times. Such a fragment cannot see + /// the reordering around it, so unless the catalog declares `reorder: 0` (or the codec + /// never reorders), a video sample with unknown duration lasts no longer than the + /// catalog frame rate's cadence. + /// /// Accepts either `Duration` or `Option` (where `None` restores /// the per-group default). pub fn with_fragment_duration(mut self, duration: impl Into>) -> Self { @@ -1073,24 +1100,39 @@ fn should_flush(track: &Fmp4Track, frame: &Frame, fragment_duration: Option) -> Result { - if frames.is_empty() { +/// +/// A run that opens a GOP (and every audio run) anchors its `tfdt` at its first frame's +/// presentation time, which re-syncs the decode timeline after a gap the durations did not +/// cover. A run cut inside a GOP continues the timeline where the track's previous fragment +/// ended, so decode times never step back however the GOP is reordered. +fn encode_fragment( + track: &mut Fmp4Track, + sequence_number: &mut u32, + frames: Vec, + opens_gop: bool, +) -> Result { + let Some(first) = frames.first() else { return Err(Error::NoFrames.into()); - } - let seq = *sequence_number; - *sequence_number += 1; + }; let timescale = moq_net::Timescale::new(track.timescale)?; + let base_dts = match track.decode_end { + Some(end) if !opens_gop => end, + _ => super::base_ticks(first, timescale)?, + }; let info = crate::container::fmp4::FragmentInfo { track_id: track.track_id, timescale, - sequence_number: seq, + sequence_number: *sequence_number, kind: if track.is_video { super::Kind::Video } else { super::Kind::Audio }, }; - Ok(crate::container::fmp4::encode_fragment(info, &frames)?) + let (data, decode_end) = super::encode_at(info, base_dts, &frames)?; + *sequence_number += 1; + track.decode_end = Some(decode_end); + Ok(data) } /// Encode a buffered run and wrap it with the metadata a segmenting consumer needs. @@ -1123,9 +1165,16 @@ fn emit_fragment( let latest = frames.iter().map(|frame| Duration::from(frame.timestamp)).max(); track.written = track.written.max(latest); let timescale = moq_net::Timescale::new(track.timescale)?; - infer_missing_durations(&mut frames, successor, track.default_frame, timescale)?; + // A run that starts or ends inside a GOP sees only part of its reordering. + let partial = !track.buffer_independent || successor.is_some_and(|next| !next.keyframe); + let run = if partial && track.config.may_reorder() { + Run::PartialGop + } else { + Run::Whole + }; + infer_missing_durations(&mut frames, successor, track.default_frame, timescale, run)?; let duration = fragment_duration(&frames, track.default_frame, timescale)?; - let data = encode_fragment(track, sequence_number, frames)?; + let data = encode_fragment(track, sequence_number, frames, independent)?; Ok(Fragment { data, independent, @@ -1204,25 +1253,53 @@ pub(crate) fn apply_codec_durations(frames: &mut [Frame], opus: bool) { /// The publisher is the one that actually knows where its group's content ends, and /// [`Producer::cut`](crate::container::Producer::cut) is how it says so: the durations it /// writes arrive already set and are left alone here. +/// +/// Frames arrive in decode order, so a gap is a decode duration only when presentation order +/// matches it. A run whose timestamps go backwards is reordered video, and every frame takes +/// the catalog cadence. A [`Run::PartialGop`] cannot rule reordering out from its own frames, +/// since the B-frames that would reveal it may sit in a neighbouring run, so no gap there may +/// exceed the cadence: in reordered video the gap to the next frame decoded spans the frames +/// decoded after it, and a longer duration would push the decode timeline past the next GOP. pub(crate) fn infer_missing_durations( frames: &mut [Frame], successor: Option<&Frame>, default_frame: Duration, timescale: moq_net::Timescale, + run: Run, ) -> Result<()> { let infer_from_pts = pts_monotonic(frames, successor); + let cadence = fallback_duration(default_frame, timescale); for i in 0..frames.len() { let successor = infer_from_pts .then(|| duration_bound(frames, successor, i)) .flatten() .cloned(); + let stated = frames[i].duration.is_some_and(|duration| !duration.is_zero()); infer_missing_duration(&mut frames[i], successor.as_ref(), default_frame, timescale)?; + if run == Run::PartialGop + && !stated + && let (Some(inferred), Some(cadence)) = (frames[i].duration, cadence) + && Duration::from(inferred) > Duration::from(cadence) + { + frames[i].duration = Some(cadence); + } } Ok(()) } +/// How much of its reordering a run of frames passed to [`infer_missing_durations`] shows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Run { + /// Whole GOPs, or media that presents in decode order, so its own timestamps show + /// whether it is reordered. + Whole, + /// Video that may reorder, cut inside a GOP, whose reordering may only show in a + /// neighbouring run. + PartialGop, +} + /// Infer one frame's duration from a known same-group successor, else the catalog cadence. pub(crate) fn infer_missing_duration( frame: &mut Frame, @@ -1373,7 +1450,14 @@ mod tests { #[test] fn infer_missing_durations_uses_default_for_trailing_sample() { let mut frames = vec![frame(0, Some(0)), frame(41_667, None), frame(83_334, None)]; - infer_missing_durations(&mut frames, None, Duration::from_millis(33), moq_net::Timescale::MICRO).unwrap(); + infer_missing_durations( + &mut frames, + None, + Duration::from_millis(33), + moq_net::Timescale::MICRO, + Run::Whole, + ) + .unwrap(); assert_eq!(frames[0].duration, Some(ts(41_667))); assert_eq!(frames[1].duration, Some(ts(41_667))); @@ -1387,7 +1471,14 @@ mod tests { #[test] fn infer_missing_duration_uses_default_for_single_frame() { let mut frames = vec![frame(83_333, Some(0))]; - infer_missing_durations(&mut frames, None, Duration::from_millis(40), moq_net::Timescale::MICRO).unwrap(); + infer_missing_durations( + &mut frames, + None, + Duration::from_millis(40), + moq_net::Timescale::MICRO, + Run::Whole, + ) + .unwrap(); assert_eq!(frames[0].duration, Some(ts(40_000))); assert_eq!( @@ -1397,7 +1488,8 @@ mod tests { } /// The duration cap splits a GOP across fragments, so the successor is a delta frame - /// in the SAME group. That one is a real bound and is still used. + /// in the SAME group. In video that presents in decode order that one is a real bound + /// and is still used. #[test] fn infer_trailing_duration_from_successor_frame() { let successor = frame(83_334, None); @@ -1407,6 +1499,7 @@ mod tests { Some(&successor), Duration::from_millis(33), moq_net::Timescale::MICRO, + Run::Whole, ) .unwrap(); @@ -1417,11 +1510,38 @@ mod tests { ); } + /// Video that may reorder, cut inside a GOP, cannot tell a reference frame's gap to the + /// B-frames decoded after it from a duration, so no inferred duration there exceeds the + /// cadence. A shorter gap still times its frame, and a stated duration is kept. + #[test] + fn a_partial_gop_caps_inferred_durations_at_the_cadence() { + let successor = frame(100_000, None); + let mut frames = vec![frame(0, Some(50_000)), frame(10_000, None), frame(20_000, None)]; + infer_missing_durations( + &mut frames, + Some(&successor), + Duration::from_millis(33), + moq_net::Timescale::MICRO, + Run::PartialGop, + ) + .unwrap(); + + let durations: Vec<_> = frames.iter().map(duration_micros).collect(); + assert_eq!(durations, vec![50_000, 10_000, 33_000]); + } + #[test] fn inferred_microsecond_clock_carries_fractional_ticks_forward() { let timescale = moq_net::Timescale::new(30_000).unwrap(); let mut frames = vec![frame(0, None), frame(33_333, None), frame(66_667, None)]; - infer_missing_durations(&mut frames, None, Duration::from_nanos(33_333_333), timescale).unwrap(); + infer_missing_durations( + &mut frames, + None, + Duration::from_nanos(33_333_333), + timescale, + Run::Whole, + ) + .unwrap(); let durations: Vec<_> = frames .iter() @@ -1446,6 +1566,7 @@ mod tests { Some(&next_group), Duration::from_millis(33), moq_net::Timescale::MICRO, + Run::Whole, ) .unwrap(); @@ -1462,7 +1583,14 @@ mod tests { #[test] fn infer_stops_at_an_interior_group_boundary() { let mut frames = vec![frame(0, None), frame(21_333, None), group_start(600_000_000)]; - infer_missing_durations(&mut frames, None, Duration::from_millis(21), moq_net::Timescale::MICRO).unwrap(); + infer_missing_durations( + &mut frames, + None, + Duration::from_millis(21), + moq_net::Timescale::MICRO, + Run::Whole, + ) + .unwrap(); assert_eq!(frames[0].duration, Some(ts(21_333)), "same group, real delta"); assert_eq!(duration_micros(&frames[1]), 21_000, "bounded by the next group"); @@ -1490,6 +1618,7 @@ mod tests { None, Duration::from_micros(21_333), moq_net::Timescale::MICRO, + Run::Whole, ) .unwrap(); @@ -1513,6 +1642,7 @@ mod tests { Some(&next_group), Duration::from_millis(50), moq_net::Timescale::MICRO, + Run::Whole, ) .unwrap(); @@ -1533,6 +1663,7 @@ mod tests { Some(&successor), Duration::from_millis(33), moq_net::Timescale::MICRO, + Run::Whole, ) .unwrap(); diff --git a/rs/moq-mux/src/container/fmp4/export_test.rs b/rs/moq-mux/src/container/fmp4/export_test.rs index 2d97ebd1793..5f15bae7312 100644 --- a/rs/moq-mux/src/container/fmp4/export_test.rs +++ b/rs/moq-mux/src/container/fmp4/export_test.rs @@ -640,6 +640,27 @@ async fn zero_fragment_duration_uses_the_successor_then_the_cut() { assert!(drain_now(&mut exporter).await.is_empty()); } +/// One sample per fragment hides the B-frames a reference frame's successor gap spans, so H.264 +/// whose catalog declares no `reorder` takes no more than the 30 fps cadence. Declaring +/// `reorder: 0` says it presents in decode order, so the 50 ms gap is a real duration. +#[tokio::test(start_paused = true)] +async fn zero_fragment_duration_trusts_a_long_gap_only_without_reordering() { + let cadence = moq_net::Timestamp::from_scale(1_000, 30_000).unwrap().into(); + for (reorder, expected) in [(None, cadence), (Some(0), std::time::Duration::from_millis(50))] { + let mut live = Live::avc3(); + for config in live.catalog.modify().unwrap().video.renditions.values_mut() { + config.reorder = reorder; + } + live.track.write(video_frame(0, true)).unwrap(); + live.track.write(video_frame(50_000, false)).unwrap(); + let mut exporter = crate::container::fmp4::Export::new(live.source().await, live.catalog_stream().await) + .with_fragment_duration(std::time::Duration::ZERO); + chunk_now(&mut exporter).await.init().expect("init"); + let first = chunk_now(&mut exporter).await.fragment().expect("first sample"); + assert_eq!(first.duration, expected, "reorder {reorder:?}"); + } +} + #[tokio::test(start_paused = true)] async fn ntsc_tail_uses_a_representable_catalog_cadence() { use hang::catalog::{Container, H264, VideoConfig}; @@ -2584,3 +2605,158 @@ async fn a_rendition_moved_to_another_sample_entry_fails() { "{err:?}" ); } + +/// Export `bbb5s_cbr.ts` (HEVC with B-pyramids, and Opus) cut at `cap`, returning the init +/// segment and every fragment it writes. +async fn export_reordered_clip(cap: Option) -> (Bytes, Vec) { + let data = include_bytes!("../ts/test_data/bbb5s_cbr.ts"); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut import = crate::container::ts::Import::new(broadcast, catalog.reserve()); + import.decode(&BytesMut::from(&data[..])).unwrap(); + import.finish().unwrap(); + + let catalog_stream = crate::catalog::Consumer::<()>::new(&consumer, crate::catalog::CatalogFormat::Hang) + .await + .expect("catalog consumer"); + let mut exporter = crate::container::fmp4::Export::new(crate::source::announced(&consumer).await, catalog_stream) + .with_max_delay(RECORDING_MAX_DELAY) + .with_fragment_duration(cap); + let mut init = None; + let mut fragments = Vec::new(); + // The tracks finished, but the broadcast stays up: the export is done once it goes quiet. + while let Ok(next) = kio::clock::timeout(RECORDING_MAX_DELAY * 4, exporter.next_chunk()).await { + let Some(chunk) = next.expect("exporter failed") else { + break; + }; + match chunk { + crate::container::fmp4::Chunk::Init(bytes) => init = Some(bytes), + crate::container::fmp4::Chunk::Fragment(fragment) => fragments.push(fragment), + } + } + (init.expect("an init segment"), fragments) +} + +/// Every track's media timescale, by track id, from an init segment. +fn track_scales(init: &Bytes) -> std::collections::BTreeMap { + let mut cursor = Cursor::new(init.as_ref()); + let mut scales = std::collections::BTreeMap::new(); + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor).expect("decode init") { + if let mp4_atom::Any::Moov(moov) = atom { + for trak in &moov.trak { + let scale = moq_net::Timescale::new(trak.mdia.mdhd.timescale.into()).unwrap(); + scales.insert(trak.tkhd.track_id, scale); + } + } + } + scales +} + +/// Every sample's presentation time in microseconds, by track id, in decode order. +fn presentation_times( + init: &Bytes, + fragments: &[crate::container::fmp4::Fragment], +) -> std::collections::BTreeMap> { + let scales = track_scales(init); + let mut times: std::collections::BTreeMap> = Default::default(); + for fragment in fragments { + let id = super::first_traf(&fragment.data).tfhd.track_id; + let kind = if id == 1 { + super::Kind::Video + } else { + super::Kind::Audio + }; + let track = super::Track::new(scales[&id], kind); + let samples = super::decode(fragment.data.clone(), None, track).expect("decode fragment"); + times + .entry(id) + .or_default() + .extend(samples.iter().map(|sample| sample.timestamp.as_micros())); + } + times +} + +/// The warnings ffprobe prints reading every packet of an fMP4 file. +fn ffprobe_warnings(file: Vec) -> String { + use std::io::Write; + + let mut child = std::process::Command::new("ffprobe") + .args([ + "-v", + "warning", + "-show_entries", + "packet=pts,dts", + "-of", + "csv", + "-i", + "pipe:0", + ]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("ffprobe is in the dev shell"); + let mut stdin = child.stdin.take().unwrap(); + let writer = std::thread::spawn(move || stdin.write_all(&file)); + let output = child.wait_with_output().unwrap(); + writer.join().unwrap().expect("feed ffprobe"); + assert!(output.status.success(), "ffprobe failed"); + String::from_utf8_lossy(&output.stderr).into_owned() +} + +/// Fragments shorter than a GOP of B-frame video continue the decode timeline instead of +/// anchoring at a reference frame that presents after the B-frames decoded next, and time each +/// sample by the cadence rather than the gap to that reference. Each track's `tfdt` then starts +/// at or after where its previous fragment ended, every sample presents when it does in the +/// whole-GOP export, and ffprobe reads the file without complaint. +#[tokio::test(start_paused = true)] +async fn reordered_video_cut_inside_a_gop_keeps_decode_times_increasing() { + let video = |fragments: &[crate::container::fmp4::Fragment]| { + fragments + .iter() + .filter(|fragment| super::first_traf(&fragment.data).tfhd.track_id == 1) + .count() + }; + let (init, whole) = export_reordered_clip(None).await; + let reference = presentation_times(&init, &whole); + assert!( + reference[&1].windows(2).any(|pair| pair[1] < pair[0]), + "the clip's video must reorder" + ); + + for cap in [std::time::Duration::ZERO, std::time::Duration::from_millis(100)] { + let (init, fragments) = export_reordered_clip(Some(cap)).await; + assert!(video(&fragments) > video(&whole), "{cap:?} must cut inside a GOP"); + + let mut ends = std::collections::BTreeMap::new(); + for fragment in &fragments { + let traf = super::first_traf(&fragment.data); + let start = traf.tfdt.expect("a tfdt").base_media_decode_time; + let span: u64 = traf.trun[0] + .samples + .iter() + .map(|sample| u64::from(sample.duration.expect("a sample duration"))) + .sum(); + if let Some(end) = ends.insert(traf.tfhd.track_id, start + span) { + assert!( + start >= end, + "{cap:?}: track {} decodes from {start} before its previous fragment ended at {end}", + traf.tfhd.track_id, + ); + } + } + + assert_eq!( + presentation_times(&init, &fragments), + reference, + "{cap:?}: every sample presents as in the whole-GOP export" + ); + + let mut file = init.to_vec(); + for fragment in &fragments { + file.extend_from_slice(&fragment.data); + } + assert_eq!(ffprobe_warnings(file), "", "{cap:?}"); + } +} diff --git a/rs/moq-mux/src/container/fmp4/fragmenter.rs b/rs/moq-mux/src/container/fmp4/fragmenter.rs index 8c19ba403d3..0da9f631737 100644 --- a/rs/moq-mux/src/container/fmp4/fragmenter.rs +++ b/rs/moq-mux/src/container/fmp4/fragmenter.rs @@ -171,15 +171,10 @@ impl Fragmenter { super::Kind::Audio }, }; - let ticks = frame - .duration - .map(|duration| super::trun_duration(duration, self.timescale)) - .transpose()? - .unwrap_or(0); // Advance by the value the trun stores, so the next tfdt continues exactly what // this fragment claimed. - let next_dts = dts.checked_add(u64::from(ticks)).ok_or(super::Error::PtsOverflow)?; - let data = super::encode_at(info, dts, std::slice::from_ref(&frame))?; + let (data, next_dts) = super::encode_at(info, dts, std::slice::from_ref(&frame))?; + let ticks = next_dts - dts; self.sequence = self.sequence.wrapping_add(1); self.dts = Some(next_dts); @@ -190,7 +185,7 @@ impl Fragmenter { independent: !self.is_video || frame.keyframe, // Describe the exact duration written into trun, including its timescale // quantization, so playlist metadata and media advance by the same amount. - duration: Duration::from_secs_f64(f64::from(ticks) / self.timescale.as_u64() as f64), + duration: Duration::from_secs_f64(ticks as f64 / self.timescale.as_u64() as f64), }) } diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index da472731c0b..9f0342d8f7a 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -674,7 +674,7 @@ pub(crate) fn encode_fragment(info: FragmentInfo, frames: &[Frame]) -> Result Resul /// signed composition offset, so a reordered frame keeps its presentation time even when the /// fragment holds a single sample. /// -/// Returns an empty `Bytes` when `frames` is empty. -fn encode_at(info: FragmentInfo, base_dts: u64, frames: &[Frame]) -> Result { +/// Returns the fragment with the decode time it ends at, which is where the next fragment on +/// the same timeline continues, or an empty `Bytes` and `base_dts` when `frames` is empty. +pub(super) fn encode_at(info: FragmentInfo, base_dts: u64, frames: &[Frame]) -> Result<(Bytes, u64)> { let FragmentInfo { track_id, timescale, @@ -721,7 +722,7 @@ fn encode_at(info: FragmentInfo, base_dts: u64, frames: &[Frame]) -> Result Result