From c9b4344883ef8a75c0d280c4a17d29dad863d2e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 12:13:53 -0700 Subject: [PATCH 1/3] chore(quest): claim quest/m1/fmp4-edit-lists.md Quest-Claim: 6532dbe2116b3c59 From e761cef62a4bfe4daacb143443ddedc84d5234a4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 12:41:42 -0700 Subject: [PATCH 2/3] fix(mux): apply fMP4 edit lists to frame timestamps The fMP4 importer reduces each track's edit list to one shift, applies it to every frame timestamp, and drops the samples before the edit (such as AAC priming), so consumers need no edit list code. The passthrough init no longer carries the edts. Opus applies only the part beyond the OpusHead pre-skip, which its timestamps already count. An edit list with several media edits or a rate other than 1 is refused. Co-Authored-By: Claude Opus 5.5 --- quest/m1/fmp4-edit-lists.md | 52 +++--- rs/moq-mux/CHANGELOG.md | 4 + rs/moq-mux/src/container/fmp4/edit.rs | 179 +++++++++++++++++++ rs/moq-mux/src/container/fmp4/import.rs | 144 +++++++++++---- rs/moq-mux/src/container/fmp4/import_test.rs | 145 +++++++++++++++ rs/moq-mux/src/container/fmp4/mod.rs | 5 + rs/moq-mux/src/container/fmp4/text.rs | 20 ++- 7 files changed, 478 insertions(+), 71 deletions(-) create mode 100644 rs/moq-mux/src/container/fmp4/edit.rs diff --git a/quest/m1/fmp4-edit-lists.md b/quest/m1/fmp4-edit-lists.md index 57853124904..17b33af0f89 100644 --- a/quest/m1/fmp4-edit-lists.md +++ b/quest/m1/fmp4-edit-lists.md @@ -1,41 +1,29 @@ -# [M] fMP4 import applies edit lists to timestamps +# [S] Interop holds the AAC tone to its phase ## Goal -An fMP4 source with an edit list (AAC priming, a non-zero `media_time`, or an -empty edit) publishes frames at their presentation time, so both players and -every export see the timeline the source meant. The interop AAC rendition -then passes `tone.py --phase` like Opus does. +The interop AAC rendition passes `tone.py --phase` like Opus does, now that +the fMP4 importer applies the edit list holding ffmpeg's 1024 AAC priming +samples to the frame timestamps. ## Plan -Today `edts` is forwarded in the catalog init but read by no Rust or JS code, -and `rs/moq-mux/src/container/fmp4/export.rs` strips it on the way out. Both -players present `tfdt` plus the composition offset. ffmpeg's fragmented -output does write `elst`: with the interop's `-movflags cmaf`, its AAC track -carries `media_time` 1024 at `tfdt` 0, so every subscriber plays the AAC tone -1024 samples late (`-880us` at 440 Hz once `--phase` checks it), and its Opus -track carries `media_time` equal to the `dOps` pre-skip. - -Decided (2026-10-07): the importer applies the edit list to each frame's -timestamp and drops `edts` from the catalog init, so no consumer needs edit -list code. This builds on decoders taking time from the frame timestamp -rather than `tfdt`. Decide in the PR whether priming samples before the edit -are dropped or published with negative-offset handling (Apple's 2112-sample -AAC priming is not a whole number of 1024-sample frames), and refuse edit -lists with more than one non-empty entry rather than guess. - -Opus frame timestamps count the OpusHead pre-skip (`draft-lcurley-moq-hang`), -so for Opus the importer applies only the part of the edit beyond the -pre-skip, or the pre-skip is trimmed twice. - -Once AAC lines up, drop the `opus` condition from `phase_checked` in -`test/interop/interop.sh` and the matching sentence in -`test/interop/README.md`. - -Public API: none. Wire: none, but a behavior change: frame timestamps now -include the edit and the init loses `edts`, so a consumer that offsets AAC -priming on its own must stop, or it shifts twice. Note it in the changelog. +Drop the `opus` condition from `phase_checked` in `test/interop/interop.sh`, +with the comment's AAC sentence, and the matching sentence in +`test/interop/README.md`. Both sit beside lines +[moq-dev/moq#6647](https://github.com/moq-dev/moq/pull/6647) rewrites, so +land this after it. + +With the condition dropped, `just test interop --all --publishers rust` +passes the AAC phase on every phase-checked subscriber except GStreamer, +which reads it 1024 samples early (`+878us`). The cause is its decoder, not +the timestamps: `faad`, which `decodebin` picks there, drops the first +frame's output and stamps the rest one frame early, while ffmpeg's decoder +keeps the priming at the input timestamps (an ffmpeg ADTS encode decodes +with its first audible sample at 2 through `faad` and at 1026 through +ffmpeg; `fdkaacdec` lands later still). Decide whether `moqsrc` compensates +for that decoder delay or the check skips GStreamer's AAC with the reason, +as it skips the native-JS subscriber. ## Related diff --git a/rs/moq-mux/CHANGELOG.md b/rs/moq-mux/CHANGELOG.md index 259c805d14b..a76ac5c3bdd 100644 --- a/rs/moq-mux/CHANGELOG.md +++ b/rs/moq-mux/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [**breaking**] Declare per-track archive durations from GOP hints or the first completed record. +### Fixed + +- fMP4 import applies each track's edit list to its frame timestamps and drops the samples before the edit, such as AAC priming, and a passthrough init no longer carries the `edts`. A consumer that offsets AAC priming itself must stop, or it shifts the track twice. An edit list it can't reduce to one shift (several media edits, or a rate other than 1) is refused with `Error::UnsupportedEditList`. + ## [0.10.11](https://github.com/moq-dev/moq/compare/moq-mux-v0.10.10...moq-mux-v0.10.11) - 2026-10-05 ### Other diff --git a/rs/moq-mux/src/container/fmp4/edit.rs b/rs/moq-mux/src/container/fmp4/edit.rs new file mode 100644 index 00000000000..06a14aeea61 --- /dev/null +++ b/rs/moq-mux/src/container/fmp4/edit.rs @@ -0,0 +1,179 @@ +//! A track's edit list (ISO/IEC 14496-12 ยง8.6.6), applied to its sample times on import. + +use super::Error; + +/// The shift a track's edit list makes to its sample times, in the track's media ticks. +/// +/// The importer applies it to every timestamp and drops the `edts` from the catalog init, so no +/// consumer needs edit list code. It covers the shape packagers write: empty edits delaying the +/// track, then one edit starting the media at `media_time` at normal rate. A sample before that +/// start is dropped, since a frame timestamp can't be negative; that includes one straddling the +/// start, such as Apple's 2112 AAC priming samples, which are not a whole number of frames. The +/// edit's duration is ignored: it trims the end, and a live source has none. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct Edit { + /// The media time presented at the edit's start. + start: u64, + /// The empty edits' duration, presented before `start`. + delay: u64, +} + +impl Edit { + /// The edit `trak` declares, in a movie counting `movie_timescale` ticks a second. + pub fn new(trak: &mp4_atom::Trak, movie_timescale: u32) -> Result { + let Some(elst) = trak.edts.as_ref().and_then(|edts| edts.elst.as_ref()) else { + return Ok(Self::default()); + }; + if elst.entries.is_empty() { + return Ok(Self::default()); + } + + let mut entries = elst.entries.iter(); + let mut empty = 0u64; + let (media_time, entry) = loop { + let entry = entries + .next() + .ok_or(Error::UnsupportedEditList("no edit presents media"))?; + match entry.media_time { + Some(media_time) => break (media_time, entry), + None => empty = empty.checked_add(entry.segment_duration).ok_or(Error::PtsOverflow)?, + } + }; + // Several media edits splice the media, which one shift can't express. + if entries.next().is_some() { + return Err(Error::UnsupportedEditList("an edit follows the media edit")); + } + // A rate of 0 dwells on one sample, and the standard defines no other. + if entry.media_rate.integer() != 1 || entry.media_rate.decimal() != 0 { + return Err(Error::UnsupportedEditList("media rate is not 1")); + } + + let timescale = u128::from(trak.mdia.mdhd.timescale); + let delay = match empty { + 0 => 0, + _ if movie_timescale == 0 => return Err(Error::UnsupportedEditList("movie timescale is zero")), + empty => u64::try_from(u128::from(empty) * timescale / u128::from(movie_timescale)) + .map_err(|_| Error::PtsOverflow)?, + }; + + // An Opus frame timestamp already counts the OpusHead pre-skip (draft-lcurley-moq-hang), and + // the edit includes it, so only the part beyond it moves the timestamps. + let skip = match trak.mdia.minf.stbl.stsd.codecs.as_slice() { + [mp4_atom::Codec::Opus(opus)] => { + u64::try_from(u128::from(opus.dops.pre_skip) * timescale / 48_000).map_err(|_| Error::PtsOverflow)? + } + _ => 0, + }; + + Ok(Self { + start: media_time.saturating_sub(skip), + delay, + }) + } + + /// Where the sample at media time `pts` presents, or `None` if it precedes the edit. + pub fn present(&self, pts: u64) -> Result, Error> { + let Some(since) = pts.checked_sub(self.start) else { + return Ok(None); + }; + since.checked_add(self.delay).map(Some).ok_or(Error::PtsOverflow) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(segment_duration: u64, media_time: Option, rate: i16) -> mp4_atom::ElstEntry { + mp4_atom::ElstEntry { + segment_duration, + media_time, + media_rate: mp4_atom::FixedPoint::from(rate), + } + } + + /// A 48 kHz track with `entries` as its edit list, Opus with a 312-sample pre-skip if `opus`. + fn trak(entries: Option>, opus: bool) -> mp4_atom::Trak { + let codec = match opus { + true => mp4_atom::Opus { + audio: mp4_atom::Audio { + data_reference_index: 1, + channel_count: 2, + sample_size: 16, + sample_rate: mp4_atom::FixedPoint::from(48_000), + }, + dops: mp4_atom::Dops { + output_channel_count: 2, + pre_skip: 312, + input_sample_rate: 48_000, + output_gain: 0, + }, + btrt: None, + } + .into(), + false => mp4_atom::Codec::Unknown(b"test".into(), Vec::new()), + }; + mp4_atom::Trak { + edts: entries.map(|entries| mp4_atom::Edts { + elst: Some(mp4_atom::Elst { entries }), + }), + ..super::super::build_audio_trak(1, 48_000, codec) + } + } + + /// Where media times 0, 959, 960, and 1000 present under `edit`. + fn presents(edit: Edit) -> [Option; 4] { + [0, 959, 960, 1000].map(|pts| edit.present(pts).unwrap()) + } + + #[test] + fn edits_shift_and_drop() { + let none = [Some(0), Some(959), Some(960), Some(1000)]; + let cases = [ + (None, false, none), + (Some(vec![]), false, none), + ( + Some(vec![entry(0, Some(960), 1)]), + false, + [None, None, Some(0), Some(40)], + ), + // One movie second (1000 ticks) is 48000 media ticks. + ( + Some(vec![entry(1000, None, 1), entry(0, Some(960), 1)]), + false, + [None, None, Some(48_000), Some(48_040)], + ), + // Opus timestamps already count the pre-skip, so an edit of just the pre-skip moves nothing. + (Some(vec![entry(0, Some(312), 1)]), true, none), + ( + Some(vec![entry(0, Some(312 + 960), 1)]), + true, + [None, None, Some(0), Some(40)], + ), + (Some(vec![entry(0, Some(100), 1)]), true, none), + ]; + for (entries, opus, expected) in cases { + let edit = Edit::new(&trak(entries.clone(), opus), 1000).unwrap(); + assert_eq!(presents(edit), expected, "{entries:?}, opus {opus}"); + } + } + + #[test] + fn unsupported_edits_are_refused() { + let cases = [ + vec![entry(1000, None, 1)], + vec![entry(0, Some(0), 1), entry(0, Some(960), 1)], + vec![entry(0, Some(0), 1), entry(1000, None, 1)], + vec![entry(1000, Some(0), 0)], + ]; + for entries in cases { + assert!( + matches!( + Edit::new(&trak(Some(entries.clone()), false), 1000), + Err(Error::UnsupportedEditList(_)) + ), + "{entries:?}" + ); + } + } +} diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 10d50aced11..b21be4b7755 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -4,7 +4,7 @@ use moq_net::Timestamp; use mp4_atom::{Any, Atom, DecodeMaybe, Encode, Mdat, Moof, Moov, Trak}; use std::collections::{HashMap, HashSet}; -use super::{Error, Kind, sample}; +use super::{Error, Kind, edit::Edit, sample}; use crate::Result; use crate::catalog::Estimator; @@ -12,7 +12,8 @@ use crate::catalog::Estimator; /// /// By default each fragment is unwrapped: every sample becomes one /// [`Legacy`](Container::Legacy) frame carrying the codec payload, timed at its presentation time, -/// so a consumer needs no ISO BMFF parser. [`with_passthrough`](Self::with_passthrough) instead +/// so a consumer needs no ISO BMFF parser. Either way, each track's edit list is applied to the +/// frame timestamps and left out of the catalog. [`with_passthrough`](Self::with_passthrough) instead /// republishes each fragment whole as one [`Cmaf`](Container::Cmaf) frame, for a source whose exact /// `trun` timing must survive. /// @@ -114,6 +115,9 @@ struct Fmp4Track { /// The `mdhd` timescale the samples count in. timescale: moq_net::Timescale, + /// The edit list, applied to every sample's presentation time. + edit: Edit, + // The decode time of the last fragment, which the next one has to advance past. last_decode_time: Option, @@ -201,11 +205,14 @@ macro_rules! media { /// One sample of a fragment, resolved against the track's defaults. struct Sample { - /// Decode time, in track ticks. + /// Decode time, in media ticks. dts: u64, - /// Presentation time, in track ticks. + /// Composition time, in media ticks. pts: u64, + /// Presentation time once the edit list applies, at the track's scale. + timestamp: Timestamp, duration: Option, + flags: u32, sync: bool, /// The sample's bytes within the `mdat` payload. range: std::ops::Range, @@ -218,6 +225,8 @@ struct Run { earliest: Timestamp, /// The furthest presentation end any sample reaches. end: Timestamp, + /// The edit list dropped some of the fragment's samples. + dropped: bool, } impl Import { @@ -396,6 +405,7 @@ impl Import { } let timescale = moq_net::Timescale::new(trak.mdia.mdhd.timescale as u64)?; + let edit = Edit::new(trak, moov.mvhd.timescale)?; let output = match self.passthrough { true => Output::Passthrough(Box::new( self.init_passthrough(trak, &moov, &reserved, kind, timescale)?, @@ -409,6 +419,7 @@ impl Import { kind, defaults: sample::Defaults::from_moov(&moov, track_id), timescale, + edit, last_decode_time: None, sample_duration: None, segment: None, @@ -519,6 +530,7 @@ impl Import { ), _ => return Ok(false), }; + let edit = Edit::new(trak, moov.mvhd.timescale)?; config.lang = Some(trak.mdia.mdhd.language.clone()); // The picker label is the HandlerBox name; `vlab` is an opaque source identifier. config.label = (!trak.mdia.hdlr.name.is_empty()).then(|| trak.mdia.hdlr.name.clone()); @@ -535,6 +547,7 @@ impl Import { format, defaults: sample::Defaults::from_moov(moov, trak.tkhd.track_id), timescale: moq_net::Timescale::new(trak.mdia.mdhd.timescale as u64)?, + edit, end_decode_time: None, discontinuity: false, }, @@ -570,9 +583,14 @@ impl Import { ..Default::default() }); + // The importer applies the edit list to every frame timestamp, so a consumer applying it + // again would shift the track twice. + let mut trak = trak.clone(); + trak.edts = None; + let single_moov = Moov { mvhd: moov.mvhd.clone(), - trak: vec![trak.clone()], + trak: vec![trak], mvex: Some(mp4_atom::Mvex { mehd: None, trex: vec![trex], @@ -942,7 +960,10 @@ impl Import { None => return Err(Error::UnknownTrack(track_id).into()), }; - let run = track.run(traf, moof_size, header_size, payload.len())?; + // A fragment wholly before the edit, such as one holding only priming samples. + let Some(run) = track.run(traf, moof_size, header_size, payload.len())? else { + continue; + }; for sample in &run.samples { claimed.take(sample.range.clone())?; } @@ -992,8 +1013,9 @@ struct Placement { } impl Fmp4Track { - /// Resolve the samples `traf` places in an mdat whose payload is `len` bytes. - fn run(&mut self, traf: &mp4_atom::Traf, moof_size: usize, header_size: usize, len: usize) -> Result { + /// Resolve the samples `traf` places in an mdat whose payload is `len` bytes, or `None` when + /// the edit list drops every one. + fn run(&mut self, traf: &mp4_atom::Traf, moof_size: usize, header_size: usize, len: usize) -> Result> { let tfdt = traf.tfdt.as_ref().ok_or(Error::MissingTfdt)?; let timescale = self.timescale; let mut dts = tfdt.base_media_decode_time; @@ -1023,6 +1045,8 @@ impl Fmp4Track { let mut end = None; let total_samples: usize = traf.trun.iter().map(|t| t.entries.len()).sum(); let mut samples = Vec::with_capacity(total_samples); + let mut index = 0; + let mut dropped = false; for trun in &traf.trun { if let Some(data_offset) = trun.data_offset { @@ -1045,7 +1069,7 @@ impl Fmp4Track { // A non-final sample with no resolvable duration leaves the rest of the // fragment's DTS ambiguous, so reject it rather than collapse timestamps. - if duration.is_none() && samples.len() + 1 != total_samples { + if duration.is_none() && index + 1 != total_samples { return Err(Error::MissingSampleDuration.into()); } @@ -1053,18 +1077,12 @@ impl Fmp4Track { let pts = dts .checked_add_signed(entry.cts.unwrap_or_default() as i64) .ok_or(Error::PtsOverflow)?; - let timestamp = Timestamp::new(pts, timescale)?; - let sample_end = if let Some(duration) = duration { - let duration = Timestamp::new(duration as u64, timescale)?; + let duration_ts = duration + .map(|duration| Timestamp::new(duration as u64, timescale)) + .transpose()?; + if let Some(duration) = duration_ts { self.sample_duration = Some(self.sample_duration.map_or(duration, |min| min.min(duration))); - timestamp.checked_add(duration)? - } else if let Some(duration) = self.sample_duration { - timestamp.checked_add(duration)? - } else { - timestamp - }; - end = Some(end.map_or(sample_end, |max: Timestamp| max.max(sample_end))); - earliest = Some(earliest.map_or(timestamp, |min: Timestamp| min.min(timestamp))); + } // Each sample must lie inside the mdat, or the sizes and offsets disagree with the // data and it cannot be addressed safely. @@ -1080,26 +1098,48 @@ impl Fmp4Track { .into()); } - samples.push(Sample { - dts, - pts, - duration, - sync: sample.is_sync(), - range: offset..next, - }); + if let Some(presented) = self.edit.present(pts)? { + let timestamp = Timestamp::new(presented, timescale)?; + let sample_end = match duration_ts.or(self.sample_duration) { + Some(duration) => timestamp.checked_add(duration)?, + None => timestamp, + }; + end = Some(end.map_or(sample_end, |max: Timestamp| max.max(sample_end))); + earliest = Some(earliest.map_or(timestamp, |min: Timestamp| min.min(timestamp))); + + samples.push(Sample { + dts, + pts, + timestamp, + duration, + flags: sample.flags, + sync: sample.is_sync(), + range: offset..next, + }); + } else { + dropped = true; + } if let Some(duration) = duration { dts = dts.checked_add(duration as u64).ok_or(Error::PtsOverflow)?; } offset = next; + index += 1; } } - Ok(Run { + let (Some(earliest), Some(end)) = (earliest, end) else { + return match dropped { + true => Ok(None), + false => Err(Error::MissingTrun.into()), + }; + }; + Ok(Some(Run { samples, - earliest: earliest.ok_or(Error::MissingTrun)?, - end: end.ok_or(Error::MissingTrun)?, - }) + earliest, + end, + dropped, + })) } } @@ -1118,7 +1158,10 @@ fn fragment( // with only the samples belonging to this track. let mut adjusted_moof = Moof { mfhd: moof.mfhd.clone(), - traf: vec![traf.clone()], + traf: vec![match run.dropped { + true => kept(traf, run)?, + false => traf.clone(), + }], }; // Apply structural (flag-presence) changes BEFORE measuring the encoded @@ -1190,6 +1233,41 @@ fn fragment( Ok(Bytes::from(moof_buf)) } +/// `traf` restated as one run of `run`'s samples, for a fragment the edit list dropped some of. +/// +/// Each sample keeps its decode and composition time: a dropped sample's duration goes to the +/// sample before it, or into `tfdt` when it led the fragment. The per-sample tables (`sbgp`, +/// `subs`, sample auxiliary information) would index the wrong samples, and no consumer reads +/// them, so they go. +fn kept(traf: &mp4_atom::Traf, run: &Run) -> Result { + let mut entries = Vec::with_capacity(run.samples.len()); + for (index, sample) in run.samples.iter().enumerate() { + let duration = match run.samples.get(index + 1) { + Some(next) => Some(u32::try_from(next.dts - sample.dts).map_err(|_| Error::PtsOverflow)?), + None => sample.duration, + }; + let cts = i128::from(sample.pts) - i128::from(sample.dts); + entries.push(mp4_atom::TrunEntry { + duration, + size: Some(sample.range.len() as u32), + flags: Some(sample.flags), + cts: Some(i32::try_from(cts).map_err(|_| Error::PtsOverflow)?), + }); + } + let first = run.samples.first().ok_or(Error::MissingTrun)?; + Ok(mp4_atom::Traf { + tfhd: traf.tfhd.clone(), + tfdt: Some(mp4_atom::Tfdt { + base_media_decode_time: first.dts, + }), + trun: vec![mp4_atom::Trun { + data_offset: None, + entries, + }], + ..Default::default() + }) +} + impl Passthrough { /// Write one per-track `fragment` as a single frame. fn write(&mut self, fragment: Bytes, run: &Run, kind: Kind, placement: Placement) -> Result<()> { @@ -1302,7 +1380,7 @@ fn unwrap( producer.reorder(Timestamp::new(sample.pts - sample.dts, timescale)?); } let frame = crate::container::Frame { - timestamp: placement.offset.apply(Timestamp::new(sample.pts, timescale)?)?, + timestamp: placement.offset.apply(sample.timestamp)?, payload: payload.slice(sample.range.clone()), keyframe, duration: sample diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index e9193feac3a..031b3a7eeed 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -819,6 +819,151 @@ async fn unwrapped_samples_present_at_their_composition_time() { ); } +/// bbb's init with `entries` as the edit list of track `track_id`. +fn bbb_init_with_edits(track_id: u32, entries: Vec) -> Vec { + let (init, _, _) = bbb_init(); + let (ftyp, mut moov) = decode_init(&init); + let trak = moov + .trak + .iter_mut() + .find(|trak| trak.tkhd.track_id == track_id) + .unwrap(); + trak.edts = Some(mp4_atom::Edts { + elst: Some(mp4_atom::Elst { entries }), + }); + let mut data = Vec::new(); + ftyp.encode(&mut data).unwrap(); + moov.encode(&mut data).unwrap(); + data +} + +/// An edit starting the media at `media_time`. +fn media_edit(media_time: u64) -> mp4_atom::ElstEntry { + mp4_atom::ElstEntry { + segment_duration: 0, + media_time: Some(media_time), + media_rate: mp4_atom::FixedPoint::from(1), + } +} + +/// The edit list moves every sample to its presentation time and drops the samples before the +/// edit, priming that fills a frame or straddles one, passthrough or unwrapped. The passthrough +/// init loses the `edts`, so no consumer applies it a second time. +#[tokio::test] +async fn audio_presents_from_its_edit() { + let (_, _, (audio_id, audio_scale)) = bbb_init(); + let micros = |ticks: u64| u128::from(ticks) * 1_000_000 / u128::from(audio_scale.as_u64()); + let empty = mp4_atom::ElstEntry { + // bbb's movie counts milliseconds. + segment_duration: 1000, + media_time: None, + media_rate: mp4_atom::FixedPoint::from(1), + }; + // The samples start at 0 and 512, then every 1024 ticks. + let cases = [ + // Priming of whole frames, as ffmpeg's AAC. + (vec![media_edit(512)], vec![0, 1024, 2048]), + // Priming that ends inside a frame drops that frame too, as Apple's AAC. + (vec![media_edit(1024)], vec![512, 1536]), + // An empty edit delays the media. + (vec![empty, media_edit(512)], vec![44_100, 45_124, 46_148]), + ]; + + for (entries, expected) in cases { + for passthrough in [false, true] { + let init = bbb_init_with_edits(audio_id, entries.clone()); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = + crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); + fmp4.decode(&init).unwrap(); + // A fragment wholly before the edit publishes nothing. + fmp4.decode(&audio_fragment(0, 512, 4)).unwrap(); + fmp4.decode(&audio_fragment_samples(512, 1024, 4, 3)).unwrap(); + fmp4.finish().unwrap(); + + let snapshot = catalog.snapshot(); + if let Container::Cmaf { init } = &snapshot.audio.renditions.values().next().unwrap().container { + let (_, moov) = decode_init(init); + assert!(moov.trak[0].edts.is_none(), "the init keeps no edit list"); + } + let [_, audio] = published(&consumer, &catalog).await; + let got: Vec<_> = audio.iter().map(|(timestamp, _, _)| *timestamp).collect(); + let expected: Vec<_> = expected.iter().map(|&ticks| micros(ticks)).collect(); + assert_eq!(got, expected, "{entries:?}, passthrough {passthrough}"); + } + } +} + +/// A dropped sample between kept ones, such as an open GOP's leading picture, leaves every kept +/// sample at its own presentation time and in decode order, passthrough or unwrapped. +#[tokio::test] +async fn video_drops_the_samples_before_its_edit() { + let (_, (video_id, video_scale), _) = bbb_init(); + let tick = video_scale.as_u64() / 24; + let init = bbb_init_with_edits(video_id, vec![media_edit(2 * tick)]); + let non_sync = 0x0001_0000; + + // I, a leading B presenting before it, P, then B, in decode order. + let entries = [ + (2 * tick, 0, 0xA), + (0, non_sync, 0xB), + (2 * tick, non_sync, 0xC), + (0, non_sync, 0xD), + ]; + let moof = mp4_atom::Moof { + mfhd: mp4_atom::Mfhd { sequence_number: 1 }, + traf: vec![mp4_atom::Traf { + tfhd: mp4_atom::Tfhd { + track_id: video_id, + ..Default::default() + }, + tfdt: Some(mp4_atom::Tfdt { + base_media_decode_time: 0, + }), + trun: vec![mp4_atom::Trun { + data_offset: None, + entries: entries + .iter() + .map(|&(cts, flags, _)| mp4_atom::TrunEntry { + duration: Some(tick as u32), + size: Some(2), + flags: Some(flags), + cts: Some(cts as i32), + }) + .collect(), + }], + ..Default::default() + }], + }; + let mut fragment = Vec::new(); + moof.encode(&mut fragment).unwrap(); + mp4_atom::Mdat { + data: entries.iter().flat_map(|&(_, _, byte)| [byte; 2]).collect(), + } + .encode(&mut fragment) + .unwrap(); + + let micros = |ticks: u64| u128::from(ticks) * 1_000_000 / u128::from(video_scale.as_u64()); + let expected = vec![ + (0, true, bytes::Bytes::from_static(&[0xA; 2])), + (micros(2 * tick), false, bytes::Bytes::from_static(&[0xC; 2])), + (micros(tick), false, bytes::Bytes::from_static(&[0xD; 2])), + ]; + for passthrough in [false, true] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); + fmp4.decode(&init).unwrap(); + fmp4.decode(&fragment).unwrap(); + fmp4.finish().unwrap(); + let [video, _] = published(&consumer, &catalog).await; + assert_eq!(video, expected, "passthrough {passthrough}"); + } +} + #[test] fn test_bbb_init_roundtrip() { let data = include_bytes!("test_data/bbb.mp4"); diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 941c186c092..b38d42b8baa 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -6,6 +6,7 @@ //! container; [`Import`] parses external fMP4 streams and [`Export`] //! produces them. +mod edit; mod export; pub mod fragment; mod fragmenter; @@ -303,6 +304,10 @@ pub enum Error { /// delivered the codec configuration their track needs. #[error("renditions {0:?} never delivered their codec configuration")] TrackUndescribed(Vec), + + /// A track's edit list does more than delay the track and start its media at one time. + #[error("unsupported edit list: {0}")] + UnsupportedEditList(&'static str), } impl From for Error { diff --git a/rs/moq-mux/src/container/fmp4/text.rs b/rs/moq-mux/src/container/fmp4/text.rs index 7f8f08f3785..ae0df610809 100644 --- a/rs/moq-mux/src/container/fmp4/text.rs +++ b/rs/moq-mux/src/container/fmp4/text.rs @@ -1,7 +1,7 @@ use moq_net::{Timescale, Timestamp}; use mp4_atom::{Any, DecodeMaybe}; -use super::{Error, sample}; +use super::{Error, edit::Edit, sample}; use crate::container::text::{self, Cue}; use crate::container::ttml; @@ -37,6 +37,8 @@ pub(super) struct Track { pub format: Format, pub defaults: sample::Defaults, pub timescale: Timescale, + /// The edit list, applied to every cue's presentation time. + pub edit: Edit, /// The exclusive decode end of the previous fragment; the next one may not start before it. pub end_decode_time: Option, pub discontinuity: bool, @@ -91,15 +93,23 @@ impl Track { let pts = dts .checked_add_signed(entry.cts.unwrap_or_default() as i64) .ok_or(Error::PtsOverflow)?; - let source = Timestamp::new(pts, self.timescale)?; - let timestamp = timebase.shift(source)?; let duration = Timestamp::new(duration as u64, self.timescale)?; - let end = timestamp.checked_add(duration)?; let next = offset .checked_add(sample.size as usize) .ok_or(Error::InvalidDataOffset)?; let data = input.data.get(offset..next).ok_or(Error::InvalidDataOffset)?; input.claimed.take(offset..next)?; + offset = next; + dts = dts.checked_add(duration.value()).ok_or(Error::PtsOverflow)?; + + // A cue starting before the edit is dropped, like a media sample. + let Some(presented) = self.edit.present(pts)? else { + continue; + }; + // TTML maps the document's media times from `source`, so they move with the edit too. + let source = Timestamp::new(pts, self.timescale)?; + let timestamp = timebase.shift(Timestamp::new(presented, self.timescale)?)?; + let end = timestamp.checked_add(duration)?; let payload = match &self.format { Format::Vtt { header } => vtt(header, data, timestamp, end)?, // The frame timestamp is serialized in hang's scale, which the body's begin must match. @@ -118,8 +128,6 @@ impl Track { payload, })?; self.producer.cut(None)?; - offset = next; - dts = dts.checked_add(duration.value()).ok_or(Error::PtsOverflow)?; } } self.end_decode_time = Some(dts); From 2de40c9836694116e303ec8331b2a8bd128e7414 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 12:55:04 -0700 Subject: [PATCH 3/3] fix(mux): keep fMP4 edits exact for Opus and dropped sync samples An Opus edit starting inside the OpusHead pre-skip saturated to no shift, misplacing every kept sample by the rest of the pre-skip; it now delays the track by that much instead. A sync sample the edit drops also drops the samples decoding from it, so a passthrough track no longer opens a group on an undecodable P frame. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/CHANGELOG.md | 2 +- rs/moq-mux/src/container/fmp4/edit.rs | 22 ++-- rs/moq-mux/src/container/fmp4/import.rs | 14 ++- rs/moq-mux/src/container/fmp4/import_test.rs | 100 +++++++++++++------ 4 files changed, 96 insertions(+), 42 deletions(-) diff --git a/rs/moq-mux/CHANGELOG.md b/rs/moq-mux/CHANGELOG.md index a76ac5c3bdd..520d6a4860e 100644 --- a/rs/moq-mux/CHANGELOG.md +++ b/rs/moq-mux/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- fMP4 import applies each track's edit list to its frame timestamps and drops the samples before the edit, such as AAC priming, and a passthrough init no longer carries the `edts`. A consumer that offsets AAC priming itself must stop, or it shifts the track twice. An edit list it can't reduce to one shift (several media edits, or a rate other than 1) is refused with `Error::UnsupportedEditList`. +- fMP4 import applies each track's edit list to its frame timestamps and drops the samples before the edit, such as AAC priming, along with the video samples decoding from a dropped sync sample, and a passthrough init no longer carries the `edts`. A consumer that offsets AAC priming itself must stop, or it shifts the track twice. An edit list it can't reduce to one shift (several media edits, or a rate other than 1) is refused with `Error::UnsupportedEditList`. ## [0.10.11](https://github.com/moq-dev/moq/compare/moq-mux-v0.10.10...moq-mux-v0.10.11) - 2026-10-05 diff --git a/rs/moq-mux/src/container/fmp4/edit.rs b/rs/moq-mux/src/container/fmp4/edit.rs index 06a14aeea61..f477338adb8 100644 --- a/rs/moq-mux/src/container/fmp4/edit.rs +++ b/rs/moq-mux/src/container/fmp4/edit.rs @@ -56,8 +56,10 @@ impl Edit { .map_err(|_| Error::PtsOverflow)?, }; - // An Opus frame timestamp already counts the OpusHead pre-skip (draft-lcurley-moq-hang), and - // the edit includes it, so only the part beyond it moves the timestamps. + // A consumer trims the OpusHead pre-skip from the first decoded frame and plays the first + // sample it keeps at the frame timestamp (draft-lcurley-moq-hang), while in ISO BMFF the edit + // alone trims. So the edit moves an Opus frame by its distance from the pre-skip: an edit + // starting inside the pre-skip delays the kept samples by the rest of it. let skip = match trak.mdia.minf.stbl.stsd.codecs.as_slice() { [mp4_atom::Codec::Opus(opus)] => { u64::try_from(u128::from(opus.dops.pre_skip) * timescale / 48_000).map_err(|_| Error::PtsOverflow)? @@ -65,9 +67,12 @@ impl Edit { _ => 0, }; - Ok(Self { - start: media_time.saturating_sub(skip), - delay, + Ok(match media_time.checked_sub(skip) { + Some(start) => Self { start, delay }, + None => Self { + start: 0, + delay: delay.checked_add(skip - media_time).ok_or(Error::PtsOverflow)?, + }, }) } @@ -150,7 +155,12 @@ mod tests { true, [None, None, Some(0), Some(40)], ), - (Some(vec![entry(0, Some(100), 1)]), true, none), + // An edit inside the pre-skip shows media 312, the first sample a consumer keeps, at 212. + ( + Some(vec![entry(0, Some(100), 1)]), + true, + [Some(212), Some(1171), Some(1172), Some(1212)], + ), ]; for (entries, opus, expected) in cases { let edit = Edit::new(&trak(entries.clone(), opus), 1000).unwrap(); diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index b21be4b7755..422a29bf064 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -118,6 +118,9 @@ struct Fmp4Track { /// The edit list, applied to every sample's presentation time. edit: Edit, + /// The edit list dropped the last sync sample, so the samples decoding from it go too. + orphaned: bool, + // The decode time of the last fragment, which the next one has to advance past. last_decode_time: Option, @@ -420,6 +423,7 @@ impl Import { defaults: sample::Defaults::from_moov(&moov, track_id), timescale, edit, + orphaned: false, last_decode_time: None, sample_duration: None, segment: None, @@ -1098,7 +1102,11 @@ impl Fmp4Track { .into()); } - if let Some(presented) = self.edit.present(pts)? { + let presented = self.edit.present(pts)?; + if sample.is_sync() { + self.orphaned = presented.is_none(); + } + if let Some(presented) = presented.filter(|_| !self.orphaned) { let timestamp = Timestamp::new(presented, timescale)?; let sample_end = match duration_ts.or(self.sample_duration) { Some(duration) => timestamp.checked_add(duration)?, @@ -1235,8 +1243,8 @@ fn fragment( /// `traf` restated as one run of `run`'s samples, for a fragment the edit list dropped some of. /// -/// Each sample keeps its decode and composition time: a dropped sample's duration goes to the -/// sample before it, or into `tfdt` when it led the fragment. The per-sample tables (`sbgp`, +/// Each sample keeps its decode and composition time: `tfdt` starts at the first kept sample, and +/// each kept sample lasts until the next one decodes. The per-sample tables (`sbgp`, /// `subs`, sample auxiliary information) would index the wrong samples, and no consumer reads /// them, so they go. fn kept(traf: &mp4_atom::Traf, run: &Run) -> Result { diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 031b3a7eeed..5e717ce4d67 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -896,31 +896,17 @@ async fn audio_presents_from_its_edit() { } } -/// A dropped sample between kept ones, such as an open GOP's leading picture, leaves every kept -/// sample at its own presentation time and in decode order, passthrough or unwrapped. -#[tokio::test] -async fn video_drops_the_samples_before_its_edit() { - let (_, (video_id, video_scale), _) = bbb_init(); - let tick = video_scale.as_u64() / 24; - let init = bbb_init_with_edits(video_id, vec![media_edit(2 * tick)]); - let non_sync = 0x0001_0000; - - // I, a leading B presenting before it, P, then B, in decode order. - let entries = [ - (2 * tick, 0, 0xA), - (0, non_sync, 0xB), - (2 * tick, non_sync, 0xC), - (0, non_sync, 0xD), - ]; +/// One video fragment decoding from `dts`, a sample per `(cts, flags, byte)` lasting `tick`. +fn video_fragment(track_id: u32, dts: u64, tick: u64, entries: &[(u64, u32, u8)]) -> Vec { let moof = mp4_atom::Moof { mfhd: mp4_atom::Mfhd { sequence_number: 1 }, traf: vec![mp4_atom::Traf { tfhd: mp4_atom::Tfhd { - track_id: video_id, + track_id, ..Default::default() }, tfdt: Some(mp4_atom::Tfdt { - base_media_decode_time: 0, + base_media_decode_time: dts, }), trun: vec![mp4_atom::Trun { data_offset: None, @@ -944,23 +930,73 @@ async fn video_drops_the_samples_before_its_edit() { } .encode(&mut fragment) .unwrap(); + fragment +} +/// The edit list drops the video samples before it and keeps every other one at its own +/// presentation time and in decode order, passthrough or unwrapped. A dropped sync sample takes the +/// samples decoding from it along, so the track resumes at the next one. +#[tokio::test] +async fn video_drops_the_samples_before_its_edit() { + let (_, (video_id, video_scale), _) = bbb_init(); + let tick = video_scale.as_u64() / 24; let micros = |ticks: u64| u128::from(ticks) * 1_000_000 / u128::from(video_scale.as_u64()); - let expected = vec![ - (0, true, bytes::Bytes::from_static(&[0xA; 2])), - (micros(2 * tick), false, bytes::Bytes::from_static(&[0xC; 2])), - (micros(tick), false, bytes::Bytes::from_static(&[0xD; 2])), + let sample = |ticks: u64, keyframe: bool, byte: u8| (micros(ticks), keyframe, bytes::Bytes::from(vec![byte; 2])); + let non_sync = 0x0001_0000; + + let cases = [ + // I, a leading B presenting before it, P, then B, in decode order. + ( + 2 * tick, + vec![video_fragment( + video_id, + 0, + tick, + &[ + (2 * tick, 0, 0xA), + (0, non_sync, 0xB), + (2 * tick, non_sync, 0xC), + (0, non_sync, 0xD), + ], + )], + vec![ + sample(0, true, 0xA), + sample(2 * tick, false, 0xC), + sample(tick, false, 0xD), + ], + ), + // An edit after the first I, as a cut at a non-sync sample writes: the P frames after the + // edit can't decode without it. + ( + tick, + vec![ + video_fragment( + video_id, + 0, + tick, + &[(0, 0, 0xA), (0, non_sync, 0xB), (0, non_sync, 0xC)], + ), + video_fragment(video_id, 3 * tick, tick, &[(0, 0, 0xD), (0, non_sync, 0xE)]), + ], + vec![sample(2 * tick, true, 0xD), sample(3 * tick, false, 0xE)], + ), ]; - for passthrough in [false, true] { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let consumer = broadcast.consume(); - let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); - fmp4.decode(&init).unwrap(); - fmp4.decode(&fragment).unwrap(); - fmp4.finish().unwrap(); - let [video, _] = published(&consumer, &catalog).await; - assert_eq!(video, expected, "passthrough {passthrough}"); + for (media_time, fragments, expected) in cases { + let init = bbb_init_with_edits(video_id, vec![media_edit(media_time)]); + for passthrough in [false, true] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = + crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); + fmp4.decode(&init).unwrap(); + for fragment in &fragments { + fmp4.decode(fragment).unwrap(); + } + fmp4.finish().unwrap(); + let [video, _] = published(&consumer, &catalog).await; + assert_eq!(video, expected, "edit at {media_time}, passthrough {passthrough}"); + } } }