From 598f29e42261449dbaf4f1b8857e2ed908bbf0ed Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 29 Oct 2026 21:22:35 -0700 Subject: [PATCH] fix(mux): preserve Opus timing in fMP4 exports (backport #7118) Hang places every decoded Opus sample one pre-skip before its packet timestamp, but the fMP4 exporter dropped every edit list, so an MP4 reader presented Opus one pre-skip late. Rebuild the Opus priming edit when the init is assembled, after the output timescales are settled. Adapted to release's exporter, which has no edit-list import. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/fmp4/export.rs | 5 +- rs/moq-mux/src/container/fmp4/mod.rs | 84 ++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index 688afdc758f..fa2f11d0ecf 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -672,6 +672,8 @@ pub(crate) fn extract_init( // composition offsets, so an edit list is redundant here, and a // browser applying an empty-edit media_time would shift the track // off the others (a black screen in Media Source Extensions). + // encode_init restores only Opus priming's presentation shift + // after the output track timescales have been settled. trak.edts = None; // tkhd.duration is in the *movie* timescale, and the merged moov // picks its own (the first trak's media scale), so whatever the @@ -1198,6 +1200,7 @@ mod tests { // A source init whose trak carries an edit list must come out of extract_init with // the edit list dropped: CMAF carries timing via tfdt + composition offsets, and a // browser applying an edit list shifts the track off the others (a black screen). + // encode_init rebuilds any codec-specific presentation shift afterwards. #[test] fn extract_init_strips_edit_lists() { use mp4_atom::Encode; @@ -1223,7 +1226,7 @@ mod tests { extract_init(&Bytes::from(init), 1, &mut ftyp, &mut traks, &mut trexs).unwrap(); assert_eq!(traks.len(), 1); - assert!(traks[0].edts.is_none(), "CMAF init must not carry an edit list"); + assert!(traks[0].edts.is_none(), "the source edit must not survive extraction"); } // tkhd.duration is in the movie timescale, which the merged moov replaces with its own. A diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index f44a39e698a..978b0f64231 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -873,7 +873,7 @@ fn build_audio_trak(track_id: u32, timescale: u32, sample_entry: mp4_atom::Codec /// [`extract_init`] normalizes a passed-through trak to match. pub(crate) fn encode_init( ftyp: Option, - traks: Vec, + mut traks: Vec, trexs: Vec, ) -> Result { use mp4_atom::Encode; @@ -883,6 +883,24 @@ pub(crate) fn encode_init( minor_version: 0x200, compatible_brands: vec![b"isom".into(), b"iso6".into(), b"mp41".into()], }); + // Hang places every decoded Opus sample one pre-skip before its packet timestamp. + // dOps alone describes decoder priming; ISO BMFF needs an edit to express that + // presentation shift. Rebuild it after the output track timescales are settled. + for trak in &mut traks { + if let [mp4_atom::Codec::Opus(opus)] = trak.mdia.minf.stbl.stsd.codecs.as_slice() { + let skip = u64::from(opus.dops.pre_skip) * u64::from(trak.mdia.mdhd.timescale) / 48_000; + trak.edts = (skip != 0).then(|| mp4_atom::Edts { + elst: Some(mp4_atom::Elst { + entries: vec![mp4_atom::ElstEntry { + // A live edit's duration is implicit in the fragments that follow. + segment_duration: 0, + media_time: Some(skip), + media_rate: mp4_atom::FixedPoint::from(1), + }], + }), + }); + } + } let timescale = traks.first().map(|t| t.mdia.mdhd.timescale).unwrap_or(1000); let next_track_id = traks.iter().map(|t| t.tkhd.track_id).max().unwrap_or(0) + 1; @@ -1204,6 +1222,70 @@ mod tests { assert_eq!(moov.trak[0].tkhd.duration, u64::MAX); } + /// Opus media time counts priming; Hang time places the first kept sample at the + /// first frame timestamp. Both init sources must express that shift as one edit in + /// the output track's timescale, replacing whatever edit the source carried. + #[test] + fn opus_init_maps_hang_timestamps_to_media() { + for timescale in [48_000, 1_000_000] { + for pre_skip in [0, 312, u16::MAX] { + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + config.description = Some( + crate::codec::opus::Config::new(48_000, 2) + .with_pre_skip(pre_skip) + .encode() + .unwrap(), + ); + for source_edit in [false, true] { + let mut trak = synthesize_audio_trak(1, timescale, &config).unwrap(); + if source_edit { + trak.edts = Some(mp4_atom::Edts { + elst: Some(mp4_atom::Elst { + entries: vec![mp4_atom::ElstEntry { + media_time: Some(96000), + ..Default::default() + }], + }), + }); + let mut init = Vec::new(); + mp4_atom::Encode::encode( + &mp4_atom::Moov { + trak: vec![trak], + ..Default::default() + }, + &mut init, + ) + .unwrap(); + let mut traks = Vec::new(); + extract_init(&Bytes::from(init), 1, &mut None, &mut traks, &mut Vec::new()).unwrap(); + trak = traks.pop().unwrap(); + } + let init = encode_init(None, vec![trak], Vec::new()).unwrap(); + let moov = moov(&init); + let trak = &moov.trak[0]; + let skip = u64::from(pre_skip) * timescale / 48_000; + if skip == 0 { + assert!(trak.edts.is_none()); + } else { + let entries = &trak.edts.as_ref().unwrap().elst.as_ref().unwrap().entries; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].media_time, Some(skip)); + assert_eq!(entries[0].segment_duration, 0, "the live duration is implicit"); + assert_eq!(entries[0].media_rate, mp4_atom::FixedPoint::from(1)); + } + } + } + } + } + + /// Only Opus carries priming: other codecs keep the edit-free init. + #[test] + fn non_opus_init_has_no_edit() { + let trak = synthesize_audio_trak(1, 44_100, &aac_config(None)).unwrap(); + let init = encode_init(None, vec![trak], Vec::new()).unwrap(); + assert!(moov(&init).trak[0].edts.is_none()); + } + // The header defaults are all the "off" value: a track the presentation doesn't claim, a // stopped playback rate, a muted volume, and a next id that collides with the track already // there. A strict player is entitled to act on any of them.