From 08da3d44eadba770924d3ba6ffaafb63cb86bc96 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 23:37:37 -0700 Subject: [PATCH 1/2] chore(quest): claim quest/m1/fmp4-passthrough-sync.md Quest-Claim: 59c0063404e1a60d From 78dac27b1d0e8ba3a8516b4fe4da394359f5489f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 00:08:11 -0700 Subject: [PATCH 2/2] fix(mux): start fMP4 passthrough video at a sync sample A passthrough video track joined mid-GOP, at the start or after a seek, published its leading non-sync samples as the start of a group. Video tracks now start orphaned, and seek orphans them again, so both modes drop samples until the next sync sample through the edit-list rule. The unwrapped path's MissingKeyframe skip is unreachable and goes. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/fmp4-passthrough-sync.md | 20 -------- rs/moq-mux/src/container/fmp4/import.rs | 38 +++++++------- rs/moq-mux/src/container/fmp4/import_test.rs | 52 ++++++++++++++++++++ 4 files changed, 70 insertions(+), 41 deletions(-) delete mode 100644 quest/m1/fmp4-passthrough-sync.md diff --git a/quest/m1/README.md b/quest/m1/README.md index e54cac761a4..f7034039c55 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -123,7 +123,6 @@ Catalog HLS field. - [More tests under load](/quest/m1/test-flakes-2/README.md) - the second round of load-only failures, one quest per flake, fixed at the cause - [Broadcast lifecycle](/quest/m1/broadcast-lifecycle/README.md) - broadcasts are online or offline with no finished state in moq-net, finished-broadcast retention is a fixed moq-mux policy, and each linger timer has one meaning - [fMP4 edit lists](/quest/m1/fmp4-edit-lists.md) - the fMP4 importer applies edit lists to frame timestamps, so the interop AAC tone passes the phase check -- [fMP4 passthrough sync start](/quest/m1/fmp4-passthrough-sync.md) - a passthrough video import that starts mid-GOP publishes nothing before its first sync sample - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate - [moq-transport 18 and 22 only](/quest/m1/ietf-drop/README.md) - drafts 14-17 and 19-21 are deleted from Rust, JS, tests, interop lanes, and docs, removing the legacy control-stream adapter and its request limit - [moq-quic's API improves on quinn's](/quest/m1/quic-api/README.md) - moq-quic and both QUIC drivers ship one mirrored API: `send::`/`recv::` namespaces, one error per direction, consuming terminal operations, plain configs, no quinn baggage diff --git a/quest/m1/fmp4-passthrough-sync.md b/quest/m1/fmp4-passthrough-sync.md deleted file mode 100644 index 2772607ea86..00000000000 --- a/quest/m1/fmp4-passthrough-sync.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] fMP4 passthrough starts at a sync sample - -## Goal - -An fMP4 passthrough video track whose import starts mid-GOP publishes nothing -until its first sync sample, as the unwrapped path already does, so no group -starts with frames a decoder cannot use. - -## Plan - -Decided with the maintainer on 2026-10-24: m1, a correctness bug in a shipped -import path. - -- Found reviewing [moq-dev/moq#6686](https://github.com/moq-dev/moq/pull/6686): - the unwrapped path skips to a sync sample, while passthrough publishes the - leading non-sync fragments as the start of a group. -- #6686 adds `Fmp4Track::orphaned` to drop samples that depend on a sync - sample its edit list dropped; reuse it for a join before the first sync - sample rather than adding a second rule. -- Start after #6686 lands. Test both modes with a fragment that starts mid-GOP. diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index a15af730b47..9e58d677b7c 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -118,7 +118,8 @@ struct Fmp4Track { /// The edit list, applied to every sample's presentation time. edit: Edit, - /// The edit list dropped the last sync sample, so the samples decoding from it go too. + /// No sync sample precedes the next sample, since the import joined mid-GOP or the edit list + /// dropped the last one, so the samples decoding from it go too. orphaned: bool, // The decode time of the last fragment, which the next one has to advance past. @@ -425,7 +426,8 @@ impl Import { defaults: sample::Defaults::from_moov(&moov, track_id), timescale, edit, - orphaned: false, + // Audio samples decode alone, whatever their sync flags say. + orphaned: kind == Kind::Video, last_decode_time: None, sample_duration: None, segment: None, @@ -966,7 +968,9 @@ impl Import { None => return Err(Error::UnknownTrack(track_id).into()), }; - // A fragment wholly before the edit, such as one holding only priming samples. + // A fragment wholly before the edit or the first sync sample, such as priming samples or + // a mid-GOP join. Skipped before the segment is counted as opened, or the next keyframe + // would roll a second segment. let Some(run) = track.run(traf, moof_size, header_size, payload.len())? else { continue; }; @@ -988,19 +992,14 @@ impl Import { declared: new_segment && self.declared, }; - let opened = match &mut track.output { + match &mut track.output { Output::Passthrough(passthrough) => { let fragment = fragment(traf, &moof, &run, &payload, &track.defaults)?; passthrough.write(fragment, &run, track.kind, placement)?; - true } output => unwrap(output, &run, &payload, track.timescale, placement)?, - }; - // A video track joined mid-GOP skips samples until a sync sample, so its segment opens - // there; counting a skipped fragment would let the next keyframe roll a second segment. - if opened { - track.segment = Some(self.segment); } + track.segment = Some(self.segment); } Ok(()) @@ -1359,15 +1358,13 @@ impl Passthrough { } /// Write each sample of `run` as its own frame on an unwrapped track. -/// -/// Returns whether the track now has a group open, which video lacks until its first sync sample. fn unwrap( output: &mut Output, run: &Run, payload: &Bytes, timescale: moq_net::Timescale, placement: Placement, -) -> Result { +) -> Result<()> { let earliest = placement.offset.apply(run.earliest)?; let end = placement.offset.apply(run.end)?; let video = matches!(output, Output::Video(_)); @@ -1398,16 +1395,12 @@ fn unwrap( .map(|duration| Timestamp::new(duration as u64, timescale)) .transpose()?, }; - match producer.write(frame) { - // A source joined mid-GOP: skip ahead to the first sync sample. - Ok(()) | Err(crate::Error::MissingKeyframe(_)) => {} - Err(err) => return Err(err), - } + producer.write(frame)?; } // The fragment arrived at once, so a consumer has to buffer its whole span. producer.burst(end.checked_sub(earliest)?.into())?; - Ok(!producer.needs_keyframe()) + Ok(()) }, _ => unreachable!("passthrough tracks write whole fragments")) } @@ -1457,13 +1450,18 @@ impl Import { /// control is intentionally not exposed. Skipping sequences is not a new timeline, so the /// next fragment must still advance past the last decode time; see /// [`discontinuity`](Self::discontinuity). Text spends a group per cue, so a text track that - /// is already past `sequence` continues from its own next group instead. + /// is already past `sequence` continues from its own next group instead. Video resumes at its + /// next sync sample, dropping the samples before it. pub fn seek(&mut self, sequence: u64) -> Result<()> { for track in self.text.values_mut() { let next = track.producer.next_sequence(); track.producer.seek(sequence.max(next))?; } for track in self.tracks.values_mut() { + // The next group opens on a sync sample, so video drops the samples before one. + if track.kind == Kind::Video { + track.orphaned = true; + } media!(&mut track.output, producer => producer.seek(sequence)?, passthrough => { passthrough.close()?; passthrough.pending_sequence = Some(sequence); diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index f056acd3774..b3904269fe0 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1000,6 +1000,58 @@ async fn video_drops_the_samples_before_its_edit() { } } +/// A video track joined mid-GOP, at the start or after a `seek`, publishes nothing before its next +/// sync sample, passthrough or unwrapped, even when that sample sits inside a fragment, so no group +/// opens on a frame its decoder cannot use. +#[tokio::test] +async fn video_joined_mid_gop_starts_at_a_sync_sample() { + let (_, (video_id, video_scale), _) = bbb_init(); + let tick = video_scale.as_u64() / 24; + let micros = |ticks: u64| u128::from(ticks) * 1_000_000 / u128::from(video_scale.as_u64()); + let sample = |ticks: u64, keyframe: bool, byte: u8| (micros(ticks), keyframe, bytes::Bytes::from(vec![byte; 2])); + let non_sync = 0x0001_0000; + + let init = bbb_init_with_edits(video_id, vec![media_edit(0)]); + let fragments = [ + video_fragment(video_id, 0, tick, &[(0, non_sync, 0xA), (0, non_sync, 0xB)]), + video_fragment( + video_id, + 2 * tick, + tick, + &[(0, non_sync, 0xC), (0, 0, 0xD), (0, non_sync, 0xE)], + ), + video_fragment(video_id, 5 * tick, tick, &[(0, 0, 0xF), (0, non_sync, 0x10)]), + ]; + let after_seek = [ + video_fragment(video_id, 7 * tick, tick, &[(0, non_sync, 0x11)]), + video_fragment(video_id, 8 * tick, tick, &[(0, non_sync, 0x12), (0, 0, 0x13)]), + ]; + let expected = vec![ + sample(3 * tick, true, 0xD), + sample(4 * tick, false, 0xE), + sample(5 * tick, true, 0xF), + sample(6 * tick, false, 0x10), + sample(9 * tick, true, 0x13), + ]; + for passthrough in [false, true] { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); + fmp4.decode(&init).unwrap(); + for fragment in &fragments { + fmp4.decode(fragment).unwrap(); + } + fmp4.seek(10).unwrap(); + for fragment in &after_seek { + fmp4.decode(fragment).unwrap(); + } + fmp4.finish().unwrap(); + let [video, _] = published(&consumer, &catalog).await; + assert_eq!(video, expected, "passthrough {passthrough}"); + } +} + #[test] fn test_bbb_init_roundtrip() { let data = include_bytes!("test_data/bbb.mp4");