Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion quest/m1/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,6 @@ per-frame CPU growth since #5926.
- [JS ordered reads wait only on owed groups](/quest/m1/js-ordered-owed.md) - @moq/net matches Rust's owed-group wait
- [Binding announcements always carry captures](/quest/m1/ffi-required-captures.md) - moq-ffi and its generated bindings make `captures` a required list, as #7266 always sets it
- [HLS export follows a resolution change](/quest/m1/hls-reinit-same-codec.md) - a decoder-identity change under the same codec string starts a new init and segment tag
- [fMP4 decode times keep increasing](/quest/m1/fmp4-bframe-decode-time.md) - fragments shorter than a GOP of B-frame video never step `tfdt` backwards
- [moq-quic deletes its remaining 0-RTT code](/quest/m1/quic-zero-rtt-leftovers.md) - only the code that drops a 0-RTT packet remains, with a test
- [Gate the no-em-dash rule](/quest/m1/em-dash-gate.md) - `just check` fails on an em dash in any tracked text file
- [Every package shares one minor version](/quest/m1/shared-version.md) - a recipe sets every published package to one minor at the cut (next: 0.22.0), and `just check` enforces it and the `0.0.x` rule for experiments
Expand Down
13 changes: 0 additions & 13 deletions quest/m1/fmp4-bframe-decode-time.md

This file was deleted.

159 changes: 145 additions & 14 deletions rs/moq-mux/src/container/fmp4/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,18 @@ impl Rendition {
matches!(self, Rendition::Video(_))
}

/// Whether frames may present out of decode order: what the catalog's `reorder` declares,
/// else whatever the codec allows.
fn may_reorder(&self) -> bool {
match self {
Rendition::Video(config) => match config.reorder {
Some(depth) => depth > 0,
None => crate::codec::video::max_reorder(config) > 0,
},
Rendition::Audio(_) => false,
}
}

/// Fallback duration for a trailing frame that carries no per-sample duration.
fn default_frame(&self) -> Duration {
match self {
Expand Down Expand Up @@ -269,6 +281,14 @@ struct Fmp4Track {
/// instead of writing the track's history twice.
written: Option<Duration>,

/// Where the last written fragment's decode timeline ended, in ticks at `timescale`.
///
/// A fragment cut inside a GOP continues from here rather than anchoring at its first
/// frame's presentation time: with reordered video that frame is often a reference
/// presented after the B-frames decoded next, so its presentation time would put the
/// fragment's `tfdt` ahead of the one that follows.
decode_end: Option<u64>,

/// The first presentation time read before the init segment, which bounds how much
/// media queues behind it.
backlog: Option<Duration>,
Expand Down Expand Up @@ -300,6 +320,7 @@ impl Fmp4Track {
resume: None,
group_finished: false,
written: None,
decode_end: None,
backlog: None,
track_id,
timescale,
Expand Down Expand Up @@ -528,6 +549,12 @@ impl<S: Stream> Export<S> {
/// marker; audio and samples with explicit durations remain immediate. Otherwise
/// the cap applies in addition to the group boundary.
///
/// A fragment cut inside a GOP continues the decode timeline where the previous one
/// ended, so B-frame video keeps increasing decode times. Such a fragment cannot see
/// the reordering around it, so unless the catalog declares `reorder: 0` (or the codec
/// never reorders), a video sample with unknown duration lasts no longer than the
/// catalog frame rate's cadence.
///
/// Accepts either `Duration` or `Option<Duration>` (where `None` restores
/// the per-group default).
pub fn with_fragment_duration(mut self, duration: impl Into<Option<Duration>>) -> Self {
Expand Down Expand Up @@ -1073,24 +1100,39 @@ fn should_flush(track: &Fmp4Track, frame: &Frame, fragment_duration: Option<Dura

/// Encode a buffered run of samples as a single CMAF moof+mdat fragment,
/// consuming the next `mfhd` sequence number of the file.
fn encode_fragment(track: &Fmp4Track, sequence_number: &mut u32, frames: Vec<Frame>) -> Result<Bytes> {
if frames.is_empty() {
///
/// A run that opens a GOP (and every audio run) anchors its `tfdt` at its first frame's
/// presentation time, which re-syncs the decode timeline after a gap the durations did not
/// cover. A run cut inside a GOP continues the timeline where the track's previous fragment
/// ended, so decode times never step back however the GOP is reordered.
fn encode_fragment(
track: &mut Fmp4Track,
sequence_number: &mut u32,
frames: Vec<Frame>,
opens_gop: bool,
) -> Result<Bytes> {
let Some(first) = frames.first() else {
return Err(Error::NoFrames.into());
}
let seq = *sequence_number;
*sequence_number += 1;
};
let timescale = moq_net::Timescale::new(track.timescale)?;
let base_dts = match track.decode_end {
Some(end) if !opens_gop => end,
_ => super::base_ticks(first, timescale)?,
};
let info = crate::container::fmp4::FragmentInfo {
track_id: track.track_id,
timescale,
sequence_number: seq,
sequence_number: *sequence_number,
kind: if track.is_video {
super::Kind::Video
} else {
super::Kind::Audio
},
};
Ok(crate::container::fmp4::encode_fragment(info, &frames)?)
let (data, decode_end) = super::encode_at(info, base_dts, &frames)?;
*sequence_number += 1;
track.decode_end = Some(decode_end);
Ok(data)
}

/// Encode a buffered run and wrap it with the metadata a segmenting consumer needs.
Expand Down Expand Up @@ -1123,9 +1165,16 @@ fn emit_fragment(
let latest = frames.iter().map(|frame| Duration::from(frame.timestamp)).max();
track.written = track.written.max(latest);
let timescale = moq_net::Timescale::new(track.timescale)?;
infer_missing_durations(&mut frames, successor, track.default_frame, timescale)?;
// A run that starts or ends inside a GOP sees only part of its reordering.
let partial = !track.buffer_independent || successor.is_some_and(|next| !next.keyframe);
let run = if partial && track.config.may_reorder() {
Run::PartialGop
} else {
Run::Whole
};
infer_missing_durations(&mut frames, successor, track.default_frame, timescale, run)?;
let duration = fragment_duration(&frames, track.default_frame, timescale)?;
let data = encode_fragment(track, sequence_number, frames)?;
let data = encode_fragment(track, sequence_number, frames, independent)?;
Ok(Fragment {
data,
independent,
Expand Down Expand Up @@ -1204,25 +1253,53 @@ pub(crate) fn apply_codec_durations(frames: &mut [Frame], opus: bool) {
/// The publisher is the one that actually knows where its group's content ends, and
/// [`Producer::cut`](crate::container::Producer::cut) is how it says so: the durations it
/// writes arrive already set and are left alone here.
///
/// Frames arrive in decode order, so a gap is a decode duration only when presentation order
/// matches it. A run whose timestamps go backwards is reordered video, and every frame takes
/// the catalog cadence. A [`Run::PartialGop`] cannot rule reordering out from its own frames,
/// since the B-frames that would reveal it may sit in a neighbouring run, so no gap there may
/// exceed the cadence: in reordered video the gap to the next frame decoded spans the frames
/// decoded after it, and a longer duration would push the decode timeline past the next GOP.
pub(crate) fn infer_missing_durations(
frames: &mut [Frame],
successor: Option<&Frame>,
default_frame: Duration,
timescale: moq_net::Timescale,
run: Run,
) -> Result<()> {
let infer_from_pts = pts_monotonic(frames, successor);
let cadence = fallback_duration(default_frame, timescale);

for i in 0..frames.len() {
let successor = infer_from_pts
.then(|| duration_bound(frames, successor, i))
.flatten()
.cloned();
let stated = frames[i].duration.is_some_and(|duration| !duration.is_zero());
infer_missing_duration(&mut frames[i], successor.as_ref(), default_frame, timescale)?;
if run == Run::PartialGop
&& !stated
&& let (Some(inferred), Some(cadence)) = (frames[i].duration, cadence)
&& Duration::from(inferred) > Duration::from(cadence)
{
frames[i].duration = Some(cadence);
}
}

Ok(())
}

/// How much of its reordering a run of frames passed to [`infer_missing_durations`] shows.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Run {
/// Whole GOPs, or media that presents in decode order, so its own timestamps show
/// whether it is reordered.
Whole,
/// Video that may reorder, cut inside a GOP, whose reordering may only show in a
/// neighbouring run.
PartialGop,
}

/// Infer one frame's duration from a known same-group successor, else the catalog cadence.
pub(crate) fn infer_missing_duration(
frame: &mut Frame,
Expand Down Expand Up @@ -1373,7 +1450,14 @@ mod tests {
#[test]
fn infer_missing_durations_uses_default_for_trailing_sample() {
let mut frames = vec![frame(0, Some(0)), frame(41_667, None), frame(83_334, None)];
infer_missing_durations(&mut frames, None, Duration::from_millis(33), moq_net::Timescale::MICRO).unwrap();
infer_missing_durations(
&mut frames,
None,
Duration::from_millis(33),
moq_net::Timescale::MICRO,
Run::Whole,
)
.unwrap();

assert_eq!(frames[0].duration, Some(ts(41_667)));
assert_eq!(frames[1].duration, Some(ts(41_667)));
Expand All @@ -1387,7 +1471,14 @@ mod tests {
#[test]
fn infer_missing_duration_uses_default_for_single_frame() {
let mut frames = vec![frame(83_333, Some(0))];
infer_missing_durations(&mut frames, None, Duration::from_millis(40), moq_net::Timescale::MICRO).unwrap();
infer_missing_durations(
&mut frames,
None,
Duration::from_millis(40),
moq_net::Timescale::MICRO,
Run::Whole,
)
.unwrap();

assert_eq!(frames[0].duration, Some(ts(40_000)));
assert_eq!(
Expand All @@ -1397,7 +1488,8 @@ mod tests {
}

/// The duration cap splits a GOP across fragments, so the successor is a delta frame
/// in the SAME group. That one is a real bound and is still used.
/// in the SAME group. In video that presents in decode order that one is a real bound
/// and is still used.
#[test]
fn infer_trailing_duration_from_successor_frame() {
let successor = frame(83_334, None);
Expand All @@ -1407,6 +1499,7 @@ mod tests {
Some(&successor),
Duration::from_millis(33),
moq_net::Timescale::MICRO,
Run::Whole,
)
.unwrap();

Expand All @@ -1417,11 +1510,38 @@ mod tests {
);
}

/// Video that may reorder, cut inside a GOP, cannot tell a reference frame's gap to the
/// B-frames decoded after it from a duration, so no inferred duration there exceeds the
/// cadence. A shorter gap still times its frame, and a stated duration is kept.
#[test]
fn a_partial_gop_caps_inferred_durations_at_the_cadence() {
let successor = frame(100_000, None);
let mut frames = vec![frame(0, Some(50_000)), frame(10_000, None), frame(20_000, None)];
infer_missing_durations(
&mut frames,
Some(&successor),
Duration::from_millis(33),
moq_net::Timescale::MICRO,
Run::PartialGop,
)
.unwrap();

let durations: Vec<_> = frames.iter().map(duration_micros).collect();
assert_eq!(durations, vec![50_000, 10_000, 33_000]);
}

#[test]
fn inferred_microsecond_clock_carries_fractional_ticks_forward() {
let timescale = moq_net::Timescale::new(30_000).unwrap();
let mut frames = vec![frame(0, None), frame(33_333, None), frame(66_667, None)];
infer_missing_durations(&mut frames, None, Duration::from_nanos(33_333_333), timescale).unwrap();
infer_missing_durations(
&mut frames,
None,
Duration::from_nanos(33_333_333),
timescale,
Run::Whole,
)
.unwrap();

let durations: Vec<_> = frames
.iter()
Expand All @@ -1446,6 +1566,7 @@ mod tests {
Some(&next_group),
Duration::from_millis(33),
moq_net::Timescale::MICRO,
Run::Whole,
)
.unwrap();

Expand All @@ -1462,7 +1583,14 @@ mod tests {
#[test]
fn infer_stops_at_an_interior_group_boundary() {
let mut frames = vec![frame(0, None), frame(21_333, None), group_start(600_000_000)];
infer_missing_durations(&mut frames, None, Duration::from_millis(21), moq_net::Timescale::MICRO).unwrap();
infer_missing_durations(
&mut frames,
None,
Duration::from_millis(21),
moq_net::Timescale::MICRO,
Run::Whole,
)
.unwrap();

assert_eq!(frames[0].duration, Some(ts(21_333)), "same group, real delta");
assert_eq!(duration_micros(&frames[1]), 21_000, "bounded by the next group");
Expand Down Expand Up @@ -1490,6 +1618,7 @@ mod tests {
None,
Duration::from_micros(21_333),
moq_net::Timescale::MICRO,
Run::Whole,
)
.unwrap();

Expand All @@ -1513,6 +1642,7 @@ mod tests {
Some(&next_group),
Duration::from_millis(50),
moq_net::Timescale::MICRO,
Run::Whole,
)
.unwrap();

Expand All @@ -1533,6 +1663,7 @@ mod tests {
Some(&successor),
Duration::from_millis(33),
moq_net::Timescale::MICRO,
Run::Whole,
)
.unwrap();

Expand Down
Loading