Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 0 additions & 20 deletions quest/m1/catalog-timestamp.md

This file was deleted.

149 changes: 132 additions & 17 deletions rs/moq-mux/src/catalog/producer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ struct State<E: CatalogExt> {
}

impl<E: CatalogExt> State<E> {
/// Whether a buffering gate opens now: no reservation is outstanding and the catalog lists a
/// track. The track scan runs last, so releasing each of many reservations stays linear.
fn gate_opens(&mut self) -> bool {
matches!(self.gate, Gate::Buffering { reservers: 0, .. }) && self.lists_track()
}

/// Whether the catalog names anything this broadcast can serve, which the initial publish
/// waits for.
///
Expand All @@ -67,12 +73,6 @@ impl<E: CatalogExt> State<E> {
/// promises no frames, so it doesn't count. A listed rendition covers application-defined
/// sections the base fields can't see, and an importer that lists tracks in the extension
/// reports them through [`ext_tracks`](Self::ext_tracks).
/// Whether a buffering gate opens now: no reservation is outstanding and the catalog lists a
/// track. The track scan runs last, so releasing each of many reservations stays linear.
fn gate_opens(&mut self) -> bool {
matches!(self.gate, Gate::Buffering { reservers: 0, .. }) && self.lists_track()
}

fn lists_track(&mut self) -> bool {
use std::any::TypeId;
// Video and audio carry `enabled`, which only the base check reads.
Expand Down Expand Up @@ -276,6 +276,9 @@ enum Gate {
/// A catalog change was made while buffering and is awaiting the initial publish. Without
/// this we'd emit an empty snapshot when the gate opens on a catalog nobody ever touched.
pending: bool,
/// The timestamp the last staged change was given (see [`Guard::set_timestamp`]), or `None`
/// to stamp the initial publish when it goes out.
at: Option<moq_net::Timestamp>,
},
/// The initial snapshot went out: every change publishes.
Published,
Expand Down Expand Up @@ -309,17 +312,36 @@ impl<E: CatalogExt> Clone for Outputs<E> {
}

impl<E: CatalogExt> Outputs<E> {
/// Emit the catalog to hang and MSF, stamped now on `clock`. Neither writes a catalog identical
/// to its last.
fn emit(&mut self, catalog: &Catalog<E>, msf: &mut Msf<E>, clock: &crate::Clock) -> crate::Result<()> {
/// Emit the catalog to hang and MSF, stamped `at` or else now on `clock`. Neither writes a
/// catalog identical to its last.
fn emit(
&mut self,
catalog: &Catalog<E>,
msf: &mut Msf<E>,
clock: &crate::Clock,
at: Option<moq_net::Timestamp>,
) -> crate::Result<()> {
// Every consumer refuses a catalog that fails this, so refuse it before it reaches one.
catalog.check_renditions()?;
// The media timeline, so the catalog in effect at a media timestamp is the one stamped at or
// before it, whichever thread or runtime publishes. Each catalog starts a group on both
// tracks, so one landing in the same millisecond as the last is stamped a unit later
// instead of refused.
let (_, at) = crate::start::next(&self.hang_track, clock.now())?;
let (_, at) = crate::start::next(&self.msf_track, at)?;
// tracks.
let at = match at {
// A caller's timestamp marks the frame the change takes effect at, so moving it would
// misplace the change: one that doesn't pass the last publish on either track is refused
// before either is written.
Some(at) => {
crate::start::after(&self.hang_track, at)?;
crate::start::after(&self.msf_track, at)?;
at
}
// One landing in the same millisecond as the last is stamped a unit later instead of
// refused.
None => {
let (_, at) = crate::start::next(&self.hang_track, clock.now())?;
crate::start::next(&self.msf_track, at)?.1
}
};
// One snapshot per group while deltas are disabled.
#[cfg(test)]
let before = self.hang_track.largest();
Expand Down Expand Up @@ -627,6 +649,7 @@ impl<E: CatalogExt> Producer<E> {
state,
outputs: &mut self.outputs,
updated: false,
at: None,
})
}

Expand Down Expand Up @@ -844,6 +867,7 @@ impl<E: CatalogExt> Producer<E> {
state.gate = Gate::Buffering {
reservers: 1,
pending: false,
at: None,
}
}
Gate::Buffering { reservers, .. } => *reservers += 1,
Expand All @@ -869,7 +893,10 @@ impl<E: CatalogExt> Producer<E> {
let mut state = take(&self.current);
// Nothing was staged (e.g. every stream was ignored): stay unpublished so a later change
// still triggers the first emit, rather than publishing an empty catalog now.
if !matches!(state.gate, Gate::Buffering { pending: true, .. }) || !state.gate_opens() {
let Gate::Buffering { pending: true, at, .. } = state.gate else {
return;
};
if !state.gate_opens() {
return;
}
state.gate = Gate::Published;
Expand All @@ -878,7 +905,7 @@ impl<E: CatalogExt> Producer<E> {
state.estimates.fold();
// Emitted under the lock, as a guard's publish is, since the MSF state belongs to it.
let state = &mut *state;
if let Err(err) = self.outputs.emit(&state.catalog, &mut state.msf, &state.clock) {
if let Err(err) = self.outputs.emit(&state.catalog, &mut state.msf, &state.clock, at) {
tracing::error!(%err, "failed to publish the catalog, aborting its tracks");
self.outputs.abort(&err);
state.closed = Some(err);
Expand Down Expand Up @@ -1147,6 +1174,8 @@ pub struct Guard<'a, E: CatalogExt = ()> {
state: MutexGuard<'a, State<E>>,
outputs: &'a mut Outputs<E>,
updated: bool,
/// The caller's timestamp for this publish, or `None` to stamp it on publishing.
at: Option<moq_net::Timestamp>,
}

impl<E: CatalogExt> Guard<'_, E> {
Expand All @@ -1160,6 +1189,15 @@ impl<E: CatalogExt> Guard<'_, E> {
self.publish()
}

/// Stamp this publish at `timestamp` on the producer's [`clock`](Producer::clock), rather than
/// the time it is written, such as the frame a layout change takes effect at.
///
/// The publish fails with [`TimestampRewind`](crate::container::TimestampRewind) unless
/// `timestamp` is past the last published catalog's, rather than reordering it.
pub fn set_timestamp(&mut self, timestamp: moq_net::Timestamp) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A refused stamp only fails commit. The guard has already written the change into state.catalog (and publish has set the gate to Published), so the next publish sends it anyway. That could be an estimate republish stamped at clock now, or the caller's next stamped change. In a_set_timestamp_that_does_not_advance_is_refused, the c publish at 61s also carries b60000 and b59000, so the change lands at a timestamp nobody picked. The guard holds the state lock, and every emit runs under that lock. So checking the stamp here, before any edits, is race-free and would let the caller find out without having changed the catalog.

Technical details
# A refused stamped change still reaches the wire with a later publish

## Affected sites
- rs/moq-mux/src/catalog/producer.rs:1197: `set_timestamp` can't fail, so the refusal only shows up in `emit`, after the catalog was already changed.
- rs/moq-mux/src/catalog/producer.rs:1231-1238: `publish` sets `gate = Published` and `anchored = true`, then `emit` refuses the stamp. The changed `state.catalog` stays in place.
- rs/moq-mux/src/catalog/producer.rs:2135: the test passes even though the refused renditions go out with `c`.

## Required outcome
- A stamp the producer refuses never puts its change on the wire at another timestamp. Either the caller learns of the refusal before changing anything, or a refused change is rolled back.
- The test asserts that the refused renditions are missing from the next publish.

## Suggested approach
- `pub fn set_timestamp(&mut self, timestamp) -> crate::Result<()>`: when the gate is `Published`, run `start::after` on `outputs.hang_track` and `outputs.msf_track` right away. The guard owns `&mut Outputs` and the lock, so the check still holds when `commit` runs. Keep the check in `emit` as a backstop, or drop it. Document that `set_timestamp` should come before any edits.

self.at = Some(timestamp);
}

/// Move the producer onto `clock`, a re-basing of the one it has, before anything fixed it.
fn retime(&mut self, clock: crate::Clock) {
self.state.clock = clock;
Expand All @@ -1184,8 +1222,9 @@ impl<E: CatalogExt> Guard<'_, E> {
// catalog names a track; the mutation stays in `current` and `pending` marks it for the
// flush once the gate opens.
if matches!(self.state.gate, Gate::Buffering { .. }) && !self.state.gate_opens() {
if let Gate::Buffering { pending, .. } = &mut self.state.gate {
if let Gate::Buffering { pending, at, .. } = &mut self.state.gate {
*pending = true;
*at = self.at;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This overwrites the stored stamp with every withheld publish. Take a change stamped at 60s, then an unstamped change staged while the reservation is still held: the second one resets at to None. The initial publish then goes out at clock now, which can be before 60s. The layout change then takes effect before its frame. Keeping the stamp when the later change is unstamped (*at = self.at.or(*at)) avoids this. The unstamped edit then lands no earlier than its stamped neighbour, which matches how the published path bumps unstamped publishes past a stamp.

}
return Ok(());
}
Expand All @@ -1195,7 +1234,8 @@ impl<E: CatalogExt> Guard<'_, E> {

// Folded only once it is on the wire, so a failed emit leaves the estimate held.
let state = &mut *self.state;
self.outputs.emit(&state.catalog, &mut state.msf, &state.clock)?;
self.outputs
.emit(&state.catalog, &mut state.msf, &state.clock, self.at)?;
self.state.estimates.fold();
Ok(())
}
Expand Down Expand Up @@ -2048,6 +2088,81 @@ mod test {
assert_eq!(sent(&catalog), Some(0));
}

/// The newest group start on the hang and MSF catalog tracks, in their timescale.
fn starts(catalog: &Producer) -> [Option<u64>; 2] {
let outputs = &catalog.outputs;
[&outputs.hang_track, &outputs.msf_track].map(|track| track.largest().map(|largest| largest.group))
}

/// `ms` in the catalog tracks' timescale.
fn start(catalog: &Producer, ms: u64) -> u64 {
let at = moq_net::Timestamp::from_millis(ms).unwrap();
catalog.outputs.hang_track.info().rescale(at).unwrap().value()
}

/// Publish an Opus rendition `name`, stamped at `at` (ms) when given.
fn opus(catalog: &mut Producer, name: &str, at: Option<u64>) -> crate::Result<()> {
let mut guard = catalog.modify().unwrap();
if let Some(at) = at {
guard.set_timestamp(moq_net::Timestamp::from_millis(at).unwrap());
}
guard
.audio
.renditions
.insert(name.to_string(), AudioConfig::new(AudioCodec::Opus, 48_000, 2));
guard.commit()
}

#[tokio::test(start_paused = true)]
async fn a_set_timestamp_stamps_both_tracks() {
let mut broadcast = moq_net::broadcast::Info::new().produce();
let mut catalog = Producer::new(&mut broadcast, Config::default()).unwrap();

// Ahead of the clock, as a frame a pipeline composes early.
opus(&mut catalog, "a", Some(60_000)).unwrap();
let at = start(&catalog, 60_000);
assert_eq!(starts(&catalog), [Some(at); 2]);

// An unstamped publish still lands after it, though the clock reads earlier.
opus(&mut catalog, "b", None).unwrap();
assert_eq!(starts(&catalog), [Some(at + 1); 2]);

opus(&mut catalog, "c", Some(90_000)).unwrap();
assert_eq!(starts(&catalog), [Some(start(&catalog, 90_000)); 2]);
}

#[tokio::test(start_paused = true)]
async fn a_set_timestamp_that_does_not_advance_is_refused() {
let mut broadcast = moq_net::broadcast::Info::new().produce();
let mut catalog = Producer::new(&mut broadcast, Config::default()).unwrap();

opus(&mut catalog, "a", Some(60_000)).unwrap();
let published = starts(&catalog);
for at in [60_000, 59_000] {
let result = opus(&mut catalog, &format!("b{at}"), Some(at));
assert!(
matches!(result, Err(crate::Error::TimestampRewind(_))),
"{at}: {result:?}"
);
assert_eq!(starts(&catalog), published, "a refused catalog reached the wire");
}
// The refusal leaves the tracks open: the next change goes out at its own timestamp.
opus(&mut catalog, "c", Some(61_000)).unwrap();
assert_eq!(starts(&catalog), [Some(start(&catalog, 61_000)); 2]);
}

#[tokio::test(start_paused = true)]
async fn a_buffered_set_timestamp_stamps_the_initial_publish() {
let mut broadcast = moq_net::broadcast::Info::new().produce();
let mut catalog = Producer::new(&mut broadcast, Config::default()).unwrap();

let reserved = catalog.reserve();
opus(&mut catalog, "a", Some(60_000)).unwrap();
assert_eq!(starts(&catalog), [None; 2], "published before the reservation resolved");
drop(reserved);
assert_eq!(starts(&catalog), [Some(start(&catalog, 60_000)); 2]);
}

/// A video rendition `video0` on `catalog`, with a keyframe group at each of `starts` (ms).
fn video0(
broadcast: &mut moq_net::broadcast::Producer,
Expand Down