From ec3028cfef63f77b3752cc81857c7a96493219e9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 22:01:31 -0700 Subject: [PATCH 01/16] quest: claim m1/archive/producer Co-Authored-By: Claude Opus 5.5 From 0cca85e34918d9553bb0a764608389dfafbf4279 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 22:24:43 -0700 Subject: [PATCH 02/16] wip(archive): archive::Producer records every track of one broadcast Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/Cargo.toml | 2 +- rs/moq-archive/benches/producer.rs | 1 + rs/moq-archive/benches/writer.rs | 81 - rs/moq-archive/src/error.rs | 37 - rs/moq-archive/src/info.rs | 44 - rs/moq-archive/src/layout.rs | 591 +++++++ rs/moq-archive/src/lib.rs | 74 +- rs/moq-archive/src/producer/mod.rs | 236 +++ rs/moq-archive/src/producer/resume.rs | 89 + rs/moq-archive/src/producer/tests.rs | 679 ++++++++ rs/moq-archive/src/producer/track.rs | 598 +++++++ rs/moq-archive/src/proof.rs | 577 ------- rs/moq-archive/src/reader/tests.rs | 61 - rs/moq-archive/src/recover.rs | 210 --- rs/moq-archive/src/store.rs | 12 +- rs/moq-archive/src/writer.rs | 2305 ------------------------- 16 files changed, 2254 insertions(+), 3343 deletions(-) create mode 100644 rs/moq-archive/benches/producer.rs delete mode 100644 rs/moq-archive/benches/writer.rs create mode 100644 rs/moq-archive/src/layout.rs create mode 100644 rs/moq-archive/src/producer/mod.rs create mode 100644 rs/moq-archive/src/producer/resume.rs create mode 100644 rs/moq-archive/src/producer/tests.rs create mode 100644 rs/moq-archive/src/producer/track.rs delete mode 100644 rs/moq-archive/src/proof.rs delete mode 100644 rs/moq-archive/src/recover.rs delete mode 100644 rs/moq-archive/src/writer.rs diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index 3a9ad66bc6e..16b8b0d7060 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -41,5 +41,5 @@ tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "net", "rt", "test-util"] } [[bench]] -name = "writer" +name = "producer" harness = false diff --git a/rs/moq-archive/benches/producer.rs b/rs/moq-archive/benches/producer.rs new file mode 100644 index 00000000000..f328e4d9d04 --- /dev/null +++ b/rs/moq-archive/benches/producer.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/rs/moq-archive/benches/writer.rs b/rs/moq-archive/benches/writer.rs deleted file mode 100644 index d342d9b7c1f..00000000000 --- a/rs/moq-archive/benches/writer.rs +++ /dev/null @@ -1,81 +0,0 @@ -// Host time measures benchmark throughput. -#![allow(clippy::disallowed_methods, clippy::disallowed_types)] - -//! `Writer::run` recording every enrolled track into memory, swept over the number of tracks and -//! the groups each one publishes. Throughput is per frame, so a per-event cost that grows with the -//! track table shows up as a slope across the track counts. -//! -//! Run with `cargo bench -p moq-archive --bench writer`. - -use std::time::{Duration, Instant}; - -use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use moq_archive::object_store::memory::InMemory; -use moq_archive::{Store, Writer, writer}; -use moq_net::{Timescale, Timestamp, broadcast, group, track}; - -const FRAMES_PER_GROUP: u64 = 2; - -/// A writer with `tracks` enrolled tracks, each already holding `groups` one-second groups. -async fn setup(tracks: usize, groups: u64) -> Writer { - let source = broadcast::Info::new().produce(); - let store = Store::new(InMemory::new(), "bench"); - let writer = Writer::new(store, source.consume(), writer::Config::default()) - .await - .unwrap(); - let control = writer.control(); - let config = moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(1)); - - for index in 0..tracks { - let name = format!("track{index}"); - let info = track::Info::default() - .with_timescale(Timescale::MILLI) - .with_max_age(Duration::from_secs(3600)); - let track = source.create_track(name.as_str(), info).unwrap(); - control.track(&name, config.clone()).await.unwrap(); - for sequence in 0..groups { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for frame in 0..FRAMES_PER_GROUP { - let pts = Timestamp::from_millis(sequence * 1_000 + frame * 500).unwrap(); - group.write_frame(pts, vec![0u8; 64]).unwrap(); - } - group.finish().unwrap(); - } - track.finish().unwrap(); - } - source.close(); - writer -} - -fn writer(c: &mut Criterion) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_time() - .build() - .unwrap(); - - let mut bench = c.benchmark_group("writer"); - bench.sample_size(10); - bench.warm_up_time(Duration::from_secs(1)); - bench.measurement_time(Duration::from_secs(2)); - for tracks in [1, 10, 100] { - for groups in [10, 100] { - bench.throughput(Throughput::Elements(tracks as u64 * groups * FRAMES_PER_GROUP)); - bench.bench_function(BenchmarkId::new(format!("{groups} groups"), tracks), |b| { - b.iter_custom(|iters| { - let mut total = Duration::ZERO; - for _ in 0..iters { - let writer = runtime.block_on(setup(tracks, groups)); - let start = Instant::now(); - runtime.block_on(writer.run()).unwrap(); - total += start.elapsed(); - } - total - }); - }); - } - } - bench.finish(); -} - -criterion_group!(benches, writer); -criterion_main!(benches); diff --git a/rs/moq-archive/src/error.rs b/rs/moq-archive/src/error.rs index e33439b030a..6bbd956dd41 100644 --- a/rs/moq-archive/src/error.rs +++ b/rs/moq-archive/src/error.rs @@ -40,20 +40,6 @@ pub enum Error { intended: Option, }, - /// An existing `.info` was recorded from another publisher instance: the source - /// restarted, so the caller starts a new prefix. - #[error( - "source epoch {} does not match the recorded {}", - epoch_text(intended), - epoch_text(existing) - )] - EpochMismatch { - /// The epoch already recorded. - existing: Option, - /// The epoch of the source being recorded now. - intended: Option, - }, - /// The object has no groups. #[error("empty object")] Empty, @@ -102,14 +88,6 @@ pub enum Error { #[error("moq: {0}")] Moq(String), - /// The track was already enrolled, or is the recording's own timeline. - #[error("track already enrolled: {0}")] - Enrolled(String), - - /// The track is untimed, so a DVR has no content time to measure its retention by. - #[error("untimed track: {0}")] - Untimed(String), - /// A frame's timestamp does not match its track: missing on a timed track, or present on an /// untimed one. #[error("frame timing does not match the track")] @@ -118,14 +96,6 @@ pub enum Error { /// The source broadcast or one of its tracks failed. #[error("source: {0}")] Source(String), - - /// The recording's timeline could not be recovered, segmented, or published. - #[error("timeline: {0}")] - Timeline(String), - - /// The writer stopped accepting commands. - #[error("writer closed")] - Closed, } impl From for Error { @@ -151,10 +121,3 @@ impl From for Error { /// A [`Result`](std::result::Result) using this crate's [`Error`]. pub type Result = std::result::Result; - -fn epoch_text(epoch: &Option) -> String { - match epoch { - Some(epoch) => epoch.to_string(), - None => "none".to_string(), - } -} diff --git a/rs/moq-archive/src/info.rs b/rs/moq-archive/src/info.rs index cfd546005b9..698fbdc8846 100644 --- a/rs/moq-archive/src/info.rs +++ b/rs/moq-archive/src/info.rs @@ -16,9 +16,6 @@ pub struct Info { /// is untimed and its frames store no timestamps. #[serde(default, skip_serializing_if = "Option::is_none")] pub timescale: Option, - /// The source's publisher epoch, when its route had one. A resume from another refuses. - #[serde(default, skip_serializing_if = "Option::is_none", with = "epoch")] - pub epoch: Option, } impl Info { @@ -28,18 +25,11 @@ impl Info { version: VERSION, priority, timescale, - epoch: None, }; info.validate()?; Ok(info) } - /// Record `epoch` as the source's publisher epoch. - pub fn with_epoch(mut self, epoch: Option) -> Self { - self.epoch = epoch; - self - } - /// Encode as compact JSON. pub fn encode(&self) -> Result { self.validate()?; @@ -67,24 +57,6 @@ impl Info { } } -/// The epoch as its UUID text. -mod epoch { - use serde::{Deserialize, Deserializer, Serializer, de::Error}; - - pub fn serialize(epoch: &Option, serializer: S) -> Result { - match epoch { - Some(epoch) => serializer.serialize_str(epoch.as_str()), - None => serializer.serialize_none(), - } - } - - pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { - let Some(text) = Option::::deserialize(deserializer)? else { - return Ok(None); - }; - text.parse().map(Some).map_err(D::Error::custom) - } -} #[cfg(test)] mod tests { @@ -126,22 +98,6 @@ mod tests { } } - #[test] - fn epoch_roundtrip() { - let epoch: moq_net::Epoch = "01900000-0000-7000-8000-000000000001".parse().unwrap(); - let info = Info::new(0, Some(1000)).unwrap().with_epoch(Some(epoch)); - let bytes = info.encode().unwrap(); - assert_eq!( - bytes.as_ref(), - br#"{"version":2,"priority":0,"timescale":1000,"epoch":"01900000-0000-7000-8000-000000000001"}"# - ); - assert_eq!(Info::decode(&bytes).unwrap(), info); - assert!(matches!( - Info::decode(br#"{"version":2,"priority":0,"timescale":1000,"epoch":"not-a-uuid"}"#), - Err(Error::Json(_)) - )); - } - #[test] fn timescale_bounds() { assert!(Info::new(0, Some(1)).is_ok()); diff --git a/rs/moq-archive/src/layout.rs b/rs/moq-archive/src/layout.rs new file mode 100644 index 00000000000..ea639128338 --- /dev/null +++ b/rs/moq-archive/src/layout.rs @@ -0,0 +1,591 @@ +//! The keys and objects of the store layout a [`Producer`](crate::Producer) writes. +//! +//! See the [crate docs](crate#layout) for the layout itself. + +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use moq_net::track::Position; +use moq_net::varint; +use object_store::path::Path; +use serde::{Deserialize, Serialize}; + +use crate::path::{check_id, decode_track, encode_track, push}; +use crate::segment::Frame; +use crate::{Error, ID_WIDTH, Result, VERSION}; + +/// The broadcast's completion marker, beside its track segments. +pub(crate) const BROADCAST_END: &str = ".end"; +/// A track's properties, which sort before every object key. +pub(crate) const TRACK_INFO: &str = ".info"; +/// A track's completion marker, which sorts after every object key. +pub(crate) const TRACK_END: &str = "end"; + +/// A track's directory under a broadcast prefix: its name as one canonical segment. +pub(crate) fn track_dir(prefix: &Path, track: &str) -> Result { + push(prefix, &encode_track(track)?) +} + +/// What one entry under a track's directory is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Name { + /// `.info` + Info, + /// `end` + End, + /// An object, named by where its content starts. + Object(Start), +} + +impl Name { + /// Parse the last segment of a key under a track's directory, refusing anything else. + pub(crate) fn parse(name: &str, timed: bool) -> Result { + match name { + TRACK_INFO => Ok(Self::Info), + TRACK_END => Ok(Self::End), + name => Ok(Self::Object(Start::parse(name, timed)?)), + } + } +} + +/// Decode a track directory's segment back into its name, refusing a non-canonical one. +pub(crate) fn parse_track(segment: &str) -> Result { + decode_track(segment) +} + +/// Where an object's content starts, which is its key: the first sequence, the first frame's +/// index within it, and, on a timed track, that frame's timestamp. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct Start { + pub group: u64, + pub frame: u64, + pub pts: Option, +} + +impl Start { + /// `--` on a timed track, `` on an untimed one, zero-padded so + /// keys sort in content order. + pub(crate) fn name(&self) -> Result { + let group = id(self.group)?; + match self.pts { + Some(pts) => Ok(format!("{group}-{}-{}", id(self.frame)?, id(pts)?)), + None if self.frame == 0 => Ok(group), + // An untimed group is never split, so no untimed object starts mid-group. + None => Err(Error::Path(format!("{group}-{}", self.frame))), + } + } + + fn parse(name: &str, timed: bool) -> Result { + let fields: Vec<&str> = name.split('-').collect(); + let invalid = || Error::Path(name.to_string()); + match (timed, fields.as_slice()) { + (true, [group, frame, pts]) => Ok(Self { + group: parse_id(group).ok_or_else(invalid)??, + frame: parse_id(frame).ok_or_else(invalid)??, + pts: Some(parse_id(pts).ok_or_else(invalid)??), + }), + (false, [group]) => Ok(Self { + group: parse_id(group).ok_or_else(invalid)??, + frame: 0, + pts: None, + }), + _ => Err(invalid()), + } + } +} + +fn id(value: u64) -> Result { + Ok(format!("{:0width$}", check_id(value)?, width = ID_WIDTH)) +} + +/// `None` when the field is not exactly [`ID_WIDTH`] digits. +fn parse_id(field: &str) -> Option> { + if field.len() != ID_WIDTH || !field.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + Some(field.parse().map_err(|_| Error::Path(field.to_string())).and_then(check_id)) +} + +/// How a track's last run ended, stored in its `end` once the broadcast ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase", deny_unknown_fields)] +pub(crate) enum End { + /// A clean end at this exclusive position. + Finish(At), + /// The stream error code it ended with. + Error(u32), +} + +/// A [`Position`] as stored. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct At { + pub group: u64, + pub frame: u64, +} + +impl From for At { + fn from(position: Position) -> Self { + Self { + group: position.group, + frame: position.frame, + } + } +} + +impl End { + pub(crate) fn encode(&self) -> Result { + Ok(Bytes::from(serde_json::to_vec(self)?)) + } + + pub(crate) fn decode(bytes: &[u8]) -> Result { + Ok(serde_json::from_slice(bytes)?) + } +} + +/// One stored sequence, in the form it was forwarded in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Entry { + /// A group's frames: all of them, or the run of them this object holds. + Group { sequence: u64, frames: Vec }, + /// A datagram: one payload and timestamp, never split. + Datagram { sequence: u64, frame: Frame }, +} + +impl Entry { + pub(crate) fn sequence(&self) -> u64 { + match self { + Self::Group { sequence, .. } | Self::Datagram { sequence, .. } => *sequence, + } + } + + fn first(&self) -> &Frame { + match self { + Self::Group { frames, .. } => &frames[0], + Self::Datagram { frame, .. } => frame, + } + } +} + +/// One object: entries in ascending sequence order, the first starting at `frame_start`. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub(crate) struct Object { + /// The index of the first group's first stored frame; zero when the first entry is a datagram. + pub frame_start: u64, + pub entries: Vec, +} + +const GROUP: u64 = 0; +const DATAGRAM: u64 = 1; + +impl Object { + /// The key this object is stored under. + pub(crate) fn start(&self) -> Result { + let first = self.entries.first().ok_or(Error::Empty)?; + Ok(Start { + group: first.sequence(), + frame: self.frame_start, + pts: first.first().timestamp, + }) + } + + /// The position just past the last stored frame or datagram. + pub(crate) fn end(&self) -> Result { + let last = self.entries.last().ok_or(Error::Empty)?; + Ok(match last { + Entry::Datagram { sequence, .. } => Position::group(sequence.checked_add(1).ok_or(Error::Overflow)?), + Entry::Group { sequence, frames } => { + let first = match self.entries.len() { + 1 => self.frame_start, + _ => 0, + }; + Position { + group: *sequence, + frame: first.checked_add(frames.len() as u64).ok_or(Error::Overflow)?, + } + } + }) + } + + /// The version, the first frame index, then each entry's sequence (the first absolute, the + /// rest `current - previous - 1`), form, and frame table, followed by every payload in order. + pub(crate) fn encode(&self, timed: bool) -> Result { + self.validate(timed)?; + + let mut table = BytesMut::new(); + write(&mut table, VERSION)?; + write(&mut table, self.frame_start)?; + write(&mut table, self.entries.len() as u64)?; + + let mut payload = BytesMut::new(); + let mut frame = |table: &mut BytesMut, frame: &Frame| -> Result<()> { + if let Some(timestamp) = frame.timestamp { + write(table, timestamp)?; + } + write(table, frame.payload.len() as u64)?; + payload.extend_from_slice(&frame.payload); + Ok(()) + }; + + let mut prev: Option = None; + for entry in &self.entries { + let sequence = entry.sequence(); + let delta = match prev { + None => sequence, + Some(prev) => sequence - prev - 1, + }; + prev = Some(sequence); + write(&mut table, delta)?; + match entry { + Entry::Group { frames, .. } => { + write(&mut table, GROUP)?; + write(&mut table, frames.len() as u64)?; + for f in frames { + frame(&mut table, f)?; + } + } + Entry::Datagram { frame: f, .. } => { + write(&mut table, DATAGRAM)?; + frame(&mut table, f)?; + } + } + } + + table.extend_from_slice(&payload); + Ok(table.freeze()) + } + + /// Decode a whole table before slicing any payload; `timed` is whether the track's `.info` + /// declares a timescale, which says whether each frame stores a timestamp. + pub(crate) fn decode(mut buf: impl Buf, timed: bool) -> Result { + let version = read(&mut buf)?; + if version != VERSION { + return Err(Error::Version(version)); + } + let frame_start = read(&mut buf)?; + // Each entry is at least a sequence, a form, and a frame count or length. + let count = read_count(&mut buf, 3)?; + + let per_frame = if timed { 2 } else { 1 }; + let mut lengths = Vec::new(); + let mut read_frame = |buf: &mut dyn Buf| -> Result> { + let timestamp = match timed { + true => Some(check_id(read(buf)?)?), + false => None, + }; + lengths.push(read(buf)?); + Ok(timestamp) + }; + + enum Raw { + Group(u64, Vec>), + Datagram(u64, Option), + } + let mut raw = Vec::with_capacity(count); + let mut prev: Option = None; + for _ in 0..count { + let delta = read(&mut buf)?; + let sequence = match prev { + None => delta, + Some(prev) => prev + .checked_add(1) + .and_then(|next| next.checked_add(delta)) + .ok_or(Error::Overflow)?, + }; + check_id(sequence)?; + prev = Some(sequence); + match read(&mut buf)? { + GROUP => { + let frames = read_count(&mut buf, per_frame)?; + let mut timestamps = Vec::with_capacity(frames); + for _ in 0..frames { + timestamps.push(read_frame(&mut buf)?); + } + raw.push(Raw::Group(sequence, timestamps)); + } + DATAGRAM => raw.push(Raw::Datagram(sequence, read_frame(&mut buf)?)), + _ => return Err(Error::Table), + } + } + + let mut payload = buf.copy_to_bytes(buf.remaining()); + let total = lengths + .iter() + .try_fold(0u64, |sum, len| sum.checked_add(*len)) + .ok_or(Error::Overflow)?; + if total != payload.len() as u64 { + return Err(Error::Table); + } + + let mut lengths = lengths.into_iter(); + let mut take = |timestamp: Option| -> Frame { + let len = lengths.next().expect("one length per frame") as usize; + Frame { + timestamp, + payload: payload.split_to(len), + } + }; + let entries = raw + .into_iter() + .map(|raw| match raw { + Raw::Group(sequence, timestamps) => Entry::Group { + sequence, + frames: timestamps.into_iter().map(&mut take).collect(), + }, + Raw::Datagram(sequence, timestamp) => Entry::Datagram { + sequence, + frame: take(timestamp), + }, + }) + .collect(); + + let object = Self { frame_start, entries }; + object.validate(timed)?; + Ok(object) + } + + fn validate(&self, timed: bool) -> Result<()> { + let first = self.entries.first().ok_or(Error::Empty)?; + if matches!(first, Entry::Datagram { .. }) && self.frame_start != 0 { + return Err(Error::Table); + } + let mut prev = None; + for entry in &self.entries { + let sequence = check_id(entry.sequence())?; + if prev.is_some_and(|prev| sequence <= prev) { + return Err(Error::Sequence); + } + prev = Some(sequence); + let frames = match entry { + Entry::Group { frames, .. } if frames.is_empty() => return Err(Error::Empty), + Entry::Group { frames, .. } => frames.as_slice(), + Entry::Datagram { frame, .. } => std::slice::from_ref(frame), + }; + for frame in frames { + match frame.timestamp { + Some(timestamp) if timed => { + check_id(timestamp)?; + } + None if !timed => {} + _ => return Err(Error::Timing), + } + } + } + Ok(()) + } +} + +fn write(buf: &mut impl BufMut, value: u64) -> Result<()> { + varint::encode_quic(value, buf).map_err(|_| Error::Overflow) +} + +fn read(mut buf: &mut (impl Buf + ?Sized)) -> Result { + varint::decode_quic(&mut buf).map_err(|_| Error::Table) +} + +/// A count of entries at least `min` bytes each, refused when the rest of the buffer can't hold it. +fn read_count(buf: &mut impl Buf, min: usize) -> Result { + let count = usize::try_from(read(buf)?).map_err(|_| Error::Overflow)?; + if count > buf.remaining() / min { + return Err(Error::Table); + } + Ok(count) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ID_MAX; + + fn frame(timestamp: Option, payload: &'static [u8]) -> Frame { + Frame { + timestamp, + payload: Bytes::from_static(payload), + } + } + + #[test] + fn timed_keys_sort_in_content_order() { + let names: Vec = [(0, 0, 9), (0, 3, 2), (1, 0, 0), (10, 0, 0)] + .into_iter() + .map(|(group, frame, pts)| { + Start { + group, + frame, + pts: Some(pts), + } + .name() + .unwrap() + }) + .collect(); + let mut sorted = names.clone(); + sorted.sort(); + assert_eq!(names, sorted); + assert_eq!(names[1], "0000000000000000000-0000000000000000003-0000000000000000002"); + assert!(names.iter().all(|name| name.as_str() < TRACK_END && name.as_str() > TRACK_INFO)); + } + + #[test] + fn names_roundtrip_and_refuse_the_other_timing() { + let timed = Start { + group: 5, + frame: 2, + pts: Some(ID_MAX), + }; + let name = timed.name().unwrap(); + assert_eq!(Name::parse(&name, true).unwrap(), Name::Object(timed)); + assert!(matches!(Name::parse(&name, false), Err(Error::Path(_)))); + + let untimed = Start { + group: 7, + frame: 0, + pts: None, + }; + let name = untimed.name().unwrap(); + assert_eq!(name, "0000000000000000007"); + assert_eq!(Name::parse(&name, false).unwrap(), Name::Object(untimed)); + assert!(matches!(Name::parse(&name, true), Err(Error::Path(_)))); + + assert_eq!(Name::parse(".info", true).unwrap(), Name::Info); + assert_eq!(Name::parse("end", false).unwrap(), Name::End); + for bad in ["7", "segments", "0000000000000000007-x", "0009007199254740992"] { + assert!(Name::parse(bad, false).is_err(), "{bad}"); + } + } + + #[test] + fn an_untimed_object_never_starts_mid_group() { + let start = Start { + group: 1, + frame: 2, + pts: None, + }; + assert!(matches!(start.name(), Err(Error::Path(_)))); + } + + #[test] + fn end_markers_are_json() { + let finish = End::Finish(At { group: 5, frame: 3 }); + assert_eq!(finish.encode().unwrap().as_ref(), br#"{"finish":{"group":5,"frame":3}}"#); + assert_eq!(End::decode(&finish.encode().unwrap()).unwrap(), finish); + let error = End::Error(0x34); + assert_eq!(error.encode().unwrap().as_ref(), br#"{"error":52}"#); + assert_eq!(End::decode(br#"{"error":52}"#).unwrap(), error); + assert!(End::decode(br#"{"finish":{"group":5}}"#).is_err()); + } + + #[test] + fn groups_and_datagrams_keep_their_form() { + let object = Object { + frame_start: 4, + entries: vec![ + Entry::Group { + sequence: 3, + frames: vec![frame(Some(10), b"a"), frame(Some(9), b"bb")], + }, + Entry::Datagram { + sequence: 4, + frame: frame(Some(11), b"d"), + }, + Entry::Group { + sequence: 7, + frames: vec![frame(Some(12), b"ccc")], + }, + ], + }; + let bytes = object.encode(true).unwrap(); + assert_eq!(Object::decode(bytes.clone(), true).unwrap(), object); + assert!(Object::decode(bytes, false).is_err()); + assert_eq!( + object.start().unwrap(), + Start { + group: 3, + frame: 4, + pts: Some(10) + } + ); + assert_eq!(object.end().unwrap(), Position { group: 7, frame: 1 }); + + let split = Object { + frame_start: 4, + entries: vec![object.entries[0].clone()], + }; + assert_eq!(split.end().unwrap(), Position { group: 3, frame: 6 }); + + let datagram = Object { + frame_start: 0, + entries: vec![object.entries[1].clone()], + }; + assert_eq!(datagram.end().unwrap(), Position::group(5)); + } + + #[test] + fn an_untimed_object_stores_no_timestamps() { + let object = Object { + frame_start: 0, + entries: vec![Entry::Group { + sequence: 2, + frames: vec![frame(None, b"x"), frame(None, b"yz")], + }], + }; + let bytes = object.encode(false).unwrap(); + // version, frame start, one entry: sequence 2, a group of two frames, then the payloads. + assert_eq!(bytes.as_ref(), &[VERSION as u8, 0, 1, 2, 0, 2, 1, 2, b'x', b'y', b'z']); + assert_eq!(Object::decode(bytes, false).unwrap(), object); + assert!(matches!(object.encode(true), Err(Error::Timing))); + } + + #[test] + fn malformed_objects_are_refused() { + let empty = Object::default(); + assert!(matches!(empty.encode(true), Err(Error::Empty))); + + let descending = Object { + frame_start: 0, + entries: vec![ + Entry::Datagram { + sequence: 5, + frame: frame(None, b"a"), + }, + Entry::Datagram { + sequence: 5, + frame: frame(None, b"b"), + }, + ], + }; + assert!(matches!(descending.encode(false), Err(Error::Sequence))); + + let datagram_mid_group = Object { + frame_start: 1, + entries: vec![Entry::Datagram { + sequence: 5, + frame: frame(None, b"a"), + }], + }; + assert!(matches!(datagram_mid_group.encode(false), Err(Error::Table))); + + let good = Object { + frame_start: 0, + entries: vec![Entry::Group { + sequence: 1, + frames: vec![frame(None, b"abc")], + }], + } + .encode(false) + .unwrap(); + // Truncated payload, trailing bytes, an unknown form, and an absurd count. + assert!(Object::decode(&good[..good.len() - 1], false).is_err()); + let mut trailing = good.to_vec(); + trailing.push(0); + assert!(Object::decode(&trailing[..], false).is_err()); + let mut form = good.to_vec(); + form[4] = 2; + assert!(matches!(Object::decode(&form[..], false), Err(Error::Table))); + assert!(matches!( + Object::decode(&[VERSION as u8, 0, 0x7f][..], false), + Err(Error::Table) + )); + assert!(matches!( + Object::decode(&[VERSION as u8 + 1, 0, 0][..], false), + Err(Error::Version(_)) + )); + } +} diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 94ec612132b..77ac384991f 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -1,17 +1,60 @@ -//! Versioned hang recording objects on any [`object_store::ObjectStore`]. +//! Broadcast archives on any [`object_store::ObjectStore`]. //! -//! The crate owns the portable layout and codecs: percent-encoded track names, `.info` JSON, -//! the binary segment envelope, and put/get/list/delete. A [`Writer`] records selected tracks of -//! a broadcast into those objects, and a [`Reader`] serves a recording back through a `moq_net` -//! broadcast. Callers that need runtime dispatch -//! supply `Arc`; the archive API itself stays generic. +//! A [`Producer`] stores every track of one broadcast under a key prefix the host picks, and a +//! [`Reader`] serves a recording back through a `moq_net` broadcast. Callers that need runtime +//! dispatch supply `Arc`; the archive API itself stays generic. //! -//! Every object is keyed by its track and a record sequence: -//! ``` -//! let key = moq_archive::Key::segments("video", 7)?; -//! assert_eq!(key.track(), "video"); -//! # Ok::<(), moq_archive::Error>(()) +//! # Layout +//! +//! A [`Producer`] writes, under its prefix: +//! +//! - `/.info`: the track's properties as JSON, `{"version":2,"priority":127,"timescale":1000}`, +//! written before its first object. `timescale` is absent on an untimed track. Properties never +//! change for a track name. The source's max age is not stored: it bounds the live cache, not +//! the archive. +//! - `/--` on a timed track, `/` on an untimed one: an +//! object, keyed by its first sequence, the index of its first frame within it, and that +//! frame's timestamp in the track's timescale. Each field is 19 zero-padded decimal digits, +//! so a sorted listing is content order and answers group and time lookups; there is no index +//! object. A timed group may split between frames across objects, which the frame index keeps +//! unique and ordered even where timestamps repeat or go backward. An untimed group never +//! splits: while it is open, its object is rewritten in place under the same key. +//! - `/end`: how the track's last run ended, `{"finish":{"group":5,"frame":0}}` for a +//! clean end at that exclusive position or `{"error":52}` for the stream error code. It sorts +//! after every object key, and DVR pruning never deletes it. +//! - `.end`, empty: the broadcast ended and no track will be added. It is written after every +//! track's `end`; without it the archive is still growing, or its producer stopped early. +//! +//! `` is the track name percent-encoded as one segment: everything but `A-Z a-z 0-9 _ -` +//! is escaped with uppercase hex, so a name holding `/` stays one segment and none starts with +//! `.`. A reader refuses a segment that is not in this canonical form. +//! +//! An object is a table followed by every payload in order, each table field a QUIC varint: +//! +//! ```text +//! Object { +//! Version (i) = 2, +//! Frame Start (i), // the first group's first frame index; 0 when it starts with a datagram +//! Entry Count (i), +//! Entry (..) ..., +//! Payload (..), // every frame's bytes, in table order +//! } +//! +//! Entry { +//! Sequence (i), // absolute for the first entry, then current - previous - 1 +//! Form (i), // 0 for a group, 1 for a datagram +//! [Frame Count (i)], // a group only, at least 1 +//! Frame (..) ..., // a datagram has exactly one +//! } +//! +//! Frame { +//! [Timestamp (i)], // exactly when the track's .info declares a timescale +//! Length (i), +//! } //! ``` +//! +//! A track's groups and datagrams share one sequence space, so each entry keeps its form, and a +//! datagram replays as a datagram, never as a cached one-frame group. #![warn(missing_docs)] @@ -20,27 +63,24 @@ pub use object_store; mod error; /// The `.info` object holding a track's immutable properties. pub mod info; +mod layout; #[cfg(test)] mod mock; mod path; -#[cfg(test)] -#[allow(clippy::disallowed_methods, clippy::disallowed_types)] -mod proof; +pub mod producer; pub mod reader; -mod recover; /// The binary segment object holding a record's groups and frames. pub mod segment; /// Put, get, list, and delete recording objects on an object store. pub mod store; -pub mod writer; pub use error::{Error, Result}; pub use info::Info; pub use path::Key; +pub use producer::Producer; pub use reader::Reader; pub use segment::{Frame, Group, Object}; pub use store::Store; -pub use writer::Writer; /// Recording format version written into `.info` and the binary envelope. pub const VERSION: u64 = 2; diff --git a/rs/moq-archive/src/producer/mod.rs b/rs/moq-archive/src/producer/mod.rs new file mode 100644 index 00000000000..f962915d306 --- /dev/null +++ b/rs/moq-archive/src/producer/mod.rs @@ -0,0 +1,236 @@ +//! Record every track of one broadcast into a [`Store`]. +//! +//! A [`Producer`] lists the broadcast's tracks (TRACKS, [`broadcast::Consumer::tracks`]) and +//! records each through its own ascending SUBSCRIBE, so it holds what the publisher's cache +//! stops serving: a group still missing once the subscription's max delay passes is a gap, never +//! a stall. Datagrams are never cached, so a track's are recorded from when its subscription +//! takes effect. The host picks the key prefix, per broadcast path and epoch say, and runs one +//! producer per broadcast; the producer stores no epoch. See the [crate docs](crate#layout) for +//! what it writes. +//! +//! A producer started under a prefix that already holds objects resumes each track at the frame +//! after its last stored one, and one under a prefix holding `.end` records nothing. +//! +//! ```no_run +//! # async fn example(broadcast: moq_net::broadcast::Consumer) -> moq_archive::Result<()> { +//! use moq_archive::object_store::memory::InMemory; +//! +//! let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); +//! moq_archive::Producer::new(store, broadcast, Default::default()).run().await?; +//! # Ok(()) +//! # } +//! ``` + +mod resume; +mod track; + +#[cfg(test)] +mod tests; + +use std::collections::HashMap; +use std::sync::Arc; +use std::task::Poll; +use std::time::Duration; + +use futures::future::BoxFuture; +use futures::stream::FuturesUnordered; +use futures::{FutureExt, StreamExt}; +use moq_net::broadcast::TracksEvent; +use moq_net::{StreamError, broadcast}; +use object_store::{ObjectStore, ObjectStoreExt, PutPayload}; + +use crate::layout::{self, End}; +use crate::path::push; +use crate::{Error, Result, Store}; + +/// How a [`Producer`] cuts, flushes, and prunes its objects. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Config { + /// How much content an object holds before it is cut at the next group boundary: media time + /// on a timed track, wall-clock time on an untimed one. A timed group whose own run in an + /// object reaches it splits between frames; an untimed group never splits. + pub duration: Duration, + /// The longest a frame waits in memory before its object is stored, however idle its track. + /// A timed object is cut there; an untimed one is rewritten in place and stays open. Keep it + /// above [`Self::duration`], or objects are cut by it instead. + pub flush: Duration, + /// The longest a missing group is waited for, capping the publisher's max age. A group still + /// missing then is a gap, and the frames behind it wait for that verdict. + pub max_delay: Duration, + /// Prune old objects (a DVR), or keep everything when `None` (an archive). + pub dvr: Option, +} + +impl Default for Config { + fn default() -> Self { + Self { + duration: Duration::from_secs(2), + flush: Duration::from_secs(4), + max_delay: Duration::from_secs(10), + dvr: None, + } + } +} + +impl Config { + /// Set [`duration`](Self::duration). + pub fn with_duration(mut self, duration: Duration) -> Self { + self.duration = duration; + self + } + + /// Set [`flush`](Self::flush). + pub fn with_flush(mut self, flush: Duration) -> Self { + self.flush = flush; + self + } + + /// Set [`max_delay`](Self::max_delay). + pub fn with_max_delay(mut self, max_delay: Duration) -> Self { + self.max_delay = max_delay; + self + } + + /// Set [`dvr`](Self::dvr). + pub fn with_dvr(mut self, dvr: impl Into>) -> Self { + self.dvr = dvr.into(); + self + } +} + +/// A DVR's pruning, for hosts without bucket lifecycle rules. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Dvr { + /// How long an object stays after a newer one of its track supersedes it. + pub window: Duration, + /// How much longer it stays, so readers that already listed it finish their GETs. + pub grace: Duration, +} + +impl Dvr { + /// Delete each superseded object `window + grace` after it was superseded. + pub fn new(window: Duration, grace: Duration) -> Self { + Self { window, grace } + } +} + +/// Records every track of one broadcast until the broadcast ends. +pub struct Producer { + shared: Arc>, +} + +struct Shared { + store: Store, + broadcast: broadcast::Consumer, + config: Config, +} + +/// What one track's recorder reports once the broadcast's listing ended and it stored everything. +type Recorded = Result<(String, Option)>; + +impl Producer { + /// Record `broadcast` under `store`'s prefix. + pub fn new(store: Store, broadcast: broadcast::Consumer, config: Config) -> Self { + Self { + shared: Arc::new(Shared { + store, + broadcast, + config, + }), + } + } + + /// Record until the broadcast ends, then write each track's `end` and the broadcast's `.end`. + /// + /// Returns at once, recording nothing, when the prefix already holds `.end`, or when TRACKS + /// is refused with not found, since the path names no broadcast. Any other refusal, a store + /// failure, or a track whose properties differ from the stored ones fails. Dropping the + /// future stops recording without markers, so the archive reads as still growing. + pub async fn run(self) -> Result<()> { + let shared = self.shared; + let Some(mut stored) = resume::load(&shared.store).await? else { + return Ok(()); + }; + + let mut listing = Some(shared.broadcast.tracks().map_err(source)?); + let mut queues: HashMap> = HashMap::new(); + let mut recorders: FuturesUnordered> = FuturesUnordered::new(); + let mut ends: HashMap> = HashMap::new(); + + enum Event { + Listed(moq_net::Result>), + Recorded(Recorded), + Done, + } + + loop { + let event = kio::wait(|waiter| { + if let Poll::Ready(Some(recorded)) = recorders.poll_next_unpin(&mut waiter.context()) { + return Poll::Ready(Event::Recorded(recorded)); + } + match listing.as_mut() { + Some(listing) => listing.poll_next(waiter).map(Event::Listed), + None if recorders.is_empty() => Poll::Ready(Event::Done), + None => Poll::Pending, + } + }) + .await; + + match event { + Event::Listed(Ok(Some(TracksEvent::Added { name, info }))) => { + let queue = queues.entry(name.to_string()).or_insert_with(|| { + let queue = kio::Queue::new(); + let record = track::record( + shared.clone(), + name.to_string(), + queue.clone(), + stored.remove(name.as_ref()), + ); + recorders.push(record.boxed()); + queue + }); + // Only fails once closed, which happens when the listing ends. + let _ = queue.try_push(info); + } + Event::Listed(Ok(Some(TracksEvent::Synced | TracksEvent::Removed { .. }))) => {} + Event::Listed(Ok(None)) => { + listing = None; + for queue in queues.values() { + queue.close(); + } + } + Event::Listed(Err(err)) if queues.is_empty() && StreamError::from(&err) == StreamError::NotFound => { + tracing::debug!(broadcast = %shared.broadcast.info().path, "not a broadcast"); + return Ok(()); + } + Event::Listed(Err(err)) => return Err(source(err)), + Event::Recorded(recorded) => { + let (name, end) = recorded?; + ends.insert(name, end); + } + Event::Done => break, + } + } + + let store = &shared.store; + for (name, end) in &ends { + if let Some(end) = end { + let path = push(&layout::track_dir(store.prefix(), name)?, layout::TRACK_END)?; + store.inner().put(&path, PutPayload::from(end.encode()?)).await?; + } + } + // A stored track that ended unseen, or was never listed again, may still grow. + let complete = ends.values().all(Option::is_some) && stored.values().all(|track| track.end.is_some()); + if complete { + let path = push(store.prefix(), layout::BROADCAST_END)?; + store.inner().put(&path, PutPayload::new()).await?; + } + Ok(()) + } +} + +fn source(err: moq_net::Error) -> Error { + Error::Source(err.to_string()) +} diff --git a/rs/moq-archive/src/producer/resume.rs b/rs/moq-archive/src/producer/resume.rs new file mode 100644 index 00000000000..f7509ecf404 --- /dev/null +++ b/rs/moq-archive/src/producer/resume.rs @@ -0,0 +1,89 @@ +//! What a prefix already holds, so a restarted producer continues each track after its last +//! stored frame. + +use std::collections::HashMap; + +use futures::TryStreamExt; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; + +use crate::layout::{self, End, Name, Object, Start}; +use crate::{Error, Info, Result, Store}; + +/// One track as stored. +pub(super) struct Stored { + pub info: Info, + /// Every object's key, oldest first. + pub objects: Vec, + /// The newest object, which says where the track continues. + pub newest: Option, + /// How its last run ended, when the producer that stored it saw that. + pub end: Option, +} + +/// Every stored track by name, or `None` when the broadcast's `.end` says it is complete. +pub(super) async fn load(store: &Store) -> Result>> { + let prefix = store.prefix(); + let scope = (!prefix.as_ref().is_empty()).then_some(prefix); + let listed = store.inner().list_with_delimiter(scope).await?; + + for object in &listed.objects { + match object.location.filename() { + Some(layout::BROADCAST_END) => return Ok(None), + _ => return Err(Error::Path(object.location.to_string())), + } + } + + let mut tracks = HashMap::new(); + for dir in listed.common_prefixes { + let segment = dir.filename().ok_or_else(|| Error::Path(dir.to_string()))?; + let name = layout::parse_track(segment)?; + let track = load_track(store, &dir).await?; + tracks.insert(name, track); + } + Ok(Some(tracks)) +} + +async fn load_track(store: &Store, dir: &Path) -> Result { + let entries: Vec<_> = store.inner().list(Some(dir)).try_collect().await?; + let info_path = crate::path::push(dir, layout::TRACK_INFO)?; + let info = Info::decode(&store.inner().get(&info_path).await?.bytes().await?)?; + let timed = info.timescale.is_some(); + + let mut objects: Vec<(Start, Path)> = Vec::new(); + let mut end = None; + for entry in entries { + // A flat listing only returns objects directly under the track; anything deeper is foreign. + if entry.location.prefix_match(dir).is_none_or(|parts| parts.count() != 1) { + return Err(Error::Path(entry.location.to_string())); + } + let filename = entry + .location + .filename() + .ok_or_else(|| Error::Path(entry.location.to_string()))?; + match Name::parse(filename, timed)? { + Name::Info => {} + Name::End => end = Some(End::decode(&store.inner().get(&entry.location).await?.bytes().await?)?), + Name::Object(start) => objects.push((start, entry.location)), + } + } + objects.sort(); + + let newest = match objects.last() { + Some((start, path)) => { + let object = Object::decode(store.inner().get(path).await?.bytes().await?, timed)?; + if object.start()? != *start { + return Err(Error::Path(path.to_string())); + } + Some(object) + } + None => None, + }; + + Ok(Stored { + info, + objects: objects.into_iter().map(|(_, path)| path).collect(), + newest, + end, + }) +} diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs new file mode 100644 index 00000000000..3c208d2d494 --- /dev/null +++ b/rs/moq-archive/src/producer/tests.rs @@ -0,0 +1,679 @@ +use std::collections::BTreeMap; + +use bytes::Bytes; +use futures::TryStreamExt; +use moq_net::track::Position; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::ObjectStoreExt; +use tokio::task::JoinHandle; +use tokio::time::sleep; + +use super::*; +use crate::layout::{At, Entry, Object}; +use crate::Info; +use crate::mock::Mock; + +const SECOND: Duration = Duration::from_secs(1); + +fn store() -> Store { + Store::new(Mock::memory(), "rec") +} + +/// No flush or gap verdict lands unless a test waits for it. +fn config() -> Config { + Config::default() + .with_duration(2 * SECOND) + .with_flush(100 * SECOND) + .with_max_delay(10 * SECOND) +} + +fn spawn(store: &Store, broadcast: &broadcast::Producer, config: Config) -> JoinHandle> { + tokio::spawn(Producer::new(store.clone(), broadcast.consume(), config).run()) +} + +fn timed() -> track::Info { + track::Info::default().with_timescale(Timescale::MILLI) +} + +fn ms(ms: u64) -> Timestamp { + Timestamp::from_millis(ms).unwrap() +} + +/// Write a whole group of `(pts, payload)` frames. +fn group(track: &track::Producer, sequence: u64, frames: &[(u64, &str)]) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for (pts, payload) in frames { + group.write_frame(ms(*pts), payload.to_string()).unwrap(); + } + group.finish().unwrap(); +} + +/// Let the producer catch up without moving time far. +async fn settle() { + for _ in 0..10 { + sleep(Duration::from_millis(1)).await; + } +} + +/// Wait for the producer to subscribe to `track`. +async fn subscribed(track: &track::Producer) { + while track.subscription().is_none() { + sleep(Duration::from_millis(1)).await; + } +} + +async fn finished(task: JoinHandle>) { + tokio::time::timeout(Duration::from_secs(1000), task) + .await + .expect("the producer ends with its broadcast") + .unwrap() + .expect("the producer succeeds"); +} + +/// Every stored key under the prefix, relative to it. +async fn keys(store: &Store) -> Vec { + let mut keys: Vec = store + .inner() + .list(Some(store.prefix())) + .map_ok(|meta| meta.location.as_ref().strip_prefix("rec/").unwrap().to_string()) + .try_collect() + .await + .unwrap(); + keys.sort(); + keys +} + +async fn get(store: &Store, key: &str) -> Option { + let path = object_store::path::Path::parse(format!("rec/{key}")).unwrap(); + match store.inner().get(&path).await { + Ok(got) => Some(got.bytes().await.unwrap()), + Err(object_store::Error::NotFound { .. }) => None, + Err(err) => panic!("{err}"), + } +} + +/// A track's objects by key name, in key order. +async fn objects(store: &Store, track: &str, timed: bool) -> BTreeMap { + let dir = format!("{}/", crate::path::encode_track(track).unwrap()); + let mut objects = BTreeMap::new(); + for key in keys(store).await { + let Some(name) = key.strip_prefix(&dir) else { + continue; + }; + if name == layout::TRACK_INFO || name == layout::TRACK_END { + continue; + } + let object = Object::decode(get(store, &key).await.unwrap(), timed).unwrap(); + objects.insert(name.to_string(), object); + } + objects +} + +/// Each object's entries as `(sequence, form, frames)`, where a frame is `pts/payload`. +fn contents(objects: &BTreeMap) -> Vec)>> { + objects + .values() + .map(|object| { + object + .entries + .iter() + .map(|entry| { + let (form, frames) = match entry { + Entry::Group { frames, .. } => ("group", frames.as_slice()), + Entry::Datagram { frame, .. } => ("datagram", std::slice::from_ref(frame)), + }; + let frames = frames + .iter() + .map(|frame| { + let payload = String::from_utf8(frame.payload.to_vec()).unwrap(); + match frame.timestamp { + Some(pts) => format!("{pts}/{payload}"), + None => payload, + } + }) + .collect(); + (entry.sequence(), form, frames) + }) + .collect() + }) + .collect() +} + +fn key(group: u64, frame: u64, pts: u64) -> String { + format!("{group:019}-{frame:019}-{pts:019}") +} + +async fn end(store: &Store, track: &str) -> Option { + let key = format!("{}/end", crate::path::encode_track(track).unwrap()); + get(store, &key).await.map(|bytes| End::decode(&bytes).unwrap()) +} + +async fn complete(store: &Store) -> bool { + get(store, layout::BROADCAST_END).await.is_some() +} + +#[tokio::test(start_paused = true)] +async fn timed_objects_cut_at_group_boundaries_by_duration() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + for sequence in 0..5 { + let pts = sequence * 1000; + group(&video, sequence, &[(pts, "a"), (pts + 500, "b")]); + } + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + settle().await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let objects = objects(&store, "video", true).await; + assert_eq!( + objects.keys().cloned().collect::>(), + [key(0, 0, 0), key(2, 0, 2000), key(4, 0, 4000)] + ); + assert_eq!( + contents(&objects)[0], + [ + (0, "group", vec!["0/a".to_string(), "500/b".to_string()]), + (1, "group", vec!["1000/a".to_string(), "1500/b".to_string()]), + ] + ); + let info = Info::decode(&get(&store, "video/.info").await.unwrap()).unwrap(); + assert_eq!(info.timescale, Some(1000)); + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 5, frame: 0 }))); + assert!(complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn a_lost_group_is_a_gap_that_never_stalls_later_groups() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let config = config().with_flush(SECOND).with_max_delay(2 * SECOND); + let task = spawn(&store, &broadcast, config); + subscribed(&video).await; + + group(&video, 0, &[(0, "a")]); + group(&video, 2, &[(2000, "c")]); + group(&video, 3, &[(3000, "d")]); + settle().await; + // Group 0 is flushed alone; 2 and 3 wait on the missing 1. + sleep(SECOND + SECOND / 2).await; + assert_eq!(objects(&store, "video", true).await.keys().cloned().collect::>(), [key(0, 0, 0)]); + + // Once the max delay passes, the gap is given up and the rest follow within the flush. + sleep(2 * SECOND).await; + let stored = objects(&store, "video", true).await; + assert_eq!(stored.keys().cloned().collect::>(), [key(0, 0, 0), key(2, 0, 2000)]); + assert_eq!( + contents(&stored)[1], + [ + (2, "group", vec!["2000/c".to_string()]), + (3, "group", vec!["3000/d".to_string()]), + ] + ); + + // The lost group arriving too late stays a gap. + group(&video, 1, &[(1000, "b")]); + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + let stored = objects(&store, "video", true).await; + assert_eq!(stored.len(), 2); + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 4, frame: 0 }))); +} + +#[tokio::test(start_paused = true)] +async fn an_oversized_timed_group_splits_into_ordered_unique_keys() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + // Timestamps repeat and go backward, as decode order allows. + let frames = [(0, "0"), (1000, "1"), (2000, "2"), (2000, "3"), (1500, "4"), (4000, "5"), (4000, "6")]; + group(&video, 0, &frames); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + settle().await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let objects = objects(&store, "video", true).await; + assert_eq!( + objects.keys().cloned().collect::>(), + [key(0, 0, 0), key(0, 2, 2000), key(0, 5, 4000)] + ); + let stored: Vec = contents(&objects) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + let expected: Vec = frames.iter().map(|(pts, payload)| format!("{pts}/{payload}")).collect(); + assert_eq!(stored, expected); + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 1, frame: 0 }))); +} + +#[tokio::test(start_paused = true)] +async fn an_open_untimed_group_is_rewritten_in_place_and_cut_by_wall_clock() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let data = broadcast.create_track("data", track::Info::default()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&data).await; + + let mut open = data.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(None, "a").unwrap(); + settle().await; + sleep(SECOND).await; + assert_eq!(contents(&objects(&store, "data", false).await), [[(0, "group", vec!["a".to_string()])]]); + + open.write_frame(None, "b").unwrap(); + settle().await; + sleep(SECOND).await; + let stored = objects(&store, "data", false).await; + assert_eq!(stored.keys().cloned().collect::>(), ["0000000000000000000"]); + assert_eq!(contents(&stored), [[(0, "group", vec!["a".to_string(), "b".to_string()])]]); + open.finish().unwrap(); + + // Past the duration, the next group starts the next object. + let mut next = data.create_group(group::Info { sequence: 1 }).unwrap(); + next.write_frame(None, "c").unwrap(); + next.finish().unwrap(); + settle().await; + data.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let stored = objects(&store, "data", false).await; + assert_eq!( + stored.keys().cloned().collect::>(), + ["0000000000000000000", "0000000000000000001"] + ); + let info = Info::decode(&get(&store, "data/.info").await.unwrap()).unwrap(); + assert_eq!(info.timescale, None); +} + +#[tokio::test(start_paused = true)] +async fn an_idle_track_is_flushed_within_the_bound() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&video).await; + + let mut open = video.create_group(group::Info { sequence: 0 }).unwrap(); + open.write_frame(ms(0), "a").unwrap(); + settle().await; + sleep(SECOND / 2).await; + assert!(objects(&store, "video", true).await.is_empty()); + sleep(SECOND / 2).await; + assert_eq!(objects(&store, "video", true).await.keys().cloned().collect::>(), [key(0, 0, 0)]); + + // The open group continues in the next object. + open.write_frame(ms(100), "b").unwrap(); + open.finish().unwrap(); + settle().await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!( + objects(&store, "video", true).await.keys().cloned().collect::>(), + [key(0, 0, 0), key(0, 1, 100)] + ); +} + +#[tokio::test(start_paused = true)] +async fn a_restart_mid_group_loses_and_duplicates_nothing() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let data = broadcast.create_track("data", track::Info::default()).unwrap(); + let config = config().with_flush(SECOND); + let task = spawn(&store, &broadcast, config.clone()); + subscribed(&video).await; + subscribed(&data).await; + + let mut timed_group = video.create_group(group::Info { sequence: 0 }).unwrap(); + let mut untimed_group = data.create_group(group::Info { sequence: 0 }).unwrap(); + for (pts, payload) in [(0, "a"), (100, "b")] { + timed_group.write_frame(ms(pts), payload).unwrap(); + untimed_group.write_frame(None, payload).unwrap(); + } + settle().await; + sleep(SECOND).await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + + // Both groups grow while nothing records them. + timed_group.write_frame(ms(200), "c").unwrap(); + untimed_group.write_frame(None, "c").unwrap(); + let task = spawn(&store, &broadcast, config); + settle().await; + timed_group.write_frame(ms(300), "d").unwrap(); + untimed_group.write_frame(None, "d").unwrap(); + timed_group.finish().unwrap(); + untimed_group.finish().unwrap(); + settle().await; + video.finish().unwrap(); + data.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let stored = objects(&store, "video", true).await; + assert_eq!(stored.keys().cloned().collect::>(), [key(0, 0, 0), key(0, 2, 200)]); + let frames: Vec = contents(&stored) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + assert_eq!(frames, ["0/a", "100/b", "200/c", "300/d"]); + + // The untimed group's object was read back and rewritten with the rest. + let stored = objects(&store, "data", false).await; + assert_eq!( + contents(&stored), + [[(0, "group", vec!["a", "b", "c", "d"].into_iter().map(String::from).collect())]] + ); + assert!(complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn changed_properties_fail_the_resume() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + group(&video, 0, &[(0, "a")]); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + video.finish().unwrap(); + drop(video); + broadcast.close(); + finished(task).await; + + // Another instance, recorded under the same prefix, which only a missing `.end` allows. + let store = Store::new(store.inner().fork(), "rec"); + store + .inner() + .delete(&object_store::path::Path::from("rec/.end")) + .await + .unwrap(); + let broadcast = broadcast::Info::new().produce(); + let _video = broadcast + .create_track("video", track::Info::default().with_timescale(Timescale::MICRO)) + .unwrap(); + let err = spawn(&store, &broadcast, config()).await.unwrap().unwrap_err(); + assert!( + matches!( + err, + Error::TimescaleMismatch { + existing: Some(1000), + intended: Some(1_000_000) + } + ), + "{err}" + ); +} + +#[tokio::test(start_paused = true)] +async fn track_names_stay_one_segment_through_a_resume() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let names = ["a/b", ".hidden", "end"]; + let tracks: Vec<_> = names + .iter() + .map(|name| broadcast.create_track(*name, track::Info::default()).unwrap()) + .collect(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + for track in &tracks { + subscribed(track).await; + let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(None, "x").unwrap(); + group.finish().unwrap(); + } + settle().await; + sleep(SECOND).await; + task.abort(); + let _ = task.await; + + let stored = keys(&store).await; + for dir in ["a%2Fb", "%2Ehidden", "end"] { + assert!(stored.contains(&format!("{dir}/0000000000000000000")), "{stored:?}"); + assert!(stored.contains(&format!("{dir}/.info")), "{stored:?}"); + } + + // The resume parses each one back to its name and continues after its stored group. + let task = spawn(&store, &broadcast, config()); + for track in &tracks { + subscribed(track).await; + track.finish().unwrap(); + } + broadcast.close(); + finished(task).await; + for name in names { + assert_eq!(end(&store, name).await, Some(End::Finish(At { group: 1, frame: 0 }))); + assert_eq!(objects(&store, name, false).await.len(), 1); + } +} + +#[tokio::test(start_paused = true)] +async fn a_dvr_prunes_superseded_objects_after_its_window_and_grace() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let dvr = Dvr::new(2 * SECOND, SECOND); + let task = spawn(&store, &broadcast, config().with_duration(SECOND).with_dvr(dvr)); + subscribed(&video).await; + + // Each group is its own object, closed when the next group starts. + for sequence in 0..4 { + group(&video, sequence, &[(sequence * 1000, "x")]); + settle().await; + sleep(SECOND).await; + } + // Object 0 was superseded when object 1 was stored, at 2s: it stays through 5s. + let names = |objects: BTreeMap| objects.values().map(|o| o.start().unwrap().group).collect::>(); + assert_eq!(names(objects(&store, "video", true).await), [0, 1, 2]); + sleep(SECOND).await; + settle().await; + assert_eq!(names(objects(&store, "video", true).await), [1, 2]); + sleep(3 * SECOND).await; + settle().await; + // The newest stored object is always kept. + assert_eq!(names(objects(&store, "video", true).await), [2]); + + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!(names(objects(&store, "video", true).await), [2, 3]); + assert!(end(&store, "video").await.is_some()); +} + +#[tokio::test(start_paused = true)] +async fn a_track_added_mid_recording_is_recorded() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + group(&video, 0, &[(0, "v")]); + settle().await; + + let audio = broadcast.create_track("audio", timed()).unwrap(); + subscribed(&audio).await; + group(&audio, 0, &[(0, "a")]); + settle().await; + video.finish().unwrap(); + audio.finish().unwrap(); + broadcast.close(); + finished(task).await; + + assert_eq!(objects(&store, "audio", true).await.len(), 1); + assert_eq!(objects(&store, "video", true).await.len(), 1); + assert!(end(&store, "audio").await.is_some()); + assert!(complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn groups_and_datagrams_keep_their_form_through_an_abrupt_end() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let mut track = broadcast.create_track("mixed", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&track).await; + + group(&track, 0, &[(0, "g0")]); + track.insert_datagram(1, ms(100), "d1").unwrap(); + group(&track, 2, &[(200, "g2")]); + track.insert_datagram(3, ms(300), "d3").unwrap(); + track.insert_datagram(4, ms(400), "d4").unwrap(); + // Ends at once: every received datagram is stored before the end. + track.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let stored = objects(&store, "mixed", true).await; + assert_eq!( + contents(&stored), + [[ + (0, "group", vec!["0/g0".to_string()]), + (1, "datagram", vec!["100/d1".to_string()]), + (2, "group", vec!["200/g2".to_string()]), + (3, "datagram", vec!["300/d3".to_string()]), + (4, "datagram", vec!["400/d4".to_string()]), + ]] + ); + assert_eq!(end(&store, "mixed").await, Some(End::Finish(At { group: 5, frame: 0 }))); +} + +#[tokio::test(start_paused = true)] +async fn markers_wait_for_the_listing_to_end() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let mut video = broadcast.create_track("video", timed()).unwrap(); + let audio = broadcast.create_track("audio", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + subscribed(&audio).await; + + group(&video, 0, &[(0, "a")]); + let mut last = video.create_group(group::Info { sequence: 1 }).unwrap(); + last.write_frame(ms(1000), "b").unwrap(); + last.write_frame(ms(1100), "c").unwrap(); + // A mid-group end: frames 0 and 1 of group 1. + video.finish_at(Position { group: 1, frame: 2 }).unwrap(); + last.finish().unwrap(); + settle().await; + sleep(SECOND).await; + // Ended, but a track re-added under the name would continue it. + assert_eq!(end(&store, "video").await, None); + assert!(!complete(&store).await); + + audio.abort(moq_net::Error::App(7)).unwrap(); + settle().await; + assert_eq!(end(&store, "audio").await, None); + + broadcast.close(); + finished(task).await; + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 1, frame: 2 }))); + // An error end is recorded, and the broadcast still completes. + assert_eq!(end(&store, "audio").await, Some(End::Error(64 + 7))); + assert!(complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn a_re_added_track_continues_and_its_marker_records_the_last_run() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let first = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&first).await; + group(&first, 0, &[(0, "a")]); + group(&first, 1, &[(1000, "b")]); + first.finish().unwrap(); + settle().await; + drop(first); + settle().await; + + let second = broadcast.create_track("video", timed()).unwrap(); + subscribed(&second).await; + group(&second, 2, &[(2000, "c")]); + settle().await; + second.abort(moq_net::Error::App(3)).unwrap(); + settle().await; + broadcast.close(); + finished(task).await; + + let frames: Vec = contents(&objects(&store, "video", true).await) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + assert_eq!(frames, ["0/a", "1000/b", "2000/c"]); + assert_eq!(end(&store, "video").await, Some(End::Error(64 + 3))); +} + +#[tokio::test(start_paused = true)] +async fn a_stopped_producer_writes_no_marker() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&video).await; + group(&video, 0, &[(0, "a")]); + settle().await; + sleep(SECOND).await; + task.abort(); + let _ = task.await; + video.finish().unwrap(); + broadcast.close(); + + assert_eq!(objects(&store, "video", true).await.len(), 1); + assert_eq!(end(&store, "video").await, None); + assert!(!complete(&store).await); +} + +#[tokio::test(start_paused = true)] +async fn a_completed_broadcast_records_nothing() { + let store = store(); + store + .inner() + .put(&object_store::path::Path::from("rec/.end"), Default::default()) + .await + .unwrap(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + group(&video, 0, &[(0, "a")]); + finished(spawn(&store, &broadcast, config())).await; + assert_eq!(keys(&store).await, [".end"]); + assert!(video.subscription().is_none()); +} + +#[tokio::test(start_paused = true)] +async fn a_track_that_ended_unseen_leaves_the_broadcast_open() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let audio = broadcast.create_track("audio", timed()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&video).await; + subscribed(&audio).await; + group(&audio, 0, &[(0, "a")]); + settle().await; + sleep(SECOND).await; + task.abort(); + let _ = task.await; + + // Audio ends while nothing records, so the resume never sees how. + audio.finish().unwrap(); + drop(audio); + let task = spawn(&store, &broadcast, config()); + subscribed(&video).await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert!(end(&store, "video").await.is_some()); + assert_eq!(end(&store, "audio").await, None); + assert!(!complete(&store).await); +} diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs new file mode 100644 index 00000000000..d7feb2aa095 --- /dev/null +++ b/rs/moq-archive/src/producer/track.rs @@ -0,0 +1,598 @@ +//! One track's recorder: its runs, in order, into duration-cut objects. + +use std::collections::{BTreeMap, VecDeque}; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Poll; +use std::time::Duration; + +use bytes::Bytes; +use moq_clock::Instant; +use moq_net::track::{Anchor, Order, Position, Range, Subscription}; +use moq_net::{Datagram, StreamError, Timescale, group}; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutPayload}; + +use super::resume::Stored; +use super::Shared; +use crate::layout::{self, End, Entry, Object}; +use crate::segment::Frame; +use crate::{Error, Info, Result}; + +/// Record each run of the track `name` reported through `queue`, until the queue closes with the +/// broadcast's listing. Returns how the last run ended, or the stored marker when no run did. +pub(super) async fn record( + shared: Arc>, + name: String, + queue: kio::Queue, + stored: Option, +) -> Result<(String, Option)> { + let mut track = Track::new(shared, &name, stored)?; + while let Some(info) = track.idle(&queue).await? { + track.properties(&info).await?; + let end = track.run(&info).await?; + track.end = Some(end); + } + Ok((name, track.end)) +} + +struct Track { + shared: Arc>, + name: String, + dir: Path, + /// The stored properties, once checked or written. + info: Option, + /// Whether `info` is known to be stored. + info_stored: bool, + timescale: Option, + /// The first position not stored yet, unknown until something is. + next: Option, + /// The object being filled. + object: Option, + /// The newest stored object, which DVR pruning always keeps. + newest: Option, + /// Objects a newer one superseded, oldest first, with when. + superseded: VecDeque<(Instant, Path)>, + /// Writes owed to the store, in order. + puts: Vec, + end: Option, +} + +struct Building { + object: Object, + /// When it was opened, which cuts an untimed object by wall clock. + opened: Instant, + /// The arrival of its oldest frame not stored yet, which bounds the flush. + unstored: Option, + /// Whether it is in the store under its key, as an untimed object rewritten in place is. + stored: bool, + /// On a timed track, the timestamp its last entry's run in this object starts at. + run_pts: u64, +} + +struct Put { + path: Path, + bytes: Bytes, + /// Rewrite an untimed object in place, rather than create an immutable one. + overwrite: bool, +} + +/// A sequence received ahead of the one being stored. +enum Pending { + Group(group::Consumer, Instant), + Datagram(Datagram, Instant), +} + +impl Pending { + fn arrived(&self) -> Instant { + match self { + Self::Group(_, at) | Self::Datagram(_, at) => *at, + } + } +} + +/// One subscription's progress. +struct Run { + max_delay: Duration, + /// Received sequences not stored yet, waiting on a lower one. + pending: BTreeMap, + /// The group being stored, frame by frame. + current: Option, + /// When the gap below the lowest pending sequence is given up. + gap: Option, + groups_done: bool, + datagrams_done: bool, +} + +enum Event { + Frame(u64, moq_net::Result>), + Group(moq_net::Result>), + Datagram(moq_net::Result>), + Timer, +} + +impl Track { + fn new(shared: Arc>, name: &str, stored: Option) -> Result { + let dir = layout::track_dir(shared.store.prefix(), name)?; + let mut track = Self { + shared, + name: name.to_string(), + dir, + info: None, + info_stored: false, + timescale: None, + next: None, + object: None, + newest: None, + superseded: VecDeque::new(), + puts: Vec::new(), + end: None, + }; + let Some(stored) = stored else { + return Ok(track); + }; + + track.timescale = timescale(&stored.info)?; + track.info = Some(stored.info); + track.end = stored.end; + let now = moq_clock::now(); + let mut objects = stored.objects; + track.newest = objects.pop(); + if track.shared.config.dvr.is_some() { + // Their age is unknown, so their window starts now. + track.superseded = objects.into_iter().map(|path| (now, path)).collect(); + } + if let Some(newest) = stored.newest { + track.next = Some(newest.end()?); + // An untimed object keeps growing in place; a timed one is immutable. + if track.timescale.is_none() { + track.object = Some(Building { + object: newest, + opened: now, + unstored: None, + stored: true, + run_pts: 0, + }); + } + } + Ok(track) + } + + /// Wait for the next run, pruning meanwhile: the newest of the queued track infos, or `None` + /// once the listing ended. + async fn idle(&mut self, queue: &kio::Queue) -> Result> { + loop { + let mut timer = self.prune_at().map(|at| Box::pin(moq_clock::sleep_until(at))); + let popped = kio::wait(|waiter| { + if let Poll::Ready(popped) = queue.poll_pop(waiter) { + return Poll::Ready(Some(popped)); + } + ready_at(&mut timer, waiter).map(|()| None) + }) + .await; + match popped { + // A track removed and re-added while a run was still draining queues it twice. + Some(Ok(mut info)) => { + while let Ok(Some(newer)) = queue.try_pop() { + info = newer; + } + return Ok(Some(info)); + } + Some(Err(kio::Closed)) => return Ok(None), + None => self.prune().await?, + } + } + } + + /// Store the track's properties before its first object, refusing ones that changed. + async fn properties(&mut self, source: &moq_net::track::Info) -> Result<()> { + let info = Info::new(source.priority, source.timescale.map(Timescale::as_u64))?; + if let Some(existing) = &self.info { + if existing.priority != info.priority { + return Err(Error::Priority { + existing: existing.priority, + intended: info.priority, + }); + } + if existing.timescale != info.timescale { + return Err(Error::TimescaleMismatch { + existing: existing.timescale, + intended: info.timescale, + }); + } + } + if !self.info_stored { + self.shared.store.put_info(&self.name, &info).await?; + self.info_stored = true; + } + self.timescale = source.timescale; + self.info = Some(info); + Ok(()) + } + + /// Record one subscription until both its groups and its datagrams end, returning how. + async fn run(&mut self, info: &moq_net::track::Info) -> Result { + let config = &self.shared.config; + let max_delay = info.max_age.map_or(config.max_delay, |age| age.min(config.max_delay)); + let range = match self.next { + Some(next) => Range { + start: Anchor::Absolute(next), + end: None, + }, + None => Range::oldest(), + }; + let subscription = Subscription::default() + .with_order(Order::Ascending) + .with_max_delay(max_delay) + .with_ranges([range]); + + let track = match self.shared.broadcast.track(&self.name) { + Ok(track) => track, + Err(err) => return Ok(error(&err)), + }; + let mut subscriber = match track.subscribe(subscription).await { + Ok(subscriber) => subscriber, + Err(err) => return Ok(error(&err)), + }; + + let mut run = Run { + max_delay, + pending: BTreeMap::new(), + current: None, + gap: None, + groups_done: false, + datagrams_done: false, + }; + loop { + self.advance(&mut run, false)?; + self.flush_due(moq_clock::now())?; + self.write().await?; + self.prune().await?; + if run.groups_done && run.datagrams_done { + break; + } + + let deadlines = [self.flush_at(), run.gap, self.prune_at()]; + let mut timer = deadlines + .into_iter() + .flatten() + .min() + .map(|at| Box::pin(moq_clock::sleep_until(at))); + let event = kio::wait(|waiter| { + if let Some(current) = run.current.as_mut() { + let index = current.index(); + if let Poll::Ready(frame) = current.poll_read_frame(waiter) { + return Poll::Ready(Event::Frame(index, frame)); + } + } + if !run.groups_done + && let Poll::Ready(group) = subscriber.poll_recv_group(waiter) + { + return Poll::Ready(Event::Group(group)); + } + if !run.datagrams_done + && let Poll::Ready(datagram) = subscriber.poll_recv_datagram(waiter) + { + return Poll::Ready(Event::Datagram(datagram)); + } + ready_at(&mut timer, waiter).map(|()| Event::Timer) + }) + .await; + + let now = moq_clock::now(); + match event { + Event::Frame(index, read) => self.frame(&mut run, index, read, now)?, + Event::Group(Ok(Some(group))) => { + if self.next.is_none_or(|next| group.sequence >= next.group) { + run.pending.insert(group.sequence, Pending::Group(group, now)); + } + } + Event::Datagram(Ok(Some(datagram))) => { + if self.next.is_none_or(|next| datagram.sequence >= next.group) { + run.pending.insert(datagram.sequence, Pending::Datagram(datagram, now)); + } + } + // How the track ended is read from the subscription below. + Event::Group(_) => run.groups_done = true, + Event::Datagram(_) => run.datagrams_done = true, + Event::Timer => {} + } + } + + // Everything received is stored, gaps and all, now that nothing more arrives. + while run.current.is_some() || !run.pending.is_empty() { + self.advance(&mut run, true)?; + if let Some(current) = run.current.as_mut() { + let index = current.index(); + let read = kio::wait(|waiter| current.poll_read_frame(waiter)).await; + self.frame(&mut run, index, read, moq_clock::now())?; + } + } + + let end = match kio::wait(|waiter| subscriber.poll_finished(waiter)).await { + Ok(end) => End::Finish(end.into()), + Err(err) => error(&err), + }; + self.flush()?; + self.write().await?; + Ok(end) + } + + /// Take pending sequences in order while the next one is here or its gap was given up. + fn advance(&mut self, run: &mut Run, draining: bool) -> Result<()> { + run.gap = None; + while run.current.is_none() { + let Some(entry) = run.pending.first_entry() else { + return Ok(()); + }; + let sequence = *entry.key(); + if let Some(next) = self.next { + if sequence < next.group { + entry.remove(); + continue; + } + let given_up = entry.get().arrived() + run.max_delay; + if sequence > next.group && !draining && moq_clock::now() < given_up { + run.gap = Some(given_up); + return Ok(()); + } + } + match entry.remove() { + Pending::Datagram(datagram, arrived) => { + let frame = Frame { + timestamp: self.stamp(datagram.timestamp)?, + payload: datagram.payload, + }; + self.push(sequence, 0, frame, true, arrived)?; + self.next = Some(Position::group(sequence + 1)); + } + Pending::Group(mut group, _) => { + // A resumed group continues after its stored frames. + if let Some(next) = self.next + && next.group == sequence + && group.index() < next.frame + { + group.set_frames(next.frame..); + } + self.next = Some(Position { + group: sequence, + frame: group.index(), + }); + run.current = Some(group); + } + } + } + Ok(()) + } + + /// Store the frame read at `index` of the group being stored, or move past that group once it + /// ended, cleanly or not (aborted, or past the budget), keeping what it stored. + fn frame( + &mut self, + run: &mut Run, + index: u64, + read: moq_net::Result>, + now: Instant, + ) -> Result<()> { + let sequence = run.current.as_ref().expect("reading a group").sequence; + match read { + Ok(Some(frame)) => { + let frame = Frame { + timestamp: self.stamp(frame.timestamp)?, + payload: frame.payload, + }; + self.push(sequence, index, frame, false, now)?; + self.next = Some(Position { + group: sequence, + frame: index + 1, + }); + } + ended => { + if let Err(err) = ended { + tracing::debug!(track = %self.name, sequence, %err, "group ended early"); + } + run.current = None; + self.next = Some(Position::group(sequence + 1)); + } + } + Ok(()) + } + + /// A frame's timestamp in the track's timescale, which it carries exactly when timed. + fn stamp(&self, timestamp: Option) -> Result> { + match (timestamp, self.timescale) { + (Some(timestamp), Some(timescale)) => { + Ok(Some(timestamp.convert(timescale).map_err(|_| Error::Timing)?.value())) + } + (None, None) => Ok(None), + _ => Err(Error::Timing), + } + } + + /// Append one frame of `sequence`, or a datagram, cutting the object before it when due. + fn push(&mut self, sequence: u64, index: u64, frame: Frame, datagram: bool, now: Instant) -> Result<()> { + let config = &self.shared.config; + if let Some(building) = &self.object { + let last = building.object.entries.last().map(Entry::sequence); + let boundary = last != Some(sequence); + let cut = match (self.timescale, frame.timestamp) { + (Some(timescale), Some(pts)) => { + let duration = units(config.duration, timescale); + let first = building.object.start()?.pts.expect("a timed object starts timed"); + match boundary { + true => pts.saturating_sub(first) >= duration, + // An oversized group splits once its own run here reaches the duration. + false => pts.saturating_sub(building.run_pts) >= duration, + } + } + _ => boundary && now.saturating_duration_since(building.opened) >= config.duration, + }; + if cut { + self.close()?; + } + } + + let building = self.object.get_or_insert_with(|| Building { + object: Object { + frame_start: index, + entries: Vec::new(), + }, + opened: now, + unstored: None, + stored: false, + run_pts: 0, + }); + let pts = frame.timestamp.unwrap_or(0); + match building.object.entries.last_mut() { + Some(Entry::Group { sequence: last, frames }) if *last == sequence && !datagram => frames.push(frame), + _ => { + building.run_pts = pts; + building.object.entries.push(match datagram { + true => Entry::Datagram { sequence, frame }, + false => Entry::Group { + sequence, + frames: vec![frame], + }, + }); + } + } + building.unstored.get_or_insert(now); + Ok(()) + } + + /// When the object's oldest unstored frame is due in the store. + fn flush_at(&self) -> Option { + let unstored = self.object.as_ref()?.unstored?; + Some(unstored + self.shared.config.flush) + } + + fn flush_due(&mut self, now: Instant) -> Result<()> { + if self.flush_at().is_some_and(|at| at <= now) { + self.flush()?; + } + Ok(()) + } + + /// Store the object: a timed one is closed, so its group continues in the next; an untimed + /// one is rewritten in place and stays open. + fn flush(&mut self) -> Result<()> { + match self.timescale { + Some(_) => self.close(), + None => { + let Some(building) = self.object.as_mut() else { + return Ok(()); + }; + if building.unstored.take().is_some() { + let bytes = building.object.encode(false)?; + building.stored = true; + let path = self.object_path()?; + self.queue(path, bytes, true); + } + Ok(()) + } + } + } + + /// Store the object for good and start the next one at the following frame. + fn close(&mut self) -> Result<()> { + let Some(building) = self.object.take() else { + return Ok(()); + }; + if building.unstored.is_none() && building.stored { + return Ok(()); + } + let timed = self.timescale.is_some(); + let bytes = building.object.encode(timed)?; + let path = push(&self.dir, &building.object.start()?.name()?)?; + self.queue(path, bytes, !timed); + Ok(()) + } + + fn object_path(&self) -> Result { + let building = self.object.as_ref().expect("an open object"); + push(&self.dir, &building.object.start()?.name()?) + } + + fn queue(&mut self, path: Path, bytes: Bytes, overwrite: bool) { + if self.newest.as_ref() != Some(&path) { + let superseded = self.newest.replace(path.clone()); + if let Some(superseded) = superseded + && self.shared.config.dvr.is_some() + { + self.superseded.push_back((moq_clock::now(), superseded)); + } + } + self.puts.push(Put { path, bytes, overwrite }); + } + + /// Perform the owed writes in order. + async fn write(&mut self) -> Result<()> { + let store = self.shared.store.inner(); + for put in std::mem::take(&mut self.puts) { + let payload = PutPayload::from(put.bytes.clone()); + if put.overwrite { + store.put(&put.path, payload).await?; + continue; + } + match store.put_opts(&put.path, payload, PutMode::Create.into()).await { + Ok(_) => {} + // Written already, by an earlier attempt that failed to report success. + Err(object_store::Error::AlreadyExists { .. }) => { + let existing = store.get(&put.path).await?.bytes().await?; + if existing != put.bytes { + return Err(Error::Conflict(put.path.to_string())); + } + } + Err(err) => return Err(err.into()), + } + } + Ok(()) + } + + fn prune_at(&self) -> Option { + let dvr = self.shared.config.dvr.as_ref()?; + let (superseded, _) = self.superseded.front()?; + Some(*superseded + dvr.window + dvr.grace) + } + + /// Delete the superseded objects whose window and grace have passed. + async fn prune(&mut self) -> Result<()> { + let now = moq_clock::now(); + while self.prune_at().is_some_and(|at| at <= now) { + let (_, path) = self.superseded.pop_front().expect("a superseded object is due"); + match self.shared.store.inner().delete(&path).await { + Ok(()) | Err(object_store::Error::NotFound { .. }) => {} + Err(err) => return Err(err.into()), + } + } + Ok(()) + } +} + +fn push(dir: &Path, name: &str) -> Result { + crate::path::push(dir, name) +} + +fn timescale(info: &Info) -> Result> { + info.timescale + .map(|timescale| Timescale::new(timescale).map_err(|_| Error::Timescale(timescale))) + .transpose() +} + +/// `duration` in `timescale` units, saturating. +fn units(duration: Duration, timescale: Timescale) -> u64 { + let units = duration.as_nanos() * u128::from(timescale.as_u64()) / 1_000_000_000; + u64::try_from(units).unwrap_or(u64::MAX) +} + +fn error(err: &moq_net::Error) -> End { + End::Error(StreamError::from(err).to_code()) +} + +/// Poll an optional timer, pending forever without one. +fn ready_at(timer: &mut Option>>, waiter: &kio::Waiter) -> Poll<()> { + match timer.as_mut() { + Some(timer) => waiter.poll_future(timer.as_mut()), + None => Poll::Pending, + } +} diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs deleted file mode 100644 index d72b8ab66c8..00000000000 --- a/rs/moq-archive/src/proof.rs +++ /dev/null @@ -1,577 +0,0 @@ -//! End to end: one multi-rendition broadcast recorded by a [`Writer`], stored byte-identically on -//! every backend, and replayed exactly through a [`Reader`]. - -use std::collections::BTreeMap; -use std::time::Duration; - -use bytes::Bytes; -use futures::TryStreamExt; -use moq_net::{Timescale, Timestamp, broadcast, group, track}; -use object_store::local::LocalFileSystem; -use object_store::path::Path; -use object_store::{ObjectStore, ObjectStoreExt}; - -use crate::mock::{Mock, Op}; -use crate::reader::Config as ReaderConfig; -use crate::writer::{Config, Retention}; -use crate::{Reader, Store, Writer}; - -const RENDITIONS: [&str; 2] = ["video/1080p", "video/360p"]; -const MEDIA: [&str; 3] = ["video/1080p", "video/360p", "audio"]; -const TRACKS: [&str; 5] = ["video/1080p", "video/360p", "audio", "catalog.json", "chat"]; - -/// Frames of one source group: (millisecond timestamp, payload). -type Frames = Vec<(u64, Bytes)>; - -/// Source segments: three 2s segments, then a one-frame tail. -const SEGMENTS: u64 = 4; - -/// The source groups of one segment: (track, sequence, frames). -/// -/// Video renditions share a 2s GOP and audio has four 500ms groups per segment, which its own -/// timeline packs into 2s records by the default minimum. The catalog and a non-media chat track publish sparsely, -/// including a sequence skip and an empty payload. -fn plan(segment: u64) -> Vec<(&'static str, u64, Frames)> { - let pts = segment * 2000; - let (frames, audio) = if segment + 1 < SEGMENTS { (4, 4) } else { (1, 1) }; - let mut groups = Vec::new(); - for track in RENDITIONS { - let frames = (0..frames) - .map(|i| (pts + i * 500, Bytes::from(format!("{track} {segment}.{i}")))) - .collect(); - groups.push((track, segment, frames)); - } - for sequence in segment * 4..segment * 4 + audio { - let frames = (0..2) - .map(|i| (sequence * 500 + i * 250, Bytes::from(format!("audio {sequence}.{i}")))) - .collect(); - groups.push(("audio", sequence, frames)); - } - let catalog = |sequence: u64| Bytes::from(format!(r#"{{"version":{sequence}}}"#)); - let chat = |text: &'static str| Bytes::from_static(text.as_bytes()); - match segment { - 0 => { - groups.push(("catalog.json", 0, vec![(0, catalog(0))])); - groups.push(("chat", 0, vec![(1200, chat("hello")), (1300, Bytes::new())])); - } - 2 => { - groups.push(("catalog.json", 1, vec![(4000, catalog(1))])); - groups.push(("chat", 3, vec![(4100, chat("skip"))])); - groups.push(("chat", 5, vec![(5100, chat("bye"))])); - } - _ => {} - } - groups -} - -fn ms(value: u64) -> Timestamp { - Timestamp::new(value, Timescale::MILLI).unwrap() -} - -fn write(track: &track::Producer, sequence: u64, frames: &Frames) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for (timestamp, payload) in frames { - group.write_frame(ms(*timestamp), payload.clone()).unwrap(); - } - group.finish().unwrap(); -} - -/// Let the writer read and report everything already published. Reads never touch the store. -async fn settle() { - for _ in 0..256 { - tokio::task::yield_now().await; - } -} - -/// Media cut at 1s, so one-second test groups get a record each. -fn media() -> moq_mux::timeline::Config { - moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(1)) -} - -fn timeline(track: &str) -> String { - hang::timeline::default_name(track) -} - -/// Every recorded track's timeline, as the catalog's `archive` entry names them. -fn config() -> ReaderConfig { - ReaderConfig::new( - TRACKS - .iter() - .map(|track| (track.to_string(), timeline(track))) - .collect(), - ) -} - -/// Record every segment of [`plan`] into `store`. -async fn record(store: &Store) { - let source = broadcast::Info::new().produce(); - let tracks: BTreeMap<&str, track::Producer> = TRACKS - .iter() - .map(|&name| { - let info = track::Info::default() - .with_timescale(Timescale::MILLI) - .with_max_age(Duration::from_secs(3600)); - (name, source.create_track(name, info).unwrap()) - }) - .collect(); - - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - for name in TRACKS { - let config = match MEDIA.contains(&name) { - // Keep this storage-layout proof at one record per 2s source segment. The default - // timeline minimum is tested in moq-mux and is not part of this fixture's contract. - true => moq_mux::timeline::Config::default().with_duration_min(Duration::from_secs(2)), - false => moq_mux::timeline::Config::default().with_duration_min(Duration::ZERO), - }; - control.track(name, config).await.unwrap(); - } - let run = tokio::spawn(writer.run()); - - // Each track cuts its own timeline, so the interleave across tracks changes nothing stored. - for segment in 0..SEGMENTS { - for (name, sequence, frames) in plan(segment) { - write(&tracks[name], sequence, &frames); - } - settle().await; - } - for track in tracks.values() { - track.finish().unwrap(); - } - source.close(); - run.await.unwrap().unwrap(); -} - -/// Every object under the store's prefix, by path relative to the store root. -async fn objects(store: &Store) -> BTreeMap { - let metas: Vec<_> = store.inner().list(Some(store.prefix())).try_collect().await.unwrap(); - let mut objects = BTreeMap::new(); - for meta in metas { - let bytes = store.inner().get(&meta.location).await.unwrap().bytes().await.unwrap(); - objects.insert(meta.location.to_string(), bytes); - } - objects -} - -fn id(value: u64) -> String { - format!("{value:019}") -} - -/// Each track's record count: a GOP per video record, four audio groups per record, and a group per -/// sparse record. -const RECORDS: [(&str, u64); 5] = [ - ("video%2F1080p", 4), - ("video%2F360p", 4), - ("audio", 4), - ("catalog%2Ejson", 2), - ("chat", 3), -]; - -/// The exact layout [`plan`] produces: no manifest, index, or completion marker. -fn layout() -> Vec { - let mut keys = Vec::new(); - for (track, records) in RECORDS { - for directory in [track.to_string(), format!("{track}%2Etimeline%2Ez")] { - keys.push(format!("rec/{directory}/.info")); - keys.extend((0..records).map(|record| format!("rec/{directory}/segments/{}", id(record)))); - } - } - keys.sort(); - keys -} - -/// FETCH one whole group, returning its frames. -async fn fetch(broadcast: &broadcast::Producer, track: &str, sequence: u64) -> moq_net::Result { - let track = broadcast.consume().track(track)?; - let mut group = track.fetch_group(sequence, group::Fetch::default()).await?; - let mut frames = Vec::new(); - while let Some(frame) = group.read_frame().await? { - frames.push(( - frame.timestamp.unwrap().convert(Timescale::MILLI).unwrap().value(), - frame.payload, - )); - } - Ok(frames) -} - -#[tokio::test] -async fn recordings_are_byte_identical_on_every_backend() { - let memory = Store::new(Mock::memory(), "rec"); - record(&memory).await; - let expected = objects(&memory).await; - assert_eq!(expected.keys().cloned().collect::>(), layout()); - assert_eq!( - &expected["rec/video%2F360p/.info"][..], - br#"{"version":2,"priority":127,"timescale":1000}"# - ); - - let dir = tempfile::tempdir().unwrap(); - let local = Store::new(Mock::new(LocalFileSystem::new_with_prefix(dir.path()).unwrap()), "rec"); - record(&local).await; - assert_eq!(objects(&local).await, expected, "local disk"); - - // A backend with no listing order, and a sibling recording sharing the prefix's stem. - let unordered = Mock::memory(); - unordered.unordered(); - let sibling = Store::new(unordered.clone(), "rec-other"); - record(&sibling).await; - let unordered = Store::new(unordered, "rec"); - record(&unordered).await; - assert_eq!(objects(&unordered).await, expected, "unordered listing"); - assert_eq!(objects(&sibling).await.len(), expected.len()); - - // The media objects hold exactly the source groups, in order, byte for byte. - let mut stored: BTreeMap<(String, u64), Frames> = BTreeMap::new(); - for (path, bytes) in &expected { - let key = crate::Key::parse(&Path::from("rec"), &Path::parse(path).unwrap()).unwrap(); - if let crate::Key::Segments { track, .. } = key - && !track.ends_with(hang::timeline::SUFFIX) - { - let object = crate::Object::decode(bytes.clone(), true).unwrap(); - assert_eq!(object.frame_start, 0, "no source group outlived the maximum"); - for group in object.groups { - let frames = group - .frames - .into_iter() - .map(|f| (f.timestamp.unwrap(), f.payload)) - .collect(); - stored.insert((track.clone(), group.sequence), frames); - } - } - } - let source: BTreeMap<(String, u64), Frames> = (0..SEGMENTS) - .flat_map(plan) - .map(|(track, sequence, frames)| ((track.to_string(), sequence), frames)) - .collect(); - assert_eq!(stored, source); -} - -#[tokio::test] -async fn fetch_replays_the_recording_and_reads_only_the_requested_rendition() { - let mock = Mock::memory(); - let store = Store::new(mock.clone(), "rec"); - record(&store).await; - - let broadcast = broadcast::Info::new().produce(); - let reader = Reader::open(store, &broadcast, config()).await.unwrap(); - tokio::spawn(reader.serve()); - mock.take(); - - // Low-rendition playback never downloads the 1080p object. - for sequence in 0..SEGMENTS { - let expected = plan(sequence) - .into_iter() - .find(|(name, ..)| *name == "video/360p") - .unwrap() - .2; - assert_eq!(fetch(&broadcast, "video/360p", sequence).await.unwrap(), expected); - } - let object = |track: &str, record: u64| format!("rec/{track}/segments/{}", id(record)); - // Each track request also GETs that track's `.info`, but nothing of another track. - let media = |gets: Vec, track: &str| { - let prefix = format!("rec/{track}/"); - assert!(gets.iter().all(|path| path.starts_with(&prefix)), "{gets:?}"); - gets.into_iter() - .filter(|path| path.contains("/segments/")) - .collect::>() - }; - assert_eq!( - media(mock.gets(), "video%2F360p"), - (0..4).map(|record| object("video%2F360p", record)).collect::>() - ); - - // Audio-only playback: four adjacent groups per GET, the rest from the cache. - for segment in 0..SEGMENTS { - for (_, sequence, frames) in plan(segment).into_iter().filter(|(name, ..)| *name == "audio") { - assert_eq!(fetch(&broadcast, "audio", sequence).await.unwrap(), frames); - } - } - assert_eq!( - media(mock.gets(), "audio"), - (0..4).map(|record| object("audio", record)).collect::>() - ); - - // Every other enrolled group replays its original sequence, timestamps, and payloads. - for segment in 0..SEGMENTS { - for (name, sequence, frames) in plan(segment) { - assert_eq!( - fetch(&broadcast, name, sequence).await.unwrap(), - frames, - "{name} {sequence}" - ); - } - } - for (track, sequence) in [("chat", 1), ("chat", 4), ("chat", 6), ("audio", 13), ("video/1080p", 4)] { - assert!( - matches!(fetch(&broadcast, track, sequence).await, Err(moq_net::Error::NotFound)), - "{track} {sequence} was never recorded" - ); - } - assert!( - !mock.take().iter().any(|op| matches!(op, Op::List { .. })), - "a group request resolves its object without listing" - ); -} - -#[tokio::test] -async fn an_offline_reader_follows_dvr_expiry() { - let mock = Mock::memory(); - let store = Store::new(mock.clone(), "rec"); - let source = broadcast::Info::new().produce(); - let info = track::Info::default() - .with_timescale(Timescale::MILLI) - .with_max_age(Duration::from_secs(3600)); - let video = source.create_track("video", info).unwrap(); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - let frames = |sequence: u64| vec![(sequence * 1000, Bytes::from(format!("video {sequence}")))]; - let committed = |segment: u64| { - let store = store.clone(); - async move { - while store.get_segments(&timeline("video"), segment, true).await.is_err() { - tokio::time::sleep(Duration::from_millis(5)).await; - } - } - }; - - for sequence in 0..3 { - write(&video, sequence, &frames(sequence)); - } - committed(1).await; - - let broadcast = broadcast::Info::new().produce(); - let config = ReaderConfig::new([("video".to_string(), timeline("video"))].into()); - let mut reader = Reader::open(store.clone(), &broadcast, config).await.unwrap(); - tokio::spawn(reader.serve()); - assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), frames(0)); - - // The reader is offline while the DVR commits and expires several records. - for sequence in 3..10 { - write(&video, sequence, &frames(sequence)); - } - committed(8).await; - // Records 7 and 8 hold the 2s window, and the newest timeline object restates both. - while store.get_segments("video", 6, true).await.is_ok() - || store.get_segments(&timeline("video"), 7, true).await.is_ok() - { - tokio::time::sleep(Duration::from_millis(5)).await; - } - - mock.take(); - reader.refresh().await.unwrap(); - let ops = mock.take(); - let (list, gets) = ops.split_first().unwrap(); - assert_eq!( - list, - &Op::List { - prefix: "rec/video%2Etimeline%2Ez/segments".to_string(), - offset: Some(format!("rec/video%2Etimeline%2Ez/segments/{}", id(1))), - } - ); - assert_eq!( - gets, - &[Op::Get(format!("rec/video%2Etimeline%2Ez/segments/{}", id(8)))], - "only the retained timeline key is read" - ); - - // Expired groups are gone from the index, so they cost no media GET. - for sequence in 0..7 { - assert!( - matches!( - fetch(&broadcast, "video", sequence).await, - Err(moq_net::Error::NotFound) - ), - "group {sequence} expired" - ); - } - let gets = mock.gets(); - assert!( - !gets.iter().any(|path| path.starts_with("rec/video/segments/")), - "{gets:?}" - ); - for sequence in 7..9 { - assert_eq!(fetch(&broadcast, "video", sequence).await.unwrap(), frames(sequence)); - } - - video.finish().unwrap(); - source.close(); - run.await.unwrap().unwrap(); -} - -/// Exercise the S3 HTTP adapter, rather than an ObjectStore listing fake. -mod s3 { - use std::num::NonZeroUsize; - use std::sync::{Arc, Mutex}; - - use axum::body::Body; - use axum::http::{Method, Uri}; - use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut}; - use object_store::{PutMode, PutPayload}; - use s3s::service::S3ServiceBuilder; - - use super::*; - - struct Request { - method: Method, - uri: Uri, - conditional: bool, - } - - struct Server { - store: Store, - requests: Arc>>, - task: tokio::task::JoinHandle<()>, - _dir: tempfile::TempDir, - } - - impl Server { - async fn new() -> Self { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir(dir.path().join("archive-proof")).unwrap(); - let service = S3ServiceBuilder::new(s3s_fs::FileSystem::new(dir.path()).unwrap()).build(); - let requests = Arc::new(Mutex::new(Vec::new())); - let seen = requests.clone(); - let app = axum::Router::new().fallback(move |request: axum::extract::Request| { - let service = service.clone(); - let seen = seen.clone(); - async move { - seen.lock().unwrap().push(Request { - method: request.method().clone(), - uri: request.uri().clone(), - conditional: request.headers().get("if-none-match").is_some_and(|value| value == "*"), - }); - service - .call(request.map(s3s::Body::http_body_unsync)) - .await - .unwrap() - .map(Body::new) - } - }); - let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) - .await - .unwrap(); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); - let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - let store = AmazonS3Builder::new() - .with_bucket_name("archive-proof") - .with_region("us-east-1") - .with_endpoint(endpoint) - .with_allow_http(true) - .with_skip_signature(true) - .with_conditional_put(S3ConditionalPut::ETagMatch) - .build() - .unwrap(); - Self { - store: Store::new(store, "rec"), - requests, - task, - _dir: dir, - } - } - } - - impl Drop for Server { - fn drop(&mut self) { - self.task.abort(); - } - } - - #[tokio::test] - async fn archive_proof_crosses_s3_http() { - let memory = Store::new(Mock::memory(), "rec"); - record(&memory).await; - let expected = objects(&memory).await; - let server = Server::new().await; - record(&server.store).await; - assert_eq!(objects(&server.store).await, expected); - let broadcast = broadcast::Info::new().produce(); - let reader = Reader::open(server.store.clone(), &broadcast, config()).await.unwrap(); - let task = tokio::spawn(reader.serve()); - for segment in 0..SEGMENTS { - for (name, sequence, frames) in plan(segment) { - assert_eq!(fetch(&broadcast, name, sequence).await.unwrap(), frames); - } - } - task.abort(); - assert!( - server - .requests - .lock() - .unwrap() - .iter() - .any(|request| request.uri.path().contains("catalog%252Ejson")), - "a percent-encoded recording key must survive URL encoding" - ); - } - - #[tokio::test] - async fn s3_create_offset_and_continuation_are_enforced() { - let server = Server::new().await; - record(&server.store).await; - let expected = objects(&server.store).await; - let (path, original) = expected.first_key_value().unwrap(); - let path = Path::parse(path).unwrap(); - let collision = server - .store - .inner() - .put_opts(&path, PutPayload::from_static(b"different"), PutMode::Create.into()) - .await; - assert!( - matches!(collision, Err(object_store::Error::AlreadyExists { .. })), - "{collision:?}" - ); - assert_eq!( - server.store.inner().get(&path).await.unwrap().bytes().await.unwrap(), - *original - ); - - let mut query = crate::store::list::Query::new().page_size(NonZeroUsize::new(2).unwrap()); - let mut listed = Vec::new(); - loop { - let page = server.store.list_paginated(&query).await.unwrap(); - assert!(page.entries.len() <= 2); - listed.extend( - page.entries - .iter() - .map(|entry| server.store.path(&entry.key).unwrap().to_string()), - ); - let Some(next) = page.next else { break }; - query = next; - } - listed.sort(); - assert_eq!(listed, expected.keys().cloned().collect::>()); - - let after = crate::Key::segments("video/1080p", 0).unwrap(); - let query = crate::store::list::Query::segments("video/1080p") - .unwrap() - .after(&after) - .unwrap(); - let later: Vec<_> = server.store.list(&query).try_collect().await.unwrap(); - assert_eq!(later.len(), (SEGMENTS - 1) as usize); - assert!(later.iter().all( - |entry| matches!(&entry.key, crate::Key::Segments { track, segment } if track == "video/1080p" && *segment > 0) - )); - let requests = server.requests.lock().unwrap(); - assert!( - requests - .iter() - .any(|request| request.method == Method::PUT && request.conditional) - ); - assert!(requests.iter().any(|request| { - request - .uri - .query() - .is_some_and(|query| query.contains("continuation-token=")) - })); - assert!( - requests - .iter() - .any(|request| request.uri.query().is_some_and(|query| query.contains("start-after="))) - ); - } -} diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs index 3cf641605e3..d3bd64b6adb 100644 --- a/rs/moq-archive/src/reader/tests.rs +++ b/rs/moq-archive/src/reader/tests.rs @@ -701,67 +701,6 @@ async fn timeline_tracks_are_republished_and_finished_on_request() { assert_eq!(records, vec![0, 1]); } -/// An untimed recording replays untimed: no media or timeline frame gains a timestamp. -#[tokio::test] -async fn an_untimed_recording_replays_untimed() { - let store = Store::new(Mock::memory(), "rec"); - let source = broadcast::Info::new().produce(); - let info = moq_net::track::Info::default().with_max_age(Duration::from_secs(3600)); - let data = source.create_track("data", info).unwrap(); - let writer = crate::Writer::new(store.clone(), source.consume(), Default::default()) - .await - .unwrap(); - writer - .control() - .track("data", moq_mux::timeline::Config::default()) - .await - .unwrap(); - for sequence in 0..2 { - let mut group = data.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, format!("data/{sequence}")).unwrap(); - group.finish().unwrap(); - } - data.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let broadcast = broadcast::Info::new().produce(); - let config = Config::new([("data".to_string(), timeline("data"))].into()); - let reader = Reader::open(store, &broadcast, config).await.unwrap(); - tokio::spawn(reader.serve()); - - let track = broadcast.consume().track("data").unwrap(); - let mut group = track.fetch_group(1, group::Fetch::default()).await.unwrap(); - assert_eq!(group.timescale(), None); - let frame = group.read_frame().await.unwrap().unwrap(); - assert_eq!((frame.timestamp, frame.payload), (None, Bytes::from("data/1"))); - - let subscriber = broadcast - .consume() - .track(&timeline("data")) - .unwrap() - .subscribe(None) - .await - .unwrap(); - assert_eq!(subscriber.info().timescale, None); - let mut timeline = - window::Consumer::::new(subscriber, window::ConsumerConfig::default().with_compression(true)); - reader.finish().unwrap(); - let mut records = Vec::new(); - while let Some(event) = timeline.next().await.unwrap() { - if let window::Event::Push { value, .. } = event { - records.push(value); - } - } - assert_eq!( - records, - vec![ - Record::untimed(0, Position::group(0), Position::group(1)), - Record::untimed(1, Position::group(1), Position::group(2)), - ] - ); -} - /// A record whose timing contradicts its timeline is not indexed, though its object is stored. #[tokio::test] async fn a_record_contradicting_its_timeline_is_not_indexed() { diff --git a/rs/moq-archive/src/recover.rs b/rs/moq-archive/src/recover.rs deleted file mode 100644 index 188e36c9043..00000000000 --- a/rs/moq-archive/src/recover.rs +++ /dev/null @@ -1,210 +0,0 @@ -//! Recover a recording so a restarted [`Writer`](crate::Writer) continues it. - -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; -use std::ops::{Range, RangeInclusive}; - -use futures::TryStreamExt; -use hang::timeline::{Position, Record}; -use moq_json::window::{self, Checkpoint}; -use object_store::ObjectStore; - -use crate::store::list::Query; -use crate::{Error, Key, Object, Result, Store}; - -/// What a restarted writer continues from. -#[derive(Default)] -pub(crate) struct Recovery { - /// Each recorded track's retained timeline, keyed by the recorded track's name. - pub tracks: HashMap, - /// Committed objects no retained record references, such as an interrupted expiration. Only - /// collected for a DVR, which deletes them after its grace. - pub orphans: Vec, - /// Objects at or past a track's next record: uploads a crash left before their commit. No - /// timeline ever referenced them, and the resumed writer reuses their keys. - pub uncommitted: Vec, - /// Each timeline's stored segments older than any recovery reads. Only collected for a DVR. - pub stale: Vec<(String, Range)>, -} - -/// One track's retained timeline. -pub(crate) struct Resume { - /// The retained window. - pub checkpoint: Checkpoint, - /// The next stored timeline group sequence, so the stored numbering keeps increasing. - pub sequence: u64, - /// The timeline objects recovery replayed, oldest first, with the first record index each - /// one's opening checkpoint restates. Only collected for a DVR. - pub checkpoints: VecDeque<(u64, u64)>, -} - -impl Resume { - /// Where the newest committed record ended: the resumed track refuses anything before it. - pub fn floor(&self) -> Option { - self.checkpoint.records.last().map(|record| record.end) - } -} - -/// List the whole recording, then replay each track's timeline from a retained checkpoint. -/// -/// A DVR (`complete`) replays far enough back to recover every retained record and reports the -/// unreferenced objects; an archive only needs the newest checkpoint. Any listing, GET, or decode -/// failure fails recovery, so a caller never acts on a partial view. -pub(crate) async fn recover(store: &Store, complete: bool) -> Result { - let entries: Vec<_> = store.list(&Query::new()).try_collect().await?; - let mut segments: BTreeMap> = BTreeMap::new(); - for entry in entries { - if let Key::Segments { track, segment } = entry.key { - segments.entry(track).or_default().push(segment); - } - } - - let mut recovery = Recovery::default(); - for (timeline, ids) in &mut segments { - let Some(track) = timeline.strip_suffix(hang::timeline::SUFFIX) else { - continue; - }; - ids.sort_unstable(); - let (first, last) = (ids[0], ids[ids.len() - 1]); - if ids.len() as u64 != last - first + 1 { - return Err(Error::Timeline(format!( - "{timeline} segments {first}..={last} are not contiguous" - ))); - } - let resume = replay(store, timeline, first..=last, complete).await?; - // The newest object is segment `last`, so the window must end on the next one. A shorter - // window would resume onto that segment and collide with it. - let next = last.checked_add(1).ok_or(Error::Overflow)?; - if resume.checkpoint.range.end != next { - return Err(Error::Timeline(format!( - "{timeline} window ends at {}, not segment {next}", - resume.checkpoint.range.end - ))); - } - if let Some(&(needed, _)) = resume.checkpoints.front() - && first < needed - { - recovery.stale.push((timeline.clone(), first..needed)); - } - recovery.tracks.insert(track.to_string(), resume); - } - - for (track, ids) in segments { - if track.ends_with(hang::timeline::SUFFIX) { - continue; - } - let resume = recovery.tracks.get(&track); - let next = resume.map_or(0, |resume| resume.checkpoint.range.end); - let retained: HashSet = resume - .iter() - .flat_map(|resume| &resume.checkpoint.records) - .map(|record| record.sequence) - .collect(); - for id in ids { - let key = Key::segments(track.clone(), id)?; - if id >= next { - recovery.uncommitted.push(key); - } else if complete && !retained.contains(&id) { - recovery.orphans.push(key); - } - } - } - - Ok(recovery) -} - -/// Replay `segments` from the newest checkpoint that restates every record `complete` needs. -async fn replay( - store: &Store, - timeline: &str, - segments: RangeInclusive, - complete: bool, -) -> Result { - // Every object opens with a checkpoint. The newest one's offset bounds what is still retained, - // so walk back until a checkpoint restates from there. The writer prunes by the same rule. - let timed = store.get_info(timeline).await?.timescale.is_some(); - let mut objects = VecDeque::new(); - let mut checkpoints = VecDeque::new(); - let mut needed = None; - for segment in segments.rev() { - let object = store.get_segments(timeline, segment, timed).await?; - let (offset, start) = checkpoint(&object)?; - let needed = *needed.get_or_insert(offset); - objects.push_front(object); - if complete { - checkpoints.push_front((segment, start)); - } - if !complete || start <= needed { - break; - } - } - - let mut decoder = decoder(); - let mut records = BTreeMap::new(); - for object in &objects { - for stored in &object.groups { - let mut group = decoder.group(); - for frame in &stored.frames { - group.decode(&frame.payload).map_err(json_error)?; - } - } - while let Some(event) = decoder.next_event() { - match event { - window::Event::Push { index, value } => { - records.insert(index, value); - } - window::Event::Pop(range) | window::Event::Skip(range) => { - let tail = records.split_off(&range.end); - records.retain(|index, _| *index < range.start); - records.extend(tail); - } - _ => {} - } - } - } - - // The records must be a contiguous suffix of the window, and all of it for a DVR. - let range = decoder.range(); - let start = records.keys().next().copied().unwrap_or(range.end); - if records.len() as u64 != range.end - start || (complete && start != range.start) { - return Err(Error::Timeline(format!( - "cannot recover {timeline} window {range:?} from the retained checkpoints" - ))); - } - - let last = objects.back().and_then(|object| object.groups.last()); - let sequence = last.map_or(Ok(0), |group| group.sequence.checked_add(1).ok_or(Error::Overflow))?; - let checkpoint = Checkpoint { - range, - records: records.into_values().collect(), - }; - Ok(Resume { - checkpoint, - sequence, - checkpoints, - }) -} - -/// The retained offset and first restated index of the checkpoint opening `object`. -pub(crate) fn checkpoint(object: &Object) -> Result<(u64, u64)> { - let frame = object - .groups - .first() - .and_then(|group| group.frames.first()) - .ok_or_else(|| Error::Timeline("timeline object has no checkpoint".into()))?; - let mut decoder = decoder(); - decoder.group().decode(&frame.payload).map_err(json_error)?; - let offset = decoder.range().start; - let start = match decoder.next_event() { - Some(window::Event::Skip(skipped)) => skipped.end, - _ => offset, - }; - Ok((offset, start)) -} - -fn decoder() -> window::Decoder { - window::Decoder::new(window::ConsumerConfig::default().with_compression(true)) -} - -fn json_error(err: moq_json::Error) -> Error { - Error::Timeline(err.to_string()) -} diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 355f338c372..d3d7cee6809 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -167,12 +167,6 @@ impl Store { intended: info.timescale, }); } - if parsed.epoch != info.epoch { - return Err(Error::EpochMismatch { - existing: parsed.epoch, - intended: info.epoch.clone(), - }); - } Ok(key) } } @@ -198,15 +192,13 @@ impl Store { Object::decode(self.get_bytes(&path).await?, timed) } - /// Each recorded track's timeline track, found by listing the track directories a [`Writer`] - /// creates: a timeline is named by [`hang::timeline::default_name`]. One listing of the + /// Each recorded track's timeline track, found by listing the recording's track directories: + /// a timeline is named by [`hang::timeline::default_name`]. One listing of the /// prefix's directories plus a HEAD per timeline, so following a long recording doesn't list /// its objects. A timeline whose `.info` isn't stored yet, such as one still enrolling, is left /// out until it is. /// /// For replaying a recording without its catalog; a catalog's `archive` entry names the same map. - /// - /// [`Writer`]: crate::Writer pub async fn timelines(&self) -> Result> { let listed = self.inner.list_with_delimiter(self.list_path(None).as_ref()).await?; let mut timelines = std::collections::BTreeMap::new(); diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs deleted file mode 100644 index 013d5cc8e62..00000000000 --- a/rs/moq-archive/src/writer.rs +++ /dev/null @@ -1,2305 +0,0 @@ -//! Record explicitly selected tracks of a [`broadcast::Consumer`] into a [`Store`]. -//! -//! The application enrolls each track through a [`Control`]; the writer never parses a catalog. -//! Every track has its own timeline: frames feed the track's segmenter as they arrive, and for -//! each closed record the writer stores the track's object, commits the record through the -//! track's own timeline encoder, and stores that record's timeline groups before committing the -//! track's next one. A timeline therefore only advertises durable objects, and no track waits for -//! another. -//! -//! An untimed track, such as any track over moq-transport without TIMESCALE, is recorded untimed: -//! each group is its own record with no `pts` or `duration`, and its objects store no timestamps. -//! -//! A writer started on a prefix that already holds a recording resumes it: see [`Writer::new`]. -//! -//! ```no_run -//! # async fn example(source: moq_net::broadcast::Consumer) -> moq_archive::Result<()> { -//! use moq_archive::object_store::memory::InMemory; -//! -//! let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); -//! let writer = moq_archive::Writer::new(store, source, Default::default()).await?; -//! let control = writer.control(); -//! let media = moq_mux::timeline::Config::default(); -//! control.track("video", media.clone()).await?; -//! control.track("audio", media.clone()).await?; -//! // Sparse data: each catalog group is its own record. -//! control.track("catalog.json", media.with_duration_min(std::time::Duration::ZERO)).await?; -//! writer.run().await?; -//! # Ok(()) -//! # } -//! ``` - -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; -use std::future::Future; -use std::ops::Range; -use std::sync::{Arc, Mutex}; -use std::task::Poll; -use std::time::Duration; - -use futures::future::BoxFuture; -use futures::stream::FuturesUnordered; -use futures::{FutureExt, StreamExt}; -use hang::timeline::{Position, Record}; -use moq_clock::Instant; -use moq_mux::timeline::{self, Segmenter}; -use moq_net::{Timescale, Timestamp, broadcast, group, track}; -use object_store::ObjectStore; -use tokio::sync::{mpsc, watch}; - -use crate::recover::{Resume, checkpoint, recover}; -use crate::segment::{Frame, Group, Object}; -use crate::{Error, Info, Key, Result, Store}; - -/// Subscribers ask for every cached group; the publisher clamps this to its own max age. -const REPLAY: Duration = Duration::from_secs(u32::MAX as u64); - -/// How a [`Writer`] retains its recording. Each track's cutting is chosen when it enrolls. -#[derive(Clone, Debug, Default)] -#[non_exhaustive] -pub struct Config { - /// Expire old records (a DVR), or keep everything when `None` (an archive). - pub retention: Option, -} - -impl Config { - /// Set [`retention`](Self::retention). - pub fn with_retention(mut self, retention: impl Into>) -> Self { - self.retention = retention.into(); - self - } -} - -/// A DVR retention policy. -#[derive(Clone, Debug)] -#[non_exhaustive] -pub struct Retention { - /// Keep at least this much of each track's content, measured from its timeline records. - pub window: Duration, - /// How long expired objects outlive the timeline that stopped advertising them, so readers - /// holding an older timeline can finish their GETs. - pub grace: Duration, -} - -impl Retention { - /// Keep `window` of content and delete expired objects `grace` after they leave the timeline. - pub fn new(window: Duration, grace: Duration) -> Self { - Self { window, grace } - } -} - -/// Records enrolled tracks until the source broadcast closes. -/// -/// Enroll tracks through [`control`](Self::control), then drive the recording with -/// [`run`](Self::run). -pub struct Writer { - control: Control, - commands: mpsc::UnboundedReceiver>, - /// Deadlines for deleting expired or orphaned objects, oldest first. - deletions: VecDeque<(Instant, Expired)>, -} - -/// Enrolls, cuts, and removes tracks on a [`Writer`]. -pub struct Control { - shared: Arc>, - commands: mpsc::UnboundedSender>, -} - -impl Clone for Control { - fn clone(&self) -> Self { - Self { - shared: self.shared.clone(), - commands: self.commands.clone(), - } - } -} - -struct Shared { - store: Store, - source: broadcast::Consumer, - retention: Option, - /// Owns the recording's timeline tracks, read back to store their groups. - timelines: broadcast::Producer, - /// Each track's retained timeline from a resumed recording, taken when it enrolls again. - recovered: Mutex>, - /// Every name ever enrolled. A name is never reused, so its records stay consecutive. - enrolled: Mutex>, - /// Enrollments between their first check and their command, so the recording does not end - /// under one that is about to arrive. - enrolling: watch::Sender, -} - -/// One [`Control::track`] in flight: drops its count, and its name unless it enrolled, however -/// the call ends. -struct Enrolling<'a, S> { - shared: &'a Shared, - name: &'a str, - enrolled: bool, -} - -impl Drop for Enrolling<'_, S> { - fn drop(&mut self) { - self.shared.enrolling.send_modify(|count| *count -= 1); - if !self.enrolled { - self.shared.enrolled.lock().unwrap().remove(self.name); - } - } -} - -enum Command { - Enroll(Box>), - Cut(Timestamp), - Remove(String), - Stop, -} - -impl Writer { - /// Start a recording under `store`'s prefix, reading tracks from `source`, or resume the one - /// already there. - /// - /// Resuming replays each track's retained timeline, and a re-enrolled track continues after - /// its newest committed record, skipping what the source replays from before it. A source - /// announced under another epoch than the recording is a restart: enrolling its tracks fails - /// with [`Error::EpochMismatch`], and the caller starts a new prefix. Objects a crash left - /// before their commit are deleted now, since the resumed records reuse their keys. A DVR - /// also deletes, one grace period after recovery, every object its retained records do not - /// reference, such as interrupted expirations, and every timeline object older than the - /// checkpoint it recovered from. The writer must own the prefix exclusively. Fails, deleting - /// nothing, when the recording cannot be listed or a timeline cannot be replayed. - pub async fn new(store: Store, source: broadcast::Consumer, config: Config) -> Result { - let recovery = recover(&store, config.retention.is_some()).await?; - for key in &recovery.uncommitted { - store.delete(key).await?; - } - - let mut deletions = VecDeque::new(); - let stale = Expired { - objects: recovery.orphans, - timelines: recovery.stale, - }; - if let Some(retention) = &config.retention - && !stale.is_empty() - { - deletions.push_back((Instant::now() + retention.grace, stale)); - } - - let shared = Arc::new(Shared { - store, - source, - retention: config.retention, - timelines: broadcast::Info::new().produce(), - recovered: Mutex::new(recovery.tracks), - enrolled: Mutex::new(HashSet::new()), - enrolling: watch::Sender::new(0), - }); - let (commands, receiver) = mpsc::unbounded_channel(); - Ok(Self { - control: Control { shared, commands }, - commands: receiver, - deletions, - }) - } - - /// A handle that enrolls, cuts, and removes tracks, before or during [`run`](Self::run). - pub fn control(&self) -> Control { - self.control.clone() - } - - /// Record until the source broadcast closes and every enrolled track ends, or [`Control::stop`]. - /// - /// Then flush each track's final record and finish its timeline. Fails when a timeline cannot - /// be committed or stored: the recording stops at each track's last durable timeline object. - /// Also fails when an enrolled track delivers a frame the recording cannot represent, but - /// only after stopping every track and committing what each already reported, so the - /// recording stays readable up to the bad frame. That includes a resumed track whose first - /// new group starts before the recorded end, the one restart a source without an epoch - /// shows. Returns the source's error, after finishing, when the broadcast aborted. - pub async fn run(self) -> Result<()> { - let Self { - control, - mut commands, - mut deletions, - } = self; - let shared = control.shared.clone(); - let source = shared.source.clone(); - let grace = shared.retention.as_ref().map(|retention| retention.grace); - let mut enrolling = shared.enrolling.subscribe(); - drop(control); - // Each timeline's oldest segment that may still be stored; pruning deletes upward from it. - let mut pruned = HashMap::new(); - - let mut tracks: HashMap>> = HashMap::new(); - let mut reads = FuturesUnordered::new(); - let mut commits: FuturesUnordered> = FuturesUnordered::new(); - let mut closed = false; - // Cleared once the channel yields nothing more: every sender dropped, or it closed and drained. - let mut accepting = true; - // Every track was stopped, by `Control::stop` or malformed input, so `run` ends once flushed. - let mut stopped = false; - // Malformed source input, returned once every track flushed. - let mut failed: Option = None; - // The tracks the last event touched. An event touches at most one, except a failure, so the - // loop never scans the whole table in steady state. - let mut touched: Vec = Vec::new(); - - loop { - for name in std::mem::take(&mut touched) { - let Some(track) = tracks.get_mut(&name) else { - continue; - }; - if let Some(commit) = track.commit()? { - commits.push(commit); - } - if track.finish() { - tracks.remove(&name); - } - } - - let idle = (closed || stopped) && tracks.is_empty() && commits.is_empty(); - if idle && *enrolling.borrow_and_update() == 0 { - // Refuse late commands, so an enrollment racing the end fails instead of vanishing. - commands.close(); - if !accepting { - break; - } - } - - let deadline = deletions.front().map(|(deadline, _)| *deadline); - tokio::select! { - biased; - command = commands.recv(), if accepting => match command { - None => accepting = false, - Some(Command::Enroll(mut track)) => { - let subscriber = track.subscriber.take().expect("an enrolling track carries its subscriber"); - // A stopped recording takes no new content; the track just finishes its timeline. - if stopped { - track.remove(); - } - reads.push(guard(track.cancelled.clone(), recv(track.name.clone(), subscriber)).boxed()); - touched.push(track.name.clone()); - tracks.insert(track.name.clone(), track); - } - Some(Command::Cut(pts)) => { - for track in tracks.values_mut() { - track.segmenter.cut(pts); - } - } - Some(Command::Remove(name)) => { - if let Some(track) = tracks.get_mut(&name) { - track.remove(); - touched.push(name); - } - } - Some(Command::Stop) => { - stopped = true; - touched.extend(stop(&mut tracks)); - commands.close(); - } - }, - Some(read) = reads.next(), if !reads.is_empty() => match handle(read, &mut tracks, &mut reads) { - Ok(name) => touched.extend(name), - Err(err) => { - tracing::error!(%err, "malformed source input; flushing every track and stopping"); - failed.get_or_insert(err); - stopped = true; - touched.extend(stop(&mut tracks)); - commands.close(); - } - }, - Some((name, committer, result)) = commits.next(), if !commits.is_empty() => { - if let Some(track) = tracks.get_mut(&name) { - track.committer = Some(committer); - touched.push(name); - } - let expired = result?; - if let Some(grace) = grace && !expired.is_empty() { - deletions.push_back((Instant::now() + grace, expired)); - } - } - _ = source.closed(), if !closed => closed = true, - // Also once commands closed: an enrollment in flight still settles its count. - _ = enrolling.changed(), if idle => {} - _ = moq_clock::sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => { - let (_, expired) = deletions.pop_front().unwrap(); - delete(&shared.store, &mut pruned, expired).await; - } - } - } - - // Complete expirations already committed to the timelines; nothing new expires after the - // final records. - for (deadline, expired) in deletions { - moq_clock::sleep_until(deadline).await; - delete(&shared.store, &mut pruned, expired).await; - } - - if let Some(err) = failed { - return Err(err); - } - if stopped { - return Ok(()); - } - // A broadcast end carries no cause. - source.closed().await; - Ok(()) - } -} - -impl Control { - /// Enroll the track `name`, its records cut by `config`. - /// - /// Media keeps the default minimum; sparse data such as a catalog uses zero, so each group is - /// stored as soon as it finishes. An untimed track ignores `config`: each group is its own - /// record, stored once it finishes. Subscribes to the track and creates its `.info` and its - /// timeline's before accepting any group. Fails when the name was already enrolled, names a - /// timeline, or `.info` conflicts, and with [`Error::Untimed`] when a DVR enrolls an untimed - /// track, whose retention it cannot measure. - pub async fn track(&self, name: &str, config: timeline::Config) -> Result<()> { - if name.ends_with(hang::timeline::SUFFIX) || !self.shared.enrolled.lock().unwrap().insert(name.to_string()) { - return Err(Error::Enrolled(name.to_string())); - } - self.shared.enrolling.send_modify(|count| *count += 1); - // Settles the count and the name even if this future is dropped mid-enrollment. - let mut enrolling = Enrolling { - shared: &self.shared, - name, - enrolled: false, - }; - self.subscribe(name, config).await?; - enrolling.enrolled = true; - Ok(()) - } - - /// Stop recording every track: store what already arrived, finish each timeline, and end - /// [`run`](Writer::run) without waiting for the source. Later commands are refused. - pub fn stop(&self) -> Result<()> { - self.send(Command::Stop) - } - - /// Declare a boundary at `pts` on every enrolled track; see [`Segmenter::cut`]. - pub fn cut(&self, pts: Timestamp) -> Result<()> { - self.send(Command::Cut(pts)) - } - - /// Stop recording `name`, storing what already arrived and dropping groups that could not be - /// indexed yet. The name cannot be enrolled again. - pub fn remove(&self, name: &str) -> Result<()> { - self.send(Command::Remove(name.to_string())) - } - - async fn subscribe(&self, name: &str, config: timeline::Config) -> Result<()> { - let shared = &self.shared; - let replay = track::Subscription::default().with_max_delay(REPLAY); - let subscriber = shared - .source - .track(name) - .map_err(source_error)? - .subscribe(replay.clone()) - .await - .map_err(source_error)?; - let timescale = subscriber.info().timescale; - if timescale.is_none() && shared.retention.is_some() { - return Err(Error::Untimed(name.to_string())); - } - let info = Info::new(subscriber.info().priority, timescale.map(Timescale::as_u64))? - .with_epoch(shared.source.info().epoch.clone()); - shared.store.put_info(name, &info).await?; - - let timeline = hang::timeline::default_name(name); - let output = shared - .timelines - .create_track(timeline.as_str(), timeline::Producer::info()) - .map_err(timeline_error_net)?; - let groups = shared - .timelines - .consume() - .track(&timeline) - .map_err(timeline_error_net)? - .subscribe(replay) - .await - .map_err(timeline_error_net)? - .ordered(); - // The stored timeline is timed exactly when its track is: an untimed track's records have no - // content time to stamp their timeline frames with. - let timeline_scale = timescale.and(groups.info().timescale); - let info = Info::new(groups.info().priority, timeline_scale.map(Timescale::as_u64))?; - shared.store.put_info(&timeline, &info).await?; - - let resume = shared.recovered.lock().unwrap().remove(name); - let (output, segmenter, window, checkpoints, offset, floor, recorded) = match resume { - Some(resume) => { - let floor = resume.floor(); - let recorded = recorded_end(&resume, timescale)?; - let output = timeline::Producer::resume(output, &resume.checkpoint).map_err(timeline_error)?; - let segmenter = Segmenter::new(config).with_sequence(resume.checkpoint.range.end); - let window = resume.checkpoint.records.into(); - ( - output, - segmenter, - window, - resume.checkpoints, - resume.sequence, - floor, - recorded, - ) - } - None => ( - timeline::Producer::new(output), - Segmenter::new(config), - VecDeque::new(), - VecDeque::new(), - 0, - None, - None, - ), - }; - - let (cancel, cancelled) = watch::channel(()); - let track = Track { - name: name.to_string(), - timescale, - segmenter, - floor, - largest: None, - // A new group must not start before the recorded end, or it would overlap it. - reported: recorded, - accepted: BTreeMap::new(), - frames: VecDeque::new(), - records: VecDeque::new(), - subscriber: Some(Box::new(subscriber)), - subscribed: true, - closed: false, - committer: Some(Committer { - shared: shared.clone(), - name: name.to_string(), - timeline, - output, - groups, - window, - checkpoints, - offset, - }), - cancel: Some(cancel), - cancelled, - }; - self.send(Command::Enroll(Box::new(track))) - } - - fn send(&self, command: Command) -> Result<()> { - self.commands.send(command).map_err(|_| Error::Closed) - } -} - -/// A group accepted from the source, not yet fully reported. -#[derive(Default)] -struct Incoming { - /// The index of the next frame to arrive. - next: u64, - /// Frames that arrived but were not reported, with their indices. - pending: Vec<(u64, Frame)>, - /// No more frames will arrive. - finished: bool, -} - -/// One enrolled track: its source reads, its segmenter, and its committer. -struct Track { - name: String, - /// `None` when the track is untimed. - timescale: Option, - segmenter: Segmenter, - /// A resumed track's committed end; earlier content is already stored. - floor: Option, - /// The newest accepted group; later groups must exceed it. - largest: Option, - /// The first-frame timestamp of the newest reported group; later groups must not precede it. - /// A resumed track starts at the recorded end. - reported: Option, - /// Accepted groups in sequence order. Reported front first, so a later group's frames wait - /// for every earlier accepted group to finish. - accepted: BTreeMap, - /// Reported frames waiting for the record that holds them. - frames: VecDeque<(Position, Frame)>, - /// Closed records waiting to be committed, oldest first. - records: VecDeque, - /// Moved into the first read once enrolled. - subscriber: Option>, - /// The subscription is still delivering groups. - subscribed: bool, - /// The segmenter was closed: every remaining record is queued. - closed: bool, - /// `None` while a commit is in flight. - committer: Option>, - /// Dropping the sender cancels every read for this track. - cancel: Option>, - cancelled: watch::Receiver<()>, -} - -impl Track { - /// Report every frame the segmenter can place: the front group's arrivals, then each later - /// group once the ones before it finish. - /// - /// Fails on a group that starts before the previous one, which the timeline cannot place. - fn report(&mut self) -> Result<()> { - while let Some(mut entry) = self.accepted.first_entry() { - let sequence = *entry.key(); - let incoming = entry.get_mut(); - for (index, frame) in incoming.pending.drain(..) { - if index == 0 - && let Some(timestamp) = frame.timestamp - { - if let Some(reported) = self.reported - && timestamp < reported - { - return Err(malformed( - &self.name, - sequence, - format!("timestamp {timestamp} precedes {reported}"), - )); - } - self.reported = Some(timestamp); - } - // Frame timestamps were validated against the track while reading, so this conversion - // succeeds, and an untimed track reports no timestamps. - let pts = frame - .timestamp - .zip(self.timescale) - .map(|(timestamp, timescale)| { - Timestamp::new(timestamp, timescale).map_err(|_| Error::Id(timestamp)) - }) - .transpose()?; - let position = Position::new(sequence, index); - self.segmenter - .frame(position, pts, index == 0) - .map_err(|err| malformed(&self.name, sequence, err))?; - self.frames.push_back((position, frame)); - } - if !incoming.finished { - break; - } - entry.remove(); - self.segmenter.finish_group(sequence); - } - Ok(()) - } - - /// Stop reading: drop the groups that were never reported and close the segmenter. - fn remove(&mut self) { - self.cancel = None; - self.subscribed = false; - self.accepted.clear(); - } - - /// Start committing the next closed record, if the committer is idle. - fn commit(&mut self) -> Result>> { - if !self.subscribed && self.accepted.is_empty() && !self.closed { - self.segmenter.close(); - self.closed = true; - } - self.records.extend(self.segmenter.by_ref()); - if self.committer.is_none() { - return Ok(None); - } - let Some(record) = self.records.pop_front() else { - return Ok(None); - }; - - let mut groups: Vec = Vec::new(); - while let Some((position, _)) = self.frames.front() - && *position < record.end - { - let (position, frame) = self.frames.pop_front().expect("checked above"); - match groups.last_mut() { - Some(group) if group.sequence == position.group => group.frames.push(frame), - _ => groups.push(Group { - sequence: position.group, - frames: vec![frame], - }), - } - } - let object = Object { - frame_start: record.start.frame, - groups, - }; - object.check_span(record.start, record.end)?; - - let committer = self.committer.take().expect("checked above"); - Ok(Some(committer.commit(record, object).boxed())) - } - - /// Finish the timeline once the track ended and every record is durable, returning whether the - /// track is done. - fn finish(&mut self) -> bool { - if !self.closed || !self.records.is_empty() { - return false; - } - let Some(committer) = self.committer.as_mut() else { - return false; - }; - if let Err(err) = committer.output.finish() { - tracing::warn!(track = %self.name, %err, "failed to finish the timeline"); - } - true - } -} - -// Each read is already boxed as a future, so the variant sizes cost nothing extra. -#[allow(clippy::large_enum_variant)] -enum Read { - Group { - name: String, - subscriber: Box, - result: moq_net::Result>, - }, - Frame { - name: String, - group: Box, - result: moq_net::Result>, - }, - Cancelled, -} - -async fn recv(name: String, mut subscriber: Box) -> Read { - let result = subscriber.recv_group().await; - Read::Group { - name, - subscriber, - result, - } -} - -async fn frame(name: String, mut group: Box) -> Read { - let result = group.read_frame().await; - Read::Frame { name, group, result } -} - -/// Stop reading every track, returning their names so each flushes and finishes. -fn stop(tracks: &mut HashMap>>) -> Vec { - tracks - .iter_mut() - .map(|(name, track)| { - track.remove(); - name.clone() - }) - .collect() -} - -/// Resolve to [`Read::Cancelled`] once the track stops reading. -async fn guard(mut cancelled: watch::Receiver<()>, read: impl Future) -> Read { - tokio::select! { - read = read => read, - _ = cancelled.changed() => Read::Cancelled, - } -} - -/// Returns the track that may have records to commit or be finished. -/// -/// Fails on malformed source input; a group the network aborted keeps the frames that arrived. -fn handle( - read: Read, - tracks: &mut HashMap>>, - reads: &mut FuturesUnordered>, -) -> Result> { - match read { - Read::Cancelled => Ok(None), - Read::Group { - name, - subscriber, - result, - } => { - let Some(track) = tracks.get_mut(&name) else { - return Ok(None); - }; - match result { - Ok(Some(group)) => { - let sequence = group.sequence; - let stored = track.floor.is_some_and(|floor| Position::group(sequence + 1) <= floor); - if track.largest.is_some_and(|largest| sequence <= largest) || stored { - tracing::debug!(track = %name, sequence, "refusing a duplicate, decreasing, or stored group"); - } else { - crate::path::check_id(sequence).map_err(|err| malformed(&name, sequence, err))?; - track.largest = Some(sequence); - track.accepted.insert(sequence, Incoming::default()); - reads.push(guard(track.cancelled.clone(), frame(name.clone(), Box::new(group))).boxed()); - } - reads.push(guard(track.cancelled.clone(), recv(name, subscriber)).boxed()); - // An accepted group holds no frames yet. - Ok(None) - } - Ok(None) => { - track.subscribed = false; - Ok(Some(name)) - } - Err(err) => { - tracing::warn!(track = %name, %err, "track ended"); - track.subscribed = false; - Ok(Some(name)) - } - } - } - Read::Frame { name, group, result } => { - let Some(track) = tracks.get_mut(&name) else { - return Ok(None); - }; - let sequence = group.sequence; - let Some(incoming) = track.accepted.get_mut(&sequence) else { - return Ok(None); - }; - let touched = name.clone(); - match result { - Ok(Some(frame)) => { - let index = incoming.next; - incoming.next += 1; - // A track is all timed or all untimed. - let timestamp = match (track.timescale, frame.timestamp) { - (Some(timescale), Some(timestamp)) => { - let timestamp = timestamp - .convert(timescale) - .map_err(|_| malformed(&name, sequence, Error::Overflow))? - .value(); - Some(crate::path::check_id(timestamp).map_err(|err| malformed(&name, sequence, err))?) - } - (None, None) => None, - _ => return Err(malformed(&name, sequence, Error::Timing)), - }; - // A resumed track already stored the head of the group it stopped inside. - if track.floor.is_none_or(|floor| Position::new(sequence, index) >= floor) { - let frame = Frame { - timestamp, - payload: frame.payload, - }; - incoming.pending.push((index, frame)); - } - reads.push(guard(track.cancelled.clone(), self::frame(name, group)).boxed()); - } - Ok(None) => incoming.finished = true, - Err(err) => { - tracing::warn!(track = %name, sequence, %err, "keeping the frames of an incomplete group"); - incoming.finished = true; - } - } - track.report()?; - Ok(Some(touched)) - } - } -} - -/// An in-flight record commit, returning the track, its committer, and the expired objects. -type Commit = BoxFuture<'static, (String, Committer, Result)>; - -/// Objects the timelines stopped needing, deleted once the grace elapses. -#[derive(Default)] -struct Expired { - /// Media objects no retained record references. - objects: Vec, - /// Timeline segments no recovery reads, by timeline. - timelines: Vec<(String, Range)>, -} - -impl Expired { - fn is_empty(&self) -> bool { - self.objects.is_empty() && self.timelines.is_empty() - } -} - -/// Commits one track's records in order: the object, the record, retention, then the timeline -/// object. -struct Committer { - shared: Arc>, - name: String, - /// The track's timeline track. - timeline: String, - output: timeline::Producer, - /// The timeline track read back, to store its complete groups. - groups: track::Ordered, - /// Committed records still in the timeline window, oldest first. - window: VecDeque, - /// A DVR's stored timeline segments that recovery still reads, oldest first, each with the - /// first record index its opening checkpoint restates. - checkpoints: VecDeque<(u64, u64)>, - /// Added to the timeline track's group sequences, continuing a resumed recording's numbering. - offset: u64, -} - -impl Committer { - /// Commit `record` with its `object`. - async fn commit(mut self, record: Record, object: Object) -> (String, Self, Result) { - let result = self.commit_inner(record, object).await; - (self.name.clone(), self, result) - } - - async fn commit_inner(&mut self, mut record: Record, object: Object) -> Result { - let shared = self.shared.clone(); - let store = &shared.store; - - // Number by the window rather than the segmenter, so a record dropped below leaves no hole. - record.sequence = self.output.range().end; - if let Err(err) = store.put_segments(&self.name, record.sequence, &object).await { - // The timeline only advertises durable objects, so this span is lost. The key may have - // landed anyway; clear it so the next record can take it. - tracing::warn!(track = %self.name, sequence = record.sequence, %err, "dropping a record that was not stored"); - let _ = store.delete(&Key::segments(self.name.clone(), record.sequence)?).await; - return Ok(Expired::default()); - } - - self.output.push(&record).map_err(timeline_error)?; - let pts = record - .pts - .map(|pts| Timestamp::new(pts, TIMESCALE).map_err(|_| Error::Id(pts))) - .transpose()?; - self.window.push_back(record.clone()); - - let expired = self.expire(); - if !expired.is_empty() { - self.output.pop(expired.len() as u64).map_err(timeline_error)?; - } - self.output.flush().map_err(timeline_error)?; - - let object = self.read_timeline(pts)?; - store.put_segments(&self.timeline, record.sequence, &object).await?; - - let objects = expired - .iter() - .map(|record| Key::segments(self.name.clone(), record.sequence)) - .collect::>()?; - let segments = self.prune(record.sequence, &object)?; - let timelines = match segments.is_empty() { - true => Vec::new(), - false => vec![(self.timeline.clone(), segments)], - }; - Ok(Expired { objects, timelines }) - } - - /// Track the newly stored timeline `object`, returning the older segments no recovery reads. - /// - /// Recovery walks back from the newest object to the first whose checkpoint restates the - /// newest one's offset, so every object before that one is no longer needed. - fn prune(&mut self, segment: u64, object: &Object) -> Result> { - if self.shared.retention.is_none() { - return Ok(segment..segment); - } - let (offset, start) = checkpoint(object)?; - self.checkpoints.push_back((segment, start)); - // Checkpoint starts never decrease, so a binary search finds the last one at or before - // `offset` without walking a long DVR window on every commit. - let keep = self - .checkpoints - .partition_point(|&(_, start)| start <= offset) - .saturating_sub(1); - let first = self.checkpoints[0].0; - self.checkpoints.drain(..keep); - Ok(first..self.checkpoints[0].0) - } - - /// Pop the oldest records while the rest still cover the retention window. The newest record - /// always stays, such as a catalog that never changes. - fn expire(&mut self) -> Vec { - let Some(retention) = self.shared.retention.as_ref().map(|retention| retention.window) else { - return Vec::new(); - }; - let target = (retention.as_micros() * TIMESCALE.as_u64() as u128).div_ceil(1_000_000); - let mut expired = Vec::new(); - while self.window.len() > 1 { - // A DVR refuses untimed tracks, so every record here is timed. - let newest = self.window.back().expect("non-empty"); - let end = newest - .pts - .zip(newest.duration) - .map(|(pts, duration)| pts as u128 + duration as u128); - let covered = end - .zip(self.window[1].pts) - .map(|(end, pts)| end.saturating_sub(pts as u128)); - if covered.is_none_or(|covered| covered < target) { - break; - } - expired.push(self.window.pop_front().expect("non-empty")); - } - expired - } - - /// Collect the timeline groups completed since the last record, stamped at `pts`, or unstamped - /// for an untimed track. - /// - /// The live timeline track stamps frames with the wall clock; storing the record's content time - /// instead keeps a recording's bytes a function of its content alone. - fn read_timeline(&mut self, pts: Option) -> Result { - let waiter = kio::Waiter::noop(); - let timestamp = match pts { - Some(pts) => { - let timescale = self - .groups - .info() - .timescale - .ok_or_else(|| Error::Timeline("untimed timeline track".to_string()))?; - let timestamp = pts.convert(timescale).map_err(|_| Error::Overflow)?.value(); - Some(crate::path::check_id(timestamp)?) - } - None => None, - }; - let mut groups = Vec::new(); - while let Poll::Ready(result) = self.groups.poll_next_group(&waiter) { - let Some(mut group) = result.map_err(timeline_error_net)? else { - break; - }; - let mut frames = Vec::new(); - loop { - match group.poll_read_frame(&waiter) { - Poll::Ready(Ok(Some(frame))) => frames.push(Frame { - timestamp, - payload: frame.payload, - }), - Poll::Ready(Ok(None)) => break, - Poll::Ready(Err(err)) => return Err(timeline_error_net(err)), - // Flushing closed every group, so an open one is a bug. - Poll::Pending => return Err(Error::Timeline("timeline group is still open".into())), - } - } - let sequence = group.sequence.checked_add(self.offset).ok_or(Error::Overflow)?; - groups.push(Group { - sequence: crate::path::check_id(sequence)?, - frames, - }); - } - Ok(Object::new(groups)) - } -} - -/// The timescale of record `pts` and `duration`, as the catalog section advertises it. -const TIMESCALE: Timescale = Timescale::MILLI; - -/// Delete `expired`'s media objects, and each timeline's objects from its `pruned` cursor up to -/// its expired segments' end. -/// -/// Timeline objects go oldest first and stop at the first failure, so the stored segments stay -/// consecutive, as recovery requires, and a later deletion retries from there. -async fn delete(store: &Store, pruned: &mut HashMap, expired: Expired) { - let objects = async { - let deletes = expired - .objects - .iter() - .map(|key| async move { (key, store.delete(key).await) }); - for (key, result) in futures::future::join_all(deletes).await { - // An object that outlives its expiry advertises nothing; recovery cleans it up. - if let Err(err) = result { - tracing::warn!(?key, %err, "failed to delete an expired object"); - } - } - }; - let timelines = async { - for (timeline, segments) in expired.timelines { - let cursor = pruned.entry(timeline.clone()).or_insert(segments.start); - while *cursor < segments.end { - let result = match Key::segments(timeline.clone(), *cursor) { - Ok(key) => store.delete(&key).await, - Err(err) => Err(err), - }; - match result { - Ok(()) | Err(Error::NotFound(_)) => *cursor += 1, - Err(err) => { - tracing::warn!(%timeline, segment = *cursor, %err, "failed to delete a pruned timeline object"); - break; - } - } - } - } - }; - futures::join!(objects, timelines); -} - -fn timeline_error(err: moq_mux::Error) -> Error { - Error::Timeline(err.to_string()) -} - -fn timeline_error_net(err: moq_net::Error) -> Error { - Error::Timeline(err.to_string()) -} - -fn source_error(err: moq_net::Error) -> Error { - Error::Source(err.to_string()) -} - -fn malformed(track: &str, sequence: u64, err: impl std::fmt::Display) -> Error { - Error::Source(format!("track {track} group {sequence}: {err}")) -} - -/// The content time the newest committed record reached, in `timescale`, or `None` when nothing -/// was recorded or the track is untimed. -/// -/// Fails when any retained record's timing disagrees with its track's, so a resume never extends a -/// mixed timeline. -fn recorded_end(resume: &Resume, timescale: Option) -> Result> { - let records = &resume.checkpoint.records; - if let Some(record) = records - .iter() - .find(|record| record.pts.is_some() != timescale.is_some() || record.duration.is_some() != timescale.is_some()) - { - return Err(Error::Timeline(format!( - "record {} timing does not match its track", - record.sequence - ))); - } - let (Some(record), Some(timescale)) = (records.last(), timescale) else { - return Ok(None); - }; - let (Some(pts), Some(duration)) = (record.pts, record.duration) else { - unreachable!("checked above"); - }; - let millis = pts.checked_add(duration).ok_or(Error::Overflow)?; - let end = Timestamp::new(millis, TIMESCALE).map_err(|_| Error::Id(millis))?; - Ok(Some(end.convert(timescale).map_err(|_| Error::Overflow)?.value())) -} - -#[cfg(test)] -#[allow(clippy::disallowed_methods, clippy::disallowed_types)] -mod tests { - use futures::TryStreamExt; - use object_store::memory::InMemory; - - use super::*; - use crate::mock::Mock; - use crate::store::list::Query; - - fn ms(v: u64) -> Timestamp { - Timestamp::from_millis(v).unwrap() - } - - /// Media cut at 1s, so one-second test groups get a record each. - fn media() -> timeline::Config { - timeline::Config::default().with_duration_min(Duration::from_secs(1)) - } - - /// Sparse data: every group is its own record. - fn sparse() -> timeline::Config { - timeline::Config::default().with_duration_min(Duration::ZERO) - } - - /// An append-only log with a three-second nominal duration and nine-second safety ceiling. - fn append_only() -> timeline::Config { - sparse().with_duration_hint(Duration::from_secs(3)) - } - - fn timeline(track: &str) -> String { - hang::timeline::default_name(track) - } - - /// A millisecond track that keeps an hour of history, so enrollment sees every group. - fn track(broadcast: &broadcast::Producer, name: &str) -> track::Producer { - let info = track::Info::default() - .with_timescale(Timescale::MILLI) - .with_max_age(Duration::from_secs(3600)); - broadcast.create_track(name, info).unwrap() - } - - /// Write one complete group with a frame at each timestamp. - fn group(track: &track::Producer, sequence: u64, timestamps: &[u64]) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for ×tamp in timestamps { - group - .write_frame(ms(timestamp), format!("{sequence}@{timestamp}")) - .unwrap(); - } - group.finish().unwrap(); - } - - /// The stored timeline segments of `track`, ascending. - async fn segments(store: &Store, track: &str) -> Vec { - let entries: Vec<_> = store - .list(&Query::segments(&timeline(track)).unwrap()) - .try_collect() - .await - .unwrap(); - let mut segments: Vec = entries - .into_iter() - .map(|entry| match entry.key { - Key::Segments { segment, .. } => segment, - key => panic!("unexpected {key:?}"), - }) - .collect(); - segments.sort(); - segments - } - - /// Replay every stored timeline object of `track`, returning the records still in its window. - async fn window(store: &Store, track: &str) -> Vec { - let timeline = timeline(track); - let segments = segments(store, track).await; - if let (Some(first), Some(last)) = (segments.first(), segments.last()) { - assert_eq!( - segments, - (*first..=*last).collect::>(), - "timeline objects are consecutive" - ); - } - - let config = moq_json::window::ConsumerConfig::default().with_compression(true); - let mut decoder = moq_json::window::Decoder::::new(config); - let mut records = BTreeMap::new(); - for segment in segments { - let object = store - .get_segments(&timeline, segment, timed(store, &timeline).await) - .await - .unwrap(); - for stored in object.groups { - let mut group = decoder.group(); - for frame in stored.frames { - group.decode(&frame.payload).unwrap(); - } - } - while let Some(event) = decoder.next_event() { - match event { - moq_json::window::Event::Push { index, value } => { - assert_eq!(index, value.sequence); - records.insert(index, value); - } - moq_json::window::Event::Pop(range) | moq_json::window::Event::Skip(range) => { - for index in range { - records.remove(&index); - } - } - _ => unreachable!(), - } - } - } - records.into_values().collect() - } - - /// Each record's groups, as `(first, last)`. - fn groups(records: &[Record]) -> Vec<(u64, u64)> { - records - .iter() - .map(|record| (*record.groups().start(), *record.groups().end())) - .collect() - } - - fn sequences(records: &[Record]) -> Vec { - records.iter().map(|record| record.sequence).collect() - } - - /// Whether `track`'s stored `.info` declares a timescale. - async fn timed(store: &Store, track: &str) -> bool { - store.get_info(track).await.unwrap().timescale.is_some() - } - - /// Every record resolves to one stored object holding exactly its frames. - async fn check_objects(store: &Store, track: &str, records: &[Record]) { - let timed = timed(store, track).await; - for record in records { - let object = store.get_segments(track, record.sequence, timed).await.unwrap(); - object.check_span(record.start, record.end).unwrap(); - } - } - - /// Every stored media object, across all tracks. - async fn stored(store: &Store) -> HashSet { - store - .list(&Query::new()) - .try_filter_map(|entry| async move { - let media = - matches!(&entry.key, Key::Segments { track, .. } if !track.ends_with(hang::timeline::SUFFIX)); - Ok(media.then_some(entry.key)) - }) - .try_collect() - .await - .unwrap() - } - - /// The media objects the retained records reference. - fn referenced(track: &str, records: &[Record]) -> HashSet { - records - .iter() - .map(|record| Key::segments(track, record.sequence).unwrap()) - .collect() - } - - #[tokio::test] - async fn records_selected_tracks() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let catalog = track(&source, "catalog.json"); - let _ignored = track(&source, "ignored"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - control.track("catalog.json", sparse()).await.unwrap(); - - group(&catalog, 0, &[0]); - for sequence in 0..6 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - catalog.finish().unwrap(); - source.close(); - - tokio::spawn(writer.run()).await.unwrap().unwrap(); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); - check_objects(&store, "video", &records).await; - let catalog = window(&store, "catalog.json").await; - assert_eq!(groups(&catalog), vec![(0, 0)]); - check_objects(&store, "catalog.json", &catalog).await; - assert!(window(&store, "ignored").await.is_empty()); - - let object = store.get_segments("video", 2, true).await.unwrap(); - assert_eq!(object.groups[0].frames[1].timestamp, Some(2500)); - assert_eq!(object.groups[0].frames[1].payload, "2@2500"); - assert_eq!( - store.get_info("video").await.unwrap(), - Info::new(127, Some(1000)).unwrap() - ); - store.get_info(&timeline("video")).await.unwrap(); - assert!(store.get_info("ignored").await.is_err()); - } - - /// An untimed track that keeps an hour of history, so enrollment sees every group. - fn untimed(broadcast: &broadcast::Producer, name: &str) -> track::Producer { - let info = track::Info::default() - .with_timescale(None) - .with_max_age(Duration::from_secs(3600)); - broadcast.create_track(name, info).unwrap() - } - - /// Write one complete untimed group of `frames` frames. - fn untimed_group(track: &track::Producer, sequence: u64, frames: u64) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for index in 0..frames { - group.write_frame(None, format!("{sequence}/{index}")).unwrap(); - } - group.finish().unwrap(); - } - - /// An untimed track records each group as its own record, with no timing anywhere. - #[tokio::test] - async fn an_untimed_track_is_recorded_untimed() { - let source = broadcast::Info::new().produce(); - let data = untimed(&source, "data"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - // The minimum measures time, so it cannot pack untimed groups. - writer.control().track("data", media()).await.unwrap(); - - untimed_group(&data, 0, 2); - untimed_group(&data, 1, 1); - untimed_group(&data, 3, 3); - data.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let records = window(&store, "data").await; - assert_eq!( - records, - vec![ - Record::untimed(0, Position::group(0), Position::group(1)), - Record::untimed(1, Position::group(1), Position::group(2)), - Record::untimed(2, Position::group(3), Position::group(4)), - ] - ); - check_objects(&store, "data", &records).await; - let object = store.get_segments("data", 2, false).await.unwrap(); - assert!(object.groups[0].frames.iter().all(|frame| frame.timestamp.is_none())); - assert_eq!(object.groups[0].frames[2].payload, "3/2"); - - assert_eq!(store.get_info("data").await.unwrap(), Info::new(127, None).unwrap()); - let timeline = timeline("data"); - assert_eq!(store.get_info(&timeline).await.unwrap().timescale, None); - let stored = store.get_segments(&timeline, 0, false).await.unwrap(); - assert!(stored.groups[0].frames.iter().all(|frame| frame.timestamp.is_none())); - } - - /// A restarted writer resumes an untimed track after its newest record. - #[tokio::test] - async fn an_untimed_track_resumes() { - let store = Store::new(InMemory::new(), "rec"); - for groups in [0..2, 0..4] { - let source = broadcast::Info::new().produce(); - let data = untimed(&source, "data"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("data", sparse()).await.unwrap(); - // The source's cache replays groups the recording already holds. - for sequence in groups { - untimed_group(&data, sequence, 1); - } - data.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - } - - let records = window(&store, "data").await; - assert_eq!(sequences(&records), (0..4).collect::>()); - assert_eq!(groups(&records), (0..4).map(|s| (s, s)).collect::>()); - check_objects(&store, "data", &records).await; - } - - /// A resume refuses a retained record whose timing disagrees with its track's, at any age. - #[test] - fn a_resume_refuses_mixed_timing() { - let resume = |records: Vec| Resume { - checkpoint: moq_json::window::Checkpoint { - range: 0..records.len() as u64, - records, - }, - sequence: 0, - checkpoints: VecDeque::new(), - }; - let timed = Record::new(1, 1_000, 1_000, Position::group(1), Position::group(2)); - let untimed = Record::untimed(0, Position::group(0), Position::group(1)); - - let mixed = resume(vec![untimed.clone(), timed.clone()]); - assert!(matches!( - recorded_end(&mixed, Some(Timescale::MILLI)), - Err(Error::Timeline(_)) - )); - assert!(matches!(recorded_end(&mixed, None), Err(Error::Timeline(_)))); - - assert_eq!(recorded_end(&resume(vec![untimed]), None).unwrap(), None); - assert_eq!( - recorded_end(&resume(vec![timed]), Some(Timescale::MILLI)).unwrap(), - Some(2_000) - ); - } - - /// A DVR measures retention in content time, which an untimed track has none of. - #[tokio::test] - async fn a_dvr_refuses_an_untimed_track() { - let source = broadcast::Info::new().produce(); - let _data = untimed(&source, "data"); - - let store = Store::new(InMemory::new(), "rec"); - let retention = Retention::new(Duration::from_secs(30), Duration::ZERO); - let config = Config::default().with_retention(retention); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - assert_eq!( - writer.control().track("data", sparse()).await, - Err(Error::Untimed("data".into())) - ); - assert!(store.get_info("data").await.is_err(), "nothing is recorded"); - } - - #[tokio::test] - async fn tracks_cut_independently() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let audio = track(&source, "audio"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", timeline::Config::default()).await.unwrap(); - control.track("audio", timeline::Config::default()).await.unwrap(); - - // Two second GOPs, 400ms audio groups: with the default 4s minimum, audio packs by its own - // boundaries, not video's GOPs. - for sequence in 0..3 { - group(&video, sequence, &[sequence * 2000, sequence * 2000 + 1000]); - } - for sequence in 0..15 { - group(&audio, sequence, &[sequence * 400]); - } - video.finish().unwrap(); - audio.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let video = window(&store, "video").await; - assert_eq!(groups(&video), vec![(0, 1), (2, 2)]); - let audio = window(&store, "audio").await; - assert_eq!(groups(&audio), vec![(0, 9), (10, 14)]); - check_objects(&store, "audio", &audio).await; - } - - #[tokio::test] - async fn an_append_only_group_spans_several_objects() { - let source = broadcast::Info::new().produce(); - let log = track(&source, "log"); - - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default(); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("log", append_only()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - // One group that never closes, a frame a second. - let mut open = log.create_group(group::Info { sequence: 0 }).unwrap(); - for second in 0..19 { - open.write_frame(ms(second * 1000), format!("line {second}")).unwrap(); - } - // The group is still open, yet its first eighteen seconds are already durable. - while window(&store, "log").await.len() < 2 { - tokio::time::sleep(Duration::from_millis(10)).await; - } - let records = window(&store, "log").await; - assert_eq!( - records - .iter() - .map(|record| (record.start, record.end)) - .collect::>(), - vec![ - (Position::new(0, 0), Position::new(0, 9)), - (Position::new(0, 9), Position::new(0, 18)), - ] - ); - check_objects(&store, "log", &records).await; - assert_eq!(store.get_segments("log", 1, true).await.unwrap().frame_start, 9); - - open.finish().unwrap(); - log.finish().unwrap(); - source.close(); - run.await.unwrap().unwrap(); - - let records = window(&store, "log").await; - assert_eq!(records.last().unwrap().start, Position::new(0, 18)); - assert_eq!(records.last().unwrap().end, Position::group(1)); - check_objects(&store, "log", &records).await; - } - - #[tokio::test] - async fn a_failed_put_omits_only_that_track() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let audio = track(&source, "audio"); - - let mock = Mock::memory(); - mock.fail_puts("/audio/segments/"); - let store = Store::new(mock, "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - control.track("audio", media()).await.unwrap(); - - for sequence in 0..3 { - group(&video, sequence, &[sequence * 1000]); - group(&audio, sequence, &[sequence * 1000]); - } - video.finish().unwrap(); - audio.finish().unwrap(); - source.close(); - - writer.run().await.unwrap(); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2)]); - check_objects(&store, "video", &records).await; - assert!(window(&store, "audio").await.is_empty()); - } - - #[tokio::test] - async fn retention_pops_then_deletes_expired_objects() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let catalog = track(&source, "catalog.json"); - - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - writer.control().track("catalog.json", sparse()).await.unwrap(); - - // A static catalog published once, then six seconds of video. - group(&catalog, 0, &[0]); - for sequence in 0..6 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - catalog.finish().unwrap(); - source.close(); - - writer.run().await.unwrap(); - - // Records 3 and 4 hold two seconds; the partial final record 5 is always kept. - let records = window(&store, "video").await; - assert_eq!(sequences(&records), vec![3, 4, 5]); - check_objects(&store, "video", &records).await; - - // The catalog's newest record outlives the video it was published with. - let catalog = window(&store, "catalog.json").await; - assert_eq!(sequences(&catalog), vec![0]); - check_objects(&store, "catalog.json", &catalog).await; - - let expected = &referenced("video", &records) | &referenced("catalog.json", &catalog); - assert_eq!(stored(&store).await, expected); - // Every checkpoint restates the whole window, so only the newest timeline object is needed. - assert_eq!(segments(&store, "video").await, vec![5]); - assert_eq!(segments(&store, "catalog.json").await, vec![0]); - } - - #[tokio::test] - async fn commands_are_refused_once_the_recording_ends() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - // A long grace keeps `run` waiting on deletions after the timeline finishes. - let config = - Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - - for sequence in 0..6 { - group(&video, sequence, &[sequence * 1000]); - } - video.finish().unwrap(); - source.close(); - - let run = tokio::spawn(writer.run()); - while window(&store, "video").await.last().map(|record| record.sequence) != Some(5) { - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert!(!run.is_finished()); - assert_eq!(control.remove("video"), Err(Error::Closed)); - run.abort(); - } - - #[tokio::test] - async fn accepted_groups_may_complete_out_of_order() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("video", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - let mut first = video.create_group(group::Info { sequence: 0 }).unwrap(); - first.write_frame(ms(0), "0@0").unwrap(); - group(&video, 1, &[1000]); - group(&video, 2, &[2000]); - tokio::time::sleep(Duration::from_millis(50)).await; - assert!(window(&store, "video").await.is_empty(), "group 1 waits for group 0"); - - first.write_frame(ms(500), "0@500").unwrap(); - first.finish().unwrap(); - video.finish().unwrap(); - source.close(); - run.await.unwrap().unwrap(); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2)]); - check_objects(&store, "video", &records).await; - let object = store.get_segments("video", 0, true).await.unwrap(); - let frames: Vec<_> = object.groups[0].frames.iter().map(|f| f.timestamp).collect(); - assert_eq!(frames, vec![Some(0), Some(500)]); - } - - #[tokio::test] - async fn decreasing_arrivals_are_refused() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("video", media()).await.unwrap(); - - group(&video, 0, &[0]); - group(&video, 2, &[1000]); - group(&video, 1, &[500]); - group(&video, 3, &[2000]); - video.finish().unwrap(); - source.close(); - - writer.run().await.unwrap(); - - // The skipped sequence closes a record, so no record holds a gap. - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(0, 0), (2, 2), (3, 3)]); - check_objects(&store, "video", &records).await; - } - - #[tokio::test] - async fn an_unrepresentable_group_fails_the_recording() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - - group(&video, 0, &[0]); - // One past the recording's largest group ID. - group(&video, 1 << 53, &[1000]); - video.finish().unwrap(); - source.close(); - - assert_eq!( - writer.run().await, - Err(Error::Source(format!( - "track video group {}: {}", - 1u64 << 53, - Error::Id(1 << 53) - ))) - ); - } - - /// Stopping a recording whose source is still live finishes every timeline and ends `run`. - #[tokio::test] - async fn stopping_ends_without_the_source() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - group(&video, 0, &[0]); - control.stop().unwrap(); - - writer.run().await.unwrap(); - let records = window(&store, "video").await; - check_objects(&store, "video", &records).await; - assert!( - control.track("audio", media()).await.is_err(), - "a stopped recording refuses enrollment" - ); - drop(source); - } - - #[tokio::test] - async fn a_stalled_track_holds_only_itself() { - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let audio = track(&source, "audio"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - control.track("audio", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - group(&audio, 0, &[0]); - // The audio group never completes, so its record never closes on its own. - let mut stalled = audio.create_group(group::Info { sequence: 1 }).unwrap(); - stalled.write_frame(ms(1000), "stalled").unwrap(); - group(&audio, 2, &[2000]); - for sequence in 0..4 { - group(&video, sequence, &[sequence * 1000]); - } - video.finish().unwrap(); - source.close(); - - // Video commits without waiting for the stalled audio. - while window(&store, "video").await.len() < 4 { - tokio::time::sleep(Duration::from_millis(10)).await; - } - assert!(!run.is_finished(), "a stalled track holds the recording open"); - - control.remove("audio").unwrap(); - run.await.unwrap().unwrap(); - - // What arrived of the stalled group is stored; the group queued behind it is dropped. - let records = window(&store, "audio").await; - assert_eq!( - records - .iter() - .map(|record| (record.start, record.end)) - .collect::>(), - vec![ - (Position::group(0), Position::group(1)), - (Position::group(1), Position::new(1, 1)) - ] - ); - check_objects(&store, "audio", &records).await; - drop(stalled); - } - - #[tokio::test] - async fn enrollment_is_refused_for_duplicates_and_timelines() { - let source = broadcast::Info::new().produce(); - let _video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - let control = writer.control(); - control.track("video", media()).await.unwrap(); - assert_eq!( - control.track("video", sparse()).await, - Err(Error::Enrolled("video".into())) - ); - let timeline = timeline("video"); - assert_eq!(control.track(&timeline, media()).await, Err(Error::Enrolled(timeline))); - } - - #[tokio::test] - async fn a_conflicting_info_fails_enrollment() { - let source = broadcast::Info::new().produce(); - let _video = track(&source, "video"); - - let store = Store::new(InMemory::new(), "rec"); - store - .put_info("video", &Info::new(7, Some(1000)).unwrap()) - .await - .unwrap(); - let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - let control = writer.control(); - assert_eq!( - control.track("video", media()).await, - Err(Error::Priority { - existing: 7, - intended: 127 - }) - ); - } - - /// Record `video` groups `sequences`, one per second, until the source ends. - async fn record(store: &Store, config: Config, sequences: std::ops::Range) { - let source = broadcast::Info::new().produce(); - record_into(store, config, source, sequences).await - } - - /// [`record`] from a source announced under `epoch`. - async fn record_from( - store: &Store, - epoch: Option, - sequences: std::ops::Range, - ) { - record_into(store, Config::default(), source(epoch), sequences).await - } - - async fn record_into( - store: &Store, - config: Config, - source: broadcast::Producer, - sequences: std::ops::Range, - ) { - let video = track(&source, "video"); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - for sequence in sequences { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - } - - /// A source broadcast resolved through a route with `epoch`. - fn source(epoch: Option) -> broadcast::Producer { - let mut info = broadcast::Info::new(); - info.epoch = epoch; - info.produce() - } - - fn epoch(n: u8) -> moq_net::Epoch { - format!("01900000-0000-7000-8000-0000000000{n:02x}").parse().unwrap() - } - - fn orphan(sequence: u64) -> Object { - Object::new(vec![Group { - sequence, - frames: vec![Frame { - timestamp: Some(sequence * 1000), - payload: "orphan".into(), - }], - }]) - } - - #[tokio::test] - async fn a_restarted_writer_resumes_the_recording() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..3).await; - // The source's cache replays groups the recording already holds. - record(&store, Config::default(), 0..6).await; - - let records = window(&store, "video").await; - assert_eq!(sequences(&records), (0..6).collect::>()); - assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); - check_objects(&store, "video", &records).await; - - // The resumed timeline groups continue the stored numbering. - let mut numbers = Vec::new(); - for segment in 0..6 { - let object = store.get_segments(&timeline("video"), segment, true).await.unwrap(); - numbers.extend(object.groups.iter().map(|group| group.sequence)); - } - assert!(numbers.windows(2).all(|pair| pair[0] < pair[1]), "{numbers:?}"); - } - - /// The group sequence continued, but content time jumped back before the recorded end. - #[tokio::test] - async fn a_backward_timestamp_is_refused() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..3).await; - - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("video", media()).await.unwrap(); - // Group 3 is the next one. Its timestamp is behind the last frame, at 2500. - group(&video, 3, &[500]); - video.finish().unwrap(); - source.close(); - - assert_eq!( - writer.run().await, - Err(Error::Source("track video group 3: timestamp 500 precedes 2500".into())) - ); - let records = window(&store, "video").await; - assert_eq!( - groups(&records), - (0..3).map(|sequence| (sequence, sequence)).collect::>() - ); - check_objects(&store, "video", &records).await; - } - - /// A source announced under another epoch restarted, whatever its groups say: enrolling it - /// fails and nothing more is recorded. - #[tokio::test] - async fn another_epoch_is_refused() { - let store = Store::new(InMemory::new(), "rec"); - record_from(&store, Some(epoch(1)), 0..3).await; - - for other in [Some(epoch(2)), None] { - let source = source(other.clone()); - let _video = track(&source, "video"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - assert_eq!( - writer.control().track("video", media()).await, - Err(Error::EpochMismatch { - existing: Some(epoch(1)), - intended: other, - }) - ); - } - - // And a recording without an epoch refuses a source with one. - let plain = Store::new(InMemory::new(), "plain"); - record(&plain, Config::default(), 0..3).await; - let source = source(Some(epoch(1))); - let _video = track(&source, "video"); - let writer = Writer::new(plain.clone(), source.consume(), Config::default()) - .await - .unwrap(); - assert_eq!( - writer.control().track("video", media()).await, - Err(Error::EpochMismatch { - existing: None, - intended: Some(epoch(1)), - }) - ); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), (0..3).map(|s| (s, s)).collect::>()); - } - - /// The same epoch resumes, skipping the groups its cache replays. - #[tokio::test] - async fn the_same_epoch_resumes() { - let store = Store::new(InMemory::new(), "rec"); - record_from(&store, Some(epoch(1)), 0..3).await; - record_from(&store, Some(epoch(1)), 0..6).await; - - let records = window(&store, "video").await; - assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); - check_objects(&store, "video", &records).await; - assert_eq!(store.get_info("video").await.unwrap().epoch, Some(epoch(1))); - } - - /// A source that jumped forward, in both group sequence and content time, keeps recording. - #[tokio::test] - async fn a_forward_restart_keeps_recording() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..3).await; - - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - writer.control().track("video", media()).await.unwrap(); - group(&video, 10, &[10_000, 10_500]); - video.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(0, 0), (1, 1), (2, 2), (10, 10)]); - check_objects(&store, "video", &records).await; - } - - #[tokio::test] - async fn a_resumed_append_only_group_continues_mid_group() { - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default(); - - // The first run stores frames 0..18 of a group that never closes, then stops. - let source = broadcast::Info::new().produce(); - let log = track(&source, "log"); - let writer = Writer::new(store.clone(), source.consume(), config.clone()) - .await - .unwrap(); - writer.control().track("log", append_only()).await.unwrap(); - let mut open = log.create_group(group::Info { sequence: 0 }).unwrap(); - for second in 0..19 { - open.write_frame(ms(second * 1000), format!("line {second}")).unwrap(); - } - let run = tokio::spawn(writer.run()); - while window(&store, "log").await.len() < 2 { - tokio::time::sleep(Duration::from_millis(10)).await; - } - run.abort(); - let _ = run.await; - drop(open); - - // The restarted source replays the whole group; the stored head is skipped. - let source = broadcast::Info::new().produce(); - let log = track(&source, "log"); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("log", append_only()).await.unwrap(); - let mut replayed = log.create_group(group::Info { sequence: 0 }).unwrap(); - for second in 0..20 { - replayed - .write_frame(ms(second * 1000), format!("line {second}")) - .unwrap(); - } - replayed.finish().unwrap(); - log.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - - let records = window(&store, "log").await; - assert_eq!( - records - .iter() - .map(|record| (record.start, record.end)) - .collect::>(), - vec![ - (Position::new(0, 0), Position::new(0, 9)), - (Position::new(0, 9), Position::new(0, 18)), - (Position::new(0, 18), Position::group(1)), - ] - ); - check_objects(&store, "log", &records).await; - } - - #[tokio::test] - async fn a_restarted_dvr_deletes_unreferenced_objects() { - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - record(&store, config.clone(), 0..6).await; - - // An interrupted expiration, an uncommitted upload, and a track no timeline names. - store.put_segments("video", 1, &orphan(1)).await.unwrap(); - store.put_segments("video", 6, &orphan(7)).await.unwrap(); - store.put_segments("audio", 0, &orphan(0)).await.unwrap(); - - record(&store, config, 6..10).await; - - let records = window(&store, "video").await; - assert_eq!(sequences(&records), vec![7, 8, 9]); - assert_eq!(groups(&records), vec![(7, 7), (8, 8), (9, 9)]); - check_objects(&store, "video", &records).await; - assert_eq!(stored(&store).await, referenced("video", &records)); - store.get_info("video").await.unwrap(); - store.get_info(&timeline("video")).await.unwrap(); - assert_eq!(segments(&store, "video").await, vec![9]); - } - - #[tokio::test] - async fn a_dvr_crash_between_pop_and_delete_is_cleaned_on_restart() { - let store = Store::new(InMemory::new(), "rec"); - - // A grace longer than the test keeps every expired object past the crash. - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let config = - Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - for sequence in 0..6 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - // Record 5 stays open, so the newest durable timeline object is record 4. - while window(&store, "video").await.last().map(|record| record.sequence) != Some(4) { - tokio::time::sleep(Duration::from_millis(10)).await; - } - run.abort(); - let _ = run.await; - - let retained = window(&store, "video").await; - assert_eq!(sequences(&retained), vec![3, 4]); - let expired: HashSet<_> = (0..3).map(|s| Key::segments("video", s).unwrap()).collect(); - assert_eq!(stored(&store).await, &referenced("video", &retained) | &expired); - assert_eq!(segments(&store, "video").await, (0..=4).collect::>()); - // An upload the crash left uncommitted. - store.put_segments("video", 5, &orphan(5)).await.unwrap(); - - let grace = Duration::from_millis(200); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let started = Instant::now(); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - assert!( - stored(&store).await.is_superset(&expired), - "expired objects outlive the grace, for readers holding the old timeline" - ); - assert!( - !stored(&store).await.contains(&Key::segments("video", 5).unwrap()), - "the uncommitted upload is cleared before its key is reused" - ); - assert_eq!(segments(&store, "video").await, (0..=4).collect::>()); - - writer.control().track("video", media()).await.unwrap(); - // The source replays groups the recording already holds; they are refused. - for sequence in 4..9 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - source.close(); - writer.run().await.unwrap(); - assert!(started.elapsed() >= grace); - - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(6, 6), (7, 7), (8, 8)]); - check_objects(&store, "video", &records).await; - assert_eq!(stored(&store).await, referenced("video", &records)); - assert_eq!(segments(&store, "video").await, vec![8]); - } - - #[tokio::test] - async fn an_archive_restart_clears_uncommitted_objects() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..3).await; - // Media stored after the last timeline commit: a crash before its record. - store.put_segments("video", 3, &orphan(3)).await.unwrap(); - - record(&store, Config::default(), 3..6).await; - let records = window(&store, "video").await; - assert_eq!(groups(&records), (0..6).map(|s| (s, s)).collect::>()); - check_objects(&store, "video", &records).await; - assert_eq!(stored(&store).await, referenced("video", &records)); - } - - #[tokio::test] - async fn a_failed_recovery_deletes_nothing() { - let mock = Mock::memory(); - let store = Store::new(mock.clone(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - record(&store, config.clone(), 0..6).await; - store.put_segments("video", 1, &orphan(1)).await.unwrap(); - let before = stored(&store).await; - - let failing = mock.fork(); - failing.fail_lists(); - let source = broadcast::Info::new().produce(); - let result = Writer::new(Store::new(failing, "rec"), source.consume(), config.clone()).await; - assert!(matches!(result, Err(Error::Store(_)))); - - // A gap in the timeline keys leaves the retained window unrecoverable. - let newest = store.get_segments(&timeline("video"), 5, true).await.unwrap(); - store.put_segments(&timeline("video"), 3, &newest).await.unwrap(); - let result = Writer::new(store.clone(), source.consume(), config).await; - assert!(matches!(result, Err(Error::Timeline(_)))); - assert_eq!(segments(&store, "video").await, vec![3, 5]); - - assert_eq!(stored(&store).await, before); - } - - #[tokio::test] - async fn a_dvr_window_longer_than_one_checkpoint_is_recovered() { - let store = Store::new(InMemory::new(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(280), Duration::ZERO)); - record(&store, config, 0..300).await; - - let recovery = recover(&store, true).await.unwrap(); - let resume = &recovery.tracks["video"]; - let records = window(&store, "video").await; - assert!(records.len() > 256, "the window outgrows one checkpoint"); - assert_eq!(resume.checkpoint.records, records); - assert_eq!(resume.checkpoint.range.end, 300); - assert!(recovery.orphans.is_empty()); - assert!(recovery.uncommitted.is_empty()); - assert_eq!(resume.floor(), Some(Position::group(300))); - - // The oldest stored timeline object is exactly the one recovery starts from. - let stored = segments(&store, "video").await; - assert!(stored.len() > 1, "the newest checkpoint omits part of the window"); - assert!(stored[0] > 0, "earlier timeline objects are pruned"); - assert_eq!(resume.checkpoints.front().unwrap().0, stored[0]); - assert!(recovery.stale.is_empty()); - } - - /// Yield to the writer until `check` holds. - async fn until>(mut check: impl FnMut() -> F) { - while !check().await { - tokio::task::yield_now().await; - } - } - - #[tokio::test(start_paused = true)] - async fn a_dvr_prunes_timeline_objects_one_grace_after_they_are_unneeded() { - let store = Store::new(InMemory::new(), "rec"); - let source = broadcast::Info::new().produce(); - let video = track(&source, "video"); - let grace = Duration::from_secs(10); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); - let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); - writer.control().track("video", media()).await.unwrap(); - let run = tokio::spawn(writer.run()); - - for sequence in 0..4 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - // Record 3 stays open, so record 2 is the newest timeline object. - until(|| async { segments(&store, "video").await.last() == Some(&2) }).await; - assert_eq!( - segments(&store, "video").await, - vec![0, 1, 2], - "unneeded timeline objects outlive the grace, for readers that listed them" - ); - - tokio::time::sleep(grace).await; - until(|| async { segments(&store, "video").await == [2] }).await; - - for sequence in 4..6 { - group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); - } - video.finish().unwrap(); - source.close(); - run.await.unwrap().unwrap(); - - assert_eq!(segments(&store, "video").await, vec![5]); - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(3, 3), (4, 4), (5, 5)]); - check_objects(&store, "video", &records).await; - let recovery = recover(&store, true).await.unwrap(); - assert_eq!(recovery.tracks["video"].checkpoint.records, records); - } - - #[tokio::test] - async fn a_failed_timeline_delete_keeps_the_keys_consecutive() { - let mock = Mock::memory(); - let store = Store::new(mock.clone(), "rec"); - let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); - let failed = Key::segments(timeline("video"), 1).unwrap(); - mock.fail_deletes(store.path(&failed).unwrap().as_ref()); - record(&store, config.clone(), 0..6).await; - - // Pruning stops at the failed object instead of deleting past it. - assert_eq!(segments(&store, "video").await, (1..=5).collect::>()); - window(&store, "video").await; - - // A restart retries from the oldest stored object. - mock.heal(); - record(&store, config, 6..8).await; - assert_eq!(segments(&store, "video").await, vec![7]); - let records = window(&store, "video").await; - assert_eq!(groups(&records), vec![(5, 5), (6, 6), (7, 7)]); - check_objects(&store, "video", &records).await; - } - - #[tokio::test] - async fn a_timeline_that_does_not_end_at_the_next_segment_fails_recovery() { - let store = Store::new(InMemory::new(), "rec"); - record(&store, Config::default(), 0..6).await; - let before = stored(&store).await; - - // `segments/5` still decodes, but it restates an earlier window, so resuming would write the - // next record on top of it. - let older = store.get_segments(&timeline("video"), 0, true).await.unwrap(); - store - .delete(&Key::segments(timeline("video"), 5).unwrap()) - .await - .unwrap(); - store.put_segments(&timeline("video"), 5, &older).await.unwrap(); - - let source = broadcast::Info::new().produce(); - match Writer::new(store.clone(), source.consume(), Config::default()).await { - Err(Error::Timeline(message)) => assert!(message.contains("not segment 6"), "{message}"), - Err(err) => panic!("expected a timeline error, got {err}"), - Ok(_) => panic!("expected recovery to fail"), - } - assert_eq!(stored(&store).await, before); - } - - /// An enrollment dropped mid-flight still settles its count, so stopping the recording ends it. - #[tokio::test(start_paused = true)] - async fn a_cancelled_enrollment_does_not_block_the_end() { - let source = broadcast::Info::new().produce(); - let mut audio = track(&source, "audio"); - - let mock = Mock::memory(); - mock.hold_puts("audio/.info"); - let store = Store::new(mock.clone(), "rec"); - let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); - let control = writer.control(); - let run = tokio::spawn(writer.run()); - let enrolling = { - let control = control.clone(); - tokio::spawn(async move { control.track("audio", media()).await }) - }; - assert!( - audio.subscription_changed().await.unwrap().is_some(), - "enrollment subscribed" - ); - enrolling.abort(); - let _ = enrolling.await; - - control.stop().unwrap(); - tokio::time::timeout(Duration::from_secs(5), run) - .await - .expect("the recording ends") - .unwrap() - .unwrap(); - // The cancelled name was never enrolled, so it isn't taken. - mock.release(); - drop(source); - } - - /// The publisher already has the subscriber, and the broadcast ends while `.info` is written. - /// - /// The enroll command is queued only after that write. The recording stays up across the gap, - /// so the track is kept instead of failing with the writer closed. - #[tokio::test] - async fn a_track_that_ends_during_enrollment_is_kept() { - let source = broadcast::Info::new().produce(); - let mut audio = track(&source, "audio"); - group(&audio, 0, &[0, 500]); - - let mock = Mock::memory(); - mock.hold_puts("audio/.info"); - let store = Store::new(mock.clone(), "rec"); - let writer = Writer::new(store.clone(), source.consume(), Config::default()) - .await - .unwrap(); - let control = writer.control(); - let run = tokio::spawn(writer.run()); - let enrolling = tokio::spawn(async move { control.track("audio", media()).await }); - - let subscription = audio.subscription_changed().await.unwrap(); - assert!(subscription.is_some(), "enrollment subscribed"); - audio.finish().unwrap(); - source.close(); - mock.release(); - - enrolling.await.unwrap().unwrap(); - run.await.unwrap().unwrap(); - - let records = window(&store, "audio").await; - assert_eq!(groups(&records), vec![(0, 0)]); - } -} From b29b857b6c521d48838d34b7b48bd4c832e99e84 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 22:37:25 -0700 Subject: [PATCH 03/16] wip(archive): moq export archive records every broadcast under its prefix Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 - doc/bin/cli.md | 39 +- doc/lib/rs/index.md | 2 +- drafts/draft-lcurley-moq-hang.md | 204 +-------- quest/m1/archive/README.md | 3 +- quest/m1/archive/consumer.md | 8 +- quest/m1/archive/producer.md | 157 ------- quest/m1/resolved-demand.md | 1 - quest/m1/sink-restart-follow.md | 2 - rs/moq-archive/Cargo.toml | 2 +- rs/moq-archive/MIGRATE.md | 9 + rs/moq-archive/README.md | 23 +- rs/moq-archive/benches/producer.rs | 91 +++- rs/moq-archive/src/producer/backends.rs | 265 +++++++++++ rs/moq-archive/src/producer/mod.rs | 2 + rs/moq-archive/src/producer/tests.rs | 72 +++ rs/moq-cli/Cargo.toml | 2 - rs/moq-cli/MIGRATE.md | 7 + rs/moq-cli/src/archive.rs | 568 ++++++++---------------- rs/moq-cli/src/args.rs | 8 +- rs/moq-cli/src/main.rs | 7 +- 21 files changed, 682 insertions(+), 791 deletions(-) delete mode 100644 quest/m1/archive/producer.md create mode 100644 rs/moq-archive/src/producer/backends.rs diff --git a/Cargo.lock b/Cargo.lock index 421472dfbaa..f4219293e2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4736,7 +4736,6 @@ dependencies = [ "moq-clock", "moq-hls", "moq-json", - "moq-msf", "moq-mux", "moq-relay", "moq-rtc", diff --git a/doc/bin/cli.md b/doc/bin/cli.md index ae5e3526323..72fc2d78526 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -24,7 +24,7 @@ Docker; see [Install](/setup/install). | `export` | `fmp4`, `mkv`, `ts`, `flv`, `h264`, `h265` | Write a container to stdout. | | `export` | `hls --listen` | Serve the broadcast as HLS over HTTP. | | `export` | `rtmp`, `srt`, `rtc` | Serve plays (`--listen`) or push to a remote (`--connect`). | -| `export` | `archive ` | Record the broadcast into an object store. | +| `export` | `archive ` | Record every broadcast under the prefix into an object store. | | `play` | | Decode and play in a native window with sound. | | `transcode` | | Publish a just-in-time rendition ladder next to a broadcast. | | `announced` | `[prefix]` | Follow the broadcasts announced on a relay. | @@ -296,26 +296,29 @@ equivalents. ## Archive ```bash -# Record a broadcast until it ends -moq --connect https://relay.example.com/anon --broadcast event.hang export archive s3://recordings/event +# Record every broadcast announced under `event` +moq --connect https://relay.example.com/anon --broadcast event export archive s3://recordings -# Replay it under another name +# Replay a timeline recording under another name moq --connect https://relay.example.com/anon --broadcast event-replay.hang import archive s3://recordings/event ``` -`export archive` records one broadcast with -[moq-archive](https://docs.rs/moq-archive), cutting each track into 2-10 s -spans at group boundaries. An [untimed](/concept/moq-lite#subscriptions) track, -as every track is over moq-lite before 05 and moq-transport 14-16, is stored -one group per span and replays untimed. A store that already holds the recording is -continued, unless the source now announces another -[epoch](/concept/moq-lite#publisher-epochs): that is a restart, so the export -fails; record it under a new prefix. `--retention 1h` keeps only the last -hour, a DVR, which refuses an untimed track since it has no time to measure. - -`import archive` republishes a recording as live tracks, fetching spans on -request. It ends where the recording ends; `--follow 2s` keeps polling a -recording still being made. +`export archive` records every broadcast announced at or under `--broadcast` +with [moq-archive](https://docs.rs/moq-archive), each publisher instance under +`/` in the store, until the connection ends. Every track is +stored as its own objects, cut at group boundaries about every 2 s of media (or +wall clock for an [untimed](/concept/moq-lite#subscriptions) track) and flushed +at least every 4 s. A broadcast restarted under a new +[epoch](/concept/moq-lite#publisher-epochs) records under its own key; a route +without an epoch gets a fresh one each time it announces, so it never resumes +or stitches into an older recording. Any recording that fails ends the export. +`--retention 1h` keeps only the last hour, a DVR, deleting older objects +`--retention-grace` (default 30 s) after they leave the window. + +`import archive` republishes a timeline recording made by an older `moq` as +live tracks, fetching spans on request. It ends where the recording ends; +`--follow 2s` keeps polling a recording still being made. It does not yet read +the per-track layout that `export archive` writes. Store URLs are `file:///absolute/path`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`, with credentials from the @@ -330,7 +333,7 @@ over one connection: moq --connect https://relay.example.com/anon \ import --broadcast event.hang srt --listen 0.0.0.0:9000 \ -- export --broadcast event.hang hls --listen 0.0.0.0:8080 \ - -- export --broadcast event.hang archive file:///recordings/event + -- export --broadcast event.hang archive file:///recordings ``` ## Publisher runs diff --git a/doc/lib/rs/index.md b/doc/lib/rs/index.md index 08059e24cea..95fc48afcbc 100644 --- a/doc/lib/rs/index.md +++ b/doc/lib/rs/index.md @@ -21,7 +21,7 @@ links cover crates awaiting their first release. | [moq-uring](https://docs.rs/moq-uring) | Experimental Linux io\_uring server for moq-lite. | | [hang](/lib/rs/hang) | The media layer: catalog, containers, ordered frame delivery. | | [moq-mux](/lib/rs/moq-mux) | Import and export fMP4/CMAF, MPEG-TS, Matroska, FLV, and Annex-B. | -| [moq-archive](https://docs.rs/moq-archive) | Record and replay hang broadcasts on any `object_store` backend. | +| [moq-archive](https://docs.rs/moq-archive) | Record and replay MoQ broadcasts on any `object_store` backend. | | [moq-video](/lib/rs/moq-video) | Native capture, hardware encode/decode (Apple, Windows, NVIDIA, VAAPI, V4L2, Android), and GPU rendering. | | [moq-v4l](https://docs.rs/moq-v4l) | Safe Video4Linux 2 bindings. | | [moq-audio](/lib/rs/moq-audio) | Microphone and speaker, Opus/PCM/AAC codecs, echo cancellation. | diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 576b331cf17..9dc32f9a561 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -955,10 +955,10 @@ The `replay` field, if present, is a relative MoQ broadcast path ({{field-broadc Absent, the timelines live on the catalog's own broadcast. A wildcard replay path names no generation. -The `store` field, if present, is the object-store URL the recording objects ({{recording}}) live under. +The `store` field, if present, is the object-store URL the recording objects live under, in the layout the moq-archive crate defines. Authorization for `replay` and `store` is external. -The `version` field is the recording object format version ({{recording-track}}). +The `version` field is the version of that recording object format. It is 2 for this specification. A publisher that exposes a store MUST set it; a publisher that does not MUST omit it. @@ -1114,199 +1114,6 @@ This is what makes a time table proxyable: each tick republishes a small snapsho Carrying the source's time rather than synthesizing one keeps the clock consistent with the EPG (event times are expressed on the source's clock) and preserves TOT's local-time-offset descriptors, which are operator policy a re-multiplexer cannot invent. -# Recording {#recording} -A live broadcast is bounded history: moq-lite {{moql}} serves the present with SUBSCRIBE and the recent past with FETCH, both ending at the publisher's cache. -A *recording* is the persistent tier, writing a broadcast to a filesystem or object store so it can be served back long after the live session ended. - -A recording is addressed by record. -Each recorded track has its own timeline ({{timeline}}), and each of its records is stored as one object named by the record's `sequence`. -A reader that wants a record of a track issues one whole-object GET, with no range header and no second request, which is what both an HLS or DASH origin and a player seeking a VOD recording need. - -A recording covers the tracks its writer selects, including the catalog, plus each one's timeline. - -## Layout {#recording-layout} -A recording is a set of objects under a common prefix: - -~~~ -//.info -//segments/ -~~~ - -The common prefix is application-defined and is not interpreted by this format. - -`` is the track's UTF-8 name with every byte outside `A-Z a-z 0-9 _ -` percent-encoded as `%` followed by two uppercase hexadecimal digits. -For example, `catalog.json` becomes `catalog%2Ejson`. -An encoded name never contains `/` or begins with `.`, so a track cannot address anything outside the prefix. -`.info` and `segments` are reserved within a track directory. - -A recording names each track's timeline by appending `.timeline.z` to the track's name, so a recording is recoverable from its objects alone; a recorded track's own name MUST NOT end in `.timeline.z`. -For a recorded track, `segments/` holds the frames of record `sequence` of its timeline. -For a timeline track, `segments/` holds the complete Window groups ({{timeline-framing}}) closed while committing record `sequence` of the track it indexes, at least one per object. - -`` is written as exactly 19 decimal digits, padded with leading zeros, so lexical and numeric order agree. -Recorded sequences, group IDs, and frame indices MUST be integers in the range 0 through 9007199254740991 (2^53 - 1), so timeline JSON preserves them exactly. -Writers and readers MUST reject IDs outside this range, including reconstructed group IDs in the binary table. -After its first record, a timeline MUST commit consecutive sequences. -A writer MUST stop before allocating a sequence beyond the limit; sequences never wrap. -Each track has its own objects so a reader can fetch one rendition without downloading the others. - -## Track Objects {#recording-track} -`/.info` is a UTF-8 JSON object containing the immutable properties needed to interpret and replay the track: - -~~~ -{ - "version": 2, - "priority": 0, - "timescale": 1000000 -} -~~~ - -`version` and `priority` are required integers. -`version` identifies this recording format and MUST be 2. -`priority` and `timescale` have the meanings of moq-lite `TRACK_INFO` {{moql}}: `priority` is in the range 0 through 255, and `timescale` is in the range 1 through 9007199254740991, so JSON consumers can preserve it exactly. -`timescale` is absent exactly when the track is untimed: its frames store no timestamps ({{recording-segments}}) and its records carry no `pts` or `duration` ({{timeline-records}}). -A timeline track's object declares a `timescale` exactly when the track it indexes does. -A reader MUST preserve integer values exactly. -An optional `epoch` string is the Epoch of the route the source was read through {{moql}}, in its canonical UUID text, and is omitted when that route had none. -`Publisher Max Age` is not stored; a reader supplies its own serving policy. - -The track object MUST be durable before its first segment object is stored, including for a timeline track. -It is immutable for the lifetime of the recording. -A reader MUST refuse an unknown version or invalid track properties. -On an existing `.info`, a writer MUST validate and compare the parsed `version`, `priority`, `timescale`, and `epoch` values, an absent `timescale` or `epoch` matching only an absent one; JSON whitespace and member order do not affect equality. -Different property values MUST fail enrollment, and the existing object MUST NOT be rewritten. -A different `epoch` means the source restarted, so the writer starts a new recording rather than continuing this one. - -## Segment Objects {#recording-segments} -A segment object holds one record's frames: - -~~~ -Segment Object { - Version (i) = 2 - Frame Start (i) - Group Count (i) - Group Table { - Sequence Delta (i) - Frame Count (i) - Frame Table { - [Timestamp (i)] - Payload Offset (i) - Payload Length (i) - } ... - } ... - Payload Bytes (..) -} -~~~ - -Fields annotated `(i)` are variable-length integers using the QUIC encoding ({{!RFC9000}}, Section 16). -`Frame Start` is the index within its group of the first group's first stored frame; every later group starts at frame 0. -`Group Count` gives the number of group entries; each `Frame Count` gives the number of frame entries in that group. -The first `Sequence Delta` is the absolute group sequence number. -Each subsequent value is the group sequence number minus the previous sequence number minus one; zero therefore means the next consecutive group. -Groups MUST have strictly ascending sequence numbers; reconstruction MUST reject values exceeding the recording ID limit ({{recording-layout}}). -Frame entries appear in their original order within each group. -`Timestamp` is the frame's absolute timestamp in the track's `timescale` units, not a delta. -It MUST be in the range 0 through 9007199254740991 (2^53 - 1), so browser replay preserves it exactly; writers and readers MUST reject larger values. -`Timestamp` is present exactly when the track object declares a `timescale` ({{recording-track}}); an untimed track's frame entries omit it, and a reader replays those frames untimed. - -`Payload Offset` is relative to the start of `Payload Bytes`, immediately after the last table entry; `Payload Length` is the frame's payload size in bytes. -Payloads are concatenated in table order, without gaps, overlap, or trailing bytes. -They are the original frame payloads, including any group-scoped compression, without moq-lite FRAME headers. -A reader can locate a group or a frame index directly from the table without parsing earlier payloads. - -A decoder MUST validate the complete table before accessing any payload. -Counts and entries MUST fit in the retrieved object, and every offset and length MUST describe a range within its payload bytes, with overflow checked before arithmetic. -An unknown version or malformed table is a missing record, not an invalid recording. - -A recorded track's object MUST hold exactly the frames of its record's span ({{timeline-records}}): its first group and `Frame Start` match `start`, its groups are consecutive and none is empty, and its last group ends at `end`. -A group may span several objects when a record was split between frames; a reader stitches it from consecutive records. -Segment objects are immutable once written and are parseable with their track object alone, without the timeline. - -The timeline objects use the same envelope with a `Frame Start` of zero, each frame stamped with the `pts` of the record it was committed with, or unstamped for an untimed track. -Their own groups are discovered by listing, not indexed by a record, avoiding a record that must index itself. -A reader replays their checkpoints and operations in group order, preserving group boundaries and their independent DEFLATE windows. -Neither timeline nor media objects are appended to or rewritten. -One writer owns a recording prefix. An existing segment object key MAY be reused only for identical object bytes; a conflicting create MUST fail. - -## Writer Behavior {#recording-writer} -A writer subscribes to the broadcast and buffers each track's open span independently of the publisher or relay cache. -It cuts each track's timeline itself ({{timeline-cutting}}) and commits each track independently: a track whose span is complete does not wait for any other track. -The recording contract MUST NOT depend on a relay retaining those groups while storage catches up. - -A writer MUST accept new groups for each track in strictly increasing sequence order and MUST refuse a duplicate or decreasing sequence; it does not reorder arrivals. -Groups already accepted MAY complete in any order; a group's frames are indexed once every earlier accepted group has finished, and a group still open is indexed frame by frame as it grows. -A writer MUST NOT insert frames before the end of a record it already committed. -A frame whose timestamp is missing on a timed track, or present on an untimed one, is malformed. - -To commit record N of a track, the writer makes the track's `segments/N` object durable, publishes the record through the track's timeline encoder, applies any retention pops, closes the timeline's current Window group, and stores the complete groups under the timeline track's `segments/N` key. -It MUST make that timeline object durable before committing the track's next record or deleting expired objects. -If a track's object cannot be made durable, the writer drops the record: its frames are lost, and the next record takes sequence N. -If the timeline object cannot be made durable, the recording stops at the preceding durable timeline object. -The recording owns its timeline encoders so they publish only what was stored. - -On a clean end, the writer MUST close each track's final record, finish its timeline track, and make its final complete groups durable. -There is no completion marker; object listing alone does not distinguish a clean end from an interruption. - -## Retention {#recording-retention} -A recording has one of two retention modes: - -- An *archive* is unbounded and retains every record until explicitly deleted. -- A *DVR* retains a configured duration of each track's records and expires each track's oldest records as newer ones become durable. - -An application offering DVR without an explicit retention value SHOULD default to at least 30 seconds. -The writer removes a track's oldest record only while the track's remaining records still span the configured duration, measured from the second-oldest record's `pts` to the newest record's end. -A track's newest record is never removed, so a catalog that never changes outlives the media recorded with it. -Retention is measured from timeline records, not from wall-clock arrival or relay cache state. -An untimed track's records have no duration, so a DVR writer MUST refuse to record an untimed track. - -Trimming occurs only as part of committing a new record of the same track, before that record's timeline object is finalized ({{recording-writer}}). -A recording MUST NOT trim after a track's final record is committed. -Expiration first pops expired records from the timeline window and makes the resulting complete timeline groups durable, then deletes the expired records' objects. -The writer MUST retain each timeline's latest object and enough earlier timeline groups to recover the retained window from a checkpoint. -It MAY delete a timeline's earlier objects, oldest first, so its remaining record IDs stay consecutive. -The index therefore never advertises media the retention process has already deleted, although media objects MAY temporarily outlive the index. -Relay cache eviction does not change the recording timeline. - -## Bootstrap and Recovery {#recording-recovery} -A reader or restarting writer lists each timeline track's `segments/` prefix and replays its objects in numeric sequence order, including retention operations, from a retained checkpoint. -The catalog supplies the timeline names ({{archive-catalog}}); a restarting writer finds them by their suffix ({{recording-layout}}). -The recovered records determine the committed object keys; a missing or malformed referenced object MUST NOT be served. -Objects not referenced by a recovered timeline do not advertise content on their own. -Listing is not an atomic snapshot across tracks; absence from an earlier listing MUST NOT override a successful GET of an object referenced by a later durable timeline record. - -Before accepting new groups, a restarting writer MUST recover each retained timeline under exclusive ownership of the recording prefix. -A track's objects at or past its next sequence were never referenced by a timeline, so the writer deletes them before reusing their keys. -It resumes each track after the end of its newest committed record, including partway through a group that record split. -A restarting DVR writer also lists every recorded track's `segments/` prefix and, after waiting the configured deletion grace period from successful recovery, deletes objects whose keys are absent from the recovered retained records, including expired objects left by interrupted expiration. -It MUST NOT perform this cleanup if timeline recovery or listing fails or is incomplete, or while another writer can create or commit objects. -This completes previously committed expiration; it does not pop additional records after a final record. -Timeline objects needed for checkpoint recovery and `.info` objects are not candidates for this cleanup. - -After replaying a timeline's record N below the recording ID limit, a reader follows an active recording by GET of that timeline's `segments/N+1`, without refreshing media listings. -At the limit there is no next key; this does not establish a clean recording end. -Not Found does not distinguish a pending commit, an expired DVR record, or an interrupted recording. -A reader that has fallen behind retention MUST bootstrap again from a retained checkpoint. -Alternatively, a reader MAY list timeline keys after its last replayed key, consume all pages, sort the results, and replay them in order. -A continuation token is used only within that enumeration; the next refresh starts from the last replayed key. -A reader MUST NOT advance its replay cursor past a missing object without recovering from a retained checkpoint. -Retention operations evict expired records from the reader's cached index; incremental listing alone does not report deletions. - -## Reader Behavior {#recording-reader} -Reading record N of track T is a GET of T's `segments/N`; the table MUST hold exactly the frames the record names. -Nothing on that path reads media the consumer did not ask for, and nothing requires a second request. - -A reader MAY serve moq-lite FETCH from a recording. -Given a group, a reader finds the retained records whose spans hold it, in order, and serves their frames from the requested `frame_start` {{moql}}. -A group whose earliest retained record starts after the requested frame is not served. -A group whose records do not reach its end yet, such as an append-log still being recorded, grows as later records commit, and ends where the recording does once the reader learns it ended. -The reader reconstructs moq-lite FRAME headers from the stored timestamps and lengths, preserving the original payload bytes. -A group absent from the recording is a normal FETCH failure. -Recording catalog groups does not establish which catalog update applies to a media group; this format does not define that correlation. - -A reader deriving a presentation-ordered format renders it from the timelines ({{timeline-derived}}) and transmuxes record objects on demand. -Nothing derived needs to be stored: the playlist or manifest is a function of the timelines, and a media segment is a function of the records its span covers. - - # Rooms A room is a broadcast path prefix. A participant publishes camera and microphone @@ -1335,12 +1142,6 @@ Clamping a reference that escapes above that root would silently redirect the su TODO Security -A consumer parsing a recording ({{recording}}) is parsing data at rest that it did not necessarily write. -It MUST validate the group/frame table against the bytes actually retrieved before allocating from its counts or accessing payloads ({{recording-segments}}), and treat a malformed object as a missing record rather than letting it invalidate the recording. -It MUST reject a track object with a zero `timescale`. -Varint fields are subject to the same limits as moq-lite {{moql}}. -A recording inherits the confidentiality and integrity properties of the storage holding it; encryption at rest is transparent to the format and out of scope. - # IANA Considerations @@ -1359,6 +1160,7 @@ This document has no IANA actions. - Added the root `version` ({{catalog-version}}): the oldest draft that reads the catalog, which a consumer refuses when above its own. A `track` other than the entry's ID or a video `warmup` needs 4. Consumers MUST ignore unknown fields. - A data track marks a discontinuity with a group of one object with no bytes, since its empty payload is data. An object with no bytes on an audio or video track, or sharing its group, is malformed. - Each archive timeline declares its own fixed nominal record duration, omitted for an untimed track; removed the broadcast-wide duration bound. +- Removed the Recording section: the store layout is a moq-archive format, documented with that crate, not part of this draft. - Added the optional video `preference` field: a consumer selects automatically among the highest preference it can decode. - Derived HLS and DASH exports hand disabled references to enabled renditions at the first record numbered past the advertised ones, keeping the record `sequence` as the HLS media sequence and starting a DASH period. - Added optional `stats` and `echo` catalog sections, publisher/viewer snapshots keyed by rendition ID, and the `.echo` feedback broadcast convention. diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index 1d940249f59..cfd62883da6 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -55,7 +55,7 @@ archive (per-track `.timeline.z` records named by the catalog's root stored keys and another epoch is another prefix. An optional DVR mode prunes old objects. Rejected: an object per whole-broadcast window (every read pulls every rendition) and size-based cuts. The - [producer](/quest/m1/archive/producer.md) records the rest. + `archive::Producer` records the rest. - **Hosts serve archives, relays do not.** `archive::Consumer` serves a stored prefix through `broadcast::Dynamic` and `track::Dynamic`, the archived catalog included as an ordinary track, and a host publishes it at a path of @@ -85,7 +85,6 @@ from a host. Each child fixes the stale docs it touches. ## Required - [TRACKS in js/net](/quest/m1/archive/tracks-js.md) - `@moq/net` lists a broadcast's tracks over lite-07's Tracks Stream and answers listings of its own broadcasts, mirroring Rust -- [Archive producer](/quest/m1/archive/producer.md) - `archive::Producer` stores every matching broadcast as duration-cut per-track objects keyed by group and pts, recording gaps instead of stalling, with an optional DVR mode - [Archive consumer](/quest/m1/archive/consumer.md) - `archive::Consumer` serves a stored prefix as a broadcast from its key listing, and `moq export/import archive` use the new pair - [Catalog HLS field](/quest/m1/archive/catalog-hls.md) - the root `archive` section is gone; an opt-in per-rendition `hls` field marks each aligned HLS segment's first group and pts and declares its nominal duration - [SUBSCRIBE_TRACKS](/quest/m1/archive/subscribe-tracks.md) - moq-transport SUBSCRIBE_TRACKS is answered through the TRACKS hook, and an IETF peer's tracks are learned with FORWARD=0, instead of NOT_SUPPORTED diff --git a/quest/m1/archive/consumer.md b/quest/m1/archive/consumer.md index a71663eee6f..9ab4f6b508b 100644 --- a/quest/m1/archive/consumer.md +++ b/quest/m1/archive/consumer.md @@ -5,8 +5,8 @@ `archive::Consumer` serves one broadcast's stored key prefix as an ordinary broadcast: every stored track, the archived catalog included, with its original groups, frames, timescale, and priority. A host publishes it at a path of its choosing; -moq-relay never uses it. `moq export archive` records through -`archive::Producer` and `moq import archive` serves through +moq-relay never uses it. `moq export archive` already records through +`archive::Producer`; `moq import archive` serves through `archive::Consumer`. It replaces `moq_archive::Reader`. ## Plan @@ -81,7 +81,3 @@ and re-added under its name replaying both runs in sequence, a growing archive with no markers staying open, and an empty or unrecognized prefix refused loudly. Public API: `archive::Consumer` replaces `Reader`. Wire: none. - -## Required - -- [Archive producer](/quest/m1/archive/producer.md) - the layout this serves diff --git a/quest/m1/archive/producer.md b/quest/m1/archive/producer.md deleted file mode 100644 index bac4caf7453..00000000000 --- a/quest/m1/archive/producer.md +++ /dev/null @@ -1,157 +0,0 @@ -# [XL] Archive producer - -## Goal - -`archive::Producer` takes a path prefix and stores every track of every -broadcast under it into an `object_store::ObjectStore`, -each broadcast under its own key prefix. It holds what the publisher's cache -stops serving: a lost group becomes a gap and never stalls the archive. -Datagrams are lossy and never cached, so a track's datagrams are recorded -from when its subscription takes effect; earlier ones are not. An -optional DVR mode prunes old objects. It replaces `moq_archive::Writer` and -its timelines and recovery machinery. - -## Plan - -Decided by the maintainer on 2026-10-17 (see the -[epic](/quest/m1/archive/README.md)): - -- **Input.** One producer records every broadcast under a prefix as it - appears: ANNOUNCE finds each broadcast announced at its own path, - TRACKS (`broadcast::Consumer::tracks`) lists its tracks, and each track - records through its own SUBSCRIBE. A covering-prefix route is capability, - not inventory, so broadcasts served under one on request without their own - announcement are not recorded, and a TRACKS refused with NOT_FOUND on an - announced path means it is not a broadcast; any other refusal, such as - UNAUTHORIZED, is that broadcast's error and reaches the caller, never a - silent skip - (decided 2026-10-19, replacing one SUBSCRIBE_ALL). The application maps - each broadcast (path and epoch) to its key prefix, so a different epoch is - a different prefix and the producer stores no epoch. Open since discovery - became per broadcast: keep the prefix input, or take one broadcast and leave - the ANNOUNCE loop to the host. Recommended: keep the prefix, so every host - does not repeat that loop. -- **Ordering.** Each track records through an ascending subscription with max - delay = min(the publisher's max age from TRACK_INFO, a configured cap). A - group still missing at the cap is skipped and stays absent from the store, - which is how a gap is recorded. Anything younger is still in the live cache. -- **Objects.** Groups coalesce in order into per-track objects, cut at a group - boundary once a configured duration passes. An oversized timed group splits - between frames. An untimed track (no timescale) cuts by wall clock at group - boundaries. Each object preserves group sequences, frame boundaries, - timestamps, and payloads. Rejected: whole-broadcast objects per window and - size-based cuts. -- **Datagrams keep their form.** A track's groups and datagrams share one - sequence space, so each stored sequence records its forwarding form, group - or datagram, and a datagram is stored as its one payload and timestamp in - sequence order beside the groups. The consumer replays a datagram as a - datagram, never as a cached one-frame group, so the replay keeps the - source's lossy contract. A datagram is never split and counts as one - sequence for keys, cuts, and resume. Datagrams sent before the track's - SUBSCRIBE took effect are not recorded, as above. -- **Keys.** `//--` for a - timed track and `//` for an untimed one, - zero-padded so keys sort in content order. The frame index keeps the - fragments of an oversized timed group unique and ordered, since they share - the group and pts can repeat or go backward in decode order; pts stays for - time lookups. `` is the name in the existing canonical percent - encoding (`rs/moq-archive/src/path.rs`, everything but `A-Z a-z 0-9 _ -` - escaped), so a name holding `/` stays one segment and no name collides with - `.info` or `.end`; a consumer refuses a non-canonical segment. A sorted LIST answers - group and time lookups; there is no index object or manifest. Rejected for - untimed: `-`, since an untimed group is never - split. -- **Untimed groups never split.** While an untimed group is open, the bounded - flush rewrites its object in place under the same key, so keys stay unique - without a frame suffix; the consumer re-reads the newest object of a growing - track. Folded from the untimed quest (#5720 landed untimed in the old writer). -- **Track properties.** A small `//.info` holds the source - properties the stored media depends on (timescale, priority), written before - the track's first object, so a consumer knows them before reading media. - Properties never change for a track name, so a resume that finds different - ones fails loud. The source's max age is not stored: it bounds the live - cache, not the archive, and replaying it would expire archived history - (timestamp age measures a group against the newest one), so the consumer - advertises its own retention instead. Rejected: properties only in each - object's header. -- **Completion markers**, decided by the maintainer on 2026-10-19 ("record - the error"). A track has ended once its ordered groups and its datagrams - (`poll_recv_datagram`) have both reached their end, so no received - datagram is lost. How it ended is decided then: a clean end at its - declared exclusive `Position` (`track::Subscriber::poll_finished`; group - and frame, the shape `track::Producer::finish_at` takes, so a mid-group - end survives), or the stream error code it ended with - (`StreamError::to_code`). The markers wait for the broadcast's listing to - end with the instance, since until then a track removed and re-added - under its name continues its sequences and a written marker would be - wrong. Then, with every frame and datagram stored, the producer writes - each track's small `//end`, recording how its last run - ended, followed by `/.end`, which says no track will be added. - Until then an ended track just stops growing. The names are chosen for the consumer's - refresh: a track's `end` sorts after its zero-padded numeric keys, so - the track's `StartAfter` LIST reaches it with no extra request, and it can - never be an object key; the broadcast's `.end` sits beside the track - segments, where the delimiter LIST that finds added tracks sees it, and - no canonical track name starts with `.`. The consumer ends each track as - its marker says and otherwise treats the archive as live. Only a crash, or a producer - stopped before the end, leaves no marker, so its archive reads as still - growing. A producer resuming mid-broadcast follows every stored track as - live, including one that ended before the restart; it writes markers only - for tracks whose end it saw, so a stored track that ended unseen leaves - the broadcast without `.end`, as a stop before the end does. A resume - under a broadcast that already has its `.end` records nothing. DVR - pruning never deletes a marker. -- **Bounded flush** (folded from the idle-flush quest): an object holding a - frame is stored within a bounded wall-clock delay of that frame's arrival, - even when its track goes idle. The bound covers closing the object, not - upload latency. -- **No manual `flush()` or `cut()`** until a caller needs one. -- **Resume.** Under a key prefix that already holds objects, each track - continues at the frame after its last stored one: its subscription starts - at `Position(last group, stored frame count)`, so a group split across - objects or still open at the restart loses nothing. A timed track stores - the rest as a new object keyed by its first frame's index and pts. An untimed track - reads its newest object back and keeps rewriting it in place with the - appended frames. A completed group yields no more frames, so this is the - same rule whether or not the restart landed mid-group; if the source has - dropped the rest, the stored frames are all the archive holds of it. The - start is per track: each track's SUBSCRIBE carries its own floor. - Rejected: resuming after the - last stored group (loses a split or open group's tail) and re-recording the - last group from frame 0 (fails once the source has dropped its head). -- **DVR mode** stays in the crate, for hosts without bucket lifecycle rules: - it deletes a track's objects older than a configured window after a grace - period, so readers in flight finish, and always keeps each track's newest - object so a static catalog outlives the media around it. Rejected: no - retention in the crate. -- **Layout docs.** Delete the hang draft's Recording section - (`drafts/draft-lcurley-moq-hang.md`): the layout is a `moq-archive` format, - not a hang one. Document it in moq-archive rustdoc and the epic's Archives - doc page. - -Test on mock time with the in-memory store: a lost group recorded as a gap -without stalling later groups, duration and wall-clock cuts, a split oversized -timed group whose fragments get unique keys in frame order even when pts -repeats or goes backward, an open untimed group rewritten in place, an idle track flushed -within the bound, resume after a restart (including mismatched properties, -and a restart mid-group for both a split timed group and an open untimed -group, with no frame lost or duplicated), a track name holding `/` or -starting with `.` round-tripping as one segment, -DVR pruning with grace, a track added mid-recording, two broadcasts under -the prefix landing under their own key prefixes, a track interleaving groups -and datagrams stored with each sequence's form, a clean end writing no -marker until the listing ends and then each track's `end` (including a -mid-group end) followed by the broadcast's `.end`, a track removed and -re-added under its name before the broadcast ends whose replay holds both -runs in sequence and whose marker records the second run's end, a track -whose groups and datagrams interleave and end at once storing every -received datagram before its `end`, a track ending with an error writing an -error marker while its broadcast still gets `.end`, and a producer stopped -mid-recording writing no marker. - -Public API: `archive::Producer` and its config replace `Writer`, `Control`, -and the timeline-driven types. Wire: none. - -## Related - -- [Archive consumer](/quest/m1/archive/consumer.md) - reads this layout back diff --git a/quest/m1/resolved-demand.md b/quest/m1/resolved-demand.md index a00ee625d33..6c945e1b0e2 100644 --- a/quest/m1/resolved-demand.md +++ b/quest/m1/resolved-demand.md @@ -174,6 +174,5 @@ max age or the range. ## Related -- [Archive producer](/quest/m1/archive/producer.md) - its resume floor crosses relays that viewers may have warmed - [Drop set](/quest/m1/drop-set.md) - closes the gap this leaves for groups already reported dropped - [JS unbounded groups](/quest/m2/unbounded-groups/js.md) - the JS byte bound for groups and the cursor runs between them diff --git a/quest/m1/sink-restart-follow.md b/quest/m1/sink-restart-follow.md index c3910d78ceb..8c354de9aee 100644 --- a/quest/m1/sink-restart-follow.md +++ b/quest/m1/sink-restart-follow.md @@ -23,8 +23,6 @@ and fix the rest the same way: streams in `moq-rtmp`, `moq-srt`, `moq-rtc`) and `--connect` (one push). SRT already rides the TS follower (`--linger`, `--stitch`); confirm its listen side does too. FLV and WebRTC need a decision. -- `export archive`: a restart already fails a continued store under a new - epoch; decide what a mid-recording restart does. - `transcode`: its source broadcast; following means restarting the ladder on the new instance. diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index 16b8b0d7060..568c480915f 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "moq-archive" -description = "Versioned hang recording objects on any object_store::ObjectStore" +description = "MoQ broadcast archives on any object_store::ObjectStore" authors = ["Luke Curley "] repository = "https://github.com/moq-dev/moq" license = "MIT OR Apache-2.0" diff --git a/rs/moq-archive/MIGRATE.md b/rs/moq-archive/MIGRATE.md index 93e45670400..df376a78891 100644 --- a/rs/moq-archive/MIGRATE.md +++ b/rs/moq-archive/MIGRATE.md @@ -4,6 +4,15 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Producer` replaces `Writer` + +`Writer`, its `Control`, and `writer::{Config, Retention}` are removed; record with `Producer::new(store, broadcast, producer::Config::default()).run().await` instead. +A producer records every track of one broadcast, found with TRACKS rather than a catalog, and needs no timeline: the layout is per-track objects keyed by group, frame, and timestamp, documented in the crate docs. +Run one producer per broadcast under a key prefix you pick, such as `/`; it stores no epoch, so `Info::epoch` and `Info::with_epoch` are removed. +`writer::Retention` becomes `producer::Dvr`, set with `Config::with_dvr`, and works on untimed tracks too. +`Error::{EpochMismatch, Enrolled, Untimed, Timeline, Closed}` are removed; a failed source listing is `Error::Source`. +The new layout is not readable by `Reader`, which still replays timeline recordings. + ### Untimed tracks record untimed (#5720) `Info::new` and `Info::timescale` take and return `Option`, and `Error::TimescaleMismatch` carries `Option` fields. diff --git a/rs/moq-archive/README.md b/rs/moq-archive/README.md index a357d1f807b..af917e9a37f 100644 --- a/rs/moq-archive/README.md +++ b/rs/moq-archive/README.md @@ -4,25 +4,26 @@ # moq-archive -Versioned [hang](https://docs.rs/hang) recordings on any +MoQ broadcast archives on any [`object_store::ObjectStore`](https://docs.rs/object_store). -The crate owns the portable layout and codecs: percent-encoded track names, -per-track `.info` JSON, the binary segment envelope, and put/get/list/delete. -`Store` wraps a store under a recording prefix and stays generic, so a caller -that needs runtime dispatch passes `Arc`. A broadcast -advertises its recording through the catalog's -[`archive`](https://doc.moq.dev/concept/hang) entry. +`Producer` records every track of one broadcast under a key prefix, with no +catalog dependency: it lists the tracks with TRACKS and subscribes to each, +storing duration-cut objects keyed by group, frame, and timestamp. A lost +group becomes a gap instead of a stall, a restarted producer resumes each +track where it stopped, and an optional DVR window prunes old objects. The +host picks the prefix and runs one producer per broadcast. ```bash cargo add moq-archive ``` -Group bounds are finite inclusive ranges in first-to-last order: - ```rust -let key = moq_archive::Key::groups("video", 5..=7)?; -assert_eq!(key.track(), "video"); +use moq_archive::object_store::memory::InMemory; + +// `broadcast` is a `moq_net::broadcast::Consumer`. +let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); +moq_archive::Producer::new(store, broadcast, Default::default()).run().await?; ``` See [docs.rs/moq-archive](https://docs.rs/moq-archive) for the object layout. diff --git a/rs/moq-archive/benches/producer.rs b/rs/moq-archive/benches/producer.rs index f328e4d9d04..12e41f8fb7e 100644 --- a/rs/moq-archive/benches/producer.rs +++ b/rs/moq-archive/benches/producer.rs @@ -1 +1,90 @@ -fn main() {} +// Host time measures benchmark throughput. +#![allow(clippy::disallowed_methods, clippy::disallowed_types)] + +//! `Producer::run` recording every track of a broadcast into memory, swept over the number of +//! tracks and the groups each one holds. Throughput is per frame, so a per-event cost that grows +//! with the track table shows up as a slope across the track counts. +//! +//! Run with `cargo bench -p moq-archive --bench producer`. + +use std::time::{Duration, Instant}; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_archive::object_store::memory::InMemory; +use moq_archive::{Producer, Store, producer}; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; + +const FRAMES_PER_GROUP: u64 = 2; + +/// A broadcast of `tracks` tracks, each already holding `groups` one-second groups. +fn setup(tracks: usize, groups: u64) -> (broadcast::Producer, Vec) { + let source = broadcast::Info::new().produce(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let tracks = (0..tracks) + .map(|index| { + let track = source.create_track(format!("track{index}"), info.clone()).unwrap(); + for sequence in 0..groups { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for frame in 0..FRAMES_PER_GROUP { + let pts = Timestamp::from_millis(sequence * 1_000 + frame * 500).unwrap(); + group.write_frame(pts, vec![0u8; 64]).unwrap(); + } + group.finish().unwrap(); + } + track + }) + .collect(); + (source, tracks) +} + +/// Record the broadcast until every track ends, from the producer's start. +async fn record(source: broadcast::Producer, tracks: Vec) -> Duration { + let config = producer::Config::default() + .with_duration(Duration::from_secs(2)) + .with_max_delay(Duration::from_secs(3600)); + let store = Store::new(InMemory::new(), "bench"); + let start = Instant::now(); + let run = tokio::spawn(Producer::new(store, source.consume(), config).run()); + for track in &tracks { + while track.subscription().is_none() { + tokio::task::yield_now().await; + } + track.finish().unwrap(); + } + source.close(); + run.await.unwrap().unwrap(); + start.elapsed() +} + +fn producer(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .unwrap(); + + let mut bench = c.benchmark_group("producer"); + bench.sample_size(10); + bench.warm_up_time(Duration::from_secs(1)); + bench.measurement_time(Duration::from_secs(2)); + for tracks in [1, 10, 100] { + for groups in [10, 100] { + bench.throughput(Throughput::Elements(tracks as u64 * groups * FRAMES_PER_GROUP)); + bench.bench_function(BenchmarkId::new(format!("{groups} groups"), tracks), |b| { + b.iter_custom(|iters| { + (0..iters) + .map(|_| { + let (source, tracks) = setup(tracks, groups); + runtime.block_on(record(source, tracks)) + }) + .sum() + }); + }); + } + } + bench.finish(); +} + +criterion_group!(benches, producer); +criterion_main!(benches); diff --git a/rs/moq-archive/src/producer/backends.rs b/rs/moq-archive/src/producer/backends.rs new file mode 100644 index 00000000000..df549ae9ad0 --- /dev/null +++ b/rs/moq-archive/src/producer/backends.rs @@ -0,0 +1,265 @@ +//! The same recording on every backend, including the S3 HTTP adapter rather than a listing fake. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use axum::body::Body; +use axum::http::{Method, Uri}; +use bytes::Bytes; +use futures::TryStreamExt; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut}; +use object_store::local::LocalFileSystem; +use object_store::memory::InMemory; +use object_store::path::Path; +use s3s::service::S3ServiceBuilder; + +use super::*; + +const VIDEO: &str = "video/1080p"; +const CHAT: &str = "chat.json"; + +/// Real time, so the S3 client's own timers behave: a flush short enough to rewrite the open +/// untimed group while the test waits, and long enough that no put outlasts it. +fn config() -> Config { + Config::default() + .with_duration(Duration::from_secs(2)) + .with_flush(Duration::from_millis(200)) + .with_max_delay(Duration::from_secs(10)) +} + +/// A source with `groups` one-second video groups, and the chat's groups as `(sequence, frames)`. +fn source(groups: u64, chat: &[(u64, &[&str])]) -> (broadcast::Producer, [track::Producer; 2]) { + let source = broadcast::Info::new().produce(); + let video = source + .create_track(VIDEO, track::Info::default().with_timescale(Timescale::MILLI)) + .unwrap(); + for sequence in 0..groups { + let mut group = video.create_group(group::Info { sequence }).unwrap(); + for frame in 0..2 { + let pts = Timestamp::from_millis(sequence * 1_000 + frame * 500).unwrap(); + group.write_frame(pts, format!("v{sequence}.{frame}")).unwrap(); + } + group.finish().unwrap(); + } + let text = source.create_track(CHAT, track::Info::default()).unwrap(); + for (sequence, frames) in chat { + let mut group = text.create_group(group::Info { sequence: *sequence }).unwrap(); + for frame in *frames { + group.write_frame(None, frame.to_string()).unwrap(); + } + group.finish().unwrap(); + } + (source, [video, text]) +} + +fn path(store: &Store, track: &str, name: &str) -> Path { + Path::parse(format!( + "{}/{}/{name}", + store.prefix(), + crate::path::encode_track(track).unwrap() + )) + .unwrap() +} + +/// Wait until every one of `paths` is stored. +async fn stored(store: &Store, paths: &[Path]) { + tokio::time::timeout(Duration::from_secs(10), async { + for path in paths { + while store.inner().head(path).await.is_err() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + }) + .await + .expect("the producer stores the objects"); +} + +/// Record in two runs: the first is stopped once it stored a split timed track and an open +/// untimed group, and the second resumes both, appending to the open group in place. +async fn record(store: &Store) { + let (first, _tracks) = source(4, &[(0, &["c0"])]); + let run = tokio::spawn(Producer::new(store.clone(), first.consume(), config()).run()); + stored( + store, + &[ + path(store, VIDEO, "0000000000000000000-0000000000000000000-0000000000000000000"), + path(store, VIDEO, "0000000000000000002-0000000000000000000-0000000000000002000"), + path(store, CHAT, "0000000000000000000"), + ], + ) + .await; + run.abort(); + assert!(run.await.unwrap_err().is_cancelled()); + + let (second, tracks) = source(6, &[(0, &["c0", "c1"]), (1, &["c2"])]); + let run = tokio::spawn(Producer::new(store.clone(), second.consume(), config()).run()); + for track in &tracks { + while track.subscription().is_none() { + tokio::time::sleep(Duration::from_millis(1)).await; + } + track.finish().unwrap(); + } + second.close(); + tokio::time::timeout(Duration::from_secs(10), run) + .await + .expect("the producer ends with its broadcast") + .unwrap() + .unwrap(); +} + +/// Every object under the store's prefix, by path relative to it. +async fn objects(store: &Store) -> BTreeMap { + let metas: Vec<_> = store.inner().list(Some(store.prefix())).try_collect().await.unwrap(); + let mut objects = BTreeMap::new(); + for meta in metas { + let bytes = store.inner().get(&meta.location).await.unwrap().bytes().await.unwrap(); + let key = meta.location.as_ref().strip_prefix("rec/").unwrap().to_string(); + objects.insert(key, bytes); + } + objects +} + +fn decode(objects: &BTreeMap, key: &str, timed: bool) -> Vec<(u64, Vec)> { + let object = crate::layout::Object::decode(objects[key].clone(), timed).unwrap(); + object + .entries + .iter() + .map(|entry| match entry { + crate::layout::Entry::Group { sequence, frames } => ( + *sequence, + frames + .iter() + .map(|frame| String::from_utf8(frame.payload.to_vec()).unwrap()) + .collect(), + ), + crate::layout::Entry::Datagram { .. } => panic!("no datagrams were sent"), + }) + .collect() +} + +#[tokio::test] +async fn recordings_are_byte_identical_on_every_backend() { + let memory = Store::new(InMemory::new(), "rec"); + record(&memory).await; + let expected = objects(&memory).await; + assert_eq!( + expected.keys().map(String::as_str).collect::>(), + [ + ".end", + "chat%2Ejson/.info", + "chat%2Ejson/0000000000000000000", + "chat%2Ejson/end", + "video%2F1080p/.info", + "video%2F1080p/0000000000000000000-0000000000000000000-0000000000000000000", + "video%2F1080p/0000000000000000002-0000000000000000000-0000000000000002000", + "video%2F1080p/0000000000000000004-0000000000000000000-0000000000000004000", + "video%2F1080p/end", + ] + ); + assert_eq!( + decode(&expected, "chat%2Ejson/0000000000000000000", false), + [(0, vec!["c0".into(), "c1".into()]), (1, vec!["c2".into()])], + "the resumed open group is appended in place" + ); + assert_eq!( + decode( + &expected, + "video%2F1080p/0000000000000000004-0000000000000000000-0000000000000004000", + true + ), + [ + (4, vec!["v4.0".into(), "v4.1".into()]), + (5, vec!["v5.0".into(), "v5.1".into()]) + ], + "the resumed timed track continues after its last stored group" + ); + + let dir = tempfile::tempdir().unwrap(); + let local = Store::new(LocalFileSystem::new_with_prefix(dir.path()).unwrap(), "rec"); + record(&local).await; + assert_eq!(objects(&local).await, expected, "local disk"); + + let server = Server::new().await; + record(&server.store).await; + assert_eq!(objects(&server.store).await, expected, "S3"); + let requests = server.requests.lock().unwrap(); + assert!( + requests + .iter() + .any(|request| request.method == Method::PUT && request.conditional), + "timed objects are created conditionally" + ); + assert!( + requests + .iter() + .any(|request| request.uri.path().contains("chat%252Ejson")), + "a percent-encoded key survives URL encoding" + ); +} + +struct Request { + method: Method, + uri: Uri, + conditional: bool, +} + +/// An S3 endpoint over a temporary directory, recording each request it serves. +struct Server { + store: Store, + requests: Arc>>, + task: tokio::task::JoinHandle<()>, + _dir: tempfile::TempDir, +} + +impl Server { + async fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("archive")).unwrap(); + let service = S3ServiceBuilder::new(s3s_fs::FileSystem::new(dir.path()).unwrap()).build(); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + let app = axum::Router::new().fallback(move |request: axum::extract::Request| { + let service = service.clone(); + let seen = seen.clone(); + async move { + seen.lock().unwrap().push(Request { + method: request.method().clone(), + uri: request.uri().clone(), + conditional: request.headers().get("if-none-match").is_some_and(|value| value == "*"), + }); + service + .call(request.map(s3s::Body::http_body_unsync)) + .await + .unwrap() + .map(Body::new) + } + }); + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let store = AmazonS3Builder::new() + .with_bucket_name("archive") + .with_region("us-east-1") + .with_endpoint(endpoint) + .with_allow_http(true) + .with_skip_signature(true) + .with_conditional_put(S3ConditionalPut::ETagMatch) + .build() + .unwrap(); + Self { + store: Store::new(store, "rec"), + requests, + task, + _dir: dir, + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} diff --git a/rs/moq-archive/src/producer/mod.rs b/rs/moq-archive/src/producer/mod.rs index f962915d306..0081d12318e 100644 --- a/rs/moq-archive/src/producer/mod.rs +++ b/rs/moq-archive/src/producer/mod.rs @@ -26,6 +26,8 @@ mod track; #[cfg(test)] mod tests; +#[cfg(test)] +mod backends; use std::collections::HashMap; use std::sync::Arc; diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index 3c208d2d494..dec1f9bb986 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -677,3 +677,75 @@ async fn a_track_that_ended_unseen_leaves_the_broadcast_open() { assert_eq!(end(&store, "audio").await, None); assert!(!complete(&store).await); } + +#[tokio::test(start_paused = true)] +async fn a_failed_put_fails_the_recording() { + let store = store(); + store.inner().fail_puts("video/0"); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + group(&video, 0, &[(0, "a")]); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + let err = task.await.unwrap().unwrap_err(); + assert!(matches!(err, Error::Store(_)), "{err}"); + assert_eq!(keys(&store).await, ["video/.info"]); +} + +#[tokio::test(start_paused = true)] +async fn a_failed_resume_listing_writes_nothing() { + let store = store(); + store.put_info("video", &Info::new(0, Some(1000)).unwrap()).await.unwrap(); + store.inner().fail_lists(); + let broadcast = broadcast::Info::new().produce(); + let _video = broadcast.create_track("video", timed()).unwrap(); + store.inner().take(); + let err = spawn(&store, &broadcast, config()).await.unwrap().unwrap_err(); + assert!(matches!(err, Error::Store(_)), "{err}"); + let ops = store.inner().take(); + assert!(!ops.iter().any(|op| matches!(op, crate::mock::Op::Put(_))), "{ops:?}"); +} + +#[tokio::test(start_paused = true)] +async fn a_failed_prune_fails_the_dvr() { + let store = store(); + store.inner().fail_deletes("video/"); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let dvr = Dvr::new(SECOND, Duration::ZERO); + let task = spawn(&store, &broadcast, config().with_duration(SECOND).with_dvr(dvr)); + subscribed(&video).await; + for sequence in 0..3 { + group(&video, sequence, &[(sequence * 1000, "x")]); + } + settle().await; + sleep(2 * SECOND).await; + let err = task.await.unwrap().unwrap_err(); + assert!(matches!(err, Error::Store(_)), "{err}"); +} + +#[tokio::test(start_paused = true)] +async fn a_stalled_put_holds_only_its_track() { + let store = store(); + store.inner().hold_puts("video/0"); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let audio = broadcast.create_track("audio", timed()).unwrap(); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + subscribed(&video).await; + subscribed(&audio).await; + group(&video, 0, &[(0, "v")]); + group(&audio, 0, &[(0, "a")]); + settle().await; + sleep(SECOND).await; + settle().await; + assert_eq!(objects(&store, "audio", true).await.len(), 1); + assert!(objects(&store, "video", true).await.is_empty()); + + store.inner().release(); + video.finish().unwrap(); + audio.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!(objects(&store, "video", true).await.len(), 1); + assert!(complete(&store).await); +} diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 0abb35dde87..2970c6dec35 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -120,8 +120,6 @@ moq-clock = { workspace = true } moq-hls = { workspace = true, features = ["server"] } # Writes and reads the stats and feedback snapshots. moq-json = { workspace = true } -# Names the MSF catalog track `export archive` records. -moq-msf = { workspace = true } moq-mux = { workspace = true } # Cluster, LAN mesh, and inbound `/.cluster` auth. default-features off so the # CLI's own feature set (iroh, quic, cluster-lan) is what selects them. diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index 0bf4c27bb0f..ebda6f10c2d 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,6 +4,13 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased +### `export archive` records a prefix + +`moq export archive ` records every broadcast announced at or under `--broadcast`, each instance under `//`, until the connection ends, instead of one broadcast at ``. +Point `` at the root you want recordings under, and drop `--catalog-format`, which it now refuses. +`--retention` now works on untimed tracks. +`import archive` does not read the new layout yet; it still replays recordings made by older `moq` versions. + ### `--listen-timeout` bounds admission (#5765) `moq --listen` now finishes auth admission within the `--listen-timeout` handshake deadline (default `10s`, counted from the first packet) instead of a separate fixed ten seconds, so a short timeout now refuses a slow auth server sooner and `0s` waits on it forever. diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 16a4611efd8..a6a556347ca 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -1,20 +1,19 @@ -//! Archive endpoints: record one broadcast into an object store (export), or -//! republish a recording from one (import), through `moq-archive`. +//! Archive endpoints: record every broadcast announced under a prefix into an object store +//! (export), or republish a recording from one (import), through `moq-archive`. -use std::collections::HashSet; +use std::collections::HashMap; +use std::sync::Arc; use std::time::Duration; use anyhow::Context; use hang::moq_net; -use moq_archive::writer::{Control, Retention}; -use moq_mux::catalog::{CatalogFormat, Stream as _}; use object_store::ObjectStore; +use object_store::path::{Path, PathPart}; use url::Url; use crate::moq::notify_ready; -/// How long expired objects outlive the timeline that dropped them, unless -/// `--retention-grace` says otherwise. +/// How long a superseded object outlives its DVR window, unless `--retention-grace` says otherwise. const GRACE: Duration = Duration::from_secs(30); /// `export archive` args. @@ -22,14 +21,15 @@ const GRACE: Duration = Duration::from_secs(30); #[usage(unknown_flags = "error", args_override_self = false)] pub struct ExportArgs { /// Where to record: `file:///dir`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`. + /// Each broadcast records under `/` beneath it. pub store: Url, - /// Keep only this much recent content (a DVR), deleting older records. Unset keeps everything. + /// Keep only this much recent content (a DVR), deleting older objects. Unset keeps everything. #[usage(long)] pub retention: Option, - /// How long expired objects outlive the timeline that dropped them, so readers can finish - /// their downloads. Defaults to 30s; needs `--retention`. + /// How long a deleted object outlives the retention, so readers can finish their downloads. + /// Defaults to 30s; needs `--retention`. #[usage(long)] pub retention_grace: Option, } @@ -38,7 +38,7 @@ pub struct ExportArgs { #[derive(usage_rs::Args, Clone)] #[usage(unknown_flags = "error", args_override_self = false)] pub struct ImportArgs { - /// The recording to replay, as `export archive` wrote it. + /// The timeline recording to replay. pub store: Url, /// Keep following a recording that is still growing, checking for new segments at this @@ -47,59 +47,106 @@ pub struct ImportArgs { pub follow: Option, } -/// Record the broadcast `name` into `args.store` until the broadcast ends. +/// One publisher instance being recorded: its path, and its epoch when its route has one. +type Instance = (moq_net::PathOwned, Option); + +/// Record every broadcast announced at or under `prefix` into `args.store`, each publisher +/// instance under `/`, until the announcements end. /// -/// Reads the broadcast's own catalog: every track gets its own timeline, cut at group boundaries -/// between 2s and 10s, while the catalog and data tracks use a zero minimum and record each group -/// as it finishes. -pub async fn export( - origin: moq_net::origin::Consumer, - name: String, - format: CatalogFormat, - args: ExportArgs, -) -> anyhow::Result<()> { - let retention = match (args.retention, args.retention_grace) { - (Some(window), grace) => Some(Retention::new( +/// A route without an epoch names no instance, so each one it announces records under an epoch +/// minted here: a restarted export never resumes it, and never stitches two instances together. +pub async fn export(origin: moq_net::origin::Consumer, prefix: String, args: ExportArgs) -> anyhow::Result<()> { + let dvr = match (args.retention, args.retention_grace) { + (Some(window), grace) => Some(moq_archive::producer::Dvr::new( window.into_std(), grace.map_or(GRACE, crate::duration::Duration::into_std), )), (None, Some(_)) => anyhow::bail!("`--retention-grace` needs `--retention`"), (None, None) => None, }; - let catalog_track = catalog_track(format)?; - let store = open(&args.store)?; - - let broadcast = origin - .routed_broadcast(&name) - .await - .with_context(|| format!("broadcast `{name}` is unavailable"))?; - let config = moq_archive::writer::Config::default().with_retention(retention); - let writer = moq_archive::Writer::new(store, broadcast.clone(), config) - .await - .with_context(|| format!("failed to start recording into {}", args.store))?; - let control = writer.control(); - control.track(catalog_track, sparse()).await?; - let catalog = moq_mux::catalog::Consumer::<()>::new(&broadcast, format).await?; - - tracing::info!(%name, store = %args.store, "recording"); + let config = moq_archive::producer::Config::default().with_dvr(dvr); + let (store, root) = open_store(&args.store)?; + + // Announcements leave hidden routes out unless asked, and a prefix naming one asks. + let hidden = prefix.split('/').any(|segment| segment.starts_with('.')); + let prefix = moq_net::Path::new(&prefix).to_owned(); + let mut announced = origin.clone().with_hidden(hidden).announced(); + let mut recordings = tokio::task::JoinSet::new(); + // How many recordings of each instance run, so a re-announced one is not recorded twice. + let mut running: HashMap = HashMap::new(); + + tracing::info!(%prefix, store = %args.store, "recording"); notify_ready(); - // The writer decides when the recording is over; the catalog only feeds it tracks. - let run = writer.run(); - tokio::pin!(run); - tokio::select! { - result = &mut run => result.context("recording failed"), - Err(err) = enroll(control.clone(), catalog) => { - // Store what already arrived before failing, so the recording stays readable. - control.stop()?; - if let Err(stopped) = run.await { - tracing::warn!(%stopped, "recording failed while stopping"); + loop { + tokio::select! { + event = announced.next() => { + let (announce, restart) = match event.context("announcements ended")? { + moq_net::announce::Event::Start(announce) => (announce, false), + // Another publisher instance: an epochless one is still a new recording. + moq_net::announce::Event::Restart(announce) => (announce, true), + _ => continue, + }; + if !announce.prefix.has_prefix(&prefix) { + continue; + } + let instance: Instance = (announce.prefix, announce.route.epoch); + let count = running.entry(instance.clone()).or_default(); + if *count > 0 && !(restart && instance.1.is_none()) { + continue; + } + *count += 1; + let key = key(&root, &instance)?; + let store = moq_archive::Store::new(store.clone(), key); + recordings.spawn(record(origin.clone(), instance, store, config.clone())); + } + Some(done) = recordings.join_next() => { + let (instance, result) = done.context("recording panicked")?; + result?; + if let Some(count) = running.get_mut(&instance) { + *count -= 1; + if *count == 0 { + running.remove(&instance); + } + } } - Err(err) } } } +/// Record one publisher instance until its broadcast ends. +async fn record( + origin: moq_net::origin::Consumer, + instance: Instance, + store: moq_archive::Store>, + config: moq_archive::producer::Config, +) -> (Instance, anyhow::Result<()>) { + let result = async { + let (path, epoch) = &instance; + let broadcast = origin + .request_broadcast(path, epoch.clone()) + .await + .with_context(|| format!("broadcast `{path}` is unavailable"))?; + tracing::info!(%path, key = %store.prefix(), "recording broadcast"); + moq_archive::Producer::new(store, broadcast, config) + .run() + .await + .with_context(|| format!("recording `{path}` failed")) + } + .await; + (instance, result) +} + +/// The key prefix one instance records under: its path's segments, then its epoch. +fn key(root: &Path, (path, epoch): &Instance) -> anyhow::Result { + let epoch = epoch.clone().unwrap_or_else(moq_net::Epoch::mint); + let mut key = root.clone(); + for segment in path.as_str().split('/').filter(|segment| !segment.is_empty()) { + key = key.join(PathPart::from(segment)); + } + Ok(key.join(PathPart::parse(epoch.as_str()).context("an epoch is one key segment")?)) +} + /// Republish the recording at `args.store` as the broadcast `name`, announced under `epoch`. /// /// Each track's timeline replays as a live track and every other track's groups are served on @@ -156,373 +203,134 @@ pub async fn import( } /// Open the object store `url` names, taking cloud credentials from the `AWS_*`, `GOOGLE_*`, -/// and `AZURE_*` environment variables. -fn open(url: &Url) -> anyhow::Result>> { +/// and `AZURE_*` environment variables, with the prefix the URL names. +fn open_store(url: &Url) -> anyhow::Result<(Arc, Path)> { let credentials = std::env::vars_os() .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))) .filter(|(key, _)| ["AWS_", "GOOGLE_", "AZURE_"].iter().any(|cloud| key.starts_with(cloud))); let (store, prefix) = object_store::parse_url_opts(url, credentials).with_context(|| format!("unsupported archive store {url}"))?; - Ok(moq_archive::Store::new(store, prefix)) -} - -/// How sparse data, such as a catalog, is cut: every group is its own record. -fn sparse() -> moq_mux::timeline::Config { - moq_mux::timeline::Config::default().with_duration_min(Duration::ZERO) -} - -/// The track carrying the catalog `format` reads. -fn catalog_track(format: CatalogFormat) -> anyhow::Result<&'static str> { - Ok(match format { - CatalogFormat::Hang => hang::Catalog::DEFAULT_NAME, - CatalogFormat::HangZ => hang::Catalog::COMPRESSED_NAME, - CatalogFormat::Msf => moq_msf::DEFAULT_NAME, - other => anyhow::bail!("`export archive` cannot record a {other:?} catalog"), - }) -} - -/// Enroll each track as the catalog lists it, until the catalog ends. -async fn enroll( - control: Control, - mut catalog: moq_mux::catalog::Consumer<()>, -) -> anyhow::Result<()> { - let mut tracks = Tracks::default(); - while let Some(snapshot) = catalog.next().await? { - for change in tracks.update(&snapshot)? { - match change { - Change::Media(name) => control.track(&name, moq_mux::timeline::Config::default()).await?, - Change::Sparse(name) => control.track(&name, sparse()).await?, - Change::Remove(name) => control.remove(&name)?, - } - } - } - Ok(()) -} - -/// A change to the recorded track set. -#[derive(Debug, PartialEq, Eq)] -enum Change { - /// Record a rendition, its records cut by duration. - Media(String), - /// Record a data track, each group its own record. - Sparse(String), - /// Stop recording a track the catalog dropped. - Remove(String), -} - -/// The catalog tracks being recorded, diffed against each new snapshot. -#[derive(Default)] -struct Tracks { - /// Every name ever enrolled. The writer never takes a name back. - enrolled: HashSet, - /// The names the latest snapshot listed. - listed: HashSet, + Ok((Arc::from(store), prefix)) } -impl Tracks { - /// The changes that bring the recording in line with `catalog`. - /// - /// Refuses a track served from another broadcast, since an archive holds exactly one, and a - /// track that returns after the catalog dropped it, since the writer cannot resume one. - fn update(&mut self, catalog: &hang::Catalog) -> anyhow::Result> { - use hang::catalog::Reference; - - // Recorded by track name, which is what the archive's timelines are keyed by. - fn entry<'a, C: Reference>( - (id, config): (&'a String, &'a C), - media: bool, - ) -> (&'a str, Option<&'a moq_net::path::RelativeOwned>, bool) { - (config.track(id), config.broadcast(), media) - } - let video = catalog.video.renditions.iter().map(|e| entry(e, true)); - let audio = catalog.audio.renditions.iter().map(|e| entry(e, true)); - let text = catalog.text.renditions.iter().map(|e| entry(e, false)); - let json = catalog.json.tracks.iter().map(|e| entry(e, false)); - let binary = catalog.binary.tracks.iter().map(|e| entry(e, false)); - - let mut listed = HashSet::new(); - let mut changes = Vec::new(); - for (name, broadcast, media) in video.chain(audio).chain(text).chain(json).chain(binary) { - if let Some(broadcast) = broadcast { - anyhow::bail!( - "track `{name}` is served from broadcast `{broadcast}`; an archive records one broadcast" - ); - } - if !listed.insert(name.to_string()) || self.listed.contains(name) { - continue; - } - anyhow::ensure!( - self.enrolled.insert(name.to_string()), - "track `{name}` returned to the catalog after it was dropped; the recording cannot resume it" - ); - changes.push(match media { - true => Change::Media(name.to_string()), - false => Change::Sparse(name.to_string()), - }); - } - - let mut dropped: Vec<_> = self.listed.difference(&listed).cloned().collect(); - dropped.sort(); - changes.extend(dropped.into_iter().map(Change::Remove)); - self.listed = listed; - Ok(changes) - } +/// [`open_store`] as one recording. +fn open(url: &Url) -> anyhow::Result>> { + let (store, prefix) = open_store(url)?; + Ok(moq_archive::Store::new(store, prefix)) } #[cfg(test)] mod tests { use super::*; - use hang::catalog::{AudioCodec, AudioConfig, JsonConfig, Mode}; use moq_net::{Timescale, Timestamp, group, track}; - fn audio() -> AudioConfig { - AudioConfig::new(AudioCodec::Opus, 48_000, 2) - } - - /// Renditions are media, data tracks are sparse, and a later snapshot only adds what is new. - #[test] - fn renditions_are_media_and_the_rest_sparse() { - let mut tracks = Tracks::default(); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert("audio".into(), audio()); - catalog - .json - .tracks - .insert("chat".into(), JsonConfig::new(Mode::Snapshot)); - - assert_eq!( - tracks.update(&catalog).unwrap(), - [Change::Media("audio".into()), Change::Sparse("chat".into())] - ); - - catalog.audio.renditions.insert("audio2".into(), audio()); - assert_eq!(tracks.update(&catalog).unwrap(), [Change::Media("audio2".into())]); - assert_eq!(tracks.update(&catalog).unwrap(), []); - } - - /// A rendition keyed apart from its track records the track. - #[test] - fn renditions_record_their_track() { - let mut rendition = audio(); - rendition.track = Some("audio".into()); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert("main".into(), rendition); - - assert_eq!( - Tracks::default().update(&catalog).unwrap(), - [Change::Media("audio".into())] - ); - } - - /// A dropped rendition stops recording, and coming back is refused rather than silently lost. - #[test] - fn a_dropped_track_is_removed_for_good() { - let mut tracks = Tracks::default(); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert("audio".into(), audio()); - tracks.update(&catalog).unwrap(); - - let removed = catalog.audio.renditions.remove("audio").unwrap(); - assert_eq!(tracks.update(&catalog).unwrap(), [Change::Remove("audio".into())]); - - catalog.audio.renditions.insert("audio".into(), removed); - let err = tracks.update(&catalog).unwrap_err().to_string(); - assert!(err.contains("cannot resume"), "{err}"); - } - - /// A rendition living in a sibling broadcast would be missing from the recording. - #[test] - fn another_broadcast_is_refused() { - let mut rendition = audio(); - rendition.broadcast = Some(moq_net::path::RelativeOwned::new("./source")); - let mut catalog = hang::Catalog::default(); - catalog.audio.renditions.insert("audio".into(), rendition); - - let err = Tracks::default().update(&catalog).unwrap_err().to_string(); - assert!(err.contains("broadcast `source`"), "{err}"); - } - - /// Wait until the export's subscription has reached `track`. - /// - /// `.info` is written after that, while the enroll command is still being queued. - /// Finishing on the file races that command and the recording ends with "writer closed". - async fn subscribed(track: &mut track::Producer, recording: &mut tokio::task::JoinHandle>) { - tokio::select! { - subscription = track.subscription_changed() => { - let subscription = subscription.expect("the rendition stays open until the export subscribes"); - assert!(subscription.is_some(), "the export subscribed"); + /// Every file under `dir`, relative to it. + fn files(dir: &std::path::Path) -> Vec { + let mut files = Vec::new(); + let mut pending = vec![dir.to_path_buf()]; + while let Some(next) = pending.pop() { + for entry in std::fs::read_dir(next).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + pending.push(path); + } else { + files.push(path.strip_prefix(dir).unwrap().to_string_lossy().into_owned()); + } } - result = &mut *recording => panic!("the export ended before it subscribed: {result:?}"), } + files.sort(); + files } - /// What `export archive` records, `import archive` serves back group for group, and the - /// export ends cleanly with its broadcast. - #[tokio::test] - async fn a_recording_replays_its_groups() { - replays_its_groups(Some(Timescale::MILLI)).await; - } - - /// An untimed track, as every track is over moq-transport without TIMESCALE, records and - /// replays untimed. + /// `export archive` records each broadcast announced under its prefix into its own + /// `/`, and leaves out what is announced elsewhere. #[tokio::test] - async fn an_untimed_recording_replays_untimed() { - replays_its_groups(None).await; - } - - async fn replays_its_groups(timescale: Option) { - // `open` reads the credential variables. + async fn each_broadcast_records_under_its_own_key_prefix() { + // `open_store` reads the credential variables. let _env = crate::test_env::EnvGuard::clear(&[]); let dir = tempfile::tempdir().unwrap(); let url = Url::from_directory_path(dir.path()).unwrap(); let origin = moq_tokio::origin::spawn(); - let mut broadcast = origin.create_broadcast("live.hang").unwrap(); - let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); - let info = track::Info::default() - .with_timescale(timescale) - .with_max_age(Duration::from_secs(3600)); - let mut track = broadcast.create_track("audio", info).unwrap(); - catalog - .mutate(|catalog| { - catalog.audio.renditions.insert("audio".into(), audio()); - }) - .unwrap(); - broadcast.announce(Default::default()).unwrap(); - let args = ExportArgs { - store: url.clone(), + store: url, retention: None, retention_grace: None, }; - let mut recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); - - for sequence in 0..3 { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for offset in [0, 500] { - let timestamp = timescale.map(|_| Timestamp::from_millis(sequence * 1000 + offset).unwrap()); - group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); - } - group.finish().unwrap(); - } - - subscribed(&mut track, &mut recording).await; - track.finish().unwrap(); - catalog.finish().unwrap(); - broadcast.close(); - - tokio::time::timeout(Duration::from_secs(10), recording) - .await - .expect("the export ends with its broadcast") - .unwrap() - .expect("the recording succeeds"); - - let replay = moq_tokio::origin::spawn(); - let args = ImportArgs { - store: url, - follow: None, - }; - let serving = tokio::spawn(import( - replay.clone(), - "replay.hang".into(), - args, - moq_net::Epoch::mint(), - )); + let recording = tokio::spawn(export(origin.consume(), "live".into(), args)); - let consumer = replay.consume().routed_broadcast("replay.hang").await.unwrap(); - let audio = consumer.track("audio").unwrap(); - for sequence in 0..3 { - let mut group = audio.fetch_group(sequence, None).await.unwrap(); - for offset in [0, 500] { - let frame = group.read_frame().await.unwrap().expect("a frame"); - assert_eq!( - frame.timestamp.map(|timestamp| timestamp.as_millis()), - timescale.map(|_| u128::from(sequence * 1000 + offset)) - ); - assert_eq!(frame.payload, format!("{sequence}+{offset}")); - } - assert!(group.read_frame().await.unwrap().is_none()); - } - - serving.abort(); - } - - /// Renditions already live in the opening catalog are all recorded from their first group. - /// - /// Groups are published before export starts, so the writer can drain the first rendition - /// while the next subscription is still in flight. - #[tokio::test] - async fn an_opening_snapshot_records_every_rendition() { - let _env = crate::test_env::EnvGuard::clear(&[]); - let dir = tempfile::tempdir().unwrap(); - let url = Url::from_directory_path(dir.path()).unwrap(); - - let origin = moq_tokio::origin::spawn(); - let mut broadcast = origin.create_broadcast("live.hang").unwrap(); - let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); let info = track::Info::default() .with_timescale(Timescale::MILLI) .with_max_age(Duration::from_secs(3600)); - let mut first = broadcast.create_track("audio", info.clone()).unwrap(); - let mut second = broadcast.create_track("audio2", info).unwrap(); - for track in [&first, &second] { - for sequence in 0..3 { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - for offset in [0, 500] { - let timestamp = Timestamp::from_millis(sequence * 1000 + offset).unwrap(); - group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); - } - group.finish().unwrap(); - } + let mut epochs = Vec::new(); + let mut broadcasts = Vec::new(); + for path in ["live/a", "live/b", "other"] { + let broadcast = origin.create_broadcast(path).unwrap(); + let track = broadcast.create_track("video", info.clone()).unwrap(); + let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(Timestamp::from_millis(0).unwrap(), path).unwrap(); + group.finish().unwrap(); + let epoch = moq_net::Epoch::mint(); + broadcast + .announce(moq_net::origin::Route::default().with_epoch(epoch.clone())) + .unwrap(); + epochs.push(epoch); + broadcasts.push((broadcast, track)); } - catalog - .mutate(|catalog| { - catalog.audio.renditions.insert("audio".into(), audio()); - catalog.audio.renditions.insert("audio2".into(), audio()); - }) - .unwrap(); - broadcast.announce(Default::default()).unwrap(); - let args = ExportArgs { - store: url.clone(), - retention: None, - retention_grace: None, - }; - let mut recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); - - subscribed(&mut first, &mut recording).await; - subscribed(&mut second, &mut recording).await; - first.finish().unwrap(); - second.finish().unwrap(); - catalog.finish().unwrap(); - broadcast.close(); - - tokio::time::timeout(Duration::from_secs(10), recording) - .await - .expect("the export ends with its broadcast") - .unwrap() - .expect("the recording succeeds"); - - // Each rendition's own timeline opens at its first group. - let store = super::open(&url).unwrap(); - for track in ["audio", "audio2"] { - let object = store - .get_segments(&hang::timeline::default_name(track), 0, true) + // Each recording ends with its broadcast, once it subscribed. + for (broadcast, mut track) in broadcasts.drain(..2) { + tokio::time::timeout(Duration::from_secs(10), track.subscription_changed()) .await - .expect("record 0"); - let config = moq_json::window::ConsumerConfig::default().with_compression(true); - let mut decoder = moq_json::window::Decoder::::new(config); - for stored in object.groups { - let mut group = decoder.group(); - for frame in stored.frames { - group.decode(&frame.payload).unwrap(); - } + .expect("the export subscribes") + .unwrap(); + track.finish().unwrap(); + broadcast.close(); + } + let complete = [ + format!("live/a/{}/.end", epochs[0]), + format!("live/b/{}/.end", epochs[1]), + ]; + tokio::time::timeout(Duration::from_secs(10), async { + while !complete.iter().all(|end| dir.path().join(end).exists()) { + tokio::time::sleep(Duration::from_millis(10)).await; } - let Some(moq_json::window::Event::Push { value, .. }) = decoder.next_event() else { - panic!("{track} record 0 is pushed first"); - }; - assert_eq!(value.start, hang::timeline::Position::group(0), "{track}"); + }) + .await + .expect("both recordings complete"); + recording.abort(); + + let files = files(dir.path()); + for (path, epoch) in [("live/a", &epochs[0]), ("live/b", &epochs[1])] { + for file in [".end", "video/.info", "video/end"] { + assert!(files.contains(&format!("{path}/{epoch}/{file}")), "{files:?}"); + } + assert!( + files.contains(&format!( + "{path}/{epoch}/video/0000000000000000000-0000000000000000000-0000000000000000000" + )), + "{files:?}" + ); } + assert!(!files.iter().any(|file| file.starts_with("other")), "{files:?}"); + } + + /// A route without an epoch records under one minted per instance, so two instances never + /// share a key prefix. + #[test] + fn an_epochless_instance_gets_a_fresh_key() { + let instance = (moq_net::Path::new("live/a").to_owned(), None); + let first = key(&Path::from("rec"), &instance).unwrap(); + let second = key(&Path::from("rec"), &instance).unwrap(); + assert_ne!(first, second); + assert!(first.as_ref().starts_with("rec/live/a/"), "{first}"); + + let epoch = moq_net::Epoch::mint(); + let pinned = (moq_net::Path::new("live/a").to_owned(), Some(epoch.clone())); + assert_eq!( + key(&Path::from("rec"), &pinned).unwrap().as_ref(), + format!("rec/live/a/{epoch}") + ); } } diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index f64ac22a7b9..29ffd380fbf 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -335,6 +335,12 @@ impl Invocation { "`export {sink}` reads no media to report on; remove --echo" ); } + if matches!(export.sink, ExportSink::Archive(_)) { + anyhow::ensure!( + export.catalog_format.is_none(), + "`export archive` records every track without reading a catalog, so it takes no --catalog-format" + ); + } if export.echo.is_some() { if export.sink.stdout().is_none() && let Some(sink) = export.sink.ignores_selection() @@ -1001,7 +1007,7 @@ pub enum ExportSink { Srt(crate::srt::ExportArgs), /// WebRTC: WHIP client pushing to a remote (`--connect`) or WHEP server serving plays (`--listen`). Rtc(crate::rtc::Args), - /// Record the broadcast into an object store until it ends. + /// Record every broadcast announced under `--broadcast` into an object store. Archive(crate::archive::ExportArgs), /// Write the broadcast's stats snapshots to stdout as JSON lines. Stats, diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 202341d706d..93313e11927 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -880,12 +880,7 @@ fn spawn_export( } } ExportSink::Archive(args) => { - let format = export - .catalog_format - .map(Into::into) - .or_else(|| moq_mux::catalog::CatalogFormat::detect(&name)) - .unwrap_or_default(); - tasks.spawn(archive::export(origin.consume(), name, format, args)); + tasks.spawn(archive::export(origin.consume(), name, args)); } ExportSink::Stats => { tasks.spawn(report::print_stats(origin.consume(), name)); From d7da5f10e7107efb7dd42dc30b9a686807a6599f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 22:45:32 -0700 Subject: [PATCH 04/16] chore(archive): drop the unused moq-mux dependency and format Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 - rs/moq-archive/Cargo.toml | 1 - rs/moq-archive/src/info.rs | 1 - rs/moq-archive/src/layout.rs | 18 ++++++-- rs/moq-archive/src/producer/backends.rs | 12 +++++- rs/moq-archive/src/producer/mod.rs | 4 +- rs/moq-archive/src/producer/resume.rs | 2 +- rs/moq-archive/src/producer/tests.rs | 56 ++++++++++++++++++++----- rs/moq-archive/src/producer/track.rs | 2 +- 9 files changed, 74 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8198915dbe..584ede7c50d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4538,7 +4538,6 @@ dependencies = [ "kio 0.6.2", "moq-clock", "moq-json", - "moq-mux", "moq-net", "object_store", "percent-encoding", diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index 568c480915f..a0b47661cbc 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -19,7 +19,6 @@ hang = { workspace = true } kio = { workspace = true } moq-clock = { workspace = true } moq-json = { workspace = true } -moq-mux = { workspace = true } moq-net = { workspace = true } object_store = { workspace = true } percent-encoding = { workspace = true } diff --git a/rs/moq-archive/src/info.rs b/rs/moq-archive/src/info.rs index 698fbdc8846..f26218a7659 100644 --- a/rs/moq-archive/src/info.rs +++ b/rs/moq-archive/src/info.rs @@ -57,7 +57,6 @@ impl Info { } } - #[cfg(test)] mod tests { use super::*; diff --git a/rs/moq-archive/src/layout.rs b/rs/moq-archive/src/layout.rs index ea639128338..7392dd8cc2b 100644 --- a/rs/moq-archive/src/layout.rs +++ b/rs/moq-archive/src/layout.rs @@ -101,7 +101,12 @@ fn parse_id(field: &str) -> Option> { if field.len() != ID_WIDTH || !field.bytes().all(|b| b.is_ascii_digit()) { return None; } - Some(field.parse().map_err(|_| Error::Path(field.to_string())).and_then(check_id)) + Some( + field + .parse() + .map_err(|_| Error::Path(field.to_string())) + .and_then(check_id), + ) } /// How a track's last run ended, stored in its `end` once the broadcast ended. @@ -420,7 +425,11 @@ mod tests { sorted.sort(); assert_eq!(names, sorted); assert_eq!(names[1], "0000000000000000000-0000000000000000003-0000000000000000002"); - assert!(names.iter().all(|name| name.as_str() < TRACK_END && name.as_str() > TRACK_INFO)); + assert!( + names + .iter() + .all(|name| name.as_str() < TRACK_END && name.as_str() > TRACK_INFO) + ); } #[test] @@ -464,7 +473,10 @@ mod tests { #[test] fn end_markers_are_json() { let finish = End::Finish(At { group: 5, frame: 3 }); - assert_eq!(finish.encode().unwrap().as_ref(), br#"{"finish":{"group":5,"frame":3}}"#); + assert_eq!( + finish.encode().unwrap().as_ref(), + br#"{"finish":{"group":5,"frame":3}}"# + ); assert_eq!(End::decode(&finish.encode().unwrap()).unwrap(), finish); let error = End::Error(0x34); assert_eq!(error.encode().unwrap().as_ref(), br#"{"error":52}"#); diff --git a/rs/moq-archive/src/producer/backends.rs b/rs/moq-archive/src/producer/backends.rs index df549ae9ad0..f8203c875cc 100644 --- a/rs/moq-archive/src/producer/backends.rs +++ b/rs/moq-archive/src/producer/backends.rs @@ -83,8 +83,16 @@ async fn record(store: &Store) { stored( store, &[ - path(store, VIDEO, "0000000000000000000-0000000000000000000-0000000000000000000"), - path(store, VIDEO, "0000000000000000002-0000000000000000000-0000000000000002000"), + path( + store, + VIDEO, + "0000000000000000000-0000000000000000000-0000000000000000000", + ), + path( + store, + VIDEO, + "0000000000000000002-0000000000000000000-0000000000000002000", + ), path(store, CHAT, "0000000000000000000"), ], ) diff --git a/rs/moq-archive/src/producer/mod.rs b/rs/moq-archive/src/producer/mod.rs index 0081d12318e..571cf592fb8 100644 --- a/rs/moq-archive/src/producer/mod.rs +++ b/rs/moq-archive/src/producer/mod.rs @@ -24,10 +24,10 @@ mod resume; mod track; -#[cfg(test)] -mod tests; #[cfg(test)] mod backends; +#[cfg(test)] +mod tests; use std::collections::HashMap; use std::sync::Arc; diff --git a/rs/moq-archive/src/producer/resume.rs b/rs/moq-archive/src/producer/resume.rs index f7509ecf404..fac29f7f4a0 100644 --- a/rs/moq-archive/src/producer/resume.rs +++ b/rs/moq-archive/src/producer/resume.rs @@ -27,7 +27,7 @@ pub(super) async fn load(store: &Store) -> Result return Ok(None), _ => return Err(Error::Path(object.location.to_string())), diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index dec1f9bb986..b0cde741b44 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -9,8 +9,8 @@ use tokio::task::JoinHandle; use tokio::time::sleep; use super::*; -use crate::layout::{At, Entry, Object}; use crate::Info; +use crate::layout::{At, Entry, Object}; use crate::mock::Mock; const SECOND: Duration = Duration::from_secs(1); @@ -201,12 +201,18 @@ async fn a_lost_group_is_a_gap_that_never_stalls_later_groups() { settle().await; // Group 0 is flushed alone; 2 and 3 wait on the missing 1. sleep(SECOND + SECOND / 2).await; - assert_eq!(objects(&store, "video", true).await.keys().cloned().collect::>(), [key(0, 0, 0)]); + assert_eq!( + objects(&store, "video", true).await.keys().cloned().collect::>(), + [key(0, 0, 0)] + ); // Once the max delay passes, the gap is given up and the rest follow within the flush. sleep(2 * SECOND).await; let stored = objects(&store, "video", true).await; - assert_eq!(stored.keys().cloned().collect::>(), [key(0, 0, 0), key(2, 0, 2000)]); + assert_eq!( + stored.keys().cloned().collect::>(), + [key(0, 0, 0), key(2, 0, 2000)] + ); assert_eq!( contents(&stored)[1], [ @@ -231,7 +237,15 @@ async fn an_oversized_timed_group_splits_into_ordered_unique_keys() { let broadcast = broadcast::Info::new().produce(); let video = broadcast.create_track("video", timed()).unwrap(); // Timestamps repeat and go backward, as decode order allows. - let frames = [(0, "0"), (1000, "1"), (2000, "2"), (2000, "3"), (1500, "4"), (4000, "5"), (4000, "6")]; + let frames = [ + (0, "0"), + (1000, "1"), + (2000, "2"), + (2000, "3"), + (1500, "4"), + (4000, "5"), + (4000, "6"), + ]; group(&video, 0, &frames); let task = spawn(&store, &broadcast, config()); subscribed(&video).await; @@ -267,14 +281,20 @@ async fn an_open_untimed_group_is_rewritten_in_place_and_cut_by_wall_clock() { open.write_frame(None, "a").unwrap(); settle().await; sleep(SECOND).await; - assert_eq!(contents(&objects(&store, "data", false).await), [[(0, "group", vec!["a".to_string()])]]); + assert_eq!( + contents(&objects(&store, "data", false).await), + [[(0, "group", vec!["a".to_string()])]] + ); open.write_frame(None, "b").unwrap(); settle().await; sleep(SECOND).await; let stored = objects(&store, "data", false).await; assert_eq!(stored.keys().cloned().collect::>(), ["0000000000000000000"]); - assert_eq!(contents(&stored), [[(0, "group", vec!["a".to_string(), "b".to_string()])]]); + assert_eq!( + contents(&stored), + [[(0, "group", vec!["a".to_string(), "b".to_string()])]] + ); open.finish().unwrap(); // Past the duration, the next group starts the next object. @@ -309,7 +329,10 @@ async fn an_idle_track_is_flushed_within_the_bound() { sleep(SECOND / 2).await; assert!(objects(&store, "video", true).await.is_empty()); sleep(SECOND / 2).await; - assert_eq!(objects(&store, "video", true).await.keys().cloned().collect::>(), [key(0, 0, 0)]); + assert_eq!( + objects(&store, "video", true).await.keys().cloned().collect::>(), + [key(0, 0, 0)] + ); // The open group continues in the next object. open.write_frame(ms(100), "b").unwrap(); @@ -362,7 +385,10 @@ async fn a_restart_mid_group_loses_and_duplicates_nothing() { finished(task).await; let stored = objects(&store, "video", true).await; - assert_eq!(stored.keys().cloned().collect::>(), [key(0, 0, 0), key(0, 2, 200)]); + assert_eq!( + stored.keys().cloned().collect::>(), + [key(0, 0, 0), key(0, 2, 200)] + ); let frames: Vec = contents(&stored) .into_iter() .flatten() @@ -374,7 +400,11 @@ async fn a_restart_mid_group_loses_and_duplicates_nothing() { let stored = objects(&store, "data", false).await; assert_eq!( contents(&stored), - [[(0, "group", vec!["a", "b", "c", "d"].into_iter().map(String::from).collect())]] + [[( + 0, + "group", + vec!["a", "b", "c", "d"].into_iter().map(String::from).collect() + )]] ); assert!(complete(&store).await); } @@ -473,7 +503,8 @@ async fn a_dvr_prunes_superseded_objects_after_its_window_and_grace() { sleep(SECOND).await; } // Object 0 was superseded when object 1 was stored, at 2s: it stays through 5s. - let names = |objects: BTreeMap| objects.values().map(|o| o.start().unwrap().group).collect::>(); + let names = + |objects: BTreeMap| objects.values().map(|o| o.start().unwrap().group).collect::>(); assert_eq!(names(objects(&store, "video", true).await), [0, 1, 2]); sleep(SECOND).await; settle().await; @@ -694,7 +725,10 @@ async fn a_failed_put_fails_the_recording() { #[tokio::test(start_paused = true)] async fn a_failed_resume_listing_writes_nothing() { let store = store(); - store.put_info("video", &Info::new(0, Some(1000)).unwrap()).await.unwrap(); + store + .put_info("video", &Info::new(0, Some(1000)).unwrap()) + .await + .unwrap(); store.inner().fail_lists(); let broadcast = broadcast::Info::new().produce(); let _video = broadcast.create_track("video", timed()).unwrap(); diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index d7feb2aa095..49e8be09a97 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -13,8 +13,8 @@ use moq_net::{Datagram, StreamError, Timescale, group}; use object_store::path::Path; use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutPayload}; -use super::resume::Stored; use super::Shared; +use super::resume::Stored; use crate::layout::{self, End, Entry, Object}; use crate::segment::Frame; use crate::{Error, Info, Result}; From d6060884ce8c8cddcfd6bfae96e7f92080d74bca Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 22:52:48 -0700 Subject: [PATCH 05/16] fix(archive): box group consumers and keep producer tests on the ambient clock Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/producer/backends.rs | 8 ++++---- rs/moq-archive/src/producer/tests.rs | 4 ++-- rs/moq-archive/src/producer/track.rs | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/rs/moq-archive/src/producer/backends.rs b/rs/moq-archive/src/producer/backends.rs index f8203c875cc..704b32654d1 100644 --- a/rs/moq-archive/src/producer/backends.rs +++ b/rs/moq-archive/src/producer/backends.rs @@ -64,10 +64,10 @@ fn path(store: &Store, track: &str, name: &str) -> Path { /// Wait until every one of `paths` is stored. async fn stored(store: &Store, paths: &[Path]) { - tokio::time::timeout(Duration::from_secs(10), async { + moq_clock::timeout(Duration::from_secs(10), async { for path in paths { while store.inner().head(path).await.is_err() { - tokio::time::sleep(Duration::from_millis(10)).await; + moq_clock::sleep(Duration::from_millis(10)).await; } } }) @@ -104,12 +104,12 @@ async fn record(store: &Store) { let run = tokio::spawn(Producer::new(store.clone(), second.consume(), config()).run()); for track in &tracks { while track.subscription().is_none() { - tokio::time::sleep(Duration::from_millis(1)).await; + moq_clock::sleep(Duration::from_millis(1)).await; } track.finish().unwrap(); } second.close(); - tokio::time::timeout(Duration::from_secs(10), run) + moq_clock::timeout(Duration::from_secs(10), run) .await .expect("the producer ends with its broadcast") .unwrap() diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index b0cde741b44..bb7393e31af 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -6,7 +6,7 @@ use moq_net::track::Position; use moq_net::{Timescale, Timestamp, broadcast, group, track}; use object_store::ObjectStoreExt; use tokio::task::JoinHandle; -use tokio::time::sleep; +use moq_clock::sleep; use super::*; use crate::Info; @@ -63,7 +63,7 @@ async fn subscribed(track: &track::Producer) { } async fn finished(task: JoinHandle>) { - tokio::time::timeout(Duration::from_secs(1000), task) + moq_clock::timeout(Duration::from_secs(1000), task) .await .expect("the producer ends with its broadcast") .unwrap() diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index 49e8be09a97..0edd6203bf7 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -79,7 +79,7 @@ struct Put { /// A sequence received ahead of the one being stored. enum Pending { - Group(group::Consumer, Instant), + Group(Box, Instant), Datagram(Datagram, Instant), } @@ -97,7 +97,7 @@ struct Run { /// Received sequences not stored yet, waiting on a lower one. pending: BTreeMap, /// The group being stored, frame by frame. - current: Option, + current: Option>, /// When the gap below the lowest pending sequence is given up. gap: Option, groups_done: bool, @@ -106,7 +106,8 @@ struct Run { enum Event { Frame(u64, moq_net::Result>), - Group(moq_net::Result>), + // Boxed, since a group consumer dwarfs every other event and waits in `Run::pending`. + Group(moq_net::Result>>), Datagram(moq_net::Result>), Timer, } @@ -268,7 +269,7 @@ impl Track { if !run.groups_done && let Poll::Ready(group) = subscriber.poll_recv_group(waiter) { - return Poll::Ready(Event::Group(group)); + return Poll::Ready(Event::Group(group.map(|group| group.map(Box::new)))); } if !run.datagrams_done && let Poll::Ready(datagram) = subscriber.poll_recv_datagram(waiter) From 7b3715b0d26423175066e1c58d3e5fa94d322396 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 22:53:35 -0700 Subject: [PATCH 06/16] style(archive): sort imports Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/producer/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index bb7393e31af..d60f0f328e1 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -2,11 +2,11 @@ use std::collections::BTreeMap; use bytes::Bytes; use futures::TryStreamExt; +use moq_clock::sleep; use moq_net::track::Position; use moq_net::{Timescale, Timestamp, broadcast, group, track}; use object_store::ObjectStoreExt; use tokio::task::JoinHandle; -use moq_clock::sleep; use super::*; use crate::Info; From bbddb043e90b6f85d6532a7120e653cc88bab1ab Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 20 Oct 2026 23:07:39 -0700 Subject: [PATCH 07/16] docs(archive): link the MIGRATE entries to #6160 Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/MIGRATE.md | 2 +- rs/moq-cli/MIGRATE.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-archive/MIGRATE.md b/rs/moq-archive/MIGRATE.md index df376a78891..d51cc4d4ca5 100644 --- a/rs/moq-archive/MIGRATE.md +++ b/rs/moq-archive/MIGRATE.md @@ -4,7 +4,7 @@ How to update your code for each breaking change, newest first. ## Unreleased -### `Producer` replaces `Writer` +### `Producer` replaces `Writer` (#6160) `Writer`, its `Control`, and `writer::{Config, Retention}` are removed; record with `Producer::new(store, broadcast, producer::Config::default()).run().await` instead. A producer records every track of one broadcast, found with TRACKS rather than a catalog, and needs no timeline: the layout is per-track objects keyed by group, frame, and timestamp, documented in the crate docs. diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index c344b86bfbd..f485e1919e3 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -4,7 +4,7 @@ How to update your invocation and config for each breaking change, newest first. ## Unreleased -### `export archive` records a prefix +### `export archive` records a prefix (#6160) `moq export archive ` records every broadcast announced at or under `--broadcast`, each instance under `//`, until the connection ends, instead of one broadcast at ``. Point `` at the root you want recordings under, and drop `--catalog-format`, which it now refuses. From 6c48d1764712d97ab621d812567005c283baece5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 11:17:39 -0700 Subject: [PATCH 08/16] fix(archive): record partial groups and covering routes A group whose head is held elsewhere starts past frame 0. The producer read it from frame 0, lagged, and stored nothing; now it starts at the group's first frame and opens its own object, since only an object's first entry carries a frame start. `export archive` now requests the selected path when a route covers it without announcing what is beneath. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 4 +- rs/moq-archive/src/producer/tests.rs | 43 +++++++++++++++++++ rs/moq-archive/src/producer/track.rs | 39 +++++++++-------- rs/moq-cli/src/archive.rs | 62 ++++++++++++++++++++++++++-- 4 files changed, 125 insertions(+), 23 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index dfcb94815df..510879e2400 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -307,7 +307,9 @@ moq --connect https://relay.example.com/anon --broadcast event-replay.hang impor `export archive` records every broadcast announced at or under `--broadcast` with [moq-archive](https://docs.rs/moq-archive), each publisher instance under -`/` in the store, until the connection ends. Every track is +`/` in the store, until the connection ends. When a route +covers `--broadcast` without listing what is beneath it, `--broadcast` itself is +recorded as the broadcast. Every track is stored as its own objects, cut at group boundaries about every 2 s of media (or wall clock for an [untimed](/concept/moq-lite#subscriptions) track) and flushed at least every 4 s. A broadcast restarted under a new diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index d60f0f328e1..75a9132d821 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -409,6 +409,49 @@ async fn a_restart_mid_group_loses_and_duplicates_nothing() { assert!(complete(&store).await); } +/// A later group that starts past frame 0, its head held elsewhere, opens its own object: an +/// entry after the first has no frame start, so a resume would otherwise rewind into it. +#[tokio::test(start_paused = true)] +async fn a_partial_group_resumes_after_its_last_stored_frame() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let config = config().with_flush(SECOND); + let task = spawn(&store, &broadcast, config.clone()); + subscribed(&video).await; + + group(&video, 0, &[(0, "a")]); + let mut partial = video.create_group(group::Info { sequence: 1 }).unwrap(); + partial.start_at(2).unwrap(); + partial.write_frame(ms(500), "c").unwrap(); + settle().await; + sleep(SECOND).await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + + let task = spawn(&store, &broadcast, config); + settle().await; + partial.write_frame(ms(600), "d").unwrap(); + partial.finish().unwrap(); + settle().await; + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let stored = objects(&store, "video", true).await; + assert_eq!( + stored.keys().cloned().collect::>(), + [key(0, 0, 0), key(1, 2, 500), key(1, 3, 600)] + ); + let frames: Vec = contents(&stored) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + assert_eq!(frames, ["0/a", "500/c", "600/d"]); + assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 2, frame: 0 }))); +} + #[tokio::test(start_paused = true)] async fn changed_properties_fail_the_resume() { let store = store(); diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index 0edd6203bf7..b2879b0b7fc 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -348,13 +348,13 @@ impl Track { self.next = Some(Position::group(sequence + 1)); } Pending::Group(mut group, _) => { - // A resumed group continues after its stored frames. - if let Some(next) = self.next - && next.group == sequence - && group.index() < next.frame - { - group.set_frames(next.frame..); - } + // A resumed group continues after its stored frames. The clamp also starts a group + // whose head is held elsewhere at its first frame, rather than lagging behind it. + let stored = self + .next + .filter(|next| next.group == sequence) + .map_or(0, |next| next.frame); + group.set_frames(stored..); self.next = Some(Position { group: sequence, frame: group.index(), @@ -416,18 +416,21 @@ impl Track { if let Some(building) = &self.object { let last = building.object.entries.last().map(Entry::sequence); let boundary = last != Some(sequence); - let cut = match (self.timescale, frame.timestamp) { - (Some(timescale), Some(pts)) => { - let duration = units(config.duration, timescale); - let first = building.object.start()?.pts.expect("a timed object starts timed"); - match boundary { - true => pts.saturating_sub(first) >= duration, - // An oversized group splits once its own run here reaches the duration. - false => pts.saturating_sub(building.run_pts) >= duration, + // Only an object's first entry carries a frame start, so a group starting past frame 0 + // opens its own object. + let cut = (boundary && index != 0) + || match (self.timescale, frame.timestamp) { + (Some(timescale), Some(pts)) => { + let duration = units(config.duration, timescale); + let first = building.object.start()?.pts.expect("a timed object starts timed"); + match boundary { + true => pts.saturating_sub(first) >= duration, + // An oversized group splits once its own run here reaches the duration. + false => pts.saturating_sub(building.run_pts) >= duration, + } } - } - _ => boundary && now.saturating_duration_since(building.opened) >= config.duration, - }; + _ => boundary && now.saturating_duration_since(building.opened) >= config.duration, + }; if cut { self.close()?; } diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index a6a556347ca..6d9c43cd0aa 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -51,7 +51,8 @@ pub struct ImportArgs { type Instance = (moq_net::PathOwned, Option); /// Record every broadcast announced at or under `prefix` into `args.store`, each publisher -/// instance under `/`, until the announcements end. +/// instance under `/`, until the announcements end. A route covering `prefix` +/// lists nothing beneath it, so `prefix` itself is recorded as its broadcast. /// /// A route without an epoch names no instance, so each one it announces records under an epoch /// minted here: a restarted export never resumes it, and never stitches two instances together. @@ -87,10 +88,16 @@ pub async fn export(origin: moq_net::origin::Consumer, prefix: String, args: Exp moq_net::announce::Event::Restart(announce) => (announce, true), _ => continue, }; - if !announce.prefix.has_prefix(&prefix) { + // A route covering the selected path serves it without listing what is beneath, so + // the selected path is requested itself. + let path = if announce.prefix.has_prefix(&prefix) { + announce.prefix + } else if prefix.has_prefix(&announce.prefix) { + prefix.clone() + } else { continue; - } - let instance: Instance = (announce.prefix, announce.route.epoch); + }; + let instance: Instance = (path, announce.route.epoch); let count = running.entry(instance.clone()).or_default(); if *count > 0 && !(restart && instance.1.is_none()) { continue; @@ -316,6 +323,53 @@ mod tests { assert!(!files.iter().any(|file| file.starts_with("other")), "{files:?}"); } + /// A route covering the selected path, which serves it on request without announcing it, is + /// still asked for that path. + #[tokio::test] + async fn a_covering_route_is_asked_for_the_selected_path() { + let _env = crate::test_env::EnvGuard::clear(&[]); + let dir = tempfile::tempdir().unwrap(); + let url = Url::from_directory_path(dir.path()).unwrap(); + + let origin = moq_tokio::origin::spawn(); + let server = origin.dynamic("live").unwrap(); + let args = ExportArgs { + store: url, + retention: None, + retention_grace: None, + }; + let recording = tokio::spawn(export(origin.consume(), "live/a".into(), args)); + + let request = tokio::time::timeout(Duration::from_secs(10), server.requested_broadcast()) + .await + .expect("the export requests the selected path") + .unwrap(); + assert_eq!(request.path().as_str(), "live/a"); + let prepared = request.prepare(moq_net::Epoch::mint()).unwrap(); + let mut track = prepared + .broadcast() + .create_track("video", track::Info::default().with_max_age(Duration::from_secs(3600))) + .unwrap(); + let broadcast = prepared.publish().unwrap(); + tokio::time::timeout(Duration::from_secs(10), track.subscription_changed()) + .await + .expect("the export subscribes") + .unwrap(); + track.finish().unwrap(); + broadcast.close(); + + tokio::time::timeout(Duration::from_secs(10), async { + while !files(dir.path()).iter().any(|file| file.ends_with("/.end")) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the recording completes"); + recording.abort(); + let files = files(dir.path()); + assert!(files.iter().all(|file| file.starts_with("live/a/")), "{files:?}"); + } + /// A route without an epoch records under one minted per instance, so two instances never /// share a key prefix. #[test] From 47c801a1f24dc9839f72d7bdef5e3139d951ed84 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 11:33:55 -0700 Subject: [PATCH 09/16] revert(cli): leave paths under a covering route unrecorded The archive producer quest settled that a covering route is capability, not inventory, so export archive records only announced broadcasts. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 4 +-- rs/moq-cli/src/archive.rs | 62 +++------------------------------------ 2 files changed, 5 insertions(+), 61 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 510879e2400..dfcb94815df 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -307,9 +307,7 @@ moq --connect https://relay.example.com/anon --broadcast event-replay.hang impor `export archive` records every broadcast announced at or under `--broadcast` with [moq-archive](https://docs.rs/moq-archive), each publisher instance under -`/` in the store, until the connection ends. When a route -covers `--broadcast` without listing what is beneath it, `--broadcast` itself is -recorded as the broadcast. Every track is +`/` in the store, until the connection ends. Every track is stored as its own objects, cut at group boundaries about every 2 s of media (or wall clock for an [untimed](/concept/moq-lite#subscriptions) track) and flushed at least every 4 s. A broadcast restarted under a new diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 6d9c43cd0aa..a6a556347ca 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -51,8 +51,7 @@ pub struct ImportArgs { type Instance = (moq_net::PathOwned, Option); /// Record every broadcast announced at or under `prefix` into `args.store`, each publisher -/// instance under `/`, until the announcements end. A route covering `prefix` -/// lists nothing beneath it, so `prefix` itself is recorded as its broadcast. +/// instance under `/`, until the announcements end. /// /// A route without an epoch names no instance, so each one it announces records under an epoch /// minted here: a restarted export never resumes it, and never stitches two instances together. @@ -88,16 +87,10 @@ pub async fn export(origin: moq_net::origin::Consumer, prefix: String, args: Exp moq_net::announce::Event::Restart(announce) => (announce, true), _ => continue, }; - // A route covering the selected path serves it without listing what is beneath, so - // the selected path is requested itself. - let path = if announce.prefix.has_prefix(&prefix) { - announce.prefix - } else if prefix.has_prefix(&announce.prefix) { - prefix.clone() - } else { + if !announce.prefix.has_prefix(&prefix) { continue; - }; - let instance: Instance = (path, announce.route.epoch); + } + let instance: Instance = (announce.prefix, announce.route.epoch); let count = running.entry(instance.clone()).or_default(); if *count > 0 && !(restart && instance.1.is_none()) { continue; @@ -323,53 +316,6 @@ mod tests { assert!(!files.iter().any(|file| file.starts_with("other")), "{files:?}"); } - /// A route covering the selected path, which serves it on request without announcing it, is - /// still asked for that path. - #[tokio::test] - async fn a_covering_route_is_asked_for_the_selected_path() { - let _env = crate::test_env::EnvGuard::clear(&[]); - let dir = tempfile::tempdir().unwrap(); - let url = Url::from_directory_path(dir.path()).unwrap(); - - let origin = moq_tokio::origin::spawn(); - let server = origin.dynamic("live").unwrap(); - let args = ExportArgs { - store: url, - retention: None, - retention_grace: None, - }; - let recording = tokio::spawn(export(origin.consume(), "live/a".into(), args)); - - let request = tokio::time::timeout(Duration::from_secs(10), server.requested_broadcast()) - .await - .expect("the export requests the selected path") - .unwrap(); - assert_eq!(request.path().as_str(), "live/a"); - let prepared = request.prepare(moq_net::Epoch::mint()).unwrap(); - let mut track = prepared - .broadcast() - .create_track("video", track::Info::default().with_max_age(Duration::from_secs(3600))) - .unwrap(); - let broadcast = prepared.publish().unwrap(); - tokio::time::timeout(Duration::from_secs(10), track.subscription_changed()) - .await - .expect("the export subscribes") - .unwrap(); - track.finish().unwrap(); - broadcast.close(); - - tokio::time::timeout(Duration::from_secs(10), async { - while !files(dir.path()).iter().any(|file| file.ends_with("/.end")) { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("the recording completes"); - recording.abort(); - let files = files(dir.path()); - assert!(files.iter().all(|file| file.starts_with("live/a/")), "{files:?}"); - } - /// A route without an epoch records under one minted per instance, so two instances never /// share a key prefix. #[test] From 5212f06018f9fc713c9f113a312a1ec90e6cca33 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 11:33:55 -0700 Subject: [PATCH 10/16] fix(archive): a fresh track waits for a group below its first arrival Arrival is not sequence order, so the first group to arrive on a track with nothing stored set where the track starts, and a lower one arriving just after it was dropped. The first arrival now waits the max delay for a lower sequence, unless it is 0. Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/producer/mod.rs | 3 ++- rs/moq-archive/src/producer/tests.rs | 37 ++++++++++++++++++++++++++++ rs/moq-archive/src/producer/track.rs | 24 +++++++++++++----- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/rs/moq-archive/src/producer/mod.rs b/rs/moq-archive/src/producer/mod.rs index 571cf592fb8..5a45ec1b76f 100644 --- a/rs/moq-archive/src/producer/mod.rs +++ b/rs/moq-archive/src/producer/mod.rs @@ -58,7 +58,8 @@ pub struct Config { /// above [`Self::duration`], or objects are cut by it instead. pub flush: Duration, /// The longest a missing group is waited for, capping the publisher's max age. A group still - /// missing then is a gap, and the frames behind it wait for that verdict. + /// missing then is a gap, and the frames behind it wait for that verdict. A track with nothing + /// stored waits this long from its first arrival for a lower sequence, unless that one is 0. pub max_delay: Duration, /// Prune old objects (a DVR), or keep everything when `None` (an archive). pub dvr: Option, diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index 75a9132d821..8c287386590 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -231,6 +231,43 @@ async fn a_lost_group_is_a_gap_that_never_stalls_later_groups() { assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 4, frame: 0 }))); } +/// Arrival is not sequence order, so a fresh track's first arrival waits for a lower sequence. +#[tokio::test(start_paused = true)] +async fn a_fresh_track_waits_for_a_group_below_its_first_arrival() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let video = broadcast.create_track("video", timed()).unwrap(); + let config = config().with_flush(SECOND).with_max_delay(2 * SECOND); + let task = spawn(&store, &broadcast, config); + subscribed(&video).await; + + group(&video, 2, &[(2000, "c")]); + settle().await; + group(&video, 1, &[(1000, "b")]); + settle().await; + // Nothing is stored until the wait from the first arrival gives up on group 0. + sleep(SECOND + SECOND / 2).await; + assert!(objects(&store, "video", true).await.is_empty()); + + sleep(2 * SECOND).await; + let stored = objects(&store, "video", true).await; + assert_eq!(stored.keys().cloned().collect::>(), [key(1, 0, 1000)]); + assert_eq!( + contents(&stored)[0], + [ + (1, "group", vec!["1000/b".to_string()]), + (2, "group", vec!["2000/c".to_string()]), + ] + ); + + // Group 0 arriving after that is too late. + group(&video, 0, &[(0, "a")]); + video.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!(objects(&store, "video", true).await.len(), 1); +} + #[tokio::test(start_paused = true)] async fn an_oversized_timed_group_splits_into_ordered_unique_keys() { let store = store(); diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index b2879b0b7fc..594c69ea360 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -100,6 +100,9 @@ struct Run { current: Option>, /// When the gap below the lowest pending sequence is given up. gap: Option, + /// When the first sequence arrived while nothing was stored, so nothing says where the track + /// starts: arrival is not sequence order, and a lower one may still be on its way. + first: Option, groups_done: bool, datagrams_done: bool, } @@ -241,6 +244,7 @@ impl Track { pending: BTreeMap::new(), current: None, gap: None, + first: None, groups_done: false, datagrams_done: false, }; @@ -327,16 +331,24 @@ impl Track { return Ok(()); }; let sequence = *entry.key(); - if let Some(next) = self.next { - if sequence < next.group { + let waiting = match self.next { + Some(next) if sequence < next.group => { entry.remove(); continue; } - let given_up = entry.get().arrived() + run.max_delay; - if sequence > next.group && !draining && moq_clock::now() < given_up { - run.gap = Some(given_up); - return Ok(()); + Some(next) => (sequence > next.group).then(|| entry.get().arrived()), + // Nothing is below 0; anything else waits from the first arrival, so a stream of + // ever lower sequences cannot hold the track back. + None => { + let first = *run.first.get_or_insert(entry.get().arrived()); + (sequence > 0).then_some(first) } + }; + if let Some(waiting) = waiting + && !draining && moq_clock::now() < waiting + run.max_delay + { + run.gap = Some(waiting + run.max_delay); + return Ok(()); } match entry.remove() { Pending::Datagram(datagram, arrived) => { From 5d4f7c4c84836912bbacddf0e0d4f7a85c1f83be Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 12:03:29 -0700 Subject: [PATCH 11/16] fix(cli)!: export archive keys each instance /@, prefix-free A publisher could announce a path that spells out another recording's key prefix and land its own recording inside it, breaking that one's resume. The epoch segment now starts with @, and a path segment's leading @ is percent-encoded (as % already is), so no key prefix lies inside another. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 9 +++--- rs/moq-archive/MIGRATE.md | 2 +- rs/moq-cli/MIGRATE.md | 2 +- rs/moq-cli/src/archive.rs | 64 ++++++++++++++++++++++++++++++++------- 4 files changed, 60 insertions(+), 17 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index dfcb94815df..2a5feb49e09 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -307,10 +307,11 @@ moq --connect https://relay.example.com/anon --broadcast event-replay.hang impor `export archive` records every broadcast announced at or under `--broadcast` with [moq-archive](https://docs.rs/moq-archive), each publisher instance under -`/` in the store, until the connection ends. Every track is -stored as its own objects, cut at group boundaries about every 2 s of media (or -wall clock for an [untimed](/concept/moq-lite#subscriptions) track) and flushed -at least every 4 s. A broadcast restarted under a new +`/@` in the store, until the connection ends. A broadcast +path segment that starts with `@` is stored as `%40`, so no recording lands +inside another. Every track is stored as its own objects, cut about every 2 s +of media (or wall clock for an [untimed](/concept/moq-lite#subscriptions) +track) and at least every 4 s, splitting a long group between frames. A broadcast restarted under a new [epoch](/concept/moq-lite#publisher-epochs) records under its own key; a route without an epoch gets a fresh one each time it announces, so it never resumes or stitches into an older recording. Any recording that fails ends the export. diff --git a/rs/moq-archive/MIGRATE.md b/rs/moq-archive/MIGRATE.md index d51cc4d4ca5..fc5a2ce96e1 100644 --- a/rs/moq-archive/MIGRATE.md +++ b/rs/moq-archive/MIGRATE.md @@ -8,7 +8,7 @@ How to update your code for each breaking change, newest first. `Writer`, its `Control`, and `writer::{Config, Retention}` are removed; record with `Producer::new(store, broadcast, producer::Config::default()).run().await` instead. A producer records every track of one broadcast, found with TRACKS rather than a catalog, and needs no timeline: the layout is per-track objects keyed by group, frame, and timestamp, documented in the crate docs. -Run one producer per broadcast under a key prefix you pick, such as `/`; it stores no epoch, so `Info::epoch` and `Info::with_epoch` are removed. +Run one producer per broadcast under a key prefix you pick, such as `/@`; it stores no epoch, so `Info::epoch` and `Info::with_epoch` are removed. `writer::Retention` becomes `producer::Dvr`, set with `Config::with_dvr`, and works on untimed tracks too. `Error::{EpochMismatch, Enrolled, Untimed, Timeline, Closed}` are removed; a failed source listing is `Error::Source`. The new layout is not readable by `Reader`, which still replays timeline recordings. diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index f485e1919e3..10ba9cc3857 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -6,7 +6,7 @@ How to update your invocation and config for each breaking change, newest first. ### `export archive` records a prefix (#6160) -`moq export archive ` records every broadcast announced at or under `--broadcast`, each instance under `//`, until the connection ends, instead of one broadcast at ``. +`moq export archive ` records every broadcast announced at or under `--broadcast`, each instance under `//@`, until the connection ends, instead of one broadcast at ``. Point `` at the root you want recordings under, and drop `--catalog-format`, which it now refuses. `--retention` now works on untimed tracks. `import archive` does not read the new layout yet; it still replays recordings made by older `moq` versions. diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index a6a556347ca..18e349b25fe 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -21,7 +21,7 @@ const GRACE: Duration = Duration::from_secs(30); #[usage(unknown_flags = "error", args_override_self = false)] pub struct ExportArgs { /// Where to record: `file:///dir`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`. - /// Each broadcast records under `/` beneath it. + /// Each broadcast records under `/@` beneath it. pub store: Url, /// Keep only this much recent content (a DVR), deleting older objects. Unset keeps everything. @@ -51,7 +51,7 @@ pub struct ImportArgs { type Instance = (moq_net::PathOwned, Option); /// Record every broadcast announced at or under `prefix` into `args.store`, each publisher -/// instance under `/`, until the announcements end. +/// instance under `/@`, until the announcements end. /// /// A route without an epoch names no instance, so each one it announces records under an epoch /// minted here: a restarted export never resumes it, and never stitches two instances together. @@ -137,14 +137,22 @@ async fn record( (instance, result) } -/// The key prefix one instance records under: its path's segments, then its epoch. +/// The key prefix one instance records under: its path's segments, then `@`. +/// +/// A path segment's key never starts with `@`, since a leading one is percent-encoded like `%` +/// always is, so no instance's prefix lies inside another's, whatever paths a publisher picks. fn key(root: &Path, (path, epoch): &Instance) -> anyhow::Result { let epoch = epoch.clone().unwrap_or_else(moq_net::Epoch::mint); let mut key = root.clone(); for segment in path.as_str().split('/').filter(|segment| !segment.is_empty()) { - key = key.join(PathPart::from(segment)); + let part = PathPart::from(segment); + key = match part.as_ref().strip_prefix('@') { + Some(rest) => key.join(PathPart::parse(&format!("%40{rest}"))?), + None => key.join(part), + }; } - Ok(key.join(PathPart::parse(epoch.as_str()).context("an epoch is one key segment")?)) + let epoch = format!("@{epoch}"); + Ok(key.join(PathPart::parse(&epoch).context("an epoch is one key segment")?)) } /// Republish the recording at `args.store` as the broadcast `name`, announced under `epoch`. @@ -244,7 +252,7 @@ mod tests { } /// `export archive` records each broadcast announced under its prefix into its own - /// `/`, and leaves out what is announced elsewhere. + /// `/@`, and leaves out what is announced elsewhere. #[tokio::test] async fn each_broadcast_records_under_its_own_key_prefix() { // `open_store` reads the credential variables. @@ -289,8 +297,8 @@ mod tests { broadcast.close(); } let complete = [ - format!("live/a/{}/.end", epochs[0]), - format!("live/b/{}/.end", epochs[1]), + format!("live/a/@{}/.end", epochs[0]), + format!("live/b/@{}/.end", epochs[1]), ]; tokio::time::timeout(Duration::from_secs(10), async { while !complete.iter().all(|end| dir.path().join(end).exists()) { @@ -304,11 +312,11 @@ mod tests { let files = files(dir.path()); for (path, epoch) in [("live/a", &epochs[0]), ("live/b", &epochs[1])] { for file in [".end", "video/.info", "video/end"] { - assert!(files.contains(&format!("{path}/{epoch}/{file}")), "{files:?}"); + assert!(files.contains(&format!("{path}/@{epoch}/{file}")), "{files:?}"); } assert!( files.contains(&format!( - "{path}/{epoch}/video/0000000000000000000-0000000000000000000-0000000000000000000" + "{path}/@{epoch}/video/0000000000000000000-0000000000000000000-0000000000000000000" )), "{files:?}" ); @@ -330,7 +338,41 @@ mod tests { let pinned = (moq_net::Path::new("live/a").to_owned(), Some(epoch.clone())); assert_eq!( key(&Path::from("rec"), &pinned).unwrap().as_ref(), - format!("rec/live/a/{epoch}") + format!("rec/live/a/@{epoch}") + ); + } + + /// A path segment's key never starts with `@`, so a path that spells out another instance's + /// key prefix records beside it, never inside it. + #[test] + fn keys_are_prefix_free() { + let epoch = moq_net::Epoch::mint(); + let victim = key( + &Path::from("rec"), + &(moq_net::Path::new("live/a").to_owned(), Some(epoch.clone())), + ) + .unwrap(); + let hostile = format!("live/a/@{epoch}"); + let nested = key( + &Path::from("rec"), + &(moq_net::Path::new(&hostile).to_owned(), Some(moq_net::Epoch::mint())), + ) + .unwrap(); + assert!(nested.prefix_match(&victim).is_none(), "{nested} is under {victim}"); + assert!( + nested.as_ref().starts_with(&format!("rec/live/a/%40{epoch}/@")), + "{nested}" ); + + // `%` is escaped too, so an escaped-looking segment keys apart from the `@` one. + let literal = key( + &Path::from("rec"), + &( + moq_net::Path::new(&format!("live/a/%40{epoch}")).to_owned(), + Some(epoch), + ), + ) + .unwrap(); + assert!(literal.as_ref().starts_with("rec/live/a/%2540"), "{literal}"); } } From b2d5a0d0e80058b44131ce7719be85dec486cb71 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 12:03:29 -0700 Subject: [PATCH 12/16] fix(archive)!: split untimed groups and order arrivals through one consumer An open untimed group was held whole in memory and rewritten in place at every flush, so a publisher that never closed one grew memory and uploads without bound. Untimed objects are now keyed - and split between frames like timed ones, at the flush or once the group's run lasts the duration; every object is written once, and the rewrite-in-place and read-back resume paths are gone. Ordering moves into producer::Ordered, which hands out each group or datagram once every lower sequence was handed out or given up, waiting at most max_delay. moq_net::track::Ordered seeks past a missing group at once and orders groups alone, so it does not fit an archive. Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/README.md | 4 +- quest/m1/archive/consumer.md | 4 +- rs/moq-archive/src/layout.rs | 38 ++- rs/moq-archive/src/lib.rs | 11 +- rs/moq-archive/src/producer/backends.rs | 23 +- rs/moq-archive/src/producer/mod.rs | 9 +- rs/moq-archive/src/producer/ordered.rs | 195 +++++++++++++++ rs/moq-archive/src/producer/resume.rs | 11 +- rs/moq-archive/src/producer/tests.rs | 83 +++++-- rs/moq-archive/src/producer/track.rs | 306 ++++++------------------ 10 files changed, 386 insertions(+), 298 deletions(-) create mode 100644 rs/moq-archive/src/producer/ordered.rs diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index cfd62883da6..d528cdb82c3 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -42,8 +42,8 @@ archive (per-track `.timeline.z` records named by the catalog's root stall. Anything younger is still in the live cache. - **Per-track objects, keyed by position.** Groups coalesce in order into objects cut by duration, keyed - `//--` (`` - for an untimed track, `` percent-encoded as one + `//--` + (`-` for an untimed track, `` percent-encoded as one segment), beside a small per-track `.info` holding its timescale and priority. Each stored sequence keeps its form, group or datagram. A track's `end` records how it ended, clean or with its error code, written diff --git a/quest/m1/archive/consumer.md b/quest/m1/archive/consumer.md index 9ab4f6b508b..351d61164cf 100644 --- a/quest/m1/archive/consumer.md +++ b/quest/m1/archive/consumer.md @@ -20,8 +20,8 @@ Decided by the maintainer on 2026-10-17 (see the track with its own cursor, a LIST of that track's key prefix StartAfter its newest known key, since one prefix-wide cursor skips a new key under a lexically earlier track. A delimiter LIST of the broadcast prefix finds - tracks added since. It also re-reads a track's newest object, which the - producer may rewrite in place while an untimed group is open. + tracks added since. Every object is written once, an open group continuing + in the next object (decided 2026-10-21). - Each served track advertises no Publisher Max Age limit (TRACK_INFO's encoded `0`), not the source's: the store's retention bounds what it holds, and DVR pruning is the publisher dropping a group early, which TRACK_INFO diff --git a/rs/moq-archive/src/layout.rs b/rs/moq-archive/src/layout.rs index 7392dd8cc2b..a2c44366129 100644 --- a/rs/moq-archive/src/layout.rs +++ b/rs/moq-archive/src/layout.rs @@ -61,15 +61,13 @@ pub(crate) struct Start { } impl Start { - /// `--` on a timed track, `` on an untimed one, zero-padded so - /// keys sort in content order. + /// `--` on a timed track, `-` on an untimed one, + /// zero-padded so keys sort in content order. pub(crate) fn name(&self) -> Result { - let group = id(self.group)?; + let (group, frame) = (id(self.group)?, id(self.frame)?); match self.pts { - Some(pts) => Ok(format!("{group}-{}-{}", id(self.frame)?, id(pts)?)), - None if self.frame == 0 => Ok(group), - // An untimed group is never split, so no untimed object starts mid-group. - None => Err(Error::Path(format!("{group}-{}", self.frame))), + Some(pts) => Ok(format!("{group}-{frame}-{}", id(pts)?)), + None => Ok(format!("{group}-{frame}")), } } @@ -82,9 +80,9 @@ impl Start { frame: parse_id(frame).ok_or_else(invalid)??, pts: Some(parse_id(pts).ok_or_else(invalid)??), }), - (false, [group]) => Ok(Self { + (false, [group, frame]) => Ok(Self { group: parse_id(group).ok_or_else(invalid)??, - frame: 0, + frame: parse_id(frame).ok_or_else(invalid)??, pts: None, }), _ => Err(invalid()), @@ -445,31 +443,27 @@ mod tests { let untimed = Start { group: 7, - frame: 0, + frame: 2, pts: None, }; let name = untimed.name().unwrap(); - assert_eq!(name, "0000000000000000007"); + assert_eq!(name, "0000000000000000007-0000000000000000002"); assert_eq!(Name::parse(&name, false).unwrap(), Name::Object(untimed)); assert!(matches!(Name::parse(&name, true), Err(Error::Path(_)))); assert_eq!(Name::parse(".info", true).unwrap(), Name::Info); assert_eq!(Name::parse("end", false).unwrap(), Name::End); - for bad in ["7", "segments", "0000000000000000007-x", "0009007199254740992"] { + for bad in [ + "7", + "segments", + "0000000000000000007", + "0000000000000000007-x", + "0009007199254740992-0000000000000000000", + ] { assert!(Name::parse(bad, false).is_err(), "{bad}"); } } - #[test] - fn an_untimed_object_never_starts_mid_group() { - let start = Start { - group: 1, - frame: 2, - pts: None, - }; - assert!(matches!(start.name(), Err(Error::Path(_)))); - } - #[test] fn end_markers_are_json() { let finish = End::Finish(At { group: 5, frame: 3 }); diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 77ac384991f..2871111019b 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -12,13 +12,12 @@ //! written before its first object. `timescale` is absent on an untimed track. Properties never //! change for a track name. The source's max age is not stored: it bounds the live cache, not //! the archive. -//! - `/--` on a timed track, `/` on an untimed one: an -//! object, keyed by its first sequence, the index of its first frame within it, and that -//! frame's timestamp in the track's timescale. Each field is 19 zero-padded decimal digits, +//! - `/--` on a timed track, `/-` on an untimed +//! one: an object, keyed by its first sequence, the index of its first frame within it, and +//! that frame's timestamp in the track's timescale. Each field is 19 zero-padded decimal digits, //! so a sorted listing is content order and answers group and time lookups; there is no index -//! object. A timed group may split between frames across objects, which the frame index keeps -//! unique and ordered even where timestamps repeat or go backward. An untimed group never -//! splits: while it is open, its object is rewritten in place under the same key. +//! object. A group may split between frames across objects, which the frame index keeps unique +//! and ordered even where timestamps repeat or go backward. Every object is written once. //! - `/end`: how the track's last run ended, `{"finish":{"group":5,"frame":0}}` for a //! clean end at that exclusive position or `{"error":52}` for the stream error code. It sorts //! after every object key, and DVR pruning never deletes it. diff --git a/rs/moq-archive/src/producer/backends.rs b/rs/moq-archive/src/producer/backends.rs index 704b32654d1..0a2218d83a6 100644 --- a/rs/moq-archive/src/producer/backends.rs +++ b/rs/moq-archive/src/producer/backends.rs @@ -19,8 +19,8 @@ use super::*; const VIDEO: &str = "video/1080p"; const CHAT: &str = "chat.json"; -/// Real time, so the S3 client's own timers behave: a flush short enough to rewrite the open -/// untimed group while the test waits, and long enough that no put outlasts it. +/// Real time, so the S3 client's own timers behave: a flush short enough to store the untimed +/// group while the test waits, and long enough that no put outlasts it. fn config() -> Config { Config::default() .with_duration(Duration::from_secs(2)) @@ -75,8 +75,8 @@ async fn stored(store: &Store, paths: &[Path]) { .expect("the producer stores the objects"); } -/// Record in two runs: the first is stopped once it stored a split timed track and an open -/// untimed group, and the second resumes both, appending to the open group in place. +/// Record in two runs: the first is stopped once it stored a split timed track and the head of +/// an untimed group, and the second resumes both, continuing that group in a new object. async fn record(store: &Store) { let (first, _tracks) = source(4, &[(0, &["c0"])]); let run = tokio::spawn(Producer::new(store.clone(), first.consume(), config()).run()); @@ -93,7 +93,7 @@ async fn record(store: &Store) { VIDEO, "0000000000000000002-0000000000000000000-0000000000000002000", ), - path(store, CHAT, "0000000000000000000"), + path(store, CHAT, "0000000000000000000-0000000000000000000"), ], ) .await; @@ -156,7 +156,8 @@ async fn recordings_are_byte_identical_on_every_backend() { [ ".end", "chat%2Ejson/.info", - "chat%2Ejson/0000000000000000000", + "chat%2Ejson/0000000000000000000-0000000000000000000", + "chat%2Ejson/0000000000000000000-0000000000000000001", "chat%2Ejson/end", "video%2F1080p/.info", "video%2F1080p/0000000000000000000-0000000000000000000-0000000000000000000", @@ -166,9 +167,13 @@ async fn recordings_are_byte_identical_on_every_backend() { ] ); assert_eq!( - decode(&expected, "chat%2Ejson/0000000000000000000", false), - [(0, vec!["c0".into(), "c1".into()]), (1, vec!["c2".into()])], - "the resumed open group is appended in place" + decode(&expected, "chat%2Ejson/0000000000000000000-0000000000000000000", false), + [(0, vec!["c0".into()])], + ); + assert_eq!( + decode(&expected, "chat%2Ejson/0000000000000000000-0000000000000000001", false), + [(0, vec!["c1".into()]), (1, vec!["c2".into()])], + "the resumed group continues in its own object" ); assert_eq!( decode( diff --git a/rs/moq-archive/src/producer/mod.rs b/rs/moq-archive/src/producer/mod.rs index 5a45ec1b76f..16f871c6e0d 100644 --- a/rs/moq-archive/src/producer/mod.rs +++ b/rs/moq-archive/src/producer/mod.rs @@ -21,6 +21,7 @@ //! # } //! ``` +mod ordered; mod resume; mod track; @@ -50,12 +51,12 @@ use crate::{Error, Result, Store}; #[non_exhaustive] pub struct Config { /// How much content an object holds before it is cut at the next group boundary: media time - /// on a timed track, wall-clock time on an untimed one. A timed group whose own run in an - /// object reaches it splits between frames; an untimed group never splits. + /// on a timed track, wall-clock time on an untimed one. A group whose own run in an object + /// reaches it splits between frames. pub duration: Duration, /// The longest a frame waits in memory before its object is stored, however idle its track. - /// A timed object is cut there; an untimed one is rewritten in place and stays open. Keep it - /// above [`Self::duration`], or objects are cut by it instead. + /// The object is cut there, mid-group if need be. Keep it above [`Self::duration`], or + /// objects are cut by it instead. pub flush: Duration, /// The longest a missing group is waited for, capping the publisher's max age. A group still /// missing then is a gap, and the frames behind it wait for that verdict. A track with nothing diff --git a/rs/moq-archive/src/producer/ordered.rs b/rs/moq-archive/src/producer/ordered.rs new file mode 100644 index 00000000000..09dd3a35571 --- /dev/null +++ b/rs/moq-archive/src/producer/ordered.rs @@ -0,0 +1,195 @@ +//! A track's groups and datagrams in sequence order, waiting a bounded time for a late one. +//! +//! `moq_net::track::Ordered` seeks past a missing group at once and orders groups alone, so it +//! fits a player, not an archive that must keep every group and datagram it can in key order. + +use std::collections::BTreeMap; +use std::pin::Pin; +use std::task::Poll; +use std::time::Duration; + +use moq_clock::Instant; +use moq_net::track::{Position, Subscriber}; +use moq_net::{Datagram, group}; + +/// One sequence, in the form it was forwarded in. +pub(super) enum Item { + // Boxed, since a group consumer dwarfs a datagram and waits in `Ordered::pending`. + Group(Box), + Datagram(Datagram), +} + +/// Hands out each sequence once every lower one was handed out or given up: at once when it is +/// the next, otherwise after `max_delay` from its arrival, or from the first arrival while no +/// start is known. Once both channels end, everything received is handed out without waiting. +pub(super) struct Ordered { + subscriber: Subscriber, + max_delay: Duration, + /// The next sequence to hand out; `None` until a start is known or the first wait ends. + next: Option, + /// Received sequences waiting on a lower one, with when they arrived. + pending: BTreeMap, + /// When the first sequence arrived while `next` was unknown, since arrival is not sequence + /// order and a lower one may still be on its way. Fixed, so ever lower arrivals can't extend + /// the wait. + first: Option, + timer: Option>>, + groups_done: bool, + datagrams_done: bool, +} + +impl Ordered { + /// Order `subscriber` from `next`, where a resume continues; otherwise from the subscription's + /// floor when it names one. + pub(super) fn new(subscriber: Subscriber, max_delay: Duration, next: Option) -> Self { + let next = next.or_else(|| subscriber.floor()); + Self { + subscriber, + max_delay, + next, + pending: BTreeMap::new(), + first: None, + timer: None, + groups_done: false, + datagrams_done: false, + } + } + + /// Take every arrival so far, registering `waiter` for the next. + pub(super) fn poll_arrivals(&mut self, waiter: &kio::Waiter) { + let now = moq_clock::now(); + while !self.groups_done { + match self.subscriber.poll_recv_group(waiter) { + Poll::Ready(Ok(Some(group))) => self.arrive(group.sequence, Item::Group(Box::new(group)), now), + // How the track ended is read from `poll_finished`. + Poll::Ready(_) => self.groups_done = true, + Poll::Pending => break, + } + } + while !self.datagrams_done { + match self.subscriber.poll_recv_datagram(waiter) { + Poll::Ready(Ok(Some(datagram))) => self.arrive(datagram.sequence, Item::Datagram(datagram), now), + Poll::Ready(_) => self.datagrams_done = true, + Poll::Pending => break, + } + } + } + + fn arrive(&mut self, sequence: u64, item: Item, now: Instant) { + // A sequence names one group, so a copy of one already here or handed out is dropped. + if self.next.is_none_or(|next| sequence >= next) { + self.pending.entry(sequence).or_insert((item, now)); + } + } + + /// The next sequence in order, or `None` once both channels ended and nothing is left. + pub(super) fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll> { + self.poll_arrivals(waiter); + let draining = self.groups_done && self.datagrams_done; + loop { + let Some(entry) = self.pending.first_entry() else { + return match draining { + true => Poll::Ready(None), + false => Poll::Pending, + }; + }; + let sequence = *entry.key(); + let arrived = entry.get().1; + let waiting = match self.next { + Some(next) if sequence < next => { + entry.remove(); + continue; + } + Some(next) => (sequence > next).then_some(arrived), + // Nothing is below 0. + None => (sequence > 0).then(|| *self.first.get_or_insert(arrived)), + }; + if let Some(waiting) = waiting + && !draining + { + let deadline = waiting + self.max_delay; + if moq_clock::now() < deadline { + let timer = self + .timer + .get_or_insert_with(|| Box::pin(moq_clock::sleep_until(deadline))); + timer.as_mut().reset(deadline); + if waiter.poll_future(timer.as_mut()).is_pending() { + return Poll::Pending; + } + } + } + let (item, _) = entry.remove(); + self.next = Some(sequence.saturating_add(1)); + return Poll::Ready(Some((sequence, item))); + } + } + + /// How the track ended, once it has. + pub(super) fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { + self.subscriber.poll_finished(waiter) + } +} + +#[cfg(test)] +mod tests { + use moq_net::track::{Order, Range, Subscription}; + use moq_net::{broadcast, track}; + + use super::*; + + const MAX_DELAY: Duration = Duration::from_secs(10); + + fn group(track: &track::Producer, sequence: u64) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + group.write_frame(None, "x").unwrap(); + group.finish().unwrap(); + } + + fn next(ordered: &mut Ordered) -> Option { + match ordered.poll_next(&kio::Waiter::noop()) { + Poll::Ready(Some((sequence, _))) => Some(sequence), + _ => None, + } + } + + /// The next sequence is handed out at once, a datagram in turn with the groups, and one past a + /// missing sequence only once the max delay from its arrival passes. + #[tokio::test(start_paused = true)] + async fn only_a_missing_sequence_waits_the_max_delay() { + let broadcast = broadcast::Info::new().produce(); + let mut track = broadcast.create_track("t", track::Info::default()).unwrap(); + let subscription = Subscription::default() + .with_order(Order::Ascending) + .with_max_delay(MAX_DELAY) + .with_ranges([Range::oldest()]); + let subscriber = broadcast + .consume() + .track("t") + .unwrap() + .subscribe(subscription) + .await + .unwrap(); + let mut ordered = Ordered::new(subscriber, MAX_DELAY, None); + + group(&track, 0); + assert_eq!(next(&mut ordered), Some(0)); + + group(&track, 2); + assert_eq!(next(&mut ordered), None); + track.insert_datagram(1, None, "d").unwrap(); + assert_eq!(next(&mut ordered), Some(1)); + assert_eq!(next(&mut ordered), Some(2)); + + group(&track, 4); + assert_eq!(next(&mut ordered), None); + moq_clock::sleep(MAX_DELAY - Duration::from_millis(1)).await; + assert_eq!(next(&mut ordered), None); + moq_clock::sleep(Duration::from_millis(1)).await; + assert_eq!(next(&mut ordered), Some(4)); + + // Too late for its turn. + group(&track, 3); + track.finish().unwrap(); + assert!(matches!(ordered.poll_next(&kio::Waiter::noop()), Poll::Ready(None))); + } +} diff --git a/rs/moq-archive/src/producer/resume.rs b/rs/moq-archive/src/producer/resume.rs index fac29f7f4a0..4d8ca41958b 100644 --- a/rs/moq-archive/src/producer/resume.rs +++ b/rs/moq-archive/src/producer/resume.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use futures::TryStreamExt; +use moq_net::track::Position; use object_store::path::Path; use object_store::{ObjectStore, ObjectStoreExt}; @@ -15,8 +16,8 @@ pub(super) struct Stored { pub info: Info, /// Every object's key, oldest first. pub objects: Vec, - /// The newest object, which says where the track continues. - pub newest: Option, + /// Where the track continues: just past its newest object's last frame. + pub next: Option, /// How its last run ended, when the producer that stored it saw that. pub end: Option, } @@ -69,13 +70,13 @@ async fn load_track(store: &Store, dir: &Path) -> Result { let object = Object::decode(store.inner().get(path).await?.bytes().await?, timed)?; if object.start()? != *start { return Err(Error::Path(path.to_string())); } - Some(object) + Some(object.end()?) } None => None, }; @@ -83,7 +84,7 @@ async fn load_track(store: &Store, dir: &Path) -> Result String { format!("{group:019}-{frame:019}-{pts:019}") } +fn untimed(group: u64, frame: u64) -> String { + format!("{group:019}-{frame:019}") +} + async fn end(store: &Store, track: &str) -> Option { let key = format!("{}/end", crate::path::encode_track(track).unwrap()); get(store, &key).await.map(|bytes| End::decode(&bytes).unwrap()) @@ -306,8 +310,10 @@ async fn an_oversized_timed_group_splits_into_ordered_unique_keys() { assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 1, frame: 0 }))); } +/// An open untimed group is cut at each flush, so it never holds more than a flush of frames in +/// memory, and a stored object is never rewritten. #[tokio::test(start_paused = true)] -async fn an_open_untimed_group_is_rewritten_in_place_and_cut_by_wall_clock() { +async fn an_open_untimed_group_is_cut_at_each_flush() { let store = store(); let broadcast = broadcast::Info::new().produce(); let data = broadcast.create_track("data", track::Info::default()).unwrap(); @@ -318,25 +324,53 @@ async fn an_open_untimed_group_is_rewritten_in_place_and_cut_by_wall_clock() { open.write_frame(None, "a").unwrap(); settle().await; sleep(SECOND).await; - assert_eq!( - contents(&objects(&store, "data", false).await), - [[(0, "group", vec!["a".to_string()])]] - ); + let head = untimed(0, 0); + let first = get(&store, &format!("data/{head}")).await.unwrap(); open.write_frame(None, "b").unwrap(); settle().await; sleep(SECOND).await; + assert_eq!(get(&store, &format!("data/{head}")).await.unwrap(), first); let stored = objects(&store, "data", false).await; - assert_eq!(stored.keys().cloned().collect::>(), ["0000000000000000000"]); + assert_eq!(stored.keys().cloned().collect::>(), [head, untimed(0, 1)]); assert_eq!( contents(&stored), - [[(0, "group", vec!["a".to_string(), "b".to_string()])]] + [ + [(0, "group", vec!["a".to_string()])], + [(0, "group", vec!["b".to_string()])] + ] ); + open.finish().unwrap(); + data.finish().unwrap(); + broadcast.close(); + finished(task).await; + assert_eq!(end(&store, "data").await, Some(End::Finish(At { group: 1, frame: 0 }))); + let info = Info::decode(&get(&store, "data/.info").await.unwrap()).unwrap(); + assert_eq!(info.timescale, None); +} + +/// Without a flush due, an untimed group splits once its own run in an object lasts the duration +/// of wall clock, and the next group starts the next object past the duration. +#[tokio::test(start_paused = true)] +async fn an_untimed_group_splits_by_wall_clock() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let data = broadcast.create_track("data", track::Info::default()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&data).await; - // Past the duration, the next group starts the next object. + let mut open = data.create_group(group::Info { sequence: 0 }).unwrap(); + for payload in ["a", "b", "c", "d", "e"] { + open.write_frame(None, payload).unwrap(); + settle().await; + sleep(SECOND / 2).await; + } + open.finish().unwrap(); + settle().await; + sleep(2 * SECOND).await; let mut next = data.create_group(group::Info { sequence: 1 }).unwrap(); - next.write_frame(None, "c").unwrap(); + next.write_frame(None, "f").unwrap(); next.finish().unwrap(); settle().await; data.finish().unwrap(); @@ -346,10 +380,17 @@ async fn an_open_untimed_group_is_rewritten_in_place_and_cut_by_wall_clock() { let stored = objects(&store, "data", false).await; assert_eq!( stored.keys().cloned().collect::>(), - ["0000000000000000000", "0000000000000000001"] + [untimed(0, 0), untimed(0, 4), untimed(1, 0)] + ); + let strings = |frames: &[&str]| frames.iter().map(|frame| frame.to_string()).collect::>(); + assert_eq!( + contents(&stored), + [ + vec![(0, "group", strings(&["a", "b", "c", "d"]))], + vec![(0, "group", strings(&["e"]))], + vec![(1, "group", strings(&["f"]))], + ] ); - let info = Info::decode(&get(&store, "data/.info").await.unwrap()).unwrap(); - assert_eq!(info.timescale, None); } #[tokio::test(start_paused = true)] @@ -433,16 +474,18 @@ async fn a_restart_mid_group_loses_and_duplicates_nothing() { .collect(); assert_eq!(frames, ["0/a", "100/b", "200/c", "300/d"]); - // The untimed group's object was read back and rewritten with the rest. + // The untimed group continues in its own object too. let stored = objects(&store, "data", false).await; assert_eq!( - contents(&stored), - [[( - 0, - "group", - vec!["a", "b", "c", "d"].into_iter().map(String::from).collect() - )]] + stored.keys().cloned().collect::>(), + [untimed(0, 0), untimed(0, 2)] ); + let frames: Vec = contents(&stored) + .into_iter() + .flatten() + .flat_map(|(_, _, frames)| frames) + .collect(); + assert_eq!(frames, ["a", "b", "c", "d"]); assert!(complete(&store).await); } @@ -549,7 +592,7 @@ async fn track_names_stay_one_segment_through_a_resume() { let stored = keys(&store).await; for dir in ["a%2Fb", "%2Ehidden", "end"] { - assert!(stored.contains(&format!("{dir}/0000000000000000000")), "{stored:?}"); + assert!(stored.contains(&format!("{dir}/{}", untimed(0, 0))), "{stored:?}"); assert!(stored.contains(&format!("{dir}/.info")), "{stored:?}"); } diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index 594c69ea360..4331fe642f0 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -1,6 +1,6 @@ //! One track's recorder: its runs, in order, into duration-cut objects. -use std::collections::{BTreeMap, VecDeque}; +use std::collections::VecDeque; use std::pin::Pin; use std::sync::Arc; use std::task::Poll; @@ -9,11 +9,12 @@ use std::time::Duration; use bytes::Bytes; use moq_clock::Instant; use moq_net::track::{Anchor, Order, Position, Range, Subscription}; -use moq_net::{Datagram, StreamError, Timescale, group}; +use moq_net::{StreamError, Timescale, group}; use object_store::path::Path; use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutPayload}; use super::Shared; +use super::ordered::{Item, Ordered}; use super::resume::Stored; use crate::layout::{self, End, Entry, Object}; use crate::segment::Frame; @@ -54,7 +55,7 @@ struct Track { /// Objects a newer one superseded, oldest first, with when. superseded: VecDeque<(Instant, Path)>, /// Writes owed to the store, in order. - puts: Vec, + puts: Vec<(Path, Bytes)>, end: Option, } @@ -62,56 +63,17 @@ struct Building { object: Object, /// When it was opened, which cuts an untimed object by wall clock. opened: Instant, - /// The arrival of its oldest frame not stored yet, which bounds the flush. - unstored: Option, - /// Whether it is in the store under its key, as an untimed object rewritten in place is. - stored: bool, - /// On a timed track, the timestamp its last entry's run in this object starts at. + /// The arrival of its oldest frame, which bounds the flush. + unstored: Instant, + /// Where its last entry's run in this object starts: the timestamp on a timed track, the + /// arrival on an untimed one, which splits an oversized group. run_pts: u64, -} - -struct Put { - path: Path, - bytes: Bytes, - /// Rewrite an untimed object in place, rather than create an immutable one. - overwrite: bool, -} - -/// A sequence received ahead of the one being stored. -enum Pending { - Group(Box, Instant), - Datagram(Datagram, Instant), -} - -impl Pending { - fn arrived(&self) -> Instant { - match self { - Self::Group(_, at) | Self::Datagram(_, at) => *at, - } - } -} - -/// One subscription's progress. -struct Run { - max_delay: Duration, - /// Received sequences not stored yet, waiting on a lower one. - pending: BTreeMap, - /// The group being stored, frame by frame. - current: Option>, - /// When the gap below the lowest pending sequence is given up. - gap: Option, - /// When the first sequence arrived while nothing was stored, so nothing says where the track - /// starts: arrival is not sequence order, and a lower one may still be on its way. - first: Option, - groups_done: bool, - datagrams_done: bool, + run_at: Instant, } enum Event { Frame(u64, moq_net::Result>), - // Boxed, since a group consumer dwarfs every other event and waits in `Run::pending`. - Group(moq_net::Result>>), - Datagram(moq_net::Result>), + Next(Option<(u64, Item)>), Timer, } @@ -139,26 +101,14 @@ impl Track { track.timescale = timescale(&stored.info)?; track.info = Some(stored.info); track.end = stored.end; - let now = moq_clock::now(); + track.next = stored.next; let mut objects = stored.objects; track.newest = objects.pop(); if track.shared.config.dvr.is_some() { // Their age is unknown, so their window starts now. + let now = moq_clock::now(); track.superseded = objects.into_iter().map(|path| (now, path)).collect(); } - if let Some(newest) = stored.newest { - track.next = Some(newest.end()?); - // An untimed object keeps growing in place; a timed one is immutable. - if track.timescale.is_none() { - track.object = Some(Building { - object: newest, - opened: now, - unstored: None, - stored: true, - run_pts: 0, - }); - } - } Ok(track) } @@ -234,51 +184,39 @@ impl Track { Ok(track) => track, Err(err) => return Ok(error(&err)), }; - let mut subscriber = match track.subscribe(subscription).await { + let subscriber = match track.subscribe(subscription).await { Ok(subscriber) => subscriber, Err(err) => return Ok(error(&err)), }; - let mut run = Run { - max_delay, - pending: BTreeMap::new(), - current: None, - gap: None, - first: None, - groups_done: false, - datagrams_done: false, - }; + let mut ordered = Ordered::new(subscriber, max_delay, self.next.map(|next| next.group)); + // The group being stored, frame by frame. + let mut current: Option> = None; loop { - self.advance(&mut run, false)?; self.flush_due(moq_clock::now())?; self.write().await?; self.prune().await?; - if run.groups_done && run.datagrams_done { - break; - } - let deadlines = [self.flush_at(), run.gap, self.prune_at()]; - let mut timer = deadlines + let mut timer = [self.flush_at(), self.prune_at()] .into_iter() .flatten() .min() .map(|at| Box::pin(moq_clock::sleep_until(at))); let event = kio::wait(|waiter| { - if let Some(current) = run.current.as_mut() { - let index = current.index(); - if let Poll::Ready(frame) = current.poll_read_frame(waiter) { - return Poll::Ready(Event::Frame(index, frame)); + match current.as_mut() { + Some(current) => { + let index = current.index(); + if let Poll::Ready(frame) = current.poll_read_frame(waiter) { + return Poll::Ready(Event::Frame(index, frame)); + } + // Arrival times decide the waits, so arrivals are taken meanwhile. + ordered.poll_arrivals(waiter); + } + None => { + if let Poll::Ready(next) = ordered.poll_next(waiter) { + return Poll::Ready(Event::Next(next)); + } } - } - if !run.groups_done - && let Poll::Ready(group) = subscriber.poll_recv_group(waiter) - { - return Poll::Ready(Event::Group(group.map(|group| group.map(Box::new)))); - } - if !run.datagrams_done - && let Poll::Ready(datagram) = subscriber.poll_recv_datagram(waiter) - { - return Poll::Ready(Event::Datagram(datagram)); } ready_at(&mut timer, waiter).map(|()| Event::Timer) }) @@ -286,82 +224,19 @@ impl Track { let now = moq_clock::now(); match event { - Event::Frame(index, read) => self.frame(&mut run, index, read, now)?, - Event::Group(Ok(Some(group))) => { - if self.next.is_none_or(|next| group.sequence >= next.group) { - run.pending.insert(group.sequence, Pending::Group(group, now)); - } - } - Event::Datagram(Ok(Some(datagram))) => { - if self.next.is_none_or(|next| datagram.sequence >= next.group) { - run.pending.insert(datagram.sequence, Pending::Datagram(datagram, now)); - } - } - // How the track ended is read from the subscription below. - Event::Group(_) => run.groups_done = true, - Event::Datagram(_) => run.datagrams_done = true, - Event::Timer => {} - } - } - - // Everything received is stored, gaps and all, now that nothing more arrives. - while run.current.is_some() || !run.pending.is_empty() { - self.advance(&mut run, true)?; - if let Some(current) = run.current.as_mut() { - let index = current.index(); - let read = kio::wait(|waiter| current.poll_read_frame(waiter)).await; - self.frame(&mut run, index, read, moq_clock::now())?; - } - } - - let end = match kio::wait(|waiter| subscriber.poll_finished(waiter)).await { - Ok(end) => End::Finish(end.into()), - Err(err) => error(&err), - }; - self.flush()?; - self.write().await?; - Ok(end) - } - - /// Take pending sequences in order while the next one is here or its gap was given up. - fn advance(&mut self, run: &mut Run, draining: bool) -> Result<()> { - run.gap = None; - while run.current.is_none() { - let Some(entry) = run.pending.first_entry() else { - return Ok(()); - }; - let sequence = *entry.key(); - let waiting = match self.next { - Some(next) if sequence < next.group => { - entry.remove(); - continue; - } - Some(next) => (sequence > next.group).then(|| entry.get().arrived()), - // Nothing is below 0; anything else waits from the first arrival, so a stream of - // ever lower sequences cannot hold the track back. - None => { - let first = *run.first.get_or_insert(entry.get().arrived()); - (sequence > 0).then_some(first) - } - }; - if let Some(waiting) = waiting - && !draining && moq_clock::now() < waiting + run.max_delay - { - run.gap = Some(waiting + run.max_delay); - return Ok(()); - } - match entry.remove() { - Pending::Datagram(datagram, arrived) => { + Event::Frame(index, read) => self.frame(&mut current, index, read, now)?, + Event::Next(Some((sequence, Item::Datagram(datagram)))) => { let frame = Frame { timestamp: self.stamp(datagram.timestamp)?, payload: datagram.payload, }; - self.push(sequence, 0, frame, true, arrived)?; + self.push(sequence, 0, frame, true, now)?; self.next = Some(Position::group(sequence + 1)); } - Pending::Group(mut group, _) => { - // A resumed group continues after its stored frames. The clamp also starts a group - // whose head is held elsewhere at its first frame, rather than lagging behind it. + Event::Next(Some((sequence, Item::Group(mut group)))) => { + // A resumed group continues after its stored frames. The clamp also starts a + // group whose head is held elsewhere at its first frame, rather than lagging + // behind it. let stored = self .next .filter(|next| next.group == sequence) @@ -371,23 +246,32 @@ impl Track { group: sequence, frame: group.index(), }); - run.current = Some(group); + current = Some(group); } + Event::Next(None) => break, + Event::Timer => {} } } - Ok(()) + + let end = match kio::wait(|waiter| ordered.poll_finished(waiter)).await { + Ok(end) => End::Finish(end.into()), + Err(err) => error(&err), + }; + self.close()?; + self.write().await?; + Ok(end) } /// Store the frame read at `index` of the group being stored, or move past that group once it /// ended, cleanly or not (aborted, or past the budget), keeping what it stored. fn frame( &mut self, - run: &mut Run, + current: &mut Option>, index: u64, read: moq_net::Result>, now: Instant, ) -> Result<()> { - let sequence = run.current.as_ref().expect("reading a group").sequence; + let sequence = current.as_ref().expect("reading a group").sequence; match read { Ok(Some(frame)) => { let frame = Frame { @@ -404,7 +288,7 @@ impl Track { if let Err(err) = ended { tracing::debug!(track = %self.name, sequence, %err, "group ended early"); } - run.current = None; + *current = None; self.next = Some(Position::group(sequence + 1)); } } @@ -441,7 +325,13 @@ impl Track { false => pts.saturating_sub(building.run_pts) >= duration, } } - _ => boundary && now.saturating_duration_since(building.opened) >= config.duration, + _ => { + let since = match boundary { + true => building.opened, + false => building.run_at, + }; + now.saturating_duration_since(since) >= config.duration + } }; if cut { self.close()?; @@ -454,15 +344,16 @@ impl Track { entries: Vec::new(), }, opened: now, - unstored: None, - stored: false, + unstored: now, run_pts: 0, + run_at: now, }); let pts = frame.timestamp.unwrap_or(0); match building.object.entries.last_mut() { Some(Entry::Group { sequence: last, frames }) if *last == sequence && !datagram => frames.push(frame), _ => { building.run_pts = pts; + building.run_at = now; building.object.entries.push(match datagram { true => Entry::Datagram { sequence, frame }, false => Entry::Group { @@ -472,91 +363,50 @@ impl Track { }); } } - building.unstored.get_or_insert(now); Ok(()) } - /// When the object's oldest unstored frame is due in the store. + /// When the object's oldest frame is due in the store. fn flush_at(&self) -> Option { - let unstored = self.object.as_ref()?.unstored?; - Some(unstored + self.shared.config.flush) + Some(self.object.as_ref()?.unstored + self.shared.config.flush) } + /// Store the object once its oldest frame is due, so its group continues in the next. fn flush_due(&mut self, now: Instant) -> Result<()> { if self.flush_at().is_some_and(|at| at <= now) { - self.flush()?; + self.close()?; } Ok(()) } - /// Store the object: a timed one is closed, so its group continues in the next; an untimed - /// one is rewritten in place and stays open. - fn flush(&mut self) -> Result<()> { - match self.timescale { - Some(_) => self.close(), - None => { - let Some(building) = self.object.as_mut() else { - return Ok(()); - }; - if building.unstored.take().is_some() { - let bytes = building.object.encode(false)?; - building.stored = true; - let path = self.object_path()?; - self.queue(path, bytes, true); - } - Ok(()) - } - } - } - /// Store the object for good and start the next one at the following frame. fn close(&mut self) -> Result<()> { let Some(building) = self.object.take() else { return Ok(()); }; - if building.unstored.is_none() && building.stored { - return Ok(()); - } - let timed = self.timescale.is_some(); - let bytes = building.object.encode(timed)?; + let bytes = building.object.encode(self.timescale.is_some())?; let path = push(&self.dir, &building.object.start()?.name()?)?; - self.queue(path, bytes, !timed); - Ok(()) - } - - fn object_path(&self) -> Result { - let building = self.object.as_ref().expect("an open object"); - push(&self.dir, &building.object.start()?.name()?) - } - - fn queue(&mut self, path: Path, bytes: Bytes, overwrite: bool) { - if self.newest.as_ref() != Some(&path) { - let superseded = self.newest.replace(path.clone()); - if let Some(superseded) = superseded - && self.shared.config.dvr.is_some() - { - self.superseded.push_back((moq_clock::now(), superseded)); - } + if let Some(superseded) = self.newest.replace(path.clone()) + && self.shared.config.dvr.is_some() + { + self.superseded.push_back((moq_clock::now(), superseded)); } - self.puts.push(Put { path, bytes, overwrite }); + self.puts.push((path, bytes)); + Ok(()) } /// Perform the owed writes in order. async fn write(&mut self) -> Result<()> { let store = self.shared.store.inner(); - for put in std::mem::take(&mut self.puts) { - let payload = PutPayload::from(put.bytes.clone()); - if put.overwrite { - store.put(&put.path, payload).await?; - continue; - } - match store.put_opts(&put.path, payload, PutMode::Create.into()).await { + for (path, bytes) in std::mem::take(&mut self.puts) { + let payload = PutPayload::from(bytes.clone()); + match store.put_opts(&path, payload, PutMode::Create.into()).await { Ok(_) => {} // Written already, by an earlier attempt that failed to report success. Err(object_store::Error::AlreadyExists { .. }) => { - let existing = store.get(&put.path).await?.bytes().await?; - if existing != put.bytes { - return Err(Error::Conflict(put.path.to_string())); + let existing = store.get(&path).await?.bytes().await?; + if existing != bytes { + return Err(Error::Conflict(path.to_string())); } } Err(err) => return Err(err.into()), From 9b4a7ed8e773980e72b1e4d1ecd7a33d3a6be1fc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 12:19:37 -0700 Subject: [PATCH 13/16] fix(archive): date a gap from its first sight, and read an older .info's epoch A gap's wait ran from the lowest pending arrival, so a publisher sending ever lower sequences above the gap could extend it forever while every higher group waited in memory. It now runs from the oldest pending arrival, when the gap was first seen. Removing Info::epoch made Reader refuse the .info of recordings the old Writer made with an epoch, which import archive still replays. Decoding ignores that member until Reader goes. Co-Authored-By: Claude Opus 5.5 --- quest/m1/archive/consumer.md | 3 +- rs/moq-archive/src/info.rs | 35 ++++++++++- rs/moq-archive/src/producer/ordered.rs | 87 +++++++++++++++++--------- 3 files changed, 95 insertions(+), 30 deletions(-) diff --git a/quest/m1/archive/consumer.md b/quest/m1/archive/consumer.md index 351d61164cf..a356218ea04 100644 --- a/quest/m1/archive/consumer.md +++ b/quest/m1/archive/consumer.md @@ -7,7 +7,8 @@ broadcast: every stored track, the archived catalog included, with its original groups, frames, timescale, and priority. A host publishes it at a path of its choosing; moq-relay never uses it. `moq export archive` already records through `archive::Producer`; `moq import archive` serves through -`archive::Consumer`. It replaces `moq_archive::Reader`. +`archive::Consumer`. It replaces `moq_archive::Reader`, and with it goes +`.info`'s tolerance of the `epoch` an older `Writer` stored. ## Plan diff --git a/rs/moq-archive/src/info.rs b/rs/moq-archive/src/info.rs index f26218a7659..ff90254914d 100644 --- a/rs/moq-archive/src/info.rs +++ b/rs/moq-archive/src/info.rs @@ -6,7 +6,7 @@ use crate::{Error, Result, VERSION}; /// Immutable track properties stored at `/.info`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] +#[serde(from = "Stored")] pub struct Info { /// Recording format version; must be [`VERSION`]. pub version: u64, @@ -18,6 +18,29 @@ pub struct Info { pub timescale: Option, } +/// [`Info`] as stored, also taking the `epoch` an older `Writer` recorded, so [`Reader`](crate::Reader) +/// still replays those recordings; it goes when `Reader` does. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Stored { + version: u64, + priority: u8, + #[serde(default)] + timescale: Option, + #[serde(default, rename = "epoch")] + _epoch: Option, +} + +impl From for Info { + fn from(stored: Stored) -> Self { + Self { + version: stored.version, + priority: stored.priority, + timescale: stored.timescale, + } + } +} + impl Info { /// The current format version with `priority` and `timescale`, `None` for an untimed track. pub fn new(priority: u8, timescale: Option) -> Result { @@ -105,6 +128,16 @@ mod tests { assert!(matches!(Info::new(0, Some(ID_MAX + 1)), Err(Error::Timescale(_)))); } + #[test] + fn an_older_writers_epoch_is_ignored() { + let info = Info::decode(br#"{"version":2,"priority":0,"timescale":1000,"epoch":"x"}"#).unwrap(); + assert_eq!(info, Info::new(0, Some(1000)).unwrap()); + assert_eq!( + info.encode().unwrap().as_ref(), + br#"{"version":2,"priority":0,"timescale":1000}"# + ); + } + #[test] fn unknown_members_and_missing_fields_are_refused() { assert!(matches!( diff --git a/rs/moq-archive/src/producer/ordered.rs b/rs/moq-archive/src/producer/ordered.rs index 09dd3a35571..f5a37f133d7 100644 --- a/rs/moq-archive/src/producer/ordered.rs +++ b/rs/moq-archive/src/producer/ordered.rs @@ -3,7 +3,7 @@ //! `moq_net::track::Ordered` seeks past a missing group at once and orders groups alone, so it //! fits a player, not an archive that must keep every group and datagram it can in key order. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, VecDeque}; use std::pin::Pin; use std::task::Poll; use std::time::Duration; @@ -20,19 +20,20 @@ pub(super) enum Item { } /// Hands out each sequence once every lower one was handed out or given up: at once when it is -/// the next, otherwise after `max_delay` from its arrival, or from the first arrival while no -/// start is known. Once both channels end, everything received is handed out without waiting. +/// the next, otherwise once `max_delay` passes from the oldest arrival still waiting, which is +/// when the gap below it was first seen. While no start is known, anything but 0 waits the same +/// way, since arrival is not sequence order. Once both channels end, everything received is +/// handed out without waiting. pub(super) struct Ordered { subscriber: Subscriber, max_delay: Duration, /// The next sequence to hand out; `None` until a start is known or the first wait ends. next: Option, - /// Received sequences waiting on a lower one, with when they arrived. - pending: BTreeMap, - /// When the first sequence arrived while `next` was unknown, since arrival is not sequence - /// order and a lower one may still be on its way. Fixed, so ever lower arrivals can't extend - /// the wait. - first: Option, + /// Received sequences waiting on a lower one. + pending: BTreeMap, + /// Each pending sequence with when it arrived, oldest first; ones handed out are dropped + /// lazily. The oldest dates the gap, so a later, lower arrival never extends the wait. + arrivals: VecDeque<(u64, Instant)>, timer: Option>>, groups_done: bool, datagrams_done: bool, @@ -48,7 +49,7 @@ impl Ordered { max_delay, next, pending: BTreeMap::new(), - first: None, + arrivals: VecDeque::new(), timer: None, groups_done: false, datagrams_done: false, @@ -77,8 +78,11 @@ impl Ordered { fn arrive(&mut self, sequence: u64, item: Item, now: Instant) { // A sequence names one group, so a copy of one already here or handed out is dropped. - if self.next.is_none_or(|next| sequence >= next) { - self.pending.entry(sequence).or_insert((item, now)); + if self.next.is_none_or(|next| sequence >= next) + && let std::collections::btree_map::Entry::Vacant(entry) = self.pending.entry(sequence) + { + entry.insert(item); + self.arrivals.push_back((sequence, now)); } } @@ -94,20 +98,26 @@ impl Ordered { }; }; let sequence = *entry.key(); - let arrived = entry.get().1; - let waiting = match self.next { + let gap = match self.next { Some(next) if sequence < next => { entry.remove(); continue; } - Some(next) => (sequence > next).then_some(arrived), + Some(next) => sequence > next, // Nothing is below 0. - None => (sequence > 0).then(|| *self.first.get_or_insert(arrived)), + None => sequence > 0, }; - if let Some(waiting) = waiting - && !draining - { - let deadline = waiting + self.max_delay; + if gap && !draining { + // The oldest arrival still pending is when this gap was first seen. + while self + .arrivals + .front() + .is_some_and(|(sequence, _)| !self.pending.contains_key(sequence)) + { + self.arrivals.pop_front(); + } + let (_, since) = self.arrivals.front().expect("a pending sequence arrived"); + let deadline = *since + self.max_delay; if moq_clock::now() < deadline { let timer = self .timer @@ -118,7 +128,7 @@ impl Ordered { } } } - let (item, _) = entry.remove(); + let item = self.pending.remove(&sequence).expect("the lowest pending sequence"); self.next = Some(sequence.saturating_add(1)); return Poll::Ready(Some((sequence, item))); } @@ -152,12 +162,7 @@ mod tests { } } - /// The next sequence is handed out at once, a datagram in turn with the groups, and one past a - /// missing sequence only once the max delay from its arrival passes. - #[tokio::test(start_paused = true)] - async fn only_a_missing_sequence_waits_the_max_delay() { - let broadcast = broadcast::Info::new().produce(); - let mut track = broadcast.create_track("t", track::Info::default()).unwrap(); + async fn ordered(broadcast: &broadcast::Producer, next: Option) -> Ordered { let subscription = Subscription::default() .with_order(Order::Ascending) .with_max_delay(MAX_DELAY) @@ -169,7 +174,16 @@ mod tests { .subscribe(subscription) .await .unwrap(); - let mut ordered = Ordered::new(subscriber, MAX_DELAY, None); + Ordered::new(subscriber, MAX_DELAY, next) + } + + /// The next sequence is handed out at once, a datagram in turn with the groups, and one past a + /// missing sequence only once the max delay from its arrival passes. + #[tokio::test(start_paused = true)] + async fn only_a_missing_sequence_waits_the_max_delay() { + let broadcast = broadcast::Info::new().produce(); + let mut track = broadcast.create_track("t", track::Info::default()).unwrap(); + let mut ordered = ordered(&broadcast, None).await; group(&track, 0); assert_eq!(next(&mut ordered), Some(0)); @@ -192,4 +206,21 @@ mod tests { track.finish().unwrap(); assert!(matches!(ordered.poll_next(&kio::Waiter::noop()), Poll::Ready(None))); } + + /// A gap's wait runs from when it was first seen, so ever lower arrivals above it can't extend it. + #[tokio::test(start_paused = true)] + async fn lower_arrivals_never_extend_a_gap() { + let broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track("t", track::Info::default()).unwrap(); + let mut ordered = ordered(&broadcast, Some(0)).await; + + group(&track, 10); + assert_eq!(next(&mut ordered), None); + moq_clock::sleep(MAX_DELAY - Duration::from_millis(1)).await; + group(&track, 9); + assert_eq!(next(&mut ordered), None); + moq_clock::sleep(Duration::from_millis(1)).await; + assert_eq!(next(&mut ordered), Some(9)); + assert_eq!(next(&mut ordered), Some(10)); + } } From 79b2e5a74cc93c07524db8a4118c560590ab4d8d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 12:45:55 -0700 Subject: [PATCH 14/16] fix(archive): store a run under its subscribed instance's properties The listing's track info could describe an instance the name no longer resolved to by the time the run subscribed, so a re-added track with other properties was stored under the old ones until its run ended. Each run now checks and stores the subscribed instance's info. Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/producer/tests.rs | 41 ++++++++++++++++++++++++++++ rs/moq-archive/src/producer/track.rs | 30 ++++++++++++++------ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index 2f6a5900f3f..ad06608f94e 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -768,6 +768,47 @@ async fn a_re_added_track_continues_and_its_marker_records_the_last_run() { assert_eq!(end(&store, "video").await, Some(End::Error(64 + 3))); } +/// A track re-added with other properties after it was listed is refused before any of its +/// frames is stored under the listed properties. +#[tokio::test(start_paused = true)] +async fn a_re_added_track_is_stored_under_its_own_properties() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let first = broadcast.create_track("video", timed().with_priority(1)).unwrap(); + store.inner().hold_puts("video/.info"); + let task = spawn(&store, &broadcast, config().with_flush(SECOND)); + settle().await; + drop(first); + settle().await; + + let second = broadcast.create_track("video", timed().with_priority(2)).unwrap(); + group(&second, 0, &[(0, "x")]); + store.inner().release(); + settle().await; + sleep(SECOND).await; + settle().await; + assert!(objects(&store, "video", true).await.is_empty()); + + second.finish().unwrap(); + broadcast.close(); + let err = moq_clock::timeout(Duration::from_secs(1000), task) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!( + matches!( + err, + Error::Priority { + existing: 1, + intended: 2 + } + ), + "{err}" + ); + assert!(objects(&store, "video", true).await.is_empty()); +} + #[tokio::test(start_paused = true)] async fn a_stopped_producer_writes_no_marker() { let store = store(); diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index 4331fe642f0..2e6c4b831bd 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -30,7 +30,6 @@ pub(super) async fn record( ) -> Result<(String, Option)> { let mut track = Track::new(shared, &name, stored)?; while let Some(info) = track.idle(&queue).await? { - track.properties(&info).await?; let end = track.run(&info).await?; track.end = Some(end); } @@ -166,8 +165,8 @@ impl Track { /// Record one subscription until both its groups and its datagrams end, returning how. async fn run(&mut self, info: &moq_net::track::Info) -> Result { - let config = &self.shared.config; - let max_delay = info.max_age.map_or(config.max_delay, |age| age.min(config.max_delay)); + let shared = self.shared.clone(); + let config = &shared.config; let range = match self.next { Some(next) => Range { start: Anchor::Absolute(next), @@ -177,17 +176,25 @@ impl Track { }; let subscription = Subscription::default() .with_order(Order::Ascending) - .with_max_delay(max_delay) + .with_max_delay(delay(config, info)) .with_ranges([range]); - let track = match self.shared.broadcast.track(&self.name) { - Ok(track) => track, - Err(err) => return Ok(error(&err)), + let subscribed = match self.shared.broadcast.track(&self.name) { + Ok(track) => track.subscribe(subscription).await, + Err(err) => Err(err), }; - let subscriber = match track.subscribe(subscription).await { + let subscriber = match subscribed { Ok(subscriber) => subscriber, - Err(err) => return Ok(error(&err)), + Err(err) => { + // The track keeps its `.info` beside the `end` this run leaves. + self.properties(info).await?; + return Ok(error(&err)); + } }; + // The listing's info may describe an instance the name no longer resolves to, so the + // subscribed one's properties are what its frames are stored under. + self.properties(subscriber.info()).await?; + let max_delay = delay(config, subscriber.info()); let mut ordered = Ordered::new(subscriber, max_delay, self.next.map(|next| next.group)); // The group being stored, frame by frame. @@ -445,6 +452,11 @@ fn timescale(info: &Info) -> Result> { .transpose() } +/// How long a missing group is waited for: the configured cap, or the publisher's max age below it. +fn delay(config: &super::Config, info: &moq_net::track::Info) -> Duration { + info.max_age.map_or(config.max_delay, |age| age.min(config.max_delay)) +} + /// `duration` in `timescale` units, saturating. fn units(duration: Duration, timescale: Timescale) -> u64 { let units = duration.as_nanos() * u128::from(timescale.as_u64()) / 1_000_000_000; From dc0e354a663baadcdbfdcbef7cddc0fa9dc82476 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 13:01:39 -0700 Subject: [PATCH 15/16] fix(archive): bound arrivals and queued infos, and record re-announced instances - Ordered drops each arrival it hands out, so a gap-free track holds none; before, every arrival stayed for the recording's life. - The listing keeps at most the newest info queued per track, however often a blocked recorder's track is re-added. - A run requests the configured cap, since the listing's max age may describe an older instance and a skipped backlog is not resent; the gap wait still takes the subscribed instance's max age. - export archive records an epochless route announced again while its last recording drains, and records an epoch-pinned one again once its recording ends, instead of dropping the announcement. Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/producer/mod.rs | 3 + rs/moq-archive/src/producer/ordered.rs | 38 ++++++++--- rs/moq-archive/src/producer/track.rs | 16 ++--- rs/moq-cli/src/archive.rs | 88 ++++++++++++++++++++------ 4 files changed, 108 insertions(+), 37 deletions(-) diff --git a/rs/moq-archive/src/producer/mod.rs b/rs/moq-archive/src/producer/mod.rs index 16f871c6e0d..ae7551ecc7a 100644 --- a/rs/moq-archive/src/producer/mod.rs +++ b/rs/moq-archive/src/producer/mod.rs @@ -195,6 +195,9 @@ impl Producer { recorders.push(record.boxed()); queue }); + // Only the newest info matters, so one still queued is replaced: a blocked + // recorder holds at most one, however often its track is re-added. + let _ = queue.try_pop(); // Only fails once closed, which happens when the listing ends. let _ = queue.try_push(info); } diff --git a/rs/moq-archive/src/producer/ordered.rs b/rs/moq-archive/src/producer/ordered.rs index f5a37f133d7..7479b78e551 100644 --- a/rs/moq-archive/src/producer/ordered.rs +++ b/rs/moq-archive/src/producer/ordered.rs @@ -31,8 +31,9 @@ pub(super) struct Ordered { next: Option, /// Received sequences waiting on a lower one. pending: BTreeMap, - /// Each pending sequence with when it arrived, oldest first; ones handed out are dropped - /// lazily. The oldest dates the gap, so a later, lower arrival never extends the wait. + /// Each pending sequence with when it arrived, oldest first. The front is always pending, + /// and the ones handed out behind it go once it does, so this holds at most what arrived + /// within a max delay. The oldest dates the gap, so a later, lower arrival never extends it. arrivals: VecDeque<(u64, Instant)>, timer: Option>>, groups_done: bool, @@ -101,6 +102,7 @@ impl Ordered { let gap = match self.next { Some(next) if sequence < next => { entry.remove(); + self.trim(); continue; } Some(next) => sequence > next, @@ -109,13 +111,6 @@ impl Ordered { }; if gap && !draining { // The oldest arrival still pending is when this gap was first seen. - while self - .arrivals - .front() - .is_some_and(|(sequence, _)| !self.pending.contains_key(sequence)) - { - self.arrivals.pop_front(); - } let (_, since) = self.arrivals.front().expect("a pending sequence arrived"); let deadline = *since + self.max_delay; if moq_clock::now() < deadline { @@ -129,11 +124,23 @@ impl Ordered { } } let item = self.pending.remove(&sequence).expect("the lowest pending sequence"); + self.trim(); self.next = Some(sequence.saturating_add(1)); return Poll::Ready(Some((sequence, item))); } } + /// Drop arrivals handed out, up to the oldest one still pending. + fn trim(&mut self) { + while self + .arrivals + .front() + .is_some_and(|(sequence, _)| !self.pending.contains_key(sequence)) + { + self.arrivals.pop_front(); + } + } + /// How the track ended, once it has. pub(super) fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { self.subscriber.poll_finished(waiter) @@ -223,4 +230,17 @@ mod tests { assert_eq!(next(&mut ordered), Some(9)); assert_eq!(next(&mut ordered), Some(10)); } + + /// A gap-free track keeps no arrival it handed out. + #[tokio::test(start_paused = true)] + async fn handed_out_arrivals_are_dropped() { + let broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track("t", track::Info::default()).unwrap(); + let mut ordered = ordered(&broadcast, None).await; + for sequence in 0..100 { + group(&track, sequence); + assert_eq!(next(&mut ordered), Some(sequence)); + } + assert!(ordered.arrivals.is_empty()); + } } diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index 2e6c4b831bd..7059a8619c7 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -111,8 +111,8 @@ impl Track { Ok(track) } - /// Wait for the next run, pruning meanwhile: the newest of the queued track infos, or `None` - /// once the listing ended. + /// Wait for the next run, pruning meanwhile: the queued track info, which the listing keeps + /// to the newest, or `None` once the listing ended. async fn idle(&mut self, queue: &kio::Queue) -> Result> { loop { let mut timer = self.prune_at().map(|at| Box::pin(moq_clock::sleep_until(at))); @@ -124,13 +124,7 @@ impl Track { }) .await; match popped { - // A track removed and re-added while a run was still draining queues it twice. - Some(Ok(mut info)) => { - while let Ok(Some(newer)) = queue.try_pop() { - info = newer; - } - return Ok(Some(info)); - } + Some(Ok(info)) => return Ok(Some(info)), Some(Err(kio::Closed)) => return Ok(None), None => self.prune().await?, } @@ -174,9 +168,11 @@ impl Track { }, None => Range::oldest(), }; + // The cap, since `info` may describe an older instance than the one subscribed, and a + // backlog a smaller budget skipped is not sent again. The wait for a gap is narrowed below. let subscription = Subscription::default() .with_order(Order::Ascending) - .with_max_delay(delay(config, info)) + .with_max_delay(config.max_delay) .with_ranges([range]); let subscribed = match self.shared.broadcast.track(&self.name) { diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 18e349b25fe..3c28bbe9c9b 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -72,43 +72,77 @@ pub async fn export(origin: moq_net::origin::Consumer, prefix: String, args: Exp let prefix = moq_net::Path::new(&prefix).to_owned(); let mut announced = origin.clone().with_hidden(hidden).announced(); let mut recordings = tokio::task::JoinSet::new(); - // How many recordings of each instance run, so a re-announced one is not recorded twice. - let mut running: HashMap = HashMap::new(); + let mut running = Running::default(); tracing::info!(%prefix, store = %args.store, "recording"); notify_ready(); loop { - tokio::select! { + let instance = tokio::select! { event = announced.next() => { - let (announce, restart) = match event.context("announcements ended")? { - moq_net::announce::Event::Start(announce) => (announce, false), - // Another publisher instance: an epochless one is still a new recording. - moq_net::announce::Event::Restart(announce) => (announce, true), + let announce = match event.context("announcements ended")? { + // Another publisher instance, or the same one announced again. + moq_net::announce::Event::Start(announce) | moq_net::announce::Event::Restart(announce) => announce, _ => continue, }; if !announce.prefix.has_prefix(&prefix) { continue; } let instance: Instance = (announce.prefix, announce.route.epoch); - let count = running.entry(instance.clone()).or_default(); - if *count > 0 && !(restart && instance.1.is_none()) { + if !running.start(&instance) { continue; } - *count += 1; - let key = key(&root, &instance)?; - let store = moq_archive::Store::new(store.clone(), key); - recordings.spawn(record(origin.clone(), instance, store, config.clone())); + instance } Some(done) = recordings.join_next() => { let (instance, result) = done.context("recording panicked")?; result?; - if let Some(count) = running.get_mut(&instance) { - *count -= 1; - if *count == 0 { - running.remove(&instance); - } + if !running.done(&instance) { + continue; } + instance + } + }; + let key = key(&root, &instance)?; + let store = moq_archive::Store::new(store.clone(), key); + recordings.spawn(record(origin.clone(), instance, store, config.clone())); + } +} + +/// The epoch-pinned instances recording, so each records under its key once at a time, and again +/// when it was announced anew meanwhile. +#[derive(Default)] +struct Running(HashMap); + +impl Running { + /// Whether to record `instance` now that it was announced. An epochless one is a new instance + /// each time, under a fresh key. + fn start(&mut self, instance: &Instance) -> bool { + if instance.1.is_none() { + return true; + } + match self.0.get_mut(instance) { + Some(again) => { + *again = true; + false + } + None => { + self.0.insert(instance.clone(), false); + true + } + } + } + + /// Whether to record `instance` again, now that its recording ended. + fn done(&mut self, instance: &Instance) -> bool { + match self.0.get_mut(instance) { + Some(again) if *again => { + *again = false; + true + } + _ => { + self.0.remove(instance); + false } } } @@ -324,6 +358,24 @@ mod tests { assert!(!files.iter().any(|file| file.starts_with("other")), "{files:?}"); } + /// An epochless route announced again while its last recording drains is a new recording, and + /// an epoch-pinned one records again once its recording ends. + #[test] + fn a_re_announced_instance_is_recorded() { + let mut running = Running::default(); + let epochless = (moq_net::Path::new("live/a").to_owned(), None); + assert!(running.start(&epochless)); + assert!(running.start(&epochless)); + assert!(!running.done(&epochless)); + + let pinned = (moq_net::Path::new("live/b").to_owned(), Some(moq_net::Epoch::mint())); + assert!(running.start(&pinned)); + assert!(!running.start(&pinned)); + assert!(running.done(&pinned)); + assert!(!running.done(&pinned)); + assert!(running.start(&pinned)); + } + /// A route without an epoch records under one minted per instance, so two instances never /// share a key prefix. #[test] From 15ae0bcac0dd9b4aa2ce2102c42e3b54f9c5f9ab Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 15:08:59 -0700 Subject: [PATCH 16/16] fix(cli): resolve each archive announcement when read, and record a broadcast once A burst of epochless announcements read after one instance took over resolved it once per announcement, recording it under two minted keys. Each announcement now requests its broadcast when the loop reads it, and a recording skips a broadcast another recording already claimed. Also drop the stale untimed-refusal MIGRATE entries this PR obsoletes, and say export archive leaves hidden routes out unless named. Co-Authored-By: Claude Opus 5.5 --- doc/bin/cli.md | 7 ++-- rs/moq-archive/MIGRATE.md | 2 +- rs/moq-cli/MIGRATE.md | 6 +-- rs/moq-cli/src/archive.rs | 80 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 80 insertions(+), 15 deletions(-) diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 2a5feb49e09..0aed2e4eb97 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -307,9 +307,10 @@ moq --connect https://relay.example.com/anon --broadcast event-replay.hang impor `export archive` records every broadcast announced at or under `--broadcast` with [moq-archive](https://docs.rs/moq-archive), each publisher instance under -`/@` in the store, until the connection ends. A broadcast -path segment that starts with `@` is stored as `%40`, so no recording lands -inside another. Every track is stored as its own objects, cut about every 2 s +`/@` in the store, until the connection ends. Like +`announced`, it leaves out a broadcast whose name starts with `.` unless +`--broadcast` names it. A broadcast path segment that starts with `@` is +stored as `%40`, so no recording lands inside another. Every track is stored as its own objects, cut about every 2 s of media (or wall clock for an [untimed](/concept/moq-lite#subscriptions) track) and at least every 4 s, splitting a long group between frames. A broadcast restarted under a new [epoch](/concept/moq-lite#publisher-epochs) records under its own key; a route diff --git a/rs/moq-archive/MIGRATE.md b/rs/moq-archive/MIGRATE.md index fc5a2ce96e1..35fa7a38048 100644 --- a/rs/moq-archive/MIGRATE.md +++ b/rs/moq-archive/MIGRATE.md @@ -18,6 +18,6 @@ The new layout is not readable by `Reader`, which still replays timeline recordi `Info::new` and `Info::timescale` take and return `Option`, and `Error::TimescaleMismatch` carries `Option` fields. `segment::Frame::timestamp` is `Option`. `Object::decode(buf, timed)` and `Store::get_segments(track, segment, timed)` take whether the track is timed. -`Error::Untimed` now means a DVR cannot record an untimed track; mismatched timing is `Error::Timing`. +Mismatched timing is `Error::Timing`. Untimed recordings use the version 2 format, which an older version 2 reader refuses. Pass `Some(timescale)` to `Info::new`, and `info.timescale.is_some()` as `timed`. diff --git a/rs/moq-cli/MIGRATE.md b/rs/moq-cli/MIGRATE.md index 10ba9cc3857..c52537abb08 100644 --- a/rs/moq-cli/MIGRATE.md +++ b/rs/moq-cli/MIGRATE.md @@ -8,7 +8,7 @@ How to update your invocation and config for each breaking change, newest first. `moq export archive ` records every broadcast announced at or under `--broadcast`, each instance under `//@`, until the connection ends, instead of one broadcast at ``. Point `` at the root you want recordings under, and drop `--catalog-format`, which it now refuses. -`--retention` now works on untimed tracks. +Untimed tracks record, and `--retention` works on them. `import archive` does not read the new layout yet; it still replays recordings made by older `moq` versions. ### `--cluster-lan-secret` is a file path only (#5910) @@ -69,10 +69,6 @@ A restarted publisher exits 1 unless `--stitch` follows it as a program switch. moq export ts --linger 10s --stitch ... ``` -### `moq export archive` refuses untimed tracks - -Recording a track with no timestamps fails with `Untimed`, so an archive over moq-lite before 05 or moq-transport drafts 14 to 16 fails instead of recording arrival times. - ### `moq export fmp4` fixes its track set at the init segment A rendition added, changed, or left undescribed after the init segment ends the export with an error instead of writing a track missing from the moov. diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 3c28bbe9c9b..4c8aa754b54 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -73,6 +73,7 @@ pub async fn export(origin: moq_net::origin::Consumer, prefix: String, args: Exp let mut announced = origin.clone().with_hidden(hidden).announced(); let mut recordings = tokio::task::JoinSet::new(); let mut running = Running::default(); + let claims = Claims::default(); tracing::info!(%prefix, store = %args.store, "recording"); notify_ready(); @@ -105,7 +106,51 @@ pub async fn export(origin: moq_net::origin::Consumer, prefix: String, args: Exp }; let key = key(&root, &instance)?; let store = moq_archive::Store::new(store.clone(), key); - recordings.spawn(record(origin.clone(), instance, store, config.clone())); + // Requested now, so an announcement resolves the instance serving it when read, not + // whichever one serves by the time its task runs. + let request = origin.request_broadcast(&instance.0, instance.1.clone()); + recordings.spawn(record(request, instance, store, config.clone(), claims.clone())); + } +} + +/// The broadcasts being recorded. Announcements read after another instance took over all +/// resolve that one, so it is recorded once. +#[derive(Clone, Default)] +struct Claims(Arc>>>); + +impl Claims { + /// Claim `broadcast` for recording, or `None` when it is recorded already. + fn claim(&self, path: &moq_net::PathOwned, broadcast: &moq_net::broadcast::Consumer) -> Option { + let mut claims = self.0.lock().unwrap(); + let claimed = claims.entry(path.clone()).or_default(); + if claimed.iter().any(|other| other.is_clone(broadcast)) { + return None; + } + claimed.push(broadcast.clone()); + Some(Claim { + claims: self.clone(), + path: path.clone(), + broadcast: broadcast.clone(), + }) + } +} + +/// A broadcast being recorded, released on drop. +struct Claim { + claims: Claims, + path: moq_net::PathOwned, + broadcast: moq_net::broadcast::Consumer, +} + +impl Drop for Claim { + fn drop(&mut self) { + let mut claims = self.claims.0.lock().unwrap(); + if let Some(claimed) = claims.get_mut(&self.path) { + claimed.retain(|other| !other.is_clone(&self.broadcast)); + if claimed.is_empty() { + claims.remove(&self.path); + } + } } } @@ -148,19 +193,22 @@ impl Running { } } -/// Record one publisher instance until its broadcast ends. +/// Record one publisher instance until its broadcast ends, unless another recording has it. async fn record( - origin: moq_net::origin::Consumer, + request: impl Future>, instance: Instance, store: moq_archive::Store>, config: moq_archive::producer::Config, + claims: Claims, ) -> (Instance, anyhow::Result<()>) { let result = async { - let (path, epoch) = &instance; - let broadcast = origin - .request_broadcast(path, epoch.clone()) + let path = &instance.0; + let broadcast = request .await .with_context(|| format!("broadcast `{path}` is unavailable"))?; + let Some(_claim) = claims.claim(path, &broadcast) else { + return Ok(()); + }; tracing::info!(%path, key = %store.prefix(), "recording broadcast"); moq_archive::Producer::new(store, broadcast, config) .run() @@ -376,6 +424,26 @@ mod tests { assert!(running.start(&pinned)); } + /// Announcements that resolve the same broadcast record it once at a time, while another + /// instance at the path records beside it. + #[tokio::test] + async fn a_resolved_broadcast_is_recorded_once() { + let origin = moq_tokio::origin::spawn(); + let path = moq_net::Path::new("live/a").to_owned(); + let newest = origin.create_broadcast(&path).unwrap(); + let older = origin.create_broadcast(&path).unwrap(); + + let claims = Claims::default(); + let claim = claims.claim(&path, &newest.consume()).expect("first claim"); + assert!(claims.claim(&path, &newest.consume()).is_none()); + let beside = claims.claim(&path, &older.consume()).expect("another instance"); + + drop(claim); + assert!(claims.claim(&path, &newest.consume()).is_some()); + drop(beside); + assert!(claims.0.lock().unwrap().is_empty()); + } + /// A route without an epoch records under one minted per instance, so two instances never /// share a key prefix. #[test]