diff --git a/doc/concept/archives.md b/doc/concept/archives.md index 3330d3764f0..c638a9f54b7 100644 --- a/doc/concept/archives.md +++ b/doc/concept/archives.md @@ -102,14 +102,19 @@ like any other broadcast; it does not look in object storage on a cache miss. Each track starts live at its newest stored group and serves older groups from storage when requested by their start. It advertises no max age, so the source's live max age cannot make archived history stale. A growing recording is -periodically listed for new tracks and objects. Once `.end` appears, refreshing +periodically listed for new tracks and objects, and each track is listed whole +again as it grows, or once a read finds an object gone, so a server attached to +a pruned recording tracks what the store retains. Once `.end` appears, refreshing stops and tracks end as recorded; without it, including when a recorder stopped early, the archive stays live. Empty or unrecognized prefixes are refused. -Serving history is not paced playback from the beginning. Time ranges currently -resolve against the served track's cache, not the archive's stored keys, so they -cannot reliably seek old stored content. There is no automatic switch from a -live broadcast to its archive at the original path. +A time range the served track's cache can't resolve is looked up from the stored +keys: its start reaches the stored group containing that time, however old, or +the oldest group the store still holds from its first frame when the time is +older, such as one retention pruned; its end stops just past the group +containing it. Serving +history is not paced playback from the beginning, and there is no automatic +switch from a live broadcast to its archive at the original path. The archived catalog is an ordinary track. [HLS](/bin/hls) serves a recording from its media tracks, numbered on the catalog's `sync.interval`, just as diff --git a/quest/m1/precut-fixes/archive-retention-index.md b/quest/m1/precut-fixes/archive-retention-index.md deleted file mode 100644 index ccb949a16fa..00000000000 --- a/quest/m1/precut-fixes/archive-retention-index.md +++ /dev/null @@ -1,9 +0,0 @@ -# [M] Bound the live archive reader index by retention - -## Goal - -A rolling archive reader tracks retained objects, and a seek across the expired boundary can reach the oldest retained group. - -## Plan - -The recorder prunes objects while the consumer index lists forward and forgets old keys after failed reads. The audit traced growth without rewinds and a seek that can stop on the first deleted object. Exercise an attached reader over repeated retention rotations and an evicted byte cache. Measure index size against retained objects, exercise a seek across the boundary, and fix any confirmed failure. Correct the archive guide's claim about time lookup. diff --git a/rs/moq-archive/src/consumer/index.rs b/rs/moq-archive/src/consumer/index.rs index e578f410ae9..64fbeb37e62 100644 --- a/rs/moq-archive/src/consumer/index.rs +++ b/rs/moq-archive/src/consumer/index.rs @@ -1,6 +1,8 @@ -//! One track's object keys, listed once and extended after its newest key. +//! One track's object keys, listed once, extended after its newest key, and listed whole again +//! to drop what the store pruned. use std::collections::BTreeMap; +use std::sync::Mutex; use futures::TryStreamExt; use moq_net::Timescale; @@ -26,43 +28,82 @@ pub(super) struct Listed { } /// A track's stored objects, by where each starts. +/// +/// Retention deletes objects that a listing after the newest key never sees, so the whole +/// directory is listed again once the objects added since its last whole listing outnumber the +/// ones that listing found, and whenever a read finds one gone. The index then holds at most +/// about twice what the store retains, for one listed key per added object. pub(super) struct Index { pub dir: Path, pub timescale: Option, /// Each object's key. A group's sequence is its start, so these sort by time too. objects: BTreeMap, - /// The newest key listed, which the next listing starts after. + /// `.info`, which a listing of the whole directory starts after. + origin: Path, + /// The newest key the live feed took, which the next listing starts after. cursor: Path, + /// The newest start ever indexed. Only objects after it are added, so one dropped up to it, + /// because the store pruned it or a read found it gone, never comes back from a stale listing. + known: Option, + /// How many objects the last whole listing left, and how many listings added since. + listed: usize, + added: usize, } impl Index { pub fn new(dir: Path, info: &Info) -> Result { - let cursor = crate::path::push(&dir, layout::TRACK_INFO)?; + let origin = crate::path::push(&dir, layout::TRACK_INFO)?; Ok(Self { dir, timescale: super::track::timescale(info)?, objects: BTreeMap::new(), - cursor, + cursor: origin.clone(), + origin, + known: None, + listed: 0, + added: 0, }) } - pub fn timed(&self) -> bool { - self.timescale.is_some() + /// How many objects it holds. + #[cfg(test)] + pub fn len(&self) -> usize { + self.objects.len() } - /// The newest key listed, for the next listing to start after. - pub fn cursor(&self) -> &Path { - &self.cursor + pub fn timed(&self) -> bool { + self.timescale.is_some() } - /// Take in the objects a listing after [`Self::cursor`] found. - pub fn extend(&mut self, objects: &[Listing]) { + /// Add the objects a listing found that are newer than any indexed before. + fn add(&mut self, objects: &[Listing]) { for (start, path) in objects { - self.objects.insert(*start, path.clone()); - self.cursor = path.clone(); + if Some(*start) > self.known { + self.objects.insert(*start, path.clone()); + self.known = Some(*start); + self.added += 1; + } } } + /// The newest start ever indexed, when the next refresh should list the whole directory, + /// which [`Self::reconcile`] then reconciles up to. + fn due(&self) -> Option> { + (self.added > self.listed).then_some(self.known) + } + + /// Take in a listing of the whole directory, `found`: drop the objects up to `through`, all + /// indexed before it began, that it no longer holds, since the store pruned them, and add + /// the newer ones. The cursor stays, so the live feed still takes the new ones. + fn reconcile(&mut self, through: Option, found: &[Listing]) { + self.objects.retain(|start, _| { + Some(*start) > through || found.binary_search_by_key(start, |(start, _)| *start).is_ok() + }); + self.add(found); + self.listed = self.objects.len(); + self.added = 0; + } + /// The newest object, which holds the newest stored sequence. pub fn newest(&self) -> Option<(Start, Path)> { self.objects @@ -123,11 +164,53 @@ impl Index { }; Some((*start, path.clone())) } +} - /// Forget an object the store no longer has, such as one a DVR pruned. - pub fn forget(&mut self, start: Start) { - self.objects.remove(&start); +/// Take in the objects new since `index`'s cursor and return them, with the track's `end` when +/// `end` is set. When the index is due, the listing covers the whole directory, dropping what +/// the store pruned. +pub(super) async fn refresh(store: &S, index: &Mutex, end: bool) -> Result { + let (dir, from, cursor, through) = { + let index = index.lock().unwrap(); + let through = index.due(); + let from = match through { + Some(_) => index.origin.clone(), + None => index.cursor.clone(), + }; + (index.dir.clone(), from, index.cursor.clone(), through) + }; + let mut listed = list(store, &dir, &from, end).await?; + let mut index = index.lock().unwrap(); + if let Some(through) = through { + index.reconcile(through, &listed.objects); + listed.objects.retain(|(_, path)| *path > cursor); + } + index.add(&listed.objects); + if let Some((_, path)) = listed.objects.last() { + index.cursor = path.clone(); + } + // The first listing starts at the origin too. + if from == index.origin { + index.listed = index.objects.len(); + index.added = 0; } + Ok(listed) +} + +/// Drop the object starting at `missing`, which a read found gone, and with it every other +/// object the store pruned, by listing the whole directory. Lists nothing when another read +/// dropped it already. A listing that still holds it never adds it back. +pub(super) async fn pruned(store: &S, index: &Mutex, missing: Start) -> Result<()> { + let (dir, from, through) = { + let mut index = index.lock().unwrap(); + if index.objects.remove(&missing).is_none() { + return Ok(()); + } + (index.dir.clone(), index.origin.clone(), index.known) + }; + let listed = list(store, &dir, &from, false).await?; + index.lock().unwrap().reconcile(through, &listed.objects); + Ok(()) } /// List a track's directory after `cursor`, refusing anything the layout does not name, and diff --git a/rs/moq-archive/src/consumer/mod.rs b/rs/moq-archive/src/consumer/mod.rs index 166b7c3c4ec..a9b3d83631a 100644 --- a/rs/moq-archive/src/consumer/mod.rs +++ b/rs/moq-archive/src/consumer/mod.rs @@ -11,7 +11,10 @@ //! however old, and a time end stops just past the group containing it. //! //! A growing archive is followed: every [`Config::refresh`], one delimiter LIST of the prefix -//! finds added tracks, and one LIST per track after its newest key finds new objects. Once the +//! finds added tracks, and one LIST per track after its newest key finds new objects. That LIST +//! covers the whole track instead once the objects added since its last whole listing outnumber +//! the ones that listing found, and a read that finds an object gone lists the whole track at +//! once, so a reader attached to a pruned archive indexes about what the store retains. Once the //! prefix holds `.end`, each track ends as recorded and refreshing stops. Without it the archive //! stays live, which is also how one whose producer stopped early reads. //! @@ -321,12 +324,8 @@ async fn refresh(shared: Arc>, mut tracks: Vec object_store::path::Path { layout::track_dir(self.store.prefix(), track).unwrap() } @@ -959,3 +971,232 @@ async fn a_scale_less_track_looks_up_a_time_by_key() { }; assert_eq!(read_range(&broadcast, "log", range).await, [2, 5]); } + +/// Serve `store` on `broadcast` with `config`, returning each track's index. +async fn open( + store: &Store, + broadcast: &broadcast::Producer, + config: Config, +) -> (HashMap>>, JoinHandle>) { + let consumer = Consumer::open(store.clone(), broadcast.clone(), config.with_refresh(REFRESH)) + .await + .unwrap(); + let indexes = consumer + .tracks + .iter() + .map(|(name, track)| (name.clone(), track.index.clone())) + .collect(); + (indexes, tokio::spawn(consumer.run())) +} + +/// A reader attached to a rolling archive indexes about what the store retains, however many +/// retention rotations pass without it reading history. +#[tokio::test(start_paused = true)] +async fn a_rolling_archive_index_tracks_retention() { + const RETAINED: u64 = 4; + let archive = Archive::new(); + archive.track("video", 0, true).await; + for sequence in 0..RETAINED { + archive.object("video", 0, vec![timed(sequence, 0..1)]).await; + } + + let broadcast = broadcast::Info::new().produce(); + let (indexes, _run) = open(&archive.store, &broadcast, Config::default()).await; + let index = &indexes["video"]; + for sequence in RETAINED..RETAINED + 100 { + archive.object("video", 0, vec![timed(sequence, 0..1)]).await; + archive.prune("video", timed(sequence - RETAINED, 0..1).sequence).await; + sleep(REFRESH).await; + let held = index.lock().unwrap().len(); + // Twice the retained objects, plus the one a refresh adds before it lists them all. + assert!( + held <= 2 * RETAINED as usize + 1, + "{held} objects indexed after {sequence}, {RETAINED} retained" + ); + } +} + +/// A time older than what retention kept reaches the oldest retained group, though the reader +/// still indexes the pruned objects and caches none of them, with one listing rather than a GET +/// per pruned object. +#[tokio::test(start_paused = true)] +async fn a_seek_across_pruned_history_reaches_the_oldest_retained_group() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + stored(&archive, 0..10).await; + + let broadcast = broadcast::Info::new().produce(); + let (_indexes, _run) = open(&archive.store, &broadcast, Config::default().with_cache(0)).await; + // Retention deletes the three oldest objects after the reader listed them. + for sequence in [0, 2, 4] { + archive.prune("video", sequence * 1000).await; + } + let mock = archive.store.inner().clone(); + mock.take(); + let range = |start| track::Range { + start: at(start), + end: Some(at(6_500)), + }; + assert_eq!(read_range(&broadcast, "video", range(4_500)).await, [6_000]); + let pruned = mock + .gets() + .into_iter() + .filter(|get| { + ["0000000000000000000-", "0000000000000002000-", "0000000000000004000-"] + .iter() + .any(|key| get.contains(key)) + }) + .count(); + assert_eq!(pruned, 1, "one pruned GET lists the track again"); + assert_eq!(read_range(&broadcast, "video", range(500)).await, [6_000]); +} + +/// Retention that replaced everything the reader indexed before it refreshed leaves a seek the +/// newer objects the store holds, which the listing the pruned GET starts adds to the index. +#[tokio::test(start_paused = true)] +async fn a_seek_finds_objects_retention_replaced_the_index_with() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + stored(&archive, 0..2).await; + + let broadcast = broadcast::Info::new().produce(); + let (indexes, _run) = open(&archive.store, &broadcast, Config::default().with_cache(0)).await; + stored(&archive, 2..6).await; + archive.prune("video", 0).await; + let mock = archive.store.inner().clone(); + mock.take(); + + let track = broadcast.consume().track("video").unwrap(); + let subscription = Subscription::default() + .with_ranges([from(500)]) + .with_order(Order::Ascending) + .with_delay(Delay::default().with_max(None)); + let mut reader = track.subscribe(subscription).await.unwrap().ordered(); + assert!(sequence(&mut reader).await.is_some()); + // Before any refresh lists after the cursor: yields, not time. + for _ in 0..1000 { + if indexes["video"].lock().unwrap().len() == 2 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!( + indexes["video"].lock().unwrap().len(), + 2, + "both retained objects are indexed" + ); + assert!( + !mock + .take() + .iter() + .any(|op| matches!(op, crate::mock::Op::List { offset: Some(offset), .. } if !offset.ends_with(".info"))), + "no refresh listed after the cursor" + ); +} + +/// A key the store still lists but no longer serves, as a listing that lags its deletes leaves +/// it, costs a seek one GET: the listing its 404 starts never adds it back, so the lookup moves +/// on to what the store serves instead of reading it again. +#[tokio::test(start_paused = true)] +async fn a_seek_reads_a_listed_but_missing_object_once() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + stored(&archive, 0..10).await; + + let broadcast = broadcast::Info::new().produce(); + let (_indexes, _run) = open(&archive.store, &broadcast, Config::default().with_cache(0)).await; + let mock = archive.store.inner().clone(); + const STALE: &str = "rec/video/0000000000000002000-"; + mock.hide_gets(STALE); + // Room for a second read, which a lookup that relisted it back would take and then wait on. + let gate = mock.stall_gets(STALE); + gate.add_permits(2); + mock.take(); + + let range = track::Range { + start: at(2_500), + end: Some(at(4_500)), + }; + // The newest group the store serves at or before the time, then the range's end. + assert_eq!(read_range(&broadcast, "video", range).await, [1_000, 4_000]); + let stale = mock.gets().iter().filter(|get| get.contains(STALE)).count(); + assert_eq!(stale, 1, "the missing object is read once"); +} + +/// Recorded through a real DVR producer while a reader stays attached for many rotations: its +/// index stays about the size of what the store retains, and a seek to the start of time +/// reaches the oldest group the store holds, as a direct listing finds it. +#[tokio::test(start_paused = true)] +async fn a_reader_follows_a_real_dvr_through_its_rotations() { + let store = Store::new(Mock::memory(), "rec"); + let source = broadcast::Info::new().produce(); + let video = source + .create_track("video", track::Info::default().with_timescale(Timescale::MILLI)) + .unwrap(); + let dvr = crate::producer::Dvr::new(Duration::from_secs(2), Duration::from_secs(1)); + let config = crate::producer::Config::default() + .with_duration(Duration::from_secs(1)) + .with_flush(Duration::from_secs(100)) + .with_dvr(dvr); + let _recording = tokio::spawn(crate::Producer::new(store.clone(), source.consume(), config).run()); + while video.subscription().is_none() { + sleep(Duration::from_millis(1)).await; + } + let write = |sequence: u64| { + let mut group = video.create_group(group::Info { start: sequence * 1000 }).unwrap(); + let pts = Timestamp::from_millis(sequence * 1000).unwrap(); + group.write_frame(pts, Bytes::from(format!("{sequence}"))).unwrap(); + group.finish().unwrap(); + }; + // Each group is its own object, stored once the next one starts. + for sequence in 0..2 { + write(sequence); + sleep(Duration::from_secs(1)).await; + } + + // Groups nobody reads leave memory a second after their last write, as an origin expires + // them, so the seek reaches the store. + let mut info = broadcast::Info::new(); + info.pool = moq_net::cache::Pool::new(moq_net::cache::Config::default().with_expiry(Duration::from_secs(1))); + let pool = info.pool.clone(); + let broadcast = info.produce(); + let (indexes, _run) = open(&store, &broadcast, Config::default().with_cache(0)).await; + let index = &indexes["video"]; + let dir = layout::track_dir(store.prefix(), "video").unwrap(); + let mut retained = Vec::new(); + for sequence in 2..300 { + write(sequence); + sleep(Duration::from_secs(1)).await; + pool.poll(kio::clock::now(), &kio::Waiter::noop()); + + let listed = index::list(store.inner(), &dir, &dir.clone().join(layout::TRACK_INFO), false) + .await + .unwrap(); + retained.push(listed.objects.len()); + // Relisting halves the index back to what the store held at that listing, so the bound + // follows the most it retained. + let most = retained.iter().max().unwrap(); + let held = index.lock().unwrap().len(); + assert!( + held <= 2 * most + 1, + "{held} objects indexed after {sequence}, at most {most} retained" + ); + + if sequence % 50 == 0 { + let oldest = listed.objects.first().unwrap().0.0; + let track = broadcast.consume().track("video").unwrap(); + let subscription = Subscription::default() + .with_ranges([from(0)]) + .with_order(Order::Ascending) + .with_delay(Delay::default().with_max(None)); + let mut reader = track.subscribe(subscription).await.unwrap().ordered(); + assert_eq!( + super::tests::sequence(&mut reader).await, + Some(oldest), + "seek after {sequence}" + ); + } + } + // Retention really rotated: the store held a few objects at a time, not all of them. + assert!(retained.iter().all(|count| *count <= 5), "{retained:?}"); +} diff --git a/rs/moq-archive/src/consumer/track.rs b/rs/moq-archive/src/consumer/track.rs index e61a5f510fb..39b33ef7040 100644 --- a/rs/moq-archive/src/consumer/track.rs +++ b/rs/moq-archive/src/consumer/track.rs @@ -8,7 +8,7 @@ use moq_net::{Timescale, Timestamp, group, track}; use object_store::ObjectStore; use super::Shared; -use super::index::Index; +use super::index::{self, Index, Start}; use crate::layout::{End, Entry, Frame, Object}; use crate::{Error, Info, Result}; @@ -183,7 +183,31 @@ impl Asker<'_> { /// as the live cache resolves it. Left unanswered when the archive holds no group, or the store /// fails. async fn look_up(shared: &Shared, index: &Mutex, lookup: &FillLookup) { - let pts = lookup.time(); + loop { + match search(shared, index, lookup.time()).await { + Search::Found(sequence) => return serve_group(shared, index, Asker::Lookup(lookup), sequence).await, + // The index dropped what the store pruned, at least the object found gone, which no + // listing adds back, so the search starts over on what it retains. Each retry drops an + // object for good, so only a store pruning what it lists faster than a GET keeps + // this going, at one LIST and GET per object it writes. + Search::Pruned => {} + Search::Unanswered => return, + } + } +} + +/// How a lookup's search ended. +enum Search { + /// At the group starting at this sequence. + Found(u64), + /// At an object the store pruned since it was listed. + Pruned, + /// With no group, or a store failure. + Unanswered, +} + +/// Search the index for the group containing `pts`; see [`look_up`]. +async fn search(shared: &Shared, index: &Mutex, pts: u64) -> Search { let (around, timed) = { let index = index.lock().unwrap(); (index.around(pts), index.timed()) @@ -191,8 +215,9 @@ async fn look_up(shared: &Shared, index: &Mutex, looku // Newest first, skipping a group whose head was pruned, which can't be served from its first // frame. for (start, path) in around { - let Some(object) = load(shared, index, start, &path, timed).await else { - return; + let object = match load(shared, index, start, &path, timed).await { + Ok(object) => object, + Err(ended) => return ended, }; let found = object .entries @@ -203,7 +228,7 @@ async fn look_up(shared: &Shared, index: &Mutex, looku .find(|(_, entry)| !entry.frames.is_empty() && entry.sequence <= pts) .map(|(_, entry)| entry.sequence); if let Some(sequence) = found { - return serve_group(shared, index, Asker::Lookup(lookup), sequence).await; + return Search::Found(sequence); } } @@ -211,8 +236,9 @@ async fn look_up(shared: &Shared, index: &Mutex, looku // one whose head was pruned, which can't be served from its first frame. let mut next = index.lock().unwrap().after(None); while let Some((start, path)) = next { - let Some(object) = load(shared, index, start, &path, timed).await else { - return; + let object = match load(shared, index, start, &path, timed).await { + Ok(object) => object, + Err(ended) => return ended, }; let found = object .entries @@ -221,34 +247,35 @@ async fn look_up(shared: &Shared, index: &Mutex, looku .find(|(position, entry)| !entry.frames.is_empty() && (*position > 0 || object.frame_start == 0)) .map(|(_, entry)| entry.sequence); if let Some(sequence) = found { - return serve_group(shared, index, Asker::Lookup(lookup), sequence).await; + return Search::Found(sequence); } next = index.lock().unwrap().after(Some(start)); } + Search::Unanswered } -/// The object a lookup searches, `None` once the store failed. +/// The object a lookup searches, or how the search ends without it. /// /// An abandoned lookup still finishes the read and answers nobody: a lookup owes no group, so /// nothing tells it when its readers stop waiting. async fn load( shared: &Shared, index: &Mutex, - start: (u64, u64), + start: Start, path: &object_store::path::Path, timed: bool, -) -> Option> { - match super::object(shared, path, start, timed).await { - Ok(object) => Some(object), - Err(err) => { - if let Error::NotFound(_) = err { - // Pruned since it was listed. - index.lock().unwrap().forget(start); - } - tracing::warn!(%path, %err, "archived object is unavailable for a time lookup"); - None - } - } +) -> std::result::Result, Search> { + let err = match super::object(shared, path, start, timed).await { + Ok(object) => return Ok(object), + // Pruned since it was listed. + Err(Error::NotFound(_)) => match index::pruned(shared.store.inner(), index, start).await { + Ok(()) => return Err(Search::Pruned), + Err(err) => err, + }, + Err(err) => err, + }; + tracing::warn!(%path, %err, "archived object is unavailable for a time lookup"); + Err(Search::Unanswered) } /// Answer group `sequence` for `asker` from the objects holding it, drop the run of sequences @@ -268,7 +295,16 @@ async fn serve_group(shared: &Shared, index: &Mutex, a let mut objects = Vec::with_capacity(holding.len()); for (start, path) in &holding { - let load = super::object(shared, path, *start, timed); + let load = async { + let loaded = super::object(shared, path, *start, timed).await; + if let Err(Error::NotFound(_)) = loaded { + // Pruned since it was listed, and likely not alone. + if let Err(err) = index::pruned(shared.store.inner(), index, *start).await { + tracing::warn!(%path, %err, "archived track is unavailable to list"); + } + } + loaded + }; let loaded = match asker { // A lookup owes no group, so nothing says when its readers stop waiting: it finishes // the read. @@ -285,8 +321,6 @@ async fn serve_group(shared: &Shared, index: &Mutex, a match loaded { Ok(object) => objects.push(object), Err(Error::NotFound(_)) => { - // Pruned since it was listed. - index.lock().unwrap().forget(*start); asker.drop_groups(sequence..sequence + 1); return; } diff --git a/rs/moq-archive/src/mock.rs b/rs/moq-archive/src/mock.rs index b0e82f80654..815e6d0ab45 100644 --- a/rs/moq-archive/src/mock.rs +++ b/rs/moq-archive/src/mock.rs @@ -244,6 +244,15 @@ impl ObjectStore for Mock { async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { self.log(Op::Get(location.to_string())); + let stall = self + .state() + .stall_gets + .iter() + .find(|(pattern, _)| location.as_ref().contains(pattern)) + .map(|(_, gate)| gate.clone()); + if let Some(gate) = stall { + gate.acquire().await.expect("stall gate closed").forget(); + } if self.state().hide_gets.iter().any(|p| location.as_ref().contains(p)) { return Err(object_store::Error::NotFound { path: location.to_string(), @@ -256,15 +265,6 @@ impl ObjectStore for Mock { source: "outage".into(), }); } - let stall = self - .state() - .stall_gets - .iter() - .find(|(pattern, _)| location.as_ref().contains(pattern)) - .map(|(_, gate)| gate.clone()); - if let Some(gate) = stall { - gate.acquire().await.expect("stall gate closed").forget(); - } self.inner.get_opts(location, options).await }