From de0b162deaecc6d5df29f87c04a519dfbcd68310 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:04:46 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m1/precut-fixes/archive-retention-index.md Quest-Claim: 89bdfbf25441792c From 2417dea1ef08c2261710172fbc064353f0663501 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:23:44 -0800 Subject: [PATCH 2/5] fix(archive): a served archive's index tracks what retention keeps A consumer attached to a pruned archive listed each track only after its newest key, so its index kept every object ever written and forgot one only when a read found it gone. A time lookup then gave up on the first pruned object and left the reader to the live cache. Each track is now listed whole again once the objects added since its last whole listing outnumber the ones that listing found, and right away when a read finds an object gone. The index stays within about twice what the store retains, for about one listed key per added object, and a lookup that meets a pruned object searches again on what the store holds, reaching the oldest retained group with one listing instead of a GET per pruned object. The archive guide no longer says time ranges resolve only against the cache. Co-Authored-By: Claude Opus 5.5 --- doc/concept/archives.md | 14 +- .../precut-fixes/archive-retention-index.md | 9 - rs/moq-archive/src/consumer/index.rs | 103 +++++++++-- rs/moq-archive/src/consumer/mod.rs | 13 +- rs/moq-archive/src/consumer/tests.rs | 169 ++++++++++++++++++ rs/moq-archive/src/consumer/track.rs | 75 +++++--- 6 files changed, 325 insertions(+), 58 deletions(-) delete mode 100644 quest/m1/precut-fixes/archive-retention-index.md diff --git a/doc/concept/archives.md b/doc/concept/archives.md index 3330d3764f0..d2d20e5ff27 100644 --- a/doc/concept/archives.md +++ b/doc/concept/archives.md @@ -102,14 +102,18 @@ like any other broadcast; it does not look in object storage on a cache miss. Each track starts live at its newest stored group and serves older groups from storage when requested by their start. It advertises no max age, so the source's live max age cannot make archived history stale. A growing recording is -periodically listed for new tracks and objects. Once `.end` appears, refreshing +periodically listed for new tracks and objects, and each track is listed whole +again as it grows, or once a read finds an object gone, so a server attached to +a pruned recording tracks what the store retains. Once `.end` appears, refreshing stops and tracks end as recorded; without it, including when a recorder stopped early, the archive stays live. Empty or unrecognized prefixes are refused. -Serving history is not paced playback from the beginning. Time ranges currently -resolve against the served track's cache, not the archive's stored keys, so they -cannot reliably seek old stored content. There is no automatic switch from a -live broadcast to its archive at the original path. +A time range the served track's cache can't resolve is looked up from the stored +keys: its start reaches the stored group containing that time, however old, or +the oldest group the store still holds when the time is older, such as one +retention pruned; its end stops just past the group containing it. Serving +history is not paced playback from the beginning, and there is no automatic +switch from a live broadcast to its archive at the original path. The archived catalog is an ordinary track. [HLS](/bin/hls) serves a recording from its media tracks, numbered on the catalog's `sync.interval`, just as diff --git a/quest/m1/precut-fixes/archive-retention-index.md b/quest/m1/precut-fixes/archive-retention-index.md deleted file mode 100644 index ccb949a16fa..00000000000 --- a/quest/m1/precut-fixes/archive-retention-index.md +++ /dev/null @@ -1,9 +0,0 @@ -# [M] Bound the live archive reader index by retention - -## Goal - -A rolling archive reader tracks retained objects, and a seek across the expired boundary can reach the oldest retained group. - -## Plan - -The recorder prunes objects while the consumer index lists forward and forgets old keys after failed reads. The audit traced growth without rewinds and a seek that can stop on the first deleted object. Exercise an attached reader over repeated retention rotations and an evicted byte cache. Measure index size against retained objects, exercise a seek across the boundary, and fix any confirmed failure. Correct the archive guide's claim about time lookup. diff --git a/rs/moq-archive/src/consumer/index.rs b/rs/moq-archive/src/consumer/index.rs index e578f410ae9..e99f723e6f2 100644 --- a/rs/moq-archive/src/consumer/index.rs +++ b/rs/moq-archive/src/consumer/index.rs @@ -1,6 +1,8 @@ -//! One track's object keys, listed once and extended after its newest key. +//! One track's object keys, listed once, extended after its newest key, and listed whole again +//! to drop what the store pruned. use std::collections::BTreeMap; +use std::sync::Mutex; use futures::TryStreamExt; use moq_net::Timescale; @@ -26,41 +28,74 @@ pub(super) struct Listed { } /// A track's stored objects, by where each starts. +/// +/// Retention deletes objects that a listing after the newest key never sees, so the whole +/// directory is listed again once the objects added since its last whole listing outnumber the +/// ones that listing found, and whenever a read finds one gone. The index then holds at most +/// about twice what the store retains, for one listed key per added object. pub(super) struct Index { pub dir: Path, pub timescale: Option, /// Each object's key. A group's sequence is its start, so these sort by time too. objects: BTreeMap, + /// `.info`, which a listing of the whole directory starts after. + origin: Path, /// The newest key listed, which the next listing starts after. cursor: Path, + /// How many objects the last whole listing left, and how many listings added since. + listed: usize, + added: usize, } impl Index { pub fn new(dir: Path, info: &Info) -> Result { - let cursor = crate::path::push(&dir, layout::TRACK_INFO)?; + let origin = crate::path::push(&dir, layout::TRACK_INFO)?; Ok(Self { dir, timescale: super::track::timescale(info)?, objects: BTreeMap::new(), - cursor, + cursor: origin.clone(), + origin, + listed: 0, + added: 0, }) } - pub fn timed(&self) -> bool { - self.timescale.is_some() + /// How many objects it holds. + #[cfg(test)] + pub fn len(&self) -> usize { + self.objects.len() } - /// The newest key listed, for the next listing to start after. - pub fn cursor(&self) -> &Path { - &self.cursor + pub fn timed(&self) -> bool { + self.timescale.is_some() } - /// Take in the objects a listing after [`Self::cursor`] found. - pub fn extend(&mut self, objects: &[Listing]) { + /// Take in the objects new since the cursor that a listing found. + fn extend(&mut self, objects: &[Listing]) { for (start, path) in objects { self.objects.insert(*start, path.clone()); self.cursor = path.clone(); } + self.added += objects.len(); + } + + /// The newest start indexed when the next refresh should list the whole directory, which + /// [`Self::retain`] then reconciles up to. + fn due(&self) -> Option { + if self.added <= self.listed { + return None; + } + self.objects.last_key_value().map(|(start, _)| *start) + } + + /// Drop the objects up to `through`, all indexed before the listing that `found` began, that it + /// no longer holds: the store pruned them. Later objects were listed since. + fn retain(&mut self, through: Start, found: &[Listing]) { + self.objects + .retain(|start, _| *start > through || found.binary_search_by_key(start, |(start, _)| *start).is_ok()); + self.listed = self.objects.len(); + self.added = 0; } /// The newest object, which holds the newest stored sequence. @@ -123,11 +158,53 @@ impl Index { }; Some((*start, path.clone())) } +} - /// Forget an object the store no longer has, such as one a DVR pruned. - pub fn forget(&mut self, start: Start) { - self.objects.remove(&start); +/// Take in the objects new since `index`'s cursor and return them, with the track's `end` when +/// `end` is set. When the index is due, the listing covers the whole directory, dropping what +/// the store pruned. +pub(super) async fn refresh(store: &S, index: &Mutex, end: bool) -> Result { + let (dir, from, cursor, through) = { + let index = index.lock().unwrap(); + let through = index.due(); + let from = match through { + Some(_) => index.origin.clone(), + None => index.cursor.clone(), + }; + (index.dir.clone(), from, index.cursor.clone(), through) + }; + let mut listed = list(store, &dir, &from, end).await?; + let mut index = index.lock().unwrap(); + if let Some(through) = through { + index.retain(through, &listed.objects); + listed.objects.retain(|(_, path)| *path > cursor); } + index.extend(&listed.objects); + // The first listing starts at the origin too. + if from == index.origin { + index.listed = index.objects.len(); + index.added = 0; + } + Ok(listed) +} + +/// Drop the object starting at `missing`, which a read found gone, and with it every other +/// object the store pruned, by listing the whole directory. Lists nothing when another read +/// dropped it already. +pub(super) async fn pruned(store: &S, index: &Mutex, missing: Start) -> Result<()> { + let (dir, from, through) = { + let mut index = index.lock().unwrap(); + if index.objects.remove(&missing).is_none() { + return Ok(()); + } + let Some((through, _)) = index.objects.last_key_value() else { + return Ok(()); + }; + (index.dir.clone(), index.origin.clone(), *through) + }; + let listed = list(store, &dir, &from, false).await?; + index.lock().unwrap().retain(through, &listed.objects); + Ok(()) } /// List a track's directory after `cursor`, refusing anything the layout does not name, and diff --git a/rs/moq-archive/src/consumer/mod.rs b/rs/moq-archive/src/consumer/mod.rs index 166b7c3c4ec..a9b3d83631a 100644 --- a/rs/moq-archive/src/consumer/mod.rs +++ b/rs/moq-archive/src/consumer/mod.rs @@ -11,7 +11,10 @@ //! however old, and a time end stops just past the group containing it. //! //! A growing archive is followed: every [`Config::refresh`], one delimiter LIST of the prefix -//! finds added tracks, and one LIST per track after its newest key finds new objects. Once the +//! finds added tracks, and one LIST per track after its newest key finds new objects. That LIST +//! covers the whole track instead once the objects added since its last whole listing outnumber +//! the ones that listing found, and a read that finds an object gone lists the whole track at +//! once, so a reader attached to a pruned archive indexes about what the store retains. Once the //! prefix holds `.end`, each track ends as recorded and refreshing stops. Without it the archive //! stays live, which is also how one whose producer stopped early reads. //! @@ -321,12 +324,8 @@ async fn refresh(shared: Arc>, mut tracks: Vec object_store::path::Path { layout::track_dir(self.store.prefix(), track).unwrap() } @@ -959,3 +971,160 @@ async fn a_scale_less_track_looks_up_a_time_by_key() { }; assert_eq!(read_range(&broadcast, "log", range).await, [2, 5]); } + +/// Serve `store` on `broadcast` with `config`, returning each track's index. +async fn open( + store: &Store, + broadcast: &broadcast::Producer, + config: Config, +) -> (HashMap>>, JoinHandle>) { + let consumer = Consumer::open(store.clone(), broadcast.clone(), config.with_refresh(REFRESH)) + .await + .unwrap(); + let indexes = consumer + .tracks + .iter() + .map(|(name, track)| (name.clone(), track.index.clone())) + .collect(); + (indexes, tokio::spawn(consumer.run())) +} + +/// A reader attached to a rolling archive indexes about what the store retains, however many +/// retention rotations pass without it reading history. +#[tokio::test(start_paused = true)] +async fn a_rolling_archive_index_tracks_retention() { + const RETAINED: u64 = 4; + let archive = Archive::new(); + archive.track("video", 0, true).await; + for sequence in 0..RETAINED { + archive.object("video", 0, vec![timed(sequence, 0..1)]).await; + } + + let broadcast = broadcast::Info::new().produce(); + let (indexes, _run) = open(&archive.store, &broadcast, Config::default()).await; + let index = &indexes["video"]; + for sequence in RETAINED..RETAINED + 100 { + archive.object("video", 0, vec![timed(sequence, 0..1)]).await; + archive.prune("video", timed(sequence - RETAINED, 0..1).sequence).await; + sleep(REFRESH).await; + let held = index.lock().unwrap().len(); + // Twice the retained objects, plus the one a refresh adds before it lists them all. + assert!( + held <= 2 * RETAINED as usize + 1, + "{held} objects indexed after {sequence}, {RETAINED} retained" + ); + } +} + +/// A time older than what retention kept reaches the oldest retained group, though the reader +/// still indexes the pruned objects and caches none of them, with one listing rather than a GET +/// per pruned object. +#[tokio::test(start_paused = true)] +async fn a_seek_across_pruned_history_reaches_the_oldest_retained_group() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + stored(&archive, 0..10).await; + + let broadcast = broadcast::Info::new().produce(); + let (_indexes, _run) = open(&archive.store, &broadcast, Config::default().with_cache(0)).await; + // Retention deletes the three oldest objects after the reader listed them. + for sequence in [0, 2, 4] { + archive.prune("video", sequence * 1000).await; + } + let mock = archive.store.inner().clone(); + mock.take(); + let range = |start| track::Range { + start: at(start), + end: Some(at(6_500)), + }; + assert_eq!(read_range(&broadcast, "video", range(4_500)).await, [6_000]); + let pruned = mock + .gets() + .into_iter() + .filter(|get| { + ["0000000000000000000-", "0000000000000002000-", "0000000000000004000-"] + .iter() + .any(|key| get.contains(key)) + }) + .count(); + assert_eq!(pruned, 1, "one pruned GET lists the track again"); + assert_eq!(read_range(&broadcast, "video", range(500)).await, [6_000]); +} + +/// Recorded through a real DVR producer while a reader stays attached for many rotations: its +/// index stays about the size of what the store retains, and a seek to the start of time +/// reaches the oldest group the store holds, as a direct listing finds it. +#[tokio::test(start_paused = true)] +async fn a_reader_follows_a_real_dvr_through_its_rotations() { + let store = Store::new(Mock::memory(), "rec"); + let source = broadcast::Info::new().produce(); + let video = source + .create_track("video", track::Info::default().with_timescale(Timescale::MILLI)) + .unwrap(); + let dvr = crate::producer::Dvr::new(Duration::from_secs(2), Duration::from_secs(1)); + let config = crate::producer::Config::default() + .with_duration(Duration::from_secs(1)) + .with_flush(Duration::from_secs(100)) + .with_dvr(dvr); + let _recording = tokio::spawn(crate::Producer::new(store.clone(), source.consume(), config).run()); + while video.subscription().is_none() { + sleep(Duration::from_millis(1)).await; + } + let write = |sequence: u64| { + let mut group = video.create_group(group::Info { start: sequence * 1000 }).unwrap(); + let pts = Timestamp::from_millis(sequence * 1000).unwrap(); + group.write_frame(pts, Bytes::from(format!("{sequence}"))).unwrap(); + group.finish().unwrap(); + }; + // Each group is its own object, stored once the next one starts. + for sequence in 0..2 { + write(sequence); + sleep(Duration::from_secs(1)).await; + } + + // Groups nobody reads leave memory a second after their last write, as an origin expires + // them, so the seek reaches the store. + let mut info = broadcast::Info::new(); + info.pool = moq_net::cache::Pool::new(moq_net::cache::Config::default().with_expiry(Duration::from_secs(1))); + let pool = info.pool.clone(); + let broadcast = info.produce(); + let (indexes, _run) = open(&store, &broadcast, Config::default().with_cache(0)).await; + let index = &indexes["video"]; + let dir = layout::track_dir(store.prefix(), "video").unwrap(); + let mut retained = Vec::new(); + for sequence in 2..300 { + write(sequence); + sleep(Duration::from_secs(1)).await; + pool.poll(kio::clock::now(), &kio::Waiter::noop()); + + let listed = index::list(store.inner(), &dir, &dir.clone().join(layout::TRACK_INFO), false) + .await + .unwrap(); + retained.push(listed.objects.len()); + // Relisting halves the index back to what the store held at that listing, so the bound + // follows the most it retained. + let most = retained.iter().max().unwrap(); + let held = index.lock().unwrap().len(); + assert!( + held <= 2 * most + 1, + "{held} objects indexed after {sequence}, at most {most} retained" + ); + + if sequence % 50 == 0 { + let oldest = listed.objects.first().unwrap().0.0; + let track = broadcast.consume().track("video").unwrap(); + let subscription = Subscription::default() + .with_ranges([from(0)]) + .with_order(Order::Ascending) + .with_delay(Delay::default().with_max(None)); + let mut reader = track.subscribe(subscription).await.unwrap().ordered(); + assert_eq!( + super::tests::sequence(&mut reader).await, + Some(oldest), + "seek after {sequence}" + ); + } + } + // Retention really rotated: the store held a few objects at a time, not all of them. + assert!(retained.iter().all(|count| *count <= 5), "{retained:?}"); +} diff --git a/rs/moq-archive/src/consumer/track.rs b/rs/moq-archive/src/consumer/track.rs index e61a5f510fb..58ba6520e2a 100644 --- a/rs/moq-archive/src/consumer/track.rs +++ b/rs/moq-archive/src/consumer/track.rs @@ -8,7 +8,7 @@ use moq_net::{Timescale, Timestamp, group, track}; use object_store::ObjectStore; use super::Shared; -use super::index::Index; +use super::index::{self, Index, Start}; use crate::layout::{End, Entry, Frame, Object}; use crate::{Error, Info, Result}; @@ -183,7 +183,29 @@ impl Asker<'_> { /// as the live cache resolves it. Left unanswered when the archive holds no group, or the store /// fails. async fn look_up(shared: &Shared, index: &Mutex, lookup: &FillLookup) { - let pts = lookup.time(); + loop { + match search(shared, index, lookup.time()).await { + Search::Found(sequence) => return serve_group(shared, index, Asker::Lookup(lookup), sequence).await, + // The index dropped what the store pruned, at least the object found gone, so the + // search starts over on what it retains. + Search::Pruned => {} + Search::Unanswered => return, + } + } +} + +/// How a lookup's search ended. +enum Search { + /// At the group starting at this sequence. + Found(u64), + /// At an object the store pruned since it was listed. + Pruned, + /// With no group, or a store failure. + Unanswered, +} + +/// Search the index for the group containing `pts`; see [`look_up`]. +async fn search(shared: &Shared, index: &Mutex, pts: u64) -> Search { let (around, timed) = { let index = index.lock().unwrap(); (index.around(pts), index.timed()) @@ -191,8 +213,9 @@ async fn look_up(shared: &Shared, index: &Mutex, looku // Newest first, skipping a group whose head was pruned, which can't be served from its first // frame. for (start, path) in around { - let Some(object) = load(shared, index, start, &path, timed).await else { - return; + let object = match load(shared, index, start, &path, timed).await { + Ok(object) => object, + Err(ended) => return ended, }; let found = object .entries @@ -203,7 +226,7 @@ async fn look_up(shared: &Shared, index: &Mutex, looku .find(|(_, entry)| !entry.frames.is_empty() && entry.sequence <= pts) .map(|(_, entry)| entry.sequence); if let Some(sequence) = found { - return serve_group(shared, index, Asker::Lookup(lookup), sequence).await; + return Search::Found(sequence); } } @@ -211,8 +234,9 @@ async fn look_up(shared: &Shared, index: &Mutex, looku // one whose head was pruned, which can't be served from its first frame. let mut next = index.lock().unwrap().after(None); while let Some((start, path)) = next { - let Some(object) = load(shared, index, start, &path, timed).await else { - return; + let object = match load(shared, index, start, &path, timed).await { + Ok(object) => object, + Err(ended) => return ended, }; let found = object .entries @@ -221,34 +245,35 @@ async fn look_up(shared: &Shared, index: &Mutex, looku .find(|(position, entry)| !entry.frames.is_empty() && (*position > 0 || object.frame_start == 0)) .map(|(_, entry)| entry.sequence); if let Some(sequence) = found { - return serve_group(shared, index, Asker::Lookup(lookup), sequence).await; + return Search::Found(sequence); } next = index.lock().unwrap().after(Some(start)); } + Search::Unanswered } -/// The object a lookup searches, `None` once the store failed. +/// The object a lookup searches, or how the search ends without it. /// /// An abandoned lookup still finishes the read and answers nobody: a lookup owes no group, so /// nothing tells it when its readers stop waiting. async fn load( shared: &Shared, index: &Mutex, - start: (u64, u64), + start: Start, path: &object_store::path::Path, timed: bool, -) -> Option> { - match super::object(shared, path, start, timed).await { - Ok(object) => Some(object), - Err(err) => { - if let Error::NotFound(_) = err { - // Pruned since it was listed. - index.lock().unwrap().forget(start); - } - tracing::warn!(%path, %err, "archived object is unavailable for a time lookup"); - None - } - } +) -> std::result::Result, Search> { + let err = match super::object(shared, path, start, timed).await { + Ok(object) => return Ok(object), + // Pruned since it was listed. + Err(Error::NotFound(_)) => match index::pruned(shared.store.inner(), index, start).await { + Ok(()) => return Err(Search::Pruned), + Err(err) => err, + }, + Err(err) => err, + }; + tracing::warn!(%path, %err, "archived object is unavailable for a time lookup"); + Err(Search::Unanswered) } /// Answer group `sequence` for `asker` from the objects holding it, drop the run of sequences @@ -285,8 +310,10 @@ async fn serve_group(shared: &Shared, index: &Mutex, a match loaded { Ok(object) => objects.push(object), Err(Error::NotFound(_)) => { - // Pruned since it was listed. - index.lock().unwrap().forget(*start); + // Pruned since it was listed, and likely not alone. + if let Err(err) = index::pruned(shared.store.inner(), index, *start).await { + tracing::warn!(%path, %err, "archived track is unavailable to list"); + } asker.drop_groups(sequence..sequence + 1); return; } From f986aa28b0d35d16a9dfb11ed774fc1202d37ea2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 14:39:22 -0800 Subject: [PATCH 3/5] fix(archive): a relist adds the retained objects it finds, and a narrowed fill cancels it A whole listing a pruned GET starts now adds every object it finds, so a seek after retention replaced the whole index searches the newer objects instead of an empty index, and it lists even when the missing object was the only one indexed. The listing runs inside the fill's race against its readers narrowing past the group, like the GET before it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/consumer/index.rs | 24 +++++++++------- rs/moq-archive/src/consumer/tests.rs | 43 ++++++++++++++++++++++++++++ rs/moq-archive/src/consumer/track.rs | 15 ++++++---- 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/rs/moq-archive/src/consumer/index.rs b/rs/moq-archive/src/consumer/index.rs index e99f723e6f2..0a2c32da914 100644 --- a/rs/moq-archive/src/consumer/index.rs +++ b/rs/moq-archive/src/consumer/index.rs @@ -81,7 +81,7 @@ impl Index { } /// The newest start indexed when the next refresh should list the whole directory, which - /// [`Self::retain`] then reconciles up to. + /// [`Self::reconcile`] then reconciles up to. fn due(&self) -> Option { if self.added <= self.listed { return None; @@ -89,11 +89,15 @@ impl Index { self.objects.last_key_value().map(|(start, _)| *start) } - /// Drop the objects up to `through`, all indexed before the listing that `found` began, that it - /// no longer holds: the store pruned them. Later objects were listed since. - fn retain(&mut self, through: Start, found: &[Listing]) { + /// Take in a listing of the whole directory, `found`: drop the objects up to `through`, all + /// indexed before it began, that it no longer holds, since the store pruned them, and add + /// every object it holds. The cursor stays, so the live feed still takes the new ones. + fn reconcile(&mut self, through: Start, found: &[Listing]) { self.objects .retain(|start, _| *start > through || found.binary_search_by_key(start, |(start, _)| *start).is_ok()); + for (start, path) in found { + self.objects.insert(*start, path.clone()); + } self.listed = self.objects.len(); self.added = 0; } @@ -176,7 +180,7 @@ pub(super) async fn refresh(store: &S, index: &Mutex, end let mut listed = list(store, &dir, &from, end).await?; let mut index = index.lock().unwrap(); if let Some(through) = through { - index.retain(through, &listed.objects); + index.reconcile(through, &listed.objects); listed.objects.retain(|(_, path)| *path > cursor); } index.extend(&listed.objects); @@ -194,16 +198,16 @@ pub(super) async fn refresh(store: &S, index: &Mutex, end pub(super) async fn pruned(store: &S, index: &Mutex, missing: Start) -> Result<()> { let (dir, from, through) = { let mut index = index.lock().unwrap(); + let Some((&through, _)) = index.objects.last_key_value() else { + return Ok(()); + }; if index.objects.remove(&missing).is_none() { return Ok(()); } - let Some((through, _)) = index.objects.last_key_value() else { - return Ok(()); - }; - (index.dir.clone(), index.origin.clone(), *through) + (index.dir.clone(), index.origin.clone(), through) }; let listed = list(store, &dir, &from, false).await?; - index.lock().unwrap().retain(through, &listed.objects); + index.lock().unwrap().reconcile(through, &listed.objects); Ok(()) } diff --git a/rs/moq-archive/src/consumer/tests.rs b/rs/moq-archive/src/consumer/tests.rs index ca732c532a2..9416a157edb 100644 --- a/rs/moq-archive/src/consumer/tests.rs +++ b/rs/moq-archive/src/consumer/tests.rs @@ -1051,6 +1051,49 @@ async fn a_seek_across_pruned_history_reaches_the_oldest_retained_group() { assert_eq!(read_range(&broadcast, "video", range(500)).await, [6_000]); } +/// Retention that replaced everything the reader indexed before it refreshed leaves a seek the +/// newer objects the store holds, which the listing the pruned GET starts adds to the index. +#[tokio::test(start_paused = true)] +async fn a_seek_finds_objects_retention_replaced_the_index_with() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + stored(&archive, 0..2).await; + + let broadcast = broadcast::Info::new().produce(); + let (indexes, _run) = open(&archive.store, &broadcast, Config::default().with_cache(0)).await; + stored(&archive, 2..6).await; + archive.prune("video", 0).await; + let mock = archive.store.inner().clone(); + mock.take(); + + let track = broadcast.consume().track("video").unwrap(); + let subscription = Subscription::default() + .with_ranges([from(500)]) + .with_order(Order::Ascending) + .with_delay(Delay::default().with_max(None)); + let mut reader = track.subscribe(subscription).await.unwrap().ordered(); + assert!(sequence(&mut reader).await.is_some()); + // Before any refresh lists after the cursor: yields, not time. + for _ in 0..1000 { + if indexes["video"].lock().unwrap().len() == 2 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!( + indexes["video"].lock().unwrap().len(), + 2, + "both retained objects are indexed" + ); + assert!( + !mock + .take() + .iter() + .any(|op| matches!(op, crate::mock::Op::List { offset: Some(offset), .. } if !offset.ends_with(".info"))), + "no refresh listed after the cursor" + ); +} + /// Recorded through a real DVR producer while a reader stays attached for many rotations: its /// index stays about the size of what the store retains, and a seek to the start of time /// reaches the oldest group the store holds, as a direct listing finds it. diff --git a/rs/moq-archive/src/consumer/track.rs b/rs/moq-archive/src/consumer/track.rs index 58ba6520e2a..9165b5d4756 100644 --- a/rs/moq-archive/src/consumer/track.rs +++ b/rs/moq-archive/src/consumer/track.rs @@ -293,7 +293,16 @@ async fn serve_group(shared: &Shared, index: &Mutex, a let mut objects = Vec::with_capacity(holding.len()); for (start, path) in &holding { - let load = super::object(shared, path, *start, timed); + let load = async { + let loaded = super::object(shared, path, *start, timed).await; + if let Err(Error::NotFound(_)) = loaded { + // Pruned since it was listed, and likely not alone. + if let Err(err) = index::pruned(shared.store.inner(), index, *start).await { + tracing::warn!(%path, %err, "archived track is unavailable to list"); + } + } + loaded + }; let loaded = match asker { // A lookup owes no group, so nothing says when its readers stop waiting: it finishes // the read. @@ -310,10 +319,6 @@ async fn serve_group(shared: &Shared, index: &Mutex, a match loaded { Ok(object) => objects.push(object), Err(Error::NotFound(_)) => { - // Pruned since it was listed, and likely not alone. - if let Err(err) = index::pruned(shared.store.inner(), index, *start).await { - tracing::warn!(%path, %err, "archived track is unavailable to list"); - } asker.drop_groups(sequence..sequence + 1); return; } From aa427d2761bbe59dff8a2342d184252d205dd48f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 15:12:24 -0800 Subject: [PATCH 4/5] fix(archive): a listing never adds back an object a read found gone The index adds only objects newer than any it held, so a key the store lists but no longer serves costs a lookup one GET, not a GET and a LIST per retry forever. Co-Authored-By: Claude Opus 5.5 --- rs/moq-archive/src/consumer/index.rs | 54 ++++++++++++++-------------- rs/moq-archive/src/consumer/tests.rs | 29 +++++++++++++++ rs/moq-archive/src/consumer/track.rs | 5 +-- rs/moq-archive/src/mock.rs | 18 +++++----- 4 files changed, 69 insertions(+), 37 deletions(-) diff --git a/rs/moq-archive/src/consumer/index.rs b/rs/moq-archive/src/consumer/index.rs index 0a2c32da914..64fbeb37e62 100644 --- a/rs/moq-archive/src/consumer/index.rs +++ b/rs/moq-archive/src/consumer/index.rs @@ -40,8 +40,11 @@ pub(super) struct Index { objects: BTreeMap, /// `.info`, which a listing of the whole directory starts after. origin: Path, - /// The newest key listed, which the next listing starts after. + /// The newest key the live feed took, which the next listing starts after. cursor: Path, + /// The newest start ever indexed. Only objects after it are added, so one dropped up to it, + /// because the store pruned it or a read found it gone, never comes back from a stale listing. + known: Option, /// How many objects the last whole listing left, and how many listings added since. listed: usize, added: usize, @@ -56,6 +59,7 @@ impl Index { objects: BTreeMap::new(), cursor: origin.clone(), origin, + known: None, listed: 0, added: 0, }) @@ -71,33 +75,31 @@ impl Index { self.timescale.is_some() } - /// Take in the objects new since the cursor that a listing found. - fn extend(&mut self, objects: &[Listing]) { + /// Add the objects a listing found that are newer than any indexed before. + fn add(&mut self, objects: &[Listing]) { for (start, path) in objects { - self.objects.insert(*start, path.clone()); - self.cursor = path.clone(); + if Some(*start) > self.known { + self.objects.insert(*start, path.clone()); + self.known = Some(*start); + self.added += 1; + } } - self.added += objects.len(); } - /// The newest start indexed when the next refresh should list the whole directory, which - /// [`Self::reconcile`] then reconciles up to. - fn due(&self) -> Option { - if self.added <= self.listed { - return None; - } - self.objects.last_key_value().map(|(start, _)| *start) + /// The newest start ever indexed, when the next refresh should list the whole directory, + /// which [`Self::reconcile`] then reconciles up to. + fn due(&self) -> Option> { + (self.added > self.listed).then_some(self.known) } /// Take in a listing of the whole directory, `found`: drop the objects up to `through`, all /// indexed before it began, that it no longer holds, since the store pruned them, and add - /// every object it holds. The cursor stays, so the live feed still takes the new ones. - fn reconcile(&mut self, through: Start, found: &[Listing]) { - self.objects - .retain(|start, _| *start > through || found.binary_search_by_key(start, |(start, _)| *start).is_ok()); - for (start, path) in found { - self.objects.insert(*start, path.clone()); - } + /// the newer ones. The cursor stays, so the live feed still takes the new ones. + fn reconcile(&mut self, through: Option, found: &[Listing]) { + self.objects.retain(|start, _| { + Some(*start) > through || found.binary_search_by_key(start, |(start, _)| *start).is_ok() + }); + self.add(found); self.listed = self.objects.len(); self.added = 0; } @@ -183,7 +185,10 @@ pub(super) async fn refresh(store: &S, index: &Mutex, end index.reconcile(through, &listed.objects); listed.objects.retain(|(_, path)| *path > cursor); } - index.extend(&listed.objects); + index.add(&listed.objects); + if let Some((_, path)) = listed.objects.last() { + index.cursor = path.clone(); + } // The first listing starts at the origin too. if from == index.origin { index.listed = index.objects.len(); @@ -194,17 +199,14 @@ pub(super) async fn refresh(store: &S, index: &Mutex, end /// Drop the object starting at `missing`, which a read found gone, and with it every other /// object the store pruned, by listing the whole directory. Lists nothing when another read -/// dropped it already. +/// dropped it already. A listing that still holds it never adds it back. pub(super) async fn pruned(store: &S, index: &Mutex, missing: Start) -> Result<()> { let (dir, from, through) = { let mut index = index.lock().unwrap(); - let Some((&through, _)) = index.objects.last_key_value() else { - return Ok(()); - }; if index.objects.remove(&missing).is_none() { return Ok(()); } - (index.dir.clone(), index.origin.clone(), through) + (index.dir.clone(), index.origin.clone(), index.known) }; let listed = list(store, &dir, &from, false).await?; index.lock().unwrap().reconcile(through, &listed.objects); diff --git a/rs/moq-archive/src/consumer/tests.rs b/rs/moq-archive/src/consumer/tests.rs index 9416a157edb..ca76f4604f0 100644 --- a/rs/moq-archive/src/consumer/tests.rs +++ b/rs/moq-archive/src/consumer/tests.rs @@ -1094,6 +1094,35 @@ async fn a_seek_finds_objects_retention_replaced_the_index_with() { ); } +/// A key the store still lists but no longer serves, as a listing that lags its deletes leaves +/// it, costs a seek one GET: the listing its 404 starts never adds it back, so the lookup moves +/// on to what the store serves instead of reading it again. +#[tokio::test(start_paused = true)] +async fn a_seek_reads_a_listed_but_missing_object_once() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + stored(&archive, 0..10).await; + + let broadcast = broadcast::Info::new().produce(); + let (_indexes, _run) = open(&archive.store, &broadcast, Config::default().with_cache(0)).await; + let mock = archive.store.inner().clone(); + const STALE: &str = "rec/video/0000000000000002000-"; + mock.hide_gets(STALE); + // Room for a second read, which a lookup that relisted it back would take and then wait on. + let gate = mock.stall_gets(STALE); + gate.add_permits(2); + mock.take(); + + let range = track::Range { + start: at(2_500), + end: Some(at(4_500)), + }; + // The newest group the store serves at or before the time, then the range's end. + assert_eq!(read_range(&broadcast, "video", range).await, [1_000, 4_000]); + let stale = mock.gets().iter().filter(|get| get.contains(STALE)).count(); + assert_eq!(stale, 1, "the missing object is read once"); +} + /// Recorded through a real DVR producer while a reader stays attached for many rotations: its /// index stays about the size of what the store retains, and a seek to the start of time /// reaches the oldest group the store holds, as a direct listing finds it. diff --git a/rs/moq-archive/src/consumer/track.rs b/rs/moq-archive/src/consumer/track.rs index 9165b5d4756..f06a4bf4a9a 100644 --- a/rs/moq-archive/src/consumer/track.rs +++ b/rs/moq-archive/src/consumer/track.rs @@ -186,8 +186,9 @@ async fn look_up(shared: &Shared, index: &Mutex, looku loop { match search(shared, index, lookup.time()).await { Search::Found(sequence) => return serve_group(shared, index, Asker::Lookup(lookup), sequence).await, - // The index dropped what the store pruned, at least the object found gone, so the - // search starts over on what it retains. + // The index dropped what the store pruned, at least the object found gone, which no + // listing adds back, so the search starts over on what it retains. Each retry drops an + // object for good, so the retries are bounded by the objects the store lists. Search::Pruned => {} Search::Unanswered => return, } diff --git a/rs/moq-archive/src/mock.rs b/rs/moq-archive/src/mock.rs index b0e82f80654..815e6d0ab45 100644 --- a/rs/moq-archive/src/mock.rs +++ b/rs/moq-archive/src/mock.rs @@ -244,6 +244,15 @@ impl ObjectStore for Mock { async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { self.log(Op::Get(location.to_string())); + let stall = self + .state() + .stall_gets + .iter() + .find(|(pattern, _)| location.as_ref().contains(pattern)) + .map(|(_, gate)| gate.clone()); + if let Some(gate) = stall { + gate.acquire().await.expect("stall gate closed").forget(); + } if self.state().hide_gets.iter().any(|p| location.as_ref().contains(p)) { return Err(object_store::Error::NotFound { path: location.to_string(), @@ -256,15 +265,6 @@ impl ObjectStore for Mock { source: "outage".into(), }); } - let stall = self - .state() - .stall_gets - .iter() - .find(|(pattern, _)| location.as_ref().contains(pattern)) - .map(|(_, gate)| gate.clone()); - if let Some(gate) = stall { - gate.acquire().await.expect("stall gate closed").forget(); - } self.inner.get_opts(location, options).await } From ea88b86334f774257745de00e5d09c36e843fce7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 18 Nov 2026 15:40:30 -0800 Subject: [PATCH 5/5] docs(archive): a time before the archive reaches the oldest whole group Co-Authored-By: Claude Opus 5.5 --- doc/concept/archives.md | 5 +++-- rs/moq-archive/src/consumer/track.rs | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/concept/archives.md b/doc/concept/archives.md index d2d20e5ff27..c638a9f54b7 100644 --- a/doc/concept/archives.md +++ b/doc/concept/archives.md @@ -110,8 +110,9 @@ early, the archive stays live. Empty or unrecognized prefixes are refused. A time range the served track's cache can't resolve is looked up from the stored keys: its start reaches the stored group containing that time, however old, or -the oldest group the store still holds when the time is older, such as one -retention pruned; its end stops just past the group containing it. Serving +the oldest group the store still holds from its first frame when the time is +older, such as one retention pruned; its end stops just past the group +containing it. Serving history is not paced playback from the beginning, and there is no automatic switch from a live broadcast to its archive at the original path. diff --git a/rs/moq-archive/src/consumer/track.rs b/rs/moq-archive/src/consumer/track.rs index f06a4bf4a9a..39b33ef7040 100644 --- a/rs/moq-archive/src/consumer/track.rs +++ b/rs/moq-archive/src/consumer/track.rs @@ -188,7 +188,8 @@ async fn look_up(shared: &Shared, index: &Mutex, looku Search::Found(sequence) => return serve_group(shared, index, Asker::Lookup(lookup), sequence).await, // The index dropped what the store pruned, at least the object found gone, which no // listing adds back, so the search starts over on what it retains. Each retry drops an - // object for good, so the retries are bounded by the objects the store lists. + // object for good, so only a store pruning what it lists faster than a GET keeps + // this going, at one LIST and GET per object it writes. Search::Pruned => {} Search::Unanswered => return, }