From c37c26327b4335ce4e92fd7f8e0f17e711e94074 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 23:14:41 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m2/hls-lazy-instances.md Quest-Claim: cd0cdebafcfcb6c2 From d157398e57aed691c9c9c77cb4e2f173fc325905 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 21:22:22 -0800 Subject: [PATCH 2/4] wip: checkpoint unpushed work from the 2026-10-29 session The session that claimed this quest stopped before committing. This preserves its uncommitted changes as found on 2026-11-02; they are unfinished and unreviewed. Co-Authored-By: Claude Opus 5.5 --- doc/bin/hls.md | 2 +- rs/moq-hls/src/export/mod.rs | 12 +- rs/moq-hls/src/server/mod.rs | 256 +++++++++++++++++++++++++++----- rs/moq-hls/src/server/routes.rs | 6 + 4 files changed, 238 insertions(+), 38 deletions(-) diff --git a/doc/bin/hls.md b/doc/bin/hls.md index 8e922ef324c..7d7210d4aba 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -55,7 +55,7 @@ What that means for you: - **Live joins cost the window.** The exporter reads the record checkpoint and lists at most 256 segments. - **Segments are timed by their records.** A segment is listed once the record after it arrives, and its duration, its DASH `t` and `d`, and its last sample's duration all come from its own record and the next, so a segment's bytes never change with a catalog estimate. A stop record (a break or the end of the content) is listed as a zero-length gap, and what follows it starts a discontinuity. - **Disabled renditions are hidden.** A rendition the catalog marks `enabled: false` leaves the multivariant playlist and DASH manifest until it re-enables. Its media playlist keeps serving, so a player already on it is not ended with `EXT-X-ENDLIST`. -- **URLs name the publisher instance.** Every URL the multivariant playlist and DASH manifest list sits under `@`, the publisher's epoch, so every edge and every reload derive the same URLs and a CDN in front of export caches them. A restarted publisher gets URLs of its own, whose numbering starts over. The multivariant playlist and manifest switch to it at once, while a player already on the old instance keeps playing until that publisher leaves; the player then has to reload the multivariant playlist. An instance without an epoch (one that reached export over moq-lite-06 or older) gets an id each export mints, so only that export serves its URLs. +- **URLs name the publisher instance.** Every URL the multivariant playlist and DASH manifest list sits under `@`, the publisher's epoch, so every edge and every reload derive the same URLs and a CDN in front of export caches them. A restarted publisher gets URLs of its own, whose numbering starts over. The multivariant playlist and manifest switch to it at once, while a player already on the old instance keeps playing until that publisher leaves; the player then has to reload the multivariant playlist. The old instance follows its catalog and records only while players read it, stopping once a `--window` passes without a request and resuming on the next. An instance without an epoch (one that reached export over moq-lite-06 or older) gets an id each export mints, so only that export serves its URLs. - **Only a clean finish ends a playlist.** `EXT-X-ENDLIST` and a `static` DASH manifest mean the records finished. A rendition whose records fail instead, such as a publisher dropping its session, answers HTTP 500 with `Cache-Control: no-store`, so a truncated recording never looks complete, and leaves the multivariant playlist and DASH manifest so players switch to a sibling. With none left, the DASH manifest answers the error too. A [`moq-archive`](https://docs.rs/moq-archive) recording is served the same diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 9a71f1afdfe..75e25ae1a4f 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -111,13 +111,19 @@ pub struct Broadcaster { impl Broadcaster { /// Start tracking the renditions of `source`'s catalog broadcast. pub fn new(source: moq_mux::Source, config: Config) -> Arc { - let broadcast = source.broadcast(); // An epoch names the instance on every edge, so each serves it under the same URLs. - let instance = broadcast.info().epoch.clone().unwrap_or_else(moq_net::Epoch::mint); + let instance = source.broadcast().info().epoch.clone().unwrap_or_else(moq_net::Epoch::mint); + Self::with_instance(source, config, Arc::from(instance.as_str())) + } + + /// [`new`](Self::new), listing every URL under `instance`, so a broadcaster rebuilt for an + /// epochless instance keeps the id its URLs were listed under. + pub(crate) fn with_instance(source: moq_mux::Source, config: Config, instance: Arc) -> Arc { + let broadcast = source.broadcast(); let renditions = renditions::Producer::new(config.window); let broadcaster = Arc::new(Self { broadcast, - instance: Arc::from(instance.as_str()), + instance, renditions: renditions.clone(), watcher: Mutex::new(None), }); diff --git a/rs/moq-hls/src/server/mod.rs b/rs/moq-hls/src/server/mod.rs index bc5e2430816..d4a91efa5f4 100644 --- a/rs/moq-hls/src/server/mod.rs +++ b/rs/moq-hls/src/server/mod.rs @@ -16,7 +16,9 @@ //! for an epochless instance. A publisher that restarts gets its own URLs, so a playlist URL //! never resets its media sequence. A replaced instance keeps serving its URLs while its //! broadcast stays open, so a player already on it keeps playing until its publisher goes, -//! then reloads the master to reach the replacement. +//! then reloads the master to reach the replacement. A replaced instance watches its catalog +//! and records only while requests reach it: once a playlist window passes without one, it +//! stops, and the next request starts over under the same URLs. //! //! `{kind}` is `video` or `audio`, so a video and an audio rendition that share a //! name address distinct resources. The two `seg/` forms name the same bytes: HLS @@ -102,46 +104,47 @@ struct Followed { struct Instances { /// The instance the master playlist and manifest list: the latest to take over. - current: Arc, + current: Arc, /// Every instance a listed URL still names, the current one included. A replaced instance /// stays until its broadcast closes, so a player already on it keeps playing. - by_id: HashMap, Arc>, + by_id: HashMap, Arc>, } impl Followed { - fn new(current: Arc) -> Self { - let by_id = HashMap::from([(current.instance().clone(), current.clone())]); + fn new(current: Arc) -> Self { + let by_id = HashMap::from([(current.id.clone(), current.clone())]); Self { instances: Mutex::new(Instances { current, by_id }), task: Mutex::new(None), } } - fn current(&self) -> Arc { + fn current(&self) -> Arc { self.instances.lock().unwrap().current.clone() } - fn instance(&self, id: &str) -> Option> { + fn instance(&self, id: &str) -> Option> { self.instances.lock().unwrap().by_id.get(id).cloned() } - /// List `broadcaster` from now on. An instance returning after a gap keeps its epoch, and so - /// its id, which then names the new broadcaster. - fn start(&self, broadcaster: Arc) { + /// List `instance` from now on, retaining the one it replaces. An instance returning after a + /// gap keeps its epoch, and so its id, which then names the new instance. + fn start(&self, instance: Arc) { let mut instances = self.instances.lock().unwrap(); - instances - .by_id - .insert(broadcaster.instance().clone(), broadcaster.clone()); - instances.current = broadcaster; + instances.by_id.insert(instance.id.clone(), instance.clone()); + std::mem::replace(&mut instances.current, instance).retire(); } /// Keep serving every instance of `replaced` still open, unless this one lists its id. fn adopt(&self, replaced: &Followed) { let replaced = replaced.instances.lock().unwrap(); let mut instances = self.instances.lock().unwrap(); - for (id, broadcaster) in &replaced.by_id { - if !broadcaster.is_closed() { - instances.by_id.entry(id.clone()).or_insert_with(|| broadcaster.clone()); + for (id, instance) in &replaced.by_id { + if !instance.is_closed() { + instances.by_id.entry(id.clone()).or_insert_with(|| { + instance.retire(); + instance.clone() + }); } } } @@ -151,7 +154,7 @@ impl Followed { let mut instances = self.instances.lock().unwrap(); instances .by_id - .retain(|_, broadcaster| broadcaster.poll_closed(waiter).is_pending()); + .retain(|_, instance| instance.broadcast.poll_closed(waiter).is_pending()); } } @@ -163,6 +166,117 @@ impl Drop for Followed { } } +/// One publisher instance, which the URLs under `@{id}` name. +/// +/// The current instance keeps its [`Broadcaster`], and so its catalog and record watchers, while +/// it is listed. A replaced one is retained only for players already on it, so it watches only +/// while read: a request starts a broadcaster, which stops once no request has reached it for a +/// playlist window, and the next request starts another under the same URLs. +struct Instance { + id: Arc, + /// What every broadcaster for this instance reads, so a rebuilt one binds the same sibling + /// broadcasts its predecessor's rows were listed against. + source: moq_mux::Source, + broadcast: moq_net::broadcast::Consumer, + config: Config, + watch: Mutex, +} + +#[derive(Default)] +struct Watch { + /// Serving this instance: held while it is current, then until its reads go idle. + broadcaster: Option>, + /// The latest read since the instance was replaced, or `None` while it is current. + read: Option, + /// Releases the broadcaster once reads go idle; running only while a replaced instance is + /// watched. + release: Option>, +} + +impl Instance { + /// The instance `source`'s broadcast serves, watched from now on as the current one. + fn current(source: moq_mux::Source, config: Config) -> Arc { + let broadcaster = Broadcaster::new(source.clone(), config.clone()); + Arc::new(Self { + id: broadcaster.instance().clone(), + broadcast: source.broadcast(), + source, + config, + watch: Mutex::new(Watch { + broadcaster: Some(broadcaster), + ..Default::default() + }), + }) + } + + fn is_closed(&self) -> bool { + self.broadcast.is_closed() + } + + /// The broadcaster serving this instance, started if it was idle, and kept a while longer if + /// it is a replaced one. + fn broadcaster(self: &Arc) -> Arc { + let mut watch = self.watch.lock().unwrap(); + let broadcaster = watch + .broadcaster + .get_or_insert_with(|| Broadcaster::with_instance(self.source.clone(), self.config.clone(), self.id.clone())) + .clone(); + if watch.read.is_some() { + self.read(&mut watch); + } + broadcaster + } + + /// Stop holding the broadcaster for listing, and keep it only while read. A player already on + /// this instance reads it again within a window, so its broadcaster carries on warm. + fn retire(self: &Arc) { + let mut watch = self.watch.lock().unwrap(); + if watch.read.is_none() { + self.read(&mut watch); + } + } + + /// Note a read now, and release the broadcaster once reads go idle. + fn read(self: &Arc, watch: &mut Watch) { + watch.read = Some(moq_clock::now()); + if watch.release.is_none() && watch.broadcaster.is_some() { + watch.release = Some(tokio::spawn(release_idle(Arc::downgrade(self), self.config.window))); + } + } +} + +impl Drop for Instance { + fn drop(&mut self) { + if let Some(release) = self.watch.get_mut().unwrap().release.take() { + release.abort(); + } + } +} + +/// Drop `instance`'s broadcaster once nothing has read it for `idle`, stopping its watchers unless +/// a response still holds it. Holds the instance weakly, so it never keeps it retained. +async fn release_idle(instance: Weak, idle: Duration) { + loop { + let deadline = { + let Some(instance) = instance.upgrade() else { return }; + let watch = instance.watch.lock().unwrap(); + let Some(read) = watch.read else { return }; + read + idle + }; + moq_clock::sleep_until(deadline).await; + + let Some(instance) = instance.upgrade() else { return }; + let mut watch = instance.watch.lock().unwrap(); + if watch.read.is_some_and(|read| read + idle <= moq_clock::now()) { + watch.release = None; + let broadcaster = watch.broadcaster.take(); + drop(watch); + drop(broadcaster); + return; + } + } +} + impl Server { /// Build a server reading broadcasts from `origin`, which it watches whole while it /// lives. Every request is served; gate access by layering middleware onto @@ -187,13 +301,13 @@ impl Server { /// instance that takes over. Returns `None` if the broadcast is not announced once the /// origin has synced with the relay. pub async fn broadcaster(&self, name: &str) -> Option> { - Some(self.followed(name).await?.current()) + Some(self.followed(name).await?.current().broadcaster()) } /// The [`Broadcaster`] for the publisher instance `id` of `name`, until its broadcast - /// closes. + /// closes. A replaced instance starts watching again if it was idle. pub(crate) async fn instance(&self, name: &str, id: &str) -> Option> { - self.followed(name).await?.instance(id) + Some(self.followed(name).await?.instance(id)?.broadcaster()) } /// Follow `name` from the instance serving it now, unless it is followed already. @@ -252,10 +366,10 @@ impl Server { } impl Inner { - /// A broadcaster serving the instance `broadcast`. - fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Arc { + /// The instance `broadcast`, watched as the current one. + fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Arc { let source = moq_mux::Source::new(self.origin.consume(), broadcast); - Broadcaster::new(source, self.config.clone()) + Instance::current(source, self.config.clone()) } } @@ -426,23 +540,23 @@ mod tests { assert_eq!(start.elapsed(), Duration::ZERO); } - async fn closed_broadcaster() -> Arc { + async fn closed_instance() -> Arc { let origin = produce_origin(); let producer = origin.create_broadcast("gone").expect("publish allowed"); producer.announce(Default::default()).expect("publish allowed"); settle().await; - let broadcaster = Broadcaster::new(crate::resolved(&origin, "gone").await, Config::default()); - // Close the publisher so the resolved broadcast (and the broadcaster) reports closed. + let instance = Instance::current(crate::resolved(&origin, "gone").await, Config::default()); + // Close the publisher so the resolved broadcast (and the instance) reports closed. producer.close(); settle().await; - broadcaster + instance } #[tokio::test] async fn broadcaster_replaces_finished_cached_instance() { let origin = produce_origin(); let server = Server::new(origin.consume(), Config::default()); - let stale = closed_broadcaster().await; + let stale = closed_instance().await; server .inner @@ -456,7 +570,7 @@ mod tests { let fresh = server.broadcaster("live").await.expect("broadcast announced"); - assert!(!Arc::ptr_eq(&stale, &fresh)); + assert!(!Arc::ptr_eq(&stale.broadcaster(), &fresh)); assert!(server.inner.broadcasts.lock().unwrap().contains_key("live")); } @@ -564,11 +678,11 @@ mod tests { .publish("live", moq_net::origin::Route::default()) .expect("publish allowed"); settle().await; - let open = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let open = Instance::current(crate::resolved(&origin, "live").await, Config::default()); // The follower has not yet seen the takeover instance close. let followed = Followed::new(open.clone()); - followed.start(closed_broadcaster().await); + followed.start(closed_instance().await); server .inner .broadcasts @@ -578,10 +692,84 @@ mod tests { let current = server.broadcaster("live").await.expect("broadcast announced"); assert!(!current.is_closed()); - let kept = server.instance("live", open.instance()).await; + let kept = server.instance("live", &open.id).await; assert!( - kept.is_some_and(|kept| Arc::ptr_eq(&kept, &open)), + kept.is_some_and(|kept| Arc::ptr_eq(&kept, &open.broadcaster())), "the open instance lost its URLs" ); } + + /// Whether a replaced `instance` holds a broadcaster, and so its watchers. It runs the task + /// that releases one exactly while it does. + fn watching(instance: &Instance) -> bool { + let watch = instance.watch.lock().unwrap(); + assert_eq!(watch.broadcaster.is_some(), watch.release.is_some()); + watch.broadcaster.is_some() + } + + /// A replaced instance watches only while read: its warm broadcaster stops once a window + /// passes without a request, a later request starts another under the same id, which reads + /// keep going, and which stops again once they stop. + #[tokio::test(start_paused = true)] + async fn a_replaced_instance_watches_only_while_read() { + let origin = produce_origin(); + let config = Config::default(); + let server = Server::new(origin.consume(), config.clone()); + let (old_epoch, new_epoch) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); + let route = |epoch: &moq_net::Epoch| moq_net::origin::Route::default().with_epoch(epoch.clone()); + + let _old = origin.publish("live", route(&old_epoch)).expect("publish allowed"); + settle().await; + let first = Arc::downgrade(&server.broadcaster("live").await.expect("broadcast announced")); + let _new = origin.publish("live", route(&new_epoch)).expect("publish allowed"); + eventually(async || { + let current = server.broadcaster("live").await.expect("broadcast announced"); + current.instance().as_ref() == new_epoch.as_str() + }) + .await; + let followed = server.followed("live").await.expect("followed"); + let old = followed.instance(old_epoch.as_str()).expect("the old instance is retained"); + assert!(watching(&old), "a player on the replaced instance keeps it warm"); + + moq_clock::sleep(config.window).await; + settle().await; + assert!(!watching(&old), "an unread replaced instance holds no watchers"); + assert_eq!(first.strong_count(), 0, "its broadcaster is gone"); + + let restarted = server.instance("live", old_epoch.as_str()).await.expect("still served"); + assert_eq!(restarted.instance().as_ref(), old_epoch.as_str()); + assert!(watching(&old), "a request restarts it"); + let restarted = Arc::downgrade(&restarted); + for _ in 0..4 { + moq_clock::sleep(config.window / 2).await; + let read = server.instance("live", old_epoch.as_str()).await.expect("still served"); + assert!( + Arc::ptr_eq(&read, &restarted.upgrade().expect("held")), + "reads keep the one broadcaster" + ); + } + + moq_clock::sleep(config.window).await; + settle().await; + assert!(!watching(&old), "it stops again once its readers go idle"); + assert_eq!(restarted.strong_count(), 0); + } + + /// A rebuilt broadcaster keeps the id an epochless instance's URLs were listed under. + #[tokio::test(start_paused = true)] + async fn an_epochless_instance_restarts_under_its_id() { + let origin = produce_origin(); + let _publication = origin + .publish("live", moq_net::origin::Route::default()) + .expect("publish allowed"); + settle().await; + let config = Config::default(); + let instance = Instance::current(crate::resolved(&origin, "live").await, config.clone()); + instance.retire(); + moq_clock::sleep(config.window).await; + settle().await; + assert!(!watching(&instance)); + + assert_eq!(instance.broadcaster().instance(), &instance.id); + } } diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index 4e8d34a6c2f..9eb95f1f5f5 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -465,6 +465,12 @@ mod tests { assert_eq!(status(&app, &old_playlist).await, StatusCode::OK); assert_eq!(status(&app, &old_segment).await, StatusCode::OK); + // Unread for a window, the old instance stops watching, and the next request restarts it + // under the same URLs. + tokio::time::sleep(crate::export::Config::default().window).await; + wait_listed(&app, &old_playlist, "seg/53e0aafd.0.m4s").await; + assert_eq!(status(&app, &old_segment).await, StatusCode::OK); + drop((old_catalog, old_track, old_media, old)); let deadline = tokio::time::Instant::now() + TIMEOUT; while status(&app, &old_playlist).await != StatusCode::NOT_FOUND { From 96d40e7f3d520b542f011fb6873dd1ac438fb566 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 11:30:15 -0800 Subject: [PATCH 3/4] Revert "wip: checkpoint unpushed work from the 2026-10-29 session" This reverts commit d157398e57aed691c9c9c77cb4e2f173fc325905. --- doc/bin/hls.md | 2 +- rs/moq-hls/src/export/mod.rs | 12 +- rs/moq-hls/src/server/mod.rs | 256 +++++--------------------------- rs/moq-hls/src/server/routes.rs | 6 - 4 files changed, 38 insertions(+), 238 deletions(-) diff --git a/doc/bin/hls.md b/doc/bin/hls.md index 7d7210d4aba..8e922ef324c 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -55,7 +55,7 @@ What that means for you: - **Live joins cost the window.** The exporter reads the record checkpoint and lists at most 256 segments. - **Segments are timed by their records.** A segment is listed once the record after it arrives, and its duration, its DASH `t` and `d`, and its last sample's duration all come from its own record and the next, so a segment's bytes never change with a catalog estimate. A stop record (a break or the end of the content) is listed as a zero-length gap, and what follows it starts a discontinuity. - **Disabled renditions are hidden.** A rendition the catalog marks `enabled: false` leaves the multivariant playlist and DASH manifest until it re-enables. Its media playlist keeps serving, so a player already on it is not ended with `EXT-X-ENDLIST`. -- **URLs name the publisher instance.** Every URL the multivariant playlist and DASH manifest list sits under `@`, the publisher's epoch, so every edge and every reload derive the same URLs and a CDN in front of export caches them. A restarted publisher gets URLs of its own, whose numbering starts over. The multivariant playlist and manifest switch to it at once, while a player already on the old instance keeps playing until that publisher leaves; the player then has to reload the multivariant playlist. The old instance follows its catalog and records only while players read it, stopping once a `--window` passes without a request and resuming on the next. An instance without an epoch (one that reached export over moq-lite-06 or older) gets an id each export mints, so only that export serves its URLs. +- **URLs name the publisher instance.** Every URL the multivariant playlist and DASH manifest list sits under `@`, the publisher's epoch, so every edge and every reload derive the same URLs and a CDN in front of export caches them. A restarted publisher gets URLs of its own, whose numbering starts over. The multivariant playlist and manifest switch to it at once, while a player already on the old instance keeps playing until that publisher leaves; the player then has to reload the multivariant playlist. An instance without an epoch (one that reached export over moq-lite-06 or older) gets an id each export mints, so only that export serves its URLs. - **Only a clean finish ends a playlist.** `EXT-X-ENDLIST` and a `static` DASH manifest mean the records finished. A rendition whose records fail instead, such as a publisher dropping its session, answers HTTP 500 with `Cache-Control: no-store`, so a truncated recording never looks complete, and leaves the multivariant playlist and DASH manifest so players switch to a sibling. With none left, the DASH manifest answers the error too. A [`moq-archive`](https://docs.rs/moq-archive) recording is served the same diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 75e25ae1a4f..9a71f1afdfe 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -111,19 +111,13 @@ pub struct Broadcaster { impl Broadcaster { /// Start tracking the renditions of `source`'s catalog broadcast. pub fn new(source: moq_mux::Source, config: Config) -> Arc { - // An epoch names the instance on every edge, so each serves it under the same URLs. - let instance = source.broadcast().info().epoch.clone().unwrap_or_else(moq_net::Epoch::mint); - Self::with_instance(source, config, Arc::from(instance.as_str())) - } - - /// [`new`](Self::new), listing every URL under `instance`, so a broadcaster rebuilt for an - /// epochless instance keeps the id its URLs were listed under. - pub(crate) fn with_instance(source: moq_mux::Source, config: Config, instance: Arc) -> Arc { let broadcast = source.broadcast(); + // An epoch names the instance on every edge, so each serves it under the same URLs. + let instance = broadcast.info().epoch.clone().unwrap_or_else(moq_net::Epoch::mint); let renditions = renditions::Producer::new(config.window); let broadcaster = Arc::new(Self { broadcast, - instance, + instance: Arc::from(instance.as_str()), renditions: renditions.clone(), watcher: Mutex::new(None), }); diff --git a/rs/moq-hls/src/server/mod.rs b/rs/moq-hls/src/server/mod.rs index d4a91efa5f4..bc5e2430816 100644 --- a/rs/moq-hls/src/server/mod.rs +++ b/rs/moq-hls/src/server/mod.rs @@ -16,9 +16,7 @@ //! for an epochless instance. A publisher that restarts gets its own URLs, so a playlist URL //! never resets its media sequence. A replaced instance keeps serving its URLs while its //! broadcast stays open, so a player already on it keeps playing until its publisher goes, -//! then reloads the master to reach the replacement. A replaced instance watches its catalog -//! and records only while requests reach it: once a playlist window passes without one, it -//! stops, and the next request starts over under the same URLs. +//! then reloads the master to reach the replacement. //! //! `{kind}` is `video` or `audio`, so a video and an audio rendition that share a //! name address distinct resources. The two `seg/` forms name the same bytes: HLS @@ -104,47 +102,46 @@ struct Followed { struct Instances { /// The instance the master playlist and manifest list: the latest to take over. - current: Arc, + current: Arc, /// Every instance a listed URL still names, the current one included. A replaced instance /// stays until its broadcast closes, so a player already on it keeps playing. - by_id: HashMap, Arc>, + by_id: HashMap, Arc>, } impl Followed { - fn new(current: Arc) -> Self { - let by_id = HashMap::from([(current.id.clone(), current.clone())]); + fn new(current: Arc) -> Self { + let by_id = HashMap::from([(current.instance().clone(), current.clone())]); Self { instances: Mutex::new(Instances { current, by_id }), task: Mutex::new(None), } } - fn current(&self) -> Arc { + fn current(&self) -> Arc { self.instances.lock().unwrap().current.clone() } - fn instance(&self, id: &str) -> Option> { + fn instance(&self, id: &str) -> Option> { self.instances.lock().unwrap().by_id.get(id).cloned() } - /// List `instance` from now on, retaining the one it replaces. An instance returning after a - /// gap keeps its epoch, and so its id, which then names the new instance. - fn start(&self, instance: Arc) { + /// List `broadcaster` from now on. An instance returning after a gap keeps its epoch, and so + /// its id, which then names the new broadcaster. + fn start(&self, broadcaster: Arc) { let mut instances = self.instances.lock().unwrap(); - instances.by_id.insert(instance.id.clone(), instance.clone()); - std::mem::replace(&mut instances.current, instance).retire(); + instances + .by_id + .insert(broadcaster.instance().clone(), broadcaster.clone()); + instances.current = broadcaster; } /// Keep serving every instance of `replaced` still open, unless this one lists its id. fn adopt(&self, replaced: &Followed) { let replaced = replaced.instances.lock().unwrap(); let mut instances = self.instances.lock().unwrap(); - for (id, instance) in &replaced.by_id { - if !instance.is_closed() { - instances.by_id.entry(id.clone()).or_insert_with(|| { - instance.retire(); - instance.clone() - }); + for (id, broadcaster) in &replaced.by_id { + if !broadcaster.is_closed() { + instances.by_id.entry(id.clone()).or_insert_with(|| broadcaster.clone()); } } } @@ -154,7 +151,7 @@ impl Followed { let mut instances = self.instances.lock().unwrap(); instances .by_id - .retain(|_, instance| instance.broadcast.poll_closed(waiter).is_pending()); + .retain(|_, broadcaster| broadcaster.poll_closed(waiter).is_pending()); } } @@ -166,117 +163,6 @@ impl Drop for Followed { } } -/// One publisher instance, which the URLs under `@{id}` name. -/// -/// The current instance keeps its [`Broadcaster`], and so its catalog and record watchers, while -/// it is listed. A replaced one is retained only for players already on it, so it watches only -/// while read: a request starts a broadcaster, which stops once no request has reached it for a -/// playlist window, and the next request starts another under the same URLs. -struct Instance { - id: Arc, - /// What every broadcaster for this instance reads, so a rebuilt one binds the same sibling - /// broadcasts its predecessor's rows were listed against. - source: moq_mux::Source, - broadcast: moq_net::broadcast::Consumer, - config: Config, - watch: Mutex, -} - -#[derive(Default)] -struct Watch { - /// Serving this instance: held while it is current, then until its reads go idle. - broadcaster: Option>, - /// The latest read since the instance was replaced, or `None` while it is current. - read: Option, - /// Releases the broadcaster once reads go idle; running only while a replaced instance is - /// watched. - release: Option>, -} - -impl Instance { - /// The instance `source`'s broadcast serves, watched from now on as the current one. - fn current(source: moq_mux::Source, config: Config) -> Arc { - let broadcaster = Broadcaster::new(source.clone(), config.clone()); - Arc::new(Self { - id: broadcaster.instance().clone(), - broadcast: source.broadcast(), - source, - config, - watch: Mutex::new(Watch { - broadcaster: Some(broadcaster), - ..Default::default() - }), - }) - } - - fn is_closed(&self) -> bool { - self.broadcast.is_closed() - } - - /// The broadcaster serving this instance, started if it was idle, and kept a while longer if - /// it is a replaced one. - fn broadcaster(self: &Arc) -> Arc { - let mut watch = self.watch.lock().unwrap(); - let broadcaster = watch - .broadcaster - .get_or_insert_with(|| Broadcaster::with_instance(self.source.clone(), self.config.clone(), self.id.clone())) - .clone(); - if watch.read.is_some() { - self.read(&mut watch); - } - broadcaster - } - - /// Stop holding the broadcaster for listing, and keep it only while read. A player already on - /// this instance reads it again within a window, so its broadcaster carries on warm. - fn retire(self: &Arc) { - let mut watch = self.watch.lock().unwrap(); - if watch.read.is_none() { - self.read(&mut watch); - } - } - - /// Note a read now, and release the broadcaster once reads go idle. - fn read(self: &Arc, watch: &mut Watch) { - watch.read = Some(moq_clock::now()); - if watch.release.is_none() && watch.broadcaster.is_some() { - watch.release = Some(tokio::spawn(release_idle(Arc::downgrade(self), self.config.window))); - } - } -} - -impl Drop for Instance { - fn drop(&mut self) { - if let Some(release) = self.watch.get_mut().unwrap().release.take() { - release.abort(); - } - } -} - -/// Drop `instance`'s broadcaster once nothing has read it for `idle`, stopping its watchers unless -/// a response still holds it. Holds the instance weakly, so it never keeps it retained. -async fn release_idle(instance: Weak, idle: Duration) { - loop { - let deadline = { - let Some(instance) = instance.upgrade() else { return }; - let watch = instance.watch.lock().unwrap(); - let Some(read) = watch.read else { return }; - read + idle - }; - moq_clock::sleep_until(deadline).await; - - let Some(instance) = instance.upgrade() else { return }; - let mut watch = instance.watch.lock().unwrap(); - if watch.read.is_some_and(|read| read + idle <= moq_clock::now()) { - watch.release = None; - let broadcaster = watch.broadcaster.take(); - drop(watch); - drop(broadcaster); - return; - } - } -} - impl Server { /// Build a server reading broadcasts from `origin`, which it watches whole while it /// lives. Every request is served; gate access by layering middleware onto @@ -301,13 +187,13 @@ impl Server { /// instance that takes over. Returns `None` if the broadcast is not announced once the /// origin has synced with the relay. pub async fn broadcaster(&self, name: &str) -> Option> { - Some(self.followed(name).await?.current().broadcaster()) + Some(self.followed(name).await?.current()) } /// The [`Broadcaster`] for the publisher instance `id` of `name`, until its broadcast - /// closes. A replaced instance starts watching again if it was idle. + /// closes. pub(crate) async fn instance(&self, name: &str, id: &str) -> Option> { - Some(self.followed(name).await?.instance(id)?.broadcaster()) + self.followed(name).await?.instance(id) } /// Follow `name` from the instance serving it now, unless it is followed already. @@ -366,10 +252,10 @@ impl Server { } impl Inner { - /// The instance `broadcast`, watched as the current one. - fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Arc { + /// A broadcaster serving the instance `broadcast`. + fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Arc { let source = moq_mux::Source::new(self.origin.consume(), broadcast); - Instance::current(source, self.config.clone()) + Broadcaster::new(source, self.config.clone()) } } @@ -540,23 +426,23 @@ mod tests { assert_eq!(start.elapsed(), Duration::ZERO); } - async fn closed_instance() -> Arc { + async fn closed_broadcaster() -> Arc { let origin = produce_origin(); let producer = origin.create_broadcast("gone").expect("publish allowed"); producer.announce(Default::default()).expect("publish allowed"); settle().await; - let instance = Instance::current(crate::resolved(&origin, "gone").await, Config::default()); - // Close the publisher so the resolved broadcast (and the instance) reports closed. + let broadcaster = Broadcaster::new(crate::resolved(&origin, "gone").await, Config::default()); + // Close the publisher so the resolved broadcast (and the broadcaster) reports closed. producer.close(); settle().await; - instance + broadcaster } #[tokio::test] async fn broadcaster_replaces_finished_cached_instance() { let origin = produce_origin(); let server = Server::new(origin.consume(), Config::default()); - let stale = closed_instance().await; + let stale = closed_broadcaster().await; server .inner @@ -570,7 +456,7 @@ mod tests { let fresh = server.broadcaster("live").await.expect("broadcast announced"); - assert!(!Arc::ptr_eq(&stale.broadcaster(), &fresh)); + assert!(!Arc::ptr_eq(&stale, &fresh)); assert!(server.inner.broadcasts.lock().unwrap().contains_key("live")); } @@ -678,11 +564,11 @@ mod tests { .publish("live", moq_net::origin::Route::default()) .expect("publish allowed"); settle().await; - let open = Instance::current(crate::resolved(&origin, "live").await, Config::default()); + let open = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); // The follower has not yet seen the takeover instance close. let followed = Followed::new(open.clone()); - followed.start(closed_instance().await); + followed.start(closed_broadcaster().await); server .inner .broadcasts @@ -692,84 +578,10 @@ mod tests { let current = server.broadcaster("live").await.expect("broadcast announced"); assert!(!current.is_closed()); - let kept = server.instance("live", &open.id).await; + let kept = server.instance("live", open.instance()).await; assert!( - kept.is_some_and(|kept| Arc::ptr_eq(&kept, &open.broadcaster())), + kept.is_some_and(|kept| Arc::ptr_eq(&kept, &open)), "the open instance lost its URLs" ); } - - /// Whether a replaced `instance` holds a broadcaster, and so its watchers. It runs the task - /// that releases one exactly while it does. - fn watching(instance: &Instance) -> bool { - let watch = instance.watch.lock().unwrap(); - assert_eq!(watch.broadcaster.is_some(), watch.release.is_some()); - watch.broadcaster.is_some() - } - - /// A replaced instance watches only while read: its warm broadcaster stops once a window - /// passes without a request, a later request starts another under the same id, which reads - /// keep going, and which stops again once they stop. - #[tokio::test(start_paused = true)] - async fn a_replaced_instance_watches_only_while_read() { - let origin = produce_origin(); - let config = Config::default(); - let server = Server::new(origin.consume(), config.clone()); - let (old_epoch, new_epoch) = (moq_net::Epoch::mint(), moq_net::Epoch::mint()); - let route = |epoch: &moq_net::Epoch| moq_net::origin::Route::default().with_epoch(epoch.clone()); - - let _old = origin.publish("live", route(&old_epoch)).expect("publish allowed"); - settle().await; - let first = Arc::downgrade(&server.broadcaster("live").await.expect("broadcast announced")); - let _new = origin.publish("live", route(&new_epoch)).expect("publish allowed"); - eventually(async || { - let current = server.broadcaster("live").await.expect("broadcast announced"); - current.instance().as_ref() == new_epoch.as_str() - }) - .await; - let followed = server.followed("live").await.expect("followed"); - let old = followed.instance(old_epoch.as_str()).expect("the old instance is retained"); - assert!(watching(&old), "a player on the replaced instance keeps it warm"); - - moq_clock::sleep(config.window).await; - settle().await; - assert!(!watching(&old), "an unread replaced instance holds no watchers"); - assert_eq!(first.strong_count(), 0, "its broadcaster is gone"); - - let restarted = server.instance("live", old_epoch.as_str()).await.expect("still served"); - assert_eq!(restarted.instance().as_ref(), old_epoch.as_str()); - assert!(watching(&old), "a request restarts it"); - let restarted = Arc::downgrade(&restarted); - for _ in 0..4 { - moq_clock::sleep(config.window / 2).await; - let read = server.instance("live", old_epoch.as_str()).await.expect("still served"); - assert!( - Arc::ptr_eq(&read, &restarted.upgrade().expect("held")), - "reads keep the one broadcaster" - ); - } - - moq_clock::sleep(config.window).await; - settle().await; - assert!(!watching(&old), "it stops again once its readers go idle"); - assert_eq!(restarted.strong_count(), 0); - } - - /// A rebuilt broadcaster keeps the id an epochless instance's URLs were listed under. - #[tokio::test(start_paused = true)] - async fn an_epochless_instance_restarts_under_its_id() { - let origin = produce_origin(); - let _publication = origin - .publish("live", moq_net::origin::Route::default()) - .expect("publish allowed"); - settle().await; - let config = Config::default(); - let instance = Instance::current(crate::resolved(&origin, "live").await, config.clone()); - instance.retire(); - moq_clock::sleep(config.window).await; - settle().await; - assert!(!watching(&instance)); - - assert_eq!(instance.broadcaster().instance(), &instance.id); - } } diff --git a/rs/moq-hls/src/server/routes.rs b/rs/moq-hls/src/server/routes.rs index 9eb95f1f5f5..4e8d34a6c2f 100644 --- a/rs/moq-hls/src/server/routes.rs +++ b/rs/moq-hls/src/server/routes.rs @@ -465,12 +465,6 @@ mod tests { assert_eq!(status(&app, &old_playlist).await, StatusCode::OK); assert_eq!(status(&app, &old_segment).await, StatusCode::OK); - // Unread for a window, the old instance stops watching, and the next request restarts it - // under the same URLs. - tokio::time::sleep(crate::export::Config::default().window).await; - wait_listed(&app, &old_playlist, "seg/53e0aafd.0.m4s").await; - assert_eq!(status(&app, &old_segment).await, StatusCode::OK); - drop((old_catalog, old_track, old_media, old)); let deadline = tokio::time::Instant::now() + TIMEOUT; while status(&app, &old_playlist).await != StatusCode::NOT_FOUND { From 88aabae35cf69ba6b22d41ff32aaf617d62d988e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 11:30:25 -0800 Subject: [PATCH 4/4] quest: delete hls-lazy-instances, moot after #7399 moq-dev/moq#7399 already drops a replaced HLS instance's catalog and record watchers on takeover, which was this quest's goal. The leftover frozen window and idle driver are small and bounded. Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 1 - quest/m2/hls-lazy-instances.md | 19 ------------------- 2 files changed, 20 deletions(-) delete mode 100644 quest/m2/hls-lazy-instances.md diff --git a/quest/m2/README.md b/quest/m2/README.md index b72e6da55be..45f8549b469 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -43,7 +43,6 @@ lite-07; bitrate enforcement is not part of this release. - [Benchmark sources](/quest/m2/bench-sources.md) - every measuring nightly benchmark feeds the regression table, and none is a smoke run - [Poll-native async](/quest/m2/ffi-poll-migrate.md) - moq-ffi's pure-kio async exports poll their state directly instead of going through `ffi::Task` - [QoS](/quest/m2/qos/README.md) - the relay's egress `lag` histogram and `dropped` counters on the stats totals and prefix tracks -- [Lazy retained HLS instances](/quest/m2/hls-lazy-instances.md) - a replaced HLS instance starts its watchers only when a request reaches it - [HLS and DASH browser playback](/quest/m2/hls-player-test.md) - hls.js and dash.js play, reload, switch, and survive reconfiguration nightly - [The DASH manifest keeps segments across a reconfigure](/quest/m2/dash-reconfigure.md) - a same-codec reconfigure keeps the old instance's segments listed in DASH, as HLS does - [Re-init retires timelines](/quest/m2/reinit-retire.md) - fMP4 and MKV re-inits retire the old timeline, so import state stays bounded diff --git a/quest/m2/hls-lazy-instances.md b/quest/m2/hls-lazy-instances.md deleted file mode 100644 index 48930111bcd..00000000000 --- a/quest/m2/hls-lazy-instances.md +++ /dev/null @@ -1,19 +0,0 @@ -# [S] Retained HLS instances watch only when read - -## Goal - -A replaced HLS instance that the server keeps for readers following a -restart starts its catalog and timeline watchers only when a request reaches -it, so an idle retained instance costs no tasks. - -## Plan - -Decided with the maintainer on 2026-10-24: m2, an efficiency gain. Retention -is already bounded by the broadcasts the publisher keeps open -([moq-dev/moq#6649](https://github.com/moq-dev/moq/pull/6649)). - -- Start the watchers on the first request for that instance, keep them - while requests continue, stop them once its readers go idle, and restart - them on a later request. Tests show an unread retained instance holds - none, including after its last reader leaves, and that a later request - restarts them.