From ddba68c13fbabb97177987698a67737ec00e8fc8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 02:25:55 -0700 Subject: [PATCH 1/6] chore(quest): claim quest/m2/hls-export-kio.md Quest-Claim: 94dabcd0e7cc3e47 From 10544f2ca61cd6012ff4fe0494a405b38bb5ef03 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 03:23:02 -0700 Subject: [PATCH 2/6] refactor(hls)!: export background work runs on a kio Driver Broadcaster::new returns a Driver alongside the broadcaster. Its kio::Tasks set runs the catalog watcher, each rendition's record watcher, and init builds, instead of tokio::spawn. Each owner holds a guard that ends its task when dropped. The catalog watcher is polled first in every pass, so a reconfigure retires its rendition before that rendition reads later records. Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 1 - quest/m2/hls-export-kio.md | 21 --- quest/m2/spawn-audit.md | 3 +- rs/moq-cli/src/hls.rs | 3 +- rs/moq-cli/src/hls/record.rs | 4 +- rs/moq-hls/src/export/archive_tests.rs | 2 +- rs/moq-hls/src/export/driver.rs | 159 ++++++++++++++++++++ rs/moq-hls/src/export/mod.rs | 191 +++++++++++++++++-------- rs/moq-hls/src/export/rendition.rs | 39 ++--- rs/moq-hls/src/export/renditions.rs | 27 +++- rs/moq-hls/src/server/mod.rs | 12 +- rs/moq-hls/tests/integration/server.rs | 3 +- 12 files changed, 340 insertions(+), 125 deletions(-) delete mode 100644 quest/m2/hls-export-kio.md create mode 100644 rs/moq-hls/src/export/driver.rs diff --git a/quest/m2/README.md b/quest/m2/README.md index 4f4f2452b0a..dd741d4d23d 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -45,7 +45,6 @@ lite-07; bitrate enforcement is not part of this release. - [QoS](/quest/m2/qos/README.md) - the relay's egress `lag` histogram and `dropped` counters on the stats totals and prefix tracks - [Lazy retained HLS instances](/quest/m2/hls-lazy-instances.md) - a replaced HLS instance starts its watchers only when a request reaches it - [HLS and DASH browser playback](/quest/m2/hls-player-test.md) - hls.js and dash.js play, reload, switch, and survive reconfiguration nightly -- [moq-hls export on kio](/quest/m2/hls-export-kio.md) - export watchers and the init build are owned kio tasks instead of tokio::spawn - [Re-init retires timelines](/quest/m2/reinit-retire.md) - fMP4 and MKV re-inits retire the old timeline, so import state stays bounded - [Replaced producers retire their records](/quest/m2/producer-retire.md) - encoders and bindings retire a replaced producer's HLS records - [Broadcast fairness](/quest/m2/broadcast-fairness.md) - broadcasts share a cluster link fairly, so one tenant's priority never starves another's diff --git a/quest/m2/hls-export-kio.md b/quest/m2/hls-export-kio.md deleted file mode 100644 index 7b02ad30530..00000000000 --- a/quest/m2/hls-export-kio.md +++ /dev/null @@ -1,21 +0,0 @@ -# [M] moq-hls export runs on kio - -## Goal - -moq-hls export's background work (the catalog, timeline, and spans watchers in -`rs/moq-hls/src/export/{mod,rendition}.rs`, and the init build) runs as -poll-driven `kio` tasks the export owns, not on `tokio::spawn`, as -`rs/AGENTS.md` prefers. - -## Plan - -Do it crate-wide in one PR. Tasks end with their owner instead of being -detached, which also removes the abort handles kept for them today. Keep the -timeline watcher's two lifetime rules: a new reference timeline replaces the -old watcher, and a recording cursor that outlives the `Broadcaster` keeps the -watcher running until the timeline ends (see `Broadcaster`'s `Drop`). Test -both. Tests -keep their own spawns where they drive a runtime. Measure export throughput -before and after with the existing moq-hls benchmarks. - -Public API: none expected. Wire: none. diff --git a/quest/m2/spawn-audit.md b/quest/m2/spawn-audit.md index 8f6dfd48caf..ef8f07a03d5 100644 --- a/quest/m2/spawn-audit.md +++ b/quest/m2/spawn-audit.md @@ -12,7 +12,8 @@ Blocking OS threads are out of scope. Planned 2026-10-25 with the maintainer as the follow-up to no-spawn. As of `main` today the spawning library crates include moq-tokio (connection, server, client), moq-relay (web, shutdown, uring), moq-mux's container -consumer, moq-hls export, moq-rtmp, moq-rtsp, moq-srt, moq-rtc, moq-transcode, +consumer, moq-hls's server (its path follower and each +broadcaster's driver), moq-rtmp, moq-rtsp, moq-srt, moq-rtc, moq-transcode, moq-video's encode producer, moq-cli's play path, moq-bench, and qmux's transport. Recount with `rg 'tokio::spawn|spawn_local|task::spawn'` outside tests, benches, and binaries before starting. diff --git a/rs/moq-cli/src/hls.rs b/rs/moq-cli/src/hls.rs index a10b1890769..d78b61cae93 100644 --- a/rs/moq-cli/src/hls.rs +++ b/rs/moq-cli/src/hls.rs @@ -304,7 +304,8 @@ pub async fn export(origin: moq_net::origin::Consumer, args: ExportArgs, name: S .await .with_context(|| format!("failed to resolve broadcast `{name}`"))?; let source = moq_mux::Source::new(scoped.clone(), broadcast); - let broadcaster = moq_hls::export::Broadcaster::new(source, config); + let (broadcaster, driver) = moq_hls::export::Broadcaster::new(source, config); + tokio::spawn(driver.run()); tracing::info!(%name, "recording HLS"); recording.run(&broadcaster).await }; diff --git a/rs/moq-cli/src/hls/record.rs b/rs/moq-cli/src/hls/record.rs index 52ddcbcc8df..ee3cf8ad602 100644 --- a/rs/moq-cli/src/hls/record.rs +++ b/rs/moq-cli/src/hls/record.rs @@ -616,7 +616,9 @@ mod tests { fn broadcaster(&self) -> Arc { let source = moq_mux::Source::new(self.origin.consume(), self.broadcast.consume()); - Broadcaster::new(source, moq_hls::export::Config::default()) + let (broadcaster, driver) = Broadcaster::new(source, moq_hls::export::Config::default()); + tokio::spawn(driver.run()); + broadcaster } /// Write GOP `n`, two seconds from `2n`, to every rendition. diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs index 59b18280362..d561f37c36f 100644 --- a/rs/moq-hls/src/export/archive_tests.rs +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -284,7 +284,7 @@ impl Replay { let consumer = tokio::spawn(consumer.run()); broadcast.announce(Default::default()).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "rec").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "rec").await, Config::default()); tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()) .await .expect("a rendition becomes playable or fails"); diff --git a/rs/moq-hls/src/export/driver.rs b/rs/moq-hls/src/export/driver.rs new file mode 100644 index 00000000000..d38b01d8b5b --- /dev/null +++ b/rs/moq-hls/src/export/driver.rs @@ -0,0 +1,159 @@ +//! The export's background work, as `kio` tasks one [`Driver`] polls. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Poll; + +/// Runs a [`Broadcaster`](super::Broadcaster)'s background work: its catalog watcher, each +/// rendition's record watcher, and the init builds. +/// +/// Poll it, or [`run`](Self::run) it, for as long as the broadcaster serves. It finishes once +/// the broadcaster and every rendition it created have dropped and their tasks ended, so a +/// recording cursor that outlives the broadcaster keeps its rendition's records flowing. +/// Dropping it early stops all of that work. +#[must_use = "the broadcaster does nothing until its driver runs"] +pub struct Driver { + /// The catalog watcher, polled before the other tasks in every pass: a reconfigure must + /// retire its rendition before that rendition's watcher reads records published after it. + lead: Option, + inbox: kio::Queue, + tasks: kio::Tasks, + /// The inbox closed: every owner that could start a task has dropped. + closed: bool, +} + +impl Driver { + /// Advance every task, ready once nothing is left to run. + pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + if let Some(lead) = &mut self.lead + && kio::Task::poll(lead, waiter).is_ready() + { + self.lead = None; + } + while !self.closed { + match self.inbox.poll_pop(waiter) { + Poll::Ready(Ok(job)) => self.tasks.push(job), + Poll::Ready(Err(kio::Closed)) => self.closed = true, + Poll::Pending => break, + } + } + match self.tasks.poll(waiter) { + Poll::Ready(()) if self.closed && self.lead.is_none() => Poll::Ready(()), + _ => Poll::Pending, + } + } + + /// Run `task` ahead of every other task (see [`lead`](Self::lead)), until it finishes or the + /// returned [`Owner`] drops. + pub(crate) fn lead(&mut self, task: impl Future + Send + 'static) -> Owner { + let (job, owner) = Job::new(task); + self.lead = Some(job); + owner + } + + /// [`poll`](Self::poll) until it is ready. + pub async fn run(mut self) { + kio::wait(|waiter| self.poll(waiter)).await + } +} + +impl Drop for Driver { + fn drop(&mut self) { + // Later starts are refused instead of queueing work nothing will run. + self.inbox.close(); + } +} + +/// Starts tasks on its [`Driver`]. Clones share it; the driver's inbox closes with the last. +#[derive(Clone)] +pub(crate) struct Spawner { + inbox: Arc, +} + +struct Inbox(kio::Queue); + +impl Drop for Inbox { + fn drop(&mut self) { + self.0.close(); + } +} + +impl Spawner { + /// A spawner and the driver that runs what it starts. + pub fn new() -> (Self, Driver) { + let inbox = kio::Queue::new(); + let driver = Driver { + lead: None, + inbox: inbox.clone(), + tasks: kio::Tasks::new(), + closed: false, + }; + ( + Self { + inbox: Arc::new(Inbox(inbox)), + }, + driver, + ) + } + + /// Run `task` on the driver until it finishes or the returned [`Owner`] drops. + pub fn spawn(&self, task: impl Future + Send + 'static) -> Owner { + let (job, owner) = Job::new(task); + // Refused once the driver is gone, which drops the task unrun. + let _ = self.inbox.0.try_push(job); + owner + } +} + +/// Ends its task when dropped. +pub(crate) struct Owner(#[expect(dead_code, reason = "held only to close on drop")] kio::Producer<()>); + +/// A task and the [`Owner`] that ends it. +struct Job { + owner: kio::Consumer<()>, + task: Pin + Send>>, +} + +impl Job { + fn new(task: impl Future + Send + 'static) -> (Self, Owner) { + let owner = kio::Producer::new(()); + let job = Self { + owner: owner.consume(), + task: Box::pin(task), + }; + (job, Owner(owner)) + } +} + +impl kio::Task for Job { + type Output = (); + + fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + if self.owner.poll_closed(waiter).is_ready() { + return Poll::Ready(()); + } + waiter.poll_future(self.task.as_mut()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The driver finishes once every spawner and task is gone, and an owner's drop ends its task + // without waiting for the task to finish on its own. + #[test] + fn tasks_end_with_their_owner() { + let (spawner, mut driver) = Spawner::new(); + let waiter = kio::Waiter::noop(); + let forever = spawner.spawn(std::future::pending()); + let lead = driver.lead(std::future::pending()); + drop(spawner); + assert!(driver.poll(&waiter).is_pending(), "the owned tasks still run"); + drop(lead); + assert!(driver.poll(&waiter).is_pending(), "the owned task still runs"); + drop(forever); + assert!(driver.poll(&waiter).is_ready(), "the task ended with its owner"); + } +} diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 207de7dc38a..c5d429945ac 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -15,6 +15,10 @@ //! group on the first playlist render (in the background on a manifest render) to build its //! init, then caches it. //! +//! That background work (the catalog and record watchers, and init builds) runs on the +//! [`Driver`] [`Broadcaster::new`] returns: poll it or spawn [`Driver::run`] wherever the +//! embedder runs tasks. +//! //! The same machinery serves two kinds of consumer: //! //! * the HTTP serve path (pull): [`Route::parse`] names the resource a request path addresses, @@ -29,6 +33,7 @@ #[cfg(test)] #[expect(clippy::disallowed_methods, reason = "tests read the host clock directly")] mod archive_tests; +mod driver; pub mod master; mod mpd; mod playlist; @@ -38,12 +43,13 @@ mod route; pub mod renditions; pub mod segments; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::Duration; use moq_mux::catalog::{self, CatalogFormat, Stream}; use rand::RngExt; +pub use driver::Driver; pub(crate) use playlist::render_media; pub use rendition::{Kind, Rendition}; pub use route::{Resource, Route}; @@ -102,29 +108,38 @@ pub struct Broadcaster { instance: Arc, /// The current rendition set, reconciled from the catalog by the watcher task. renditions: renditions::Producer, - /// Aborts the catalog watcher when the broadcaster is dropped; `Drop` then releases the + /// Ends the catalog watcher when the broadcaster is dropped; `Drop` also releases the /// renditions themselves (see [`renditions::Producer::clear`]), each of which stops following - /// its records once no recording cursor holds it. Set once, right after construction. - watcher: Mutex>>, + /// its records once no recording cursor holds it. + _watcher: driver::Owner, } impl Broadcaster { - /// Start tracking the renditions of `source`'s catalog broadcast. - pub fn new(source: moq_mux::Source, config: Config) -> Arc { + /// Start tracking the renditions of `source`'s catalog broadcast, once the returned + /// [`Driver`] runs. + pub fn new(source: moq_mux::Source, config: Config) -> (Arc, Driver) { let broadcast = source.broadcast(); // An epoch names the instance on every edge, so each serves it under the same URLs. let instance = broadcast.info().epoch.clone().unwrap_or_else(moq_net::Epoch::mint); - let renditions = renditions::Producer::new(config.window); + let (spawner, mut driver) = driver::Spawner::new(); + let renditions = renditions::Producer::new(config.window, spawner); + // The watcher owns its own producer clone; dropping the `Broadcaster` ends it so the + // standing catalog subscription stops when nobody's serving from this broadcaster. + let watcher = driver.lead(watch_catalog(source, renditions.clone())); let broadcaster = Arc::new(Self { broadcast, instance: Arc::from(instance.as_str()), - renditions: renditions.clone(), - watcher: Mutex::new(None), + renditions, + _watcher: watcher, }); - // The watcher owns its own producer clone; the `Broadcaster`'s `Drop` aborts it so the - // standing catalog subscription stops when nobody's serving from this broadcaster. - let watcher = tokio::spawn(watch_catalog(source, renditions)); - *broadcaster.watcher.lock().unwrap() = Some(watcher); + (broadcaster, driver) + } + + /// [`new`](Self::new), with its driver spawned on the ambient runtime. + #[cfg(test)] + pub(crate) fn spawned(source: moq_mux::Source, config: Config) -> Arc { + let (broadcaster, driver) = Self::new(source, config); + tokio::spawn(driver.run()); broadcaster } @@ -382,9 +397,6 @@ impl Broadcaster { impl Drop for Broadcaster { fn drop(&mut self) { - if let Some(watcher) = self.watcher.lock().unwrap().take() { - watcher.abort(); - } // Release the map: a surviving `renditions::Consumer` would otherwise pin every // `Rendition` forever. A recording cursor's own `Arc` keeps that rendition's // records flowing, so its final segments are not truncated. @@ -643,7 +655,7 @@ mod tests { let mut catalog = moq_mux::catalog::hang::Catalog::default(); catalog.video.renditions.insert("video".to_string(), config); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(&upstream, &catalog); assert!(renditions.get(Kind::Video, "video").is_none()); } @@ -667,7 +679,7 @@ mod tests { config.hls = Some(hls(name)); catalog.audio.renditions.insert(name.to_string(), config); } - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(&source, &catalog); let listed = || { renditions @@ -697,7 +709,7 @@ mod tests { settle().await; let upstream = crate::resolved(&origin, "live").await; - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); let mut cursor = renditions.subscribe(); let mut catalog = catalog_with_both(); catalog.audio.renditions.get_mut("audio0").unwrap().hls = None; @@ -734,7 +746,7 @@ mod tests { let upstream = empty_upstream(&origin, "live").await; let mut catalog = catalog_with_both(); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); let mut cursor = renditions.subscribe(); renditions.sync(&upstream, &catalog); @@ -804,7 +816,7 @@ mod tests { let mut catalog = catalog_with_both(); catalog.video.renditions.get_mut("video0").unwrap().framerate = None; - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); let mut cursor = renditions.subscribe(); renditions.sync(&upstream, &catalog); let video = renditions.get(Kind::Video, "video0").expect("video rendition"); @@ -833,7 +845,7 @@ mod tests { let mut catalog = catalog_with_both(); catalog.audio.renditions.get_mut("audio0").unwrap().bitrate = None; - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(&upstream, &catalog); let audio = renditions.get(Kind::Audio, "audio0").expect("audio rendition"); @@ -865,7 +877,7 @@ mod tests { let upstream = empty_upstream(&origin, "live").await; let mut catalog = catalog_with_both(); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); let mut cursor = renditions.subscribe(); renditions.sync(&upstream, &catalog); @@ -907,7 +919,7 @@ mod tests { let upstream = empty_upstream(&origin, "live").await; let mut catalog = catalog_with_both(); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); for width in 1..=8 { catalog.video.renditions.get_mut("video0").unwrap().coded_width = Some(width); renditions.sync(&upstream, &catalog); @@ -928,7 +940,7 @@ mod tests { let catalog = hls_catalog(&mut broadcast.broadcast().clone()); let mut media = video(&catalog, &broadcast, "video0", video_config()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let rendition = tokio::time::timeout(Duration::from_secs(5), async { loop { if let Some(rendition) = broadcaster.rendition(Kind::Video, "video0") { @@ -1011,7 +1023,7 @@ mod tests { records.push(&gop(sequence)).unwrap(); } } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); tokio::time::timeout(Duration::from_secs(5), broadcaster.advertised()) .await .expect("video0 is advertised"); @@ -1044,7 +1056,7 @@ mod tests { media.write(vp8_frame(i * 2_000_000, true)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); tokio::time::timeout(Duration::from_secs(5), broadcaster.advertised()) .await .expect("the rendition is advertised"); @@ -1119,7 +1131,7 @@ mod tests { let (mut catalog, _media) = publish_vp8(&mut broadcast.broadcast().clone(), inline_vp8()); broadcast.announce(moq_net::origin::Route::default()).unwrap(); settle().await; - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.ready().await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let mut cursor = rendition.segments(); @@ -1152,7 +1164,7 @@ mod tests { records.push(&stop(1, 2_000)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.ready().await; let rendition = broadcaster.rendition(Kind::Video, "video1").expect("rendition"); until_listed(&rendition, 1).await; @@ -1296,7 +1308,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; listed_in_manifest(&broadcaster).await; @@ -1313,7 +1325,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; tokio::time::timeout(Duration::from_millis(1), broadcaster.video_init()) @@ -1368,7 +1380,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; let broken = broadcaster.rendition(Kind::Video, "video0").unwrap(); until_listed(&broken, 3).await; @@ -1401,7 +1413,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; let stuck = broadcaster.rendition(Kind::Video, "video1").unwrap(); @@ -1426,7 +1438,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; let render = broadcaster.clone(); @@ -1448,7 +1460,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; broadcaster.set_generation(Some("run1")).unwrap(); until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; @@ -1473,7 +1485,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; let rendition = broadcaster.rendition(Kind::Video, "video0").unwrap(); until_listed(&rendition, 3).await; @@ -1514,7 +1526,7 @@ mod tests { let mut open = media.create_group(1u64.into()).unwrap(); write_keyframe(&mut open, 1); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 2).await; catalog.finish().unwrap(); @@ -1540,7 +1552,7 @@ mod tests { let (origin, broadcast, _catalog, mut records) = inline_pair("video0").await; let media = broadcast.create_track("video0", timed()).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); while broadcaster.rendition(Kind::Video, "video0").is_none() { moq_clock::sleep(Duration::from_millis(10)).await; } @@ -1577,7 +1589,7 @@ mod tests { group.finish().unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; for _ in 0..3 { @@ -1607,7 +1619,7 @@ mod tests { settle().await; let (_catalog, mut media) = publish_vp8(&mut broadcast.broadcast().clone(), video_config()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let old = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let old_playlist = old @@ -1646,6 +1658,49 @@ mod tests { assert!(new.init(&old_hash).await.unwrap().is_none()); } + // Background work ends with its owners, not with the publisher: once a reconfigure replaced + // the rendition, dropping the broadcaster and both renditions finishes the driver while the + // records stay open. + #[tokio::test] + async fn the_driver_finishes_with_its_owners() { + let origin = produce_origin(); + let broadcast = origin.create_broadcast("live").expect("publish allowed"); + broadcast.announce(Default::default()).expect("publish allowed"); + settle().await; + let (_catalog, mut media) = publish_vp8(&mut broadcast.broadcast().clone(), video_config()); + + let (broadcaster, driver) = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let driving = tokio::spawn(driver.run()); + let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; + let old = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); + + let mut config = video_config(); + config.coded_width = Some(640); + media.set(config).unwrap(); + media.write(vp8_frame(6_000_000, true)).unwrap(); + let new = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let current = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); + if !Arc::ptr_eq(¤t, &old) { + break current; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the reconfigure rebuilt the rendition"); + + drop((broadcaster, new)); + settle().await; + assert!(!driving.is_finished(), "a held rendition keeps the driver running"); + drop(old); + tokio::time::timeout(Duration::from_secs(5), driving) + .await + .expect("the driver finishes once its owners drop") + .unwrap(); + drop(media); + } + // A generation rides every segment URL. The first one labels the run already listed; // replacing it drops that run's rows, and the next run lists under the new generation. #[tokio::test] @@ -1656,7 +1711,7 @@ mod tests { settle().await; let (_catalog, mut media) = publish_vp8(&mut broadcast.broadcast().clone(), video_config()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let tag = rendition.tag().to_string(); @@ -1718,7 +1773,7 @@ mod tests { media.write(vp8_frame(micros, true)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -1753,7 +1808,7 @@ mod tests { // Three GOPs, 2s apart: segments 0 and 1 are complete, segment 2 is the live edge. let (catalog, media) = publish_vp8(&mut broadcast.broadcast().clone(), inline_vp8()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster .rendition(Kind::Video, "video0") @@ -1815,7 +1870,7 @@ mod tests { settle().await; let (catalog, media) = publish_vp8(&mut broadcast.broadcast().clone(), video_config()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -1853,7 +1908,7 @@ mod tests { media.write(vp8_frame(2_000_000, true)).unwrap(); media.write(vp8_frame(4_000_000, true)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -1892,7 +1947,7 @@ mod tests { video.write(frame(4_000_000, true)).unwrap(); // live edge audio_write(4_000_000); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let video_rendition = broadcaster.rendition(Kind::Video, "video0").expect("video discovered"); let audio_rendition = broadcaster.rendition(Kind::Audio, "audio0").expect("audio discovered"); @@ -1973,7 +2028,7 @@ mod tests { media.write(frame(2_000_000, true)).unwrap(); media.write(frame(4_000_000, true)).unwrap(); // live edge - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("video discovered"); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -2034,7 +2089,7 @@ mod tests { records.push(&gop(sequence)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("video discovered"); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -2079,7 +2134,7 @@ mod tests { // Watch before the first fragment, as a live viewer would, so a catalog the moov // published on a provisional clock would stick. - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); assert!( next_event(&mut broadcaster.renditions()).await.is_none(), "the moov alone lists no rendition" @@ -2205,7 +2260,7 @@ mod tests { .await .unwrap(); let source = crate::resolved(consuming, "live").await; - let broadcaster = (!late).then(|| Broadcaster::new(source.clone(), Config::default())); + let broadcaster = (!late).then(|| Broadcaster::spawned(source.clone(), Config::default())); for i in 0..20u64 { let fragment = video_muxer .fragment(i as u32, &[vp8_frame(i * 1_000_000, i % 2 == 0)], None) @@ -2225,7 +2280,7 @@ mod tests { import.decode(&fragment).unwrap(); settle().await; } - let broadcaster = broadcaster.unwrap_or_else(|| Broadcaster::new(source, Config::default())); + let broadcaster = broadcaster.unwrap_or_else(|| Broadcaster::spawned(source, Config::default())); tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()) .await .unwrap(); @@ -2274,7 +2329,7 @@ mod tests { media.write(frame(0, true)).unwrap(); media.write(frame(2_000_000, true)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.playable()).await; @@ -2322,7 +2377,7 @@ mod tests { audio.cut(None).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Audio, "audio0").expect("audio discovered"); let _ = tokio::time::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -2367,7 +2422,7 @@ mod tests { video.write(frame(4_000_000, true)).unwrap(); // segment 2 (live edge) audio_write(4_000_000); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let video_rendition = broadcaster.rendition(Kind::Video, "video0").expect("video discovered"); let audio_rendition = broadcaster.rendition(Kind::Audio, "audio0").expect("audio discovered"); @@ -2416,7 +2471,8 @@ mod tests { media.write(frame(2_000_000, true)).unwrap(); media.write(frame(3_000_000, false)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let (broadcaster, driver) = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let driving = tokio::spawn(driver.run()); let _ = tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster .rendition(Kind::Video, "video0") @@ -2444,7 +2500,14 @@ mod tests { "the final segment survives dropping the broadcaster" ); - drop((catalog, media, broadcast, rendition)); + // The cursor holds its rendition, and the rendition its share of the driver. + assert!(!driving.is_finished(), "the driver outlives a held rendition"); + drop((segments, rendition)); + tokio::time::timeout(Duration::from_secs(5), driving) + .await + .expect("the driver finishes once the cursor drops") + .unwrap(); + drop((catalog, media, broadcast)); } /// A one-video-rendition catalog snapshot reconciled by hand, which is what `watch_catalog` @@ -2457,7 +2520,7 @@ mod tests { config.hls = Some(hls("video0")); let mut catalog = moq_mux::catalog::hang::Catalog::default(); catalog.video.renditions.insert("video0".to_string(), config); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(upstream, &catalog); let rendition = renditions.get(Kind::Video, "video0").expect("rendition synced"); (renditions, rendition) @@ -3121,7 +3184,7 @@ mod tests { media.write(frame(0, true)).unwrap(); media.write(frame(2_000_000, true)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let mut renditions = broadcaster.renditions(); let added = tokio::time::timeout(Duration::from_secs(5), renditions.next()) @@ -3167,7 +3230,7 @@ mod tests { media.write(frame(2_000_000, true)).unwrap(); media.write(frame(4_000_000, true)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let mut renditions = broadcaster.renditions(); let rendition = match tokio::time::timeout(Duration::from_secs(5), renditions.next()) @@ -3207,8 +3270,12 @@ mod tests { let end = tokio::time::timeout(Duration::from_secs(5), segments.next()) .await .expect("segment cursor resolves after the source is lost"); + // The record and catalog watchers race to see the close; either fails the recording. assert!( - matches!(end, Err(crate::Error::RecordsFailed { .. })), + matches!( + end, + Err(crate::Error::RecordsFailed { .. } | crate::Error::CatalogFailed { .. }) + ), "an abrupt end fails the recording: {:?}", end.map(|segment| segment.map(|segment| segment.segment)) ); @@ -3270,7 +3337,7 @@ mod tests { broadcast: &moq_net::broadcast::Producer, catalog: &moq_mux::catalog::hang::Catalog, ) -> renditions::Producer { - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(&Self::source(broadcast), catalog); renditions } @@ -3520,7 +3587,7 @@ mod tests { video0.push(&gop(sequence)).unwrap(); video1.push(&gop(sequence)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; let healthy = broadcaster.rendition(Kind::Video, "video0").unwrap(); let broken = broadcaster.rendition(Kind::Video, "video1").unwrap(); diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 86e3fc297fa..37e22bd829a 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -11,6 +11,7 @@ use hang::catalog::{AudioConfig, Clock, Hls, Reference, VideoConfig}; use moq_mux::container::fmp4::Muxer; use sha2::{Digest, Sha256}; +use super::driver::Owner; use super::playlist::{Segment, Snapshot}; use super::{mpd, renditions, segments}; use crate::Result; @@ -341,8 +342,8 @@ pub struct Rendition { fanout: renditions::Fanout, /// The broadcast carrying the catalog and every record track. catalog: moq_net::broadcast::Consumer, - /// Follows the records into [`live`](Self::live); aborted when the rendition drops. - watcher: Mutex>>, + /// Follows the records into [`live`](Self::live); ends when the rendition drops. + watcher: Mutex>, /// Set once the catalog drops or replaces the rendition (see [`retire`](Self::retire)). retiring: tokio::sync::watch::Sender, /// Which records this instance lists. @@ -359,11 +360,8 @@ pub struct Rendition { /// Serializes init builds, so concurrent first requests fetch the keyframe group once. Shared /// so a background build (see [`start_init`](Self::start_init)) holds it for its whole run. building: Arc>, - /// The background init build; aborted when the rendition drops. - build: Mutex>>, - /// The runtime the rendition was built on, which runs the background init build even when a - /// synchronous render calls from another thread. - runtime: tokio::runtime::Handle, + /// The background init build; ends when the rendition drops. + build: Mutex>, } impl Rendition { @@ -465,7 +463,6 @@ impl Rendition { run: kio::Producer::default(), building: Arc::new(tokio::sync::Mutex::new(())), build: Mutex::new(None), - runtime: tokio::runtime::Handle::current(), }) } @@ -505,13 +502,12 @@ impl Rendition { run: kio::Producer::default(), building: Arc::new(tokio::sync::Mutex::new(())), build: Mutex::new(None), - runtime: tokio::runtime::Handle::current(), }) } /// Start following this rendition's own HLS records. pub(crate) fn watch(self: &Arc) { - let watcher = tokio::spawn(watch_records( + let watcher = self.fanout.spawn(watch_records( self.catalog.clone(), self.hls.clone(), Arc::downgrade(self), @@ -627,9 +623,7 @@ impl Rendition { pub(crate) fn restart(&self, generation: Option>) { // A build parked on the old run's open group would hold `building`, so the new run could // never start its own. - if let Some(build) = self.build.lock().expect("build lock poisoned").take() { - build.abort(); - } + self.build.lock().expect("build lock poisoned").take(); self.live.clear(); let mut run = self.run_mut(); *run = Run { @@ -1023,7 +1017,7 @@ impl Rendition { /// later [`representation`](Self::representation) lists this rendition without any render /// waiting on its media. /// - /// The build never holds the rendition itself, and is aborted when the rendition drops, so + /// The build never holds the rendition itself, and ends when the rendition drops, so /// one stuck on an open group cannot keep a torn-down export's state alive. pub(crate) fn start_init(self: &Arc) { let Ok(binding) = self.authorize() else { @@ -1048,7 +1042,7 @@ impl Rendition { let media = self.media.clone(); let rendition = Arc::downgrade(self); let name = self.name.clone(); - let build = self.runtime.spawn(async move { + let build = self.fanout.spawn(async move { let _building = building; match read.read(&media, &binding).await { Ok(Some(bytes)) => { @@ -1413,17 +1407,6 @@ impl InitRead { } } -impl Drop for Rendition { - fn drop(&mut self) { - if let Some(watcher) = self.watcher.lock().expect("watcher lock poisoned").take() { - watcher.abort(); - } - if let Some(build) = self.build.lock().expect("build lock poisoned").take() { - build.abort(); - } - } -} - /// Fetch one group, mapping "no longer (or not yet) servable" to the error that says so /// rather than a bare `None`: a caller abandoning a segment can only explain itself with it. async fn fetch( @@ -1542,7 +1525,7 @@ mod tests { let source = moq_mux::Source::new(origin.consume(), broadcast.consume()); let mut config = VideoConfig::new(hang::catalog::VideoCodec::VP8); config.broadcast = Some(moq_net::path::Relative::new("./secret").unwrap()); - let renditions = renditions::Producer::new(Duration::from_secs(30)); + let renditions = renditions::Producer::spawned(Duration::from_secs(30)); let rendition = Arc::new(Rendition::video("video".into(), &config, &source, hls(), None, renditions.fanout()).unwrap()); let request = dynamic.requested_broadcast().await.unwrap(); @@ -1714,7 +1697,7 @@ mod tests { let broadcast = origin.create_broadcast("live").unwrap(); let source = moq_mux::Source::new(origin.consume(), broadcast.consume()); let config = VideoConfig::new(hang::catalog::VideoCodec::VP8); - let renditions = renditions::Producer::new(Duration::from_secs(30)); + let renditions = renditions::Producer::spawned(Duration::from_secs(30)); let rendition = Arc::new(Rendition::video("video".into(), &config, &source, hls(), None, renditions.fanout()).unwrap()); // On its own task, so only a registered wake polls it again. diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index d276fc51800..0c31d6e7ebd 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -17,6 +17,7 @@ use std::time::Duration; use moq_mux::catalog::hang::Catalog; +use super::driver::{Owner, Spawner}; use super::master; use super::rendition::{Kind, Rendition}; @@ -131,10 +132,12 @@ struct State { #[derive(Clone)] pub(crate) struct Fanout { feed: kio::Producer, + /// Starts each rendition's background work on the broadcaster's driver. + spawner: Spawner, } impl Producer { - pub fn new(window: Duration) -> Self { + pub fn new(window: Duration, spawner: Spawner) -> Self { Self { state: kio::Producer::new(State::default()), fanout: Fanout { @@ -147,10 +150,19 @@ impl Producer { generation: None, window, }), + spawner, }, } } + /// A producer whose renditions run on a driver spawned on the ambient runtime. + #[cfg(test)] + pub(super) fn spawned(window: Duration) -> Self { + let (spawner, driver) = Spawner::new(); + tokio::spawn(driver.run()); + Self::new(window, spawner) + } + /// How many renditions the fanout still reaches, dropped ones not yet pruned included. #[cfg(test)] pub(super) fn targets(&self) -> usize { @@ -259,6 +271,11 @@ impl Producer { } impl Fanout { + /// Run `task` on the broadcaster's driver until it finishes or the returned [`Owner`] drops. + pub fn spawn(&self, task: impl Future + Send + 'static) -> Owner { + self.spawner.spawn(task) + } + /// The playlist window duration every rendition's window is trimmed to. pub fn window(&self) -> Duration { self.feed.read().window @@ -680,20 +697,20 @@ mod tests { #[tokio::test] async fn cursors_end_only_with_a_finished_catalog() { tokio::time::pause(); - let producer = Producer::new(Duration::from_secs(30)); + let producer = Producer::spawned(Duration::from_secs(30)); let mut cursor = producer.subscribe(); assert!(cursor.poll_next(&kio::Waiter::noop()).is_pending()); producer.finish(Ok(())); assert!(cursor.next().await.unwrap().is_none()); assert!(cursor.next().await.unwrap().is_none()); - let producer = Producer::new(Duration::from_secs(30)); + let producer = Producer::spawned(Duration::from_secs(30)); let mut cursor = producer.subscribe(); producer.close(); assert!(matches!(cursor.next().await, Err(crate::Error::IncompleteRecording))); assert!(matches!(cursor.next().await, Err(crate::Error::IncompleteRecording))); - let producer = Producer::new(Duration::from_secs(30)); + let producer = Producer::spawned(Duration::from_secs(30)); let mut cursor = producer.subscribe(); drop(producer); assert!(matches!(cursor.next().await, Err(crate::Error::IncompleteRecording))); @@ -729,7 +746,7 @@ mod tests { #[tokio::test] async fn a_failed_catalog_never_completes_a_cursor() { tokio::time::pause(); - let producer = Producer::new(Duration::from_secs(30)); + let producer = Producer::spawned(Duration::from_secs(30)); let mut cursor = producer.subscribe(); producer.finish(Err(crate::Error::NoVariants)); assert!(matches!( diff --git a/rs/moq-hls/src/server/mod.rs b/rs/moq-hls/src/server/mod.rs index bc5e2430816..044554dc86a 100644 --- a/rs/moq-hls/src/server/mod.rs +++ b/rs/moq-hls/src/server/mod.rs @@ -253,9 +253,15 @@ impl Server { impl Inner { /// A broadcaster serving the instance `broadcast`. + /// + /// Its driver runs on its own task rather than the path's follower: a replaced instance + /// keeps serving after the follower that started it moves on, and the driver ends with the + /// broadcaster. fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Arc { let source = moq_mux::Source::new(self.origin.consume(), broadcast); - Broadcaster::new(source, self.config.clone()) + let (broadcaster, driver) = Broadcaster::new(source, self.config.clone()); + tokio::spawn(driver.run()); + broadcaster } } @@ -431,7 +437,7 @@ mod tests { let producer = origin.create_broadcast("gone").expect("publish allowed"); producer.announce(Default::default()).expect("publish allowed"); settle().await; - let broadcaster = Broadcaster::new(crate::resolved(&origin, "gone").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "gone").await, Config::default()); // Close the publisher so the resolved broadcast (and the broadcaster) reports closed. producer.close(); settle().await; @@ -564,7 +570,7 @@ mod tests { .publish("live", moq_net::origin::Route::default()) .expect("publish allowed"); settle().await; - let open = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let open = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); // The follower has not yet seen the takeover instance close. let followed = Followed::new(open.clone()); diff --git a/rs/moq-hls/tests/integration/server.rs b/rs/moq-hls/tests/integration/server.rs index 083358e29a1..e2bd69fb84a 100644 --- a/rs/moq-hls/tests/integration/server.rs +++ b/rs/moq-hls/tests/integration/server.rs @@ -54,7 +54,8 @@ async fn a_cursor_records_a_publisher_without_fetch() { .await .expect("request timed out") .expect("remote broadcast"); - let broadcaster = Broadcaster::new(moq_mux::Source::new(origin, remote), Default::default()); + let (broadcaster, driver) = Broadcaster::new(moq_mux::Source::new(origin, remote), Default::default()); + tokio::spawn(driver.run()); // Recorded as soon as the catalog lists it with its records, as a recorder does. let mut renditions = broadcaster.renditions(); let rendition = match moq_clock::timeout(TIMEOUT, renditions.next()).await { From c27e5e0be859563e42b3c32ba32cdf11985b5976 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 03:27:26 -0700 Subject: [PATCH 3/6] fix(hls): a dropped driver drops its queued tasks Co-Authored-By: Claude Opus 5.5 --- rs/moq-hls/src/export/driver.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/rs/moq-hls/src/export/driver.rs b/rs/moq-hls/src/export/driver.rs index d38b01d8b5b..7b219071fff 100644 --- a/rs/moq-hls/src/export/driver.rs +++ b/rs/moq-hls/src/export/driver.rs @@ -60,8 +60,10 @@ impl Driver { impl Drop for Driver { fn drop(&mut self) { - // Later starts are refused instead of queueing work nothing will run. + // Later starts are refused instead of queueing work nothing will run, and queued ones drop + // now, releasing what they hold (such as an init build's lock). self.inbox.close(); + while let Ok(Some(_)) = self.inbox.try_pop() {} } } @@ -156,4 +158,17 @@ mod tests { drop(forever); assert!(driver.poll(&waiter).is_ready(), "the task ended with its owner"); } + + // Dropping the driver drops the tasks it never polled, so nothing they hold leaks. + #[test] + fn a_dropped_driver_drops_queued_tasks() { + let (spawner, driver) = Spawner::new(); + let held = Arc::new(()); + let _owner = spawner.spawn({ + let held = held.clone(); + async move { drop(held) } + }); + drop(driver); + assert_eq!(Arc::strong_count(&held), 1, "the queued task was dropped"); + } } From 53a31d0034a3f8b4732c08f7384e7909ba48bd36 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 13:32:57 -0700 Subject: [PATCH 4/6] refactor(hls)!: Broadcaster is a cheap-clone handle Broadcaster::new returns (Broadcaster, Driver) instead of an Arc, matching the other handles: clones share a guard whose drop ends the catalog watcher. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/hls/record.rs | 4 +- rs/moq-hls/MIGRATE.md | 12 +++++ rs/moq-hls/src/export/archive_tests.rs | 2 +- rs/moq-hls/src/export/mod.rs | 64 ++++++++++++++++++------- rs/moq-hls/src/server/mod.rs | 30 ++++++------ rs/moq-hls/tests/integration/restart.rs | 2 +- 6 files changed, 78 insertions(+), 36 deletions(-) diff --git a/rs/moq-cli/src/hls/record.rs b/rs/moq-cli/src/hls/record.rs index ee3cf8ad602..9b15d1e96eb 100644 --- a/rs/moq-cli/src/hls/record.rs +++ b/rs/moq-cli/src/hls/record.rs @@ -614,7 +614,7 @@ mod tests { } } - fn broadcaster(&self) -> Arc { + fn broadcaster(&self) -> Broadcaster { let source = moq_mux::Source::new(self.origin.consume(), self.broadcast.consume()); let (broadcaster, driver) = Broadcaster::new(source, moq_hls::export::Config::default()); tokio::spawn(driver.run()); @@ -681,7 +681,7 @@ mod tests { .collect() } - fn spawn(dir: &Path, broadcaster: &Arc) -> tokio::task::JoinHandle> { + fn spawn(dir: &Path, broadcaster: &Broadcaster) -> tokio::task::JoinHandle> { let recording = Recording::create(dir.to_path_buf()).unwrap(); let broadcaster = broadcaster.clone(); tokio::spawn(async move { recording.run(&broadcaster).await }) diff --git a/rs/moq-hls/MIGRATE.md b/rs/moq-hls/MIGRATE.md index 6d96505d428..a09b81f0899 100644 --- a/rs/moq-hls/MIGRATE.md +++ b/rs/moq-hls/MIGRATE.md @@ -4,6 +4,18 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Broadcaster::new` returns a cloneable handle and its `Driver` + +`Broadcaster::new(source, config)` returns `(Broadcaster, Driver)` instead of `Arc`. +`Broadcaster` is a cheap handle: clone it instead of an `Arc`, and compare two with `is_clone` instead of `Arc::ptr_eq`. +The export no longer spawns its background work: run the driver wherever you run tasks, for example `tokio::spawn(driver.run())`, or poll it with `Driver::poll`. +Nothing is watched until it runs. + +```rust +let (broadcaster, driver) = moq_hls::export::Broadcaster::new(source, config); +tokio::spawn(driver.run()); +``` + ### Renditions list their own HLS records Each rendition reads the records its catalog `hls` field names, and segment `n` is its record `n`; a rendition without `hls` gets no playlist. diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs index d561f37c36f..8b5a58c0a33 100644 --- a/rs/moq-hls/src/export/archive_tests.rs +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -265,7 +265,7 @@ fn catalog() -> hang::Catalog { /// An archive served back through an origin by a consumer, and a broadcaster exporting it. struct Replay { - broadcaster: Arc, + broadcaster: Broadcaster, /// Keeps the served broadcast's origin open. _origin: moq_net::origin::Producer, consumer: tokio::task::JoinHandle>, diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index c5d429945ac..5989052fed0 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -94,6 +94,10 @@ impl Default for Config { } /// All renditions of one broadcast, kept in sync with its catalog. +/// +/// A cheap handle: clones share the one broadcaster, which stops watching the catalog once the +/// last clone drops. +#[derive(Clone)] pub struct Broadcaster { #[cfg_attr( not(feature = "server"), @@ -108,16 +112,31 @@ pub struct Broadcaster { instance: Arc, /// The current rendition set, reconciled from the catalog by the watcher task. renditions: renditions::Producer, - /// Ends the catalog watcher when the broadcaster is dropped; `Drop` also releases the - /// renditions themselves (see [`renditions::Producer::clear`]), each of which stops following - /// its records once no recording cursor holds it. + /// Shared by every clone; its drop with the last one ends the catalog watcher. + _guard: Arc, +} + +/// What the last [`Broadcaster`] clone releases. +struct Guard { + renditions: renditions::Producer, + /// Ends the catalog watcher when dropped. _watcher: driver::Owner, } +impl Drop for Guard { + fn drop(&mut self) { + // Release the map: a surviving `renditions::Consumer` would otherwise pin every + // `Rendition` forever. A recording cursor's own `Arc` keeps that rendition's + // records flowing, so its final segments are not truncated. Each rendition stops + // following its records once no recording cursor holds it. + self.renditions.clear(); + } +} + impl Broadcaster { /// Start tracking the renditions of `source`'s catalog broadcast, once the returned /// [`Driver`] runs. - pub fn new(source: moq_mux::Source, config: Config) -> (Arc, Driver) { + pub fn new(source: moq_mux::Source, config: Config) -> (Self, Driver) { let broadcast = source.broadcast(); // An epoch names the instance on every edge, so each serves it under the same URLs. let instance = broadcast.info().epoch.clone().unwrap_or_else(moq_net::Epoch::mint); @@ -126,23 +145,43 @@ impl Broadcaster { // The watcher owns its own producer clone; dropping the `Broadcaster` ends it so the // standing catalog subscription stops when nobody's serving from this broadcaster. let watcher = driver.lead(watch_catalog(source, renditions.clone())); - let broadcaster = Arc::new(Self { + let broadcaster = Self { broadcast, instance: Arc::from(instance.as_str()), + _guard: Arc::new(Guard { + renditions: renditions.clone(), + _watcher: watcher, + }), renditions, - _watcher: watcher, - }); + }; (broadcaster, driver) } /// [`new`](Self::new), with its driver spawned on the ambient runtime. #[cfg(test)] - pub(crate) fn spawned(source: moq_mux::Source, config: Config) -> Arc { + pub(crate) fn spawned(source: moq_mux::Source, config: Config) -> Self { let (broadcaster, driver) = Self::new(source, config); tokio::spawn(driver.run()); broadcaster } + /// Return true if this is a clone of the same broadcaster. + pub fn is_clone(&self, other: &Self) -> bool { + Arc::ptr_eq(&self._guard, &other._guard) + } + + /// How many clones of this broadcaster are alive. + #[cfg(test)] + pub(crate) fn clones(&self) -> usize { + Arc::strong_count(&self._guard) + } + + /// A weak reference that reports, through its `strong_count`, whether any clone is alive. + #[cfg(test)] + pub(crate) fn downgrade(&self) -> std::sync::Weak> { + Arc::downgrade(&self._guard) + } + /// The publisher instance every listed URL is under (`@{instance}/{kind}/{rendition}/...`). #[cfg_attr( not(feature = "server"), @@ -395,15 +434,6 @@ impl Broadcaster { } } -impl Drop for Broadcaster { - fn drop(&mut self) { - // Release the map: a surviving `renditions::Consumer` would otherwise pin every - // `Rendition` forever. A recording cursor's own `Arc` keeps that rendition's - // records flowing, so its final segments are not truncated. - self.renditions.clear(); - } -} - async fn watch_catalog(source: moq_mux::Source, renditions: renditions::Producer) { // The source's catalog broadcast, so the catalog, the record subscriptions, and the closed // check below all refer to the one broadcast it was built from. diff --git a/rs/moq-hls/src/server/mod.rs b/rs/moq-hls/src/server/mod.rs index 044554dc86a..0037a6d2927 100644 --- a/rs/moq-hls/src/server/mod.rs +++ b/rs/moq-hls/src/server/mod.rs @@ -102,14 +102,14 @@ struct Followed { struct Instances { /// The instance the master playlist and manifest list: the latest to take over. - current: Arc, + current: Broadcaster, /// Every instance a listed URL still names, the current one included. A replaced instance /// stays until its broadcast closes, so a player already on it keeps playing. - by_id: HashMap, Arc>, + by_id: HashMap, Broadcaster>, } impl Followed { - fn new(current: Arc) -> Self { + fn new(current: Broadcaster) -> Self { let by_id = HashMap::from([(current.instance().clone(), current.clone())]); Self { instances: Mutex::new(Instances { current, by_id }), @@ -117,17 +117,17 @@ impl Followed { } } - fn current(&self) -> Arc { + fn current(&self) -> Broadcaster { self.instances.lock().unwrap().current.clone() } - fn instance(&self, id: &str) -> Option> { + fn instance(&self, id: &str) -> Option { self.instances.lock().unwrap().by_id.get(id).cloned() } /// List `broadcaster` from now on. An instance returning after a gap keeps its epoch, and so /// its id, which then names the new broadcaster. - fn start(&self, broadcaster: Arc) { + fn start(&self, broadcaster: Broadcaster) { let mut instances = self.instances.lock().unwrap(); instances .by_id @@ -186,13 +186,13 @@ impl Server { /// The [`Broadcaster`] for the publisher instance serving `name` now, following each /// instance that takes over. Returns `None` if the broadcast is not announced once the /// origin has synced with the relay. - pub async fn broadcaster(&self, name: &str) -> Option> { + pub async fn broadcaster(&self, name: &str) -> Option { Some(self.followed(name).await?.current()) } /// The [`Broadcaster`] for the publisher instance `id` of `name`, until its broadcast /// closes. - pub(crate) async fn instance(&self, name: &str, id: &str) -> Option> { + pub(crate) async fn instance(&self, name: &str, id: &str) -> Option { self.followed(name).await?.instance(id) } @@ -257,7 +257,7 @@ impl Inner { /// Its driver runs on its own task rather than the path's follower: a replaced instance /// keeps serving after the follower that started it moves on, and the driver ends with the /// broadcaster. - fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Arc { + fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Broadcaster { let source = moq_mux::Source::new(self.origin.consume(), broadcast); let (broadcaster, driver) = Broadcaster::new(source, self.config.clone()); tokio::spawn(driver.run()); @@ -432,7 +432,7 @@ mod tests { assert_eq!(start.elapsed(), Duration::ZERO); } - async fn closed_broadcaster() -> Arc { + async fn closed_broadcaster() -> Broadcaster { let origin = produce_origin(); let producer = origin.create_broadcast("gone").expect("publish allowed"); producer.announce(Default::default()).expect("publish allowed"); @@ -462,7 +462,7 @@ mod tests { let fresh = server.broadcaster("live").await.expect("broadcast announced"); - assert!(!Arc::ptr_eq(&stale, &fresh)); + assert!(!stale.is_clone(&fresh)); assert!(server.inner.broadcasts.lock().unwrap().contains_key("live")); } @@ -479,7 +479,7 @@ mod tests { for _ in 0..4 { let server = Server::new(origin.consume(), Config::default()); let broadcaster = server.broadcaster("live").await.expect("broadcast announced"); - let released = Arc::downgrade(&broadcaster); + let released = broadcaster.downgrade(); drop(broadcaster); drop(server); settle().await; @@ -490,7 +490,7 @@ mod tests { let held = server.broadcaster("live").await.expect("broadcast announced"); drop(server); settle().await; - assert_eq!(Arc::strong_count(&held), 1, "only the response holds it now"); + assert_eq!(held.clones(), 1, "only the response holds it now"); assert!(!held.is_closed(), "a held broadcaster keeps serving"); producer.close(); } @@ -529,7 +529,7 @@ mod tests { .await; let kept = server.instance("live", old_epoch.as_str()).await; assert!( - kept.is_some_and(|kept| Arc::ptr_eq(&kept, &first)), + kept.is_some_and(|kept| kept.is_clone(&first)), "the old instance serves" ); @@ -586,7 +586,7 @@ mod tests { assert!(!current.is_closed()); let kept = server.instance("live", open.instance()).await; assert!( - kept.is_some_and(|kept| Arc::ptr_eq(&kept, &open)), + kept.is_some_and(|kept| kept.is_clone(&open)), "the open instance lost its URLs" ); } diff --git a/rs/moq-hls/tests/integration/restart.rs b/rs/moq-hls/tests/integration/restart.rs index 2e7112dfe04..30d6d4d44aa 100644 --- a/rs/moq-hls/tests/integration/restart.rs +++ b/rs/moq-hls/tests/integration/restart.rs @@ -71,7 +71,7 @@ async fn takeover(version: &str, case: Case) { let deadline = moq_clock::now() + TIMEOUT; let current = loop { let current = server.broadcaster("live").await.expect("announced"); - if !std::sync::Arc::ptr_eq(¤t, &replaced) { + if !current.is_clone(&replaced) { break current; } assert!(moq_clock::now() < deadline, "the new instance never took over"); From bf302afe98a975363e639aad7f0f1ab1be455998 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 13:43:51 -0700 Subject: [PATCH 5/6] fix(hls): gate the broadcaster's test helpers on the server feature Co-Authored-By: Claude Opus 5.5 --- rs/moq-hls/src/export/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 5989052fed0..17bdc4a3a30 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -171,13 +171,13 @@ impl Broadcaster { } /// How many clones of this broadcaster are alive. - #[cfg(test)] + #[cfg(all(test, feature = "server"))] pub(crate) fn clones(&self) -> usize { Arc::strong_count(&self._guard) } /// A weak reference that reports, through its `strong_count`, whether any clone is alive. - #[cfg(test)] + #[cfg(all(test, feature = "server"))] pub(crate) fn downgrade(&self) -> std::sync::Weak> { Arc::downgrade(&self._guard) } From 3f5e0532d7e986cfd5fdb221f8caf46f16016d2d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 00:14:17 -0700 Subject: [PATCH 6/6] test(hls): pause time in the driver lifecycle tests Co-Authored-By: Claude Opus 5.5 --- rs/moq-hls/src/export/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 17bdc4a3a30..7363b85a0d0 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -1691,7 +1691,7 @@ mod tests { // Background work ends with its owners, not with the publisher: once a reconfigure replaced // the rendition, dropping the broadcaster and both renditions finishes the driver while the // records stay open. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn the_driver_finishes_with_its_owners() { let origin = produce_origin(); let broadcast = origin.create_broadcast("live").expect("publish allowed"); @@ -2488,7 +2488,7 @@ mod tests { // Dropping the broadcaster must not truncate a recording in progress. A cursor holds its // rendition alive, so the rendition's own watcher still lists the final segment once the // publisher finishes. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn dropping_the_broadcaster_keeps_a_cursor_drainable() { let origin = produce_origin(); let broadcast = origin.create_broadcast("live").expect("publish allowed");