diff --git a/quest/m2/README.md b/quest/m2/README.md index 48e2b0d911a..3361f12852b 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -44,7 +44,6 @@ lite-07; bitrate enforcement is not part of this release. - [QoS](/quest/m2/qos/README.md) - the relay's egress `lag` histogram and `dropped` counters on the stats totals and prefix tracks - [Lazy retained HLS instances](/quest/m2/hls-lazy-instances.md) - a replaced HLS instance starts its watchers only when a request reaches it - [HLS and DASH browser playback](/quest/m2/hls-player-test.md) - hls.js and dash.js play, reload, switch, and survive reconfiguration nightly -- [moq-hls export on kio](/quest/m2/hls-export-kio.md) - export watchers and the init build are owned kio tasks instead of tokio::spawn - [Re-init retires timelines](/quest/m2/reinit-retire.md) - fMP4 and MKV re-inits retire the old timeline, so import state stays bounded - [Replaced producers retire their records](/quest/m2/producer-retire.md) - encoders and bindings retire a replaced producer's HLS records - [Broadcast fairness](/quest/m2/broadcast-fairness.md) - broadcasts share a cluster link fairly, so one tenant's priority never starves another's diff --git a/quest/m2/hls-export-kio.md b/quest/m2/hls-export-kio.md deleted file mode 100644 index 7b02ad30530..00000000000 --- a/quest/m2/hls-export-kio.md +++ /dev/null @@ -1,21 +0,0 @@ -# [M] moq-hls export runs on kio - -## Goal - -moq-hls export's background work (the catalog, timeline, and spans watchers in -`rs/moq-hls/src/export/{mod,rendition}.rs`, and the init build) runs as -poll-driven `kio` tasks the export owns, not on `tokio::spawn`, as -`rs/AGENTS.md` prefers. - -## Plan - -Do it crate-wide in one PR. Tasks end with their owner instead of being -detached, which also removes the abort handles kept for them today. Keep the -timeline watcher's two lifetime rules: a new reference timeline replaces the -old watcher, and a recording cursor that outlives the `Broadcaster` keeps the -watcher running until the timeline ends (see `Broadcaster`'s `Drop`). Test -both. Tests -keep their own spawns where they drive a runtime. Measure export throughput -before and after with the existing moq-hls benchmarks. - -Public API: none expected. Wire: none. diff --git a/quest/m2/spawn-audit.md b/quest/m2/spawn-audit.md index 8f6dfd48caf..ef8f07a03d5 100644 --- a/quest/m2/spawn-audit.md +++ b/quest/m2/spawn-audit.md @@ -12,7 +12,8 @@ Blocking OS threads are out of scope. Planned 2026-10-25 with the maintainer as the follow-up to no-spawn. As of `main` today the spawning library crates include moq-tokio (connection, server, client), moq-relay (web, shutdown, uring), moq-mux's container -consumer, moq-hls export, moq-rtmp, moq-rtsp, moq-srt, moq-rtc, moq-transcode, +consumer, moq-hls's server (its path follower and each +broadcaster's driver), moq-rtmp, moq-rtsp, moq-srt, moq-rtc, moq-transcode, moq-video's encode producer, moq-cli's play path, moq-bench, and qmux's transport. Recount with `rg 'tokio::spawn|spawn_local|task::spawn'` outside tests, benches, and binaries before starting. diff --git a/rs/moq-cli/src/hls.rs b/rs/moq-cli/src/hls.rs index 7dac0cfb3c8..d3ef63503af 100644 --- a/rs/moq-cli/src/hls.rs +++ b/rs/moq-cli/src/hls.rs @@ -304,7 +304,8 @@ pub async fn export(origin: moq_net::origin::Consumer, args: ExportArgs, name: S .await .with_context(|| format!("failed to resolve broadcast `{name}`"))?; let source = moq_mux::Source::new(scoped.clone(), broadcast); - let broadcaster = moq_hls::export::Broadcaster::new(source, config); + let (broadcaster, driver) = moq_hls::export::Broadcaster::new(source, config); + tokio::spawn(driver.run()); tracing::info!(%name, "recording HLS"); recording.run(&broadcaster).await }; diff --git a/rs/moq-cli/src/hls/record.rs b/rs/moq-cli/src/hls/record.rs index 6a8225eaf97..9735a8da12e 100644 --- a/rs/moq-cli/src/hls/record.rs +++ b/rs/moq-cli/src/hls/record.rs @@ -637,9 +637,11 @@ mod tests { } } - fn broadcaster(&self) -> Arc { + fn broadcaster(&self) -> Broadcaster { let source = moq_mux::Source::new(self.origin.consume(), self.broadcast.consume()); - Broadcaster::new(source, moq_hls::export::Config::default()) + let (broadcaster, driver) = Broadcaster::new(source, moq_hls::export::Config::default()); + tokio::spawn(driver.run()); + broadcaster } /// Write GOP `n`, two seconds from `2n`, to every rendition. @@ -702,7 +704,7 @@ mod tests { .collect() } - fn spawn(dir: &Path, broadcaster: &Arc) -> tokio::task::JoinHandle> { + fn spawn(dir: &Path, broadcaster: &Broadcaster) -> tokio::task::JoinHandle> { let recording = Recording::create(dir.to_path_buf()).unwrap(); let broadcaster = broadcaster.clone(); tokio::spawn(async move { recording.run(&broadcaster).await }) diff --git a/rs/moq-hls/MIGRATE.md b/rs/moq-hls/MIGRATE.md index 6d96505d428..a09b81f0899 100644 --- a/rs/moq-hls/MIGRATE.md +++ b/rs/moq-hls/MIGRATE.md @@ -4,6 +4,18 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `Broadcaster::new` returns a cloneable handle and its `Driver` + +`Broadcaster::new(source, config)` returns `(Broadcaster, Driver)` instead of `Arc`. +`Broadcaster` is a cheap handle: clone it instead of an `Arc`, and compare two with `is_clone` instead of `Arc::ptr_eq`. +The export no longer spawns its background work: run the driver wherever you run tasks, for example `tokio::spawn(driver.run())`, or poll it with `Driver::poll`. +Nothing is watched until it runs. + +```rust +let (broadcaster, driver) = moq_hls::export::Broadcaster::new(source, config); +tokio::spawn(driver.run()); +``` + ### Renditions list their own HLS records Each rendition reads the records its catalog `hls` field names, and segment `n` is its record `n`; a rendition without `hls` gets no playlist. diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs index 5146e888736..2184c22c3c5 100644 --- a/rs/moq-hls/src/export/archive_tests.rs +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -265,7 +265,7 @@ fn catalog() -> hang::Catalog { /// An archive served back through an origin by a consumer, and a broadcaster exporting it. struct Replay { - broadcaster: Arc, + broadcaster: Broadcaster, /// Keeps the served broadcast's origin open. _origin: moq_net::origin::Producer, consumer: tokio::task::JoinHandle>, @@ -284,7 +284,7 @@ impl Replay { let consumer = tokio::spawn(consumer.run()); broadcast.announce(Default::default()).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "rec").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "rec").await, Config::default()); kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()) .await .expect("a rendition becomes playable or fails"); diff --git a/rs/moq-hls/src/export/driver.rs b/rs/moq-hls/src/export/driver.rs new file mode 100644 index 00000000000..7b219071fff --- /dev/null +++ b/rs/moq-hls/src/export/driver.rs @@ -0,0 +1,174 @@ +//! The export's background work, as `kio` tasks one [`Driver`] polls. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Poll; + +/// Runs a [`Broadcaster`](super::Broadcaster)'s background work: its catalog watcher, each +/// rendition's record watcher, and the init builds. +/// +/// Poll it, or [`run`](Self::run) it, for as long as the broadcaster serves. It finishes once +/// the broadcaster and every rendition it created have dropped and their tasks ended, so a +/// recording cursor that outlives the broadcaster keeps its rendition's records flowing. +/// Dropping it early stops all of that work. +#[must_use = "the broadcaster does nothing until its driver runs"] +pub struct Driver { + /// The catalog watcher, polled before the other tasks in every pass: a reconfigure must + /// retire its rendition before that rendition's watcher reads records published after it. + lead: Option, + inbox: kio::Queue, + tasks: kio::Tasks, + /// The inbox closed: every owner that could start a task has dropped. + closed: bool, +} + +impl Driver { + /// Advance every task, ready once nothing is left to run. + pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + if let Some(lead) = &mut self.lead + && kio::Task::poll(lead, waiter).is_ready() + { + self.lead = None; + } + while !self.closed { + match self.inbox.poll_pop(waiter) { + Poll::Ready(Ok(job)) => self.tasks.push(job), + Poll::Ready(Err(kio::Closed)) => self.closed = true, + Poll::Pending => break, + } + } + match self.tasks.poll(waiter) { + Poll::Ready(()) if self.closed && self.lead.is_none() => Poll::Ready(()), + _ => Poll::Pending, + } + } + + /// Run `task` ahead of every other task (see [`lead`](Self::lead)), until it finishes or the + /// returned [`Owner`] drops. + pub(crate) fn lead(&mut self, task: impl Future + Send + 'static) -> Owner { + let (job, owner) = Job::new(task); + self.lead = Some(job); + owner + } + + /// [`poll`](Self::poll) until it is ready. + pub async fn run(mut self) { + kio::wait(|waiter| self.poll(waiter)).await + } +} + +impl Drop for Driver { + fn drop(&mut self) { + // Later starts are refused instead of queueing work nothing will run, and queued ones drop + // now, releasing what they hold (such as an init build's lock). + self.inbox.close(); + while let Ok(Some(_)) = self.inbox.try_pop() {} + } +} + +/// Starts tasks on its [`Driver`]. Clones share it; the driver's inbox closes with the last. +#[derive(Clone)] +pub(crate) struct Spawner { + inbox: Arc, +} + +struct Inbox(kio::Queue); + +impl Drop for Inbox { + fn drop(&mut self) { + self.0.close(); + } +} + +impl Spawner { + /// A spawner and the driver that runs what it starts. + pub fn new() -> (Self, Driver) { + let inbox = kio::Queue::new(); + let driver = Driver { + lead: None, + inbox: inbox.clone(), + tasks: kio::Tasks::new(), + closed: false, + }; + ( + Self { + inbox: Arc::new(Inbox(inbox)), + }, + driver, + ) + } + + /// Run `task` on the driver until it finishes or the returned [`Owner`] drops. + pub fn spawn(&self, task: impl Future + Send + 'static) -> Owner { + let (job, owner) = Job::new(task); + // Refused once the driver is gone, which drops the task unrun. + let _ = self.inbox.0.try_push(job); + owner + } +} + +/// Ends its task when dropped. +pub(crate) struct Owner(#[expect(dead_code, reason = "held only to close on drop")] kio::Producer<()>); + +/// A task and the [`Owner`] that ends it. +struct Job { + owner: kio::Consumer<()>, + task: Pin + Send>>, +} + +impl Job { + fn new(task: impl Future + Send + 'static) -> (Self, Owner) { + let owner = kio::Producer::new(()); + let job = Self { + owner: owner.consume(), + task: Box::pin(task), + }; + (job, Owner(owner)) + } +} + +impl kio::Task for Job { + type Output = (); + + fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { + if self.owner.poll_closed(waiter).is_ready() { + return Poll::Ready(()); + } + waiter.poll_future(self.task.as_mut()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // The driver finishes once every spawner and task is gone, and an owner's drop ends its task + // without waiting for the task to finish on its own. + #[test] + fn tasks_end_with_their_owner() { + let (spawner, mut driver) = Spawner::new(); + let waiter = kio::Waiter::noop(); + let forever = spawner.spawn(std::future::pending()); + let lead = driver.lead(std::future::pending()); + drop(spawner); + assert!(driver.poll(&waiter).is_pending(), "the owned tasks still run"); + drop(lead); + assert!(driver.poll(&waiter).is_pending(), "the owned task still runs"); + drop(forever); + assert!(driver.poll(&waiter).is_ready(), "the task ended with its owner"); + } + + // Dropping the driver drops the tasks it never polled, so nothing they hold leaks. + #[test] + fn a_dropped_driver_drops_queued_tasks() { + let (spawner, driver) = Spawner::new(); + let held = Arc::new(()); + let _owner = spawner.spawn({ + let held = held.clone(); + async move { drop(held) } + }); + drop(driver); + assert_eq!(Arc::strong_count(&held), 1, "the queued task was dropped"); + } +} diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 7b7f4ceafa0..45d611a0328 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -15,6 +15,10 @@ //! group on the first playlist render (in the background on a manifest render) to build its //! init, then caches it. //! +//! That background work (the catalog and record watchers, and init builds) runs on the +//! [`Driver`] [`Broadcaster::new`] returns: poll it or spawn [`Driver::run`] wherever the +//! embedder runs tasks. +//! //! The same machinery serves two kinds of consumer: //! //! * the HTTP serve path (pull): [`Route::parse`] names the resource a request path addresses, @@ -28,6 +32,7 @@ #[cfg(test)] mod archive_tests; +mod driver; pub mod master; mod mpd; mod playlist; @@ -37,12 +42,13 @@ mod route; pub mod renditions; pub mod segments; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::Duration; use moq_mux::catalog::{self, CatalogFormat, Stream}; use rand::RngExt; +pub use driver::Driver; pub(crate) use playlist::render_media; pub use rendition::{Kind, Rendition}; pub use route::{Resource, Route}; @@ -87,6 +93,10 @@ impl Default for Config { } /// All renditions of one broadcast, kept in sync with its catalog. +/// +/// A cheap handle: clones share the one broadcaster, which stops watching the catalog once the +/// last clone drops. +#[derive(Clone)] pub struct Broadcaster { #[cfg_attr( not(feature = "server"), @@ -101,32 +111,76 @@ pub struct Broadcaster { instance: Arc, /// The current rendition set, reconciled from the catalog by the watcher task. renditions: renditions::Producer, - /// Aborts the catalog watcher when the broadcaster is dropped; `Drop` then releases the - /// renditions themselves (see [`renditions::Producer::clear`]), each of which stops following - /// its records once no recording cursor holds it. Set once, right after construction. - watcher: Mutex>>, + /// Shared by every clone; its drop with the last one ends the catalog watcher. + _guard: Arc, +} + +/// What the last [`Broadcaster`] clone releases. +struct Guard { + renditions: renditions::Producer, + /// Ends the catalog watcher when dropped. + _watcher: driver::Owner, +} + +impl Drop for Guard { + fn drop(&mut self) { + // Release the map: a surviving `renditions::Consumer` would otherwise pin every + // `Rendition` forever. A recording cursor's own `Arc` keeps that rendition's + // records flowing, so its final segments are not truncated. Each rendition stops + // following its records once no recording cursor holds it. + self.renditions.clear(); + } } impl Broadcaster { - /// Start tracking the renditions of `source`'s catalog broadcast. - pub fn new(source: moq_mux::Source, config: Config) -> Arc { + /// Start tracking the renditions of `source`'s catalog broadcast, once the returned + /// [`Driver`] runs. + pub fn new(source: moq_mux::Source, config: Config) -> (Self, Driver) { let broadcast = source.broadcast(); // An epoch names the instance on every edge, so each serves it under the same URLs. let instance = broadcast.info().epoch.clone().unwrap_or_else(moq_net::Epoch::mint); - let renditions = renditions::Producer::new(config.window); - let broadcaster = Arc::new(Self { + let (spawner, mut driver) = driver::Spawner::new(); + let renditions = renditions::Producer::new(config.window, spawner); + // The watcher owns its own producer clone; dropping the `Broadcaster` ends it so the + // standing catalog subscription stops when nobody's serving from this broadcaster. + let watcher = driver.lead(watch_catalog(source, renditions.clone())); + let broadcaster = Self { broadcast, instance: Arc::from(instance.as_str()), - renditions: renditions.clone(), - watcher: Mutex::new(None), - }); - // The watcher owns its own producer clone; the `Broadcaster`'s `Drop` aborts it so the - // standing catalog subscription stops when nobody's serving from this broadcaster. - let watcher = tokio::spawn(watch_catalog(source, renditions)); - *broadcaster.watcher.lock().unwrap() = Some(watcher); + _guard: Arc::new(Guard { + renditions: renditions.clone(), + _watcher: watcher, + }), + renditions, + }; + (broadcaster, driver) + } + + /// [`new`](Self::new), with its driver spawned on the ambient runtime. + #[cfg(test)] + pub(crate) fn spawned(source: moq_mux::Source, config: Config) -> Self { + let (broadcaster, driver) = Self::new(source, config); + tokio::spawn(driver.run()); broadcaster } + /// Return true if this is a clone of the same broadcaster. + pub fn is_clone(&self, other: &Self) -> bool { + Arc::ptr_eq(&self._guard, &other._guard) + } + + /// How many clones of this broadcaster are alive. + #[cfg(all(test, feature = "server"))] + pub(crate) fn clones(&self) -> usize { + Arc::strong_count(&self._guard) + } + + /// A weak reference that reports, through its `strong_count`, whether any clone is alive. + #[cfg(all(test, feature = "server"))] + pub(crate) fn downgrade(&self) -> std::sync::Weak> { + Arc::downgrade(&self._guard) + } + /// The publisher instance every listed URL is under (`@{instance}/{kind}/{rendition}/...`). #[cfg_attr( not(feature = "server"), @@ -379,18 +433,6 @@ impl Broadcaster { } } -impl Drop for Broadcaster { - fn drop(&mut self) { - if let Some(watcher) = self.watcher.lock().unwrap().take() { - watcher.abort(); - } - // Release the map: a surviving `renditions::Consumer` would otherwise pin every - // `Rendition` forever. A recording cursor's own `Arc` keeps that rendition's - // records flowing, so its final segments are not truncated. - self.renditions.clear(); - } -} - async fn watch_catalog(source: moq_mux::Source, renditions: renditions::Producer) { // The source's catalog broadcast, so the catalog, the record subscriptions, and the closed // check below all refer to the one broadcast it was built from. @@ -637,7 +679,7 @@ mod tests { let mut catalog = moq_mux::catalog::hang::Catalog::default(); catalog.video.renditions.insert("video".to_string(), config); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(&upstream, &catalog); assert!(renditions.get(Kind::Video, "video").is_none()); } @@ -661,7 +703,7 @@ mod tests { config.hls = Some(hls(name)); catalog.audio.renditions.insert(name.to_string(), config); } - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(&source, &catalog); let listed = || { renditions @@ -691,7 +733,7 @@ mod tests { settle().await; let upstream = crate::resolved(&origin, "live").await; - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); let mut cursor = renditions.subscribe(); let mut catalog = catalog_with_both(); catalog.audio.renditions.get_mut("audio0").unwrap().hls = None; @@ -728,7 +770,7 @@ mod tests { let upstream = empty_upstream(&origin, "live").await; let mut catalog = catalog_with_both(); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); let mut cursor = renditions.subscribe(); renditions.sync(&upstream, &catalog); @@ -798,7 +840,7 @@ mod tests { let mut catalog = catalog_with_both(); catalog.video.renditions.get_mut("video0").unwrap().framerate = None; - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); let mut cursor = renditions.subscribe(); renditions.sync(&upstream, &catalog); let video = renditions.get(Kind::Video, "video0").expect("video rendition"); @@ -827,7 +869,7 @@ mod tests { let mut catalog = catalog_with_both(); catalog.audio.renditions.get_mut("audio0").unwrap().bitrate = None; - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(&upstream, &catalog); let audio = renditions.get(Kind::Audio, "audio0").expect("audio rendition"); @@ -859,7 +901,7 @@ mod tests { let upstream = empty_upstream(&origin, "live").await; let mut catalog = catalog_with_both(); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); let mut cursor = renditions.subscribe(); renditions.sync(&upstream, &catalog); @@ -901,7 +943,7 @@ mod tests { let upstream = empty_upstream(&origin, "live").await; let mut catalog = catalog_with_both(); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); for width in 1..=8 { catalog.video.renditions.get_mut("video0").unwrap().coded_width = Some(width); renditions.sync(&upstream, &catalog); @@ -922,7 +964,7 @@ mod tests { let catalog = hls_catalog(&mut broadcast.broadcast().clone()); let mut media = video(&catalog, &broadcast, "video0", video_config()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let rendition = kio::clock::timeout(Duration::from_secs(5), async { loop { if let Some(rendition) = broadcaster.rendition(Kind::Video, "video0") { @@ -1005,7 +1047,7 @@ mod tests { records.push(&gop(sequence)).unwrap(); } } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); kio::clock::timeout(Duration::from_secs(5), broadcaster.advertised()) .await .expect("video0 is advertised"); @@ -1038,7 +1080,7 @@ mod tests { media.write(vp8_frame(i * 2_000_000, true)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); kio::clock::timeout(Duration::from_secs(5), broadcaster.advertised()) .await .expect("the rendition is advertised"); @@ -1113,7 +1155,7 @@ mod tests { let (mut catalog, _media) = publish_vp8(&mut broadcast.broadcast().clone(), inline_vp8()); broadcast.announce(moq_net::origin::Route::default()).unwrap(); settle().await; - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.ready().await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let mut cursor = rendition.segments(); @@ -1146,7 +1188,7 @@ mod tests { records.push(&stop(1, 2_000)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.ready().await; let rendition = broadcaster.rendition(Kind::Video, "video1").expect("rendition"); until_listed(&rendition, 1).await; @@ -1290,7 +1332,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; listed_in_manifest(&broadcaster).await; @@ -1307,7 +1349,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; kio::clock::timeout(Duration::from_millis(1), broadcaster.video_init()) @@ -1362,7 +1404,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; let broken = broadcaster.rendition(Kind::Video, "video0").unwrap(); until_listed(&broken, 3).await; @@ -1395,7 +1437,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; let stuck = broadcaster.rendition(Kind::Video, "video1").unwrap(); @@ -1420,7 +1462,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; let render = broadcaster.clone(); @@ -1442,7 +1484,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; broadcaster.set_generation(Some("run1")).unwrap(); until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; @@ -1467,7 +1509,7 @@ mod tests { let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; let rendition = broadcaster.rendition(Kind::Video, "video0").unwrap(); until_listed(&rendition, 3).await; @@ -1508,7 +1550,7 @@ mod tests { let mut open = media.create_group(1u64.into()).unwrap(); write_keyframe(&mut open, 1); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 2).await; catalog.finish().unwrap(); @@ -1534,7 +1576,7 @@ mod tests { let (origin, broadcast, _catalog, mut records) = inline_pair("video0").await; let media = broadcast.create_track("video0", timed()).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); while broadcaster.rendition(Kind::Video, "video0").is_none() { kio::clock::sleep(Duration::from_millis(10)).await; } @@ -1571,7 +1613,7 @@ mod tests { group.finish().unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; until_listed(&broadcaster.rendition(Kind::Video, "video0").unwrap(), 3).await; for _ in 0..3 { @@ -1601,7 +1643,7 @@ mod tests { settle().await; let (_catalog, mut media) = publish_vp8(&mut broadcast.broadcast().clone(), video_config()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let old = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let old_playlist = old @@ -1640,6 +1682,49 @@ mod tests { assert!(new.init(&old_hash).await.unwrap().is_none()); } + // Background work ends with its owners, not with the publisher: once a reconfigure replaced + // the rendition, dropping the broadcaster and both renditions finishes the driver while the + // records stay open. + #[tokio::test(start_paused = true)] + async fn the_driver_finishes_with_its_owners() { + let origin = produce_origin(); + let broadcast = origin.create_broadcast("live").expect("publish allowed"); + broadcast.announce(Default::default()).expect("publish allowed"); + settle().await; + let (_catalog, mut media) = publish_vp8(&mut broadcast.broadcast().clone(), video_config()); + + let (broadcaster, driver) = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let driving = tokio::spawn(driver.run()); + let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; + let old = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); + + let mut config = video_config(); + config.coded_width = Some(640); + media.set(config).unwrap(); + media.write(vp8_frame(6_000_000, true)).unwrap(); + let new = kio::clock::timeout(Duration::from_secs(5), async { + loop { + let current = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); + if !Arc::ptr_eq(¤t, &old) { + break current; + } + kio::clock::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the reconfigure rebuilt the rendition"); + + drop((broadcaster, new)); + settle().await; + assert!(!driving.is_finished(), "a held rendition keeps the driver running"); + drop(old); + kio::clock::timeout(Duration::from_secs(5), driving) + .await + .expect("the driver finishes once its owners drop") + .unwrap(); + drop(media); + } + // A generation rides every segment URL. The first one labels the run already listed; // replacing it drops that run's rows, and the next run lists under the new generation. #[tokio::test] @@ -1650,7 +1735,7 @@ mod tests { settle().await; let (_catalog, mut media) = publish_vp8(&mut broadcast.broadcast().clone(), video_config()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let tag = rendition.tag().to_string(); @@ -1712,7 +1797,7 @@ mod tests { media.write(vp8_frame(micros, true)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let _ = kio::clock::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -1747,7 +1832,7 @@ mod tests { // Three GOPs, 2s apart: segments 0 and 1 are complete, segment 2 is the live edge. let (catalog, media) = publish_vp8(&mut broadcast.broadcast().clone(), inline_vp8()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster .rendition(Kind::Video, "video0") @@ -1809,7 +1894,7 @@ mod tests { settle().await; let (catalog, media) = publish_vp8(&mut broadcast.broadcast().clone(), video_config()); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let _ = kio::clock::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -1847,7 +1932,7 @@ mod tests { media.write(vp8_frame(2_000_000, true)).unwrap(); media.write(vp8_frame(4_000_000, true)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("rendition"); let _ = kio::clock::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -1886,7 +1971,7 @@ mod tests { video.write(frame(4_000_000, true)).unwrap(); // live edge audio_write(4_000_000); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let video_rendition = broadcaster.rendition(Kind::Video, "video0").expect("video discovered"); let audio_rendition = broadcaster.rendition(Kind::Audio, "audio0").expect("audio discovered"); @@ -1967,7 +2052,7 @@ mod tests { media.write(frame(2_000_000, true)).unwrap(); media.write(frame(4_000_000, true)).unwrap(); // live edge - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("video discovered"); let _ = kio::clock::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -2028,7 +2113,7 @@ mod tests { records.push(&gop(sequence)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Video, "video0").expect("video discovered"); let _ = kio::clock::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -2073,7 +2158,7 @@ mod tests { // Watch before the first fragment, as a live viewer would, so a catalog the moov // published on a provisional clock would stick. - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); assert!( next_event(&mut broadcaster.renditions()).await.is_none(), "the moov alone lists no rendition" @@ -2199,7 +2284,7 @@ mod tests { .await .unwrap(); let source = crate::resolved(consuming, "live").await; - let broadcaster = (!late).then(|| Broadcaster::new(source.clone(), Config::default())); + let broadcaster = (!late).then(|| Broadcaster::spawned(source.clone(), Config::default())); for i in 0..20u64 { let fragment = video_muxer .fragment(i as u32, &[vp8_frame(i * 1_000_000, i % 2 == 0)], None) @@ -2219,7 +2304,7 @@ mod tests { import.decode(&fragment).unwrap(); settle().await; } - let broadcaster = broadcaster.unwrap_or_else(|| Broadcaster::new(source, Config::default())); + let broadcaster = broadcaster.unwrap_or_else(|| Broadcaster::spawned(source, Config::default())); kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()) .await .unwrap(); @@ -2268,7 +2353,7 @@ mod tests { media.write(frame(0, true)).unwrap(); media.write(frame(2_000_000, true)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.playable()).await; @@ -2316,7 +2401,7 @@ mod tests { audio.cut(None).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster.rendition(Kind::Audio, "audio0").expect("audio discovered"); let _ = kio::clock::timeout(Duration::from_secs(5), rendition.playable()).await; @@ -2361,7 +2446,7 @@ mod tests { video.write(frame(4_000_000, true)).unwrap(); // segment 2 (live edge) audio_write(4_000_000); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let video_rendition = broadcaster.rendition(Kind::Video, "video0").expect("video discovered"); let audio_rendition = broadcaster.rendition(Kind::Audio, "audio0").expect("audio discovered"); @@ -2397,7 +2482,7 @@ mod tests { // Dropping the broadcaster must not truncate a recording in progress. A cursor holds its // rendition alive, so the rendition's own watcher still lists the final segment once the // publisher finishes. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn dropping_the_broadcaster_keeps_a_cursor_drainable() { let origin = produce_origin(); let broadcast = origin.create_broadcast("live").expect("publish allowed"); @@ -2410,7 +2495,8 @@ mod tests { media.write(frame(2_000_000, true)).unwrap(); media.write(frame(3_000_000, false)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let (broadcaster, driver) = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let driving = tokio::spawn(driver.run()); let _ = kio::clock::timeout(Duration::from_secs(5), broadcaster.ready()).await; let rendition = broadcaster .rendition(Kind::Video, "video0") @@ -2438,7 +2524,14 @@ mod tests { "the final segment survives dropping the broadcaster" ); - drop((catalog, media, broadcast, rendition)); + // The cursor holds its rendition, and the rendition its share of the driver. + assert!(!driving.is_finished(), "the driver outlives a held rendition"); + drop((segments, rendition)); + kio::clock::timeout(Duration::from_secs(5), driving) + .await + .expect("the driver finishes once the cursor drops") + .unwrap(); + drop((catalog, media, broadcast)); } /// A one-video-rendition catalog snapshot reconciled by hand, which is what `watch_catalog` @@ -2451,7 +2544,7 @@ mod tests { config.hls = Some(hls("video0")); let mut catalog = moq_mux::catalog::hang::Catalog::default(); catalog.video.renditions.insert("video0".to_string(), config); - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(upstream, &catalog); let rendition = renditions.get(Kind::Video, "video0").expect("rendition synced"); (renditions, rendition) @@ -3115,7 +3208,7 @@ mod tests { media.write(frame(0, true)).unwrap(); media.write(frame(2_000_000, true)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let mut renditions = broadcaster.renditions(); let added = kio::clock::timeout(Duration::from_secs(5), renditions.next()) @@ -3161,7 +3254,7 @@ mod tests { media.write(frame(2_000_000, true)).unwrap(); media.write(frame(4_000_000, true)).unwrap(); - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); let mut renditions = broadcaster.renditions(); let rendition = match kio::clock::timeout(Duration::from_secs(5), renditions.next()) @@ -3201,8 +3294,12 @@ mod tests { let end = kio::clock::timeout(Duration::from_secs(5), segments.next()) .await .expect("segment cursor resolves after the source is lost"); + // The record and catalog watchers race to see the close; either fails the recording. assert!( - matches!(end, Err(crate::Error::RecordsFailed { .. })), + matches!( + end, + Err(crate::Error::RecordsFailed { .. } | crate::Error::CatalogFailed { .. }) + ), "an abrupt end fails the recording: {:?}", end.map(|segment| segment.map(|segment| segment.segment)) ); @@ -3264,7 +3361,7 @@ mod tests { broadcast: &moq_net::broadcast::Producer, catalog: &moq_mux::catalog::hang::Catalog, ) -> renditions::Producer { - let renditions = renditions::Producer::new(Config::default().window); + let renditions = renditions::Producer::spawned(Config::default().window); renditions.sync(&Self::source(broadcast), catalog); renditions } @@ -3514,7 +3611,7 @@ mod tests { video0.push(&gop(sequence)).unwrap(); video1.push(&gop(sequence)).unwrap(); } - let broadcaster = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); broadcaster.advertised().await; let healthy = broadcaster.rendition(Kind::Video, "video0").unwrap(); let broken = broadcaster.rendition(Kind::Video, "video1").unwrap(); diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index f87107e9c80..349405073cb 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -11,6 +11,7 @@ use hang::catalog::{AudioConfig, Clock, Hls, Reference, VideoConfig}; use moq_mux::container::fmp4::Muxer; use sha2::{Digest, Sha256}; +use super::driver::Owner; use super::playlist::{Segment, Snapshot}; use super::{mpd, renditions, segments}; use crate::Result; @@ -341,8 +342,8 @@ pub struct Rendition { fanout: renditions::Fanout, /// The broadcast carrying the catalog and every record track. catalog: moq_net::broadcast::Consumer, - /// Follows the records into [`live`](Self::live); aborted when the rendition drops. - watcher: Mutex>>, + /// Follows the records into [`live`](Self::live); ends when the rendition drops. + watcher: Mutex>, /// Set once the catalog drops or replaces the rendition (see [`retire`](Self::retire)). retiring: tokio::sync::watch::Sender, /// Which records this instance lists. @@ -359,11 +360,8 @@ pub struct Rendition { /// Serializes init builds, so concurrent first requests fetch the keyframe group once. Shared /// so a background build (see [`start_init`](Self::start_init)) holds it for its whole run. building: Arc>, - /// The background init build; aborted when the rendition drops. - build: Mutex>>, - /// The runtime the rendition was built on, which runs the background init build even when a - /// synchronous render calls from another thread. - runtime: tokio::runtime::Handle, + /// The background init build; ends when the rendition drops. + build: Mutex>, } impl Rendition { @@ -465,7 +463,6 @@ impl Rendition { run: kio::Producer::default(), building: Arc::new(tokio::sync::Mutex::new(())), build: Mutex::new(None), - runtime: tokio::runtime::Handle::current(), }) } @@ -505,13 +502,12 @@ impl Rendition { run: kio::Producer::default(), building: Arc::new(tokio::sync::Mutex::new(())), build: Mutex::new(None), - runtime: tokio::runtime::Handle::current(), }) } /// Start following this rendition's own HLS records. pub(crate) fn watch(self: &Arc) { - let watcher = tokio::spawn(watch_records( + let watcher = self.fanout.spawn(watch_records( self.catalog.clone(), self.hls.clone(), Arc::downgrade(self), @@ -627,9 +623,7 @@ impl Rendition { pub(crate) fn restart(&self, generation: Option>) { // A build parked on the old run's open group would hold `building`, so the new run could // never start its own. - if let Some(build) = self.build.lock().expect("build lock poisoned").take() { - build.abort(); - } + self.build.lock().expect("build lock poisoned").take(); self.live.clear(); let mut run = self.run_mut(); *run = Run { @@ -1023,7 +1017,7 @@ impl Rendition { /// later [`representation`](Self::representation) lists this rendition without any render /// waiting on its media. /// - /// The build never holds the rendition itself, and is aborted when the rendition drops, so + /// The build never holds the rendition itself, and ends when the rendition drops, so /// one stuck on an open group cannot keep a torn-down export's state alive. pub(crate) fn start_init(self: &Arc) { let Ok(binding) = self.authorize() else { @@ -1048,7 +1042,7 @@ impl Rendition { let media = self.media.clone(); let rendition = Arc::downgrade(self); let name = self.name.clone(); - let build = self.runtime.spawn(async move { + let build = self.fanout.spawn(async move { let _building = building; match read.read(&media, &binding).await { Ok(Some(bytes)) => { @@ -1413,17 +1407,6 @@ impl InitRead { } } -impl Drop for Rendition { - fn drop(&mut self) { - if let Some(watcher) = self.watcher.lock().expect("watcher lock poisoned").take() { - watcher.abort(); - } - if let Some(build) = self.build.lock().expect("build lock poisoned").take() { - build.abort(); - } - } -} - /// Fetch one group, mapping "no longer (or not yet) servable" to the error that says so /// rather than a bare `None`: a caller abandoning a segment can only explain itself with it. async fn fetch( @@ -1542,7 +1525,7 @@ mod tests { let source = moq_mux::Source::new(origin.consume(), broadcast.consume()); let mut config = VideoConfig::new(hang::catalog::VideoCodec::VP8); config.broadcast = Some(moq_net::path::Relative::new("./secret").unwrap()); - let renditions = renditions::Producer::new(Duration::from_secs(30)); + let renditions = renditions::Producer::spawned(Duration::from_secs(30)); let rendition = Arc::new(Rendition::video("video".into(), &config, &source, hls(), None, renditions.fanout()).unwrap()); let request = dynamic.requested_broadcast().await.unwrap(); @@ -1714,7 +1697,7 @@ mod tests { let broadcast = origin.create_broadcast("live").unwrap(); let source = moq_mux::Source::new(origin.consume(), broadcast.consume()); let config = VideoConfig::new(hang::catalog::VideoCodec::VP8); - let renditions = renditions::Producer::new(Duration::from_secs(30)); + let renditions = renditions::Producer::spawned(Duration::from_secs(30)); let rendition = Arc::new(Rendition::video("video".into(), &config, &source, hls(), None, renditions.fanout()).unwrap()); // On its own task, so only a registered wake polls it again. diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index ac7b094dae1..89f2bec547b 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -17,6 +17,7 @@ use std::time::Duration; use moq_mux::catalog::hang::Catalog; +use super::driver::{Owner, Spawner}; use super::master; use super::rendition::{Kind, Rendition}; @@ -131,10 +132,12 @@ struct State { #[derive(Clone)] pub(crate) struct Fanout { feed: kio::Producer, + /// Starts each rendition's background work on the broadcaster's driver. + spawner: Spawner, } impl Producer { - pub fn new(window: Duration) -> Self { + pub fn new(window: Duration, spawner: Spawner) -> Self { Self { state: kio::Producer::new(State::default()), fanout: Fanout { @@ -147,10 +150,19 @@ impl Producer { generation: None, window, }), + spawner, }, } } + /// A producer whose renditions run on a driver spawned on the ambient runtime. + #[cfg(test)] + pub(super) fn spawned(window: Duration) -> Self { + let (spawner, driver) = Spawner::new(); + tokio::spawn(driver.run()); + Self::new(window, spawner) + } + /// How many renditions the fanout still reaches, dropped ones not yet pruned included. #[cfg(test)] pub(super) fn targets(&self) -> usize { @@ -259,6 +271,11 @@ impl Producer { } impl Fanout { + /// Run `task` on the broadcaster's driver until it finishes or the returned [`Owner`] drops. + pub fn spawn(&self, task: impl Future + Send + 'static) -> Owner { + self.spawner.spawn(task) + } + /// The playlist window duration every rendition's window is trimmed to. pub fn window(&self) -> Duration { self.feed.read().window @@ -680,20 +697,20 @@ mod tests { #[tokio::test] async fn cursors_end_only_with_a_finished_catalog() { tokio::time::pause(); - let producer = Producer::new(Duration::from_secs(30)); + let producer = Producer::spawned(Duration::from_secs(30)); let mut cursor = producer.subscribe(); assert!(cursor.poll_next(&kio::Waiter::noop()).is_pending()); producer.finish(Ok(())); assert!(cursor.next().await.unwrap().is_none()); assert!(cursor.next().await.unwrap().is_none()); - let producer = Producer::new(Duration::from_secs(30)); + let producer = Producer::spawned(Duration::from_secs(30)); let mut cursor = producer.subscribe(); producer.close(); assert!(matches!(cursor.next().await, Err(crate::Error::IncompleteRecording))); assert!(matches!(cursor.next().await, Err(crate::Error::IncompleteRecording))); - let producer = Producer::new(Duration::from_secs(30)); + let producer = Producer::spawned(Duration::from_secs(30)); let mut cursor = producer.subscribe(); drop(producer); assert!(matches!(cursor.next().await, Err(crate::Error::IncompleteRecording))); @@ -729,7 +746,7 @@ mod tests { #[tokio::test] async fn a_failed_catalog_never_completes_a_cursor() { tokio::time::pause(); - let producer = Producer::new(Duration::from_secs(30)); + let producer = Producer::spawned(Duration::from_secs(30)); let mut cursor = producer.subscribe(); producer.finish(Err(crate::Error::NoVariants)); assert!(matches!( diff --git a/rs/moq-hls/src/server/mod.rs b/rs/moq-hls/src/server/mod.rs index a648959c15f..28b9b2ce5ef 100644 --- a/rs/moq-hls/src/server/mod.rs +++ b/rs/moq-hls/src/server/mod.rs @@ -102,14 +102,14 @@ struct Followed { struct Instances { /// The instance the master playlist and manifest list: the latest to take over. - current: Arc, + current: Broadcaster, /// Every instance a listed URL still names, the current one included. A replaced instance /// stays until its broadcast closes, so a player already on it keeps playing. - by_id: HashMap, Arc>, + by_id: HashMap, Broadcaster>, } impl Followed { - fn new(current: Arc) -> Self { + fn new(current: Broadcaster) -> Self { let by_id = HashMap::from([(current.instance().clone(), current.clone())]); Self { instances: Mutex::new(Instances { current, by_id }), @@ -117,17 +117,17 @@ impl Followed { } } - fn current(&self) -> Arc { + fn current(&self) -> Broadcaster { self.instances.lock().unwrap().current.clone() } - fn instance(&self, id: &str) -> Option> { + fn instance(&self, id: &str) -> Option { self.instances.lock().unwrap().by_id.get(id).cloned() } /// List `broadcaster` from now on. An instance returning after a gap keeps its epoch, and so /// its id, which then names the new broadcaster. - fn start(&self, broadcaster: Arc) { + fn start(&self, broadcaster: Broadcaster) { let mut instances = self.instances.lock().unwrap(); instances .by_id @@ -186,13 +186,13 @@ impl Server { /// The [`Broadcaster`] for the publisher instance serving `name` now, following each /// instance that takes over. Returns `None` if the broadcast is not announced once the /// origin has synced with the relay. - pub async fn broadcaster(&self, name: &str) -> Option> { + pub async fn broadcaster(&self, name: &str) -> Option { Some(self.followed(name).await?.current()) } /// The [`Broadcaster`] for the publisher instance `id` of `name`, until its broadcast /// closes. - pub(crate) async fn instance(&self, name: &str, id: &str) -> Option> { + pub(crate) async fn instance(&self, name: &str, id: &str) -> Option { self.followed(name).await?.instance(id) } @@ -253,9 +253,15 @@ impl Server { impl Inner { /// A broadcaster serving the instance `broadcast`. - fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Arc { + /// + /// Its driver runs on its own task rather than the path's follower: a replaced instance + /// keeps serving after the follower that started it moves on, and the driver ends with the + /// broadcaster. + fn serve(&self, broadcast: moq_net::broadcast::Consumer) -> Broadcaster { let source = moq_mux::Source::new(self.origin.consume(), broadcast); - Broadcaster::new(source, self.config.clone()) + let (broadcaster, driver) = Broadcaster::new(source, self.config.clone()); + tokio::spawn(driver.run()); + broadcaster } } @@ -426,12 +432,12 @@ mod tests { assert_eq!(start.elapsed(), Duration::ZERO); } - async fn closed_broadcaster() -> Arc { + async fn closed_broadcaster() -> Broadcaster { let origin = produce_origin(); let producer = origin.create_broadcast("gone").expect("publish allowed"); producer.announce(Default::default()).expect("publish allowed"); settle().await; - let broadcaster = Broadcaster::new(crate::resolved(&origin, "gone").await, Config::default()); + let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "gone").await, Config::default()); // Close the publisher so the resolved broadcast (and the broadcaster) reports closed. producer.close(); settle().await; @@ -456,7 +462,7 @@ mod tests { let fresh = server.broadcaster("live").await.expect("broadcast announced"); - assert!(!Arc::ptr_eq(&stale, &fresh)); + assert!(!stale.is_clone(&fresh)); assert!(server.inner.broadcasts.lock().unwrap().contains_key("live")); } @@ -473,7 +479,7 @@ mod tests { for _ in 0..4 { let server = Server::new(origin.consume(), Config::default()); let broadcaster = server.broadcaster("live").await.expect("broadcast announced"); - let released = Arc::downgrade(&broadcaster); + let released = broadcaster.downgrade(); drop(broadcaster); drop(server); settle().await; @@ -484,7 +490,7 @@ mod tests { let held = server.broadcaster("live").await.expect("broadcast announced"); drop(server); settle().await; - assert_eq!(Arc::strong_count(&held), 1, "only the response holds it now"); + assert_eq!(held.clones(), 1, "only the response holds it now"); assert!(!held.is_closed(), "a held broadcaster keeps serving"); producer.close(); } @@ -523,7 +529,7 @@ mod tests { .await; let kept = server.instance("live", old_epoch.as_str()).await; assert!( - kept.is_some_and(|kept| Arc::ptr_eq(&kept, &first)), + kept.is_some_and(|kept| kept.is_clone(&first)), "the old instance serves" ); @@ -564,7 +570,7 @@ mod tests { .publish("live", moq_net::origin::Route::default()) .expect("publish allowed"); settle().await; - let open = Broadcaster::new(crate::resolved(&origin, "live").await, Config::default()); + let open = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); // The follower has not yet seen the takeover instance close. let followed = Followed::new(open.clone()); @@ -580,7 +586,7 @@ mod tests { assert!(!current.is_closed()); let kept = server.instance("live", open.instance()).await; assert!( - kept.is_some_and(|kept| Arc::ptr_eq(&kept, &open)), + kept.is_some_and(|kept| kept.is_clone(&open)), "the open instance lost its URLs" ); } diff --git a/rs/moq-hls/tests/integration/restart.rs b/rs/moq-hls/tests/integration/restart.rs index d2c3826ad1f..a4c005fd72b 100644 --- a/rs/moq-hls/tests/integration/restart.rs +++ b/rs/moq-hls/tests/integration/restart.rs @@ -71,7 +71,7 @@ async fn takeover(version: &str, case: Case) { let deadline = kio::clock::now() + TIMEOUT; let current = loop { let current = server.broadcaster("live").await.expect("announced"); - if !std::sync::Arc::ptr_eq(¤t, &replaced) { + if !current.is_clone(&replaced) { break current; } assert!(kio::clock::now() < deadline, "the new instance never took over"); diff --git a/rs/moq-hls/tests/integration/server.rs b/rs/moq-hls/tests/integration/server.rs index b4c6c93256e..0ee1785d45d 100644 --- a/rs/moq-hls/tests/integration/server.rs +++ b/rs/moq-hls/tests/integration/server.rs @@ -54,7 +54,8 @@ async fn a_cursor_records_a_publisher_without_fetch() { .await .expect("request timed out") .expect("remote broadcast"); - let broadcaster = Broadcaster::new(moq_mux::Source::new(origin, remote), Default::default()); + let (broadcaster, driver) = Broadcaster::new(moq_mux::Source::new(origin, remote), Default::default()); + tokio::spawn(driver.run()); // Recorded as soon as the catalog lists it with its records, as a recorder does. let mut renditions = broadcaster.renditions(); let rendition = match kio::clock::timeout(TIMEOUT, renditions.next()).await {