From 6ac1f6159a9ac9bb698454efcc033db87122dd72 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 03:48:02 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m1/poll-first/no-spawn/cluster.md Quest-Claim: 36abfa25ebce8563 From 7c464b0fa3d9458aa17cc2485aecdb52939f532f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 09:58:40 -0800 Subject: [PATCH 2/3] refactor(relay): the cluster polls its own work instead of spawning `Cluster::new` returns `(Cluster, Driver)`: the origin driver and cache governor run as `kio::Tasks` children of a `Driver` the owner polls, and `Relay::run` polls it for its whole run, reporting a panic through the cluster's critical slot so teardown still runs. `Started::run` polls every peer dial, the `api.url` reconciler, and LAN discovery as `kio::Tasks` children; the sources queue dial-set changes the mesh applies, and `DialMap` owns its dial tasks by `TaskKey`. A dropped LAN dial's client close joins a polled set instead of a detached spawn. Each child's panic becomes an error naming it, so the relay still exits with "origin driver panicked: ..." and similar. Closes the quest quest/m1/poll-first/no-spawn/cluster.md. Co-Authored-By: Claude Opus 5.5 --- quest/m1/poll-first/no-spawn/README.md | 1 - quest/m1/poll-first/no-spawn/admissions.md | 16 +- quest/m1/poll-first/no-spawn/cluster.md | 52 -- quest/m1/poll-first/no-spawn/sim-tests.md | 1 - rs/moq-cli/src/args.rs | 8 +- rs/moq-cli/src/main.rs | 26 +- rs/moq-relay/CHANGELOG.md | 1 + rs/moq-relay/MIGRATE.md | 12 + rs/moq-relay/benches/auth.rs | 6 +- rs/moq-relay/benches/http_auth.rs | 3 +- rs/moq-relay/benches/session.rs | 3 +- rs/moq-relay/src/auth/admission_tests.rs | 14 +- rs/moq-relay/src/cache.rs | 80 +- rs/moq-relay/src/cluster.rs | 884 +++++++++++------- rs/moq-relay/src/config.rs | 2 +- rs/moq-relay/src/connection.rs | 2 +- rs/moq-relay/src/critical.rs | 69 +- rs/moq-relay/src/relay.rs | 42 +- rs/moq-relay/src/uring.rs | 2 +- rs/moq-relay/src/web.rs | 34 +- .../tests/integration/auth_lifetime.rs | 12 +- .../tests/integration/cluster_close.rs | 8 +- .../tests/integration/cluster_drain.rs | 5 +- .../tests/integration/cluster_idle.rs | 4 +- .../tests/integration/goaway_cluster.rs | 15 +- .../tests/integration/hidden_cluster.rs | 5 +- rs/moq-relay/tests/integration/lan_mesh.rs | 3 +- rs/moq-relay/tests/integration/main.rs | 12 + rs/moq-relay/tests/integration/memory.rs | 2 +- rs/moq-relay/tests/integration/peer_limits.rs | 4 +- .../tests/integration/public_rules.rs | 2 +- .../tests/integration/session_revalidate.rs | 4 +- rs/moq-relay/tests/integration/smoke.rs | 4 +- .../tests/integration/upstream_links.rs | 4 +- 34 files changed, 789 insertions(+), 553 deletions(-) delete mode 100644 quest/m1/poll-first/no-spawn/cluster.md diff --git a/quest/m1/poll-first/no-spawn/README.md b/quest/m1/poll-first/no-spawn/README.md index ca02cfbac92..ad8a5a53c05 100644 --- a/quest/m1/poll-first/no-spawn/README.md +++ b/quest/m1/poll-first/no-spawn/README.md @@ -49,7 +49,6 @@ binding boundary. ## Required -- [Cluster polls its own work](/quest/m1/poll-first/no-spawn/cluster.md) - `Cluster::new` returns a `Driver` the relay polls, and the dial loop runs as `kio::Tasks` children - [Admissions and stats are polled](/quest/m1/poll-first/no-spawn/admissions.md) - HTTP admissions run inside their waiter, and the stats publisher runs in the cluster's driver - [moq-ffi spawns only its root](/quest/m1/poll-first/no-spawn/ffi.md) - one runtime-owned driver task holds every `Background` as `kio::Tasks` children - [The relay's in-memory tests run on kio's simulator](/quest/m1/poll-first/no-spawn/sim-tests.md) - `rs/moq-relay/tests/integration/connection.rs` runs on the simulator instead of paused tokio diff --git a/quest/m1/poll-first/no-spawn/admissions.md b/quest/m1/poll-first/no-spawn/admissions.md index 86c63132c59..4ada3dda796 100644 --- a/quest/m1/poll-first/no-spawn/admissions.md +++ b/quest/m1/poll-first/no-spawn/admissions.md @@ -28,14 +28,12 @@ panic then ends the session being admitted, the same scope the per-admission polled by `Relay::run`, which brings back the long-lived decider task that #7360 removed. -The publisher joins `Driver`, the type -[Cluster polls its own work](/quest/m1/poll-first/no-spawn/cluster.md) -introduces, so a publisher panic still ends the relay. `Publisher::run` is an -`async fn`; giving `Publisher` a `poll` is optional and only worth it if -`Driver` needs it. +The publisher joins the cluster's `Driver` (`rs/moq-relay/src/cluster.rs`, +from [moq-dev/moq#7626](https://github.com/moq-dev/moq/pull/7626)), so a +publisher panic still ends the relay. `Cluster::new` has already handed the +driver out by the time `Cluster::with_stats` runs, so the publisher either +reaches it through a queue the driver polls or moves into `Options` for +`Cluster::new` to add. `Publisher::run` is an `async fn`; giving `Publisher` a +`poll` is optional and only worth it if `Driver` needs it. Public API: `Cluster::with_stats` no longer needs a tokio runtime. Wire: none. - -## Required - -- [Cluster polls its own work](/quest/m1/poll-first/no-spawn/cluster.md) - provides the `Driver` the publisher joins diff --git a/quest/m1/poll-first/no-spawn/cluster.md b/quest/m1/poll-first/no-spawn/cluster.md deleted file mode 100644 index b7cde935748..00000000000 --- a/quest/m1/poll-first/no-spawn/cluster.md +++ /dev/null @@ -1,52 +0,0 @@ -# [L] Cluster polls its own work - -## Goal - -moq-relay's `Cluster` spawns no tasks. `Cluster::new` returns the origin -driver and cache governor as work its owner polls, and `Started::run` drives -every dial, the `api.url` reconciler, and mDNS discovery as `kio::Tasks` -children instead of a tokio `JoinSet` and detached tasks. The relay still exits -with the panic message when a task it cannot serve without panics. - -## Plan - -Spawns in scope on `main` (2026-11-01), all in `rs/moq-relay/src/cluster.rs`: - -- `Cluster::new`: the cache governor and the origin driver, through `Critical`. -- `run_work`: the dial `JoinSet`, the `api.url` reconciler, and mDNS. -- `run_mdns` and `apply_peer_list`: one detached task per peer, tracked by an - `AbortHandle` in `DialMap`. These become a queue into the `run` loop's - `Tasks`, so `DialMap` holds `TaskKey`s. -- `Held::drop`: the close of a LAN dial's client. `Dialers` polls its existing - close set instead. - -✅ **`Cluster::new` returns a driver** (maintainer, 2026-11-01, D2 on -[moq-dev/moq#7519](https://github.com/moq-dev/moq/pull/7519)): -`Cluster::new(options) -> anyhow::Result<(Cluster, Driver)>`, the same shape as -`origin::Producer::new`. `Relay::run` polls `Driver` next to `Started::run`. -`Driver` catches each child's panic and turns it into an error, so the relay -still exits with "origin driver panicked: ..." the way `Critical` does now. -Tests spawn the driver, because they are edges. Rejected: keeping the signature -and moving the driver into `Started::run`, which breaks `Cluster::new`'s -documented promise that the origin serves sessions whether or not `start` is -ever called. There are about 68 `Cluster::new(` call sites across moq-relay, -moq-cli, the benches, and moq-net's `mesh_withdraw` test. - -✅ **Peer dials stop running in parallel** (maintainer, 2026-11-01, D5 on -[moq-dev/moq#7519](https://github.com/moq-dev/moq/pull/7519)): as `kio::Tasks` -children, every peer dial supervisor runs on the task that polls -`Started::run` instead of its own tokio task. The heavy session I/O is still -spawned inside moq-tokio, which the spawn audit owns, so the cost is probably -small. Measure it once in this PR and report the result. - -`Critical` stays for ACME renewal and the web certificate reload, which belong -to the spawn audit; `Cluster::with_stats` keeps spawning the publisher until -[the admissions child](/quest/m1/poll-first/no-spawn/admissions.md) moves it -into `Driver`. - -Public API: `Cluster::new` returns `(Cluster, Driver)`; the PR's Impact section -describes the migration. Wire: none. - -## Related - -- [The remaining spawns](/quest/m2/spawn-audit.md) - owns moq-tokio's session spawns and `Critical`'s remaining tasks diff --git a/quest/m1/poll-first/no-spawn/sim-tests.md b/quest/m1/poll-first/no-spawn/sim-tests.md index f5f4c7e581c..e317ad695f5 100644 --- a/quest/m1/poll-first/no-spawn/sim-tests.md +++ b/quest/m1/poll-first/no-spawn/sim-tests.md @@ -19,5 +19,4 @@ Public API: none. Wire: none. ## Required -- [Cluster polls its own work](/quest/m1/poll-first/no-spawn/cluster.md) - the cluster's driver and dials stop spawning on tokio - [Admissions and stats are polled](/quest/m1/poll-first/no-spawn/admissions.md) - admissions stop spawning on tokio diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 539143e1c68..6da9a4d3759 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -556,10 +556,10 @@ impl MoqSide { found } - /// The cluster this process publishes and subscribes on. Built once; the - /// origin is its origin, whose Hop ID is random per run unless `--cluster-id` - /// names the node. - pub fn cluster(&self) -> anyhow::Result { + /// The cluster this process publishes and subscribes on, with the driver that + /// must run for as long as it serves. Built once; the origin is its origin, whose + /// Hop ID is random per run unless `--cluster-id` names the node. + pub fn cluster(&self) -> anyhow::Result<(moq_relay::cluster::Cluster, moq_relay::cluster::Driver)> { moq_relay::cluster::Cluster::new(moq_relay::cluster::Options::new(self.cluster.clone())) } diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 89dadceef91..9c07cce95de 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -499,7 +499,7 @@ async fn spawn_moq( }; let (started, shutdown) = spawn_server(tasks, moq, &cluster, net, directions).await?; - // Even standalone: it is what reports a panic in the origin driver. + // Even standalone: it is what reports a panic in a critical task. tasks.spawn(async move { started.run().await }); let mut bandwidth = moq_net::bandwidth::Allocator::unlimited(); @@ -557,9 +557,10 @@ async fn run_play(moq: MoqSide, args: play::Args, net: Net) -> anyhow::Result<() // Before anything dials: a codec we can't decode is a blank window otherwise. args.validate()?; - let cluster = moq.cluster()?; + let (cluster, driver) = moq.cluster()?; let name = moq.broadcast.clone().unwrap_or_default(); let mut tasks: JoinSet> = JoinSet::new(); + tasks.spawn(driver.run()); let directions = Directions { consume: true, @@ -584,8 +585,9 @@ async fn run_play(moq: MoqSide, args: play::Args, net: Net) -> anyhow::Result<() /// and the first endpoint to finish ends the process. A shutdown signal keeps /// accepted sessions serving through the configured drain window. async fn run_stages(moq: MoqSide, stages: Vec, net: Net) -> anyhow::Result<()> { - let cluster = moq.cluster()?; + let (cluster, driver) = moq.cluster()?; let mut tasks: JoinSet> = JoinSet::new(); + tasks.spawn(driver.run()); // The stdin/capture pipelines run on this thread instead of the JoinSet: the // platform capture stream is not Send, so their futures cannot be spawned. let mut locals: Vec = Vec::new(); @@ -1298,7 +1300,8 @@ mod tests { .unwrap(); let listener = cli.moq.server_config().init_streams().unwrap().listen().await.unwrap(); let addr = listener.tcp_local_addr().unwrap(); - let cluster = moq_relay::cluster::Cluster::new(Default::default()).unwrap(); + let (cluster, driver) = moq_relay::cluster::Cluster::new(Default::default()).unwrap(); + tokio::spawn(driver.run()); let broadcast = cluster.origin.create_broadcast("probe").unwrap(); broadcast.announce(Default::default()).unwrap(); let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI); @@ -1565,7 +1568,8 @@ mod tests { "ts", ]) .expect("parse"); - let cluster = invocation.moq.cluster().expect("create cluster"); + let (cluster, driver) = invocation.moq.cluster().expect("create cluster"); + tokio::spawn(driver.run()); let net = Net { quic: invocation.moq.quic.clone(), #[cfg(feature = "iroh")] @@ -1589,7 +1593,8 @@ mod tests { async fn a_listener_without_auth_stops_startup() { let invocation = Invocation::try_parse_from(["moq", "--listen-tcp-bind", "127.0.0.1:0", "import", "ts"]) .expect("parse TCP-only invocation"); - let cluster = invocation.moq.cluster().expect("create cluster"); + let (cluster, driver) = invocation.moq.cluster().expect("create cluster"); + tokio::spawn(driver.run()); let net = Net { quic: invocation.moq.quic.clone(), #[cfg(feature = "iroh")] @@ -1629,7 +1634,8 @@ mod tests { "ts", ]) .expect("parse TCP-only invocation"); - let cluster = invocation.moq.cluster().expect("create cluster"); + let (cluster, driver) = invocation.moq.cluster().expect("create cluster"); + tokio::spawn(driver.run()); let net = Net { quic: invocation.moq.quic.clone(), #[cfg(feature = "iroh")] @@ -1670,7 +1676,8 @@ mod tests { Invocation::try_parse_from(["moq", "--cluster-api-url", "https://api.example/peers", "import", "ts"]) .expect("parse"); assert!(invocation.moq.validate().is_ok()); - let cluster = invocation.moq.cluster().expect("cluster"); + let (cluster, driver) = invocation.moq.cluster().expect("cluster"); + tokio::spawn(driver.run()); let err = cluster .clone() .start() @@ -1707,7 +1714,8 @@ mod tests { ]) .expect("parse"); invocation.moq.validate().expect("validate"); - let cluster = invocation.moq.cluster().expect("cluster"); + let (cluster, driver) = invocation.moq.cluster().expect("cluster"); + tokio::spawn(driver.run()); let net = Net { quic: invocation.moq.quic.clone(), #[cfg(feature = "iroh")] diff --git a/rs/moq-relay/CHANGELOG.md b/rs/moq-relay/CHANGELOG.md index 65be2e610b6..b41d4d9bc39 100644 --- a/rs/moq-relay/CHANGELOG.md +++ b/rs/moq-relay/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - [**breaking**] Auth token roles hold path patterns directly, matching release; a capped role is enforced or refused, never dropped. +- [**breaking**] `Cluster::new` returns `(Cluster, Driver)` instead of spawning the origin driver and cache governor, and `Started::run` polls every peer dial, the `--cluster-api-url` reconciler, and LAN discovery on its own task ([#7626](https://github.com/moq-dev/moq/pull/7626)). - Auth requests distinguish `webtransport` from native `quic`. Upgrade auth servers before relays: older servers reject the new value. Rust and JavaScript auth parsers now map future transport names to `unknown`. - `listen.timeout` bounds auth admission as part of the handshake, replacing the fixed ten-second admission timeout, so an auth server answering `connect` late is admitted within the deadline and `"0"` waits through admission too. HTTP reads bound admission by the web listener's timeout. `Auth::admit` and `Cluster::admit` no longer time out on their own; the caller owns the deadline. In-band token verification keeps its ten-second bound. diff --git a/rs/moq-relay/MIGRATE.md b/rs/moq-relay/MIGRATE.md index 0f943eedb71..87eae779068 100644 --- a/rs/moq-relay/MIGRATE.md +++ b/rs/moq-relay/MIGRATE.md @@ -5,6 +5,18 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq. ## Unreleased +### `Cluster::new` returns a driver to poll ([#7626](https://github.com/moq-dev/moq/pull/7626)) + +`Cluster::new` no longer spawns the origin's lifecycle work or the cache governor; it returns `(Cluster, Driver)`, and nothing runs until the driver is polled. +`Relay` polls it for you. An embedder building a `Cluster` itself keeps the driver for as long as the cluster serves, and polls it with `Driver::poll` or runs it with `Driver::run`, for example `tokio::spawn(driver.run())`: + +```rust +let (cluster, driver) = moq_relay::cluster::Cluster::new(options)?; +tokio::spawn(driver.run()); +``` + +Dropping the driver tears the origin down. `Driver::run` returns the first panic among its tasks, such as `origin driver panicked: ...`, which `Started::run` used to report. + ### `tls://` links verify client certificates ([#7358](https://github.com/moq-dev/moq/pull/7358)) The `listen.tcp.tls` listener applies `listen.tls.root` and pinned `tls.peers` as the QUIC listener does, so a stream-only relay with that listener no longer refuses them. diff --git a/rs/moq-relay/benches/auth.rs b/rs/moq-relay/benches/auth.rs index 645987afa8e..820540fc0be 100644 --- a/rs/moq-relay/benches/auth.rs +++ b/rs/moq-relay/benches/auth.rs @@ -58,7 +58,8 @@ impl Fixture { config.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); let mut server = config.init(Default::default()).unwrap().listen().await.unwrap(); let port = server.tcp_local_addr().unwrap().port(); - let cluster = cluster::Cluster::new(Default::default()).unwrap(); + let (cluster, driver) = cluster::Cluster::new(Default::default()).unwrap(); + tokio::spawn(driver.run()); let listener = tokio::spawn(async move { let mut serving = tokio::task::JoinSet::new(); while let Ok(request) = server.accept().await { @@ -170,7 +171,8 @@ impl Queue { config.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); let mut server = config.init(Default::default()).unwrap().listen().await.unwrap(); let port = server.tcp_local_addr().unwrap().port(); - let cluster = cluster::Cluster::new(Default::default()).unwrap(); + let (cluster, driver) = cluster::Cluster::new(Default::default()).unwrap(); + tokio::spawn(driver.run()); let listener = tokio::spawn(async move { let mut serving = tokio::task::JoinSet::new(); while let Ok(request) = server.accept().await { diff --git a/rs/moq-relay/benches/http_auth.rs b/rs/moq-relay/benches/http_auth.rs index cc3c4429086..21d0f8bbe8c 100644 --- a/rs/moq-relay/benches/http_auth.rs +++ b/rs/moq-relay/benches/http_auth.rs @@ -57,7 +57,8 @@ impl Fixture { let certificates = moq_tokio::tls::Certificates::from_pem(pem.as_bytes()).unwrap(); let mut config = web::Config::default(); config.http.listen = Some("127.0.0.1:0".parse().unwrap()); - let cluster = cluster::Cluster::new(Default::default()).unwrap(); + let (cluster, driver) = cluster::Cluster::new(Default::default()).unwrap(); + tokio::spawn(driver.run()); // No admission timeout, so held requests stay waiting however long setup and the // measurement take. let web = web::Web::new(auth, cluster, certificates, config) diff --git a/rs/moq-relay/benches/session.rs b/rs/moq-relay/benches/session.rs index fe85e9a9035..c3e2c4ba0ea 100644 --- a/rs/moq-relay/benches/session.rs +++ b/rs/moq-relay/benches/session.rs @@ -43,7 +43,8 @@ impl Backend for Relay { options.config.id = Some(id); // The shared sweeps include more announcements than a default ordinary client. options.session.announces = options.peer.announces; - let cluster = cluster::Cluster::new(options.with_cache(cache)).unwrap(); + let (cluster, driver) = cluster::Cluster::new(options.with_cache(cache)).unwrap(); + tokio::spawn(driver.run()); let mut stats = stats::Config::default(); stats.enabled = true; stats.node = Some(id.to_string()); diff --git a/rs/moq-relay/src/auth/admission_tests.rs b/rs/moq-relay/src/auth/admission_tests.rs index 70467ef5e31..4d1b08a96f1 100644 --- a/rs/moq-relay/src/auth/admission_tests.rs +++ b/rs/moq-relay/src/auth/admission_tests.rs @@ -9,7 +9,7 @@ fn patterns(texts: &[&str]) -> Patterns { #[tokio::test] async fn gateway_admission_scopes_origins_and_revoke_stops_work() { - let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap(); let (auth, mut admissions) = auth::Auth::embedded("edge"); let mut grant = Grant::new(patterns(&["media/**"]), patterns(&["watch/**"])); grant.root = Some("canonical".into()); @@ -42,7 +42,7 @@ async fn gateway_admission_scopes_origins_and_revoke_stops_work() { #[tokio::test] async fn a_gateway_role_without_scope_is_forbidden() { - let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap(); let (auth, mut admissions) = auth::Auth::embedded("edge"); tokio::spawn(async move { let admission = admissions.next().await.unwrap(); @@ -61,7 +61,7 @@ async fn a_gateway_role_without_scope_is_forbidden() { #[tokio::test] async fn completed_gateway_work_closes_its_lease() { - let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap(); let (auth, mut admissions) = auth::Auth::embedded("edge"); let decider = tokio::spawn(async move { let admission = admissions.next().await.unwrap(); @@ -80,7 +80,7 @@ async fn completed_gateway_work_closes_its_lease() { #[tokio::test] async fn a_narrowed_gateway_grant_stops_work() { - let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap(); let (auth, mut admissions) = auth::Auth::embedded("edge"); let decider = tokio::spawn(async move { let admission = admissions.next().await.unwrap(); @@ -103,7 +103,7 @@ async fn a_narrowed_gateway_grant_stops_work() { async fn a_moved_gateway_tier_retags_its_stats() { use moq_net::stats; - let mut cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); + let mut cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap(); cluster.stats = stats::Registry::new(stats::Config::new()); let registry = cluster.stats.clone(); let active = |tier: &'static str| { @@ -150,7 +150,7 @@ async fn a_moved_gateway_tier_retags_its_stats() { #[tokio::test(start_paused = true)] async fn gateway_admission_refuses_once_in_the_shared_stats_registry() { - let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap(); let (producer, publisher) = moq_stats::Producer::new(moq_stats::produce::Config::new().with_origin(cluster.origin.clone())); let registry = producer.registry().clone(); @@ -184,7 +184,7 @@ async fn gateway_admission_refuses_once_in_the_shared_stats_registry() { #[tokio::test] async fn embedded_expired_refusals_keep_verified_root_and_tier() { tokio::time::pause(); - let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap(); let config = crate::stats::Config { enabled: true, ..Default::default() diff --git a/rs/moq-relay/src/cache.rs b/rs/moq-relay/src/cache.rs index 381a2cec0f8..bdd92ac0e41 100644 --- a/rs/moq-relay/src/cache.rs +++ b/rs/moq-relay/src/cache.rs @@ -201,6 +201,11 @@ async fn governor(pool: cache::PoolWeak, capacity: Option, headroom: u64) { #[cfg(test)] mod tests { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use super::*; #[test] @@ -253,8 +258,7 @@ mod tests { } } - /// Spawned tasks currently alive in this test's runtime. The governor is the - /// only thing these tests spawn, so a delta of one is the governor. + /// Spawned tasks currently alive in this test's runtime. fn spawned() -> usize { tokio::runtime::Handle::current().metrics().num_alive_tasks() } @@ -269,21 +273,36 @@ mod tests { /// The prefix of [`crate::Relay::load`] that owns the cache: resolve it, then /// hand it to a cluster whose construction can still fail. - fn attach(cache: &Config, cluster: crate::cluster::Config) -> anyhow::Result { + fn attach( + cache: &Config, + cluster: crate::cluster::Config, + ) -> anyhow::Result<(crate::cluster::Cluster, crate::cluster::Driver)> { let cache = cache.init()?; crate::cluster::Cluster::new(crate::cluster::Options::new(cluster).with_cache(cache)) } - /// The governor runs as a critical task: its panic stops the cluster's - /// [`run`](crate::cluster::Started::run) rather than leaving the budget fixed. + /// Drive `driver` on its own task, reporting how many of its tasks still run. + fn drive(mut driver: crate::cluster::Driver) -> Arc { + let running = Arc::new(AtomicUsize::new(driver.len())); + let report = running.clone(); + tokio::spawn(moq_net::kio::wait(move |waiter| { + let poll = driver.poll(waiter); + report.store(driver.len(), Ordering::SeqCst); + poll + })); + running + } + + /// The governor runs in the cluster's driver: its panic fails the driver, which + /// stops [`Relay::run`](crate::Relay::run), rather than leaving the budget fixed. #[tokio::test(start_paused = true)] - async fn governor_panic_stops_the_cluster() { + async fn governor_panic_fails_the_driver() { let mut cache = governed().init().unwrap(); cache.governor = Some(async { panic!("injected") }.boxed()); - let cluster = crate::cluster::Cluster::new(crate::cluster::Options::default().with_cache(cache)).unwrap(); + let (_cluster, driver) = + crate::cluster::Cluster::new(crate::cluster::Options::default().with_cache(cache)).unwrap(); - let started = cluster.start().await.unwrap(); - let err = started.run().await.expect_err("the cluster kept running"); + let err = driver.run().await.expect_err("the driver kept running"); assert_eq!(err.to_string(), "cache governor panicked: injected"); } @@ -328,53 +347,48 @@ mod tests { assert_eq!(spawned(), before, "a failed setup should not strand the governor"); } - /// Tasks a cluster spawns on its own (its origin driver), measured rather than - /// assumed so the governor's task can be told apart from them. - async fn cluster_tasks() -> usize { - let before = spawned(); - let cluster = attach(&Config::default(), crate::cluster::Config::default()).unwrap(); - settle().await; - let own = spawned() - before; - drop(cluster); - settle().await; - own - } - + /// The governor and the origin driver both run until the last cluster handle drops. #[tokio::test(start_paused = true)] async fn last_owner_drop_stops_governor() { - let own = cluster_tasks().await; - let before = spawned(); - let cluster = attach(&governed(), crate::cluster::Config::default()).unwrap(); + let (cluster, driver) = attach(&governed(), crate::cluster::Config::default()).unwrap(); + let running = drive(driver); settle().await; assert_eq!( - spawned(), - before + own + 1, + running.load(Ordering::SeqCst), + 2, "the cluster should keep the governor running" ); drop(cluster); settle().await; - assert_eq!(spawned(), before, "the last cluster handle should stop the governor"); + assert_eq!( + running.load(Ordering::SeqCst), + 0, + "the last cluster handle should stop the governor" + ); } #[tokio::test(start_paused = true)] async fn extra_cluster_handle_keeps_governor() { - let own = cluster_tasks().await; - let before = spawned(); - let cluster = attach(&governed(), crate::cluster::Config::default()).unwrap(); + let (cluster, driver) = attach(&governed(), crate::cluster::Config::default()).unwrap(); + let running = drive(driver); let session = cluster.clone(); drop(cluster); settle().await; assert_eq!( - spawned(), - before + own + 1, + running.load(Ordering::SeqCst), + 2, "a surviving cluster clone keeps the governor" ); drop(session); settle().await; - assert_eq!(spawned(), before, "the last cluster handle should stop the governor"); + assert_eq!( + running.load(Ordering::SeqCst), + 0, + "the last cluster handle should stop the governor" + ); } } diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index afa4d4a83e9..fd9e7c0173c 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -13,7 +13,9 @@ use futures::FutureExt as _; use moq_net::origin; use moq_net::{Hop, Path, stats::Tier}; use reqwest_middleware::ClientWithMiddleware; -use tokio::task::AbortHandle; +use std::task::Poll; + +use crate::critical::Child; use tracing::Instrument as _; use url::Url; @@ -438,73 +440,59 @@ impl DialSources { } } -/// A dial task that ends once its [`DialEntry`] is replaced or the [`DialMap`] -/// drops, so a cancelled `run` takes the dynamic dials, spawned outside its -/// `JoinSet`, down with it. -struct DialTask(AbortHandle); - -impl Drop for DialTask { - fn drop(&mut self) { - self.0.abort(); - } -} - /// One entry in [`DialMap`]. struct DialEntry { - task: DialTask, + /// The live dial, or a finished one's stale key, which names nothing. + task: kio::TaskKey, sources: DialSources, active: DialSource, } -/// Map of in-flight cluster dials, keyed by canonical peer identity, owning each -/// dial's task. Cloneable: the inner map is shared via `Arc>` so the -/// discovery tasks and the static-seed phase write to the same set of entries. -#[derive(Clone, Default)] -struct DialMap { - inner: Arc>>, +/// The in-flight cluster dials, keyed by canonical peer identity, and the tasks +/// running them. A replaced or released dial's task is removed, which cancels it, +/// and dropping the map cancels them all. +/// +/// Generic over the dial task so tests can stand in a placeholder. +struct DialMap { + entries: HashMap, + tasks: kio::Tasks, } -impl DialMap { - /// True if `peer` is already being dialed. - fn contains(&self, peer: &str) -> bool { - self.inner - .lock() - .unwrap_or_else(PoisonError::into_inner) - .contains_key(peer) +impl Default for DialMap { + fn default() -> Self { + Self { + entries: HashMap::new(), + tasks: kio::Tasks::new(), + } } +} - /// Record an already-spawned dial under `source`. If the source is redundant, - /// abort its task instead of leaking a second session. - fn insert(&self, target: DialTarget, handle: AbortHandle, source: DialSource) { - let mut handle = Some(handle); - self.upsert(target, source, &mut |_| handle.take().expect("dial handle used once")); - if let Some(handle) = handle { - handle.abort(); - } +impl DialMap { + /// True if `peer` is already being dialed. + fn contains(&self, peer: &str) -> bool { + self.entries.contains_key(peer) } /// Add or update one source's target, replacing the live dial only when that /// source already owns it. Other sources retain their latest target as a - /// fallback without changing the first source's active configuration. - fn upsert(&self, target: DialTarget, source: DialSource, spawn: &mut F) - where - F: FnMut(DialTarget) -> AbortHandle, - { - let mut map = self.inner.lock().unwrap_or_else(PoisonError::into_inner); - if let Some(entry) = map.get_mut(&target.key) { + /// fallback without changing the first source's active configuration. `dial` + /// builds the task for a target that needs one. + fn upsert(&mut self, target: DialTarget, source: DialSource, dial: &mut impl FnMut(DialTarget) -> T) { + if let Some(entry) = self.entries.get_mut(&target.key) { let replace = entry.active == source && entry.sources.get(source) != Some(&target); entry.sources.set(source, target.clone()); if replace { - entry.task = DialTask(spawn(target)); + self.tasks.remove(entry.task); + entry.task = self.tasks.insert(dial(target)); } return; } let key = target.key.clone(); - let task = DialTask(spawn(target.clone())); + let task = self.tasks.insert(dial(target.clone())); let mut sources = DialSources::default(); sources.set(source, target); - map.insert( + self.entries.insert( key, DialEntry { task, @@ -516,16 +504,13 @@ impl DialMap { /// Release one source. If it owned the live dial, switch to a remaining /// source's latest target or abandon the peer when no source remains. - fn release(&self, peer: &str, source: DialSource, spawn: &mut F) - where - F: FnMut(DialTarget) -> AbortHandle, - { - let mut map = self.inner.lock().unwrap_or_else(PoisonError::into_inner); - let Some(entry) = map.get_mut(peer) else { return }; - if entry.sources.get(source).is_none() { + fn release(&mut self, peer: &str, source: DialSource, dial: &mut impl FnMut(DialTarget) -> T) { + let Some(entry) = self.entries.get_mut(peer) else { return; - } - let active_target = entry.sources.get(source).cloned(); + }; + let Some(active_target) = entry.sources.get(source).cloned() else { + return; + }; entry.sources.clear(source); if entry.active != source { return; @@ -534,39 +519,54 @@ impl DialMap { if let Some((next_source, next_target)) = entry.sources.fallback() { let next_target = next_target.clone(); entry.active = next_source; - if active_target.as_ref() != Some(&next_target) { - entry.task = DialTask(spawn(next_target)); + if active_target != next_target { + self.tasks.remove(entry.task); + entry.task = self.tasks.insert(dial(next_target)); } return; } - map.remove(peer); + if let Some(entry) = self.entries.remove(peer) { + self.tasks.remove(entry.task); + } } /// Reconcile the API source against `desired`, including changes to a peer's /// dial-affecting URL or link cost while its canonical identity stays fixed. - fn reconcile_api(&self, desired: &HashMap, mut spawn: F) - where - F: FnMut(DialTarget) -> AbortHandle, - { + fn reconcile_api(&mut self, desired: &HashMap, dial: &mut impl FnMut(DialTarget) -> T) { for target in desired.values() { - self.upsert(target.clone(), DialSource::Api, &mut spawn); + self.upsert(target.clone(), DialSource::Api, dial); } let removed: Vec = self - .inner - .lock() - .unwrap_or_else(PoisonError::into_inner) + .entries .iter() .filter(|(peer, entry)| entry.sources.api.is_some() && !desired.contains_key(*peer)) .map(|(peer, _)| peer.clone()) .collect(); for peer in removed { - self.release(&peer, DialSource::Api, &mut spawn); + self.release(&peer, DialSource::Api, dial); } } } +/// A change to the dial set from a peer source running beside the dials, which +/// [`Mesh`] applies on its own poll: the sources cannot reach the [`DialMap`] +/// whose task set is polling them. +enum DialCommand { + /// The latest valid `--cluster-api-url` peer list. + Api(HashMap), + /// A LAN peer this node should dial appeared or changed. + #[cfg(feature = "cluster-lan")] + Found(DialTarget), + /// A LAN peer's advertisement went away, by canonical key. + #[cfg(feature = "cluster-lan")] + Lost(String), +} + +/// Where the peer sources queue their [`DialCommand`]s. +type DialCommands = Arc>; + /// Every dial task and QUIC client the cluster dials peers with, so /// [`Cluster::close`] can end each link and send the peer its close before the /// runtime stops: a dropped QUIC link only queues the close, and the peer then @@ -592,21 +592,19 @@ struct DialersPhase { struct DialersState { clients: HashMap, next: u64, - /// Cloned into each background close, so `finished` ends once they all have. - closing: tokio::sync::mpsc::Sender<()>, - finished: tokio::sync::mpsc::Receiver<()>, + /// The closes of clients whose [`Held`] guard dropped, polled by + /// [`Started::run`] and drained by the shutdown. + closing: kio::Tasks>, } impl Dialers { fn new() -> Self { - let (closing, finished) = tokio::sync::mpsc::channel(1); Self { state: Arc::new(Mutex::new(DialersPhase { open: Some(DialersState { clients: HashMap::new(), next: 0, - closing, - finished, + closing: kio::Tasks::new(), }), closing: None, })), @@ -628,8 +626,8 @@ impl Dialers { Ok(id) } - /// Keep `client` until [`Self::close`] or until the guard drops, which closes it - /// in the background. + /// Keep `client` until [`Self::close`] or until the guard drops, which queues its + /// close for [`Self::poll_closing`]. fn hold(&self, client: &moq_tokio::Client) -> anyhow::Result { Ok(Held { id: self.insert(client)?, @@ -648,6 +646,18 @@ impl Dialers { } } + /// Advance the closes of clients whose guard dropped. Never resolves the dials' + /// lifetime: an empty set is only drained for now. + fn poll_closing(&self, waiter: &kio::Waiter) { + if let Some(state) = self.lock().open.as_mut() { + let _ = state.closing.poll_each(waiter, |_, res| { + if let Err(err) = res { + tracing::warn!(%err, "cluster client close failed"); + } + }); + } + } + /// Stop every dial task and close every client, including those a dropped /// guard is still closing, and refuse further dials. async fn close(&self) { @@ -668,15 +678,10 @@ impl Dialers { async fn shutdown(state: DialersState, stop: Arc>) { stop.send_replace(true); let DialersState { - clients, - closing, - mut finished, - .. + clients, mut closing, .. } = state; futures::future::join_all(clients.into_values().map(moq_tokio::Client::close)).await; - drop(closing); - // Nothing is ever sent: this returns `None` once every sender has dropped. - let _ = finished.recv().await; + kio::wait(|waiter| closing.poll_each(waiter, |_, _| {})).await; // Each stopped task dropped its link, which ends a non-QUIC one. stop.closed().await; } @@ -698,14 +703,7 @@ impl Drop for Held { let Some(client) = state.clients.remove(&self.id) else { return; }; - let closing = state.closing.clone(); - // Without a runtime nothing could send the close anyway. - if let Ok(runtime) = tokio::runtime::Handle::try_current() { - runtime.spawn(async move { - client.close().await; - drop(closing); - }); - } + state.closing.push(Child::new("cluster client close", client.close())); } } @@ -1021,6 +1019,48 @@ impl Options { } } +/// The work a [`Cluster`] needs to serve at all: its origin's lifecycle driver and, +/// when the cache has one, the headroom governor. Returned by [`Cluster::new`]. +/// +/// Nothing runs on its own: poll it, or await [`run`](Self::run), for as long as the +/// cluster serves. Dropping it tears the origin down, failing every later mutation. +#[must_use = "the origin makes no progress unless its driver is polled"] +pub struct Driver { + work: kio::Tasks>, +} + +impl Driver { + /// Drive the cluster's work until all of it has finished, which the origin only + /// does once every owner of it is gone, or until any of it panics. + /// + /// Resolves `Ok` once finished, or with the first panic as + /// `" panicked: "`, the error [`Relay::run`](crate::Relay::run) + /// exits with. Polling on after that keeps driving the rest. + pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + let mut failed = None; + let poll = self.work.poll_each(waiter, |_, res| { + if let Err(err) = res { + failed.get_or_insert(err); + } + }); + match failed { + Some(err) => Poll::Ready(Err(err)), + None => poll.map(Ok), + } + } + + /// [`poll`](Self::poll) until it resolves. + pub async fn run(mut self) -> anyhow::Result<()> { + kio::wait(move |waiter| self.poll(waiter)).await + } +} + +impl std::fmt::Debug for Driver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Driver").field("tasks", &self.work.len()).finish() + } +} + /// A [`Cluster`] whose config is validated and whose resources are bound, /// produced by [`Cluster::start`] and run with [`run`](Self::run). /// @@ -1044,13 +1084,16 @@ impl Started { self.work.is_none() } - /// Run the cluster until it stops, or until a task it cannot serve without (its - /// origin driver, cache governor, or certificate reload) panics. + /// Run the cluster's dials and peer sources on this task until they stop, or + /// until one of them or a task the relay cannot serve without (its stats + /// publisher, ACME renewal, or certificate reload) panics. The origin's own work + /// is the [`Driver`]'s. /// /// Standalone, this only waits for such a panic. Returning, a failure /// included, closes every dial as [`Cluster::close`] does. A cancelled `run` - /// starts each link's close but cannot wait for it, so a caller that must have - /// peers told before exiting awaits [`Cluster::close`] or lets `run` return. + /// drops its dials but leaves their links' close to [`Cluster::close`], so a + /// caller that must have peers told before exiting awaits it or lets `run` + /// return. pub async fn run(self) -> anyhow::Result<()> { // Registered only now, so a start that fails or never runs drops its client // and frees the bind. A closed cluster has nothing left to run. @@ -1063,7 +1106,10 @@ impl Started { let dialers = self.cluster.dialers.clone(); let work = async move { match self.work { - Some(work) => self.cluster.run_work(work).await, + Some(work) => { + let mut mesh = self.cluster.mesh(work)?; + kio::wait(move |waiter| mesh.poll(waiter)).await + } None => std::future::pending().await, } }; @@ -1099,6 +1145,78 @@ struct Work { discovery: Option, } +/// The peer dials and the sources feeding them, all polled on the task running +/// [`Started::run`] rather than spawned. +struct Mesh { + cluster: Cluster, + /// The shared cluster token, for dials whose URL carries none. + token: String, + /// Every peer dial, however many sources want it. + dialed: DialMap>, + /// The `api.url` reconciler and LAN discovery, which feed `commands`. + sources: kio::Tasks>>, + commands: DialCommands, +} + +impl Mesh { + /// Drive the dials and their sources. + /// + /// Resolves `Ok` once nothing is left to run: every dial ended and no source + /// can add another. Resolves with an error once a source fails or any of it + /// panics. + fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { + let mut failed = None; + // Sources first, so whatever a finishing source queued is applied below. + let sources = self.sources.poll_each(waiter, |_, res| { + if let Err(err) = res.and_then(|res| res) { + failed.get_or_insert(err); + } + }); + while let Poll::Ready(Ok(command)) = self.commands.poll_pop(waiter) { + self.apply(command); + } + // Each dial attempt contains its own panic, so one here is in the dial loop itself. + let dials = self.dialed.tasks.poll_each(waiter, |_, res| { + if let Err(err) = res { + failed.get_or_insert(err); + } + }); + self.cluster.dialers.poll_closing(waiter); + + if let Some(err) = failed { + return Poll::Ready(Err(err)); + } + match (sources, dials) { + (Poll::Ready(()), Poll::Ready(())) => Poll::Ready(Ok(())), + _ => Poll::Pending, + } + } + + /// Apply one source's change to the dial set. + fn apply(&mut self, command: DialCommand) { + let mut dial = dialer(&self.cluster, &self.token); + match command { + DialCommand::Api(desired) => self.dialed.reconcile_api(&desired, &mut dial), + #[cfg(feature = "cluster-lan")] + DialCommand::Found(target) => self.dialed.upsert(target, DialSource::Mdns, &mut dial), + #[cfg(feature = "cluster-lan")] + DialCommand::Lost(peer) => self.dialed.release(&peer, DialSource::Mdns, &mut dial), + } + } +} + +/// Build the task that dials and redials one peer. +fn dialer<'a>(cluster: &'a Cluster, token: &'a str) -> impl FnMut(DialTarget) -> Child<()> + 'a { + move |target| { + // Logged by key, never by the target's URL, so an inline query stays out of the logs. + tracing::info!(peer = %target.key, lan = target.lan, "dialing cluster peer"); + Child::new( + "cluster dial", + cluster.clone().supervise_remote(target, token.to_string()), + ) + } +} + /// A relay cluster built around a single [`origin::Producer`]. /// /// Local sessions and remote cluster connections all publish into the same @@ -1143,7 +1261,7 @@ pub struct Cluster { /// view always points at the same session in the logs. connection_ids: Arc, - /// The origin driver, cache governor, and certificate reload: a panic in any + /// The stats publisher, ACME renewal, and certificate reload: a panic in any /// of them stops [`Relay::run`](crate::Relay::run). pub(crate) critical: crate::critical::Critical, @@ -1216,14 +1334,16 @@ impl Cluster { /// [`with_stats`](Self::with_stats) to enable metrics publishing. Those /// builders do not rebuild the origin. /// - /// Must be called within a tokio runtime: the origin's lifecycle driver is - /// spawned here, so the origin serves sessions whether or not - /// [`start`](Self::start) (the mesh half) is ever called. + /// Returns the cluster with its [`Driver`]: the origin's lifecycle work and the + /// cache governor, which run only while the owner polls it. Poll it for as long + /// as the cluster serves, whether or not [`start`](Self::start) (the mesh half) + /// is ever called; [`Relay::run`](crate::Relay::run) does. Dropping it tears the + /// origin down. /// /// Errors if `config.id` is set to 0. An unset id picks a fresh random origin. /// Errors too if the `peer.quic` receive window does not fit a QUIC varint, or /// `config.tier` is not a valid stats tier. - pub fn new(options: Options) -> anyhow::Result { + pub fn new(options: Options) -> anyhow::Result<(Self, Driver)> { let Options { config, cache, @@ -1245,7 +1365,7 @@ impl Cluster { } crate::configured_tier(config.tier.clone()).context("invalid --cluster-tier")?; peer.quic.validate()?; - let critical = crate::critical::Critical::default(); + let mut work = kio::Tasks::new(); let mut origin_config = origin::Config::new(id); if let Some(max) = config.max_hops { origin_config.max_hops = origin::HopLimit::new(max).context("--cluster-max-hops must be from 1 to 32")?; @@ -1254,16 +1374,16 @@ impl Cluster { origin_config.pool = cache.pool; origin_config.cache_duration = cache.duration; if let Some(governor) = cache.governor { - critical.spawn("cache governor", governor); + work.push(Child::new("cache governor", governor)); } } let (origin, driver) = origin::Producer::new(origin_config); - critical.spawn("origin driver", async move { + work.push(Child::new("origin driver", async move { moq_net::time::run(driver).await; - }); + })); let nodes = crate::nodes::Nodes::default(); tracing::info!(hop_id = %origin.hop(), configured = config.id.is_some(), "cluster initialized"); - Ok(Cluster { + let cluster = Cluster { session_limits: session.resolve(), peer_limits: peer.resolve(), peer_quic: peer.quic, @@ -1280,11 +1400,12 @@ impl Cluster { nodes, refusals: Default::default(), connection_ids: Arc::default(), - critical, + critical: Default::default(), origin, stats: moq_net::stats::Registry::disabled(), _stats_publisher: None, - }) + }; + Ok((cluster, Driver { work })) } /// Admit a gateway session and scope both origin directions from its grant. @@ -1706,10 +1827,10 @@ impl Cluster { }) } - /// Runs the cluster event loop against the work [`start`](Self::start) - /// resolved: dials the static peers, then keeps the `api.url` list and - /// LAN discovery reconciled into the same dial set. - async fn run_work(self, work: Work) -> anyhow::Result<()> { + /// The dials and peer sources for the work [`start`](Self::start) resolved: + /// the static peers dialed at once, and the `api.url` reconciler and LAN + /// discovery queued to feed the same dial set. + fn mesh(self, work: Work) -> anyhow::Result { let Work { http, node, @@ -1718,60 +1839,41 @@ impl Cluster { discovery, } = work; - // Every source shares one dial map so a peer reached via several only - // opens a single dial. - let dialed = DialMap::default(); - let mut tasks = tokio::task::JoinSet::new(); - // Tasks whose ending is a failure rather than an ordinary lifecycle event, - // so their result has to be looked at instead of drained. - #[allow(unused_mut, reason = "only the cluster-lan feature spawns into it")] - let mut supervised: tokio::task::JoinSet> = tokio::task::JoinSet::new(); + let commands = DialCommands::default(); + let mut mesh = Mesh { + cluster: self.clone(), + token, + dialed: DialMap::default(), + sources: kio::Tasks::new(), + commands: commands.clone(), + }; for peer in &self.config.connect { let target = DialTarget::from_peer(peer).context("invalid --cluster-connect peer URL")?; - if dialed.contains(&target.key) { - continue; + if !mesh.dialed.contains(&target.key) { + let mut dial = dialer(&mesh.cluster, &mesh.token); + mesh.dialed.upsert(target, DialSource::Static, &mut dial); } - let this = self.clone(); - let token = token.clone(); - let peer_for_task = target.clone(); - let handle = tasks.spawn(this.supervise_remote(peer_for_task, token)); - dialed.insert(target, handle, DialSource::Static); } if let Some(source) = self.config.api.url.clone() { - let this = self.clone(); - let token = token.clone(); - let dialed = dialed.clone(); - let node = node.clone(); - tasks.spawn(async move { - this.run_connect_api(source, node, token, dialed, http).await; - }); + let commands = commands.clone(); + // Its ending is ordinary: it only stops when updates are disabled. + mesh.sources.push(Child::new("cluster.api.url reconciler", async move { + Self::run_connect_api(source, node, commands, http).await; + Ok(()) + })); } #[cfg(feature = "cluster-lan")] if let Some(discovery) = discovery { - let this = self.clone(); - let dialed = dialed.clone(); - // Supervised rather than fire-and-forget: a dial task ending is ordinary - // (that peer went away), but discovery ending is the relay going blind. - supervised.spawn(async move { this.run_mdns(dialed, discovery).await }); + // A dial ending is ordinary (that peer went away), but discovery ending is + // the relay going blind, so `run_mdns` only returns an error. + mesh.sources + .push(Child::new("LAN discovery", Self::run_mdns(commands, discovery))); } - loop { - tokio::select! { - // A supervised task ending at all is a failure, so its result decides - // the cluster's. A panic surfaces too rather than being swallowed. - Some(res) = supervised.join_next() => res.context("cluster task panicked")??, - // A dial task ending is ordinary: that peer went away. A panic is not: - // each dial attempt contains its own, so one here is in the loop itself. - Some(res) = tasks.join_next() => if let Err(err) = res && err.is_panic() { - return Err(err).context("cluster task panicked"); - }, - else => break, - } - } - Ok(()) + Ok(mesh) } /// Dial every process that advertises itself on the LAN. @@ -1782,16 +1884,9 @@ impl Cluster { /// never carries `?jwt=`: [`Config::token`] is for static and API peers /// only. #[cfg(feature = "cluster-lan")] - async fn run_mdns(self, dialed: DialMap, mut discovery: moq_tokio::mdns::Discovery) -> anyhow::Result<()> { + async fn run_mdns(commands: DialCommands, mut discovery: moq_tokio::mdns::Discovery) -> anyhow::Result<()> { use moq_tokio::mdns::Event; - // Logged by key, never by the target's URL, so an inline query stays out of - // the logs. Empty token: LAN authenticates with the mDNS credential. - let mut spawn = |target: DialTarget| { - tracing::info!(peer = %target.key, "dialing LAN cluster peer"); - tokio::spawn(self.clone().supervise_remote(target, String::new())).abort_handle() - }; - while let Some(event) = discovery.recv().await { match event { Event::Found(peer) => { @@ -1805,9 +1900,11 @@ impl Cluster { continue; } }; - dialed.upsert(target, DialSource::Mdns, &mut spawn); + let _ = commands.try_push(DialCommand::Found(target)); + } + Event::Lost(id) => { + let _ = commands.try_push(DialCommand::Lost(canonicalize_peer_key(&id))); } - Event::Lost(id) => dialed.release(&canonicalize_peer_key(&id), DialSource::Mdns, &mut spawn), _ => continue, } } @@ -1823,27 +1920,22 @@ impl Cluster { /// `file://` URL) is watched for changes. Either way the source yields a JSON /// array of peers that's reconciled into the shared dial map. async fn run_connect_api( - self, source: String, node: Option, - token: String, - dialed: DialMap, + commands: DialCommands, http: Option, ) { match Url::parse(&source) { Ok(url) if matches!(url.scheme(), "http" | "https") => { let http = http.expect("http(s) connect_api client built at startup"); - self.run_connect_api_http(url, node, token, dialed, http).await; + Self::run_connect_api_http(url, node, commands, http).await; } Ok(url) if url.scheme() == "file" => match url.to_file_path() { - Ok(path) => self.run_connect_api_file(path, node, token, dialed).await, + Ok(path) => Self::run_connect_api_file(path, node, commands).await, Err(()) => tracing::error!(%source, "cluster.api.url file URL is not a valid local path"), }, // Anything that isn't a URL we recognize is treated as a filesystem path. - _ => { - self.run_connect_api_file(PathBuf::from(&source), node, token, dialed) - .await - } + _ => Self::run_connect_api_file(PathBuf::from(&source), node, commands).await, } } @@ -1854,14 +1946,7 @@ impl Cluster { /// (`ETag` / `Last-Modified`) and serves the cached body if revalidation fails. /// Fails static: a failed fetch logs and keeps the current dials rather than /// tearing the cluster down. - async fn run_connect_api_http( - &self, - url: Url, - node: Option, - token: String, - dialed: DialMap, - http: ClientWithMiddleware, - ) { + async fn run_connect_api_http(url: Url, node: Option, commands: DialCommands, http: ClientWithMiddleware) { let mut tick = kio::clock::interval(CONNECT_API_POLL_INTERVAL); // A slow fetch must not bank missed ticks into a catch-up burst; just resume // the cadence from the next whole interval. @@ -1875,7 +1960,7 @@ impl Cluster { } match Self::fetch_peer_list(&http, req_url).await { - Ok(list) => self.apply_peer_list(list, &node, &token, &dialed), + Ok(list) => Self::apply_peer_list(list, &node, &commands), Err(err) => tracing::warn!(%err, "cluster.api.url fetch failed; keeping current peers"), } } @@ -1885,8 +1970,8 @@ impl Cluster { /// [`moq_tokio::watch::Files`] (OS notifications with a polling fallback). /// Fails static: a missing or malformed file keeps the current dials, and the /// next change triggers a fresh attempt. - async fn run_connect_api_file(&self, path: PathBuf, node: Option, token: String, dialed: DialMap) { - self.reload_connect_api_file(&path, &node, &token, &dialed); + async fn run_connect_api_file(path: PathBuf, node: Option, commands: DialCommands) { + Self::reload_connect_api_file(&path, &node, &commands); let mut watcher = match moq_tokio::watch::Files::new(std::slice::from_ref(&path)) { Ok(watcher) => watcher, @@ -1898,7 +1983,7 @@ impl Cluster { loop { watcher.changed().await; - self.reload_connect_api_file(&path, &node, &token, &dialed); + Self::reload_connect_api_file(&path, &node, &commands); } } @@ -1906,10 +1991,10 @@ impl Cluster { /// current dials; the [`Files`](moq_tokio::watch::Files) only /// re-invokes this on a real change, so a malformed file isn't re-warned on a /// loop. - fn reload_connect_api_file(&self, path: &std::path::Path, node: &Option, token: &str, dialed: &DialMap) { + fn reload_connect_api_file(path: &std::path::Path, node: &Option, commands: &DialCommands) { match std::fs::read_to_string(path) { Ok(body) => match serde_json::from_str::>(&body) { - Ok(list) => self.apply_peer_list(list, node, token, dialed), + Ok(list) => Self::apply_peer_list(list, node, commands), Err(err) => { tracing::warn!(%err, ?path, "cluster.api.url file is not a JSON array of peers; keeping current peers") } @@ -1933,10 +2018,11 @@ impl Cluster { serde_json::from_str(&body).context("cluster.api.url response is not a JSON array of peers") } - /// Reconcile a freshly fetched peer list into the dial map: dial peers that - /// are new and drop API peers that disappeared. The relay's own [`node`] URL - /// is filtered out so it never dials itself. - fn apply_peer_list(&self, list: Vec, node: &Option, token: &str, dialed: &DialMap) { + /// Queue a freshly fetched peer list for the dial map to reconcile: dial peers + /// that are new and drop API peers that disappeared. The relay's own [`node`] + /// URL is filtered out so it never dials itself, and an invalid list queues + /// nothing, keeping the current dials. + fn apply_peer_list(list: Vec, node: &Option, commands: &DialCommands) { // Dedupe against the shared dial map (and filter out self) on stable identity, // while retaining the full dial configuration so a cost or credential update // replaces the existing session. @@ -1948,11 +2034,7 @@ impl Cluster { } }; - dialed.reconcile_api(&desired, |target| { - tracing::info!(peer = %target.key, "cluster.api.url peer; dialing"); - let handle = tokio::spawn(self.clone().supervise_remote(target, token.to_string())); - handle.abort_handle() - }); + let _ = commands.try_push(DialCommand::Api(desired)); } async fn supervise_remote(self, target: DialTarget, token: String) { @@ -2348,6 +2430,34 @@ impl Default for Redial { } } +#[cfg(test)] +impl Driver { + /// How many of its tasks are still running. + pub(crate) fn len(&self) -> usize { + self.work.len() + } + + /// Add a task named `name` that panics on its first poll. + pub(crate) fn inject_panic(&mut self, name: &'static str) { + self.work.push(Child::new(name, async { panic!("injected") })); + } +} + +#[cfg(test)] +impl Cluster { + /// [`Self::new`] with its [`Driver`] spawned onto the ambient runtime, the way a + /// test edge runs it. A failed driver panics its task. + pub(crate) fn spawn(options: Options) -> anyhow::Result { + let (cluster, driver) = Self::new(options)?; + tokio::spawn(async move { + if let Err(err) = driver.run().await { + panic!("cluster driver failed: {err:#}"); + } + }); + Ok(cluster) + } +} + #[cfg(test)] mod tests { use super::*; @@ -2358,7 +2468,7 @@ mod tests { let mut options = Options::default(); options.session.announces = 2; options.peer.announces = 20; - let cluster = Cluster::new(options).unwrap(); + let cluster = Cluster::spawn(options).unwrap(); for (peer, upstream, expected) in [(false, false, 2), (true, false, 20), (true, true, 20)] { let mut grant = moq_auth::Grant::new(moq_auth::Patterns::new(), moq_auth::Patterns::new()); grant.peer = peer; @@ -2432,7 +2542,7 @@ mod tests { } fn new_cluster(config: Config) -> anyhow::Result { - Cluster::new(Options::new(config)) + Cluster::spawn(Options::new(config)) } /// A grant's mount reaches the target for subscribe and refuses publish, both @@ -2595,9 +2705,24 @@ mod tests { assert!(format!("{err:#}").contains("--cluster-tier"), "{err:#}"); } - /// Stand-in dial task: never makes progress, exposes an AbortHandle. - fn placeholder_handle() -> AbortHandle { - tokio::spawn(std::future::pending::<()>()).abort_handle() + /// Stand-in dial task: never makes progress. + type Placeholder = fn(&kio::Waiter) -> Poll<()>; + + fn placeholder(_: &kio::Waiter) -> Poll<()> { + Poll::Pending + } + + /// Dials nothing: fails the test if the dial map starts a dial. + fn no_dial(target: DialTarget) -> Placeholder { + panic!("must not dial {}", target.key) + } + + /// Records each dial the map starts. + fn recording(dialed: &mut Vec) -> impl FnMut(DialTarget) -> Placeholder + '_ { + |target| { + dialed.push(target); + placeholder + } } fn target(key: &str) -> DialTarget { @@ -2616,210 +2741,185 @@ mod tests { keys.iter().map(|key| ((*key).to_string(), target(key))).collect() } + /// Seed a dial for `target` under `source`. + fn seeded(dialed: &mut DialMap, target: DialTarget, source: DialSource) { + dialed.upsert(target, source, &mut |_| placeholder); + } + /// A peer wanted by both a static seed and the API survives losing either /// source: the dial is only torn down once the last source releases it. - #[tokio::test] - async fn multi_source_peer_survives_until_last_release() { - let dialed = DialMap::default(); + #[test] + fn multi_source_peer_survives_until_last_release() { + let mut dialed = DialMap::default(); // Seeded first, then also appears in the API list. - dialed.insert(target("both:4443"), placeholder_handle(), DialSource::Static); - dialed.reconcile_api(&desired(&["both:4443"]), |_| panic!("already dialed")); + seeded(&mut dialed, target("both:4443"), DialSource::Static); + dialed.reconcile_api(&desired(&["both:4443"]), &mut no_dial); // Dropped from the API list -> still wanted by the seed. - dialed.reconcile_api(&HashMap::new(), |_| panic!("static dial stays active")); + dialed.reconcile_api(&HashMap::new(), &mut no_dial); assert!(dialed.contains("both:4443"), "the seed still wants it"); + assert_eq!(dialed.tasks.len(), 1); - dialed.release("both:4443", DialSource::Static, &mut |_| panic!("must not redial")); + dialed.release("both:4443", DialSource::Static, &mut no_dial); assert!(!dialed.contains("both:4443")); + assert!(dialed.tasks.is_empty(), "the abandoned dial is cancelled"); } - /// `insert` for an already-dialed peer merges the source onto the existing - /// entry (and aborts the redundant handle) rather than opening a second dial. - #[tokio::test] - async fn insert_merges_redundant_dial() { - let dialed = DialMap::default(); - dialed.insert(target("p:4443"), placeholder_handle(), DialSource::Static); - dialed.insert(target("p:4443"), placeholder_handle(), DialSource::Api); + /// A second source for an already-dialed peer merges onto the existing entry + /// rather than opening a second dial. + #[test] + fn upsert_merges_redundant_dial() { + let mut dialed = DialMap::default(); + seeded(&mut dialed, target("p:4443"), DialSource::Static); + dialed.upsert(target("p:4443"), DialSource::Api, &mut no_dial); + assert_eq!(dialed.tasks.len(), 1); // Dropping the API source leaves the static source holding the dial. - dialed.reconcile_api(&HashMap::new(), |_| panic!("static dial stays active")); + dialed.reconcile_api(&HashMap::new(), &mut no_dial); assert!(dialed.contains("p:4443"), "static source still holds the dial"); } - /// `reconcile_api` drops API dials missing from the desired set, reports the - /// newly desired ones for the caller to spawn, and never touches Static dials - /// (even when they're absent from the API list). - #[tokio::test] - async fn reconcile_api_adds_and_removes_only_api() { - let dialed = DialMap::default(); - dialed.insert(target("static:4443"), placeholder_handle(), DialSource::Static); - dialed.insert(target("api-keep:4443"), placeholder_handle(), DialSource::Api); - dialed.insert(target("api-drop:4443"), placeholder_handle(), DialSource::Api); + /// `reconcile_api` drops API dials missing from the desired set, dials the + /// newly desired ones, and never touches Static dials (even when they're absent + /// from the API list). + #[test] + fn reconcile_api_adds_and_removes_only_api() { + let mut dialed = DialMap::default(); + seeded(&mut dialed, target("static:4443"), DialSource::Static); + seeded(&mut dialed, target("api-keep:4443"), DialSource::Api); + seeded(&mut dialed, target("api-drop:4443"), DialSource::Api); // Desired: keep one existing API peer, drop the other, add a new one. // The static peer is not in the list but must survive. - let mut to_add = Vec::new(); - dialed.reconcile_api(&desired(&["api-keep:4443", "api-new:4443"]), |target| { - to_add.push(target.key); - placeholder_handle() - }); - to_add.sort(); + let mut added = Vec::new(); + dialed.reconcile_api(&desired(&["api-keep:4443", "api-new:4443"]), &mut recording(&mut added)); - assert_eq!(to_add, vec!["api-new:4443".to_string()]); + let added: Vec<_> = added.into_iter().map(|target| target.key).collect(); + assert_eq!(added, ["api-new:4443"]); assert!(dialed.contains("api-keep:4443")); assert!(!dialed.contains("api-drop:4443"), "dropped API peer must be removed"); assert!(dialed.contains("static:4443"), "static peer must survive reconcile"); + assert_eq!(dialed.tasks.len(), 3, "the dropped dial is cancelled"); } - /// A peer already dialed via another source is not re-reported for dialing, - /// so the API reconcile can't open a duplicate connection. - #[tokio::test] - async fn reconcile_api_dedupes_against_other_sources() { - let dialed = DialMap::default(); - dialed.insert(target("shared:4443"), placeholder_handle(), DialSource::Static); + /// A peer already dialed via another source is not dialed again, so the API + /// reconcile can't open a duplicate connection. + #[test] + fn reconcile_api_dedupes_against_other_sources() { + let mut dialed = DialMap::default(); + seeded(&mut dialed, target("shared:4443"), DialSource::Static); - dialed.reconcile_api(&desired(&["shared:4443"]), |_| panic!("already dialed")); + dialed.reconcile_api(&desired(&["shared:4443"]), &mut no_dial); assert!(dialed.contains("shared:4443")); } /// A secondary source can update its target without disturbing the source that /// opened the session. If the active source disappears, its latest fallback /// configuration is used for the replacement. - #[tokio::test] - async fn inactive_source_update_applies_on_takeover() { - let dialed = DialMap::default(); - let seeded = DialTarget::parse("https://peer.example/?cost=1").unwrap(); + #[test] + fn inactive_source_update_applies_on_takeover() { + let mut dialed = DialMap::default(); + let seed = DialTarget::parse("https://peer.example/?cost=1").unwrap(); let api = DialTarget::parse("https://peer.example/?cost=3").unwrap(); - let old_task = tokio::spawn(std::future::pending::<()>()); - dialed.insert(seeded.clone(), old_task.abort_handle(), DialSource::Static); + seeded(&mut dialed, seed.clone(), DialSource::Static); + let old = dialed.entries[&seed.key].task; let desired = [(api.key.clone(), api.clone())].into_iter().collect(); - dialed.reconcile_api(&desired, |_| panic!("inactive source must not redial")); - assert!(!old_task.is_finished()); + dialed.reconcile_api(&desired, &mut no_dial); + assert_eq!(dialed.entries[&seed.key].task, old, "inactive source must not redial"); let mut spawned = Vec::new(); - dialed.release(&seeded.key, DialSource::Static, &mut |target| { - spawned.push(target); - placeholder_handle() - }); + dialed.release(&seed.key, DialSource::Static, &mut recording(&mut spawned)); - tokio::task::yield_now().await; - assert!(old_task.is_finished(), "old source must be aborted"); + assert!(dialed.tasks.remove(old).is_none(), "old source must be cancelled"); + assert_eq!(dialed.tasks.len(), 1); assert_eq!(spawned.len(), 1); assert!(spawned[0] == api); } /// If the fallback source requests the same target, ownership transfers without /// interrupting the healthy session. - #[tokio::test] - async fn identical_fallback_takeover_keeps_task() { - let dialed = DialMap::default(); + #[test] + fn identical_fallback_takeover_keeps_task() { + let mut dialed = DialMap::default(); let target = DialTarget::parse("https://peer.example/?cost=1").unwrap(); - let task = tokio::spawn(std::future::pending::<()>()); - dialed.insert(target.clone(), task.abort_handle(), DialSource::Static); + seeded(&mut dialed, target.clone(), DialSource::Static); + let task = dialed.entries[&target.key].task; let desired = [(target.key.clone(), target.clone())].into_iter().collect(); - dialed.reconcile_api(&desired, |_| panic!("inactive source must not redial")); + dialed.reconcile_api(&desired, &mut no_dial); + dialed.release(&target.key, DialSource::Static, &mut no_dial); - dialed.release(&target.key, DialSource::Static, &mut |_| { - panic!("identical fallback must not redial") - }); - - assert!(!task.is_finished(), "healthy dial must be preserved"); - let map = dialed.inner.lock().expect("dial map"); - let entry = map.get(&target.key).expect("current dial"); + let entry = dialed.entries.get(&target.key).expect("current dial"); + assert_eq!(entry.task, task, "healthy dial must be preserved"); assert_eq!(entry.active, DialSource::Api); assert!(entry.sources.get(entry.active) == Some(&target)); - drop(map); - task.abort(); + assert_eq!(dialed.tasks.len(), 1); } - /// A cost-only API update has the same identity but different SETUP input, so - /// it must replace the live task instead of being mistaken for an unchanged - /// peer. - #[tokio::test] - async fn reconcile_api_replaces_cost_only_change() { - let dialed = DialMap::default(); - let old = DialTarget::parse("https://peer.example/?cost=1").unwrap(); - let new = DialTarget::parse("https://peer.example/?cost=2").unwrap(); + /// Replacing `old` with `new` through the API redials exactly once and cancels + /// the old dial. + fn assert_api_replaces(old: DialTarget, new: DialTarget) { assert_eq!(old.key, new.key); - assert!(old != new); + let mut dialed = DialMap::default(); + seeded(&mut dialed, old.clone(), DialSource::Api); + let old_task = dialed.entries[&old.key].task; - let old_task = tokio::spawn(std::future::pending::<()>()); - dialed.insert(old, old_task.abort_handle(), DialSource::Api); let desired = [(new.key.clone(), new.clone())].into_iter().collect(); let mut spawned = Vec::new(); - dialed.reconcile_api(&desired, |target| { - spawned.push(target); - placeholder_handle() - }); + dialed.reconcile_api(&desired, &mut recording(&mut spawned)); - tokio::task::yield_now().await; - assert!(old_task.is_finished(), "old dial must be aborted"); + assert!(dialed.tasks.remove(old_task).is_none(), "old dial must be cancelled"); + assert_eq!(dialed.tasks.len(), 1); assert_eq!(spawned.len(), 1); assert!(spawned[0] == new); } + /// A cost-only API update has the same identity but different SETUP input, so + /// it must replace the live task instead of being mistaken for an unchanged + /// peer. + #[test] + fn reconcile_api_replaces_cost_only_change() { + let old = DialTarget::parse("https://peer.example/?cost=1").unwrap(); + let new = DialTarget::parse("https://peer.example/?cost=2").unwrap(); + assert!(old != new); + assert_api_replaces(old, new); + } + /// An identical API render keeps the live task, avoiding connection churn. - #[tokio::test] - async fn reconcile_api_identical_target_is_noop() { - let dialed = DialMap::default(); + #[test] + fn reconcile_api_identical_target_is_noop() { + let mut dialed = DialMap::default(); let target = DialTarget::parse("https://peer.example/?cost=2&jwt=secret").unwrap(); - let task = tokio::spawn(std::future::pending::<()>()); - dialed.insert(target.clone(), task.abort_handle(), DialSource::Api); - let desired = [(target.key.clone(), target)].into_iter().collect(); + seeded(&mut dialed, target.clone(), DialSource::Api); + let task = dialed.entries[&target.key].task; + let desired = [(target.key.clone(), target.clone())].into_iter().collect(); - dialed.reconcile_api(&desired, |_| panic!("identical target must not redial")); - assert!(!task.is_finished(), "live dial must be preserved"); - task.abort(); + dialed.reconcile_api(&desired, &mut no_dial); + assert_eq!(dialed.entries[&target.key].task, task, "live dial must be preserved"); + assert_eq!(dialed.tasks.len(), 1); } /// Inline credentials are dial-affecting even though they are excluded from /// peer identity, so rotating one replaces the session too. - #[tokio::test] - async fn reconcile_api_replaces_inline_credential() { - let dialed = DialMap::default(); - let old = DialTarget::parse("https://peer.example/?jwt=old").unwrap(); - let new = DialTarget::parse("https://peer.example/?jwt=new").unwrap(); - assert_eq!(old.key, new.key); - - let old_task = tokio::spawn(std::future::pending::<()>()); - dialed.insert(old, old_task.abort_handle(), DialSource::Api); - let desired = [(new.key.clone(), new.clone())].into_iter().collect(); - let mut spawned = Vec::new(); - dialed.reconcile_api(&desired, |target| { - spawned.push(target); - placeholder_handle() - }); - - tokio::task::yield_now().await; - assert!(old_task.is_finished(), "old dial must be aborted"); - assert_eq!(spawned.len(), 1); - assert!(spawned[0] == new); + #[test] + fn reconcile_api_replaces_inline_credential() { + assert_api_replaces( + DialTarget::parse("https://peer.example/?jwt=old").unwrap(), + DialTarget::parse("https://peer.example/?jwt=new").unwrap(), + ); } /// A malformed replacement is rejected before reconciliation, so it cannot /// tear down or reconfigure the last-known-good dial set. - #[tokio::test] - async fn malformed_peer_list_preserves_current_dial() { - let cluster = new_cluster(Config::default()).expect("cluster"); - let dialed = DialMap::default(); - let current = DialTarget::parse("https://peer.example/?cost=1").unwrap(); - let task = tokio::spawn(std::future::pending::<()>()); - dialed.insert(current.clone(), task.abort_handle(), DialSource::Api); - - cluster.apply_peer_list( - vec![Peer::new("https://peer.example/?cost=invalid")], - &None, - "", - &dialed, - ); + #[test] + fn malformed_peer_list_preserves_current_dial() { + let commands = DialCommands::default(); + Cluster::apply_peer_list(vec![Peer::new("https://peer.example/?cost=invalid")], &None, &commands); + assert!(commands.is_empty(), "an invalid list must not reach the dial map"); - assert!(!task.is_finished(), "last-known-good dial must stay active"); - let map = dialed.inner.lock().expect("dial map"); - let entry = map.get(¤t.key).expect("current dial"); - assert!(entry.sources.get(entry.active) == Some(¤t)); - task.abort(); + Cluster::apply_peer_list(vec![Peer::new("https://peer.example/?cost=1")], &None, &commands); + assert_eq!(commands.len(), 1, "a valid list is queued"); } /// The same identity cannot appear twice with different SETUP inputs because @@ -3040,28 +3140,78 @@ mod tests { new_cluster(equivalent).expect("equivalent static peers dedupe"); } - /// A malformed object entry keeps the last-good dials, exactly like a - /// malformed URL does. + fn test_mesh() -> (Mesh, DialCommands) { + let commands = DialCommands::default(); + let mesh = Mesh { + cluster: new_cluster(Config::default()).expect("cluster"), + token: String::new(), + dialed: DialMap::default(), + sources: kio::Tasks::new(), + commands: commands.clone(), + }; + (mesh, commands) + } + + /// A peer source feeds the dial set through the mesh's queue, on the mesh's own + /// poll, and a source failing ends the mesh with its error. #[tokio::test] - async fn malformed_object_peer_list_preserves_current_dial() { - let cluster = new_cluster(Config::default()).expect("cluster"); - let dialed = DialMap::default(); - let current = DialTarget::from_peer(&Peer::new("https://peer.example/?cost=1")).unwrap(); - let task = tokio::spawn(std::future::pending::<()>()); - dialed.insert(current.clone(), task.abort_handle(), DialSource::Api); + async fn mesh_dials_what_a_source_queues_and_fails_with_it() { + let (mut mesh, commands) = test_mesh(); + let (stop, stopped) = tokio::sync::oneshot::channel::<()>(); + mesh.sources.push(Child::new("test source", async move { + Cluster::apply_peer_list(vec![Peer::new("https://peer.example/")], &None, &commands); + let _ = stopped.await; + anyhow::bail!("source stopped") + })); + + let waiter = kio::Waiter::noop(); + assert!(mesh.poll(&waiter).is_pending()); + assert!( + mesh.dialed.contains("https://peer.example/"), + "the queued peer is dialed" + ); + assert_eq!(mesh.dialed.tasks.len(), 1); - let asymmetric = Peer::new("https://peer.example/").with_cost(1).with_egress(2); - cluster.apply_peer_list(vec![asymmetric], &None, "", &dialed); + drop(stop); + let Poll::Ready(Err(err)) = mesh.poll(&waiter) else { + panic!("a failed source must end the mesh"); + }; + assert_eq!(err.to_string(), "source stopped"); + } - assert!(!task.is_finished(), "last-known-good dial must stay active"); - let map = dialed.inner.lock().expect("dial map"); - let entry = map.get(¤t.key).expect("current dial"); - assert!(entry.sources.get(entry.active) == Some(¤t)); - task.abort(); + /// A dial task's panic ends the mesh with it, rather than unwinding the task + /// polling the mesh. + #[tokio::test] + async fn mesh_reports_a_dial_panic() { + let (mut mesh, _commands) = test_mesh(); + mesh.dialed + .tasks + .insert(Child::new("cluster dial", async { panic!("injected") })); + + let Poll::Ready(Err(err)) = mesh.poll(&kio::Waiter::noop()) else { + panic!("a dial panic must end the mesh"); + }; + assert_eq!(err.to_string(), "cluster dial panicked: injected"); + } + + /// With every source finished and no dial left, the mesh has nothing to run. + #[tokio::test] + async fn mesh_finishes_once_nothing_is_left() { + let (mut mesh, _commands) = test_mesh(); + mesh.sources.push(Child::new("test source", async { Ok(()) })); + assert!(matches!(mesh.poll(&kio::Waiter::noop()), Poll::Ready(Ok(())))); + } + + /// A malformed object entry keeps the last-good dials, exactly like a + /// malformed URL does. + #[test] + fn malformed_object_peer_list_preserves_current_dial() { + let commands = DialCommands::default(); + let asymmetric = Peer::new("https://peer.example/").with_cost(1).with_egress(2); + Cluster::apply_peer_list(vec![asymmetric], &None, &commands); + assert!(commands.is_empty(), "an invalid list must not reach the dial map"); } - /// A valid `cluster.id`, anywhere in the 64-bit range, is used verbatim as the - /// relay's Hop ID, giving the node a stable identity across restarts. #[tokio::test] async fn cluster_id_sets_origin() { for id in [42, u64::MAX] { @@ -3086,7 +3236,7 @@ mod tests { let cache = cache.init().expect("cache"); let pool = cache.pool.clone(); - let cluster = Cluster::new( + let cluster = Cluster::spawn( Options::new(Config { id: Some(42), ..Default::default() @@ -3248,18 +3398,18 @@ mod tests { /// The same relay spelled as a bare `host:port`, a full URL, or a URL with an /// inline jwt all canonicalize to one key, so they share a single dial entry. - #[tokio::test] - async fn canonicalize_peer_key_dedupes_spellings() { + #[test] + fn canonicalize_peer_key_dedupes_spellings() { let key = canonicalize_peer_key("host:4443"); assert_eq!(key, "https://host:4443/"); assert_eq!(canonicalize_peer_key("https://host:4443/"), key); assert_eq!(canonicalize_peer_key("https://host:4443/?jwt=abc"), key); // A URL form and the legacy host:port form dedupe against each other. - let dialed = DialMap::default(); - dialed.insert( + let mut dialed = DialMap::default(); + seeded( + &mut dialed, DialTarget::parse("https://host:4443/?jwt=abc").unwrap(), - placeholder_handle(), DialSource::Static, ); assert!(dialed.contains(&canonicalize_peer_key("host:4443"))); @@ -3474,22 +3624,23 @@ mod tests { /// its dial when the advertisement goes away, and only the last release /// tears it down. #[cfg(feature = "cluster-lan")] - #[tokio::test] - async fn mdns_release_respects_the_other_sources() { - let dialed = DialMap::default(); + #[test] + fn mdns_release_respects_the_other_sources() { + let mut dialed = DialMap::default(); let peer = target("peer:4443"); - dialed.insert(peer.clone(), placeholder_handle(), DialSource::Mdns); - dialed.insert(peer.clone(), placeholder_handle(), DialSource::Static); + seeded(&mut dialed, peer.clone(), DialSource::Mdns); + dialed.upsert(peer.clone(), DialSource::Static, &mut no_dial); // The static seed wants the same target, so the takeover reuses the dial. - dialed.release(&peer.key, DialSource::Mdns, &mut |_| panic!("must not redial")); + dialed.release(&peer.key, DialSource::Mdns, &mut no_dial); assert!(dialed.contains(&peer.key), "the static seed still wants it"); - dialed.release(&peer.key, DialSource::Static, &mut |_| panic!("must not redial")); + dialed.release(&peer.key, DialSource::Static, &mut no_dial); assert!(!dialed.contains(&peer.key), "the last release abandons the dial"); + assert!(dialed.tasks.is_empty(), "the abandoned dial is cancelled"); // Releasing an unknown peer is a no-op, not a panic. - dialed.release("never-dialed", DialSource::Mdns, &mut |_| panic!("must not redial")); + dialed.release("never-dialed", DialSource::Mdns, &mut no_dial); } #[test] @@ -3844,8 +3995,17 @@ mod tests { #[tokio::test] async fn concurrent_closes_share_one_shutdown() { let dialers = Dialers::new(); - // Stands in for a dropped guard's background close, which holds the shutdown open. - let background = dialers.lock().open.as_ref().expect("open").closing.clone(); + // Stands in for a dropped guard's close, which holds the shutdown open. + let (background, closed) = tokio::sync::oneshot::channel::<()>(); + dialers + .lock() + .open + .as_mut() + .expect("open") + .closing + .push(Child::new("held close", async move { + let _ = closed.await; + })); let first = tokio::spawn({ let dialers = dialers.clone(); async move { dialers.close().await } diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 232b06bdbb3..6f5570caa5c 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -946,7 +946,7 @@ root = ["ca.pem"] ]; let config = Config::parse_and_merge(args).expect("config load"); assert!(config.stats.enabled); - let cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster"); + let cluster = cluster::Cluster::spawn(cluster::Options::default()).expect("cluster"); let _ = config.stats.build(cluster.origin.clone()).expect("stats build"); } diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index dfb47c73870..38a1d716b74 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -455,7 +455,7 @@ mod tests { tracing_subscriber::filter::Targets::new().with_target("moq_relay::connection", tracing::Level::INFO), ); let _logging = tracing::subscriber::set_default(subscriber); - let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap(); let (auth, _admissions) = auth::Auth::embedded("test"); let lifecycle = Lifecycle::new(cluster, auth, 42, None); lifecycle diff --git a/rs/moq-relay/src/critical.rs b/rs/moq-relay/src/critical.rs index b9384cca6b6..d9feeef4397 100644 --- a/rs/moq-relay/src/critical.rs +++ b/rs/moq-relay/src/critical.rs @@ -5,12 +5,50 @@ //! certificate reload that dies leaves a relay that still accepts connections but //! serves nothing, which is worse than exiting. So their panic is not contained; //! [`Relay::run`](crate::Relay::run) returns it. +//! +//! [`Critical`] spawns such a task; [`Child`] is the same guard for one polled as a +//! [`kio::Tasks`] child by its owner. use std::{future::Future, panic::AssertUnwindSafe, task::Poll}; -use futures::FutureExt; +use futures::{FutureExt, future::BoxFuture}; use moq_net::kio; +/// A named future polled as a [`kio::Tasks`] child, whose panic becomes an error +/// instead of unwinding through the owner's poll. +pub(crate) struct Child { + name: &'static str, + future: BoxFuture<'static, T>, +} + +impl Child { + pub(crate) fn new(name: &'static str, future: impl Future + Send + 'static) -> Self { + Self { + name, + future: future.boxed(), + } + } +} + +impl kio::Task for Child { + /// The future's output, or `" panicked: "` once it panics. + type Output = anyhow::Result; + + fn poll(&mut self, waiter: &kio::Waiter) -> Poll { + match std::panic::catch_unwind(AssertUnwindSafe(|| waiter.poll_future(self.future.as_mut()))) { + Ok(poll) => poll.map(Ok), + Err(payload) => Poll::Ready(Err(panicked(self.name, payload.as_ref()))), + } + } +} + +/// Log a critical task's panic and describe it as `" panicked: "`. +fn panicked(name: &'static str, payload: &(dyn std::any::Any + Send)) -> anyhow::Error { + let panic = moq_tokio::panic::message(payload); + tracing::error!(task = name, panic, "critical task panicked"); + anyhow::anyhow!("{name} panicked: {panic}") +} + /// Spawns long-lived tasks and reports the first one to panic. #[derive(Clone, Default)] pub(crate) struct Critical { @@ -26,14 +64,20 @@ impl Critical { let Err(payload) = AssertUnwindSafe(task).catch_unwind().await else { return; }; - let panic = moq_tokio::panic::message(payload.as_ref()); - tracing::error!(task = name, panic, "critical task panicked"); + let panic = panicked(name, payload.as_ref()).to_string(); if let Ok(mut failed) = failed.write() { - failed.get_or_insert_with(|| format!("{name} panicked: {panic}")); + failed.get_or_insert(panic); } }); } + /// Report a failure from a task polled elsewhere, as though a spawned one panicked. + pub(crate) fn fail(&self, err: anyhow::Error) { + if let Ok(mut failed) = self.failed.write() { + failed.get_or_insert_with(|| err.to_string()); + } + } + /// Resolve once any task panics, with that panic. pub(crate) async fn failed(&self) -> anyhow::Error { let failed = self.failed.consume(); @@ -64,4 +108,21 @@ mod tests { let err = critical.failed().await; assert_eq!(err.to_string(), "broken panicked: injected"); } + + /// A child's panic resolves it with the error instead of unwinding the owner. + #[test] + fn a_child_panic_becomes_its_output() { + let mut tasks = kio::Tasks::new(); + tasks.push(Child::new("quiet", async {})); + tasks.push(Child::new("broken", async { panic!("injected") })); + + let mut errors = Vec::new(); + let poll = tasks.poll_each(&kio::Waiter::noop(), |_, res| { + if let Err(err) = res { + errors.push(err.to_string()); + } + }); + assert!(poll.is_ready(), "both children finished"); + assert_eq!(errors, ["broken panicked: injected"]); + } } diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 8ca21030be1..3dca346aaa8 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -20,6 +20,8 @@ //! Calling `load` means a new step arrives with the update, and a reshaped API //! is a compile error. +use std::task::Poll; + use anyhow::Context; use axum::Router; @@ -87,6 +89,9 @@ pub struct Relay { /// config named a source; `run` refuses to start while this is still held. admissions: Option, cluster: cluster::Cluster, + /// The cluster's origin and cache work, polled for as long as [`Self::run`] + /// runs. `None` only once `run` took it. + driver: Option, stats: moq_stats::Producer, internal: internal::Internal, web: web::Web, @@ -339,7 +344,8 @@ impl Relay { let mut cluster_options = cluster::Options::new(config.cluster).with_cache(cache); cluster_options.session = config.limits.client; cluster_options.peer = config.limits.peer; - let cluster = cluster::Cluster::new(cluster_options)? + let (cluster, driver) = cluster::Cluster::new(cluster_options)?; + let cluster = cluster .with_connect(config.connect.clone(), config.quic.clone()) .with_advertise(advertise); #[cfg(feature = "iroh")] @@ -413,6 +419,7 @@ impl Relay { auth, admissions, cluster, + driver: Some(driver), stats, internal, web, @@ -603,11 +610,26 @@ impl Relay { /// /// This is also the embedding loop. Extra routes go on via [`Self::with_web`] /// / [`Self::with_internal`] before calling this; cloned handles outlive it. - pub async fn run(self) -> anyhow::Result<()> { + pub async fn run(mut self) -> anyhow::Result<()> { // Held past `run_inner`, so a waiter sees the failure before the close. let ready = self.ready.clone(); + // Polled beside everything `run_inner` does, startup and teardown included. A + // panic is reported like any critical task's, so the cluster's `run` returns + // it and `run_inner` still tears down on the way out. + let mut driver = self.driver.take().expect("only run takes the driver"); + let critical = self.cluster.critical.clone(); + let driving = kio::wait(move |waiter| { + if let Poll::Ready(Err(err)) = driver.poll(waiter) { + critical.fail(err); + } + Poll::::Pending + }); // Serving branches must not inflate the caller's stack frame. - let result = Box::pin(self.run_inner()).await; + let result = tokio::select! { + biased; + result = Box::pin(self.run_inner()) => result, + never = driving => match never {}, + }; if let Err(err) = &result { ready.send_if_modified(|startup| { let starting = matches!(startup, Startup::Starting); @@ -1090,6 +1112,20 @@ mod tests { assert_eq!(err.root_cause().to_string(), "injected task panicked: injected"); } + /// A panic in the cluster's driver, which `run` polls rather than spawns, ends + /// `run` with that panic the same way. + #[tokio::test] + async fn a_cluster_driver_panic_stops_the_relay() { + let mut config = crate::Config::default(); + config.listen.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); + let mut relay = super::Relay::load(config).await.unwrap().with_signals(false); + let _admissions = relay.admissions().unwrap(); + relay.driver.as_mut().expect("driver").inject_panic("origin driver"); + + let err = relay.run().await.expect_err("the relay kept running"); + assert_eq!(err.root_cause().to_string(), "origin driver panicked: injected"); + } + #[cfg(not(feature = "_quic"))] #[tokio::test] async fn workers_require_a_quic_backend() { diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index e905f9772e2..f2a871d3cdb 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -1137,7 +1137,7 @@ mod tests { server.listen.bind = Some("127.0.0.1:0".parse().unwrap()); server.listen.tls.generate = vec!["localhost".into()]; let workers = Workers::bind(server, moq_tokio::worker::Config::new(2)).unwrap(); - let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap(); let workers = workers .spawn( cluster, diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index 7579027206d..2a445f6c0da 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -1238,7 +1238,7 @@ mod tests { /// The state an acceptor built outside [`Web::serve`] hands a `tls://` session. #[cfg(all(feature = "websocket", any(unix, windows)))] fn acceptor_state() -> Arc { - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); let certificates = moq_tokio::tls::Certificates::from_pem(cert.cert.pem().as_bytes()).unwrap(); Web::new( @@ -1265,7 +1265,7 @@ mod tests { } .init("test", &moq_tokio::tls::Connect::default(), false) .unwrap(); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let broadcast = cluster.origin.create_broadcast("room").unwrap(); broadcast.announce(Default::default()).unwrap(); let track = broadcast @@ -1358,7 +1358,7 @@ mod tests { } else { auth::Auth::refuse("test") }; - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let web = Web::new(auth, cluster, certificates.clone(), Config::default()); let state = web.state; let remote = ConnectInfo("127.0.0.1:1234".parse().unwrap()); @@ -1426,7 +1426,7 @@ mod tests { } }); let registry = Registry::new(Default::default()); - let mut cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let mut cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); cluster.refusals = crate::refusals::Refusals::new(®istry); let state = Web::new(auth, cluster, certificates, Config::default()).state; let fetched = serve_fetch( @@ -1495,7 +1495,7 @@ mod tests { } leases }); - let mut cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let mut cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); cluster.stats = Registry::new(Default::default()); let broadcast = cluster.origin.create_broadcast("room").unwrap(); broadcast.announce(Default::default()).unwrap(); @@ -1598,7 +1598,7 @@ mod tests { } .init("test", &moq_tokio::tls::Connect::default(), false) .unwrap(); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let broadcast = cluster.origin.create_broadcast("room").unwrap(); broadcast.announce(Default::default()).unwrap(); let track = broadcast @@ -1656,7 +1656,7 @@ mod tests { } leases }); - let mut cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let mut cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); cluster.stats = moq_net::stats::Registry::new(Default::default()); let state = Web::new(auth, cluster, certificates, Config::default()).state; let peer = || ConnectInfo("127.0.0.1:1234".parse().unwrap()); @@ -2002,7 +2002,7 @@ mod tests { let auth = auth_config .init("test", &moq_tokio::tls::Connect::default(), false) .unwrap(); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); let web = Web::new(auth, cluster, certificates, config).bind().unwrap(); @@ -2053,7 +2053,7 @@ mod tests { config.https.key = vec![key]; config.ws = true; let (auth, mut admissions) = crate::auth::Auth::embedded("test"); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); let web = Web::new(auth, cluster, certificates, config).bind().unwrap(); let Addrs { @@ -2289,7 +2289,7 @@ mod tests { } .init("test", &moq_tokio::tls::Connect::default(), false) .unwrap(); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); // No idle deadline, so a stalled connection outlives serving below. let mut web = Web::new(auth, cluster, certificates, config) @@ -2391,7 +2391,7 @@ mod tests { } .init("test", &moq_tokio::tls::Connect::default(), false) .unwrap(); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); let web = Web::new(auth, cluster, certificates, config).bind().unwrap(); let http = web.addrs().http.unwrap(); @@ -2430,7 +2430,7 @@ mod tests { for (flag, set) in cases { let mut config = Config::default(); set(&mut config.https); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let web = Web::new(crate::auth::Auth::refuse("test"), cluster, certificates.clone(), config); let err = match web.bind() { Ok(_) => panic!("{flag} was ignored"), @@ -2451,7 +2451,7 @@ mod tests { let auth = auth_config .init("test", &moq_tokio::tls::Connect::default(), false) .unwrap(); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let dir = TempDir::new().unwrap(); let (_, cert, _) = make_certs(&dir); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); @@ -2476,7 +2476,7 @@ mod tests { let auth = auth_config .init("test", &moq_tokio::tls::Connect::default(), false) .unwrap(); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let dir = TempDir::new().unwrap(); let (_, cert, _) = make_certs(&dir); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); @@ -2513,7 +2513,7 @@ mod tests { config.http.listen = Some("127.0.0.1:0".parse().unwrap()); // Held and never answered: the decider that never decides. let (auth, _admissions) = crate::auth::Auth::embedded("test"); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let dir = TempDir::new().unwrap(); let (_, cert, _) = make_certs(&dir); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); @@ -2554,7 +2554,7 @@ mod tests { config.http.listen = Some("127.0.0.1:0".parse().unwrap()); let (auth, _admissions) = crate::auth::Auth::embedded("test"); let auth = auth.with_verifications(7); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let dir = TempDir::new().unwrap(); let (_, cert, _) = make_certs(&dir); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); @@ -2607,7 +2607,7 @@ mod tests { let auth = auth_config .init("test", &moq_tokio::tls::Connect::default(), false) .unwrap(); - let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let cluster = cluster::Cluster::spawn(crate::cluster::Options::default()).unwrap(); let dir = TempDir::new().unwrap(); let (_, cert, _) = make_certs(&dir); let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index 4eca08a15ab..06ad9a296d1 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -166,7 +166,7 @@ fn build_auth(url: url::Url) -> moq_relay::auth::Auth { /// Stand up the relay's accept loop on a plain-TCP qmux listener and return the /// port plus an abort handle. async fn spawn_relay(auth: moq_relay::auth::Auth) -> (u16, tokio::task::JoinHandle<()>) { - let cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster init"); + let cluster = crate::spawn_cluster(cluster::Options::default()); spawn_relay_with(auth, cluster).await } @@ -206,7 +206,7 @@ async fn spawn_relay_with( /// Stand up the relay's axum web stack with WebSocket enabled and return the /// port plus an abort handle. async fn spawn_ws_relay(auth: moq_relay::auth::Auth) -> (u16, tokio::task::JoinHandle<()>) { - let cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster init"); + let cluster = crate::spawn_cluster(cluster::Options::default()); spawn_ws_relay_with(auth, cluster).await } @@ -363,7 +363,7 @@ async fn spawn_quic_relay( auth: moq_relay::auth::Auth, root: Option, ) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { - let cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster init"); + let cluster = crate::spawn_cluster(cluster::Options::default()); spawn_quic_relay_with(auth, root, cluster).await } @@ -596,7 +596,7 @@ async fn a_moved_tier_retags_the_live_session() { let mut labeled = grant(Duration::from_secs(3600)); labeled.label = Some("project/key/1".into()); let script = Script::new(labeled); - let mut cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster init"); + let mut cluster = crate::spawn_cluster(cluster::Options::default()); cluster.stats = stats::Registry::new(stats::Config::new()); let registry = cluster.stats.clone(); let (port, relay) = spawn_relay_with(build_auth(script.spawn().await), cluster).await; @@ -1311,7 +1311,7 @@ async fn mtls_route_source(mtls_peer: bool) -> moq_net::origin::Source { let server = moq_auth::serve::Server::new(policy).unwrap(); tokio::spawn(async move { server.serve(listener).await }); - let cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster init"); + let cluster = crate::spawn_cluster(cluster::Options::default()); let (addr, relay) = spawn_quic_relay_with(build_auth(auth_url), Some(root), cluster.clone()).await; let mut config = moq_tokio::connect::Config::default(); @@ -1849,7 +1849,7 @@ async fn relay_request_limits_distinguish_clients_and_peers() { let mut options = cluster::Options::default(); options.session.announces = 1; options.peer.announces = 2; - let cluster = cluster::Cluster::new(options).unwrap(); + let cluster = crate::spawn_cluster(options); let consumer = cluster.origin.consume(); let mut observed = consumer.announced(); let (port, relay) = match scheme { diff --git a/rs/moq-relay/tests/integration/cluster_close.rs b/rs/moq-relay/tests/integration/cluster_close.rs index 333b9ad6452..290d07f752b 100644 --- a/rs/moq-relay/tests/integration/cluster_close.rs +++ b/rs/moq-relay/tests/integration/cluster_close.rs @@ -69,9 +69,7 @@ async fn closing_a_cluster_closes_its_static_dials() { let runtime = dialer_runtime(); let cluster = runtime .spawn(async move { - let cluster = moq_relay::cluster::Cluster::new(options) - .expect("cluster init") - .with_connect(connect, quic); + let cluster = crate::spawn_cluster(options).with_connect(connect, quic); let started = cluster.clone().start().await.expect("cluster start"); tokio::spawn(started.run()); cluster @@ -101,9 +99,7 @@ async fn closing_a_cluster_ends_its_websocket_dials() { let mut options = moq_relay::cluster::Options::new(Default::default()); options.config.connect = vec![moq_relay::cluster::Peer::new(format!("ws://127.0.0.1:{port}/"))]; let (connect, quic) = dial_template(); - let cluster = moq_relay::cluster::Cluster::new(options) - .expect("cluster init") - .with_connect(connect, quic); + let cluster = crate::spawn_cluster(options).with_connect(connect, quic); let started = cluster.clone().start().await.expect("cluster start"); let running = tokio::spawn(started.run()); diff --git a/rs/moq-relay/tests/integration/cluster_drain.rs b/rs/moq-relay/tests/integration/cluster_drain.rs index 372f00cb8a7..1776c40515d 100644 --- a/rs/moq-relay/tests/integration/cluster_drain.rs +++ b/rs/moq-relay/tests/integration/cluster_drain.rs @@ -125,9 +125,8 @@ async fn relay(name: &str) -> (Relay, SocketAddr, moq_net::origin::Publication) async fn peer(name: &str, forwarder: &Forwarder) -> (cluster::Cluster, moq_net::origin::Publication) { let mut config = cluster::Config::default(); config.connect = vec![Peer::new(format!("tcp://127.0.0.1:{}/", forwarder.port))]; - let cluster = cluster::Cluster::new(cluster::Options::new(config)) - .expect("cluster init") - .with_connect(Default::default(), Default::default()); + let cluster = + crate::spawn_cluster(cluster::Options::new(config)).with_connect(Default::default(), Default::default()); let broadcast = cluster.origin.create_broadcast(name).expect("create broadcast"); broadcast.announce(Default::default()).expect("announce"); let started = cluster.clone().start().await.expect("cluster start"); diff --git a/rs/moq-relay/tests/integration/cluster_idle.rs b/rs/moq-relay/tests/integration/cluster_idle.rs index f1fa012016e..9841f4a75c4 100644 --- a/rs/moq-relay/tests/integration/cluster_idle.rs +++ b/rs/moq-relay/tests/integration/cluster_idle.rs @@ -107,9 +107,7 @@ async fn silent_peer_detected() { // The keep-alive derives from the idle timeout, so only the silenced peer idles out. let mut quic = moq_tokio::quic::Config::default(); quic.idle_timeout = Some((IDLE).into()); - let cluster = cluster::Cluster::new(cluster::Options::new(config)) - .expect("cluster init") - .with_connect(connect, quic); + let cluster = crate::spawn_cluster(cluster::Options::new(config)).with_connect(connect, quic); let mut announced = cluster.origin.consume().announced(); let run = tokio::spawn(cluster.clone().start().await.expect("cluster start").run()); diff --git a/rs/moq-relay/tests/integration/goaway_cluster.rs b/rs/moq-relay/tests/integration/goaway_cluster.rs index a91dc438658..993c9e2d30f 100644 --- a/rs/moq-relay/tests/integration/goaway_cluster.rs +++ b/rs/moq-relay/tests/integration/goaway_cluster.rs @@ -163,9 +163,8 @@ async fn cluster_continues_a_group_split_by_goaway_inner() { let mut cluster_config = cluster::Config::default(); cluster_config.connect = vec![Peer::new(format!("tcp://127.0.0.1:{port_a}/"))]; - let cluster = cluster::Cluster::new(cluster::Options::new(cluster_config)) - .expect("cluster init") - .with_connect(client_config, Default::default()); + let cluster = + crate::spawn_cluster(cluster::Options::new(cluster_config)).with_connect(client_config, Default::default()); let started = cluster.clone().start().await.expect("cluster start"); let cluster_run = tokio::spawn(started.run()); @@ -304,9 +303,8 @@ async fn cluster_migrates_on_upstream_goaway_inner() { let mut cluster_config = cluster::Config::default(); cluster_config.connect = vec![Peer::new(format!("tcp://127.0.0.1:{port_a}/"))]; - let cluster = cluster::Cluster::new(cluster::Options::new(cluster_config)) - .expect("cluster init") - .with_connect(client_config, Default::default()); + let cluster = + crate::spawn_cluster(cluster::Options::new(cluster_config)).with_connect(client_config, Default::default()); let started = cluster.clone().start().await.expect("cluster start"); let cluster_run = tokio::spawn(started.run()); @@ -426,9 +424,8 @@ async fn spawn_relay_with_upstream( // Only lite-07 carries the epoch that lets a failover resume the subscription. client_config.version = vec![LITE_07.parse().unwrap()]; - let cluster = cluster::Cluster::new(cluster::Options::new(cluster_config)) - .expect("cluster init") - .with_connect(client_config, Default::default()); + let cluster = + crate::spawn_cluster(cluster::Options::new(cluster_config)).with_connect(client_config, Default::default()); let started = cluster.clone().start().await.expect("cluster start"); let handle = tokio::spawn(async move { diff --git a/rs/moq-relay/tests/integration/hidden_cluster.rs b/rs/moq-relay/tests/integration/hidden_cluster.rs index de66627b93d..91e6ef17893 100644 --- a/rs/moq-relay/tests/integration/hidden_cluster.rs +++ b/rs/moq-relay/tests/integration/hidden_cluster.rs @@ -65,9 +65,8 @@ async fn old_peer_keeps_hidden_paths(version: moq_net::Version) { client_config.tls.insecure = Some(true); let mut cluster_config = cluster::Config::default(); cluster_config.connect = vec![Peer::new(format!("tcp://127.0.0.1:{port}/"))]; - let cluster = cluster::Cluster::new(cluster::Options::new(cluster_config)) - .expect("cluster init") - .with_connect(client_config, Default::default()); + let cluster = + crate::spawn_cluster(cluster::Options::new(cluster_config)).with_connect(client_config, Default::default()); let node = cluster.origin.create_broadcast(".hidden/test").expect("create hidden"); node.announce(Default::default()).expect("announce hidden"); let cluster_run = tokio::spawn(cluster.clone().start().await.expect("cluster start").run()); diff --git a/rs/moq-relay/tests/integration/lan_mesh.rs b/rs/moq-relay/tests/integration/lan_mesh.rs index c11707d6829..bf792e127b2 100644 --- a/rs/moq-relay/tests/integration/lan_mesh.rs +++ b/rs/moq-relay/tests/integration/lan_mesh.rs @@ -49,8 +49,7 @@ async fn moq_import_cluster_lan_beside_a_relay() { config.lan = lan; config.node = Some(format!("https://127.0.0.1:{port}/")); - let cluster = cluster::Cluster::new(cluster::Options::new(config)) - .expect("cluster") + let cluster = crate::spawn_cluster(cluster::Options::new(config)) .with_connect(connect, Default::default()) .with_advertise(cluster::LanAdvertise::new(port).with_fingerprint(fingerprint)); diff --git a/rs/moq-relay/tests/integration/main.rs b/rs/moq-relay/tests/integration/main.rs index 757a5f02814..c5d6607ff1b 100644 --- a/rs/moq-relay/tests/integration/main.rs +++ b/rs/moq-relay/tests/integration/main.rs @@ -37,3 +37,15 @@ mod stack; mod tenant_matrix; mod upstream_links; mod wire_stats; + +/// Build a cluster and spawn its driver, the way an embedding edge runs one. A +/// failed driver panics its task. +fn spawn_cluster(options: moq_relay::cluster::Options) -> moq_relay::cluster::Cluster { + let (cluster, driver) = moq_relay::cluster::Cluster::new(options).expect("cluster init"); + tokio::spawn(async move { + if let Err(err) = driver.run().await { + panic!("cluster driver failed: {err:#}"); + } + }); + cluster +} diff --git a/rs/moq-relay/tests/integration/memory.rs b/rs/moq-relay/tests/integration/memory.rs index 4ff69f4c170..1ec83a7e86f 100644 --- a/rs/moq-relay/tests/integration/memory.rs +++ b/rs/moq-relay/tests/integration/memory.rs @@ -65,5 +65,5 @@ pub fn public(public: &[&str], publish: &[&str], subscribe: &[&str]) -> auth::Au /// A standalone cluster with default options. #[cfg(feature = "test-support")] pub fn cluster() -> cluster::Cluster { - cluster::Cluster::new(cluster::Options::default()).expect("cluster init") + crate::spawn_cluster(cluster::Options::default()) } diff --git a/rs/moq-relay/tests/integration/peer_limits.rs b/rs/moq-relay/tests/integration/peer_limits.rs index 700ffbfb4b2..3bf7f4c4f4d 100644 --- a/rs/moq-relay/tests/integration/peer_limits.rs +++ b/rs/moq-relay/tests/integration/peer_limits.rs @@ -264,9 +264,7 @@ async fn static_dial( let mut options = moq_relay::cluster::Options::new(Default::default()); options.config.connect = vec![moq_relay::cluster::Peer::new(format!("moql://127.0.0.1:{port}/"))]; options.peer.quic.max_streams = std::num::NonZeroU64::new(4 * PROBE as u64).unwrap(); - let cluster = moq_relay::cluster::Cluster::new(options) - .expect("cluster init") - .with_connect(connect, quic); + let cluster = crate::spawn_cluster(options).with_connect(connect, quic); let running = tokio::spawn(cluster.start().await.expect("cluster start").run()); let conn = kio::clock::timeout(TIMEOUT, server.accept()) .await diff --git a/rs/moq-relay/tests/integration/public_rules.rs b/rs/moq-relay/tests/integration/public_rules.rs index 9add0d508dc..7b58ac99172 100644 --- a/rs/moq-relay/tests/integration/public_rules.rs +++ b/rs/moq-relay/tests/integration/public_rules.rs @@ -38,7 +38,7 @@ async fn spawn_relay(auth: auth::Config) -> (u16, tokio::task::JoinHandle<()>) { let auth = auth .init("test", &moq_tokio::tls::Connect::default(), false) .expect("auth init"); - let cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster init"); + let cluster = crate::spawn_cluster(cluster::Options::default()); // Only the certificate handle is used; stream listeners bind lazily. let mut server_config = moq_tokio::listen::Config::default(); diff --git a/rs/moq-relay/tests/integration/session_revalidate.rs b/rs/moq-relay/tests/integration/session_revalidate.rs index 69e4b240eb1..a171ad624ac 100644 --- a/rs/moq-relay/tests/integration/session_revalidate.rs +++ b/rs/moq-relay/tests/integration/session_revalidate.rs @@ -143,7 +143,7 @@ impl Fixture { }); let (url, handle) = if websocket { - let cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster init"); + let cluster = crate::spawn_cluster(cluster::Options::default()); let mut server_config = moq_tokio::listen::Config::default(); server_config.bind = Some("[::]:0".parse().unwrap()); server_config.tls.generate = vec!["localhost".into()]; @@ -169,7 +169,7 @@ impl Fixture { let server = config.init(Default::default()).expect("server init"); let mut server = server.listen().await.expect("listen"); let port = server.tcp_local_addr().expect("TCP listener is configured").port(); - let cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster init"); + let cluster = crate::spawn_cluster(cluster::Options::default()); let sessions = sessions.clone(); let handle = tokio::spawn(async move { let mut id = 0; diff --git a/rs/moq-relay/tests/integration/smoke.rs b/rs/moq-relay/tests/integration/smoke.rs index 7066499b3ae..76527c4e647 100644 --- a/rs/moq-relay/tests/integration/smoke.rs +++ b/rs/moq-relay/tests/integration/smoke.rs @@ -49,7 +49,7 @@ async fn build_web(ws: bool) -> web::Web { /// A standalone cluster with default options. fn new_cluster() -> cluster::Cluster { - cluster::Cluster::new(cluster::Options::default()).expect("cluster init") + crate::spawn_cluster(cluster::Options::default()) } /// [`build_web`] for a test that configures the listeners or the cluster itself (e.g. HTTPS). @@ -569,7 +569,7 @@ async fn spawn_accept_relay( .init("test", &moq_tokio::tls::Connect::default(), false) .expect("auth init"); - let cluster = cluster::Cluster::new(cluster::Options::default()).expect("cluster init"); + let cluster = crate::spawn_cluster(cluster::Options::default()); let mut server = server.listen().await.expect("listen"); let quic = server.local_addr().ok(); let tcp = server.tcp_local_addr(); diff --git a/rs/moq-relay/tests/integration/upstream_links.rs b/rs/moq-relay/tests/integration/upstream_links.rs index 6d754c20578..841e91388fc 100644 --- a/rs/moq-relay/tests/integration/upstream_links.rs +++ b/rs/moq-relay/tests/integration/upstream_links.rs @@ -102,9 +102,7 @@ async fn node(id: u64, connect: Vec) -> Node { let mut config = cluster::Config::default(); config.id = Some(id); config.connect = connect; - let cluster = cluster::Cluster::new(cluster::Options::new(config)) - .expect("cluster init") - .with_connect(client, Default::default()); + let cluster = crate::spawn_cluster(cluster::Options::new(config)).with_connect(client, Default::default()); let (auth, mut admissions) = auth::Auth::embedded(format!("node-{id}")); tasks.spawn(async move { From 89d2aefbf63c0d609b8b14b00b691a2266d453ad Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 11:53:44 -0800 Subject: [PATCH 3/3] docs(relay): the cache governor starts when the cluster's driver is polled Cache and Relay::load still described the governor as spawned at construction; it now lives in the Driver that Cluster::new returns and runs only while Relay::run polls it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/cache.rs | 16 +++++++++------- rs/moq-relay/src/relay.rs | 8 +++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/rs/moq-relay/src/cache.rs b/rs/moq-relay/src/cache.rs index bdd92ac0e41..bfbf68907f4 100644 --- a/rs/moq-relay/src/cache.rs +++ b/rs/moq-relay/src/cache.rs @@ -68,14 +68,15 @@ pub struct Config { /// The relay's resolved cache settings: the shared byte-budget pool plus the /// retention ceiling applied to every track. -#[non_exhaustive] /// -/// The headroom governor, when configured, starts once this is handed to -/// [`Cluster::new`](crate::cluster::Cluster::new), which runs it as a critical task, and -/// is then owned by [`Self::pool`]: it holds only a [`cache::PoolWeak`] and stops on +/// The headroom governor, when configured, moves into the +/// [`Driver`](crate::cluster::Driver) that [`Cluster::new`](crate::cluster::Cluster::new) +/// returns once this is handed to it, and starts sampling only while that driver is +/// polled ([`Relay::run`](crate::Relay::run) does). It is then owned by [`Self::pool`]: it holds only a [`cache::PoolWeak`] and stops on /// its next tick once every [`cache::Pool`] clone has dropped. Holding a clone of /// [`Self::pool`] past the relay keeps the governor running, deliberately: as long as /// anything can still cache into the budget, resizing it is still the right thing to do. +#[non_exhaustive] pub struct Cache { /// The shared pool every session's groups register with: the byte budget plus /// the wall-clock LRU window that reclaims idle groups. @@ -96,9 +97,10 @@ impl Config { /// Resolve the size knobs into a shared [`cache::Pool`] and age ceiling, /// plus the headroom governor when configured. /// - /// The governor starts with the cluster the [`Cache`] is attached to, so a - /// caller that drops it without attaching it (a setup step failing after this - /// point, say) leaves nothing sampling memory behind. + /// The governor starts only once the [`Cache`] is attached to a cluster and + /// that cluster's [`Driver`](crate::cluster::Driver) is polled, so a caller that + /// drops it first (a setup step failing after this point, say) leaves nothing + /// sampling memory behind. pub fn init(&self) -> anyhow::Result { let capacity = self.capacity.as_deref().map(parse_limit).transpose()?; let duration = self.duration.map(Into::into); diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 3dca346aaa8..2da85e07b91 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -134,9 +134,11 @@ impl Relay { /// auth, and build the cluster with its cache and stats attached. /// /// This performs the side effects of starting up (binding every socket, - /// reading key material, spawning the cache governor), so a returned `Relay` - /// reports its ephemeral ports and a taken port fails here; no session is - /// admitted until [`Self::run`] drives it. + /// reading key material), so a returned `Relay` reports its ephemeral ports and + /// a taken port fails here. Nothing runs yet: the cluster's + /// [`Driver`](crate::cluster::Driver), which holds the origin's lifecycle work and + /// the cache governor, is only stored, and no session is admitted until + /// [`Self::run`] drives both. pub async fn load(config: Config) -> anyhow::Result { // Keep the large debug state machine off the embedder's main-thread stack. Box::pin(Self::load_inner(config)).await