Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion quest/m1/poll-first/no-spawn/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ binding boundary.

## Required

- [Cluster polls its own work](/quest/m1/poll-first/no-spawn/cluster.md) - `Cluster::new` returns a `Driver` the relay polls, and the dial loop runs as `kio::Tasks` children
- [Admissions and stats are polled](/quest/m1/poll-first/no-spawn/admissions.md) - HTTP admissions run inside their waiter, and the stats publisher runs in the cluster's driver
- [moq-ffi spawns only its root](/quest/m1/poll-first/no-spawn/ffi.md) - one runtime-owned driver task holds every `Background` as `kio::Tasks` children
- [The relay's in-memory tests run on kio's simulator](/quest/m1/poll-first/no-spawn/sim-tests.md) - `rs/moq-relay/tests/integration/connection.rs` runs on the simulator instead of paused tokio
Expand Down
16 changes: 7 additions & 9 deletions quest/m1/poll-first/no-spawn/admissions.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,12 @@ panic then ends the session being admitted, the same scope the per-admission
polled by `Relay::run`, which brings back the long-lived decider task that
#7360 removed.

The publisher joins `Driver`, the type
[Cluster polls its own work](/quest/m1/poll-first/no-spawn/cluster.md)
introduces, so a publisher panic still ends the relay. `Publisher::run` is an
`async fn`; giving `Publisher` a `poll` is optional and only worth it if
`Driver` needs it.
The publisher joins the cluster's `Driver` (`rs/moq-relay/src/cluster.rs`,
from [moq-dev/moq#7626](https://github.com/moq-dev/moq/pull/7626)), so a
publisher panic still ends the relay. `Cluster::new` has already handed the
driver out by the time `Cluster::with_stats` runs, so the publisher either
reaches it through a queue the driver polls or moves into `Options` for
`Cluster::new` to add. `Publisher::run` is an `async fn`; giving `Publisher` a
`poll` is optional and only worth it if `Driver` needs it.

Public API: `Cluster::with_stats` no longer needs a tokio runtime. Wire: none.

## Required

- [Cluster polls its own work](/quest/m1/poll-first/no-spawn/cluster.md) - provides the `Driver` the publisher joins
52 changes: 0 additions & 52 deletions quest/m1/poll-first/no-spawn/cluster.md

This file was deleted.

1 change: 0 additions & 1 deletion quest/m1/poll-first/no-spawn/sim-tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,4 @@ Public API: none. Wire: none.

## Required

- [Cluster polls its own work](/quest/m1/poll-first/no-spawn/cluster.md) - the cluster's driver and dials stop spawning on tokio
- [Admissions and stats are polled](/quest/m1/poll-first/no-spawn/admissions.md) - admissions stop spawning on tokio
8 changes: 4 additions & 4 deletions rs/moq-cli/src/args.rs
Original file line number Diff line number Diff line change
Expand Up @@ -556,10 +556,10 @@ impl MoqSide {
found
}

/// The cluster this process publishes and subscribes on. Built once; the
/// origin is its origin, whose Hop ID is random per run unless `--cluster-id`
/// names the node.
pub fn cluster(&self) -> anyhow::Result<moq_relay::cluster::Cluster> {
/// The cluster this process publishes and subscribes on, with the driver that
/// must run for as long as it serves. Built once; the origin is its origin, whose
/// Hop ID is random per run unless `--cluster-id` names the node.
pub fn cluster(&self) -> anyhow::Result<(moq_relay::cluster::Cluster, moq_relay::cluster::Driver)> {
moq_relay::cluster::Cluster::new(moq_relay::cluster::Options::new(self.cluster.clone()))
}

Expand Down
26 changes: 17 additions & 9 deletions rs/moq-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ async fn spawn_moq(
};

let (started, shutdown) = spawn_server(tasks, moq, &cluster, net, directions).await?;
// Even standalone: it is what reports a panic in the origin driver.
// Even standalone: it is what reports a panic in a critical task.
tasks.spawn(async move { started.run().await });

let mut bandwidth = moq_net::bandwidth::Allocator::unlimited();
Expand Down Expand Up @@ -557,9 +557,10 @@ async fn run_play(moq: MoqSide, args: play::Args, net: Net) -> anyhow::Result<()
// Before anything dials: a codec we can't decode is a blank window otherwise.
args.validate()?;

let cluster = moq.cluster()?;
let (cluster, driver) = moq.cluster()?;
let name = moq.broadcast.clone().unwrap_or_default();
let mut tasks: JoinSet<anyhow::Result<()>> = JoinSet::new();
tasks.spawn(driver.run());

let directions = Directions {
consume: true,
Expand All @@ -584,8 +585,9 @@ async fn run_play(moq: MoqSide, args: play::Args, net: Net) -> anyhow::Result<()
/// and the first endpoint to finish ends the process. A shutdown signal keeps
/// accepted sessions serving through the configured drain window.
async fn run_stages(moq: MoqSide, stages: Vec<Command>, net: Net) -> anyhow::Result<()> {
let cluster = moq.cluster()?;
let (cluster, driver) = moq.cluster()?;
let mut tasks: JoinSet<anyhow::Result<()>> = JoinSet::new();
tasks.spawn(driver.run());
// The stdin/capture pipelines run on this thread instead of the JoinSet: the
// platform capture stream is not Send, so their futures cannot be spawned.
let mut locals: Vec<Publish> = Vec::new();
Expand Down Expand Up @@ -1298,7 +1300,8 @@ mod tests {
.unwrap();
let listener = cli.moq.server_config().init_streams().unwrap().listen().await.unwrap();
let addr = listener.tcp_local_addr().unwrap();
let cluster = moq_relay::cluster::Cluster::new(Default::default()).unwrap();
let (cluster, driver) = moq_relay::cluster::Cluster::new(Default::default()).unwrap();
tokio::spawn(driver.run());
let broadcast = cluster.origin.create_broadcast("probe").unwrap();
broadcast.announce(Default::default()).unwrap();
let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI);
Expand Down Expand Up @@ -1565,7 +1568,8 @@ mod tests {
"ts",
])
.expect("parse");
let cluster = invocation.moq.cluster().expect("create cluster");
let (cluster, driver) = invocation.moq.cluster().expect("create cluster");
tokio::spawn(driver.run());
let net = Net {
quic: invocation.moq.quic.clone(),
#[cfg(feature = "iroh")]
Expand All @@ -1589,7 +1593,8 @@ mod tests {
async fn a_listener_without_auth_stops_startup() {
let invocation = Invocation::try_parse_from(["moq", "--listen-tcp-bind", "127.0.0.1:0", "import", "ts"])
.expect("parse TCP-only invocation");
let cluster = invocation.moq.cluster().expect("create cluster");
let (cluster, driver) = invocation.moq.cluster().expect("create cluster");
tokio::spawn(driver.run());
let net = Net {
quic: invocation.moq.quic.clone(),
#[cfg(feature = "iroh")]
Expand Down Expand Up @@ -1629,7 +1634,8 @@ mod tests {
"ts",
])
.expect("parse TCP-only invocation");
let cluster = invocation.moq.cluster().expect("create cluster");
let (cluster, driver) = invocation.moq.cluster().expect("create cluster");
tokio::spawn(driver.run());
let net = Net {
quic: invocation.moq.quic.clone(),
#[cfg(feature = "iroh")]
Expand Down Expand Up @@ -1670,7 +1676,8 @@ mod tests {
Invocation::try_parse_from(["moq", "--cluster-api-url", "https://api.example/peers", "import", "ts"])
.expect("parse");
assert!(invocation.moq.validate().is_ok());
let cluster = invocation.moq.cluster().expect("cluster");
let (cluster, driver) = invocation.moq.cluster().expect("cluster");
tokio::spawn(driver.run());
let err = cluster
.clone()
.start()
Expand Down Expand Up @@ -1707,7 +1714,8 @@ mod tests {
])
.expect("parse");
invocation.moq.validate().expect("validate");
let cluster = invocation.moq.cluster().expect("cluster");
let (cluster, driver) = invocation.moq.cluster().expect("cluster");
tokio::spawn(driver.run());
let net = Net {
quic: invocation.moq.quic.clone(),
#[cfg(feature = "iroh")]
Expand Down
1 change: 1 addition & 0 deletions rs/moq-relay/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed

- [**breaking**] Auth token roles hold path patterns directly, matching release; a capped role is enforced or refused, never dropped.
- [**breaking**] `Cluster::new` returns `(Cluster, Driver)` instead of spawning the origin driver and cache governor, and `Started::run` polls every peer dial, the `--cluster-api-url` reconciler, and LAN discovery on its own task ([#7626](https://github.com/moq-dev/moq/pull/7626)).

- Auth requests distinguish `webtransport` from native `quic`. Upgrade auth servers before relays: older servers reject the new value. Rust and JavaScript auth parsers now map future transport names to `unknown`.
- `listen.timeout` bounds auth admission as part of the handshake, replacing the fixed ten-second admission timeout, so an auth server answering `connect` late is admitted within the deadline and `"0"` waits through admission too. HTTP reads bound admission by the web listener's timeout. `Auth::admit` and `Cluster::admit` no longer time out on their own; the caller owns the deadline. In-band token verification keeps its ten-second bound.
Expand Down
12 changes: 12 additions & 0 deletions rs/moq-relay/MIGRATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,18 @@ Rolling a cluster across a release also follows the [upgrade guide](https://moq.

## Unreleased

### `Cluster::new` returns a driver to poll ([#7626](https://github.com/moq-dev/moq/pull/7626))

`Cluster::new` no longer spawns the origin's lifecycle work or the cache governor; it returns `(Cluster, Driver)`, and nothing runs until the driver is polled.
`Relay` polls it for you. An embedder building a `Cluster` itself keeps the driver for as long as the cluster serves, and polls it with `Driver::poll` or runs it with `Driver::run`, for example `tokio::spawn(driver.run())`:

```rust
let (cluster, driver) = moq_relay::cluster::Cluster::new(options)?;
tokio::spawn(driver.run());
```

Dropping the driver tears the origin down. `Driver::run` returns the first panic among its tasks, such as `origin driver panicked: ...`, which `Started::run` used to report.

### `tls://` links verify client certificates ([#7358](https://github.com/moq-dev/moq/pull/7358))

The `listen.tcp.tls` listener applies `listen.tls.root` and pinned `tls.peers` as the QUIC listener does, so a stream-only relay with that listener no longer refuses them.
Expand Down
6 changes: 4 additions & 2 deletions rs/moq-relay/benches/auth.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ impl Fixture {
config.tcp.bind = Some("127.0.0.1:0".parse().unwrap());
let mut server = config.init(Default::default()).unwrap().listen().await.unwrap();
let port = server.tcp_local_addr().unwrap().port();
let cluster = cluster::Cluster::new(Default::default()).unwrap();
let (cluster, driver) = cluster::Cluster::new(Default::default()).unwrap();
tokio::spawn(driver.run());
let listener = tokio::spawn(async move {
let mut serving = tokio::task::JoinSet::new();
while let Ok(request) = server.accept().await {
Expand Down Expand Up @@ -170,7 +171,8 @@ impl Queue {
config.tcp.bind = Some("127.0.0.1:0".parse().unwrap());
let mut server = config.init(Default::default()).unwrap().listen().await.unwrap();
let port = server.tcp_local_addr().unwrap().port();
let cluster = cluster::Cluster::new(Default::default()).unwrap();
let (cluster, driver) = cluster::Cluster::new(Default::default()).unwrap();
tokio::spawn(driver.run());
let listener = tokio::spawn(async move {
let mut serving = tokio::task::JoinSet::new();
while let Ok(request) = server.accept().await {
Expand Down
3 changes: 2 additions & 1 deletion rs/moq-relay/benches/http_auth.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ impl Fixture {
let certificates = moq_tokio::tls::Certificates::from_pem(pem.as_bytes()).unwrap();
let mut config = web::Config::default();
config.http.listen = Some("127.0.0.1:0".parse().unwrap());
let cluster = cluster::Cluster::new(Default::default()).unwrap();
let (cluster, driver) = cluster::Cluster::new(Default::default()).unwrap();
tokio::spawn(driver.run());
// No admission timeout, so held requests stay waiting however long setup and the
// measurement take.
let web = web::Web::new(auth, cluster, certificates, config)
Expand Down
3 changes: 2 additions & 1 deletion rs/moq-relay/benches/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ impl Backend for Relay {
options.config.id = Some(id);
// The shared sweeps include more announcements than a default ordinary client.
options.session.announces = options.peer.announces;
let cluster = cluster::Cluster::new(options.with_cache(cache)).unwrap();
let (cluster, driver) = cluster::Cluster::new(options.with_cache(cache)).unwrap();
tokio::spawn(driver.run());
let mut stats = stats::Config::default();
stats.enabled = true;
stats.node = Some(id.to_string());
Expand Down
14 changes: 7 additions & 7 deletions rs/moq-relay/src/auth/admission_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ fn patterns(texts: &[&str]) -> Patterns {

#[tokio::test]
async fn gateway_admission_scopes_origins_and_revoke_stops_work() {
let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap();
let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap();
let (auth, mut admissions) = auth::Auth::embedded("edge");
let mut grant = Grant::new(patterns(&["media/**"]), patterns(&["watch/**"]));
grant.root = Some("canonical".into());
Expand Down Expand Up @@ -42,7 +42,7 @@ async fn gateway_admission_scopes_origins_and_revoke_stops_work() {

#[tokio::test]
async fn a_gateway_role_without_scope_is_forbidden() {
let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap();
let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap();
let (auth, mut admissions) = auth::Auth::embedded("edge");
tokio::spawn(async move {
let admission = admissions.next().await.unwrap();
Expand All @@ -61,7 +61,7 @@ async fn a_gateway_role_without_scope_is_forbidden() {

#[tokio::test]
async fn completed_gateway_work_closes_its_lease() {
let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap();
let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap();
let (auth, mut admissions) = auth::Auth::embedded("edge");
let decider = tokio::spawn(async move {
let admission = admissions.next().await.unwrap();
Expand All @@ -80,7 +80,7 @@ async fn completed_gateway_work_closes_its_lease() {

#[tokio::test]
async fn a_narrowed_gateway_grant_stops_work() {
let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap();
let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap();
let (auth, mut admissions) = auth::Auth::embedded("edge");
let decider = tokio::spawn(async move {
let admission = admissions.next().await.unwrap();
Expand All @@ -103,7 +103,7 @@ async fn a_narrowed_gateway_grant_stops_work() {
async fn a_moved_gateway_tier_retags_its_stats() {
use moq_net::stats;

let mut cluster = cluster::Cluster::new(cluster::Options::default()).unwrap();
let mut cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap();
cluster.stats = stats::Registry::new(stats::Config::new());
let registry = cluster.stats.clone();
let active = |tier: &'static str| {
Expand Down Expand Up @@ -150,7 +150,7 @@ async fn a_moved_gateway_tier_retags_its_stats() {

#[tokio::test(start_paused = true)]
async fn gateway_admission_refuses_once_in_the_shared_stats_registry() {
let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap();
let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap();
let (producer, publisher) =
moq_stats::Producer::new(moq_stats::produce::Config::new().with_origin(cluster.origin.clone()));
let registry = producer.registry().clone();
Expand Down Expand Up @@ -184,7 +184,7 @@ async fn gateway_admission_refuses_once_in_the_shared_stats_registry() {
#[tokio::test]
async fn embedded_expired_refusals_keep_verified_root_and_tier() {
tokio::time::pause();
let cluster = cluster::Cluster::new(cluster::Options::default()).unwrap();
let cluster = cluster::Cluster::spawn(cluster::Options::default()).unwrap();
let config = crate::stats::Config {
enabled: true,
..Default::default()
Expand Down
Loading
Loading