Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .config/nextest.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ default-filter = "not kind(bench)"
# rings; integration workers keep production geometry. sh/rs/nextest.sh raises
# hosted CI's locked-memory limit and uses the host's hard limit locally.
# Several invocations still share that user-wide budget; see rs/moq-uring/README.md.
# A relay test joins by naming `uring` as a whole module or word, not "during".
[[profile.default.overrides]]
filter = "package(moq-uring) | (binary_id(moq-relay::integration) & (test(/^runtime_uring::/) | test(/^embed::.*uring/)))"
filter = "package(moq-uring) | (package(moq-relay) & test(/(^|[:_])uring([:_]|$)/))"
test-group = "io-uring"

# The vivid tests share one capture node, and a second process configuring its
Expand Down
1 change: 0 additions & 1 deletion quest/m0/ci/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,6 @@ The historical samples locate candidates; they are not controlled benchmarks.
- [Check on ready PRs](/quest/m0/ci/pr-check.md) - Check runs again on non-draft main PRs as a signal for `/quest-merge` and the land fixer, never as a gate
- [Land main through GitHub's merge queue](/quest/m0/ci/land.md) - one PR at a time on the self-hosted runner; the host's Codex fixer fixes and re-enqueues every dropped PR
- [A cache error is a miss](/quest/m0/ci/nix-cache-5xx.md) - a transient `nix.moq.dev` 5xx no longer fails `just ci`
- [Every io_uring test in the io-uring group](/quest/m0/ci/io-uring-test-group.md) - the parallel tests of one `just ci` stop exhausting the shared `RLIMIT_MEMLOCK`, and a too-low limit fails clearly
- [Cheap checks skip the queue](/quest/m0/ci/gate-quick.md) - condition: quest ships `quest gate --quick`; then cheap modules wait at most a minute
- [Gate each diff in under five minutes](/quest/m0/ci/five-minutes.md) - `just check` and `just test` each finish within a worker's 5-minute prompt cache, admission wait included
- [Measure the parallel frontend](/quest/m0/ci/nightly-levers.md) - a pinned nightly's parallel frontend, adopted only if the numbers beat its risks; Cranelift is ruled out
Expand Down
21 changes: 0 additions & 21 deletions quest/m0/ci/io-uring-test-group.md

This file was deleted.

62 changes: 48 additions & 14 deletions rs/moq-relay/src/relay.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,33 @@ use crate::{Config, Connection, auth, auth::Admissions, cluster, internal, shutd
/// A handle that waits until a relay has finished startup.
#[derive(Clone)]
pub struct Ready {
receiver: tokio::sync::watch::Receiver<bool>,
receiver: tokio::sync::watch::Receiver<Startup>,
}

impl Ready {
/// Wait until `Relay::run` starts serving, or fail if it exits first.
/// Wait until `Relay::run` starts serving, or fail with its error if it exits first.
pub async fn wait(mut self) -> anyhow::Result<()> {
loop {
if *self.receiver.borrow() {
return Ok(());
}
self.receiver
.changed()
.await
.context("relay stopped before becoming ready")?;
let startup = self
.receiver
.wait_for(|startup| !matches!(startup, Startup::Starting))
.await
.context("relay stopped before becoming ready")?;
if let Startup::Failed(err) = &*startup {
return Err(anyhow::anyhow!("{err}").context("relay stopped before becoming ready"));
}
Ok(())
}
}

/// How far [`Relay::run`] got through startup.
#[derive(Clone)]
enum Startup {
Starting,
Serving,
// The error's whole chain, since `run` returns the error itself.
Failed(std::sync::Arc<str>),
}

/// A fully assembled relay: the owner of every listener, worker group, and
/// shutdown join.
///
Expand All @@ -69,7 +78,7 @@ impl Ready {
/// running.await??;
/// ```
pub struct Relay {
ready: tokio::sync::watch::Sender<bool>,
ready: tokio::sync::watch::Sender<Startup>,
// Keep the inspection snapshot out of every future that moves a Relay.
config: Box<Config>,
server: moq_tokio::Listener,
Expand Down Expand Up @@ -344,7 +353,7 @@ impl Relay {
let cluster = cluster.with_stats(stats.clone(), publisher);

let sessions = crate::session::Registry::new();
let (ready, _) = tokio::sync::watch::channel(false);
let (ready, _) = tokio::sync::watch::channel(Startup::Starting);
let web = web::Web::new(auth.clone(), cluster.clone(), certificates, config.web)
.with_shutdown(shutdown.clone())
.with_versions(server_versions)
Expand Down Expand Up @@ -595,8 +604,20 @@ impl Relay {
/// This is also the embedding loop. Extra routes go on via [`Self::with_web`]
/// / [`Self::with_internal`] before calling this; cloned handles outlive it.
pub async fn run(self) -> anyhow::Result<()> {
// Held past `run_inner`, so a waiter sees the failure before the close.
let ready = self.ready.clone();
// Serving branches must not inflate the caller's stack frame.
Box::pin(self.run_inner()).await
let result = Box::pin(self.run_inner()).await;
if let Err(err) = &result {
ready.send_if_modified(|startup| {
let starting = matches!(startup, Startup::Starting);
if starting {
*startup = Startup::Failed(format!("{err:#}").into());
}
starting
});
}
result
}

async fn run_inner(self) -> anyhow::Result<()> {
Expand Down Expand Up @@ -691,7 +712,7 @@ impl Relay {

// An embedder that already stopped the relay gets a drain, not a READY.
if shutdown.deadline().is_none() {
ready.send_replace(true);
ready.send_replace(Startup::Serving);

#[cfg(unix)]
// Notify systemd that we're ready after all initialization is complete
Expand Down Expand Up @@ -994,6 +1015,19 @@ mod tests {
assert!(ready.wait().await.is_err(), "reported ready after the stop");
}

/// A waiter learns why startup failed, not only that it ended.
#[tokio::test]
async fn a_startup_failure_reaches_the_waiter() {
let mut config = crate::Config::default();
config.listen.tcp.bind = Some("127.0.0.1:0".parse().unwrap());
let relay = super::Relay::load(config).await.unwrap().with_signals(false);
let ready = relay.ready();
let running = tokio::spawn(relay.run());
let error = format!("{:#}", ready.wait().await.expect_err("no auth refuses startup"));
assert!(error.contains("nobody can authenticate"), "{error}");
running.await.unwrap().expect_err("run fails with the waiter");
}

/// A connection arriving as the last admitted one leaves is either counted
/// before the drain commits to exit, or refused at accept; never admitted
/// into a relay that has already decided to exit under it.
Expand Down
4 changes: 4 additions & 0 deletions rs/moq-relay/src/uring.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1131,7 +1131,9 @@ mod tests {
let port = relay.addr().unwrap().port();
let mut admissions = relay.admissions().unwrap();
let trigger = relay.shutdown_trigger().clone();
let ready = relay.ready();
let running = tokio::spawn(relay.run());
ready.wait().await.expect("relay ready");

let mut client = moq_tokio::connect::Config::default();
client.tls.insecure = Some(true);
Expand Down Expand Up @@ -1189,7 +1191,9 @@ mod tests {
let port = relay.addr().unwrap().port();
let mut admissions = relay.admissions().unwrap();
let trigger = relay.shutdown_trigger().clone();
let ready = relay.ready();
let running = tokio::spawn(relay.run());
ready.wait().await.expect("relay ready");

let mut client = moq_tokio::connect::Config::default();
client.tls.insecure = Some(true);
Expand Down
4 changes: 2 additions & 2 deletions rs/moq-relay/tests/integration/embed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,6 @@ async fn embed_and_stop(mut config: Config) {
// are cloned before `run` consumes the relay.
let origin = relay.cluster().origin.clone();
let trigger = relay.shutdown_trigger().clone();
let ready = relay.ready();
let web = relay
.web()
.routes()
Expand All @@ -178,6 +177,7 @@ async fn embed_and_stop(mut config: Config) {
axum::routing::post(|| async { ([("access-control-allow-origin", "https://trusted.example")], "private") }),
)
.route("/plain-post", axum::routing::post(|| async { "plain" }));
let ready = relay.ready();
let running = tokio::spawn(relay.with_web(web).run());
ready.wait().await.expect("relay ready");
assert!(!running.is_finished(), "the relay stopped while serving");
Expand Down Expand Up @@ -503,9 +503,9 @@ async fn embedded_listener_health_reaches_metrics() {
config.drain_timeout = Some((Duration::ZERO).into());
let relay = Relay::load(config).await.expect("load relay").with_signals(false);
let internal = relay.internal().addr().expect("internal listener bound");
let ready = relay.ready();
let trigger = relay.shutdown_trigger().clone();
let health = moq_tokio::accept::Health::new("embedded");
let ready = relay.ready();
let running = tokio::spawn(relay.with_listeners([health]).run());
ready.wait().await.expect("relay ready");
let body = reqwest::get(format!("http://{internal}/metrics"))
Expand Down
2 changes: 2 additions & 0 deletions rs/moq-relay/tests/integration/peer_limits.rs
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,9 @@ async fn peer_widens_and_viewer_does_not(mut relay: Relay, port: u16) {
}
leases
});
let ready = relay.ready();
let running = tokio::spawn(relay.run());
ready.wait().await.expect("relay ready");

let peer = probe(port, true).await;
assert_eq!(peer.streams, PROBE, "a peer opens past the viewer stream limit");
Expand Down
4 changes: 3 additions & 1 deletion rs/moq-relay/tests/integration/refusals.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ async fn start(config: Config) -> Running {
let quic = relay.quic_addr().expect("quic listener bound");
let http = relay.web_addrs().http.expect("http listener bound");
let internal = relay.internal().addr().expect("internal listener bound");
let ready = relay.ready();
let trigger = relay.shutdown_trigger().clone();
let ready = relay.ready();
let running = tokio::spawn(relay.run());
ready.wait().await.expect("relay ready");
Running {
Expand Down Expand Up @@ -473,7 +473,9 @@ async fn uri_setup_paths(io_uring: bool) {
let addr = relay.quic_addr().unwrap();
let mut admissions = relay.admissions().unwrap();
let trigger = relay.shutdown_trigger().clone();
let ready = relay.ready();
let running = tokio::spawn(relay.run());
ready.wait().await.expect("relay ready");
for version in versions {
for path in [
Some(""),
Expand Down
Loading