diff --git a/.config/nextest.toml b/.config/nextest.toml index 63ba5a810e1..1e7c5d30b59 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -27,8 +27,9 @@ default-filter = "not kind(bench)" # rings; integration workers keep production geometry. sh/rs/nextest.sh raises # hosted CI's locked-memory limit and uses the host's hard limit locally. # Several invocations still share that user-wide budget; see rs/moq-uring/README.md. +# A relay test joins by naming `uring` as a whole module or word, not "during". [[profile.default.overrides]] -filter = "package(moq-uring) | (binary_id(moq-relay::integration) & (test(/^runtime_uring::/) | test(/^embed::.*uring/)))" +filter = "package(moq-uring) | (package(moq-relay) & test(/(^|[:_])uring([:_]|$)/))" test-group = "io-uring" # The vivid tests share one capture node, and a second process configuring its diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 4542d9ac160..2d14465aee6 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -155,7 +155,6 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Check on ready PRs](/quest/m0/ci/pr-check.md) - Check runs again on non-draft main PRs as a signal for `/quest-merge` and the land fixer, never as a gate - [Land main through GitHub's merge queue](/quest/m0/ci/land.md) - one PR at a time on the self-hosted runner; the host's Codex fixer fixes and re-enqueues every dropped PR - [A cache error is a miss](/quest/m0/ci/nix-cache-5xx.md) - a transient `nix.moq.dev` 5xx no longer fails `just ci` -- [Every io_uring test in the io-uring group](/quest/m0/ci/io-uring-test-group.md) - the parallel tests of one `just ci` stop exhausting the shared `RLIMIT_MEMLOCK`, and a too-low limit fails clearly - [Cheap checks skip the queue](/quest/m0/ci/gate-quick.md) - condition: quest ships `quest gate --quick`; then cheap modules wait at most a minute - [Gate each diff in under five minutes](/quest/m0/ci/five-minutes.md) - `just check` and `just test` each finish within a worker's 5-minute prompt cache, admission wait included - [Measure the parallel frontend](/quest/m0/ci/nightly-levers.md) - a pinned nightly's parallel frontend, adopted only if the numbers beat its risks; Cranelift is ruled out diff --git a/quest/m0/ci/io-uring-test-group.md b/quest/m0/ci/io-uring-test-group.md deleted file mode 100644 index 2b3a88b5a08..00000000000 --- a/quest/m0/ci/io-uring-test-group.md +++ /dev/null @@ -1,21 +0,0 @@ -# [XS] Every io_uring test runs in the nextest io-uring group - -## Goal - -Every test that opens an io_uring, including the relay's `wire_stats::uring_*`, `peer_limits::uring_*`, and `refusals::uring_*` integration tests and the `bitrate_caps::uring_*` tests [moq-dev/moq#7535](https://github.com/moq-dev/moq/pull/7535) adds, runs inside the `io-uring` nextest test group, so the tests of one `just ci` running in parallel no longer exhaust the user's shared `RLIMIT_MEMLOCK`. The group caps rings only within one nextest invocation; separate invocations still share the budget, as `rs/moq-uring/README.md` documents, and this quest does not change that. An io_uring test that still hits too low a limit fails with a message naming `RLIMIT_MEMLOCK` and how to raise it, not a generic setup error or a timeout. - -## Plan - -Found by [moq-dev/moq#7535](https://github.com/moq-dev/moq/pull/7535) and [moq-dev/moq#7211](https://github.com/moq-dev/moq/pull/7211): #7535's io_uring tests passed serially but could hit the shared limit in parallel, and #7211's `just ci` failed once in `uring_peer_session_widens_quic_limits` on `RLIMIT_MEMLOCK`. - -Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-afk run): m0, since each such failure fails a whole merge-gate batch. - -- The group's filter in `.config/nextest.toml` names only `runtime_uring::` and `embed::.*uring` in `moq-relay::integration`. Match every io_uring test instead, for example `test(/uring/)` in that binary plus moq-relay's own unit tests that start ring workers, so a new test joins by its name. Check that the inventory of uring tests (`rg 'fn uring_' rs/moq-relay`) all land in the group with `cargo nextest show-config test-groups`, which lists each group's members; plain `cargo nextest list` shows no group. -- moq-uring already names the limit on a ring's `ENOMEM` (`rs/moq-uring/src/error.rs`). Check that this error reaches the failing test's output on the relay paths (worker startup, `start_relay`) instead of being wrapped or swallowed; if a test sees only a timeout, surface the startup error there. -- Keep the group's `max-threads` unless measured otherwise; the limit is shared with other checkouts, which `rs/moq-uring/README.md` documents. - -Public API: none. Wire: none. - -## Related - -- [More tests under load](/quest/m0/test-flakes-2/README.md) - the flake epic this would otherwise land in diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 36a5451e2ae..59939de0b62 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -28,24 +28,33 @@ use crate::{Config, Connection, auth, auth::Admissions, cluster, internal, shutd /// A handle that waits until a relay has finished startup. #[derive(Clone)] pub struct Ready { - receiver: tokio::sync::watch::Receiver, + receiver: tokio::sync::watch::Receiver, } impl Ready { - /// Wait until `Relay::run` starts serving, or fail if it exits first. + /// Wait until `Relay::run` starts serving, or fail with its error if it exits first. pub async fn wait(mut self) -> anyhow::Result<()> { - loop { - if *self.receiver.borrow() { - return Ok(()); - } - self.receiver - .changed() - .await - .context("relay stopped before becoming ready")?; + let startup = self + .receiver + .wait_for(|startup| !matches!(startup, Startup::Starting)) + .await + .context("relay stopped before becoming ready")?; + if let Startup::Failed(err) = &*startup { + return Err(anyhow::anyhow!("{err}").context("relay stopped before becoming ready")); } + Ok(()) } } +/// How far [`Relay::run`] got through startup. +#[derive(Clone)] +enum Startup { + Starting, + Serving, + // The error's whole chain, since `run` returns the error itself. + Failed(std::sync::Arc), +} + /// A fully assembled relay: the owner of every listener, worker group, and /// shutdown join. /// @@ -69,7 +78,7 @@ impl Ready { /// running.await??; /// ``` pub struct Relay { - ready: tokio::sync::watch::Sender, + ready: tokio::sync::watch::Sender, // Keep the inspection snapshot out of every future that moves a Relay. config: Box, server: moq_tokio::Listener, @@ -344,7 +353,7 @@ impl Relay { let cluster = cluster.with_stats(stats.clone(), publisher); let sessions = crate::session::Registry::new(); - let (ready, _) = tokio::sync::watch::channel(false); + let (ready, _) = tokio::sync::watch::channel(Startup::Starting); let web = web::Web::new(auth.clone(), cluster.clone(), certificates, config.web) .with_shutdown(shutdown.clone()) .with_versions(server_versions) @@ -595,8 +604,20 @@ impl Relay { /// This is also the embedding loop. Extra routes go on via [`Self::with_web`] /// / [`Self::with_internal`] before calling this; cloned handles outlive it. pub async fn run(self) -> anyhow::Result<()> { + // Held past `run_inner`, so a waiter sees the failure before the close. + let ready = self.ready.clone(); // Serving branches must not inflate the caller's stack frame. - Box::pin(self.run_inner()).await + let result = Box::pin(self.run_inner()).await; + if let Err(err) = &result { + ready.send_if_modified(|startup| { + let starting = matches!(startup, Startup::Starting); + if starting { + *startup = Startup::Failed(format!("{err:#}").into()); + } + starting + }); + } + result } async fn run_inner(self) -> anyhow::Result<()> { @@ -691,7 +712,7 @@ impl Relay { // An embedder that already stopped the relay gets a drain, not a READY. if shutdown.deadline().is_none() { - ready.send_replace(true); + ready.send_replace(Startup::Serving); #[cfg(unix)] // Notify systemd that we're ready after all initialization is complete @@ -994,6 +1015,19 @@ mod tests { assert!(ready.wait().await.is_err(), "reported ready after the stop"); } + /// A waiter learns why startup failed, not only that it ended. + #[tokio::test] + async fn a_startup_failure_reaches_the_waiter() { + let mut config = crate::Config::default(); + config.listen.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); + let relay = super::Relay::load(config).await.unwrap().with_signals(false); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + let error = format!("{:#}", ready.wait().await.expect_err("no auth refuses startup")); + assert!(error.contains("nobody can authenticate"), "{error}"); + running.await.unwrap().expect_err("run fails with the waiter"); + } + /// A connection arriving as the last admitted one leaves is either counted /// before the drain commits to exit, or refused at accept; never admitted /// into a relay that has already decided to exit under it. diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 217e2432788..cdb70d2f4ce 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -1131,7 +1131,9 @@ mod tests { let port = relay.addr().unwrap().port(); let mut admissions = relay.admissions().unwrap(); let trigger = relay.shutdown_trigger().clone(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut client = moq_tokio::connect::Config::default(); client.tls.insecure = Some(true); @@ -1189,7 +1191,9 @@ mod tests { let port = relay.addr().unwrap().port(); let mut admissions = relay.admissions().unwrap(); let trigger = relay.shutdown_trigger().clone(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut client = moq_tokio::connect::Config::default(); client.tls.insecure = Some(true); diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 6d0b7d74db5..3ff968e2a8c 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -168,7 +168,6 @@ async fn embed_and_stop(mut config: Config) { // are cloned before `run` consumes the relay. let origin = relay.cluster().origin.clone(); let trigger = relay.shutdown_trigger().clone(); - let ready = relay.ready(); let web = relay .web() .routes() @@ -178,6 +177,7 @@ async fn embed_and_stop(mut config: Config) { axum::routing::post(|| async { ([("access-control-allow-origin", "https://trusted.example")], "private") }), ) .route("/plain-post", axum::routing::post(|| async { "plain" })); + let ready = relay.ready(); let running = tokio::spawn(relay.with_web(web).run()); ready.wait().await.expect("relay ready"); assert!(!running.is_finished(), "the relay stopped while serving"); @@ -503,9 +503,9 @@ async fn embedded_listener_health_reaches_metrics() { config.drain_timeout = Some((Duration::ZERO).into()); let relay = Relay::load(config).await.expect("load relay").with_signals(false); let internal = relay.internal().addr().expect("internal listener bound"); - let ready = relay.ready(); let trigger = relay.shutdown_trigger().clone(); let health = moq_tokio::accept::Health::new("embedded"); + let ready = relay.ready(); let running = tokio::spawn(relay.with_listeners([health]).run()); ready.wait().await.expect("relay ready"); let body = reqwest::get(format!("http://{internal}/metrics")) diff --git a/rs/moq-relay/tests/integration/peer_limits.rs b/rs/moq-relay/tests/integration/peer_limits.rs index c71d7cdd147..0947b85b043 100644 --- a/rs/moq-relay/tests/integration/peer_limits.rs +++ b/rs/moq-relay/tests/integration/peer_limits.rs @@ -128,7 +128,9 @@ async fn peer_widens_and_viewer_does_not(mut relay: Relay, port: u16) { } leases }); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let peer = probe(port, true).await; assert_eq!(peer.streams, PROBE, "a peer opens past the viewer stream limit"); diff --git a/rs/moq-relay/tests/integration/refusals.rs b/rs/moq-relay/tests/integration/refusals.rs index ac37c36d97e..a3c7193fa72 100644 --- a/rs/moq-relay/tests/integration/refusals.rs +++ b/rs/moq-relay/tests/integration/refusals.rs @@ -60,8 +60,8 @@ async fn start(config: Config) -> Running { let quic = relay.quic_addr().expect("quic listener bound"); let http = relay.web_addrs().http.expect("http listener bound"); let internal = relay.internal().addr().expect("internal listener bound"); - let ready = relay.ready(); let trigger = relay.shutdown_trigger().clone(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); ready.wait().await.expect("relay ready"); Running { @@ -473,7 +473,9 @@ async fn uri_setup_paths(io_uring: bool) { let addr = relay.quic_addr().unwrap(); let mut admissions = relay.admissions().unwrap(); let trigger = relay.shutdown_trigger().clone(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); for version in versions { for path in [ Some(""), diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index ba739f62337..c22d213fcdf 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -126,7 +126,9 @@ async fn uring_workers_serve_webtransport_and_raw_quic() { // The stock loop serves everything: the uring workers own QUIC, the shared // runtime owns auth and supervision. + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // A WebTransport publisher (https, the browser path)... let wt_url: url::Url = format!("https://127.0.0.1:{port}/uring").parse().expect("parse url"); @@ -231,7 +233,9 @@ async fn uring_workers_serve_every_configured_version() { .expect("load relay") .with_signals(false); let port = relay.addr().expect("workers bound an address").port(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let origin = moq_tokio::origin::spawn(); let broadcast = origin.create_broadcast("test").expect("create broadcast"); @@ -317,7 +321,9 @@ async fn uring_workers_report_link_facts() { let local = relay.addr().expect("bound address"); let port = local.port(); let sessions = relay.sessions().clone(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut connections = Vec::new(); for scheme in ["moql", "https"] { @@ -380,7 +386,9 @@ async fn uring_workers_drain_on_the_trigger() { let port = relay.addr().expect("workers bound an address").port(); let trigger = relay.shutdown_trigger().clone(); let mut admissions = relay.admissions().expect("embedded admission"); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // One-shot (see `client`), so a session leaves on its GOAWAY rather than // migrating, and closing cleanly shows it was one. @@ -474,7 +482,9 @@ async fn workers_publish_their_rotated_certificate_fingerprint() { let addr = listener.local_addr().expect("local addr"); let routes = relay.web().routes(); let serving = tokio::spawn(async move { axum::serve(listener, routes).await }); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let result = kio::clock::timeout(TIMEOUT, async { for rotated in [false, true] { if rotated { @@ -543,7 +553,9 @@ async fn workers_share_a_generated_identity() { let addr = listener.local_addr().unwrap(); let routes = relay.web().routes(); let serving = tokio::spawn(async move { axum::serve(listener, routes).await }); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let fingerprint = http_get(addr, "/certificate.sha256").await; let mut dial = moq_tokio::connect::Config::default(); dial.tls.fingerprint = vec![fingerprint]; @@ -585,7 +597,9 @@ async fn an_mtls_client_authenticates_without_a_token() { let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let client = || { let mut dial = moq_tokio::connect::Config::default(); @@ -677,7 +691,9 @@ async fn uring_workers_write_qlog_traces() { config.quic.qlog = Some(traces.clone()); let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // A real session, so a trace covers a handshake and application data // rather than a connection that only ever exchanged Initials. @@ -763,7 +779,9 @@ async fn uring_refusals_reach_metrics() { let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); let internal = relay.internal().addr().expect("internal listener bound"); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // `/rooms` is outside the public rules. A client may finish connecting before // the verdict lands, so a session that closes promptly counts as refused. @@ -872,7 +890,9 @@ async fn uring_forwards_the_setup_token() { admission.grant(moq_auth::lease::Consumer::fixed(moq_auth::Grant::new(all.clone(), all))); } }); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut config = moq_tokio::connect::Config::default(); config.tls.insecure = Some(true); let tls = config.tls.build().unwrap(); @@ -963,7 +983,9 @@ async fn uring_auth_reports_distinct_quic_and_webtransport() { } leases }); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut connections = Vec::new(); for scheme in ["moql", "https"] { let url = format!("{scheme}://127.0.0.1:{port}/transport").parse().unwrap(); @@ -996,7 +1018,9 @@ async fn uring_workers_offer_the_configured_extensions() { let mut relay = Relay::load(config).await.unwrap().with_signals(false); let port = relay.addr().unwrap().port(); let mut admissions = relay.admissions().expect("embedded auth"); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut dial = moq_tokio::connect::Config::default(); dial.tls.insecure = Some(true); @@ -1073,7 +1097,9 @@ fn uring_workers_bound_stalled_handshakes() { let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); let addr = SocketAddr::from(([127, 0, 0, 1], port)); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut config = moq_tokio::connect::Config::default(); config.tls.insecure = Some(true); @@ -1198,7 +1224,9 @@ async fn uring_sessions_log_their_close_under_a_connection_span() { .with_signals(false); let port = relay.addr().unwrap().port(); let trigger = relay.shutdown_trigger().clone(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut connections = Vec::new(); for (id, scheme) in ["moql", "https"].into_iter().enumerate() { let url = format!("{scheme}://127.0.0.1:{port}/logs").parse().unwrap(); @@ -1240,3 +1268,59 @@ async fn uring_sessions_log_their_close_under_a_connection_span() { } } } + +/// A ring refused at worker startup reaches a `Ready` waiter with the actionable error. +#[tokio::test] +async fn uring_startup_reports_the_memlock_limit() { + // Resource limits belong to the process, so isolate even under libtest. + if !test_env::Env::default().set("MOQ_TEST_MEMLOCK", "0").isolate() { + return; + } + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + if !supported() { + return; + } + // CAP_IPC_LOCK, which root holds, exempts io_uring from the limit. + let status = std::fs::read_to_string("/proc/self/status").expect("process status"); + let effective = status + .lines() + .find_map(|line| line.strip_prefix("CapEff:")) + .expect("effective capabilities"); + let effective = u64::from_str_radix(effective.trim(), 16).expect("capability mask"); + const CAP_IPC_LOCK: u32 = 14; + if effective & (1 << CAP_IPC_LOCK) != 0 { + eprintln!("skipping the memlock test: CAP_IPC_LOCK lifts the limit"); + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + let (cert, key) = certificate(dir.path()); + let relay = Relay::load(uring_config(&cert, &key)) + .await + .expect("load relay") + .with_signals(false); + let mut limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: valid out-pointer in an isolated test process. + assert_eq!(unsafe { libc::getrlimit(libc::RLIMIT_MEMLOCK, &mut limit) }, 0); + limit.rlim_cur = 0; + // SAFETY: valid limit in an isolated test process. + assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_MEMLOCK, &limit) }, 0); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + let error = ready.wait().await.expect_err("a worker needs locked memory"); + let message = format!("{error:#}"); + for expected in [ + "io_uring worker 0 failed to start", + "RLIMIT_MEMLOCK", + "ulimit -l", + "LimitMEMLOCK=", + ] { + assert!(message.contains(expected), "{message}"); + } + running + .await + .expect("join relay") + .expect_err("run fails with the waiter"); +} diff --git a/rs/moq-relay/tests/integration/tenant_matrix.rs b/rs/moq-relay/tests/integration/tenant_matrix.rs index 6f72f1fb9e6..fe5575701d7 100644 --- a/rs/moq-relay/tests/integration/tenant_matrix.rs +++ b/rs/moq-relay/tests/integration/tenant_matrix.rs @@ -142,8 +142,8 @@ async fn start(io_uring: bool) -> Running { let quic = relay.quic_addr().expect("quic listener bound"); let http = relay.web_addrs().http.expect("http listener bound"); let internal = relay.internal().addr().expect("internal listener bound"); - let ready = relay.ready(); let trigger = relay.shutdown_trigger().clone(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); ready.wait().await.expect("relay ready"); Running { diff --git a/rs/moq-relay/tests/integration/wire_stats.rs b/rs/moq-relay/tests/integration/wire_stats.rs index 36797b4159e..267f937a880 100644 --- a/rs/moq-relay/tests/integration/wire_stats.rs +++ b/rs/moq-relay/tests/integration/wire_stats.rs @@ -48,7 +48,9 @@ async fn wire_sources(io_uring: bool) { let metrics = format!("http://{}/metrics", relay.internal().addr().unwrap()); let registry = relay.cluster().stats.clone(); let trigger = relay.shutdown_trigger().clone(); + let ready = relay.ready(); let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // Both adapters must expose UDP totals without asking Session::stats(), // which would arm a periodic sampler. Snapshot reads the source directly. for scheme in ["moql", "https"] {