From 284296b8547540bb8d7e0740e8ae081e5a206f21 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 17:50:35 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m0/ci/io-uring-test-group.md Quest-Claim: 2ad98b29ec8de91d From 016e3e3ebd8783214b7ea1c458569216a7c69f0a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 17:56:42 -0800 Subject: [PATCH 2/6] test(relay): group io_uring tests and preserve startup errors Co-Authored-By: GPT-6.1 Sol --- .config/nextest.toml | 2 +- quest/m0/ci/README.md | 1 - quest/m0/ci/io-uring-test-group.md | 21 ------ rs/moq-relay/src/lib.rs | 3 + rs/moq-relay/src/uring.rs | 4 +- rs/moq-relay/tests/integration/embed.rs | 12 ++-- rs/moq-relay/tests/integration/main.rs | 5 ++ rs/moq-relay/tests/integration/peer_limits.rs | 2 +- rs/moq-relay/tests/integration/refusals.rs | 6 +- .../tests/integration/runtime_uring.rs | 69 +++++++++++++++---- rs/moq-relay/tests/integration/start.rs | 13 ++++ .../tests/integration/tenant_matrix.rs | 4 +- rs/moq-relay/tests/integration/wire_stats.rs | 2 +- 13 files changed, 90 insertions(+), 54 deletions(-) delete mode 100644 quest/m0/ci/io-uring-test-group.md create mode 100644 rs/moq-relay/tests/integration/start.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 1a5f8ed28b2..e0516f295e4 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -28,7 +28,7 @@ default-filter = "not kind(bench)" # hosted CI's locked-memory limit and uses the host's hard limit locally. # Several invocations still share that user-wide budget; see rs/moq-uring/README.md. [[profile.default.overrides]] -filter = "package(moq-uring) | (binary_id(moq-relay::integration) & (test(/^runtime_uring::/) | test(/^embed::.*uring/)))" +filter = "package(moq-uring) | (package(moq-relay) & test(/uring/))" test-group = "io-uring" # The vivid tests share one capture node, and a second process configuring its diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 4c3cb465c0e..44bd0ef72af 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -154,7 +154,6 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Read-only pull-request CI](/quest/m0/ci/read-only-secrets.md) - write secrets and publisher identities reach only protected branches and tags, never a pushed branch - [Land main through GitHub's merge queue](/quest/m0/ci/land.md) - one PR at a time on the self-hosted runner; the host's Codex fixer fixes and re-enqueues every dropped PR - [A cache error is a miss](/quest/m0/ci/nix-cache-5xx.md) - a transient `nix.moq.dev` 5xx no longer fails `just ci` -- [Every io_uring test in the io-uring group](/quest/m0/ci/io-uring-test-group.md) - the parallel tests of one `just ci` stop exhausting the shared `RLIMIT_MEMLOCK`, and a too-low limit fails clearly - [Cheap checks skip the queue](/quest/m0/ci/gate-quick.md) - condition: quest ships `quest gate --quick`; then cheap modules wait at most a minute - [Gate each diff in under five minutes](/quest/m0/ci/five-minutes.md) - `just check` and `just test` each finish within a worker's 5-minute prompt cache, admission wait included - [Measure the parallel frontend](/quest/m0/ci/nightly-levers.md) - a pinned nightly's parallel frontend, adopted only if the numbers beat its risks; Cranelift is ruled out diff --git a/quest/m0/ci/io-uring-test-group.md b/quest/m0/ci/io-uring-test-group.md deleted file mode 100644 index 2b3a88b5a08..00000000000 --- a/quest/m0/ci/io-uring-test-group.md +++ /dev/null @@ -1,21 +0,0 @@ -# [XS] Every io_uring test runs in the nextest io-uring group - -## Goal - -Every test that opens an io_uring, including the relay's `wire_stats::uring_*`, `peer_limits::uring_*`, and `refusals::uring_*` integration tests and the `bitrate_caps::uring_*` tests [moq-dev/moq#7535](https://github.com/moq-dev/moq/pull/7535) adds, runs inside the `io-uring` nextest test group, so the tests of one `just ci` running in parallel no longer exhaust the user's shared `RLIMIT_MEMLOCK`. The group caps rings only within one nextest invocation; separate invocations still share the budget, as `rs/moq-uring/README.md` documents, and this quest does not change that. An io_uring test that still hits too low a limit fails with a message naming `RLIMIT_MEMLOCK` and how to raise it, not a generic setup error or a timeout. - -## Plan - -Found by [moq-dev/moq#7535](https://github.com/moq-dev/moq/pull/7535) and [moq-dev/moq#7211](https://github.com/moq-dev/moq/pull/7211): #7535's io_uring tests passed serially but could hit the shared limit in parallel, and #7211's `just ci` failed once in `uring_peer_session_widens_quic_limits` on `RLIMIT_MEMLOCK`. - -Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-afk run): m0, since each such failure fails a whole merge-gate batch. - -- The group's filter in `.config/nextest.toml` names only `runtime_uring::` and `embed::.*uring` in `moq-relay::integration`. Match every io_uring test instead, for example `test(/uring/)` in that binary plus moq-relay's own unit tests that start ring workers, so a new test joins by its name. Check that the inventory of uring tests (`rg 'fn uring_' rs/moq-relay`) all land in the group with `cargo nextest show-config test-groups`, which lists each group's members; plain `cargo nextest list` shows no group. -- moq-uring already names the limit on a ring's `ENOMEM` (`rs/moq-uring/src/error.rs`). Check that this error reaches the failing test's output on the relay paths (worker startup, `start_relay`) instead of being wrapped or swallowed; if a test sees only a timeout, surface the startup error there. -- Keep the group's `max-threads` unless measured otherwise; the limit is shared with other checkouts, which `rs/moq-uring/README.md` documents. - -Public API: none. Wire: none. - -## Related - -- [More tests under load](/quest/m0/test-flakes-2/README.md) - the flake epic this would otherwise land in diff --git a/rs/moq-relay/src/lib.rs b/rs/moq-relay/src/lib.rs index fd896f264e6..6d8b0d1c3a4 100644 --- a/rs/moq-relay/src/lib.rs +++ b/rs/moq-relay/src/lib.rs @@ -33,6 +33,9 @@ pub mod runtime; pub mod session; mod settings; pub mod shutdown; +#[cfg(all(test, target_os = "linux", feature = "_uring"))] +#[path = "../tests/integration/start.rs"] +mod start; pub mod stats; #[cfg(any(feature = "stun", test))] pub mod stun; diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 217e2432788..0079fb0b6b8 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -1131,7 +1131,7 @@ mod tests { let port = relay.addr().unwrap().port(); let mut admissions = relay.admissions().unwrap(); let trigger = relay.shutdown_trigger().clone(); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let mut client = moq_tokio::connect::Config::default(); client.tls.insecure = Some(true); @@ -1189,7 +1189,7 @@ mod tests { let port = relay.addr().unwrap().port(); let mut admissions = relay.admissions().unwrap(); let trigger = relay.shutdown_trigger().clone(); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let mut client = moq_tokio::connect::Config::default(); client.tls.insecure = Some(true); diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 005f8ff41bf..3e2a605a43c 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -168,7 +168,6 @@ async fn embed_and_stop(mut config: Config) { // are cloned before `run` consumes the relay. let origin = relay.cluster().origin.clone(); let trigger = relay.shutdown_trigger().clone(); - let ready = relay.ready(); let web = relay .web() .routes() @@ -178,8 +177,9 @@ async fn embed_and_stop(mut config: Config) { axum::routing::post(|| async { ([("access-control-allow-origin", "https://trusted.example")], "private") }), ) .route("/plain-post", axum::routing::post(|| async { "plain" })); - let running = tokio::spawn(relay.with_web(web).run()); - ready.wait().await.expect("relay ready"); + let running = crate::start::start_relay(relay.with_web(web)) + .await + .expect("start relay"); assert!(!running.is_finished(), "the relay stopped while serving"); let body = reqwest::get(format!("http://127.0.0.1:{}/embedded", http.port())) @@ -435,11 +435,11 @@ async fn embedded_listener_health_reaches_metrics() { config.drain_timeout = Some((Duration::ZERO).into()); let relay = Relay::load(config).await.expect("load relay").with_signals(false); let internal = relay.internal().addr().expect("internal listener bound"); - let ready = relay.ready(); let trigger = relay.shutdown_trigger().clone(); let health = moq_tokio::accept::Health::new("embedded"); - let running = tokio::spawn(relay.with_listeners([health]).run()); - ready.wait().await.expect("relay ready"); + let running = crate::start::start_relay(relay.with_listeners([health])) + .await + .expect("start relay"); let body = reqwest::get(format!("http://{internal}/metrics")) .await .expect("fetch metrics") diff --git a/rs/moq-relay/tests/integration/main.rs b/rs/moq-relay/tests/integration/main.rs index 9c4d360edda..5dd42ca6c65 100644 --- a/rs/moq-relay/tests/integration/main.rs +++ b/rs/moq-relay/tests/integration/main.rs @@ -33,6 +33,11 @@ mod session_revalidate; mod shutdown_signal; mod smoke; mod stack; +#[cfg(any(feature = "_quic", all(target_os = "linux", feature = "_uring")))] +mod start; mod tenant_matrix; mod upstream_links; mod wire_stats; + +#[cfg(any(feature = "_quic", all(target_os = "linux", feature = "_uring")))] +use moq_relay::Relay; diff --git a/rs/moq-relay/tests/integration/peer_limits.rs b/rs/moq-relay/tests/integration/peer_limits.rs index c71d7cdd147..c864b0197c1 100644 --- a/rs/moq-relay/tests/integration/peer_limits.rs +++ b/rs/moq-relay/tests/integration/peer_limits.rs @@ -128,7 +128,7 @@ async fn peer_widens_and_viewer_does_not(mut relay: Relay, port: u16) { } leases }); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let peer = probe(port, true).await; assert_eq!(peer.streams, PROBE, "a peer opens past the viewer stream limit"); diff --git a/rs/moq-relay/tests/integration/refusals.rs b/rs/moq-relay/tests/integration/refusals.rs index ac37c36d97e..562d5240119 100644 --- a/rs/moq-relay/tests/integration/refusals.rs +++ b/rs/moq-relay/tests/integration/refusals.rs @@ -60,10 +60,8 @@ async fn start(config: Config) -> Running { let quic = relay.quic_addr().expect("quic listener bound"); let http = relay.web_addrs().http.expect("http listener bound"); let internal = relay.internal().addr().expect("internal listener bound"); - let ready = relay.ready(); let trigger = relay.shutdown_trigger().clone(); - let running = tokio::spawn(relay.run()); - ready.wait().await.expect("relay ready"); + let running = crate::start::start_relay(relay).await.expect("start relay"); Running { stats, origin, @@ -473,7 +471,7 @@ async fn uri_setup_paths(io_uring: bool) { let addr = relay.quic_addr().unwrap(); let mut admissions = relay.admissions().unwrap(); let trigger = relay.shutdown_trigger().clone(); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); for version in versions { for path in [ Some(""), diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index ba739f62337..827b73d7ee0 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -126,7 +126,7 @@ async fn uring_workers_serve_webtransport_and_raw_quic() { // The stock loop serves everything: the uring workers own QUIC, the shared // runtime owns auth and supervision. - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); // A WebTransport publisher (https, the browser path)... let wt_url: url::Url = format!("https://127.0.0.1:{port}/uring").parse().expect("parse url"); @@ -231,7 +231,7 @@ async fn uring_workers_serve_every_configured_version() { .expect("load relay") .with_signals(false); let port = relay.addr().expect("workers bound an address").port(); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let origin = moq_tokio::origin::spawn(); let broadcast = origin.create_broadcast("test").expect("create broadcast"); @@ -317,7 +317,7 @@ async fn uring_workers_report_link_facts() { let local = relay.addr().expect("bound address"); let port = local.port(); let sessions = relay.sessions().clone(); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let mut connections = Vec::new(); for scheme in ["moql", "https"] { @@ -380,7 +380,7 @@ async fn uring_workers_drain_on_the_trigger() { let port = relay.addr().expect("workers bound an address").port(); let trigger = relay.shutdown_trigger().clone(); let mut admissions = relay.admissions().expect("embedded admission"); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); // One-shot (see `client`), so a session leaves on its GOAWAY rather than // migrating, and closing cleanly shows it was one. @@ -474,7 +474,7 @@ async fn workers_publish_their_rotated_certificate_fingerprint() { let addr = listener.local_addr().expect("local addr"); let routes = relay.web().routes(); let serving = tokio::spawn(async move { axum::serve(listener, routes).await }); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let result = kio::clock::timeout(TIMEOUT, async { for rotated in [false, true] { if rotated { @@ -543,7 +543,7 @@ async fn workers_share_a_generated_identity() { let addr = listener.local_addr().unwrap(); let routes = relay.web().routes(); let serving = tokio::spawn(async move { axum::serve(listener, routes).await }); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let fingerprint = http_get(addr, "/certificate.sha256").await; let mut dial = moq_tokio::connect::Config::default(); dial.tls.fingerprint = vec![fingerprint]; @@ -585,7 +585,7 @@ async fn an_mtls_client_authenticates_without_a_token() { let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let client = || { let mut dial = moq_tokio::connect::Config::default(); @@ -677,7 +677,7 @@ async fn uring_workers_write_qlog_traces() { config.quic.qlog = Some(traces.clone()); let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); // A real session, so a trace covers a handshake and application data // rather than a connection that only ever exchanged Initials. @@ -763,7 +763,7 @@ async fn uring_refusals_reach_metrics() { let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); let internal = relay.internal().addr().expect("internal listener bound"); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); // `/rooms` is outside the public rules. A client may finish connecting before // the verdict lands, so a session that closes promptly counts as refused. @@ -872,7 +872,7 @@ async fn uring_forwards_the_setup_token() { admission.grant(moq_auth::lease::Consumer::fixed(moq_auth::Grant::new(all.clone(), all))); } }); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let mut config = moq_tokio::connect::Config::default(); config.tls.insecure = Some(true); let tls = config.tls.build().unwrap(); @@ -963,7 +963,7 @@ async fn uring_auth_reports_distinct_quic_and_webtransport() { } leases }); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let mut connections = Vec::new(); for scheme in ["moql", "https"] { let url = format!("{scheme}://127.0.0.1:{port}/transport").parse().unwrap(); @@ -996,7 +996,7 @@ async fn uring_workers_offer_the_configured_extensions() { let mut relay = Relay::load(config).await.unwrap().with_signals(false); let port = relay.addr().unwrap().port(); let mut admissions = relay.admissions().expect("embedded auth"); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let mut dial = moq_tokio::connect::Config::default(); dial.tls.insecure = Some(true); @@ -1073,7 +1073,7 @@ fn uring_workers_bound_stalled_handshakes() { let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); let addr = SocketAddr::from(([127, 0, 0, 1], port)); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let mut config = moq_tokio::connect::Config::default(); config.tls.insecure = Some(true); @@ -1198,7 +1198,7 @@ async fn uring_sessions_log_their_close_under_a_connection_span() { .with_signals(false); let port = relay.addr().unwrap().port(); let trigger = relay.shutdown_trigger().clone(); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); let mut connections = Vec::new(); for (id, scheme) in ["moql", "https"].into_iter().enumerate() { let url = format!("{scheme}://127.0.0.1:{port}/logs").parse().unwrap(); @@ -1240,3 +1240,44 @@ async fn uring_sessions_log_their_close_under_a_connection_span() { } } } + +/// A ring refused at worker startup preserves the actionable error through the +/// same fixture startup path used by the session tests. +#[tokio::test] +async fn uring_startup_reports_the_memlock_limit() { + // Resource limits belong to the process, so isolate even under libtest. + if !test_env::Env::default().set("MOQ_TEST_MEMLOCK", "0").isolate() { + return; + } + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + if !supported() { + return; + } + let dir = tempfile::tempdir().expect("tempdir"); + let (cert, key) = certificate(dir.path()); + let relay = Relay::load(uring_config(&cert, &key)) + .await + .expect("load relay") + .with_signals(false); + let mut limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: valid out-pointer in an isolated test process. + assert_eq!(unsafe { libc::getrlimit(libc::RLIMIT_MEMLOCK, &mut limit) }, 0); + limit.rlim_cur = 0; + // SAFETY: valid limit in an isolated test process. + assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_MEMLOCK, &limit) }, 0); + let error = crate::start::start_relay(relay) + .await + .expect_err("a worker needs locked memory"); + let message = format!("{error:#}"); + for expected in [ + "io_uring worker 0 failed to start", + "RLIMIT_MEMLOCK", + "ulimit -l", + "LimitMEMLOCK=", + ] { + assert!(message.contains(expected), "{message}"); + } +} diff --git a/rs/moq-relay/tests/integration/start.rs b/rs/moq-relay/tests/integration/start.rs new file mode 100644 index 00000000000..c4ee300e441 --- /dev/null +++ b/rs/moq-relay/tests/integration/start.rs @@ -0,0 +1,13 @@ +//! Start relay fixtures before clients connect, preserving any startup error. + +pub(crate) async fn start_relay(relay: crate::Relay) -> anyhow::Result>> { + let ready = relay.ready(); + let mut run = Box::pin(relay.run()); + tokio::select! { + Ok(()) = ready.wait() => Ok(tokio::spawn(run)), + result = &mut run => { + result?; + anyhow::bail!("relay stopped before becoming ready"); + } + } +} diff --git a/rs/moq-relay/tests/integration/tenant_matrix.rs b/rs/moq-relay/tests/integration/tenant_matrix.rs index 6f72f1fb9e6..c222855be58 100644 --- a/rs/moq-relay/tests/integration/tenant_matrix.rs +++ b/rs/moq-relay/tests/integration/tenant_matrix.rs @@ -142,10 +142,8 @@ async fn start(io_uring: bool) -> Running { let quic = relay.quic_addr().expect("quic listener bound"); let http = relay.web_addrs().http.expect("http listener bound"); let internal = relay.internal().addr().expect("internal listener bound"); - let ready = relay.ready(); let trigger = relay.shutdown_trigger().clone(); - let running = tokio::spawn(relay.run()); - ready.wait().await.expect("relay ready"); + let running = crate::start::start_relay(relay).await.expect("start relay"); Running { tenants, quic, diff --git a/rs/moq-relay/tests/integration/wire_stats.rs b/rs/moq-relay/tests/integration/wire_stats.rs index 36797b4159e..b2019e359f9 100644 --- a/rs/moq-relay/tests/integration/wire_stats.rs +++ b/rs/moq-relay/tests/integration/wire_stats.rs @@ -48,7 +48,7 @@ async fn wire_sources(io_uring: bool) { let metrics = format!("http://{}/metrics", relay.internal().addr().unwrap()); let registry = relay.cluster().stats.clone(); let trigger = relay.shutdown_trigger().clone(); - let running = tokio::spawn(relay.run()); + let running = crate::start::start_relay(relay).await.expect("start relay"); // Both adapters must expose UDP totals without asking Session::stats(), // which would arm a periodic sampler. Snapshot reads the source directly. for scheme in ["moql", "https"] { From 1ad616386eddeabc3556b55108e58d22d9a82b56 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 18:01:18 -0800 Subject: [PATCH 3/6] docs(quest): retain blocked validation and review Co-Authored-By: GPT-6.1 Sol --- quest/m0/ci/README.md | 1 + quest/m0/ci/io-uring-test-group.md | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 quest/m0/ci/io-uring-test-group.md diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 44bd0ef72af..4c3cb465c0e 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -154,6 +154,7 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Read-only pull-request CI](/quest/m0/ci/read-only-secrets.md) - write secrets and publisher identities reach only protected branches and tags, never a pushed branch - [Land main through GitHub's merge queue](/quest/m0/ci/land.md) - one PR at a time on the self-hosted runner; the host's Codex fixer fixes and re-enqueues every dropped PR - [A cache error is a miss](/quest/m0/ci/nix-cache-5xx.md) - a transient `nix.moq.dev` 5xx no longer fails `just ci` +- [Every io_uring test in the io-uring group](/quest/m0/ci/io-uring-test-group.md) - the parallel tests of one `just ci` stop exhausting the shared `RLIMIT_MEMLOCK`, and a too-low limit fails clearly - [Cheap checks skip the queue](/quest/m0/ci/gate-quick.md) - condition: quest ships `quest gate --quick`; then cheap modules wait at most a minute - [Gate each diff in under five minutes](/quest/m0/ci/five-minutes.md) - `just check` and `just test` each finish within a worker's 5-minute prompt cache, admission wait included - [Measure the parallel frontend](/quest/m0/ci/nightly-levers.md) - a pinned nightly's parallel frontend, adopted only if the numbers beat its risks; Cranelift is ruled out diff --git a/quest/m0/ci/io-uring-test-group.md b/quest/m0/ci/io-uring-test-group.md new file mode 100644 index 00000000000..7498696655e --- /dev/null +++ b/quest/m0/ci/io-uring-test-group.md @@ -0,0 +1,21 @@ +# [XS] Every io_uring test runs in the nextest io-uring group + +## Goal + +Every test that opens an io_uring, including the relay's `wire_stats::uring_*`, `peer_limits::uring_*`, and `refusals::uring_*` integration tests and the `bitrate_caps::uring_*` tests [moq-dev/moq#7535](https://github.com/moq-dev/moq/pull/7535) adds, runs inside the `io-uring` nextest test group, so the tests of one `just ci` running in parallel no longer exhaust the user's shared `RLIMIT_MEMLOCK`. The group caps rings only within one nextest invocation; separate invocations still share the budget, as `rs/moq-uring/README.md` documents, and this quest does not change that. An io_uring test that still hits too low a limit fails with a message naming `RLIMIT_MEMLOCK` and how to raise it, not a generic setup error or a timeout. + +## Plan + +The implementation is checkpointed in draft [moq-dev/moq#7593](https://github.com/moq-dev/moq/pull/7593): name-based grouping covers the relay's integration and unit tests, and shared fixture startup preserves worker failures before clients connect. The four-test group limit is unchanged. + +Remaining work is validation and local review: + +- Run the full `just ci` successfully, then `/quest-review` and mark the claim PR ready. The current environment fails in Cargo documentation and `sh/rs/test-lints.test.py` with mbx `E2BIG`; the separate [mbx quest](/quest/m0/ci/mbx-arg-max.md) owns that root cause. +- Preserve the completed inventory check: `cargo nextest show-config test-groups` with `moq-relay/io-uring,moq-relay/qlog` lists all 19 named `uring_*` tests and all three ring-opening relay unit tests in `io-uring`. The pending `bitrate_caps::uring_*` names from [moq-dev/moq#7535](https://github.com/moq-dev/moq/pull/7535) match the same filter. +- The full relay suite with `io-uring,qlog` passes: 437 tests, 9 ignored. A constrained-memory reproduction previously reported only `relay stopped before becoming ready: channel closed`; the new isolated-process test exercises the real worker startup and verifies `RLIMIT_MEMLOCK`, `ulimit -l`, and `LimitMEMLOCK=` survive the error chain. + +Public API: none. Wire: none. + +## Related + +- [More tests under load](/quest/m0/test-flakes-2/README.md) - the flake epic this would otherwise land in From 94ada66c5ff8ca773095a291cce44edd1e1906bf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 22:17:00 -0800 Subject: [PATCH 4/6] test(relay): match uring as a whole name segment `test(/uring/)` also grouped tests whose names contain "during". The quest is complete, so remove it. Co-Authored-By: Claude Opus 5.5 --- .config/nextest.toml | 3 ++- quest/m0/ci/README.md | 1 - quest/m0/ci/io-uring-test-group.md | 21 --------------------- 3 files changed, 2 insertions(+), 23 deletions(-) delete mode 100644 quest/m0/ci/io-uring-test-group.md diff --git a/.config/nextest.toml b/.config/nextest.toml index e0516f295e4..11b3f70b991 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -27,8 +27,9 @@ default-filter = "not kind(bench)" # rings; integration workers keep production geometry. sh/rs/nextest.sh raises # hosted CI's locked-memory limit and uses the host's hard limit locally. # Several invocations still share that user-wide budget; see rs/moq-uring/README.md. +# A relay test joins by naming `uring` as a whole module or word, not "during". [[profile.default.overrides]] -filter = "package(moq-uring) | (package(moq-relay) & test(/uring/))" +filter = "package(moq-uring) | (package(moq-relay) & test(/(^|[:_])uring([:_]|$)/))" test-group = "io-uring" # The vivid tests share one capture node, and a second process configuring its diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index d7a6897dee1..3e0137afdac 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -155,7 +155,6 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Check on ready PRs](/quest/m0/ci/pr-check.md) - Check runs again on non-draft main PRs as a signal for `/quest-merge` and the land fixer, never as a gate - [Land main through GitHub's merge queue](/quest/m0/ci/land.md) - one PR at a time on the self-hosted runner; the host's Codex fixer fixes and re-enqueues every dropped PR - [A cache error is a miss](/quest/m0/ci/nix-cache-5xx.md) - a transient `nix.moq.dev` 5xx no longer fails `just ci` -- [Every io_uring test in the io-uring group](/quest/m0/ci/io-uring-test-group.md) - the parallel tests of one `just ci` stop exhausting the shared `RLIMIT_MEMLOCK`, and a too-low limit fails clearly - [Cheap checks skip the queue](/quest/m0/ci/gate-quick.md) - condition: quest ships `quest gate --quick`; then cheap modules wait at most a minute - [Gate each diff in under five minutes](/quest/m0/ci/five-minutes.md) - `just check` and `just test` each finish within a worker's 5-minute prompt cache, admission wait included - [Measure the parallel frontend](/quest/m0/ci/nightly-levers.md) - a pinned nightly's parallel frontend, adopted only if the numbers beat its risks; Cranelift is ruled out diff --git a/quest/m0/ci/io-uring-test-group.md b/quest/m0/ci/io-uring-test-group.md deleted file mode 100644 index 7498696655e..00000000000 --- a/quest/m0/ci/io-uring-test-group.md +++ /dev/null @@ -1,21 +0,0 @@ -# [XS] Every io_uring test runs in the nextest io-uring group - -## Goal - -Every test that opens an io_uring, including the relay's `wire_stats::uring_*`, `peer_limits::uring_*`, and `refusals::uring_*` integration tests and the `bitrate_caps::uring_*` tests [moq-dev/moq#7535](https://github.com/moq-dev/moq/pull/7535) adds, runs inside the `io-uring` nextest test group, so the tests of one `just ci` running in parallel no longer exhaust the user's shared `RLIMIT_MEMLOCK`. The group caps rings only within one nextest invocation; separate invocations still share the budget, as `rs/moq-uring/README.md` documents, and this quest does not change that. An io_uring test that still hits too low a limit fails with a message naming `RLIMIT_MEMLOCK` and how to raise it, not a generic setup error or a timeout. - -## Plan - -The implementation is checkpointed in draft [moq-dev/moq#7593](https://github.com/moq-dev/moq/pull/7593): name-based grouping covers the relay's integration and unit tests, and shared fixture startup preserves worker failures before clients connect. The four-test group limit is unchanged. - -Remaining work is validation and local review: - -- Run the full `just ci` successfully, then `/quest-review` and mark the claim PR ready. The current environment fails in Cargo documentation and `sh/rs/test-lints.test.py` with mbx `E2BIG`; the separate [mbx quest](/quest/m0/ci/mbx-arg-max.md) owns that root cause. -- Preserve the completed inventory check: `cargo nextest show-config test-groups` with `moq-relay/io-uring,moq-relay/qlog` lists all 19 named `uring_*` tests and all three ring-opening relay unit tests in `io-uring`. The pending `bitrate_caps::uring_*` names from [moq-dev/moq#7535](https://github.com/moq-dev/moq/pull/7535) match the same filter. -- The full relay suite with `io-uring,qlog` passes: 437 tests, 9 ignored. A constrained-memory reproduction previously reported only `relay stopped before becoming ready: channel closed`; the new isolated-process test exercises the real worker startup and verifies `RLIMIT_MEMLOCK`, `ulimit -l`, and `LimitMEMLOCK=` survive the error chain. - -Public API: none. Wire: none. - -## Related - -- [More tests under load](/quest/m0/test-flakes-2/README.md) - the flake epic this would otherwise land in From 35ae59674651b1e8a95d38341e5bb72d2a531bee Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 23:11:12 -0800 Subject: [PATCH 5/6] fix(relay): report the startup error to Ready waiters `Ready::wait` turned every early exit from `Relay::run` into "channel closed", so each embedder and fixture lost the cause, such as a refused io_uring worker naming RLIMIT_MEMLOCK. `run` now records its error in the readiness watch before returning. Fixtures wait on `Ready` again, which drops the shared start helper and its `#[path]` include. The memlock test skips under CAP_IPC_LOCK, which lifts the limit. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/lib.rs | 3 - rs/moq-relay/src/relay.rs | 62 +++++++++++---- rs/moq-relay/src/uring.rs | 8 +- rs/moq-relay/tests/integration/embed.rs | 12 +-- rs/moq-relay/tests/integration/main.rs | 5 -- rs/moq-relay/tests/integration/peer_limits.rs | 4 +- rs/moq-relay/tests/integration/refusals.rs | 8 +- .../tests/integration/runtime_uring.rs | 78 ++++++++++++++----- rs/moq-relay/tests/integration/start.rs | 13 ---- .../tests/integration/tenant_matrix.rs | 4 +- rs/moq-relay/tests/integration/wire_stats.rs | 4 +- 11 files changed, 134 insertions(+), 67 deletions(-) delete mode 100644 rs/moq-relay/tests/integration/start.rs diff --git a/rs/moq-relay/src/lib.rs b/rs/moq-relay/src/lib.rs index 6d8b0d1c3a4..fd896f264e6 100644 --- a/rs/moq-relay/src/lib.rs +++ b/rs/moq-relay/src/lib.rs @@ -33,9 +33,6 @@ pub mod runtime; pub mod session; mod settings; pub mod shutdown; -#[cfg(all(test, target_os = "linux", feature = "_uring"))] -#[path = "../tests/integration/start.rs"] -mod start; pub mod stats; #[cfg(any(feature = "stun", test))] pub mod stun; diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 36a5451e2ae..59939de0b62 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -28,24 +28,33 @@ use crate::{Config, Connection, auth, auth::Admissions, cluster, internal, shutd /// A handle that waits until a relay has finished startup. #[derive(Clone)] pub struct Ready { - receiver: tokio::sync::watch::Receiver, + receiver: tokio::sync::watch::Receiver, } impl Ready { - /// Wait until `Relay::run` starts serving, or fail if it exits first. + /// Wait until `Relay::run` starts serving, or fail with its error if it exits first. pub async fn wait(mut self) -> anyhow::Result<()> { - loop { - if *self.receiver.borrow() { - return Ok(()); - } - self.receiver - .changed() - .await - .context("relay stopped before becoming ready")?; + let startup = self + .receiver + .wait_for(|startup| !matches!(startup, Startup::Starting)) + .await + .context("relay stopped before becoming ready")?; + if let Startup::Failed(err) = &*startup { + return Err(anyhow::anyhow!("{err}").context("relay stopped before becoming ready")); } + Ok(()) } } +/// How far [`Relay::run`] got through startup. +#[derive(Clone)] +enum Startup { + Starting, + Serving, + // The error's whole chain, since `run` returns the error itself. + Failed(std::sync::Arc), +} + /// A fully assembled relay: the owner of every listener, worker group, and /// shutdown join. /// @@ -69,7 +78,7 @@ impl Ready { /// running.await??; /// ``` pub struct Relay { - ready: tokio::sync::watch::Sender, + ready: tokio::sync::watch::Sender, // Keep the inspection snapshot out of every future that moves a Relay. config: Box, server: moq_tokio::Listener, @@ -344,7 +353,7 @@ impl Relay { let cluster = cluster.with_stats(stats.clone(), publisher); let sessions = crate::session::Registry::new(); - let (ready, _) = tokio::sync::watch::channel(false); + let (ready, _) = tokio::sync::watch::channel(Startup::Starting); let web = web::Web::new(auth.clone(), cluster.clone(), certificates, config.web) .with_shutdown(shutdown.clone()) .with_versions(server_versions) @@ -595,8 +604,20 @@ impl Relay { /// This is also the embedding loop. Extra routes go on via [`Self::with_web`] /// / [`Self::with_internal`] before calling this; cloned handles outlive it. pub async fn run(self) -> anyhow::Result<()> { + // Held past `run_inner`, so a waiter sees the failure before the close. + let ready = self.ready.clone(); // Serving branches must not inflate the caller's stack frame. - Box::pin(self.run_inner()).await + let result = Box::pin(self.run_inner()).await; + if let Err(err) = &result { + ready.send_if_modified(|startup| { + let starting = matches!(startup, Startup::Starting); + if starting { + *startup = Startup::Failed(format!("{err:#}").into()); + } + starting + }); + } + result } async fn run_inner(self) -> anyhow::Result<()> { @@ -691,7 +712,7 @@ impl Relay { // An embedder that already stopped the relay gets a drain, not a READY. if shutdown.deadline().is_none() { - ready.send_replace(true); + ready.send_replace(Startup::Serving); #[cfg(unix)] // Notify systemd that we're ready after all initialization is complete @@ -994,6 +1015,19 @@ mod tests { assert!(ready.wait().await.is_err(), "reported ready after the stop"); } + /// A waiter learns why startup failed, not only that it ended. + #[tokio::test] + async fn a_startup_failure_reaches_the_waiter() { + let mut config = crate::Config::default(); + config.listen.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); + let relay = super::Relay::load(config).await.unwrap().with_signals(false); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + let error = format!("{:#}", ready.wait().await.expect_err("no auth refuses startup")); + assert!(error.contains("nobody can authenticate"), "{error}"); + running.await.unwrap().expect_err("run fails with the waiter"); + } + /// A connection arriving as the last admitted one leaves is either counted /// before the drain commits to exit, or refused at accept; never admitted /// into a relay that has already decided to exit under it. diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 0079fb0b6b8..cdb70d2f4ce 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -1131,7 +1131,9 @@ mod tests { let port = relay.addr().unwrap().port(); let mut admissions = relay.admissions().unwrap(); let trigger = relay.shutdown_trigger().clone(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut client = moq_tokio::connect::Config::default(); client.tls.insecure = Some(true); @@ -1189,7 +1191,9 @@ mod tests { let port = relay.addr().unwrap().port(); let mut admissions = relay.admissions().unwrap(); let trigger = relay.shutdown_trigger().clone(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut client = moq_tokio::connect::Config::default(); client.tls.insecure = Some(true); diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index dd569ad1fc1..3ff968e2a8c 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -177,9 +177,9 @@ async fn embed_and_stop(mut config: Config) { axum::routing::post(|| async { ([("access-control-allow-origin", "https://trusted.example")], "private") }), ) .route("/plain-post", axum::routing::post(|| async { "plain" })); - let running = crate::start::start_relay(relay.with_web(web)) - .await - .expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.with_web(web).run()); + ready.wait().await.expect("relay ready"); assert!(!running.is_finished(), "the relay stopped while serving"); let body = reqwest::get(format!("http://127.0.0.1:{}/embedded", http.port())) @@ -505,9 +505,9 @@ async fn embedded_listener_health_reaches_metrics() { let internal = relay.internal().addr().expect("internal listener bound"); let trigger = relay.shutdown_trigger().clone(); let health = moq_tokio::accept::Health::new("embedded"); - let running = crate::start::start_relay(relay.with_listeners([health])) - .await - .expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.with_listeners([health]).run()); + ready.wait().await.expect("relay ready"); let body = reqwest::get(format!("http://{internal}/metrics")) .await .expect("fetch metrics") diff --git a/rs/moq-relay/tests/integration/main.rs b/rs/moq-relay/tests/integration/main.rs index 5dd42ca6c65..9c4d360edda 100644 --- a/rs/moq-relay/tests/integration/main.rs +++ b/rs/moq-relay/tests/integration/main.rs @@ -33,11 +33,6 @@ mod session_revalidate; mod shutdown_signal; mod smoke; mod stack; -#[cfg(any(feature = "_quic", all(target_os = "linux", feature = "_uring")))] -mod start; mod tenant_matrix; mod upstream_links; mod wire_stats; - -#[cfg(any(feature = "_quic", all(target_os = "linux", feature = "_uring")))] -use moq_relay::Relay; diff --git a/rs/moq-relay/tests/integration/peer_limits.rs b/rs/moq-relay/tests/integration/peer_limits.rs index c864b0197c1..0947b85b043 100644 --- a/rs/moq-relay/tests/integration/peer_limits.rs +++ b/rs/moq-relay/tests/integration/peer_limits.rs @@ -128,7 +128,9 @@ async fn peer_widens_and_viewer_does_not(mut relay: Relay, port: u16) { } leases }); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let peer = probe(port, true).await; assert_eq!(peer.streams, PROBE, "a peer opens past the viewer stream limit"); diff --git a/rs/moq-relay/tests/integration/refusals.rs b/rs/moq-relay/tests/integration/refusals.rs index 562d5240119..a3c7193fa72 100644 --- a/rs/moq-relay/tests/integration/refusals.rs +++ b/rs/moq-relay/tests/integration/refusals.rs @@ -61,7 +61,9 @@ async fn start(config: Config) -> Running { let http = relay.web_addrs().http.expect("http listener bound"); let internal = relay.internal().addr().expect("internal listener bound"); let trigger = relay.shutdown_trigger().clone(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); Running { stats, origin, @@ -471,7 +473,9 @@ async fn uri_setup_paths(io_uring: bool) { let addr = relay.quic_addr().unwrap(); let mut admissions = relay.admissions().unwrap(); let trigger = relay.shutdown_trigger().clone(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); for version in versions { for path in [ Some(""), diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index 827b73d7ee0..f50d1a99232 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -126,7 +126,9 @@ async fn uring_workers_serve_webtransport_and_raw_quic() { // The stock loop serves everything: the uring workers own QUIC, the shared // runtime owns auth and supervision. - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // A WebTransport publisher (https, the browser path)... let wt_url: url::Url = format!("https://127.0.0.1:{port}/uring").parse().expect("parse url"); @@ -231,7 +233,9 @@ async fn uring_workers_serve_every_configured_version() { .expect("load relay") .with_signals(false); let port = relay.addr().expect("workers bound an address").port(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let origin = moq_tokio::origin::spawn(); let broadcast = origin.create_broadcast("test").expect("create broadcast"); @@ -317,7 +321,9 @@ async fn uring_workers_report_link_facts() { let local = relay.addr().expect("bound address"); let port = local.port(); let sessions = relay.sessions().clone(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut connections = Vec::new(); for scheme in ["moql", "https"] { @@ -380,7 +386,9 @@ async fn uring_workers_drain_on_the_trigger() { let port = relay.addr().expect("workers bound an address").port(); let trigger = relay.shutdown_trigger().clone(); let mut admissions = relay.admissions().expect("embedded admission"); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // One-shot (see `client`), so a session leaves on its GOAWAY rather than // migrating, and closing cleanly shows it was one. @@ -474,7 +482,9 @@ async fn workers_publish_their_rotated_certificate_fingerprint() { let addr = listener.local_addr().expect("local addr"); let routes = relay.web().routes(); let serving = tokio::spawn(async move { axum::serve(listener, routes).await }); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let result = kio::clock::timeout(TIMEOUT, async { for rotated in [false, true] { if rotated { @@ -543,7 +553,9 @@ async fn workers_share_a_generated_identity() { let addr = listener.local_addr().unwrap(); let routes = relay.web().routes(); let serving = tokio::spawn(async move { axum::serve(listener, routes).await }); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let fingerprint = http_get(addr, "/certificate.sha256").await; let mut dial = moq_tokio::connect::Config::default(); dial.tls.fingerprint = vec![fingerprint]; @@ -585,7 +597,9 @@ async fn an_mtls_client_authenticates_without_a_token() { let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let client = || { let mut dial = moq_tokio::connect::Config::default(); @@ -677,7 +691,9 @@ async fn uring_workers_write_qlog_traces() { config.quic.qlog = Some(traces.clone()); let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // A real session, so a trace covers a handshake and application data // rather than a connection that only ever exchanged Initials. @@ -763,7 +779,9 @@ async fn uring_refusals_reach_metrics() { let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); let internal = relay.internal().addr().expect("internal listener bound"); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // `/rooms` is outside the public rules. A client may finish connecting before // the verdict lands, so a session that closes promptly counts as refused. @@ -872,7 +890,9 @@ async fn uring_forwards_the_setup_token() { admission.grant(moq_auth::lease::Consumer::fixed(moq_auth::Grant::new(all.clone(), all))); } }); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut config = moq_tokio::connect::Config::default(); config.tls.insecure = Some(true); let tls = config.tls.build().unwrap(); @@ -963,7 +983,9 @@ async fn uring_auth_reports_distinct_quic_and_webtransport() { } leases }); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut connections = Vec::new(); for scheme in ["moql", "https"] { let url = format!("{scheme}://127.0.0.1:{port}/transport").parse().unwrap(); @@ -996,7 +1018,9 @@ async fn uring_workers_offer_the_configured_extensions() { let mut relay = Relay::load(config).await.unwrap().with_signals(false); let port = relay.addr().unwrap().port(); let mut admissions = relay.admissions().expect("embedded auth"); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut dial = moq_tokio::connect::Config::default(); dial.tls.insecure = Some(true); @@ -1073,7 +1097,9 @@ fn uring_workers_bound_stalled_handshakes() { let relay = Relay::load(config).await.expect("load relay").with_signals(false); let port = relay.addr().expect("workers bound an address").port(); let addr = SocketAddr::from(([127, 0, 0, 1], port)); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut config = moq_tokio::connect::Config::default(); config.tls.insecure = Some(true); @@ -1198,7 +1224,9 @@ async fn uring_sessions_log_their_close_under_a_connection_span() { .with_signals(false); let port = relay.addr().unwrap().port(); let trigger = relay.shutdown_trigger().clone(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); let mut connections = Vec::new(); for (id, scheme) in ["moql", "https"].into_iter().enumerate() { let url = format!("{scheme}://127.0.0.1:{port}/logs").parse().unwrap(); @@ -1241,8 +1269,7 @@ async fn uring_sessions_log_their_close_under_a_connection_span() { } } -/// A ring refused at worker startup preserves the actionable error through the -/// same fixture startup path used by the session tests. +/// A ring refused at worker startup reaches a `Ready` waiter with the actionable error. #[tokio::test] async fn uring_startup_reports_the_memlock_limit() { // Resource limits belong to the process, so isolate even under libtest. @@ -1253,6 +1280,18 @@ async fn uring_startup_reports_the_memlock_limit() { if !supported() { return; } + // CAP_IPC_LOCK, which root holds, exempts io_uring from the limit. + let status = std::fs::read_to_string("/proc/self/status").expect("process status"); + let effective = status + .lines() + .find_map(|line| line.strip_prefix("CapEff:")) + .expect("effective capabilities"); + let effective = u64::from_str_radix(effective.trim(), 16).expect("capability mask"); + const CAP_IPC_LOCK: u32 = 14; + if effective & (1 << CAP_IPC_LOCK) != 0 { + eprintln!("skipping the memlock test: CAP_IPC_LOCK lifts the limit"); + return; + } let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = certificate(dir.path()); let relay = Relay::load(uring_config(&cert, &key)) @@ -1268,9 +1307,9 @@ async fn uring_startup_reports_the_memlock_limit() { limit.rlim_cur = 0; // SAFETY: valid limit in an isolated test process. assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_MEMLOCK, &limit) }, 0); - let error = crate::start::start_relay(relay) - .await - .expect_err("a worker needs locked memory"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + let error = ready.wait().await.expect_err("a worker needs locked memory"); let message = format!("{error:#}"); for expected in [ "io_uring worker 0 failed to start", @@ -1280,4 +1319,5 @@ async fn uring_startup_reports_the_memlock_limit() { ] { assert!(message.contains(expected), "{message}"); } + running.await.expect("join relay").expect_err("run fails with the waiter"); } diff --git a/rs/moq-relay/tests/integration/start.rs b/rs/moq-relay/tests/integration/start.rs deleted file mode 100644 index c4ee300e441..00000000000 --- a/rs/moq-relay/tests/integration/start.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Start relay fixtures before clients connect, preserving any startup error. - -pub(crate) async fn start_relay(relay: crate::Relay) -> anyhow::Result>> { - let ready = relay.ready(); - let mut run = Box::pin(relay.run()); - tokio::select! { - Ok(()) = ready.wait() => Ok(tokio::spawn(run)), - result = &mut run => { - result?; - anyhow::bail!("relay stopped before becoming ready"); - } - } -} diff --git a/rs/moq-relay/tests/integration/tenant_matrix.rs b/rs/moq-relay/tests/integration/tenant_matrix.rs index c222855be58..fe5575701d7 100644 --- a/rs/moq-relay/tests/integration/tenant_matrix.rs +++ b/rs/moq-relay/tests/integration/tenant_matrix.rs @@ -143,7 +143,9 @@ async fn start(io_uring: bool) -> Running { let http = relay.web_addrs().http.expect("http listener bound"); let internal = relay.internal().addr().expect("internal listener bound"); let trigger = relay.shutdown_trigger().clone(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); Running { tenants, quic, diff --git a/rs/moq-relay/tests/integration/wire_stats.rs b/rs/moq-relay/tests/integration/wire_stats.rs index b2019e359f9..267f937a880 100644 --- a/rs/moq-relay/tests/integration/wire_stats.rs +++ b/rs/moq-relay/tests/integration/wire_stats.rs @@ -48,7 +48,9 @@ async fn wire_sources(io_uring: bool) { let metrics = format!("http://{}/metrics", relay.internal().addr().unwrap()); let registry = relay.cluster().stats.clone(); let trigger = relay.shutdown_trigger().clone(); - let running = crate::start::start_relay(relay).await.expect("start relay"); + let ready = relay.ready(); + let running = tokio::spawn(relay.run()); + ready.wait().await.expect("relay ready"); // Both adapters must expose UDP totals without asking Session::stats(), // which would arm a periodic sampler. Snapshot reads the source directly. for scheme in ["moql", "https"] { From 9de598d89ceec0a39d0be42f8238d10b3ff48354 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 23:16:07 -0800 Subject: [PATCH 6/6] style(relay): format the memlock test Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/tests/integration/runtime_uring.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index f50d1a99232..c22d213fcdf 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -1319,5 +1319,8 @@ async fn uring_startup_reports_the_memlock_limit() { ] { assert!(message.contains(expected), "{message}"); } - running.await.expect("join relay").expect_err("run fails with the waiter"); + running + .await + .expect("join relay") + .expect_err("run fails with the waiter"); }