diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cbf7d58875..2e5bccee9e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -405,6 +405,7 @@ jobs: # `test route-sweep` checks random lite-07 meshes against the route rules from a fresh seed; PRs run a fixed 8. # It also runs the router's handoff sweeps over 20,000 fixed scenarios; PRs run 150. # It also runs the bandwidth probe's acceptance suite, of which PRs run the short cases. + # `test route-race` races a lite-07 relay's session tasks in seeded orders from a fresh seed; PRs run a fixed 16. # `test drill impaired` runs the relay drills over a freshly seeded shaper; PRs run loopback. # `rs cli-export` runs moq-cli's export process tests, which take tens of seconds of real time. # `rs hls-conformance` probes every segment moq-hls serves with ffprobe, against its playlists and manifest. @@ -415,7 +416,7 @@ jobs: # `rs v4l2` streams moq-video's V4L2 capture from the kernel's vivid # virtual camera, a module load no pull request job should make. # `js worklet-matrix` loads `?worklet` output through the bundlers pinned in test/worklet. - recipe: ["gh test-integration", "rs doctest --workspace", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "test route-sweep", "rs quic-sim", "rs rung-disable", "test drill impaired", "rs cli-export", "rs hls-conformance", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux", "rs v4l2", "js worklet-matrix"] + recipe: ["gh test-integration", "rs doctest --workspace", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "test route-sweep", "test route-race", "rs quic-sim", "rs rung-disable", "test drill impaired", "rs cli-export", "rs hls-conformance", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux", "rs v4l2", "js worklet-matrix"] include: # Miri interprets kio's and moq-net's in-memory unsafe and moq-sock's and moq-uring's # control-message walks; too slow for the merge gate. diff --git a/quest/m0/test-flakes-3/js-stale-budget.md b/quest/m0/test-flakes-3/js-stale-budget.md new file mode 100644 index 0000000000..c067c39429 --- /dev/null +++ b/quest/m0/test-flakes-3/js-stale-budget.md @@ -0,0 +1,20 @@ +# [S] A JS lite-07 stale drop narrows its budget every run + +## Goal + +`js/net`'s `a stale drop is recorded under the narrowest budget asked on lite-07` +(`src/lite/subscriber.test.ts`) passes every run. + +## Plan + +Seen 2026-11-18 while validating +[moq-dev/moq#9128](https://github.com/moq-dev/moq/pull/9128), on a base that +touched no JS: run alone with `bun test src/lite/subscriber.test.ts -t 'stale drop +is recorded under the narrowest'` from `js/net`, it failed 4 of 5 runs. The second +narrowing (`track.update({ delay: { max: 200ms } })`) leaves `entry.staleBudget` at +500 after `drainReactions()`. The update reaches the subscriber (`subscribe update` +is logged), so either the narrowing is applied after the drain returns, or the +entry it narrows is not the one the test holds. Find which and fix that, not the +drain. + +Public API: none. Wire: none. diff --git a/quest/m3/route-race-stress.md b/quest/m3/route-race-stress.md deleted file mode 100644 index fcca3a8b4a..0000000000 --- a/quest/m3/route-race-stress.md +++ /dev/null @@ -1,44 +0,0 @@ -# [M] Nightly races the route layer across threads - -## Goal - -Nightly runs a seeded stress of a publisher, a relay, and readers over mock -moq-lite-07 sessions on a multi-threaded runtime, mixing the steps whose -order two tasks decide: a broadcast ending as another starts on the same -session, a request arriving as its route lands, a grant widening, and a -session closing during an edit. A reader that hangs, or a request refused -for a claim the reader was shown, fails the run with its seed, its round, -and the steps in flight. PRs run a short fixed count. - -## Plan - -Decided 2026-11-18 (maintainer, in the coordinating session): ✅ planned -(rejected: leaving it out). - -[moq-dev/moq#8987](https://github.com/moq-dev/moq/pull/8987) put a claim in -front of a reader before its route reached the origin, so a request could be -refused as unroutable. Every route test scripts its order on one thread, and -only the interop `finish` lane saw it, intermittently. -[moq-dev/moq#9052](https://github.com/moq-dev/moq/pull/9052) found it with a -scratch stress that was not kept: one publisher, one relay, and 8 readers, -ending one broadcast and announcing another on the same session. It failed -within 72 to 827 rounds before the fix and ran 18,000 clean after. - -Agent's choices, for the implementer to revisit: - -- m3: it tests code, not the wire, so a bug it finds after the cut is a - patch release. -- Show it has teeth: with #9052's fix reverted it fails inside the nightly - budget. Measure the rounds that takes and report it. -- Real threads do not replay from a seed. Try a controlled scheduler first, - such as `shuttle`, if the mock sessions and `kio` run under one; otherwise - log the steps in flight so a failure can be read without a replay. -- [Random fault schedules](/quest/m4/cluster-fault-schedules.md) searches - fault timings on the simulated clock and may come to cover this. Say in - the PR which orders it cannot reach. - -Public API: none. Wire: none. - -## Related - -- [Random fault schedules](/quest/m4/cluster-fault-schedules.md) - the simulated search this sits beside diff --git a/rs/moq-net/benches/session/mod.rs b/rs/moq-net/benches/session/mod.rs index 7b699d8d9e..d727fa54b9 100644 --- a/rs/moq-net/benches/session/mod.rs +++ b/rs/moq-net/benches/session/mod.rs @@ -88,6 +88,8 @@ mod harness; #[path = "../../tests/integration/support/mock.rs"] pub(crate) mod mock; +#[path = "../../tests/integration/support/schedule.rs"] +mod schedule; use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 0e34b4ebcc..7a2a39995e 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -75,6 +75,7 @@ mod route_change; mod route_cluster; mod route_failover; mod route_health; +mod route_race; mod route_stats; mod segmented_frame; mod session_close; diff --git a/rs/moq-net/tests/integration/route_race.rs b/rs/moq-net/tests/integration/route_race.rs new file mode 100644 index 0000000000..30815d9dff --- /dev/null +++ b/rs/moq-net/tests/integration/route_race.rs @@ -0,0 +1,510 @@ +#![cfg(feature = "async")] + +//! Races the lite-07 route layer: a publisher, a relay, and readers over mock sessions, +//! with every task run in a seeded order. +//! +//! Each session drains its own outbox on its own task, so the route layer's correctness +//! can hang on which task runs first, an order a test scripted on one thread never +//! varies. Each round sets up a relay carrying a few broadcasts, then races these steps, +//! each its own task under a seeded [`schedule`] that holds tasks back the way +//! descheduled threads would: +//! +//! - a broadcast ending as another starts on the same session, +//! - more broadcasts announced on that session, and every reader requesting each claim +//! the moment it hears of it, so a request arrives as its route lands, +//! - a reader's grant widening, showing it claims it could not see, +//! - another publishing session closing, and a reader's leaving, during those edits. +//! +//! Once things settle, every reader must read each broadcast still live, show none that +//! ended, and have no request left unanswered. A failure prints the seed and round that +//! replay it, the steps taken before each failed request, and the round's trace. The +//! replay runs the same schedule, though moq-net's hash maps may reorder events within +//! one batch, so its trace can differ in that order. +//! +//! The schedule reorders whole task polls, so it reaches an order inside one poll only +//! where the poll yields, as at a [`kio::coop::Budget`]. Threads can interleave +//! anywhere. A session's announce loop yields between taking its peer's claims into the +//! node and serving them into the origin, which is where moq-dev/moq#9052's reader hung; +//! the extra announcements give that yield more chances to land between the two. +//! +//! `MOQ_RACE_SEED`, `MOQ_RACE_ROUND`, and `MOQ_RACE_ROUNDS` pick the rounds. Nightly +//! sweeps a fresh seed (`just test route-race`). + +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::rc::Rc; +use std::time::Duration; + +use moq_net::{ + Client, Error, Pattern, Patterns, Server, Version, + auth::Grant, + origin::{self, RouteKind}, + track, +}; + +use crate::support::harness::{self, MockConnectOptions, MockPair, connect_mock}; +use crate::support::mock::create_mock_session_pair; +use crate::support::schedule::{self, Schedule}; +use crate::support::{self, AppendNext, AtStart, Rng}; + +const VERSION: &str = "moq-lite-07-wip"; + +/// The only track each broadcast carries, holding one frame. +const TRACK: &str = "t"; + +/// Long enough, in simulated time, for anything in flight to land. +const SETTLE: Duration = Duration::from_secs(10); + +/// Live from the start, ended by [`Step::Replace`]. +const OLD: &str = "live/old"; +/// Announced by [`Step::Replace`] on the same session. +const NEW: &str = "live/new"; +/// Live from the start on the second session, which [`Step::Close`] closes. +const GONE: &str = "live/gone"; +/// Live from the start, outside a narrow reader's grant until [`Step::Widen`]. +const STAY: &str = "wide/stay"; +/// What a narrow reader may see before its grant widens. +const NARROW: &str = "live"; + +/// One step a round races against the others. +#[derive(Clone, Copy, Debug)] +enum Step { + /// End [`OLD`] and announce [`NEW`] on the same session, in either order. + Replace { drop: bool, new_first: bool }, + /// Widen a narrow reader's grant to everything. + Widen(usize), + /// Close the second publishing session, carrying [`GONE`]. + Close(Close), + /// A reader's session closes. + Leave(usize), + /// Announce another broadcast on the publisher's session, `live/`. + Announce(usize), +} + +/// How [`Step::Close`] ends the session. +#[derive(Clone, Copy, Debug)] +enum Close { + /// The publisher aborts it. + Publisher, + /// The relay aborts it. + Relay, + /// Both ends drop their last handle. + Drop, +} + +/// What a round runs, drawn from its seed. +#[derive(Debug)] +struct Shape { + readers: usize, + /// Which readers start with the grant of [`NARROW`] only. + narrow: Vec, + /// Whether the relay has settled before the steps start. + settled: bool, + /// Each step, after the executor passes it waits once started. + steps: Vec<(Step, u64)>, + /// The schedule's chance of holding a poll back, in 1/1024ths. + chance: u64, + /// The schedule's longest hold, in executor passes. + longest: u64, +} + +impl Shape { + fn draw(rng: &mut Rng) -> Self { + let readers = rng.range(2, 8) as usize; + let narrow: Vec = (0..readers).map(|_| rng.next().is_multiple_of(2)).collect(); + let mut steps = vec![ + Step::Replace { + drop: rng.next().is_multiple_of(2), + new_first: rng.next().is_multiple_of(2), + }, + Step::Close([Close::Publisher, Close::Relay, Close::Drop][rng.pick(3)]), + ]; + steps.extend((0..readers).filter(|reader| narrow[*reader]).map(Step::Widen)); + if rng.next().is_multiple_of(2) { + steps.push(Step::Leave(rng.pick(readers))); + } + steps.extend((0..rng.range(0, 16) as usize).map(Step::Announce)); + // Started in a random order, each step's task then runs when the schedule lets it. + for index in (1..steps.len()).rev() { + steps.swap(index, rng.pick(index + 1)); + } + let steps = steps.into_iter().map(|step| (step, rng.range(0, 64))).collect(); + Self { + readers, + narrow, + settled: rng.next().is_multiple_of(4), + steps, + chance: rng.range(32, 512), + longest: rng.range(1, 32), + } + } +} + +/// How one reader's request for one claim it was shown went. +#[derive(Clone, Debug, PartialEq)] +enum Outcome { + /// Still waiting. + Pending, + /// Read the broadcast's frame. + Read, + /// Ended cleanly with no group. + Ended, + /// Failed. + Failed(String), +} + +/// One reader's request for one claim it was shown. +struct Request { + reader: usize, + path: String, + outcome: Outcome, + /// The steps already taken when it was made: their effects were still in flight. + after: Vec, +} + +/// What a round saw, shared by its tasks. +#[derive(Default)] +struct Log { + trace: Vec, + /// The steps taken so far, in order. Each acts at once, but what it sets off lands later. + taken: Vec, + /// Every request a reader made, in order. + requests: Vec, + /// Each reader's view: the paths it shows, with the request made for the latest showing. + shown: BTreeMap<(usize, String), usize>, + /// Readers whose session closed. + left: Vec, +} + +type Shared = Rc>; + +fn note(log: &Shared, line: String) { + log.borrow_mut().trace.push(line); +} + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(std::num::NonZeroU64::new(hop).unwrap())); + harness::spawn(driver); + producer.with_proxy(true) +} + +/// A broadcast with one frame on [`TRACK`], announced. +struct Live { + route: origin::Route, + _track: track::Producer, +} + +fn publish(origin: &origin::Producer, path: &str) -> Live { + let route = origin.route(path, origin::RouteOptions::hidden()).unwrap(); + let info = track::Info::default().with_timescale(moq_net::Timescale::MILLI); + let track = route.broadcast().create_track(TRACK, info).unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), path.as_bytes()).unwrap(); + group.finish().unwrap(); + route.announce().unwrap(); + Live { route, _track: track } +} + +fn grant(subscribe: &str) -> Grant { + let mut grant = Grant::default(); + grant.subscribe = [Pattern::subtree(subscribe).unwrap()].into_iter().collect::(); + grant +} + +/// A reader's session to the relay, its grant set before the relay's side first runs. +async fn connect_reader(relay: &origin::Producer, reader: &origin::Producer, narrow: bool) -> MockPair { + let version: Version = VERSION.parse().unwrap(); + let mut mock = moq_net::transport::memory::Config::default(); + mock.protocol = Some(version.alpn().to_string()); + let (client_transport, server_transport) = create_mock_session_pair(mock); + let transports = (client_transport.clone(), server_transport.clone()); + let client = Client::new() + .with_versions(version.into()) + .with_subscriber(reader.clone()); + let server = Server::new() + .with_versions(version.into()) + .with_publisher(relay.consume()); + let (client, server) = futures::join!( + async { + let (session, driver) = client.connect(harness::now(), client_transport).await.unwrap(); + harness::spawn(driver); + session + }, + async { + let (session, driver) = server + .handshake(harness::now(), server_transport) + .await + .unwrap() + .ok() + .await + .unwrap(); + session.auth().authorize(&grant(if narrow { NARROW } else { "" })); + harness::spawn(driver); + session + }, + ); + MockPair { + client, + server, + client_transport: transports.0, + server_transport: transports.1, + client_discovery: None, + } +} + +/// Request `path` and read its frame, as a reader does the moment it hears of a claim. +async fn read(consumer: origin::Consumer, path: String) -> Outcome { + let result = async { + let broadcast = consumer.request(path.as_str(), None).await?; + let subscription = track::Subscription::default().with_ranges([support::keys(0..)]); + let mut sub = broadcast.track(TRACK)?.subscribe(subscription).await?; + let Some(mut group) = sub.recv_group().await? else { + return Ok(Outcome::Ended); + }; + Ok::<_, Error>(match group.read_frame().await? { + Some(_) => Outcome::Read, + None => Outcome::Ended, + }) + }; + result.await.unwrap_or_else(|err| Outcome::Failed(err.to_string())) +} + +/// Follow every claim `reader` shows, requesting each the moment it appears. +fn follow(index: usize, reader: &origin::Producer, log: Shared) { + let consumer = reader.consume(); + let mut routes = consumer.routes(); + drop(schedule::spawn(async move { + while let Some(event) = routes.next().await { + let path = event.announce.prefix.to_string(); + note(&log, format!("reader {index} hears {:?} {path}", event.kind)); + match event.kind { + RouteKind::Start | RouteKind::Restart => { + let request = { + let mut log = log.borrow_mut(); + let request = log.requests.len(); + let after = log.taken.clone(); + log.requests.push(Request { + reader: index, + path: path.clone(), + outcome: Outcome::Pending, + after, + }); + log.shown.insert((index, path.clone()), request); + request + }; + let (consumer, log) = (consumer.clone(), log.clone()); + drop(schedule::spawn(async move { + let outcome = read(consumer, path.clone()).await; + note( + &log, + format!("reader {index} request {request} for {path}: {outcome:?}"), + ); + log.borrow_mut().requests[request].outcome = outcome; + })); + } + RouteKind::Update => {} + RouteKind::End | RouteKind::Offline => { + log.borrow_mut().shown.remove(&(index, path)); + } + } + } + })); +} + +/// The broadcasts and sessions the steps act on. +struct World { + old: Option, + new: Option, + /// What [`Step::Announce`] published. + more: Vec, + publisher: origin::Producer, + second: Option, + readers: Vec>, +} + +fn run(step: Step, world: &Rc>, log: &Shared) { + let mut world = world.borrow_mut(); + match step { + Step::Replace { drop: ended, new_first } => { + let old = world.old.take().expect("replaced once"); + let end = |old: Live| match ended { + true => drop(old), + false => old.route.withdraw(), + }; + let new = match new_first { + true => { + let new = publish(&world.publisher, NEW); + end(old); + new + } + false => { + end(old); + publish(&world.publisher, NEW) + } + }; + world.new = Some(new); + } + Step::Widen(reader) => { + if let Some(pair) = &world.readers[reader] { + pair.server.auth().authorize(&grant("")); + } + } + Step::Close(close) => { + let pair = world.second.take().expect("closed once"); + match close { + Close::Publisher => pair.client.abort(Error::Cancel), + Close::Relay => pair.server.abort(Error::Cancel), + Close::Drop => drop(pair), + } + } + Step::Announce(index) => { + let live = publish(&world.publisher, &format!("{NARROW}/{index}")); + world.more.push(live); + } + Step::Leave(reader) => { + drop(world.readers[reader].take()); + log.borrow_mut().left.push(reader); + } + } +} + +/// One round: set up, race the steps, settle, and check every reader. +async fn round(shape: &Shape) -> Result<(), String> { + let _schedule = schedule::install(Schedule { + chance: shape.chance, + longest: shape.longest, + }); + let log = Shared::default(); + let version: Version = VERSION.parse().unwrap(); + + let publisher = produce_origin(1); + let second = produce_origin(1); + let relay = produce_origin(2); + let old = publish(&publisher, OLD); + let stay = publish(&publisher, STAY); + let gone = publish(&second, GONE); + + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(relay.clone()); + let _upstream = connect_mock(options).await; + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(second.consume()); + options.server_subscribe = Some(relay.clone()); + let second_pair = connect_mock(options).await; + + let mut readers = Vec::new(); + for index in 0..shape.readers { + let reader = produce_origin(3); + follow(index, &reader, log.clone()); + readers.push(Some(connect_reader(&relay, &reader, shape.narrow[index]).await)); + } + if shape.settled { + kio::clock::sleep(SETTLE).await; + } + + let world = Rc::new(RefCell::new(World { + old: Some(old), + new: None, + more: Vec::new(), + publisher, + second: Some(second_pair), + readers, + })); + let mut steps = Vec::new(); + for (step, lead) in &shape.steps { + let (step, lead, world, log) = (*step, *lead, world.clone(), log.clone()); + let name = format!("{step:?}"); + steps.push(schedule::spawn(async move { + for _ in 0..lead { + kio::clock::sim::yield_now().await; + } + note(&log, format!("step {name}")); + run(step, &world, &log); + log.borrow_mut().taken.push(name); + })); + } + for step in steps { + step.await.expect("a step panicked"); + } + kio::clock::sleep(SETTLE).await; + + let log = log.borrow(); + let mut failures = Vec::new(); + for (index, request) in log.requests.iter().enumerate() { + if request.outcome == Outcome::Pending { + failures.push(format!( + "reader {}'s request {index} for {} hangs, made after steps {:?}", + request.reader, request.path, request.after + )); + } + } + for reader in (0..shape.readers).filter(|reader| !log.left.contains(reader)) { + for path in [OLD, GONE] { + if log.shown.contains_key(&(reader, path.to_string())) { + failures.push(format!("reader {reader} still shows {path}, which ended")); + } + } + let more = shape.steps.iter().filter_map(|(step, _)| match step { + Step::Announce(index) => Some(format!("{NARROW}/{index}")), + _ => None, + }); + for path in [NEW.to_string(), STAY.to_string()].into_iter().chain(more) { + match log.shown.get(&(reader, path.clone())) { + None => failures.push(format!("reader {reader} never shows {path}")), + Some(index) => { + let request = &log.requests[*index]; + if !matches!(request.outcome, Outcome::Read | Outcome::Pending) { + failures.push(format!( + "reader {reader}'s request {index} for {path}, a claim it shows: {:?}, made after steps {:?}", + request.outcome, request.after + )); + } + } + } + } + } + drop((stay, gone)); + if failures.is_empty() { + return Ok(()); + } + let mut report = failures.join("\n"); + let _ = write!(report, "\n\ntrace:\n {}", log.trace.join("\n ")); + Err(report) +} + +/// The rounds of a sweep from `seed`, each with its own seed. +fn round_seed(seed: u64, index: u64) -> u64 { + Rng(seed ^ index.wrapping_mul(0xA24B_AED4_963E_E407)).next() +} + +fn setting(name: &str, fallback: u64) -> u64 { + std::env::var(name).map_or(fallback, |value| value.parse().expect(name)) +} + +/// Runs the rounds `MOQ_RACE_SEED`, `MOQ_RACE_ROUND`, and `MOQ_RACE_ROUNDS` pick. +#[test] +fn seeded_races_hold_every_claim_shown() { + let seed = setting("MOQ_RACE_SEED", 0x726163); + let first = setting("MOQ_RACE_ROUND", 0); + let rounds = setting("MOQ_RACE_ROUNDS", 16); + eprintln!("MOQ_RACE_SEED={seed} MOQ_RACE_ROUNDS={rounds}"); + for index in first..first + rounds { + let round_seed = round_seed(seed, index); + let mut rng = Rng(round_seed); + let shape = Shape::draw(&mut rng); + let replay = format!("replay it with MOQ_RACE_SEED={seed} MOQ_RACE_ROUND={index} MOQ_RACE_ROUNDS=1"); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + kio::clock::sim::run_seeded(round_seed, round(&shape)) + })); + match result { + Ok(Ok(())) => {} + Ok(Err(report)) => panic!("round {index} failed; {replay}\n{shape:#?}\n{report}"), + Err(panic) => { + eprintln!("round {index} panicked; {replay}\n{shape:#?}"); + std::panic::resume_unwind(panic); + } + } + } +} diff --git a/rs/moq-net/tests/integration/support/harness.rs b/rs/moq-net/tests/integration/support/harness.rs index 6c69c47f20..6d919641b6 100644 --- a/rs/moq-net/tests/integration/support/harness.rs +++ b/rs/moq-net/tests/integration/support/harness.rs @@ -26,7 +26,7 @@ pub fn now() -> moq_net::time::Instant { /// the benches. pub fn spawn(mut driver: D) { match kio::clock::sim::is_running() { - true => drop(kio::clock::sim::spawn(kio::clock::drive(move |now, waiter| { + true => drop(super::schedule::spawn(kio::clock::drive(move |now, waiter| { moq_net::time::Driver::poll(&mut driver, now, waiter) }))), false => drop(tokio::spawn(moq_net::time::run(driver))), diff --git a/rs/moq-net/tests/integration/support/mod.rs b/rs/moq-net/tests/integration/support/mod.rs index 744262fe4c..26dee79172 100644 --- a/rs/moq-net/tests/integration/support/mod.rs +++ b/rs/moq-net/tests/integration/support/mod.rs @@ -2,6 +2,7 @@ pub mod harness; pub mod mock; +pub mod schedule; use std::ops::{Bound, RangeBounds}; diff --git a/rs/moq-net/tests/integration/support/schedule.rs b/rs/moq-net/tests/integration/support/schedule.rs new file mode 100644 index 0000000000..ece5d9a9f6 --- /dev/null +++ b/rs/moq-net/tests/integration/support/schedule.rs @@ -0,0 +1,114 @@ +//! A seeded task schedule for the simulator. +//! +//! The simulator polls ready tasks in the order they woke, so two sessions' tasks always +//! run in the order the test happened to wake them. Separate threads give no such +//! order. While a schedule is [`install`]ed, every task [`spawn`] starts (the harness's +//! session and origin drivers included) may be held back for a few executor passes +//! before a poll, letting every other ready task run first, the way a thread descheduled +//! for a moment would. Holds are drawn from the run's [`kio::clock::seed`], so the +//! run's seed replays its schedule. +//! +//! It reorders whole task polls. An interleaving inside one poll, between two threads +//! taking the same lock, is out of its reach. + +#![allow(dead_code, reason = "shared by several test binaries, each using only part of it")] + +use std::cell::RefCell; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; + +thread_local! { + static SCHEDULE: RefCell> = const { RefCell::new(None) }; +} + +/// How often and how long a poll is held back. +pub struct Schedule { + /// The chance, in 1/1024ths, that a poll is held back at all. + pub chance: u64, + /// The most executor passes one hold lasts. + pub longest: u64, +} + +/// Holds the schedule installed until dropped. +pub struct Installed(()); + +impl Drop for Installed { + fn drop(&mut self) { + SCHEDULE.with(|schedule| schedule.borrow_mut().take()); + } +} + +/// Hold back the polls of every task [`spawn`] starts on this thread, as `schedule` +/// draws from the running simulator's seed. +pub fn install(schedule: Schedule) -> Installed { + SCHEDULE.with(|installed| { + let mut installed = installed.borrow_mut(); + assert!(installed.is_none(), "a schedule is already installed"); + // Salted, so a test drawing from the same seed draws other numbers. + *installed = Some((schedule, kio::clock::seed() ^ 0x5CED_0000_5CED)); + }); + Installed(()) +} + +/// Spawn on the simulator, under the installed schedule if there is one. +pub fn spawn(future: F) -> kio::clock::sim::JoinHandle +where + F: Future + 'static, + F::Output: 'static, +{ + match SCHEDULE.with(|schedule| schedule.borrow().is_some()) { + true => kio::clock::sim::spawn(Held { + future: Box::pin(future), + hold: None, + }), + false => kio::clock::sim::spawn(future), + } +} + +/// The executor passes the next poll waits, 0 to run it now. +fn draw() -> u64 { + SCHEDULE.with(|schedule| { + let mut schedule = schedule.borrow_mut(); + let Some((schedule, state)) = schedule.as_mut() else { + return 0; + }; + match next(state) % 1024 < schedule.chance { + true => 1 + next(state) % schedule.longest, + false => 0, + } + }) +} + +/// splitmix64. +fn next(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +struct Held { + future: Pin>, + /// Passes left before the held poll runs. + hold: Option, +} + +impl Future for Held { + type Output = F::Output; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let left = match self.hold.take() { + Some(left) => left, + None => draw(), + }; + if left > 0 { + self.hold = Some(left - 1); + // Woken now, it goes behind every task already ready. + cx.waker().wake_by_ref(); + return Poll::Pending; + } + self.future.as_mut().poll(cx) + } +} diff --git a/test/justfile b/test/justfile index 61fd9ef73a..652fc958d6 100644 --- a/test/justfile +++ b/test/justfile @@ -323,6 +323,10 @@ route-proof: route-sweep: bash ../sh/local.sh bash route-sweep.sh +# Race a lite-07 relay's session tasks in seeded orders from a fresh replayable seed. +route-race: + bash ../sh/local.sh bash route-race.sh + # The TypeScript qmux peer as client and server against the in-tree Rust qmux # interop examples, per draft. The manifest path names the in-tree crate, which # shares its package name with the 0.6 release its tests interoperate with. diff --git a/test/route-race.sh b/test/route-race.sh new file mode 100755 index 0000000000..d95935fd33 --- /dev/null +++ b/test/route-race.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Nightly races fresh rounds; the printed settings replay a failing round. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +MOQ_RACE_SEED=${MOQ_RACE_SEED:-$(python3 -c 'import secrets; print(secrets.randbits(31))')} +# With moq-dev/moq#9052's fix reverted, a round fails about once in 820 (40 to 1464 over 10 seeds), +# so 10000 rounds miss it less than once in 100,000 nights; about 2.5 minutes. +MOQ_RACE_ROUNDS=${MOQ_RACE_ROUNDS:-10000} +export MOQ_RACE_SEED MOQ_RACE_ROUNDS +echo "Route race: MOQ_RACE_SEED=$MOQ_RACE_SEED MOQ_RACE_ROUNDS=$MOQ_RACE_ROUNDS" +# Release keeps the rounds quick; the checks stay on so it still catches what a debug run would. +export CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS=true CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS=true +just rs test -P sweep -p moq-net --release --features async --test integration -E 'test(=route_race::seeded_races_hold_every_claim_shown)'